def __init__(self, dbName, dbLite=False):
volDict = {
"binance": "takerBaseAssetVol",
"poloniex": "quoteVolume",
"bitmex": "volume",
}
self.volField = volDict[dbName]
self.TU = TimeUtil()
if not dbLite:
self.db = MongoClient("localhost", 27017)[dbName]
def __init__(self, EntryStrat, periodList=(5, 10, 15), verbose=False):
self.periodList = periodList
self.E = EntryStrat
self.enterAtDict = self.E.run()
self.ts2Bin = TimeUtil().ts2Bin
self.assetList = self.E.assetList
self.dfDict = self.E.dfDict
self.verbose = verbose
self.period = str(periodList[0])
def __init__(self, Entrance, exitStrat, isVisual=False, verbose=False):
self.E = Entrance
self.positionDict = Exit(self.E, exitStrat).run()
print(self.positionDict)
self.assetList = self.E.assetList
self.dfDict = self.E.dfDict
self.ts2Bin = TimeUtil().ts2Bin
self.isVisual = isVisual
self.verbose = verbose
class MongoUtil:
"""
* insert_one/ query method is crazy slow when doing initial propagation but only a one time run and
guarantees no duplication of data
"""
def __init__(self, dbName):
self.dbName = dbName
self.client = MongoClient("localhost", 27017)
self.db = self.client[self.dbName]
self.TU = TimeUtil()
def toMongo(self, data, colName, id, parameters={}):
for val in data:
try:
if len(list(self.db[colName].find({id: val[id]}))) == 0:
for param in parameters.keys():
if (param == "TS" and self.dbName == "binance"
or self.dbName == "bitmex"):
val["TS"] = int(
self.TU.getTS(
val[parameters["TS"][0]],
timeFormat=parameters["TS"][1],
))
self.db[colName].insert_one(val)
else:
try:
val[param] = val[parameters[param]]
self.db[colName].insert_one(val)
except TypeError:
print(val)
raise SystemExit
except TypeError:
print(val)
print(list(self.db[colName].find({id: val[id]})))
break
def lastVal(self, colName):
return list(self.db[colName].find({}, {
"_id": 0,
"TS": 1
}).sort("TS", DESCENDING).limit(1))[0]["TS"]
def count(self, colName):
return self.db[colName].count()
def index(self, colName):
self.db[colName].ensure_index("TS", ASCENDING)
class BitmexDS:
"""
Datasource for Bitmex exchange
"""
def __init__(self):
self.TU = TimeUtil()
self.MU = MongoUtil(dbName="bitmex")
self.baseUrl = "https://www.bitmex.com/api/v1/"
self.bin2Time = {"1d": 60 * 60 * 24, "1h": 60 * 60, "5m": 60 * 5}
def pullData(self, endPoint, params=None):
data = requests.get(self.baseUrl + endPoint, params=params).content.decode(
"utf-8"
)
return json.loads(data)
def getInst(self):
return [inst["symbol"] for inst in self.pullData("instrument/active")]
def pullCandles(self, asset, binSize, startTime, endTime=None, isDemo=False):
data = []
tmpTime = (
startTime if type(startTime) == str else self.TU.getDatetime(startTime)
)
endTS = self.TU.getTS(endTime) if endTime else time.time() - self.bin2Time["1d"]
while self.TU.getTS(tmpTime) + 500 * self.bin2Time[binSize] < endTS:
try:
tmpData = self.pullData(
"trade/bucketed?binSize=%s&symbol=%s&count=500&startTime=%s"
% (binSize, asset, tmpTime)
)
except OSError:
time.sleep(30)
tmpData = self.pullData(
"trade/bucketed?binSize=%s&symbol=%s&count=500&startTime=%s"
% (binSize, asset, tmpTime)
)
tmpTime = tmpData[-1]["timestamp"]
data += tmpData
time.sleep(2)
data += self.pullData(
"trade/bucketed?binSize=%s&symbol=%s&count=500&startTime=%s&endTime=%s"
% (binSize, asset, tmpTime, endTime)
)
if data != ["error"]:
if not isDemo:
self.MU.toMongo(
data=data,
colName="%s_%s" % (asset, binSize),
parameters={
"binSize": self.bin2Time[binSize],
"TS": ("timestamp", "%Y-%m-%dT%H:%M:%S.000Z"),
},
id="timestamp",
)
else:
df = pd.DataFrame(
data,
columns=[
"timestamp",
"symbol",
"open",
"high",
"low",
"close",
"trades",
"volume",
"vmap",
"lastSize",
"turnover",
"homeNotional",
"foreignNotional",
],
).drop_duplicates("timestamp")
df["binSize"] = self.bin2Time[binSize]
return df
else:
print("Error for asset: %s and bin size: %s" % (asset, binSize))
def updateDB(self):
for inst in self.getInst():
print("For instrument: %s" % inst)
for bin in self.bin2Time.keys():
col = "%s_%s" % (inst, bin)
print("For col: %s" % col)
try:
print(self.MU.count(col))
startTime = (
self.MU.lastVal(col)
if self.MU.count(col) != 0
else self.TU.getTS(
self.pullData(
"trade/bucketed?binSize=%s&symbol=%s&count=1"
% (bin, inst)
)[0]["timestamp"]
)
)
if int(time.time() - self.bin2Time["1d"]) - startTime > 1000:
print("Starting bin size: %s" % bin)
print(
"Start time: %s, End time: %s"
% (
self.TU.getDatetime(startTime),
self.TU.getDatetime(time.time() - self.bin2Time["1d"]),
)
)
self.pullCandles(asset=inst, binSize=bin, startTime=startTime)
self.MU.index(colName=col)
print("%s updated" % col)
else:
print("Already up to date for bin size: %s" % bin)
except IndexError:
print("Not enough date for inst: %s and bin: %s" % (inst, bin))
def __init__(self):
self.TU = TimeUtil()
self.MU = MongoUtil(dbName="bitmex")
self.baseUrl = "https://www.bitmex.com/api/v1/"
self.bin2Time = {"1d": 60 * 60 * 24, "1h": 60 * 60, "5m": 60 * 5}
def __init__(self):
self.MU = MongoUtil(dbName="poloniex")
self.TU = TimeUtil()
self.baseUrl = "https://poloniex.com/public?command="
from Backtest.main.Utils.TimeUtil import TimeUtil
"""
Tests for TimeUtil class
"""
TU = TimeUtil()
def test_getTS():
pass # assert TU.getTS('2012-08-29 11:38:22', timeFormat='%Y-%m-%d %H:%M:%S') == 1346236702
class BinanceDS:
"""
Datasource for Binance exchange
"""
def __init__(self):
self.MU = MongoUtil(dbName="binance")
self.TU = TimeUtil()
self.baseUrl = "https://api.binance.com"
self.bin2TS = self.TU.bin2TS
def pullData(self, endPoint, params=None):
data = requests.get(self.baseUrl + endPoint,
params=params).content.decode("utf-8")
return json.loads(data)
def getBTCAssets(self):
dayStats = self.pullData("/api/v1/ticker/24hr")
return [stat["symbol"] for stat in dayStats if "BTC" in stat["symbol"]]
def getUSDTAssets(self):
dayStats = self.pullData("/api/v1/ticker/24hr")
return [
stat["symbol"] for stat in dayStats if "USDT" in stat["symbol"]
]
def getFirstTrade(self, asset, binSize):
return (self.pullData(
"/api/v1/klines?symbol=%s&interval=%s&limit=1&startTime=0" %
(asset, binSize))[0][0] / 1000)
def list2Dict(self, val):
return {
"milliTimestamp": val[0],
"open": val[1],
"high": val[2],
"low": val[3],
"close": val[4],
"volume": val[5],
"quoteVol": val[7],
"numTrades": val[8],
"takerBaseAssetVol": val[9],
"takerQuoteAssetVol": val[10],
"TS": val[0] / 1000,
}
def pullCandles(self,
asset,
binSize,
startTime,
endTime=None,
isDemo=False):
data = []
tmpTime = startTime * 1000
endTime = (endTime * 1000 if endTime else
(time.time() - self.bin2TS["1d"]) * 1000)
while tmpTime + 500 * self.bin2TS[binSize] * 1000 < endTime:
tmpData = []
print(tmpTime + 500 * self.bin2TS[binSize] * 1000 - endTime)
try:
tmpData = self.pullData(
"/api/v1/klines?symbol=%s&interval=%s&limit=500&startTime=%s"
% (asset, binSize, int(tmpTime)))
except OSError:
time.sleep(30)
tmpData = self.pullData(
"/api/v1/klines?symbol=%s&interval=%s&limit=500&startTime=%s"
% (asset, binSize, int(tmpTime)))
try:
tmpTime = tmpData[-1][6]
except IndexError:
break
data += [self.list2Dict(val) for val in tmpData]
time.sleep(1)
lastData = self.pullData(
"/api/v1/klines?symbol=%s&interval=%s&limit=500&startTime=%s&endTime=%s"
% (asset, binSize, int(tmpTime), int(endTime)))
data += [self.list2Dict(val) for val in lastData]
if not isDemo:
print("Add mongo implementation")
self.MU.toMongo(data=data,
colName="%s_%s" % (asset, binSize),
id="milliTimestamp")
else:
return pd.DataFrame(
data,
columns=[
"milliTimestamp",
"open",
"high",
"low",
"close",
"volume",
"quoteVol",
"numTrades",
"takerBaseAssetVol",
"takerQuoteAssetVol",
"TS",
],
)
def updateDB(self):
for asset in ["ETHUSDT"
]: # self.getBTCAssets() + self.getUSDTAssets():
print("For asset: %s" % asset)
for bin in ["5m"]:
col = "%s_%s" % (asset, bin)
startTime = (self.MU.lastVal(col) if self.MU.count(col) != 0
else self.getFirstTrade(asset, bin))
if int(time.time() - self.bin2TS["1d"]) - startTime > 1000:
print("Starting pull of bin size: %s" % bin)
print(self.TU.getDatetime(startTime))
self.pullCandles(asset=asset,
binSize=bin,
startTime=startTime)
self.MU.index(colName=col)
print("%s updated" % col)
else:
print("Already up to date for bin size: %s" % bin)
def createCSV(self,
asset,
binSize,
startTime,
location="../csv/",
endTime=None):
df = self.pullCandles(
asset=asset,
binSize=binSize,
startTime=startTime,
endTime=endTime,
isDemo=True,
)
df.to_csv("%s%s.csv.gz" % (location, asset),
compression="gzip",
index=False)
def __init__(self):
self.MU = MongoUtil(dbName="binance")
self.TU = TimeUtil()
self.baseUrl = "https://api.binance.com"
self.bin2TS = self.TU.bin2TS
class Load:
def __init__(self, dbName, dbLite=False):
volDict = {
"binance": "takerBaseAssetVol",
"poloniex": "quoteVolume",
"bitmex": "volume",
}
self.volField = volDict[dbName]
self.TU = TimeUtil()
if not dbLite:
self.db = MongoClient("localhost", 27017)[dbName]
def listAssets(self, contains=""):
return [
col for col in self.db.list_collection_names() if contains in col
]
def loadOne(self, col, timeStart, timeEnd=None, limit=100, paramList=None):
"""
timeStart in format:
-> TS
-> 'dd/mm/yyyy'
"""
timeStart = (self.TU.getTS(timeStart, timeFormat="%d/%m/%Y")
if type(timeStart) == str else timeStart)
timeEnd = (self.TU.getTS(timeEnd, timeFormat="%d/%m/%Y")
if type(timeEnd) == str else timeEnd)
paramDict = {} if not paramList else {val: 1 for val in paramList}
if timeEnd:
data = list(self.db[col].find(
{
"TS": {
"$gte": timeStart,
"$lte": timeEnd
}
},
{
**paramDict, "_id": 0
},
).sort("TS", ASCENDING))
else:
data = list(self.db[col].find({
"TS": {
"$gte": timeStart
}
}, {
**paramDict, "_id": 0
}).sort("TS", ASCENDING).limit(limit))
try:
key = (list(self.db[col].find_one({}, {
"_id": 0
}).keys()) if not paramList else paramList)
df = pd.DataFrame(data, columns=key)
except AttributeError:
print(self.db[col].count())
df = pd.DataFrame(
[], columns=["open", "close", "high", "low", self.volField])
return self.makeNumeric(
df, fieldList=["open", "close", "high", "low", self.volField])
def makeNumeric(self, df, fieldList):
df[fieldList] = df[fieldList].apply(pd.to_numeric)
return df
def loadCSV(self, file, location="./csv/"):
df = pd.read_csv("%s%s.csv.gz" % (location, file))
return self.makeNumeric(
df, fieldList=["open", "close", "high", "low", self.volField])
def __init__(self, dbName):
self.dbName = dbName
self.client = MongoClient("localhost", 27017)
self.db = self.client[self.dbName]
self.TU = TimeUtil()