def load(self, file_path: str, symbol: str, exchange: Exchange,
interval: Interval, datetime_head: str, open_head: str,
close_head: str, low_head: str, high_head: str, volume_head: str,
datetime_format: str):
""""""
vt_symbol = f"{symbol}.{exchange.value}"
start = None
end = None
count = 0
with open(file_path, "rt") as f:
reader = csv.DictReader(f)
db_bars = []
for item in reader:
dt = datetime.strptime(item[datetime_head], datetime_format)
db_bar = {
"symbol": symbol,
"exchange": exchange.value,
"datetime": dt,
"interval": interval.value,
"volume": item[volume_head],
"open_price": item[open_head],
"high_price": item[high_head],
"low_price": item[low_head],
"close_price": item[close_head],
"vt_symbol": vt_symbol,
"gateway_name": "DB"
}
db_bars.append(db_bar)
# do some statistics
count += 1
if not start:
start = db_bar["datetime"]
end = db_bar["datetime"]
# Insert into DB
with DB.atomic():
for batch in chunked(db_bars, 50):
DbBarData.insert_many(batch).on_conflict_replace().execute()
return start, end, count
def load_data(self):
""""""
self.output("开始加载历史数据")
if self.mode == BacktestingMode.BAR:
s = (
DbBarData.select()
.where(
(DbBarData.vt_symbol == self.vt_symbol) &
(DbBarData.interval == self.interval)
& (DbBarData.datetime >= self.start)
& (DbBarData.datetime <= self.end)
) .order_by(DbBarData.datetime)
)
self.history_data = [db_bar.to_bar() for db_bar in s]
else:
s = (
DbTickData.select()
.where(
(DbTickData.vt_symbol == self.vt_symbol)
& (DbTickData.datetime >= self.start)
& (DbTickData.datetime <= self.end)
)
.order_by(DbTickData.datetime)
)
self.history_data = [db_tick.to_tick() for db_tick in s]
self.output(f"历史数据加载完成，数据量：{len(self.history_data)}")
def load_data(self):
""""""
self.output("开始加载历史数据")
if self.mode == BacktestingMode.BAR:
s = (
DbBarData.select()
.where(
(DbBarData.vt_symbol == self.vt_symbol)
& (DbBarData.interval == self.interval)
& (DbBarData.datetime >= self.start)
& (DbBarData.datetime <= self.end)
)
.order_by(DbBarData.datetime)
)
self.history_data = [db_bar.to_bar() for db_bar in s]
else:
s = (
DbTickData.select()
.where(
(DbTickData.vt_symbol == self.vt_symbol)
& (DbTickData.datetime >= self.start)
& (DbTickData.datetime <= self.end)
)
.order_by(DbTickData.datetime)
)
self.history_data = [db_tick.to_tick() for db_tick in s]
self.output(f"历史数据加载完成，数据量：{len(self.history_data)}")
def load_bar_data(vt_symbol: str, interval: str, start: datetime,
end: datetime):
""""""
s = (DbBarData.select().where((DbBarData.vt_symbol == vt_symbol)
& (DbBarData.interval == interval)
& (DbBarData.datetime >= start)
& (DbBarData.datetime <= end)).order_by(
DbBarData.datetime))
data = [db_bar.to_bar() for db_bar in s]
return data
def download_minute_bar(vt_symbol):
"""下载某一合约的分钟线数据"""
print(f"开始下载合约数据{vt_symbol}")
symbol, exchange = vt_symbol.split(".")
start = time()
df = rq.get_price(symbol, frequency="1m", fields=FIELDS)
with DB.atomic():
for ix, row in df.iterrows():
print(row.name)
bar = generate_bar_from_row(row, symbol, exchange)
DbBarData.replace(bar.__data__).execute()
end = time()
cost = (end - start) * 1000
print("合约%s的分钟K线数据下载完成%s - %s，耗时%s毫秒" %
(symbol, df.index[0], df.index[-1], cost))
def downHourBarBySymbol(symbol, period, start, end):
"""下载hour线数据"""
startTime = time.time()
startDt = datetime.datetime.strptime(start, '%Y%m%d')
endDt = datetime.datetime.strptime(end, '%Y%m%d')
url = 'https://www.bitmex.com/api/v1/trade/bucketed'
params = {
'binSize': period,
'partial': 'false',
'symbol': 'XBTUSD',
'count': 500,
'reverse': 'false',
'startTime': startDt.strftime('%Y-%m-%dT%H:%M:%S.%f0Z'),
'endTime': endDt.strftime('%Y-%m-%dT%H:%M:%S.%f0Z'),
}
resp = requests.get(url, headers=headers, params=params)
if resp.status_code != 200:
print(u'%s数据下载失败' % symbol)
return
l = resp.json()
for d in l:
# {'timestamp': '2018-07-26T02:00:00.000Z', 'symbol': 'XBTUSD', 'open': 8240.5, 'high': 8255.5, 'low': 8206, 'close': 8224.5, 'trades': 17097, 'volume': 73042086, 'vwap': 8229.0981, 'lastSize': 6000, 'turnover': 887640041277, 'homeNotional': 8876.40041276999, 'foreignNotional': 73042086}
# print(d)
data = DbBarData(symbol="XBTUSD", exchange="BITMEX",
datetime=datetime.datetime.strptime(d['timestamp'], '%Y-%m-%dT%H:%M:%S.%f0Z'),
interval="1h", volume=d['volume'], open_price=d['open'], high_price=d['high'],
low_price=d['low'],
close_price=d['close'], vt_symbol="BITMEX", gateway_name="bitmexgateway")
data.save()
endTime = time.time()
cost = (endTime - startTime) * 1000
print(u'合约%s数据下载完成，耗时%s毫秒' % (symbol, cost))
def download_minute_bar(vt_symbol):
"""下载某一合约的分钟线数据"""
print(f"开始下载合约数据{vt_symbol}")
symbol, exchange = vt_symbol.split(".")
start = time()
df = rq.get_price(symbol, frequency="1m", fields=FIELDS)
with DB.atomic():
for ix, row in df.iterrows():
print(row.name)
bar = generate_bar_from_row(row, symbol, exchange)
DbBarData.replace(bar.__data__).execute()
end = time()
cost = (end - start) * 1000
print(
"合约%s的分钟K线数据下载完成%s - %s，耗时%s毫秒"
% (symbol, df.index[0], df.index[-1], cost)
)
def load_bar(self, vt_symbol: str, days: int, interval: Interval,
callback: Callable):
"""callback就一般是on_bar"""
end = datetime.now()
start = end - timedelta(days)
# Query data from RQData by default, if not found, load from database.
data = self.query_bar_from_rq(vt_symbol, interval, start, end)
if not data:
s = (DbBarData.select().where(
(DbBarData.vt_symbol == vt_symbol)
& (DbBarData.interval == interval)
& (DbBarData.datetime >= start)
& (DbBarData.datetime <= end)).order_by(DbBarData.datetime))
data = [db_bar.to_bar() for db_bar in s]
for bar in data:
callback(bar)
def load_bar(
self, vt_symbol: str, days: int, interval: Interval, callback: Callable
):
""""""
end = datetime.now()
start = end - timedelta(days)
# Query data from RQData by default, if not found, load from database.
data = self.query_bar_from_rq(vt_symbol, interval, start, end)
if not data:
s = (
DbBarData.select()
.where(
(DbBarData.vt_symbol == vt_symbol)
& (DbBarData.interval == interval)
& (DbBarData.datetime >= start)
& (DbBarData.datetime <= end)
)
.order_by(DbBarData.datetime)
)
data = [db_bar.to_bar() for db_bar in s]
for bar in data:
callback(bar)
def generate_bar_from_row(row, symbol, exchange):
""""""
bar = DbBarData()
bar.symbol = symbol
bar.exchange = exchange
bar.interval = "1m"
bar.open_price = row["open"]
bar.high_price = row["high"]
bar.low_price = row["low"]
bar.close_price = row["close"]
bar.volume = row["volume"]
bar.datetime = row.name.to_pydatetime()
bar.gateway_name = "DB"
bar.vt_symbol = f"{symbol}.{exchange}"
return bar
def load(self, file_path: str, symbol: str, exchange: Exchange,
interval: Interval, datetime_head: str, open_head: str,
close_head: str, low_head: str, high_head: str, volume_head: str,
datetime_format: str):
""""""
vt_symbol = f"{symbol}.{exchange.value}"
start = None
end = None
count = 0
with open(file_path, 'rt') as f:
reader = csv.DictReader(f)
for item in reader:
db_bar = DbBarData()
db_bar.symbol = symbol
db_bar.exchange = exchange.value
db_bar.datetime = datetime.strptime(item[datetime_head],
datetime_format)
db_bar.interval = interval.value
db_bar.volume = item[volume_head]
db_bar.open_price = item[open_head]
db_bar.high_price = item[high_head]
db_bar.low_price = item[low_head]
db_bar.close_price = item[close_head]
db_bar.vt_symbol = vt_symbol
db_bar.gateway_name = "DB"
db_bar.replace()
# do some statistics
count += 1
if not start:
start = db_bar.datetime
end = db_bar.datetime
return start, end, count
def load(
self,
file_path: str,
symbol: str,
exchange: Exchange,
interval: Interval,
datetime_head: str,
open_head: str,
close_head: str,
low_head: str,
high_head: str,
volume_head: str,
datetime_format: str
):
""""""
vt_symbol = f"{symbol}.{exchange.value}"
start = None
end = None
count = 0
with open(file_path, 'rt') as f:
reader = csv.DictReader(f)
for item in reader:
db_bar = DbBarData()
db_bar.symbol = symbol
db_bar.exchange = exchange.value
db_bar.datetime = datetime.strptime(
item[datetime_head], datetime_format
)
db_bar.interval = interval.value
db_bar.volume = item[volume_head]
db_bar.open_price = item[open_head]
db_bar.high_price = item[high_head]
db_bar.low_price = item[low_head]
db_bar.close_price = item[close_head]
db_bar.vt_symbol = vt_symbol
db_bar.gateway_name = "DB"
db_bar.replace()
# do some statistics
count += 1
if not start:
start = db_bar.datetime
end = db_bar.datetime
return start, end, count
def generate_bar_from_row(row, symbol, exchange):
""""""
bar = DbBarData()
bar.symbol = symbol
bar.exchange = exchange
bar.interval = "1m"
bar.open_price = row["open"]
bar.high_price = row["high"]
bar.low_price = row["low"]
bar.close_price = row["close"]
bar.volume = row["volume"]
bar.datetime = row.name.to_pydatetime()
bar.gateway_name = "DB"
bar.vt_symbol = f"{symbol}.{exchange}"
return bar