def import_data_from_csv(self, file_path: str, symbol: str,
exchange: Exchange, interval: Interval,
datetime_head: str, open_head: str,
high_head: str, low_head: str, close_head: str,
volume_head: str, open_interest_head: str,
datetime_format: str) -> Tuple:
""""""
with open(file_path, "rt") as f:
buf = [line.replace("\0", "") for line in f]
reader = csv.DictReader(buf, delimiter=",")
bars = []
start = None
count = 0
for item in reader:
if datetime_format:
dt = datetime.strptime(item[datetime_head], datetime_format)
else:
dt = datetime.fromisoformat(item[datetime_head])
open_interest = item.get(open_interest_head, 0)
bar = BarData(
symbol=symbol,
exchange=exchange,
datetime=dt,
interval=interval,
volume=float(item[volume_head]),
open_price=float(item[open_head]),
high_price=float(item[high_head]),
low_price=float(item[low_head]),
close_price=float(item[close_head]),
open_interest=float(open_interest),
gateway_name="DB",
)
bars.append(bar)
# do some statistics
count += 1
if not start:
start = bar.datetime
# insert into database
database_manager.save_bar_data(bars)
end = bar.datetime
return start, end, count
def request1():
start = datetime(2020, 10, 1, tzinfo=get_localzone())
end = datetime(2020, 10, 2, tzinfo=get_localzone())
req = HistoryRequest(symbol=symbol,
exchange=exchange,
interval=Interval(interval),
start=start,
end=end)
data = gate_way.query_history(req)
print(data)
if data:
database_manager.save_bar_data(data)
def run(self):
""""""
while self.active:
try:
task = self.queue.get(timeout=1)
task_type, data = task
if task_type == "tick":
database_manager.save_tick_data([data])
elif task_type == "bar":
database_manager.save_bar_data([data])
except Empty:
continue
except Exception:
self.active = False
info = sys.exc_info()
event = Event(EVENT_RECORDER_EXCEPTION, info)
self.event_engine.put(event)
def run_downloading(self, vt_symbol: str, interval: str, start: datetime,
end: datetime):
"""
Query bar data from RQData.
"""
self.write_log(f"{vt_symbol}-{interval}开始下载历史数据")
try:
symbol, exchange = extract_vt_symbol(vt_symbol)
except ValueError:
self.write_log(f"{vt_symbol}解析失败，请检查交易所后缀")
self.thread = None
return
req = HistoryRequest(symbol=symbol,
exchange=exchange,
interval=Interval(interval),
start=start,
end=end)
contract = self.main_engine.get_contract(vt_symbol)
try:
# If history data provided in gateway, then query
if contract and contract.history_data:
data = self.main_engine.query_history(req,
contract.gateway_name)
if data:
database_manager.save_bar_data(data)
self.write_log(f"{vt_symbol}-{interval}历史数据下载完成")
else:
self.write_log(f"数据下载失败，无法获取{vt_symbol}的历史数据")
except Exception:
msg = f"数据下载失败，触发异常：\n{traceback.format_exc()}"
self.write_log(msg)
# Clear thread object handler.
self.thread = None
def download_bar_data(self, symbol: str, exchange: Exchange, interval: str,
start: datetime) -> int:
"""
Query bar data from RQData.
"""
req = HistoryRequest(symbol=symbol,
exchange=exchange,
interval=Interval(interval),
start=start,
end=datetime.now())
vt_symbol = f"{symbol}.{exchange.value}"
contract = self.main_engine.get_contract(vt_symbol)
# If history data provided in gateway, then query
if contract and contract.history_data:
data = self.main_engine.query_history(req, contract.gateway_name)
if data:
database_manager.save_bar_data(data)
return (len(data))
return 0
def get_binance_data(symbol: str, exchanges: str, start_time: str,
end_time: str):
"""
爬取币安交易所的数据
:param symbol: BTCUSDT.
:param exchanges: 现货、USDT合约, 或者币币合约.
:param start_time: 格式如下:2020-1-1 或者2020-01-01
:param end_time: 格式如下:2020-1-1 或者2020-01-01
:return:
"""
api_url = ''
save_symbol = symbol
gate_way = 'BINANCES'
if exchanges == 'spot':
print("spot")
limit = BINANCE_SPOT_LIMIT
save_symbol = symbol.lower()
gate_way = 'BINANCE'
api_url = f'https://api.binance.com/api/v3/klines?symbol={symbol}&interval=1m&limit={limit}'
elif exchanges == 'future':
print('future')
limit = BINANCE_FUTURE_LIMIT
api_url = f'https://fapi.binance.com/fapi/v1/klines?symbol={symbol}&interval=1m&limit={limit}'
elif exchanges == 'coin_future':
print("coin_future")
limit = BINANCE_FUTURE_LIMIT
f'https://dapi.binance.com/dapi/v1/klines?symbol={symbol}&interval=1m&limit={limit}'
else:
raise Exception('交易所名称请输入以下其中一个：spot, future, coin_future')
start_time = int(
datetime.strptime(start_time, '%Y-%m-%d').timestamp() * 1000)
end_time = int(datetime.strptime(end_time, '%Y-%m-%d').timestamp() * 1000)
while True:
try:
print(start_time)
url = f'{api_url}&startTime={start_time}'
print(url)
data = requests.get(url=url, timeout=10, proxies=proxies).json()
"""
[
[
1591258320000, // 开盘时间
"9640.7", // 开盘价
"9642.4", // 最高价
"9640.6", // 最低价
"9642.0", // 收盘价(当前K线未结束的即为最新价)
"206", // 成交量
1591258379999, // 收盘时间
"2.13660389", // 成交额(标的数量)
48, // 成交笔数
"119", // 主动买入成交量
"1.23424865", // 主动买入成交额(标的数量)
"0" // 请忽略该参数
]
"""
buf = []
for l in data:
bar = BarData(symbol=save_symbol,
exchange=Exchange.BINANCE,
datetime=generate_datetime(l[0]),
interval=Interval.MINUTE,
volume=float(l[5]),
open_price=float(l[1]),
high_price=float(l[2]),
low_price=float(l[3]),
close_price=float(l[4]),
gateway_name=gate_way)
buf.append(bar)
database_manager.save_bar_data(buf)
# 到结束时间就退出, 后者收盘价大于当前的时间.
if (data[-1][0] > end_time
) or data[-1][6] >= (int(time.time() * 1000) - 60 * 1000):
break
start_time = data[-1][0]
except Exception as error:
print(error)
time.sleep(10)