def import_data_from_csv(self, file_path: str, symbol: str, exchange: Exchange, interval: Interval, datetime_head: str, open_head: str, high_head: str, low_head: str, close_head: str, volume_head: str, open_interest_head: str, datetime_format: str) -> Tuple: """""" with open(file_path, "rt") as f: buf = [line.replace("\0", "") for line in f] reader = csv.DictReader(buf, delimiter=",") bars = [] start = None count = 0 for item in reader: if datetime_format: dt = datetime.strptime(item[datetime_head], datetime_format) else: dt = datetime.fromisoformat(item[datetime_head]) open_interest = item.get(open_interest_head, 0) bar = BarData( symbol=symbol, exchange=exchange, datetime=dt, interval=interval, volume=float(item[volume_head]), open_price=float(item[open_head]), high_price=float(item[high_head]), low_price=float(item[low_head]), close_price=float(item[close_head]), open_interest=float(open_interest), gateway_name="DB", ) bars.append(bar) # do some statistics count += 1 if not start: start = bar.datetime # insert into database database_manager.save_bar_data(bars) end = bar.datetime return start, end, count
def request1(): start = datetime(2020, 10, 1, tzinfo=get_localzone()) end = datetime(2020, 10, 2, tzinfo=get_localzone()) req = HistoryRequest(symbol=symbol, exchange=exchange, interval=Interval(interval), start=start, end=end) data = gate_way.query_history(req) print(data) if data: database_manager.save_bar_data(data)
def run_downloading( self, vt_symbol: str, interval: str, start: datetime, end: datetime ): """ Query bar data from RQData. """ self.write_log(f"{vt_symbol}-{interval}开始下载历史数据") try: symbol, exchange = extract_vt_symbol(vt_symbol) except ValueError: self.write_log(f"{vt_symbol}解析失败,请检查交易所后缀") self.thread = None return req = HistoryRequest( symbol=symbol, exchange=exchange, interval=Interval(interval), start=start, end=end ) contract = self.main_engine.get_contract(vt_symbol) try: # If history data provided in gateway, then query if contract and contract.history_data: data = self.main_engine.query_history( req, contract.gateway_name ) if data: database_manager.save_bar_data(data) self.write_log(f"{vt_symbol}-{interval}历史数据下载完成") else: self.write_log(f"数据下载失败,无法获取{vt_symbol}的历史数据") except Exception: msg = f"数据下载失败,触发异常:\n{traceback.format_exc()}" self.write_log(msg) # Clear thread object handler. self.thread = None
def run(self): """""" while self.active: try: task = self.queue.get(timeout=1) task_type, data = task if task_type == "tick": database_manager.save_tick_data([data]) elif task_type == "bar": database_manager.save_bar_data([data]) except Empty: continue except Exception: self.active = False info = sys.exc_info() event = Event(EVENT_RECORDER_EXCEPTION, info) self.event_engine.put(event)
def download_bar_data(self, symbol: str, exchange: Exchange, interval: str, start: datetime) -> int: """ Query bar data from RQData. """ req = HistoryRequest(symbol=symbol, exchange=exchange, interval=Interval(interval), start=start, end=datetime.now()) vt_symbol = f"{symbol}.{exchange.value}" contract = self.main_engine.get_contract(vt_symbol) # If history data provided in gateway, then query if contract and contract.history_data: data = self.main_engine.query_history(req, contract.gateway_name) if data: database_manager.save_bar_data(data) return (len(data)) return 0
def get_binance_data(symbol: str, exchanges: str, start_time: str, end_time: str): """ 爬取币安交易所的数据 :param symbol: BTCUSDT. :param exchanges: 现货、USDT合约, 或者币币合约. :param start_time: 格式如下:2020-1-1 或者2020-01-01 :param end_time: 格式如下:2020-1-1 或者2020-01-01 :return: """ api_url = '' save_symbol = symbol gate_way = 'BINANCES' if exchanges == 'spot': print("spot") limit = BINANCE_SPOT_LIMIT save_symbol = symbol.lower() gate_way = 'BINANCE' api_url = f'https://api.binance.com/api/v3/klines?symbol={symbol}&interval=1m&limit={limit}' elif exchanges == 'future': print('future') limit = BINANCE_FUTURE_LIMIT api_url = f'https://fapi.binance.com/fapi/v1/klines?symbol={symbol}&interval=1m&limit={limit}' elif exchanges == 'coin_future': print("coin_future") limit = BINANCE_FUTURE_LIMIT f'https://dapi.binance.com/dapi/v1/klines?symbol={symbol}&interval=1m&limit={limit}' else: raise Exception('交易所名称请输入以下其中一个:spot, future, coin_future') start_time = int(datetime.strptime(start_time, '%Y-%m-%d').timestamp() * 1000) end_time = int(datetime.strptime(end_time, '%Y-%m-%d').timestamp() * 1000) while True: try: print(start_time) url = f'{api_url}&startTime={start_time}' print(url) data = requests.get(url=url, timeout=10, proxies=proxies).json() """ [ [ 1591258320000, // 开盘时间 "9640.7", // 开盘价 "9642.4", // 最高价 "9640.6", // 最低价 "9642.0", // 收盘价(当前K线未结束的即为最新价) "206", // 成交量 1591258379999, // 收盘时间 "2.13660389", // 成交额(标的数量) 48, // 成交笔数 "119", // 主动买入成交量 "1.23424865", // 主动买入成交额(标的数量) "0" // 请忽略该参数 ] """ buf = [] for l in data: bar = BarData( symbol=save_symbol, exchange=Exchange.BINANCE, datetime=generate_datetime(l[0]), interval=Interval.MINUTE, volume=float(l[5]), open_price=float(l[1]), high_price=float(l[2]), low_price=float(l[3]), close_price=float(l[4]), gateway_name=gate_way ) buf.append(bar) database_manager.save_bar_data(buf) # 到结束时间就退出, 后者收盘价大于当前的时间. if (data[-1][0] > end_time) or data[-1][6] >= (int(time.time() * 1000) - 60 * 1000): break start_time = data[-1][0] except Exception as error: print(error) time.sleep(10)