def get_bar(timeout=1, sleep=1): sleep = int(sleep) _time1 = datetime.datetime.now() from QUANTAXIS.QAFetch.QAQuery_Advance import QA_fetch_stock_block_adv code = QA_fetch_stock_block_adv().code print(len(code)) x = QA_Tdx_Executor(timeout=float(timeout)) print(x._queue.qsize()) print(x.get_available()) while True: _time = datetime.datetime.now() if QA_util_if_tradetime(_time): # 如果在交易时间 data = x.get_security_bar_concurrent(code, 'day', 1) print('Time {}'.format( (datetime.datetime.now() - _time).total_seconds())) time.sleep(sleep) print('Connection Pool NOW LEFT {} Available IP'.format( x._queue.qsize())) print('Program Last Time {}'.format( (datetime.datetime.now() - _time1).total_seconds())) return data else: print('Not Trading time {}'.format(_time)) time.sleep(sleep)
def run(self): """generator driven data flow """ # 如果出现了日期的改变 才会进行结算的事件 _date = None while QA_util_if_tradetime(self.now): for data in self.ingest_data: # 对于在ingest_data中的数据 # <class 'QUANTAXIS.QAData.QADataStruct.QA_DataStruct_Stock_day'> date = data.date[0] if self.market_type is MARKET_TYPE.STOCK_CN: # 如果是股票市场 if _date != date: # 如果新的date # 前一天的交易日已经过去 # 往 broker 和 account 发送 settle 事件 try: self.market.trade_engine.join() # time.sleep(2) self.market._settle(self.broker_name) except Exception as e: raise e # 基金 指数 期货 elif self.market_type in [MARKET_TYPE.FUND_CN, MARKET_TYPE.INDEX_CN, MARKET_TYPE.FUTURE_CN]: self.market._settle(self.broker_name) # print(data) self.broker.run( QA_Event(event_type=ENGINE_EVENT.UPCOMING_DATA, market_data=data)) # 生成 UPCOMING_DATA 事件放到 队列中去执行 self.market.upcoming_data(self.broker_name, data) self.market.trade_engine.join() _date = date
def run(self): sleep = int(self.sleep_time) _time1 = datetime.datetime.now() database = DATABASE.get_collection('realtime_{}'.format( datetime.date.today())) database.create_index([('code', QA_util_sql_mongo_sort_ASCENDING)]) database.create_index([('datetime', QA_util_sql_mongo_sort_ASCENDING)]) from QUANTAXIS.QAFetch.QAQuery_Advance import QA_fetch_stock_block_adv code = QA_fetch_stock_block_adv().code while True: _time = datetime.datetime.now() if QA_util_if_tradetime(_time): # 如果在交易时间 data = x.get_realtime_concurrent(code) data[0]['datetime'] = data[1] x.save_mongo(data[0]) print('Time {}'.format( (datetime.datetime.now() - _time).total_seconds())) time.sleep(sleep) print('Connection Pool NOW LEFT {} Available IP'.format( x._queue.qsize())) print('Program Last Time {}'.format( (datetime.datetime.now() - _time1).total_seconds())) else: print('Not Trading time {}'.format(_time)) time.sleep(sleep)
def bat(timeout): _time1 = datetime.datetime.now() from QUANTAXIS.QAFetch.QAQuery_Advance import QA_fetch_stock_block_adv code = QA_fetch_stock_block_adv().code print(len(code)) x = QA_Tdx_Executor(timeout=timeout) print(x._queue.qsize()) print(x.get_available()) database = DATABASE.get_collection('realtime_{}'.format( datetime.date.today())) print(database) database.create_index([('code', QA_util_sql_mongo_sort_ASCENDING), ('datetime', QA_util_sql_mongo_sort_ASCENDING)]) for i in range(100000): _time = datetime.datetime.now() if QA_util_if_tradetime(_time): # 如果在交易时间 data = x.get_realtime_concurrent(code) data[0]['datetime'] = data[1] x.save_mongo(data[0]) print('Time {}'.format( (datetime.datetime.now() - _time).total_seconds())) time.sleep(1) print('Connection Pool NOW LEFT {} Available IP'.format( x._queue.qsize())) print('Program Last Time {}'.format( (datetime.datetime.now() - _time1).total_seconds())) else: print('Not Trading time {}'.format(_time)) time.sleep(1)
def main(): df = down_market_snapshot() print(df.shape) print(df.head()) if QA_util_if_tradetime() or debug: df1 = df.loc[df['code'].isin(pg_util.get_instere_stock()), columns] print(df1.to_html()) mail_util.send_mail_table(df1) """ return '' quotation = easyquotation.use('sina') easyquotation.update_stock_codes() indexs = quotation.stocks(['sh000001', 'sz399001', 'sh000300', 'sz399006'], prefix=True) df_indexs = pd.DataFrame.from_dict(indexs).T print(df_indexs.to_dict()) """ # dict = quotation.market_snapshot(prefix=True) # df = pd.DataFrame.from_dict(dict).T # print(df[df.index == 'sh000001']) # print(type(dict)) pass else: QA.QA_util_log_info('非交易时间') print(QA.__version__) print('hello world')
def down_market_snapshot(save=True): """ 每半小时获取下市场快照 :return: 市场快照 """ if QA_util_if_tradetime(): date = datetime.datetime.now() dateStr = date.strftime('%Y-%m-%d_%H-%M') quotation = easyquotation.use('sina') snapshot = quotation.market_snapshot(prefix=True) # market_snapshot = MarketSnapshot(date=date, context=snapshot) # market_snapshot.save() df_snapshot = pd.DataFrame.from_dict(snapshot) df_snapshot = df_snapshot.T df_snapshot.index.name = 'code' if save: df_snapshot.to_csv(ls.LocalSetting.data_path + "mark_snapshot/" + dateStr + ".csv", compression='gzip') return df_snapshot else: return pd.read_csv( '/Users/momantang/work/cobrass/data/mark_snapshot/2018-11-09_15-05.csv', compression='gzip')
def test_QA_SU_save_stock_day_lastdate(self): # 交易时间QA.QAFetch.QATdx.QA_fetch_get_stock_day返回的数据会实时更新 codelist = QA.QA_fetch_stock_list_adv().code.tolist() days = 300 start = datetime.datetime.now() - datetime.timedelta(days) end = datetime.datetime.now() if QA_util_if_tradetime(end): end2 = end - datetime.timedelta(1) else: end2 = end codeListCount = 200 a = time.time() # ps = Parallelism(cpu_count()) data1 = QA.QAFetch.QATdx.QA_fetch_get_stock_day( codelist[0], start, end) data2 = QA.QAFetch.QATdx.QA_fetch_get_stock_day( codelist[0], start, end2) # 交易时间段 self.assertTrue( len(data1) == len(data2), '数据长度:{} {} 日期: {} {}'.format(len(data1), len(data2), end, end2)) print('数据长度:{} {} 日期: {} {}'.format(len(data1), len(data2), end, end2)) print('当前数据: {} {}'.format(data1.close[-1], data2.close[-1])) # 设定在交易时间内 end = datetime.datetime(end.year, end.month, end.day, 10, 0) end2 = end while not QA_util_if_tradetime(end): end = end - datetime.timedelta(1) end = end - datetime.timedelta(7) codeListCount = 200 a = time.time() # ps = Parallelism(cpu_count()) data1 = QA.QAFetch.QATdx.QA_fetch_get_stock_day( codelist[0], start, end) data2 = QA.QAFetch.QATdx.QA_fetch_get_stock_day( codelist[0], start, end2) # 交易时间段 self.assertFalse( len(data1) == len(data2), '数据长度应不等:{} {} 日期: {} {}'.format(len(data1), len(data2), end, end2)) print('数据长度:{} {} 日期: {} {}'.format(len(data1), len(data2), end, end2)) print('当前数据: {} {}'.format(data1.close[-1], data2.close[-1]))
def bat(timeout=0.2, sleep=1, thread=2, size=80): sleep = int(sleep) _time1 = datetime.datetime.now() from QUANTAXIS.QAFetch.QAQuery_Advance import QA_fetch_stock_block_adv code = QA_fetch_stock_block_adv().code print(len(code)) x = QA_Tdx_Executor(timeout=float(timeout), thread_num=int(thread), max_size=int(size)) print(x._queue.qsize()) print(x.get_available()) database = DATABASE.get_collection('realtime_{}'.format( datetime.date.today())) print(database) # mongodb在排序时,复合索引并不能像关系型数据库一样提升效率,反而会降低排序性能 # 在测试的过程,发现datetime倒序索引在倒序查询时性能更高,但写入性能较低 # 此表属于写多读少,所以索引继续采用正序 database.create_index([('code', QA_util_sql_mongo_sort_ASCENDING)]) database.create_index([('datetime', QA_util_sql_mongo_sort_ASCENDING)]) while True: _time = datetime.datetime.now() if QA_util_if_tradetime(_time): # 如果在交易时间 data = x.get_realtime_concurrent(code) data[0]['datetime'] = data[1] x.save_mongo(data[0]) print('Time {}'.format( (datetime.datetime.now() - _time).total_seconds())) time.sleep(sleep) print('Connection Pool NOW LEFT {} Available IP'.format( x._queue.qsize())) print('Program Last Time {}'.format( (datetime.datetime.now() - _time1).total_seconds())) else: print('Not Trading time {}'.format(_time)) time.sleep(sleep)
def bat(): _time1 = datetime.datetime.now() from QUANTAXIS.QAFetch.QAQuery_Advance import QA_fetch_stock_block_adv code = QA_fetch_stock_block_adv().code print(len(code)) x = QA_Tdx_Executor() print(x._queue.qsize()) print(x.get_available()) #data = x.get_security_bars(code[0], '15min', 20) # print(data) # for i in range(5): # print(x.get_realtime_concurrent(code)) for i in range(100000): _time = datetime.datetime.now() if QA_util_if_tradetime(_time): # 如果在交易时间 #data = x.get_realtime(code) data = x.get_realtime_concurrent(code) data[0]['datetime'] = data[1] x.save_mongo(data[0]) # print(code[0]) #data = x.get_security_bars(code, '15min', 20) # if data is not None: print(len(data[0])) # print(data) print('Time {}'.format( (datetime.datetime.now() - _time).total_seconds())) time.sleep(1) print('Connection Pool NOW LEFT {} Available IP'.format( x._queue.qsize())) print('Program Last Time {}'.format( (datetime.datetime.now() - _time1).total_seconds())) else: print('Not Trading time {}'.format(_time)) time.sleep(1)
def sub_l1_from_sina(): """ 从新浪获取L1数据,3秒更新一次,建议mongodb数据库存放在企业级SSD上面 (我用Intel DC P3600 800GB SSD,锐龙 3900X,每个tick 保存时间 < 0.6s) """ def collections_of_today(): database = DATABASE.get_collection('realtime_{}'.format(datetime.date.today())) database.create_index([('code', QA_util_sql_mongo_sort_ASCENDING)]) database.create_index([('datetime', QA_util_sql_mongo_sort_ASCENDING)]) database.create_index( [ ("code", QA_util_sql_mongo_sort_ASCENDING), ("datetime", QA_util_sql_mongo_sort_ASCENDING) ], #unique=True, ) return database def save_X_func(): """ QUANTAXIS的save X,建议每天下午四~五点收盘后执行 """ pass quotation = easyquotation.use('sina') # 新浪 ['sina'] 腾讯 ['tencent', 'qq'] sleep_time = 2.0 sleep = int(sleep_time) _time1 = datetime.datetime.now() database = collections_of_today() get_once = True # 开盘/收盘时间 end_time = datetime.datetime.strptime(str(datetime.datetime.now().date())+' 16:30', '%Y-%m-%d %H:%M') start_time = datetime.datetime.strptime(str(datetime.datetime.now().date()) + ' 09:15', '%Y-%m-%d %H:%M') day_changed_time = datetime.datetime.strptime(str(datetime.datetime.now().date()) + ' 01:00', '%Y-%m-%d %H:%M') while (datetime.datetime.now() < end_time): # 开盘/收盘时间 end_time = datetime.datetime.strptime(str(datetime.datetime.now().date()) + ' 16:30', '%Y-%m-%d %H:%M') start_time = datetime.datetime.strptime(str(datetime.datetime.now().date()) + ' 09:15', '%Y-%m-%d %H:%M') day_changed_time = datetime.datetime.strptime(str(datetime.datetime.now().date()) + ' 01:00', '%Y-%m-%d %H:%M') _time = datetime.datetime.now() if (datetime.datetime.now() < day_changed_time): # 日期变更,写入表也会相应变更,这是为了防止用户永不退出一直执行 print(u'当前日期更新~! {} '.format(datetime.date.today())) database = collections_of_today() if QA_util_if_tradetime(_time) or \ (get_once): # 如果在交易时间 l1_ticks = quotation.market_snapshot(prefix=False) l1_ticks_data = formater_l1_ticks(l1_ticks) if (datetime.datetime.now() < start_time): print(u'Not Trading time 现在是中国A股收盘时间 {}'.format(_time)) timer.sleep(sleep) continue # 查询是否新 tick query_id = { "code": [l1_tick['code'] for l1_tick in l1_ticks_data], "datetime": { '$in': [l1_tick['datetime'] for l1_tick in l1_ticks_data] } } refcount = database.count_documents(query_id) if refcount > 0: if (len(l1_ticks_data) > 1): # 删掉重复数据 database.delete_many(query_id) database.insert_many(l1_ticks_data) else: # 持续更新模式,更新单条记录 database.replace_one(query_id, l1_ticks_data[0]) else: # 新 tick,插入记录 database.insert_many(l1_ticks_data) if (get_once != True): print(u'Trading time now 现在是中国A股交易时间 {}\nProcessing ticks data cost:{:.3f}s'.format(datetime.datetime.now(), (datetime.datetime.now() - _time).total_seconds())) timer.sleep(sleep) print('Program Last Time {:.3f}s'.format( (datetime.datetime.now() - _time1).total_seconds())) get_once = False else: print(u'Not Trading time 现在是中国A股收盘时间 {}'.format(_time)) timer.sleep(sleep) # 每天下午5点,代码就会执行到这里,如有必要,再次执行收盘行情下载,也就是 QUANTAXIS/save X save_time = datetime.datetime.strptime(str(datetime.datetime.now().date()) + ' 17:00', '%Y-%m-%d %H:%M') if (datetime.datetime.now() > end_time) and \ (datetime.datetime.now() < save_time): # 收盘时间 下午16:00到17:00 更新收盘数据 QA_SU_save_stock_day('tdx') QA_SU_save_stock_xdxr('tdx') QA_SU_save_stock_min('tdx') QA_SU_save_index_day('tdx') QA_SU_save_index_min('tdx') QA_SU_save_etf_list('tdx') QA_SU_save_etf_day('tdx') QA_SU_save_etf_min('tdx') QA_SU_save_stock_list('tdx') QA_SU_save_index_list('tdx') QA_SU_save_stock_block('tdx') QA_SU_save_future_list('tdx') # While循环每天下午5点自动结束,在此等待13小时,大概早上六点结束程序自动重启 print(u'While循环每天下午5点自动结束,在此等待13小时,大概早上六点结束程序自动重启,这样只要窗口不关,永远每天自动收取 tick') timer.sleep(30000)
import time _time1 = datetime.datetime.now() from QUANTAXIS.QAFetch.QAQuery_Advance import QA_fetch_stock_block_adv code = QA_fetch_stock_block_adv().code print(len(code)) x = QA_Tdx_Executor() print(x._queue.qsize()) print(x.get_available()) #data = x.get_security_bars(code[0], '15min', 20) # print(data) # for i in range(5): # print(x.get_realtime_concurrent(code)) for i in range(100000): _time = datetime.datetime.now() if QA_util_if_tradetime(_time): # 如果在交易时间 #data = x.get_realtime(code) data = x.get_realtime_concurrent(code) data[0]['datetime'] = data[1] x.save_mongo(data[0]) # print(code[0]) #data = x.get_security_bars(code, '15min', 20) # if data is not None: print(len(data[0])) # print(data) print('Time {}'.format( (datetime.datetime.now() - _time).total_seconds())) time.sleep(1) print('Connection Pool NOW LEFT {} Available IP'.format( x._queue.qsize()))
def sub_codelist_l1_from_sina(codelist: list = None): """ 从新浪获取L1数据,3秒更新一次,建议mongodb数据库存放在企业级SSD上面 (我用Intel DC P3600 800GB SSD,锐龙 3600,每个tick 保存时间 < 0.6s) """ def collections_of_today(): database = DATABASE.get_collection('realtime_{}'.format( datetime.date.today())) database.create_index([('code', QA_util_sql_mongo_sort_ASCENDING)]) database.create_index([('datetime', QA_util_sql_mongo_sort_ASCENDING)]) database.create_index([("code", QA_util_sql_mongo_sort_ASCENDING), ("datetime", QA_util_sql_mongo_sort_ASCENDING)], #unique=True, ) return database quotation = easyquotation.use('sina') # 新浪 ['sina'] 腾讯 ['tencent', 'qq'] sleep_time = 2.0 sleep = int(sleep_time) _time1 = dt.now() database = collections_of_today() get_once = True # 开盘/收盘时间 end_time = dt.strptime(str(dt.now().date()) + ' 16:30', '%Y-%m-%d %H:%M') start_time = dt.strptime(str(dt.now().date()) + ' 09:15', '%Y-%m-%d %H:%M') day_changed_time = dt.strptime( str(dt.now().date()) + ' 01:00', '%Y-%m-%d %H:%M') while (dt.now() < end_time): # 开盘/收盘时间 end_time = dt.strptime( str(dt.now().date()) + ' 16:30', '%Y-%m-%d %H:%M') start_time = dt.strptime( str(dt.now().date()) + ' 09:15', '%Y-%m-%d %H:%M') day_changed_time = dt.strptime( str(dt.now().date()) + ' 01:00', '%Y-%m-%d %H:%M') _time = dt.now() if QA_util_if_tradetime(_time) and \ (dt.now() < day_changed_time): # 日期变更,写入表也会相应变更,这是为了防止用户永不退出一直执行 print(u'当前日期更新~! {} '.format(datetime.date.today())) database = collections_of_today() print(u'Not Trading time 现在是中国A股收盘时间 {}'.format(_time)) timer.sleep(sleep) continue if QA_util_if_tradetime(_time) or \ (get_once): # 如果在交易时间 l1_ticks = quotation.market_snapshot(prefix=True) l1_ticks_data, symbol_list = formater_l1_ticks(l1_ticks, codelist=codelist) if (dt.now() < start_time) or \ ((len(l1_ticks_data) > 0) and \ (dt.strptime(l1_ticks_data[-1]['datetime'], '%Y-%m-%d %H:%M:%S') < dt.strptime(str(dt.now().date()) + ' 00:00', '%Y-%m-%d %H:%M'))): print(u'Not Trading time 现在是中国A股收盘时间 {}'.format(_time)) timer.sleep(sleep) continue # 获取第二遍,包含上证指数信息 l1_ticks = quotation.market_snapshot(prefix=False) l1_ticks_data, symbol_list = formater_l1_ticks( l1_ticks, codelist=codelist, stacks=l1_ticks_data, symbol_list=symbol_list) # 查询是否新 tick query_id = { "code": { '$in': list(set([l1_tick['code'] for l1_tick in l1_ticks_data])) }, "datetime": sorted( list( set([l1_tick['datetime'] for l1_tick in l1_ticks_data])))[-1] } #print(symbol_list, len(symbol_list)) refcount = database.count_documents(query_id) if refcount > 0: if (len(l1_ticks_data) > 1): # 删掉重复数据 database.delete_many(query_id) database.insert_many(l1_ticks_data) else: # 持续更新模式,更新单条记录 database.replace_one(query_id, l1_ticks_data[0]) else: # 新 tick,插入记录 database.insert_many(l1_ticks_data) if (get_once != True): print( u'Trading time now 现在是中国A股交易时间 {}\nProcessing ticks data cost:{:.3f}s' .format(dt.now(), (dt.now() - _time).total_seconds())) timer.sleep(sleep) print('Program Last Time {:.3f}s'.format( (dt.now() - _time1).total_seconds())) get_once = False else: print(u'Not Trading time 现在是中国A股收盘时间 {}'.format(_time)) timer.sleep(sleep) # 每天下午5点,代码就会执行到这里,如有必要,再次执行收盘行情下载,也就是 QUANTAXIS/save X save_time = dt.strptime(str(dt.now().date()) + ' 17:00', '%Y-%m-%d %H:%M') if (dt.now() > end_time) and \ (dt.now() < save_time): # 收盘时间 下午16:00到17:00 更新收盘数据 # 我不建议整合,因为少数情况会出现 程序执行阻塞 block, # 本进程被阻塞后无人干预第二天影响实盘行情接收。 # save_X_func() pass # While循环每天下午5点自动结束,在此等待13小时,大概早上六点结束程序自动重启 print(u'While循环每天下午5点自动结束,在此等待13小时,大概早上六点结束程序自动重启,这样只要窗口不关,永远每天自动收取 tick') timer.sleep(40000)