def get_bar(timeout=1, sleep=1):
sleep = int(sleep)
_time1 = datetime.datetime.now()
from QUANTAXIS.QAFetch.QAQuery_Advance import QA_fetch_stock_block_adv
code = QA_fetch_stock_block_adv().code
print(len(code))
x = QA_Tdx_Executor(timeout=float(timeout))
print(x._queue.qsize())
print(x.get_available())
while True:
_time = datetime.datetime.now()
if QA_util_if_tradetime(_time): # 如果在交易时间
data = x.get_security_bar_concurrent(code, 'day', 1)
print('Time {}'.format(
(datetime.datetime.now() - _time).total_seconds()))
time.sleep(sleep)
print('Connection Pool NOW LEFT {} Available IP'.format(
x._queue.qsize()))
print('Program Last Time {}'.format(
(datetime.datetime.now() - _time1).total_seconds()))
return data
else:
print('Not Trading time {}'.format(_time))
time.sleep(sleep)
def run(self):
"""generator driven data flow
"""
# 如果出现了日期的改变 才会进行结算的事件
_date = None
while QA_util_if_tradetime(self.now):
for data in self.ingest_data: # 对于在ingest_data中的数据
# <class 'QUANTAXIS.QAData.QADataStruct.QA_DataStruct_Stock_day'>
date = data.date[0]
if self.market_type is MARKET_TYPE.STOCK_CN: # 如果是股票市场
if _date != date: # 如果新的date
# 前一天的交易日已经过去
# 往 broker 和 account 发送 settle 事件
try:
self.market.trade_engine.join()
# time.sleep(2)
self.market._settle(self.broker_name)
except Exception as e:
raise e
# 基金 指数 期货
elif self.market_type in [MARKET_TYPE.FUND_CN, MARKET_TYPE.INDEX_CN, MARKET_TYPE.FUTURE_CN]:
self.market._settle(self.broker_name)
# print(data)
self.broker.run(
QA_Event(event_type=ENGINE_EVENT.UPCOMING_DATA, market_data=data))
# 生成 UPCOMING_DATA 事件放到 队列中去执行
self.market.upcoming_data(self.broker_name, data)
self.market.trade_engine.join()
_date = date
def run(self):
sleep = int(self.sleep_time)
_time1 = datetime.datetime.now()
database = DATABASE.get_collection('realtime_{}'.format(
datetime.date.today()))
database.create_index([('code', QA_util_sql_mongo_sort_ASCENDING)])
database.create_index([('datetime', QA_util_sql_mongo_sort_ASCENDING)])
from QUANTAXIS.QAFetch.QAQuery_Advance import QA_fetch_stock_block_adv
code = QA_fetch_stock_block_adv().code
while True:
_time = datetime.datetime.now()
if QA_util_if_tradetime(_time): # 如果在交易时间
data = x.get_realtime_concurrent(code)
data[0]['datetime'] = data[1]
x.save_mongo(data[0])
print('Time {}'.format(
(datetime.datetime.now() - _time).total_seconds()))
time.sleep(sleep)
print('Connection Pool NOW LEFT {} Available IP'.format(
x._queue.qsize()))
print('Program Last Time {}'.format(
(datetime.datetime.now() - _time1).total_seconds()))
else:
print('Not Trading time {}'.format(_time))
time.sleep(sleep)
def bat(timeout):
_time1 = datetime.datetime.now()
from QUANTAXIS.QAFetch.QAQuery_Advance import QA_fetch_stock_block_adv
code = QA_fetch_stock_block_adv().code
print(len(code))
x = QA_Tdx_Executor(timeout=timeout)
print(x._queue.qsize())
print(x.get_available())
database = DATABASE.get_collection('realtime_{}'.format(
datetime.date.today()))
print(database)
database.create_index([('code', QA_util_sql_mongo_sort_ASCENDING),
('datetime', QA_util_sql_mongo_sort_ASCENDING)])
for i in range(100000):
_time = datetime.datetime.now()
if QA_util_if_tradetime(_time): # 如果在交易时间
data = x.get_realtime_concurrent(code)
data[0]['datetime'] = data[1]
x.save_mongo(data[0])
print('Time {}'.format(
(datetime.datetime.now() - _time).total_seconds()))
time.sleep(1)
print('Connection Pool NOW LEFT {} Available IP'.format(
x._queue.qsize()))
print('Program Last Time {}'.format(
(datetime.datetime.now() - _time1).total_seconds()))
else:
print('Not Trading time {}'.format(_time))
time.sleep(1)
def main():
df = down_market_snapshot()
print(df.shape)
print(df.head())
if QA_util_if_tradetime() or debug:
df1 = df.loc[df['code'].isin(pg_util.get_instere_stock()), columns]
print(df1.to_html())
mail_util.send_mail_table(df1)
"""
return ''
quotation = easyquotation.use('sina')
easyquotation.update_stock_codes()
indexs = quotation.stocks(['sh000001', 'sz399001', 'sh000300', 'sz399006'], prefix=True)
df_indexs = pd.DataFrame.from_dict(indexs).T
print(df_indexs.to_dict())
"""
# dict = quotation.market_snapshot(prefix=True)
# df = pd.DataFrame.from_dict(dict).T
# print(df[df.index == 'sh000001'])
# print(type(dict))
pass
else:
QA.QA_util_log_info('非交易时间')
print(QA.__version__)
print('hello world')
def down_market_snapshot(save=True):
"""
每半小时获取下市场快照
:return: 市场快照
"""
if QA_util_if_tradetime():
date = datetime.datetime.now()
dateStr = date.strftime('%Y-%m-%d_%H-%M')
quotation = easyquotation.use('sina')
snapshot = quotation.market_snapshot(prefix=True)
# market_snapshot = MarketSnapshot(date=date, context=snapshot)
# market_snapshot.save()
df_snapshot = pd.DataFrame.from_dict(snapshot)
df_snapshot = df_snapshot.T
df_snapshot.index.name = 'code'
if save:
df_snapshot.to_csv(ls.LocalSetting.data_path + "mark_snapshot/" +
dateStr + ".csv",
compression='gzip')
return df_snapshot
else:
return pd.read_csv(
'/Users/momantang/work/cobrass/data/mark_snapshot/2018-11-09_15-05.csv',
compression='gzip')
def test_QA_SU_save_stock_day_lastdate(self):
# 交易时间QA.QAFetch.QATdx.QA_fetch_get_stock_day返回的数据会实时更新
codelist = QA.QA_fetch_stock_list_adv().code.tolist()
days = 300
start = datetime.datetime.now() - datetime.timedelta(days)
end = datetime.datetime.now()
if QA_util_if_tradetime(end):
end2 = end - datetime.timedelta(1)
else:
end2 = end
codeListCount = 200
a = time.time()
# ps = Parallelism(cpu_count())
data1 = QA.QAFetch.QATdx.QA_fetch_get_stock_day(
codelist[0], start, end)
data2 = QA.QAFetch.QATdx.QA_fetch_get_stock_day(
codelist[0], start, end2)
# 交易时间段
self.assertTrue(
len(data1) == len(data2),
'数据长度：{} {} 日期： {} {}'.format(len(data1), len(data2), end, end2))
print('数据长度：{} {} 日期： {} {}'.format(len(data1), len(data2), end, end2))
print('当前数据： {} {}'.format(data1.close[-1], data2.close[-1]))
# 设定在交易时间内
end = datetime.datetime(end.year, end.month, end.day, 10, 0)
end2 = end
while not QA_util_if_tradetime(end):
end = end - datetime.timedelta(1)
end = end - datetime.timedelta(7)
codeListCount = 200
a = time.time()
# ps = Parallelism(cpu_count())
data1 = QA.QAFetch.QATdx.QA_fetch_get_stock_day(
codelist[0], start, end)
data2 = QA.QAFetch.QATdx.QA_fetch_get_stock_day(
codelist[0], start, end2)
# 交易时间段
self.assertFalse(
len(data1) == len(data2),
'数据长度应不等：{} {} 日期： {} {}'.format(len(data1), len(data2), end,
end2))
print('数据长度：{} {} 日期： {} {}'.format(len(data1), len(data2), end, end2))
print('当前数据： {} {}'.format(data1.close[-1], data2.close[-1]))
def bat(timeout=0.2, sleep=1, thread=2, size=80):
sleep = int(sleep)
_time1 = datetime.datetime.now()
from QUANTAXIS.QAFetch.QAQuery_Advance import QA_fetch_stock_block_adv
code = QA_fetch_stock_block_adv().code
print(len(code))
x = QA_Tdx_Executor(timeout=float(timeout),
thread_num=int(thread),
max_size=int(size))
print(x._queue.qsize())
print(x.get_available())
database = DATABASE.get_collection('realtime_{}'.format(
datetime.date.today()))
print(database)
# mongodb在排序时，复合索引并不能像关系型数据库一样提升效率，反而会降低排序性能
# 在测试的过程，发现datetime倒序索引在倒序查询时性能更高，但写入性能较低
# 此表属于写多读少，所以索引继续采用正序
database.create_index([('code', QA_util_sql_mongo_sort_ASCENDING)])
database.create_index([('datetime', QA_util_sql_mongo_sort_ASCENDING)])
while True:
_time = datetime.datetime.now()
if QA_util_if_tradetime(_time): # 如果在交易时间
data = x.get_realtime_concurrent(code)
data[0]['datetime'] = data[1]
x.save_mongo(data[0])
print('Time {}'.format(
(datetime.datetime.now() - _time).total_seconds()))
time.sleep(sleep)
print('Connection Pool NOW LEFT {} Available IP'.format(
x._queue.qsize()))
print('Program Last Time {}'.format(
(datetime.datetime.now() - _time1).total_seconds()))
else:
print('Not Trading time {}'.format(_time))
time.sleep(sleep)
def bat():
_time1 = datetime.datetime.now()
from QUANTAXIS.QAFetch.QAQuery_Advance import QA_fetch_stock_block_adv
code = QA_fetch_stock_block_adv().code
print(len(code))
x = QA_Tdx_Executor()
print(x._queue.qsize())
print(x.get_available())
#data = x.get_security_bars(code[0], '15min', 20)
# print(data)
# for i in range(5):
# print(x.get_realtime_concurrent(code))
for i in range(100000):
_time = datetime.datetime.now()
if QA_util_if_tradetime(_time): # 如果在交易时间
#data = x.get_realtime(code)
data = x.get_realtime_concurrent(code)
data[0]['datetime'] = data[1]
x.save_mongo(data[0])
# print(code[0])
#data = x.get_security_bars(code, '15min', 20)
# if data is not None:
print(len(data[0]))
# print(data)
print('Time {}'.format(
(datetime.datetime.now() - _time).total_seconds()))
time.sleep(1)
print('Connection Pool NOW LEFT {} Available IP'.format(
x._queue.qsize()))
print('Program Last Time {}'.format(
(datetime.datetime.now() - _time1).total_seconds()))
else:
print('Not Trading time {}'.format(_time))
time.sleep(1)
def sub_l1_from_sina():
"""
从新浪获取L1数据，3秒更新一次，建议mongodb数据库存放在企业级SSD上面
（我用Intel DC P3600 800GB SSD，锐龙 3900X，每个tick 保存时间 < 0.6s）
"""
def collections_of_today():
database = DATABASE.get_collection('realtime_{}'.format(datetime.date.today()))
database.create_index([('code', QA_util_sql_mongo_sort_ASCENDING)])
database.create_index([('datetime', QA_util_sql_mongo_sort_ASCENDING)])
database.create_index(
[
("code",
QA_util_sql_mongo_sort_ASCENDING),
("datetime",
QA_util_sql_mongo_sort_ASCENDING)
],
#unique=True,
)
return database
def save_X_func():
"""
QUANTAXIS的save X，建议每天下午四~五点收盘后执行
"""
pass
quotation = easyquotation.use('sina') # 新浪 ['sina'] 腾讯 ['tencent', 'qq']
sleep_time = 2.0
sleep = int(sleep_time)
_time1 = datetime.datetime.now()
database = collections_of_today()
get_once = True
# 开盘/收盘时间
end_time = datetime.datetime.strptime(str(datetime.datetime.now().date())+' 16:30', '%Y-%m-%d %H:%M')
start_time = datetime.datetime.strptime(str(datetime.datetime.now().date()) + ' 09:15', '%Y-%m-%d %H:%M')
day_changed_time = datetime.datetime.strptime(str(datetime.datetime.now().date()) + ' 01:00', '%Y-%m-%d %H:%M')
while (datetime.datetime.now() < end_time):
# 开盘/收盘时间
end_time = datetime.datetime.strptime(str(datetime.datetime.now().date()) + ' 16:30', '%Y-%m-%d %H:%M')
start_time = datetime.datetime.strptime(str(datetime.datetime.now().date()) + ' 09:15', '%Y-%m-%d %H:%M')
day_changed_time = datetime.datetime.strptime(str(datetime.datetime.now().date()) + ' 01:00', '%Y-%m-%d %H:%M')
_time = datetime.datetime.now()
if (datetime.datetime.now() < day_changed_time):
# 日期变更，写入表也会相应变更，这是为了防止用户永不退出一直执行
print(u'当前日期更新~！ {} '.format(datetime.date.today()))
database = collections_of_today()
if QA_util_if_tradetime(_time) or \
(get_once): # 如果在交易时间
l1_ticks = quotation.market_snapshot(prefix=False)
l1_ticks_data = formater_l1_ticks(l1_ticks)
if (datetime.datetime.now() < start_time):
print(u'Not Trading time 现在是中国A股收盘时间 {}'.format(_time))
timer.sleep(sleep)
continue
# 查询是否新 tick
query_id = {
"code": [l1_tick['code'] for l1_tick in l1_ticks_data],
"datetime": {
'$in': [l1_tick['datetime'] for l1_tick in l1_ticks_data]
}
}
refcount = database.count_documents(query_id)
if refcount > 0:
if (len(l1_ticks_data) > 1):
# 删掉重复数据
database.delete_many(query_id)
database.insert_many(l1_ticks_data)
else:
# 持续更新模式，更新单条记录
database.replace_one(query_id, l1_ticks_data[0])
else:
# 新 tick，插入记录
database.insert_many(l1_ticks_data)
if (get_once != True):
print(u'Trading time now 现在是中国A股交易时间 {}\nProcessing ticks data cost:{:.3f}s'.format(datetime.datetime.now(),
(datetime.datetime.now() - _time).total_seconds()))
timer.sleep(sleep)
print('Program Last Time {:.3f}s'.format(
(datetime.datetime.now() - _time1).total_seconds()))
get_once = False
else:
print(u'Not Trading time 现在是中国A股收盘时间 {}'.format(_time))
timer.sleep(sleep)
# 每天下午5点，代码就会执行到这里，如有必要，再次执行收盘行情下载，也就是 QUANTAXIS/save X
save_time = datetime.datetime.strptime(str(datetime.datetime.now().date()) + ' 17:00', '%Y-%m-%d %H:%M')
if (datetime.datetime.now() > end_time) and \
(datetime.datetime.now() < save_time):
# 收盘时间 下午16:00到17:00 更新收盘数据
QA_SU_save_stock_day('tdx')
QA_SU_save_stock_xdxr('tdx')
QA_SU_save_stock_min('tdx')
QA_SU_save_index_day('tdx')
QA_SU_save_index_min('tdx')
QA_SU_save_etf_list('tdx')
QA_SU_save_etf_day('tdx')
QA_SU_save_etf_min('tdx')
QA_SU_save_stock_list('tdx')
QA_SU_save_index_list('tdx')
QA_SU_save_stock_block('tdx')
QA_SU_save_future_list('tdx')
# While循环每天下午5点自动结束，在此等待13小时，大概早上六点结束程序自动重启
print(u'While循环每天下午5点自动结束，在此等待13小时，大概早上六点结束程序自动重启，这样只要窗口不关，永远每天自动收取 tick')
timer.sleep(30000)
import time
_time1 = datetime.datetime.now()
from QUANTAXIS.QAFetch.QAQuery_Advance import QA_fetch_stock_block_adv
code = QA_fetch_stock_block_adv().code
print(len(code))
x = QA_Tdx_Executor()
print(x._queue.qsize())
print(x.get_available())
#data = x.get_security_bars(code[0], '15min', 20)
# print(data)
# for i in range(5):
# print(x.get_realtime_concurrent(code))
for i in range(100000):
_time = datetime.datetime.now()
if QA_util_if_tradetime(_time): # 如果在交易时间
#data = x.get_realtime(code)
data = x.get_realtime_concurrent(code)
data[0]['datetime'] = data[1]
x.save_mongo(data[0])
# print(code[0])
#data = x.get_security_bars(code, '15min', 20)
# if data is not None:
print(len(data[0]))
# print(data)
print('Time {}'.format(
(datetime.datetime.now() - _time).total_seconds()))
time.sleep(1)
print('Connection Pool NOW LEFT {} Available IP'.format(
x._queue.qsize()))
def sub_codelist_l1_from_sina(codelist: list = None):
"""
从新浪获取L1数据，3秒更新一次，建议mongodb数据库存放在企业级SSD上面
（我用Intel DC P3600 800GB SSD，锐龙 3600，每个tick 保存时间 < 0.6s）
"""
def collections_of_today():
database = DATABASE.get_collection('realtime_{}'.format(
datetime.date.today()))
database.create_index([('code', QA_util_sql_mongo_sort_ASCENDING)])
database.create_index([('datetime', QA_util_sql_mongo_sort_ASCENDING)])
database.create_index([("code", QA_util_sql_mongo_sort_ASCENDING),
("datetime", QA_util_sql_mongo_sort_ASCENDING)],
#unique=True,
)
return database
quotation = easyquotation.use('sina') # 新浪 ['sina'] 腾讯 ['tencent', 'qq']
sleep_time = 2.0
sleep = int(sleep_time)
_time1 = dt.now()
database = collections_of_today()
get_once = True
# 开盘/收盘时间
end_time = dt.strptime(str(dt.now().date()) + ' 16:30', '%Y-%m-%d %H:%M')
start_time = dt.strptime(str(dt.now().date()) + ' 09:15', '%Y-%m-%d %H:%M')
day_changed_time = dt.strptime(
str(dt.now().date()) + ' 01:00', '%Y-%m-%d %H:%M')
while (dt.now() < end_time):
# 开盘/收盘时间
end_time = dt.strptime(
str(dt.now().date()) + ' 16:30', '%Y-%m-%d %H:%M')
start_time = dt.strptime(
str(dt.now().date()) + ' 09:15', '%Y-%m-%d %H:%M')
day_changed_time = dt.strptime(
str(dt.now().date()) + ' 01:00', '%Y-%m-%d %H:%M')
_time = dt.now()
if QA_util_if_tradetime(_time) and \
(dt.now() < day_changed_time):
# 日期变更，写入表也会相应变更，这是为了防止用户永不退出一直执行
print(u'当前日期更新~！ {} '.format(datetime.date.today()))
database = collections_of_today()
print(u'Not Trading time 现在是中国A股收盘时间 {}'.format(_time))
timer.sleep(sleep)
continue
if QA_util_if_tradetime(_time) or \
(get_once): # 如果在交易时间
l1_ticks = quotation.market_snapshot(prefix=True)
l1_ticks_data, symbol_list = formater_l1_ticks(l1_ticks,
codelist=codelist)
if (dt.now() < start_time) or \
((len(l1_ticks_data) > 0) and \
(dt.strptime(l1_ticks_data[-1]['datetime'],
'%Y-%m-%d %H:%M:%S') < dt.strptime(str(dt.now().date()) + ' 00:00',
'%Y-%m-%d %H:%M'))):
print(u'Not Trading time 现在是中国A股收盘时间 {}'.format(_time))
timer.sleep(sleep)
continue
# 获取第二遍，包含上证指数信息
l1_ticks = quotation.market_snapshot(prefix=False)
l1_ticks_data, symbol_list = formater_l1_ticks(
l1_ticks,
codelist=codelist,
stacks=l1_ticks_data,
symbol_list=symbol_list)
# 查询是否新 tick
query_id = {
"code": {
'$in':
list(set([l1_tick['code'] for l1_tick in l1_ticks_data]))
},
"datetime":
sorted(
list(
set([l1_tick['datetime']
for l1_tick in l1_ticks_data])))[-1]
}
#print(symbol_list, len(symbol_list))
refcount = database.count_documents(query_id)
if refcount > 0:
if (len(l1_ticks_data) > 1):
# 删掉重复数据
database.delete_many(query_id)
database.insert_many(l1_ticks_data)
else:
# 持续更新模式，更新单条记录
database.replace_one(query_id, l1_ticks_data[0])
else:
# 新 tick，插入记录
database.insert_many(l1_ticks_data)
if (get_once != True):
print(
u'Trading time now 现在是中国A股交易时间 {}\nProcessing ticks data cost:{:.3f}s'
.format(dt.now(), (dt.now() - _time).total_seconds()))
timer.sleep(sleep)
print('Program Last Time {:.3f}s'.format(
(dt.now() - _time1).total_seconds()))
get_once = False
else:
print(u'Not Trading time 现在是中国A股收盘时间 {}'.format(_time))
timer.sleep(sleep)
# 每天下午5点，代码就会执行到这里，如有必要，再次执行收盘行情下载，也就是 QUANTAXIS/save X
save_time = dt.strptime(str(dt.now().date()) + ' 17:00', '%Y-%m-%d %H:%M')
if (dt.now() > end_time) and \
(dt.now() < save_time):
# 收盘时间 下午16:00到17:00 更新收盘数据
# 我不建议整合，因为少数情况会出现 程序执行阻塞 block，
# 本进程被阻塞后无人干预第二天影响实盘行情接收。
# save_X_func()
pass
# While循环每天下午5点自动结束，在此等待13小时，大概早上六点结束程序自动重启
print(u'While循环每天下午5点自动结束，在此等待13小时，大概早上六点结束程序自动重启，这样只要窗口不关，永远每天自动收取 tick')
timer.sleep(40000)
