def get_list(type_=''):
"""
return data list or dataframe depends on given type_
"""
type_list = ['stock', 'index', 'etf', 'future']
'''todo: option list'''
type_ = type_.split('_')
frequence = type_[1]
if type_[0] in type_list:
if type_[0] == 'future':
lst = QA_fetch_get_future_list()
lst = lst if frequence == 'list' else lst.code.unique().tolist()
if 'all' not in type_ and frequence != 'list':
lst = [item for item in lst if str(item)[-2:] in ['L8', 'L9']]
else:
lst = QA_fetch_get_stock_list(type_=type_[0])
lst = lst if frequence == 'list' else lst.code.unique().tolist()
if len(lst) > 0:
return lst, frequence
else:
return None, None
return None, None
def QA_SU_save_future_list(client=DATABASE, ui_log=None, ui_progress=None):
future_list=QA_fetch_get_future_list()
coll_future_list = client.future_list
coll_future_list.create_index("code", unique=True)
try:
coll_future_list.insert_many(QA_util_to_json_from_pandas(future_list),ordered=False)
except:
pass
def get(self):
currentlist = pd.concat([
QA_fetch_get_stock_list().assign(market='stock_cn'),
# QA_fetch_get_index_list().assign(market='index_cn'),
# QA_fetch_get_hkstock_list().assign(market='stock_hk'),
QA_fetch_get_future_list().assign(market='future_cn')], sort=False)
data = (currentlist.code + '/' + currentlist.name + '/' + currentlist.market).tolist()
self.write({'result': data})
def get(self):
bond_list = QA_fetch_get_bond_list() #.assign(market='bond_cn'),
ts_bond_list = QA_fetch_bond_list_adv(DATABASE.bond_list_ts)
cbond = list(ts_bond_list[ts_bond_list.list_date > '2002-01-01'].code)
missing_cb_l = [
cb for cb in cbond if cb[:6] not in list(bond_list.code)
]
missing_cb = ts_bond_list[ts_bond_list.code.isin(missing_cb_l)][[
'code', 'name', 'sse'
]].drop_duplicates()
missing_cb.code = missing_cb.code.apply(lambda x: x[:6])
currentlist = pd.concat(
[
QA_fetch_get_stock_list().assign(market='stock_cn'),
QA_fetch_get_bond_list().assign(market='bond_cn'),
missing_cb.assign(market='bond_cn'),
QA_fetch_get_index_list().assign(market='index_cn'),
# QA_fetch_get_hkstock_list().assign(market='stock_hk'),
QA_fetch_get_future_list().assign(market='future_cn')
],
sort=False)
data = (currentlist.code + '/' + currentlist.name + '/' +
currentlist.market).tolist()
self.write({'result': data})
def QA_SU_save_future_min(client=DATABASE, ui_log=None, ui_progress=None):
"""save future_min
Keyword Arguments:
client {[type]} -- [description] (default: {DATABASE})
"""
future_list = [
item for item in QA_fetch_get_future_list().code.unique().tolist()
if str(item)[-2:] in ['L8', 'L9']
]
coll = client.future_min
coll.create_index([('code', pymongo.ASCENDING),
('time_stamp', pymongo.ASCENDING),
('date_stamp', pymongo.ASCENDING)])
err = []
def __saving_work(code, coll):
QA_util_log_info('##JOB13 Now Saving Future_MIN ==== {}'.format(
str(code)),
ui_log=ui_log)
try:
for type in ['1min', '5min', '15min', '30min', '60min']:
ref_ = coll.find({'code': str(code)[0:6], 'type': type})
end_time = str(now_time())[0:19]
if ref_.count() > 0:
start_time = ref_[ref_.count() - 1]['datetime']
QA_util_log_info(
'##JOB13.{} Now Saving Future {} from {} to {} =={} '.
format(['1min', '5min',
'15min', '30min', '60min'].index(type),
str(code), start_time, end_time, type),
ui_log=ui_log)
if start_time != end_time:
__data = QA_fetch_get_future_min(
str(code), start_time, end_time, type)
if len(__data) > 1:
coll.insert_many(
QA_util_to_json_from_pandas(__data[1::]))
else:
start_time = '2015-01-01'
QA_util_log_info(
'##JOB13.{} Now Saving Future {} from {} to {} =={} '.
format(['1min', '5min',
'15min', '30min', '60min'].index(type),
str(code), start_time, end_time, type),
ui_log=ui_log)
if start_time != end_time:
__data = QA_fetch_get_future_min(
str(code), start_time, end_time, type)
if len(__data) > 1:
coll.insert_many(
QA_util_to_json_from_pandas(__data))
except:
err.append(code)
executor = ThreadPoolExecutor(max_workers=4)
res = {
executor.submit(__saving_work, future_list[i_], coll)
for i_ in range(len(future_list))
} # multi index ./.
count = 0
for i_ in concurrent.futures.as_completed(res):
QA_util_log_info('The {} of Total {}'.format(count, len(future_list)),
ui_log=ui_log)
strLogProgress = 'DOWNLOAD PROGRESS {} '.format(
str(float(count / len(future_list) * 100))[0:4] + '%')
intLogProgress = int(float(count / len(future_list) * 10000.0))
QA_util_log_info(strLogProgress,
ui_log=ui_log,
ui_progress=ui_progress,
ui_progress_int_value=intLogProgress)
count = count + 1
if len(err) < 1:
QA_util_log_info('SUCCESS', ui_log=ui_log)
else:
QA_util_log_info(' ERROR CODE \n ', ui_log=ui_log)
QA_util_log_info(err, ui_log=ui_log)
def QA_SU_save_future_day(client=DATABASE, ui_log=None, ui_progress=None):
'''
save future_day
保存日线数据
:param client:
:param ui_log: 给GUI qt 界面使用
:param ui_progress: 给GUI qt 界面使用
:param ui_progress_int_value: 给GUI qt 界面使用
:return:
'''
future_list = [
item for item in QA_fetch_get_future_list().code.unique().tolist()
if str(item)[-2:] in ['L8', 'L9']
]
coll_future_day = client.future_day
coll_future_day.create_index([("code", pymongo.ASCENDING),
("date_stamp", pymongo.ASCENDING)])
err = []
def __saving_work(code, coll_future_day):
try:
QA_util_log_info(
'##JOB12 Now Saving Future_DAY==== {}'.format(str(code)),
ui_log)
# 首选查找数据库 是否 有 这个代码的数据
ref = coll_future_day.find({'code': str(code)[0:4]})
end_date = str(now_time())[0:10]
# 当前数据库已经包含了这个代码的数据， 继续增量更新
# 加入这个判断的原因是因为如果股票是刚上市的 数据库会没有数据 所以会有负索引问题出现
if ref.count() > 0:
# 接着上次获取的日期继续更新
start_date = ref[ref.count() - 1]['date']
QA_util_log_info(
'UPDATE_Future_DAY \n Trying updating {} from {} to {}'.
format(code, start_date, end_date), ui_log)
if start_date != end_date:
coll_future_day.insert_many(
QA_util_to_json_from_pandas(
QA_fetch_get_future_day(
str(code), QA_util_get_next_day(start_date),
end_date)))
# 当前数据库中没有这个代码的股票数据， 从1990-01-01 开始下载所有的数据
else:
start_date = '2001-01-01'
QA_util_log_info(
'UPDATE_Future_DAY \n Trying updating {} from {} to {}'.
format(code, start_date, end_date), ui_log)
if start_date != end_date:
coll_future_day.insert_many(
QA_util_to_json_from_pandas(
QA_fetch_get_future_day(str(code), start_date,
end_date)))
except Exception as error0:
print(error0)
err.append(str(code))
for item in range(len(future_list)):
QA_util_log_info('The {} of Total {}'.format(item, len(future_list)))
strProgressToLog = 'DOWNLOAD PROGRESS {} {}'.format(
str(float(item / len(future_list) * 100))[0:4] + '%', ui_log)
intProgressToLog = int(float(item / len(future_list) * 100))
QA_util_log_info(strProgressToLog,
ui_log=ui_log,
ui_progress=ui_progress,
ui_progress_int_value=intProgressToLog)
__saving_work(future_list[item], coll_future_day)
if len(err) < 1:
QA_util_log_info('SUCCESS save future day ^_^', ui_log)
else:
QA_util_log_info(' ERROR CODE \n ', ui_log)
QA_util_log_info(err, ui_log)