def QA_data_stock_to_fq(__data, type_='01'):
def __QA_fetch_stock_xdxr(code,
format_='pd',
collections=DATABASE.stock_xdxr):
'获取股票除权信息/数据库'
try:
data = pd.DataFrame([
item for item in collections.find({'code': code})
]).drop(['_id'], axis=1)
data['date'] = pd.to_datetime(data['date'])
return data.set_index(['date', 'code'], drop=False)
except:
return pd.DataFrame(data=[],
columns=[
'category', 'category_meaning', 'code',
'date', 'fenhong', 'fenshu',
'liquidity_after', 'liquidity_before',
'name', 'peigu', 'peigujia',
'shares_after', 'shares_before',
'songzhuangu', 'suogu', 'xingquanjia'
])
'股票 日线/分钟线 动态复权接口'
code = __data.index.remove_unused_levels().levels[1][0] if isinstance(
__data.index, pd.core.indexes.multi.MultiIndex) else __data['code'][0]
if type_ in ['01', 'qfq']:
return QA_data_make_qfq(__data, __QA_fetch_stock_xdxr(code))
elif type_ in ['02', 'hfq']:
return QA_data_make_hfq(__data, __QA_fetch_stock_xdxr(code))
else:
util_log_info('wrong fq type! Using qfq')
return QA_data_make_qfq(__data, __QA_fetch_stock_xdxr(code))
def __saving_work(code, coll):
try:
ref_ = coll.find({'code': str(code)[0:6]})
end_time = str(now_time())[0:10]
if ref_.count() > 0:
start_time = ref_[ref_.count() - 1]['date']
util_log_info('##JOB06 Now Saving ETF_DAY==== \n Trying updating {} from {} to {}'.format
(code, start_time, end_time), ui_log=ui_log)
if start_time != end_time:
coll.insert_many(
util_to_json_from_pandas(
fetch_get_index_day(str(code), util_get_next_day(start_time), end_time)))
else:
start_time = '1990-01-01'
util_log_info('##JOB06 Now Saving ETF_DAY==== \n Trying updating {} from {} to {}'.format
(code, start_time, end_time), ui_log=ui_log)
if start_time != end_time:
coll.insert_many(
util_to_json_from_pandas(
fetch_get_index_day(str(code), start_time, end_time)))
except:
err.append(str(code))
def SU_save_stock_block(client=STOCKDATA, ui_log=None, ui_progress=None):
"""save stock_block
Keyword Arguments:
client {[type]} -- [description] (default: {STOCKDATA})
"""
client.drop_collection('stock_block')
coll = client.stock_block
coll.create_index('code')
try:
util_log_info('##JOB09 Now Saving STOCK_BlOCK ====', ui_log=ui_log,
ui_progress=ui_progress, ui_progress_int_value=5000)
coll.insert_many(util_to_json_from_pandas(
fetch_get_stock_block('tdx')))
util_log_info('tdx Block ====', ui_log=ui_log,
ui_progress=ui_progress, ui_progress_int_value=5000)
# 🛠todo fixhere here 获取同花顺板块， 还是调用tdx的
coll.insert_many(util_to_json_from_pandas(
fetch_get_stock_block('ths')))
util_log_info('ths Block ====', ui_log=ui_log,
ui_progress=ui_progress, ui_progress_int_value=8000)
util_log_info('完成股票板块获取=', ui_log=ui_log,
ui_progress=ui_progress, ui_progress_int_value=10000)
except Exception as e:
util_log_info(e, ui_log=ui_log)
print(" Error save_tdx.SU_save_stock_block exception!")
pass
def SU_save_etf_day(client=STOCKDATA, ui_log=None, ui_progress=None):
"""save etf_day
Keyword Arguments:
client {[type]} -- [description] (default: {STOCKDATA})
"""
__index_list = fetch_get_stock_list('etf')
coll = client.index_day
coll.create_index([('code', pymongo.ASCENDING),
('date_stamp', pymongo.ASCENDING)])
err = []
def __saving_work(code, coll):
try:
ref_ = coll.find({'code': str(code)[0:6]})
end_time = str(now_time())[0:10]
if ref_.count() > 0:
start_time = ref_[ref_.count() - 1]['date']
util_log_info('##JOB06 Now Saving ETF_DAY==== \n Trying updating {} from {} to {}'.format
(code, start_time, end_time), ui_log=ui_log)
if start_time != end_time:
coll.insert_many(
util_to_json_from_pandas(
fetch_get_index_day(str(code), util_get_next_day(start_time), end_time)))
else:
start_time = '1990-01-01'
util_log_info('##JOB06 Now Saving ETF_DAY==== \n Trying updating {} from {} to {}'.format
(code, start_time, end_time), ui_log=ui_log)
if start_time != end_time:
coll.insert_many(
util_to_json_from_pandas(
fetch_get_index_day(str(code), start_time, end_time)))
except:
err.append(str(code))
for i_ in range(len(__index_list)):
# __saving_work('000001')
util_log_info('The {} of Total {}'.format(
i_, len(__index_list)), ui_log=ui_log)
strLogProgress = 'DOWNLOAD PROGRESS {} '.format(
str(float(i_ / len(__index_list) * 100))[0:4] + '%')
intLogProgress = int(float(i_ / len(__index_list) * 10000.0))
util_log_info(strLogProgress, ui_log=ui_log,
ui_progress=ui_progress, ui_progress_int_value=intLogProgress)
__saving_work(__index_list.index[i_][0], coll)
if len(err) < 1:
util_log_info('SUCCESS', ui_log=ui_log)
else:
util_log_info(' ERROR CODE \n ', ui_log=ui_log)
util_log_info(err, ui_log=ui_log)
def __saving_work(code):
util_log_info(
'##JOB11 Now Saving STOCK_TRANSACTION ==== {}'.format(str(code)), ui_log=ui_log)
try:
coll.insert_many(
util_to_json_from_pandas(
# 🛠todo str(stock_list[code]) 参数不对？
fetch_get_stock_transaction(str(code), '1990-01-01', str(now_time())[0:10])))
except:
err.append(str(code))
def __saving_work(code, coll):
util_log_info(
'##JOB010 Now Saving STOCK INFO ==== {}'.format(str(code)), ui_log=ui_log)
try:
coll.insert_many(
util_to_json_from_pandas(
fetch_get_stock_info(str(code))))
except:
err.append(str(code))
def _save_stock_data(client=STOCKDATA, stock_list=None, ui_log=None, ui_progress=None, frequence='day'):
if stock_list is None:
stock_list = fetch_get_stock_list().code.unique().tolist()
data_cli = client
err = []
def __saving_work_job(code, tb, frequence):
try:
util_log_info(
'##JOB01 Now Saving STOCK_'+ frequence +'==== {}'.format(str(code)), ui_log)
# 首选查找数据库 是否 有 这个代码的数据
tb.create_table(_sel_market_code(code)+str(code))
end_date = str(now_time())[0:10]
# 当前数据库已经包含了这个代码的数据， 继续增量更新
# 加入这个判断的原因是因为如果股票是刚上市的 数据库会没有数据 所以会有负索引问题出现
if tb.nrows > 0:
# 接着上次获取的日期继续更新
start_date = tb[-1]['datetime']
start_date=util_get_next_day(parse(str(start_date)).date().strftime('%Y-%m-%d'))
util_log_info('UPDATE_STOCK_'+ frequence+' \n Trying updating {} from {} to {}'.format(
code, start_date, end_date), ui_log)
# 当前数据库中没有这个代码的股票数据， 从1990-01-01 开始下载所有的数据
else:
start_date = '1990-01-01'
util_log_info('UPDATE_STOCK_'+frequence +'\n Trying updating {} from {} to {}'.format
(code, start_date, end_date), ui_log)
if (start_date < end_date):
__data=fetch_get_stock_day(str(code), start_date, end_date, '00', frequence)
if (__data is None)==False:
if len(__data)>0 :
tb.save_data(__data)
except Exception as error0:
if (error0.__str__() !='ERROR CODE'):
print(error0)
err.append(str(code))
for item in range(len(stock_list)):
util_log_info('The {} of Total {}'.format
(item, len(stock_list)))
strProgressToLog = 'DOWNLOAD PROGRESS {} {}'.format(
str(float(item / len(stock_list) * 100))[0:4] + '%', ui_log)
intProgressToLog = int(float(item / len(stock_list) * 100))
util_log_info(strProgressToLog, ui_log=ui_log,
ui_progress=ui_progress, ui_progress_int_value=intProgressToLog)
__saving_work_job(stock_list[item],
data_cli[_sel_market_code(stock_list[item])+frequenceDict[frequence]],frequence)
if len(err) < 1:
util_log_info('SUCCESS save stock '+ frequence +' ^_^', ui_log)
else:
util_log_info(' ERROR CODE \n ', ui_log)
util_log_info(err, ui_log)
def __saving_work(code, coll):
util_log_info('##JOB02 Now Saving XDXR INFO ==== {}'.format(
str(code)), ui_log=ui_log)
try:
coll.insert_many(
util_to_json_from_pandas(
fetch_get_stock_xdxr(str(code))), ordered=False)
except:
err.append(str(code))
def to_hfq(self):
if self.if_fq is 'bfq':
if len(self.code) < 1:
self.if_fq = 'hfq'
return self
else:
return self.new(
self.groupby(level=1).apply(data_stock_to_fq, 'hfq'),
self.type, 'hfq')
# return self.new(pd.concat(list(map(lambda x: QA_data_stock_to_fq(
# self.data[self.data['code'] == x], 'hfq'), self.code))), self.type, 'hfq')
else:
util_log_info('none support type for qfq Current type is: %s' %
self.if_fq)
return self
def to_hfq(self):
if self.if_fq is 'bfq':
if len(self.code) < 1:
self.if_fq = 'hfq'
return self
else:
return self.new(
self.groupby(level=1).apply(data_stock_to_fq, 'hfq'),
self.type, 'hfq')
# data = QA_DataStruct_Stock_min(pd.concat(list(map(lambda x: QA_data_stock_to_fq(
# self.data[self.data['code'] == x], 'hfq'), self.code))).set_index(['datetime', 'code'], drop=False))
# data.if_fq = 'hfq'
# return data
else:
util_log_info('none support type for qfq Current type is:%s' %
self.if_fq)
return self
def __saving_work_job(code, tb, frequence):
try:
util_log_info(
'##JOB01 Now Saving STOCK_'+ frequence +'==== {}'.format(str(code)), ui_log)
# 首选查找数据库 是否 有 这个代码的数据
tb.create_table(_sel_market_code(code)+str(code))
end_date = str(now_time())[0:10]
# 当前数据库已经包含了这个代码的数据， 继续增量更新
# 加入这个判断的原因是因为如果股票是刚上市的 数据库会没有数据 所以会有负索引问题出现
if tb.nrows > 0:
# 接着上次获取的日期继续更新
start_date = tb[-1]['datetime']
start_date=util_get_next_day(parse(str(start_date)).date().strftime('%Y-%m-%d'))
util_log_info('UPDATE_STOCK_'+ frequence+' \n Trying updating {} from {} to {}'.format(
code, start_date, end_date), ui_log)
# 当前数据库中没有这个代码的股票数据， 从1990-01-01 开始下载所有的数据
else:
start_date = '1990-01-01'
util_log_info('UPDATE_STOCK_'+frequence +'\n Trying updating {} from {} to {}'.format
(code, start_date, end_date), ui_log)
if (start_date < end_date):
__data=fetch_get_stock_day(str(code), start_date, end_date, '00', frequence)
if (__data is None)==False:
if len(__data)>0 :
tb.save_data(__data)
except Exception as error0:
if (error0.__str__() !='ERROR CODE'):
print(error0)
err.append(str(code))
def SU_save_stock_xdxr(client=STOCKDATA, ui_log=None, ui_progress=None):
"""[summary]
Keyword Arguments:
client {[type]} -- [description] (default: {STOCKDATA})
"""
stock_list = fetch_get_stock_list().code.unique().tolist()
#client.drop_collection('stock_xdxr')
try:
coll = client.stock_xdxr
coll.create_index([('code', pymongo.ASCENDING),
('date', pymongo.ASCENDING)], unique=True)
except:
client.drop_collection('stock_xdxr')
coll = client.stock_xdxr
coll.create_index([('code', pymongo.ASCENDING),
('date', pymongo.ASCENDING)], unique=True)
err = []
def __saving_work(code, coll):
util_log_info('##JOB02 Now Saving XDXR INFO ==== {}'.format(
str(code)), ui_log=ui_log)
try:
coll.insert_many(
util_to_json_from_pandas(
fetch_get_stock_xdxr(str(code))), ordered=False)
except:
err.append(str(code))
for i_ in range(len(stock_list)):
util_log_info('The {} of Total {}'.format(
i_, len(stock_list)), ui_log=ui_log)
strLogInfo = 'DOWNLOAD PROGRESS {} '.format(
str(float(i_ / len(stock_list) * 100))[0:4] + '%')
intLogProgress = int(float(i_ / len(stock_list) * 100))
util_log_info(strLogInfo, ui_log=ui_log,
ui_progress=ui_progress, ui_progress_int_value=intLogProgress)
__saving_work(stock_list[i_], coll)
def SU_save_stock_transaction(client=STOCKDATA, ui_log=None, ui_progress=None):
"""save stock_transaction
Keyword Arguments:
client {[type]} -- [description] (default: {STOCKDATA})
"""
stock_list = fetch_get_stock_list().code.unique().tolist()
coll = client.stock_transaction
coll.create_index('code')
err = []
def __saving_work(code):
util_log_info(
'##JOB11 Now Saving STOCK_TRANSACTION ==== {}'.format(str(code)), ui_log=ui_log)
try:
coll.insert_many(
util_to_json_from_pandas(
# 🛠todo str(stock_list[code]) 参数不对？
fetch_get_stock_transaction(str(code), '1990-01-01', str(now_time())[0:10])))
except:
err.append(str(code))
for i_ in range(len(stock_list)):
# __saving_work('000001')
util_log_info('The {} of Total {}'.format(
i_, len(stock_list)), ui_log=ui_log)
strLogProgress = 'DOWNLOAD PROGRESS {} '.format(
str(float(i_ / len(stock_list) * 100))[0:4] + '%')
intLogProgress = int(float(i_ / len(stock_list) * 10000.0))
util_log_info(strLogProgress, ui_log=ui_log,
ui_progress=ui_progress, ui_progress_int_value=intLogProgress)
__saving_work(stock_list[i_])
if len(err) < 1:
util_log_info('SUCCESS', ui_log=ui_log)
else:
util_log_info(' ERROR CODE \n ', ui_log=ui_log)
util_log_info(err, ui_log=ui_log)
def SU_save_stock_info(client=STOCKDATA, ui_log=None, ui_progress=None):
"""save stock_info
Keyword Arguments:
client {[type]} -- [description] (default: {STOCKDATA})
"""
client.drop_collection('stock_info')
stock_list = fetch_get_stock_list().code.unique().tolist()
coll = client.stock_info
coll.create_index('code')
err = []
def __saving_work(code, coll):
util_log_info(
'##JOB010 Now Saving STOCK INFO ==== {}'.format(str(code)), ui_log=ui_log)
try:
coll.insert_many(
util_to_json_from_pandas(
fetch_get_stock_info(str(code))))
except:
err.append(str(code))
for i_ in range(len(stock_list)):
# __saving_work('000001')
strLogProgress = 'DOWNLOAD PROGRESS {} '.format(
str(float(i_ / len(stock_list) * 100))[0:4] + '%')
intLogProgress = int(float(i_ / len(stock_list) * 10000.0))
util_log_info('The {} of Total {}'.format(i_, len(stock_list)))
util_log_info(strLogProgress, ui_log=ui_log,
ui_progress=ui_progress, ui_progress_int_value=intLogProgress)
__saving_work(stock_list[i_], coll)
if len(err) < 1:
util_log_info('SUCCESS', ui_log=ui_log)
else:
util_log_info(' ERROR CODE \n ', ui_log=ui_log)
util_log_info(err, ui_log=ui_log)
def __saving_work(code, tb):
util_log_info(
'##JOB03 Now Saving STOCK_MIN ==== {}'.format(str(code)), ui_log=ui_log)
try:
for type in ['1min', '5min', '15min', '30min', '60min']:
cli=tb[_sel_market_code(code)+frequenceDict[type]]\
.create_table(_sel_market_code(code)+str(code))
end_time = str(now_time())[0:19]
if cli.count() > 0:
start_time = cli[-1]['datetime']
util_log_info(
'##JOB03.{} Now Saving {} from {} to {} =={} '.format(['1min', '5min', '15min', '30min', '60min'].index(type),
str(code), start_time, end_time, type),
ui_log=ui_log)
if start_time != end_time:
__data = fetch_get_stock_min(
str(code), start_time, end_time, type)
if len(__data) > 1:
cli.insert_many(__data[1::])
else:
start_time = '2015-01-01'
util_log_info(
'##JOB03.{} Now Saving {} from {} to {} =={} '.format(['1min', '5min', '15min', '30min', '60min'].index(type),
str(code), start_time, end_time, type),
ui_log=ui_log)
if start_time != end_time:
__data = fetch_get_stock_min(
str(code), start_time, end_time, type)
if len(__data) > 1:
cli.save_data(__data)
except Exception as e:
util_log_info(e, ui_log=ui_log)
err.append(code)
util_log_info(err, ui_log=ui_log)
def __saving_work(code, coll):
util_log_info(
'##JOB05 Now Saving Index_MIN ==== {}'.format(str(code)), ui_log=ui_log)
try:
for type in ['1min', '5min', '15min', '30min', '60min']:
ref_ = coll.find(
{'code': str(code)[0:6], 'type': type})
end_time = str(now_time())[0:19]
if ref_.count() > 0:
start_time = ref_[ref_.count() - 1]['datetime']
util_log_info(
'##JOB05.{} Now Saving {} from {} to {} =={} '.
format(['1min', '5min', '15min', '30min', '60min'].
index(type), str(code), start_time, end_time, type),
ui_log=ui_log)
if start_time != end_time:
__data = fetch_get_index_min(
str(code), start_time, end_time, type)
if len(__data) > 1:
coll.insert_many(
util_to_json_from_pandas(__data[1::]))
else:
start_time = '2015-01-01'
util_log_info(
'##JOB05.{} Now Saving {} from {} to {} =={} '.
format(['1min', '5min', '15min', '30min', '60min'].
index(type), str(code), start_time, end_time, type),
ui_log=ui_log)
if start_time != end_time:
__data = fetch_get_index_min(
str(code), start_time, end_time, type)
if len(__data) > 1:
coll.insert_many(
util_to_json_from_pandas(__data))
except:
err.append(code)
def SU_save_stock_list(client=STOCKDATA, ui_log=None, ui_progress=None):
"""save stock_list
Keyword Arguments:
client {[type]} -- [description] (default: {STOCKDATA})
"""
client.drop_collection('stock_list')
coll = client.stock_list
coll.create_index('code')
try:
# 🛠todo 这个应该是第一个任务 JOB01， 先更新股票列表！！
util_log_info('##JOB08 Now Saving STOCK_LIST ====', ui_log=ui_log,
ui_progress=ui_progress, ui_progress_int_value=5000)
stock_list_from_tdx = fetch_get_stock_list()
pandas_data = util_to_json_from_pandas(stock_list_from_tdx)
coll.insert_many(pandas_data)
util_log_info("完成股票列表获取", ui_log=ui_log,
ui_progress=ui_progress, ui_progress_int_value=10000)
except Exception as e:
util_log_info(e, ui_log=ui_log)
print(" Error save_tdx.SU_save_stock_list exception!")
pass
def _save_stock_min(client=STOCKDATA, stock_list=None, ui_log=None, ui_progress=None):
"""save stock_min
Keyword Arguments:
client {[type]} -- [description] (default: {STOCKDATA})
"""
if stock_list is None :
stock_list = fetch_get_stock_list().code.unique().tolist()
coll = client
err = []
def __saving_work(code, tb):
util_log_info(
'##JOB03 Now Saving STOCK_MIN ==== {}'.format(str(code)), ui_log=ui_log)
try:
for type in ['1min', '5min', '15min', '30min', '60min']:
cli=tb[_sel_market_code(code)+frequenceDict[type]]\
.create_table(_sel_market_code(code)+str(code))
end_time = str(now_time())[0:19]
if cli.count() > 0:
start_time = cli[-1]['datetime']
util_log_info(
'##JOB03.{} Now Saving {} from {} to {} =={} '.format(['1min', '5min', '15min', '30min', '60min'].index(type),
str(code), start_time, end_time, type),
ui_log=ui_log)
if start_time != end_time:
__data = fetch_get_stock_min(
str(code), start_time, end_time, type)
if len(__data) > 1:
cli.insert_many(__data[1::])
else:
start_time = '2015-01-01'
util_log_info(
'##JOB03.{} Now Saving {} from {} to {} =={} '.format(['1min', '5min', '15min', '30min', '60min'].index(type),
str(code), start_time, end_time, type),
ui_log=ui_log)
if start_time != end_time:
__data = fetch_get_stock_min(
str(code), start_time, end_time, type)
if len(__data) > 1:
cli.save_data(__data)
except Exception as e:
util_log_info(e, ui_log=ui_log)
err.append(code)
util_log_info(err, ui_log=ui_log)
executor = ThreadPoolExecutor(max_workers=4)
#executor.map((__saving_work, stock_list[i_], coll),URLS)
res = {executor.submit(
__saving_work, stock_list[i_], coll) for i_ in range(len(stock_list))}
count = 0
for i_ in concurrent.futures.as_completed(res):
util_log_info('The {} of Total {}'.format(
count, len(stock_list)), ui_log=ui_log)
strProgress = 'DOWNLOAD PROGRESS {} '.format(
str(float(count / len(stock_list) * 100))[0:4] + '%')
intProgress = int(count / len(stock_list) * 10000.0)
util_log_info(strProgress, ui_log, ui_progress=ui_progress,
ui_progress_int_value=intProgress)
count = count + 1
if len(err) < 1:
util_log_info('SUCCESS', ui_log=ui_log)
else:
util_log_info(' ERROR CODE \n ', ui_log=ui_log)
util_log_info(err, ui_log=ui_log)
def show(self):
"""
打印数据包的内容
"""
return util_log_info(self.data)
def plot(self, code=None):
def kline_formater(param):
return param.name + ':' + vars(param)
"""plot the market_data"""
if code is None:
path_name = '.' + os.sep + 'QA_' + self.type + \
'_codepackage_' + self.if_fq + '.html'
kline = Kline('CodePackage_' + self.if_fq + '_' + self.type,
width=1360, height=700, page_title='QUANTAXIS')
bar = Bar()
data_splits = self.splits()
for ds in data_splits:
data = []
axis = []
if ds.data_type[-3:] == 'day':
datetime = np.array(ds.date.map(str))
else:
datetime = np.array(ds.datetime.map(str))
ohlc = np.array(
ds.data.loc[:, ['open', 'close', 'low', 'high']])
#amount = np.array(ds.amount)
#vol = np.array(ds.volume)
kline.add(ds.code[0], datetime, ohlc, mark_point=[
"max", "min"], is_datazoom_show=False, datazoom_orient='horizontal')
kline.render(path_name)
webbrowser.open(path_name)
util_log_info(
'The Pic has been saved to your path: %s' % path_name)
else:
data = []
axis = []
ds = self.select_code(code)
data = []
#axis = []
if self.data_type[-3:] == 'day':
datetime = np.array(ds.date.map(str))
else:
datetime = np.array(ds.datetime.map(str))
ohlc = np.array(ds.data.loc[:, ['open', 'close', 'low', 'high']])
#amount = np.array(ds.amount)
vol = np.array(ds.volume)
kline = Kline('{}__{}__{}'.format(code, self.if_fq, self.type),
width=1360, height=700, page_title='QUANTAXIS')
bar = Bar()
kline.add(self.code, datetime, ohlc,
mark_point=["max", "min"],
# is_label_show=True,
is_datazoom_show=True,
is_xaxis_show=False,
# is_toolbox_show=True,
tooltip_formatter='{b}:{c}', # kline_formater,
# is_more_utils=True,
datazoom_orient='horizontal')
bar.add(self.code, datetime, vol,
is_datazoom_show=True,
datazoom_xaxis_index=[0, 1])
path_name = '.{}QA_{}_{}_{}.html'.format(
os.sep, self.type, code, self.if_fq)
# kline.add(code, axis, data, mark_point=[
# "max", "min"], is_datazoom_show=True, datazoom_orient='horizontal')
grid = Grid(width=1360, height=700, page_title='QUANTAXIS')
grid.add(bar, grid_top="80%")
grid.add(kline, grid_bottom="30%")
grid.render(path_name)
webbrowser.open(path_name)
util_log_info(
'The Pic has been saved to your path: {}'.format(path_name))
def __saving_work(code, coll_option_day):
try:
util_log_info('##JOB12 Now Saving OPTION_DAY==== {}'.format(
str(code)), ui_log=ui_log)
# 首选查找数据库 是否 有 这个代码的数据
# 期权代码 从 10000001 开始编码 10001228
ref = coll_option_day.find({'code': str(code)[0:8]})
end_date = str(now_time())[0:10]
# 当前数据库已经包含了这个代码的数据， 继续增量更新
# 加入这个判断的原因是因为如果是刚上市的 数据库会没有数据 所以会有负索引问题出现
if ref.count() > 0:
# 接着上次获取的日期继续更新
start_date = ref[ref.count() - 1]['date']
util_log_info(' 上次获取期权日线数据的最后日期是 {}'.format(
start_date), ui_log=ui_log)
util_log_info('UPDATE_OPTION_DAY \n 从上一次下载数据开始继续 Trying update {} from {} to {}'.format(
code, start_date, end_date), ui_log=ui_log)
if start_date != end_date:
start_date0 = util_get_next_day(start_date)
df0 = fetch_get_option_day(code=code, start_date=start_date0, end_date=end_date,
frequence='day', ip=None, port=None)
retCount = df0.iloc[:, 0].size
util_log_info("日期从开始{}-结束{} , 合约代码{} , 返回了{}条记录 , 准备写入数据库"
.format(start_date0, end_date, code, retCount), ui_log=ui_log)
coll_option_day.insert_many(
util_to_json_from_pandas(df0))
else:
util_log_info("^已经获取过这天的数据了^ {}".format(
start_date), ui_log=ui_log)
else:
start_date = '1990-01-01'
util_log_info('UPDATE_OPTION_DAY \n 从新开始下载数据 Trying update {} from {} to {}'.format
(code, start_date, end_date), ui_log=ui_log)
if start_date != end_date:
df0 = fetch_get_option_day(code=code, start_date=start_date, end_date=end_date,
frequence='day', ip=None, port=None)
retCount = df0.iloc[:, 0].size
util_log_info("日期从开始{}-结束{} , 合约代码{} , 获取了{}条记录 , 准备写入数据库^_^ "
.format(start_date, end_date, code, retCount),
ui_log=ui_log)
coll_option_day.insert_many(
util_to_json_from_pandas(df0))
else:
util_log_info(
"*已经获取过这天的数据了* {}".format(start_date), ui_log=ui_log)
except Exception as error0:
print(error0)
err.append(str(code))
def SU_save_option_day(client=STOCKDATA, ui_log=None, ui_progress=None):
'''
:param client:
:return:
'''
option_contract_list = fetch_get_50etf_option_contract_time_to_market()
coll_option_day = client.option_day
coll_option_day.create_index(
[("code", pymongo.ASCENDING), ("date_stamp", pymongo.ASCENDING)])
err = []
# 索引 code
def __saving_work(code, coll_option_day):
try:
util_log_info('##JOB12 Now Saving OPTION_DAY==== {}'.format(
str(code)), ui_log=ui_log)
# 首选查找数据库 是否 有 这个代码的数据
# 期权代码 从 10000001 开始编码 10001228
ref = coll_option_day.find({'code': str(code)[0:8]})
end_date = str(now_time())[0:10]
# 当前数据库已经包含了这个代码的数据， 继续增量更新
# 加入这个判断的原因是因为如果是刚上市的 数据库会没有数据 所以会有负索引问题出现
if ref.count() > 0:
# 接着上次获取的日期继续更新
start_date = ref[ref.count() - 1]['date']
util_log_info(' 上次获取期权日线数据的最后日期是 {}'.format(
start_date), ui_log=ui_log)
util_log_info('UPDATE_OPTION_DAY \n 从上一次下载数据开始继续 Trying update {} from {} to {}'.format(
code, start_date, end_date), ui_log=ui_log)
if start_date != end_date:
start_date0 = util_get_next_day(start_date)
df0 = fetch_get_option_day(code=code, start_date=start_date0, end_date=end_date,
frequence='day', ip=None, port=None)
retCount = df0.iloc[:, 0].size
util_log_info("日期从开始{}-结束{} , 合约代码{} , 返回了{}条记录 , 准备写入数据库"
.format(start_date0, end_date, code, retCount), ui_log=ui_log)
coll_option_day.insert_many(
util_to_json_from_pandas(df0))
else:
util_log_info("^已经获取过这天的数据了^ {}".format(
start_date), ui_log=ui_log)
else:
start_date = '1990-01-01'
util_log_info('UPDATE_OPTION_DAY \n 从新开始下载数据 Trying update {} from {} to {}'.format
(code, start_date, end_date), ui_log=ui_log)
if start_date != end_date:
df0 = fetch_get_option_day(code=code, start_date=start_date, end_date=end_date,
frequence='day', ip=None, port=None)
retCount = df0.iloc[:, 0].size
util_log_info("日期从开始{}-结束{} , 合约代码{} , 获取了{}条记录 , 准备写入数据库^_^ "
.format(start_date, end_date, code, retCount),
ui_log=ui_log)
coll_option_day.insert_many(
util_to_json_from_pandas(df0))
else:
util_log_info(
"*已经获取过这天的数据了* {}".format(start_date), ui_log=ui_log)
except Exception as error0:
print(error0)
err.append(str(code))
for item in range(len(option_contract_list)):
util_log_info('The {} of Total {}'.format(
item, len(option_contract_list)), ui_log=ui_log)
strLogProgress = 'DOWNLOAD PROGRESS {} '.format(
str(float(item / len(option_contract_list) * 100))[0:4] + '%')
intLogProgress = int(float(item / len(option_contract_list) * 10000.0))
util_log_info(strLogProgress, ui_log=ui_log,
ui_progress=ui_progress, ui_progress_int_value=intLogProgress)
__saving_work(option_contract_list[item].code, coll_option_day)
if len(err) < 1:
util_log_info('SUCCESS save option day ^_^ ', ui_log=ui_log)
else:
util_log_info(' ERROR CODE \n ', ui_log=ui_log)
util_log_info(err, ui_log=ui_log)
def SU_save_etf_min(client=STOCKDATA, ui_log=None, ui_progress=None):
"""save etf_min
Keyword Arguments:
client {[type]} -- [description] (default: {STOCKDATA})
"""
__index_list = fetch_get_stock_list('etf')
coll = client.index_min
coll.create_index([('code', pymongo.ASCENDING), ('time_stamp',
pymongo.ASCENDING), ('date_stamp', pymongo.ASCENDING)])
err = []
def __saving_work(code, coll):
util_log_info(
'##JOB07 Now Saving ETF_MIN ==== {}'.format(str(code)), ui_log=ui_log)
try:
for type in ['1min', '5min', '15min', '30min', '60min']:
ref_ = coll.find(
{'code': str(code)[0:6], 'type': type})
end_time = str(now_time())[0:19]
if ref_.count() > 0:
start_time = ref_[ref_.count() - 1]['datetime']
util_log_info(
'##JOB07.{} Now Saving {} from {} to {} =={} '
.format(['1min', '5min', '15min', '30min', '60min']
.index(type), str(code), start_time, end_time, type),
ui_log=ui_log)
if start_time != end_time:
__data = fetch_get_index_min(
str(code), start_time, end_time, type)
if len(__data) > 1:
coll.insert_many(
util_to_json_from_pandas(__data[1::]))
else:
start_time = '2015-01-01'
util_log_info(
'##JOB07.{} Now Saving {} from {} to {} =={} '
.format(['1min', '5min', '15min', '30min', '60min']
.index(type), str(code), start_time, end_time, type), ui_log=ui_log)
if start_time != end_time:
__data = fetch_get_index_min(
str(code), start_time, end_time, type)
if len(__data) > 1:
coll.insert_many(
util_to_json_from_pandas(__data))
except:
err.append(code)
executor = ThreadPoolExecutor(max_workers=4)
res = {executor.submit(
__saving_work, __index_list.index[i_][0], coll) for i_ in range(len(__index_list))} # multi index ./.
count = 0
for i_ in concurrent.futures.as_completed(res):
util_log_info('The {} of Total {}'.format(
count, len(__index_list)), ui_log=ui_log)
strLogProgress = 'DOWNLOAD PROGRESS {} '.format(
str(float(count / len(__index_list) * 100))[0:4] + '%')
intLogProgress = int(float(count / len(__index_list) * 10000.0))
util_log_info(strLogProgress, ui_log=ui_log,
ui_progress=ui_progress, ui_progress_int_value=intLogProgress)
count = count + 1
if len(err) < 1:
util_log_info('SUCCESS', ui_log=ui_log)
else:
util_log_info(' ERROR CODE \n ', ui_log=ui_log)
util_log_info(err, ui_log=ui_log)
