def QA_SU_save_stock_transaction(client=DATABASE):
"""save stock_transaction
Keyword Arguments:
client {[type]} -- [description] (default: {DATABASE})
"""
stock_list = QA_fetch_get_stock_time_to_market()
coll = client.stock_transaction
coll.create_index('code')
err = []
def __saving_work(code):
QA_util_log_info('##JOB10 Now Saving STOCK_TRANSACTION ==== {}'.format(
str(code)))
try:
coll.insert_many(
QA_util_to_json_from_pandas(
QA_fetch_get_stock_transaction(str(code),
str(stock_list[code]),
str(now_time())[0:10])))
except:
err.append(str(code))
for i_ in range(len(stock_list)):
#__saving_work('000001')
QA_util_log_info('The {} of Total {}'.format(i_, len(stock_list)))
QA_util_log_info('DOWNLOAD PROGRESS {} '.format(
str(float(i_ / len(stock_list) * 100))[0:4] + '%'))
__saving_work(stock_list.index[i_])
if len(err) < 1:
QA_util_log_info('SUCCESS')
else:
QA_util_log_info(' ERROR CODE \n ')
QA_util_log_info(err)
def QA_SU_save_stock_info(client=DATABASE):
"""save stock_info
Keyword Arguments:
client {[type]} -- [description] (default: {DATABASE})
"""
client.drop_collection('stock_info')
stock_list = QA_fetch_get_stock_time_to_market()
coll = client.stock_info
coll.create_index('code')
err = []
def __saving_work(code, coll):
QA_util_log_info('##JOB010 Now Saving STOCK INFO ==== %s' %
(str(code)))
try:
coll.insert_many(
QA_util_to_json_from_pandas(QA_fetch_get_stock_info(
str(code))))
except:
err.append(str(code))
for i_ in range(len(stock_list)):
#__saving_work('000001')
QA_util_log_info('The %s of Total %s' % (i_, len(stock_list)))
QA_util_log_info('DOWNLOAD PROGRESS %s ' %
str(float(i_ / len(stock_list) * 100))[0:4] + '%')
__saving_work(stock_list.index[i_], coll)
if len(err) < 1:
QA_util_log_info('SUCCESS')
else:
QA_util_log_info('ERROR CODE \n ')
QA_util_log_info(err)
def QA_SU_update_stock_day(client=QA_Setting.client):
def save_stock_day(code, start, end, coll):
QA_util_log_info('##JOB01 Now Updating STOCK_DAY==== %s' % (str(code)))
data = QA_util_to_json_from_pandas(
QA_fetch_get_stock_day(str(code), start, end, '00'))
if len(data) > 0:
coll.insert_many(data)
else:
pass
coll_stock_day = client.quantaxis.stock_day
for item in QA_fetch_get_stock_time_to_market().index:
if coll_stock_day.find({'code': str(item)[0:6]}).count() > 0:
# 加入这个判断的原因是因为如果股票是刚上市的 数据库会没有数据 所以会有负索引问题出现
start_date = str(coll_stock_day.find({'code': str(item)[0:6]})[
coll_stock_day.find({'code': str(item)[0:6]}).count() - 1]['date'])
print('*' * 20)
end_date = str(now_time())[0:10]
start_date = trade_date_sse[trade_date_sse.index(
start_date) + 1]
QA_util_log_info(' UPDATE_STOCK_DAY \n Trying updating %s from %s to %s' %
(item, start_date, end_date))
save_stock_day(item, start_date, end_date, coll_stock_day)
else:
save_stock_day(item, '1990-01-01',
str(now_time())[0:10], coll_stock_day)
QA_util_log_info('Done == \n')
def QA_SU_save_stock_xdxr(client=QA_Setting.client):
client.quantaxis.drop_collection('stock_xdxr')
__stock_list = QA_fetch_get_stock_time_to_market()
__coll = client.quantaxis.stock_xdxr
__coll.ensure_index('code')
__err = []
def __saving_work(code, __coll):
QA_util_log_info('##JOB02 Now Saving XDXR INFO ==== %s' % (str(code)))
try:
__coll.insert_many(
QA_util_to_json_from_pandas(QA_fetch_get_stock_xdxr(
str(code))))
except:
__err.append(str(code))
for i_ in range(len(__stock_list)):
#__saving_work('000001')
QA_util_log_info('The %s of Total %s' % (i_, len(__stock_list)))
QA_util_log_info('DOWNLOAD PROGRESS %s ' %
str(float(i_ / len(__stock_list) * 100))[0:4] + '%')
__saving_work(__stock_list.index[i_], __coll)
QA_util_log_info('ERROR CODE \n ')
QA_util_log_info(__err)
def QA_SU_save_stock_transaction(client=QA_Setting.client):
__stock_list = QA_fetch_get_stock_time_to_market()
__coll = client.quantaxis.stock_transaction
__coll.ensure_index('code')
__err = []
def __saving_work(code):
QA_util_log_info('##JOB07 Now Saving STOCK_TRANSACTION ==== %s' %
(str(code)))
try:
__coll.insert_many(
QA_util_to_json_from_pandas(
QA_fetch_get_stock_transaction(str(code),
str(__stock_list[code]),
str(now_time())[0:10])))
except:
__err.append(str(code))
for i_ in range(len(__stock_list)):
#__saving_work('000001')
QA_util_log_info('The %s of Total %s' % (i_, len(__stock_list)))
QA_util_log_info('DOWNLOAD PROGRESS %s ' %
str(float(i_ / len(__stock_list) * 100))[0:4] + '%')
__saving_work(__stock_list.index[i_])
QA_util_log_info('ERROR CODE \n ')
QA_util_log_info(__err)
def QA_SU_save_stock_xdxr(client=QA_Setting.client):
client.quantaxis.drop_collection('stock_xdxr')
stock_list = QA_fetch_get_stock_time_to_market()
coll = client.quantaxis.stock_xdxr
coll.create_index([('code', pymongo.ASCENDING),
('date', pymongo.ASCENDING)])
err = []
def __saving_work(code, coll):
QA_util_log_info('##JOB02 Now Saving XDXR INFO ==== %s' % (str(code)))
try:
coll.insert_many(
QA_util_to_json_from_pandas(
QA_fetch_get_stock_xdxr(str(code))))
except:
err.append(str(code))
for i_ in range(len(stock_list)):
#__saving_work('000001')
QA_util_log_info('The %s of Total %s' % (i_, len(stock_list)))
QA_util_log_info('DOWNLOAD PROGRESS %s ' % str(
float(i_ / len(stock_list) * 100))[0:4] + '%')
__saving_work(stock_list.index[i_], coll)
if len(err) < 1:
QA_util_log_info('SUCCESS')
else:
QA_util_log_info('ERROR CODE \n ')
QA_util_log_info(err)
def QA_SU_save_stock_transaction(code=None, client=QA_Setting.client):
codelist = QA_fetch_get_stock_time_to_market() if code is None else code
codelist = pd.DataFrame(
[codelist], columns=['code']).set_index('code') if isinstance(
codelist, str) else pd.DataFrame(
codelist, columns=['code']).set_index('code')
coll = client.quantaxis.stock_transaction
coll.create_index('code')
err = []
def __saving_work(code):
QA_util_log_info('##JOB10 Now Saving STOCK_TRANSACTION ==== %s' %
(str(code)))
try:
coll.insert_many(
QA_util_to_json_from_pandas(
QA_fetch_get_stock_transaction(str(code),
str(codelist[code]),
str(now_time())[0:10])))
except:
err.append(str(code))
for i_ in range(len(codelist)):
#__saving_work('000001')
QA_util_log_info('The %s of Total %s' % (i_, len(codelist)))
QA_util_log_info('DOWNLOAD PROGRESS %s ' %
str(float(i_ / len(codelist) * 100))[0:4] + '%')
__saving_work(codelist.index[i_])
if len(err) < 1:
QA_util_log_info('SUCCESS')
else:
QA_util_log_info('ERROR CODE \n ')
QA_util_log_info(err)
def QA_SU_save_stock_day(client=DATABASE):
"""save stock_day
Keyword Arguments:
client {[type]} -- [description] (default: {DATABASE})
"""
stock_list = QA_fetch_get_stock_time_to_market()
coll_stock_day = client.stock_day
coll_stock_day.create_index([("code", pymongo.ASCENDING),
("date_stamp", pymongo.ASCENDING)])
err = []
def __saving_work(code, coll_stock_day):
try:
QA_util_log_info('##JOB01 Now Saving STOCK_DAY==== %s' %
(str(code)))
ref = coll_stock_day.find({'code': str(code)[0:6]})
end_date = str(now_time())[0:10]
if ref.count() > 0:
# 加入这个判断的原因是因为如果股票是刚上市的 数据库会没有数据 所以会有负索引问题出现
start_date = ref[ref.count() - 1]['date']
QA_util_log_info(
' UPDATE_STOCK_DAY \n Trying updating %s from %s to %s' %
(code, start_date, end_date))
if start_date != end_date:
coll_stock_day.insert_many(
QA_util_to_json_from_pandas(
QA_fetch_get_stock_day(
str(code), QA_util_get_next_day(start_date),
end_date, '00')))
else:
start_date = '1990-01-01'
QA_util_log_info(
' UPDATE_STOCK_DAY \n Trying updating %s from %s to %s' %
(code, start_date, end_date))
if start_date != end_date:
coll_stock_day.insert_many(
QA_util_to_json_from_pandas(
QA_fetch_get_stock_day(str(code), start_date,
end_date, '00')))
except:
err.append(str(code))
for item in range(len(stock_list)):
QA_util_log_info('The %s of Total %s' % (item, len(stock_list)))
QA_util_log_info('DOWNLOAD PROGRESS %s ' %
str(float(item / len(stock_list) * 100))[0:4] + '%')
__saving_work(stock_list.index[item], coll_stock_day)
if len(err) < 1:
QA_util_log_info('SUCCESS')
else:
QA_util_log_info('ERROR CODE \n ')
QA_util_log_info(err)
def QA_SU_save_stock_min(client=DATABASE):
stock_list = QA_fetch_get_stock_time_to_market()
coll = client.stock_min
coll.create_index([('code', pymongo.ASCENDING), ('time_stamp',
pymongo.ASCENDING), ('date_stamp', pymongo.ASCENDING)])
err = []
def __saving_work(code, coll):
QA_util_log_info('##JOB03 Now Saving STOCK_MIN ==== %s' % (str(code)))
try:
for type in ['1min', '5min', '15min', '30min', '60min']:
ref_ = coll.find(
{'code': str(code)[0:6], 'type': type})
end_time = str(now_time())[0:19]
if ref_.count() > 0:
start_time = ref_[ref_.count() - 1]['datetime']
QA_util_log_info(
'##JOB03.%s Now Saving %s from %s to %s ==%s ' % (['1min', '5min', '15min', '30min', '60min'].index(type), str(code), start_time, end_time, type))
if start_time != end_time:
__data = QA_fetch_get_stock_min(
str(code), start_time, end_time, type)
if len(__data) > 1:
coll.insert_many(
QA_util_to_json_from_pandas(__data[1::]))
else:
start_time = '2015-01-01'
QA_util_log_info(
'##JOB03.%s Now Saving %s from %s to %s ==%s ' % (['1min', '5min', '15min', '30min', '60min'].index(type), str(code), start_time, end_time, type))
if start_time != end_time:
__data = QA_fetch_get_stock_min(
str(code), start_time, end_time, type)
if len(__data) > 1:
coll.insert_many(
QA_util_to_json_from_pandas(__data))
except Exception as e:
QA_util_log_info(e)
err.append(code)
executor = ThreadPoolExecutor(max_workers=4)
#executor.map((__saving_work, stock_list.index[i_], coll),URLS)
res = {executor.submit(
__saving_work, stock_list.index[i_], coll) for i_ in range(len(stock_list))}
count = 0
for i_ in concurrent.futures.as_completed(res):
QA_util_log_info('The %s of Total %s' % (count, len(stock_list)))
QA_util_log_info('DOWNLOAD PROGRESS %s ' % str(
float(count / len(stock_list) * 100))[0:4] + '%')
count = count + 1
if len(err) < 1:
QA_util_log_info('SUCCESS')
else:
QA_util_log_info('ERROR CODE \n ')
QA_util_log_info(err)
def QA_SU_save_stock_xdxr(client=DATABASE):
"""[summary]
Keyword Arguments:
client {[type]} -- [description] (default: {DATABASE})
"""
client.drop_collection('stock_xdxr')
stock_list = QA_fetch_get_stock_time_to_market()
coll = client.stock_xdxr
coll.create_index([('code', pymongo.ASCENDING),
('date', pymongo.ASCENDING)])
err = []
def __saving_work(code, coll):
QA_util_log_info('##JOB02 Now Saving XDXR INFO ==== {}'.format(
str(code)))
try:
coll.insert_many(
QA_util_to_json_from_pandas(QA_fetch_get_stock_xdxr(
str(code))))
except:
err.append(str(code))
for i_ in range(len(stock_list)):
QA_util_log_info('The {} of Total {}'.format(i_, len(stock_list)))
QA_util_log_info('DOWNLOAD PROGRESS {} '.format(
str(float(i_ / len(stock_list) * 100))[0:4] + '%'))
__saving_work(stock_list.index[i_], coll)
if len(err) < 1:
QA_util_log_info('SUCCESS')
else:
try_code = err
err = []
QA_util_log_info('Try to get stock xdxr info in erro list! \n')
for i__ in range(len(try_code)):
QA_util_log_info('The {} of Total {}'.format(i__, len(try_code)))
QA_util_log_info('DOWNLOAD PROGRESS {} '.format(
str(float(i__ / len(try_code) * 100))[0:4] + '%'))
__saving_work(try_code[i__], coll)
if len(err) < 1:
QA_util_log_info('SUCCESS')
else:
QA_util_log_info(' ERROR CODE \n ')
QA_util_log_info(err)
def QA_SU_save_stock_min(client=QA_Setting.client):
__stock_list = QA_fetch_get_stock_time_to_market()
__coll = client.quantaxis.stock_min
__coll.ensure_index('code')
__err = []
def __saving_work(code, __coll):
QA_util_log_info('##JOB03 Now Saving STOCK_MIN ==== %s' % (str(code)))
try:
for type in ['1min', '5min', '15min', '30min', '60min']:
ref_ = __coll.find({'code': str(code)[0:6], 'type': type})
end_time = str(datetime.datetime.now())[0:19]
if ref_.count() > 0:
start_time = ref_[ref_.count() - 1]['datetime']
else:
start_time = '2015-01-01'
QA_util_log_info(
'##JOB03.%s Now Saving %s from %s to %s ==%s ' %
(['1min', '5min', '15min', '30min', '60min'
].index(type), str(code), start_time, end_time, type))
if start_time != end_time:
__data = QA_fetch_get_stock_min(str(code), start_time,
end_time, type)
if len(__data) > 1:
__coll.insert_many(
QA_util_to_json_from_pandas(__data[1::]))
except Exception as e:
QA_util_log_info(e)
__err.append(code)
executor = ThreadPoolExecutor(max_workers=4)
res = {
executor.submit(__saving_work, __stock_list.index[i_], __coll)
for i_ in range(len(__stock_list))
}
count = 0
for i_ in concurrent.futures.as_completed(res):
QA_util_log_info('The %s of Total %s' % (count, len(__stock_list)))
QA_util_log_info('DOWNLOAD PROGRESS %s ' %
str(float(count / len(__stock_list) * 100))[0:4] +
'%')
count = count + 1
QA_util_log_info('ERROR CODE \n ')
QA_util_log_info(__err)
def QA_SU_save_stock_day(code=None, client=QA_Setting.client):
codelist = QA_fetch_get_stock_time_to_market() if code is None else code
codelist = pd.DataFrame(
[codelist], columns=['code']).set_index('code') if isinstance(
codelist, str) else pd.DataFrame(
codelist, columns=['code']).set_index('code')
coll_stock_day = client.quantaxis.stock_day
coll_stock_day.create_index([("code", pymongo.ASCENDING),
("date_stamp", pymongo.ASCENDING)])
err = []
def __saving_work(code, coll_stock_day):
try:
QA_util_log_info('##JOB01 Now Saving STOCK_DAY==== %s' %
(str(code)))
ref = coll_stock_day.find({'code': str(code)[0:6]})
end_date = str(now_time())[0:10]
if ref.count() > 0:
# 加入这个判断的原因是因为如果股票是刚上市的 数据库会没有数据 所以会有负索引问题出现
start_date = ref[ref.count() - 1]['date']
else:
start_date = '1990-01-01'
QA_util_log_info(
' UPDATE_STOCK_DAY \n Trying updating %s from %s to %s' %
(code, start_date, end_date))
if start_date != end_date:
coll_stock_day.insert_many(
QA_util_to_json_from_pandas(
QA_fetch_get_stock_day(str(code), start_date, end_date,
'00')[1::]))
except:
err.append(str(code))
for item in range(len(codelist)):
QA_util_log_info('The %s of Total %s' % (item, len(codelist)))
QA_util_log_info('DOWNLOAD PROGRESS %s ' %
str(float(item / len(codelist) * 100))[0:4] + '%')
__saving_work(codelist.index[item], coll_stock_day)
if len(err) < 1:
QA_util_log_info('SUCCESS')
else:
QA_util_log_info('ERROR CODE \n ')
QA_util_log_info(err)
def QA_SU_save_stock_day(client=QA_Setting.client):
__stock_list = QA_fetch_get_stock_time_to_market()
coll_stock_day = client.quantaxis.stock_day
coll_stock_day.ensure_index('code')
__err = []
def __saving_work(code, coll_stock_day):
try:
QA_util_log_info('##JOB01 Now Saving STOCK_DAY==== %s' %
(str(code)))
ref = coll_stock_day.find({'code': str(code)[0:6]})
end_date = str(now_time())[0:10]
if ref.count() > 0:
# 加入这个判断的原因是因为如果股票是刚上市的 数据库会没有数据 所以会有负索引问题出现
start_date = ref[ref.count() - 1]['date']
else:
start_date = '1990-01-01'
QA_util_log_info(
' UPDATE_STOCK_DAY \n Trying updating %s from %s to %s' %
(code, start_date, end_date))
if start_date != end_date:
coll_stock_day.insert_many(
QA_util_to_json_from_pandas(
QA_fetch_get_stock_day(str(code), start_date, end_date,
'00')[1::]))
except:
__err.append(str(code))
for item in range(len(__stock_list)):
QA_util_log_info('The %s of Total %s' % (item, len(__stock_list)))
QA_util_log_info('DOWNLOAD PROGRESS %s ' %
str(float(item / len(__stock_list) * 100))[0:4] + '%')
__saving_work(__stock_list.index[item], coll_stock_day)
QA_util_log_info('ERROR CODE \n ')
QA_util_log_info(__err)
def QA_SU_save_stock_day(client=DATABASE):
"""save stock_day
Keyword Arguments:
client {[type]} -- [description] (default: {DATABASE})
"""
stock_list = QA_fetch_get_stock_time_to_market()
coll_stock_day = client.stock_day
coll_stock_day.create_index([("code", pymongo.ASCENDING),
("date_stamp", pymongo.ASCENDING)])
err = []
def __saving_work(code, coll_stock_day):
try:
QA_util_log_info('##JOB01 Now Saving STOCK_DAY==== {}'.format(
str(code)))
#首选查找数据库 是否 有 这个代码的数据
ref = coll_stock_day.find({'code': str(code)[0:6]})
end_date = str(now_time())[0:10]
#当前数据库已经包含了这个代码的数据， 继续增量更新
# 加入这个判断的原因是因为如果股票是刚上市的 数据库会没有数据 所以会有负索引问题出现
if ref.count() > 0:
# 接着上次获取的日期继续更新
start_date = ref[ref.count() - 1]['date']
QA_util_log_info(
'UPDATE_STOCK_DAY \n Trying updating {} from {} to {}'.
format(code, start_date, end_date))
if start_date != end_date:
coll_stock_day.insert_many(
QA_util_to_json_from_pandas(
QA_fetch_get_stock_day(
str(code), QA_util_get_next_day(start_date),
end_date, '00')))
#当前数据库中没有这个代码的股票数据， 从1990-01-01 开始下载所有的数据
else:
start_date = '1990-01-01'
QA_util_log_info(
'UPDATE_STOCK_DAY \n Trying updating {} from {} to {}'.
format(code, start_date, end_date))
if start_date != end_date:
coll_stock_day.insert_many(
QA_util_to_json_from_pandas(
QA_fetch_get_stock_day(str(code), start_date,
end_date, '00')))
except Exception as error0:
print(error0)
err.append(str(code))
for item in range(len(stock_list)):
QA_util_log_info('The {} of Total {}'.format(item, len(stock_list)))
QA_util_log_info('DOWNLOAD PROGRESS {} '.format(
str(float(item / len(stock_list) * 100))[0:4] + '%'))
__saving_work(stock_list.index[item], coll_stock_day)
if len(err) < 1:
QA_util_log_info('SUCCESS save stock day ^_^')
else:
QA_util_log_info(' ERROR CODE \n ')
QA_util_log_info(err)
