def QA_SU_save_stock_transaction(client=DATABASE): """save stock_transaction Keyword Arguments: client {[type]} -- [description] (default: {DATABASE}) """ stock_list = QA_fetch_get_stock_time_to_market() coll = client.stock_transaction coll.create_index('code') err = [] def __saving_work(code): QA_util_log_info('##JOB10 Now Saving STOCK_TRANSACTION ==== {}'.format( str(code))) try: coll.insert_many( QA_util_to_json_from_pandas( QA_fetch_get_stock_transaction(str(code), str(stock_list[code]), str(now_time())[0:10]))) except: err.append(str(code)) for i_ in range(len(stock_list)): #__saving_work('000001') QA_util_log_info('The {} of Total {}'.format(i_, len(stock_list))) QA_util_log_info('DOWNLOAD PROGRESS {} '.format( str(float(i_ / len(stock_list) * 100))[0:4] + '%')) __saving_work(stock_list.index[i_]) if len(err) < 1: QA_util_log_info('SUCCESS') else: QA_util_log_info(' ERROR CODE \n ') QA_util_log_info(err)
def QA_SU_save_stock_info(client=DATABASE): """save stock_info Keyword Arguments: client {[type]} -- [description] (default: {DATABASE}) """ client.drop_collection('stock_info') stock_list = QA_fetch_get_stock_time_to_market() coll = client.stock_info coll.create_index('code') err = [] def __saving_work(code, coll): QA_util_log_info('##JOB010 Now Saving STOCK INFO ==== %s' % (str(code))) try: coll.insert_many( QA_util_to_json_from_pandas(QA_fetch_get_stock_info( str(code)))) except: err.append(str(code)) for i_ in range(len(stock_list)): #__saving_work('000001') QA_util_log_info('The %s of Total %s' % (i_, len(stock_list))) QA_util_log_info('DOWNLOAD PROGRESS %s ' % str(float(i_ / len(stock_list) * 100))[0:4] + '%') __saving_work(stock_list.index[i_], coll) if len(err) < 1: QA_util_log_info('SUCCESS') else: QA_util_log_info('ERROR CODE \n ') QA_util_log_info(err)
def QA_SU_update_stock_day(client=QA_Setting.client): def save_stock_day(code, start, end, coll): QA_util_log_info('##JOB01 Now Updating STOCK_DAY==== %s' % (str(code))) data = QA_util_to_json_from_pandas( QA_fetch_get_stock_day(str(code), start, end, '00')) if len(data) > 0: coll.insert_many(data) else: pass coll_stock_day = client.quantaxis.stock_day for item in QA_fetch_get_stock_time_to_market().index: if coll_stock_day.find({'code': str(item)[0:6]}).count() > 0: # 加入这个判断的原因是因为如果股票是刚上市的 数据库会没有数据 所以会有负索引问题出现 start_date = str(coll_stock_day.find({'code': str(item)[0:6]})[ coll_stock_day.find({'code': str(item)[0:6]}).count() - 1]['date']) print('*' * 20) end_date = str(now_time())[0:10] start_date = trade_date_sse[trade_date_sse.index( start_date) + 1] QA_util_log_info(' UPDATE_STOCK_DAY \n Trying updating %s from %s to %s' % (item, start_date, end_date)) save_stock_day(item, start_date, end_date, coll_stock_day) else: save_stock_day(item, '1990-01-01', str(now_time())[0:10], coll_stock_day) QA_util_log_info('Done == \n')
def QA_SU_save_stock_xdxr(client=QA_Setting.client): client.quantaxis.drop_collection('stock_xdxr') __stock_list = QA_fetch_get_stock_time_to_market() __coll = client.quantaxis.stock_xdxr __coll.ensure_index('code') __err = [] def __saving_work(code, __coll): QA_util_log_info('##JOB02 Now Saving XDXR INFO ==== %s' % (str(code))) try: __coll.insert_many( QA_util_to_json_from_pandas(QA_fetch_get_stock_xdxr( str(code)))) except: __err.append(str(code)) for i_ in range(len(__stock_list)): #__saving_work('000001') QA_util_log_info('The %s of Total %s' % (i_, len(__stock_list))) QA_util_log_info('DOWNLOAD PROGRESS %s ' % str(float(i_ / len(__stock_list) * 100))[0:4] + '%') __saving_work(__stock_list.index[i_], __coll) QA_util_log_info('ERROR CODE \n ') QA_util_log_info(__err)
def QA_SU_save_stock_transaction(client=QA_Setting.client): __stock_list = QA_fetch_get_stock_time_to_market() __coll = client.quantaxis.stock_transaction __coll.ensure_index('code') __err = [] def __saving_work(code): QA_util_log_info('##JOB07 Now Saving STOCK_TRANSACTION ==== %s' % (str(code))) try: __coll.insert_many( QA_util_to_json_from_pandas( QA_fetch_get_stock_transaction(str(code), str(__stock_list[code]), str(now_time())[0:10]))) except: __err.append(str(code)) for i_ in range(len(__stock_list)): #__saving_work('000001') QA_util_log_info('The %s of Total %s' % (i_, len(__stock_list))) QA_util_log_info('DOWNLOAD PROGRESS %s ' % str(float(i_ / len(__stock_list) * 100))[0:4] + '%') __saving_work(__stock_list.index[i_]) QA_util_log_info('ERROR CODE \n ') QA_util_log_info(__err)
def QA_SU_save_stock_xdxr(client=QA_Setting.client): client.quantaxis.drop_collection('stock_xdxr') stock_list = QA_fetch_get_stock_time_to_market() coll = client.quantaxis.stock_xdxr coll.create_index([('code', pymongo.ASCENDING), ('date', pymongo.ASCENDING)]) err = [] def __saving_work(code, coll): QA_util_log_info('##JOB02 Now Saving XDXR INFO ==== %s' % (str(code))) try: coll.insert_many( QA_util_to_json_from_pandas( QA_fetch_get_stock_xdxr(str(code)))) except: err.append(str(code)) for i_ in range(len(stock_list)): #__saving_work('000001') QA_util_log_info('The %s of Total %s' % (i_, len(stock_list))) QA_util_log_info('DOWNLOAD PROGRESS %s ' % str( float(i_ / len(stock_list) * 100))[0:4] + '%') __saving_work(stock_list.index[i_], coll) if len(err) < 1: QA_util_log_info('SUCCESS') else: QA_util_log_info('ERROR CODE \n ') QA_util_log_info(err)
def QA_SU_save_stock_transaction(code=None, client=QA_Setting.client): codelist = QA_fetch_get_stock_time_to_market() if code is None else code codelist = pd.DataFrame( [codelist], columns=['code']).set_index('code') if isinstance( codelist, str) else pd.DataFrame( codelist, columns=['code']).set_index('code') coll = client.quantaxis.stock_transaction coll.create_index('code') err = [] def __saving_work(code): QA_util_log_info('##JOB10 Now Saving STOCK_TRANSACTION ==== %s' % (str(code))) try: coll.insert_many( QA_util_to_json_from_pandas( QA_fetch_get_stock_transaction(str(code), str(codelist[code]), str(now_time())[0:10]))) except: err.append(str(code)) for i_ in range(len(codelist)): #__saving_work('000001') QA_util_log_info('The %s of Total %s' % (i_, len(codelist))) QA_util_log_info('DOWNLOAD PROGRESS %s ' % str(float(i_ / len(codelist) * 100))[0:4] + '%') __saving_work(codelist.index[i_]) if len(err) < 1: QA_util_log_info('SUCCESS') else: QA_util_log_info('ERROR CODE \n ') QA_util_log_info(err)
def QA_SU_save_stock_day(client=DATABASE): """save stock_day Keyword Arguments: client {[type]} -- [description] (default: {DATABASE}) """ stock_list = QA_fetch_get_stock_time_to_market() coll_stock_day = client.stock_day coll_stock_day.create_index([("code", pymongo.ASCENDING), ("date_stamp", pymongo.ASCENDING)]) err = [] def __saving_work(code, coll_stock_day): try: QA_util_log_info('##JOB01 Now Saving STOCK_DAY==== %s' % (str(code))) ref = coll_stock_day.find({'code': str(code)[0:6]}) end_date = str(now_time())[0:10] if ref.count() > 0: # 加入这个判断的原因是因为如果股票是刚上市的 数据库会没有数据 所以会有负索引问题出现 start_date = ref[ref.count() - 1]['date'] QA_util_log_info( ' UPDATE_STOCK_DAY \n Trying updating %s from %s to %s' % (code, start_date, end_date)) if start_date != end_date: coll_stock_day.insert_many( QA_util_to_json_from_pandas( QA_fetch_get_stock_day( str(code), QA_util_get_next_day(start_date), end_date, '00'))) else: start_date = '1990-01-01' QA_util_log_info( ' UPDATE_STOCK_DAY \n Trying updating %s from %s to %s' % (code, start_date, end_date)) if start_date != end_date: coll_stock_day.insert_many( QA_util_to_json_from_pandas( QA_fetch_get_stock_day(str(code), start_date, end_date, '00'))) except: err.append(str(code)) for item in range(len(stock_list)): QA_util_log_info('The %s of Total %s' % (item, len(stock_list))) QA_util_log_info('DOWNLOAD PROGRESS %s ' % str(float(item / len(stock_list) * 100))[0:4] + '%') __saving_work(stock_list.index[item], coll_stock_day) if len(err) < 1: QA_util_log_info('SUCCESS') else: QA_util_log_info('ERROR CODE \n ') QA_util_log_info(err)
def QA_SU_save_stock_min(client=DATABASE): stock_list = QA_fetch_get_stock_time_to_market() coll = client.stock_min coll.create_index([('code', pymongo.ASCENDING), ('time_stamp', pymongo.ASCENDING), ('date_stamp', pymongo.ASCENDING)]) err = [] def __saving_work(code, coll): QA_util_log_info('##JOB03 Now Saving STOCK_MIN ==== %s' % (str(code))) try: for type in ['1min', '5min', '15min', '30min', '60min']: ref_ = coll.find( {'code': str(code)[0:6], 'type': type}) end_time = str(now_time())[0:19] if ref_.count() > 0: start_time = ref_[ref_.count() - 1]['datetime'] QA_util_log_info( '##JOB03.%s Now Saving %s from %s to %s ==%s ' % (['1min', '5min', '15min', '30min', '60min'].index(type), str(code), start_time, end_time, type)) if start_time != end_time: __data = QA_fetch_get_stock_min( str(code), start_time, end_time, type) if len(__data) > 1: coll.insert_many( QA_util_to_json_from_pandas(__data[1::])) else: start_time = '2015-01-01' QA_util_log_info( '##JOB03.%s Now Saving %s from %s to %s ==%s ' % (['1min', '5min', '15min', '30min', '60min'].index(type), str(code), start_time, end_time, type)) if start_time != end_time: __data = QA_fetch_get_stock_min( str(code), start_time, end_time, type) if len(__data) > 1: coll.insert_many( QA_util_to_json_from_pandas(__data)) except Exception as e: QA_util_log_info(e) err.append(code) executor = ThreadPoolExecutor(max_workers=4) #executor.map((__saving_work, stock_list.index[i_], coll),URLS) res = {executor.submit( __saving_work, stock_list.index[i_], coll) for i_ in range(len(stock_list))} count = 0 for i_ in concurrent.futures.as_completed(res): QA_util_log_info('The %s of Total %s' % (count, len(stock_list))) QA_util_log_info('DOWNLOAD PROGRESS %s ' % str( float(count / len(stock_list) * 100))[0:4] + '%') count = count + 1 if len(err) < 1: QA_util_log_info('SUCCESS') else: QA_util_log_info('ERROR CODE \n ') QA_util_log_info(err)
def QA_SU_save_stock_xdxr(client=DATABASE): """[summary] Keyword Arguments: client {[type]} -- [description] (default: {DATABASE}) """ client.drop_collection('stock_xdxr') stock_list = QA_fetch_get_stock_time_to_market() coll = client.stock_xdxr coll.create_index([('code', pymongo.ASCENDING), ('date', pymongo.ASCENDING)]) err = [] def __saving_work(code, coll): QA_util_log_info('##JOB02 Now Saving XDXR INFO ==== {}'.format( str(code))) try: coll.insert_many( QA_util_to_json_from_pandas(QA_fetch_get_stock_xdxr( str(code)))) except: err.append(str(code)) for i_ in range(len(stock_list)): QA_util_log_info('The {} of Total {}'.format(i_, len(stock_list))) QA_util_log_info('DOWNLOAD PROGRESS {} '.format( str(float(i_ / len(stock_list) * 100))[0:4] + '%')) __saving_work(stock_list.index[i_], coll) if len(err) < 1: QA_util_log_info('SUCCESS') else: try_code = err err = [] QA_util_log_info('Try to get stock xdxr info in erro list! \n') for i__ in range(len(try_code)): QA_util_log_info('The {} of Total {}'.format(i__, len(try_code))) QA_util_log_info('DOWNLOAD PROGRESS {} '.format( str(float(i__ / len(try_code) * 100))[0:4] + '%')) __saving_work(try_code[i__], coll) if len(err) < 1: QA_util_log_info('SUCCESS') else: QA_util_log_info(' ERROR CODE \n ') QA_util_log_info(err)
def QA_SU_save_stock_min(client=QA_Setting.client): __stock_list = QA_fetch_get_stock_time_to_market() __coll = client.quantaxis.stock_min __coll.ensure_index('code') __err = [] def __saving_work(code, __coll): QA_util_log_info('##JOB03 Now Saving STOCK_MIN ==== %s' % (str(code))) try: for type in ['1min', '5min', '15min', '30min', '60min']: ref_ = __coll.find({'code': str(code)[0:6], 'type': type}) end_time = str(datetime.datetime.now())[0:19] if ref_.count() > 0: start_time = ref_[ref_.count() - 1]['datetime'] else: start_time = '2015-01-01' QA_util_log_info( '##JOB03.%s Now Saving %s from %s to %s ==%s ' % (['1min', '5min', '15min', '30min', '60min' ].index(type), str(code), start_time, end_time, type)) if start_time != end_time: __data = QA_fetch_get_stock_min(str(code), start_time, end_time, type) if len(__data) > 1: __coll.insert_many( QA_util_to_json_from_pandas(__data[1::])) except Exception as e: QA_util_log_info(e) __err.append(code) executor = ThreadPoolExecutor(max_workers=4) res = { executor.submit(__saving_work, __stock_list.index[i_], __coll) for i_ in range(len(__stock_list)) } count = 0 for i_ in concurrent.futures.as_completed(res): QA_util_log_info('The %s of Total %s' % (count, len(__stock_list))) QA_util_log_info('DOWNLOAD PROGRESS %s ' % str(float(count / len(__stock_list) * 100))[0:4] + '%') count = count + 1 QA_util_log_info('ERROR CODE \n ') QA_util_log_info(__err)
def QA_SU_save_stock_day(code=None, client=QA_Setting.client): codelist = QA_fetch_get_stock_time_to_market() if code is None else code codelist = pd.DataFrame( [codelist], columns=['code']).set_index('code') if isinstance( codelist, str) else pd.DataFrame( codelist, columns=['code']).set_index('code') coll_stock_day = client.quantaxis.stock_day coll_stock_day.create_index([("code", pymongo.ASCENDING), ("date_stamp", pymongo.ASCENDING)]) err = [] def __saving_work(code, coll_stock_day): try: QA_util_log_info('##JOB01 Now Saving STOCK_DAY==== %s' % (str(code))) ref = coll_stock_day.find({'code': str(code)[0:6]}) end_date = str(now_time())[0:10] if ref.count() > 0: # 加入这个判断的原因是因为如果股票是刚上市的 数据库会没有数据 所以会有负索引问题出现 start_date = ref[ref.count() - 1]['date'] else: start_date = '1990-01-01' QA_util_log_info( ' UPDATE_STOCK_DAY \n Trying updating %s from %s to %s' % (code, start_date, end_date)) if start_date != end_date: coll_stock_day.insert_many( QA_util_to_json_from_pandas( QA_fetch_get_stock_day(str(code), start_date, end_date, '00')[1::])) except: err.append(str(code)) for item in range(len(codelist)): QA_util_log_info('The %s of Total %s' % (item, len(codelist))) QA_util_log_info('DOWNLOAD PROGRESS %s ' % str(float(item / len(codelist) * 100))[0:4] + '%') __saving_work(codelist.index[item], coll_stock_day) if len(err) < 1: QA_util_log_info('SUCCESS') else: QA_util_log_info('ERROR CODE \n ') QA_util_log_info(err)
def QA_SU_save_stock_day(client=QA_Setting.client): __stock_list = QA_fetch_get_stock_time_to_market() coll_stock_day = client.quantaxis.stock_day coll_stock_day.ensure_index('code') __err = [] def __saving_work(code, coll_stock_day): try: QA_util_log_info('##JOB01 Now Saving STOCK_DAY==== %s' % (str(code))) ref = coll_stock_day.find({'code': str(code)[0:6]}) end_date = str(now_time())[0:10] if ref.count() > 0: # 加入这个判断的原因是因为如果股票是刚上市的 数据库会没有数据 所以会有负索引问题出现 start_date = ref[ref.count() - 1]['date'] else: start_date = '1990-01-01' QA_util_log_info( ' UPDATE_STOCK_DAY \n Trying updating %s from %s to %s' % (code, start_date, end_date)) if start_date != end_date: coll_stock_day.insert_many( QA_util_to_json_from_pandas( QA_fetch_get_stock_day(str(code), start_date, end_date, '00')[1::])) except: __err.append(str(code)) for item in range(len(__stock_list)): QA_util_log_info('The %s of Total %s' % (item, len(__stock_list))) QA_util_log_info('DOWNLOAD PROGRESS %s ' % str(float(item / len(__stock_list) * 100))[0:4] + '%') __saving_work(__stock_list.index[item], coll_stock_day) QA_util_log_info('ERROR CODE \n ') QA_util_log_info(__err)
def QA_SU_save_stock_day(client=DATABASE): """save stock_day Keyword Arguments: client {[type]} -- [description] (default: {DATABASE}) """ stock_list = QA_fetch_get_stock_time_to_market() coll_stock_day = client.stock_day coll_stock_day.create_index([("code", pymongo.ASCENDING), ("date_stamp", pymongo.ASCENDING)]) err = [] def __saving_work(code, coll_stock_day): try: QA_util_log_info('##JOB01 Now Saving STOCK_DAY==== {}'.format( str(code))) #首选查找数据库 是否 有 这个代码的数据 ref = coll_stock_day.find({'code': str(code)[0:6]}) end_date = str(now_time())[0:10] #当前数据库已经包含了这个代码的数据, 继续增量更新 # 加入这个判断的原因是因为如果股票是刚上市的 数据库会没有数据 所以会有负索引问题出现 if ref.count() > 0: # 接着上次获取的日期继续更新 start_date = ref[ref.count() - 1]['date'] QA_util_log_info( 'UPDATE_STOCK_DAY \n Trying updating {} from {} to {}'. format(code, start_date, end_date)) if start_date != end_date: coll_stock_day.insert_many( QA_util_to_json_from_pandas( QA_fetch_get_stock_day( str(code), QA_util_get_next_day(start_date), end_date, '00'))) #当前数据库中没有这个代码的股票数据, 从1990-01-01 开始下载所有的数据 else: start_date = '1990-01-01' QA_util_log_info( 'UPDATE_STOCK_DAY \n Trying updating {} from {} to {}'. format(code, start_date, end_date)) if start_date != end_date: coll_stock_day.insert_many( QA_util_to_json_from_pandas( QA_fetch_get_stock_day(str(code), start_date, end_date, '00'))) except Exception as error0: print(error0) err.append(str(code)) for item in range(len(stock_list)): QA_util_log_info('The {} of Total {}'.format(item, len(stock_list))) QA_util_log_info('DOWNLOAD PROGRESS {} '.format( str(float(item / len(stock_list) * 100))[0:4] + '%')) __saving_work(stock_list.index[item], coll_stock_day) if len(err) < 1: QA_util_log_info('SUCCESS save stock day ^_^') else: QA_util_log_info(' ERROR CODE \n ') QA_util_log_info(err)