def saving_work(code, coll_stock_day): try: print("entering saving_work {}".format(code)) ref = coll_stock_day.find({'code': str(code)[:6]}) end_date = str(now_time())[:10] if ref.count() > 0: start_date = ref[ref.count() - 1]['date'] QA_util_log_info( 'UPDATE_STOCK_DAY \n Trying updating {} from {} to {}'. format(code, start_date, end_date)) if start_date != end_date: start_date = QA_util_get_next_day(start_date) data_json = QA_fetch_get_stock_day(str(code), start=start_date, end=end_date, type_='json') coll_stock_day.insert_many(data_json) else: start_date = '1990-01-01' data_json = QA_fetch_get_stock_day(str(code), start=start_date, type_='json') coll_stock_day.insert_many(data_json) print("done saving_work {}".format(code)) except Exception as e: QA_util_log_info('error in saving ==== %s' % str(code)) print("The exception is {}".format(str(e)))
def saving_work(i): QA_util_log_info('Now Saving ==== %s' % (i)) try: df = QA_fetch_get_stock_day(i, start='1990-01-01', if_fq='bfq') __coll.insert_many(json.loads(df.to_json(orient='records'))) except Exception as e: print(e) QA_util_log_info('error in saving ==== %s' % str(i))
def _saving_work(code, coll_stock_day, ui_log=None, err=[]): try: QA_util_log_info( '##JOB01 Now Saving STOCK_DAY==== {}'.format(str(code)), ui_log) # 首选查找数据库 是否 有 这个代码的数据 ref = coll_stock_day.find({'ts_code': str(code)}, sort=[('trade_date', 1)]) end_date = now_time() # 当前数据库已经包含了这个代码的数据, 继续增量更新 # 加入这个判断的原因是因为如果股票是刚上市的 数据库会没有数据 所以会有负索引问题出现 if ref.count() > 0: # 接着上次获取的日期继续更新 start_date_new_format = ref[ref.count() - 1]['trade_date'] start_date = ref[ref.count() - 1]['date'] if date_conver_to_new_format( QA_util_get_next_day(start_date, n=100)) < end_date: QA_util_log_info( 'LAST STOCK_DAY 100 days ago {}, skip. To resume please delete {}' .format(start_date_new_format, code), ui_log) else: QA_util_log_info( 'UPDATE_BOND_DAY \n Trying updating {} from {} to {}'. format(code, start_date_new_format, end_date), ui_log) if start_date_new_format != end_date: pandas_data = QA_util_to_json_from_pandas( QA_fetch_get_stock_day( str(code), date_conver_to_new_format( QA_util_get_next_day(start_date)), end_date, 'bfq')) if len(pandas_data) > 0: coll_stock_day.insert_many(pandas_data) # 当前数据库中没有这个代码的股票数据, 从1990-01-01 开始下载所有的数据 else: start_date = '19900101' QA_util_log_info( 'UPDATE_STOCK_DAY \n Trying updating {} from {} to {}'. format(code, start_date, end_date), ui_log) if start_date != end_date: pandas_data = QA_util_to_json_from_pandas( QA_fetch_get_stock_day(str(code), start_date, end_date, 'bfq')) if len(pandas_data) > 0: coll_stock_day.insert_many(pandas_data) except Exception as e: print(e) err.append(str(code))
def saving_work(i): QA_util_log_info('Now Saving ==== %s' % (i)) try: data_json = QA_fetch_get_stock_day(i, startDate='1990-01-01') __coll.insert_many(data_json) except: QA_util_log_info('error in saving ==== %s' % str(i))
def saving_work(i): QA_util_log_info('Now Saving ==== %s' % (i)) try: data_hfq = QA_fetch_get_stock_day( i, start='1990-01-01', if_fq='02', type_='pd') data_json = QA_util_to_json_from_pandas(data_hfq) __coll.insert_many(data_json) except: QA_util_log_info('error in saving ==== %s' % str(i))
def _saving_work(code, coll_stock_day, ui_log=None, err=[]): try: QA_util_log_info( '##JOB01 Now Saving STOCK_DAY==== {}'.format(str(code)), ui_log) # 首选查找数据库 是否 有 这个代码的数据 ref = coll_stock_day.find({'code': str(code)[0:9]}) end_date = now_time() # 当前数据库已经包含了这个代码的数据, 继续增量更新 # 加入这个判断的原因是因为如果股票是刚上市的 数据库会没有数据 所以会有负索引问题出现 if ref.count() > 0: # 接着上次获取的日期继续更新 start_date_new_format = ref[ref.count() - 1]['trade_date'] start_date = ref[ref.count() - 1]['date'] QA_util_log_info( 'UPDATE_STOCK_DAY \n Trying updating {} from {} to {}'.format( code, start_date, end_date), ui_log) if start_date_new_format != end_date: coll_stock_day.insert_many( QA_util_to_json_from_pandas( QA_fetch_get_stock_day( str(code), date_conver_to_new_format( QA_util_get_next_day(start_date)), end_date, 'qfq'))) # 当前数据库中没有这个代码的股票数据, 从1990-01-01 开始下载所有的数据 else: start_date = '19900101' QA_util_log_info( 'UPDATE_STOCK_DAY \n Trying updating {} from {} to {}'.format( code, start_date, end_date), ui_log) if start_date != end_date: coll_stock_day.insert_many( QA_util_to_json_from_pandas( QA_fetch_get_stock_day(str(code), start_date, end_date, 'qfq'))) except Exception as e: print(e) err.append(str(code))
def __saving_work(self, code): try: QA_util_log_info( '##JOB01 Now Saving STOCK_DAY==== {}'.format(str(code)), self.ui_log) time.sleep(0.2) # 首选查找数据库 是否 有 这个代码的数据 search_cond = {'code': str(code)[0:6]} ref = get_coll(self.client).find(search_cond) ref_count = get_coll(self.client).count_documents(search_cond) end_date = now_time() # 当前数据库已经包含了这个代码的数据, 继续增量更新 # 加入这个判断的原因是因为如果股票是刚上市的 数据库会没有数据 所以会有负索引问题出现 if ref_count > 0: # 接着上次获取的日期继续更新 start_date_new_format = ref[ref_count - 1]['trade_date'] start_date = ref[ref_count - 1]['date'] QA_util_log_info( 'UPDATE_STOCK_DAY: Trying updating {} {} from {} to {}'. format(ref_count, code, start_date_new_format, end_date), self.ui_log) if start_date_new_format != end_date: df = QA_fetch_get_stock_day( str(code), date_conver_to_new_format( QA_util_get_next_day(start_date)), end_date, 'bfq') if not (df is None) and len(df) > 0: get_coll(self.client).insert_many( QA_util_to_json_from_pandas(df)) else: QA_util_log_info( 'UPDATE_STOCK_DAY: Trying updating {} {} from {} to {} ---- No Data' .format(ref_count, code, start_date_new_format, end_date), self.ui_log) # 当前数据库中没有这个代码的股票数据, 从1990-01-01 开始下载所有的数据 # 一次只返回 5000 条,所以有些股票得获取两次 else: start_date = '19900101' QA_util_log_info( 'FETCH_ALL_STOCK_DAY: Trying fetching all data {} from {} to {}' .format(code, start_date, end_date), self.ui_log) if start_date != end_date: # 第一次获取 df = QA_fetch_get_stock_day(str(code), start_date, end_date, 'bfq') if not (df is None): if len(df) >= 5000: again_date = QA_util_get_last_day(df['date'].min()) QA_util_log_info( 'FETCH_ALL_STOCK_DAY: Trying updating again {} from {} to {}' .format(code, start_date, again_date), self.ui_log) df2 = QA_fetch_get_stock_day( str(code), start_date, again_date, 'bfq') df = df.append(df2) get_coll(self.client).insert_many( QA_util_to_json_from_pandas(df)) else: QA_util_log_info( 'FETCH_ALL_STOCK_DAY: Trying fetching all data {} from {} to {} ---- No Data' .format(code, start_date, end_date), self.ui_log) except Exception as e: print(e) self.err.append(str(code))