def __saving_work(code, __coll): QA_util_log_info('##JOB02 Now Saving XDXR INFO ==== %s' % (str(code))) try: __coll.insert_many( QA_util_to_json_from_pandas(QA_fetch_get_stock_xdxr( str(code)))) except: __err.append(str(code))
def __saving_work(code, coll): QA_util_log_info('##JOB02 Now Saving XDXR INFO ==== {}'.format( str(code)), ui_log=ui_log) try: coll.insert_many( QA_util_to_json_from_pandas( QA_fetch_get_stock_xdxr(str(code)))) except: err.append(str(code))
def _saving_work(self, code): def __QA_log_info(code, end_time, start_time): def loginfo(prefix='', astr='', listCounts=5): if len(self._loginfolist) < listCounts: self._loginfolist.append(astr) else: str = '' for i in range(len(self._loginfolist)): str += self._loginfolist[i] + ' ' str += astr QA_util_log_info(prefix.format(str), self.ui_log) self._loginfolist.clear() prefix = '##JOB02 Saving parallelism {}_DAY ==== Trying updating\n{}'.format( "xdxr", '{}') loginfo(prefix, ' {} from {} to {}'.format(code, start_time, end_time)) try: search_cond = {'code': str(code)[0:6]} _col = get_coll(client=None, cacheName="stock_xdxr", tableName="stock_xdxr") ref_ = _col.find(search_cond) ref_count = _col.count_documents(search_cond) end_time = str(now_time())[0:10] _xdxr = QA_fetch_get_stock_xdxr(str(code)) if ref_count > 0: start_time = ref_[ref_count - 1]['date'] __QA_log_info(code, end_time, start_time) if start_time != end_time: # 最后一次保存的数据之后的数据 xdxrdata = _xdxr[_xdxr['date'] > start_time] if len(xdxrdata) > 0: _col.insert_many(QA_util_to_json_from_pandas(xdxrdata), ordered=False) else: # 第一次保存数据 try: start_time = '1990-01-01' __QA_log_info(code, end_time, start_time) _col.insert_many(QA_util_to_json_from_pandas(_xdxr), ordered=False) except Exception as e: pass except Exception as e: QA_util_log_info(e.args, ui_log=self.ui_log) self.err.append(str(code)) QA_util_log_info(self.err, ui_log=self.ui_log) try: # 插入前复权数据 coll_adj = get_coll(client=None, cacheName="stock_adj", tableName="stock_adj") data = QA_fetch_stock_day(str(code), '1990-01-01', str(datetime.date.today()), 'pd') qfq = _QA_data_stock_to_fq(data, _xdxr, 'qfq') qfq = qfq.assign(date=qfq.date.apply(lambda x: str(x)[0:10])) adjdata = QA_util_to_json_from_pandas( qfq.loc[:, ['date', 'code', 'adj']]) # 如果没有新增幅圈数据,插入新增数据 ref_ = coll_adj.find(search_cond) ref_count = coll_adj.count_documents(search_cond) if ref_count > 0: # 历史保存过数据 lastref = ref_[ref_count - 1] del lastref['_id'] try: # 存在相同的数据,则保存最近数据 adjdata.index(lastref) # 找不到对应的数据,会出发异常 start_time = lastref['date'] adjdata2 = QA_util_to_json_from_pandas(qfq.loc[slice( pd.Timestamp(start_time), pd.Timestamp( end_time))].loc[:, ['date', 'code', 'adj']][1:]) if (ref_count + len(adjdata2)) == len(adjdata): # if len(adjdata2) > 0: coll_adj.insert_many(adjdata2) else: raise Exception("数据总量不匹配,重新存储") except Exception as e: # 分红数据有变化,删除以前保存的数据,重新保存新的数据 coll_adj.delete_many({'code': code}) coll_adj.insert_many(adjdata) else: # 第一次保存前复权数据 # print(adjdata) coll_adj.insert_many(adjdata) except Exception as e: print(e)