def record(self, entity, start, end, size, timestamps): start = to_time_str(start, fmt=TIME_FORMAT_DAY1) end = now_time_str(fmt=TIME_FORMAT_DAY1) if entity.exchange == 'sh': exchange_flag = 0 else: exchange_flag = 1 url = self.url.format(exchange_flag, entity.code, start, end) response = requests.get(url=url) df = read_csv(io.BytesIO(response.content), encoding='GB2312', na_values='None') if df is None: return [] df['name'] = entity.name # 指数数据 if entity.entity_type == 'index': df = df.loc[:, ['日期', 'name', '最低价', '开盘价', '收盘价', '最高价', '成交量', '成交金额', '涨跌幅']] df.columns = ['timestamp', 'name', 'low', 'open', 'close', 'high', 'volume', 'turnover', 'change_pct'] # 股票数据 else: df = df.loc[:, ['日期', 'name', '最低价', '开盘价', '收盘价', '最高价', '成交量', '成交金额', '涨跌幅', '换手率']] df.columns = ['timestamp', 'name', 'low', 'open', 'close', 'high', 'volume', 'turnover', 'change_pct', 'turnover_rate'] df['timestamp'] = pd.to_datetime(df['timestamp']) df['provider'] = 'netease' df['level'] = self.level.value return df.to_dict(orient='records')
def on_finish_entity(self, entity): kdatas = get_kdata(provider=self.provider, entity_id=entity.id, level=self.level.value, order=self.data_schema.timestamp.asc(), return_type='domain', session=self.session, filters=[self.data_schema.hfq_close.is_(None), self.data_schema.timestamp >= to_pd_timestamp('2005-01-01')]) if kdatas: start = kdatas[0].timestamp end = kdatas[-1].timestamp # get hfq from joinquant df = get_price(to_jq_entity_id(entity), start_date=to_time_str(start), end_date=now_time_str(), frequency='daily', fields=['factor', 'open', 'close', 'low', 'high'], skip_paused=True, fq='post') if df is not None and not df.empty: # fill hfq data for kdata in kdatas: time_str = to_time_str(kdata.timestamp) if time_str in df.index: kdata.hfq_open = df.loc[time_str, 'open'] kdata.hfq_close = df.loc[time_str, 'close'] kdata.hfq_high = df.loc[time_str, 'high'] kdata.hfq_low = df.loc[time_str, 'low'] kdata.factor = df.loc[time_str, 'factor'] self.session.commit() latest_factor = df.factor[-1] # factor not change yet, no need to reset the qfq past if latest_factor == self.current_factors.get(entity.id): sql = 'UPDATE {} SET qfq_close=hfq_close/{},qfq_high=hfq_high/{}, qfq_open= hfq_open/{}, qfq_low= hfq_low/{} where ' \ 'entity_id=\'{}\' and level=\'{}\' and (qfq_close isnull or qfq_high isnull or qfq_low isnull or qfq_open isnull)'.format( self.data_schema.__table__, latest_factor, latest_factor, latest_factor, latest_factor, entity.id, self.level.value) else: sql = 'UPDATE {} SET qfq_close=hfq_close/{},qfq_high=hfq_high/{}, qfq_open= hfq_open/{}, qfq_low= hfq_low/{} where ' \ 'entity_id=\'{}\' and level=\'{}\''.format(self.data_schema.__table__, latest_factor, latest_factor, latest_factor, latest_factor, entity.id, self.level.value) self.logger.info(sql) self.session.execute(sql) self.session.commit() # use netease provider to get turnover_rate query_url = 'http://quotes.money.163.com/service/chddata.html?code={}{}&start={}&end={}&fields=PCHG;TURNOVER' if entity.exchange == 'sh': exchange_flag = 0 else: exchange_flag = 1 url = query_url.format(exchange_flag, entity.code, to_time_str(start), to_time_str(end)) response = requests.get(url=url) df = read_csv(io.BytesIO(response.content), encoding='GB2312', na_values='None') df['日期'] = pd.to_datetime(df['日期']) df.set_index('日期', drop=True, inplace=True) if df is not None and not df.empty: # fill turnover_rate, pct_change for kdata in kdatas: if kdata.timestamp in df.index: kdata.turnover_rate = df.loc[kdata.timestamp, '换手率'] kdata.change_pct = df.loc[kdata.timestamp, '涨跌幅'] self.session.commit()