def record(self, entity, start, end, size, timestamps):
start = to_time_str(start, fmt=TIME_FORMAT_DAY1)
end = now_time_str(fmt=TIME_FORMAT_DAY1)
if entity.exchange == 'sh':
exchange_flag = 0
else:
exchange_flag = 1
url = self.url.format(exchange_flag, entity.code, start, end)
response = requests.get(url=url)
df = read_csv(io.BytesIO(response.content), encoding='GB2312', na_values='None')
if df is None:
return []
df['name'] = entity.name
# 指数数据
if entity.entity_type == 'index':
df = df.loc[:,
['日期', 'name', '最低价', '开盘价', '收盘价', '最高价', '成交量', '成交金额', '涨跌幅']]
df.columns = ['timestamp', 'name', 'low', 'open', 'close', 'high', 'volume', 'turnover', 'change_pct']
# 股票数据
else:
df = df.loc[:,
['日期', 'name', '最低价', '开盘价', '收盘价', '最高价', '成交量', '成交金额', '涨跌幅', '换手率']]
df.columns = ['timestamp', 'name', 'low', 'open', 'close', 'high', 'volume', 'turnover', 'change_pct',
'turnover_rate']
df['timestamp'] = pd.to_datetime(df['timestamp'])
df['provider'] = 'netease'
df['level'] = self.level.value
return df.to_dict(orient='records')
def on_finish_entity(self, entity):
kdatas = get_kdata(provider=self.provider, entity_id=entity.id, level=self.level.value,
order=self.data_schema.timestamp.asc(),
return_type='domain',
session=self.session,
filters=[self.data_schema.hfq_close.is_(None),
self.data_schema.timestamp >= to_pd_timestamp('2005-01-01')])
if kdatas:
start = kdatas[0].timestamp
end = kdatas[-1].timestamp
# get hfq from joinquant
df = get_price(to_jq_entity_id(entity), start_date=to_time_str(start), end_date=now_time_str(),
frequency='daily',
fields=['factor', 'open', 'close', 'low', 'high'],
skip_paused=True, fq='post')
if df is not None and not df.empty:
# fill hfq data
for kdata in kdatas:
time_str = to_time_str(kdata.timestamp)
if time_str in df.index:
kdata.hfq_open = df.loc[time_str, 'open']
kdata.hfq_close = df.loc[time_str, 'close']
kdata.hfq_high = df.loc[time_str, 'high']
kdata.hfq_low = df.loc[time_str, 'low']
kdata.factor = df.loc[time_str, 'factor']
self.session.commit()
latest_factor = df.factor[-1]
# factor not change yet, no need to reset the qfq past
if latest_factor == self.current_factors.get(entity.id):
sql = 'UPDATE {} SET qfq_close=hfq_close/{},qfq_high=hfq_high/{}, qfq_open= hfq_open/{}, qfq_low= hfq_low/{} where ' \
'entity_id=\'{}\' and level=\'{}\' and (qfq_close isnull or qfq_high isnull or qfq_low isnull or qfq_open isnull)'.format(
self.data_schema.__table__, latest_factor, latest_factor, latest_factor, latest_factor,
entity.id, self.level.value)
else:
sql = 'UPDATE {} SET qfq_close=hfq_close/{},qfq_high=hfq_high/{}, qfq_open= hfq_open/{}, qfq_low= hfq_low/{} where ' \
'entity_id=\'{}\' and level=\'{}\''.format(self.data_schema.__table__, latest_factor,
latest_factor, latest_factor, latest_factor,
entity.id,
self.level.value)
self.logger.info(sql)
self.session.execute(sql)
self.session.commit()
# use netease provider to get turnover_rate
query_url = 'http://quotes.money.163.com/service/chddata.html?code={}{}&start={}&end={}&fields=PCHG;TURNOVER'
if entity.exchange == 'sh':
exchange_flag = 0
else:
exchange_flag = 1
url = query_url.format(exchange_flag, entity.code, to_time_str(start), to_time_str(end))
response = requests.get(url=url)
df = read_csv(io.BytesIO(response.content), encoding='GB2312', na_values='None')
df['日期'] = pd.to_datetime(df['日期'])
df.set_index('日期', drop=True, inplace=True)
if df is not None and not df.empty:
# fill turnover_rate, pct_change
for kdata in kdatas:
if kdata.timestamp in df.index:
kdata.turnover_rate = df.loc[kdata.timestamp, '换手率']
kdata.change_pct = df.loc[kdata.timestamp, '涨跌幅']
self.session.commit()