def record(self, entity, start, end, size, timestamps):
if self.start_timestamp:
start = max(self.start_timestamp, to_pd_timestamp(start))
end = now_pd_timestamp() + timedelta(days=1)
start_timestamp = to_time_str(start)
end_timestamp = to_time_str(end)
# 不复权
df = get_price(to_jq_entity_id(entity), start_date=to_time_str(start_timestamp),
end_date=end_timestamp,
frequency=self.jq_trading_level,
fields=['open', 'close', 'low', 'high', 'volume', 'money'],
skip_paused=True, fq=None)
df.index.name = 'timestamp'
df.reset_index(inplace=True)
df['name'] = entity.name
df.rename(columns={'money': 'turnover'}, inplace=True)
df['timestamp'] = pd.to_datetime(df['timestamp'])
df['provider'] = 'joinquant'
df['level'] = self.level.value
# remove the unfinished kdata
if is_in_trading(entity_type='stock', exchange='sh', timestamp=df.iloc[-1, :]['timestamp']):
df = df.iloc[:-1, :]
return df.to_dict(orient='records')
def on_finish_entity(self, entity):
kdatas = get_kdata(
entity_id=entity.id,
level=self.level.value,
order=Stock1dKdata.timestamp.asc(),
return_type='domain',
session=self.session,
filters=[
Stock1dKdata.factor.is_(None),
Stock1dKdata.timestamp >= to_pd_timestamp('2005-01-01')
])
if kdatas:
start = kdatas[0].timestamp
end = kdatas[-1].timestamp
# get hfq from joinquant
df = get_price(to_jq_entity_id(entity),
start_date=to_time_str(start),
end_date=now_time_str(),
frequency='daily',
fields=['factor', 'open', 'close', 'low', 'high'],
skip_paused=True,
fq='post')
if df is not None and not df.empty:
# fill hfq data
for kdata in kdatas:
if kdata.timestamp in df.index:
kdata.hfq_open = df.loc[kdata.timestamp, 'open']
kdata.hfq_close = df.loc[kdata.timestamp, 'close']
kdata.hfq_high = df.loc[kdata.timestamp, 'high']
kdata.hfq_low = df.loc[kdata.timestamp, 'low']
kdata.factor = df.loc[kdata.timestamp, 'factor']
self.session.commit()
latest_factor = df.factor[-1]
# factor not change yet, no need to reset the qfq past
if latest_factor == self.current_factors.get(entity.id):
sql = 'UPDATE stock_1d_kdata SET qfq_close=hfq_close/{},qfq_high=hfq_high/{}, qfq_open= hfq_open/{}, qfq_low= hfq_low/{} where ' \
'entity_id=\'{}\' and level=\'{}\' and (qfq_close isnull or qfq_high isnull or qfq_low isnull or qfq_open isnull)'.format(
latest_factor, latest_factor, latest_factor, latest_factor, entity.id, self.level.value)
else:
sql = 'UPDATE stock_1d_kdata SET qfq_close=hfq_close/{},qfq_high=hfq_high/{}, qfq_open= hfq_open/{}, qfq_low= hfq_low/{} where ' \
'entity_id=\'{}\' and level=\'{}\''.format(latest_factor,
latest_factor,
latest_factor,
latest_factor,
entity.id,
self.level.value)
self.logger.info(sql)
self.session.execute(sql)
self.session.commit()
def fill_timestamp_with_jq(self, security_item, the_data):
# get report published date from jq
q = query(indicator.pubDate).filter(
indicator.code == to_jq_entity_id(security_item), )
df = get_fundamentals(q,
statDate=to_jq_report_period(
the_data.report_date))
if not df.empty:
the_data.timestamp = to_pd_timestamp(df['pubDate'][0])
self.logger.info(
'jq fill {} {} timestamp:{} for report_date:{}'.format(
self.data_schema, security_item.id, the_data.timestamp,
the_data.report_date))
self.session.commit()
def on_finish_entity(self, entity):
kdatas = get_kdata(provider=self.provider, entity_id=entity.id, level=self.level.value,
order=self.data_schema.timestamp.asc(),
return_type='domain',
session=self.session,
filters=[self.data_schema.hfq_close.is_(None),
self.data_schema.timestamp >= to_pd_timestamp('2005-01-01')])
if kdatas:
start = kdatas[0].timestamp
end = kdatas[-1].timestamp
# get hfq from joinquant
df = get_price(to_jq_entity_id(entity), start_date=to_time_str(start), end_date=now_time_str(),
frequency='daily',
fields=['factor', 'open', 'close', 'low', 'high'],
skip_paused=True, fq='post')
if df is not None and not df.empty:
# fill hfq data
for kdata in kdatas:
time_str = to_time_str(kdata.timestamp)
if time_str in df.index:
kdata.hfq_open = df.loc[time_str, 'open']
kdata.hfq_close = df.loc[time_str, 'close']
kdata.hfq_high = df.loc[time_str, 'high']
kdata.hfq_low = df.loc[time_str, 'low']
kdata.factor = df.loc[time_str, 'factor']
self.session.commit()
latest_factor = df.factor[-1]
# factor not change yet, no need to reset the qfq past
if latest_factor == self.current_factors.get(entity.id):
sql = 'UPDATE {} SET qfq_close=hfq_close/{},qfq_high=hfq_high/{}, qfq_open= hfq_open/{}, qfq_low= hfq_low/{} where ' \
'entity_id=\'{}\' and level=\'{}\' and (qfq_close isnull or qfq_high isnull or qfq_low isnull or qfq_open isnull)'.format(
self.data_schema.__table__, latest_factor, latest_factor, latest_factor, latest_factor,
entity.id, self.level.value)
else:
sql = 'UPDATE {} SET qfq_close=hfq_close/{},qfq_high=hfq_high/{}, qfq_open= hfq_open/{}, qfq_low= hfq_low/{} where ' \
'entity_id=\'{}\' and level=\'{}\''.format(self.data_schema.__table__, latest_factor,
latest_factor, latest_factor, latest_factor,
entity.id,
self.level.value)
self.logger.info(sql)
self.session.execute(sql)
self.session.commit()
# use netease provider to get turnover_rate
query_url = 'http://quotes.money.163.com/service/chddata.html?code={}{}&start={}&end={}&fields=PCHG;TURNOVER'
if entity.exchange == 'sh':
exchange_flag = 0
else:
exchange_flag = 1
url = query_url.format(exchange_flag, entity.code, to_time_str(start), to_time_str(end))
response = requests.get(url=url)
df = read_csv(io.BytesIO(response.content), encoding='GB2312', na_values='None')
df['日期'] = pd.to_datetime(df['日期'])
df.set_index('日期', drop=True, inplace=True)
if df is not None and not df.empty:
# fill turnover_rate, pct_change
for kdata in kdatas:
if kdata.timestamp in df.index:
kdata.turnover_rate = df.loc[kdata.timestamp, '换手率']
kdata.change_pct = df.loc[kdata.timestamp, '涨跌幅']
self.session.commit()