def generate_request_param(self, security_item, start, end, size,
timestamps, http_session):
comp_type = company_type_flag(security_item, http_session)
if len(timestamps) <= 10:
param = {
"color": "w",
"fc": get_fc(security_item),
"corpType": comp_type,
# 0 means get all types
"reportDateType": 0,
"endDate": '',
"latestCount": size
}
else:
param = {
"color": "w",
"fc": get_fc(security_item),
"corpType": comp_type,
# 0 means get all types
"reportDateType": 0,
"endDate": to_time_str(timestamps[10]),
"latestCount": 10
}
if self.finance_report_type == 'LiRunBiaoList' or self.finance_report_type == 'XianJinLiuLiangBiaoList':
param['reportType'] = 1
return param
async def persist(self, entity, http_session, db_session, para):
start_point = time.time()
(ref_record, df_record) = para
saved_counts = 0
is_finished = False
if pd_valid(df_record):
assert 'id' in df_record.columns
saved_counts = await df_to_db(region=self.region,
provider=self.provider,
data_schema=self.data_schema,
db_session=db_session,
df=df_record,
ref_df=ref_record,
fix_duplicate_way=self.fix_duplicate_way)
if saved_counts == 0:
is_finished = True
# could not get more data
else:
# not realtime
if not self.real_time:
is_finished = True
# realtime and to the close time
elif (self.close_hour is not None) and (self.close_minute is not None):
now = now_pd_timestamp(self.region)
if now.hour >= self.close_hour:
if now.minute - self.close_minute >= 5:
self.logger.info(f'{entity.id} now is the close time: {now}')
is_finished = True
if isinstance(self, KDataRecorder):
is_finished = True
start_timestamp = to_time_str(df_record['timestamp'].min(axis=0))
end_timestamp = to_time_str(df_record['timestamp'].max(axis=0))
self.result = [saved_counts, start_timestamp, end_timestamp]
return is_finished, time.time() - start_point, saved_counts
async def record(self, entity, http_session, db_session, para):
start_point = time.time()
calendar = mcal.get_calendar(entity.upper())
trade_day, column_names = StockTradeDay.query_data(
region=self.region,
provider=self.provider,
db_session=db_session,
func=func.max(StockTradeDay.timestamp))
start = to_time_str(trade_day) if trade_day else "1980-01-01"
dates = calendar.schedule(start_date=start, end_date=to_time_str(now_pd_timestamp(Region.US)))
dates = dates.index.to_list()
self.logger.info(f'add dates:{dates}')
if len(dates) > 0:
df = pd.DataFrame(dates, columns=['timestamp'])
return False, time.time() - start_point, self.format(entity, df)
return True, time.time() - start_point, None
async def record(self, entity, http_session, db_session, para):
start_point = time.time()
(ref_record, start, end, size, timestamps) = para
start = to_time_str(start)
if self.bao_trading_level in ['d', 'w', 'm']:
start = max(start, "1990-12-19")
else:
start = max(start, "1999-07-26")
df = self.bao_get_bars(to_bao_entity_id(entity),
start=start,
end=end if end is None else to_time_str(end),
frequency=self.bao_trading_level,
fields=to_bao_trading_field(self.bao_trading_level),
adjustflag=to_bao_adjust_flag(self.adjust_type))
if pd_valid(df):
return False, time.time() - start_point, (ref_record, self.format(entity, df))
return True, time.time() - start_point, None
async def record(self, entity, http_session, db_session, para):
start_point = time.time()
(ref_record, start, end, size, timestamps) = para
end_timestamp = to_time_str(
self.end_timestamp) if self.end_timestamp else None
df = await self.yh_get_bars(http_session,
entity,
start=start,
end=end_timestamp)
if pd_valid(df):
return False, time.time() - start_point, (ref_record,
self.format(entity, df))
return True, time.time() - start_point, None
async def record(self, entity, http_session, db_session, para):
start_point = time.time()
trade_day, column_names = StockTradeDay.query_data(
region=self.region,
provider=self.provider,
db_session=db_session,
func=func.max(StockTradeDay.timestamp))
start = to_time_str(trade_day) if trade_day else "1990-12-19"
df = self.bao_get_trade_days(start_date=start)
if pd_valid(df):
return False, time.time() - start_point, self.format(entity, df)
return True, time.time() - start_point, None
async def record(self, entity, http_session, db_session, para):
start_point = time.time()
(ref_record, start, end, size, timestamps) = para
json_results = []
for timestamp in timestamps:
timestamp_str = to_time_str(timestamp)
url = self.url.format(timestamp_str)
async with http_session.get(
url, headers=DEFAULT_SH_SUMMARY_HEADER) as response:
if response.status != 200:
return
text = await response.text()
if text is None:
continue
results = demjson.decode(text[text.index("(") +
1:text.index(")")])['result']
result = [
result for result in results
if result['productType'] == '1'
]
if result and len(result) == 1:
result_json = result[0]
# 有些较老的数据不存在,默认设为0.0
json_results.append({
'timestamp':
timestamp,
'pe':
to_float(result_json['profitRate'], 0.0),
'total_value':
to_float(result_json['marketValue1'] + '亿', 0.0),
'total_tradable_vaule':
to_float(result_json['negotiableValue1'] + '亿', 0.0),
'volume':
to_float(result_json['trdVol1'] + '万', 0.0),
'turnover':
to_float(result_json['trdAmt1'] + '亿', 0.0),
'turnover_rate':
to_float(result_json['exchangeRate'], 0.0),
})
if len(json_results) > self.batch_size:
df = pd.DataFrame.from_records(json_results)
df['entity_id'] = entity.id
df['provider'] = Provider.Exchange.value
df['timestamp'] = pd.to_datetime(df['timestamp'])
df['name'] = '上证指数'
df = self.format(df)
return False, time.time() - start_point, (ref_record,
df)
if len(json_results) > 0:
df = pd.DataFrame.from_records(json_results)
df = self.format(df)
return False, time.time() - start_point, (ref_record, df)
return True, time.time() - start_point, None