def record(self, entity, start, end, size, timestamps):
param = self.generate_request_param(entity, start, end, size,
timestamps)
# to_time_str(
# self.data_schema.query_data(filters=[self.data_schema.report_date>='20200101'],
# entity_id=entity.id, columns=['report_date']).report_date.max()) >= to_time_str(
# '20200101')
columns_map = {
key: value[0]
for key, value in self.get_data_map().items()
}
columns_list = list(columns_map.values())
em_code = to_em_entity_id(entity)
df = pd.DataFrame()
for reportdate in param:
em_data = c.css(
em_code, columns_list,
"ispandas=1,TtmType=1,TradeDate=" + to_time_str(reportdate) +
",ReportDate=" + to_time_str(reportdate))
if type(em_data) == pd.DataFrame:
em_data['report_date'] = to_time_str(reportdate)
df = df.append(em_data)
if df.empty:
return None
df.rename(columns={value: key
for key, value in columns_map.items()},
inplace=True)
df = df.sort_values("report_date", ascending=True)
if pd_is_not_null(df):
df.rename(
columns={value: key
for key, value in columns_map.items()},
inplace=True)
df['entity_id'] = entity.id
df['provider'] = 'emquantapi'
df['code'] = entity.code
df['report_period'] = df['report_date'].apply(
lambda x: to_report_period_type(x))
def generate_id(se):
return "{}_{}".format(
se['entity_id'],
to_time_str(se['report_date'], fmt=TIME_FORMAT_DAY))
df['id'] = df[['entity_id', 'report_date']].apply(generate_id,
axis=1)
data_pub_date = BalanceSheet.query_data(entity_id=entity.id,
columns=['pub_date', 'id'])
del data_pub_date['timestamp']
df = pd.merge(df, data_pub_date, on=['id'])
df['timestamp'] = df['pub_date']
df_to_db(df=df,
data_schema=self.data_schema,
provider=self.provider,
force_update=True)
return None
def record(self, entity, start, end, size, timestamps):
param = self.generate_request_param(entity, start, end, size,
timestamps)
if not end:
end = to_time_str(now_pd_timestamp())
start = to_time_str(start)
em_code = to_em_entity_id(entity)
df = pd.DataFrame()
columns_map = {
key: value[0]
for key, value in self.get_data_map().items()
}
columns_list = list(columns_map.values())
for reportdate in param:
# 获取数据
# 三大财务报表 使用ctr方法读取表名
if self.data_type < 4:
em_data = c.ctr(
self.finance_report_type, columns_list, "secucode=" +
em_code + ",ReportDate=" + reportdate + ",ReportType=1")
if em_data.Data == {}:
continue
data = pd.DataFrame(em_data.Data['0']).T
data.columns = em_data.Indicators
data['report_date'] = reportdate
df = df.append(data)
if df.empty:
return None
df.rename(columns={value: key
for key, value in columns_map.items()},
inplace=True)
df = df.sort_values("report_date", ascending=True)
if pd_is_not_null(df):
df.rename(
columns={value: key
for key, value in columns_map.items()},
inplace=True)
df['entity_id'] = entity.id
df['timestamp'] = pd.to_datetime(df.report_date)
df['provider'] = 'emquantapi'
df['code'] = entity.code
df['report_period'] = df['report_date'].apply(
lambda x: to_report_period_type(x))
def generate_id(se):
return "{}_{}".format(
se['entity_id'],
to_time_str(se['timestamp'], fmt=TIME_FORMAT_DAY))
df['id'] = df[['entity_id', 'timestamp']].apply(generate_id,
axis=1)
df_to_db(df=df,
data_schema=self.data_schema,
provider=self.provider,
force_update=self.force_update)
return None
def record(self, entity, start, end, size, timestamps):
for timestamp in timestamps:
the_date = to_time_str(timestamp)
result = get_free_holders(code=entity.code, end_date=the_date)
if result:
holders = []
new_actors = []
for item in result:
# {'END_DATE': '2021-03-31 00:00:00',
# 'FREE_HOLDNUM_RATIO': 0.631949916991,
# 'FREE_RATIO_QOQ': '-5.33046217',
# 'HOLDER_CODE': '161606',
# 'HOLDER_CODE_OLD': '161606',
# 'HOLDER_NAME': '交通银行-融通行业景气证券投资基金',
# 'HOLDER_RANK': 10,
# 'HOLD_NUM': 39100990,
# 'IS_HOLDORG': '1',
# 'SECUCODE': '000338.SZ'}
# 机构
if item['IS_HOLDORG'] == '1':
domains: List[ActorMeta] = ActorMeta.query_data(filters=[ActorMeta.code == item['HOLDER_CODE']],
return_type='domain')
if not domains:
actor_type = ActorType.corporation.value
actor = ActorMeta(entity_id=f'{actor_type}_cn_{item["HOLDER_CODE"]}',
id=f'{actor_type}_cn_{item["HOLDER_CODE"]}',
entity_type=actor_type,
exchange='cn',
code=item["HOLDER_CODE"],
name=item["HOLDER_NAME"])
else:
actor = domains[0]
else:
actor_type = ActorType.individual.value
actor = ActorMeta(entity_id=f'{actor_type}_cn_{item["HOLDER_NAME"]}',
id=f'{actor_type}_cn_{item["HOLDER_NAME"]}',
entity_type=actor_type,
exchange='cn',
code=item["HOLDER_NAME"],
name=item["HOLDER_NAME"])
new_actors.append(actor.__dict__)
holder = {'id': f'{entity.entity_id}_{the_date}_{actor.entity_id}',
'entity_id': entity.entity_id,
'timestamp': timestamp,
'code': entity.code,
'name': entity.name,
'actor_id': actor.entity_id,
'actor_type': actor.entity_type,
'actor_code': actor.code,
'actor_name': actor.name,
'report_date': timestamp,
'report_period': to_report_period_type(timestamp),
'holding_numbers': item['HOLD_NUM'],
'holding_ratio': item['FREE_HOLDNUM_RATIO']}
holders.append(holder)
if holders:
df = pd.DataFrame.from_records(holders)
df_to_db(data_schema=self.data_schema, df=df, provider=self.provider,
force_update=True)
if new_actors:
df = pd.DataFrame.from_records(new_actors)
df_to_db(data_schema=ActorMeta, df=df, provider=self.provider,
force_update=False)
def record(self, entity, start, end, size, timestamps):
for timestamp in timestamps:
the_date = to_time_str(timestamp)
result = get_holders(code=entity.code, end_date=the_date)
if result:
holders = []
new_actors = []
for item in result:
# 机构
if item['IS_HOLDORG'] == '1':
domains: List[ActorMeta] = ActorMeta.query_data(
filters=[ActorMeta.code == item['HOLDER_CODE']],
return_type='domain')
if not domains:
actor_type = ActorType.corporation.value
actor = ActorMeta(
entity_id=
f'{actor_type}_cn_{item["HOLDER_CODE"]}',
id=f'{actor_type}_cn_{item["HOLDER_CODE"]}',
entity_type=actor_type,
exchange='cn',
code=item["HOLDER_CODE"],
name=item["HOLDER_NAME"])
else:
actor = domains[0]
else:
actor_type = ActorType.individual.value
actor = ActorMeta(
entity_id=f'{actor_type}_cn_{item["HOLDER_NAME"]}',
id=f'{actor_type}_cn_{item["HOLDER_NAME"]}',
entity_type=actor_type,
exchange='cn',
code=item["HOLDER_NAME"],
name=item["HOLDER_NAME"])
new_actors.append(actor.__dict__)
holder = {
'id':
f'{entity.entity_id}_{the_date}_{actor.entity_id}',
'entity_id': entity.entity_id,
'timestamp': timestamp,
'code': entity.code,
'name': entity.name,
'actor_id': actor.entity_id,
'actor_type': actor.entity_type,
'actor_code': actor.code,
'actor_name': actor.name,
'report_date': timestamp,
'report_period': to_report_period_type(timestamp),
'holding_numbers': item['HOLD_NUM'],
'holding_ratio': item['HOLD_NUM_RATIO']
}
holders.append(holder)
if holders:
df = pd.DataFrame.from_records(holders)
df_to_db(data_schema=self.data_schema,
df=df,
provider=self.provider,
force_update=True)
if new_actors:
df = pd.DataFrame.from_records(new_actors)
df_to_db(data_schema=ActorMeta,
df=df,
provider=self.provider,
force_update=False)
