def record(self, entity, start, end, size, timestamps): param = self.generate_request_param(entity, start, end, size, timestamps) # to_time_str( # self.data_schema.query_data(filters=[self.data_schema.report_date>='20200101'], # entity_id=entity.id, columns=['report_date']).report_date.max()) >= to_time_str( # '20200101') columns_map = { key: value[0] for key, value in self.get_data_map().items() } columns_list = list(columns_map.values()) em_code = to_em_entity_id(entity) df = pd.DataFrame() for reportdate in param: em_data = c.css( em_code, columns_list, "ispandas=1,TtmType=1,TradeDate=" + to_time_str(reportdate) + ",ReportDate=" + to_time_str(reportdate)) if type(em_data) == pd.DataFrame: em_data['report_date'] = to_time_str(reportdate) df = df.append(em_data) if df.empty: return None df.rename(columns={value: key for key, value in columns_map.items()}, inplace=True) df = df.sort_values("report_date", ascending=True) if pd_is_not_null(df): df.rename( columns={value: key for key, value in columns_map.items()}, inplace=True) df['entity_id'] = entity.id df['provider'] = 'emquantapi' df['code'] = entity.code df['report_period'] = df['report_date'].apply( lambda x: to_report_period_type(x)) def generate_id(se): return "{}_{}".format( se['entity_id'], to_time_str(se['report_date'], fmt=TIME_FORMAT_DAY)) df['id'] = df[['entity_id', 'report_date']].apply(generate_id, axis=1) data_pub_date = BalanceSheet.query_data(entity_id=entity.id, columns=['pub_date', 'id']) del data_pub_date['timestamp'] df = pd.merge(df, data_pub_date, on=['id']) df['timestamp'] = df['pub_date'] df_to_db(df=df, data_schema=self.data_schema, provider=self.provider, force_update=True) return None
def record(self, entity, start, end, size, timestamps): param = self.generate_request_param(entity, start, end, size, timestamps) if not end: end = to_time_str(now_pd_timestamp()) start = to_time_str(start) em_code = to_em_entity_id(entity) df = pd.DataFrame() columns_map = { key: value[0] for key, value in self.get_data_map().items() } columns_list = list(columns_map.values()) for reportdate in param: # 获取数据 # 三大财务报表 使用ctr方法读取表名 if self.data_type < 4: em_data = c.ctr( self.finance_report_type, columns_list, "secucode=" + em_code + ",ReportDate=" + reportdate + ",ReportType=1") if em_data.Data == {}: continue data = pd.DataFrame(em_data.Data['0']).T data.columns = em_data.Indicators data['report_date'] = reportdate df = df.append(data) if df.empty: return None df.rename(columns={value: key for key, value in columns_map.items()}, inplace=True) df = df.sort_values("report_date", ascending=True) if pd_is_not_null(df): df.rename( columns={value: key for key, value in columns_map.items()}, inplace=True) df['entity_id'] = entity.id df['timestamp'] = pd.to_datetime(df.report_date) df['provider'] = 'emquantapi' df['code'] = entity.code df['report_period'] = df['report_date'].apply( lambda x: to_report_period_type(x)) def generate_id(se): return "{}_{}".format( se['entity_id'], to_time_str(se['timestamp'], fmt=TIME_FORMAT_DAY)) df['id'] = df[['entity_id', 'timestamp']].apply(generate_id, axis=1) df_to_db(df=df, data_schema=self.data_schema, provider=self.provider, force_update=self.force_update) return None
def record(self, entity, start, end, size, timestamps): for timestamp in timestamps: the_date = to_time_str(timestamp) result = get_free_holders(code=entity.code, end_date=the_date) if result: holders = [] new_actors = [] for item in result: # {'END_DATE': '2021-03-31 00:00:00', # 'FREE_HOLDNUM_RATIO': 0.631949916991, # 'FREE_RATIO_QOQ': '-5.33046217', # 'HOLDER_CODE': '161606', # 'HOLDER_CODE_OLD': '161606', # 'HOLDER_NAME': '交通银行-融通行业景气证券投资基金', # 'HOLDER_RANK': 10, # 'HOLD_NUM': 39100990, # 'IS_HOLDORG': '1', # 'SECUCODE': '000338.SZ'} # 机构 if item['IS_HOLDORG'] == '1': domains: List[ActorMeta] = ActorMeta.query_data(filters=[ActorMeta.code == item['HOLDER_CODE']], return_type='domain') if not domains: actor_type = ActorType.corporation.value actor = ActorMeta(entity_id=f'{actor_type}_cn_{item["HOLDER_CODE"]}', id=f'{actor_type}_cn_{item["HOLDER_CODE"]}', entity_type=actor_type, exchange='cn', code=item["HOLDER_CODE"], name=item["HOLDER_NAME"]) else: actor = domains[0] else: actor_type = ActorType.individual.value actor = ActorMeta(entity_id=f'{actor_type}_cn_{item["HOLDER_NAME"]}', id=f'{actor_type}_cn_{item["HOLDER_NAME"]}', entity_type=actor_type, exchange='cn', code=item["HOLDER_NAME"], name=item["HOLDER_NAME"]) new_actors.append(actor.__dict__) holder = {'id': f'{entity.entity_id}_{the_date}_{actor.entity_id}', 'entity_id': entity.entity_id, 'timestamp': timestamp, 'code': entity.code, 'name': entity.name, 'actor_id': actor.entity_id, 'actor_type': actor.entity_type, 'actor_code': actor.code, 'actor_name': actor.name, 'report_date': timestamp, 'report_period': to_report_period_type(timestamp), 'holding_numbers': item['HOLD_NUM'], 'holding_ratio': item['FREE_HOLDNUM_RATIO']} holders.append(holder) if holders: df = pd.DataFrame.from_records(holders) df_to_db(data_schema=self.data_schema, df=df, provider=self.provider, force_update=True) if new_actors: df = pd.DataFrame.from_records(new_actors) df_to_db(data_schema=ActorMeta, df=df, provider=self.provider, force_update=False)
def record(self, entity, start, end, size, timestamps): for timestamp in timestamps: the_date = to_time_str(timestamp) result = get_holders(code=entity.code, end_date=the_date) if result: holders = [] new_actors = [] for item in result: # 机构 if item['IS_HOLDORG'] == '1': domains: List[ActorMeta] = ActorMeta.query_data( filters=[ActorMeta.code == item['HOLDER_CODE']], return_type='domain') if not domains: actor_type = ActorType.corporation.value actor = ActorMeta( entity_id= f'{actor_type}_cn_{item["HOLDER_CODE"]}', id=f'{actor_type}_cn_{item["HOLDER_CODE"]}', entity_type=actor_type, exchange='cn', code=item["HOLDER_CODE"], name=item["HOLDER_NAME"]) else: actor = domains[0] else: actor_type = ActorType.individual.value actor = ActorMeta( entity_id=f'{actor_type}_cn_{item["HOLDER_NAME"]}', id=f'{actor_type}_cn_{item["HOLDER_NAME"]}', entity_type=actor_type, exchange='cn', code=item["HOLDER_NAME"], name=item["HOLDER_NAME"]) new_actors.append(actor.__dict__) holder = { 'id': f'{entity.entity_id}_{the_date}_{actor.entity_id}', 'entity_id': entity.entity_id, 'timestamp': timestamp, 'code': entity.code, 'name': entity.name, 'actor_id': actor.entity_id, 'actor_type': actor.entity_type, 'actor_code': actor.code, 'actor_name': actor.name, 'report_date': timestamp, 'report_period': to_report_period_type(timestamp), 'holding_numbers': item['HOLD_NUM'], 'holding_ratio': item['HOLD_NUM_RATIO'] } holders.append(holder) if holders: df = pd.DataFrame.from_records(holders) df_to_db(data_schema=self.data_schema, df=df, provider=self.provider, force_update=True) if new_actors: df = pd.DataFrame.from_records(new_actors) df_to_db(data_schema=ActorMeta, df=df, provider=self.provider, force_update=False)