def run(self): # 抓取fund列表 df = finance.run_query(query(finance.FUND_MAIN_INFO)) df.index.name = 'entity_id' df = df.reset_index() # 上市日期 df.rename(columns={'start_date': 'timestamp'}, inplace=True) df['timestamp'] = pd.to_datetime(df['timestamp']) df['list_date'] = df['timestamp'] df['end_date'] = pd.to_datetime(df['end_date']) df['entity_id'] = df.main_code.apply(lambda x: normalize_code(x)) df['entity_id'] = df['entity_id'].apply( lambda x: to_entity_id(entity_type='fund', jq_code=x)) df['id'] = df['entity_id'] df['entity_type'] = 'fund' df['exchange'] = df['entity_id'].apply( lambda x: get_entity_exchange(x)) df['code'] = df['entity_id'].apply(lambda x: get_entity_code(x)) df['category'] = 'fund' # df['choice_code'] = df.apply(lambda x:x.main_code+'.'+x.exchange.upper(),axis=1) # loginResult = c.start("ForceLogin=1", '') # df['underlying_index_code'] = df.apply(lambda x:c.css(x.choice_code, "BMINDEXCODE", "Rank=1").Data if x.operate_mode == 'ETF' else None,axis=1) # df['underlying_index_code'] = df['underlying_index_code'].apply(lambda x:[i for i in x.values()][0][0].lower().replace(".","_") if x else None) # c.stop() df_to_db(df, data_schema=FundDetail, provider=self.provider, force_update=self.force_update) # self.logger.info(df_index) self.logger.info("persist etf list success") logout()
def run(self): # 抓取fund列表 df = finance.run_query(query(finance.FUND_MAIN_INFO)) df.index.name = 'entity_id' df = df.reset_index() # 上市日期 df.rename(columns={'start_date': 'timestamp'}, inplace=True) df['timestamp'] = pd.to_datetime(df['timestamp']) df['list_date'] = df['timestamp'] df['end_date'] = pd.to_datetime(df['end_date']) df['entity_id'] = df.main_code.apply(lambda x: normalize_code(x)) df['entity_id'] = df['entity_id'].apply( lambda x: to_entity_id(entity_type='fund', jq_code=x)) df['id'] = df['entity_id'] df['entity_type'] = 'fund' df['exchange'] = df['entity_id'].apply( lambda x: get_entity_exchange(x)) df['code'] = df['entity_id'].apply(lambda x: get_entity_code(x)) df['category'] = 'fund' df_to_db(df, data_schema=FundDetail, provider=self.provider, force_update=self.force_update) # self.logger.info(df_index) self.logger.info("persist etf list success") logout()
def to_zvt_entity(self, df, entity_type, category=None): df.index.name = 'entity_id' df = df.reset_index() # 上市日期 df.rename(columns={'start_date': 'timestamp'}, inplace=True) df['timestamp'] = pd.to_datetime(df['timestamp']) df['list_date'] = df['timestamp'] df['end_date'] = pd.to_datetime(df['end_date']) df['entity_id'] = df['entity_id'].apply(lambda x: to_entity_id(entity_type=entity_type, jq_code=x)) df['id'] = df['entity_id'] df['entity_type'] = entity_type df['exchange'] = df['entity_id'].apply(lambda x: get_entity_exchange(x)) df['code'] = df['entity_id'].apply(lambda x: get_entity_code(x)) df['name'] = df['display_name'] if entity_type == 'etf': # ETF 查询标的指数 df['choice_code'] = df.apply(lambda x: x.code + '.' + x.exchange.upper(), axis=1) loginResult = c.start("ForceLogin=1", '') df['underlying_index_code'] = df.apply(lambda x: c.css(x.choice_code, "BMINDEXCODE", "Rank=1").Data, axis=1) df['index_codes'] = df['underlying_index_code'].apply(lambda x: [i for i in x.values()][0][0]) df['index_exchange'] = df['index_codes'].apply(lambda x: str(x).split('.')) df['index_code'] = df['index_codes'].apply(lambda x: str(x).split('.')[0]) df['index_exchange'] = df['index_exchange'].apply( lambda x: x[1] if isinstance(x, list) and len(x) > 1 else None) df['underlying_index_code'] = df.apply( lambda x: 'index_' + x.index_exchange.lower() + '_' + x.index_code if x.index_exchange else None, axis=1) loginResult = c.stop() if category: df['category'] = category return df
def to_zvt_entity(self, df, entity_type, category=None): df.index.name = 'entity_id' df = df.reset_index() # 上市日期 df.rename(columns={'start_date': 'timestamp'}, inplace=True) df['timestamp'] = pd.to_datetime(df['timestamp']) df['list_date'] = df['timestamp'] df['end_date'] = pd.to_datetime(df['end_date']) df['entity_id'] = df['entity_id'].apply(lambda x: x.lower()) df['entity_id'] = df['entity_id'].apply(lambda x: to_entity_id(entity_type=entity_type, jq_code=x)) df['id'] = df['entity_id'] df['entity_type'] = entity_type df['exchange'] = df['entity_id'].apply(lambda x: get_entity_exchange(x)) df['code'] = df['entity_id'].apply(lambda x: get_entity_code(x)) df['end_date'].fillna(pd.to_datetime("22000101"),inplace=True) if category: df['category'] = category return df
def to_zvt_entity(self, df, entity_type: EntityType, category=None): df.index.name = 'entity_id' df = df.reset_index() # 上市日期 df.rename(columns={'start_date': 'timestamp'}, inplace=True) df['timestamp'] = pd.to_datetime(df['timestamp']) df['list_date'] = df['timestamp'] df['end_date'] = pd.to_datetime(df['end_date']) df['entity_id'] = df['entity_id'].apply(lambda x: to_entity_id(entity_type=entity_type, jq_code=x)) df['id'] = df['entity_id'] df['entity_type'] = entity_type.value df['exchange'] = df['entity_id'].apply(lambda x: get_entity_exchange(x)) df['code'] = df['entity_id'].apply(lambda x: get_entity_code(x)) df['name'] = df['display_name'] if category: df['category'] = category return df
def to_zvt_entity(self, df, entity_type, category=None): df = df.set_index("code") df.index.name = "entity_id" df = df.reset_index() # 上市日期 df.rename(columns={"start_date": "timestamp"}, inplace=True) df["timestamp"] = pd.to_datetime(df["timestamp"]) df["list_date"] = df["timestamp"] df["end_date"] = pd.to_datetime(df["end_date"]) df["entity_id"] = df["entity_id"].apply( lambda x: to_entity_id(entity_type=entity_type, jq_code=x)) df["id"] = df["entity_id"] df["entity_type"] = entity_type df["exchange"] = df["entity_id"].apply( lambda x: get_entity_exchange(x)) df["code"] = df["entity_id"].apply(lambda x: get_entity_code(x)) df["name"] = df["display_name"] if category: df["category"] = category return df