def record(self, entity, start, end, size, timestamps): # 不复权 try: df = get_bars(to_jq_entity_id(entity), count=size, unit=self.jq_trading_level, fields=[ 'date', 'open', 'close', 'low', 'high', 'volume', 'money' ], include_now=False) except Exception as e: # just ignore the error,for some new stocks not in the index self.logger.exception(e) return None df['name'] = entity.name df.rename(columns={'money': 'turnover'}, inplace=True) df['timestamp'] = pd.to_datetime(df['date']) df['provider'] = 'joinquant' df['level'] = self.level.value # 前复权 end_timestamp = to_time_str(now_pd_timestamp()) qfq_df = get_bars(to_jq_entity_id(entity), count=size, unit=self.jq_trading_level, fields=['date', 'open', 'close', 'low', 'high'], fq_ref_date=end_timestamp, include_now=False) # not need to update past df['qfq_close'] = qfq_df['close'] df['qfq_open'] = qfq_df['open'] df['qfq_high'] = qfq_df['high'] df['qfq_low'] = qfq_df['low'] check_df = qfq_df.head(1) check_date = check_df['date'][0] current_df = get_kdata(entity_id=entity.id, provider=self.provider, start_timestamp=check_date, end_timestamp=check_date, limit=1, level=self.level) if df_is_not_null(current_df): old = current_df.iloc[0, :]['qfq_close'] new = check_df['close'][0] # 相同时间的close不同,表明前复权需要重新计算 if old != new: self.factor = new / old self.last_timestamp = pd.Timestamp(check_date) return df.to_dict(orient='records')
def record(self, entity, start, end, size, timestamps): # 只要前复权数据 if not self.end_timestamp: df = get_bars(to_jq_entity_id(entity), count=size, unit=self.jq_trading_level, fields=['date', 'open', 'close', 'low', 'high', 'volume', 'money'], fq_ref_date=to_time_str(now_pd_timestamp()), include_now=True) else: end_timestamp = to_time_str(self.end_timestamp) df = get_bars(to_jq_entity_id(entity), count=size, unit=self.jq_trading_level, fields=['date', 'open', 'close', 'low', 'high', 'volume', 'money'], end_dt=end_timestamp, fq_ref_date=to_time_str(now_pd_timestamp()), include_now=False) if pd_is_not_null(df): df['name'] = entity.name df.rename(columns={'money': 'turnover', 'date': 'timestamp'}, inplace=True) df['entity_id'] = entity.id df['timestamp'] = pd.to_datetime(df['timestamp']) df['provider'] = 'joinquant' df['level'] = self.level.value df['code'] = entity.code # 判断是否需要重新计算之前保存的前复权数据 check_df = df.head(1) check_date = check_df['timestamp'][0] current_df = get_kdata(entity_id=entity.id, provider=self.provider, start_timestamp=check_date, end_timestamp=check_date, limit=1, level=self.level) if pd_is_not_null(current_df): old = current_df.iloc[0, :]['close'] new = check_df['close'][0] # 相同时间的close不同,表明前复权需要重新计算 if round(old, 2) != round(new, 2): self.factor = new / old self.last_timestamp = pd.Timestamp(check_date) def generate_kdata_id(se): if self.level >= IntervalLevel.LEVEL_1DAY: return "{}_{}".format(se['entity_id'], to_time_str(se['timestamp'], fmt=TIME_FORMAT_DAY)) else: return "{}_{}".format(se['entity_id'], to_time_str(se['timestamp'], fmt=TIME_FORMAT_ISO8601)) df['id'] = df[['entity_id', 'timestamp']].apply(generate_kdata_id, axis=1) df_to_db(df=df, data_schema=self.data_schema, provider=self.provider, force_update=self.force_update) return None
def record(self, entity, start, end, size, timestamps): # 只要前复权数据 q = query(valuation).filter(valuation.code == to_jq_entity_id(entity)) count: pd.Timedelta = now_pd_timestamp() - start df = get_fundamentals_continuously(q, end_date=now_time_str(), count=count.days + 1, panel=False) df['entity_id'] = entity.id df['timestamp'] = pd.to_datetime(df['day']) df['code'] = entity.code df['name'] = entity.name df['id'] = df['timestamp'].apply( lambda x: "{}_{}".format(entity.id, to_time_str(x))) df = df.rename( { 'pe_ratio_lyr': 'pe', 'pe_ratio': 'pe_ttm', 'pb_ratio': 'pb', 'ps_ratio': 'ps', 'pcf_ratio': 'pcf' }, axis='columns') df['market_cap'] = df['market_cap'] * 100000000 df['circulating_cap'] = df['circulating_cap'] * 100000000 df['capitalization'] = df['capitalization'] * 10000 df['circulating_cap'] = df['circulating_cap'] * 10000 df_to_db(df=df, data_schema=self.data_schema, provider=self.provider, force_update=self.force_update) return None
def record(self, entity, start, end, size, timestamps): if self.start_timestamp: start = max(self.start_timestamp, to_pd_timestamp(start)) end = now_pd_timestamp() + timedelta(days=1) start_timestamp = to_time_str(start) end_timestamp = to_time_str(end) # 不复权 df = get_price( to_jq_entity_id(entity), start_date=to_time_str(start_timestamp), end_date=end_timestamp, frequency=self.jq_trading_level, fields=['open', 'close', 'low', 'high', 'volume', 'money'], skip_paused=True, fq=None) df.index.name = 'timestamp' df.reset_index(inplace=True) df['name'] = entity.name df.rename(columns={'money': 'turnover'}, inplace=True) df['timestamp'] = pd.to_datetime(df['timestamp']) df['provider'] = 'joinquant' df['level'] = self.level.value # remove the unfinished kdata if is_in_trading(entity_type='stock', exchange='sh', timestamp=df.iloc[-1, :]['timestamp']): df = df.iloc[:-1, :] return df.to_dict(orient='records')
def on_finish_entity(self, entity): kdatas = get_kdata( provider=self.provider, entity_id=entity.id, level=self.level.value, order=self.data_schema.timestamp.asc(), return_type='domain', session=self.session, filters=[ self.data_schema.hfq_close.is_(None), self.data_schema.timestamp >= to_pd_timestamp('2005-01-01') ]) if kdatas: start = kdatas[0].timestamp end = kdatas[-1].timestamp # get hfq from joinquant df = get_price(to_jq_entity_id(entity), start_date=to_time_str(start), end_date=now_time_str(), frequency='daily', fields=['factor', 'open', 'close', 'low', 'high'], skip_paused=True, fq='post') if df_is_not_null(df): # fill hfq data for kdata in kdatas: time_str = to_time_str(kdata.timestamp) if time_str in df.index: kdata.hfq_open = df.loc[time_str, 'open'] kdata.hfq_close = df.loc[time_str, 'close'] kdata.hfq_high = df.loc[time_str, 'high'] kdata.hfq_low = df.loc[time_str, 'low'] kdata.factor = df.loc[time_str, 'factor'] self.session.add_all(kdatas) self.session.commit() latest_factor = df.factor[-1] # factor not change yet, no need to reset the qfq past if latest_factor == self.current_factors.get(entity.id): sql = 'UPDATE {} SET qfq_close=hfq_close/{},qfq_high=hfq_high/{}, qfq_open= hfq_open/{}, qfq_low= hfq_low/{} where ' \ 'entity_id=\'{}\' and level=\'{}\' and (qfq_close isnull or qfq_high isnull or qfq_low isnull or qfq_open isnull)'.format( self.data_schema.__table__, latest_factor, latest_factor, latest_factor, latest_factor, entity.id, self.level.value) else: sql = 'UPDATE {} SET qfq_close=hfq_close/{},qfq_high=hfq_high/{}, qfq_open= hfq_open/{}, qfq_low= hfq_low/{} where ' \ 'entity_id=\'{}\' and level=\'{}\''.format(self.data_schema.__table__, latest_factor, latest_factor, latest_factor, latest_factor, entity.id, self.level.value) self.logger.info(sql) self.session.execute(sql) self.session.commit()
def record(self, entity, start, end, size, timestamps): if self.start_timestamp: start = max(self.start_timestamp, to_pd_timestamp(start)) # if self.level < IntervalLevel.LEVEL_1HOUR: # start = '2019-01-01' end = now_pd_timestamp() start_timestamp = to_time_str(start) # 聚宽get_price函数必须指定结束时间,否则会有未来数据 end_timestamp = to_time_str(end, fmt=TIME_FORMAT_MINUTE2) # 不复权 df = get_price( to_jq_entity_id(entity), start_date=to_time_str(start_timestamp), end_date=end_timestamp, frequency=self.jq_trading_level, fields=['open', 'close', 'low', 'high', 'volume', 'money'], skip_paused=True, fq=None) if df_is_not_null(df): df.index.name = 'timestamp' df.reset_index(inplace=True) df['name'] = entity.name df.rename(columns={'money': 'turnover'}, inplace=True) df['entity_id'] = entity.id df['timestamp'] = pd.to_datetime(df['timestamp']) df['provider'] = 'joinquant' df['level'] = self.level.value df['code'] = entity.code def generate_kdata_id(se): if self.level >= IntervalLevel.LEVEL_1DAY: return "{}_{}".format( se['entity_id'], to_time_str(se['timestamp'], fmt=TIME_FORMAT_DAY)) else: return "{}_{}".format( se['entity_id'], to_time_str(se['timestamp'], fmt=TIME_FORMAT_ISO8601)) df['id'] = df[['entity_id', 'timestamp']].apply(generate_kdata_id, axis=1) df_to_db(df=df, data_schema=self.data_schema, provider=self.provider, force=self.force_update) return None
def fill_timestamp_with_jq(self, security_item, the_data): # get report published date from jq q = query(indicator.pubDate).filter( indicator.code == to_jq_entity_id(security_item), ) df = get_fundamentals(q, statDate=to_jq_report_period( the_data.report_date)) if not df.empty: the_data.timestamp = to_pd_timestamp(df['pubDate'][0]) self.logger.info( 'jq fill {} {} timestamp:{} for report_date:{}'.format( self.data_schema, security_item.id, the_data.timestamp, the_data.report_date)) self.session.commit()