def test_basic_get_securities(): items = get_entities(region=Region.CHN, entity_type=EntityType.Stock, provider=Provider.EastMoney) print(items) items = get_entities(region=Region.CHN, entity_type=EntityType.Index, provider=Provider.EastMoney) print(items)
def update_trader_details(trader_index, entity_type, entity_provider): if trader_index is not None: # change entity_type options entity_type = traders[trader_index].entity_type if not entity_type: entity_type = 'stock' entity_type_options = [{'label': entity_type, 'value': entity_type}] # account stats account_stats = get_account_stats_figure( account_stats_reader=account_readers[trader_index]) providers = zvt_context.tradable_schema_map.get(entity_type).providers entity_provider_options = [{ 'label': name, 'value': name } for name in providers] # entities entity_ids = get_order_securities( trader_name=trader_names[trader_index]) df = get_entities(provider=entity_provider, entity_type=entity_type, entity_ids=entity_ids, columns=['entity_id', 'code', 'name'], index='entity_id') entity_options = [{ 'label': f'{entity_id}({entity["name"]})', 'value': entity_id } for entity_id, entity in df.iterrows()] return account_stats, entity_type_options, entity_provider_options, entity_options else: entity_type_options = [{ 'label': name, 'value': name } for name in zvt_context.tradable_schema_map.keys()] account_stats = None providers = zvt_context.tradable_schema_map.get(entity_type).providers entity_provider_options = [{ 'label': name, 'value': name } for name in providers] df = get_entities(provider=entity_provider, entity_type=entity_type, columns=['entity_id', 'code', 'name'], index='entity_id') entity_options = [{ 'label': f'{entity_id}({entity["name"]})', 'value': entity_id } for entity_id, entity in df.iterrows()] return account_stats, entity_type_options, entity_provider_options, entity_options
def __init__(self, force_update=False, sleeping_time=5, code=None, codes=None) -> None: super().__init__(force_update, sleeping_time) # get list at first EastmoneyStockRecorder().run() if codes is None and code is not None: self.codes = [code] else: self.codes = codes filters = None if not self.force_update: filters = [StockDetail.profile.is_(None)] self.entities = get_entities( session=self.session, entity_schema=StockDetail, exchanges=None, codes=self.codes, filters=filters, return_type="domain", provider=self.provider, )
def update_code_selector(entity_type): if entity_type is not None: return [{'label': schema.__name__, 'value': schema.__name__} for schema in zvt_context.entity_map_schemas.get(entity_type)], \ [{'label': code, 'value': code} for code in get_entities(entity_type=entity_type, columns=['code']).index] raise dash.PreventUpdate()
def __init__(self, batch_size=10, force_update=False, sleeping_time=5, codes=None, share_para=None) -> None: super().__init__(batch_size, force_update, sleeping_time) # get list at first # EastmoneyChinaStockListRecorder().run() self.codes = codes self.share_para = share_para if not self.force_update: assert self.region is not None self.entities = get_entities( region=self.region, session=self.session, entity_type=EntityType.StockDetail, exchanges=['sh', 'sz'], codes=self.codes, filters=[StockDetail.profile.is_(None)], return_type='domain', provider=self.provider)
def cal_ma_states(region: Region, start='000001', end='002000'): logger.info(f'start cal day ma stats {start}:{end}') entities = get_entities(region=region, provider=Provider.Default, entity_type=EntityType.Stock, columns=[Stock.entity_id, Stock.code], filters=[Stock.code >= start, Stock.code < end]) codes = entities.index.to_list() ma_1d_stats = MaStateStatsFactor(region=region, codes=codes, start_timestamp='2005-01-01', end_timestamp=now_pd_timestamp( Region.CHN), level=IntervalLevel.LEVEL_1DAY) ma_1d_factor = MaFactor(region=region, codes=codes, start_timestamp='2005-01-01', end_timestamp=now_pd_timestamp(Region.CHN), level=IntervalLevel.LEVEL_1DAY) logger.info(f'finish cal day ma stats {start}:{end}') ma_1wk_stats = MaStateStatsFactor(region=region, codes=codes, start_timestamp='2005-01-01', end_timestamp=now_pd_timestamp( Region.CHN), level=IntervalLevel.LEVEL_1WEEK) logger.info(f'finish cal week ma stats {start}:{end}')
def inform(action: EmailInformer, entity_ids, target_date, title, entity_provider, entity_type, em_group, em_group_over_write): msg = "no targets" if entity_ids: entities = get_entities(provider=entity_provider, entity_type=entity_type, entity_ids=entity_ids, return_type="domain") if em_group: try: codes = [entity.code for entity in entities] add_to_eastmoney(codes=codes, entity_type=entity_type, group=em_group, over_write=em_group_over_write) except Exception as e: action.send_message( zvt_config["email_username"], f"{target_date} {title} error", f"{target_date} {title} error: {e}", ) infos = [f"{entity.name}({entity.code})" for entity in entities] msg = "\n".join(infos) + "\n" logger.info(msg) action.send_message(zvt_config["email_username"], f"{target_date} {title}", msg)
def cal_ma_states(start='000001', end='002000'): logger.info(f'start cal day ma stats {start}:{end}') entities = get_entities(provider='eastmoney', entity_type='stock', columns=[Stock.entity_id, Stock.code], filters=[Stock.code >= start, Stock.code < end]) codes = entities.index.to_list() ma_1d_stats = MaStateStatsFactor(codes=codes, start_timestamp='2005-01-01', end_timestamp=now_pd_timestamp(), level=IntervalLevel.LEVEL_1DAY) ma_1d_factor = MaFactor(codes=codes, start_timestamp='2005-01-01', end_timestamp=now_pd_timestamp(), level=IntervalLevel.LEVEL_1DAY) logger.info(f'finish cal day ma stats {start}:{end}') ma_1wk_stats = MaStateStatsFactor(codes=codes, start_timestamp='2005-01-01', end_timestamp=now_pd_timestamp(), level=IntervalLevel.LEVEL_1WEEK) logger.info(f'finish cal week ma stats {start}:{end}')
def init_entities(self): """ init the entities which we would record data for """ if self.entity_provider == self.provider and self.entity_schema == self.data_schema: self.entity_session = self.session else: self.entity_session = get_db_session(provider=self.entity_provider, data_schema=self.entity_schema) filters = None if self.day_data: df = self.data_schema.query_data(start_timestamp=now_time_str(), columns=['entity_id', 'timestamp'], provider=self.provider) if pd_is_not_null(df): entity_ids = df['entity_id'].tolist() self.logger.info(f'ignore entity_ids:{entity_ids}') filters = [self.entity_schema.entity_id.notin_(entity_ids)] # init the entity list self.entities = get_entities(session=self.entity_session, entity_schema=self.entity_schema, entity_type=self.entity_type, exchanges=self.exchanges, entity_ids=self.entity_ids, codes=self.codes, return_type='domain', provider=self.entity_provider, filters=filters)
def entity_ids_to_msg(entity_ids): if entity_ids: stocks = get_entities(provider='joinquant', entity_schema=Stock, entity_ids=entity_ids, return_type='domain') info = [f'{stock.name}({stock.code})' for stock in stocks] return ' '.join(info) return ''
def entity_ids_to_msg(region: Region, entity_ids): if entity_ids: stocks = get_entities(region=region, provider=Provider.JoinQuant, entity_schema=Stock, entity_ids=entity_ids, return_type='domain') info = [f'{stock.name}({stock.code})' for stock in stocks] return ' '.join(info) return ''
def report_core_company(): while True: error_count = 0 email_action = EmailInformer() try: # StockTradeDay.record_data(provider='joinquant') # Stock.record_data(provider='joinquant') # FinanceFactor.record_data(provider='eastmoney') # BalanceSheet.record_data(provider='eastmoney') target_date = to_time_str(now_pd_timestamp()) my_selector: TargetSelector = FundamentalSelector( start_timestamp='2015-01-01', end_timestamp=target_date) my_selector.run() long_targets = my_selector.get_open_long_targets( timestamp=target_date) if long_targets: stocks = get_entities(provider='joinquant', entity_schema=Stock, entity_ids=long_targets, return_type='domain') # add them to eastmoney try: try: eastmoneypy.del_group('core') except: pass eastmoneypy.create_group('core') for stock in stocks: eastmoneypy.add_to_group(stock.code, group_name='core') except Exception as e: email_action.send_message( "*****@*****.**", f'report_core_company error', 'report_core_company error:{}'.format(e)) info = [f'{stock.name}({stock.code})' for stock in stocks] msg = ' '.join(info) else: msg = 'no targets' logger.info(msg) email_action.send_message(get_subscriber_emails(), f'{to_time_str(target_date)} 核心资产选股结果', msg) break except Exception as e: logger.exception('report_core_company error:{}'.format(e)) time.sleep(60 * 3) error_count = error_count + 1 if error_count == 10: email_action.send_message( "*****@*****.**", f'report_core_company error', 'report_core_company error:{}'.format(e))
def report_bull(): while True: error_count = 0 email_action = EmailInformer() try: latest_day: Stock1dHfqKdata = Stock1dHfqKdata.query_data(order=Stock1dHfqKdata.timestamp.desc(), limit=1, return_type='domain') target_date = latest_day[0].timestamp # 计算均线 start = '2019-01-01' my_selector = TargetSelector(start_timestamp=start, end_timestamp=target_date) # add the factors factor1 = BullFactor(start_timestamp=start, end_timestamp=target_date) factor2 = CrossMaVolumeFactor(start_timestamp=start, end_timestamp=target_date, windows=[5, 120, 250]) my_selector.add_factor(factor1) my_selector.add_factor(factor2) my_selector.run() long_stocks = my_selector.get_open_long_targets(timestamp=target_date) msg = 'no targets' if long_stocks: stocks = get_entities(provider='joinquant', entity_schema=Stock, entity_ids=long_stocks, return_type='domain') # add them to eastmoney try: try: eastmoneypy.del_group('bull') except: pass eastmoneypy.create_group('bull') for stock in stocks: eastmoneypy.add_to_group(stock.code, group_name='bull') except Exception as e: email_action.send_message(zvt_config['email_username'], f'report_bull error', 'report_bull error:{}'.format(e)) infos = stocks_with_info(stocks) msg = '\n'.join(infos) + '\n' logger.info(msg) email_action.send_message(zvt_config['email_username'], f'{target_date} bull选股结果', msg) break except Exception as e: logger.exception('report_bull error:{}'.format(e)) time.sleep(60 * 3) error_count = error_count + 1 if error_count == 10: email_action.send_message(zvt_config['email_username'], f'report_bull error', 'report_bull error:{}'.format(e))
def update_code_selector(entity_type): if entity_type is not None: return [{ 'label': code, 'value': code } for code in get_entities(entity_type=entity_type, columns=['code']).index] raise dash.PreventUpdate()
def report_cross_ma(): while True: error_count = 0 email_action = EmailInformer() try: # 抓取k线数据 # StockTradeDay.record_data(provider='joinquant') # Stock1dKdata.record_data(provider='joinquant') latest_day: StockTradeDay = StockTradeDay.query_data( order=StockTradeDay.timestamp.desc(), limit=1, return_type='domain') if latest_day: target_date = latest_day[0].timestamp else: target_date = now_pd_timestamp() # 计算均线 my_selector = TargetSelector(start_timestamp='2018-01-01', end_timestamp=target_date) # add the factors ma_factor = CrossMaFactor(start_timestamp='2018-01-01', end_timestamp=target_date) my_selector.add_filter_factor(ma_factor) my_selector.run() long_targets = my_selector.get_open_long_targets( timestamp=target_date) if long_targets: stocks = get_entities(provider='joinquant', entity_schema=Stock, entity_ids=long_targets, return_type='domain') info = [f'{stock.name}({stock.code})' for stock in stocks] msg = ' '.join(info) else: msg = 'no targets' logger.info(msg) email_action.send_message("*****@*****.**", f'{target_date} 均线选股结果', msg) break except Exception as e: logger.exception('report_cross_ma error:{}'.format(e)) time.sleep(60 * 3) error_count = error_count + 1 if error_count == 10: email_action.send_message("*****@*****.**", f'report_cross_ma error', 'report_cross_ma error:{}'.format(e))
def test_iterate_entities(): data = get_entities(entity_type='stock') first = None last = None for sub_data in iterate_with_step(data): if first is None: first = sub_data last = sub_data assert len(first) == 100 assert len(last) <= 100
def update_trader_details(region: Region, trader_index, entity_type): if trader_index is not None: # change entity_type options entity_type = traders[trader_index].entity_type if not entity_type: entity_type = EntityType.Stock entity_type_options = [{'label': entity_type, 'value': entity_type}] # account stats account_stats = get_account_stats_figure( account_stats_reader=account_readers[trader_index]) # entities entity_ids = get_order_securities( trader_name=trader_names[trader_index]) df = get_entities(region=region, entity_type=entity_type, entity_ids=entity_ids, columns=['entity_id', 'code', 'name'], index='entity_id') entity_options = [{ 'label': f'{entity_id}({entity["name"]})', 'value': entity_id } for entity_id, entity in df.iterrows()] return account_stats, entity_type_options, entity_options else: entity_type_options = [{ 'label': name, 'value': name } for name in zvt_context.entity_schema_map.keys()] account_stats = None df = get_entities(region=region, entity_type=entity_type, columns=['entity_id', 'code', 'name'], index='entity_id') entity_options = [{ 'label': f'{entity_id}({entity["name"]})', 'value': entity_id } for entity_id, entity in df.iterrows()] return account_stats, entity_type_options, entity_options
def update_entity_selector(entity_type, related): if entity_type is not None: if related: schemas = zvt_context.entity_map_schemas.get(entity_type) else: schemas = zvt_context.schemas df = get_entities(entity_type=entity_type, columns=['entity_id', 'code', 'name'], index='entity_id') return [{'label': schema.__name__, 'value': schema.__name__} for schema in schemas], \ [{'label': f'{entity_id}({entity["name"]})', 'value': entity_id} for entity_id, entity in df.iterrows()] raise dash.PreventUpdate()
def report_core_company(): while True: error_count = 0 email_action = EmailInformer() try: # StockTradeDay.record_data(provider='joinquant') # Stock.record_data(provider='joinquant') # FinanceFactor.record_data(provider='eastmoney') # BalanceSheet.record_data(provider='eastmoney') target_date = to_time_str(now_pd_timestamp()) my_selector: TargetSelector = FundamentalSelector(start_timestamp="2016-01-01", end_timestamp=target_date) my_selector.run() long_targets = my_selector.get_open_long_targets(timestamp=target_date) if long_targets: stocks = get_entities( provider="joinquant", entity_schema=Stock, entity_ids=long_targets, return_type="domain" ) # add them to eastmoney try: codes = [stock.code for stock in stocks] add_to_eastmoney(codes=codes, entity_type="stock", group="core") except Exception as e: email_action.send_message( zvt_config["email_username"], f"report_core_company error", "report_core_company error:{}".format(e), ) infos = stocks_with_info(stocks) msg = "\n".join(infos) else: msg = "no targets" logger.info(msg) email_action.send_message(get_subscriber_emails(), f"{to_time_str(target_date)} 核心资产选股结果", msg) break except Exception as e: logger.exception("report_core_company error:{}".format(e)) time.sleep(60 * 3) error_count = error_count + 1 if error_count == 10: email_action.send_message( zvt_config["email_username"], f"report_core_company error", "report_core_company error:{}".format(e) )
def __init__(self, batch_size=10, force_update=False, sleeping_time=5, codes=None, share_para=None) -> None: exchanges = ['nyse', 'nasdaq', 'amex'] if not force_update: assert self.region is not None self.entities = get_entities(region=self.region, entity_type=EntityType.StockDetail, exchanges=exchanges, codes=codes, filters=[StockDetail.profile.is_(None)], return_type='domain', provider=self.provider) super().__init__(entity_type=EntityType.StockDetail, exchanges=exchanges, batch_size=batch_size, force_update=force_update, sleeping_time=sleeping_time, codes=codes, share_para=share_para)
def init_entities(self): """ init the entities which we would record data for """ assert self.region is not None # init the entity list self.entities = get_entities(region=self.region, entity_schema=self.entity_schema, entity_type=self.entity_type, exchanges=self.exchanges, entity_ids=self.entity_ids, codes=self.codes, return_type='domain', provider=self.provider)
def __init__(self, batch_size=10, force_update=False, sleeping_time=5, codes=None) -> None: super().__init__(batch_size, force_update, sleeping_time) loginResult = c.start("ForceLogin=1", '', mainCallback) if (loginResult.ErrorCode != 0): print("login in fail") exit() self.codes = codes if not self.force_update: self.entities = get_entities(session=self.session, entity_type='stock_detail', exchanges=['hk'], codes=self.codes, # filters=[StockDetail.profile.is_(None)], return_type='domain', provider=self.provider)
def init_entities(self): assert self.region is not None self.entities = get_entities( region=self.region, entity_type=EntityType.Index, exchanges=self.exchanges, codes=self.codes, entity_ids=self.entity_ids, return_type='domain', provider=self.provider, # 只抓概念和行业 filters=[ Index.category.in_([ BlockCategory.industry.value, BlockCategory.concept.value ]) ])
def __init__(self, batch_size=10, force_update=False, sleeping_time=5, codes=None) -> None: super().__init__(batch_size, force_update, sleeping_time) # get list at first # JqChinaStockRecorder().run() auth(zvt_env['jq_username'], zvt_env['jq_password']) print(f"剩余{get_query_count()['spare'] / 10000}万") self.codes = codes if not self.force_update: self.entities = get_entities(session=self.session, entity_type='stock_detail', exchanges=['sh', 'sz'], codes=self.codes, filters=[StockDetail.issues.is_(None)], return_type='domain', provider=self.provider)
def run(): while True: try: items = get_entities(entity_type='stock', provider='joinquant') entity_ids = items['entity_id'].to_list() StockActorSummary.record_data(entity_ids=entity_ids[0:], provider='em', sleeping_time=0.5) StockInstitutionalInvestorHolder.record_data(provider='em', sleeping_time=1) StockTopTenHolder.record_data(provider='em', sleeping_time=1) break except Exception as e: msg = f'em_data_runner error:{e}' logger.exception(msg) time.sleep(60)
def init_entities(self): self.entity_session = get_db_session(provider=self.entity_provider, data_schema=self.entity_schema) self.entities = get_entities( session=self.entity_session, entity_type='index', exchanges=self.exchanges, codes=self.codes, entity_ids=self.entity_ids, return_type='domain', provider=self.provider, # 只抓概念和行业 filters=[ Index.category.in_([ BlockCategory.industry.value, BlockCategory.concept.value ]) ])
def __init__(self, batch_size=10, force_update=False, sleeping_time=5, codes=None) -> None: super().__init__(batch_size, force_update, sleeping_time) # get list at first EastmoneyChinaStockListRecorder().run() self.codes = codes if not self.force_update: self.entities = get_entities( session=self.session, entity_type='stock_detail', exchanges=['sh', 'sz'], codes=self.codes, filters=[StockDetail.profile.is_(None)], return_type='domain', provider=self.provider)
def init_entities(self): """ init the entities which we would record data for """ if self.entity_provider == self.provider and self.entity_schema == self.data_schema: self.entity_session = self.session else: self.entity_session = get_db_session( provider=self.entity_provider, data_schema=self.entity_schema) # init the entity list self.entities = get_entities(session=self.entity_session, entity_schema=self.entity_schema, entity_type=self.entity_type, exchanges=self.exchanges, entity_ids=self.entity_ids, codes=self.codes, return_type='domain', provider=self.entity_provider)
def init_entities(self): """ init the entities which we would record data for """ if self.entity_provider == self.provider and self.entity_schema == self.data_schema: self.entity_session = self.session else: self.entity_session = get_db_session( provider=self.entity_provider, data_schema=self.entity_schema) if self.day_data: df = self.data_schema.query_data( start_timestamp=now_time_str(), columns=["entity_id", "timestamp"], provider=self.provider) if pd_is_not_null(df): entity_ids = df["entity_id"].tolist() self.logger.info(f"ignore entity_ids:{entity_ids}") if self.entity_filters: self.entity_filters.append( self.entity_schema.entity_id.notin_(entity_ids)) else: self.entity_filters = [ self.entity_schema.entity_id.notin_(entity_ids) ] #: init the entity list self.entities = get_entities( session=self.entity_session, entity_schema=self.entity_schema, exchanges=self.exchanges, entity_ids=self.entity_ids, codes=self.codes, return_type="domain", provider=self.entity_provider, filters=self.entity_filters, )
def __init__(self, region: Region, data_schema: Type[Mixin], entity_schema: Type[EntityMixin], provider: Provider = Provider.Default, entity_ids: List[str] = None, exchanges: List[str] = None, codes: List[str] = None, the_timestamp: Union[str, pd.Timestamp] = None, start_timestamp: Union[str, pd.Timestamp] = None, end_timestamp: Union[str, pd.Timestamp] = None, columns: List = None, filters: List = None, order: object = None, limit: int = None, level: IntervalLevel = None, category_field: str = 'entity_id', time_field: str = 'timestamp', computing_window: int = None) -> None: super().__init__() self.logger = logging.getLogger(self.__class__.__name__) self.data_schema = data_schema self.entity_schema = entity_schema self.region = region self.provider = provider if end_timestamp is None: end_timestamp = now_pd_timestamp(self.region) self.the_timestamp = the_timestamp if the_timestamp: self.start_timestamp = the_timestamp self.end_timestamp = the_timestamp else: self.start_timestamp = start_timestamp self.end_timestamp = end_timestamp self.start_timestamp = to_pd_timestamp(self.start_timestamp) self.end_timestamp = to_pd_timestamp(self.end_timestamp) self.exchanges = exchanges if codes: if type(codes) == str: codes = codes.replace(' ', '') if codes.startswith('[') and codes.endswith(']'): codes = json.loads(codes) else: codes = codes.split(',') self.codes = codes self.entity_ids = entity_ids # 转换成标准entity_id if entity_schema and not self.entity_ids: df = get_entities(region=self.region, entity_schema=entity_schema, provider=self.provider, exchanges=self.exchanges, codes=self.codes) if pd_is_not_null(df): self.entity_ids = df['entity_id'].to_list() self.filters = filters self.order = order self.limit = limit if level: self.level = IntervalLevel(level) else: self.level = level self.category_field = category_field self.time_field = time_field self.computing_window = computing_window self.category_col = eval('self.data_schema.{}'.format( self.category_field)) self.time_col = eval('self.data_schema.{}'.format(self.time_field)) self.columns = columns # we store the data in a multiple index(category_column,timestamp) Dataframe if self.columns: # support str if type(columns[0]) == str: self.columns = [] for col in columns: self.columns.append(eval('data_schema.{}'.format(col))) # always add category_column and time_field for normalizing self.columns = list( set(self.columns) | {self.category_col, self.time_col}) self.data_listeners: List[DataListener] = [] self.data_df: pd.DataFrame = None self.load_data()