def init_entities(self): """ init the entities which we would record data for """ if self.entity_provider == self.provider and self.entity_schema == self.data_schema: self.entity_session = self.session else: self.entity_session = get_db_session(provider=self.entity_provider, data_schema=self.entity_schema) filters = None if self.day_data: df = self.data_schema.query_data(start_timestamp=now_time_str(), columns=['entity_id', 'timestamp'], provider=self.provider) if pd_is_not_null(df): entity_ids = df['entity_id'].tolist() self.logger.info(f'ignore entity_ids:{entity_ids}') filters = [self.entity_schema.entity_id.notin_(entity_ids)] # init the entity list self.entities = get_entities(session=self.entity_session, entity_schema=self.entity_schema, entity_type=self.entity_type, exchanges=self.exchanges, entity_ids=self.entity_ids, codes=self.codes, return_type='domain', provider=self.entity_provider, filters=filters)
def persist_factor(self): if self.states: session = get_db_session(provider='zvt', data_schema=FactorState) for entity_id in self.states: state = self.states[entity_id] if state: domain_id = f'{self.factor_name}_{entity_id}' factor_state: FactorState = session.query(FactorState).get( domain_id) state_str = json.dumps(state, cls=FactorStateEncoder) if factor_state: factor_state.state = state_str else: factor_state = FactorState( id=domain_id, entity_id=entity_id, factor_name=self.factor_name, state=state_str) session.add(factor_state) session.commit() df = self.factor_df.copy() df['zhongshu'] = df['zhongshu'].apply( lambda x: json.dumps(x, cls=FactorStateEncoder)) df_to_db(df=df, data_schema=self.factor_schema, provider='zvt', force_update=False)
def process_loop(self, entity, http_session): assert isinstance(entity, StockDetail) # get stock info info = yh_get_info(entity.code) if info is None or len(info) == 0: return None if not entity.sector: entity.sector = info.get('sector', None) if not entity.industry: entity.industry = info.get('industry', None) if not entity.market_cap or entity.market_cap == 0: entity.market_cap = info.get('market_cap', 0) entity.profile = info.get('longBusinessSummary', None) entity.state = info.get('state', None) entity.city = info.get('city', None) entity.zip_code = info.get('zip', None) entity.last_sale = info.get('previousClose', None) session = get_db_session(region=self.region, provider=self.provider, data_schema=self.data_schema) session.commit() self.sleep()
def init_account(self) -> AccountStats: trader_info = get_trader_info(self.region, trader_name=self.trader_name, return_type='domain', limit=1) if trader_info: self.logger.warning( "trader:{} has run before,old result would be deleted".format( self.trader_name)) session = get_db_session(region=Region.CHN, provider=Provider.ZVT, data_schema=TraderInfo) session.query(TraderInfo).filter( TraderInfo.trader_name == self.trader_name).delete() session.query(AccountStats).filter( AccountStats.trader_name == self.trader_name).delete() session.query(Position).filter( Position.trader_name == self.trader_name).delete() session.query(Order).filter( Order.trader_name == self.trader_name).delete() session.commit() return AccountStats(entity_id=f'trader_zvt_{self.trader_name}', timestamp=self.start_timestamp, trader_name=self.trader_name, cash=self.base_capital, input_money=self.base_capital, all_value=self.base_capital, value=0, closing=False)
def __init__(self, batch_size: int = 10, force_update: bool = False, sleeping_time: int = 10) -> None: """ :param batch_size:batch size to saving to db :type batch_size:int :param force_update: whether force update the data even if it exists,please set it to True if the data need to be refreshed from the provider :type force_update:bool :param sleeping_time:sleeping seconds for recoding loop :type sleeping_time:int """ self.logger = logging.getLogger(self.__class__.__name__) assert self.provider is not None assert self.data_schema is not None assert self.provider in self.data_schema.providers self.batch_size = batch_size self.force_update = force_update self.sleeping_time = sleeping_time # using to do db operations self.session = get_db_session(provider=self.provider, data_schema=self.data_schema)
def __init__(self, entity_schema: EntityMixin, trader_name, timestamp, provider=None, level=IntervalLevel.LEVEL_1DAY, base_capital=1000000, buy_cost=0.001, sell_cost=0.001, slippage=0.001, rich_mode=True, adjust_type: AdjustType = None): self.entity_schema = entity_schema self.base_capital = base_capital self.buy_cost = buy_cost self.sell_cost = sell_cost self.slippage = slippage self.rich_mode = rich_mode self.adjust_type = adjust_type self.trader_name = trader_name self.session = get_db_session('zvt', data_schema=TraderInfo) self.provider = provider self.level = level self.start_timestamp = timestamp self.account: AccountStats = self.init_account()
def __init__(self) -> None: assert self.state_schema is not None self.state_session = get_db_session(data_schema=self.state_schema, provider='zvt') self.state_domain = self.state_schema.get_one( id=self.get_state_entity_id()) if self.state_domain: self.state: dict = self.decode_state(self.state_domain.state)
def clear_trader(trader_name, session=None): if not session: session = get_db_session("zvt", data_schema=TraderInfo) session.query(TraderInfo).filter(TraderInfo.trader_name == trader_name).delete() session.query(AccountStats).filter(AccountStats.trader_name == trader_name).delete() session.query(Position).filter(Position.trader_name == trader_name).delete() session.query(Order).filter(Order.trader_name == trader_name).delete() session.commit()
def get_order_securities(trader_name): items = get_db_session(region=Region.CHN, provider=Provider.ZVT, data_schema=Order).query(Order.entity_id).filter( Order.trader_name == trader_name).group_by( Order.entity_id).all() return [item[0] for item in items]
def get_order_securities(trader_name): items = ( get_db_session(provider="zvt", data_schema=Order) .query(Order.entity_id) .filter(Order.trader_name == trader_name) .group_by(Order.entity_id) .all() ) return [item[0] for item in items]
def __init__(self, batch_size: int = 10, force_update: bool = False, sleeping_time: int = 10) -> None: """ :param batch_size:batch size to saving to db :type batch_size:int :param force_update: whether force update the data even if it exists,please set it to True if the data need to be refreshed from the provider :type force_update:bool :param sleeping_time:sleeping seconds for recoding loop :type sleeping_time:int """ super().__init__() self.logger = logging.getLogger(self.__class__.__name__) assert self.provider is not None assert self.data_schema is not None assert self.provider in self.data_schema.providers self.batch_size = batch_size self.force_update = force_update self.sleeping_time = sleeping_time # using to do db operations self.session = get_db_session(provider=self.provider, data_schema=self.data_schema) self.state_session = get_db_session(provider='zvt', data_schema=RecorderState) if not self.name: self.name = type(self).__name__.lower() self.recorder_state = None self.state = None states = RecorderState.query_data( provider='zvt', filters=[RecorderState.recoder_name == self.name], return_type='domain') if states: self.recorder_state = states[0] self.state: dict = self.decode_state(self.recorder_state.state)
def __init__(self, force=False) -> None: super().__init__() assert self.entity_schema is not None assert self.data_schema is not None self.force = force self.session = get_db_session(provider="zvt", data_schema=self.data_schema) if self.state and not self.force: logger.info(f"get start_timestamp from state") self.start_timestamp = self.state["current_timestamp"] logger.info(f"tag start_timestamp: {self.start_timestamp}")
def persist_factor(self): df = self.factor_df.copy() # encode json columns if pd_is_not_null(df) and self.factor_col_map_object_hook(): for col in self.factor_col_map_object_hook(): if col in df.columns: df[col] = df[col].apply( lambda x: json.dumps(x, cls=self.factor_encoder())) if self.states: session = get_db_session(region=self.region, provider=Provider.ZVT, data_schema=FactorState) g = df.groupby(level=0) for entity_id in self.states: state = self.states[entity_id] try: if state: domain_id = f'{self.factor_name}_{entity_id}' factor_state: FactorState = session.query( FactorState).get(domain_id) state_str = self.encode_state(state) if factor_state: factor_state.state = state_str else: factor_state = FactorState( id=domain_id, entity_id=entity_id, factor_name=self.factor_name, state=state_str) session.add(factor_state) session.commit() if entity_id in g.groups: df_to_db(df=df.loc[(entity_id, )], ref_df=None, region=self.region, data_schema=self.factor_schema, provider=Provider.ZVT) except Exception as e: self.logger.error( f'{self.factor_name} {entity_id} save state error') self.logger.exception(e) # clear them if error happen self.clear_state_data(self.region, entity_id) else: df_to_db(df=df, ref_df=None, region=self.region, data_schema=self.factor_schema, provider=Provider.ZVT)
def __init__(self, entity_schema: EntityMixin, trader_name, timestamp, provider=None, level=IntervalLevel.LEVEL_1DAY, base_capital=1000000, buy_cost=0.001, sell_cost=0.001, slippage=0.001): self.entity_schema = entity_schema self.base_capital = base_capital self.buy_cost = buy_cost self.sell_cost = sell_cost self.slippage = slippage self.trader_name = trader_name self.session = get_db_session('zvt', data_schema=TraderInfo) self.provider = provider self.level = level self.start_timestamp = timestamp account = get_trader_info(session=self.session, trader_name=self.trader_name, return_type='domain', limit=1) if account: self.logger.warning( "trader:{} has run before,old result would be deleted".format( trader_name)) self.session.query(TraderInfo).filter( TraderInfo.trader_name == self.trader_name).delete() self.session.query(AccountStats).filter( AccountStats.trader_name == self.trader_name).delete() self.session.query(Position).filter( Position.trader_name == self.trader_name).delete() self.session.query(Order).filter( Order.trader_name == self.trader_name).delete() self.session.commit() account = AccountStats(entity_id=f'trader_zvt_{self.trader_name}', trader_name=self.trader_name, cash=self.base_capital, all_value=self.base_capital, value=0, closing=False, timestamp=timestamp) self.latest_account = account_stats_schema.dump(account)
def persist_factor(self): df_to_db(df=self.factor_df, data_schema=self.factor_schema, provider='zvt', force_update=False) if self.states: session = get_db_session(provider='zvt', data_schema=FactorState) for entity_id, state in self.states: domain_id = f'{self.factor_name}_{entity_id}' factor_state: FactorState = session.query(FactorState).get(domain_id) state_str = json.dumps(state) if factor_state: factor_state.state = state_str else: factor_state = FactorState(id=domain_id, entity_id=entity_id, factor_name=self.factor_name, state=json.dumps(state)) session.add(factor_state) session.commit()
def on_start(self): # run all the selectors for selector in self.selectors: # run for the history data at first selector.run() if self.entity_ids: entity_ids = json.dumps(self.entity_ids) else: entity_ids = None if self.exchanges: exchanges = json.dumps(self.exchanges) else: exchanges = None if self.codes: codes = json.dumps(self.codes) else: codes = None entity_type = self.entity_schema.__name__.lower() sim_account = TraderInfo( id=self.trader_name, entity_id=f'trader_zvt_{self.trader_name}', timestamp=self.start_timestamp, trader_name=self.trader_name, entity_type=entity_type, entity_ids=entity_ids, exchanges=exchanges, codes=codes, start_timestamp=self.start_timestamp, end_timestamp=self.end_timestamp, provider=self.provider, level=self.level.value, real_time=self.real_time, kdata_use_begin_time=self.kdata_use_begin_time, kdata_adjust_type=self.adjust_type.value) session = get_db_session(region=Region.CHN, provider=Provider.ZVT, data_schema=TraderInfo) session.add(sim_account) session.commit()
def init_entities(self): self.entity_session = get_db_session(provider=self.entity_provider, data_schema=self.entity_schema) self.entities = get_entities( session=self.entity_session, entity_type='index', exchanges=self.exchanges, codes=self.codes, entity_ids=self.entity_ids, return_type='domain', provider=self.provider, # 只抓概念和行业 filters=[ Index.category.in_([ BlockCategory.industry.value, BlockCategory.concept.value ]) ])
def __init__( self, entity_schema: TradableEntity, trader_name, timestamp, provider=None, level=IntervalLevel.LEVEL_1DAY, base_capital=1000000, buy_cost=0.001, sell_cost=0.001, slippage=0.001, rich_mode=True, adjust_type: AdjustType = None, keep_history=False, real_time=False, kdata_use_begin_time=False, ): self.entity_schema = entity_schema self.base_capital = base_capital self.buy_cost = buy_cost self.sell_cost = sell_cost self.slippage = slippage self.rich_mode = rich_mode self.adjust_type = adjust_type self.trader_name = trader_name self.session = get_db_session("zvt", data_schema=TraderInfo) self.provider = provider self.level = level self.start_timestamp = timestamp self.keep_history = keep_history self.real_time = real_time self.kdata_use_begin_time = kdata_use_begin_time self.account = None self.account = self.init_account() account_info = ( f"init_account,holding size:{len(self.account.positions)} profit:{self.account.profit} input_money:{self.account.input_money} " f"cash:{self.account.cash} value:{self.account.value} all_value:{self.account.all_value}" ) self.logger.info(account_info)
def init_entities(self): """ init the entities which we would record data for """ if self.entity_provider == self.provider and self.entity_schema == self.data_schema: self.entity_session = self.session else: self.entity_session = get_db_session( provider=self.entity_provider, data_schema=self.entity_schema) # init the entity list self.entities = get_entities(session=self.entity_session, entity_schema=self.entity_schema, entity_type=self.entity_type, exchanges=self.exchanges, entity_ids=self.entity_ids, codes=self.codes, return_type='domain', provider=self.entity_provider)
def __init__(self, force_update: bool = False, sleeping_time: int = 10) -> None: super().__init__() self.logger = logging.getLogger(self.__class__.__name__) assert self.provider is not None assert self.data_schema is not None if self.provider not in self.data_schema.providers: self.logger.error( f"provider: {self.provider} is not registered for {self.data_schema}({self.data_schema.providers})" ) assert False self.force_update = force_update self.sleeping_time = sleeping_time #: using to do db operations self.session = get_db_session(provider=self.provider, data_schema=self.data_schema)
def init_entities(self): assert self.region is not None self.entity_session = get_db_session(region=self.region, provider=self.entity_provider, data_schema=self.entity_schema) self.entities = get_entities( region=self.region, session=self.entity_session, entity_type=EntityType.Index, exchanges=self.exchanges, codes=self.codes, entity_ids=self.entity_ids, return_type='domain', provider=self.provider, # 只抓概念和行业 filters=[ Index.category.in_([ BlockCategory.industry.value, BlockCategory.concept.value ]) ])
def init_entities(self): """ init the entities which we would record data for """ if self.entity_provider == self.provider and self.entity_schema == self.data_schema: self.entity_session = self.session else: self.entity_session = get_db_session( provider=self.entity_provider, data_schema=self.entity_schema) if self.day_data: df = self.data_schema.query_data( start_timestamp=now_time_str(), columns=["entity_id", "timestamp"], provider=self.provider) if pd_is_not_null(df): entity_ids = df["entity_id"].tolist() self.logger.info(f"ignore entity_ids:{entity_ids}") if self.entity_filters: self.entity_filters.append( self.entity_schema.entity_id.notin_(entity_ids)) else: self.entity_filters = [ self.entity_schema.entity_id.notin_(entity_ids) ] #: init the entity list self.entities = get_entities( session=self.entity_session, entity_schema=self.entity_schema, exchanges=self.exchanges, entity_ids=self.entity_ids, codes=self.codes, return_type="domain", provider=self.entity_provider, filters=self.entity_filters, )
from zvt.contract.api import get_db_session from ..context import init_test_context init_test_context() from typing import List from zvt.domain import HolderTrading, ManagerTrading session = get_db_session(provider='eastmoney', db_name='trading') # type: sqlalchemy.orm.Session # 股东交易 def test_000778_holder_trading(): result: List[HolderTrading] = HolderTrading.query_data( session=session, provider='eastmoney', return_type='domain', codes=['000778'], end_timestamp='2018-09-30', start_timestamp='2018-09-30', order=HolderTrading.holding_pct.desc()) assert len(result) == 6 assert result[0].holder_name == '新兴际华集团有限公司' assert result[0].change_pct == 0.0205 assert result[0].volume == 32080000 assert result[0].holding_pct == 0.3996 # 高管交易
from ..context import init_test_context init_test_context() from zvt.domain import FinanceFactor, BalanceSheet, IncomeStatement, CashFlowStatement from zvt.contract.api import get_db_session from zvt.utils.time_utils import to_time_str session = get_db_session(provider='eastmoney', db_name='finance') # type: sqlalchemy.orm.Session # 银行指标 def test_000001_finance_factor(): correct_timestamps = [ '2018-09-30', '2018-06-30', '2018-03-31', '2017-12-31', '2017-09-30', '2017-06-30', '2017-03-31', '2016-12-31', '2016-09-30', '2016-06-30', '2016-03-31', '2015-12-31', '2015-09-30', '2015-06-30', '2015-03-31', '2014-12-31', '2014-09-30', '2014-06-30', '2014-03-31', '2013-12-31', '2013-09-30', '2013-06-30', '2013-03-31', '2012-12-31', '2012-09-30', '2012-06-30', '2012-03-31', '2011-12-31', '2011-09-30', '2011-06-30', '2011-03-31', '2010-12-31', '2010-09-30', '2010-06-30', '2010-03-31', '2009-12-31', '2009-09-30', '2009-06-30', '2009-03-31', '2008-12-31', '2008-09-30', '2008-06-30', '2008-03-31', '2007-12-31', '2007-09-30', '2007-06-30', '2007-03-31', '2006-12-31', '2006-09-30', '2006-06-30', '2006-03-31', '2005-12-31', '2005-09-30', '2005-06-30', '2005-03-31', '2004-12-31', '2004-09-30', '2004-06-30', '2004-03-31', '2003-12-31', '2003-09-30', '2003-06-30', '2003-03-31', '2002-12-31', '2002-09-30', '2002-06-30', '2002-03-31', '2001-12-31', '2001-09-30', '2001-06-30', '2001-03-31', '2000-12-31', '2000-06-30', '1999-12-31', '1999-06-30', '1998-12-31', '1998-06-30', '1997-12-31', '1997-06-30', '1996-12-31',
def __init__(self, entity_ids: List[str] = None, exchanges: List[str] = None, codes: List[str] = None, start_timestamp: Union[str, pd.Timestamp] = None, end_timestamp: Union[str, pd.Timestamp] = None, provider: str = None, level: Union[str, IntervalLevel] = IntervalLevel.LEVEL_1DAY, trader_name: str = None, real_time: bool = False, kdata_use_begin_time: bool = False, draw_result: bool = True) -> None: assert self.entity_schema is not None self.logger = logging.getLogger(__name__) if trader_name: self.trader_name = trader_name else: self.trader_name = type(self).__name__.lower() self.trading_signal_listeners: List[TradingListener] = [] self.selectors: List[TargetSelector] = [] self.entity_ids = entity_ids self.exchanges = exchanges self.codes = codes self.provider = provider # make sure the min level selector correspond to the provider and level self.level = IntervalLevel(level) self.real_time = real_time if start_timestamp and end_timestamp: self.start_timestamp = to_pd_timestamp(start_timestamp) self.end_timestamp = to_pd_timestamp(end_timestamp) else: assert False self.trading_dates = self.entity_schema.get_trading_dates( start_date=self.start_timestamp, end_date=self.end_timestamp) if real_time: logger.info( 'real_time mode, end_timestamp should be future,you could set it big enough for running forever' ) assert self.end_timestamp >= now_pd_timestamp() self.kdata_use_begin_time = kdata_use_begin_time self.draw_result = draw_result self.account_service = SimAccountService( entity_schema=self.entity_schema, trader_name=self.trader_name, timestamp=self.start_timestamp, provider=self.provider, level=self.level) self.add_trading_signal_listener(self.account_service) self.init_selectors(entity_ids=entity_ids, entity_schema=self.entity_schema, exchanges=self.exchanges, codes=self.codes, start_timestamp=self.start_timestamp, end_timestamp=self.end_timestamp) if self.selectors: self.trading_level_asc = list( set([ IntervalLevel(selector.level) for selector in self.selectors ])) self.trading_level_asc.sort() self.logger.info( f'trader level:{self.level},selectors level:{self.trading_level_asc}' ) if self.level != self.trading_level_asc[0]: raise Exception( "trader level should be the min of the selectors") self.trading_level_desc = list(self.trading_level_asc) self.trading_level_desc.reverse() self.targets_slot: TargetsSlot = TargetsSlot() self.session = get_db_session('zvt', data_schema=TraderInfo) self.on_start()
from zvt.api import get_kdata from zvt.contract import IntervalLevel from zvt.contract.api import get_db_session from ..context import init_test_context init_test_context() day_k_session = get_db_session(provider='joinquant', db_name='stock_1d_kdata') # type: sqlalchemy.orm.Session day_1h_session = get_db_session(provider='joinquant', db_name='stock_1h_kdata') # type: sqlalchemy.orm.Session def test_jq_603220_kdata(): df = get_kdata(entity_id='stock_sh_603220', session=day_k_session, level=IntervalLevel.LEVEL_1DAY, provider='joinquant') print(df) df = get_kdata(entity_id='stock_sh_603220', session=day_1h_session, level=IntervalLevel.LEVEL_1HOUR, provider='joinquant') print(df)
from ..context import init_test_context init_test_context() from zvt.domain import SpoDetail, RightsIssueDetail, DividendFinancing from zvt.contract.api import get_db_session from zvt.utils.time_utils import to_pd_timestamp session = get_db_session( provider='eastmoney', db_name='dividend_financing') # type: sqlalchemy.orm.Session # 增发详情 def test_000778_spo_detial(): result = SpoDetail.query_data(session=session, provider='eastmoney', return_type='domain', codes=['000778'], end_timestamp='2018-09-30', order=SpoDetail.timestamp.desc()) assert len(result) == 4 latest: SpoDetail = result[0] assert latest.timestamp == to_pd_timestamp('2017-04-01') assert latest.spo_issues == 347600000 assert latest.spo_price == 5.15 assert latest.spo_raising_fund == 1766000000 # 配股详情 def test_000778_rights_issue_detail():
from zvt.contract.api import get_db_session from ..context import init_test_context init_test_context() from typing import List from zvt.domain import TopTenHolder, TopTenTradableHolder session = get_db_session(provider="eastmoney", db_name="holder") # type: sqlalchemy.orm.Session # 十大股东 def test_000778_top_ten_holder(): result: List[TopTenHolder] = TopTenHolder.query_data( session=session, provider="eastmoney", return_type="domain", codes=["000778"], end_timestamp="2018-09-30", start_timestamp="2018-09-30", order=TopTenHolder.shareholding_ratio.desc(), ) assert len(result) == 10 assert result[0].holder_name == "新兴际华集团有限公司" assert result[0].shareholding_numbers == 1595000000 assert result[0].shareholding_ratio == 0.3996 assert result[0].change == 32080000 assert result[0].change_ratio == 0.0205
from ..context import init_test_context init_test_context() from zvt.domain import SpoDetail, RightsIssueDetail, DividendFinancing from zvt.contract.api import get_db_session from zvt.utils.time_utils import to_pd_timestamp session = get_db_session( provider='eastmoney', db_name='shares_change') # type: sqlalchemy.orm.Session # 增发详情 def test_000778_spo_detial(): result = SpoDetail.query_data(session=session, provider='eastmoney', return_type='domain', codes=['000778'], end_timestamp='2018-09-30', order=SpoDetail.timestamp.desc()) assert len(result) == 4 latest: SpoDetail = result[0] assert latest.timestamp == to_pd_timestamp('2017-04-01') assert latest.spo_issues == 347600000 assert latest.spo_price == 5.15 assert latest.spo_raising_fund == 1766000000 # 配股详情 def test_000778_rights_issue_detail():
def __init__(self) -> None: assert self.state_schema is not None if self.name is None: self.name = to_snake_str(type(self).__name__) self.state_session = get_db_session(data_schema=self.state_schema, provider="zvt")