def every_day_report(): while True: try: today = now_pd_timestamp() long_targets = select_by_finance(today) logger.info(f'selected:{len(long_targets)}') if long_targets: long_targets = list(set(long_targets)) df = get_entities(provider='eastmoney', entity_schema=Stock, entity_ids=long_targets, columns=['code', 'name']) info = [df.loc[i, 'code'] + ' ' + df.loc[i, 'name'] for i in df.index] msg = ' '.join(info) else: msg = 'no targets' logger.info(msg) email_action = EmailInformer() email_action.send_message("*****@*****.**", f'{today} 基本面选股结果', msg) break except Exception as e: logger.exception('report2 sched error:{}'.format(e)) time.sleep(60 * 3)
def get_securities_in_blocks( provider: str = 'eastmoney', categories: List[Union[str, BlockCategory]] = ['concept', 'industry'], names=None, codes=None, ids=None): session = get_db_session(provider=provider, data_schema=Index) categories = [BlockCategory(category).value for category in categories] filters = [Index.category.in_(categories)] # add name filters if names: filters.append(Index.name.in_(names)) blocks = get_entities(entity_ids=ids, codes=codes, entity_type='index', provider=provider, filters=filters, return_type='domain', session=session) securities = [] for block in blocks: securities += [item.stock_id for item in block.stocks] return securities
def cal_ma_states(start='000001', end='002000'): logger.info(f'start cal day ma stats {start}:{end}') entities = get_entities(provider='eastmoney', entity_type='stock', columns=[Stock.entity_id, Stock.code], filters=[Stock.code >= start, Stock.code < end]) codes = entities.index.to_list() ma_1d_stats = MaStateStas(codes=codes, start_timestamp='2005-01-01', end_timestamp=now_pd_timestamp(), level=IntervalLevel.LEVEL_1DAY) ma_1d_factor = MaFactor(codes=codes, start_timestamp='2005-01-01', end_timestamp=now_pd_timestamp(), level=IntervalLevel.LEVEL_1DAY) logger.info(f'finish cal day ma stats {start}:{end}') ma_1wk_stats = MaStateStas(codes=codes, start_timestamp='2005-01-01', end_timestamp=now_pd_timestamp(), level=IntervalLevel.LEVEL_1WEEK) logger.info(f'finish cal week ma stats {start}:{end}')
def get_indices(provider: str = 'sina', block_category: Union[str, StockCategory] = 'concept', return_type: str = 'df') -> object: """ get indices/blocks on block_category :param provider: :type provider: :param block_category: :type block_category: :param return_type: :type return_type: :return: :rtype: """ if type(block_category) == StockCategory: block_category = block_category.value session = get_db_session(provider=provider, data_schema=Index) filters = [Index.category == block_category] blocks = get_entities(entity_type='index', provider=provider, filters=filters, session=session, return_type=return_type) return blocks
def report_core_company(): while True: error_count = 0 email_action = EmailInformer() try: # StockTradeDay.record_data(provider='joinquant') # Stock.record_data(provider='joinquant') # FinanceFactor.record_data(provider='eastmoney') # BalanceSheet.record_data(provider='eastmoney') target_date = to_time_str(now_pd_timestamp()) my_selector: TargetSelector = FundamentalSelector( start_timestamp='2015-01-01', end_timestamp=target_date) my_selector.run() long_targets = my_selector.get_open_long_targets( timestamp=target_date) if long_targets: stocks = get_entities(provider='joinquant', entity_schema=Stock, entity_ids=long_targets, return_type='domain') # add them to eastmoney try: try: eastmoneypy.del_group('core') except: pass eastmoneypy.create_group('core') for stock in stocks: eastmoneypy.add_to_group(stock.code, group_name='core') except Exception as e: email_action.send_message( "*****@*****.**", f'report_core_company error', 'report_core_company error:{}'.format(e)) info = [f'{stock.name}({stock.code})' for stock in stocks] msg = ' '.join(info) else: msg = 'no targets' logger.info(msg) email_action.send_message(get_subscriber_emails(), f'{to_time_str(target_date)} 核心资产选股结果', msg) break except Exception as e: logger.exception('report_core_company error:{}'.format(e)) time.sleep(60 * 3) error_count = error_count + 1 if error_count == 10: email_action.send_message( "*****@*****.**", f'report_core_company error', 'report_core_company error:{}'.format(e))
def report_vol_up_250(): while True: error_count = 0 email_action = EmailInformer() try: # 抓取k线数据 # StockTradeDay.record_data(provider='joinquant') # Stock1dKdata.record_data(provider='joinquant') latest_day: Stock1dKdata = Stock1dKdata.query_data(order=Stock1dKdata.timestamp.desc(), limit=1, return_type='domain') target_date = latest_day[0].timestamp # 计算均线 my_selector = TargetSelector(start_timestamp='2018-01-01', end_timestamp=target_date) # add the factors factor1 = VolumeUpMa250Factor(start_timestamp='2018-01-01', end_timestamp=target_date) my_selector.add_filter_factor(factor1) my_selector.run() long_targets = my_selector.get_open_long_targets(timestamp=target_date) if long_targets: stocks = get_entities(provider='joinquant', entity_schema=Stock, entity_ids=long_targets, return_type='domain') # add them to eastmoney try: try: eastmoneypy.del_group('tech') except: pass eastmoneypy.create_group('tech') for stock in stocks: eastmoneypy.add_to_group(stock.code, group_name='tech') except Exception as e: email_action.send_message("*****@*****.**", f'report_vol_up_250 error', 'report_vol_up_250 error:{}'.format(e)) info = [f'{stock.name}({stock.code})' for stock in stocks] msg = ' '.join(info) else: msg = 'no targets' logger.info(msg) email_action.send_message(get_subscriber_emails(), f'{target_date} 放量突破年线选股结果', msg) break except Exception as e: logger.exception('report_vol_up_250 error:{}'.format(e)) time.sleep(60 * 3) error_count = error_count + 1 if error_count == 10: email_action.send_message("*****@*****.**", f'report_vol_up_250 error', 'report_vol_up_250 error:{}'.format(e))
def report_cross_ma(): while True: error_count = 0 email_action = EmailInformer() try: # 抓取k线数据 # StockTradeDay.record_data(provider='joinquant') # Stock1dKdata.record_data(provider='joinquant') latest_day: StockTradeDay = StockTradeDay.query_data( order=StockTradeDay.timestamp.desc(), limit=1, return_type='domain') if latest_day: target_date = latest_day[0].timestamp else: target_date = now_pd_timestamp() # 计算均线 my_selector = TargetSelector(start_timestamp='2018-01-01', end_timestamp=target_date) # add the factors ma_factor = CrossMaFactor(start_timestamp='2018-01-01', end_timestamp=target_date) my_selector.add_filter_factor(ma_factor) my_selector.run() long_targets = my_selector.get_open_long_targets( timestamp=target_date) if long_targets: stocks = get_entities(provider='joinquant', entity_schema=Stock, entity_ids=long_targets, return_type='domain') info = [f'{stock.name}({stock.code})' for stock in stocks] msg = ' '.join(info) else: msg = 'no targets' logger.info(msg) email_action.send_message("*****@*****.**", f'{target_date} 均线选股结果', msg) break except Exception as e: logger.exception('report_cross_ma error:{}'.format(e)) time.sleep(60 * 3) error_count = error_count + 1 if error_count == 10: email_action.send_message("*****@*****.**", f'report_cross_ma error', 'report_cross_ma error:{}'.format(e))
def init_entities(self): self.entity_session = get_db_session(provider=self.entity_provider, data_schema=self.entity_schema) self.entities = get_entities(session=self.entity_session, entity_type='index', exchanges=self.exchanges, codes=self.codes, entity_ids=self.entity_ids, return_type='domain', provider=self.provider, # 只抓概念和行业 filters=[Index.category.in_( [StockCategory.industry.value, StockCategory.concept.value])])
def report_core_company(): while True: error_count = 0 email_action = EmailInformer() try: StockTradeDay.record_data(provider='joinquant') Stock.record_data(provider='joinquant') FinanceFactor.record_data(provider='eastmoney') BalanceSheet.record_data(provider='eastmoney') latest_day: StockTradeDay = StockTradeDay.query_data( order=StockTradeDay.timestamp.desc(), limit=1, return_type='domain') if latest_day: target_date = latest_day[0].timestamp else: target_date = now_pd_timestamp() my_selector: TargetSelector = FundamentalSelector( start_timestamp='2015-01-01', end_timestamp=target_date) my_selector.run() long_targets = my_selector.get_open_long_targets( timestamp=target_date) if long_targets: stocks = get_entities(provider='joinquant', entity_schema=Stock, entity_ids=long_targets, return_type='domain') info = [f'{stock.name}({stock.code})' for stock in stocks] msg = ' '.join(info) else: msg = 'no targets' logger.info(msg) email_action.send_message([ '*****@*****.**', '*****@*****.**', '*****@*****.**', '*****@*****.**', '*****@*****.**' ], f'{to_time_str(target_date)} 核心资产选股结果', msg) break except Exception as e: logger.exception('report_core_company error:{}'.format(e)) time.sleep(60 * 3) error_count = error_count + 1 if error_count == 10: email_action.send_message( "*****@*****.**", f'report_core_company error', 'report_core_company error:{}'.format(e))
def every_day_report(): while True: try: t = now_pd_timestamp() if t.dayofweek in (5, 6): logger.info(f'today:{t} is {t.day_name()},just ignore') today = to_time_str(t) # 抓取k线数据 JqChinaStockKdataRecorder(level=IntervalLevel.LEVEL_1DAY).run() JqChinaStockKdataRecorder(level=IntervalLevel.LEVEL_1WEEK).run() JqChinaStockKdataRecorder(level=IntervalLevel.LEVEL_1MON).run() # 计算均线 my_selector = TargetSelector(start_timestamp='2016-01-01', end_timestamp=today) # add the factors # 设置dry_run为True,因为我们只需要最近的数据,不需要加载全量数据进行回测 ma_factor = CrossMaFactor(start_timestamp='2016-01-01', end_timestamp=today, dry_run=True) my_selector.add_filter_factor(ma_factor) my_selector.run() long_targets = my_selector.get_open_long_targets(timestamp=today) if long_targets: df = get_entities(provider='eastmoney', entity_schema=Stock, entity_ids=long_targets, columns=['code', 'name']) info = [ df.loc[i, 'code'] + ' ' + df.loc[i, 'name'] for i in df.index ] msg = ' '.join(info) else: msg = 'no targets' logger.info(msg) email_action = EmailInformer() email_action.send_message("*****@*****.**", f'{today} 均线选股结果', msg) break except Exception as e: logger.exception('report1 sched error:{}'.format(e)) time.sleep(60 * 3)
def init_entities(self): if self.entity_provider == self.provider and self.entity_schema == self.data_schema: self.entity_session = self.session else: self.entity_session = get_db_session(provider=self.entity_provider, data_schema=self.entity_schema) # init the entity list self.entities = get_entities(session=self.entity_session, entity_type=self.entity_type, exchanges=self.exchanges, entity_ids=self.entity_ids, codes=self.codes, return_type='domain', provider=self.entity_provider)
def every_day_report(): while True: try: today = now_pd_timestamp() long_targets = select_by_finance(today) logger.info(f'selected:{len(long_targets)}') if long_targets: ma_factor = CrossMaFactor(start_timestamp='2015-01-01', end_timestamp=today, dry_run=True, persist_factor=False, entity_ids=long_targets, windows=[5, 30, 120]) my_selector = TargetSelector(start_timestamp='2015-01-01', end_timestamp=today, entity_ids=long_targets) my_selector.add_filter_factor(ma_factor) my_selector.run() final_targets = my_selector.get_open_long_targets(today) final_targets = list(set(final_targets)) logger.info(f'final selected:{len(final_targets)}') if final_targets: df = get_entities(provider='eastmoney', entity_schema=Stock, entity_ids=final_targets, columns=['code', 'name']) info = [ df.loc[i, 'code'] + ' ' + df.loc[i, 'name'] for i in df.index ] msg = ' '.join(info) else: msg = 'no targets' logger.info(msg) email_action = EmailInformer() email_action.send_message("*****@*****.**", f'{today} 基本面 + 技术面选股结果', msg) break except Exception as e: logger.exception('report3 sched error:{}'.format(e)) time.sleep(60 * 3)
def init_entities(self): if self.entity_provider == self.provider and self.entity_schema == self.data_schema: self.entity_session = self.session else: self.entity_session = get_db_session( provider=self.entity_provider, data_schema=self.entity_schema) # init the entity list self.entities = get_entities( session=self.entity_session, entity_type=self.entity_type, entity_ids=self.entity_ids, codes=self.codes, return_type='domain', provider=self.entity_provider, # 最近7天更新过的跳过 filters=[(GithubUser.updated_timestamp < day_offset_today(-7)) | (GithubUser.updated_timestamp.is_(None))], start_timestamp=self.start_timestamp, end_timestamp=self.end_timestamp)
def get_securities_in_blocks(block_names=['HS300_'], block_category='concept', provider='eastmoney'): session = get_db_session(provider=provider, data_schema=Index) filters = [Index.category == block_category] name_filters = None for block_name in block_names: if name_filters: name_filters |= (Index.name == block_name) else: name_filters = (Index.name == block_name) filters.append(name_filters) blocks = get_entities(entity_type='index', provider='eastmoney', filters=filters, return_type='domain', session=session) securities = [] for block in blocks: securities += [item.stock_id for item in block.stocks] return securities
limit=limit) def get_current_price(entity_ids=None, entity_type='coin'): result = {} if entity_type == 'coin': if entity_ids: for entity_id in entity_ids: a, exchange, code = decode_entity_id(entity_id) assert a == entity_type ccxt_exchange = CCXTAccount.get_ccxt_exchange( exchange_str=exchange) if not ccxt_exchange: raise Exception('{} not support'.format(exchange)) orderbook = ccxt_exchange.fetch_order_book(code) bid = orderbook['bids'][0][0] if len( orderbook['bids']) > 0 else None ask = orderbook['asks'][0][0] if len( orderbook['asks']) > 0 else None entity_id = f'coin_{exchange}_{code}' result[entity_id] = (bid, ask) return result if __name__ == '__main__': get_entities(provider='linkedin', entity_type='user', limit=10)
def report_real(): while True: error_count = 0 email_action = EmailInformer(ssl=True) try: latest_day: Stock1dKdata = Stock1dKdata.query_data( order=Stock1dKdata.timestamp.desc(), limit=1, return_type='domain') target_date = latest_day[0].timestamp # target_date = '2020-02-04' # 计算均线 my_selector = TargetSelector(start_timestamp='2018-01-01', end_timestamp=target_date) # add the factors factor1 = VolumeUpMa250Factor(start_timestamp='2018-01-01', end_timestamp=target_date) my_selector.add_filter_factor(factor1) my_selector.run() long_stocks = my_selector.get_open_long_targets( timestamp=target_date) msg = 'no targets' # 过滤亏损股 # check StockValuation data pe_date = target_date - datetime.timedelta(10) if StockValuation.query_data(start_timestamp=pe_date, limit=1, return_type='domain'): positive_df = StockValuation.query_data( provider='joinquant', entity_ids=long_stocks, start_timestamp=pe_date, filters=[StockValuation.pe > 0], columns=['entity_id']) bad_stocks = set(long_stocks) - set( positive_df['entity_id'].tolist()) if bad_stocks: stocks = get_entities(provider='joinquant', entity_schema=Stock, entity_ids=bad_stocks, return_type='domain') info = [f'{stock.name}({stock.code})' for stock in stocks] msg = '亏损股:' + ' '.join(info) + '\n' long_stocks = set(positive_df['entity_id'].tolist()) if long_stocks: # use block to filter block_selector = BlockSelector(start_timestamp='2020-01-01', long_threshold=0.8) block_selector.run() long_blocks = block_selector.get_open_long_targets( timestamp=target_date) if long_blocks: blocks: List[Block] = Block.query_data( provider='sina', entity_ids=long_blocks, return_type='domain') info = [f'{block.name}({block.code})' for block in blocks] msg = ' '.join(info) + '\n' block_stocks: List[BlockStock] = BlockStock.query_data( provider='sina', filters=[BlockStock.stock_id.in_(long_stocks)], entity_ids=long_blocks, return_type='domain') if block_stocks: # add them to eastmoney try: try: eastmoneypy.del_group('real') except: pass eastmoneypy.create_group('real') for block_stock in block_stocks: eastmoneypy.add_to_group( block_stock.stock_code, group_name='real') except Exception as e: email_action.send_message( "*****@*****.**", f'report_real error', 'report_real error:{}'.format(e)) block_map_stocks = {} for block_stock in block_stocks: stocks = block_map_stocks.get(block_stock.name) if not stocks: stocks = [] block_map_stocks[block_stock.name] = stocks stocks.append( f'{block_stock.stock_name}({block_stock.stock_code})' ) for block in block_map_stocks: stocks = block_map_stocks[block] stock_msg = ' '.join(stocks) msg = msg + f'{block}:\n' + stock_msg + '\n' logger.info(msg) email_action.send_message('*****@*****.**', f'{target_date} 放量突破年线real选股结果', msg) break except Exception as e: logger.exception('report_real error:{}'.format(e)) time.sleep(60 * 3) error_count = error_count + 1 if error_count == 10: email_action.send_message("*****@*****.**", f'report_real error', 'report_real error:{}'.format(e))
def report_state(): while True: error_count = 0 email_action = EmailInformer(ssl=True) try: latest_day: Stock1dKdata = Stock1dKdata.query_data(order=Stock1dKdata.timestamp.desc(), limit=1, return_type='domain') target_date = latest_day[0].timestamp # target_date = to_pd_timestamp('2020-01-02') # 计算均线 my_selector = TargetSelector(start_timestamp='2018-01-01', end_timestamp=target_date) # add the factors factor1 = VolumeUpMa250Factor(start_timestamp='2018-01-01', end_timestamp=target_date) my_selector.add_filter_factor(factor1) my_selector.run() long_stocks = my_selector.get_open_long_targets(timestamp=target_date) msg = 'no targets' # 过滤亏损股 # check StockValuation data pe_date = target_date - datetime.timedelta(10) if StockValuation.query_data(start_timestamp=pe_date, limit=1, return_type='domain'): positive_df = StockValuation.query_data(provider='joinquant', entity_ids=long_stocks, start_timestamp=pe_date, filters=[StockValuation.pe > 0], columns=['entity_id']) bad_stocks = set(long_stocks) - set(positive_df['entity_id'].tolist()) if bad_stocks: stocks = get_entities(provider='joinquant', entity_schema=Stock, entity_ids=bad_stocks, return_type='domain') info = [f'{stock.name}({stock.code})' for stock in stocks] msg = '亏损股:' + ' '.join(info) + '\n' long_stocks = set(positive_df['entity_id'].tolist()) if long_stocks: pre_date = target_date - datetime.timedelta(3 * 365) ma_state = MaStateStatsFactor(entity_ids=long_stocks, start_timestamp=pre_date, end_timestamp=target_date, persist_factor=False) bad_stocks = [] for entity_id, df in ma_state.factor_df.groupby(level=0): if df['current_pct'].max() >= 0.35: bad_stocks.append(entity_id) long_stocks.remove(entity_id) if bad_stocks: stocks = get_entities(provider='joinquant', entity_schema=Stock, entity_ids=bad_stocks, return_type='domain') info = [f'{stock.name}({stock.code})' for stock in stocks] msg = msg + '3年内高潮过:' + ' '.join(info) + '\n' # 过滤风险股 if long_stocks: risky_codes = risky_company(the_date=target_date, entity_ids=long_stocks) if risky_codes: long_stocks = [entity_id for entity_id in long_stocks if get_entity_code(entity_id) not in risky_codes] stocks = get_entities(provider='joinquant', entity_schema=Stock, entity_ids=risky_codes, return_type='domain') info = [f'{stock.name}({stock.code})' for stock in stocks] msg = msg + '风险股:' + ' '.join(info) + '\n' if long_stocks: stocks = get_entities(provider='joinquant', entity_schema=Stock, entity_ids=long_stocks, return_type='domain') # add them to eastmoney try: try: eastmoneypy.del_group('real') except: pass eastmoneypy.create_group('real') for stock in stocks: eastmoneypy.add_to_group(stock.code, group_name='real') except Exception as e: email_action.send_message("*****@*****.**", f'report state error', 'report state error:{}'.format(e)) info = [f'{stock.name}({stock.code})' for stock in stocks] msg = msg + '盈利股:' + ' '.join(info) + '\n' logger.info(msg) email_action.send_message('*****@*****.**', f'{target_date} 放量突破年线state选股结果', msg) break except Exception as e: logger.exception('report state error:{}'.format(e)) time.sleep(60 * 3) error_count = error_count + 1 if error_count == 10: email_action.send_message("*****@*****.**", f'report state error', 'report state error:{}'.format(e))
def test_get_entities(): df = get_entities(entity_type='stock', provider='sina') assert '000001' in df.index assert '12345' not in df.index
assert a == entity_type ccxt_exchange = CCXTAccount.get_ccxt_exchange(exchange_str=exchange) if not ccxt_exchange: raise Exception('{} not support'.format(exchange)) orderbook = ccxt_exchange.fetch_order_book(code) bid = orderbook['bids'][0][0] if len(orderbook['bids']) > 0 else None ask = orderbook['asks'][0][0] if len(orderbook['asks']) > 0 else None entity_id = f'coin_{exchange}_{code}' result[entity_id] = (bid, ask) return result if __name__ == '__main__': money_flow_session = get_db_session(provider='sina', data_schema=IndexMoneyFlow) entities = get_entities(entity_type='index', return_type='domain', provider='sina', # 只抓概念和行业 filters=[Index.category.in_( [StockCategory.industry.value, StockCategory.concept.value])]) for entity in entities: sql = 'UPDATE index_money_flow SET name="{}" where code="{}"'.format( entity.name, entity.code) money_flow_session.execute(sql) money_flow_session.commit()
def __init__(self, data_schema: Mixin, entity_schema: EntityMixin, provider: str = None, entity_provider: str = None, entity_ids: List[str] = None, exchanges: List[str] = None, codes: List[str] = None, the_timestamp: Union[str, pd.Timestamp] = None, start_timestamp: Union[str, pd.Timestamp] = None, end_timestamp: Union[str, pd.Timestamp] = now_pd_timestamp(), columns: List = None, filters: List = None, order: object = None, limit: int = None, level: IntervalLevel = IntervalLevel.LEVEL_1DAY, category_field: str = 'entity_id', time_field: str = 'timestamp', computing_window: int = None) -> None: self.logger = logging.getLogger(self.__class__.__name__) self.data_schema = data_schema self.entity_schema = entity_schema self.provider = provider self.entity_provider = entity_provider self.the_timestamp = the_timestamp if the_timestamp: self.start_timestamp = the_timestamp self.end_timestamp = the_timestamp else: self.start_timestamp = start_timestamp self.end_timestamp = end_timestamp self.start_timestamp = to_pd_timestamp(self.start_timestamp) self.end_timestamp = to_pd_timestamp(self.end_timestamp) self.exchanges = exchanges if codes: if type(codes) == str: codes = codes.replace(' ', '') if codes.startswith('[') and codes.endswith(']'): codes = json.loads(codes) else: codes = codes.split(',') self.codes = codes self.entity_ids = entity_ids # 转换成标准entity_id if entity_schema and not self.entity_ids: df = get_entities(entity_schema=entity_schema, provider=self.entity_provider, exchanges=self.exchanges, codes=self.codes) if pd_is_not_null(df): self.entity_ids = df['entity_id'].to_list() self.filters = filters self.order = order self.limit = limit if level: self.level = IntervalLevel(level) else: self.level = level self.category_field = category_field self.time_field = time_field self.computing_window = computing_window self.category_col = eval('self.data_schema.{}'.format(self.category_field)) self.time_col = eval('self.data_schema.{}'.format(self.time_field)) self.columns = columns # we store the data in a multiple index(category_column,timestamp) Dataframe if self.columns: # support str if type(columns[0]) == str: self.columns = [] for col in columns: self.columns.append(eval('data_schema.{}'.format(col))) # always add category_column and time_field for normalizing self.columns = list(set(self.columns) | {self.category_col, self.time_col}) self.data_listeners: List[DataListener] = [] self.data_df: pd.DataFrame = None self.load_data()