def get_plotly_annotations(self): annotations = [] if df_is_not_null(self.get_annotation_df()): for trace_name, df in self.annotation_df.groupby(level=0): if df_is_not_null(df): for (_, timestamp), item in df.iterrows(): if 'color' in item: color = item['color'] else: color = '#ec0000' value = round(item['value'], 2) annotations.append( dict( x=timestamp, y=value, xref='x', yref='y', text=item['flag'], showarrow=True, align='center', arrowhead=2, arrowsize=1, arrowwidth=2, # arrowcolor='#030813', ax=-10, ay=-30, bordercolor='#c7c7c7', borderwidth=1, bgcolor=color, opacity=0.8)) return annotations
def test_technical_selector(): selector = TechnicalSelector(security_type=SecurityType.stock, start_timestamp='2019-01-01', end_timestamp='2019-06-10', level=TradingLevel.LEVEL_1DAY, provider=Provider.JOINQUANT) selector.run() print(selector.get_result_df()) targets = selector.get_targets('2019-06-04') if df_is_not_null(targets): assert 'stock_sz_000338' not in targets['security_id'].tolist() assert 'stock_sz_000338' not in targets['security_id'].tolist() assert 'stock_sz_002572' not in targets['security_id'].tolist() assert 'stock_sz_002572' not in targets['security_id'].tolist() selector.move_on(timeout=0) targets = selector.get_targets('2019-06-19') if df_is_not_null(targets): assert 'stock_sz_000338' in targets['security_id'].tolist() assert 'stock_sz_002572' not in targets['security_id'].tolist()
def get_trading_signals_figure(order_reader: OrderReader, security_id: str, provider: Union[str, Provider], level): security_type, _, _ = decode_security_id(security_id) security_factor = TechnicalFactor(security_type=security_type, security_list=[security_id], level=level, provider=provider) if df_is_not_null(security_factor.get_data_df()): print(security_factor.get_data_df().tail()) # generate the annotation df order_reader.move_on(timeout=0) df = order_reader.get_data_df().copy() if df_is_not_null(df): df['value'] = df['order_price'] df['flag'] = df['order_type'].apply(lambda x: order_type_flag(x)) df['color'] = df['order_type'].apply(lambda x: order_type_color(x)) print(df.tail()) data, layout = security_factor.draw(render=None, figures=go.Candlestick, annotation_df=df) return go.Figure(data=data, layout=layout)
def load_data(self): if self.security_list: self.data_df = get_data(data_schema=self.data_schema, security_list=self.security_list, provider=self.provider, columns=self.columns, start_timestamp=self.start_timestamp, end_timestamp=self.end_timestamp, filters=self.filters, level=self.level) else: self.data_df = get_data(data_schema=self.data_schema, codes=self.codes, provider=self.provider, columns=self.columns, start_timestamp=self.start_timestamp, end_timestamp=self.end_timestamp, filters=self.filters, level=self.level) if df_is_not_null(self.data_df): self.data_df = index_df_with_category_time( self.data_df, category=self.category_field) for listener in self.data_listeners: listener.on_data_loaded(self.data_df)
def register_data_listener(self, listener): if listener not in self.data_listeners: self.data_listeners.append(listener) # notify it once after registered if df_is_not_null(self.data_df): listener.on_data_loaded(self.data_df)
def on_finish(self): last_year = str(now_pd_timestamp().year) codes = [item.code for item in self.entities] need_filleds = get_dividend_financing( provider=self.provider, codes=codes, return_type='domain', session=self.session, filters=[DividendFinancing.rights_raising_fund.is_(None)], end_timestamp=last_year) for item in need_filleds: df = get_rights_issue_detail( provider=self.provider, entity_id=item.entity_id, columns=[ RightsIssueDetail.timestamp, RightsIssueDetail.rights_raising_fund ], start_timestamp=item.timestamp, end_timestamp="{}-12-31".format(item.timestamp.year)) if df_is_not_null(df): item.rights_raising_fund = df['rights_raising_fund'].sum() self.session.commit() super().on_finish()
def get_traders() -> List[str]: df = get_group(provider='zvt', data_schema=SimAccount, column=SimAccount.trader_name, group_func=None) if df_is_not_null(df): return df['trader_name'].tolist() return []
def run(self): if self.filter_factors: musts = [] for factor in self.filter_factors: df = factor.get_result_df() if len(df.columns) > 1: s = df.agg("and", axis="columns") s.name = 'score' musts.append(s.to_frame(name='score')) else: df.columns = ['score'] musts.append(df) self.must_result = list(accumulate(musts, func=operator.__and__))[-1] if self.score_factors: scores = [] for factor in self.score_factors: df = factor.get_result_df() if len(df.columns) > 1: s = df.agg("mean", axis="columns") s.name = 'score' scores.append(s.to_frame(name='score')) else: df.columns = ['score'] scores.append(df) self.score_result = list(accumulate(scores, func=operator.__add__))[-1] if df_is_not_null(self.must_result) and df_is_not_null( self.score_result): result1 = self.must_result[self.must_result.score] result2 = self.score_result[ self.score_result.score >= self.threshold] result = result2.loc[result1.index, :] elif df_is_not_null(self.score_result): result = self.score_result[ self.score_result.score >= self.threshold] else: result = self.must_result[self.must_result.score] self.result_df = result.reset_index() self.result_df = index_df(self.result_df)
def get_trader_details(account_reader: AccountReader, order_reader: OrderReader, provider: Union[str, Provider]): graph_list = [] account_data, account_layout = account_reader.draw(render=None, value_field='all_value', keep_ui_state=False) for trader_name in account_reader.trader_names: graph_list.append( dcc.Graph(id='{}_account'.format(trader_name), figure={ 'data': account_data, 'layout': account_layout })) df_account = account_reader.get_data_df() if df_is_not_null(df_account): df_orders = order_reader.get_data_df() if df_is_not_null(df_orders): grouped = df_orders.groupby('security_id') for security_id, order_df in grouped: security_type, _, _ = decode_security_id(security_id) security_factor = TechnicalFactor(security_type=security_type, security_list=[security_id], level=account_reader.level, provider=provider) if df_is_not_null(security_factor.get_data_df()): print(security_factor.get_data_df().tail()) data, layout = security_factor.draw(figure=go.Candlestick, render=None, keep_ui_state=False) graph_list.append( dcc.Graph(id='{}_signals'.format(security_id), figure={ 'data': data, 'layout': layout })) return graph_list
def get_plotly_data(self): if not df_is_not_null(self.get_data_df()): return [] df_list: List[pd.DataFrame] = [] for _, df_item in self.get_data_df().groupby(self.category_field): df = df_item.copy() df.reset_index(inplace=True, level=self.category_field) df_list.append(df) if len(df_list) > 1: df_list = fill_with_same_index(df_list=df_list) data = [] for df in df_list: series_name = df[df[self.category_field].notna()][ self.category_field][0] xdata = [timestamp for timestamp in df.index] # draw all figures for one category for i, figure in enumerate(self.figures): if figure == go.Candlestick: security_type, _, _ = decode_security_id(series_name) trace_name = '{}_kdata'.format(series_name) if security_type == SecurityType.stock: open = df.loc[:, 'qfq_open'] close = df.loc[:, 'qfq_close'] high = df.loc[:, 'qfq_high'] low = df.loc[:, 'qfq_low'] else: open = df.loc[:, 'open'] close = df.loc[:, 'close'] high = df.loc[:, 'high'] low = df.loc[:, 'low'] data.append( go.Candlestick(x=xdata, open=open, close=close, low=low, high=high, name=trace_name)) else: trace_name = '{}_{}'.format(series_name, self.value_fields[i]) ydata = df.loc[:, self.value_fields[i]].values.tolist() data.append( figure(x=xdata, y=ydata, mode=self.modes[i], name=trace_name)) return data
def run(self): """ """ if self.filter_factors: musts = [] for factor in self.filter_factors: df = factor.get_result_df() if not df_is_not_null(df): raise Exception('no data for factor:{},{}'.format( factor.factor_name, factor)) if len(df.columns) > 1: s = df.agg("and", axis="columns") s.name = 'score' musts.append(s.to_frame(name='score')) else: df.columns = ['score'] musts.append(df) self.filter_result = list(accumulate(musts, func=operator.__and__))[-1] if self.score_factors: scores = [] for factor in self.score_factors: df = factor.get_result_df() if not df_is_not_null(df): raise Exception('no data for factor:{],{}'.format( factor.factor_name, factor)) if len(df.columns) > 1: s = df.agg("mean", axis="columns") s.name = 'score' scores.append(s.to_frame(name='score')) else: df.columns = ['score'] scores.append(df) self.score_result = list(accumulate(scores, func=operator.__add__))[-1] self.generate_targets()
def generate_targets(self): if df_is_not_null(self.filter_result) and df_is_not_null(self.score_result): # for long result1 = self.filter_result[self.filter_result.score] result2 = self.score_result[self.score_result.score >= self.long_threshold] long_result = result2.loc[result1.index, :] # for short result1 = self.filter_result[~self.filter_result.score] result2 = self.score_result[self.score_result.score <= self.short_threshold] short_result = result2.loc[result1.index, :] elif df_is_not_null(self.score_result): long_result = self.score_result[self.score_result.score >= self.long_threshold] short_result = self.score_result[self.score_result.score <= self.short_threshold] else: long_result = self.filter_result[self.filter_result.score] short_result = self.filter_result[~self.filter_result.score] self.open_long_df = self.normalize_result_df(long_result) self.open_short_df = self.normalize_result_df(short_result)
def df_to_db(df, data_schema, provider): store_category = get_store_category(data_schema) db_engine = get_db_engine(provider, store_category=store_category) current = get_data(data_schema=data_schema, columns=[data_schema.id], provider=provider) if df_is_not_null(current): df = df[~df['id'].isin(current['id'])] df.to_sql(data_schema.__tablename__, db_engine, index=False, if_exists='append')
def get_targets(self, timestamp, target_type='open_long') -> pd.DataFrame: if target_type == 'open_long': df = self.open_long_df if target_type == 'open_short': df = self.open_short_df if target_type == 'keep_long': df = self.keep_long_df if target_type == 'keep_short': df = self.keep_short_df if df_is_not_null(df): if timestamp in df.index: target_df = df.loc[[to_pd_timestamp(timestamp)], :] return target_df['security_id'].tolist() return []
def get_trader_detail_figures(trader_domain: business.Trader, account_reader: AccountReader, order_reader: OrderReader): graph_list = [] if account_reader: account_data, account_layout = account_reader.draw( render=None, value_fields=['all_value'], keep_ui_state=False) for trader_name in account_reader.trader_names: graph_list.append( dcc.Graph(id='{}-account'.format(trader_name), figure={ 'data': account_data, 'layout': account_layout })) order_reader.move_on(timeout=0) df_orders = order_reader.get_data_df().copy() if df_is_not_null(df_orders): grouped = df_orders.groupby('security_id') for security_id, order_df in grouped: security_type, _, _ = decode_security_id(security_id) indicators = [] indicators_param = [] indicator_cols = [] if trader_domain.technical_factors: tech_factors = simplejson.loads( trader_domain.technical_factors) for factor in tech_factors: indicators += factor['indicators'] indicators_param += factor['indicators_param'] indicator_cols += factor['indicator_cols'] security_factor = TechnicalFactor( security_type=security_type, security_list=[security_id], start_timestamp=trader_domain.start_timestamp, end_timestamp=trader_domain.end_timestamp, level=trader_domain.level, provider=trader_domain.provider, indicators=indicators, indicators_param=indicators_param) # generate the annotation df df = order_df.copy() if df_is_not_null(df): df['value'] = df['order_price'] df['flag'] = df['order_type'].apply( lambda x: order_type_flag(x)) df['color'] = df['order_type'].apply( lambda x: order_type_color(x)) print(df.tail()) data, layout = security_factor.draw_with_indicators( render=None, annotation_df=df, indicators=indicator_cols, height=620) if trader_domain.real_time: result = get_current_price(security_list=[security_id]) bid_ask = result.get(security_id) if bid_ask: graph_list.append( daq.LEDDisplay(id='ask', label=f'ask price', value=bid_ask[0], color="#00da3c")) graph_list.append( daq.LEDDisplay(id='bid', label=f'bid price', value=bid_ask[1], color="#FF5E5E")) graph_list.append( dcc.Graph(id='{}-{}-signals'.format(trader_domain.trader_name, security_id), figure={ 'data': data, 'layout': layout })) return graph_list
def get_trader_detail_figures(trader_domain: business.Trader, account_reader: AccountReader, order_reader: OrderReader): graph_list = [] account_data, account_layout = account_reader.draw( render=None, value_fields=['all_value'], keep_ui_state=False) for trader_name in account_reader.trader_names: graph_list.append( dcc.Graph(id='{}-account'.format(trader_name), figure={ 'data': account_data, 'layout': account_layout })) df_orders = order_reader.get_data_df() if df_is_not_null(df_orders): grouped = df_orders.groupby('security_id') for security_id, order_df in grouped: security_type, _, _ = decode_security_id(security_id) # TODO:just show the indicators used by the trader security_factor = TechnicalFactor( security_type=security_type, security_list=[security_id], start_timestamp=trader_domain.start_timestamp, end_timestamp=trader_domain.end_timestamp, level=trader_domain.level, provider=trader_domain.provider, indicators=['ma', 'ma'], indicators_param=[{ 'window': 5 }, { 'window': 10 }]) # if df_is_not_null(security_factor.get_data_df()): # print(security_factor.get_data_df().tail()) # generate the annotation df order_reader.move_on(timeout=0) df = order_reader.get_data_df().copy() if df_is_not_null(df): df['value'] = df['order_price'] df['flag'] = df['order_type'].apply( lambda x: order_type_flag(x)) df['color'] = df['order_type'].apply( lambda x: order_type_color(x)) print(df.tail()) data, layout = security_factor.draw_with_indicators( render=None, annotation_df=df) graph_list.append( dcc.Graph(id='{}-{}-signals'.format(trader_domain.trader_name, security_id), figure={ 'data': data, 'layout': layout })) return graph_list
def get_data(data_schema, security_list=None, security_id=None, codes=None, level=None, provider='eastmoney', columns=None, return_type='df', start_timestamp=None, end_timestamp=None, filters=None, session=None, order=None, limit=None, index='timestamp', index_is_time=True): local_session = False if not session: store_category = get_store_category(data_schema) session = get_db_session(provider=provider, store_category=store_category) local_session = True try: if columns: if data_schema.timestamp not in columns: columns.append(data_schema.timestamp) query = session.query(*columns) else: query = session.query(data_schema) if security_id: query = query.filter(data_schema.security_id == security_id) if codes: query = query.filter(data_schema.code.in_(codes)) if security_list: query = query.filter(data_schema.security_id.in_(security_list)) # we always store different level in different schema,the level param is not useful now if level: try: # some schema has no level,just ignore it data_schema.level if type(level) == TradingLevel: level = level.value query = query.filter(data_schema.level == level) except Exception as e: pass query = common_filter(query, data_schema=data_schema, start_timestamp=start_timestamp, end_timestamp=end_timestamp, filters=filters, order=order, limit=limit) if return_type == 'df': df = pd.read_sql(query.statement, query.session.bind) if df_is_not_null(df): return index_df(df, drop=False, index=index, index_is_time=index_is_time) elif return_type == 'domain': return query.all() elif return_type == 'dict': return [item.__dict__ for item in query.all()] except Exception: raise finally: if local_session: session.close()
def move_on(self, to_timestamp: Union[str, pd.Timestamp] = None, timeout: int = 20) -> bool: """ get the data happened before to_timestamp,if not set,get all the data which means to now Parameters ---------- to_timestamp : timeout : the time waiting the data ready in seconds Returns ------- whether got data """ if not df_is_not_null(self.data_df): self.load_data() return False df = self.data_df.reset_index(level='timestamp') recorded_timestamps = df.groupby(level=0)['timestamp'].max() self.logger.info('level:{},current_timestamps:\n{}'.format( self.level, recorded_timestamps)) changed = False # FIXME:we suppose history data should be there at first start_time = time.time() for category, recorded_timestamp in recorded_timestamps.iteritems(): while True: category_filter = [self.category_column == category] if self.filters: filters = self.filters + category_filter else: filters = category_filter added = get_data(data_schema=self.data_schema, provider=self.provider, columns=self.columns, start_timestamp=recorded_timestamp, end_timestamp=to_timestamp, filters=filters, level=self.level) if df_is_not_null(added): would_added = added[ added['timestamp'] != recorded_timestamp].copy() if not would_added.empty: added = index_df_with_category_time( would_added, category=self.category_field) self.logger.info('category:{},added:\n{}'.format( category, added)) self.data_df = self.data_df.append(added) self.data_df = self.data_df.sort_index(level=[0, 1]) for listener in self.data_listeners: listener.on_category_data_added(category=category, added_data=added) changed = True # if got data,just move to another category break cost_time = time.time() - start_time if cost_time > timeout: self.logger.warning( 'category:{} level:{} getting data timeout,to_timestamp:{},now:{}' .format(category, self.level, to_timestamp, now_pd_timestamp())) break if changed: for listener in self.data_listeners: listener.on_data_changed(self.data_df) return changed
def normalize_result_df(self, df): if df_is_not_null(df): df = df.reset_index() df = index_df(df) df = df.sort_values(by=['score', 'security_id']) return df