def get_plotly_annotations(self):
annotations = []
if df_is_not_null(self.get_annotation_df()):
for trace_name, df in self.annotation_df.groupby(level=0):
if df_is_not_null(df):
for (_, timestamp), item in df.iterrows():
if 'color' in item:
color = item['color']
else:
color = '#ec0000'
value = round(item['value'], 2)
annotations.append(
dict(
x=timestamp,
y=value,
xref='x',
yref='y',
text=item['flag'],
showarrow=True,
align='center',
arrowhead=2,
arrowsize=1,
arrowwidth=2,
# arrowcolor='#030813',
ax=-10,
ay=-30,
bordercolor='#c7c7c7',
borderwidth=1,
bgcolor=color,
opacity=0.8))
return annotations
def test_technical_selector():
selector = TechnicalSelector(security_type=SecurityType.stock,
start_timestamp='2019-01-01',
end_timestamp='2019-06-10',
level=TradingLevel.LEVEL_1DAY,
provider=Provider.JOINQUANT)
selector.run()
print(selector.get_result_df())
targets = selector.get_targets('2019-06-04')
if df_is_not_null(targets):
assert 'stock_sz_000338' not in targets['security_id'].tolist()
assert 'stock_sz_000338' not in targets['security_id'].tolist()
assert 'stock_sz_002572' not in targets['security_id'].tolist()
assert 'stock_sz_002572' not in targets['security_id'].tolist()
selector.move_on(timeout=0)
targets = selector.get_targets('2019-06-19')
if df_is_not_null(targets):
assert 'stock_sz_000338' in targets['security_id'].tolist()
assert 'stock_sz_002572' not in targets['security_id'].tolist()
def get_trading_signals_figure(order_reader: OrderReader, security_id: str,
provider: Union[str, Provider], level):
security_type, _, _ = decode_security_id(security_id)
security_factor = TechnicalFactor(security_type=security_type,
security_list=[security_id],
level=level,
provider=provider)
if df_is_not_null(security_factor.get_data_df()):
print(security_factor.get_data_df().tail())
# generate the annotation df
order_reader.move_on(timeout=0)
df = order_reader.get_data_df().copy()
if df_is_not_null(df):
df['value'] = df['order_price']
df['flag'] = df['order_type'].apply(lambda x: order_type_flag(x))
df['color'] = df['order_type'].apply(lambda x: order_type_color(x))
print(df.tail())
data, layout = security_factor.draw(render=None,
figures=go.Candlestick,
annotation_df=df)
return go.Figure(data=data, layout=layout)
def load_data(self):
if self.security_list:
self.data_df = get_data(data_schema=self.data_schema,
security_list=self.security_list,
provider=self.provider,
columns=self.columns,
start_timestamp=self.start_timestamp,
end_timestamp=self.end_timestamp,
filters=self.filters,
level=self.level)
else:
self.data_df = get_data(data_schema=self.data_schema,
codes=self.codes,
provider=self.provider,
columns=self.columns,
start_timestamp=self.start_timestamp,
end_timestamp=self.end_timestamp,
filters=self.filters,
level=self.level)
if df_is_not_null(self.data_df):
self.data_df = index_df_with_category_time(
self.data_df, category=self.category_field)
for listener in self.data_listeners:
listener.on_data_loaded(self.data_df)
def register_data_listener(self, listener):
if listener not in self.data_listeners:
self.data_listeners.append(listener)
# notify it once after registered
if df_is_not_null(self.data_df):
listener.on_data_loaded(self.data_df)
def on_finish(self):
last_year = str(now_pd_timestamp().year)
codes = [item.code for item in self.entities]
need_filleds = get_dividend_financing(
provider=self.provider,
codes=codes,
return_type='domain',
session=self.session,
filters=[DividendFinancing.rights_raising_fund.is_(None)],
end_timestamp=last_year)
for item in need_filleds:
df = get_rights_issue_detail(
provider=self.provider,
entity_id=item.entity_id,
columns=[
RightsIssueDetail.timestamp,
RightsIssueDetail.rights_raising_fund
],
start_timestamp=item.timestamp,
end_timestamp="{}-12-31".format(item.timestamp.year))
if df_is_not_null(df):
item.rights_raising_fund = df['rights_raising_fund'].sum()
self.session.commit()
super().on_finish()
def get_traders() -> List[str]:
df = get_group(provider='zvt',
data_schema=SimAccount,
column=SimAccount.trader_name,
group_func=None)
if df_is_not_null(df):
return df['trader_name'].tolist()
return []
def run(self):
if self.filter_factors:
musts = []
for factor in self.filter_factors:
df = factor.get_result_df()
if len(df.columns) > 1:
s = df.agg("and", axis="columns")
s.name = 'score'
musts.append(s.to_frame(name='score'))
else:
df.columns = ['score']
musts.append(df)
self.must_result = list(accumulate(musts,
func=operator.__and__))[-1]
if self.score_factors:
scores = []
for factor in self.score_factors:
df = factor.get_result_df()
if len(df.columns) > 1:
s = df.agg("mean", axis="columns")
s.name = 'score'
scores.append(s.to_frame(name='score'))
else:
df.columns = ['score']
scores.append(df)
self.score_result = list(accumulate(scores,
func=operator.__add__))[-1]
if df_is_not_null(self.must_result) and df_is_not_null(
self.score_result):
result1 = self.must_result[self.must_result.score]
result2 = self.score_result[
self.score_result.score >= self.threshold]
result = result2.loc[result1.index, :]
elif df_is_not_null(self.score_result):
result = self.score_result[
self.score_result.score >= self.threshold]
else:
result = self.must_result[self.must_result.score]
self.result_df = result.reset_index()
self.result_df = index_df(self.result_df)
def get_trader_details(account_reader: AccountReader,
order_reader: OrderReader, provider: Union[str,
Provider]):
graph_list = []
account_data, account_layout = account_reader.draw(render=None,
value_field='all_value',
keep_ui_state=False)
for trader_name in account_reader.trader_names:
graph_list.append(
dcc.Graph(id='{}_account'.format(trader_name),
figure={
'data': account_data,
'layout': account_layout
}))
df_account = account_reader.get_data_df()
if df_is_not_null(df_account):
df_orders = order_reader.get_data_df()
if df_is_not_null(df_orders):
grouped = df_orders.groupby('security_id')
for security_id, order_df in grouped:
security_type, _, _ = decode_security_id(security_id)
security_factor = TechnicalFactor(security_type=security_type,
security_list=[security_id],
level=account_reader.level,
provider=provider)
if df_is_not_null(security_factor.get_data_df()):
print(security_factor.get_data_df().tail())
data, layout = security_factor.draw(figure=go.Candlestick,
render=None,
keep_ui_state=False)
graph_list.append(
dcc.Graph(id='{}_signals'.format(security_id),
figure={
'data': data,
'layout': layout
}))
return graph_list
def get_plotly_data(self):
if not df_is_not_null(self.get_data_df()):
return []
df_list: List[pd.DataFrame] = []
for _, df_item in self.get_data_df().groupby(self.category_field):
df = df_item.copy()
df.reset_index(inplace=True, level=self.category_field)
df_list.append(df)
if len(df_list) > 1:
df_list = fill_with_same_index(df_list=df_list)
data = []
for df in df_list:
series_name = df[df[self.category_field].notna()][
self.category_field][0]
xdata = [timestamp for timestamp in df.index]
# draw all figures for one category
for i, figure in enumerate(self.figures):
if figure == go.Candlestick:
security_type, _, _ = decode_security_id(series_name)
trace_name = '{}_kdata'.format(series_name)
if security_type == SecurityType.stock:
open = df.loc[:, 'qfq_open']
close = df.loc[:, 'qfq_close']
high = df.loc[:, 'qfq_high']
low = df.loc[:, 'qfq_low']
else:
open = df.loc[:, 'open']
close = df.loc[:, 'close']
high = df.loc[:, 'high']
low = df.loc[:, 'low']
data.append(
go.Candlestick(x=xdata,
open=open,
close=close,
low=low,
high=high,
name=trace_name))
else:
trace_name = '{}_{}'.format(series_name,
self.value_fields[i])
ydata = df.loc[:, self.value_fields[i]].values.tolist()
data.append(
figure(x=xdata,
y=ydata,
mode=self.modes[i],
name=trace_name))
return data
def run(self):
"""
"""
if self.filter_factors:
musts = []
for factor in self.filter_factors:
df = factor.get_result_df()
if not df_is_not_null(df):
raise Exception('no data for factor:{},{}'.format(
factor.factor_name, factor))
if len(df.columns) > 1:
s = df.agg("and", axis="columns")
s.name = 'score'
musts.append(s.to_frame(name='score'))
else:
df.columns = ['score']
musts.append(df)
self.filter_result = list(accumulate(musts,
func=operator.__and__))[-1]
if self.score_factors:
scores = []
for factor in self.score_factors:
df = factor.get_result_df()
if not df_is_not_null(df):
raise Exception('no data for factor:{],{}'.format(
factor.factor_name, factor))
if len(df.columns) > 1:
s = df.agg("mean", axis="columns")
s.name = 'score'
scores.append(s.to_frame(name='score'))
else:
df.columns = ['score']
scores.append(df)
self.score_result = list(accumulate(scores,
func=operator.__add__))[-1]
self.generate_targets()
def generate_targets(self):
if df_is_not_null(self.filter_result) and df_is_not_null(self.score_result):
# for long
result1 = self.filter_result[self.filter_result.score]
result2 = self.score_result[self.score_result.score >= self.long_threshold]
long_result = result2.loc[result1.index, :]
# for short
result1 = self.filter_result[~self.filter_result.score]
result2 = self.score_result[self.score_result.score <= self.short_threshold]
short_result = result2.loc[result1.index, :]
elif df_is_not_null(self.score_result):
long_result = self.score_result[self.score_result.score >= self.long_threshold]
short_result = self.score_result[self.score_result.score <= self.short_threshold]
else:
long_result = self.filter_result[self.filter_result.score]
short_result = self.filter_result[~self.filter_result.score]
self.open_long_df = self.normalize_result_df(long_result)
self.open_short_df = self.normalize_result_df(short_result)
def df_to_db(df, data_schema, provider):
store_category = get_store_category(data_schema)
db_engine = get_db_engine(provider, store_category=store_category)
current = get_data(data_schema=data_schema,
columns=[data_schema.id],
provider=provider)
if df_is_not_null(current):
df = df[~df['id'].isin(current['id'])]
df.to_sql(data_schema.__tablename__,
db_engine,
index=False,
if_exists='append')
def get_targets(self, timestamp, target_type='open_long') -> pd.DataFrame:
if target_type == 'open_long':
df = self.open_long_df
if target_type == 'open_short':
df = self.open_short_df
if target_type == 'keep_long':
df = self.keep_long_df
if target_type == 'keep_short':
df = self.keep_short_df
if df_is_not_null(df):
if timestamp in df.index:
target_df = df.loc[[to_pd_timestamp(timestamp)], :]
return target_df['security_id'].tolist()
return []
def get_trader_detail_figures(trader_domain: business.Trader,
account_reader: AccountReader,
order_reader: OrderReader):
graph_list = []
if account_reader:
account_data, account_layout = account_reader.draw(
render=None, value_fields=['all_value'], keep_ui_state=False)
for trader_name in account_reader.trader_names:
graph_list.append(
dcc.Graph(id='{}-account'.format(trader_name),
figure={
'data': account_data,
'layout': account_layout
}))
order_reader.move_on(timeout=0)
df_orders = order_reader.get_data_df().copy()
if df_is_not_null(df_orders):
grouped = df_orders.groupby('security_id')
for security_id, order_df in grouped:
security_type, _, _ = decode_security_id(security_id)
indicators = []
indicators_param = []
indicator_cols = []
if trader_domain.technical_factors:
tech_factors = simplejson.loads(
trader_domain.technical_factors)
for factor in tech_factors:
indicators += factor['indicators']
indicators_param += factor['indicators_param']
indicator_cols += factor['indicator_cols']
security_factor = TechnicalFactor(
security_type=security_type,
security_list=[security_id],
start_timestamp=trader_domain.start_timestamp,
end_timestamp=trader_domain.end_timestamp,
level=trader_domain.level,
provider=trader_domain.provider,
indicators=indicators,
indicators_param=indicators_param)
# generate the annotation df
df = order_df.copy()
if df_is_not_null(df):
df['value'] = df['order_price']
df['flag'] = df['order_type'].apply(
lambda x: order_type_flag(x))
df['color'] = df['order_type'].apply(
lambda x: order_type_color(x))
print(df.tail())
data, layout = security_factor.draw_with_indicators(
render=None,
annotation_df=df,
indicators=indicator_cols,
height=620)
if trader_domain.real_time:
result = get_current_price(security_list=[security_id])
bid_ask = result.get(security_id)
if bid_ask:
graph_list.append(
daq.LEDDisplay(id='ask',
label=f'ask price',
value=bid_ask[0],
color="#00da3c"))
graph_list.append(
daq.LEDDisplay(id='bid',
label=f'bid price',
value=bid_ask[1],
color="#FF5E5E"))
graph_list.append(
dcc.Graph(id='{}-{}-signals'.format(trader_domain.trader_name,
security_id),
figure={
'data': data,
'layout': layout
}))
return graph_list
def get_trader_detail_figures(trader_domain: business.Trader,
account_reader: AccountReader,
order_reader: OrderReader):
graph_list = []
account_data, account_layout = account_reader.draw(
render=None, value_fields=['all_value'], keep_ui_state=False)
for trader_name in account_reader.trader_names:
graph_list.append(
dcc.Graph(id='{}-account'.format(trader_name),
figure={
'data': account_data,
'layout': account_layout
}))
df_orders = order_reader.get_data_df()
if df_is_not_null(df_orders):
grouped = df_orders.groupby('security_id')
for security_id, order_df in grouped:
security_type, _, _ = decode_security_id(security_id)
# TODO:just show the indicators used by the trader
security_factor = TechnicalFactor(
security_type=security_type,
security_list=[security_id],
start_timestamp=trader_domain.start_timestamp,
end_timestamp=trader_domain.end_timestamp,
level=trader_domain.level,
provider=trader_domain.provider,
indicators=['ma', 'ma'],
indicators_param=[{
'window': 5
}, {
'window': 10
}])
# if df_is_not_null(security_factor.get_data_df()):
# print(security_factor.get_data_df().tail())
# generate the annotation df
order_reader.move_on(timeout=0)
df = order_reader.get_data_df().copy()
if df_is_not_null(df):
df['value'] = df['order_price']
df['flag'] = df['order_type'].apply(
lambda x: order_type_flag(x))
df['color'] = df['order_type'].apply(
lambda x: order_type_color(x))
print(df.tail())
data, layout = security_factor.draw_with_indicators(
render=None, annotation_df=df)
graph_list.append(
dcc.Graph(id='{}-{}-signals'.format(trader_domain.trader_name,
security_id),
figure={
'data': data,
'layout': layout
}))
return graph_list
def get_data(data_schema,
security_list=None,
security_id=None,
codes=None,
level=None,
provider='eastmoney',
columns=None,
return_type='df',
start_timestamp=None,
end_timestamp=None,
filters=None,
session=None,
order=None,
limit=None,
index='timestamp',
index_is_time=True):
local_session = False
if not session:
store_category = get_store_category(data_schema)
session = get_db_session(provider=provider,
store_category=store_category)
local_session = True
try:
if columns:
if data_schema.timestamp not in columns:
columns.append(data_schema.timestamp)
query = session.query(*columns)
else:
query = session.query(data_schema)
if security_id:
query = query.filter(data_schema.security_id == security_id)
if codes:
query = query.filter(data_schema.code.in_(codes))
if security_list:
query = query.filter(data_schema.security_id.in_(security_list))
# we always store different level in different schema,the level param is not useful now
if level:
try:
# some schema has no level,just ignore it
data_schema.level
if type(level) == TradingLevel:
level = level.value
query = query.filter(data_schema.level == level)
except Exception as e:
pass
query = common_filter(query,
data_schema=data_schema,
start_timestamp=start_timestamp,
end_timestamp=end_timestamp,
filters=filters,
order=order,
limit=limit)
if return_type == 'df':
df = pd.read_sql(query.statement, query.session.bind)
if df_is_not_null(df):
return index_df(df,
drop=False,
index=index,
index_is_time=index_is_time)
elif return_type == 'domain':
return query.all()
elif return_type == 'dict':
return [item.__dict__ for item in query.all()]
except Exception:
raise
finally:
if local_session:
session.close()
def move_on(self,
to_timestamp: Union[str, pd.Timestamp] = None,
timeout: int = 20) -> bool:
"""
get the data happened before to_timestamp,if not set,get all the data which means to now
Parameters
----------
to_timestamp :
timeout : the time waiting the data ready in seconds
Returns
-------
whether got data
"""
if not df_is_not_null(self.data_df):
self.load_data()
return False
df = self.data_df.reset_index(level='timestamp')
recorded_timestamps = df.groupby(level=0)['timestamp'].max()
self.logger.info('level:{},current_timestamps:\n{}'.format(
self.level, recorded_timestamps))
changed = False
# FIXME:we suppose history data should be there at first
start_time = time.time()
for category, recorded_timestamp in recorded_timestamps.iteritems():
while True:
category_filter = [self.category_column == category]
if self.filters:
filters = self.filters + category_filter
else:
filters = category_filter
added = get_data(data_schema=self.data_schema,
provider=self.provider,
columns=self.columns,
start_timestamp=recorded_timestamp,
end_timestamp=to_timestamp,
filters=filters,
level=self.level)
if df_is_not_null(added):
would_added = added[
added['timestamp'] != recorded_timestamp].copy()
if not would_added.empty:
added = index_df_with_category_time(
would_added, category=self.category_field)
self.logger.info('category:{},added:\n{}'.format(
category, added))
self.data_df = self.data_df.append(added)
self.data_df = self.data_df.sort_index(level=[0, 1])
for listener in self.data_listeners:
listener.on_category_data_added(category=category,
added_data=added)
changed = True
# if got data,just move to another category
break
cost_time = time.time() - start_time
if cost_time > timeout:
self.logger.warning(
'category:{} level:{} getting data timeout,to_timestamp:{},now:{}'
.format(category, self.level, to_timestamp,
now_pd_timestamp()))
break
if changed:
for listener in self.data_listeners:
listener.on_data_changed(self.data_df)
return changed
def normalize_result_df(self, df):
if df_is_not_null(df):
df = df.reset_index()
df = index_df(df)
df = df.sort_values(by=['score', 'security_id'])
return df
