def test_basic_get_securities():
items = get_entities(region=Region.CHN,
entity_type=EntityType.Stock,
provider=Provider.EastMoney)
print(items)
items = get_entities(region=Region.CHN,
entity_type=EntityType.Index,
provider=Provider.EastMoney)
print(items)
def update_trader_details(trader_index, entity_type, entity_provider):
if trader_index is not None:
# change entity_type options
entity_type = traders[trader_index].entity_type
if not entity_type:
entity_type = 'stock'
entity_type_options = [{'label': entity_type, 'value': entity_type}]
# account stats
account_stats = get_account_stats_figure(
account_stats_reader=account_readers[trader_index])
providers = zvt_context.tradable_schema_map.get(entity_type).providers
entity_provider_options = [{
'label': name,
'value': name
} for name in providers]
# entities
entity_ids = get_order_securities(
trader_name=trader_names[trader_index])
df = get_entities(provider=entity_provider,
entity_type=entity_type,
entity_ids=entity_ids,
columns=['entity_id', 'code', 'name'],
index='entity_id')
entity_options = [{
'label': f'{entity_id}({entity["name"]})',
'value': entity_id
} for entity_id, entity in df.iterrows()]
return account_stats, entity_type_options, entity_provider_options, entity_options
else:
entity_type_options = [{
'label': name,
'value': name
} for name in zvt_context.tradable_schema_map.keys()]
account_stats = None
providers = zvt_context.tradable_schema_map.get(entity_type).providers
entity_provider_options = [{
'label': name,
'value': name
} for name in providers]
df = get_entities(provider=entity_provider,
entity_type=entity_type,
columns=['entity_id', 'code', 'name'],
index='entity_id')
entity_options = [{
'label': f'{entity_id}({entity["name"]})',
'value': entity_id
} for entity_id, entity in df.iterrows()]
return account_stats, entity_type_options, entity_provider_options, entity_options
def __init__(self,
force_update=False,
sleeping_time=5,
code=None,
codes=None) -> None:
super().__init__(force_update, sleeping_time)
# get list at first
EastmoneyStockRecorder().run()
if codes is None and code is not None:
self.codes = [code]
else:
self.codes = codes
filters = None
if not self.force_update:
filters = [StockDetail.profile.is_(None)]
self.entities = get_entities(
session=self.session,
entity_schema=StockDetail,
exchanges=None,
codes=self.codes,
filters=filters,
return_type="domain",
provider=self.provider,
)
def update_code_selector(entity_type):
if entity_type is not None:
return [{'label': schema.__name__, 'value': schema.__name__} for schema in
zvt_context.entity_map_schemas.get(entity_type)], \
[{'label': code, 'value': code} for code in
get_entities(entity_type=entity_type, columns=['code']).index]
raise dash.PreventUpdate()
def __init__(self,
batch_size=10,
force_update=False,
sleeping_time=5,
codes=None,
share_para=None) -> None:
super().__init__(batch_size, force_update, sleeping_time)
# get list at first
# EastmoneyChinaStockListRecorder().run()
self.codes = codes
self.share_para = share_para
if not self.force_update:
assert self.region is not None
self.entities = get_entities(
region=self.region,
session=self.session,
entity_type=EntityType.StockDetail,
exchanges=['sh', 'sz'],
codes=self.codes,
filters=[StockDetail.profile.is_(None)],
return_type='domain',
provider=self.provider)
def cal_ma_states(region: Region, start='000001', end='002000'):
logger.info(f'start cal day ma stats {start}:{end}')
entities = get_entities(region=region,
provider=Provider.Default,
entity_type=EntityType.Stock,
columns=[Stock.entity_id, Stock.code],
filters=[Stock.code >= start, Stock.code < end])
codes = entities.index.to_list()
ma_1d_stats = MaStateStatsFactor(region=region,
codes=codes,
start_timestamp='2005-01-01',
end_timestamp=now_pd_timestamp(
Region.CHN),
level=IntervalLevel.LEVEL_1DAY)
ma_1d_factor = MaFactor(region=region,
codes=codes,
start_timestamp='2005-01-01',
end_timestamp=now_pd_timestamp(Region.CHN),
level=IntervalLevel.LEVEL_1DAY)
logger.info(f'finish cal day ma stats {start}:{end}')
ma_1wk_stats = MaStateStatsFactor(region=region,
codes=codes,
start_timestamp='2005-01-01',
end_timestamp=now_pd_timestamp(
Region.CHN),
level=IntervalLevel.LEVEL_1WEEK)
logger.info(f'finish cal week ma stats {start}:{end}')
def inform(action: EmailInformer, entity_ids, target_date, title,
entity_provider, entity_type, em_group, em_group_over_write):
msg = "no targets"
if entity_ids:
entities = get_entities(provider=entity_provider,
entity_type=entity_type,
entity_ids=entity_ids,
return_type="domain")
if em_group:
try:
codes = [entity.code for entity in entities]
add_to_eastmoney(codes=codes,
entity_type=entity_type,
group=em_group,
over_write=em_group_over_write)
except Exception as e:
action.send_message(
zvt_config["email_username"],
f"{target_date} {title} error",
f"{target_date} {title} error: {e}",
)
infos = [f"{entity.name}({entity.code})" for entity in entities]
msg = "\n".join(infos) + "\n"
logger.info(msg)
action.send_message(zvt_config["email_username"], f"{target_date} {title}",
msg)
def cal_ma_states(start='000001', end='002000'):
logger.info(f'start cal day ma stats {start}:{end}')
entities = get_entities(provider='eastmoney',
entity_type='stock',
columns=[Stock.entity_id, Stock.code],
filters=[Stock.code >= start, Stock.code < end])
codes = entities.index.to_list()
ma_1d_stats = MaStateStatsFactor(codes=codes,
start_timestamp='2005-01-01',
end_timestamp=now_pd_timestamp(),
level=IntervalLevel.LEVEL_1DAY)
ma_1d_factor = MaFactor(codes=codes,
start_timestamp='2005-01-01',
end_timestamp=now_pd_timestamp(),
level=IntervalLevel.LEVEL_1DAY)
logger.info(f'finish cal day ma stats {start}:{end}')
ma_1wk_stats = MaStateStatsFactor(codes=codes,
start_timestamp='2005-01-01',
end_timestamp=now_pd_timestamp(),
level=IntervalLevel.LEVEL_1WEEK)
logger.info(f'finish cal week ma stats {start}:{end}')
def init_entities(self):
"""
init the entities which we would record data for
"""
if self.entity_provider == self.provider and self.entity_schema == self.data_schema:
self.entity_session = self.session
else:
self.entity_session = get_db_session(provider=self.entity_provider, data_schema=self.entity_schema)
filters = None
if self.day_data:
df = self.data_schema.query_data(start_timestamp=now_time_str(), columns=['entity_id', 'timestamp'],
provider=self.provider)
if pd_is_not_null(df):
entity_ids = df['entity_id'].tolist()
self.logger.info(f'ignore entity_ids:{entity_ids}')
filters = [self.entity_schema.entity_id.notin_(entity_ids)]
# init the entity list
self.entities = get_entities(session=self.entity_session,
entity_schema=self.entity_schema,
entity_type=self.entity_type,
exchanges=self.exchanges,
entity_ids=self.entity_ids,
codes=self.codes,
return_type='domain',
provider=self.entity_provider,
filters=filters)
def entity_ids_to_msg(entity_ids):
if entity_ids:
stocks = get_entities(provider='joinquant', entity_schema=Stock, entity_ids=entity_ids,
return_type='domain')
info = [f'{stock.name}({stock.code})' for stock in stocks]
return ' '.join(info)
return ''
def entity_ids_to_msg(region: Region, entity_ids):
if entity_ids:
stocks = get_entities(region=region, provider=Provider.JoinQuant, entity_schema=Stock, entity_ids=entity_ids,
return_type='domain')
info = [f'{stock.name}({stock.code})' for stock in stocks]
return ' '.join(info)
return ''
def report_core_company():
while True:
error_count = 0
email_action = EmailInformer()
try:
# StockTradeDay.record_data(provider='joinquant')
# Stock.record_data(provider='joinquant')
# FinanceFactor.record_data(provider='eastmoney')
# BalanceSheet.record_data(provider='eastmoney')
target_date = to_time_str(now_pd_timestamp())
my_selector: TargetSelector = FundamentalSelector(
start_timestamp='2015-01-01', end_timestamp=target_date)
my_selector.run()
long_targets = my_selector.get_open_long_targets(
timestamp=target_date)
if long_targets:
stocks = get_entities(provider='joinquant',
entity_schema=Stock,
entity_ids=long_targets,
return_type='domain')
# add them to eastmoney
try:
try:
eastmoneypy.del_group('core')
except:
pass
eastmoneypy.create_group('core')
for stock in stocks:
eastmoneypy.add_to_group(stock.code, group_name='core')
except Exception as e:
email_action.send_message(
"*****@*****.**", f'report_core_company error',
'report_core_company error:{}'.format(e))
info = [f'{stock.name}({stock.code})' for stock in stocks]
msg = ' '.join(info)
else:
msg = 'no targets'
logger.info(msg)
email_action.send_message(get_subscriber_emails(),
f'{to_time_str(target_date)} 核心资产选股结果',
msg)
break
except Exception as e:
logger.exception('report_core_company error:{}'.format(e))
time.sleep(60 * 3)
error_count = error_count + 1
if error_count == 10:
email_action.send_message(
"*****@*****.**", f'report_core_company error',
'report_core_company error:{}'.format(e))
def report_bull():
while True:
error_count = 0
email_action = EmailInformer()
try:
latest_day: Stock1dHfqKdata = Stock1dHfqKdata.query_data(order=Stock1dHfqKdata.timestamp.desc(), limit=1,
return_type='domain')
target_date = latest_day[0].timestamp
# 计算均线
start = '2019-01-01'
my_selector = TargetSelector(start_timestamp=start, end_timestamp=target_date)
# add the factors
factor1 = BullFactor(start_timestamp=start, end_timestamp=target_date)
factor2 = CrossMaVolumeFactor(start_timestamp=start, end_timestamp=target_date,
windows=[5, 120, 250])
my_selector.add_factor(factor1)
my_selector.add_factor(factor2)
my_selector.run()
long_stocks = my_selector.get_open_long_targets(timestamp=target_date)
msg = 'no targets'
if long_stocks:
stocks = get_entities(provider='joinquant', entity_schema=Stock, entity_ids=long_stocks,
return_type='domain')
# add them to eastmoney
try:
try:
eastmoneypy.del_group('bull')
except:
pass
eastmoneypy.create_group('bull')
for stock in stocks:
eastmoneypy.add_to_group(stock.code, group_name='bull')
except Exception as e:
email_action.send_message(zvt_config['email_username'], f'report_bull error',
'report_bull error:{}'.format(e))
infos = stocks_with_info(stocks)
msg = '\n'.join(infos) + '\n'
logger.info(msg)
email_action.send_message(zvt_config['email_username'], f'{target_date} bull选股结果', msg)
break
except Exception as e:
logger.exception('report_bull error:{}'.format(e))
time.sleep(60 * 3)
error_count = error_count + 1
if error_count == 10:
email_action.send_message(zvt_config['email_username'], f'report_bull error',
'report_bull error:{}'.format(e))
def update_code_selector(entity_type):
if entity_type is not None:
return [{
'label': code,
'value': code
}
for code in get_entities(entity_type=entity_type,
columns=['code']).index]
raise dash.PreventUpdate()
def report_cross_ma():
while True:
error_count = 0
email_action = EmailInformer()
try:
# 抓取k线数据
# StockTradeDay.record_data(provider='joinquant')
# Stock1dKdata.record_data(provider='joinquant')
latest_day: StockTradeDay = StockTradeDay.query_data(
order=StockTradeDay.timestamp.desc(),
limit=1,
return_type='domain')
if latest_day:
target_date = latest_day[0].timestamp
else:
target_date = now_pd_timestamp()
# 计算均线
my_selector = TargetSelector(start_timestamp='2018-01-01',
end_timestamp=target_date)
# add the factors
ma_factor = CrossMaFactor(start_timestamp='2018-01-01',
end_timestamp=target_date)
my_selector.add_filter_factor(ma_factor)
my_selector.run()
long_targets = my_selector.get_open_long_targets(
timestamp=target_date)
if long_targets:
stocks = get_entities(provider='joinquant',
entity_schema=Stock,
entity_ids=long_targets,
return_type='domain')
info = [f'{stock.name}({stock.code})' for stock in stocks]
msg = ' '.join(info)
else:
msg = 'no targets'
logger.info(msg)
email_action.send_message("*****@*****.**",
f'{target_date} 均线选股结果', msg)
break
except Exception as e:
logger.exception('report_cross_ma error:{}'.format(e))
time.sleep(60 * 3)
error_count = error_count + 1
if error_count == 10:
email_action.send_message("*****@*****.**",
f'report_cross_ma error',
'report_cross_ma error:{}'.format(e))
def test_iterate_entities():
data = get_entities(entity_type='stock')
first = None
last = None
for sub_data in iterate_with_step(data):
if first is None:
first = sub_data
last = sub_data
assert len(first) == 100
assert len(last) <= 100
def update_trader_details(region: Region, trader_index, entity_type):
if trader_index is not None:
# change entity_type options
entity_type = traders[trader_index].entity_type
if not entity_type:
entity_type = EntityType.Stock
entity_type_options = [{'label': entity_type, 'value': entity_type}]
# account stats
account_stats = get_account_stats_figure(
account_stats_reader=account_readers[trader_index])
# entities
entity_ids = get_order_securities(
trader_name=trader_names[trader_index])
df = get_entities(region=region,
entity_type=entity_type,
entity_ids=entity_ids,
columns=['entity_id', 'code', 'name'],
index='entity_id')
entity_options = [{
'label': f'{entity_id}({entity["name"]})',
'value': entity_id
} for entity_id, entity in df.iterrows()]
return account_stats, entity_type_options, entity_options
else:
entity_type_options = [{
'label': name,
'value': name
} for name in zvt_context.entity_schema_map.keys()]
account_stats = None
df = get_entities(region=region,
entity_type=entity_type,
columns=['entity_id', 'code', 'name'],
index='entity_id')
entity_options = [{
'label': f'{entity_id}({entity["name"]})',
'value': entity_id
} for entity_id, entity in df.iterrows()]
return account_stats, entity_type_options, entity_options
def update_entity_selector(entity_type, related):
if entity_type is not None:
if related:
schemas = zvt_context.entity_map_schemas.get(entity_type)
else:
schemas = zvt_context.schemas
df = get_entities(entity_type=entity_type,
columns=['entity_id', 'code', 'name'],
index='entity_id')
return [{'label': schema.__name__, 'value': schema.__name__} for schema in schemas], \
[{'label': f'{entity_id}({entity["name"]})', 'value': entity_id} for entity_id, entity in df.iterrows()]
raise dash.PreventUpdate()
def report_core_company():
while True:
error_count = 0
email_action = EmailInformer()
try:
# StockTradeDay.record_data(provider='joinquant')
# Stock.record_data(provider='joinquant')
# FinanceFactor.record_data(provider='eastmoney')
# BalanceSheet.record_data(provider='eastmoney')
target_date = to_time_str(now_pd_timestamp())
my_selector: TargetSelector = FundamentalSelector(start_timestamp="2016-01-01", end_timestamp=target_date)
my_selector.run()
long_targets = my_selector.get_open_long_targets(timestamp=target_date)
if long_targets:
stocks = get_entities(
provider="joinquant", entity_schema=Stock, entity_ids=long_targets, return_type="domain"
)
# add them to eastmoney
try:
codes = [stock.code for stock in stocks]
add_to_eastmoney(codes=codes, entity_type="stock", group="core")
except Exception as e:
email_action.send_message(
zvt_config["email_username"],
f"report_core_company error",
"report_core_company error:{}".format(e),
)
infos = stocks_with_info(stocks)
msg = "\n".join(infos)
else:
msg = "no targets"
logger.info(msg)
email_action.send_message(get_subscriber_emails(), f"{to_time_str(target_date)} 核心资产选股结果", msg)
break
except Exception as e:
logger.exception("report_core_company error:{}".format(e))
time.sleep(60 * 3)
error_count = error_count + 1
if error_count == 10:
email_action.send_message(
zvt_config["email_username"], f"report_core_company error", "report_core_company error:{}".format(e)
)
def __init__(self, batch_size=10, force_update=False, sleeping_time=5, codes=None, share_para=None) -> None:
exchanges = ['nyse', 'nasdaq', 'amex']
if not force_update:
assert self.region is not None
self.entities = get_entities(region=self.region,
entity_type=EntityType.StockDetail,
exchanges=exchanges,
codes=codes,
filters=[StockDetail.profile.is_(None)],
return_type='domain',
provider=self.provider)
super().__init__(entity_type=EntityType.StockDetail, exchanges=exchanges, batch_size=batch_size, force_update=force_update, sleeping_time=sleeping_time, codes=codes, share_para=share_para)
def init_entities(self):
"""
init the entities which we would record data for
"""
assert self.region is not None
# init the entity list
self.entities = get_entities(region=self.region,
entity_schema=self.entity_schema,
entity_type=self.entity_type,
exchanges=self.exchanges,
entity_ids=self.entity_ids,
codes=self.codes,
return_type='domain',
provider=self.provider)
def __init__(self, batch_size=10, force_update=False, sleeping_time=5, codes=None) -> None:
super().__init__(batch_size, force_update, sleeping_time)
loginResult = c.start("ForceLogin=1", '', mainCallback)
if (loginResult.ErrorCode != 0):
print("login in fail")
exit()
self.codes = codes
if not self.force_update:
self.entities = get_entities(session=self.session,
entity_type='stock_detail',
exchanges=['hk'],
codes=self.codes,
# filters=[StockDetail.profile.is_(None)],
return_type='domain',
provider=self.provider)
def init_entities(self):
assert self.region is not None
self.entities = get_entities(
region=self.region,
entity_type=EntityType.Index,
exchanges=self.exchanges,
codes=self.codes,
entity_ids=self.entity_ids,
return_type='domain',
provider=self.provider,
# 只抓概念和行业
filters=[
Index.category.in_([
BlockCategory.industry.value, BlockCategory.concept.value
])
])
def __init__(self, batch_size=10, force_update=False, sleeping_time=5, codes=None) -> None:
super().__init__(batch_size, force_update, sleeping_time)
# get list at first
# JqChinaStockRecorder().run()
auth(zvt_env['jq_username'], zvt_env['jq_password'])
print(f"剩余{get_query_count()['spare'] / 10000}万")
self.codes = codes
if not self.force_update:
self.entities = get_entities(session=self.session,
entity_type='stock_detail',
exchanges=['sh', 'sz'],
codes=self.codes,
filters=[StockDetail.issues.is_(None)],
return_type='domain',
provider=self.provider)
def run():
while True:
try:
items = get_entities(entity_type='stock', provider='joinquant')
entity_ids = items['entity_id'].to_list()
StockActorSummary.record_data(entity_ids=entity_ids[0:],
provider='em',
sleeping_time=0.5)
StockInstitutionalInvestorHolder.record_data(provider='em',
sleeping_time=1)
StockTopTenHolder.record_data(provider='em', sleeping_time=1)
break
except Exception as e:
msg = f'em_data_runner error:{e}'
logger.exception(msg)
time.sleep(60)
def init_entities(self):
self.entity_session = get_db_session(provider=self.entity_provider,
data_schema=self.entity_schema)
self.entities = get_entities(
session=self.entity_session,
entity_type='index',
exchanges=self.exchanges,
codes=self.codes,
entity_ids=self.entity_ids,
return_type='domain',
provider=self.provider,
# 只抓概念和行业
filters=[
Index.category.in_([
BlockCategory.industry.value, BlockCategory.concept.value
])
])
def __init__(self,
batch_size=10,
force_update=False,
sleeping_time=5,
codes=None) -> None:
super().__init__(batch_size, force_update, sleeping_time)
# get list at first
EastmoneyChinaStockListRecorder().run()
self.codes = codes
if not self.force_update:
self.entities = get_entities(
session=self.session,
entity_type='stock_detail',
exchanges=['sh', 'sz'],
codes=self.codes,
filters=[StockDetail.profile.is_(None)],
return_type='domain',
provider=self.provider)
def init_entities(self):
"""
init the entities which we would record data for
"""
if self.entity_provider == self.provider and self.entity_schema == self.data_schema:
self.entity_session = self.session
else:
self.entity_session = get_db_session(
provider=self.entity_provider, data_schema=self.entity_schema)
# init the entity list
self.entities = get_entities(session=self.entity_session,
entity_schema=self.entity_schema,
entity_type=self.entity_type,
exchanges=self.exchanges,
entity_ids=self.entity_ids,
codes=self.codes,
return_type='domain',
provider=self.entity_provider)
def init_entities(self):
"""
init the entities which we would record data for
"""
if self.entity_provider == self.provider and self.entity_schema == self.data_schema:
self.entity_session = self.session
else:
self.entity_session = get_db_session(
provider=self.entity_provider, data_schema=self.entity_schema)
if self.day_data:
df = self.data_schema.query_data(
start_timestamp=now_time_str(),
columns=["entity_id", "timestamp"],
provider=self.provider)
if pd_is_not_null(df):
entity_ids = df["entity_id"].tolist()
self.logger.info(f"ignore entity_ids:{entity_ids}")
if self.entity_filters:
self.entity_filters.append(
self.entity_schema.entity_id.notin_(entity_ids))
else:
self.entity_filters = [
self.entity_schema.entity_id.notin_(entity_ids)
]
#: init the entity list
self.entities = get_entities(
session=self.entity_session,
entity_schema=self.entity_schema,
exchanges=self.exchanges,
entity_ids=self.entity_ids,
codes=self.codes,
return_type="domain",
provider=self.entity_provider,
filters=self.entity_filters,
)
def __init__(self,
region: Region,
data_schema: Type[Mixin],
entity_schema: Type[EntityMixin],
provider: Provider = Provider.Default,
entity_ids: List[str] = None,
exchanges: List[str] = None,
codes: List[str] = None,
the_timestamp: Union[str, pd.Timestamp] = None,
start_timestamp: Union[str, pd.Timestamp] = None,
end_timestamp: Union[str, pd.Timestamp] = None,
columns: List = None,
filters: List = None,
order: object = None,
limit: int = None,
level: IntervalLevel = None,
category_field: str = 'entity_id',
time_field: str = 'timestamp',
computing_window: int = None) -> None:
super().__init__()
self.logger = logging.getLogger(self.__class__.__name__)
self.data_schema = data_schema
self.entity_schema = entity_schema
self.region = region
self.provider = provider
if end_timestamp is None:
end_timestamp = now_pd_timestamp(self.region)
self.the_timestamp = the_timestamp
if the_timestamp:
self.start_timestamp = the_timestamp
self.end_timestamp = the_timestamp
else:
self.start_timestamp = start_timestamp
self.end_timestamp = end_timestamp
self.start_timestamp = to_pd_timestamp(self.start_timestamp)
self.end_timestamp = to_pd_timestamp(self.end_timestamp)
self.exchanges = exchanges
if codes:
if type(codes) == str:
codes = codes.replace(' ', '')
if codes.startswith('[') and codes.endswith(']'):
codes = json.loads(codes)
else:
codes = codes.split(',')
self.codes = codes
self.entity_ids = entity_ids
# 转换成标准entity_id
if entity_schema and not self.entity_ids:
df = get_entities(region=self.region,
entity_schema=entity_schema,
provider=self.provider,
exchanges=self.exchanges,
codes=self.codes)
if pd_is_not_null(df):
self.entity_ids = df['entity_id'].to_list()
self.filters = filters
self.order = order
self.limit = limit
if level:
self.level = IntervalLevel(level)
else:
self.level = level
self.category_field = category_field
self.time_field = time_field
self.computing_window = computing_window
self.category_col = eval('self.data_schema.{}'.format(
self.category_field))
self.time_col = eval('self.data_schema.{}'.format(self.time_field))
self.columns = columns
# we store the data in a multiple index(category_column,timestamp) Dataframe
if self.columns:
# support str
if type(columns[0]) == str:
self.columns = []
for col in columns:
self.columns.append(eval('data_schema.{}'.format(col)))
# always add category_column and time_field for normalizing
self.columns = list(
set(self.columns) | {self.category_col, self.time_col})
self.data_listeners: List[DataListener] = []
self.data_df: pd.DataFrame = None
self.load_data()
