def __init__(self,
security_type=SecurityType.stock,
exchanges=['sh', 'sz'],
codes=None,
batch_size=10,
force_update=False,
sleeping_time=10) -> None:
"""
:param security_type:
:type security_type:SecurityType
:param exchanges:the exchanges for recording
:type exchanges:list of str
:param codes:the codes for recording
:type codes:list of str
:param batch_size:batch size to saving to db
:type batch_size:int
:param force_update: whether force update the data even if it exists
:type force_update:bool
:param sleeping_time:sleeping seconds for recoding loop
:type sleeping_time:int
"""
# setup the securities you want to record
self.security_type = security_type
self.exchanges = exchanges
self.codes = codes
self.batch_size = batch_size
self.force_update = force_update
self.sleeping_time = sleeping_time
# using to do db operations
self.session = get_db_session(
provider=self.provider,
store_category=self.store_category) # type: sqlalchemy.orm.Session
if self.need_securities:
if self.store_category != StoreCategory.meta:
self.meta_session = get_db_session(
provider=self.meta_provider,
store_category=StoreCategory.meta)
else:
self.meta_session = self.session
self.securities = get_securities(session=self.meta_session,
security_type=self.security_type,
exchanges=self.exchanges,
codes=self.codes,
return_type='domain',
provider=self.meta_provider)
def __init__(self,
batch_size=10,
force_update=False,
sleeping_time=10) -> None:
"""
:param batch_size:batch size to saving to db
:type batch_size:int
:param force_update: whether force update the data even if it exists
:type force_update:bool
:param sleeping_time:sleeping seconds for recoding loop
:type sleeping_time:int
"""
assert self.provider is not None
assert self.data_schema is not None
self.store_category = get_store_category(data_schema=self.data_schema)
self.batch_size = batch_size
self.force_update = force_update
self.sleeping_time = sleeping_time
# using to do db operations
self.session = get_db_session(provider=self.provider,
store_category=self.store_category)
def __init__(self, trader_name,
timestamp,
provider=Provider.NETEASE,
level=TradingLevel.LEVEL_1DAY,
base_capital=1000000,
buy_cost=0.001,
sell_cost=0.001,
slippage=0.001):
self.base_capital = base_capital
self.buy_cost = buy_cost
self.sell_cost = sell_cost
self.slippage = slippage
self.trader_name = trader_name
self.session = get_db_session('zvt', StoreCategory.business)
self.provider = provider
self.level = level
self.start_timestamp = timestamp
account = get_account(session=self.session, trader_name=self.trader_name, return_type='domain', limit=1)
if account:
self.logger.warning("trader:{} has run before,old result would be deleted".format(trader_name))
self.session.query(SimAccount).filter(SimAccount.trader_name == self.trader_name).delete()
self.session.query(Position).filter(Position.trader_name == self.trader_name).delete()
self.session.query(Order).filter(Order.trader_name == self.trader_name).delete()
self.session.commit()
account = SimAccount(trader_name=self.trader_name, cash=self.base_capital,
positions=[], all_value=self.base_capital, value=0, closing=False,
timestamp=timestamp)
self.latest_account = sim_account_schema.dump(account).data
def __init__(self, trader_name,
timestamp,
base_capital=1000000,
buy_cost=0.001,
sell_cost=0.001,
slippage=0.001):
self.base_capital = base_capital
self.buy_cost = buy_cost
self.sell_cost = sell_cost
self.slippage = slippage
self.trader_name = trader_name
self.session = get_db_session('zvt', StoreCategory.business)
self.provider = 'netease'
account = get_account(session=self.session, trader_name=self.trader_name, return_type='domain', limit=1)
if account:
self.logger.warning("trader:{} has run before,old result would be deleted".format(trader_name))
self.session.query(SimAccount).filter(SimAccount.trader_name == self.trader_name).delete()
self.session.query(Position).filter(Position.trader_name == self.trader_name).delete()
self.latest_account = SimAccount(trader_name=self.trader_name, cash=self.base_capital,
positions=[], all_value=self.base_capital, value=0, timestamp=timestamp)
self.save_account(self.latest_account)
def __init__(self,
security_type=SecurityType.stock,
exchanges=['sh', 'sz'],
codes=None,
batch_size=10,
force_update=False,
sleeping_time=10) -> None:
super().__init__(batch_size=batch_size,
force_update=force_update,
sleeping_time=sleeping_time)
assert self.meta_provider is not None
assert self.meta_schema is not None
self.meta_category = get_store_category(data_schema=self.meta_schema)
# setup the securities you want to record
self.security_type = security_type
self.exchanges = exchanges
self.codes = codes
self.meta_session = get_db_session(provider=self.meta_provider,
store_category=self.meta_category)
# init the security list
self.securities = get_securities(session=self.meta_session,
security_type=self.security_type,
exchanges=self.exchanges,
codes=self.codes,
return_type='domain',
provider=self.meta_provider)
def get_securities(security_type='stock',
exchanges=None,
codes=None,
columns=None,
return_type='df',
session=None,
start_timestamp=None,
end_timestamp=None,
filters=None,
order=None,
limit=None,
provider='eastmoney'):
local_session = False
data_schema = get_security_schema(security_type)
store_category = get_store_category(data_schema=data_schema)
if not session:
session = get_db_session(provider=provider,
store_category=store_category)
local_session = True
if not order:
order = data_schema.code.asc()
try:
if columns:
query = session.query(*columns)
else:
query = session.query(data_schema)
# filters
if exchanges:
query = query.filter(data_schema.exchange.in_(exchanges))
if codes:
query = query.filter(data_schema.code.in_(codes))
query = common_filter(query,
data_schema=data_schema,
start_timestamp=start_timestamp,
end_timestamp=end_timestamp,
filters=filters,
order=order,
limit=limit)
if return_type == 'df':
# TODO:add indices info
return pd.read_sql(query.statement, query.session.bind)
elif return_type == 'domain':
return query.all()
elif return_type == 'dict':
return [item.to_json() for item in query.all()]
except Exception as e:
raise
finally:
if local_session:
session.close()
def get_data(data_schema,
security_id=None,
codes=None,
level=None,
provider='eastmoney',
columns=None,
return_type='df',
start_timestamp=None,
end_timestamp=None,
filters=None,
session=None,
order=None,
limit=None):
# assert (security_id is None) != (codes is None)
if isinstance(data_schema, StockKdataBase):
assert level != None
local_session = False
if not session:
store_category = get_store_category(data_schema)
session = get_db_session(store_category=store_category)
local_session = True
try:
if columns:
query = session.query(*columns)
else:
query = session.query(data_schema)
if security_id:
query = query.filter(data_schema.security_id == security_id)
if codes:
query = query.filter(data_schema.code.in_(codes))
if isinstance(data_schema, StockKdataBase):
query = query.filter(data_schema.level == level)
query = common_filter(query,
data_schema=data_schema,
start_timestamp=start_timestamp,
end_timestamp=end_timestamp,
filters=filters,
order=order,
limit=limit,
provider=provider)
if return_type == 'df':
return pd.read_sql(query.statement, query.session.bind)
elif return_type == 'domain':
return query.all()
elif return_type == 'dict':
return [item.to_json() for item in query.all()]
except Exception:
raise
finally:
if local_session:
session.close()
def get_data(data_schema,
security_id=None,
codes=None,
level=None,
provider='eastmoney',
columns=None,
return_type='df',
start_timestamp=None,
end_timestamp=None,
filters=None,
session=None,
order=None,
limit=None):
local_session = False
if not session:
store_category = get_store_category(data_schema)
session = get_db_session(provider=provider,
store_category=store_category)
local_session = True
try:
if columns:
query = session.query(*columns)
else:
query = session.query(data_schema)
if security_id:
query = query.filter(data_schema.security_id == security_id)
if codes:
query = query.filter(data_schema.code.in_(codes))
if level:
query = query.filter(data_schema.level == level)
query = common_filter(query,
data_schema=data_schema,
start_timestamp=start_timestamp,
end_timestamp=end_timestamp,
filters=filters,
order=order,
limit=limit)
if return_type == 'df':
df = pd.read_sql(query.statement, query.session.bind)
if not df.empty:
return index_df_with_time(df, drop=False)
elif return_type == 'domain':
return query.all()
elif return_type == 'dict':
return [item.to_json() for item in query.all()]
except Exception:
raise
finally:
if local_session:
session.close()
def get_stock_category(stock_id, session=None):
local_session = False
if not session:
session = get_db_session(store_category=StoreCategory.meta)
local_session = True
try:
return session.query(Index).filter(Index.stocks.any(id=stock_id)).all()
except Exception:
raise
finally:
if local_session:
session.close()
def __init__(self,
trader_name,
model_name,
timestamp,
base_capital=1000000,
buy_cost=0.001,
sell_cost=0.001,
slippage=0.001):
self.base_capital = base_capital
self.buy_cost = buy_cost
self.sell_cost = sell_cost
self.slippage = slippage
self.trader_name = trader_name
self.model_name = model_name
self.session = get_db_session('trader')
account = get_account(trader_name=self.trader_name,
model_name=self.model_name,
limit=1)
if account:
self.logger.warning(
"trader:{} has run before,old result would be deleted".format(
trader_name))
self.session.query(SimAccount).filter(
SimAccount.trader_name == self.trader_name).delete()
self.account = SimAccount()
self.account.trader_name = trader_name
self.account.model_name = model_name
self.account.cash = self.base_capital
self.account.positions = []
self.account.all_value = self.base_capital
self.account.value = 0
self.account.timestamp = timestamp
self.account_to_queue()
t = threading.Thread(target=self.saving_account_worker)
t.start()
def security_id_in_index(security_id,
index_id,
session=None,
data_schema=StockIndex,
provider='eastmoney'):
the_id = '{}_{}'.format(index_id, security_id)
local_session = False
if not session:
store_category = get_store_category(data_schema)
session = get_db_session(provider=provider,
store_category=store_category)
local_session = True
try:
return data_exist(session=session, schema=data_schema, id=the_id)
except Exception:
raise
finally:
if local_session:
session.close()
def get_group(provider,
data_schema,
column,
group_func=func.count,
session=None):
local_session = False
if not session:
store_category = get_store_category(data_schema)
session = get_db_session(provider=provider,
store_category=store_category)
local_session = True
try:
if group_func:
query = session.query(column, group_func(column)).group_by(column)
else:
query = session.query(column).group_by(column)
df = pd.read_sql(query.statement, query.session.bind)
return df
except Exception:
raise
finally:
if local_session:
session.close()
def get_securities_in_blocks(block_names=['HS300_'],
block_category='concept',
provider='eastmoney'):
session = get_db_session(provider=provider,
store_category=StoreCategory.meta)
filters = [Index.category == block_category]
name_filters = None
for block_name in block_names:
if name_filters:
name_filters |= (Index.name == block_name)
else:
name_filters = (Index.name == block_name)
filters.append(name_filters)
blocks = get_securities(security_type='index',
provider='eastmoney',
filters=filters,
return_type='domain',
session=session)
securities = []
for block in blocks:
securities += [item.stock_id for item in block.stocks]
return securities
def __init__(self,
security_list: List[str] = None,
exchanges: List[str] = ['sh', 'sz'],
codes: List[str] = None,
start_timestamp: Union[str, pd.Timestamp] = None,
end_timestamp: Union[str, pd.Timestamp] = None,
provider: Union[str, Provider] = Provider.JOINQUANT,
level: Union[str, TradingLevel] = TradingLevel.LEVEL_1DAY,
trader_name: str = None,
real_time: bool = False,
kdata_use_begin_time: bool = False) -> None:
assert self.security_type is not None
if trader_name:
self.trader_name = trader_name
else:
self.trader_name = type(self).__name__.lower()
self.trading_signal_listeners = []
self.state_listeners = []
self.selectors: List[TargetSelector] = []
self.security_list = security_list
self.exchanges = exchanges
self.codes = codes
# FIXME:handle this case gracefully
if self.security_list:
security_type, exchange, code = decode_security_id(self.security_list[0])
if not self.security_type:
self.security_type = security_type
if not self.exchanges:
self.exchanges = [exchange]
self.provider = Provider(provider)
# make sure the min level selector correspond to the provider and level
self.level = TradingLevel(level)
self.real_time = real_time
if start_timestamp and end_timestamp:
self.start_timestamp = to_pd_timestamp(start_timestamp)
self.end_timestamp = to_pd_timestamp(end_timestamp)
else:
assert False
if real_time:
logger.info(
'real_time mode, end_timestamp should be future,you could set it big enough for running forever')
assert self.end_timestamp >= now_pd_timestamp()
self.kdata_use_begin_time = kdata_use_begin_time
self.account_service = SimAccountService(trader_name=self.trader_name,
timestamp=self.start_timestamp,
provider=self.provider,
level=self.level)
self.add_trading_signal_listener(self.account_service)
self.init_selectors(security_list=security_list, security_type=self.security_type, exchanges=self.exchanges,
codes=self.codes, start_timestamp=self.start_timestamp, end_timestamp=self.end_timestamp)
self.selectors_comparator = self.init_selectors_comparator()
self.trading_level_asc = list(set([TradingLevel(selector.level) for selector in self.selectors]))
self.trading_level_asc.sort()
self.trading_level_desc = list(self.trading_level_asc)
self.trading_level_desc.reverse()
self.targets_slot: TargetsSlot = TargetsSlot()
self.session = get_db_session('zvt', StoreCategory.business)
trader = get_trader(session=self.session, trader_name=self.trader_name, return_type='domain', limit=1)
if trader:
self.logger.warning("trader:{} has run before,old result would be deleted".format(self.trader_name))
self.session.query(business.Trader).filter(business.Trader.trader_name == self.trader_name).delete()
self.session.commit()
self.on_start()
from ..context import init_context
init_context()
from zvt.api import technical
from zvt.domain import get_db_session, StoreCategory, TradingLevel
day_k_session = get_db_session(provider='joinquant',
store_category=StoreCategory.stock_1d_kdata
) # type: sqlalchemy.orm.Session
day_1h_session = get_db_session(provider='joinquant',
store_category=StoreCategory.stock_1h_kdata
) # type: sqlalchemy.orm.Session
def test_jq_603220_kdata():
df = technical.get_kdata(security_id='stock_sh_603220',
session=day_k_session,
level=TradingLevel.LEVEL_1DAY,
provider='joinquant')
print(df)
df = technical.get_kdata(security_id='stock_sh_603220',
session=day_1h_session,
level=TradingLevel.LEVEL_1HOUR,
provider='joinquant')
print(df)
index_id,
session=None,
data_schema=StockIndex,
provider='eastmoney'):
the_id = '{}_{}'.format(index_id, security_id)
local_session = False
if not session:
store_category = get_store_category(data_schema)
session = get_db_session(provider=provider,
store_category=store_category)
local_session = True
try:
return data_exist(session=session, schema=data_schema, id=the_id)
except Exception:
raise
finally:
if local_session:
session.close()
if __name__ == '__main__':
store_category = get_store_category(Stock)
session = get_db_session(provider='eastmoney',
store_category=store_category)
print(
get_count(data_schema=Stock,
filters=[Stock.exchange == 'sh'],
session=session))
from .context import init_context
init_context()
from zvt.api import fundamental
from zvt.domain import get_db_session, StoreCategory, Provider, SPODetail, RightsIssueDetail, DividendFinancing
from zvt.utils.time_utils import to_pd_timestamp
session = get_db_session(store_category=StoreCategory.dividend_financing
) # type: sqlalchemy.orm.Session
# 增发详情
def test_000778_spo_detial():
result = fundamental.get_spo_detail(session=session,
provider=Provider.EASTMONEY,
return_type='domain',
codes=['000778'],
end_timestamp='2018-12-31',
order=SPODetail.timestamp.desc())
assert len(result) == 4
latest: SPODetail = result[0]
assert latest.timestamp == to_pd_timestamp('2017-04-01')
assert latest.spo_issues == 347600000
assert latest.spo_price == 5.15
assert latest.spo_raising_fund == 1766000000
# 配股详情
def test_000778_rights_issue_detail():
result = fundamental.get_rights_issue_detail(
def get_data(data_schema,
security_list=None,
security_id=None,
codes=None,
level=None,
provider='eastmoney',
columns=None,
return_type='df',
start_timestamp=None,
end_timestamp=None,
filters=None,
session=None,
order=None,
limit=None,
index='timestamp',
index_is_time=True):
local_session = False
if not session:
store_category = get_store_category(data_schema)
session = get_db_session(provider=provider,
store_category=store_category)
local_session = True
try:
if columns:
if data_schema.timestamp not in columns:
columns.append(data_schema.timestamp)
query = session.query(*columns)
else:
query = session.query(data_schema)
if security_id:
query = query.filter(data_schema.security_id == security_id)
if codes:
query = query.filter(data_schema.code.in_(codes))
if security_list:
query = query.filter(data_schema.security_id.in_(security_list))
# we always store different level in different schema,the level param is not useful now
if level:
try:
# some schema has no level,just ignore it
data_schema.level
if type(level) == TradingLevel:
level = level.value
query = query.filter(data_schema.level == level)
except Exception as e:
pass
query = common_filter(query,
data_schema=data_schema,
start_timestamp=start_timestamp,
end_timestamp=end_timestamp,
filters=filters,
order=order,
limit=limit)
if return_type == 'df':
df = pd.read_sql(query.statement, query.session.bind)
if df_is_not_null(df):
return index_df(df,
drop=False,
index=index,
index_is_time=index_is_time)
elif return_type == 'domain':
return query.all()
elif return_type == 'dict':
return [item.__dict__ for item in query.all()]
except Exception:
raise
finally:
if local_session:
session.close()
close_hour, close_minute = get_close_time(security_id)
return the_timestamp.hour == close_hour and the_timestamp.minute == close_minute
def get_open_time(security_id: str):
security_type, _, _ = decode_security_id(security_id)
if security_type == SecurityType.coin:
return 0, 0
if security_type == SecurityType.stock:
return 9, 30
def generate_kdata_id(security_id, timestamp, level):
if level == TradingLevel.LEVEL_1DAY:
return "{}_{}".format(security_id,
to_time_str(timestamp, fmt=TIME_FORMAT_DAY))
else:
return "{}_{}".format(timestamp,
to_time_str(timestamp, fmt=TIME_FORMAT_ISO8601))
if __name__ == '__main__':
store_category = get_store_category(Stock)
session = get_db_session(store_category=store_category)
print(
get_count(data_schema=Stock,
filters=[Stock.exchange == 'sh'],
session=session))
from .context import init_context
init_context()
from typing import List
from zvt.api import fundamental
from zvt.domain import get_db_session, StoreCategory, Provider, HolderTrading, ManagerTrading
session = get_db_session(
store_category=StoreCategory.trading) # type: sqlalchemy.orm.Session
# 股东交易
def test_000778_holder_trading():
result: List[HolderTrading] = fundamental.get_holder_trading(
session=session,
provider=Provider.EASTMONEY,
return_type='domain',
codes=['000778'],
end_timestamp='2018-12-31',
start_timestamp='2018-09-30',
order=HolderTrading.holding_pct.desc())
assert len(result) == 6
assert result[0].holder_name == '新兴际华集团有限公司'
assert result[0].change_pct == 0.0205
assert result[0].volume == 32080000
assert result[0].holding_pct == 0.3996
# 高管交易
from .context import init_context
init_context()
from zvt.api import fundamental
from zvt.domain import get_db_session, StoreCategory, Provider, SPODetail, RightsIssueDetail, DividendFinancing
from zvt.utils.time_utils import to_pd_timestamp
session = get_db_session(provider='eastmoney',
store_category=StoreCategory.dividend_financing) # type: sqlalchemy.orm.Session
# 增发详情
def test_000778_spo_detial():
result = fundamental.get_spo_detail(session=session, provider=Provider.EASTMONEY, return_type='domain',
codes=['000778'], end_timestamp='2018-09-30',
order=SPODetail.timestamp.desc())
assert len(result) == 4
latest: SPODetail = result[0]
assert latest.timestamp == to_pd_timestamp('2017-04-01')
assert latest.spo_issues == 347600000
assert latest.spo_price == 5.15
assert latest.spo_raising_fund == 1766000000
# 配股详情
def test_000778_rights_issue_detail():
result = fundamental.get_rights_issue_detail(session=session, provider=Provider.EASTMONEY, return_type='domain',
codes=['000778'], end_timestamp='2018-09-30',
order=RightsIssueDetail.timestamp.desc())
assert len(result) == 2
from ..context import init_context
init_context()
from typing import List
from zvt.api import fundamental
from zvt.domain import get_db_session, StoreCategory, Provider, TopTenHolder, TopTenTradableHolder
session = get_db_session(
provider='eastmoney',
store_category=StoreCategory.holder) # type: sqlalchemy.orm.Session
# 十大股东
def test_000778_top_ten_holder():
result: List[TopTenHolder] = fundamental.get_top_ten_holder(
session=session,
provider=Provider.EASTMONEY,
return_type='domain',
codes=['000778'],
end_timestamp='2018-09-30',
start_timestamp='2018-09-30',
order=TopTenHolder.shareholding_ratio.desc())
assert len(result) == 10
assert result[0].holder_name == '新兴际华集团有限公司'
assert result[0].shareholding_numbers == 1595000000
assert result[0].shareholding_ratio == 0.3996
assert result[0].change == 32080000
assert result[0].change_ratio == 0.0205
from .context import init_context
init_context()
from zvt.api import fundamental
from zvt.domain import Provider, get_db_session, StoreCategory, FinanceFactor, BalanceSheet, IncomeStatement, \
CashFlowStatement
from zvt.utils.time_utils import to_time_str
session = get_db_session(
store_category=StoreCategory.finance) # type: sqlalchemy.orm.Session
# 银行指标
def test_000001_finance_factor():
correct_timestamps = [
'2018-09-30', '2018-06-30', '2018-03-31', '2017-12-31', '2017-09-30',
'2017-06-30', '2017-03-31', '2016-12-31', '2016-09-30', '2016-06-30',
'2016-03-31', '2015-12-31', '2015-09-30', '2015-06-30', '2015-03-31',
'2014-12-31', '2014-09-30', '2014-06-30', '2014-03-31', '2013-12-31',
'2013-09-30', '2013-06-30', '2013-03-31', '2012-12-31', '2012-09-30',
'2012-06-30', '2012-03-31', '2011-12-31', '2011-09-30', '2011-06-30',
'2011-03-31', '2010-12-31', '2010-09-30', '2010-06-30', '2010-03-31',
'2009-12-31', '2009-09-30', '2009-06-30', '2009-03-31', '2008-12-31',
'2008-09-30', '2008-06-30', '2008-03-31', '2007-12-31', '2007-09-30',
'2007-06-30', '2007-03-31', '2006-12-31', '2006-09-30', '2006-06-30',
'2006-03-31', '2005-12-31', '2005-09-30', '2005-06-30', '2005-03-31',
'2004-12-31', '2004-09-30', '2004-06-30', '2004-03-31', '2003-12-31',
'2003-09-30', '2003-06-30', '2003-03-31', '2002-12-31', '2002-09-30',
'2002-06-30', '2002-03-31', '2001-12-31', '2001-09-30', '2001-06-30',
'2001-03-31', '2000-12-31', '2000-06-30', '1999-12-31', '1999-06-30',
from .context import init_context
init_context()
from typing import List
from zvt.api import fundamental
from zvt.domain import get_db_session, StoreCategory, Provider, TopTenHolder, TopTenTradableHolder
session = get_db_session(
store_category=StoreCategory.holder) # type: sqlalchemy.orm.Session
# 十大股东
def test_000778_top_ten_holder():
result: List[TopTenHolder] = fundamental.get_top_ten_holder(
session=session,
provider=Provider.EASTMONEY,
return_type='domain',
codes=['000778'],
end_timestamp='2018-12-31',
start_timestamp='2018-09-30',
order=TopTenHolder.shareholding_ratio.desc())
assert len(result) == 10
assert result[0].holder_name == '新兴际华集团有限公司'
assert result[0].shareholding_numbers == 1595000000
assert result[0].shareholding_ratio == 0.3996
assert result[0].change == 32080000
assert result[0].change_ratio == 0.0205
