def record(self, entity, start, end, size, timestamps):
for timestamp in timestamps:
df = run_query(
table='finance.STK_HK_HOLD_INFO',
conditions=
f'link_id#=#{entity.code}&day#=#{to_time_str(timestamp)}')
print(df)
if pd_is_not_null(df):
df.rename(columns={
'day': 'timestamp',
'link_id': 'holder_code',
'link_name': 'holder_name'
},
inplace=True)
df['timestamp'] = pd.to_datetime(df['timestamp'])
df['entity_id'] = df['code'].apply(
lambda x: to_entity_id(entity_type='stock', jq_code=x))
df['code'] = df['code'].apply(lambda x: x.split('.')[0])
# id格式为:{holder_name}_{entity_id}_{timestamp}
df['id'] = df[['holder_name', 'entity_id', 'timestamp']].apply(
lambda se: "{}_{}_{}".format(
se['holder_name'], se['entity_id'],
to_time_str(se['timestamp'], fmt=TIME_FORMAT_DAY)),
axis=1)
df_to_db(df=df,
data_schema=self.data_schema,
provider=self.provider,
force_update=self.force_update)
def record(self, entity, start, end, size, timestamps):
jq_code = code_map_jq.get(entity.code)
q = query(finance.STK_MT_TOTAL).filter(
finance.STK_MT_TOTAL.exchange_code == jq_code,
finance.STK_MT_TOTAL.date >= to_time_str(start)).limit(2000)
df = finance.run_query(q)
print(df)
json_results = []
for item in df.to_dict(orient='records'):
result = {
'provider': self.provider,
'timestamp': item['date'],
'name': entity.name,
'margin_value': item['fin_value'],
'margin_buy': item['fin_buy_value'],
'short_value': item['sec_value'],
'short_volume': item['sec_sell_volume'],
'total_value': item['fin_sec_value']
}
json_results.append(result)
if len(json_results) < 100:
self.one_shot = True
return json_results
def record(self, entity, start, end, size, timestamps):
q = query(
valuation
).filter(
valuation.code == to_jq_entity_id(entity)
)
count: pd.Timedelta = now_pd_timestamp() - start
df = get_fundamentals_continuously(q, end_date=now_time_str(), count=count.days + 1, panel=False)
df['entity_id'] = entity.id
df['timestamp'] = pd.to_datetime(df['day'])
df['code'] = entity.code
df['name'] = entity.name
df['id'] = df['timestamp'].apply(lambda x: "{}_{}".format(entity.id, to_time_str(x)))
df = df.rename({'pe_ratio_lyr': 'pe',
'pe_ratio': 'pe_ttm',
'pb_ratio': 'pb',
'ps_ratio': 'ps',
'pcf_ratio': 'pcf'},
axis='columns')
df['market_cap'] = df['market_cap'] * 100000000
df['circulating_market_cap'] = df['circulating_market_cap'] * 100000000
df['capitalization'] = df['capitalization'] * 10000
df['circulating_cap'] = df['circulating_cap'] * 10000
df['turnover_ratio'] = df['turnover_ratio'] * 0.01
df_to_db(df=df, data_schema=self.data_schema, provider=self.provider, force_update=self.force_update)
return None
def generate_request_param(self, security_item, start, end, size, timestamps):
if len(timestamps) <= 10:
param = {
"color": "w",
"fc": get_fc(security_item),
"corpType": company_type_flag(security_item),
# 0 means get all types
"reportDateType": 0,
"endDate": '',
"latestCount": size
}
else:
param = {
"color": "w",
"fc": get_fc(security_item),
"corpType": company_type_flag(security_item),
# 0 means get all types
"reportDateType": 0,
"endDate": to_time_str(timestamps[10]),
"latestCount": 10
}
if self.finance_report_type == 'LiRunBiaoList' or self.finance_report_type == 'XianJinLiuLiangBiaoList':
param['reportType'] = 1
return param
def common_draw(df_list: List[pd.DataFrame], chart_type=Line, columns=[], name_field='security_id', render='html',
file_name=None):
if len(df_list) > 1:
df_list = fill_with_same_index(df_list=df_list)
chart = None
if chart_type == Line:
chart = get_default_line()
assert len(columns) == 1
xdata = [to_time_str(timestamp) for timestamp in df_list[0].index]
chart.add_xaxis(xdata)
for df in df_list:
series_name = df[df[name_field].notna()][name_field][0]
if len(columns) == 1:
ydata = df.loc[:, columns[0]].values.tolist()
chart.add_yaxis(series_name, ydata, is_smooth=True,
markpoint_opts=opts.MarkPointOpts(
data=[opts.MarkPointItem(type_="min"), opts.MarkPointItem(type_="max")]))
if render == 'html':
chart.render(get_ui_path(file_name))
elif render == 'notebook':
chart.render_notebook()
return chart
def on_trading_open(self, timestamp):
self.logger.info('on_trading_open:{}'.format(timestamp))
if is_same_date(timestamp, self.start_timestamp):
return
# get the account for trading at the date
accounts = get_account_stats(session=self.session,
trader_name=self.trader_name,
return_type='domain',
end_timestamp=to_time_str(timestamp),
limit=1,
order=AccountStats.timestamp.desc())
if accounts:
account = accounts[0]
else:
return
positions = []
# FIXME:dump all directly
for position_domain in account.positions:
position_dict = position_schema.dump(position_domain)
self.logger.info('current position:{}'.format(position_dict))
del position_dict['account_stats']
positions.append(position_dict)
self.latest_account = account_stats_schema.dump(account)
self.latest_account['positions'] = positions
self.logger.info('on_trading_open:{},latest_account:{}'.format(
timestamp, self.latest_account))
def on_finish(self, security_item):
kdatas = get_kdata(provider=self.provider, security_id=security_item.id, level=self.level.value,
order=self.data_schema.timestamp.asc(),
return_type='domain',
session=self.session,
filters=[self.data_schema.hfq_close.is_(None),
self.data_schema.timestamp >= to_pd_timestamp('2005-01-01')])
if kdatas:
start = kdatas[0].timestamp
end = kdatas[-1].timestamp
# get hfq from joinquant
df = get_price(to_jq_security_id(security_item), start_date=to_time_str(start), end_date=now_time_str(),
frequency='daily',
fields=['factor', 'open', 'close', 'low', 'high'],
skip_paused=True, fq='post')
if df is not None and not df.empty:
# fill hfq data
for kdata in kdatas:
time_str = to_time_str(kdata.timestamp)
if time_str in df.index:
kdata.hfq_open = df.loc[time_str, 'open']
kdata.hfq_close = df.loc[time_str, 'close']
kdata.hfq_high = df.loc[time_str, 'high']
kdata.hfq_low = df.loc[time_str, 'low']
kdata.factor = df.loc[time_str, 'factor']
self.session.commit()
latest_factor = df.factor[-1]
# factor not change yet, no need to reset the qfq past
if latest_factor == self.current_factors.get(security_item.id):
sql = 'UPDATE {} SET qfq_close=hfq_close/{},qfq_high=hfq_high/{}, qfq_open= hfq_open/{}, qfq_low= hfq_low/{} where ' \
'security_id=\'{}\' and level=\'{}\' and (qfq_close isnull or qfq_high isnull or qfq_low isnull or qfq_open isnull)'.format(
self.data_schema.__table__, latest_factor, latest_factor, latest_factor, latest_factor,
security_item.id, self.level.value)
else:
sql = 'UPDATE {} SET qfq_close=hfq_close/{},qfq_high=hfq_high/{}, qfq_open= hfq_open/{}, qfq_low= hfq_low/{} where ' \
'security_id=\'{}\' and level=\'{}\''.format(self.data_schema.__table__, latest_factor,
latest_factor, latest_factor, latest_factor,
security_item.id,
self.level.value)
self.logger.info(sql)
self.session.execute(sql)
self.session.commit()
# TODO:use netease provider to get turnover_rate
self.logger.info('use netease provider to get turnover_rate')
def test_000778_cash_flow_statement():
correct_timestamps = ['2018-09-30', '2018-06-30', '2018-03-31', '2017-12-31', '2017-09-30', '2017-06-30',
'2017-03-31', '2016-12-31', '2016-09-30', '2016-06-30', '2016-03-31', '2015-12-31',
'2015-09-30', '2015-06-30', '2015-03-31', '2014-12-31', '2014-09-30', '2014-06-30',
'2014-03-31', '2013-12-31', '2013-09-30', '2013-06-30', '2013-03-31', '2012-12-31',
'2012-09-30', '2012-06-30', '2012-03-31', '2011-12-31', '2011-09-30', '2011-06-30',
'2011-03-31', '2010-12-31', '2010-09-30', '2010-06-30', '2010-03-31', '2009-12-31',
'2009-09-30', '2009-06-30', '2009-03-31', '2008-12-31', '2008-09-30', '2008-06-30',
'2008-03-31', '2007-12-31', '2007-09-30', '2007-06-30', '2007-03-31', '2006-12-31',
'2006-09-30', '2006-06-30', '2006-03-31', '2005-12-31', '2005-09-30', '2005-06-30',
'2005-03-31', '2004-12-31', '2004-09-30', '2004-06-30', '2004-03-31', '2003-12-31',
'2003-09-30', '2003-06-30', '2003-03-31', '2002-12-31', '2002-06-30', '2001-12-31',
'2001-06-30', '2000-12-31', '2000-06-30', '1999-12-31', '1998-12-31', '1998-06-30']
result = fundamental.get_cash_flow_statement(session=session, provider=Provider.EASTMONEY, return_type='domain',
codes=['000778'], end_timestamp='2018-12-30',
order=CashFlowStatement.timestamp.desc())
assert len(correct_timestamps) == len(result)
timestamps = [to_time_str(item.report_date) for item in result]
assert set(correct_timestamps) == set(timestamps)
latest: CashFlowStatement = result[0]
assert latest.cash_from_selling == 27784000000
assert latest.tax_refund == 60700000
assert latest.cash_from_other_op == 1463000000
assert latest.total_op_cash_inflows == 29310000000
assert latest.cash_to_goods_services == 21210000000
assert latest.cash_to_employees == 1460000000
assert latest.taxes_and_surcharges == 2016000000
assert latest.cash_to_other_related_op == 573700000
assert latest.total_op_cash_outflows == 25260000000
assert latest.net_op_cash_flows == 4050000000
assert latest.cash_from_disposal_of_investments == 556500000
assert latest.cash_from_returns_on_investments == 44180000
assert latest.cash_from_disposal_fixed_intangible_assets == 457200
assert latest.cash_from_disposal_subsidiaries == 1046000000
assert latest.cash_from_other_investing == 553000000
assert latest.total_investing_cash_inflows == 2201000000
assert latest.cash_to_acquire_fixed_intangible_assets == 2521000000
assert latest.cash_to_investments == 1808000000
assert latest.total_investing_cash_outflows == 4329000000
assert latest.net_investing_cash_flows == -2128000000
assert latest.cash_from_accepting_investment == 24500000
assert latest.cash_from_subsidiaries_accepting_minority_interest == 24500000
assert latest.cash_from_borrowings == 10080000000
assert latest.cash_from_issuing_bonds == 997000000
assert latest.cash_from_other_financing == 200000000
assert latest.total_financing_cash_inflows == 11300000000
assert latest.cash_to_repay_borrowings == 11940000000
assert latest.cash_to_pay_interest_dividend == 892100000
assert latest.cash_to_other_financing == 328500000
assert latest.total_financing_cash_outflows == 13160000000
assert latest.net_financing_cash_flows == -1862000000
assert latest.foreign_exchange_rate_effect == 21350000
assert latest.net_cash_increase == 81240000
assert latest.cash_at_beginning == 5078000000
assert latest.cash == 5159000000
def generate_request_param(self, security_item, start, end, size,
timestamp):
return {
'security_item': security_item,
'start': to_time_str(start, fmt=TIME_FORMAT_DAY1),
'end': now_time_str(fmt=TIME_FORMAT_DAY1),
'level': self.level.value
}
def generate_id(se):
holdname = se['holder_name']
if len(holdname) > 20:
holdname = str(holdname).split('、')[0]
return "{}_{}_{}".format(
se['entity_id'],
to_time_str(se['timestamp'], fmt=TIME_FORMAT_DAY),
holdname)
def record(self, entity, start, end, size, timestamps):
if not end:
end = to_time_str(now_pd_timestamp())
if (pd.to_datetime(end) - start).days >= 800:
from datetime import timedelta
end = to_time_str(start + timedelta(days=800))
start = to_time_str(start)
exchange = 'SH' if 'sh' in entity.id else 'SZ'
em_code = entity.code + '.' + exchange
if em_code == '000022.SH':
return None
columns_list = {
'MV': 'market_cap', #总市值
'LIQMV': 'circulating_market_cap', #流通市值
'TURN': 'turnover_ratio', #换手率
'PELYR': 'pe', # 静态pe
'PETTM': 'pe_ttm', # 动态pe
'PBLYR': 'pb', # 市净率PB(最新年报)
'PBMRQ': 'pb_mrq', # 市净率PB(MRQ)
'PSTTM': 'ps_ttm', #市销率PS(TTM)
'PCFTTM': 'pcf_ttm', #市现率PCF(最新年报，经营性现金流)
'DIVIDENDYIELD': 'div_yield', #股息率
}
df = c.csd(em_code, [i for i in columns_list.keys()], start, end,
"ispandas=1,DelType=1")
if not isinstance(df, pd.DataFrame):
return None
# 该方法会获取到未来数据,总市值为Nan时删除该条
df = df.dropna(subset=['MV'])
if df.empty:
return None
df = df.rename(columns=columns_list)
df['entity_id'] = entity.id[:15]
df['timestamp'] = pd.to_datetime(df['DATES'])
df['code'] = entity.code
df['name'] = entity.name
df['id'] = df.apply(
lambda x: "{}_{}".format(x.entity_id, to_time_str(x.timestamp)),
axis=1)
df_to_db(df=df,
data_schema=self.data_schema,
provider=self.provider,
force_update=self.force_update)
return None
def marshal_object_for_ui(object):
if isinstance(object, Enum):
return object.value
if isinstance(object, pd.Timestamp):
return to_time_str(object)
return object
def report_core_company():
while True:
error_count = 0
email_action = EmailInformer()
try:
# StockTradeDay.record_data(provider='joinquant')
# Stock.record_data(provider='joinquant')
# FinanceFactor.record_data(provider='eastmoney')
# BalanceSheet.record_data(provider='eastmoney')
target_date = to_time_str(now_pd_timestamp())
my_selector: TargetSelector = FundamentalSelector(
start_timestamp='2015-01-01', end_timestamp=target_date)
my_selector.run()
long_targets = my_selector.get_open_long_targets(
timestamp=target_date)
if long_targets:
stocks = get_entities(provider='joinquant',
entity_schema=Stock,
entity_ids=long_targets,
return_type='domain')
# add them to eastmoney
try:
try:
eastmoneypy.del_group('core')
except:
pass
eastmoneypy.create_group('core')
for stock in stocks:
eastmoneypy.add_to_group(stock.code, group_name='core')
except Exception as e:
email_action.send_message(
"*****@*****.**", f'report_core_company error',
'report_core_company error:{}'.format(e))
info = [f'{stock.name}({stock.code})' for stock in stocks]
msg = ' '.join(info)
else:
msg = 'no targets'
logger.info(msg)
email_action.send_message(get_subscriber_emails(),
f'{to_time_str(target_date)} 核心资产选股结果',
msg)
break
except Exception as e:
logger.exception('report_core_company error:{}'.format(e))
time.sleep(60 * 3)
error_count = error_count + 1
if error_count == 10:
email_action.send_message(
"*****@*****.**", f'report_core_company error',
'report_core_company error:{}'.format(e))
def record(self, entity, start, end, size, timestamps):
now_date = to_time_str(now_pd_timestamp())
sql_bars = 'select a.TRADE_DATE,a.TICKER_SYMBOL,b.SEC_SHORT_NAME,a.PRE_CLOSE_PRICE_1,' \
' a.OPEN_PRICE_1,a.CLOSE_PRICE_1,a.HIGHEST_PRICE_1,a.LOWEST_PRICE_1,c.TURNOVER_VOL,' \
'c.TURNOVER_VALUE,c.DEAL_AMOUNT,c.CHG_PCT,c.NEG_MARKET_VALUE,c.MARKET_VALUE, c.TURNOVER_RATE ' \
'from mkt_equd_adj a,md_security b,mkt_equd c ' \
'where a.SECURITY_ID=b.SECURITY_ID ' \
'and a.TICKER_SYMBOL=c.TICKER_SYMBOL ' \
'and a.TRADE_DATE=c.TRADE_DATE ' \
'and a.TRADE_DATE>=%s ' \
'and a.TRADE_DATE<=%s and a.HIGHEST_PRICE_1!=0 and a.TICKER_SYMBOL=%s'
if not self.end_timestamp:
prev_trade_date = self.get_prev_trade_date(self.tonglian_conn, now_date)
df = pd.read_sql(sql_bars, self.tonglian_conn, params=(prev_trade_date, now_date,entity.code))
else:
end_timestamp = to_time_str(self.end_timestamp)
prev_end_timestamp = self.get_prev_trade_date(self.tonglian_conn, end_timestamp)
df = pd.read_sql(sql_bars, self.tonglian_conn, params=(prev_end_timestamp, end_timestamp,entity.code))
if pd_is_not_null(df):
df.rename(columns={'TICKER_SYMBOL': 'code',
'TRADE_DATE': 'timestamp',
'OPEN_PRICE_1': 'open', #开盘价
'CLOSE_PRICE_1': 'close', #收盘价
'HIGHEST_PRICE_1': 'high', #最高价
'LOWEST_PRICE_1': 'low', #最低价
'TURNOVER_VOL': 'volume', #成交量
'TURNOVER_VALUE': 'money', #成交额
'SEC_SHORT_NAME': 'name', #证券名称
}, inplace=True)
df = df[['timestamp', 'code','name','open', 'close', 'low', 'high', 'volume', 'money']]
df['entity_id'] = entity.id
df['timestamp'] = pd.to_datetime(df['timestamp'])
df['provider'] = 'tonglian'
df['level'] = self.level.value
def generate_kdata_id(se):
if self.level >= IntervalLevel.LEVEL_1DAY:
return "{}_{}".format(se['entity_id'], to_time_str(se['timestamp'], fmt=TIME_FORMAT_DAY))
else:
return "{}_{}".format(se['entity_id'], to_time_str(se['timestamp'], fmt=TIME_FORMAT_ISO8601))
df['id'] = df[['entity_id', 'timestamp']].apply(generate_kdata_id, axis=1)
df_to_db(df=df, data_schema=self.data_schema, provider=self.provider, force_update=self.force_update)
return None
def test_000001_income_statement():
correct_timestamps = [
'2018-09-30', '2018-06-30', '2018-03-31', '2017-12-31', '2017-09-30',
'2017-06-30', '2017-03-31', '2016-12-31', '2016-09-30', '2016-06-30',
'2016-03-31', '2015-12-31', '2015-09-30', '2015-06-30', '2015-03-31',
'2014-12-31', '2014-09-30', '2014-06-30', '2014-03-31', '2013-12-31',
'2013-09-30', '2013-06-30', '2013-03-31', '2012-12-31', '2012-09-30',
'2012-06-30', '2012-03-31', '2011-12-31', '2011-09-30', '2011-06-30',
'2011-03-31', '2010-12-31', '2010-09-30', '2010-06-30', '2010-03-31',
'2009-12-31', '2009-09-30', '2009-06-30', '2009-03-31', '2008-12-31',
'2008-09-30', '2008-06-30', '2008-03-31', '2007-12-31', '2007-09-30',
'2007-06-30', '2007-03-31', '2006-12-31', '2006-09-30', '2006-06-30',
'2006-03-31', '2005-12-31', '2005-09-30', '2005-06-30', '2005-03-31',
'2004-12-31', '2004-09-30', '2004-06-30', '2004-03-31', '2003-12-31',
'2003-09-30', '2003-06-30', '2003-03-31', '2002-12-31', '2002-09-30',
'2002-06-30', '2002-03-31', '2001-12-31', '2001-09-30', '2001-06-30',
'2001-03-31', '2000-12-31', '2000-06-30', '1999-12-31', '1999-06-30',
'1998-12-31', '1998-06-30', '1997-12-31', '1997-06-30', '1996-12-31',
'1996-06-30', '1995-12-31', '1995-06-30', '1994-12-31', '1994-06-30',
'1993-12-31', '1993-06-30', '1992-12-31', '1991-12-31', '1990-12-31',
'1989-12-31'
]
result = IncomeStatement.query_data(
region=Region.CHN,
session=session,
provider=Provider.EastMoney,
return_type='domain',
codes=['000001'],
end_timestamp='2018-12-30',
order=IncomeStatement.report_date.desc(),
time_field='report_date')
assert len(correct_timestamps) == len(result)
timestamps = [to_time_str(item.report_date) for item in result]
assert set(correct_timestamps) == set(timestamps)
latest: IncomeStatement = result[0]
assert latest.operating_income == 86660000000
assert latest.fi_net_interest_income == 54530000000
assert latest.fi_interest_income == 121700000000
assert latest.fi_interest_expenses == 67130000000
assert latest.fi_net_incomes_from_fees_and_commissions == 23710000000
assert latest.fi_incomes_from_fees_and_commissions == 28920000000
assert latest.fi_expenses_for_fees_and_commissions == 5218000000
assert latest.investment_income == 7099000000
assert latest.fi_income_from_fair_value_change == 1047000000
assert latest.fi_income_from_exchange == -40000000
assert latest.fi_other_income == 144000000
assert latest.operating_costs == 26430000000
assert latest.business_taxes_and_surcharges == 847000000
assert latest.fi_operate_and_manage_expenses == 25580000000
assert latest.operating_profit == 26610000000
assert latest.non_operating_income == 14000000
assert latest.non_operating_costs == 62000000
assert latest.total_profits == 26570000000
def record(self, entity, start, end, size, timestamps):
df = pd.DataFrame()
dates = get_trade_days(start_date=start)
self.logger.info(f'add dates:{dates}')
df['timestamp'] = pd.to_datetime(dates)
df['id'] = [to_time_str(date) for date in dates]
df['entity_id'] = 'stock_sz_000001'
df_to_db(df=df, data_schema=self.data_schema, provider=self.provider, force_update=self.force_update)
def is_in_finished_timestamps(security_type, exchange, timestamp, level: TradingLevel):
if type(security_type) == str:
security_type = SecurityType(security_type)
if security_type == SecurityType.stock and exchange in ('sh', 'sz'):
if to_time_str(timestamp, fmt=TIME_FORMAT_MINUTE1) in china_stock_level_map_finished_timestamps.get(
level.value):
return True
return False
def test_000778_income_statement():
correct_timestamps = ['2018-09-30', '2018-06-30', '2018-03-31', '2017-12-31', '2017-09-30', '2017-06-30',
'2017-03-31', '2016-12-31', '2016-09-30', '2016-06-30', '2016-03-31', '2015-12-31',
'2015-09-30', '2015-06-30', '2015-03-31', '2014-12-31', '2014-09-30', '2014-06-30',
'2014-03-31', '2013-12-31', '2013-09-30', '2013-06-30', '2013-03-31', '2012-12-31',
'2012-09-30', '2012-06-30', '2012-03-31', '2011-12-31', '2011-09-30', '2011-06-30',
'2011-03-31', '2010-12-31', '2010-09-30', '2010-06-30', '2010-03-31', '2009-12-31',
'2009-09-30', '2009-06-30', '2009-03-31', '2008-12-31', '2008-09-30', '2008-06-30',
'2008-03-31', '2007-12-31', '2007-09-30', '2007-06-30', '2007-03-31', '2006-12-31',
'2006-09-30', '2006-06-30', '2006-03-31', '2005-12-31', '2005-09-30', '2005-06-30',
'2005-03-31', '2004-12-31', '2004-09-30', '2004-06-30', '2004-03-31', '2003-12-31',
'2003-09-30', '2003-06-30', '2003-03-31', '2002-12-31', '2002-09-30', '2002-06-30',
'2002-03-31', '2001-12-31', '2001-06-30', '2000-12-31', '2000-06-30', '1999-12-31',
'1999-06-30', '1998-12-31', '1998-06-30', '1997-12-31', '1997-06-30', '1996-12-31',
'1995-12-31', '1994-12-31']
result = fundamental.get_income_statement(session=session, provider=Provider.EASTMONEY, return_type='domain',
codes=['000778'], end_timestamp='2018-12-30',
order=IncomeStatement.timestamp.desc())
assert len(correct_timestamps) == len(result)
timestamps = [to_time_str(item.report_date) for item in result]
assert set(correct_timestamps) == set(timestamps)
latest: IncomeStatement = result[0]
assert latest.operating_income == 31710000000
assert latest.total_operating_costs == 29230000000
assert latest.operating_costs == 26220000000
assert latest.rd_costs == 185500000
assert latest.net_change_in_insurance_contract_reserves == 0
assert latest.business_taxes_and_surcharges == 359700000
assert latest.sales_costs == 771400000
assert latest.managing_costs == 472900000
assert latest.financing_costs == 397500000
assert latest.assets_devaluation == 824400000
assert latest.investment_income == 104100000
assert latest.investment_income_from_related_enterprise == 61290000
assert latest.operating_profit == 2637000000
assert latest.non_operating_income == 38340000
assert latest.non_operating_costs == 221700000
assert latest.total_profits == 2454000000
assert latest.tax_expense == 579600000
assert latest.net_profit == 1874000000
assert latest.net_profit_as_parent == 1811000000
assert latest.net_profit_as_minority_interest == 63570000
assert latest.deducted_net_profit == 1897000000
assert latest.eps == 0.4537
assert latest.diluted_eps == 0.4537
assert latest.other_comprehensive_income == -521000000
assert latest.other_comprehensive_income_as_parent == -522400000
assert latest.other_comprehensive_income_as_minority_interest == 1403000
assert latest.total_comprehensive_income == 1353000000
assert latest.total_comprehensive_income_as_parent == 1288000000
assert latest.total_comprehensive_income_as_minority_interest == 64980000
def test_000001_finance_factor():
correct_timestamps = ['2018-09-30', '2018-06-30', '2018-03-31', '2017-12-31', '2017-09-30', '2017-06-30',
'2017-03-31', '2016-12-31', '2016-09-30', '2016-06-30', '2016-03-31', '2015-12-31',
'2015-09-30', '2015-06-30', '2015-03-31', '2014-12-31', '2014-09-30', '2014-06-30',
'2014-03-31', '2013-12-31', '2013-09-30', '2013-06-30', '2013-03-31', '2012-12-31',
'2012-09-30', '2012-06-30', '2012-03-31', '2011-12-31', '2011-09-30', '2011-06-30',
'2011-03-31', '2010-12-31', '2010-09-30', '2010-06-30', '2010-03-31', '2009-12-31',
'2009-09-30', '2009-06-30', '2009-03-31', '2008-12-31', '2008-09-30', '2008-06-30',
'2008-03-31', '2007-12-31', '2007-09-30', '2007-06-30', '2007-03-31', '2006-12-31',
'2006-09-30', '2006-06-30', '2006-03-31', '2005-12-31', '2005-09-30', '2005-06-30',
'2005-03-31', '2004-12-31', '2004-09-30', '2004-06-30', '2004-03-31', '2003-12-31',
'2003-09-30', '2003-06-30', '2003-03-31', '2002-12-31', '2002-09-30', '2002-06-30',
'2002-03-31', '2001-12-31', '2001-09-30', '2001-06-30', '2001-03-31', '2000-12-31',
'2000-06-30', '1999-12-31', '1999-06-30', '1998-12-31', '1998-06-30', '1997-12-31',
'1997-06-30', '1996-12-31', '1996-06-30', '1995-12-31', '1995-06-30', '1994-12-31',
'1994-06-30', '1993-12-31', '1993-06-30', '1992-12-31', '1991-12-31', '1990-12-31',
'1989-12-31']
result = fundamental.get_finance_factor(session=session, provider=Provider.EASTMONEY, return_type='domain',
codes=['000001'], end_timestamp='2018-12-30',
order=FinanceFactor.timestamp.desc())
assert len(correct_timestamps) == len(result)
timestamps = [to_time_str(item.report_date) for item in result]
assert set(correct_timestamps) == set(timestamps)
latest: FinanceFactor = result[0]
assert latest.basic_eps == 1.14
assert latest.deducted_eps == 1.13
assert latest.diluted_eps == 1.14
assert latest.bps == 12.538
assert latest.capital_reserve_ps == 3.2886
assert latest.undistributed_profit_ps == 5.3566
assert latest.op_cash_flow_ps == -0.6587
assert latest.total_op_income == 86660000000
assert latest.net_profit == 20460000000
assert latest.deducted_net_profit == 20350000000
assert latest.op_income_growth_yoy == 0.0856
assert latest.net_profit_growth_yoy == 0.068
assert latest.deducted_net_profit_growth_yoy == 0.0636
assert latest.op_income_growth_qoq == 0.0336
assert latest.net_profit_growth_qoq == 0.0202
assert latest.deducted_net_profit_growth_qoq == 0.0168
assert latest.roe == 0.0948
assert latest.deducted_roe == 0.0943
assert latest.rota == 0.0062
assert latest.net_margin == 0.2360
assert latest.debt_asset_ratio == 0.9298
assert latest.em == 14.25
assert latest.equity_ratio == 13.25
assert latest.fi_total_deposit == 2130000000000
assert latest.fi_total_loan == 1920000000000
assert latest.fi_loan_deposit_ratio == 0.9004
assert latest.fi_npl_ratio == 0.0168
assert latest.fi_npl_provision_coverage == 1.6914
def china_stock_finished_timestamp(timestamp: pd.Timestamp,
level: TradingLevel):
timestamp = to_pd_timestamp(timestamp)
if timestamp.microsecond != 0:
return False
return to_time_str(timestamp, fmt=TIME_FORMAT_MINUTE1
) in china_stock_level_map_finished_timestamps.get(
level.value)
def request(self, url=None, method='get', param=None, path_fields=None):
security_item = param['security_item']
start_timestamp = param['start_timestamp']
size = param['size']
ccxt_level = param['ccxt_level']
level = param['level']
ccxt_account: CCXTAccount = param['ccxt_account']
ccxt_exchange = ccxt_account.get_ccxt_exchange(security_item.exchange)
if ccxt_exchange.has['fetchOHLCV']:
limit = ccxt_account.get_kdata_limit(security_item.exchange)
limit = min(size, limit)
kdata_list = []
try:
if ccxt_account.exchange_conf[
security_item.exchange]['support_since']:
kdatas = ccxt_exchange.fetch_ohlcv(security_item.code,
timeframe=ccxt_level,
since=start_timestamp)
else:
kdatas = ccxt_exchange.fetch_ohlcv(security_item.code,
timeframe=ccxt_level,
limit=limit)
# always ignore the latest one,because it's not finished
for kdata in kdatas[0:-1]:
current_timestamp = kdata[0]
if level == TradingLevel.LEVEL_1DAY:
current_timestamp = to_time_str(current_timestamp)
kdata_json = {
'timestamp': to_pd_timestamp(current_timestamp),
'open': kdata[1],
'high': kdata[2],
'low': kdata[3],
'close': kdata[4],
'volume': kdata[5],
'name': security_item.name,
'provider': 'ccxt',
'level': level.value
}
kdata_list.append(kdata_json)
return kdata_list
except Exception as e:
logger.exception("record_kdata for security:{} failed".format(
security_item.id))
else:
logger.warning("exchange:{} not support fetchOHLCV".format(
security_item.exchange))
def record(self, entity, start, end, size, timestamps, http_session):
if self.end_timestamp:
end_timestamp = to_time_str(self.end_timestamp)
df = yh_get_bars(code=entity.code, interval=self.yahoo_trading_level, start=start, end=end_timestamp, actions=False)
else:
end_timestamp = None
df = yh_get_bars(code=entity.code, interval=self.yahoo_trading_level, start=start, actions=False)
if pd_is_not_null(df):
return df
return None
def default(self, object):
if isinstance(object, pd.Series):
return object.to_dict()
elif isinstance(object, pd.Timestamp):
return to_time_str(object, fmt=TIME_FORMAT_ISO8601)
elif isinstance(object, Enum):
return object.value
elif isinstance(object, Bean):
return object.dict()
else:
return super().default(object)
def generate_finished_timestamps(security_type, exchange, level):
return [
to_time_str(timestamp, fmt=TIME_FORMAT_MINUTE1)
for timestamp in iterate_timestamps(security_type=security_type,
exchange=exchange,
start_timestamp='1999-01-01',
end_timestamp='1999-01-01',
level=level,
contain_all_timestamp=False,
kdata_use_begin_time=False)
]
def run(self):
from zvt.api import get_kdata
bond_data = get_kdata(entity_id='bond_cn_EMM00166466')
now_date = to_time_str(now_pd_timestamp())
if bond_data.empty:
# 初始时间定在2007年
start = '2007-01-01'
else:
start = to_time_str(bond_data.timestamp.max())
# EMM00166466 中债国债到期收益率：10年
df = c.edb(
"EMM00166466",
f"IsLatest=0,StartDate={start},EndDate={now_date},ispandas=1")
if pd_is_not_null(df):
df['name'] = "中债国债到期收益率：10年"
df.rename(columns={
'RESULT': 'data_value',
'DATES': 'timestamp'
},
inplace=True)
df['entity_id'] = 'bond_cn_EMM00166466'
df['timestamp'] = pd.to_datetime(df['timestamp'])
df['provider'] = 'emquantapi'
df['exchange'] = 'cn'
df['level'] = '1d'
df['code'] = "EMM00166466"
def generate_kdata_id(se):
return "{}_{}".format(
se['entity_id'],
to_time_str(se['timestamp'], fmt=TIME_FORMAT_DAY))
df['id'] = df[['entity_id', 'timestamp']].apply(generate_kdata_id,
axis=1)
df_to_db(df=df,
data_schema=self.data_schema,
provider=self.provider,
force_update=self.force_update)
def record(self, entity, start, end, size, timestamps):
# if entity.exchange == "swl1":
# return None
if not end:
if (now_pd_timestamp() - start).days > 365:
from datetime import timedelta
end = to_time_str(start + timedelta(days=365))
else:
end = to_time_str(now_pd_timestamp())
start = to_time_str(start)
df = c.csd(f"{entity.code}.SWI", "OPEN,CLOSE,HIGH,LOW,VOLUME,AMOUNT", start, end,
"period=1,adjustflag=1,curtype=1,order=1,ispandas=1")
if type(df) != pd.DataFrame:
return None
df.rename(columns={
'DATES': 'timestamp',
'OPEN': 'open',
'CLOSE': 'close',
'HIGH': 'high',
'LOW': 'low',
'VOLUME': 'volume',
'AMOUNT': 'turnover',
}, inplace=True)
if pd_is_not_null(df):
df['name'] = entity.name
df['entity_id'] = entity.id
df['timestamp'] = pd.to_datetime(df['timestamp'])
df['provider'] = 'joinquant'
df['level'] = '1d'
df['code'] = entity.code
def generate_kdata_id(se):
return "{}_{}".format(se['entity_id'], to_time_str(se['timestamp'], fmt=TIME_FORMAT_DAY))
df['id'] = df[['entity_id', 'timestamp']].apply(generate_kdata_id, axis=1)
df_to_db(df=df, data_schema=self.data_schema, provider=self.provider, force_update=self.force_update)
return None
def fetch_cumulative_net_value(self, security_item, start, end,
http_session) -> pd.DataFrame:
query_url = 'http://api.fund.eastmoney.com/f10/lsjz?' \
'fundCode={}&pageIndex={}&pageSize=200&startDate={}&endDate={}'
page = 1
df = pd.DataFrame()
while True:
url = query_url.format(security_item.code, page,
to_time_str(start), to_time_str(end))
text = sync_get(http_session,
url,
headers=EASTMONEY_ETF_NET_VALUE_HEADER,
return_type='text')
if text is None:
break
try:
response_json = demjson.decode(text)
response_df = pd.DataFrame(response_json['Data']['LSJZList'])
except:
break
# 最后一页
if not pd_is_not_null(response_df):
break
response_df['FSRQ'] = pd.to_datetime(response_df['FSRQ'])
response_df['JZZZL'] = pd.to_numeric(response_df['JZZZL'],
errors='coerce')
response_df['LJJZ'] = pd.to_numeric(response_df['LJJZ'],
errors='coerce')
response_df = response_df.fillna(0)
response_df.set_index('FSRQ', inplace=True, drop=True)
df = pd.concat([df, response_df])
page += 1
self.sleep()
return df
def persist(self, ref_record, df_record, entity_item, http_session):
saved_counts = 0
entity_finished = False
if pd_is_not_null(df_record):
assert 'id' in df_record.columns
saved_counts = df_to_db(df=df_record,
ref_df=ref_record,
region=self.region,
data_schema=self.data_schema,
provider=self.provider,
fix_duplicate_way=self.fix_duplicate_way)
if saved_counts == 0:
entity_finished = True
# could not get more data
else:
# not realtime
if not self.real_time:
entity_finished = True
# realtime and to the close time
elif (self.close_hour is not None) and (self.close_minute is not None):
now = now_pd_timestamp(self.region)
if now.hour >= self.close_hour:
if now.minute - self.close_minute >= 5:
self.logger.info(f'{entity_item.id} now is the close time: {now}')
entity_finished = True
if isinstance(self, FixedCycleDataRecorder):
entity_finished = True
start_timestamp = to_time_str(df_record['timestamp'].min(axis=0))
end_timestamp = to_time_str(df_record['timestamp'].max(axis=0))
self.result = [saved_counts, start_timestamp, end_timestamp]
# add finished entity to finished_items
if entity_finished:
self.on_finish_entity(entity_item, http_session)
return entity_finished, saved_counts
def generate_request_param(self, security_item, start, end, size,
timestamp):
if self.start_timestamp:
start = max(self.start_timestamp, to_pd_timestamp(start))
return {
'security_item': security_item,
'start_timestamp': to_time_str(start),
'size': size,
'level': self.level,
'ccxt_level': self.ccxt_trading_level
}
def generate_request_param(self, security_item, start, end, size,
timestamp):
if self.start_timestamp:
start = max(self.start_timestamp, to_pd_timestamp(start))
if size < 20:
size = 20
return {
'security_item': security_item,
'start_timestamp': to_time_str(start),
'size': size
}