def create_index_figure_column(code, dtype, start_date, end_date):
delta_days = (end_date - start_date).days
if delta_days <= 0: return None
start_date = start_date.strftime('%Y-%m-%d')
end_date = end_date.strftime('%Y-%m-%d')
if dtype == 'K线图':
obj = CIndex(code)
df = obj.get_k_data_in_range(start_date, end_date)
source = ColumnDataSource(df)
mapper = linear_cmap(field_name='pchange', palette=['red', 'green'], low=0, high=0, low_color = 'green', high_color = 'red')
p = figure(plot_height=500, plot_width=1200, tools="", toolbar_location=None, sizing_mode="scale_both", x_range=(0, len(df)))
p.xaxis.axis_label = "时间"
p.yaxis.axis_label = "点数"
p.xaxis.major_label_overrides = {i: mdate for i, mdate in enumerate(df['date'])}
p.segment(x0='index', y0='low', x1='date', y1='high', line_width=2, color='black', source=source)
p.vbar(x='index', bottom='open', top='close', width = 50 / delta_days, color=mapper, source=source)
p.add_tools(HoverTool(tooltips=[("数量", "@close"), ("时间", "@date")]))
volume_p = figure(plot_height=150, plot_width=1200, tools="", toolbar_location=None, sizing_mode="scale_both")
volume_p.x_range = p.x_range
volume_p.vbar(x='date', top='volume', width = 50 / delta_days, color=mapper, source=source)
return column(p, volume_p)
else:
obj = BullStockRatio(code)
df = obj.get_k_data_between(start_date, end_date)
source = ColumnDataSource(df)
p = figure(plot_height=500, plot_width=1200, tools="", toolbar_location=None, sizing_mode="scale_both")
p.xaxis.axis_label = "时间"
p.yaxis.axis_label = "比例"
p.line(x = 'index', y = 'ratio', line_width = 3, line_alpha = 1.0, source = source)
p.xaxis.major_label_overrides = {i: mdate for i, mdate in enumerate(df['date'])}
p.add_tools(HoverTool(tooltips=[("比例", "@ratio"), ("时间", "@date")]))
return column(p)
def get_index_data(start_date, end_date, index_code):
iobj = CIndex(index_code, dbinfo=ct.OUT_DB_INFO, redis_host='127.0.0.1')
i_data = iobj.get_k_data_in_range(start_date, end_date)
i_data = i_data[[
'open', 'high', 'low', 'close', 'volume', 'amount', 'date'
]]
return i_data
def choose_plate(edate = '2016-10-11', ndays = 90):
rindustry_info_client = RIndexIndustryInfo(redis_host='127.0.0.1')
today_industry_df = rindustry_info_client.get_k_data(edate)
pchange_df = today_industry_df.sort_values(by = 'pchange', ascending = False).head(3)
mchange_df = today_industry_df.sort_values(by = 'mchange', ascending = False).head(3)
plate_code_list = list(set(pchange_df.code.tolist()).intersection(pchange_df.code.tolist()))
if len(plate_code_list) == 0:
logger.info("no interested plate for date:%s" % edate)
return list()
sdate = get_day_nday_ago(edate, ndays, '%Y-%m-%d')
#get sh index data
sh_index_obj = CIndex('000001', redis_host='127.0.0.1')
sh_index_info = sh_index_obj.get_k_data_in_range(sdate, edate)
sh_index_pchange = 100 * (sh_index_info.loc[len(sh_index_info) - 1, 'close'] - sh_index_info.loc[0, 'preclose']) / sh_index_info.loc[0, 'preclose']
#get industry data
all_industry_df = rindustry_info_client.get_k_data_in_range(sdate, edate)
all_industry_df = all_industry_df.loc[all_industry_df.code.isin(plate_code_list)]
industry_static_info = DataFrame(columns={'code', 'sai', 'pchange', ct.KL, ct.QL, ct.JL, ct.FL})
#choose better industry
redisobj = create_redis_obj("127.0.0.1")
today_industry_info = IndustryInfo.get(redisobj)
for code, industry in all_industry_df.groupby('code'):
industry = industry.reset_index(drop = True)
industry['sri'] = 0
industry['sri'] = industry['pchange'] - sh_index_info['pchange']
industry['sai'] = 0
industry.at[(industry.pchange > 0) & (sh_index_info.pchange < 0), 'sai'] = industry.loc[(industry.pchange > 0) & (sh_index_info.pchange < 0), 'sri']
industry_sai = len(industry.loc[industry.sai > 0])
industry_pchange = 100 * (industry.loc[len(industry) - 1, 'close'] - industry.loc[0, 'preclose']) / industry.loc[0, 'preclose']
code_list = json.loads(today_industry_info.loc[today_industry_info.code == code, 'content'].values[0])
info_dict, good_code_list = choose_stock(code_list, sdate, edate)
industry_static_info = industry_static_info.append(DataFrame([[code, industry_sai, industry_pchange, info_dict[ct.KL], info_dict[ct.QL], info_dict[ct.JL], info_dict[ct.FL]]], columns = ['code', 'sai', 'pchange', ct.KL, ct.QL, ct.JL, ct.FL]), sort = 'True')
industry_static_info = industry_static_info.reset_index(drop = True)
industry_static_info = industry_static_info.sort_values(by=['pchange'], ascending=False)
return good_code_list
def get_overview_data():
#cdate = (datetime.now() - timedelta(days = 1)).strftime('%Y-%m-%d')
cdate = datetime.now().strftime('%Y-%m-%d')
code_dict = {
'000001': '上证指数',
'399001': '深证成指',
'399006': '创业板指',
'000016': '上证50',
'000905': '中证500'
}
overview_dict = {
'上证指数' : {'icon': 'dollar-sign', 'value': 0, 'change': 0, 'label': '上证指数', 'cdate': cdate},
'深证成指' : {'icon': 'dollar-sign', 'value': 0, 'change': 0, 'label': '深证成指', 'cdate': cdate},
'创业板指' : {'icon': 'dollar-sign', 'value': 0, 'change': 0, 'label': '创业板指', 'cdate': cdate},
'上证50' : {'icon': 'dollar-sign', 'value': 0, 'change': 0, 'label': '上证50', 'cdate': cdate},
'中证500' : {'icon': 'dollar-sign', 'value': 0, 'change': 0, 'label': '中证500', 'cdate': cdate},
'科创板指' : {'icon': 'dollar-sign', 'value': 0, 'change': 0, 'label': '科创板指', 'cdate': cdate}
}
for code, name in code_dict.items():
df = CIndex(code).get_k_data(cdate)
if df is None: return overview_dict
if df.empty:
overview_dict[name]['value'] = 0
overview_dict[name]['change'] = 0
else:
row = df.to_dict('records')[0]
overview_dict[name]['value'] = round(row['close'], 2)
overview_dict[name]['change'] = round(row['pchange'], 2)
return overview_dict
def _set_index_info(code_id):
if code_id in self.index_objs:
_obj = self.index_objs[code_id]
else:
_obj = CIndex(code_id) if code_id in list(
ct.INDEX_DICT.keys()) else TdxFgIndex(code_id)
return (code_id, _obj.set_components_data(cdate))
def get_index_df(code, start_date, end_date):
df = CIndex(code).get_k_data_in_range(start_date, end_date)
df['time'] = df.index.tolist()
df = df[[
'time', 'open', 'high', 'low', 'close', 'volume', 'amount', 'date'
]]
return df
def read_data(self, needStore = False):
if os.path.exists('i_data.json'):
with open('k_data.json', 'r') as f:
k_data = pd.read_json(f.read(), orient = 'records', lines = True)
k_data.date = k_data.date.dt.strftime('%Y-%m-%d')
with open('d_data.json', 'r') as f:
d_data = pd.read_json(f.read(), orient = 'records', lines = True)
with open('i_data.json', 'r') as f:
i_data = pd.read_json(f.read(), orient = 'records', lines = True)
i_data.date = i_data.date.dt.strftime('%Y-%m-%d')
else:
obj = CStock(self.code, dbinfo = ct.OUT_DB_INFO, redis_host = '127.0.0.1')
k_data = obj.get_k_data()
k_data.date = pd.to_datetime(k_data.date).dt.strftime('%Y-%m-%d')
d_data = obj.get_chip_distribution()
iobj = CIndex(self.index_code, dbinfo = ct.OUT_DB_INFO, redis_host = '127.0.0.1')
i_data = iobj.get_k_data()
cdates = k_data.date.tolist()
i_data = i_data.loc[i_data.date.isin(cdates)]
i_data = i_data.reset_index(drop = True)
k_data = k_data[['date', 'open', 'high', 'low', 'close', 'volume', 'amount', 'outstanding', 'totals', 'adj', 'aprice', 'uprice', 'sprice', 'mprice', 'lprice', 'profit']]
k_data = k_data.rename(columns = {"date": "time"})
i_data = i_data[['date', 'open', 'high', 'low', 'close', 'volume', 'amount']]
i_data = i_data.rename(columns = {"date": "time"})
if needStore:
with open('k_data.json', 'w') as f:
f.write(k_data.to_json(orient='records', lines=True))
with open('d_data.json', 'w') as f:
f.write(d_data.to_json(orient='records', lines=True))
with open('i_data.json', 'w') as f:
f.write(i_data.to_json(orient='records', lines=True))
return k_data, d_data, i_data
def _set_index_info(cdate, code_id):
try:
_obj = self.index_objs[
code_id] if code_id in self.index_objs else CIndex(code_id)
return (code_id, _obj.set_k_data(cdate))
except Exception as e:
self.logger.error(e)
return (code_id, False)
def create_obj(self, code):
try:
CIndex(code,
should_create_influxdb=True,
should_create_mysqldb=True)
return (code, True)
except Exception as e:
return (code, False)
def get_index_data(self, start_date, end_date, index_code):
iobj = CIndex(index_code, dbinfo = self.dbinfo, redis_host = self.redis_host)
i_data = iobj.get_k_data_in_range(start_date, end_date)
i_data = i_data.sort_values(by=['date'], ascending=True)
i_data = i_data.reset_index(drop = True)
i_data['time'] = i_data.index.tolist()
i_data = i_data[['time', 'open', 'high', 'low', 'close', 'volume', 'amount', 'date']]
return i_data
def get_data(code, start_date, end_date):
cstock_obj = CIndex(code, dbinfo=ct.OUT_DB_INFO, redis_host='127.0.0.1')
data = cstock_obj.get_k_data_in_range(start_date, end_date)
data = data.set_index('date')
if is_df_has_unexpected_data(data): return None
data.index = pd.to_datetime(data.index)
data = data.dropna(how='any')
return data
def get_index_data(self, cdate):
df = pd.DataFrame()
for code, name in ct.TDX_INDEX_DICT.items():
data = CIndex(code).get_k_data(cdate)
data['name'] = name
data['code'] = code
df = df.append(data)
df = df.reset_index(drop=True)
return df
def get_valuation_data(code_dict, start_date, end_date):
data_dict = dict()
for code, name in code_dict.items():
df = CIndex(code).get_val_data()
df = df.loc[(df.date >= start_date) & (df.date <= end_date)]
df = df.sort_values(by=['date'], ascending = True)
df = df.reset_index(drop = True)
data_dict[code] = df
data_dict['date'] = df.date.tolist()
return data_dict
def get_data(self, start_date, end_date, index_code):
df = self.ris.get_k_data_in_range(start_date, end_date)
iobj = CIndex(index_code,
dbinfo=ct.OUT_DB_INFO,
redis_host='127.0.0.1')
i_data = iobj.get_k_data_in_range(start_date, end_date)
i_data['time'] = i_data.index.tolist()
i_data = i_data[[
'time', 'open', 'high', 'low', 'close', 'volume', 'amount', 'date'
]]
return df, i_data
def get_industry_data(self, _date):
df = pd.DataFrame()
df_info = IndustryInfo.get()
for _, code in df_info.code.iteritems():
data = CIndex(code).get_k_data(date=_date)
df = df.append(data)
df = df.reset_index(drop=True)
df['name'] = df_info['name']
df = df.sort_values(by='amount', ascending=False)
df = df.reset_index(drop=True)
return df
def _set_index_info(cdate, code_id):
try:
if code_id in self.index_objs:
_obj = self.index_objs[code_id]
else:
_obj = CIndex(code_id) if code_id in list(
ct.TDX_INDEX_DICT.keys()) else TdxFgIndex(code_id)
return (code_id, _obj.set_k_data(cdate))
except Exception as e:
self.logger.error(e)
return (code_id, False)
def __init__(self, index_code, dbinfo = ct.DB_INFO, redis_host = None):
self.dbinfo = dbinfo
self.index_code = index_code
self.index_obj = CIndex(index_code, dbinfo = self.dbinfo, redis_host = redis_host)
self.db_name = self.index_obj.get_dbname(index_code)
self.logger = getLogger(__name__)
self.ris = RIndexStock(dbinfo, redis_host)
self.bull_stock_ratio_table = self.get_table_name()
self.redis = create_redis_obj() if redis_host is None else create_redis_obj(redis_host)
self.mysql_client = CMySQL(self.dbinfo, dbname = self.db_name, iredis = self.redis)
if not self.create(): raise Exception("create emotion table failed")
def init_index_info(self):
index_list = ct.INDEX_DICT.keys()
prefix_index_list = [add_index_prefix(code) for code in index_list]
ret = self.subscriber.subscribe(prefix_index_list, SubType.QUOTE,
self.quote_handler)
if 0 != ret:
self.logger.error("subscribe for index list failed")
return ret
for code in index_list:
if code not in self.index_objs:
self.index_objs[code] = CIndex(code,
should_create_influxdb=True,
should_create_mysqldb=True)
def init_stock_info(self, cdate=None):
def _set_stock_info(_date, bonus_info, index_info, code_id):
try:
if CStock(code_id).set_k_data(bonus_info, index_info, _date):
self.logger.info("%s set k data success" % code_id)
return (code_id, True)
else:
self.logger.error("%s set k data failed" % code_id)
return (code_id, False)
except Exception as e:
self.logger.error("%s set k data exception:%s" % (code_id, e))
return (code_id, False)
#get stock bonus info
bonus_info = pd.read_csv("/data/tdx/base/bonus.csv",
sep=',',
dtype={
'code': str,
'market': int,
'type': int,
'money': float,
'price': float,
'count': float,
'rate': float,
'date': int
})
index_info = CIndex('000001').get_k_data()
if index_info is None or index_info.empty: return False
df = self.stock_info_client.get()
failed_list = df.code.tolist()
if cdate is None:
cfunc = partial(_set_stock_info, cdate, bonus_info, index_info)
return process_concurrent_run(cfunc, failed_list, num=5)
else:
succeed = True
start_date = get_day_nday_ago(cdate, num=10, dformat="%Y-%m-%d")
for mdate in get_dates_array(start_date, cdate, asending=True):
if self.cal_client.is_trading_day(mdate):
cfunc = partial(_set_stock_info, mdate, bonus_info,
index_info)
if not process_concurrent_run(cfunc, failed_list, num=500):
succeed = False
return succeed
def init_stock_info(self, cdate=None):
def _set_stock_info(mdate, bonus_info, index_info, code_id):
try:
if CStock(code_id).set_k_data(bonus_info, index_info, mdate):
self.logger.info("%s set k data success for date:%s",
code_id, mdate)
return (code_id, True)
else:
self.logger.error("%s set k data failed for date:%s",
code_id, mdate)
return (code_id, False)
except Exception as e:
self.logger.error("%s set k data for date %s exception:%s",
code_id, mdate, e)
return (code_id, False)
#get stock bonus info
bonus_info = pd.read_csv("/data/tdx/base/bonus.csv",
sep=',',
dtype={
'code': str,
'market': int,
'type': int,
'money': float,
'price': float,
'count': float,
'rate': float,
'date': int
})
index_info = CIndex('000001').get_k_data()
if index_info is None or index_info.empty: return False
df = self.stock_info_client.get()
if df.empty: return False
failed_list = df.code.tolist()
if cdate is None:
cfunc = partial(_set_stock_info, cdate, bonus_info, index_info)
return process_concurrent_run(cfunc, failed_list, num=8)
else:
cfunc = partial(_set_stock_info, cdate, bonus_info, index_info)
succeed = True
if not process_concurrent_run(cfunc, failed_list, num=8):
succeed = False
return succeed
# -*- coding: utf-8 -*-
import os
import sys
import pandas as pd
from os.path import abspath, dirname
sys.path.insert(0, dirname(dirname(dirname(abspath(__file__)))))
from cindex import CIndex
pd.set_option('display.max_columns', None)
pd.set_option('display.max_rows', None)
code = '399006'
start_date = '2019-01-01'
end_date = '2019-08-11'
df = CIndex(code).get_val_data()
df = df.sort_values(by=['date'], ascending=True)
df = df.reset_index(drop=True)
print(df.loc[(df.date > start_date) & (df.date < end_date)])
return new_tmp_df
if __name__ == '__main__':
if not os.path.exists('norm.json'):
creview = CReivew(ct.DB_INFO)
start_date = '2018-02-09'
end_date = '2018-09-10'
stock_info = CStockInfo.get()
stock_info = stock_info[['code', 'name', 'industry', 'timeToMarket']]
stock_info = stock_info[(stock_info.timeToMarket < 20180327)
& (stock_info.timeToMarket > 0)]
if not os.path.exists('index.json'):
#上证指数的数据
logger.info("start get index data")
szzs_df = CIndex('000001').get_k_data_in_range(
start_date, end_date)
szzs_df = szzs_df.sort_values(by='date', ascending=True)
szzs_df['code'] = 'i000001'
szzs_df['name'] = "上证指数"
szzs_df['industry'] = "所有"
szzs_df['preclose'] = szzs_df['close'].shift(1)
szzs_df = szzs_df[szzs_df.date != start_date]
szzs_df['pchange'] = 100 * (szzs_df.close -
szzs_df.preclose) / szzs_df.preclose
#write data to json file
with open('index.json', 'w') as f:
f.write(szzs_df.to_json(orient='records', lines=True))
else:
logger.info("begin read index file")
with open('index.json', 'r') as f:
szzs_df = pd.read_json(f.read(),
def _set_industry_info(cdate, code_id):
return (code_id, CIndex(code_id).set_k_data(cdate))
base_color = '#e6daa6'
name = "shanghai_%s_image" % cdate
i_data = i_data[[
'date', 'open', 'high', 'low', 'close', 'volume', 'amount'
]]
fig, index_ax = plt.subplots(figsize=(16, 10))
fig.subplots_adjust(bottom=0.2)
candlestick2_ochl(index_ax,
i_data['open'],
i_data['close'],
i_data['high'],
i_data['low'],
width=1.0,
colorup='r',
colordown='g')
index_ax.set_ylabel(name)
index_ax.set_xticks(range(0, len(i_data['date']), 10))
plt.plot()
fig.autofmt_xdate()
plt.show()
index_obj = CIndex('000001', dbinfo=ct.OUT_DB_INFO, redis_host='127.0.0.1')
df = index_obj.get_k_data()
cdates = df[(df.pchange > 4) & (df.date > '1999-01-01')].date.tolist()
for mdate in cdates:
start_date = mdate
end_date = get_day_nday_after(start_date, num=40, dformat="%Y-%m-%d")
index_data = index_obj.get_k_data_in_range(start_date, end_date)
plot_index(start_date, index_data)
def _set_index_info(code_id):
_obj = self.index_objs[
code_id] if code_id in self.index_objs else CIndex(code_id)
return (code_id, _obj.set_components_data(cdate))
from ccalendar import CCalendar
from cstock_info import CStockInfo
from base.cdate import get_day_nday_ago, get_dates_array
if __name__ == '__main__':
num = 5500
end_date = '2019-08-13'
stock_info_client = CStockInfo()
df = stock_info_client.get()
code_list = df['code'].tolist()
name_list = df['name'].tolist()
code2namedict = dict(zip(code_list, name_list))
start_date = get_day_nday_ago(end_date, num=num, dformat="%Y-%m-%d")
date_array = get_dates_array(start_date, end_date)
auth('18701683341', '52448oo78')
for code in [
'000001', '000016', '000300', '000905', '399001', '399005',
'399673'
]:
obj = CIndex(code)
for mdate in date_array:
if CCalendar.is_trading_day(mdate):
table_name = obj.get_components_table_name(mdate)
if obj.is_table_exists(table_name):
obj.mysql_client.delete(table_name)
for mdate in date_array:
if CCalendar.is_trading_day(mdate):
if not obj.set_components_data_from_joinquant(
code2namedict, mdate):
print("{} for {} set failed".format(code, mdate))
def get_industry_data(self, cdate, code):
return (code, CIndex(code).get_k_data(cdate))
raise Exception(
"tmp_df.volume.sum() is not equal to outstanding")
tmp_df = tmp_df.reset_index(drop=True)
pre_outstanding = outstanding
df = df.append(tmp_df)
df = df[df.volume != 0]
df = df.reset_index(drop=True)
return df
if __name__ == '__main__':
from cindex import CIndex
from cstock import CStock
cdate = None
cstock = CStock('601318')
index_info = CIndex('000001').get_k_data(cdate)
bonus_info = pd.read_csv("/data/tdx/base/bonus.csv",
sep=',',
dtype={
'code': str,
'market': int,
'type': int,
'money': float,
'price': float,
'count': float,
'rate': float,
'date': int
})
quantity_change_info, price_change_info = cstock.collect_right_info(
bonus_info)
