def get_index_data(cdate):
df = pd.DataFrame()
for code, name in ct.TDX_INDEX_DICT.items():
data = CIndex(code).get_k_data(cdate)
data['name'] = name
data['code'] = code
df = df.append(data)
df = df.reset_index(drop = True)
return df
def get_index_data(self, _date):
df = pd.DataFrame()
for code, name in ct.TDX_INDEX_DICT.items():
self.mysql_client.changedb(CIndex.get_dbname(code))
data = self.mysql_client.get(
"select * from day where date=\"%s\";" % _date)
data['name'] = name
df = df.append(data)
self.mysql_client.changedb()
return df
def get_industry_data(self, _date):
df = pd.DataFrame()
df_info = IndustryInfo.get()
for _, code in df_info.code.iteritems():
data = CIndex(code).get_k_data(date=_date)
df = df.append(data)
df = df.reset_index(drop=True)
df['name'] = df_info['name']
df = df.sort_values(by='amount', ascending=False)
df = df.reset_index(drop=True)
return df
def read_data(self):
if not os.path.exists('i_data.json'):
obj = CStock(self.code, redis_host='127.0.0.1')
k_data = obj.get_k_data()
k_data.date = pd.to_datetime(k_data.date).dt.strftime('%Y-%m-%d')
with open('k_data.json', 'w') as f:
f.write(k_data.to_json(orient='records', lines=True))
d_data = obj.get_chip_distribution()
with open('d_data.json', 'w') as f:
f.write(d_data.to_json(orient='records', lines=True))
iobj = CIndex(self.index_code)
i_data = iobj.get_k_data()
cdates = k_data.date.tolist()
i_data = i_data.loc[i_data.date.isin(cdates)]
i_data = i_data.reset_index(drop=True)
with open('i_data.json', 'w') as f:
f.write(i_data.to_json(orient='records', lines=True))
else:
with open('k_data.json', 'r') as f:
k_data = pd.read_json(f.read(), orient='records', lines=True)
k_data.date = k_data.date.dt.strftime('%Y-%m-%d')
with open('d_data.json', 'r') as f:
d_data = pd.read_json(f.read(), orient='records', lines=True)
with open('i_data.json', 'r') as f:
i_data = pd.read_json(f.read(), orient='records', lines=True)
i_data.date = i_data.date.dt.strftime('%Y-%m-%d')
k_data = k_data[[
'date', 'open', 'high', 'low', 'close', 'volume', 'amount',
'outstanding', 'totals', 'adj', 'aprice', 'uprice'
]]
k_data = k_data.rename(columns={"date": "time"})
i_data = i_data[[
'date', 'open', 'high', 'low', 'close', 'volume', 'amount'
]]
i_data = i_data.rename(columns={"date": "time"})
return k_data, d_data, i_data
def init_index_info(self):
index_list = ct.INDEX_DICT.keys()
prefix_index_list = [add_index_prefix(code) for code in index_list]
ret = self.subscriber.subscribe(prefix_index_list, SubType.QUOTE,
self.quote_handler)
if 0 != ret:
self.logger.error("subscribe for index list failed")
return ret
for code in index_list:
if code not in self.index_objs:
self.index_objs[code] = CIndex(code,
should_create_influxdb=True,
should_create_mysqldb=True)
def collect(self):
cdata = dict()
cdata['name'] = list()
cdata['price'] = list()
cdata['volume'] = list()
cdata['amount'] = list()
cdict = self.get_combination_dict()
if 0 == len(cdict): return False
for code in ct.INDEX_DICT:
df_byte = self.redis.get(
CIndex.get_redis_name(CIndex.get_dbname(code)))
if df_byte is None: continue
df = _pickle.loads(df_byte)
price = float(df.last_price.tolist()[0])
p_volume = float(df.volume.tolist()[0])
p_amount = float(df.turnover.tolist()[0])
cdata['name'].append(ct.INDEX_DICT[code])
cdata['price'].append(price)
cdata['volume'].append(p_volume)
cdata['amount'].append(p_amount)
for code in cdict:
key = cdict[code]
df_byte = self.redis.get(Combination.get_redis_name(code))
if df_byte is None: continue
df = _pickle.loads(df_byte)
price = float(df.price.tolist()[0])
p_volume = float(df.volume.tolist()[0])
p_amount = float(df.amount.tolist()[0])
cdata['name'].append(key)
cdata['price'].append(price)
cdata['volume'].append(p_volume)
cdata['amount'].append(p_amount)
df = pd.DataFrame.from_dict(cdata)
df['time'] = datetime.fromtimestamp(time.time()).strftime('%H:%M:%S')
df['date'] = datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d')
self.mysql_client.set(df, self.table)
return True
def init_stock_info(self, cdate=None):
def _set_stock_info(_date, bonus_info, index_info, code_id):
try:
if CStock(code_id).set_k_data(bonus_info, index_info, _date):
self.logger.info("%s set k data success" % code_id)
return (code_id, True)
else:
self.logger.error("%s set k data failed" % code_id)
return (code_id, False)
except Exception as e:
self.logger.error("%s set k data exception:%s" % (code_id, e))
return (code_id, False)
#get stock bonus info
bonus_info = pd.read_csv("/data/tdx/base/bonus.csv",
sep=',',
dtype={
'code': str,
'market': int,
'type': int,
'money': float,
'price': float,
'count': float,
'rate': float,
'date': int
})
index_info = CIndex('000001').get_k_data()
if index_info is None or index_info.empty: return False
df = self.stock_info_client.get()
failed_list = df.code.tolist()
if cdate is None:
cfunc = partial(_set_stock_info, cdate, bonus_info, index_info)
return process_concurrent_run(cfunc, failed_list, num=5)
else:
succeed = True
start_date = get_day_nday_ago(cdate, num=10, dformat="%Y-%m-%d")
for mdate in get_dates_array(start_date, cdate, asending=True):
if self.cal_client.is_trading_day(mdate):
cfunc = partial(_set_stock_info, mdate, bonus_info,
index_info)
if not process_concurrent_run(cfunc, failed_list, num=500):
succeed = False
return succeed
def init_stock_info(self, cdate=None):
def _set_stock_info(mdate, bonus_info, index_info, code_id):
try:
if CStock(code_id).set_k_data(bonus_info, index_info, mdate):
self.logger.info("%s set k data success for date:%s",
code_id, mdate)
return (code_id, True)
else:
self.logger.error("%s set k data failed for date:%s",
code_id, mdate)
return (code_id, False)
except Exception as e:
self.logger.error("%s set k data for date %s exception:%s",
code_id, mdate, e)
return (code_id, False)
#get stock bonus info
bonus_info = pd.read_csv("/data/tdx/base/bonus.csv",
sep=',',
dtype={
'code': str,
'market': int,
'type': int,
'money': float,
'price': float,
'count': float,
'rate': float,
'date': int
})
index_info = CIndex('000001').get_k_data()
if index_info is None or index_info.empty: return False
df = self.stock_info_client.get()
if df.empty: return False
failed_list = df.code.tolist()
if cdate is None:
cfunc = partial(_set_stock_info, cdate, bonus_info, index_info)
return process_concurrent_run(cfunc, failed_list, num=8)
else:
cfunc = partial(_set_stock_info, cdate, bonus_info, index_info)
succeed = True
if not process_concurrent_run(cfunc, failed_list, num=8):
succeed = False
return succeed
def create_obj(self, code):
try:
CIndex(code, should_create_influxdb = True, should_create_mysqldb = True)
return (code, True)
except Exception as e:
return (code, False)
def get_index_data(start_date, end_date, index_code):
iobj = CIndex(index_code, dbinfo = ct.OUT_DB_INFO, redis_host = '127.0.0.1')
i_data = iobj.get_k_data_in_range(start_date, end_date)
i_data = i_data[['open', 'high', 'low', 'close', 'volume', 'amount', 'date']]
return i_data
# -*- coding: utf-8 -*-
import os
import sys
import pandas as pd
from os.path import abspath, dirname
sys.path.insert(0, dirname(dirname(dirname(abspath(__file__)))))
from cindex import CIndex
pd.set_option('display.max_columns', None)
pd.set_option('display.max_rows', None)
code = '399006'
start_date = '2019-01-01'
end_date = '2019-08-11'
df = CIndex(code).get_val_data()
df = df.sort_values(by=['date'], ascending=True)
df = df.reset_index(drop=True)
print(df.loc[(df.date > start_date) & (df.date < end_date)])
from ccalendar import CCalendar
from cstock_info import CStockInfo
from base.cdate import get_day_nday_ago, get_dates_array
if __name__ == '__main__':
num = 5500
end_date = '2019-08-13'
stock_info_client = CStockInfo()
df = stock_info_client.get()
code_list = df['code'].tolist()
name_list = df['name'].tolist()
code2namedict = dict(zip(code_list, name_list))
start_date = get_day_nday_ago(end_date, num=num, dformat="%Y-%m-%d")
date_array = get_dates_array(start_date, end_date)
auth('18701683341', '52448oo78')
for code in [
'000001', '000016', '000300', '000905', '399001', '399005',
'399673'
]:
obj = CIndex(code)
for mdate in date_array:
if CCalendar.is_trading_day(mdate):
table_name = obj.get_components_table_name(mdate)
if obj.is_table_exists(table_name):
obj.mysql_client.delete(table_name)
for mdate in date_array:
if CCalendar.is_trading_day(mdate):
if not obj.set_components_data_from_joinquant(
code2namedict, mdate):
print("{} for {} set failed".format(code, mdate))
def _set_index_info(code_id):
_obj = self.index_objs[
code_id] if code_id in self.index_objs else CIndex(code_id)
return (code_id, _obj.set_components_data(cdate))
return new_tmp_df
if __name__ == '__main__':
if not os.path.exists('norm.json'):
creview = CReivew(ct.DB_INFO)
start_date = '2018-02-09'
end_date = '2018-09-10'
stock_info = CStockInfo.get()
stock_info = stock_info[['code', 'name', 'industry', 'timeToMarket']]
stock_info = stock_info[(stock_info.timeToMarket < 20180327)
& (stock_info.timeToMarket > 0)]
if not os.path.exists('index.json'):
#上证指数的数据
logger.info("start get index data")
szzs_df = CIndex('000001').get_k_data_in_range(
start_date, end_date)
szzs_df = szzs_df.sort_values(by='date', ascending=True)
szzs_df['code'] = 'i000001'
szzs_df['name'] = "上证指数"
szzs_df['industry'] = "所有"
szzs_df['preclose'] = szzs_df['close'].shift(1)
szzs_df = szzs_df[szzs_df.date != start_date]
szzs_df['pchange'] = 100 * (szzs_df.close -
szzs_df.preclose) / szzs_df.preclose
#write data to json file
with open('index.json', 'w') as f:
f.write(szzs_df.to_json(orient='records', lines=True))
else:
logger.info("begin read index file")
with open('index.json', 'r') as f:
szzs_df = pd.read_json(f.read(),
class BullStockRatio:
def __init__(self, index_code, dbinfo=ct.DB_INFO, redis_host=None):
self.dbinfo = dbinfo
self.index_code = index_code
self.index_obj = CIndex(index_code,
dbinfo=self.dbinfo,
redis_host=redis_host)
self.db_name = self.index_obj.get_dbname(index_code)
self.logger = getLogger(__name__)
self.ris = RIndexStock(dbinfo, redis_host)
self.bull_stock_ratio_table = self.get_table_name()
self.redis = create_redis_obj(
) if redis_host is None else create_redis_obj(redis_host)
self.mysql_client = CMySQL(self.dbinfo,
dbname=self.db_name,
iredis=self.redis)
if not self.create(): raise Exception("create emotion table failed")
def delete(self):
self.mysql_client.delete(self.bull_stock_ratio_table)
def get_table_name(self):
return "%s_%s" % (self.db_name, ct.BULLSTOCKRATIO_TABLE)
def create(self):
if self.bull_stock_ratio_table not in self.mysql_client.get_all_tables(
):
sql = 'create table if not exists %s(date varchar(10) not null, ratio float, PRIMARY KEY (date))' % self.bull_stock_ratio_table
if not self.mysql_client.create(sql, self.bull_stock_ratio_table):
return False
return True
def is_date_exists(self, table_name, cdate):
if self.redis.exists(table_name):
return cdate in set(
str(tdate, encoding=ct.UTF8)
for tdate in self.redis.smembers(table_name))
return False
def get_k_data_between(self, start_date, end_date):
sql = "select * from %s where date between \"%s\" and \"%s\"" % (
self.get_table_name(), start_date, end_date)
return self.mysql_client.get(sql)
def get_components(self, cdate):
df = self.index_obj.get_components_data(cdate)
if df is None: return list()
if df.empty: return list()
if self.index_code == '000001': df = df[df.code.str.startswith('6')]
return df.code.tolist()
def get_data(self, cdate):
return self.ris.get_data(cdate)
def update(self, end_date=None, num=30):
if end_date is None: end_date = datetime.now().strftime('%Y-%m-%d')
#start_date = "1997-12-30"
start_date = get_day_nday_ago(end_date, num=num, dformat="%Y-%m-%d")
succeed = True
code_list = self.get_components(end_date)
if 0 == len(code_list):
self.logger.error("%s code_list for %s is empty" %
(end_date, self.index_code))
return False
for mdate in get_dates_array(start_date, end_date):
if CCalendar.is_trading_day(mdate, redis=self.redis):
if not self.set_ratio(code_list, mdate):
self.logger.error("set %s score for %s set failed" %
(self.index_code, mdate))
succeed = False
return succeed
def get_profit_stocks(self, df):
data = df[df.profit >= 0]
return data.code.tolist()
def set_ratio(self,
now_code_list,
cdate=datetime.now().strftime('%Y-%m-%d')):
if self.is_date_exists(self.bull_stock_ratio_table, cdate):
self.logger.debug("existed date:%s, date:%s" %
(self.bull_stock_ratio_table, cdate))
return True
code_list = self.get_components(cdate)
if len(code_list) == 0: code_list = now_code_list
df = self.get_data(cdate)
if df is None: return False
if df.empty: return False
df = df[df.code.isin(code_list)]
if df.empty: return True
profit_code_list = self.get_profit_stocks(df)
bull_stock_num = len(profit_code_list)
bull_ration = 100 * bull_stock_num / len(df)
data = {'date': [cdate], 'ratio': [bull_ration]}
df = pd.DataFrame.from_dict(data)
if self.mysql_client.set(df, self.bull_stock_ratio_table):
return self.redis.sadd(self.bull_stock_ratio_table, cdate)
return False
raise Exception(
"tmp_df.volume.sum() is not equal to outstanding")
tmp_df = tmp_df.reset_index(drop=True)
pre_outstanding = outstanding
df = df.append(tmp_df)
df = df[df.volume != 0]
df = df.reset_index(drop=True)
return df
if __name__ == '__main__':
from cindex import CIndex
from cstock import CStock
cdate = None
cstock = CStock('601318')
index_info = CIndex('000001').get_k_data(cdate)
bonus_info = pd.read_csv("/data/tdx/base/bonus.csv",
sep=',',
dtype={
'code': str,
'market': int,
'type': int,
'money': float,
'price': float,
'count': float,
'rate': float,
'date': int
})
quantity_change_info, price_change_info = cstock.collect_right_info(
bonus_info)
base_color = '#e6daa6'
name = "shanghai_%s_image" % cdate
i_data = i_data[[
'date', 'open', 'high', 'low', 'close', 'volume', 'amount'
]]
fig, index_ax = plt.subplots(figsize=(16, 10))
fig.subplots_adjust(bottom=0.2)
candlestick2_ochl(index_ax,
i_data['open'],
i_data['close'],
i_data['high'],
i_data['low'],
width=1.0,
colorup='r',
colordown='g')
index_ax.set_ylabel(name)
index_ax.set_xticks(range(0, len(i_data['date']), 10))
plt.plot()
fig.autofmt_xdate()
plt.show()
index_obj = CIndex('000001', dbinfo=ct.OUT_DB_INFO, redis_host='127.0.0.1')
df = index_obj.get_k_data()
cdates = df[(df.pchange > 4) & (df.date > '1999-01-01')].date.tolist()
for mdate in cdates:
start_date = mdate
end_date = get_day_nday_after(start_date, num=40, dformat="%Y-%m-%d")
index_data = index_obj.get_k_data_in_range(start_date, end_date)
plot_index(start_date, index_data)
def get_overview_data():
#cdate = (datetime.now() - timedelta(days = 1)).strftime('%Y-%m-%d')
cdate = datetime.now().strftime('%Y-%m-%d')
code_dict = {
'000001': '上证指数',
'399001': '深证成指',
'399006': '创业板指',
'000016': '上证50',
'000905': '中证500'
}
overview_dict = {
'上证指数': {
'icon': 'dollar-sign',
'value': 0,
'change': 0,
'label': '上证指数',
'cdate': cdate
},
'深证成指': {
'icon': 'dollar-sign',
'value': 0,
'change': 0,
'label': '深证成指',
'cdate': cdate
},
'创业板指': {
'icon': 'dollar-sign',
'value': 0,
'change': 0,
'label': '创业板指',
'cdate': cdate
},
'上证50': {
'icon': 'dollar-sign',
'value': 0,
'change': 0,
'label': '上证50',
'cdate': cdate
},
'中证500': {
'icon': 'dollar-sign',
'value': 0,
'change': 0,
'label': '中证500',
'cdate': cdate
},
'科创板指': {
'icon': 'dollar-sign',
'value': 0,
'change': 0,
'label': '科创板指',
'cdate': cdate
}
}
for code, name in code_dict.items():
df = CIndex(code).get_k_data(cdate)
if df is None: return overview_dict
if df.empty:
overview_dict[name]['value'] = 0
overview_dict[name]['change'] = 0
else:
row = df.to_dict('records')[0]
overview_dict[name]['value'] = round(row['close'], 2)
overview_dict[name]['change'] = round(row['pchange'], 2)
return overview_dict
def create_index_figure_column(code, dtype, start_date, end_date):
delta_days = (end_date - start_date).days
if delta_days <= 0: return None
start_date = start_date.strftime('%Y-%m-%d')
end_date = end_date.strftime('%Y-%m-%d')
if dtype == 'K线图':
obj = CIndex(code)
df = obj.get_k_data_in_range(start_date, end_date)
source = ColumnDataSource(df)
mapper = linear_cmap(field_name='pchange',
palette=['red', 'green'],
low=0,
high=0,
low_color='green',
high_color='red')
p = figure(plot_height=500,
plot_width=1200,
tools="",
toolbar_location=None,
sizing_mode="scale_both",
x_range=(0, len(df)))
p.xaxis.axis_label = "时间"
p.yaxis.axis_label = "点数"
p.xaxis.major_label_overrides = {
i: mdate
for i, mdate in enumerate(df['date'])
}
p.segment(x0='index',
y0='low',
x1='date',
y1='high',
line_width=2,
color='black',
source=source)
p.vbar(x='index',
bottom='open',
top='close',
width=50 / delta_days,
color=mapper,
source=source)
p.add_tools(HoverTool(tooltips=[("数量", "@close"), ("时间", "@date")]))
volume_p = figure(plot_height=150,
plot_width=1200,
tools="",
toolbar_location=None,
sizing_mode="scale_both")
volume_p.x_range = p.x_range
volume_p.vbar(x='date',
top='volume',
width=50 / delta_days,
color=mapper,
source=source)
return column(p, volume_p)
else:
obj = BullStockRatio(code)
df = obj.get_k_data_between(start_date, end_date)
source = ColumnDataSource(df)
p = figure(plot_height=500,
plot_width=1200,
tools="",
toolbar_location=None,
sizing_mode="scale_both")
p.xaxis.axis_label = "时间"
p.yaxis.axis_label = "比例"
p.line(x='index',
y='ratio',
line_width=3,
line_alpha=1.0,
source=source)
p.xaxis.major_label_overrides = {
i: mdate
for i, mdate in enumerate(df['date'])
}
p.add_tools(HoverTool(tooltips=[("比例", "@ratio"), ("时间", "@date")]))
return column(p)
def get_industry_data(self, cdate, code):
return (code, CIndex(code).get_k_data(cdate))
def get_index_df(self, code, start_date, end_date):
cindex_client = CIndex(code)
df = cindex_client.get_k_data_in_range(start_date, end_date)
df['time'] = df.index.tolist()
df = df[['time', 'open', 'high', 'low', 'close', 'volume', 'amount', 'date']]
return df
def _set_industry_info(cdate, code_id):
return (code_id, CIndex(code_id).set_k_data(cdate))
from os.path import abspath, dirname
sys.path.insert(0, dirname(dirname(abspath(__file__))))
import const as ct
from cindex import CIndex
from jqdatasdk import auth
from ccalendar import CCalendar
from cstock_info import CStockInfo
from base.cdate import get_day_nday_ago, get_dates_array
if __name__ == '__main__':
num = 5500
end_date = '2019-08-13'
stock_info_client = CStockInfo()
df = stock_info_client.get()
code_list = df['code'].tolist()
name_list = df['name'].tolist()
code2namedict = dict(zip(code_list, name_list))
start_date = get_day_nday_ago(end_date, num = num, dformat = "%Y-%m-%d")
date_array = get_dates_array(start_date, end_date)
auth('18701683341', '52448oo78')
for code in ['399006']:
obj = CIndex(code)
#for mdate in date_array:
# if CCalendar.is_trading_day(mdate):
# table_name = obj.get_components_table_name(mdate)
# if obj.is_table_exists(table_name): obj.mysql_client.delete(table_name)
for mdate in date_array:
if CCalendar.is_trading_day(mdate):
if not obj.set_components_data_from_joinquant(code2namedict, mdate):
print("{} for {} set failed".format(code, mdate))
