def get_index_data(cdate): df = pd.DataFrame() for code, name in ct.TDX_INDEX_DICT.items(): data = CIndex(code).get_k_data(cdate) data['name'] = name data['code'] = code df = df.append(data) df = df.reset_index(drop = True) return df
def get_index_data(self, _date): df = pd.DataFrame() for code, name in ct.TDX_INDEX_DICT.items(): self.mysql_client.changedb(CIndex.get_dbname(code)) data = self.mysql_client.get( "select * from day where date=\"%s\";" % _date) data['name'] = name df = df.append(data) self.mysql_client.changedb() return df
def get_industry_data(self, _date): df = pd.DataFrame() df_info = IndustryInfo.get() for _, code in df_info.code.iteritems(): data = CIndex(code).get_k_data(date=_date) df = df.append(data) df = df.reset_index(drop=True) df['name'] = df_info['name'] df = df.sort_values(by='amount', ascending=False) df = df.reset_index(drop=True) return df
def read_data(self): if not os.path.exists('i_data.json'): obj = CStock(self.code, redis_host='127.0.0.1') k_data = obj.get_k_data() k_data.date = pd.to_datetime(k_data.date).dt.strftime('%Y-%m-%d') with open('k_data.json', 'w') as f: f.write(k_data.to_json(orient='records', lines=True)) d_data = obj.get_chip_distribution() with open('d_data.json', 'w') as f: f.write(d_data.to_json(orient='records', lines=True)) iobj = CIndex(self.index_code) i_data = iobj.get_k_data() cdates = k_data.date.tolist() i_data = i_data.loc[i_data.date.isin(cdates)] i_data = i_data.reset_index(drop=True) with open('i_data.json', 'w') as f: f.write(i_data.to_json(orient='records', lines=True)) else: with open('k_data.json', 'r') as f: k_data = pd.read_json(f.read(), orient='records', lines=True) k_data.date = k_data.date.dt.strftime('%Y-%m-%d') with open('d_data.json', 'r') as f: d_data = pd.read_json(f.read(), orient='records', lines=True) with open('i_data.json', 'r') as f: i_data = pd.read_json(f.read(), orient='records', lines=True) i_data.date = i_data.date.dt.strftime('%Y-%m-%d') k_data = k_data[[ 'date', 'open', 'high', 'low', 'close', 'volume', 'amount', 'outstanding', 'totals', 'adj', 'aprice', 'uprice' ]] k_data = k_data.rename(columns={"date": "time"}) i_data = i_data[[ 'date', 'open', 'high', 'low', 'close', 'volume', 'amount' ]] i_data = i_data.rename(columns={"date": "time"}) return k_data, d_data, i_data
def init_index_info(self): index_list = ct.INDEX_DICT.keys() prefix_index_list = [add_index_prefix(code) for code in index_list] ret = self.subscriber.subscribe(prefix_index_list, SubType.QUOTE, self.quote_handler) if 0 != ret: self.logger.error("subscribe for index list failed") return ret for code in index_list: if code not in self.index_objs: self.index_objs[code] = CIndex(code, should_create_influxdb=True, should_create_mysqldb=True)
def collect(self): cdata = dict() cdata['name'] = list() cdata['price'] = list() cdata['volume'] = list() cdata['amount'] = list() cdict = self.get_combination_dict() if 0 == len(cdict): return False for code in ct.INDEX_DICT: df_byte = self.redis.get( CIndex.get_redis_name(CIndex.get_dbname(code))) if df_byte is None: continue df = _pickle.loads(df_byte) price = float(df.last_price.tolist()[0]) p_volume = float(df.volume.tolist()[0]) p_amount = float(df.turnover.tolist()[0]) cdata['name'].append(ct.INDEX_DICT[code]) cdata['price'].append(price) cdata['volume'].append(p_volume) cdata['amount'].append(p_amount) for code in cdict: key = cdict[code] df_byte = self.redis.get(Combination.get_redis_name(code)) if df_byte is None: continue df = _pickle.loads(df_byte) price = float(df.price.tolist()[0]) p_volume = float(df.volume.tolist()[0]) p_amount = float(df.amount.tolist()[0]) cdata['name'].append(key) cdata['price'].append(price) cdata['volume'].append(p_volume) cdata['amount'].append(p_amount) df = pd.DataFrame.from_dict(cdata) df['time'] = datetime.fromtimestamp(time.time()).strftime('%H:%M:%S') df['date'] = datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d') self.mysql_client.set(df, self.table) return True
def init_stock_info(self, cdate=None): def _set_stock_info(_date, bonus_info, index_info, code_id): try: if CStock(code_id).set_k_data(bonus_info, index_info, _date): self.logger.info("%s set k data success" % code_id) return (code_id, True) else: self.logger.error("%s set k data failed" % code_id) return (code_id, False) except Exception as e: self.logger.error("%s set k data exception:%s" % (code_id, e)) return (code_id, False) #get stock bonus info bonus_info = pd.read_csv("/data/tdx/base/bonus.csv", sep=',', dtype={ 'code': str, 'market': int, 'type': int, 'money': float, 'price': float, 'count': float, 'rate': float, 'date': int }) index_info = CIndex('000001').get_k_data() if index_info is None or index_info.empty: return False df = self.stock_info_client.get() failed_list = df.code.tolist() if cdate is None: cfunc = partial(_set_stock_info, cdate, bonus_info, index_info) return process_concurrent_run(cfunc, failed_list, num=5) else: succeed = True start_date = get_day_nday_ago(cdate, num=10, dformat="%Y-%m-%d") for mdate in get_dates_array(start_date, cdate, asending=True): if self.cal_client.is_trading_day(mdate): cfunc = partial(_set_stock_info, mdate, bonus_info, index_info) if not process_concurrent_run(cfunc, failed_list, num=500): succeed = False return succeed
def init_stock_info(self, cdate=None): def _set_stock_info(mdate, bonus_info, index_info, code_id): try: if CStock(code_id).set_k_data(bonus_info, index_info, mdate): self.logger.info("%s set k data success for date:%s", code_id, mdate) return (code_id, True) else: self.logger.error("%s set k data failed for date:%s", code_id, mdate) return (code_id, False) except Exception as e: self.logger.error("%s set k data for date %s exception:%s", code_id, mdate, e) return (code_id, False) #get stock bonus info bonus_info = pd.read_csv("/data/tdx/base/bonus.csv", sep=',', dtype={ 'code': str, 'market': int, 'type': int, 'money': float, 'price': float, 'count': float, 'rate': float, 'date': int }) index_info = CIndex('000001').get_k_data() if index_info is None or index_info.empty: return False df = self.stock_info_client.get() if df.empty: return False failed_list = df.code.tolist() if cdate is None: cfunc = partial(_set_stock_info, cdate, bonus_info, index_info) return process_concurrent_run(cfunc, failed_list, num=8) else: cfunc = partial(_set_stock_info, cdate, bonus_info, index_info) succeed = True if not process_concurrent_run(cfunc, failed_list, num=8): succeed = False return succeed
def create_obj(self, code): try: CIndex(code, should_create_influxdb = True, should_create_mysqldb = True) return (code, True) except Exception as e: return (code, False)
def get_index_data(start_date, end_date, index_code): iobj = CIndex(index_code, dbinfo = ct.OUT_DB_INFO, redis_host = '127.0.0.1') i_data = iobj.get_k_data_in_range(start_date, end_date) i_data = i_data[['open', 'high', 'low', 'close', 'volume', 'amount', 'date']] return i_data
# -*- coding: utf-8 -*- import os import sys import pandas as pd from os.path import abspath, dirname sys.path.insert(0, dirname(dirname(dirname(abspath(__file__))))) from cindex import CIndex pd.set_option('display.max_columns', None) pd.set_option('display.max_rows', None) code = '399006' start_date = '2019-01-01' end_date = '2019-08-11' df = CIndex(code).get_val_data() df = df.sort_values(by=['date'], ascending=True) df = df.reset_index(drop=True) print(df.loc[(df.date > start_date) & (df.date < end_date)])
from ccalendar import CCalendar from cstock_info import CStockInfo from base.cdate import get_day_nday_ago, get_dates_array if __name__ == '__main__': num = 5500 end_date = '2019-08-13' stock_info_client = CStockInfo() df = stock_info_client.get() code_list = df['code'].tolist() name_list = df['name'].tolist() code2namedict = dict(zip(code_list, name_list)) start_date = get_day_nday_ago(end_date, num=num, dformat="%Y-%m-%d") date_array = get_dates_array(start_date, end_date) auth('18701683341', '52448oo78') for code in [ '000001', '000016', '000300', '000905', '399001', '399005', '399673' ]: obj = CIndex(code) for mdate in date_array: if CCalendar.is_trading_day(mdate): table_name = obj.get_components_table_name(mdate) if obj.is_table_exists(table_name): obj.mysql_client.delete(table_name) for mdate in date_array: if CCalendar.is_trading_day(mdate): if not obj.set_components_data_from_joinquant( code2namedict, mdate): print("{} for {} set failed".format(code, mdate))
def _set_index_info(code_id): _obj = self.index_objs[ code_id] if code_id in self.index_objs else CIndex(code_id) return (code_id, _obj.set_components_data(cdate))
return new_tmp_df if __name__ == '__main__': if not os.path.exists('norm.json'): creview = CReivew(ct.DB_INFO) start_date = '2018-02-09' end_date = '2018-09-10' stock_info = CStockInfo.get() stock_info = stock_info[['code', 'name', 'industry', 'timeToMarket']] stock_info = stock_info[(stock_info.timeToMarket < 20180327) & (stock_info.timeToMarket > 0)] if not os.path.exists('index.json'): #上证指数的数据 logger.info("start get index data") szzs_df = CIndex('000001').get_k_data_in_range( start_date, end_date) szzs_df = szzs_df.sort_values(by='date', ascending=True) szzs_df['code'] = 'i000001' szzs_df['name'] = "上证指数" szzs_df['industry'] = "所有" szzs_df['preclose'] = szzs_df['close'].shift(1) szzs_df = szzs_df[szzs_df.date != start_date] szzs_df['pchange'] = 100 * (szzs_df.close - szzs_df.preclose) / szzs_df.preclose #write data to json file with open('index.json', 'w') as f: f.write(szzs_df.to_json(orient='records', lines=True)) else: logger.info("begin read index file") with open('index.json', 'r') as f: szzs_df = pd.read_json(f.read(),
class BullStockRatio: def __init__(self, index_code, dbinfo=ct.DB_INFO, redis_host=None): self.dbinfo = dbinfo self.index_code = index_code self.index_obj = CIndex(index_code, dbinfo=self.dbinfo, redis_host=redis_host) self.db_name = self.index_obj.get_dbname(index_code) self.logger = getLogger(__name__) self.ris = RIndexStock(dbinfo, redis_host) self.bull_stock_ratio_table = self.get_table_name() self.redis = create_redis_obj( ) if redis_host is None else create_redis_obj(redis_host) self.mysql_client = CMySQL(self.dbinfo, dbname=self.db_name, iredis=self.redis) if not self.create(): raise Exception("create emotion table failed") def delete(self): self.mysql_client.delete(self.bull_stock_ratio_table) def get_table_name(self): return "%s_%s" % (self.db_name, ct.BULLSTOCKRATIO_TABLE) def create(self): if self.bull_stock_ratio_table not in self.mysql_client.get_all_tables( ): sql = 'create table if not exists %s(date varchar(10) not null, ratio float, PRIMARY KEY (date))' % self.bull_stock_ratio_table if not self.mysql_client.create(sql, self.bull_stock_ratio_table): return False return True def is_date_exists(self, table_name, cdate): if self.redis.exists(table_name): return cdate in set( str(tdate, encoding=ct.UTF8) for tdate in self.redis.smembers(table_name)) return False def get_k_data_between(self, start_date, end_date): sql = "select * from %s where date between \"%s\" and \"%s\"" % ( self.get_table_name(), start_date, end_date) return self.mysql_client.get(sql) def get_components(self, cdate): df = self.index_obj.get_components_data(cdate) if df is None: return list() if df.empty: return list() if self.index_code == '000001': df = df[df.code.str.startswith('6')] return df.code.tolist() def get_data(self, cdate): return self.ris.get_data(cdate) def update(self, end_date=None, num=30): if end_date is None: end_date = datetime.now().strftime('%Y-%m-%d') #start_date = "1997-12-30" start_date = get_day_nday_ago(end_date, num=num, dformat="%Y-%m-%d") succeed = True code_list = self.get_components(end_date) if 0 == len(code_list): self.logger.error("%s code_list for %s is empty" % (end_date, self.index_code)) return False for mdate in get_dates_array(start_date, end_date): if CCalendar.is_trading_day(mdate, redis=self.redis): if not self.set_ratio(code_list, mdate): self.logger.error("set %s score for %s set failed" % (self.index_code, mdate)) succeed = False return succeed def get_profit_stocks(self, df): data = df[df.profit >= 0] return data.code.tolist() def set_ratio(self, now_code_list, cdate=datetime.now().strftime('%Y-%m-%d')): if self.is_date_exists(self.bull_stock_ratio_table, cdate): self.logger.debug("existed date:%s, date:%s" % (self.bull_stock_ratio_table, cdate)) return True code_list = self.get_components(cdate) if len(code_list) == 0: code_list = now_code_list df = self.get_data(cdate) if df is None: return False if df.empty: return False df = df[df.code.isin(code_list)] if df.empty: return True profit_code_list = self.get_profit_stocks(df) bull_stock_num = len(profit_code_list) bull_ration = 100 * bull_stock_num / len(df) data = {'date': [cdate], 'ratio': [bull_ration]} df = pd.DataFrame.from_dict(data) if self.mysql_client.set(df, self.bull_stock_ratio_table): return self.redis.sadd(self.bull_stock_ratio_table, cdate) return False
raise Exception( "tmp_df.volume.sum() is not equal to outstanding") tmp_df = tmp_df.reset_index(drop=True) pre_outstanding = outstanding df = df.append(tmp_df) df = df[df.volume != 0] df = df.reset_index(drop=True) return df if __name__ == '__main__': from cindex import CIndex from cstock import CStock cdate = None cstock = CStock('601318') index_info = CIndex('000001').get_k_data(cdate) bonus_info = pd.read_csv("/data/tdx/base/bonus.csv", sep=',', dtype={ 'code': str, 'market': int, 'type': int, 'money': float, 'price': float, 'count': float, 'rate': float, 'date': int }) quantity_change_info, price_change_info = cstock.collect_right_info( bonus_info)
base_color = '#e6daa6' name = "shanghai_%s_image" % cdate i_data = i_data[[ 'date', 'open', 'high', 'low', 'close', 'volume', 'amount' ]] fig, index_ax = plt.subplots(figsize=(16, 10)) fig.subplots_adjust(bottom=0.2) candlestick2_ochl(index_ax, i_data['open'], i_data['close'], i_data['high'], i_data['low'], width=1.0, colorup='r', colordown='g') index_ax.set_ylabel(name) index_ax.set_xticks(range(0, len(i_data['date']), 10)) plt.plot() fig.autofmt_xdate() plt.show() index_obj = CIndex('000001', dbinfo=ct.OUT_DB_INFO, redis_host='127.0.0.1') df = index_obj.get_k_data() cdates = df[(df.pchange > 4) & (df.date > '1999-01-01')].date.tolist() for mdate in cdates: start_date = mdate end_date = get_day_nday_after(start_date, num=40, dformat="%Y-%m-%d") index_data = index_obj.get_k_data_in_range(start_date, end_date) plot_index(start_date, index_data)
def get_overview_data(): #cdate = (datetime.now() - timedelta(days = 1)).strftime('%Y-%m-%d') cdate = datetime.now().strftime('%Y-%m-%d') code_dict = { '000001': '上证指数', '399001': '深证成指', '399006': '创业板指', '000016': '上证50', '000905': '中证500' } overview_dict = { '上证指数': { 'icon': 'dollar-sign', 'value': 0, 'change': 0, 'label': '上证指数', 'cdate': cdate }, '深证成指': { 'icon': 'dollar-sign', 'value': 0, 'change': 0, 'label': '深证成指', 'cdate': cdate }, '创业板指': { 'icon': 'dollar-sign', 'value': 0, 'change': 0, 'label': '创业板指', 'cdate': cdate }, '上证50': { 'icon': 'dollar-sign', 'value': 0, 'change': 0, 'label': '上证50', 'cdate': cdate }, '中证500': { 'icon': 'dollar-sign', 'value': 0, 'change': 0, 'label': '中证500', 'cdate': cdate }, '科创板指': { 'icon': 'dollar-sign', 'value': 0, 'change': 0, 'label': '科创板指', 'cdate': cdate } } for code, name in code_dict.items(): df = CIndex(code).get_k_data(cdate) if df is None: return overview_dict if df.empty: overview_dict[name]['value'] = 0 overview_dict[name]['change'] = 0 else: row = df.to_dict('records')[0] overview_dict[name]['value'] = round(row['close'], 2) overview_dict[name]['change'] = round(row['pchange'], 2) return overview_dict
def create_index_figure_column(code, dtype, start_date, end_date): delta_days = (end_date - start_date).days if delta_days <= 0: return None start_date = start_date.strftime('%Y-%m-%d') end_date = end_date.strftime('%Y-%m-%d') if dtype == 'K线图': obj = CIndex(code) df = obj.get_k_data_in_range(start_date, end_date) source = ColumnDataSource(df) mapper = linear_cmap(field_name='pchange', palette=['red', 'green'], low=0, high=0, low_color='green', high_color='red') p = figure(plot_height=500, plot_width=1200, tools="", toolbar_location=None, sizing_mode="scale_both", x_range=(0, len(df))) p.xaxis.axis_label = "时间" p.yaxis.axis_label = "点数" p.xaxis.major_label_overrides = { i: mdate for i, mdate in enumerate(df['date']) } p.segment(x0='index', y0='low', x1='date', y1='high', line_width=2, color='black', source=source) p.vbar(x='index', bottom='open', top='close', width=50 / delta_days, color=mapper, source=source) p.add_tools(HoverTool(tooltips=[("数量", "@close"), ("时间", "@date")])) volume_p = figure(plot_height=150, plot_width=1200, tools="", toolbar_location=None, sizing_mode="scale_both") volume_p.x_range = p.x_range volume_p.vbar(x='date', top='volume', width=50 / delta_days, color=mapper, source=source) return column(p, volume_p) else: obj = BullStockRatio(code) df = obj.get_k_data_between(start_date, end_date) source = ColumnDataSource(df) p = figure(plot_height=500, plot_width=1200, tools="", toolbar_location=None, sizing_mode="scale_both") p.xaxis.axis_label = "时间" p.yaxis.axis_label = "比例" p.line(x='index', y='ratio', line_width=3, line_alpha=1.0, source=source) p.xaxis.major_label_overrides = { i: mdate for i, mdate in enumerate(df['date']) } p.add_tools(HoverTool(tooltips=[("比例", "@ratio"), ("时间", "@date")])) return column(p)
def get_industry_data(self, cdate, code): return (code, CIndex(code).get_k_data(cdate))
def get_index_df(self, code, start_date, end_date): cindex_client = CIndex(code) df = cindex_client.get_k_data_in_range(start_date, end_date) df['time'] = df.index.tolist() df = df[['time', 'open', 'high', 'low', 'close', 'volume', 'amount', 'date']] return df
def _set_industry_info(cdate, code_id): return (code_id, CIndex(code_id).set_k_data(cdate))
from os.path import abspath, dirname sys.path.insert(0, dirname(dirname(abspath(__file__)))) import const as ct from cindex import CIndex from jqdatasdk import auth from ccalendar import CCalendar from cstock_info import CStockInfo from base.cdate import get_day_nday_ago, get_dates_array if __name__ == '__main__': num = 5500 end_date = '2019-08-13' stock_info_client = CStockInfo() df = stock_info_client.get() code_list = df['code'].tolist() name_list = df['name'].tolist() code2namedict = dict(zip(code_list, name_list)) start_date = get_day_nday_ago(end_date, num = num, dformat = "%Y-%m-%d") date_array = get_dates_array(start_date, end_date) auth('18701683341', '52448oo78') for code in ['399006']: obj = CIndex(code) #for mdate in date_array: # if CCalendar.is_trading_day(mdate): # table_name = obj.get_components_table_name(mdate) # if obj.is_table_exists(table_name): obj.mysql_client.delete(table_name) for mdate in date_array: if CCalendar.is_trading_day(mdate): if not obj.set_components_data_from_joinquant(code2namedict, mdate): print("{} for {} set failed".format(code, mdate))