def set_components_data_from_joinquant(self, basic_dict, cdate):
table_name = self.get_components_table_name(cdate)
if not self.is_table_exists(table_name):
if not self.create_components_table(table_name):
logger.error("create components table failed")
return False
if self.is_date_exists(table_name, cdate):
logger.debug("existed table:%s, date:%s" % (table_name, cdate))
return True
code = "{}.XSHG".format(self.code) if self.code.startswith("6") else "{}.XSHE".format(self.code)
stocks = get_index_stocks(code, date = cdate)
df = pd.DataFrame(stocks, columns =['code'])
df['code'] = df['code'].str.split('.', expand=True)[0]
if not df[~df.code.isin(basic_dict)].empty:
logger.error("code from juquant is not in basic in data {}".format(df[~df.code.isin(basic_dict)]))
return False
df['name'] = df['code'].apply(lambda x: basic_dict[x])
df['date'] = cdate
if 'wieight' not in df.columns:
df['weight'] = 1/len(df)
if 'flag' not in df.columns:
df['flag'] = 1
df = df.reset_index(drop = True)
if is_df_has_unexpected_data(df):
logger.error("data for %s is not clear" % self.code)
return False
if self.mysql_client.set(df, table_name):
if self.redis.sadd(table_name, cdate): return True
return False
def set_chip_distribution(self, df, zdate=None):
if zdate is None:
time2Market = self.get('timeToMarket')
start_year = int(time2Market / 10000)
end_year = int(datetime.now().strftime('%Y'))
year_list = get_years_between(start_year, end_year)
cfunc = partial(self.set_chip_table, df)
return concurrent_run(cfunc, year_list, num=20)
else:
chip_table = self.get_chip_distribution_table(zdate)
if not self.is_table_exists(chip_table):
if not self.create_chip_table(chip_table):
logger.error("create chip table:%s failed" % chip_table)
return False
if self.is_date_exists(chip_table, zdate):
logger.debug("existed chip for code:%s, date:%s" %
(self.code, zdate))
return True
if is_df_has_unexpected_data(df):
logger.error("data for %s is not clear" % self.code)
return False
if self.mysql_client.set(df, chip_table):
if self.redis.sadd(chip_table, zdate):
logger.debug("finish record chip:%s. table:%s" %
(self.code, chip_table))
return True
return False
def set_ticket(self, cdate=None):
cdate = datetime.now().strftime('%Y-%m-%d') if cdate is None else cdate
if not self.has_on_market(cdate):
logger.debug("not on market code:%s, date:%s" % (self.code, cdate))
return True
tick_table = self.get_redis_tick_table(cdate)
if not self.is_table_exists(tick_table):
if not self.create_ticket_table(tick_table):
logger.error("create tick table failed")
return False
if not self.redis.sadd(self.dbname, tick_table):
logger.error("add tick table to redis failed")
return False
if self.is_date_exists(tick_table, cdate):
logger.debug("existed code:%s, date:%s" % (self.code, cdate))
return True
logger.debug("%s read code from file %s" % (self.code, cdate))
df = ts.get_tick_data(self.code, date=cdate)
df_tdx = read_tick(
os.path.join(
ct.TIC_DIR, '%s.tic' %
datetime.strptime(cdate, "%Y-%m-%d").strftime("%Y%m%d")),
self.code)
if not df_tdx.empty:
if df is not None and not df.empty and df.loc[0]['time'].find(
"当天没有数据") == -1:
net_volume = df.volume.sum()
tdx_volume = df_tdx.volume.sum()
if net_volume != tdx_volume:
logger.error(
"code:%s, date:%s, net volume:%s, tdx volume:%s not equal"
% (self.code, cdate, net_volume, tdx_volume))
df = df_tdx
else:
if df is None:
logger.debug("nonedata code:%s, date:%s" % (self.code, cdate))
return True
if df.empty:
logger.debug("emptydata code:%s, date:%s" % (self.code, cdate))
return True
if df.loc[0]['time'].find("当天没有数据") != -1:
logger.debug("nodata code:%s, date:%s" % (self.code, cdate))
return True
df.columns = ['time', 'price', 'cchange', 'volume', 'amount', 'ctype']
logger.debug("merge ticket code:%s date:%s" % (self.code, cdate))
df = self.merge_ticket(df)
df['date'] = cdate
logger.debug("write data code:%s, date:%s, table:%s" %
(self.code, cdate, tick_table))
if is_df_has_unexpected_data(df):
logger.error("data for %s is not clear" % self.code)
return False
if self.mysql_client.set(df, tick_table):
logger.debug("finish record:%s. table:%s" %
(self.code, tick_table))
if self.redis.sadd(tick_table, cdate):
return True
return False
def set_components_data(self, cdate = datetime.now().strftime('%Y-%m-%d')):
table_name = self.get_components_table_name(cdate)
if not self.is_table_exists(table_name):
if not self.create_components_table(table_name):
logger.error("create components table failed")
return False
if self.is_date_exists(table_name, cdate):
logger.debug("existed table:%s, date:%s" % (table_name, cdate))
return True
url = self.INDEX_URLS[self.code][0]
columns = self.INDEX_URLS[self.code][1]
column_names = self.INDEX_URLS[self.code][2]
df = smart_get(pd.read_excel, url, usecols = columns)
if df is None:
logger.error("data for %s is empty" % self.code)
return False
df.columns = column_names
df.code = df.code.astype('str').str.zfill(6)
df['date'] = cdate
if 'wieight' not in df.columns:
df['weight'] = 1/len(df)
if 'flag' not in df.columns:
df['flag'] = 1
df = df.reset_index(drop = True)
if is_df_has_unexpected_data(df):
logger.error("data for %s is not clear" % self.code)
return False
if self.mysql_client.set(df, table_name):
if self.redis.sadd(table_name, cdate): return True
return False
def get_data(code, start_date, end_date):
cstock_obj = CIndex(code, dbinfo=ct.OUT_DB_INFO, redis_host='127.0.0.1')
data = cstock_obj.get_k_data_in_range(start_date, end_date)
data = data.set_index('date')
if is_df_has_unexpected_data(data): return None
data.index = pd.to_datetime(data.index)
data = data.dropna(how='any')
return data
def set_all_data(self, quantity_change_info, price_change_info,
index_info):
df, _ = self.read()
if df.empty:
logger.error("read empty file for:%s" % self.code)
return False
#modify price and quanity for all split-adjusted share prices
df = self.adjust_share(df, quantity_change_info)
if df.empty:
logger.error("adjust share %s failed" % self.code)
return False
df = self.qfq(df, price_change_info)
if df.empty:
logger.error("qfq %s failed" % self.code)
return False
#transfer data to split-adjusted share prices
df = self.transfer2adjusted(df)
#compute strength relative index
df = self.relative_index_strength(df, index_info)
if df is None:
logger.error("length of code %s is not equal to index." %
self.code)
return False
#set chip distribution
dist_data = self.compute_distribution(df)
if dist_data.empty:
logger.error("%s is empty distribution." % self.code)
return False
if not self.set_chip_distribution(dist_data):
logger.error("store %s distribution failed" % self.code)
return False
df['uprice'] = mac(dist_data, 0)
df['sprice'] = mac(dist_data, 5)
df['mprice'] = mac(dist_data, 13)
df['lprice'] = mac(dist_data, 37)
df = pro_nei_chip(df, dist_data)
if is_df_has_unexpected_data(df):
logger.error("data for %s is not clean." % self.code)
return False
day_table = self.get_day_table()
if not self.mysql_client.delsert(df, day_table):
logger.error("save %s data to mysql failed." % self.code)
return False
self.redis.sadd(day_table, *set(df.date.tolist()))
return True
def get_feed(all_df, codes, start_date, end_date, peried, duaration):
feed = dataFramefeed.Feed()
for code in codes:
data = all_df.loc[all_df.code == code]
data = data.sort_values(by=['date'], ascending=True)
data = data.reset_index(drop=True)
data = data.set_index('date')
if is_df_has_unexpected_data(data): return None, None
data.index = pd.to_datetime(data.index)
data = data.dropna(how='any')
data = atr(data, ndays=duaration)
feed.addBarsFromDataFrame(code, data)
return feed
def set_today_data(self, df, index_df, pre_date, cdate):
day_table = self.get_day_table()
if self.is_date_exists(day_table, cdate):
logger.debug("existed data for code:%s, date:%s" %
(self.code, cdate))
return True
index_df = index_df.loc[index_df.date == cdate]
preday_df = self.get_k_data(date=pre_date)
if preday_df is None:
logger.error("%s get pre date data failed." % self.code)
return False
if preday_df.empty:
logger.error("%s get pre date data empty." % self.code)
return False
df['adj'] = 1.0
df['preclose'] = preday_df['close'][0]
df['totals'] = preday_df['totals'][0]
df['outstanding'] = preday_df['outstanding'][0]
#transfer data to split-adjusted share prices
df = self.transfer2adjusted(df)
df = self.relative_index_strength(df, index_df, cdate)
if df is None: return False
#set chip distribution
dist_df = df.append(preday_df, sort=False)
dist_df = dist_df.sort_values(by='date', ascending=True)
dist_data = self.compute_distribution(dist_df, cdate)
if dist_data.empty:
logger.error("%s chip distribution compute failed." % self.code)
return False
if self.set_chip_distribution(dist_data, zdate=cdate):
df['uprice'] = mac(dist_data, 0)
df['sprice'] = mac(dist_data, 5)
df['mprice'] = mac(dist_data, 13)
df['lprice'] = mac(dist_data, 37)
df = pro_nei_chip(df, dist_data, preday_df, cdate)
if is_df_has_unexpected_data(df):
logger.error("data for %s is not clean." % self.code)
return False
if self.mysql_client.set(df, self.get_day_table()):
return self.redis.sadd(day_table, cdate)
return False
def get_feed(all_df, code, start_date, end_date, peried):
feed = dataFramefeed.Feed()
#cindex_obj = CIndex(code, dbinfo = ct.OUT_DB_INFO, redis_host = '127.0.0.1')
#cstock_obj = CStock(code, dbinfo = ct.OUT_DB_INFO, redis_host = '127.0.0.1')
#data = cstock_obj.get_k_data_in_range(start_date, end_date)
#data = data.set_index('date')
data = all_df.loc[all_df.code == code]
data = data.sort_values(by=['date'], ascending=True)
data = data.reset_index(drop=True)
data = data.set_index('date')
if is_df_has_unexpected_data(data): return None, None
data.index = pd.to_datetime(data.index)
data = data.dropna(how='any')
data = atr(data)
feed.addBarsFromDataFrame(code, data)
return feed
def set_day_data(self, cdate):
table_name = self.get_table_name(cdate)
if not self.is_table_exists(table_name):
if not self.create_table(table_name):
self.logger.error("create tick table failed")
return False
self.redis.sadd(self.dbname, table_name)
if self.is_date_exists(table_name, cdate):
self.logger.debug("existed table:%s, date:%s" % (table_name, cdate))
return True
df = self.generate_all_data(cdate)
if df is None: return False
if is_df_has_unexpected_data(df): return False
if self.mysql_client.set(df, table_name):
self.redis.sadd(table_name, cdate)
return True
return False
def set_day_data(self, cdate, fpath):
day_table = self.get_day_table()
if self.is_date_exists(day_table, cdate):
logger.debug("existed data for code:%s, date:%s" %
(self.code, cdate))
return True
df = self.read(fpath)
if 0 == len(df):
logger.error("no data for %s:%s" % (cdate, self.code))
return False
elif 1 == len(df):
if len(df.loc[df.date == cdate]) == 0:
logger.error("no data:%s for %s" % (cdate, self.code))
return False
df.at[0, 'preclose'] = df.at[0, 'open']
df.at[0,
'pchange'] = 100 * (df.at[0, 'close'] - df.at[0, 'preclose']
) / df.at[0, 'preclose']
df.at[0, 'preamount'] = df.at[0, 'amount']
df.at[0, 'mchange'] = 0
else:
index_list = df.loc[df.date == cdate].index.values
if len(index_list) == 0:
logger.error("no data:%s for %s" % (cdate, self.code))
return False
preday_index = index_list[0] - 1
preday_df = df.loc[preday_index]
df = df.loc[df.date == cdate]
df['preclose'] = preday_df['close']
df['pchange'] = 100 * (df['close'] -
df['preclose']) / df['preclose']
df['preamount'] = preday_df['amount']
df['mchange'] = 100 * (df['amount'] -
df['preamount']) / df['preamount']
df = df.reset_index(drop=True)
day_table = self.get_day_table()
if is_df_has_unexpected_data(df):
logger.error("data for %s is not clear" % self.code)
return False
if self.mysql_client.set(df, day_table):
if self.redis.sadd(day_table, cdate):
return True
return False
def set_components_data(self, cdate = datetime.now().strftime('%Y-%m-%d')):
table_name = self.get_components_table_name(cdate)
if not self.is_table_exists(table_name):
if not self.create_components_table(table_name):
logger.error("create components table failed")
return False
if self.is_date_exists(table_name, cdate):
logger.debug("existed table:%s, date:%s" % (table_name, cdate))
return True
df = self.get_fg_index_df(cdate)
if is_df_has_unexpected_data(df):
logger.error("data for %s is not clear" % self.code)
return False
if self.mysql_client.set(df, table_name):
if self.redis.sadd(table_name, cdate): return True
return False
def set_all_data(self, fpath):
df = self.read(fpath)
if 0 == len(df):
logger.error("no data for %s" % self.code)
return False
else:
if 1 == len(df):
df.at[0, 'preclose'] = df.loc[0, 'open']
df.at[0, 'pchange'] = 100 * (
df.at[0, 'close'] -
df.at[0, 'preclose']) / df.at[0, 'preclose']
df.at[0, 'preamount'] = df.loc[0, 'amount']
df.at[0, 'mchange'] = 0
else:
df['preclose'] = df['close'].shift(1)
df.at[0, 'preclose'] = df.loc[0, 'open']
df['pchange'] = 100 * (df['close'] -
df['preclose']) / df['preclose']
df['preamount'] = df['amount'].shift(1)
df.at[0, 'preamount'] = df.loc[0, 'amount']
df['mchange'] = 100 * (df['amount'] -
df['preamount']) / df['preamount']
day_table = self.get_day_table()
existed_date_list = self.get_existed_keys_list(day_table)
df = df[~df.date.isin(existed_date_list)]
df = df.reset_index(drop=True)
if df.empty: return True
if is_df_has_unexpected_data(df):
logger.error("data for %s is not clear" % self.code)
return False
if not self.mysql_client.set(df, day_table):
logger.error("set data %s %s failed" % (self.code, day_table))
return False
if 0 == self.redis.sadd(day_table, *set(df.date.tolist())):
logger.error("sadd %s for %s failed" % (self.code, day_table))
return False
return True
