def set_components_data_from_joinquant(self, basic_dict, cdate): table_name = self.get_components_table_name(cdate) if not self.is_table_exists(table_name): if not self.create_components_table(table_name): logger.error("create components table failed") return False if self.is_date_exists(table_name, cdate): logger.debug("existed table:%s, date:%s" % (table_name, cdate)) return True code = "{}.XSHG".format(self.code) if self.code.startswith("6") else "{}.XSHE".format(self.code) stocks = get_index_stocks(code, date = cdate) df = pd.DataFrame(stocks, columns =['code']) df['code'] = df['code'].str.split('.', expand=True)[0] if not df[~df.code.isin(basic_dict)].empty: logger.error("code from juquant is not in basic in data {}".format(df[~df.code.isin(basic_dict)])) return False df['name'] = df['code'].apply(lambda x: basic_dict[x]) df['date'] = cdate if 'wieight' not in df.columns: df['weight'] = 1/len(df) if 'flag' not in df.columns: df['flag'] = 1 df = df.reset_index(drop = True) if is_df_has_unexpected_data(df): logger.error("data for %s is not clear" % self.code) return False if self.mysql_client.set(df, table_name): if self.redis.sadd(table_name, cdate): return True return False
def set_chip_distribution(self, df, zdate=None): if zdate is None: time2Market = self.get('timeToMarket') start_year = int(time2Market / 10000) end_year = int(datetime.now().strftime('%Y')) year_list = get_years_between(start_year, end_year) cfunc = partial(self.set_chip_table, df) return concurrent_run(cfunc, year_list, num=20) else: chip_table = self.get_chip_distribution_table(zdate) if not self.is_table_exists(chip_table): if not self.create_chip_table(chip_table): logger.error("create chip table:%s failed" % chip_table) return False if self.is_date_exists(chip_table, zdate): logger.debug("existed chip for code:%s, date:%s" % (self.code, zdate)) return True if is_df_has_unexpected_data(df): logger.error("data for %s is not clear" % self.code) return False if self.mysql_client.set(df, chip_table): if self.redis.sadd(chip_table, zdate): logger.debug("finish record chip:%s. table:%s" % (self.code, chip_table)) return True return False
def set_ticket(self, cdate=None): cdate = datetime.now().strftime('%Y-%m-%d') if cdate is None else cdate if not self.has_on_market(cdate): logger.debug("not on market code:%s, date:%s" % (self.code, cdate)) return True tick_table = self.get_redis_tick_table(cdate) if not self.is_table_exists(tick_table): if not self.create_ticket_table(tick_table): logger.error("create tick table failed") return False if not self.redis.sadd(self.dbname, tick_table): logger.error("add tick table to redis failed") return False if self.is_date_exists(tick_table, cdate): logger.debug("existed code:%s, date:%s" % (self.code, cdate)) return True logger.debug("%s read code from file %s" % (self.code, cdate)) df = ts.get_tick_data(self.code, date=cdate) df_tdx = read_tick( os.path.join( ct.TIC_DIR, '%s.tic' % datetime.strptime(cdate, "%Y-%m-%d").strftime("%Y%m%d")), self.code) if not df_tdx.empty: if df is not None and not df.empty and df.loc[0]['time'].find( "当天没有数据") == -1: net_volume = df.volume.sum() tdx_volume = df_tdx.volume.sum() if net_volume != tdx_volume: logger.error( "code:%s, date:%s, net volume:%s, tdx volume:%s not equal" % (self.code, cdate, net_volume, tdx_volume)) df = df_tdx else: if df is None: logger.debug("nonedata code:%s, date:%s" % (self.code, cdate)) return True if df.empty: logger.debug("emptydata code:%s, date:%s" % (self.code, cdate)) return True if df.loc[0]['time'].find("当天没有数据") != -1: logger.debug("nodata code:%s, date:%s" % (self.code, cdate)) return True df.columns = ['time', 'price', 'cchange', 'volume', 'amount', 'ctype'] logger.debug("merge ticket code:%s date:%s" % (self.code, cdate)) df = self.merge_ticket(df) df['date'] = cdate logger.debug("write data code:%s, date:%s, table:%s" % (self.code, cdate, tick_table)) if is_df_has_unexpected_data(df): logger.error("data for %s is not clear" % self.code) return False if self.mysql_client.set(df, tick_table): logger.debug("finish record:%s. table:%s" % (self.code, tick_table)) if self.redis.sadd(tick_table, cdate): return True return False
def set_components_data(self, cdate = datetime.now().strftime('%Y-%m-%d')): table_name = self.get_components_table_name(cdate) if not self.is_table_exists(table_name): if not self.create_components_table(table_name): logger.error("create components table failed") return False if self.is_date_exists(table_name, cdate): logger.debug("existed table:%s, date:%s" % (table_name, cdate)) return True url = self.INDEX_URLS[self.code][0] columns = self.INDEX_URLS[self.code][1] column_names = self.INDEX_URLS[self.code][2] df = smart_get(pd.read_excel, url, usecols = columns) if df is None: logger.error("data for %s is empty" % self.code) return False df.columns = column_names df.code = df.code.astype('str').str.zfill(6) df['date'] = cdate if 'wieight' not in df.columns: df['weight'] = 1/len(df) if 'flag' not in df.columns: df['flag'] = 1 df = df.reset_index(drop = True) if is_df_has_unexpected_data(df): logger.error("data for %s is not clear" % self.code) return False if self.mysql_client.set(df, table_name): if self.redis.sadd(table_name, cdate): return True return False
def get_data(code, start_date, end_date): cstock_obj = CIndex(code, dbinfo=ct.OUT_DB_INFO, redis_host='127.0.0.1') data = cstock_obj.get_k_data_in_range(start_date, end_date) data = data.set_index('date') if is_df_has_unexpected_data(data): return None data.index = pd.to_datetime(data.index) data = data.dropna(how='any') return data
def set_all_data(self, quantity_change_info, price_change_info, index_info): df, _ = self.read() if df.empty: logger.error("read empty file for:%s" % self.code) return False #modify price and quanity for all split-adjusted share prices df = self.adjust_share(df, quantity_change_info) if df.empty: logger.error("adjust share %s failed" % self.code) return False df = self.qfq(df, price_change_info) if df.empty: logger.error("qfq %s failed" % self.code) return False #transfer data to split-adjusted share prices df = self.transfer2adjusted(df) #compute strength relative index df = self.relative_index_strength(df, index_info) if df is None: logger.error("length of code %s is not equal to index." % self.code) return False #set chip distribution dist_data = self.compute_distribution(df) if dist_data.empty: logger.error("%s is empty distribution." % self.code) return False if not self.set_chip_distribution(dist_data): logger.error("store %s distribution failed" % self.code) return False df['uprice'] = mac(dist_data, 0) df['sprice'] = mac(dist_data, 5) df['mprice'] = mac(dist_data, 13) df['lprice'] = mac(dist_data, 37) df = pro_nei_chip(df, dist_data) if is_df_has_unexpected_data(df): logger.error("data for %s is not clean." % self.code) return False day_table = self.get_day_table() if not self.mysql_client.delsert(df, day_table): logger.error("save %s data to mysql failed." % self.code) return False self.redis.sadd(day_table, *set(df.date.tolist())) return True
def get_feed(all_df, codes, start_date, end_date, peried, duaration): feed = dataFramefeed.Feed() for code in codes: data = all_df.loc[all_df.code == code] data = data.sort_values(by=['date'], ascending=True) data = data.reset_index(drop=True) data = data.set_index('date') if is_df_has_unexpected_data(data): return None, None data.index = pd.to_datetime(data.index) data = data.dropna(how='any') data = atr(data, ndays=duaration) feed.addBarsFromDataFrame(code, data) return feed
def set_today_data(self, df, index_df, pre_date, cdate): day_table = self.get_day_table() if self.is_date_exists(day_table, cdate): logger.debug("existed data for code:%s, date:%s" % (self.code, cdate)) return True index_df = index_df.loc[index_df.date == cdate] preday_df = self.get_k_data(date=pre_date) if preday_df is None: logger.error("%s get pre date data failed." % self.code) return False if preday_df.empty: logger.error("%s get pre date data empty." % self.code) return False df['adj'] = 1.0 df['preclose'] = preday_df['close'][0] df['totals'] = preday_df['totals'][0] df['outstanding'] = preday_df['outstanding'][0] #transfer data to split-adjusted share prices df = self.transfer2adjusted(df) df = self.relative_index_strength(df, index_df, cdate) if df is None: return False #set chip distribution dist_df = df.append(preday_df, sort=False) dist_df = dist_df.sort_values(by='date', ascending=True) dist_data = self.compute_distribution(dist_df, cdate) if dist_data.empty: logger.error("%s chip distribution compute failed." % self.code) return False if self.set_chip_distribution(dist_data, zdate=cdate): df['uprice'] = mac(dist_data, 0) df['sprice'] = mac(dist_data, 5) df['mprice'] = mac(dist_data, 13) df['lprice'] = mac(dist_data, 37) df = pro_nei_chip(df, dist_data, preday_df, cdate) if is_df_has_unexpected_data(df): logger.error("data for %s is not clean." % self.code) return False if self.mysql_client.set(df, self.get_day_table()): return self.redis.sadd(day_table, cdate) return False
def get_feed(all_df, code, start_date, end_date, peried): feed = dataFramefeed.Feed() #cindex_obj = CIndex(code, dbinfo = ct.OUT_DB_INFO, redis_host = '127.0.0.1') #cstock_obj = CStock(code, dbinfo = ct.OUT_DB_INFO, redis_host = '127.0.0.1') #data = cstock_obj.get_k_data_in_range(start_date, end_date) #data = data.set_index('date') data = all_df.loc[all_df.code == code] data = data.sort_values(by=['date'], ascending=True) data = data.reset_index(drop=True) data = data.set_index('date') if is_df_has_unexpected_data(data): return None, None data.index = pd.to_datetime(data.index) data = data.dropna(how='any') data = atr(data) feed.addBarsFromDataFrame(code, data) return feed
def set_day_data(self, cdate): table_name = self.get_table_name(cdate) if not self.is_table_exists(table_name): if not self.create_table(table_name): self.logger.error("create tick table failed") return False self.redis.sadd(self.dbname, table_name) if self.is_date_exists(table_name, cdate): self.logger.debug("existed table:%s, date:%s" % (table_name, cdate)) return True df = self.generate_all_data(cdate) if df is None: return False if is_df_has_unexpected_data(df): return False if self.mysql_client.set(df, table_name): self.redis.sadd(table_name, cdate) return True return False
def set_day_data(self, cdate, fpath): day_table = self.get_day_table() if self.is_date_exists(day_table, cdate): logger.debug("existed data for code:%s, date:%s" % (self.code, cdate)) return True df = self.read(fpath) if 0 == len(df): logger.error("no data for %s:%s" % (cdate, self.code)) return False elif 1 == len(df): if len(df.loc[df.date == cdate]) == 0: logger.error("no data:%s for %s" % (cdate, self.code)) return False df.at[0, 'preclose'] = df.at[0, 'open'] df.at[0, 'pchange'] = 100 * (df.at[0, 'close'] - df.at[0, 'preclose'] ) / df.at[0, 'preclose'] df.at[0, 'preamount'] = df.at[0, 'amount'] df.at[0, 'mchange'] = 0 else: index_list = df.loc[df.date == cdate].index.values if len(index_list) == 0: logger.error("no data:%s for %s" % (cdate, self.code)) return False preday_index = index_list[0] - 1 preday_df = df.loc[preday_index] df = df.loc[df.date == cdate] df['preclose'] = preday_df['close'] df['pchange'] = 100 * (df['close'] - df['preclose']) / df['preclose'] df['preamount'] = preday_df['amount'] df['mchange'] = 100 * (df['amount'] - df['preamount']) / df['preamount'] df = df.reset_index(drop=True) day_table = self.get_day_table() if is_df_has_unexpected_data(df): logger.error("data for %s is not clear" % self.code) return False if self.mysql_client.set(df, day_table): if self.redis.sadd(day_table, cdate): return True return False
def set_components_data(self, cdate = datetime.now().strftime('%Y-%m-%d')): table_name = self.get_components_table_name(cdate) if not self.is_table_exists(table_name): if not self.create_components_table(table_name): logger.error("create components table failed") return False if self.is_date_exists(table_name, cdate): logger.debug("existed table:%s, date:%s" % (table_name, cdate)) return True df = self.get_fg_index_df(cdate) if is_df_has_unexpected_data(df): logger.error("data for %s is not clear" % self.code) return False if self.mysql_client.set(df, table_name): if self.redis.sadd(table_name, cdate): return True return False
def set_all_data(self, fpath): df = self.read(fpath) if 0 == len(df): logger.error("no data for %s" % self.code) return False else: if 1 == len(df): df.at[0, 'preclose'] = df.loc[0, 'open'] df.at[0, 'pchange'] = 100 * ( df.at[0, 'close'] - df.at[0, 'preclose']) / df.at[0, 'preclose'] df.at[0, 'preamount'] = df.loc[0, 'amount'] df.at[0, 'mchange'] = 0 else: df['preclose'] = df['close'].shift(1) df.at[0, 'preclose'] = df.loc[0, 'open'] df['pchange'] = 100 * (df['close'] - df['preclose']) / df['preclose'] df['preamount'] = df['amount'].shift(1) df.at[0, 'preamount'] = df.loc[0, 'amount'] df['mchange'] = 100 * (df['amount'] - df['preamount']) / df['preamount'] day_table = self.get_day_table() existed_date_list = self.get_existed_keys_list(day_table) df = df[~df.date.isin(existed_date_list)] df = df.reset_index(drop=True) if df.empty: return True if is_df_has_unexpected_data(df): logger.error("data for %s is not clear" % self.code) return False if not self.mysql_client.set(df, day_table): logger.error("set data %s %s failed" % (self.code, day_table)) return False if 0 == self.redis.sadd(day_table, *set(df.date.tolist())): logger.error("sadd %s for %s failed" % (self.code, day_table)) return False return True