def delItemsFromRemove(self): try: gLogger.info("start delItemsFromRemove") indexList = list(set(self.removeList)) self.df = self.df.drop(indexList, axis=0) except Exception as e: gLogger.exception("Exception : %s" % e)
def gen1minKData(self, vtSymbol, df_data): try: gLogger.info("start gen1minKData , vtSymbol is %s" % vtSymbol) c = 1 self.barDict[vtSymbol] = {} self.barDict[vtSymbol][c] = [] self.df["structTime"] = self.df["time"].map( lambda x: datetime.datetime.strptime(x, "%H%M%S%f")) for i in zip( *[iter(self.splitDict[vtSymbol][c][i:]) for i in range(2)]): start = datetime.datetime.strptime( str(i[0]).strip(), '%H:%M:%S') end = datetime.datetime.strptime(str(i[1]).strip(), '%H:%M:%S') if (start - datetime.timedelta(minutes=1) ).strftime('%H:%M') in self.AucTime: start = start - datetime.timedelta(minutes=1) p1 = df_data["structTime"] >= start p2 = df_data["structTime"] < end dfTemp = df_data.loc[p1 & p2] if len(dfTemp) > 2: self.barDict[vtSymbol][c].append(self.aggMethod(dfTemp)) dbNew = self.db.get_db("localhost", 27017, 'WIND_1_MIN_DB') self.db.insert2db(dbNew, vtSymbol, self.barDict[vtSymbol][c]) except Exception as e: gLogger.exception("Exception : %s" % e)
def recordExceptionalPrice(self): gLogger.info("start recordExceptionalPrice") self.estimateExceptional("lastPrice") self.estimateExceptional("highPrice") self.estimateExceptional("lowPrice") self.estimateExceptional("bidPrice1") self.estimateExceptional("askPrice1")
def cleanSameTimestamp(self): """清除重复时间戳,记录""" try: gLogger.info("start cleanSameTimestamp") dfTemp = self.df.sort_values(by=['datetime'], ascending=False) idList = dfTemp[dfTemp["datetime"].duplicated()].index for i in idList.values: self.removeList.append(i) gLogger.debug('remove index = %d' % i) except Exception as e: gLogger.exception("Exception : %s" % e)
def parse2CycleData(self): self.dateList = [ datetime.datetime(2017, 5, 31, 0, 0), datetime.datetime(2017, 6, 1, 0, 0), datetime.datetime(2017, 6, 2, 0, 0) ] for i in list(set(self.dateList)): gLogger.info("start parse cycle data —— %s" % i) self.date = i dfInfo = self.loadInformation() AggregateTickData(dfInfo, i, self.AucTime)
def cleanIllegalTradingTime(self): """删除非交易时段数据""" try: gLogger.info("start cleanIllegalTradingTime ") self.df['illegalTime'] = self.df["time"].map( self.StandardizeTimePeriod) self.df['illegalTime'] = self.df['illegalTime'].fillna(False) for i, row in self.df[self.df['illegalTime'] == False].iterrows(): self.removeList.append(i) gLogger.debug('remove index = %d' % (i)) del self.df["illegalTime"] except Exception as e: gLogger.exception("Exception: %s" % e)
def gen1DayKData(self, vtSymbol): try: gLogger.info("start gen1DayKData , vtSymbol = %s" % vtSymbol) c = '1Day' self.barDict[vtSymbol][c] = [] items = self.barDict[vtSymbol][1] dfTemp = pd.DataFrame(items) if not dfTemp.empty: self.barDict[vtSymbol][c].append(self.aggMethod(dfTemp)) dbNew = self.db.get_db("localhost", 27017, 'WIND_' + str(c) + '_MIN_DB') self.db.insert2db(dbNew, vtSymbol, self.barDict[vtSymbol][c]) except Exception as e: gLogger.exception("Exception : %s" % e)
def estimateExceptional(self, field): try: gLogger.info("start estimateExceptional, field = %s" % field) dfTemp = pd.DataFrame(self.df[field]) dfTemp["shift"] = self.df[field].shift(1) dfTemp["delta"] = abs(dfTemp[field] - dfTemp["shift"]) dfTemp = dfTemp.dropna(axis=0, how='any') dfTemp["IsExcept"] = dfTemp["delta"] >= dfTemp["shift"] * 0.12 for i, row in dfTemp.loc[dfTemp["IsExcept"]].iterrows(): if i not in self.removeList: self.logList.append(i) gLogger.debug('Field = %s, log index = %d' % (field, i)) except Exception as e: gLogger.exception("Exception : %s" % e)
def paddingWithPrevious(self, field): try: gLogger.info("start paddingWithPrevious, field = %s" % field) for i, row in self.df.loc[self.df[field] == 0.0].iterrows(): if i not in self.removeList: preIndex = i - 1 if preIndex >= 0 and i not in self.removeList: row[field] = self.df.loc[preIndex, field] self.df.loc[i, field] = row[field] self.updateList.append(i) gLogger.debug('Field = %s, update index = %d' % (field, i)) except Exception as e: gLogger.exception("Exception : %s" % e)
def processTickData(self): self.fileList = self.parseMatFile() self.fileList = [ "E:\\windDataOriginal\\commodity\\20170531\\bb1805\\bb1805_20170531.mat" ] for i in self.fileList: sym = i.split('\\')[-2] if "SP-" in sym or "SPC-" in sym or "IMCI" in sym: continue gLogger.info("start process tick data —— %s" % i) self.date = datetime.datetime.strptime( i.split('\\')[-1].split('_')[-1][:-4], '%Y%m%d') self.dateList.append(self.date) dfInfo = self.loadInformation() dfData = LoadMatFile(i).dfData CleanData(dfData, dfInfo, self.AucTime)
def getTimeList(self, cycle): try: gLogger.info("start getTimeList") if not os.path.exists(self.timeFilePath): os.makedirs(self.timeFilePath) # filePath = self.timeFilePath + 'timeSeries_' + self.Symbol + '.pickle' # if os.path.exists(filePath) and datetime.datetime.fromtimestamp(os.path.getmtime(filePath)).replace(hour=0,minute=0,second=0,microsecond=0) == \ # datetime.datetime.today().replace(hour=0,minute=0,second=0,microsecond=0): # gLogger.info("splitDict is load from pickle file") # with open(filePath, 'rb') as handle: # self.splitDict[self.Symbol] = pickle.load(handle) # else: self.genTimeList(self.Symbol, cycle) self.saveTimeList() except Exception as e: gLogger.exception("Exception : %s" % e)
def normalizeData(self, df): try: gLogger.info("start normalize df Data") f = lambda x: str(int(x)) df["date"] = df["date"].map(f) df["time"] = df["time"].map(f) df["time"] = df["time"].map(lambda x: x.zfill(9)) vtSymbol = self.matFile.split('\\')[-1].split('_')[0] symbol = "".join([a for a in vtSymbol if a.isalpha()]).lower() date = self.matFile.split('\\')[-1].split('_')[-1][:-4] df["vtSymbol"] = vtSymbol df["symbol"] = symbol df["DT"] = df["date"] + ' ' + df["time"] df["datetime"] = df["DT"].map(self.convert2datetime) del df["DT"] df["date"] = date return df except Exception as e: gLogger.exception("Exception when convert to df: %s" % e)
def reserveLastTickInAuc(self): """保留集合竞价期间最后一个tick数据""" try: gLogger.info("start reserveLastTickInAuc") self.df["structTime"] = self.df["time"].map( lambda x: datetime.datetime.strptime(x, "%H%M%S%f")) for st in self.AucTime: start = datetime.datetime.strptime(st, '%H:%M') end = start + datetime.timedelta(minutes=1) p1 = self.df["structTime"] >= start p2 = self.df["structTime"] < end dfTemp = self.df.loc[p1 & p2] dfTemp = dfTemp.sort_values(by=["structTime"], ascending=False) for i in dfTemp.index.values[1:]: self.removeList.append(i) gLogger.debug('remove index = %d' % i) del self.df["structTime"] except Exception as e: gLogger.exception("Exception : %s" % e)
def cleanNullPriceIndicator(self): gLogger.info("start cleanNullPriceIndicator") lastP = self.df["lastPrice"] == 0.0 high = self.df["highPrice"] == 0.0 low = self.df["lowPrice"] == 0.0 bidP = self.df["bidPrice1"] == 0.0 askP = self.df["askPrice1"] == 0.0 # 如果均为0,删除 if self.df.loc[lastP & high & low & bidP & askP]._values.any(): gLogger.debug("process data that all price indicators are null") for i in self.df.loc[lastP & high & low & bidP & askP].index.values: if i not in self.removeList: self.removeList.append(i) gLogger.debug('All Price is Null, remove index = %d' % i) # 某些为0,填充 self.paddingWithPrevious("lastPrice") self.paddingWithPrevious("highPrice") self.paddingWithPrevious("lowPrice") self.paddingWithPrevious("bidPrice1") self.paddingWithPrevious("askPrice1")
def initCleanRegulation(self): gLogger.info("start initCleanRegulation") dbNew = self.db.get_db("localhost", 27017, 'WIND_TICK_DB') i = self.df["vtSymbol"][0] try: if "IFC" in i or "IHC" in i or "ICC" in i or "TFC" in i: i = i[:2] self.Symbol = "".join([a for a in i if a.isalpha()]).lower() self.initList() if not self.df.empty: self.cleanIllegalTradingTime() self.cleanSameTimestamp() self.reserveLastTickInAuc() self.cleanNullVolTurn() self.cleanNullPriceIndicator() self.cleanNullOpenInter() self.recordExceptionalPrice() self.delItemsFromRemove() self.db.insert2db(dbNew, i, self.df) except Exception as e: gLogger.exception("Exception: %s" % e)
def genOtherKData(self, vtSymbol, cycle): for c in cycle: try: gLogger.info("start genOtherKData cycle = %d" % c) self.barDict[vtSymbol][c] = [] for i in zip( * [iter(self.splitDict[vtSymbol][c][i:]) for i in range(2)]): self.start1 = time.strptime(str(i[0]).strip(), '%H:%M:%S') self.end1 = time.strptime(str(i[1]).strip(), '%H:%M:%S') items = list( map(self.selectItems, self.barDict[vtSymbol][1])) items = list(filter(lambda x: x is not None, items)) dfTemp = pd.DataFrame(items) if len(dfTemp) > 2: self.barDict[vtSymbol][c].append( self.aggMethod(dfTemp)) dbNew = self.db.get_db("localhost", 27017, 'WIND_' + str(c) + '_MIN_DB') self.db.insert2db(dbNew, vtSymbol, self.barDict[vtSymbol][c]) except Exception as e: gLogger.exception("Exception : %s" % e)
def genTimeList(self, symbol, cycle): try: tempDict = {} self.splitDict[symbol] = {} for c in cycle: gLogger.info("start genTimeList, cycle = %d" % c) tempDict[c] = [] self.splitDict[symbol][c] = [] tp = self.dfInfo.loc[symbol]["CurrPeriod"] time1 = [t for i in tp.split(',') for t in i.split('-')] for i in zip(*([iter(time1)] * 2)): start = str(i[0]).strip() end = str(i[1]).strip() if start in self.AucTime: start1 = datetime.datetime.strptime( start, "%H:%M") + datetime.timedelta(minutes=1) start = start1.strftime("%H:%M") else: while ([ 60 if datetime.datetime.strptime( start, "%H:%M").minute == 0 else datetime. datetime.strptime(start, "%H:%M").minute ][0] % int(c) != 0): start1 = datetime.datetime.strptime( start, "%H:%M") + datetime.timedelta(minutes=10) start = start1.strftime("%H:%M") tempList = pd.date_range( start, end, freq=(str(c) + 'min')).time.tolist() tempDict[c].extend(tempList) # tempDict[c].extend(pd.date_range(end, end, freq=(str(c) + 'min')).time.tolist()) lst = list(set(tempDict[c])) lst.sort() self.splitDict[symbol][c] = lst except Exception as e: gLogger.exception("Exception : %s" % e)
def __init__(self, matFile): self.matFile = matFile self.data = sio.loadmat(self.matFile) gLogger.info("load mat file completed") self.dfData = self.convert2df()
def cleanNullVolTurn(self): """Tick有成交,但volume和turnover为0""" gLogger.info("start cleanNullVolTurn") f = lambda x: float(x) self.df["lastVolume"] = self.df["lastVolume"].map(f) self.df["lastTurnover"] = self.df["lastTurnover"].map(f) self.df["volume"] = self.df["volume"].map(f) self.df["turnover"] = self.df["turnover"].map(f) self.df["openInterest"] = self.df["openInterest"].map(f) self.df["lastPrice"] = self.df["lastPrice"].map(f) lastVol = self.df["lastVolume"] != 0.0 lastTurn = self.df["lastTurnover"] != 0.0 Vol = self.df["volume"] == 0.0 Turn = self.df["turnover"] == 0.0 openIn = self.df["openInterest"] == 0.0 lastP = self.df["lastPrice"] != 0.0 tu = self.dfInfo.loc[self.Symbol]["TradingUnits"] # lastTurn为0,lastVolume和lastPrice不为0 dfTemp = self.df.loc[~lastTurn & lastVol & lastP] if not dfTemp.empty: gLogger.debug( "process data that lastTurn is null but lastVol and lastP are not" ) dfTemp["lastTurnover"] = dfTemp["lastVolume"] * dfTemp[ "lastPrice"] * float(tu) for i, row in dfTemp.iterrows(): if i not in self.removeList: self.df.loc[i, "lastTurnover"] = row["lastTurnover"] self.updateList.append(i) gLogger.debug('lastTurn = 0, update index = %d' % (i)) # lastVolume为0,lastTurnover和lastPrice不为0 dfTemp = self.df.loc[lastTurn & ~lastVol & lastP] if not dfTemp.empty: dfTemp["lastVolume"] = dfTemp["lastTurnover"] / ( dfTemp["lastPrice"] * float(tu)) dfTemp["lastVolume"].map(lambda x: int(round(x))) for i, row in dfTemp.iterrows(): if i not in self.removeList: self.df.loc[i, "lastVolume"] = row["lastVolume"] self.updateList.append(i) gLogger.debug('lastVol = 0, update index = %d' % (i)) # lastPrice为0,lastVolume和lastTurnover不为0 dfTemp = self.df.loc[lastTurn & lastVol & ~lastP] if not dfTemp.empty: dfTemp["lastPrice"] = dfTemp["lastTurnover"] / ( dfTemp["lastVolume"] * float(tu)) for i, row in dfTemp.iterrows(): if i not in self.removeList: self.df.loc[i, "lastPrice"] = row["lastPrice"] self.updateList.append(i) gLogger.debug('lastPrice = 0, update index = %d' % (i)) # lastVolume和lastTurnover均不为0 dfTemp = self.df.loc[lastVol & lastTurn & (Vol | Turn | openIn)] if not dfTemp.empty: # volume、openInterest、turnover均为0,删除并记录 if dfTemp.loc[Vol & Turn & openIn]._values.any(): for i in dfTemp.loc[Vol & Turn & openIn].index.values: if i not in self.removeList: self.removeList.append(i) self.logList.append(i) gLogger.debug( 'Vol & openInterest & turn = 0, remove index = %d' % i) # turnover为0,lastVol不为0 for i, row in self.df[Turn & lastVol].iterrows(): preIndex = i - 1 if preIndex >= 0 and i not in self.removeList: row["turnover"] = self.df.loc[ preIndex, "turnover"] + row["lastTurnover"] self.df.loc[i, "turnover"] = row["turnover"] self.updateList.append(i) gLogger.debug( 'Turn = 0 & lastTurn != 0, update index = %d' % (i)) # volume为0,lastVol不为0 for i, row in self.df[Vol & lastVol].iterrows(): preIndex = i - 1 if preIndex >= 0 and i not in self.removeList: row["volume"] = self.df.loc[preIndex, "volume"] + row["lastVolume"] self.df.loc[i, "volume"] = row["volume"] self.updateList.append(i) gLogger.debug('Vol = 0 & lastVol != 0, update index = %d' % (i))
def cleanNullOpenInter(self): """持仓量为0,用上一个填充""" gLogger.info("start cleanNullOpenInter") self.paddingWithPrevious("openInterest")