def getMinDate(self, historyData): df = historyData.copy() ######## from monlan.mods.VSASpread import VSASpread from monlan.mods.HeikenAshiMod import HeikenAshiMod from monlan.mods.EnergyMod import EnergyMod historyMod = VSASpread() df = historyMod.modHistory(df) historyMod = HeikenAshiMod() df = historyMod.modHistory(df) historyMod = EnergyMod() df = historyMod.modHistory(df, featList=["open", "close", "low", "high"]) ######## for i in range(self.nDiffs): for feat in self.featureList: notShifted = df[feat] shiftedData = df[feat].shift(periods=1) df[feat] = notShifted - shiftedData iter = next(df.iterrows()) df = df.drop(iter[0]) iter = df.iterrows() obs = next(iter) #acceleration #for i in range(self.nPoints + self.nPoints): # for i in range(self.nPoints - 1): obs = next(iter) return obs[0]
def getManyPointsFeat(self, datetimeStr, historyData): obsList = None if self.fitMode: obsList = self.fitData.loc[:str(datetimeStr)].tail(self.nPoints).copy() else: df = historyData.copy() ######## df = df[:str(datetimeStr)].tail(500) from monlan.mods.VSASpread import VSASpread from monlan.mods.HeikenAshiMod import HeikenAshiMod from monlan.mods.EnergyMod import EnergyMod historyMod = VSASpread() df = historyMod.modHistory(df) historyMod = HeikenAshiMod() df = historyMod.modHistory(df) historyMod = EnergyMod() df = historyMod.modHistory(df, featList=["open", "close", "low", "high"]) ######## for i in range(self.nDiffs): for feat in self.featureList: notShifted = df[feat] shiftedData = df[feat].shift(periods=1) df[feat] = notShifted - shiftedData iter = next(df.iterrows()) df = df.drop(iter[0]) obsList = df.loc[:str(datetimeStr)].tail(self.nPoints).copy() obsListTransformed = {} for obs in obsList.iterrows(): for feat in self.featureList: data = obs[1][feat] data = data.reshape(-1, 1) data = self.scalerDict[feat].transform(data) obs[1][feat] = data obsListTransformed[obs[0]] = obs[1] obsList = pd.DataFrame(obsListTransformed.values(), index=list(obsListTransformed.keys())) obs = [] if self.flatStack: localObs = np.zeros((self.nPoints,)) # for feat in self.featureList: # localObs = np.vstack([localObs, vectorPrices[feat]]) # obs = np.array(localObs[1:]) else: localObs = np.zeros((self.nPoints,)) for feat in self.featureList: localObs = np.vstack([localObs, obsList[feat].values]) obs.append(localObs[1:]) obs = np.array(obs) obs = np.reshape(obs, (obs.shape[1], obs.shape[2], 1)) return obs
"open", "close", "low", "high", "spread", "vsa_spread", "tick_volume", "hkopen", "hkclose", "enopen", "enclose", "enlow", "enhigh" ] #terminal = MT5Terminal(login=123456, server="broker-server", password="******") #dataUpdater = SymbolDataUpdater("../data/raw/") dataManager = SymbolDataManager("../data/raw/") #dataUpdater.fullUpdate(terminal, symbol, timeframe, startDate="2008-01-01 00:00:00") df = dataManager.getData(symbol, timeframe) df = df.tail(400000) ######## historyMod = VSASpread() modDf = historyMod.modHistory(df) historyMod = HeikenAshiMod() modDf = historyMod.modHistory(modDf) historyMod = EnergyMod() modDf = historyMod.modHistory(modDf, featList=["open", "close", "low", "high"]) ######## priceDiffGenerator = MultiScalerDiffGenerator(featureList=hkFeatList, nDiffs=1, nPoints=256, flatStack=False, fitOnStep=False) priceDiffGenerator.setFitMode(True) priceDiffGenerator = priceDiffGenerator.globalFit(modDf) priceDiffGenerator.saveGenerator("../models/MSDiffGen.pkl") ###############################
def getManyPointsFeat(self, datetimeStr, historyData): obsList = None if self.fitMode: obsList = self.fitData.loc[:datetimeStr].tail(self.nPoints).copy() else: df = historyData.copy() ######## df = df[:str(datetimeStr)].tail(500) from monlan.mods.VSASpread import VSASpread from monlan.mods.HeikenAshiMod import HeikenAshiMod from monlan.mods.EnergyMod import EnergyMod historyMod = VSASpread() df = historyMod.modHistory(df) historyMod = HeikenAshiMod() df = historyMod.modHistory(df) historyMod = EnergyMod() df = historyMod.modHistory( df, featList=["open", "close", "low", "high"]) ######## for i in range(self.nDiffs): for feat in self.featureList: notShifted = df[feat] shiftedData = df[feat].shift(periods=1) df[feat] = notShifted - shiftedData iter = next(df.iterrows()) df = df.drop(iter[0]) obsList = df.loc[:str(datetimeStr)].tail(self.nPoints).copy() obsListTransformed = {} for obs in obsList.iterrows(): for feat in self.featureList: data = obs[1][feat] data = data.reshape(-1, 1) data = self.scalerDict[feat].transform(data) obs[1][feat] = data obsListTransformed[obs[0]] = obs[1] obsList = pd.DataFrame(obsListTransformed.values(), index=list(obsListTransformed.keys())) ################################# # Scale each feature series to (0, 1) individually # to remove big picture properties from current window of local features for feat in self.featureList: tmp = obsList[feat].values.reshape((-1, 1)) tmp = MinMaxScaler(feature_range=(0.0, 1.0)).fit_transform(tmp) #tmp = StandardScaler().fit_transform(tmp) tmp = tmp.reshape((-1, )) obsList[feat] = tmp ################################# obs = [] if self.flatStack: localObs = np.zeros((self.nPoints, )) #for feat in self.featureList: # localObs = np.vstack([localObs, vectorPrices[feat]]) #obs = np.array(localObs[1:]) else: localObs = np.zeros((self.nPoints, )) for feat in self.featureList: localObs = np.vstack([localObs, obsList[feat].values]) obs.append(localObs[1:]) obs = np.array(obs) #2DConv features shape obs = np.reshape(obs, (obs.shape[1], obs.shape[2], 1)) #1DConv features shape #obs = np.reshape(obs, (obs.shape[1], obs.shape[2])) #obs = obs.T ################## return obs