def getMinDate(self, historyData):
df = historyData.copy()
########
from avera.mods.VSASpread import VSASpread
from avera.mods.HeikenAshiMod import HeikenAshiMod
from avera.mods.EnergyMod import EnergyMod
historyMod = VSASpread()
df = historyMod.modHistory(df)
historyMod = HeikenAshiMod()
df = historyMod.modHistory(df)
historyMod = EnergyMod()
df = historyMod.modHistory(df,
featList=["open", "close", "low", "high"])
########
for i in range(self.nDiffs):
for feat in self.featureList:
notShifted = df[feat]
shiftedData = df[feat].shift(periods=1)
df[feat] = notShifted - shiftedData
iter = next(df.iterrows())
df = df.drop(iter[0])
iter = df.iterrows()
obs = next(iter)
#acceleration
#for i in range(self.nPoints + self.nPoints):
#
for i in range(self.nPoints - 1):
obs = next(iter)
return obs[0]
def getManyPointsFeat(self, datetimeStr, historyData):
obsList = None
if self.fitMode:
obsList = self.fitData.loc[:str(datetimeStr)].tail(
self.nPoints).copy()
else:
df = historyData.copy()
########
df = df[:str(datetimeStr)].tail(500)
from avera.mods.VSASpread import VSASpread
from avera.mods.HeikenAshiMod import HeikenAshiMod
from avera.mods.EnergyMod import EnergyMod
historyMod = VSASpread()
df = historyMod.modHistory(df)
historyMod = HeikenAshiMod()
df = historyMod.modHistory(df)
historyMod = EnergyMod()
df = historyMod.modHistory(
df, featList=["open", "close", "low", "high"])
########
for i in range(self.nDiffs):
for feat in self.featureList:
notShifted = df[feat]
shiftedData = df[feat].shift(periods=1)
df[feat] = notShifted - shiftedData
iter = next(df.iterrows())
df = df.drop(iter[0])
obsList = df.loc[:str(datetimeStr)].tail(self.nPoints).copy()
obsListTransformed = {}
for obs in obsList.iterrows():
for feat in self.featureList:
data = obs[1][feat]
data = data.reshape(-1, 1)
data = self.scalerDict[feat].transform(data)
obs[1][feat] = data
obsListTransformed[obs[0]] = obs[1]
obsList = pd.DataFrame(obsListTransformed.values(),
index=list(obsListTransformed.keys()))
obs = []
if self.flatStack:
localObs = np.zeros((self.nPoints, ))
# for feat in self.featureList:
# localObs = np.vstack([localObs, vectorPrices[feat]])
# obs = np.array(localObs[1:])
else:
localObs = np.zeros((self.nPoints, ))
for feat in self.featureList:
localObs = np.vstack([localObs, obsList[feat].values])
obs.append(localObs[1:])
obs = np.array(obs)
obs = np.reshape(obs, (obs.shape[1], obs.shape[2], 1))
return obs
def preprocData(df):
historyMod = VSASpread()
modDf = historyMod.modHistory(df)
historyMod = HeikenAshiMod()
modDf = historyMod.modHistory(modDf)
historyMod = EnergyMod()
modDf = historyMod.modHistory(modDf, featList=["open", "close", "low", "high"])
return modDf
def createGenerators(df, priceFeatList):
########
historyMod = VSASpread()
df = historyMod.modHistory(df)
historyMod = HeikenAshiMod()
df = historyMod.modHistory(df)
historyMod = EnergyMod()
df = historyMod.modHistory(df, featList=["open", "close", "low", "high"])
########
priceDiffGenerator = MultiScalerDiffGenerator(featureList=priceFeatList, nDiffs=1, nPoints=32, flatStack=False,fitOnStep=False)
priceDiffGenerator.setFitMode(True)
priceDiffGenerator = priceDiffGenerator.globalFit(df)
priceDiffGenerator.saveGenerator("./MSDiffGen.pkl")
pass
symbol = "EURUSD_i"
timeframe = "H1"
hkFeatList = ["open", "close", "low", "high", "vsa_spread", "tick_volume",
"hkopen", "hkclose", "enopen", "enclose", "enlow", "enhigh"]
#terminal = MT5Terminal()
dataUpdater = SymbolDataUpdater("../../data/raw/")
dataManager = SymbolDataManager("../../data/raw/")
#dataUpdater.fullUpdate(terminal, symbol, timeframe, startDate="2015-01-01 00:00:00")
df = SymbolDataManager().getData(symbol, timeframe)
########
historyMod = VSASpread()
modDf = historyMod.modHistory(df)
historyMod = HeikenAshiMod()
modDf = historyMod.modHistory(modDf)
historyMod = EnergyMod()
modDf = historyMod.modHistory(modDf, featList=["open", "close", "low", "high"])
########
genList = []
for i in range(2):
gen = MultiScalerDiffGenerator(featureList=hkFeatList, nDiffs=i+1, nPoints = 96, flatStack = False, fitOnStep = False)
genList.append( gen )
compositeGenerator = CompositeGenerator( genList=genList, flatStack=False)
compositeGenerator.setFitMode(True)
compositeGenerator.globalFit(modDf)
compositeGenerator.saveGenerator("./CompositeMSDiffGen.pkl")
startTime = datetime.now()
print("start time: {}".format(startTime))
symbol = "EURUSD_i"
timeframe = "H1"
hkFeatList = [
"open", "close", "low", "high", "vsa_spread", "tick_volume", "hkopen",
"hkclose", "enopen", "enclose", "enlow", "enhigh"
]
dataManager = SymbolDataManager()
df = SymbolDataManager().getData(symbol, timeframe)
df = df.tail(10000)
########
historyMod = HeikenAshiMod()
modDf = historyMod.modHistory(df)
########
nIntervals = 100
nLevels = 8
featVals = modDf["hkclose"].values
minVal = np.min(featVals)
maxVal = np.max(featVals)
step = (maxVal - minVal) / nIntervals
borders = []
intervalDict = {}
for i in range(nIntervals):
border = minVal + i * step
borders.append(border)
intervalDict[border] = [[], []]
symbol = "EURUSD_i"
timeframe = "M15"
hkFeatList = ["hkopen", "hkclose", "low", "high"]
volumeFeatList = ["tick_volume"]
#terminal = MT5Terminal()
#dataUpdater = SymbolDataUpdater()
dataManager = SymbolDataManager()
#dataUpdater.fullUpdate(terminal, symbol, timeframe, startDate="2015-01-01 00:00:00")
df = SymbolDataManager().getData(symbol, timeframe)
########
#df = df.tail(10000)
print("applying heiken ashi mod...")
enMod = HeikenAshiMod()
df = enMod.modHistory(df)
#enMod.checkQuality(df)
########
priceDiffGenerator = W2VDiffGenerator(featureList=hkFeatList, nDiffs=1, nPoints = 21, flatStack = False, fitOnStep = False,
nIntervals = 10000, w2vSize=32, window=3, iter=100, min_count=0, sample=0.0, sg=0)
volumeDiffGenerator = W2VDiffGenerator(featureList=volumeFeatList, nDiffs=1, nPoints = 21, flatStack = False, fitOnStep = False,
nIntervals = 10000, w2vSize=32, window=3, iter=100, min_count=0, sample=0.0, sg=0)
priceDiffGenerator.setFitMode(True)
priceDiffGenerator = priceDiffGenerator.globalFit(df)
#priceDiffGenerator.checkReconstructionQuality(df)
priceDiffGenerator.saveGenerator("./w2vPriceDiffGen.pkl")
