def cycleTrain(symbol, timeframe, dataUpdater, saveDir, genPath, saveAgentName): while True: dataUpdater.partialUpdate(terminal, symbol, timeframe) df = SymbolDataManager().getData(symbol, timeframe) preprocDf = preprocData(df) createGenerator(preprocDf, featureList=hkFeatList, saveDir=saveDir, saveName=genName) trainDf = preprocDf.tail(2032).tail(1032) backTestDf = preprocDf.tail(2032).head(1032) lastSaveEp = trainAgent(trainDf=trainDf, backTestDf=backTestDf, genPath=genPath, saveAgentDir=saveDir, saveAgentName=saveAgentName) ############## reset_keras() ############## ##### # break ##### sumReward = evaluateAgent(backTestDf=backTestDf, genPath=genPath, agentDir=saveDir, agentName=saveAgentName) if lastSaveEp > 0 and sumReward > -110: print("Backtest's succesfull. Stop Training.") break else: print("Backtest failed. Retrain.") ############# #weighted train/back reward """backReward = evaluateAgent(backTestDf=backTestDf, genPath=genPath, agentDir=saveDir, agentName=saveAgentName) trainReward = evaluateAgent(backTestDf=trainDf, genPath=genPath, agentDir=saveDir, agentName=saveAgentName) backW = 0.8 #1.0 - (len(backTestDf) / (len(backTestDf) + len(trainDf))) trainW = 0.2 #1.0 - (len(trainDf) / (len(backTestDf) + len(trainDf))) weightedReward = (backW * backReward + trainW * trainReward) / 2 if lastSaveEp > 0 and weightedReward > 0: print("Backtest's succesfull. Stop Training.") break else: print("Backtest failed. Retrain.")""" ############# reset_keras()
from avera.datamanagement.SymbolDataManager import SymbolDataManager symbolDM = SymbolDataManager() readedData = symbolDM.getData(symbol="TEST", timeFrame="H1") print(readedData.tail(2)) readedData = symbolDM.getData(symbol="TEST", timeFrame="H1", normalizeNames=True) print(readedData.tail(2)) readedData = symbolDM.getData(symbol="TEST", timeFrame="H1", normalizeNames=True, normalizeDateTime=True) print(readedData.tail(2))
from avera.mods.EnergyMod import EnergyMod from avera.mods.VSASpread import VSASpread import matplotlib.pyplot as plt from datetime import datetime startTime = datetime.now() print("start time: {}".format(startTime)) symbol = "EURUSD_i" timeframe = "H1" hkFeatList = ["open", "close", "low", "high", "vsa_spread", "tick_volume", "hkopen", "hkclose", "enopen", "enclose", "enlow", "enhigh"] #terminal = MT5Terminal() dataUpdater = SymbolDataUpdater("../../data/raw/") dataManager = SymbolDataManager("../../data/raw/") #dataUpdater.fullUpdate(terminal, symbol, timeframe, startDate="2015-01-01 00:00:00") df = SymbolDataManager().getData(symbol, timeframe) ######## historyMod = VSASpread() modDf = historyMod.modHistory(df) historyMod = HeikenAshiMod() modDf = historyMod.modHistory(modDf) historyMod = EnergyMod() modDf = historyMod.modHistory(modDf, featList=["open", "close", "low", "high"]) ######## genList = [] for i in range(2):
from avera.feature_generators.DiffGenerator import DiffGenerator from avera.feature_generators.FeatureScaler import FeatureScaler from avera.datamanagement.SymbolDataManager import SymbolDataManager from matplotlib import pyplot as plt from dateutil import parser featureList = ["open", "close"] df = SymbolDataManager().getData("TEST", "H1", normalizeDateTime=True, normalizeNames=True) df.set_index("datetime", drop=True, inplace=True) featureScaler = DiffGenerator(featureList=featureList, nDiffs=1, nPoints=4, flatStack=True) featureScaler = featureScaler.globalFit(df) print(df.tail(1)) lastDt = list(df.tail(1).index)[0] print(lastDt) print(df.loc[lastDt]) print(featureScaler.getFeatByDatetime(lastDt, df)) featureScaler = FeatureScaler() df = featureScaler.extractFeature(df, featureList) print(df.loc[lastDt])
from avera.envs.RealEnv import RealEnv from avera.terminal.MT5Terminal import MT5Terminal #from avera.feature_generators.FeatureScaler import FeatureScaler from avera.feature_generators.ScalerGenerator import ScalerGenerator from avera.feature_generators.DiffGenerator import DiffGenerator from avera.feature_generators.CompositeGenerator import CompositeGenerator from avera.datamanagement.SymbolDataManager import SymbolDataManager from avera.datamanagement.SymbolDataUpdater import SymbolDataUpdater symbol = "EURUSD_i" timeframe = "H1" obsFeatList = ["open", "close"] #terminal = MT5Terminal() #dataUpdater = SymbolDataUpdater() dataManager = SymbolDataManager() #dataUpdater.fullUpdate(terminal, symbol, timeframe, startDate="2015-01-01 00:00:00") df = SymbolDataManager().getData( symbol, timeframe) #, normalizeNames=True, normalizeDateTime=True) df = df.tail(1000) openerScaleGen = ScalerGenerator(featureList=obsFeatList, nPoints=50, flatStack=True, fitOnStep=False) openerDiffGen = DiffGenerator(featureList=obsFeatList, nDiffs=1, nPoints=50, flatStack=True, fitOnStep=False)
from avera.datamanagement.SymbolDataManager import SymbolDataManager from datetime import datetime from avera_core.CloserFeatureGenerator import CloserFeatureGenerator from avera.utils.save_load import * from tqdm import tqdm startTime = datetime.now() print("start time: {}".format(startTime)) symbol = "EURUSD_i" timeframe = "M10" dataManager = SymbolDataManager("../data/raw/") df = dataManager.getData(symbol, timeframe) df = df.tail(50000) print(df.shape) print(df.head(2)) print(df.tail(2)) df.set_index("datetime", drop=True, inplace=True) featGen = CloserFeatureGenerator(featureList=["open", "close"], nFeatRows=1, nPoints=110, nLevels=5, flatStack=True, fitOnStep=True) featGen = featGen.globalFit(df) startTime = datetime.now()
from avera.envs.CompositeEnv import CompositeEnv #from avera.feature_generators.FeatureScaler import FeatureScaler from avera.feature_generators.W2VCompositeGenerator import W2VCompositeGenerator from avera.feature_generators.W2VScaleGenerator import W2VScaleGenerator from avera.feature_generators.W2VDiffGenerator import W2VDiffGenerator from avera.datamanagement.SymbolDataManager import SymbolDataManager from avera.mods.EnergyMod import EnergyMod symbol = "EURUSD_i" timeframe = "H1" energyFeatList = ["enopen", "enclose", "enlow", "enhigh"] volumeFeatList = ["tick_volume"] #terminal = MT5Terminal() #dataUpdater = SymbolDataUpdater() dataManager = SymbolDataManager() #dataUpdater.fullUpdate(terminal, symbol, timeframe, startDate="2015-01-01 00:00:00") df = SymbolDataManager().getData(symbol, timeframe) ######## #df = df.tail(1000) print("applying energy mod...") enMod = EnergyMod() df = enMod.modHistory(df, ["open", "close", "low", "high"]) #enMod.checkQuality(df) ######## """energyScaleGenerator = W2VScaleGenerator(featureList=energyFeatList, nPoints = 21, flatStack = False, fitOnStep = False, nIntervals = 10000, w2vSize=32, window=20, iter=100, min_count=0, sample=0.0, sg=0) energyDiffGenerator = W2VDiffGenerator(featureList=energyFeatList, nDiffs=1, nPoints = 21, flatStack = False, fitOnStep = False, nIntervals = 10000, w2vSize=32, window=20, iter=100, min_count=0, sample=0.0, sg=0)"""
hkFeatList = [ "open", "close", "low", "high", "vsa_spread", "tick_volume", "hkopen", "hkclose", "enopen", "enclose", "enlow", "enhigh" ] saveDir = "../models/" genName = "MSDiffGen" genPath = saveDir + genName + ".pkl" agentName = "best_composite" timeConstraint = timedelta(days=5, hours=12, minutes=0) terminal = MT5Terminal(login=123456, server="broker-server", password="******") #terminal = MT5Terminal(login=123456, server="broker-server", password="******") dataUpdater = SymbolDataUpdater() dataManager = SymbolDataManager() dataUpdater.fullUpdate(terminal, symbol, timeframe, startDate="2008-01-01 00:00:00") while True: ################# """symbolList = ["EURUSD_i", "USDCHF_i", "USDCAD_i", "AUDUSD_i"] for sym in symbolList: print("Partial update: {}".format(sym)) #dataUpdater.fullUpdate(terminal, sym, timeframe, startDate="2015-01-01 00:00:00") dataUpdater.partialUpdate(terminal, sym, timeframe) symbol = selectSymbol(symbolList, dataManager, period=18) print("Selected symbol: {}".format(symbol))""" #################
from avera.feature_generators.FeatureDiff import FeatureDiff from avera.feature_generators.FeatureScaler import FeatureScaler from avera.datamanagement.SymbolDataManager import SymbolDataManager from matplotlib import pyplot as plt df = SymbolDataManager().getData("EURUSD_i", "H1") df = FeatureDiff().extractFeature(df, featList=["open"], nDiffs=1) df = FeatureScaler().extractFeature(df, featList=["open"]) print(df) print(df["open"].mean()) print(df["open"].std()) y = df["open"].values x = [x for x in range(y.shape[0])] plt.plot(x, y) plt.show()
from avera.datamanagement.SymbolDataManager import SymbolDataManager from avera.envs.SimpleEnv import SimpleEnv df = SymbolDataManager().getData("TEST", "H1") #featureScaler = FeatureScaler() #df = featureScaler.extractFeature(df) simpleEnv = SimpleEnv(df) obs = simpleEnv.reset() print(str(obs[0]) + " | " + str(obs[1])) #check buy obs, reward, done, info = simpleEnv.step(0) print(str(obs[0]) + " | " + str(obs[1])) print(reward) #check hold obs, reward, done, info = simpleEnv.step(1) print(str(obs[0]) + " | " + str(obs[1])) print(reward) #check sell obs, reward, done, info = simpleEnv.step(2) print(str(obs[0]) + " | " + str(obs[1])) print(reward)