from avera.feature_generators.W2VCompositeGenerator import W2VCompositeGenerator
from avera.feature_generators.W2VScaleGenerator import W2VScaleGenerator
from avera.feature_generators.W2VRelativeDiffGenerator import W2VRelativeDiffGenerator
from avera.datamanagement.SymbolDataManager import SymbolDataManager
from avera.datamanagement.SymbolDataUpdater import SymbolDataUpdater
from avera.terminal.MT5Terminal import MT5Terminal
from avera.mods.HeikenAshiMod import HeikenAshiMod
symbol = "EURUSD_i"
timeframe = "M15"
hkFeatList = ["open", "close", "low", "high"]
volumeFeatList = ["tick_volume"]
#terminal = MT5Terminal()
#dataUpdater = SymbolDataUpdater("../../data/raw/")
dataManager = SymbolDataManager("../../data/raw/")
#dataUpdater.fullUpdate(terminal, symbol, timeframe, startDate="2015-01-01 00:00:00")
df = SymbolDataManager().getData(symbol, timeframe)
df = df.tail(10000)
########
#df = df.tail(10000)
#print("applying heiken ashi mod...")
#enMod = HeikenAshiMod()
#df = enMod.modHistory(df)
#enMod.checkQuality(df)
########
"""priceDiffGenerator = W2VRelativeDiffGenerator(featureList=hkFeatList, nDiffs=1, nPoints = 21, flatStack = False, fitOnStep = False,
nIntervals = 10000, w2vSize=32, window=3, iter=100, min_count=0, sample=0.0, sg=0)
volumeDiffGenerator = W2VRelativeDiffGenerator(featureList=volumeFeatList, nDiffs=1, nPoints = 21, flatStack = False, fitOnStep = False,
sellerAgent = DQNAgent(testEnv.observation_space["seller"], testEnv.action_space["seller"].n)
agent = CompositeAgent(openerAgent, buyerAgent, sellerAgent)
agent = agent.load_agent("./", "best_composite")
print("start using agent")
dealsStatistics = agent.use_agent(testEnv)
return dealsStatistics
#########################################################################################################
timeframe = "M15"
symbolList = ["GBPCHF_i", "NZDJPY_i", "AUDCAD_i", "EURUSD_i"]
priceFeatList = ["open", "close", "low", "high"]
volumeFeatList = ["tick_volume"]
spreadDict = { "GBPCHF_i": 83, "NZDJPY_i": 37, "AUDCAD_i": 83, "EURUSD_i": 18 }
spreadCoefDict = { "GBPCHF_i": 0.00001, "NZDJPY_i": 0.001, "AUDCAD_i": 0.00001, "EURUSD_i": 0.00001 }
terminal = MT5Terminal()
dataUpdater = SymbolDataUpdater("../../data/raw/")
dataManager = SymbolDataManager("../../data/raw/")
###########################################################################################################
for symbol in symbolList:
dataUpdater.fullUpdate(terminal, symbol, timeframe, startDate="2016-01-01 00:00:00")
df = SymbolDataManager("../../data/raw/").getData(symbol, timeframe)
createGenerators(df, priceFeatList, volumeFeatList)
trainTestSets = getTrainTestSets(df, nExperiments=10, mainPackSize=2044, trainSize=0.5)
trainDealsStatistics = []
testDealsStatistics = []
nStep = 0
for trainX, testX in trainTestSets:
nStep += 1
print("Experiment №{}".format(nStep))
trainAgent(trainX.copy(), spread=spreadDict[symbol], spreadCoef=spreadCoefDict[symbol])
"EURUSD_i": 18,
"AUDUSD_i": 18,
"GBPUSD_i": 18,
"USDCHF_i": 18,
"USDCAD_i": 18
}
spreadCoefDict = {
"EURUSD_i": 0.00001,
"AUDUSD_i": 0.00001,
"GBPUSD_i": 0.00001,
"USDCHF_i": 0.00001,
"USDCAD_i": 0.00001
}
terminal = MT5Terminal()
dataUpdater = SymbolDataUpdater()
dataManager = SymbolDataManager()
###########################################################################################################
for symbol in symbolList:
dataUpdater.fullUpdate(terminal,
symbol,
timeframe,
startDate="2015-01-01 00:00:00")
saveEpInfo = []
for symbol in symbolList:
df = SymbolDataManager().getData(symbol, timeframe)
########
historyMod = VSASpread()
df = historyMod.modHistory(df)
historyMod = HeikenAshiMod()
import matplotlib.pyplot as plt
from datetime import datetime
import numpy as np
from sklearn.linear_model import HuberRegressor
startTime = datetime.now()
print("start time: {}".format(startTime))
symbol = "EURUSD_i"
timeframe = "H1"
hkFeatList = [
"open", "close", "low", "high", "vsa_spread", "tick_volume", "hkopen",
"hkclose", "enopen", "enclose", "enlow", "enhigh"
]
dataManager = SymbolDataManager()
df = SymbolDataManager().getData(symbol, timeframe)
df = df.tail(10000)
########
historyMod = HeikenAshiMod()
modDf = historyMod.modHistory(df)
########
nIntervals = 100
nLevels = 8
featVals = modDf["hkclose"].values
minVal = np.min(featVals)
maxVal = np.max(featVals)
step = (maxVal - minVal) / nIntervals
borders = []
"EURUSD_i": 18,
"AUDUSD_i": 18,
"GBPUSD_i": 18,
"USDCHF_i": 18,
"USDCAD_i": 18
}
spreadCoefDict = {
"EURUSD_i": 0.00001,
"AUDUSD_i": 0.00001,
"GBPUSD_i": 0.00001,
"USDCHF_i": 0.00001,
"USDCAD_i": 0.00001
}
#terminal = MT5Terminal()
dataUpdater = SymbolDataUpdater()
dataManager = SymbolDataManager()
################################################
def getCumulativeReward(dealsStatistics):
sumRew = 0
cumulativeReward = []
for i in range(len(dealsStatistics)):
sumRew += dealsStatistics[i]
cumulativeReward.append(sumRew)
return cumulativeReward
def convertToLinearSet(dealsStatistics):
dealsStatistics = np.reshape(dealsStatistics, (-1, ))
X = np.array([x for x in range(len(dealsStatistics))])
from avera.feature_generators.FeatureDiff import FeatureDiff
from avera.feature_generators.FeatureScaler import FeatureScaler
from avera.datamanagement.SymbolDataManager import SymbolDataManager
from matplotlib import pyplot as plt
df = SymbolDataManager().getData("EURUSD_i", "H1")
df = FeatureDiff().extractFeature(df, featList=["open"], nDiffs=1)
df = FeatureScaler().extractFeature(df, featList=["open"])
print(df)
print(df["open"].mean())
print(df["open"].std())
y = df["open"].values
x = [x for x in range(y.shape[0])]
plt.plot(x, y)
plt.show()
from avera.feature_generators.W2VScaleGenerator import W2VScaleGenerator
from avera.feature_generators.W2VDiffGenerator import W2VDiffGenerator
from avera.datamanagement.SymbolDataManager import SymbolDataManager
from avera.datamanagement.SymbolDataUpdater import SymbolDataUpdater
from avera.terminal.MT5Terminal import MT5Terminal
from avera.mods.HeikenAshiMod import HeikenAshiMod
symbol = "EURUSD_i"
timeframe = "M15"
hkFeatList = ["hkopen", "hkclose", "low", "high"]
volumeFeatList = ["tick_volume"]
#terminal = MT5Terminal()
#dataUpdater = SymbolDataUpdater()
dataManager = SymbolDataManager()
#dataUpdater.fullUpdate(terminal, symbol, timeframe, startDate="2015-01-01 00:00:00")
df = SymbolDataManager().getData(symbol, timeframe)
########
#df = df.tail(10000)
print("applying heiken ashi mod...")
enMod = HeikenAshiMod()
df = enMod.modHistory(df)
#enMod.checkQuality(df)
########
priceDiffGenerator = W2VDiffGenerator(featureList=hkFeatList, nDiffs=1, nPoints = 21, flatStack = False, fitOnStep = False,
nIntervals = 10000, w2vSize=32, window=3, iter=100, min_count=0, sample=0.0, sg=0)
from avera.datamanagement.SymbolDataManager import SymbolDataManager
from avera.envs.SimpleEnv import SimpleEnv
df = SymbolDataManager().getData("TEST", "H1")
#featureScaler = FeatureScaler()
#df = featureScaler.extractFeature(df)
simpleEnv = SimpleEnv(df)
obs = simpleEnv.reset()
print(str(obs[0]) + " | " + str(obs[1]))
#check buy
obs, reward, done, info = simpleEnv.step(0)
print(str(obs[0]) + " | " + str(obs[1]))
print(reward)
#check hold
obs, reward, done, info = simpleEnv.step(1)
print(str(obs[0]) + " | " + str(obs[1]))
print(reward)
#check sell
obs, reward, done, info = simpleEnv.step(2)
print(str(obs[0]) + " | " + str(obs[1]))
print(reward)
from avera.feature_generators.W2VScaleGenerator import W2VScaleGenerator
from avera.feature_generators.W2VDiffGenerator import W2VDiffGenerator
from avera.datamanagement.SymbolDataManager import SymbolDataManager
from avera.mods.VSASpread import VSASpread
from datetime import datetime
import matplotlib.pyplot as plt
symbol = "EURUSD_i"
timeframe = "H1"
priceFeatList = ["open", "close", "low", "high"]
#energyFeatList = ["vsa_spread"]
volumeFeatList = ["tick_volume"]
#terminal = MT5Terminal()
#dataUpdater = SymbolDataUpdater("../../data/raw/")
dataManager = SymbolDataManager("../../data/raw/")
#dataUpdater.fullUpdate(terminal, symbol, timeframe, startDate="2015-01-01 00:00:00")
df = SymbolDataManager("../../data/raw/").getData(symbol, timeframe)
priceDiffGenerator = W2VDiffGenerator(featureList=priceFeatList,
nDiffs=1,
nPoints=21,
flatStack=False,
fitOnStep=False,
nIntervals=10000,
w2vSize=32,
window=3,
iter=100,
min_count=0,
sample=0.0,
from avera.agents.CompositeAgent import CompositeAgent
from avera.envs.CompositeEnv import CompositeEnv
from avera.envs.RealEnv import RealEnv
from avera.terminal.MT5Terminal import MT5Terminal
#from avera.feature_generators.FeatureScaler import FeatureScaler
from avera.feature_generators.ScalerGenerator import ScalerGenerator
from avera.datamanagement.SymbolDataManager import SymbolDataManager
from avera.datamanagement.SymbolDataUpdater import SymbolDataUpdater
symbol = "TEST"
timeframe = "H1"
obsFeatList = ["open", "close"]
#terminal = MT5Terminal()
#dataUpdater = SymbolDataUpdater()
dataManager = SymbolDataManager()
#dataUpdater.fullUpdate(terminal, symbol, timeframe, startDate="2015-01-01 00:00:00")
df = SymbolDataManager().getData(symbol, timeframe, normalizeNames=True, normalizeDateTime=True)
openerFeatureFactory = ScalerGenerator(featureList=obsFeatList, nPoints=10, flatStack=True, fitOnStep=False)
buyerFeatureFactory = ScalerGenerator(featureList=obsFeatList, nPoints=10, flatStack=True, fitOnStep=False)
sellerFatureFactory = ScalerGenerator(featureList=obsFeatList, nPoints=10, flatStack=True, fitOnStep=False)
openerFeatureFactory.globalFit(df[:int(len(df)*0.9)])
buyerFeatureFactory.globalFit(df[:int(len(df)*0.9)])
sellerFatureFactory.globalFit(df[:int(len(df)*0.9)])
print("data updated")
print("start train")
################################
# train agent
################################
from avera.terminal.MT5Terminal import MT5Terminal
from avera.datamanagement.SymbolDataUpdater import SymbolDataUpdater
from avera.datamanagement.SymbolDataManager import SymbolDataManager
from datetime import datetime, timedelta
from dateutil import parser
import time
terminal = MT5Terminal()
symbolUpdater = SymbolDataUpdater()
symbolManager = SymbolDataManager()
symbol = "EURUSD_i"
timeframe = "M5"
symbolUpdater.partialUpdate(terminal, symbol, timeframe)
rawData = symbolManager.getData(symbol, timeframe)
lastRow = rawData.tail(1)
lastDateTime = parser.parse(lastRow["datetime"].values[0])
while (True):
symbolUpdater.partialUpdate(terminal, symbol, timeframe)
rawData = symbolManager.getData(symbol, timeframe)
lastRow = rawData.tail(1)
freshDateTime = parser.parse(lastRow["datetime"].values[0])
if freshDateTime != lastDateTime:
lastDateTime = freshDateTime
break
else:
time.sleep(10)
# place order (buy/sell by market)
orderType = "sell"
lotSize = 0.01
from avera.feature_generators.ScalerGenerator import ScalerGenerator
from avera.feature_generators.FeatureScaler import FeatureScaler
from avera.datamanagement.SymbolDataManager import SymbolDataManager
from matplotlib import pyplot as plt
from dateutil import parser
featureList = ["open", "close"]
df = SymbolDataManager().getData("TEST",
"H1",
normalizeDateTime=True,
normalizeNames=True)
df.set_index("datetime", drop=True, inplace=True)
featureScaler = ScalerGenerator(featureList=featureList,
nPoints=4,
flatStack=True)
featureScaler = featureScaler.globalFit(df)
print(df.tail(1))
lastDt = list(df.tail(1).index)[0]
print(lastDt)
print(df.loc[lastDt])
print(featureScaler.getFeatByDatetime(lastDt, df))
featureScaler = FeatureScaler()
df = featureScaler.extractFeature(df, featureList)
print(df.loc[lastDt])