def _generateConfigToRoot(self, ainstPkgDir, aicfInfo, settingMap, confDict):
if aicfInfo:
for path, configInfo in aicfInfo.configs.iteritems():
srcConfigPath = ainstPkgDir + '/' + path
destConfigPath = self._ainstRoot.getRoot() + '/' + path
if not file_util.isFile(srcConfigPath):
Log.cout(Log.ERROR, 'Config file %s is not exists' % srcConfigPath)
return False
if not file_util.exists(destConfigPath):
Log.cout(Log.ERROR, 'Dest config file %s is not exists' % destConfigPath)
return False
tmpDirName = self._ainstRoot.getRootVarAinstDir('tmp')
tmpPath = tmpDirName + '/' + os.path.basename(destConfigPath) + '.tmp.set'
if not file_util.move(destConfigPath, tmpPath):
Log.cout(Log.ERROR, 'Backup config file %s failed' % destConfigPath)
return False
confDict[path] = (tmpPath, destConfigPath)
configGenerator = ConfigGenerator()
if not configGenerator.generateConfig(srcConfigPath,
destConfigPath,
configInfo.mode,
configInfo.noReplace,
settingMap):
Log.cout(Log.ERROR, 'Generate Config file %s failed' % path)
return False
else:
Log.cout(Log.DEBUG, 'No aicf file, so no config will be changed')
return True
def grid_search(env_name, params_grid, load):
conf_gen = ConfigGenerator()
procs = []
train_runner = TrainRunner(env_name)
params_grid = list(ParameterGrid(params_grid))
for params in params_grid:
run_id = str(params).strip("{}").replace(': ',
'').replace('\'', '').replace(
', ', '_')
conf_path = conf_gen.generate(env_name,
params,
run_id,
params_dict_format=True)
proc = train_runner.start_train_process(conf_path, run_id, load)
procs.append(proc)
def signal_handler(sig, frame):
print('You pressed Ctrl+C!')
for proc in procs:
terminate_proc(sig, proc)
sys.exit(0)
signal.signal(signal.SIGINT, signal_handler)
pool = ThreadPool(len(procs))
pool.map(output_reader, procs)
pool.close()
pool.join()
def __init__(self):
self.config_generator = ConfigGenerator()
# Path to MT4 launcher
self.mt4_demo_launcher = r"C:\MT4\mt4_demo\terminal.exe"
self.mt4_live_launcher = r"C:\MT4\mt4_live\terminal.exe"
# Path to MT5 launcher
self.mt5_demo_launcher = r"C:\MT5\mt5_demo\terminal64.exe"
self.mt5_live_launcher = r"C:\MT5\mt5_live\terminal64.exe"
def _generateConfigToRoot(self, rpmFileInfoList, aicfInfo, settingMap,
confSet):
configGenerator = ConfigGenerator()
if aicfInfo:
for path, configInfo in aicfInfo.configs.iteritems():
srcConfigPath = self._ainstPkgDir + '/' + path
destConfigPath = self._ainstRoot.getRoot() + '/' + path
if not file_util.isFile(srcConfigPath):
Log.cout(
Log.ERROR,
'Config file %s is not exists in pkg' % srcConfigPath)
return False
if not file_util.remove(destConfigPath):
Log.cout(Log.ERROR,
'Remove path %s failed in pkg' % destConfigPath)
return False
if not configGenerator.generateConfig(
srcConfigPath, destConfigPath, configInfo.mode,
configInfo.noReplace, settingMap):
Log.cout(Log.ERROR,
'Generate Config file %s failed' % path)
return False
confSet.add(path)
for rpmFileInfo in rpmFileInfoList:
if rpmFileInfo.isConfigFile():
path = rpmFileInfo.relativePath
if path in confSet:
continue
srcConfigPath = self._ainstPkgDir + '/' + path
destConfigPath = self._ainstRoot.getRoot() + '/' + path
if not file_util.isFile(srcConfigPath):
Log.cout(
Log.ERROR,
'Config file %s is not exists in pkg' % srcConfigPath)
return False
if not configGenerator.generateConfig(srcConfigPath,
destConfigPath):
Log.cout(Log.ERROR,
'Generate Config file %s failed' % path)
return False
confSet.add(path)
return True
def config_generator():
"""Create a ConfigGenerator instance."""
# read ConfigGenerator config file
try:
with open(config_file) as f:
# parse config JSON with original order of keys
config = json.load(f, object_pairs_hook=OrderedDict)
except Exception as e:
msg = "Error loading ConfigGenerator config:\n%s" % e
app.logger.error(msg)
raise Exception(msg)
# create ConfigGenerator
return ConfigGenerator(config, app.logger)
class TrainRunner(object):
def __init__(self, env_name):
self.env_name = env_name
self.conf_gen = ConfigGenerator()
def f(self, params):
"""
Function to optimize
Runs a training process and wait for its termination
"""
run_id = self.env_name + '_' + datetime.now().strftime(
"%Y-%m-%d_%H-%M-%S.%f")
conf_path = self.conf_gen.generate(self.env_name,
params,
run_id,
params_dict_format=False)
proc = self.start_train_process(conf_path, run_id)
for line in iter(proc.stdout.readline, b''):
print('[{0}] {1}'.format(proc.pid, line.decode('utf-8')), end='')
proc.wait()
reward = SummariesReader(run_id).get_scalar(
'Info/cumulative_reward')[-1].value
return reward
def start_train_process(self, conf_path, run_id, options=None):
unused_port = portpicker.pick_unused_port()
command = [
'python', 'learn.py', self.env_name, '--train',
'--worker-id=' + str(unused_port),
'--trainer-config-path=' + str(conf_path), '--run-id=' + run_id
]
if options:
command.append(options)
proc = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
return proc
def __init__(self, env_name):
self.env_name = env_name
self.conf_gen = ConfigGenerator()
class MTLauncher():
def __init__(self):
self.config_generator = ConfigGenerator()
# Path to MT4 launcher
self.mt4_demo_launcher = r"C:\MT4\mt4_demo\terminal.exe"
self.mt4_live_launcher = r"C:\MT4\mt4_live\terminal.exe"
# Path to MT5 launcher
self.mt5_demo_launcher = r"C:\MT5\mt5_demo\terminal64.exe"
self.mt5_live_launcher = r"C:\MT5\mt5_live\terminal64.exe"
def launchMT4(self, mt4_type, mode):
# 0 means first time launch during installation (blocking mode)
# 1 means launch during strategy testing (blocking mode)
# 2 means persistent launch (non-blocking mode)
# mt_version = Account:mt4_demo, Account:mt4_live
time.sleep(5)
if (mode == 0
): # During Installation. No need of configuration file here
# Launch MT4 in portable mode for the first time without configuration file
mt4_demo_process = subprocess.Popen(
[self.mt4_demo_launcher, "/portable"],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
# For debugging
print("[Debug] MT4 process id is " + str(mt4_demo_process.pid) +
". Please continue to wait...")
# TODO: Launch MT4 Live account for the first time
# Sleep for 30 seconds to allow default directories to be created
# within portable MT4 directory
time.sleep(30)
mt4_demo_process.kill()
# TODO: Kill MT4 Live account launched for the first time
# Sleep for 60 seconds more to allow safe deletion of default directories
time.sleep(60)
elif (
mode == 1
): # During Strategy Testing. Uses existing configuration file with shutdown set to true
mt4_demo_process = subprocess.call([
self.mt4_demo_launcher, self.config_generator.config_file,
"/portable"
])
else: # During normal launch of MT4. Uses configuration file with shutdown set to false
self.config_generator.generateConfigFile(mt4_type)
mt4_demo_process = subprocess.Popen([
self.mt4_demo_launcher, self.config_generator.config_file,
"/portable"
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
def launchMT5(self, mt5_type, mode):
# 0 means first time launch during installation (blocking mode)
# 1 means launch during strategy testing (blocking mode)
# 2 means persistent launch (non-blocking mode)
# mt_version =Account:mt5_demo, Account:mt5_live
# Sleep a little before and after launchig the application
time.sleep(5)
if (mode == 0
): # During Installation. No need of configuration file here
# Launch MT5 in portable mode for the first time without configuration file for mt5
mt5_demo_process = subprocess.Popen(
[self.mt5_demo_launcher, "/portable"],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
# For debugging
print("[Debug] MT5 process id is " + str(mt5_demo_process.pid) +
". Please continue to wait...")
# TODO: Launch MT5 Live account for the first time
# Sleep for 30 seconds to allow default directories to be created
# within portable MT5 directory
time.sleep(30)
mt5_demo_process.kill()
# TODO: Kill MT5 Live account launched for the first time
# Sleep for 60 seconds more to allow safe deletion of default directories
time.sleep(60)
elif (
mode == 1
): # During Strategy Testing. Uses existing configuration file with shutdown set to true
mt5_demo_process = subprocess.call([
self.mt5_demo_launcher, "/config:",
self.config_generator.config_file, "/portable"
])
else: # During normal launch of MT5. Uses configuration file with shutdown set to false
self.config_generator.generateConfigFile(mt5_type)
mt5_demo_process = subprocess.Popen([
self.mt5_demo_launcher, "/config:",
self.config_generator.config_file, "/portable"
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
def launchStrategyTester(self):
print("Strategy Tester Running")
# Open and read all test cases
f = open(self.strategyTests, "r")
strategyTests= (f.read()).splitlines()
count =1
# Loop theough each test case and generate configuration files
for i in strategyTests:
platform = (i.split("|")[0]).lower()
account = "Account:"+platform+"_demo"
expert = "Expert:"+i.split("|")[1]
period = "Period:"+i.split("|")[2]
symbol = "Symbol:"+i.split("|")[3]
mode = "StrategyTester:-"
# Source of input must be in .txt format.
# Destination must be in .set format
if(platform=="mt4"):
# Get all inputs for current Expert
custom_inputs=glob.glob(os.path.join(self.mt4_custom_inputs_folder,i.split("|")[1],"*.set"))
# input file for testing
test_input = os.path.join(self.mt4_inputs,i.split("|")[1]+".set")
elif(platform=="mt5"):
# Get all inputs for current Expert
custom_inputs=glob.glob(os.path.join(self.mt5_custom_inputs_folder,i.split("|")[1],"*.set"))
# input file for testing
test_input = os.path.join(self.mt5_inputs,i.split("|")[1]+".set")
if(self.currentExpertName != i.split("|")[1]):
self.currentExpertName=i.split("|")[1]
#Save the report
if(self.reportGenerator!=None):
self.reportGenerator.saveReport()
# Generate new report
self.reportGenerator=ReportGenerator(i.split("|")[3],self.currentExpertName,custom_inputs,platform)
for j in custom_inputs:
shutil.copy(j,test_input)
_input="Input: "+i.split("|")[1]
print(_input)
# Generate configuration file for current account, symbol, period and expert
configGenerator = ConfigGenerator()
configGenerator.generateConfigFile(account, period, expert, symbol,mode,_input,platform)
# Now launch MT4 or MT5
if(platform=="mt4"):
self.mtLauncher.launchMT4(account,1) # Launch MT4
elif(platform=="mt5"):
self.mtLauncher.launchMT5(account,1) # Launch MT5
# TODO: Update report
self.reportGenerator.updateReport(i.split("|")[3],i.split("|")[2],j)
print("Test "+str(count)+" of "+str(len(strategyTests))+" completed")
count+=1
# Move this out to save report only once.
self.reportGenerator.saveReport()
print("All tests complete...")
repeated_list = self.reportGenerator.visualization_data["currency_pairs"]
self.reportGenerator.visualization_data["currency_pairs"] = repeated_list[0:int(len(repeated_list)/2)]
with open(self.reportGenerator.json_report, 'w') as outfile:
json.dump(self.reportGenerator.visualization_data, outfile)