def attempt_test(self):
if self.is_local_simulation():
pass # No test submissions for local simulations
else:
monitor_thread = regression_hpc_monitor.HpcMonitor(
"TestJob",
None,
None,
self.params,
self.params.label,
None,
None,
priority="Highest")
# Test whether we can submit jobs to the cluster as specified
try:
monitor_thread.test_submission()
except (subprocess.TimeoutExpired, AssertionError) as ex:
print(
"FAILED to submit test job to the cluster. Make sure that you have valid credentials "
"cached with the cluster and that the cluster at the specified address is available."
)
raise ex
def commissionFromConfigJson(self,
sim_id,
reply_json,
config_id,
report,
compare_results_to_baseline=True):
# compare_results_to_baseline means we're running regression, compare results to reference.
# opposite/alternative is sweep, which means we don't do comparison check at end
# now we have the config_json, find out if we're commissioning locally or on HPC
def is_local_simulation(some_json, id):
if os.name == "posix":
return True
return self.params.local_execution
sim_dir = os.path.join(self.params.sim_root, sim_id)
bin_dir = os.path.join(self.params.bin_root,
self.dtk_hash) # may not exist yet
is_local = is_local_simulation(reply_json, config_id)
if is_local:
print("Commissioning locally (not on cluster)!")
sim_dir = os.path.join(self.params.local_sim_root, sim_id)
bin_dir = os.path.join(self.params.local_bin_root,
self.dtk_hash) # may not exist yet
# else:
# print( "HPC!" )
# create unique simulation directory
self.sim_dir_sem.acquire()
os.makedirs(sim_dir)
self.sim_dir_sem.release()
# only copy binary if new to us; copy to bin/<md5>/Eradication.exe and run from there
# Would like to create a symlink and run from the sim dir, but can't do that on cluster; no permissions!
# JPS - can't we just check for existence of that file? This seems overly complicated...
# check in bin_dir to see if our binary exists there...
foundit = False
bin_path = os.path.join(
bin_dir,
"Eradication" if os.name == "posix" else "Eradication.exe")
if os.path.exists(bin_dir):
if os.path.exists(bin_path):
foundit = True
else:
os.makedirs(bin_dir)
if not foundit:
print("We didn't have it, copy it up...")
shutil.copy(self.params.executable_path, bin_path)
print("Copied!")
reply_json["bin_path"] = bin_path
reply_json["executable_hash"] = self.dtk_hash
# JPS - not sure what some of the rest of this stuff does
# campaign_json is non, and config.json contains double the stuff it needs in the sim dir... :-(
# tease out campaign json, save separately
campaign_json = json.dumps(reply_json["campaign_json"]).replace(
"u'", "'").replace("'", '"').strip('"')
reply_json["campaign_json"] = None
# tease out custom_reports json, save separately
if reply_json["custom_reports_json"] is not None:
reports_json = json.dumps(
reply_json["custom_reports_json"]).replace("u'", "'").replace(
"'", '"').strip('"')
reply_json["custom_reports_json"] = None
# save custom_reports.json
f = open(sim_dir + "/custom_reports.json", 'w')
#f.write( json.dumps( reports_json, sort_keys=True, indent=4 ) )
f.write(str(reports_json))
f.close()
# Use a local variable here because we don't want the PSP in the config.json that gets written out to disk
# but we need it passed through to the monitor thread execution in the reply_json/config_json.
py_input = None
if "Python_Script_Path" in reply_json["parameters"]:
psp_param = reply_json["parameters"]["Python_Script_Path"]
if psp_param == "LOCAL" or psp_param == ".": # or . is for new usecase when using existing config.json (SFT)
py_input = "."
for py_file in glob.glob(os.path.join(config_id, "dtk_*.py")):
self.copy_sim_file(config_id, sim_dir,
os.path.basename(py_file))
elif psp_param == "SHARED":
py_input = self.params.py_input
for py_file in glob.glob(os.path.join(config_id, "dtk_*.py")):
self.copy_sim_file(config_id, sim_dir,
os.path.basename(py_file))
for py_file in glob.glob(
os.path.join("shared_embedded_py_scripts",
"dtk_*.py")):
self.copy_sim_file("shared_embedded_py_scripts", py_input,
os.path.basename(py_file))
elif psp_param != "NO":
print(
psp_param +
" is not a valid value for Python_Script_Path. Valid values are NO, LOCAL, SHARED. Exiting."
)
sys.exit()
del (reply_json["parameters"]["Python_Script_Path"])
self.copy_input_files_to_user_input(sim_id, config_id, reply_json,
is_local)
#print "Writing out config and campaign.json."
# save config.json
f = open(sim_dir + "/config.json", 'w')
f.write(json.dumps(reply_json, sort_keys=True, indent=4))
f.close()
# now that config.json is written out, add Py Script Path back (if non-empty)
if py_input is not None:
reply_json["PSP"] = py_input
# save campaign.json
f = open(sim_dir + "/campaign.json", 'w')
#f.write( json.dumps( campaign_json, sort_keys=True, indent=4 ) )
f.write(str(campaign_json))
f.close()
f = open(sim_dir + "/emodules_map.json", 'w')
f.write(json.dumps(self.emodules_map, sort_keys=True, indent=4))
f.close()
# ------------------------------------------------------------------
# If you uncomment the following line, it will copy the program database
# file to the directory where a simulation will run (i.e. with the config.json file).
# This will help you get a stack trace with files and line numbers.
# ------------------------------------------------------------------
#print( "Copying PDB file...." )
#shutil.copy( "../Eradication/x64/Release/Eradication.pdb", sim_dir )
# ------------------------------------------------------------------
if os.path.exists(os.path.join(config_id, "dtk_post_process.py")):
self.copy_sim_file(config_id, sim_dir, "dtk_post_process.py")
monitorThread = None # need scoped here
#print "Creating run & monitor thread."
if is_local_simulation(reply_json, config_id):
monitorThread = regression_local_monitor.Monitor(
sim_id, config_id, report, self.params, reply_json,
compare_results_to_baseline)
else:
monitorThread = regression_hpc_monitor.HpcMonitor(
sim_id, config_id, report, self.params, self.params.label,
reply_json, compare_results_to_baseline)
#monitorThread.daemon = True
monitorThread.daemon = False
#print "Starting run & monitor thread."
monitorThread.start()
#print "Monitor thread started, notify data service, and return."
return monitorThread