def _submitSimCleanupJob(tracker_host, tracker_port, hq_server, parent_job_id,
child_ids):
"""Submits a job to cleanup the simulation when it is complete.
The new job will be a sibling of this job and will have the same children.
"""
hq_cmds = _getHQueueCommands()
commands = hqrop.getJobCommands(hq_cmds, "pythonCommands",
"hq_stop_sim.py")
cleanup_job = {
"name": "Stop Tracker",
"environment": {
"HQCOMMANDS":
hutil.json.utf8Dumps(hq_cmds),
"HQPARMS":
hutil.json.utf8Dumps({
"tracker_host": tracker_host,
"tracker_port": tracker_port
})
},
"tags": ["excludeProgress", "single"],
"command": commands
}
if parent_job_id is not None:
cleanup_job["parentIds"] = [
parent_job_id,
]
cleanup_job["childrenIds"] = child_ids
new_job_ids = _submitHQJobs(hq_server, cleanup_job)
if len(new_job_ids) != 0:
print "Submitted cleanup job"
def _createClusterJobSpec(cluster_number, number_of_clusters, hip_file,
output_driver, cluster_node, enable_perf_mon,
hq_server):
"""Creates the job spec for the cluster job.
"""
hip_name = os.path.basename(hip_file)
rop_name = os.path.basename(output_driver)
cluster_string = _getFormattedIndex(cluster_number, number_of_clusters - 1)
name = "Simulate -> HIP: %s ROP: %s (Cluster %s) " % (hip_name, rop_name,
cluster_string)
hq_cmds = _getHQueueCommands()
commands = hqrop.getJobCommands(hq_cmds, "hythonCommands",
"hq_sim_cluster.py")
hq_parms = {
"cluster_number": cluster_number,
"hip_file": hip_file,
"output_driver": output_driver,
"cluster_node": cluster_node,
"enable_perf_mon": enable_perf_mon,
}
job_spec = {
"name": name,
"environment": {
"HQCOMMANDS": hutil.json.utf8Dumps(hq_cmds),
"HQPARMS": hutil.json.utf8Dumps(hq_parms),
},
"command": commands,
}
# Set the number of cpus that the job will use.
job_info = _getJobCpuAndTagInfo()
if "single" in job_info["tags"]:
job_spec["tags"] = ["single"]
else:
job_spec["cpus"] = job_info["cpus"]
# We set this jobs parent to be parent of the current job
# This parent job will be the empty job that just contains the active jobs
parent_job_id = _getParentJobID(hq_server)
if parent_job_id is not None:
job_spec["parentIds"] = [
parent_job_id,
]
return job_spec
def _byu_troubleshoot_hq(parms):
"""Build and submit the top-level render job."""
if parms["use_output_driver"]:
hip_file, hfs = hqrop.getHipFileAndHFS(parms)
if hip_file is None or hfs is None:
return
else:
hfs = hqrop.getHFS(parms)
if hfs is None:
return
# We submit a top-level job that doesn't do anything except wait for its
# children to finish. This job initially has a single child that we also
# submit at the same time. When we're generating IFDs, that child job
# generates them and then submits render jobs as children of the top-level
# job. When we're not generating IFDs, that child will submit individual
# render jobs to render each batch of frames directly from the hip file.
if parms["batch_all_frames"]:
frames_per_job = -1
else:
frames_per_job = parms["frames_per_job"]
# Build the list of parameters to pass to the child job.
hq_parms = {
"frames_per_job": frames_per_job,
"render_frame_order": parms["render_frame_order"],
"dirs_to_create": parms["dirs_to_create"],
"min_hosts_per_job": parms["min_hosts_per_job"],
"max_hosts_per_job": parms["max_hosts_per_job"],
"use_render_tracker" : parms["use_render_tracker"],
}
if parms["make_ifds"]:
hq_parms.update({
"ifd_path": parms["output_ifd"],
"delete_ifds": parms["delete_ifds"]
})
elif parms["use_output_driver"]:
# The render single tile option is only available
# when rendering from an output driver and not generating IFDs.
hq_parms["render_single_tile"] = parms["render_single_tile"]
if parms["use_output_driver"]:
hq_parms["output_driver"] = parms["output_driver"]
hq_parms["hip_file"] = hip_file
hq_parms["project_name"] = _getProjectName(
parms["hq_server"], hip_file)
else:
hq_parms.update({
"ifd_path" : parms["ifd_path"],
"start_frame" : parms["start_frame"],
"end_frame" : parms["end_frame"],
"frame_skip" : parms["frame_skip"],
"project_name" : _getProjectName(
parms["hq_server"], parms["ifd_path"])
})
# Determine number of cpus per job.
# Zero means to use the maximum number of CPUs.
cpus_per_job = 0
if bool(parms["is_CPU_number_set"]):
cpus_per_job = int(parms["CPUs_to_use"])
# Now build the spec for the child job.
apply_conditions_to_children = True
if parms["make_ifds"]:
script_name = "hq_make_ifds.py"
command_type = "hythonCommands"
child_job = {"name": "Generate IFDs"}
# Set client conditions.
assign_to = parms["assign_ifdgen_to"]
if assign_to == "clients":
child_job["host"] = parms["ifdgen_clients"]
apply_conditions_to_children = False
elif assign_to == "client_groups":
child_job["hostgroup"] = parms["ifdgen_client_groups"]
apply_conditions_to_children = False
elif not parms["use_output_driver"]:
script_name = "hq_prepare_ifd_render.py"
command_type = "pythonCommands"
child_job = {"name": "Prepare IFD Render Jobs"}
else:
script_name = "hq_submit_renders.py"
command_type = "hythonCommands"
child_job = {"name": "Prepare Render Jobs"}
# Set the number of cpus in the child job spec.
if cpus_per_job > 0:
child_job["cpus"] = cpus_per_job
else:
child_job["tags"] = ["single"]
# Build job commands.
hq_cmds = hqrop.getHQueueCommands(hfs, cpus_per_job)
if hq_cmds is None:
return
commands = hqrop.getJobCommands(hq_cmds, command_type, script_name)
# Build the environment that the job will run in.
env_vars = {
"HQCOMMANDS": hutil.json.utf8Dumps(hq_cmds),
"HQPARMS": hutil.json.utf8Dumps(hq_parms),
}
if len(parms["environment"]) > 0:
env_vars["HQ_PRESERVE_ENV_VARS"] = ",".join(parms["environment"].keys())
env_vars.update(parms["environment"])
# Update job spec.
child_job.update({
"environment": env_vars,
"priority" : parms["priority"],
"command": commands
})
hqrop.setEnvironmentVariablesInJobSpec(child_job)
if (parms["name"] != "<default>") and parms["name"]:
name = parms["name"]
elif parms["use_output_driver"]:
name = "Render -> HIP: %s ROP: %s" % (hip_file, parms['output_driver'])
else:
name = 'Render -> IFDs: %s' % (parms["ifd_path"])
# Build and submit the top-level job.
main_job = hqrop.buildContainingJobSpec(
name, hq_cmds, parms, child_job, apply_conditions_to_children)
if cpus_per_job > 0:
main_job["cpus"] = cpus_per_job
else:
main_job["tags"] = ["single"]
hqrop.setEnvironmentVariablesInJobSpec(main_job)
fname = os.path.expandvars('${JOB}/tmp/hq_jobs_troubleshoot.log')
fp = open(fname, 'a')
fp.write('\n\nWriting log for job "' + main_job['name'] + '" at ' + str(datetime.now()) + ':\n')
json.dump(main_job, fp, indent=4)
fp.close()
hqrop.sendJob(parms["hq_server"], main_job, parms["open_browser"])
def _byu_troubleshoot_hq(parms):
"""Build and submit the top-level render job."""
if parms["use_output_driver"]:
hip_file, hfs = hqrop.getHipFileAndHFS(parms)
if hip_file is None or hfs is None:
return
else:
hfs = hqrop.getHFS(parms)
if hfs is None:
return
# We submit a top-level job that doesn't do anything except wait for its
# children to finish. This job initially has a single child that we also
# submit at the same time. When we're generating IFDs, that child job
# generates them and then submits render jobs as children of the top-level
# job. When we're not generating IFDs, that child will submit individual
# render jobs to render each batch of frames directly from the hip file.
if parms["batch_all_frames"]:
frames_per_job = -1
else:
frames_per_job = parms["frames_per_job"]
# Build the list of parameters to pass to the child job.
hq_parms = {
"frames_per_job": frames_per_job,
"render_frame_order": parms["render_frame_order"],
"dirs_to_create": parms["dirs_to_create"],
"min_hosts_per_job": parms["min_hosts_per_job"],
"max_hosts_per_job": parms["max_hosts_per_job"],
"use_render_tracker": parms["use_render_tracker"],
}
if parms["make_ifds"]:
hq_parms.update({
"ifd_path": parms["output_ifd"],
"delete_ifds": parms["delete_ifds"]
})
elif parms["use_output_driver"]:
# The render single tile option is only available
# when rendering from an output driver and not generating IFDs.
hq_parms["render_single_tile"] = parms["render_single_tile"]
if parms["use_output_driver"]:
hq_parms["output_driver"] = parms["output_driver"]
hq_parms["hip_file"] = hip_file
hq_parms["project_name"] = _getProjectName(parms["hq_server"],
hip_file)
else:
hq_parms.update({
"ifd_path":
parms["ifd_path"],
"start_frame":
parms["start_frame"],
"end_frame":
parms["end_frame"],
"frame_skip":
parms["frame_skip"],
"project_name":
_getProjectName(parms["hq_server"], parms["ifd_path"])
})
# Determine number of cpus per job.
# Zero means to use the maximum number of CPUs.
cpus_per_job = 0
if bool(parms["is_CPU_number_set"]):
cpus_per_job = int(parms["CPUs_to_use"])
# Now build the spec for the child job.
apply_conditions_to_children = True
if parms["make_ifds"]:
script_name = "hq_make_ifds.py"
command_type = "hythonCommands"
child_job = {"name": "Generate IFDs"}
# Set client conditions.
assign_to = parms["assign_ifdgen_to"]
if assign_to == "clients":
child_job["host"] = parms["ifdgen_clients"]
apply_conditions_to_children = False
elif assign_to == "client_groups":
child_job["hostgroup"] = parms["ifdgen_client_groups"]
apply_conditions_to_children = False
elif not parms["use_output_driver"]:
script_name = "hq_prepare_ifd_render.py"
command_type = "pythonCommands"
child_job = {"name": "Prepare IFD Render Jobs"}
else:
script_name = "hq_submit_renders.py"
command_type = "hythonCommands"
child_job = {"name": "Prepare Render Jobs"}
# Set the number of cpus in the child job spec.
if cpus_per_job > 0:
child_job["cpus"] = cpus_per_job
else:
child_job["tags"] = ["single"]
# Build job commands.
hq_cmds = hqrop.getHQueueCommands(hfs, cpus_per_job)
if hq_cmds is None:
return
commands = hqrop.getJobCommands(hq_cmds, command_type, script_name)
# Build the environment that the job will run in.
env_vars = {
"HQCOMMANDS": hutil.json.utf8Dumps(hq_cmds),
"HQPARMS": hutil.json.utf8Dumps(hq_parms),
}
if len(parms["environment"]) > 0:
env_vars["HQ_PRESERVE_ENV_VARS"] = ",".join(
parms["environment"].keys())
env_vars.update(parms["environment"])
# Update job spec.
child_job.update({
"environment": env_vars,
"priority": parms["priority"],
"command": commands
})
hqrop.setEnvironmentVariablesInJobSpec(child_job)
if (parms["name"] != "<default>") and parms["name"]:
name = parms["name"]
elif parms["use_output_driver"]:
name = "Render -> HIP: %s ROP: %s" % (hip_file, parms['output_driver'])
else:
name = 'Render -> IFDs: %s' % (parms["ifd_path"])
# Build and submit the top-level job.
main_job = hqrop.buildContainingJobSpec(name, hq_cmds, parms, child_job,
apply_conditions_to_children)
if cpus_per_job > 0:
main_job["cpus"] = cpus_per_job
else:
main_job["tags"] = ["single"]
hqrop.setEnvironmentVariablesInJobSpec(main_job)
fname = os.path.expandvars('${JOB}/tmp/hq_jobs_troubleshoot.log')
fp = open(fname, 'a')
fp.write('\n\nWriting log for job "' + main_job['name'] + '" at ' +
str(datetime.now()) + ':\n')
json.dump(main_job, fp, indent=4)
fp.close()
hqrop.sendJob(parms["hq_server"], main_job, parms["open_browser"])
def _submitSliceJobs(tracker_host, tracker_port, parent_job_id, hq_server,
hip_file, output_driver, controls_node, slice_divs,
num_slices, slice_type, enable_perf_mon, dirs_to_create):
"""Submits child jobs that will process slices in the simulation.
The slice information is obtained from the controls node.
"""
# Figure out how many jobs we need to submit based on the
# number of slices in the simulation.
if slice_type == "volume":
num_slices = slice_divs[0] * slice_divs[1] * slice_divs[2]
# Set slice job name.
hip_name = os.path.basename(hip_file)
rop_name = os.path.basename(output_driver)
# Submit a child job for each slice.
child_ids = []
hq_cmds = _getHQueueCommands()
commands = hqrop.getJobCommands(hq_cmds, "hythonCommands",
"hq_sim_slice.py")
# Keep track of the parent job's cpu and tag info.
# We need to pass them down to the children.
job_info = _getJobCpuAndTagInfo(parent_job_id)
for i in range(num_slices):
# We subtract 1 because slices are zero-based
slice_string = _getFormattedIndex(i, num_slices - 1)
name = ("Simulate -> HIP: %s ROP: %s (Slice %s)" %
(hip_name, rop_name, slice_string))
child_job = {
"name": name,
"environment": {
"HQCOMMANDS":
hutil.json.utf8Dumps(hq_cmds),
"HQPARMS":
hutil.json.utf8Dumps({
"tracker_host": tracker_host,
"tracker_port": tracker_port,
"hip_file": hip_file,
"output_driver": output_driver,
"controls_node": controls_node,
"slice_divs": slice_divs,
"slice_type": slice_type,
"enable_perf_mon": enable_perf_mon,
"slice_num": i,
"dirs_to_create": dirs_to_create
})
},
"command": commands
}
# Set the number of cpus that the job will use.
if "single" in job_info["tags"]:
child_job["tags"] = ["single"]
else:
child_job["cpus"] = job_info["cpus"]
if parent_job_id is not None:
child_job["parentIds"] = [
parent_job_id,
]
new_job_ids = _submitHQJobs(hq_server, child_job)
if len(new_job_ids) != 0:
print "Submitted job for slice ", i
child_ids.append(new_job_ids[0])
return child_ids
def _submitRenderFromIFDsJob(base_name, ifd_path, frames, ifds, image_paths,
commandline_args, min_hosts_per_job,
max_hosts_per_job, project_name, delete_ifds,
use_render_tracker, cross_platform_ifd):
"""Submit a new job to HQueue to render an IFD using Mantra.
Note that job invokes hq_mantra.py with a standard Python interpreter.
"""
frames_str = " ".join(str(f) for f in frames)
name = _generateRenderJobName(base_name, frames, ifd_path)
hq_cmds = _getHQueueCommands()
commands = hqrop.getJobCommands(hq_cmds, "mantraCommands")
job = {
"name": name,
"environment": {
"HQCOMMANDS":
hutil.json.utf8Dumps(hq_cmds),
"HQPARMS":
hutil.json.utf8Dumps({
"ifds": ifds,
"frames": frames,
"image_paths": image_paths,
"delete_ifds": delete_ifds,
"project_name": project_name,
"commandline_args": commandline_args,
"use_render_tracker": use_render_tracker,
}),
},
"minHosts": str(min_hosts_per_job),
"maxHosts": str(max_hosts_per_job),
"command": commands
}
# Restrict the render job to machines that have the same
# operating system as the machine generating the IFDs.
# We have to place this restriction because IFDs embed hard-coded
# file paths which differ between OS's.
hq_client_arch = os.environ["HQCLIENTARCH"]
operating_system = hq_client_arch.split("-")[0]
if cross_platform_ifd:
job["conditions"] = []
else:
job["conditions"] = [{
"type": "client",
"name": "os",
"op": "==",
"value": operating_system
}]
# Set the number of cpus that the render job will have.
job_info = _getJobCpuAndTagInfo()
if "single" in job_info["tags"]:
job["tags"] = ["single"]
else:
job["cpus"] = job_info["cpus"]
hq_server = _newHQServerConnection()
parent_job_id = _getParentJobID()
if parent_job_id is not None:
# Make the new job a child of this job's parent.
job["parentIds"] = [
parent_job_id,
]
# Also add any conditions coming from the parent job.
parent_job = hq_server.getinfo(parent_job_id)
job["conditions"].extend(parent_job["conditions"])
_submitHQJobs(hq_server, job)