def add_node(self, node):
""" Add a node to this workflow
This function adds nodes to the workflow. It also determines
parent/child relations from the DataStorage inputs to this job.
Parameters
----------
node : pycbc.workflow.pegasus_workflow.Node
A node that should be executed as part of this workflow.
"""
node._finalize()
node.in_workflow = self
# Record the executable that this node uses
if not node.executable.in_workflow:
for exe in self._executables:
if node.executable.is_same_as(exe):
node.executable.in_workflow = True
node._dax_node.name = exe.logical_name
node.executable.logical_name = exe.logical_name
break
else:
node.executable.in_workflow = True
self._executables += [node.executable]
# Add the node itself
self._adag.addJob(node._dax_node)
# Determine the parent child relationships based on the inputs that
# this node requires.
added_nodes = []
for inp in node._inputs:
if inp.node is not None and inp.node.in_workflow == self:
if inp.node not in added_nodes:
parent = inp.node._dax_node
child = node._dax_node
dep = dax.Dependency(parent=parent, child=child)
self._adag.addDependency(dep)
added_nodes.append(inp.node)
elif inp.node is not None and not inp.node.in_workflow:
raise ValueError('Parents of this node must be added to the '
'workflow first.')
elif inp.node is None and not inp.workflow_input:
self._inputs += [inp]
inp.workflow_input = True
elif inp.node is not None and inp.node.in_workflow != self and inp not in self._inputs:
self._inputs += [inp]
self._external_workflow_inputs += [inp]
# Record the outputs that this node generates
self._outputs += node._outputs
return self
def make_results_web_page(workflow, results_dir, explicit_dependencies=None):
template_path = 'templates/orange.html'
out_dir = workflow.cp.get('results_page', 'output-path')
makedir(out_dir)
node = PlotExecutable(workflow.cp, 'results_page', ifos=workflow.ifos,
out_dir=out_dir).create_node()
node.add_opt('--plots-dir', results_dir)
node.add_opt('--template-file', template_path)
workflow += node
if explicit_dependencies is not None:
import Pegasus.DAX3 as dax
for dep in explicit_dependencies:
dax_dep = dax.Dependency(parent=dep._dax_node,
child=node._dax_node)
workflow._adag.addDependency(dax_dep)
def _make_root_dependency(self, inp):
def root_path(v):
path = [v]
while v.in_workflow:
path += [v.in_workflow]
v = v.in_workflow
return path
workflow_root = root_path(self)
input_root = root_path(inp)
for step in workflow_root:
if step in input_root:
common = step
break
dep = dax.Dependency(
parent=input_root[input_root.index(common) - 1].as_job,
child=workflow_root[workflow_root.index(common) - 1].as_job)
common._adag.addDependency(dep)
def add_node(self, node):
""" Add a node to this workflow
This function adds nodes to the workflow. It also determines
parent/child relations from the DataStorage inputs to this job.
Parameters
----------
node : Node
A node that should be exectuded as part of this workflow.
"""
node._finalize()
node.in_workflow = True
self._adag.addJob(node._dax_node)
# Determine the parent child relationships based on the inputs that
# this node requires.
for inp in node._inputs:
if inp.node is not None and inp.node.in_workflow:
parent = inp.node._dax_node
child = node._dax_node
dep = dax.Dependency(parent=parent, child=child)
self._adag.addDependency(dep)
elif inp.node is not None and not inp.node.in_workflow:
raise ValueError('Parents of this node must be added to the '
'workflow first.')
elif inp.node is None and inp.workflow_input is False:
self._inputs += [inp]
inp.workflow_input = True
# Record the outputs that this node generates
self._outputs += node._outputs
# Record the executable that this node uses
if not node.executable.in_workflow:
node.executable.in_workflow = True
self._executables += [node.executable]
return self
def setup_foreground_minifollowups(workflow, coinc_file, single_triggers,
tmpltbank_file, insp_segs, insp_data_name,
insp_anal_name, dax_output, out_dir, tags=None):
""" Create plots that followup the Nth loudest coincident injection
from a statmap produced HDF file.
Parameters
----------
workflow: pycbc.workflow.Workflow
The core workflow instance we are populating
coinc_file:
single_triggers: list of pycbc.workflow.File
A list cointaining the file objects associated with the merged
single detector trigger files for each ifo.
tmpltbank_file: pycbc.workflow.File
The file object pointing to the HDF format template bank
insp_segs: SegFile
The segment file containing the data read and analyzed by each inspiral
job.
insp_data_name: str
The name of the segmentlist storing data read.
insp_anal_name: str
The name of the segmentlist storing data analyzed.
out_dir: path
The directory to store minifollowups result plots and files
tags: {None, optional}
Tags to add to the minifollowups executables
Returns
-------
layout: list
A list of tuples which specify the displayed file layout for the
minifollops plots.
"""
logging.info('Entering minifollowups module')
if not workflow.cp.has_section('workflow-minifollowups'):
logging.info('There is no [workflow-minifollowups] section in configuration file')
logging.info('Leaving minifollowups')
return
tags = [] if tags is None else tags
makedir(dax_output)
# turn the config file into a File class
config_path = os.path.abspath(dax_output + '/' + '_'.join(tags) + 'foreground_minifollowup.ini')
workflow.cp.write(open(config_path, 'w'))
config_file = wdax.File(os.path.basename(config_path))
config_file.PFN(urljoin('file:', pathname2url(config_path)), site='local')
exe = Executable(workflow.cp, 'foreground_minifollowup',
ifos=workflow.ifos, out_dir=dax_output, tags=tags)
node = exe.create_node()
node.add_input_opt('--config-files', config_file)
node.add_input_opt('--bank-file', tmpltbank_file)
node.add_input_opt('--statmap-file', coinc_file)
node.add_multiifo_input_list_opt('--single-detector-triggers',
single_triggers)
node.add_input_opt('--inspiral-segments', insp_segs)
node.add_opt('--inspiral-data-read-name', insp_data_name)
node.add_opt('--inspiral-data-analyzed-name', insp_anal_name)
if tags:
node.add_list_opt('--tags', tags)
node.new_output_file_opt(workflow.analysis_time, '.dax', '--output-file')
node.new_output_file_opt(workflow.analysis_time, '.dax.map', '--output-map')
node.new_output_file_opt(workflow.analysis_time, '.tc.txt',
'--transformation-catalog')
name = node.output_files[0].name
map_file = node.output_files[1]
tc_file = node.output_files[2]
node.add_opt('--workflow-name', name)
node.add_opt('--output-dir', out_dir)
workflow += node
# execute this in a sub-workflow
fil = node.output_files[0]
# determine if a staging site has been specified
try:
staging_site = workflow.cp.get('workflow-foreground_minifollowups',
'staging-site')
except:
staging_site = None
job = dax.DAX(fil)
job.addArguments('--basename %s' % os.path.splitext(os.path.basename(name))[0])
Workflow.set_job_properties(job, map_file, tc_file, staging_site=staging_site)
workflow._adag.addJob(job)
dep = dax.Dependency(parent=node._dax_node, child=job)
workflow._adag.addDependency(dep)
logging.info('Leaving minifollowups module')
def setup_single_det_minifollowups(workflow, single_trig_file, tmpltbank_file,
insp_segs, insp_data_name, insp_anal_name,
dax_output, out_dir, veto_file=None,
veto_segment_name=None, statfiles=None,
tags=None):
""" Create plots that followup the Nth loudest clustered single detector
triggers from a merged single detector trigger HDF file.
Parameters
----------
workflow: pycbc.workflow.Workflow
The core workflow instance we are populating
single_trig_file: pycbc.workflow.File
The File class holding the single detector triggers.
tmpltbank_file: pycbc.workflow.File
The file object pointing to the HDF format template bank
insp_segs: SegFile
The segment file containing the data read by each inspiral job.
insp_data_name: str
The name of the segmentlist storing data read.
insp_anal_name: str
The name of the segmentlist storing data analyzed.
out_dir: path
The directory to store minifollowups result plots and files
statfiles: FileList (optional, default=None)
Supplementary files necessary for computing the single-detector
statistic.
tags: {None, optional}
Tags to add to the minifollowups executables
Returns
-------
layout: list
A list of tuples which specify the displayed file layout for the
minifollops plots.
"""
logging.info('Entering minifollowups module')
if not workflow.cp.has_section('workflow-sngl_minifollowups'):
msg = 'There is no [workflow-sngl_minifollowups] section in '
msg += 'configuration file'
logging.info(msg)
logging.info('Leaving minifollowups')
return
tags = [] if tags is None else tags
makedir(dax_output)
# turn the config file into a File class
curr_ifo = single_trig_file.ifo
config_path = os.path.abspath(dax_output + '/' + curr_ifo + \
'_'.join(tags) + 'singles_minifollowup.ini')
workflow.cp.write(open(config_path, 'w'))
config_file = wdax.File(os.path.basename(config_path))
config_file.PFN(urljoin('file:', pathname2url(config_path)), site='local')
exe = Executable(workflow.cp, 'singles_minifollowup',
ifos=curr_ifo, out_dir=dax_output, tags=tags)
wikifile = curr_ifo + '_'.join(tags) + 'loudest_table.txt'
node = exe.create_node()
node.add_input_opt('--config-files', config_file)
node.add_input_opt('--bank-file', tmpltbank_file)
node.add_input_opt('--single-detector-file', single_trig_file)
node.add_input_opt('--inspiral-segments', insp_segs)
node.add_opt('--inspiral-data-read-name', insp_data_name)
node.add_opt('--inspiral-data-analyzed-name', insp_anal_name)
node.add_opt('--instrument', curr_ifo)
node.add_opt('--wiki-file', wikifile)
if veto_file is not None:
assert(veto_segment_name is not None)
node.add_input_opt('--veto-file', veto_file)
node.add_opt('--veto-segment-name', veto_segment_name)
if statfiles:
statfiles = statfiles.find_output_with_ifo(curr_ifo)
node.add_input_list_opt('--statistic-files', statfiles)
node.new_output_file_opt(workflow.analysis_time, '.dax', '--output-file')
node.new_output_file_opt(workflow.analysis_time, '.dax.map',
'--output-map')
node.new_output_file_opt(workflow.analysis_time, '.tc.txt',
'--transformation-catalog')
name = node.output_files[0].name
map_file = node.output_files[1]
tc_file = node.output_files[2]
node.add_opt('--workflow-name', name)
node.add_opt('--output-dir', out_dir)
workflow += node
# execute this in a sub-workflow
fil = node.output_files[0]
# determine if a staging site has been specified
try:
staging_site = workflow.cp.get('workflow-sngl_minifollowups',
'staging-site')
except:
staging_site = None
job = dax.DAX(fil)
job.addArguments('--basename %s' \
% os.path.splitext(os.path.basename(name))[0])
Workflow.set_job_properties(job, map_file, tc_file,
staging_site=staging_site)
workflow._adag.addJob(job)
dep = dax.Dependency(parent=node._dax_node, child=job)
workflow._adag.addDependency(dep)
logging.info('Leaving minifollowups module')
def setup_injection_minifollowups(workflow,
injection_file,
inj_xml_file,
single_triggers,
tmpltbank_file,
insp_segs,
insp_seg_name,
dax_output,
out_dir,
tags=None):
""" Create plots that followup the closest missed injections
Parameters
----------
workflow: pycbc.workflow.Workflow
The core workflow instance we are populating
coinc_file:
single_triggers: list of pycbc.workflow.File
A list cointaining the file objects associated with the merged
single detector trigger files for each ifo.
tmpltbank_file: pycbc.workflow.File
The file object pointing to the HDF format template bank
insp_segs: dict
A dictionary, keyed by ifo name, of the data read by each inspiral job.
insp_segs_name: str
The name of the segmentlist to read from the inspiral segment file
out_dir: path
The directory to store minifollowups result plots and files
tags: {None, optional}
Tags to add to the minifollowups executables
Returns
-------
layout: list
A list of tuples which specify the displayed file layout for the
minifollops plots.
"""
logging.info('Entering injection minifollowups module')
if not workflow.cp.has_section('workflow-injection_minifollowups'):
logging.info(
'There is no [workflow-injection_minifollowups] section in configuration file'
)
logging.info('Leaving minifollowups')
return
tags = [] if tags is None else tags
makedir(dax_output)
# turn the config file into a File class
config_path = os.path.abspath(dax_output + '/' + '_'.join(tags) +
'injection_minifollowup.ini')
workflow.cp.write(open(config_path, 'w'))
config_file = wdax.File(os.path.basename(config_path))
config_file.PFN(config_path, 'local')
exe = Executable(workflow.cp,
'injection_minifollowup',
ifos=workflow.ifos,
out_dir=dax_output)
node = exe.create_node()
node.add_input_opt('--config-files', config_file)
node.add_input_opt('--bank-file', tmpltbank_file)
node.add_input_opt('--injection-file', injection_file)
node.add_input_opt('--injection-xml-file', inj_xml_file)
node.add_multiifo_input_list_opt('--single-detector-triggers',
single_triggers)
node.add_multiifo_input_list_opt('--inspiral-segments', insp_segs.values())
node.add_opt('--inspiral-segment-name', insp_seg_name)
node.new_output_file_opt(workflow.analysis_time,
'.dax',
'--output-file',
tags=tags)
node.new_output_file_opt(workflow.analysis_time,
'.dax.map',
'--output-map',
tags=tags)
name = node.output_files[0].name
map_loc = node.output_files[1].name
node.add_opt('--workflow-name', name)
node.add_opt('--output-dir', out_dir)
workflow += node
# execute this is a sub-workflow
fil = node.output_files[0]
job = dax.DAX(fil)
job.addArguments('--basename %s' %
os.path.splitext(os.path.basename(name))[0])
Workflow.set_job_properties(job, map_loc)
workflow._adag.addJob(job)
dep = dax.Dependency(parent=node._dax_node, child=job)
workflow._adag.addDependency(dep)
logging.info('Leaving injection minifollowups module')
def setup_foreground_inference(workflow,
coinc_file,
single_triggers,
tmpltbank_file,
insp_segs,
insp_data_name,
insp_anal_name,
dax_output,
out_dir,
tags=None):
""" Creates workflow node that will run the inference workflow.
Parameters
----------
workflow: pycbc.workflow.Workflow
The core workflow instance we are populating
coinc_file: pycbc.workflow.File
The file associated with coincident triggers.
single_triggers: list of pycbc.workflow.File
A list cointaining the file objects associated with the merged
single detector trigger files for each ifo.
tmpltbank_file: pycbc.workflow.File
The file object pointing to the HDF format template bank
insp_segs: SegFile
The segment file containing the data read and analyzed by each inspiral
job.
insp_data_name: str
The name of the segmentlist storing data read.
insp_anal_name: str
The name of the segmentlist storing data analyzed.
dax_output : str
The name of the output DAX file.
out_dir: path
The directory to store inference result plots and files
tags: {None, optional}
Tags to add to the inference executables
"""
logging.info("Entering inference module")
# check if configuration file has inference section
if not workflow.cp.has_section("workflow-inference"):
logging.info(
"There is no [workflow-inference] section in configuration file")
logging.info("Leaving inference module")
return
# default tags is a list
tags = [] if tags is None else tags
# make the directory that will contain the dax file
makedir(dax_output)
# turn the config file into a File class
config_path = os.path.abspath(dax_output + "/" + "_".join(tags) \
+ "foreground_inference.ini")
workflow.cp.write(open(config_path, "w"))
config_file = wdax.File(os.path.basename(config_path))
config_file.PFN(config_path, "local")
# create an Executable for the inference workflow generator
exe = Executable(workflow.cp,
"foreground_inference",
ifos=workflow.ifos,
out_dir=dax_output)
# create the node that will run in the workflow
node = exe.create_node()
node.add_input_opt("--config-files", config_file)
node.add_input_opt("--bank-file", tmpltbank_file)
node.add_input_opt("--statmap-file", coinc_file)
node.add_multiifo_input_list_opt("--single-detector-triggers",
single_triggers)
node.new_output_file_opt(workflow.analysis_time,
".dax",
"--output-file",
tags=tags)
node.new_output_file_opt(workflow.analysis_time,
".dax.map",
"--output-map",
tags=tags)
node.new_output_file_opt(workflow.analysis_time,
".tc.txt",
"--transformation-catalog",
tags=tags)
# get dax name and use it for the workflow name
name = node.output_files[0].name
node.add_opt("--workflow-name", name)
# get output map name and use it for the output dir name
map_file = node.output_files[1]
node.add_opt("--output-dir", out_dir)
# get the transformation catalog name
tc_file = node.output_files[2]
# add this node to the workflow
workflow += node
# create job for dax that will run a sub-workflow
# and add it to the workflow
fil = node.output_files[0]
job = dax.DAX(fil)
job.addArguments("--basename %s" %
os.path.splitext(os.path.basename(name))[0])
Workflow.set_job_properties(job, map_file, tc_file)
workflow._adag.addJob(job)
# make dax a child of the inference workflow generator node
dep = dax.Dependency(parent=node._dax_node, child=job)
workflow._adag.addDependency(dep)
logging.info("Leaving inference module")
def setup_single_det_minifollowups(workflow,
single_trig_file,
tmpltbank_file,
insp_segs,
insp_seg_name,
dax_output,
out_dir,
veto_file=None,
veto_segment_name=None,
tags=None):
""" Create plots that followup the Nth loudest clustered single detector
triggers from a merged single detector trigger HDF file.
Parameters
----------
workflow: pycbc.workflow.Workflow
The core workflow instance we are populating
single_trig_file: pycbc.workflow.File
The File class holding the single detector triggers.
tmpltbank_file: pycbc.workflow.File
The file object pointing to the HDF format template bank
insp_segs: dict
A dictionary, keyed by ifo name, of the data read by each inspiral job.
insp_segs_name: str
The name of the segmentlist to read from the inspiral segment file
out_dir: path
The directory to store minifollowups result plots and files
tags: {None, optional}
Tags to add to the minifollowups executables
Returns
-------
layout: list
A list of tuples which specify the displayed file layout for the
minifollops plots.
"""
logging.info('Entering minifollowups module')
if not workflow.cp.has_section('workflow-minifollowups'):
msg = 'There is no [workflow-minifollowups] section in '
msg += 'configuration file'
logging.info(msg)
logging.info('Leaving minifollowups')
return
tags = [] if tags is None else tags
makedir(dax_output)
# turn the config file into a File class
curr_ifo = single_trig_file.ifo
config_path = os.path.abspath(dax_output + '/' + curr_ifo + \
'_'.join(tags) + 'singles_minifollowup.ini')
workflow.cp.write(open(config_path, 'w'))
config_file = wdax.File(os.path.basename(config_path))
config_file.PFN(config_path, 'local')
exe = Executable(workflow.cp,
'singles_minifollowup',
ifos=curr_ifo,
out_dir=dax_output)
node = exe.create_node()
node.add_input_opt('--config-files', config_file)
node.add_input_opt('--bank-file', tmpltbank_file)
node.add_input_opt('--single-detector-file', single_trig_file)
node.add_input_opt('--inspiral-segments', insp_segs[curr_ifo])
node.add_opt('--inspiral-segment-name', insp_seg_name)
node.add_opt('--instrument', curr_ifo)
if veto_file is not None:
assert (veto_segment_name is not None)
node.add_input_opt('--veto-file', veto_file)
node.add_opt('--veto-segment-name', veto_segment_name)
node.new_output_file_opt(workflow.analysis_time,
'.dax',
'--output-file',
tags=tags)
node.new_output_file_opt(workflow.analysis_time,
'.dax.map',
'--output-map',
tags=tags)
name = node.output_files[0].name
map_loc = node.output_files[1].name
node.add_opt('--workflow-name', name)
node.add_opt('--output-dir', out_dir)
workflow += node
# execute this is a sub-workflow
fil = node.output_files[0]
job = dax.DAX(fil)
job.addArguments('--basename %s' \
% os.path.splitext(os.path.basename(name))[0])
Workflow.set_job_properties(job, map_loc)
workflow._adag.addJob(job)
dep = dax.Dependency(parent=node._dax_node, child=job)
workflow._adag.addDependency(dep)
logging.info('Leaving minifollowups module')
def setup_postproc_coh_PTF_workflow(workflow,
trig_files,
trig_cache,
inj_trig_files,
inj_files,
inj_trig_caches,
inj_caches,
config_file,
output_dir,
html_dir,
segment_dir,
ifos,
inj_tags=[],
tags=[]):
"""
This module sets up the post-processing stage in the workflow, using a
coh_PTF style set up. This consists of running trig_combiner to find
coherent triggers, and injfinder to look for injections. It then runs
a horizon_dist job, trig_cluster to cluster triggers, and injcombiner to
calculate injection statistics. Finally, efficiency and sbv_plotter jobs
calculate efficiency and signal based veto statistics and make plots.
workflow : pycbc.workflow.core.Workflow
The Workflow instance that the jobs will be added to.
trig_files : pycbc.workflow.core.FileList
A FileList containing the combined databases.
Returns
--------
"""
cp = workflow.cp
full_segment = trig_files[0].segment
trig_name = cp.get("workflow", "trigger-name")
grb_string = "GRB" + trig_name
num_trials = int(cp.get("trig_combiner", "num-trials"))
pp_outs = FileList([])
pp_nodes = []
# Set up needed exe classes
trig_combiner_exe = os.path.basename(cp.get("executables",
"trig_combiner"))
trig_combiner_class = select_generic_executable(workflow, "trig_combiner")
trig_cluster_exe = os.path.basename(cp.get("executables", "trig_cluster"))
trig_cluster_class = select_generic_executable(workflow, "trig_cluster")
sbv_plotter_exe = os.path.basename(cp.get("executables", "sbv_plotter"))
sbv_plotter_class = select_generic_executable(workflow, "sbv_plotter")
efficiency_exe = os.path.basename(cp.get("executables", "efficiency"))
efficiency_class = select_generic_executable(workflow, "efficiency")
"""
horizon_dist_exe = os.path.basename(cp.get("executables",
"horizon_dist"))
horizon_dist_class = select_generic_executable(workflow,
"horizon_dist")
"""
html_summary_exe = os.path.basename(cp.get("executables", "html_summary"))
html_summary_class = select_generic_executable(workflow, "html_summary")
# Set up trig_combiner job
trig_combiner_out_tags = ["OFFSOURCE", "ONSOURCE", "ALL_TIMES"]
trig_combiner_jobs = trig_combiner_class(cp,
"trig_combiner",
ifo=ifos,
out_dir=output_dir,
tags=tags)
trig_combiner_node, trig_combiner_outs = trig_combiner_jobs.create_node(\
trig_files, segment_dir, out_tags=trig_combiner_out_tags,
tags=tags)
pp_nodes.append(trig_combiner_node)
workflow.add_node(trig_combiner_node)
pp_outs.extend(trig_combiner_outs)
# Initialise trig_cluster class
trig_cluster_outs = FileList([])
trig_cluster_jobs = trig_cluster_class(cp,
"trig_cluster",
ifo=ifos,
out_dir=output_dir,
tags=tags)
# Set up injfinder jobs
if cp.has_section("workflow-injections"):
injfinder_nodes = []
injcombiner_parent_nodes = []
inj_sbv_plotter_parent_nodes = []
injfinder_exe = os.path.basename(cp.get("executables", "injfinder"))
injfinder_class = select_generic_executable(workflow, "injfinder")
injfinder_jobs = injfinder_class(cp,
"injfinder",
ifo=ifos,
out_dir=output_dir,
tags=tags)
injcombiner_exe = os.path.basename(cp.get("executables",
"injcombiner"))
injcombiner_class = select_generic_executable(workflow, "injcombiner")
injcombiner_jobs = injcombiner_class(cp,
"injcombiner",
ifo=ifos,
out_dir=output_dir,
tags=tags)
injfinder_outs = FileList([])
for inj_tag in inj_tags:
triggers = FileList([file for file in inj_trig_files \
if inj_tag in file.tag_str])
injections = FileList([file for file in inj_files \
if inj_tag in file.tag_str])
trig_cache = [file for file in inj_trig_caches \
if inj_tag in file.tag_str][0]
inj_cache = [file for file in inj_caches \
if inj_tag in file.tag_str][0]
injfinder_node, curr_outs = injfinder_jobs.create_node(\
triggers, injections, segment_dir, tags=[inj_tag])
injfinder_nodes.append(injfinder_node)
pp_nodes.append(injfinder_node)
workflow.add_node(injfinder_node)
injfinder_outs.extend(curr_outs)
if "DETECTION" not in curr_outs[0].tagged_description:
injcombiner_parent_nodes.append(injfinder_node)
else:
inj_sbv_plotter_parent_nodes.append(injfinder_node)
pp_outs.extend(injfinder_outs)
# Make injfinder output cache
fm_cache = File(ifos,
"foundmissed",
full_segment,
extension="lcf",
directory=output_dir)
fm_cache.PFN(fm_cache.cache_entry.path, site="local")
injfinder_outs.convert_to_lal_cache().tofile(\
open(fm_cache.storage_path, "w"))
pp_outs.extend(FileList([fm_cache]))
# Set up injcombiner jobs
injcombiner_outs = FileList([file for file in injfinder_outs \
if "DETECTION" in file.tag_str])
injcombiner_tags = [inj_tag for inj_tag in inj_tags \
if "DETECTION" not in inj_tag]
injcombiner_out_tags = [injcombiner_outs[0].tag_str.rsplit('_', 1)[0]]
injcombiner_nodes = []
for injcombiner_tag in injcombiner_tags:
max_inc = cp.get_opt_tags("injections", "max-inc",
[injcombiner_tag])
inj_str = injcombiner_tag[:4]
inputs = FileList([file for file in injfinder_outs \
if injcombiner_tag in file.tagged_description])
# if any(tag in file.tagged_description \
# for tag in injcombiner_tags)])
injcombiner_node, curr_outs = injcombiner_jobs.create_node(\
fm_cache, inputs, inj_str, max_inc, workflow.analysis_time)
injcombiner_nodes.append(injcombiner_node)
injcombiner_out_tags.append("%s_FILTERED_%s" % (inj_str, max_inc))
injcombiner_outs.extend(curr_outs)
pp_outs.extend(curr_outs)
pp_nodes.append(injcombiner_node)
workflow.add_node(injcombiner_node)
for parent_node in injcombiner_parent_nodes:
dep = dax.Dependency(parent=parent_node._dax_node,
child=injcombiner_node._dax_node)
workflow._adag.addDependency(dep)
# Initialise injection_efficiency class
inj_efficiency_jobs = efficiency_class(cp,
"inj_efficiency",
ifo=ifos,
out_dir=output_dir,
tags=tags)
# Initialise sbv_plotter class
sbv_plotter_outs = FileList([])
sbv_plotter_jobs = sbv_plotter_class(cp,
"sbv_plotter",
ifo=ifos,
out_dir=output_dir,
tags=tags)
# Initialise efficiency class
efficiency_outs = FileList([])
efficiency_jobs = efficiency_class(cp,
"efficiency",
ifo=ifos,
out_dir=output_dir,
tags=tags)
# Add trig_cluster jobs and their corresponding plotting jobs
for out_tag in trig_combiner_out_tags:
unclust_file = [file for file in trig_combiner_outs \
if out_tag in file.tag_str][0]
trig_cluster_node, curr_outs = trig_cluster_jobs.create_node(\
unclust_file)
trig_cluster_outs.extend(curr_outs)
clust_file = curr_outs[0]
if out_tag != "ONSOURCE":
# Add memory requirememnt for jobs with potentially large files
trig_cluster_node.set_memory(1300)
pp_nodes.append(trig_cluster_node)
workflow.add_node(trig_cluster_node)
dep = dax.Dependency(parent=trig_combiner_node._dax_node,
child=trig_cluster_node._dax_node)
workflow._adag.addDependency(dep)
# Add sbv_plotter job
sbv_out_tags = [out_tag, "_clustered"]
sbv_plotter_node = sbv_plotter_jobs.create_node(clust_file,
segment_dir,
tags=sbv_out_tags)
pp_nodes.append(sbv_plotter_node)
workflow.add_node(sbv_plotter_node)
dep = dax.Dependency(parent=trig_cluster_node._dax_node,
child=sbv_plotter_node._dax_node)
workflow._adag.addDependency(dep)
# Add injection sbv_plotter nodes if appropriate
if out_tag == "OFFSOURCE" and \
cp.has_section("workflow-injections"):
offsource_clustered = clust_file
off_node = sbv_plotter_node
found_inj_files = FileList([file for file in injcombiner_outs \
if "FOUND" in file.tag_str])
for curr_injs in found_inj_files:
curr_tags = [tag for tag in injcombiner_out_tags \
if tag in curr_injs.name]
curr_tags.append("_clustered")
sbv_plotter_node = sbv_plotter_jobs.create_node(
clust_file,
segment_dir,
inj_file=curr_injs,
tags=curr_tags)
pp_nodes.append(sbv_plotter_node)
workflow.add_node(sbv_plotter_node)
dep = dax.Dependency(parent=trig_cluster_node._dax_node,
child=sbv_plotter_node._dax_node)
workflow._adag.addDependency(dep)
if "DETECTION" in curr_injs.tagged_description:
for parent_node in inj_sbv_plotter_parent_nodes:
dep = dax.Dependency(
parent=parent_node._dax_node,
child=sbv_plotter_node._dax_node)
workflow._adag.addDependency(dep)
else:
for parent_node in injcombiner_nodes:
dep = dax.Dependency(
parent=parent_node._dax_node,
child=sbv_plotter_node._dax_node)
workflow._adag.addDependency(dep)
# Also add sbv_plotter job for unclustered triggers
sbv_plotter_node = sbv_plotter_jobs.create_node(
unclust_file, segment_dir, tags=[out_tag, "_unclustered"])
sbv_plotter_node.set_memory(1300)
pp_nodes.append(sbv_plotter_node)
workflow.add_node(sbv_plotter_node)
dep = dax.Dependency(parent=trig_combiner_node._dax_node,
child=sbv_plotter_node._dax_node)
workflow._adag.addDependency(dep)
else:
pp_nodes.append(trig_cluster_node)
workflow.add_node(trig_cluster_node)
dep = dax.Dependency(parent=trig_combiner_node._dax_node,
child=trig_cluster_node._dax_node)
workflow._adag.addDependency(dep)
# Add efficiency job for on/off
efficiency_node = efficiency_jobs.create_node(clust_file,
offsource_clustered,
segment_dir,
tags=[out_tag])
pp_nodes.append(efficiency_node)
workflow.add_node(efficiency_node)
dep = dax.Dependency(parent=off_node._dax_node,
child=efficiency_node._dax_node)
workflow._adag.addDependency(dep)
if cp.has_section("workflow-injections"):
for tag in injcombiner_out_tags:
if "_FILTERED_" in tag:
inj_set_tag = [t for t in inj_tags if \
str(tag).replace("_FILTERED_", "") \
in t][0]
else:
inj_set_tag = str(tag)
found_file = [file for file in injcombiner_outs \
if tag + "_FOUND" in file.tag_str][0]
missed_file = [file for file in injcombiner_outs \
if tag + "_MISSED" in file.tag_str][0]
inj_efficiency_node = inj_efficiency_jobs.create_node(\
clust_file, offsource_clustered, segment_dir,
found_file, missed_file, tags=[out_tag, tag,
inj_set_tag])
pp_nodes.append(inj_efficiency_node)
workflow.add_node(inj_efficiency_node)
dep = dax.Dependency(parent=off_node._dax_node,
child=inj_efficiency_node._dax_node)
workflow._adag.addDependency(dep)
for injcombiner_node in injcombiner_nodes:
dep = dax.Dependency(
parent=injcombiner_node._dax_node,
child=inj_efficiency_node._dax_node)
workflow._adag.addDependency(dep)
for injfinder_node in injfinder_nodes:
dep = dax.Dependency(
parent=injfinder_node._dax_node,
child=inj_efficiency_node._dax_node)
workflow._adag.addDependency(dep)
# Add further trig_cluster jobs for trials
trial = 1
while trial <= num_trials:
trial_tag = "OFFTRIAL_%d" % trial
unclust_file = [file for file in trig_combiner_outs \
if trial_tag in file.tag_str][0]
trig_cluster_node, clust_outs = trig_cluster_jobs.create_node(\
unclust_file)
clust_file = clust_outs[0]
trig_cluster_outs.extend(clust_outs)
pp_nodes.append(trig_cluster_node)
workflow.add_node(trig_cluster_node)
dep = dax.Dependency(parent=trig_combiner_node._dax_node,
child=trig_cluster_node._dax_node)
workflow._adag.addDependency(dep)
# Add efficiency job
efficiency_node = efficiency_jobs.create_node(clust_file,
offsource_clustered,
segment_dir,
tags=[trial_tag])
pp_nodes.append(efficiency_node)
workflow.add_node(efficiency_node)
dep = dax.Dependency(parent=off_node._dax_node,
child=efficiency_node._dax_node)
workflow._adag.addDependency(dep)
dep = dax.Dependency(parent=trig_cluster_node._dax_node,
child=efficiency_node._dax_node)
workflow._adag.addDependency(dep)
# Adding inj_efficiency job
if cp.has_section("workflow-injections"):
for tag in injcombiner_out_tags:
if "_FILTERED_" in tag:
inj_set_tag = [t for t in inj_tags if \
str(tag).replace("_FILTERED_", "") in t][0]
else:
inj_set_tag = str(tag)
found_file = [file for file in injcombiner_outs \
if tag + "_FOUND" in file.tag_str][0]
missed_file = [file for file in injcombiner_outs \
if tag + "_MISSED" in file.tag_str][0]
inj_efficiency_node = inj_efficiency_jobs.create_node(\
clust_file, offsource_clustered, segment_dir,
found_file, missed_file, tags=[trial_tag, tag,
inj_set_tag])
pp_nodes.append(inj_efficiency_node)
workflow.add_node(inj_efficiency_node)
dep = dax.Dependency(parent=off_node._dax_node,
child=inj_efficiency_node._dax_node)
workflow._adag.addDependency(dep)
for injcombiner_node in injcombiner_nodes:
dep = dax.Dependency(parent=injcombiner_node._dax_node,
child=inj_efficiency_node._dax_node)
workflow._adag.addDependency(dep)
for injfinder_node in injfinder_nodes:
dep = dax.Dependency(parent=injfinder_node._dax_node,
child=inj_efficiency_node._dax_node)
workflow._adag.addDependency(dep)
trial += 1
# Initialise html_summary class and set up job
#FIXME: We may want this job to run even if some jobs fail
html_summary_jobs = html_summary_class(cp,
"html_summary",
ifo=ifos,
out_dir=output_dir,
tags=tags)
if cp.has_section("workflow-injections"):
tuning_tags = [inj_tag for inj_tag in injcombiner_out_tags \
if "DETECTION" in inj_tag]
exclusion_tags = [inj_tag for inj_tag in injcombiner_out_tags \
if "DETECTION" not in inj_tag]
html_summary_node = html_summary_jobs.create_node(
c_file=config_file,
tuning_tags=tuning_tags,
exclusion_tags=exclusion_tags,
html_dir=html_dir)
else:
html_summary_node = html_summary_jobs.create_node(c_file=config_file,
html_dir=html_dir)
workflow.add_node(html_summary_node)
for pp_node in pp_nodes:
dep = dax.Dependency(parent=pp_node._dax_node,
child=html_summary_node._dax_node)
workflow._adag.addDependency(dep)
# Make the open box shell script
open_box_cmd = html_summary_node.executable.get_pfn() + " "
open_box_cmd += ' '.join(html_summary_node._args + \
html_summary_node._options)
open_box_cmd += " --open-box"
open_box_path = "%s/open_the_box.sh" % output_dir
f = open(open_box_path, "w")
f.write("#!/bin/sh\n%s" % open_box_cmd)
f.close()
os.chmod(open_box_path, 0500)
pp_outs.extend(trig_cluster_outs)
return pp_outs
def setup_postproc_coh_PTF_online_workflow(workflow, trig_files, trig_cache,
inj_trig_files, inj_files, inj_trig_caches, inj_caches, config_file,
output_dir, html_dir, segment_dir, segs_plot, ifos, inj_tags=None,
tags=None):
"""
This module sets up a stripped down post-processing stage for the online
workflow, using a coh_PTF style set up. This consists of running
trig_combiner to find coherent triggers, and trig_cluster to cluster them.
This process may be done in two stages to reduce memory requirements. It
also runs injfinder to look for injections, and injcombiner to calculate
injection statistics. Finally, efficiency and sbv_plotter jobs calculate
efficiency and signal based veto statistics and make plots.
Parameters
-----------
workflow : pycbc.workflow.core.Workflow
The Workflow instance that the coincidence jobs will be added to.
trig_files : pycbc.workflow.core.FileList
A FileList of the trigger files from the on/off source analysis jobs.
trig_cache : pycbc.workflow.core.File
A cache file pointing to the trigger files.
inj_trig_files : pycbc.workflow.core.FileList
A FileList of the trigger files produced by injection jobs.
inj_files : pycbc.workflow.core.FileList
A FileList of the injection set files.
inj_trig_caches : pycbc.workflow.core.FileList
A FileList containing the cache files that point to the injection
trigger files.
inj_caches : pycbc.workflow.core.FileList
A FileList containing cache files that point to the injection files.
config_file : pycbc.workflow.core.File
The parsed configuration file.
output_dir : path
The directory in which output files will be stored.
html_dir : path
The directory where the result webpage will be placed.
segment_dir : path
The directory in which data segment information is stored.
segs_plot : pycbc.workflow.core.File
The plot showing the analysis segments for each IFO around the GRB time.
This is produced at the time of workflow generation.
ifos : list
A list containing the analysis interferometers.
inj_tags : list
List containing the strings used to uniquely identify the injection
sets included in the analysis.
tags : list of strings (optional, default = [])
A list of the tagging strings that will be used for all jobs created
by this call to the workflow. An example might be ['POSTPROC1'] or
['DENTYSNEWPOSTPROC']. This will be used in output names.
Returns
--------
pp_outs : pycbc.workflow.core.FileList
A list of the output from this stage.
"""
if inj_tags is None:
inj_tags = []
if tags is None:
tags = []
cp = workflow.cp
full_segment = trig_files[0].segment
trig_name = cp.get("workflow", "trigger-name")
grb_string = "GRB" + trig_name
num_trials = int(cp.get("trig_combiner", "num-trials"))
pp_outs = FileList([])
pp_nodes = []
# Set up needed exe classes
trig_combiner_class = select_generic_executable(workflow, "trig_combiner")
trig_cluster_class = select_generic_executable(workflow, "trig_cluster")
sbv_plotter_class = select_generic_executable(workflow, "sbv_plotter")
efficiency_class = select_generic_executable(workflow, "efficiency")
#horizon_dist_class = select_generic_executable(workflow, "horizon_dist")
html_summary_class = select_generic_executable(workflow, "html_summary")
# Set up trig_combiner job
trig_combiner_out_tags = ["OFFSOURCE", "ONSOURCE", "ALL_TIMES"]
if all("COHERENT_NO_INJECTIONS" in t.name for t in trig_files) and \
cp.has_option_tag("inspiral", "do-short-slides",
"coherent_no_injections"):
trig_combiner_out_tags.extend(["ZEROLAG_OFF", "ZEROLAG_ALL"])
trig_combiner_jobs = trig_combiner_class(cp, "trig_combiner", ifo=ifos,
out_dir=output_dir, tags=tags)
# Do first stage of trig_combiner and trig_cluster jobs if desired
if workflow.cp.has_option("workflow-postproc", "do-two-stage-clustering"):
logging.info("Doing two-stage clustering.")
trig_combiner_s1_jobs = trig_combiner_class(cp, "trig_combiner",
ifo=ifos, out_dir=output_dir, tags=tags+["INTERMEDIATE"])
num_stage_one_jobs = int(workflow.cp.get("workflow-postproc",
"num-stage-one-cluster-jobs"))
num_inputs_per_job = -(-len(trig_files) // num_stage_one_jobs)
split_trig_files = (trig_files[p:p + num_inputs_per_job] for p in \
xrange(0, len(trig_files), num_inputs_per_job))
trig_cluster_s1_jobs = trig_cluster_class(cp, "trig_cluster", ifo=ifos,
out_dir=output_dir, tags=tags+["INTERMEDIATE"])
trig_cluster_s1_nodes = []
trig_cluster_s1_outs = FileList([])
for j, s1_inputs in zip(range(num_stage_one_jobs), split_trig_files):
trig_combiner_s1_node, trig_combiner_s1_outs = \
trig_combiner_s1_jobs.create_node(s1_inputs,
segment_dir, workflow.analysis_time,
out_tags=trig_combiner_out_tags, tags=tags+[str(j)])
pp_nodes.append(trig_combiner_s1_node)
workflow.add_node(trig_combiner_s1_node)
unclust_file = [f for f in trig_combiner_s1_outs \
if "ALL_TIMES" in f.tag_str][0]
trig_cluster_s1_node, curr_outs = trig_cluster_s1_jobs.create_node(\
unclust_file)
trig_cluster_s1_outs.extend(curr_outs)
clust_file = curr_outs[0]
trig_cluster_s1_node.set_memory(1300)
trig_cluster_s1_nodes.append(trig_cluster_s1_node)
pp_nodes.append(trig_cluster_s1_node)
workflow.add_node(trig_cluster_s1_node)
dep = dax.Dependency(parent=trig_combiner_s1_node._dax_node,
child=trig_cluster_s1_node._dax_node)
workflow._adag.addDependency(dep)
trig_combiner_node, trig_combiner_outs = \
trig_combiner_jobs.create_node(trig_cluster_s1_outs,
segment_dir, workflow.analysis_time,
out_tags=trig_combiner_out_tags, tags=tags)
pp_nodes.append(trig_combiner_node)
workflow.add_node(trig_combiner_node)
pp_outs.extend(trig_combiner_outs)
for trig_cluster_s1_node in trig_cluster_s1_nodes:
dep = dax.Dependency(parent=trig_cluster_s1_node._dax_node,
child=trig_combiner_node._dax_node)
workflow._adag.addDependency(dep)
else:
trig_combiner_node, trig_combiner_outs = \
trig_combiner_jobs.create_node(trig_files, segment_dir,
workflow.analysis_time, out_tags=trig_combiner_out_tags,
tags=tags)
pp_nodes.append(trig_combiner_node)
workflow.add_node(trig_combiner_node)
pp_outs.extend(trig_combiner_outs)
# Initialise trig_cluster class
trig_cluster_outs = FileList([])
trig_cluster_jobs = trig_cluster_class(cp, "trig_cluster", ifo=ifos,
out_dir=output_dir, tags=tags)
# Set up injfinder jobs
if cp.has_section("workflow-injections"):
injfinder_nodes = []
injcombiner_parent_nodes = []
inj_sbv_plotter_parent_nodes = []
injfinder_exe = os.path.basename(cp.get("executables", "injfinder"))
injfinder_class = select_generic_executable(workflow, "injfinder")
injfinder_jobs = injfinder_class(cp, "injfinder", ifo=ifos,
out_dir=output_dir, tags=tags)
injcombiner_exe = os.path.basename(cp.get("executables",
"injcombiner"))
injcombiner_class = select_generic_executable(workflow, "injcombiner")
injcombiner_jobs = injcombiner_class(cp, "injcombiner", ifo=ifos,
out_dir=output_dir, tags=tags)
injfinder_outs = FileList([])
for inj_tag in inj_tags:
triggers = FileList([file for file in inj_trig_files \
if inj_tag in file.tag_str])
injections = FileList([file for file in inj_files \
if inj_tag in file.tag_str])
trig_cache = [file for file in inj_trig_caches \
if inj_tag in file.tag_str][0]
inj_cache = [file for file in inj_caches \
if inj_tag in file.tag_str][0]
injfinder_node, curr_outs = injfinder_jobs.create_node(\
triggers, injections, segment_dir, tags=[inj_tag])
injfinder_nodes.append(injfinder_node)
pp_nodes.append(injfinder_node)
workflow.add_node(injfinder_node)
injfinder_outs.extend(curr_outs)
if "DETECTION" not in curr_outs[0].tagged_description:
injcombiner_parent_nodes.append(injfinder_node)
else:
inj_sbv_plotter_parent_nodes.append(injfinder_node)
pp_outs.extend(injfinder_outs)
# Make injfinder output cache
fm_cache = File(ifos, "foundmissed", full_segment,
extension="lcf", directory=output_dir)
fm_cache.PFN(fm_cache.cache_entry.path, site="local")
injfinder_outs.convert_to_lal_cache().tofile(\
open(fm_cache.storage_path, "w"))
pp_outs.extend(FileList([fm_cache]))
# Set up injcombiner jobs
injcombiner_outs = FileList([f for f in injfinder_outs \
if "DETECTION" in f.tag_str])
injcombiner_tags = [inj_tag for inj_tag in inj_tags \
if "DETECTION" not in inj_tag]
injcombiner_out_tags = [i.tag_str.rsplit('_', 1)[0] for i in \
injcombiner_outs if "FOUND" in i.tag_str]
injcombiner_nodes = []
for injcombiner_tag in injcombiner_tags:
max_inc = cp.get_opt_tags("injections", "max-inc",
[injcombiner_tag])
inj_str = injcombiner_tag.replace("INJ", "")
inputs = FileList([f for f in injfinder_outs \
if injcombiner_tag in f.tagged_description])
injcombiner_node, curr_outs = injcombiner_jobs.create_node(\
fm_cache, inputs, inj_str, max_inc, workflow.analysis_time)
injcombiner_nodes.append(injcombiner_node)
injcombiner_out_tags.append("%s_FILTERED_%s"
% (inj_str.split(max_inc)[0], max_inc))
injcombiner_outs.extend(curr_outs)
pp_outs.extend(curr_outs)
pp_nodes.append(injcombiner_node)
workflow.add_node(injcombiner_node)
for parent_node in injcombiner_parent_nodes:
dep = dax.Dependency(parent=parent_node._dax_node,
child=injcombiner_node._dax_node)
workflow._adag.addDependency(dep)
# Initialise injection_efficiency class
inj_efficiency_jobs = efficiency_class(cp, "inj_efficiency", ifo=ifos,
out_dir=output_dir, tags=tags)
# Initialise sbv_plotter class
sbv_plotter_outs = FileList([])
sbv_plotter_jobs = sbv_plotter_class(cp, "sbv_plotter", ifo=ifos,
out_dir=output_dir, tags=tags)
# Initialise efficiency class
efficiency_outs = FileList([])
efficiency_jobs = efficiency_class(cp, "efficiency", ifo=ifos,
out_dir=output_dir, tags=tags)
# Add trig_cluster jobs and their corresponding plotting jobs
for out_tag in trig_combiner_out_tags:
unclust_file = [f for f in trig_combiner_outs \
if out_tag in f.tag_str][0]
trig_cluster_node, curr_outs = trig_cluster_jobs.create_node(\
unclust_file)
trig_cluster_outs.extend(curr_outs)
clust_file = curr_outs[0]
if out_tag != "ONSOURCE":
# Add memory requirememnt for jobs with potentially large files
trig_cluster_node.set_memory(1300)
pp_nodes.append(trig_cluster_node)
workflow.add_node(trig_cluster_node)
dep = dax.Dependency(parent=trig_combiner_node._dax_node,
child=trig_cluster_node._dax_node)
workflow._adag.addDependency(dep)
# Add sbv_plotter job
sbv_out_tags = [out_tag, "_clustered"]
sbv_plotter_node = sbv_plotter_jobs.create_node(clust_file,
segment_dir,
tags=sbv_out_tags)
pp_nodes.append(sbv_plotter_node)
workflow.add_node(sbv_plotter_node)
dep = dax.Dependency(parent=trig_cluster_node._dax_node,
child=sbv_plotter_node._dax_node)
workflow._adag.addDependency(dep)
# Add injection sbv_plotter nodes if appropriate
if out_tag == "OFFSOURCE":
offsource_clustered = clust_file
off_node = sbv_plotter_node
if out_tag == "OFFSOURCE" and \
cp.has_section("workflow-injections"):
found_inj_files = FileList([file for file in injcombiner_outs \
if "FOUND" in file.tag_str])
for curr_injs in found_inj_files:
curr_tags = [tag for tag in injcombiner_out_tags \
if tag in curr_injs.name]
curr_tags.append("_clustered")
sbv_plotter_node = sbv_plotter_jobs.create_node(clust_file,
segment_dir, inj_file=curr_injs, tags=curr_tags)
pp_nodes.append(sbv_plotter_node)
workflow.add_node(sbv_plotter_node)
dep = dax.Dependency(parent=trig_cluster_node._dax_node,
child=sbv_plotter_node._dax_node)
workflow._adag.addDependency(dep)
if "DETECTION" in curr_injs.tagged_description:
for parent_node in inj_sbv_plotter_parent_nodes:
dep = dax.Dependency(parent=parent_node._dax_node,
child=sbv_plotter_node._dax_node)
workflow._adag.addDependency(dep)
else:
for parent_node in injcombiner_nodes:
dep = dax.Dependency(parent=parent_node._dax_node,
child=sbv_plotter_node._dax_node)
workflow._adag.addDependency(dep)
else:
pp_nodes.append(trig_cluster_node)
workflow.add_node(trig_cluster_node)
dep = dax.Dependency(parent=trig_combiner_node._dax_node,
child=trig_cluster_node._dax_node)
workflow._adag.addDependency(dep)
# Add efficiency job for on/off
efficiency_node = efficiency_jobs.create_node(clust_file,
offsource_clustered, segment_dir, tags=[out_tag])
pp_nodes.append(efficiency_node)
workflow.add_node(efficiency_node)
dep = dax.Dependency(parent=off_node._dax_node,
child=efficiency_node._dax_node)
workflow._adag.addDependency(dep)
if cp.has_section("workflow-injections"):
for tag in injcombiner_out_tags:
if "_FILTERED_" in tag:
inj_set_tag = [t for t in inj_tags if \
str(tag).replace("_FILTERED_", "") \
in t][0]
else:
inj_set_tag = str(tag)
found_file = [file for file in injcombiner_outs \
if tag + "_FOUND" in file.tag_str][0]
missed_file = [file for file in injcombiner_outs \
if tag + "_MISSED" in file.tag_str][0]
inj_efficiency_node = inj_efficiency_jobs.create_node(\
clust_file, offsource_clustered, segment_dir,
found_file, missed_file, tags=[out_tag, tag,
inj_set_tag])
pp_nodes.append(inj_efficiency_node)
workflow.add_node(inj_efficiency_node)
dep = dax.Dependency(parent=off_node._dax_node,
child=inj_efficiency_node._dax_node)
workflow._adag.addDependency(dep)
for injcombiner_node in injcombiner_nodes:
dep = dax.Dependency(parent=injcombiner_node._dax_node,
child=inj_efficiency_node._dax_node)
workflow._adag.addDependency(dep)
for injfinder_node in injfinder_nodes:
dep = dax.Dependency(parent=injfinder_node._dax_node,
child=inj_efficiency_node._dax_node)
workflow._adag.addDependency(dep)
# Add further trig_cluster jobs for trials
trial = 1
while trial <= num_trials:
trial_tag = "OFFTRIAL_%d" % trial
unclust_file = [f for f in trig_combiner_outs \
if trial_tag in f.tag_str][0]
trig_cluster_node, clust_outs = trig_cluster_jobs.create_node(\
unclust_file)
clust_file = clust_outs[0]
trig_cluster_outs.extend(clust_outs)
pp_nodes.append(trig_cluster_node)
workflow.add_node(trig_cluster_node)
dep = dax.Dependency(parent=trig_combiner_node._dax_node,
child=trig_cluster_node._dax_node)
workflow._adag.addDependency(dep)
# Add efficiency job
efficiency_node = efficiency_jobs.create_node(clust_file,
offsource_clustered, segment_dir, tags=[trial_tag])
pp_nodes.append(efficiency_node)
workflow.add_node(efficiency_node)
dep = dax.Dependency(parent=off_node._dax_node,
child=efficiency_node._dax_node)
workflow._adag.addDependency(dep)
dep = dax.Dependency(parent=trig_cluster_node._dax_node,
child=efficiency_node._dax_node)
workflow._adag.addDependency(dep)
# Adding inj_efficiency job
if cp.has_section("workflow-injections"):
for tag in injcombiner_out_tags:
if "_FILTERED_" in tag:
inj_set_tag = [t for t in inj_tags if \
str(tag).replace("_FILTERED_", "") in t][0]
else:
inj_set_tag = str(tag)
found_file = [file for file in injcombiner_outs \
if tag + "_FOUND" in file.tag_str][0]
missed_file = [file for file in injcombiner_outs \
if tag + "_MISSED" in file.tag_str][0]
inj_efficiency_node = inj_efficiency_jobs.create_node(\
clust_file, offsource_clustered, segment_dir,
found_file, missed_file, tags=[trial_tag, tag,
inj_set_tag])
pp_nodes.append(inj_efficiency_node)
workflow.add_node(inj_efficiency_node)
dep = dax.Dependency(parent=off_node._dax_node,
child=inj_efficiency_node._dax_node)
workflow._adag.addDependency(dep)
for injcombiner_node in injcombiner_nodes:
dep = dax.Dependency(parent=injcombiner_node._dax_node,
child=inj_efficiency_node._dax_node)
workflow._adag.addDependency(dep)
for injfinder_node in injfinder_nodes:
dep = dax.Dependency(parent=injfinder_node._dax_node,
child=inj_efficiency_node._dax_node)
workflow._adag.addDependency(dep)
trial += 1
# Initialise html_summary class and set up job
#FIXME: We may want this job to run even if some jobs fail
html_summary_jobs = html_summary_class(cp, "html_summary", ifo=ifos,
out_dir=output_dir, tags=tags)
if cp.has_section("workflow-injections"):
tuning_tags = [inj_tag for inj_tag in injcombiner_out_tags \
if "DETECTION" in inj_tag]
exclusion_tags = [inj_tag for inj_tag in injcombiner_out_tags \
if "DETECTION" not in inj_tag]
html_summary_node = html_summary_jobs.create_node(c_file=config_file,
tuning_tags=tuning_tags, exclusion_tags=exclusion_tags,
seg_plot=segs_plot, html_dir=html_dir)
else:
html_summary_node = html_summary_jobs.create_node(c_file=config_file,
seg_plot=segs_plot, html_dir=html_dir)
workflow.add_node(html_summary_node)
for pp_node in pp_nodes:
dep = dax.Dependency(parent=pp_node._dax_node,
child=html_summary_node._dax_node)
workflow._adag.addDependency(dep)
# Make the open box shell script
try:
open_box_cmd = html_summary_node.executable.get_pfn() + " "
except:
exe_path = html_summary_node.executable.get_pfn('nonlocal').replace(\
"https", "http")
exe_name = exe_path.rsplit('/', 1)[-1]
open_box_cmd = "wget %s\n" % exe_path
open_box_cmd += "chmod 500 ./%s\n./%s " % (exe_name, exe_name)
open_box_cmd += ' '.join(html_summary_node._args + \
html_summary_node._options)
open_box_cmd += " --open-box"
open_box_path = "%s/open_the_box.sh" % output_dir
f = open(open_box_path, "w")
f.write("#!/bin/sh\n%s" % open_box_cmd)
f.close()
os.chmod(open_box_path, 0500)
pp_outs.extend(trig_cluster_outs)
return pp_outs
def setup_postproc_coh_PTF_offline_workflow(workflow,
trig_files,
trig_cache,
ts_trig_files,
inj_trig_files,
inj_files,
inj_trig_caches,
inj_caches,
config_file,
output_dir,
html_dir,
segment_dir,
segs_plot,
ifos,
inj_tags=None,
tags=None):
"""
This module sets up the post-processing stage in the workflow, using a
coh_PTF style set up. This consists of running trig_combiner to find
coherent triggers, and injfinder to look for injections. It then runs
a horizon_dist job, trig_cluster to cluster triggers, and injcombiner to
calculate injection statistics. Finally, efficiency and sbv_plotter jobs
calculate efficiency and signal based veto statistics and make plots.
Parameters
-----------
workflow : pycbc.workflow.core.Workflow
The Workflow instance that the coincidence jobs will be added to.
trig_files : pycbc.workflow.core.FileList
A FileList of the trigger files from the on/off source analysis jobs.
trig_cache : pycbc.workflow.core.File
A cache file pointing to the trigger files.
ts_trig_files : pycbc.workflow.core.FileList
A FileList of the trigger files from the timeslide analysis jobs.
inj_trig_files : pycbc.workflow.core.FileList
A FileList of the trigger files produced by injection jobs.
inj_files : pycbc.workflow.core.FileList
A FileList of the injection set files.
inj_trig_caches : pycbc.workflow.core.FileList
A FileList containing the cache files that point to the injection
trigger files.
inj_caches : pycbc.workflow.core.FileList
A FileList containing cache files that point to the injection files.
config_file : pycbc.workflow.core.File
The parsed configuration file.
output_dir : path
The directory in which output files will be stored.
html_dir : path
The directory where the result webpage will be placed.
segment_dir : path
The directory in which data segment information is stored.
segs_plot : pycbc.workflow.core.File
The plot showing the analysis segments for each IFO around the GRB time.
This is produced at the time of workflow generation.
ifos : list
A list containing the analysis interferometers.
inj_tags : list
List containing the strings used to uniquely identify the injection
sets included in the analysis.
tags : list of strings (optional, default = [])
A list of the tagging strings that will be used for all jobs created
by this call to the workflow. An example might be ['POSTPROC1'] or
['DENTYSNEWPOSTPROC']. This will be used in output names.
Returns
--------
pp_outs : pycbc.workflow.core.FileList
A list of the output from this stage.
"""
if inj_tags is None:
inj_tags = []
if tags is None:
tags = []
cp = workflow.cp
full_segment = trig_files[0].segment
trig_name = cp.get("workflow", "trigger-name")
grb_string = "GRB" + trig_name
num_trials = int(cp.get("trig_combiner", "num-trials"))
do_injections = cp.has_section("workflow-injections")
pp_outs = FileList([])
pp_nodes = []
# Set up needed exe classes
trig_combiner_class = select_generic_executable(workflow, "trig_combiner")
trig_cluster_class = select_generic_executable(workflow, "trig_cluster")
sbv_plotter_class = select_generic_executable(workflow, "sbv_plotter")
efficiency_class = select_generic_executable(workflow, "efficiency")
#horizon_dist_class = select_generic_executable(workflow, "horizon_dist")
html_summary_class = select_generic_executable(workflow, "html_summary")
# Set up injection jobs if desired
if do_injections:
workflow, injfinder_nodes, injfinder_outs, fm_cache, \
injcombiner_nodes, injcombiner_outs, injcombiner_out_tags, \
inj_sbv_plotter_parent_nodes, pp_nodes, pp_outs = \
setup_coh_PTF_injections_pp(workflow, inj_trig_files,
inj_files, inj_trig_caches, inj_caches, pp_nodes,
pp_outs, inj_tags, output_dir, segment_dir, ifos,
tags=tags)
# Initialise injection_efficiency class
inj_efficiency_jobs = efficiency_class(cp,
"inj_efficiency",
ifo=ifos,
out_dir=output_dir,
tags=tags)
# Set up main trig_combiner class and tags
trig_combiner_out_tags = ["OFFSOURCE", "ONSOURCE", "ALL_TIMES"]
slides = all("COHERENT_NO_INJECTIONS" in t.name for t in trig_files) and \
cp.has_option_tag("inspiral", "do-short-slides",
"coherent_no_injections")
if slides:
trig_combiner_out_tags.extend(["ZEROLAG_OFF", "ZEROLAG_ALL"])
trig_combiner_jobs = trig_combiner_class(cp,
"trig_combiner",
ifo=ifos,
out_dir=output_dir,
tags=tags)
# Do first stage of trig_combiner and trig_cluster jobs if desired
if workflow.cp.has_option("workflow-postproc", "do-two-stage-clustering"):
logging.info("Doing two-stage clustering.")
trig_combiner_s1_jobs = trig_combiner_class(cp,
"trig_combiner",
ifo=ifos,
out_dir=output_dir,
tags=tags +
["INTERMEDIATE"])
num_stage_one_jobs = int(
workflow.cp.get("workflow-postproc", "num-stage-one-cluster-jobs"))
num_inputs_per_job = -(-len(trig_files) // num_stage_one_jobs)
split_trig_files = (trig_files[p:p + num_inputs_per_job] for p in \
xrange(0, len(trig_files), num_inputs_per_job))
trig_cluster_s1_jobs = trig_cluster_class(cp,
"trig_cluster",
ifo=ifos,
out_dir=output_dir,
tags=tags + ["INTERMEDIATE"])
trig_cluster_s1_nodes = []
trig_cluster_s1_outs = FileList([])
for j, s1_inputs in zip(range(num_stage_one_jobs), split_trig_files):
trig_combiner_s1_node, trig_combiner_s1_outs = \
trig_combiner_s1_jobs.create_node(s1_inputs,
segment_dir, workflow.analysis_time,
out_tags=trig_combiner_out_tags, tags=tags+[str(j)])
pp_nodes.append(trig_combiner_s1_node)
workflow.add_node(trig_combiner_s1_node)
unclust_file = [f for f in trig_combiner_s1_outs \
if "ALL_TIMES" in f.tag_str][0]
trig_cluster_s1_node, curr_outs = trig_cluster_s1_jobs.create_node(\
unclust_file)
trig_cluster_s1_outs.extend(curr_outs)
clust_file = curr_outs[0]
trig_cluster_s1_nodes.append(trig_cluster_s1_node)
pp_nodes.append(trig_cluster_s1_node)
workflow.add_node(trig_cluster_s1_node)
dep = dax.Dependency(parent=trig_combiner_s1_node._dax_node,
child=trig_cluster_s1_node._dax_node)
workflow._adag.addDependency(dep)
trig_combiner_node, trig_combiner_outs = \
trig_combiner_jobs.create_node(trig_cluster_s1_outs,
segment_dir, workflow.analysis_time,
out_tags=trig_combiner_out_tags, tags=tags)
pp_nodes.append(trig_combiner_node)
workflow.add_node(trig_combiner_node)
pp_outs.extend(trig_combiner_outs)
for trig_cluster_s1_node in trig_cluster_s1_nodes:
dep = dax.Dependency(parent=trig_cluster_s1_node._dax_node,
child=trig_combiner_node._dax_node)
workflow._adag.addDependency(dep)
else:
trig_combiner_node, trig_combiner_outs = \
trig_combiner_jobs.create_node(trig_files, segment_dir,
workflow.analysis_time, out_tags=trig_combiner_out_tags,
tags=tags)
pp_nodes.append(trig_combiner_node)
workflow.add_node(trig_combiner_node)
pp_outs.extend(trig_combiner_outs)
# Initialise trig_cluster class
trig_cluster_outs = FileList([])
trig_cluster_jobs = trig_cluster_class(cp,
"trig_cluster",
ifo=ifos,
out_dir=output_dir,
tags=tags)
# Initialise sbv_plotter class
sbv_plotter_outs = FileList([])
sbv_plotter_jobs = sbv_plotter_class(cp,
"sbv_plotter",
ifo=ifos,
out_dir=output_dir,
tags=tags)
# Initialise efficiency class
efficiency_outs = FileList([])
efficiency_jobs = efficiency_class(cp,
"efficiency",
ifo=ifos,
out_dir=output_dir,
tags=tags)
# Set up trig_cluster jobs
trig_cluster_nodes = []
for out_tag in trig_combiner_out_tags:
unclust_file = [f for f in trig_combiner_outs \
if out_tag in f.tag_str][0]
trig_cluster_node, curr_outs = trig_cluster_jobs.create_node(\
unclust_file)
trig_cluster_outs.extend(curr_outs)
clust_file = curr_outs[0]
trig_cluster_nodes.append(trig_cluster_node)
pp_nodes.append(trig_cluster_node)
workflow.add_node(trig_cluster_node)
dep = dax.Dependency(parent=trig_combiner_node._dax_node,
child=trig_cluster_node._dax_node)
workflow._adag.addDependency(dep)
# Are we not doing time slides?
if ts_trig_files is None:
if out_tag == "OFFSOURCE":
off_node = trig_cluster_node
offsource_clustered = clust_file
# Add sbv_plotter and efficiency jobs
workflow, pp_nodes = setup_coh_PTF_plotting_jobs(
workflow,
unclust_file,
clust_file,
sbv_plotter_jobs,
efficiency_jobs,
inj_efficiency_jobs,
off_node,
trig_cluster_node,
offsource_clustered,
injfinder_nodes,
injcombiner_nodes,
injcombiner_outs,
inj_sbv_plotter_parent_nodes,
inj_tags,
injcombiner_out_tags,
pp_nodes,
output_dir,
segment_dir,
ifos,
out_tag,
do_injs=do_injections,
tags=tags)
# If doing time slides
if ts_trig_files is not None:
trig_combiner_ts_nodes = []
trig_cluster_ts_nodes = []
trig_cluster_all_times_nodes = []
ts_all_times_outs = FileList(
[out for out in trig_cluster_outs if "ALL_TIMES" in out.tag_str])
trig_combiner_ts_out_tags = ["ALL_TIMES", "OFFSOURCE"]
ts_tags = list(
set([[ts_tag for ts_tag in ts_trig_file.tags
if "SLIDE" in ts_tag][0] for ts_trig_file in ts_trig_files]))
for ts_tag in ts_tags:
# Do one slide at a time
ts_trigs = FileList([
ts_trig_file for ts_trig_file in ts_trig_files
if ts_tag in ts_trig_file.tags
])
# And do two-stage clustering if desired
if workflow.cp.has_option("workflow-postproc",
"do-two-stage-clustering"):
split_trig_files = (
ts_trigs[p:p + num_inputs_per_job]
for p in xrange(0, len(ts_trigs), num_inputs_per_job))
trig_cluster_s1_nodes = []
trig_cluster_s1_outs = FileList([])
for j, s1_inputs in zip(range(num_stage_one_jobs),
split_trig_files):
trig_combiner_s1_node, trig_combiner_s1_outs = \
trig_combiner_s1_jobs.create_node(s1_inputs,
segment_dir, workflow.analysis_time,
out_tags=trig_combiner_ts_out_tags,
slide_tag=ts_tag, tags=tags+[str(j)])
pp_nodes.append(trig_combiner_s1_node)
workflow.add_node(trig_combiner_s1_node)
unclust_file = [f for f in trig_combiner_s1_outs \
if "ALL_TIMES" in f.tag_str][0]
trig_cluster_s1_node, curr_outs = \
trig_cluster_s1_jobs.create_node(unclust_file)
trig_cluster_s1_outs.extend(curr_outs)
clust_file = curr_outs[0]
trig_cluster_s1_nodes.append(trig_cluster_s1_node)
pp_nodes.append(trig_cluster_s1_node)
workflow.add_node(trig_cluster_s1_node)
dep = dax.Dependency(
parent=trig_combiner_s1_node._dax_node,
child=trig_cluster_s1_node._dax_node)
workflow._adag.addDependency(dep)
trig_combiner_ts_node, trig_combiner_ts_outs = \
trig_combiner_jobs.create_node(trig_cluster_s1_outs,
segment_dir, workflow.analysis_time,
slide_tag=ts_tag,
out_tags=trig_combiner_ts_out_tags, tags=tags)
trig_combiner_ts_nodes.append(trig_combiner_ts_node)
pp_nodes.append(trig_combiner_ts_node)
workflow.add_node(trig_combiner_ts_node)
pp_outs.extend(trig_combiner_ts_outs)
for trig_cluster_s1_node in trig_cluster_s1_nodes:
dep = dax.Dependency(parent=trig_cluster_s1_node._dax_node,
child=trig_combiner_ts_node._dax_node)
workflow._adag.addDependency(dep)
else:
trig_combiner_ts_node, trig_combiner_ts_outs = \
trig_combiner_jobs.create_node(ts_trigs, segment_dir,
workflow.analysis_time, slide_tag=ts_tag,
out_tags=trig_combiner_ts_out_tags, tags=tags)
trig_combiner_ts_nodes.append(trig_combiner_ts_node)
pp_nodes.append(trig_combiner_ts_node)
workflow.add_node(trig_combiner_ts_node)
pp_outs.extend(trig_combiner_ts_outs)
# Set up trig cluster jobs for each timeslide
for ts_out_tag in trig_combiner_ts_out_tags:
unclust_file = [f for f in trig_combiner_ts_outs \
if ts_out_tag in f.tag_str][0]
trig_cluster_node, curr_outs = trig_cluster_jobs.create_node(\
unclust_file)
trig_cluster_outs.extend(curr_outs)
clust_file = curr_outs[0]
trig_cluster_ts_nodes.append(trig_cluster_node)
pp_nodes.append(trig_cluster_node)
workflow.add_node(trig_cluster_node)
dep = dax.Dependency(parent=trig_combiner_ts_node._dax_node,
child=trig_cluster_node._dax_node)
workflow._adag.addDependency(dep)
if ts_out_tag == "ALL_TIMES":
trig_cluster_all_times_nodes.append(trig_cluster_node)
ts_all_times_outs.extend(FileList([clust_file]))
# Combine all timeslides
trig_combiner_all_node, trig_combiner_all_outs = \
trig_combiner_jobs.create_node(ts_all_times_outs, segment_dir,
workflow.analysis_time, slide_tag="ALL_SLIDES",
out_tags=trig_combiner_ts_out_tags, tags=tags)
pp_nodes.append(trig_combiner_all_node)
workflow.add_node(trig_combiner_all_node)
for trig_cluster_ts_node in trig_cluster_all_times_nodes:
dep = dax.Dependency(parent=trig_cluster_ts_node._dax_node,
child=trig_combiner_all_node._dax_node)
workflow._adag.addDependency(dep)
for out_tag in trig_combiner_ts_out_tags:
trig_cluster_outs = FileList(
[f for f in trig_cluster_outs if out_tag not in f.tag_str])
trig_cluster_outs.extend(trig_combiner_all_outs)
off_node = trig_combiner_all_node
offsource_clustered = [
f for f in trig_cluster_outs
if "OFFSOURCE" in f.tag_str and "ZERO_LAG" not in f.tag_str
][0]
# Add sbv_plotter and efficiency jobs
for out_tag in trig_combiner_out_tags:
clust_file = [f for f in trig_cluster_outs \
if out_tag in f.tag_str][0]
workflow, pp_nodes = setup_coh_PTF_plotting_jobs(
workflow,
unclust_file,
clust_file,
sbv_plotter_jobs,
efficiency_jobs,
inj_efficiency_jobs,
off_node,
off_node,
offsource_clustered,
injfinder_nodes,
injcombiner_nodes,
injcombiner_outs,
inj_sbv_plotter_parent_nodes,
inj_tags,
injcombiner_out_tags,
pp_nodes,
output_dir,
segment_dir,
ifos,
out_tag,
do_injs=do_injections,
tags=tags)
trial = 1
while trial <= num_trials:
trial_tag = "OFFTRIAL_%d" % trial
unclust_file = [f for f in trig_combiner_outs \
if trial_tag in f.tag_str][0]
trig_cluster_node, clust_outs = trig_cluster_jobs.create_node(\
unclust_file)
clust_file = clust_outs[0]
trig_cluster_outs.extend(clust_outs)
pp_nodes.append(trig_cluster_node)
workflow.add_node(trig_cluster_node)
dep = dax.Dependency(parent=trig_combiner_node._dax_node,
child=trig_cluster_node._dax_node)
workflow._adag.addDependency(dep)
# Add efficiency job
efficiency_node = efficiency_jobs.create_node(clust_file,
offsource_clustered,
segment_dir,
tags=[trial_tag])
pp_nodes.append(efficiency_node)
workflow.add_node(efficiency_node)
dep = dax.Dependency(parent=off_node._dax_node,
child=efficiency_node._dax_node)
workflow._adag.addDependency(dep)
dep = dax.Dependency(parent=trig_cluster_node._dax_node,
child=efficiency_node._dax_node)
workflow._adag.addDependency(dep)
# Adding inj_efficiency job
if do_injections:
for tag in injcombiner_out_tags:
if "_FILTERED_" in tag:
inj_set_tag = [t for t in inj_tags if \
str(tag).replace("_FILTERED_", "") in t][0]
else:
inj_set_tag = str(tag)
found_file = [file for file in injcombiner_outs \
if tag + "_FOUND" in file.tag_str][0]
missed_file = [file for file in injcombiner_outs \
if tag + "_MISSED" in file.tag_str][0]
inj_efficiency_node = inj_efficiency_jobs.create_node(\
clust_file, offsource_clustered, segment_dir,
found_file, missed_file, tags=[trial_tag, tag,
inj_set_tag])
pp_nodes.append(inj_efficiency_node)
workflow.add_node(inj_efficiency_node)
dep = dax.Dependency(parent=off_node._dax_node,
child=inj_efficiency_node._dax_node)
workflow._adag.addDependency(dep)
for injcombiner_node in injcombiner_nodes:
dep = dax.Dependency(parent=injcombiner_node._dax_node,
child=inj_efficiency_node._dax_node)
workflow._adag.addDependency(dep)
for injfinder_node in injfinder_nodes:
dep = dax.Dependency(parent=injfinder_node._dax_node,
child=inj_efficiency_node._dax_node)
workflow._adag.addDependency(dep)
trial += 1
# Initialise html_summary class and set up job
#FIXME: We may want this job to run even if some jobs fail
html_summary_jobs = html_summary_class(cp,
"html_summary",
ifo=ifos,
out_dir=output_dir,
tags=tags)
if do_injections:
tuning_tags = [inj_tag for inj_tag in injcombiner_out_tags \
if "DETECTION" in inj_tag]
exclusion_tags = [inj_tag for inj_tag in injcombiner_out_tags \
if "DETECTION" not in inj_tag]
html_summary_node = html_summary_jobs.create_node(
c_file=config_file,
tuning_tags=tuning_tags,
exclusion_tags=exclusion_tags,
seg_plot=segs_plot,
html_dir=html_dir,
time_slides=slides)
else:
html_summary_node = html_summary_jobs.create_node(c_file=config_file,
seg_plot=segs_plot,
html_dir=html_dir,
time_slides=slides)
workflow.add_node(html_summary_node)
for pp_node in pp_nodes:
dep = dax.Dependency(parent=pp_node._dax_node,
child=html_summary_node._dax_node)
workflow._adag.addDependency(dep)
# Make the open box shell script
try:
open_box_cmd = html_summary_node.executable.get_pfn() + " "
except:
exe_path = html_summary_node.executable.get_pfn('nonlocal').replace(\
"https", "http")
exe_name = exe_path.rsplit('/', 1)[-1]
open_box_cmd = "wget %s\n" % exe_path
open_box_cmd += "chmod 500 ./%s\n./%s " % (exe_name, exe_name)
open_box_cmd += ' '.join(html_summary_node._args + \
html_summary_node._options)
open_box_cmd += " --open-box"
open_box_path = "%s/open_the_box.sh" % output_dir
f = open(open_box_path, "w")
f.write("#!/bin/sh\n%s" % open_box_cmd)
f.close()
os.chmod(open_box_path, 0500)
pp_outs.extend(trig_cluster_outs)
return pp_outs
def setup_coh_PTF_plotting_jobs(workflow,
unclust_file,
clust_file,
sbv_plotter_jobs,
efficiency_jobs,
inj_efficiency_jobs,
off_node,
dep_node,
offsource_clustered,
injfinder_nodes,
injcombiner_nodes,
injcombiner_outs,
inj_sbv_plotter_parent_nodes,
inj_tags,
injcombiner_out_tags,
pp_nodes,
output_dir,
segment_dir,
ifos,
out_tag,
do_injs=False,
tags=None):
"""
Creates signal-based veto and efficiency jobs
"""
if out_tag != "ONSOURCE":
# Add sbv_plotter job
sbv_out_tags = [out_tag, "_clustered"]
sbv_plotter_node = sbv_plotter_jobs.create_node(clust_file,
segment_dir,
tags=sbv_out_tags)
pp_nodes.append(sbv_plotter_node)
workflow.add_node(sbv_plotter_node)
for n in set((off_node, dep_node)):
dep = dax.Dependency(parent=n._dax_node,
child=sbv_plotter_node._dax_node)
workflow._adag.addDependency(dep)
# Add injection sbv_plotter nodes if appropriate
if out_tag == "OFFSOURCE" and do_injs:
found_inj_files = FileList([file for file in injcombiner_outs \
if "FOUND" in file.tag_str])
for curr_injs in found_inj_files:
curr_tags = [tag for tag in injcombiner_out_tags \
if tag in curr_injs.name]
curr_tags.append("_clustered")
sbv_plotter_node = sbv_plotter_jobs.create_node(
clust_file,
segment_dir,
inj_file=curr_injs,
tags=curr_tags)
pp_nodes.append(sbv_plotter_node)
workflow.add_node(sbv_plotter_node)
dep = dax.Dependency(parent=dep_node._dax_node,
child=sbv_plotter_node._dax_node)
workflow._adag.addDependency(dep)
if "DETECTION" in curr_injs.tagged_description:
for parent_node in inj_sbv_plotter_parent_nodes:
dep = dax.Dependency(parent=parent_node._dax_node,
child=sbv_plotter_node._dax_node)
workflow._adag.addDependency(dep)
else:
for parent_node in injcombiner_nodes:
dep = dax.Dependency(parent=parent_node._dax_node,
child=sbv_plotter_node._dax_node)
workflow._adag.addDependency(dep)
# Also add sbv_plotter job for unclustered triggers
#sbv_plotter_node = sbv_plotter_jobs.create_node(unclust_file,
# segment_dir, tags=[out_tag, "_unclustered"])
#pp_nodes.append(sbv_plotter_node)
#workflow.add_node(sbv_plotter_node)
#dep = dax.Dependency(parent=trig_combiner_node._dax_node,
# child=sbv_plotter_node._dax_node)
#workflow._adag.addDependency(dep)
else:
# Add efficiency job for on/off
efficiency_node = efficiency_jobs.create_node(clust_file,
offsource_clustered,
segment_dir,
tags=[out_tag])
pp_nodes.append(efficiency_node)
workflow.add_node(efficiency_node)
dep = dax.Dependency(parent=dep_node._dax_node,
child=efficiency_node._dax_node)
workflow._adag.addDependency(dep)
if do_injs:
for tag in injcombiner_out_tags:
if "_FILTERED_" in tag:
inj_set_tag = [t for t in inj_tags if \
str(tag).replace("_FILTERED_", "") \
in t][0]
else:
inj_set_tag = str(tag)
found_file = [file for file in injcombiner_outs \
if tag + "_FOUND" in file.tag_str][0]
missed_file = [file for file in injcombiner_outs \
if tag + "_MISSED" in file.tag_str][0]
inj_efficiency_node = inj_efficiency_jobs.create_node(\
clust_file, offsource_clustered, segment_dir,
found_file, missed_file, tags=[out_tag, tag,
inj_set_tag])
pp_nodes.append(inj_efficiency_node)
workflow.add_node(inj_efficiency_node)
dep = dax.Dependency(parent=dep_node._dax_node,
child=inj_efficiency_node._dax_node)
workflow._adag.addDependency(dep)
for injcombiner_node in injcombiner_nodes:
dep = dax.Dependency(parent=injcombiner_node._dax_node,
child=inj_efficiency_node._dax_node)
workflow._adag.addDependency(dep)
for injfinder_node in injfinder_nodes:
dep = dax.Dependency(parent=injfinder_node._dax_node,
child=inj_efficiency_node._dax_node)
workflow._adag.addDependency(dep)
return workflow, pp_nodes
def setup_coh_PTF_injections_pp(wf,
inj_trigger_files,
inj_files,
inj_trigger_caches,
inj_caches,
pp_nodes,
pp_outs,
inj_tags,
out_dir,
seg_dir,
ifos,
tags=None):
"""
Set up post processing for injections
"""
injfinder_nodes = []
injcombiner_parent_nodes = []
inj_sbv_plotter_parent_nodes = []
full_segment = inj_trigger_files[0].segment
injfinder_exe = os.path.basename(wf.cp.get("executables", "injfinder"))
injfinder_class = select_generic_executable(wf, "injfinder")
injfinder_jobs = injfinder_class(wf.cp,
"injfinder",
ifo=ifos,
out_dir=out_dir,
tags=tags)
injcombiner_exe = os.path.basename(wf.cp.get("executables", "injcombiner"))
injcombiner_class = select_generic_executable(wf, "injcombiner")
injcombiner_jobs = injcombiner_class(wf.cp,
"injcombiner",
ifo=ifos,
out_dir=out_dir,
tags=tags)
injfinder_outs = FileList([])
for inj_tag in inj_tags:
triggers = FileList([file for file in inj_trigger_files \
if inj_tag in file.tag_str])
injections = FileList([file for file in inj_files \
if inj_tag in file.tag_str])
trig_cache = [file for file in inj_trigger_caches \
if inj_tag in file.tag_str][0]
inj_cache = [file for file in inj_caches \
if inj_tag in file.tag_str][0]
injfinder_node, curr_outs = injfinder_jobs.create_node(\
triggers, injections, seg_dir, tags=[inj_tag])
injfinder_nodes.append(injfinder_node)
pp_nodes.append(injfinder_node)
wf.add_node(injfinder_node)
injfinder_outs.extend(curr_outs)
if "DETECTION" not in curr_outs[0].tagged_description:
injcombiner_parent_nodes.append(injfinder_node)
else:
inj_sbv_plotter_parent_nodes.append(injfinder_node)
pp_outs.extend(injfinder_outs)
# Make injfinder output cache
fm_cache = File(ifos,
"foundmissed",
full_segment,
extension="lcf",
directory=out_dir)
fm_cache.PFN(fm_cache.cache_entry.path, site="local")
injfinder_outs.convert_to_lal_cache().tofile(\
open(fm_cache.storage_path, "w"))
pp_outs.extend(FileList([fm_cache]))
# Set up injcombiner jobs
injcombiner_outs = FileList([f for f in injfinder_outs \
if "DETECTION" in f.tag_str])
injcombiner_tags = [inj_tag for inj_tag in inj_tags \
if "DETECTION" not in inj_tag]
injcombiner_out_tags = [i.tag_str.rsplit('_', 1)[0] for i in \
injcombiner_outs if "FOUND" in i.tag_str]
injcombiner_nodes = []
for injcombiner_tag in injcombiner_tags:
max_inc = wf.cp.get_opt_tags("injections", "max-inc",
[injcombiner_tag])
inj_str = injcombiner_tag.replace("INJ", "")
inputs = FileList([f for f in injfinder_outs \
if injcombiner_tag in f.tagged_description])
injcombiner_node, curr_outs = injcombiner_jobs.create_node(\
fm_cache, inputs, inj_str, max_inc, wf.analysis_time)
injcombiner_nodes.append(injcombiner_node)
injcombiner_out_tags.append("%s_FILTERED_%s" %
(inj_str.split(max_inc)[0], max_inc))
injcombiner_outs.extend(curr_outs)
pp_outs.extend(curr_outs)
pp_nodes.append(injcombiner_node)
wf.add_node(injcombiner_node)
for parent_node in injcombiner_parent_nodes:
dep = dax.Dependency(parent=parent_node._dax_node,
child=injcombiner_node._dax_node)
wf._adag.addDependency(dep)
return (wf, injfinder_nodes, injfinder_outs, fm_cache, injcombiner_nodes,
injcombiner_outs, injcombiner_out_tags,
inj_sbv_plotter_parent_nodes, pp_nodes, pp_outs)
