def create_node(self, stat_files, tags=None):
if tags is None:
tags = []
node = Node(self)
node.add_input_list_opt('--statmap-files', stat_files)
node.new_output_file_opt(stat_files[0].segment, '.hdf', '--output-file', tags=tags)
return node
def create_node(self, parent, inj_trigs, inj_string, max_inc, segment):
node = Node(self)
trig_name = self.cp.get('workflow', 'trigger-name')
node.add_opt('--inj-string', inj_string)
node.add_opt('--max-inclination', max_inc)
node.add_opt('--inj-cache', '%s' % parent.storage_path)
out_files = FileList([])
for inj_trig in inj_trigs:
out_file_tag = [
inj_string, "FILTERED", max_inc,
inj_trig.tag_str.rsplit('_', 1)[-1]
]
out_file = File(
self.ifos,
inj_trig.description,
inj_trig.segment,
extension="xml",
directory=self.out_dir,
tags=out_file_tag)
out_file.PFN(out_file.cache_entry.path, site="local")
out_files.append(out_file)
node.add_opt('--output-dir', self.out_dir)
return node, out_files
def create_node(self, parent, inj_trigs, inj_string, max_inc, segment):
node = Node(self)
trig_name = self.cp.get("workflow", "trigger-name")
node.add_opt("--inj-string", inj_string)
node.add_opt("--max-inclination", max_inc)
node.add_opt("--inj-cache", "%s" % parent.storage_path)
out_files = FileList([])
for inj_trig in inj_trigs:
out_string = inj_string.split(max_inc)[0]
out_file_tag = [out_string, "FILTERED", max_inc, inj_trig.tag_str.rsplit("_", 1)[-1]]
out_file = File(
self.ifos,
inj_trig.description,
inj_trig.segment,
extension="xml",
directory=self.out_dir,
tags=out_file_tag,
)
out_file.PFN(out_file.cache_entry.path, site="local")
out_files.append(out_file)
node.add_opt("--output-dir", self.out_dir)
return node, out_files
def create_node(self, bank):
"""
Set up a CondorDagmanNode class to run lalapps_splitbank code
Parameters
----------
bank : pycbc.workflow.core.File
The OutFile containing the template bank to be split
Returns
--------
node : pycbc.workflow.core.Node
The node to run the job
"""
node = Node(self)
# FIXME: This is a hack because SplitBank fails if given an input file
# whose path contains the character '-' or if the input file is not in
# the same directory as the output. Therefore we just set the path to
# be the local path
fullPath = bank.cache_entry.path
bank.cache_entry.path = os.path.basename(fullPath)
node.add_input_opt('--bank-file', bank)
# FIXME: Set the path back to what it was. This is part of the hack
# above and should be removed if possible.
bank.cache_entry.path = fullPath
# Get the output (taken from inspiral.py)
url_list = []
x = bank.filename.split('-')
if len(x) != 4:
errMsg = "Input file name is not compatible with splitbank. Name "
errMsg += "must follow the lal cache standard, for example "
errMsg += "H1-TMPLTBANK-900000000-1000.xml. "
errMsg += "Got %s." % (bank.filename, )
raise ValueError(errMsg)
for i in range(0, self.num_banks):
out_file = "%s-%s_%2.2d-%s-%s" % (x[0], x[1], i, x[2], x[3])
out_url = urlparse.urlunparse([
'file', 'localhost',
os.path.join(self.out_dir, out_file), None, None, None
])
url_list.append(out_url)
job_tag = bank.description + "_" + self.name.upper()
out_file = File(
bank.ifo,
job_tag,
bank.segment,
file_url=out_url,
tags=bank.tags,
store_file=self.retain_files)
node._add_output(out_file)
return node
def create_node(self, inj_coinc_file, inj_xml_file, veto_file, veto_name, tags=[]):
node = Node(self)
node.add_input_list_opt('--trigger-file', inj_coinc_file)
node.add_input_list_opt('--injection-file', inj_xml_file)
if veto_name is not None:
node.add_input_opt('--veto-file', veto_file)
node.add_opt('--segment-name', veto_name)
node.new_output_file_opt(inj_xml_file[0].segment, '.hdf', '--output-file',
tags=tags)
return node
def create_node(self, coinc_files, ifos, tags=None):
if tags is None:
tags = []
segs = coinc_files.get_times_covered_by_files()
seg = segments.segment(segs[0][0], segs[-1][1])
node = Node(self)
node.set_memory(5000)
node.add_input_list_opt('--coinc-files', coinc_files)
node.add_opt('--ifos', ifos)
node.new_output_file_opt(seg, '.hdf', '--output-file', tags=tags)
return node
def create_node(self, coinc_files, bank_file, background_bins, tags=[]):
node = Node(self)
node.add_input_list_opt('--coinc-files', coinc_files)
node.add_input_opt('--bank-file', bank_file)
node.add_opt('--background-bins', ' '.join(background_bins))
names = [b.split(':')[0] for b in background_bins]
output_files = [File(coinc_files[0].ifo_list,
self.name,
coinc_files[0].segment,
directory=self.out_dir,
tags = tags + ['mbin-%s' % i],
extension='.hdf') for i in range(len(background_bins))]
node.add_output_list_opt('--output-files', output_files)
node.names = names
return node
def create_node(self, parent, tags=None):
import Pegasus.DAX3 as dax
if tags is None:
tags = []
node = Node(self)
# Set input / output options
node.add_opt('--trig-file', '%s' % parent.storage_path)
#node._dax_node.uses(parent, link=dax.Link.INPUT, register=False,
# transfer=False)
#node._inputs += [parent]
node.add_opt('--output-dir', self.out_dir)
node.add_profile('condor', 'request_cpus', self.num_threads)
# Adding output files as pycbc.workflow.core.File objects
out_file = File(self.ifos, 'INSPIRAL', parent.segment,
directory=self.out_dir, extension='xml.gz',
tags=[parent.tag_str, 'CLUSTERED'],
store_file=self.retain_files)
out_file.PFN(out_file.cache_entry.path, site="local")
#node._dax_node.uses(out_file, link=dax.Link.OUTPUT, register=False,
# transfer=False)
#node._outputs += [out_file]
out_file.node = node
#node._add_output(out_file)
return node, FileList([out_file])
def create_node(self, trig_files, bank_file):
node = Node(self)
node.add_input_opt('--bank-file', bank_file)
node.add_input_list_opt('--trigger-files', trig_files)
node.new_output_file_opt(trig_files[0].segment, '.hdf',
'--output-file', use_tmp_subdirs=True)
return node
def create_node(self, parent, tags=[]):
node = Node(self)
# Set input / output options
node.add_opt('--trig-file', '%s' % parent.storage_path)
node.add_opt('--output-dir', self.out_dir)
node.add_profile('condor', 'request_cpus', self.num_threads)
# Adding output files as pycbc.workflow.core.File objects
out_file = File(self.ifos, 'INSPIRAL', parent.segment,
directory=self.out_dir, extension='xml.gz',
tags=[parent.tag_str, 'CLUSTERED'],
store_file=self.retain_files)
#out_file.PFN(out_file.cache_entry.path, site="local")
return node, FileList([out_file])
def create_node(self, parent, tags=[]):
node = Node(self)
# Set input / output options
node.add_opt("--trig-file", "%s" % parent.storage_path)
node.add_opt("--output-dir", self.out_dir)
node.add_profile("condor", "request_cpus", self.num_threads)
# Adding output files as pycbc.workflow.core.File objects
out_file = File(
self.ifos,
"INSPIRAL",
parent.segment,
directory=self.out_dir,
extension="xml.gz",
tags=[parent.tag_str, "CLUSTERED"],
store_file=self.retain_files,
)
# out_file.PFN(out_file.cache_entry.path, site="local")
return node, FileList([out_file])
def create_node(self,
parent=None,
c_file=None,
open_box=False,
seg_plot=None,
tuning_tags=None,
exclusion_tags=None,
html_dir=None,
tags=[]):
node = Node(self)
node.add_opt('--grb-name', self.cp.get('workflow', 'trigger-name'))
node.add_opt('--start-time', self.cp.get('workflow', 'trigger-time'))
node.add_opt('--ra', self.cp.get('workflow', 'ra'))
node.add_opt('--dec', self.cp.get('workflow', 'dec'))
node.add_opt('--ifo-tag', self.ifos)
if tuning_tags is not None:
node.add_opt('--tuning-injections', ','.join(tuning_tags))
if exclusion_tags is not None:
node.add_opt('--exclusion-injections', ','.join(exclusion_tags))
if seg_plot is not None:
node.add_opt('--seg-plot', seg_plot.storage_path)
if open_box:
node.add_opt('--open-box')
if html_dir is not None:
node.add_opt('--html-path', html_dir)
# Set input / output options
node.add_opt('--config-file', '%s' % c_file.storage_path)
node.add_opt('--output-path', "%s/output" % self.out_dir)
node.add_profile('condor', 'request_cpus', self.num_threads)
return node
def create_node(self, trig_file, bank_file, veto_file, veto_name):
node = Node(self)
# Executable objects are initialized with ifo information
node.add_opt('--ifo', self.ifo_string)
node.add_input_opt('--trigger-file', trig_file)
node.add_input_opt('--bank-file', bank_file)
node.add_input_opt('--veto-file', veto_file)
node.add_opt('--veto-segment-name', veto_name)
node.new_output_file_opt(trig_file.segment, '.hdf', '--output')
return node
def create_node(self, trig_files, inj_files, seg_dir, tags=[]):
node = Node(self)
# Set input / output options
node.add_input_list_opt('--input-files', trig_files)
node.add_input_list_opt('--inj-files', inj_files)
node.add_opt('--ifo-tag', self.ifos)
node.add_opt('--exclude-segments', '%s/bufferSeg.txt' % seg_dir)
node.add_opt('--output-dir', self.out_dir)
# Create output files as File objects
name_string = inj_files[0].description
seg = trig_files[0].segment
f_file = File(
self.ifos,
name_string,
seg,
extension="xml",
directory=self.out_dir,
store_file=self.retain_files,
tags=[inj_files[0].tag_str.replace("split0", "FOUND")])
m_file = File(
self.ifos,
name_string,
seg,
extension="xml",
directory=self.out_dir,
store_file=self.retain_files,
tags=[inj_files[0].tag_str.replace("split0", "MISSED")])
return node, FileList([f_file, m_file])
def create_node(self, parent=None, seg_dir=None, inj_file=None, tags=[]):
node = Node(self)
if not parent:
raise ValueError(
"%s must be supplied with trigger files" % self.name)
if isinstance(parent, str) and tags[1] == "_unclustered":
node.add_opt('--trig-file', '%s' % parent.storage_path)
tags[1] = ""
else:
node.add_opt('--trig-file', '%s' % parent.storage_path)
node.add_opt('--grb-name', self.cp.get('workflow', 'trigger-name'))
# Set input / output options
node.add_opt('--veto-directory', seg_dir)
node.add_opt('--segment-dir', seg_dir)
out_dir = "%s/output/%s/plots%s" % (self.out_dir, tags[0], tags[1])
node.add_opt('--output-path', out_dir)
if inj_file is not None:
node.add_opt('--inj-file', inj_file.storage_path)
node.add_profile('condor', 'request_cpus', self.num_threads)
return node
def get_veto_segs(workflow, ifo, category, start_time, end_time, out_dir,
vetoGenJob, tag=None, execute_now=False):
"""
Obtain veto segments for the selected ifo and veto category and add the job
to generate this to the workflow.
Parameters
-----------
workflow: pycbc.workflow.core.Workflow
An instance of the Workflow class that manages the workflow.
ifo : string
The string describing the ifo to generate vetoes for.
category : int
The veto category to generate vetoes for.
start_time : gps time (either int/LIGOTimeGPS)
The time at which to begin searching for segments.
end_time : gps time (either int/LIGOTimeGPS)
The time at which to stop searching for segments.
out_dir : path
The directory in which output will be stored.
vetoGenJob : Job
The veto generation Job class that will be used to create the Node.
tag : string, optional (default=None)
Use this to specify a tag. This can be used if this module is being
called more than once to give call specific configuration (by setting
options in [workflow-datafind-${TAG}] rather than [workflow-datafind]). This
is also used to tag the Files returned by the class to uniqueify
the Files and uniqueify the actual filename.
FIXME: Filenames may not be unique with current codes!
execute_now : boolean, optional
If true, jobs are executed immediately. If false, they are added to the
workflow to be run later.
Returns
--------
veto_def_file : pycbc.workflow.core.OutSegFile
The workflow File object corresponding to this DQ veto file.
"""
segValidSeg = segments.segment([start_time,end_time])
node = Node(vetoGenJob)
node.add_opt('--veto-categories', str(category))
node.add_opt('--ifo-list', ifo)
node.add_opt('--gps-start-time', str(start_time))
node.add_opt('--gps-end-time', str(end_time))
vetoXmlFileName = "%s-VETOTIME_CAT%d-%d-%d.xml" \
%(ifo, category, start_time, end_time-start_time)
vetoXmlFilePath = os.path.abspath(os.path.join(out_dir, vetoXmlFileName))
currUrl = urlparse.urlunparse(['file', 'localhost',
vetoXmlFilePath, None, None, None])
if tag:
currTags = [tag, 'VETO_CAT%d' %(category)]
else:
currTags = ['VETO_CAT%d' %(category)]
vetoXmlFile = OutSegFile(ifo, 'SEGMENTS', segValidSeg, currUrl,
tags=currTags)
node._add_output(vetoXmlFile)
if execute_now:
if file_needs_generating(vetoXmlFile.cache_entry.path):
workflow.execute_node(node)
else:
node.executed = True
for fil in node._outputs:
fil.node = None
fil.PFN(fil.storage_path, site='local')
else:
workflow.add_node(node)
return vetoXmlFile
def create_node(self,
data_seg,
valid_seg,
parent=None,
inj_file=None,
dfParents=None,
bankVetoBank=None,
tags=[]):
node = Node(self)
if not dfParents:
raise ValueError(
"%s must be supplied with frame files" % self.name)
# If doing single IFO search, make sure slides are disabled
if len(self.ifo_list) < 2 and \
('--do-short-slides' in node._options or \
'--short-slide-offset' in node._options):
raise ValueError("Cannot run with time slides in a single IFO "
"configuration! Please edit your configuration "
"file accordingly.")
pad_data = self.get_opt('pad-data')
if pad_data is None:
raise ValueError("The option pad-data is a required option of "
"%s. Please check the ini file." % self.name)
# Feed in bank_veto_bank.xml
if self.cp.has_option('inspiral', 'do-bank-veto'):
if not bankVetoBank:
raise ValueError(
"%s must be given a bank veto file if the "
"argument 'do-bank-veto' is given" % self.name)
node.add_input_opt('--bank-veto-templates', bankVetoBank)
# Set time options
node.add_opt('--gps-start-time', data_seg[0] + int(pad_data))
node.add_opt('--gps-end-time', data_seg[1] - int(pad_data))
node.add_opt('--trig-start-time', valid_seg[0])
node.add_opt('--trig-end-time', valid_seg[1])
node.add_profile('condor', 'request_cpus', self.num_threads)
# Set the input and output files
node.new_output_file_opt(
data_seg,
'.xml.gz',
'--output-file',
tags=tags,
store_file=self.retain_files)
node.add_input_opt(
'--non-spin-bank',
parent,
)
for frameCache in dfParents:
node.add_input_opt('--%s-frame-cache' % frameCache.ifo.lower(),
frameCache)
node.add_arg('--%s-data' % frameCache.ifo.lower())
if inj_file is not None:
if ('--do-short-slides' in node._options or \
'--short-slide-offset' in node._options):
raise ValueError("Cannot run with short slides in an "
"injection job. Please edit your "
"configuration file accordingly.")
node.add_input_opt('--injection-file', inj_file)
return node
def setup_hardware_injection_page(workflow, input_files, cache_filename,
inspiral_cachepattern, output_dir, tags=[], **kwargs):
"""
This function sets up the nodes that will create the hardware injection page.
Parameters
-----------
Workflow : ahope.Workflow
The ahope workflow instance that the coincidence jobs will be added to.
input_files : ahope.FileList
An FileList of files that are used as input at this stage.
cache_filename : str
Filename of the ihope cache.
inspiral_cachepattern : str
The pattern that will be used to find inspiral filenames in the cache.
output_dir : path
The directory in which output files will be stored.
tags : list of strings (optional, default = [])
A list of the tagging strings that will be used for all jobs created
by this call to the workflow. An example might be ['full_data'].
This will be used to search the cache.
Returns
--------
plot_files : ahope.FileList
A list of the output files from this stage.
"""
logging.info("Entering hardware injection page setup.")
out_files = FileList([])
# check if hardware injection section exists
# if not then do not do add hardware injection job to the workflow
if not workflow.cp.has_section('workflow-hardware-injections'):
msg = "There is no workflow-hardware-injections section. "
msg += "The hardware injection page will not be added to the workflow."
logging.info(msg)
logging.info("Leaving hardware injection page setup.")
return out_files
# make the output dir
if not os.path.exists(output_dir):
os.makedirs(output_dir)
# create executable
hwinjpage_job = Executable(workflow.cp, 'hardware_injection_page',
'vanilla', workflow.ifos, output_dir, tags)
# retrieve hardware injection file
hwinjDefUrl = workflow.cp.get_opt_tags('workflow-hardware-injections',
'hwinj-definer-url', tags)
hwinjDefBaseName = os.path.basename(hwinjDefUrl)
hwinjDefNewPath = os.path.join(output_dir, hwinjDefBaseName)
urllib.urlretrieve (hwinjDefUrl, hwinjDefNewPath)
# update hwinj definer file location
workflow.cp.set("workflow-hardware-injections", "hwinj-definer-file",
hwinjDefNewPath)
# query for the hardware injection segments
get_hardware_injection_segment_files(workflow, output_dir, hwinjDefNewPath)
# create node
node = Node(hwinjpage_job)
node.add_opt('--gps-start-time', workflow.analysis_time[0])
node.add_opt('--gps-end-time', workflow.analysis_time[1])
node.add_opt('--source-xml', hwinjDefNewPath)
node.add_opt('--segment-dir', output_dir)
node.add_opt('--cache-file', cache_filename)
node.add_opt('--cache-pattern', inspiral_cachepattern)
node.add_opt('--analyze-injections', '')
for ifo in workflow.ifos:
node.add_opt('--%s-injections'%ifo.lower(), '')
outfile = File(node.executable.ifo_string, 'HWINJ_SUMMARY',
workflow.analysis_time, extension='html', directory=output_dir)
node.add_opt('--outfile', outfile.storage_path)
# add node to workflow
workflow.add_node(node)
# make all input_files parents
#for f in input_files:
# dep = dax.Dependency(parent=f.node._dax_node, child=node._dax_node)
# workflow._adag.addDependency(dep)
out_files += node.output_files
logging.info("Leaving hardware injection page setup.")
return out_files
def create_node(self, raw_fit_file, bank_file):
node = Node(self)
node.add_input_opt('--template-fit-file', raw_fit_file)
node.add_input_opt('--template-file', bank_file)
node.new_output_file_opt(raw_fit_file.segment, '.hdf', '--output')
return node
def create_node(self, parent=None, offsource_file=None, seg_dir=None,
found_file=None, missed_file=None, tags=[]):
node = Node(self)
if not parent:
raise ValueError("%s must be supplied with trigger files"
% self.name)
# Set input / output options
node.add_opt('--onsource-file', '%s' % parent.storage_path)
node.add_opt('--offsource-file', '%s' % offsource_file.storage_path)
node.add_opt('--veto-directory', seg_dir)
node.add_opt('--segment-dir', seg_dir)
if found_file and missed_file:
node.add_opt('--found-file', '%s' % found_file.storage_path)
node.add_opt('--missed-file', '%s' % missed_file.storage_path)
out_dir = "%s/output/%s/efficiency_%s" % (self.out_dir, tags[1],
tags[0])
elif found_file or missed_file:
if found_file:
present = found_file
else:
present = missed_file
raise ValueError("Must either be supplied with no injection files "
"or both missed and found injection files. "
"Received only %s" % present.name)
else:
out_dir = "%s/output/%s/efficiency" % (self.out_dir, tags[0])
node.add_opt('--output-path', out_dir)
node.add_profile('condor', 'request_cpus', self.num_threads)
return node
def setup_plotthinca(workflow, input_files, cache_filename, coinc_cachepattern,
slide_cachepattern, output_dir, tags=[], **kwargs):
"""
This function sets up the nodes that will generate summary from a list of
thinca files.
Parameters
-----------
Workflow : ahope.Workflow
The ahope workflow instance that the coincidence jobs will be added to.
input_files : ahope.FileList
An FileList of files that are used as input at this stage.
cache_filename : str
Filename of the ihope cache.
coinc_cachepattern : str
The pattern that will be used to find zero-lag coincidence filenames in the cache.
slide_cachepattern : str
The pattern that will be used to find time slide filenames in the cache.
output_dir : path
The directory in which output files will be stored.
tags : list of strings (optional, default = [])
A list of the tagging strings that will be used for all jobs created
by this call to the workflow. An example might be ['full_data'].
This will be used in output names and directories.
Returns
--------
plot_files : ahope.FileList
A list of the output files from this stage.
"""
plot_files = FileList([])
# create executable
plotthinca_job = Executable(workflow.cp, 'plotthinca', 'vanilla',
workflow.ifos, output_dir, tags)
# get all ifo combinations of at least 2 coincident ifos
ifo_combos = []
for n in xrange(len(plotthinca_job.ifo_list)+1):
for ifo_list in itertools.combinations(plotthinca_job.ifo_list, n+2):
ifo_combos.append(ifo_list)
for tag in tags:
for ifo_list in ifo_combos:
ifo_string = ''.join(ifo_list)
# create node
node = Node(plotthinca_job)
node.add_opt('--gps-start-time', workflow.analysis_time[0])
node.add_opt('--gps-end-time', workflow.analysis_time[1])
node.add_opt('--cache-file', cache_filename)
node.add_opt('--ifo-times', ifo_string)
node.add_opt('--ifo-tag', 'SECOND_'+ifo_string)
for ifo in ifo_list:
node.add_opt('--%s-triggers'%ifo.lower(), '')
node.add_opt('--user-tag', tag.upper()+'_SUMMARY_PLOTS')
node.add_opt('--output-path', output_dir)
node.add_opt('--coinc-pattern', coinc_cachepattern)
node.add_opt('--slide-pattern', slide_cachepattern)
node.add_opt('--enable-output')
# add node to workflow
workflow.add_node(node)
# make all input_files parents
#for f in input_files:
# dep = dax.Dependency(parent=f.node._dax_node, child=node._dax_node)
# workflow._adag.addDependency(dep)
return plot_files
def create_node(self, zerolag, full_data, injfull, fullinj, tags=None):
if tags is None:
tags = []
segs = zerolag.get_times_covered_by_files()
seg = segments.segment(segs[0][0], segs[-1][1])
node = Node(self)
node.set_memory(5000)
node.add_input_list_opt('--zero-lag-coincs', zerolag)
node.add_input_list_opt('--full-data-background', full_data)
node.add_input_list_opt('--mixed-coincs-inj-full', injfull)
node.add_input_list_opt('--mixed-coincs-full-inj', fullinj)
node.new_output_file_opt(seg, '.hdf', '--output-file', tags=tags)
return node
def create_node(self,
trig_files,
bank_file,
stat_files,
veto_file,
veto_name,
template_str,
pivot_ifo,
fixed_ifo,
tags=None):
if tags is None:
tags = []
segs = trig_files.get_times_covered_by_files()
seg = segments.segment(segs[0][0], segs[-1][1])
node = Node(self)
node.add_input_opt('--template-bank', bank_file)
node.add_input_list_opt('--trigger-files', trig_files)
if len(stat_files) > 0:
node.add_input_list_opt('--statistic-files', stat_files)
if veto_file is not None:
node.add_input_opt('--veto-files', veto_file)
node.add_opt('--segment-name', veto_name)
node.add_opt('--pivot-ifo', pivot_ifo)
node.add_opt('--fixed-ifo', fixed_ifo)
node.add_opt('--template-fraction-range', template_str)
node.new_output_file_opt(seg, '.hdf', '--output-file', tags=tags)
return node
def create_node(self, raw_fit_file, bank_file):
node = Node(self)
node.add_input_opt('--template-fit-file', raw_fit_file)
node.add_input_opt('--bank-file', bank_file)
node.new_output_file_opt(raw_fit_file.segment, '.hdf', '--output')
return node
def create_node(self, parent=None, seg_dir=None, inj_file=None, tags=[]):
node = Node(self)
if not parent:
raise ValueError("%s must be supplied with trigger files" %
self.name)
if isinstance(parent, str) and tags[1] == "_unclustered":
node.add_opt('--trig-file', '%s' % parent.storage_path)
tags[1] = ""
else:
node.add_opt('--trig-file', '%s' % parent.storage_path)
node.add_opt('--grb-name', self.cp.get('workflow', 'trigger-name'))
# Set input / output options
node.add_opt('--veto-directory', seg_dir)
node.add_opt('--segment-dir', seg_dir)
out_dir = "%s/output/%s/plots%s" % (self.out_dir, tags[0], tags[1])
node.add_opt('--output-path', out_dir)
if inj_file is not None:
node.add_opt('--inj-file', inj_file.storage_path)
node.add_profile('condor', 'request_cpus', self.num_threads)
return node
def setup_plotnumtemplates(workflow, input_files, cache_filename,
tmpltbank_cachepattern, output_dir, tags=[],
**kwargs):
"""
This function sets up the nodes that will generate a plot of the number
of templates against time.
Parameters
-----------
Workflow : ahope.Workflow
The ahope workflow instance that the coincidence jobs will be added to.
input_files : ahope.FileList
An FileList of files that are used as input at this stage.
cache_filename : str
Filename of the ihope cache.
tmpltbank_cachepattern : str
The pattern that will be used to find template_bank filenames in the cache.
output_dir : path
The directory in which output files will be stored.
tags : list of strings (optional, default = [])
A list of the tagging strings that will be used for all jobs created
by this call to the workflow. An example might be ['full_data'].
This will be used in output names and directories.
Returns
--------
plot_files : ahope.FileList
A list of the output files from this stage.
"""
plot_files = FileList([])
# create executable
plotnumtemplates_job = Executable(workflow.cp, 'plotnumtemplates',
'vanilla', workflow.ifos, output_dir, tags)
for tag in tags:
# create node
node = Node(plotnumtemplates_job)
node.add_opt('--gps-start-time', workflow.analysis_time[0])
node.add_opt('--gps-end-time', workflow.analysis_time[1])
node.add_opt('--cache-file', cache_filename)
node.add_opt('--ifo-times', node.executable.ifo_string)
node.add_opt('--user-tag', tag.upper()+'_SUMMARY_PLOTS')
node.add_opt('--output-path', output_dir)
node.add_opt('--bank-pattern', tmpltbank_cachepattern)
node.add_opt('--enable-output')
# add node to workflow
workflow.add_node(node)
# make all input_files parents
#for f in input_files:
# dep = dax.Dependency(parent=f.node._dax_node, child=node._dax_node)
# workflow._adag.addDependency(dep)
return plot_files
def create_node(self, trig_file, bank_file, veto_file, veto_name):
node = Node(self)
# Executable objects are initialized with ifo information
node.add_opt('--ifo', self.ifo_string)
node.add_input_opt('--trigger-file', trig_file)
node.add_input_opt('--template-file', bank_file)
node.add_input_opt('--veto-file', veto_file)
node.add_opt('--veto-segment-name', veto_name)
node.new_output_file_opt(trig_file.segment, '.hdf', '--output')
return node
def create_node(self, bank_file):
node = Node(self)
node.add_input_opt('--bank-file', bank_file)
node.new_output_file_opt(bank_file.segment, '.hdf', '--output-file')
return node
def create_node(self, workflow, input_files):
node = Node(self)
node.add_input_list_opt('--injection-files', input_files)
node.new_output_file_opt(workflow.analysis_time, '.hdf',
'--output-file')
return node
def create_node(self, trig_files, bank_file, veto_file, veto_name, template_str, tags=None):
if tags is None:
tags = []
segs = trig_files.get_times_covered_by_files()
seg = segments.segment(segs[0][0], segs[-1][1])
node = Node(self)
node.set_memory(10000)
node.add_input_opt('--template-bank', bank_file)
node.add_input_list_opt('--trigger-files', trig_files)
if veto_file is not None:
node.add_input_opt('--veto-files', veto_file)
node.add_opt('--segment-name', veto_name)
node.add_opt('--template-fraction-range', template_str)
node.new_output_file_opt(seg, '.hdf', '--output-file', tags=tags)
return node
def create_node(self, trig_files, inj_files, seg_dir, tags=[]):
node = Node(self)
# Set input / output options
node.add_input_list_opt('--input-files', trig_files)
node.add_input_list_opt('--inj-files', inj_files)
node.add_opt('--ifo-tag', self.ifos)
node.add_opt('--exclude-segments', '%s/bufferSeg.txt' % seg_dir)
node.add_opt('--output-dir', self.out_dir)
# Create output files as File objects
name_string = inj_files[0].description
seg = trig_files[0].segment
f_file = File(self.ifos,
name_string,
seg,
extension="xml",
directory=self.out_dir,
store_file=self.retain_files,
tags=[inj_files[0].tag_str.replace("split0", "FOUND")])
m_file = File(self.ifos,
name_string,
seg,
extension="xml",
directory=self.out_dir,
store_file=self.retain_files,
tags=[inj_files[0].tag_str.replace("split0", "MISSED")])
return node, FileList([f_file, m_file])
def create_node(self,
data_seg,
valid_seg,
parent=None,
inj_file=None,
dfParents=None,
bankVetoBank=None,
ipn_file=None,
tags=[]):
node = Node(self)
if not dfParents:
raise ValueError("%s must be supplied with frame files" %
self.name)
# If doing single IFO search, make sure slides are disabled
if len(self.ifo_list) < 2 and \
('--do-short-slides' in node._options or \
'--short-slide-offset' in node._options):
raise ValueError("Cannot run with time slides in a single IFO "
"configuration! Please edit your configuration "
"file accordingly.")
pad_data = self.get_opt('pad-data')
if pad_data is None:
raise ValueError("The option pad-data is a required option of "
"%s. Please check the ini file." % self.name)
# Feed in bank_veto_bank.xml
if self.cp.has_option('inspiral', 'do-bank-veto'):
if not bankVetoBank:
raise ValueError("%s must be given a bank veto file if the "
"argument 'do-bank-veto' is given" %
self.name)
node.add_input_opt('--bank-veto-templates', bankVetoBank)
# Set time options
node.add_opt('--gps-start-time', data_seg[0] + int(pad_data))
node.add_opt('--gps-end-time', data_seg[1] - int(pad_data))
node.add_opt('--trig-start-time', valid_seg[0])
node.add_opt('--trig-end-time', valid_seg[1])
node.add_profile('condor', 'request_cpus', self.num_threads)
# Set the input and output files
node.new_output_file_opt(data_seg,
'.xml.gz',
'--output-file',
tags=tags,
store_file=self.retain_files)
node.add_input_opt(
'--non-spin-bank',
parent,
)
for frameCache in dfParents:
node.add_input_opt('--%s-frame-cache' % frameCache.ifo.lower(),
frameCache)
node.add_arg('--%s-data' % frameCache.ifo.lower())
if ipn_file is not None:
node.add_input_opt('--sky-positions-file', ipn_file)
if inj_file is not None:
if ('--do-short-slides' in node._options or \
'--short-slide-offset' in node._options):
raise ValueError("Cannot run with short slides in an "
"injection job. Please edit your "
"configuration file accordingly.")
node.add_input_opt('--injection-file', inj_file)
return node
def create_node(self,
trig_files=None,
segment_dir=None,
out_tags=[],
tags=[]):
node = Node(self)
if not trig_files:
raise ValueError(
"%s must be supplied with trigger files" % self.name)
# Data options
pad_data = self.cp.get('inspiral', 'pad-data')
if pad_data is None:
raise ValueError("The option pad-data is a required option of "
"%s. Please check the ini file." % self.name)
num_trials = int(self.cp.get("trig_combiner", "num-trials"))
trig_name = self.cp.get('workflow', 'trigger-name')
node.add_opt('--grb-name', trig_name)
node.add_opt('--pad-data', pad_data)
node.add_opt('--segment-length',
self.cp.get('inspiral', 'segment-duration'))
node.add_opt('--ifo-tag', self.ifos)
node.add_opt('--user-tag', 'INSPIRAL')
# Set input / output options
node.add_input_list_opt('--input-files', trig_files)
node.add_opt('--segment-dir', segment_dir)
node.add_opt('--output-dir', self.out_dir)
out_files = FileList([])
for out_tag in out_tags:
out_file = File(
self.ifos,
'INSPIRAL',
trig_files[0].segment,
directory=self.out_dir,
extension='xml.gz',
tags=["GRB%s" % trig_name, out_tag],
store_file=self.retain_files)
#out_file.PFN(out_file.cache_entry.path, site="local")
out_files.append(out_file)
for trial in range(1, num_trials + 1):
out_file = File(
self.ifos,
'INSPIRAL',
trig_files[0].segment,
directory=self.out_dir,
extension='xml.gz',
tags=["GRB%s" % trig_name,
"OFFTRIAL_%d" % trial],
store_file=self.retain_files)
#out_file.PFN(out_file.cache_entry.path, site="local")
out_files.append(out_file)
node.add_profile('condor', 'request_cpus', self.num_threads)
return node, out_files
def create_node(self,
parent=None,
c_file=None,
open_box=False,
seg_plot=None,
tuning_tags=None,
exclusion_tags=None,
html_dir=None,
tags=[]):
node = Node(self)
node.add_opt('--grb-name', self.cp.get('workflow', 'trigger-name'))
node.add_opt('--start-time', self.cp.get('workflow', 'trigger-time'))
node.add_opt('--ra', self.cp.get('workflow', 'ra'))
node.add_opt('--dec', self.cp.get('workflow', 'dec'))
node.add_opt('--ifo-tag', self.ifos)
if tuning_tags is not None:
node.add_opt('--tuning-injections', ','.join(tuning_tags))
if exclusion_tags is not None:
node.add_opt('--exclusion-injections', ','.join(exclusion_tags))
if seg_plot is not None:
node.add_opt('--seg-plot', seg_plot.storage_path)
if open_box:
node.add_opt('--open-box')
if html_dir is not None:
node.add_opt('--html-path', html_dir)
# Set input / output options
node.add_opt('--config-file', '%s' % c_file.storage_path)
node.add_opt('--output-path', "%s/output" % self.out_dir)
node.add_profile('condor', 'request_cpus', self.num_threads)
return node
def create_node(self,
trig_files=None,
segment_dir=None,
analysis_seg=None,
out_tags=[],
tags=[]):
node = Node(self)
if not trig_files:
raise ValueError("%s must be supplied with trigger files" %
self.name)
# Data options
num_trials = int(self.cp.get("trig_combiner", "num-trials"))
trig_name = self.cp.get('workflow', 'trigger-name')
if all("COHERENT_NO_INJECTIONS" in t.name for t in trig_files) and \
self.cp.has_option_tag('inspiral', 'do-short-slides',
'coherent_no_injections'):
node.add_opt('--short-slides')
node.add_opt('--grb-name', trig_name)
node.add_opt('--trig-start-time', analysis_seg[0])
node.add_opt('--ifo-tag', self.ifos)
node.add_opt('--user-tag', 'INSPIRAL')
# Set input / output options
node.add_input_list_opt('--input-files', trig_files)
node.add_opt('--segment-dir', segment_dir)
node.add_opt('--output-dir', self.out_dir)
out_files = FileList([])
for out_tag in out_tags:
out_file = File(self.ifos,
'INSPIRAL',
trig_files[0].segment,
directory=self.out_dir,
extension='xml.gz',
tags=["GRB%s" % trig_name, out_tag],
store_file=self.retain_files)
out_files.append(out_file)
for trial in range(1, num_trials + 1):
out_file = File(self.ifos,
'INSPIRAL',
trig_files[0].segment,
directory=self.out_dir,
extension='xml.gz',
tags=["GRB%s" % trig_name,
"OFFTRIAL_%d" % trial],
store_file=self.retain_files)
out_files.append(out_file)
node.add_profile('condor', 'request_cpus', self.num_threads)
return node, out_files
def create_node(self, trig_files=None, segment_dir=None, analysis_seg=None,
slide_tag=None, out_tags=None, tags=None):
import Pegasus.DAX3 as dax
if out_tags is None:
out_tags = []
if tags is None:
tags = []
node = Node(self)
if not trig_files:
raise ValueError("%s must be supplied with trigger files"
% self.name)
# Data options
num_trials = int(self.cp.get("trig_combiner", "num-trials"))
trig_name = self.cp.get('workflow', 'trigger-name')
if all("COHERENT_NO_INJECTIONS" in t.name for t in trig_files) and \
self.cp.has_option_tag('inspiral', 'do-short-slides',
'coherent_no_injections'):
node.add_opt('--short-slides')
node.add_opt('--grb-name', trig_name)
node.add_opt('--trig-start-time', analysis_seg[0])
node.add_opt('--ifo-tag', self.ifos)
node.add_opt('--user-tag', 'INSPIRAL')
if tags:
node.add_opt('--job-tag', '_'.join(tags))
if slide_tag is not None:
node.add_opt('--slide-tag', slide_tag)
node.add_opt('--long-slides')
tag_start=["TIMESLIDES_GRB%s_%s" % (trig_name, slide_tag)]+tags
else:
tag_start=["GRB%s" % trig_name]+tags
# Set input / output options
if all(hasattr(t.node, "executable") for t in trig_files):
if all(t.node.executable.name == "trig_cluster"
for t in trig_files):
node.add_opt('--input-files',
" ".join([t.storage_path for t in trig_files]))
if self.cp.has_option_tag('inspiral', 'do-short-slides',
'coherent_no_injections'):
node.add_opt('--short-slides')
else:
node.add_input_list_opt('--input-files', trig_files)
else:
node.add_opt('--input-files',
" ".join([t.storage_path for t in trig_files]))
node.add_opt('--segment-dir', segment_dir)
node.add_opt('--output-dir', self.out_dir)
out_files = FileList([])
for out_tag in out_tags:
out_file = File(self.ifos, 'INSPIRAL', trig_files[0].segment,
directory=self.out_dir, extension='xml.gz',
tags=tag_start+[out_tag],
store_file=self.retain_files)
out_files.append(out_file)
#node._dax_node.uses(out_file, link=dax.Link.OUTPUT, register=False,
# transfer=False)
#node._outputs += [out_file]
#out_file.node = node
#node._add_output(out_file)
for trial in range(1, num_trials + 1):
out_file = File(self.ifos, 'INSPIRAL', trig_files[0].segment,
directory=self.out_dir, extension='xml.gz',
tags=tag_start+["OFFTRIAL_%d" % trial],
store_file=self.retain_files)
out_files.append(out_file)
#node._dax_node.uses(out_file, link=dax.Link.OUTPUT, register=False,
# transfer=False)
#node._outputs += [out_file]
#out_file.node = node
#node._add_output(out_file)
node.add_profile('condor', 'request_cpus', self.num_threads)
return node, out_files
def create_node(self,
parent=None,
offsource_file=None,
seg_dir=None,
found_file=None,
missed_file=None,
tags=[]):
node = Node(self)
if not parent:
raise ValueError(
"%s must be supplied with trigger files" % self.name)
# Set input / output options
node.add_opt('--onsource-file', '%s' % parent.storage_path)
node.add_opt('--offsource-file', '%s' % offsource_file.storage_path)
node.add_opt('--veto-directory', seg_dir)
node.add_opt('--segment-dir', seg_dir)
if found_file and missed_file:
node.add_opt('--found-file', '%s' % found_file.storage_path)
node.add_opt('--missed-file', '%s' % missed_file.storage_path)
out_dir = "%s/output/%s/efficiency_%s" % (self.out_dir, tags[1],
tags[0])
if self.cp.has_option_tag('injections', 'min-distance', tags[-1]):
lower_dist = float(
self.cp.get_opt_tag('injections', 'min-distance',
tags[-1]))
#Convert distance from kpc to Mpc then add as option
lower_dist /= 1e3
node.add_opt('--lower-inj-dist', lower_dist)
if self.cp.has_option_tag('injections', 'max-distance', tags[-1]):
upper_dist = float(
self.cp.get_opt_tag('injections', 'max-distance',
tags[-1]))
#Convert distance from kpc to Mpc then add as option
upper_dist /= 1e3
node.add_opt('--upper-inj-dist', upper_dist)
elif found_file or missed_file:
if found_file:
present = found_file
else:
present = missed_file
raise ValueError("Must either be supplied with no injection files "
"or both missed and found injection files. "
"Received only %s" % present.name)
else:
out_dir = "%s/output/%s/efficiency" % (self.out_dir, tags[0])
node.add_opt('--output-path', out_dir)
node.add_profile('condor', 'request_cpus', self.num_threads)
return node
def create_node(self, coinc_files, bank_file, background_bins, tags=None):
if tags is None:
tags = []
node = Node(self)
node.add_input_list_opt('--coinc-files', coinc_files)
node.add_input_opt('--bank-file', bank_file)
node.add_opt('--background-bins', ' '.join(background_bins))
names = [b.split(':')[0] for b in background_bins]
output_files = [
File(coinc_files[0].ifo_list,
self.name,
coinc_files[0].segment,
directory=self.out_dir,
tags=tags + ['mbin-%s' % i],
extension='.hdf') for i in range(len(background_bins))
]
node.add_output_list_opt('--output-files', output_files)
node.names = names
return node
def create_node(self,
parent=None,
offsource_file=None,
seg_dir=None,
found_file=None,
missed_file=None,
tags=[]):
node = Node(self)
if not parent:
raise ValueError("%s must be supplied with trigger files" %
self.name)
# Set input / output options
node.add_opt('--onsource-file', '%s' % parent.storage_path)
node.add_opt('--offsource-file', '%s' % offsource_file.storage_path)
node.add_opt('--veto-directory', seg_dir)
node.add_opt('--segment-dir', seg_dir)
if found_file and missed_file:
node.add_opt('--found-file', '%s' % found_file.storage_path)
node.add_opt('--missed-file', '%s' % missed_file.storage_path)
out_dir = "%s/output/%s/efficiency_%s" % (self.out_dir, tags[1],
tags[0])
if self.cp.has_option_tag('injections', 'min-distance', tags[-1]):
lower_dist = float(
self.cp.get_opt_tag('injections', 'min-distance',
tags[-1]))
#Convert distance from kpc to Mpc then add as option
lower_dist /= 1e3
node.add_opt('--lower-inj-dist', lower_dist)
if self.cp.has_option_tag('injections', 'max-distance', tags[-1]):
upper_dist = float(
self.cp.get_opt_tag('injections', 'max-distance',
tags[-1]))
#Convert distance from kpc to Mpc then add as option
upper_dist /= 1e3
node.add_opt('--upper-inj-dist', upper_dist)
elif found_file or missed_file:
if found_file:
present = found_file
else:
present = missed_file
raise ValueError("Must either be supplied with no injection files "
"or both missed and found injection files. "
"Received only %s" % present.name)
else:
out_dir = "%s/output/%s/efficiency" % (self.out_dir, tags[0])
node.add_opt('--output-path', out_dir)
node.add_profile('condor', 'request_cpus', self.num_threads)
return node
def create_node(self, bank_file):
node = Node(self)
node.add_input_opt('--bank-file', bank_file)
node.new_output_file_opt(bank_file.segment, '.hdf', '--output-file')
return node
def create_node(self, trig_files=None, segment_dir=None, analysis_seg=None,
out_tags=[], tags=[]):
node = Node(self)
if not trig_files:
raise ValueError("%s must be supplied with trigger files"
% self.name)
# Data options
num_trials = int(self.cp.get("trig_combiner", "num-trials"))
trig_name = self.cp.get('workflow', 'trigger-name')
if all("COHERENT_NO_INJECTIONS" in t.name for t in trig_files) and \
self.cp.has_option_tag('inspiral', 'do-short-slides',
'coherent_no_injections'):
node.add_opt('--short-slides')
node.add_opt('--grb-name', trig_name)
node.add_opt('--trig-start-time', analysis_seg[0])
node.add_opt('--ifo-tag', self.ifos)
node.add_opt('--user-tag', 'INSPIRAL')
# Set input / output options
node.add_input_list_opt('--input-files', trig_files)
node.add_opt('--segment-dir', segment_dir)
node.add_opt('--output-dir', self.out_dir)
out_files = FileList([])
for out_tag in out_tags:
out_file = File(self.ifos, 'INSPIRAL', trig_files[0].segment,
directory=self.out_dir, extension='xml.gz',
tags=["GRB%s" % trig_name, out_tag],
store_file=self.retain_files)
out_files.append(out_file)
for trial in range(1, num_trials + 1):
out_file = File(self.ifos, 'INSPIRAL', trig_files[0].segment,
directory=self.out_dir, extension='xml.gz',
tags=["GRB%s" % trig_name, "OFFTRIAL_%d" % trial],
store_file=self.retain_files)
out_files.append(out_file)
node.add_profile('condor', 'request_cpus', self.num_threads)
return node, out_files
def create_node(self, trig_files, bank_file, stat_files, veto_file,
veto_name, template_str, pivot_ifo, fixed_ifo, tags=None):
if tags is None:
tags = []
segs = trig_files.get_times_covered_by_files()
seg = segments.segment(segs[0][0], segs[-1][1])
node = Node(self)
node.add_input_opt('--template-bank', bank_file)
node.add_input_list_opt('--trigger-files', trig_files)
if len(stat_files) > 0:
node.add_input_list_opt('--statistic-files', stat_files)
if veto_file is not None:
node.add_input_opt('--veto-files', veto_file)
node.add_opt('--segment-name', veto_name)
node.add_opt('--pivot-ifo', pivot_ifo)
node.add_opt('--fixed-ifo', fixed_ifo)
node.add_opt('--template-fraction-range', template_str)
node.new_output_file_opt(seg, '.hdf', '--output-file', tags=tags)
return node
def create_node(self, zerolag, full_data, injfull, fullinj, tags=None):
if tags is None:
tags = []
segs = zerolag.get_times_covered_by_files()
seg = segments.segment(segs[0][0], segs[-1][1])
node = Node(self)
node.set_memory(5000)
node.add_input_list_opt('--zero-lag-coincs', zerolag)
node.add_input_list_opt('--full-data-background', full_data)
node.add_input_list_opt('--mixed-coincs-inj-full', injfull)
node.add_input_list_opt('--mixed-coincs-full-inj', fullinj)
node.new_output_file_opt(seg, '.hdf', '--output-file', tags=tags)
return node
def create_node(self,
trig_files,
bank_file,
veto_file,
veto_name,
template_str,
tags=[]):
segs = trig_files.get_times_covered_by_files()
seg = segments.segment(segs[0][0], segs[-1][1])
node = Node(self)
node.set_memory(10000)
node.add_input_opt('--template-bank', bank_file)
node.add_input_list_opt('--trigger-files', trig_files)
if veto_file is not None:
node.add_input_opt('--veto-files', veto_file)
node.add_opt('--segment-name', veto_name)
node.add_opt('--template-fraction-range', template_str)
node.new_output_file_opt(seg, '.hdf', '--output-file', tags=tags)
return node