def create_node(self, parent, tags=None):
import Pegasus.DAX3 as dax
if tags is None:
tags = []
node = Node(self)
# Set input / output options
node.add_opt('--trig-file', '%s' % parent.storage_path)
#node._dax_node.uses(parent, link=dax.Link.INPUT, register=False,
# transfer=False)
#node._inputs += [parent]
node.add_opt('--output-dir', self.out_dir)
node.add_profile('condor', 'request_cpus', self.num_threads)
# Adding output files as pycbc.workflow.core.File objects
out_file = File(self.ifos, 'INSPIRAL', parent.segment,
directory=self.out_dir, extension='xml.gz',
tags=[parent.tag_str, 'CLUSTERED'],
store_file=self.retain_files)
out_file.PFN(out_file.cache_entry.path, site="local")
#node._dax_node.uses(out_file, link=dax.Link.OUTPUT, register=False,
# transfer=False)
#node._outputs += [out_file]
out_file.node = node
#node._add_output(out_file)
return node, FileList([out_file])
def create_node(self,
trig_files=None,
segment_dir=None,
out_tags=[],
tags=[]):
node = Node(self)
if not trig_files:
raise ValueError("%s must be supplied with trigger files" %
self.name)
# Data options
pad_data = self.cp.get('inspiral', 'pad-data')
if pad_data is None:
raise ValueError("The option pad-data is a required option of "
"%s. Please check the ini file." % self.name)
num_trials = int(self.cp.get("trig_combiner", "num-trials"))
trig_name = self.cp.get('workflow', 'trigger-name')
node.add_opt('--grb-name', trig_name)
node.add_opt('--pad-data', pad_data)
node.add_opt('--segment-length',
self.cp.get('inspiral', 'segment-duration'))
node.add_opt('--ifo-tag', self.ifos)
node.add_opt('--user-tag', 'INSPIRAL')
# Set input / output options
node.add_input_list_opt('--input-files', trig_files)
node.add_opt('--segment-dir', segment_dir)
node.add_opt('--output-dir', self.out_dir)
out_files = FileList([])
for out_tag in out_tags:
out_file = File(self.ifos,
'INSPIRAL',
trig_files[0].segment,
directory=self.out_dir,
extension='xml.gz',
tags=["GRB%s" % trig_name, out_tag],
store_file=self.retain_files)
#out_file.PFN(out_file.cache_entry.path, site="local")
out_files.append(out_file)
for trial in range(1, num_trials + 1):
out_file = File(self.ifos,
'INSPIRAL',
trig_files[0].segment,
directory=self.out_dir,
extension='xml.gz',
tags=["GRB%s" % trig_name,
"OFFTRIAL_%d" % trial],
store_file=self.retain_files)
#out_file.PFN(out_file.cache_entry.path, site="local")
out_files.append(out_file)
node.add_profile('condor', 'request_cpus', self.num_threads)
return node, out_files
def create_node(self, trig_files, bank_file):
node = Node(self)
node.add_input_opt('--bank-file', bank_file)
node.add_input_list_opt('--trigger-files', trig_files)
node.new_output_file_opt(trig_files[0].segment, '.hdf',
'--output-file', use_tmp_subdirs=True)
return node
def create_node(self,
trig_files,
bank_file,
stat_files,
veto_file,
veto_name,
template_str,
pivot_ifo,
fixed_ifo,
tags=None):
if tags is None:
tags = []
segs = trig_files.get_times_covered_by_files()
seg = segments.segment(segs[0][0], segs[-1][1])
node = Node(self)
node.add_input_opt('--template-bank', bank_file)
node.add_input_list_opt('--trigger-files', trig_files)
if len(stat_files) > 0:
node.add_input_list_opt('--statistic-files', stat_files)
if veto_file is not None:
node.add_input_opt('--veto-files', veto_file)
node.add_opt('--segment-name', veto_name)
node.add_opt('--pivot-ifo', pivot_ifo)
node.add_opt('--fixed-ifo', fixed_ifo)
node.add_opt('--template-fraction-range', template_str)
node.new_output_file_opt(seg, '.hdf', '--output-file', tags=tags)
return node
def create_node(self, trig_files, inj_files, seg_dir, tags=[]):
node = Node(self)
# Set input / output options
node.add_input_list_opt('--input-files', trig_files)
node.add_input_list_opt('--inj-files', inj_files)
node.add_opt('--ifo-tag', self.ifos)
node.add_opt('--exclude-segments', '%s/bufferSeg.txt' % seg_dir)
node.add_opt('--output-dir', self.out_dir)
# Create output files as File objects
name_string = inj_files[0].description
seg = trig_files[0].segment
f_file = File(self.ifos,
name_string,
seg,
extension="xml",
directory=self.out_dir,
store_file=self.retain_files,
tags=[inj_files[0].tag_str.replace("split0", "FOUND")])
m_file = File(self.ifos,
name_string,
seg,
extension="xml",
directory=self.out_dir,
store_file=self.retain_files,
tags=[inj_files[0].tag_str.replace("split0", "MISSED")])
return node, FileList([f_file, m_file])
def create_node(self,
zerolag,
full_data,
injfull,
fullinj,
ifos,
tags=None):
if tags is None:
tags = []
segs = zerolag.get_times_covered_by_files()
seg = segments.segment(segs[0][0], segs[-1][1])
node = Node(self)
node.add_input_list_opt('--zero-lag-coincs', zerolag)
if isinstance(full_data, list):
node.add_input_list_opt('--full-data-background', full_data)
else:
node.add_input_opt('--full-data-background', full_data)
node.add_input_list_opt('--mixed-coincs-inj-full', injfull)
node.add_input_list_opt('--mixed-coincs-full-inj', fullinj)
node.add_opt('--ifos', ifos)
node.new_output_file_opt(seg, '.hdf', '--output-file', tags=tags)
return node
def create_node(self, parent, inj_trigs, inj_string, max_inc, segment):
node = Node(self)
trig_name = self.cp.get('workflow', 'trigger-name')
node.add_opt('--inj-string', inj_string)
node.add_opt('--max-inclination', max_inc)
node.add_opt('--inj-cache', '%s' % parent.storage_path)
out_files = FileList([])
for inj_trig in inj_trigs:
out_file_tag = [
inj_string, "FILTERED", max_inc,
inj_trig.tag_str.rsplit('_', 1)[-1]
]
out_file = File(self.ifos,
inj_trig.description,
inj_trig.segment,
extension="xml",
directory=self.out_dir,
tags=out_file_tag)
out_file.PFN(out_file.cache_entry.path, site="local")
out_files.append(out_file)
node.add_opt('--output-dir', self.out_dir)
return node, out_files
def create_node(self, parent=None, seg_dir=None, inj_file=None, tags=[]):
node = Node(self)
if not parent:
raise ValueError("%s must be supplied with trigger files" %
self.name)
if isinstance(parent, str) and tags[1] == "_unclustered":
node.add_opt('--trig-file', '%s' % parent.storage_path)
tags[1] = ""
else:
node.add_opt('--trig-file', '%s' % parent.storage_path)
node.add_opt('--grb-name', self.cp.get('workflow', 'trigger-name'))
# Set input / output options
node.add_opt('--veto-directory', seg_dir)
node.add_opt('--segment-dir', seg_dir)
out_dir = "%s/output/%s/plots%s" % (self.out_dir, tags[0], tags[1])
node.add_opt('--output-path', out_dir)
if inj_file is not None:
node.add_opt('--inj-file', inj_file.storage_path)
node.add_profile('condor', 'request_cpus', self.num_threads)
return node
def create_node(self,
parent=None,
c_file=None,
open_box=False,
tuning_tags=None,
exclusion_tags=None,
html_dir=None,
tags=[]):
node = Node(self)
node.add_opt('--grb-name', self.cp.get('workflow', 'trigger-name'))
node.add_opt('--start-time', self.cp.get('workflow', 'trigger-time'))
node.add_opt('--ra', self.cp.get('workflow', 'ra'))
node.add_opt('--dec', self.cp.get('workflow', 'dec'))
node.add_opt('--ifo-tag', self.ifos)
if tuning_tags is not None:
node.add_opt('--tuning-injections', ','.join(tuning_tags))
if exclusion_tags is not None:
node.add_opt('--exclusion-injections', ','.join(exclusion_tags))
if open_box:
node.add_opt('--open-box')
if html_dir is not None:
node.add_opt('--html-path', html_dir)
# Set input / output options
node.add_opt('--config-file', '%s' % c_file.storage_path)
node.add_opt('--output-path', "%s/output" % self.output_dir)
node.add_profile('condor', 'request_cpus', self.num_threads)
return node
def create_node(self, workflow, inj_file, precalc_psd_files, group_str):
node = Node(self)
node.add_input_opt('--input-file', inj_file)
node.add_opt('--injection-fraction-range', group_str)
node.add_input_list_opt('--time-varying-psds', precalc_psd_files)
node.new_output_file_opt(workflow.analysis_time, '.xml',
'--output-file')
return node
def create_node(self, stat_files, tags=[]):
node = Node(self)
node.add_input_list_opt('--statmap-files', stat_files)
node.new_output_file_opt(stat_files[0].segment,
'.hdf',
'--output-file',
tags=tags)
return node
def create_node(self,
data_seg,
valid_seg,
parent=None,
inj_file=None,
dfParents=None,
bankVetoBank=None,
tags=[]):
node = Node(self)
if not dfParents:
raise ValueError("%s must be supplied with frame files" %
self.name)
pad_data = self.get_opt('pad-data')
if pad_data is None:
raise ValueError("The option pad-data is a required option of "
"%s. Please check the ini file." % self.name)
# Feed in bank_veto_bank.xml
if self.cp.has_option('inspiral', 'do-bank-veto'):
if not bankVetoBank:
raise ValueError("%s must be given a bank veto file if the"
"argument 'do-bank-veto' is given" %
self.name)
node.add_input_opt('--bank-veto-templates', bankVetoBank)
node.add_opt('--gps-start-time', data_seg[0] + int(pad_data))
node.add_opt('--gps-end-time', data_seg[1] - int(pad_data))
node.add_opt('--trig-start-time', valid_seg[0])
node.add_opt('--trig-end-time', valid_seg[1])
node.add_profile('condor', 'request_cpus', self.num_threads)
# Set the input and output files
node.new_output_file_opt(data_seg,
'.xml.gz',
'--output-file',
tags=tags,
store_file=self.retain_files)
node.add_input_opt(
'--non-spin-bank',
parent,
)
for frameCache in dfParents:
node.add_input_opt('--%s-frame-cache' % frameCache.ifo.lower(),
frameCache)
node.add_arg('--%s-data' % frameCache.ifo.lower())
node.add_opt(
'--%s-channel-name' % frameCache.ifo.lower(),
self.cp.get('workflow',
'%s-channel-name' % frameCache.ifo.lower()))
if inj_file is not None:
node.add_input_opt('--injection-file', inj_file)
return node
def create_node(self, coinc_files, tags=[]):
segs = coinc_files.get_times_covered_by_files()
seg = segments.segment(segs[0][0], segs[-1][1])
node = Node(self)
node.set_memory(5000)
node.add_input_list_opt('--coinc-files', coinc_files)
node.new_output_file_opt(seg, '.hdf', '--output-file', tags=tags)
return node
def create_node(self,
parent=None,
offsource_file=None,
seg_dir=None,
found_file=None,
missed_file=None,
tags=None):
if tags is None:
tags = []
node = Node(self)
if not parent:
raise ValueError("%s must be supplied with trigger files" %
self.name)
# Set input / output options
node.add_opt('--onsource-file', '%s' % parent.storage_path)
node.add_opt('--offsource-file', '%s' % offsource_file.storage_path)
node.add_opt('--veto-directory', seg_dir)
node.add_opt('--segment-dir', seg_dir)
if found_file and missed_file:
node.add_opt('--found-file', '%s' % found_file.storage_path)
node.add_opt('--missed-file', '%s' % missed_file.storage_path)
out_dir = "%s/output/%s/efficiency_%s" % (self.out_dir, tags[1],
tags[0])
if self.cp.has_option_tag('injections', 'min-distance', tags[-1]):
lower_dist = float(
self.cp.get_opt_tag('injections', 'min-distance',
tags[-1]))
#Convert distance from kpc to Mpc then add as option
lower_dist /= 1e3
node.add_opt('--lower-inj-dist', lower_dist)
if self.cp.has_option_tag('injections', 'max-distance', tags[-1]):
upper_dist = float(
self.cp.get_opt_tag('injections', 'max-distance',
tags[-1]))
#Convert distance from kpc to Mpc then add as option
upper_dist /= 1e3
node.add_opt('--upper-inj-dist', upper_dist)
elif found_file or missed_file:
if found_file:
present = found_file
else:
present = missed_file
raise ValueError("Must either be supplied with no injection files "
"or both missed and found injection files. "
"Received only %s" % present.name)
else:
out_dir = "%s/output/%s/efficiency" % (self.out_dir, tags[0])
node.add_opt('--output-path', out_dir)
node.add_profile('condor', 'request_cpus', self.num_threads)
return node
def create_node(self, statmap_files, cluster_window, tags=None):
if tags is None:
tags = []
node = Node(self)
node.add_input_list_opt('--statmap-files', statmap_files)
node.new_output_file_opt(statmap_files[0].segment, '.hdf',
'--output-file', tags=tags)
node.add_opt('--cluster-window', cluster_window)
return node
def create_node(self,
trig_files=None,
segment_dir=None,
analysis_seg=None,
out_tags=[],
tags=[]):
node = Node(self)
if not trig_files:
raise ValueError("%s must be supplied with trigger files" %
self.name)
# Data options
num_trials = int(self.cp.get("trig_combiner", "num-trials"))
trig_name = self.cp.get('workflow', 'trigger-name')
if all("COHERENT_NO_INJECTIONS" in t.name for t in trig_files) and \
self.cp.has_option_tag('inspiral', 'do-short-slides',
'coherent_no_injections'):
node.add_opt('--short-slides')
node.add_opt('--grb-name', trig_name)
node.add_opt('--trig-start-time', analysis_seg[0])
node.add_opt('--ifo-tag', self.ifos)
node.add_opt('--user-tag', 'INSPIRAL')
# Set input / output options
node.add_input_list_opt('--input-files', trig_files)
node.add_opt('--segment-dir', segment_dir)
node.add_opt('--output-dir', self.out_dir)
out_files = FileList([])
for out_tag in out_tags:
out_file = File(self.ifos,
'INSPIRAL',
trig_files[0].segment,
directory=self.out_dir,
extension='xml.gz',
tags=["GRB%s" % trig_name, out_tag],
store_file=self.retain_files)
out_files.append(out_file)
for trial in range(1, num_trials + 1):
out_file = File(self.ifos,
'INSPIRAL',
trig_files[0].segment,
directory=self.out_dir,
extension='xml.gz',
tags=["GRB%s" % trig_name,
"OFFTRIAL_%d" % trial],
store_file=self.retain_files)
out_files.append(out_file)
node.add_profile('condor', 'request_cpus', self.num_threads)
return node, out_files
def create_node(self, data_seg, valid_seg, parent=None, inj_file=None,
dfParents=None, bankVetoBank=None, tags=[]):
node = Node(self)
if not dfParents:
raise ValueError("%s must be supplied with frame files"
% self.name)
# If doing single IFO search, make sure slides are disabled
if len(self.ifo_list) < 2 and \
('--do-short-slides' in node._options or \
'--short-slide-offset' in node._options):
raise ValueError("Cannot run with time slides in a single IFO "
"configuration! Please edit your configuration "
"file accordingly.")
pad_data = self.get_opt('pad-data')
if pad_data is None:
raise ValueError("The option pad-data is a required option of "
"%s. Please check the ini file." % self.name)
# Feed in bank_veto_bank.xml
if self.cp.has_option('inspiral', 'do-bank-veto'):
if not bankVetoBank:
raise ValueError("%s must be given a bank veto file if the "
"argument 'do-bank-veto' is given"
% self.name)
node.add_input_opt('--bank-veto-templates', bankVetoBank)
# Set time options
node.add_opt('--gps-start-time', data_seg[0] + int(pad_data))
node.add_opt('--gps-end-time', data_seg[1] - int(pad_data))
node.add_opt('--trig-start-time', valid_seg[0])
node.add_opt('--trig-end-time', valid_seg[1])
node.add_profile('condor', 'request_cpus', self.num_threads)
# Set the input and output files
node.new_output_file_opt(data_seg, '.xml.gz', '--output-file',
tags=tags, store_file=self.retain_files)
node.add_input_opt('--non-spin-bank', parent, )
for frameCache in dfParents:
node.add_input_opt('--%s-frame-cache' % frameCache.ifo.lower(),
frameCache)
node.add_arg('--%s-data' % frameCache.ifo.lower())
if inj_file is not None:
if ('--do-short-slides' in node._options or \
'--short-slide-offset' in node._options):
raise ValueError("Cannot run with short slides in an "
"injection job. Please edit your "
"configuration file accordingly.")
node.add_input_opt('--injection-file', inj_file)
return node
def create_node(self, trig_file, bank_file, veto_file, veto_name):
node = Node(self)
# Executable objects are initialized with ifo information
node.add_opt('--ifo', self.ifo_string)
node.add_input_opt('--trigger-file', trig_file)
node.add_input_opt('--bank-file', bank_file)
node.add_input_opt('--veto-file', veto_file)
node.add_opt('--veto-segment-name', veto_name)
node.new_output_file_opt(trig_file.segment, '.hdf', '--output')
return node
def create_node(self, statmap_file, other_statmap_files, tags=None):
if tags is None:
tags = []
node = Node(self)
node.add_input_opt('--statmap-file', statmap_file)
node.add_input_list_opt('--other-statmap-files',
other_statmap_files)
node.new_output_file_opt(statmap_file.segment, '.hdf',
'--output-file', tags=None)
return node
def create_node(self, coinc_files, ifos, tags=None):
if tags is None:
tags = []
segs = coinc_files.get_times_covered_by_files()
seg = segments.segment(segs[0][0], segs[-1][1])
node = Node(self)
node.add_input_list_opt('--coinc-files', coinc_files)
node.add_opt('--ifos', ifos)
node.new_output_file_opt(seg, '.hdf', '--output-file', tags=tags)
return node
def create_node(self, statmap_files, background_files, tags=None):
if tags is None:
tags = []
node = Node(self)
node.add_input_list_opt('--statmap-files', statmap_files)
if 'injections' in tags:
node.add_input_list_opt('--background-files', background_files)
node.new_output_file_opt(statmap_files[0].segment,
'.hdf',
'--output-file',
tags=tags)
return node
def create_node(self, inj_coinc_file, inj_xml_file, veto_file, veto_name, tags=None):
if tags is None:
tags = []
node = Node(self)
node.add_input_list_opt('--trigger-file', inj_coinc_file)
node.add_input_list_opt('--injection-file', inj_xml_file)
if veto_name is not None:
node.add_input_opt('--veto-file', veto_file)
node.add_opt('--segment-name', veto_name)
node.new_output_file_opt(inj_xml_file[0].segment, '.hdf', '--output-file',
tags=tags)
return node
def create_node(self, zerolag, full_data, injfull, fullinj, tags=[]):
segs = zerolag.get_times_covered_by_files()
seg = segments.segment(segs[0][0], segs[-1][1])
node = Node(self)
node.set_memory(5000)
node.add_input_list_opt('--zero-lag-coincs', zerolag)
node.add_input_list_opt('--full-data-background', full_data)
node.add_input_list_opt('--mixed-coincs-inj-full', injfull)
node.add_input_list_opt('--mixed-coincs-full-inj', fullinj)
node.new_output_file_opt(seg, '.hdf', '--output-file', tags=tags)
return node
def create_node(self, bank):
"""
Set up a CondorDagmanNode class to run lalapps_splitbank code
Parameters
----------
bank : pycbc.workflow.core.File
The OutFile containing the template bank to be split
Returns
--------
node : pycbc.workflow.core.Node
The node to run the job
"""
node = Node(self)
# FIXME: This is a hack because SplitBank fails if given an input file
# whose path contains the character '-' or if the input file is not in
# the same directory as the output. Therefore we just set the path to
# be the local path
fullPath = bank.cache_entry.path
bank.cache_entry.path = os.path.basename(fullPath)
node.add_input_opt('--bank-file', bank)
# FIXME: Set the path back to what it was. This is part of the hack
# above and should be removed if possible.
bank.cache_entry.path = fullPath
# Get the output (taken from inspiral.py)
url_list = []
x = bank.filename.split('-')
if len(x) != 4:
errMsg = "Input file name is not compatible with splitbank. Name "
errMsg += "must follow the lal cache standard, for example "
errMsg += "H1-TMPLTBANK-900000000-1000.xml. "
errMsg += "Got %s." % (bank.filename, )
raise ValueError(errMsg)
for i in range(0, self.num_banks):
out_file = "%s-%s_%2.2d-%s-%s" % (x[0], x[1], i, x[2], x[3])
out_url = urlparse.urlunparse([
'file', 'localhost',
os.path.join(self.out_dir, out_file), None, None, None
])
url_list.append(out_url)
job_tag = bank.description + "_" + self.name.upper()
out_file = File(bank.ifo,
job_tag,
bank.segment,
file_url=out_url,
tags=bank.tags,
store_file=self.retain_files)
node._add_output(out_file)
return node
def create_node(self, coinc_files, tags=None):
if tags is None:
tags = []
segs = coinc_files.get_times_covered_by_files()
seg = segments.segment(segs[0][0], segs[-1][1])
node = Node(self)
node.set_memory(5000)
if isinstance(coinc_files, list):
node.add_input_list_opt('--coinc-files', coinc_files)
else:
node.add_input_opt('--coinc-files', coinc_files)
node.new_output_file_opt(seg, '.hdf', '--output-file', tags=tags)
return node
def create_node(self, parent, tags=[]):
node = Node(self)
# Set input / output options
node.add_opt('--trig-file', '%s' % parent.storage_path)
node.add_opt('--output-dir', self.out_dir)
node.add_profile('condor', 'request_cpus', self.num_threads)
# Adding output files as pycbc.workflow.core.File objects
out_file = File(self.ifos, 'INSPIRAL', parent.segment,
directory=self.out_dir, extension='xml.gz',
tags=[parent.tag_str, 'CLUSTERED'],
store_file=self.retain_files)
#out_file.PFN(out_file.cache_entry.path, site="local")
return node, FileList([out_file])
def create_node(self,
trig_files,
bank_file,
veto_file,
veto_name,
template_str,
tags=[]):
segs = trig_files.get_times_covered_by_files()
seg = segments.segment(segs[0][0], segs[-1][1])
node = Node(self)
node.set_memory(10000)
node.add_input_opt('--template-bank', bank_file)
node.add_input_list_opt('--trigger-files', trig_files)
if veto_file is not None:
node.add_input_opt('--veto-files', veto_file)
node.add_opt('--segment-name', veto_name)
node.add_opt('--template-fraction-range', template_str)
node.new_output_file_opt(seg, '.hdf', '--output-file', tags=tags)
return node
def create_node(self, coinc_files, bank_file, background_bins, tags=[]):
node = Node(self)
node.add_input_list_opt('--coinc-files', coinc_files)
node.add_input_opt('--bank-file', bank_file)
node.add_opt('--background-bins', ' '.join(background_bins))
names = [b.split(':')[0] for b in background_bins]
output_files = [
File(coinc_files[0].ifo_list,
self.name,
coinc_files[0].segment,
directory=self.out_dir,
tags=tags + ['mbin-%s' % i],
extension='.hdf') for i in range(len(background_bins))
]
node.add_output_list_opt('--output-files', output_files)
node.names = names
return node
def create_node(self, bank_file):
node = Node(self)
node.add_input_opt('--bank-file', bank_file)
node.new_output_file_opt(bank_file.segment, '.hdf', '--output-file')
return node
def create_node(self, raw_fit_file, bank_file):
node = Node(self)
node.add_input_opt('--template-fit-file', raw_fit_file)
node.add_input_opt('--bank-file', bank_file)
node.new_output_file_opt(raw_fit_file.segment, '.hdf', '--output')
return node
