def setup_psd_pregenerated(workflow, tags=None): ''' Setup CBC workflow to use pregenerated psd files. The file given in cp.get('workflow','pregenerated-psd-file-(ifo)') will be used as the --psd-file argument to geom_nonspinbank, geom_aligned_bank and pycbc_plot_psd_file. Parameters ---------- workflow: pycbc.workflow.core.Workflow An instanced class that manages the constructed workflow. tags : list of strings If given these tags are used to uniquely name and identify output files that would be produced in multiple calls to this function. Returns -------- psd_files : pycbc.workflow.core.FileList The FileList holding the gating files ''' if tags is None: tags = [] psd_files = FileList([]) cp = workflow.cp global_seg = workflow.analysis_time file_attrs = {'segs': global_seg, 'tags': tags} # Check for one psd for all ifos try: pre_gen_file = cp.get_opt_tags('workflow-psd', 'psd-pregenerated-file', tags) file_attrs['ifos'] = workflow.ifos curr_file = resolve_url_to_file(pre_gen_file, attrs=file_attrs) psd_files.append(curr_file) except ConfigParser.Error: # Check for one psd per ifo for ifo in workflow.ifos: try: pre_gen_file = cp.get_opt_tags('workflow-psd', 'psd-pregenerated-file-%s' % ifo.lower(), tags) file_attrs['ifos'] = [ifo] curr_file = resolve_url_to_file(pre_gen_file, attrs=file_attrs) psd_files.append(curr_file) except ConfigParser.Error: # It's unlikely, but not impossible, that only some ifos # will have pregenerated PSDs logging.warn("No psd file specified for IFO %s." % (ifo,)) pass return psd_files
def get_ipn_sky_files(workflow, file_url, tags=None): ''' Retreive the sky point files for searching over the IPN error box and populating it with injections. Parameters ---------- workflow: pycbc.workflow.core.Workflow An instanced class that manages the constructed workflow. file_url : string The URL of the IPN sky points file. tags : list of strings If given these tags are used to uniquely name and identify output files that would be produced in multiple calls to this function. Returns -------- sky_points_file : pycbc.workflow.core.File File object representing the IPN sky points file. ''' tags = tags or [] file_attrs = { 'ifos': workflow.ifos, 'segs': workflow.analysis_time, 'exe_name': "IPN_SKY_POINTS", 'tags': tags } sky_points_file = resolve_url_to_file(file_url, attrs=file_attrs) return sky_points_file
def build_veto_filelist(workflow): """Construct a FileList instance containing all veto xml files""" veto_dir = workflow.cp.get('workflow', 'veto-directory') veto_files = glob.glob(veto_dir + '/*CAT*.xml') veto_files = [resolve_url_to_file(vf) for vf in veto_files] veto_files = _workflow.FileList(veto_files) return veto_files
def build_segment_filelist(workflow): """Construct a FileList instance containing all segments txt files""" seg_dir = workflow.cp.get('workflow', 'segment-dir') file_names = ["bufferSeg.txt", "offSourceSeg.txt", "onSourceSeg.txt"] seg_files = [os.path.join(seg_dir, file_name) for file_name in file_names] seg_files = [resolve_url_to_file(sf) for sf in seg_files] seg_files = _workflow.FileList(seg_files) return seg_files
def setup_injection_workflow(workflow, output_dir=None, inj_section_name='injections', exttrig_file=None, tags=None): """ This function is the gateway for setting up injection-generation jobs in a workflow. It should be possible for this function to support a number of different ways/codes that could be used for doing this, however as this will presumably stay as a single call to a single code (which need not be inspinj) there are currently no subfunctions in this moudle. Parameters ----------- workflow : pycbc.workflow.core.Workflow The Workflow instance that the coincidence jobs will be added to. output_dir : path The directory in which injection files will be stored. inj_section_name : string (optional, default='injections') The string that corresponds to the option describing the exe location in the [executables] section of the .ini file and that corresponds to the section (and sub-sections) giving the options that will be given to the code at run time. tags : list of strings (optional, default = []) A list of the tagging strings that will be used for all jobs created by this call to the workflow. This will be used in output names. Returns -------- inj_files : pycbc.workflow.core.FileList The list of injection files created by this call. inj_tags : list of strings The tag corresponding to each injection file and used to uniquely identify them. The FileList class contains functions to search based on tags. """ if tags is None: tags = [] logging.info("Entering injection module.") make_analysis_dir(output_dir) # Get full analysis segment for output file naming full_segment = workflow.analysis_time ifos = workflow.ifos # Identify which injections to do by presence of sub-sections in # the configuration file inj_tags = [] inj_files = FileList([]) for section in workflow.cp.get_subsections(inj_section_name): inj_tag = section.upper() curr_tags = tags + [inj_tag] # Parse for options in ini file injection_method = workflow.cp.get_opt_tags("workflow-injections", "injections-method", curr_tags) if injection_method in ["IN_WORKFLOW", "AT_RUNTIME"]: # FIXME: Add ability to specify different exes inj_job = LalappsInspinjExecutable(workflow.cp, inj_section_name, out_dir=output_dir, ifos='HL', tags=curr_tags) node = inj_job.create_node(full_segment) if injection_method == "AT_RUNTIME": workflow.execute_node(node) else: workflow.add_node(node) inj_file = node.output_files[0] inj_files.append(inj_file) elif injection_method == "PREGENERATED": file_attrs = { 'ifos': ['HL'], 'segs': full_segment, 'tags': curr_tags } injection_path = workflow.cp.get_opt_tags( "workflow-injections", "injections-pregenerated-file", curr_tags) curr_file = resolve_url_to_file(injection_path, attrs=file_attrs) inj_files.append(curr_file) elif injection_method in ["IN_COH_PTF_WORKFLOW", "AT_COH_PTF_RUNTIME"]: inj_job = LalappsInspinjExecutable(workflow.cp, inj_section_name, out_dir=output_dir, ifos=ifos, tags=curr_tags) node = inj_job.create_node(full_segment, exttrig_file) if injection_method == "AT_COH_PTF_RUNTIME": workflow.execute_node(node) else: workflow.add_node(node) inj_file = node.output_files[0] if workflow.cp.has_option("workflow-injections", "em-bright-only"): em_filter_job = PycbcDarkVsBrightInjectionsExecutable( workflow.cp, 'em_bright_filter', tags=curr_tags, out_dir=output_dir, ifos=ifos) node = em_filter_job.create_node(inj_file, full_segment, curr_tags) if injection_method == "AT_COH_PTF_RUNTIME": workflow.execute_node(node) else: workflow.add_node(node) inj_file = node.output_files[0] if workflow.cp.has_option("workflow-injections", "do-jitter-skyloc"): jitter_job = LigolwCBCJitterSkylocExecutable( workflow.cp, 'jitter_skyloc', tags=curr_tags, out_dir=output_dir, ifos=ifos) node = jitter_job.create_node(inj_file, full_segment, curr_tags) if injection_method == "AT_COH_PTF_RUNTIME": workflow.execute_node(node) else: workflow.add_node(node) inj_file = node.output_files[0] if workflow.cp.has_option("workflow-injections", "do-align-total-spin"): align_job = LigolwCBCAlignTotalSpinExecutable( workflow.cp, 'align_total_spin', tags=curr_tags, out_dir=output_dir, ifos=ifos) node = align_job.create_node(inj_file, full_segment, curr_tags) if injection_method == "AT_COH_PTF_RUNTIME": workflow.execute_node(node) else: workflow.add_node(node) inj_file = node.output_files[0] inj_files.append(inj_file) else: err = "Injection method must be one of IN_WORKFLOW, " err += "AT_RUNTIME or PREGENERATED. Got %s." % (injection_method) raise ValueError(err) inj_tags.append(inj_tag) logging.info("Leaving injection module.") return inj_files, inj_tags
def setup_tmpltbank_pregenerated(workflow, tags=None): ''' Setup CBC workflow to use a pregenerated template bank. The bank given in cp.get('workflow','pregenerated-template-bank') will be used as the input file for all matched-filtering jobs. If this option is present, workflow will assume that it should be used and not generate template banks within the workflow. Parameters ---------- workflow: pycbc.workflow.core.Workflow An instanced class that manages the constructed workflow. tags : list of strings If given these tags are used to uniquely name and identify output files that would be produced in multiple calls to this function. Returns -------- tmplt_banks : pycbc.workflow.core.FileList The FileList holding the details of the template bank. ''' if tags is None: tags = [] # Currently this uses the *same* fixed bank for all ifos. # Maybe we want to add capability to analyse separate banks in all ifos? # Set up class for holding the banks tmplt_banks = FileList([]) cp = workflow.cp global_seg = workflow.analysis_time file_attrs = {'segs': global_seg, 'tags': tags} try: # First check if we have a bank for all ifos pre_gen_bank = cp.get_opt_tags('workflow-tmpltbank', 'tmpltbank-pregenerated-bank', tags) file_attrs['ifos'] = workflow.ifos curr_file = resolve_url_to_file(pre_gen_bank, attrs=file_attrs) tmplt_banks.append(curr_file) except ConfigParser.Error: # Okay then I must have banks for each ifo for ifo in workflow.ifos: try: pre_gen_bank = cp.get_opt_tags( 'workflow-tmpltbank', 'tmpltbank-pregenerated-bank-%s' % ifo.lower(), tags) file_attrs['ifos'] = [ifo] curr_file = resolve_url_to_file(pre_gen_bank, attrs=file_attrs) tmplt_banks.append(curr_file) except ConfigParser.Error: err_msg = "Cannot find pregerated template bank in section " err_msg += "[workflow-tmpltbank] or any tagged sections. " if tags: tagged_secs = " ".join("[workflow-tmpltbank-%s]" \ %(ifo,) for ifo in workflow.ifos) err_msg += "Tagged sections are %s. " % (tagged_secs, ) err_msg += "I looked for 'tmpltbank-pregenerated-bank' option " err_msg += "and 'tmpltbank-pregenerated-bank-%s'." % (ifo, ) raise ConfigParser.Error(err_msg) return tmplt_banks
def setup_foreground_minifollowups(workflow, coinc_file, single_triggers, tmpltbank_file, insp_segs, insp_data_name, insp_anal_name, dax_output, out_dir, tags=None): """ Create plots that followup the Nth loudest coincident injection from a statmap produced HDF file. Parameters ---------- workflow: pycbc.workflow.Workflow The core workflow instance we are populating coinc_file: single_triggers: list of pycbc.workflow.File A list cointaining the file objects associated with the merged single detector trigger files for each ifo. tmpltbank_file: pycbc.workflow.File The file object pointing to the HDF format template bank insp_segs: SegFile The segment file containing the data read and analyzed by each inspiral job. insp_data_name: str The name of the segmentlist storing data read. insp_anal_name: str The name of the segmentlist storing data analyzed. out_dir: path The directory to store minifollowups result plots and files tags: {None, optional} Tags to add to the minifollowups executables Returns ------- layout: list A list of tuples which specify the displayed file layout for the minifollops plots. """ logging.info('Entering minifollowups module') if not workflow.cp.has_section('workflow-minifollowups'): logging.info( 'There is no [workflow-minifollowups] section in configuration file' ) logging.info('Leaving minifollowups') return tags = [] if tags is None else tags makedir(dax_output) # turn the config file into a File class config_path = os.path.abspath(dax_output + '/' + '_'.join(tags) + 'foreground_minifollowup.ini') workflow.cp.write(open(config_path, 'w')) config_file = resolve_url_to_file(config_path) exe = Executable(workflow.cp, 'foreground_minifollowup', ifos=workflow.ifos, out_dir=dax_output, tags=tags) node = exe.create_node() node.add_input_opt('--config-files', config_file) node.add_input_opt('--bank-file', tmpltbank_file) node.add_input_opt('--statmap-file', coinc_file) node.add_multiifo_input_list_opt('--single-detector-triggers', single_triggers) node.add_input_opt('--inspiral-segments', insp_segs) node.add_opt('--inspiral-data-read-name', insp_data_name) node.add_opt('--inspiral-data-analyzed-name', insp_anal_name) if tags: node.add_list_opt('--tags', tags) node.new_output_file_opt(workflow.analysis_time, '.dax', '--dax-file') node.new_output_file_opt(workflow.analysis_time, '.dax.map', '--output-map') name = node.output_files[0].name map_file = node.output_files[1] node.add_opt('--workflow-name', name) node.add_opt('--output-dir', out_dir) workflow += node # execute this in a sub-workflow fil = node.output_files[0] # determine if a staging site has been specified job = SubWorkflow(fil.name, is_planned=False) input_files = [tmpltbank_file, coinc_file, insp_segs] + single_triggers job.add_inputs(*input_files) job.set_subworkflow_properties(map_file, staging_site=workflow.staging_site, cache_file=workflow.cache_file) job.add_into_workflow(workflow, parents=[node]) logging.info('Leaving minifollowups module')
def setup_single_det_minifollowups(workflow, single_trig_file, tmpltbank_file, insp_segs, insp_data_name, insp_anal_name, dax_output, out_dir, veto_file=None, veto_segment_name=None, statfiles=None, tags=None): """ Create plots that followup the Nth loudest clustered single detector triggers from a merged single detector trigger HDF file. Parameters ---------- workflow: pycbc.workflow.Workflow The core workflow instance we are populating single_trig_file: pycbc.workflow.File The File class holding the single detector triggers. tmpltbank_file: pycbc.workflow.File The file object pointing to the HDF format template bank insp_segs: SegFile The segment file containing the data read by each inspiral job. insp_data_name: str The name of the segmentlist storing data read. insp_anal_name: str The name of the segmentlist storing data analyzed. out_dir: path The directory to store minifollowups result plots and files statfiles: FileList (optional, default=None) Supplementary files necessary for computing the single-detector statistic. tags: {None, optional} Tags to add to the minifollowups executables Returns ------- layout: list A list of tuples which specify the displayed file layout for the minifollops plots. """ logging.info('Entering minifollowups module') if not workflow.cp.has_section('workflow-sngl_minifollowups'): msg = 'There is no [workflow-sngl_minifollowups] section in ' msg += 'configuration file' logging.info(msg) logging.info('Leaving minifollowups') return tags = [] if tags is None else tags makedir(dax_output) # turn the config file into a File class curr_ifo = single_trig_file.ifo config_path = os.path.abspath(dax_output + '/' + curr_ifo + \ '_'.join(tags) + 'singles_minifollowup.ini') workflow.cp.write(open(config_path, 'w')) config_file = resolve_url_to_file(config_path) exe = Executable(workflow.cp, 'singles_minifollowup', ifos=curr_ifo, out_dir=dax_output, tags=tags) node = exe.create_node() node.add_input_opt('--config-files', config_file) node.add_input_opt('--bank-file', tmpltbank_file) node.add_input_opt('--single-detector-file', single_trig_file) node.add_input_opt('--inspiral-segments', insp_segs) node.add_opt('--inspiral-data-read-name', insp_data_name) node.add_opt('--inspiral-data-analyzed-name', insp_anal_name) node.add_opt('--instrument', curr_ifo) if veto_file is not None: assert (veto_segment_name is not None) node.add_input_opt('--veto-file', veto_file) node.add_opt('--veto-segment-name', veto_segment_name) if statfiles: statfiles = statfiles.find_output_with_ifo(curr_ifo) node.add_input_list_opt('--statistic-files', statfiles) if tags: node.add_list_opt('--tags', tags) node.new_output_file_opt(workflow.analysis_time, '.dax', '--dax-file') node.new_output_file_opt(workflow.analysis_time, '.dax.map', '--output-map') name = node.output_files[0].name map_file = node.output_files[1] node.add_opt('--workflow-name', name) node.add_opt('--output-dir', out_dir) workflow += node # execute this in a sub-workflow fil = node.output_files[0] job = SubWorkflow(fil.name, is_planned=False) input_files = [tmpltbank_file, insp_segs, single_trig_file] if veto_file is not None: input_files.append(veto_file) if statfiles: input_files += statfiles job.add_inputs(*input_files) job.set_subworkflow_properties(map_file, staging_site=workflow.staging_site, cache_file=workflow.cache_file) job.add_into_workflow(workflow, parents=[node]) logging.info('Leaving minifollowups module')