def make_veto_table(workflow, out_dir, vetodef_file=None, tags=None): """ Creates a node in the workflow for writing the veto_definer table. Returns a File instances for the output file. """ if vetodef_file is None: if not workflow.cp.has_option_tags("workflow-segments", "segments-veto-definer-file", []): return None vetodef_file = workflow.cp.get_opt_tags("workflow-segments", "segments-veto-definer-file", []) file_url = urljoin('file:', pathname2url(vetodef_file)) vdf_file = File(workflow.ifos, 'VETO_DEFINER', workflow.analysis_time, file_url=file_url) vdf_file.add_pfn(file_url, site='local') else: vdf_file = vetodef_file if tags is None: tags = [] makedir(out_dir) node = PlotExecutable(workflow.cp, 'page_vetotable', ifos=workflow.ifos, out_dir=out_dir, tags=tags).create_node() node.add_input_opt('--veto-definer-file', vdf_file) node.new_output_file_opt(workflow.analysis_time, '.html', '--output-file') workflow += node return node.output_files[0]
def get_coh_PTF_files(cp, ifos, run_dir, bank_veto=False, summary_files=False): """ Retrieve files needed to run coh_PTF jobs within a PyGRB workflow Parameters ---------- cp : pycbc.workflow.configuration.WorkflowConfigParser object The parsed configuration options of a pycbc.workflow.core.Workflow. ifos : str String containing the analysis interferometer IDs. run_dir : str The run directory, destination for retrieved files. bank_veto : Boolean If true, will retrieve the bank_veto_bank.xml file. summary_files : Boolean If true, will retrieve the summary page style files. Returns ------- file_list : pycbc.workflow.FileList object A FileList containing the retrieved files. """ if os.getenv("LAL_SRC") is None: raise ValueError("The environment variable LAL_SRC must be set to a " "location containing the file lalsuite.git") else: lalDir = os.getenv("LAL_SRC") sci_seg = segments.segment(int(cp.get("workflow", "start-time")), int(cp.get("workflow", "end-time"))) file_list = FileList([]) # Bank veto if bank_veto: shutil.copy("%s/lalapps/src/ring/coh_PTF_config_files/" \ "bank_veto_bank.xml" % lalDir, "%s" % run_dir) bank_veto_url = "file://localhost%s/bank_veto_bank.xml" % run_dir bank_veto = File(ifos, "bank_veto_bank", sci_seg, file_url=bank_veto_url) # FIXME: Is this an input file? If so use the from_path classmethod bank_veto.add_pfn(bank_veto.cache_entry.path, site="local") file_list.extend(FileList([bank_veto])) if summary_files: # summary.js file shutil.copy("%s/lalapps/src/ring/coh_PTF_config_files/" \ "coh_PTF_html_summary.js" % lalDir, "%s" % run_dir) summary_js_url = "file://localhost%s/coh_PTF_html_summary.js" \ % run_dir summary_js = File(ifos, "coh_PTF_html_summary_js", sci_seg, file_url=summary_js_url) summary_js.add_pfn(summary_js.cache_entry.path, site="local") file_list.extend(FileList([summary_js])) # summary.css file shutil.copy("%s/lalapps/src/ring/coh_PTF_config_files/" \ "coh_PTF_html_summary.css" % lalDir, "%s" % run_dir) summary_css_url = "file://localhost%s/coh_PTF_html_summary.css" \ % run_dir summary_css = File(ifos, "coh_PTF_html_summary_css", sci_seg, file_url=summary_css_url) summary_css.add_pfn(summary_css.cache_entry.path, site="local") file_list.extend(FileList([summary_css])) return file_list
def make_exttrig_file(cp, ifos, sci_seg, out_dir): ''' Make an ExtTrig xml file containing information on the external trigger Parameters ---------- cp : pycbc.workflow.configuration.WorkflowConfigParser object The parsed configuration options of a pycbc.workflow.core.Workflow. ifos : str String containing the analysis interferometer IDs. sci_seg : ligo.segments.segment The science segment for the analysis run. out_dir : str The output directory, destination for xml file. Returns ------- xml_file : pycbc.workflow.File object The xml file with external trigger information. ''' # Initialise objects xmldoc = ligolw.Document() xmldoc.appendChild(ligolw.LIGO_LW()) tbl = lsctables.New(lsctables.ExtTriggersTable) cols = tbl.validcolumns xmldoc.childNodes[-1].appendChild(tbl) row = tbl.appendRow() # Add known attributes for this GRB setattr(row, "event_ra", float(cp.get("workflow", "ra"))) setattr(row, "event_dec", float(cp.get("workflow", "dec"))) setattr(row, "start_time", int(cp.get("workflow", "trigger-time"))) setattr(row, "event_number_grb", str(cp.get("workflow", "trigger-name"))) # Fill in all empty rows for entry in cols.keys(): if not hasattr(row, entry): if cols[entry] in ['real_4', 'real_8']: setattr(row, entry, 0.) elif cols[entry] == 'int_4s': setattr(row, entry, 0) elif cols[entry] == 'lstring': setattr(row, entry, '') elif entry == 'process_id': row.process_id = ilwd.ilwdchar("external_trigger:process_id:0") elif entry == 'event_id': row.event_id = ilwd.ilwdchar("external_trigger:event_id:0") else: print("Column %s not recognized" % (entry), file=sys.stderr) raise ValueError # Save file xml_file_name = "triggerGRB%s.xml" % str(cp.get("workflow", "trigger-name")) xml_file_path = os.path.join(out_dir, xml_file_name) utils.write_filename(xmldoc, xml_file_path) xml_file_url = urljoin("file:", pathname2url(xml_file_path)) xml_file = File(ifos, xml_file_name, sci_seg, file_url=xml_file_url) xml_file.add_pfn(xml_file_url, site="local") return xml_file
def run_datafind_instance(cp, outputDir, connection, observatory, frameType, startTime, endTime, ifo, tags=None): """ This function will query the datafind server once to find frames between the specified times for the specified frame type and observatory. Parameters ---------- cp : ConfigParser instance Source for any kwargs that should be sent to the datafind module outputDir : Output cache files will be written here. We also write the commands for reproducing what is done in this function to this directory. connection : datafind connection object Initialized through the `gwdatafind` module, this is the open connection to the datafind server. observatory : string The observatory to query frames for. Ex. 'H', 'L' or 'V'. NB: not 'H1', 'L1', 'V1' which denote interferometers. frameType : string The frame type to query for. startTime : int Integer start time to query the datafind server for frames. endTime : int Integer end time to query the datafind server for frames. ifo : string The interferometer to use for naming output. Ex. 'H1', 'L1', 'V1'. Maybe this could be merged with the observatory string, but this could cause issues if running on old 'H2' and 'H1' data. tags : list of string, optional (default=None) Use this to specify tags. This can be used if this module is being called more than once to give call specific configuration (by setting options in [workflow-datafind-${TAG}] rather than [workflow-datafind]). This is also used to tag the Files returned by the class to uniqueify the Files and uniquify the actual filename. FIXME: Filenames may not be unique with current codes! Returns -------- dfCache : glue.lal.Cache instance The glue.lal.Cache representation of the call to the datafind server and the returned frame files. cacheFile : pycbc.workflow.core.File Cache file listing all of the datafind output files for use later in the pipeline. """ if tags is None: tags = [] seg = segments.segment([startTime, endTime]) # Take the datafind kwargs from config (usually urltype=file is # given). dfKwargs = {} # By default ignore missing frames, this case is dealt with outside of here dfKwargs['on_gaps'] = 'ignore' if cp.has_section("datafind"): for item, value in cp.items("datafind"): dfKwargs[item] = value for tag in tags: if cp.has_section('datafind-%s' %(tag)): for item, value in cp.items("datafind-%s" %(tag)): dfKwargs[item] = value # It is useful to print the corresponding command to the logs # directory to check if this was expected. log_datafind_command(observatory, frameType, startTime, endTime, os.path.join(outputDir,'logs'), **dfKwargs) logging.debug("Asking datafind server for frames.") dfCache = lal.Cache.from_urls( connection.find_frame_urls(observatory, frameType, startTime, endTime, **dfKwargs), ) logging.debug("Frames returned") # workflow format output file cache_file = File(ifo, 'DATAFIND', seg, extension='lcf', directory=outputDir, tags=tags) cache_file.add_pfn(cache_file.cache_entry.path, site='local') dfCache.ifo = ifo # Dump output to file fP = open(cache_file.storage_path, "w") # FIXME: CANNOT use dfCache.tofile because it will print 815901601.00000 # as a gps time which is incompatible with the lal cache format # (and the C codes) which demand an integer. #dfCache.tofile(fP) for entry in dfCache: start = str(int(entry.segment[0])) duration = str(int(abs(entry.segment))) print("%s %s %s %s %s" \ % (entry.observatory, entry.description, start, duration, entry.url), file=fP) entry.segment = segments.segment(int(entry.segment[0]), int(entry.segment[1])) fP.close() return dfCache, cache_file
def convert_cachelist_to_filelist(datafindcache_list): """ Take as input a list of glue.lal.Cache objects and return a pycbc FileList containing all frames within those caches. Parameters ----------- datafindcache_list : list of glue.lal.Cache objects The list of cache files to convert. Returns -------- datafind_filelist : FileList of frame File objects The list of frame files. """ prev_file = None prev_name = None this_name = None datafind_filelist = FileList([]) for cache in datafindcache_list: # sort the cache into time sequential order cache.sort() curr_ifo = cache.ifo for frame in cache: # Pegasus doesn't like "localhost" in URLs. frame.url = frame.url.replace('file://localhost','file://') # Create one File() object for each unique frame file that we # get back in the cache. if prev_file: prev_name = os.path.basename(prev_file.cache_entry.url) this_name = os.path.basename(frame.url) if (prev_file is None) or (prev_name != this_name): currFile = File(curr_ifo, frame.description, frame.segment, file_url=frame.url, use_tmp_subdirs=True) datafind_filelist.append(currFile) prev_file = currFile # Populate the PFNs for the File() we just created if frame.url.startswith('file://'): currFile.add_pfn(frame.url, site='local') if frame.url.startswith( 'file:///cvmfs/oasis.opensciencegrid.org/ligo/frames'): # Datafind returned a URL valid on the osg as well # so add the additional PFNs to allow OSG access. currFile.add_pfn(frame.url, site='osg') currFile.add_pfn(frame.url.replace( 'file:///cvmfs/oasis.opensciencegrid.org/', 'root://xrootd-local.unl.edu/user/'), site='osg') currFile.add_pfn(frame.url.replace( 'file:///cvmfs/oasis.opensciencegrid.org/', 'gsiftp://red-gridftp.unl.edu/user/'), site='osg') currFile.add_pfn(frame.url.replace( 'file:///cvmfs/oasis.opensciencegrid.org/', 'gsiftp://ldas-grid.ligo.caltech.edu/hdfs/'), site='osg') elif frame.url.startswith( 'file:///cvmfs/gwosc.osgstorage.org/'): # Datafind returned a URL valid on the osg as well # so add the additional PFNs to allow OSG access. for s in ['osg', 'orangegrid', 'osgconnect']: currFile.add_pfn(frame.url, site=s) currFile.add_pfn(frame.url, site="{}-scratch".format(s)) else: currFile.add_pfn(frame.url, site='notlocal') return datafind_filelist