def create_cache(ifo, start, end): """Find .gwf files and create cache. Will first look in the llhoft, and if the frames have expired from llhoft, will call gwdatafind. Parameters ---------- ifo : str Interferometer name (e.g. ``H1``). start, end: int or float GPS start and end times desired. Returns ------- :class:`glue.lal.Cache` Example ------- >>> create_cache('H1', 1198800018, 1198800618) [<glue.lal.CacheEntry at 0x7fbae6b71278>, <glue.lal.CacheEntry at 0x7fbae6ae5b38>, <glue.lal.CacheEntry at 0x7fbae6ae5c50>, ... <glue.lal.CacheEntry at 0x7fbae6b15080>, <glue.lal.CacheEntry at 0x7fbae6b15828>] """ pattern = app.conf['llhoft_glob'].format(detector=ifo) filenames = glob.glob(pattern) cache = Cache.from_urls(filenames) try: cache_starttime = int( list(cache.to_segmentlistdict().values())[0][0][0]) except IndexError: log.exception('Files do not exist in llhoft_glob') return cache # returns empty cache if start >= cache_starttime: # required data is in llhoft return cache # otherwise, required data has left llhoft high_latency = app.conf['high_latency_frame_types'][ifo] urls = find_urls(ifo[0], high_latency, start, end) if urls: return Cache.from_urls(urls) # required data not in high latency frames low_latency = app.conf['low_latency_frame_types'][ifo] urls = find_urls(ifo[0], low_latency, start, end) if not urls: # required data not in low latency frames log.error('This data cannot be found, or does not exist.') return Cache.from_urls(urls)
def _find_frames_datafind(obs, frametype, start, end, **kwargs): kwargs.setdefault('urltype', 'file') cache = list(map( path_from_file_url, gwdatafind.find_urls(obs[0], frametype, start, end, **kwargs), )) # use latest frame to find more recent frames that aren't in # datafind yet, this is quite hacky, and isn't guaranteed to # work at any point, but it shouldn't break anything try: latest = cache[-1] except IndexError: # no frames, `cache` is list() latestgps = start else: cache.extend(_find_more_files(latest)) latestgps = file_segment(cache[-1])[1] # if we're searching for aggregated h(t), find more files # for the equivalent short h(t) type: if frametype in AGGREGATED_HOFT and latestgps < end: cache.extend(_find_frames_datafind( obs, AGGREGATED_HOFT[frametype], latestgps, end, **kwargs )) return cache
def _find_frames_datafind(obs, frametype, start, end, **kwargs): kwargs.setdefault('urltype', 'file') cache = list( map( path_from_file_url, gwdatafind.find_urls(obs[0], frametype, start, end, **kwargs), )) # use latest frame to find more recent frames that aren't in # datafind yet, this is quite hacky, and isn't guaranteed to # work at any point, but it shouldn't break anything try: latest = cache[-1] except IndexError: # no frames, `cache` is list() latestgps = start else: cache.extend(_find_more_files(latest)) latestgps = file_segment(cache[-1])[1] # if we're searching for aggregated h(t), find more files # for the equivalent short h(t) type: if frametype in AGGREGATED_HOFT and latestgps < end: cache.extend( _find_frames_datafind(obs, AGGREGATED_HOFT[frametype], latestgps, end, **kwargs)) return cache
def checkifo(event): server, repository = connect_gitlab() gitlab_events = gitlab.find_events(repository, subset=event) for event in gitlab_events: if "event time" not in event.event_object.meta: print(f"Time not found {event.event_object.name}") time = event.event_object.meta['event time'] gpsstart = time - 600 gpsend = time + 600 bits = ['Bit 0', 'Bit 1', 'Bit 2'] active_ifo = [] for ifo in ["L1", "H1", "V1"]: frametypes = event.event_object.meta['data']['frame-types'] urls = find_urls(site=f"{ifo[0]}", frametype=frametypes[ifo], gpsstart=gpsstart, gpsend=gpsend) datacache = Cache.from_urls(urls) if len(datacache) == 0: print(f"No {ifo} data found.") continue if "state vector" in event.meta: state_vector_channel = event.meta['state vector'] else: state_vector_channel = ast.literal_eval( config.get("data", "state-vector")) state = gwpy.timeseries.StateVector.read( datacache, state_vector_channel[ifo], start=gpsstart, end=gpsend, pad= 0 # padding data so that errors are not raised even if found data are not continuous. ) if not np.issubdtype(state.dtype, np.unsignedinteger): # if data are not unsigned integers, cast to them now so that # we can determine the bit content for the flags state = state.astype( "uint32", casting="unsafe", subok=True, copy=False, ) flags = state.to_dqflags() segments = flags[bits[0]].active for bit in bits: segments -= ~flags[bit].active if len(segments) > 0: active_ifo += [ifo] print(event.event_object.name) if event.event_object.meta['interferometers'] != active_ifo: print(f"Gitlab data\t{event.event_object.meta['interferometers']}") print(f"Recommended IFOS\t{active_ifo}") event.event_object.meta['interferometers'] = active_ifo event.update_data()
def _data_exists(end, frametype_dict): """Check whether data at end time can be found with gwdatafind and return true it it is found. """ return min( len(find_urls(ifo[0], frametype_dict[ifo], end, end + 1)) for ifo in frametype_dict.keys()) > 0
def _discover_data_source(obs, frametype, start, end, preview): """Determine filepaths to local gravitational-wave frame files """ # get paths to frame files cache1 = gwdatafind.find_urls( obs, frametype, start - preview, start, ) cache2 = gwdatafind.find_urls( obs, frametype, end, end + 1, ) return (cache1, cache2)
def _query(): return gwdatafind.find_urls(ifo[0].upper(), frametype, gpsstart, gpsend, urltype=urltype, on_gaps=gaps, match=match, host=host, port=port)
def _ligo_model_overflow_channels_gwf(dcuid, ifo, frametype, gpstime): try: framefile = find_urls(ifo[0], frametype, gpstime, gpstime)[0] except IndexError as e: e.args = ('No %s-%s frames found at GPS %d' % (ifo[0], frametype, gpstime),) raise try: return _CHANNELS[framefile] except KeyError: _CHANNELS[framefile] = get_channel_names(framefile) return _CHANNELS[framefile]
def get_urls(self, host, args): urls = [] # Parse the input arguments for arg in args: attr, value = arg.split("=") if attr == "observatory": observatory = value if attr == "type": type = value if attr == "s": start_frame = value if attr == "e": end_frame = value # If any input arguments are missing, report error and exit try: observatory, type, end_frame, start_frame except NameError: with open(self.outfile, 'w') as outfile: outfile_dict = get_error_dict( "gwdata:// urls must include arguments 'observatory', 'type', 's' (start frame), 'e' (end frame)" ) outfile = open(self.outfile_path, 'w') outfile.write(str(classad.ClassAd(outfile_dict))) sys.exit(-1) # Retrieve the list of URLs try: urls = gwdatafind.find_urls(host=host, site=observatory, frametype=type, gpsstart=int(start_frame), gpsend=int(end_frame)) except Exception as e: outfile_dict = get_error_dict( f"Error retrieving gwdatafind URLs: {sys.exc_info()[0]} ({e})") outfile = open(self.outfile_path, 'w') outfile.write(str(classad.ClassAd(outfile_dict))) sys.exit(-1) return urls
def check_data_availability(obs, frametype, start, end): """Check for the full data availability for this frame type Parameters ---------- obs : `str` the initial for the observatory frametype : `str` the name of the frame type for which to search start : `int` the GPS start time of this search end : `int` the GPS end time of this search Raises ------ ValueError if gaps are found in the frame archive for the given frame type """ return gwdatafind.find_urls(obs[0], frametype, start, end, on_gaps='error')
def get_urls(self, host, args): urls = [] # Parse the input arguments for arg in args: attr, value = arg.split("=") if attr == "observatory": observatory = value if attr == "type": type = value if attr == "s": start_frame = value if attr == "e": end_frame = value # Retrieve the list of URLs try: urls = gwdatafind.find_urls(host=host, site=observatory, frametype=type, gpsstart=int(start_frame), gpsend=int(end_frame)) except Exception as e: print("Error retrieving gwdatafind URLs: " + str(sys.exc_info()[0]) + ": " + str(e)) return urls
def find_frames(ifo, frametype, gpsstart, gpsend, config=GWSummConfigParser(), urltype='file', gaps='warn', onerror='raise'): """Query the datafind server for GWF files for the given type Parameters ---------- ifo : `str` prefix for the IFO of interest (either one or two characters) frametype : `str` name of the frametype to find gpsstart : `int` GPS start time of the query gpsend : `int` GPS end time of the query config : `~ConfigParser.ConfigParser`, optional configuration with `[datafind]` section containing `server` specification, otherwise taken from the environment urltype : `str`, optional what type of file paths to return, default: `file` gaps : `str`, optional what to do when gaps are detected, one of - `ignore` : do nothing - `warn` : display the existence of gaps but carry on - `raise` : raise an exception onerror : `str`, optional what to do when the `gwdatafind` query itself fails, same options as for ``gaps`` Returns ------- cache : `list` of `str` a list of file paths pointing at GWF files matching the request """ vprint(' Finding %s-%s frames for [%d, %d)...' % (ifo[0], frametype, int(gpsstart), int(gpsend))) # find datafind host:port try: host = config.get('datafind', 'server') except (NoOptionError, NoSectionError): host = None port = None else: port = config.getint('datafind', 'port') # XXX HACK: LLO changed frame types on Dec 6 2013: LLOCHANGE = 1070291904 if re.match(r'L1_{CRMT}', frametype) and gpsstart < LLOCHANGE: frametype = frametype[-1] # query frames ifo = ifo[0].upper() gpsstart = int(floor(gpsstart)) gpsend = int(ceil(min(globalv.NOW, gpsend))) if gpsend <= gpsstart: return [] # parse match try: frametype, match = frametype.split('|', 1) except ValueError: match = None def _query(): return gwdatafind.find_urls(ifo[0].upper(), frametype, gpsstart, gpsend, urltype=urltype, on_gaps=gaps, match=match, host=host, port=port) try: cache = _query() except RuntimeError as e: sleep(1) try: cache = _query() except RuntimeError: if 'Invalid GPS times' in str(e): e.args = ('%s: %d ... %s' % (str(e), gpsstart, gpsend), ) if onerror in ['ignore', None]: pass elif onerror in ['warn']: warnings.warn('Caught %s: %s' % (type(e).__name__, str(e))) else: raise cache = [] # XXX: if querying for day of LLO frame type change, do both if (ifo[0].upper() == 'L' and frametype in ['C', 'R', 'M', 'T'] and gpsstart < LLOCHANGE < gpsend): start = len(cache) and cache[-1].segment[1] or gpsstart if start < gpsend: cache.extend( gwdatafind.find_urls(ifo[0].upper(), 'L1_%s' % frametype, start, gpsend, urltype=urltype, on_gaps=gaps, host=host, port=port)[1:]) # extend cache beyond datafind's knowledge to reduce latency try: latest = cache[-1] ngps = len( re_gwf_gps_epoch.search( os.path.dirname(latest)).groupdict()['gpsepoch']) except (IndexError, AttributeError): pass else: while True: s, e = file_segment(latest) if s >= gpsend: break # replace GPS time of file basename new = latest.replace('-%d-' % s, '-%d-' % e) # replace GPS epoch in dirname new = new.replace('%s/' % str(s)[:ngps], '%s/' % str(e)[:ngps]) if os.path.isfile(new): cache.append(new) else: break # validate files existing and return cache = list(filter(os.path.exists, map(_urlpath, cache))) vprint(' %d found.\n' % len(cache)) return cache
def main(args=None): """Run the software saturation command-line interface """ parser = create_parser() args = parser.parse_args(args=args) # get IFO ifo = args.ifo.upper() site = ifo[0] frametype = args.frametype or '%s_R' % ifo # let's go LOGGER.info('{} Software saturations {}-{}'.format( args.ifo, int(args.gpsstart), int(args.gpsend))) # get segments span = Segment(args.gpsstart, args.gpsend) if args.state_flag: state = DataQualityFlag.query(args.state_flag, int(args.gpsstart), int(args.gpsend), url=const.DEFAULT_SEGMENT_SERVER) for i, seg in enumerate(state.active): state.active[i] = type(seg)(seg[0], seg[1]-args.pad_state_end) segs = state.active.coalesce() LOGGER.debug("Recovered %d seconds of time for %s" % (abs(segs), args.state_flag)) else: segs = SegmentList([Segment(args.gpsstart, args.gpsend)]) # find frames cache = gwdatafind.find_urls( site, frametype, int(args.gpsstart), int(args.gpsend)) # find channels if not os.getenv('LIGO_DATAFIND_SERVER'): raise RuntimeError("No LIGO_DATAFIND_SERVER variable set, don't know " "how to discover channels") else: LOGGER.debug("Identifying channels in frame files") if len(cache) == 0: raise RuntimeError( "No frames recovered for %s in interval [%s, %s)" % (frametype, int(args.gpsstart), int(args.gpsend))) allchannels = get_channel_names(cache[0]) LOGGER.debug(" Found %d channels" % len(allchannels)) sys.stdout.flush() channels = core.find_limit_channels(allchannels, skip=args.skip) LOGGER.info( " Parsed %d channels with '_LIMIT' and '_LIMEN' or '_SWSTAT'" % sum(map(len, channels))) # -- read channels and check limits ------------- saturations = DataQualityDict() bad = set() # TODO: use multiprocessing to separate channel list into discrete chunks # should give a factor of X for X processes # check limens for suffix, clist in zip(['LIMEN', 'SWSTAT'], channels): nchans = len(clist) # group channels in sets for batch processing # min of <number of channels>, user group size (sensible number), # and 512 Mb of RAM for single-precision EPICS try: dur = max([float(abs(s)) for s in segs]) except ValueError: ngroup = args.group_size else: ngroup = int( min(nchans, args.group_size, 2 * 1024**3 / 4. / 16. / dur)) LOGGER.info('Processing %s channels in groups of %d' % ( suffix, ngroup)) sys.stdout.flush() sets = core.grouper(clist, ngroup) for i, cset in enumerate(sets): # remove empty entries use to pad the list to 8 elements cset = list(cset) while cset[-1] is None: cset.pop(-1) for seg in segs: cache2 = sieve_cache(cache, segment=seg) if not len(cache2): continue saturated = core.is_saturated( cset, cache2, seg[0], seg[1], indicator=suffix, nproc=args.nproc) for new in saturated: try: saturations[new.name] += new except KeyError: saturations[new.name] = new for j, c in enumerate(cset): try: sat = saturations[c] except KeyError: LOGGER.debug('%40s: SKIP [%d/%d]' % (c, i*ngroup + j + 1, nchans)) else: if abs(sat.active): LOGGER.debug('%40s: ---- FAIL ---- [%d/%d]' % (c, i*ngroup + j + 1, nchans)) for seg in sat.active: LOGGER.debug(" " * 42 + str(seg)) bad.add(c) else: LOGGER.debug('%40s: PASS [%d/%d]' % (c, i*ngroup + j + 1, nchans)) sys.stdout.flush() # -- log results and exit ----------------------- if len(bad): LOGGER.info("Saturations were found for all of the following:\n\n") for c in bad: print(c) print('\n\n') else: LOGGER.info("No software saturations were found in any channels") # write segments to file outfile = ('%s-SOFTWARE_SATURATIONS-%d-%d.h5' % (ifo, int(args.gpsstart), int(args.gpsend) - int(args.gpsstart))) LOGGER.info("Writing saturation segments to %s" % outfile) saturations.write(outfile, path="segments", overwrite=True) if args.html: # get base path base = os.path.dirname(args.html) os.chdir(base) if args.plot: args.plot = os.path.curdir segfile = os.path.relpath(outfile, os.path.dirname(args.html)) if os.path.basename(args.html) == 'index.html': links = [ '%d-%d' % (int(args.gpsstart), int(args.gpsend)), ('Parameters', '#parameters'), ('Segments', [('Software saturations', '#software-saturations')]), ('Results', '#results'), ] if args.state_flag: links[2][1].insert(0, ('State flag', '#state-flag')) (brand, class_) = htmlio.get_brand(ifo, 'Saturations', args.gpsstart) navbar = htmlio.navbar(links, class_=class_, brand=brand) page = htmlio.new_bootstrap_page( navbar=navbar, title='%s Saturations | %d-%d' % ( ifo, int(args.gpsstart), int(args.gpsend))) else: page = markup.page() page.div(class_='container') # -- header page.div(class_='pb-2 mt-3 mb-2 border-bottom') page.h1('%s Software Saturations: %d-%d' % (ifo, int(args.gpsstart), int(args.gpsend))) page.div.close() # -- paramters content = [ ('State end padding', args.pad_state_end), ('Skip', ', '.join(map(repr, args.skip)))] page.h2('Parameters', class_='mt-4 mb-4', id_='parameters') page.div(class_='row') page.div(class_='col-md-9 col-sm-12') page.add(htmlio.parameter_table( content, start=args.gpsstart, end=args.gpsend, flag=args.state_flag)) page.div.close() # col-md-9 col-sm-12 page.div(class_='col-md-3 col-sm-12') page.add(htmlio.download_btn( [('Segments (HDF)', segfile)], btnclass='btn btn-%s dropdown-toggle' % ifo.lower(), )) page.div.close() # col-md-9 col-sm-12 page.div.close() # row page.h5('Command-line:') page.add(htmlio.get_command_line(about=False, prog=PROG)) # -- segments page.h2('Segments', class_='mt-4', id_='segments') msg = ("This analysis searched {0} filter bank readback channels for " "time periods during which their OUTPUT value matched or " "exceeded the LIMIT value set in software. Signals that " "achieve saturation are shown below, and saturation segments " "are available by expanding a given panel.").format( sum(map(len, channels))) page.add(htmlio.alert(msg, context=ifo.lower())) # record state segments if args.state_flag: page.h3('State flag', class_='mt-3', id_='state-flag') page.div(id_='accordion1') page.add(htmlio.write_flag_html( state, span, 'state', parent='accordion1', context='success', plotdir=args.plot, facecolor=(0.2, 0.8, 0.2), edgecolor='darkgreen', known={ 'facecolor': 'red', 'edgecolor': 'darkred', 'height': 0.4}, )) page.div.close() # record saturation segments if len(bad): page.h3('Software saturations', class_='mt-3', id_='software-saturations') page.div(id_='accordion2') for i, (c, flag) in enumerate(saturations.items()): if abs(flag.active) > 0: title = '%s [%d]' % (flag.name, len(flag.active)) page.add(htmlio.write_flag_html( flag, span=span, id=i, parent='accordion2', title=title, plotdir=args.plot)) page.div.close() else: page.add(htmlio.alert('No software saturations were found in this ' 'analysis', context=ifo.lower(), dismiss=False)) # -- results table page.h2('Results summary', class_='mt-4', id_='results') page.add(htmlio.alert('All channels for which the LIMIT setting was ' 'active are shown below.', context=ifo.lower())) page.table(class_='table table-striped table-hover') # write table header page.thead() page.tr() for header in ['Channel', 'Result', 'Num. saturations']: page.th(header) page.thead.close() # write body page.tbody() for c, seglist in saturations.items(): passed = abs(seglist.active) == 0 if passed: page.tr() else: page.tr(class_='table-warning') page.td(c) page.td(passed and 'Pass' or 'Fail') page.td(len(seglist.active)) page.tr.close() page.tbody.close() page.table.close() # close and write htmlio.close_page(page, args.html)
def get_data(channel, start, end, frametype=None, source=None, nproc=1, verbose=False, **kwargs): """Retrieve data for given channels within a certain time range Parameters ---------- channel : `str` or `list` either a single channel name, or a list of channel names start : `float` GPS start time of requested data end : `float` GPS end time of requested data frametype : `str`, optional name of frametype in which channel(s) are stored, default: `None` source : `str`, `list`, optional path(s) of a LAL-format cache file or individual data file, default: `None` nproc : `int`, optional number of parallel processes to use, uses serial process by default verbose : `bool`, optional print verbose output about NDS progress, default: False **kwargs : `dict`, optional additional keyword arguments to `~gwpy.timeseries.TimeSeries.read` or `~gwpy.timeseries.TimeSeries.get` Returns ------- data : `~gwpy.timeseries.TimeSeries` or `~gwpy.timeseries.TimeSeriesDict` collection of data for the requested channels in the requested time range Notes ----- If `channel` is a `str`, then a `TimeSeries` object will be returned, else the result is a `TimeSeriesDict`. The `frametype` argument should be used to read from archived frame files, while `source` should be used to read from a local cache or specific data file. If either fails, or if neither is passed, this function will attempt to get data over an NDS server. If `frametype` is used to read from the archive, any channels missing from the first or last frame file in the requested time range will be ignored. See Also -------- remove_missing_channels a utility that removes channels missing from the frame archive gwpy.timeseries.TimeSeries.get the underlying method to read data over an NDS server gwpy.timeseries.TimeSeries.read the underlying method to read data from local files """ # get TimeSeries class if isinstance(channel, (list, tuple)): series_class = TimeSeriesDict else: series_class = TimeSeries if frametype is not None: try: # locate frame files ifo = re.search('[A-Z]1', frametype).group(0) obs = ifo[0] source = gwdatafind.find_urls(obs, frametype, start, end) except AttributeError: raise AttributeError( 'Could not determine observatory from frametype') except (HTTPError, JSONDecodeError): # frame files not found pass if isinstance(source, list) and isinstance(channel, (list, tuple)): channel = remove_missing_channels(channel, source) if source is not None: # read from frame files return series_class.read( source, channel, start=start, end=end, nproc=nproc, verbose=verbose, **kwargs) # read single channel from NDS if not isinstance(channel, (list, tuple)): return series_class.get( channel, start, end, verbose=verbose, **kwargs) # if all else fails, process channels in groups of 60 data = series_class() for group in [channel[i:i + 60] for i in range(0, len(channel), 60)]: data.append(series_class.get( group, start, end, verbose=verbose, **kwargs)) return data
def main(args=None): """Run the zero-crossing counter tool """ parser = create_parser() args = parser.parse_args(args=args) span = Segment(args.gpsstart, args.gpsend) LOGGER.info('-- Processing channel %s over span %d - %d' % (args.channel, args.gpsstart, args.gpsend)) if args.state_flag: state = DataQualityFlag.query( args.state_flag, int(args.gpsstart), int(args.gpsend), url=const.DEFAULT_SEGMENT_SERVER, ) statea = state.active else: statea = SegmentList([span]) duration = abs(span) # initialize output files for each threshold and store them in a dict outfiles = {} for thresh in args.threshold: outfiles[str(thresh)] = (os.path.join( args.output_path, '%s_%s_DAC-%d-%d.h5' % (args.channel.replace('-', '_').replace(':', '-'), str( int(thresh)).replace('-', 'n'), int(args.gpsstart), duration))) # get frame cache cache = gwdatafind.find_urls(args.ifo[0], args.frametype, int(args.gpsstart), int(args.gpsend)) cachesegs = statea & cache_segments(cache) if not os.path.exists(args.output_path): os.makedirs(args.output_path) # initialize a ligolw table for each threshold and store them in a dict names = ("time", "frequency", "snr") dtypes = ("f8", ) * len(names) tables = {} for thresh in args.threshold: tables[str(thresh)] = EventTable( names=names, dtype=dtypes, meta={"channel": args.channel}, ) # for each science segment, read in the data from frames, check for # threshold crossings, and if the rate of crossings is less than # rate_thresh, write to a sngl_burst table for seg in cachesegs: LOGGER.debug("Processing {}:".format(seg)) c = sieve_cache(cache, segment=seg) if not c: LOGGER.warning(" No {} data files for this segment, " "skipping".format(args.frametype)) continue data = get_data(args.channel, seg[0], seg[1], nproc=args.nproc, source=c, verbose="Reading data:".rjust(30)) for thresh in args.threshold: times = find_crossings(data, thresh) rate = float(times.size) / abs(seg) if times.size else 0 LOGGER.info(" Found {0} crossings of {1}, rate: {2} Hz".format( times.size, thresh, rate, )) if times.size and rate < args.rate_thresh: existing = tables[str(thresh)] tables[str(thresh)] = vstack_tables( ( existing, table_from_times(times, snr=10., frequency=100., names=existing.colnames), ), join_type="exact", ) n = max(map(len, tables.values())) for thresh, outfile in outfiles.items(): tables[thresh].write( outfile, path="triggers", format="hdf5", overwrite=True, ) LOGGER.info("{0} events written to {1}".format( str(len(tables[thresh])).rjust(len(str(n))), outfile, ))
def find_frames(ifo, frametype, gpsstart, gpsend, config=GWSummConfigParser(), urltype='file', gaps='warn', onerror='raise'): """Query the datafind server for GWF files for the given type Parameters ---------- ifo : `str` prefix for the IFO of interest (either one or two characters) frametype : `str` name of the frametype to find gpsstart : `int` GPS start time of the query gpsend : `int` GPS end time of the query config : `~ConfigParser.ConfigParser`, optional configuration with `[datafind]` section containing `server` specification, otherwise taken from the environment urltype : `str`, optional what type of file paths to return, default: `file` gaps : `str`, optional what to do when gaps are detected, one of - `ignore` : do nothing - `warn` : display the existence of gaps but carry on - `raise` : raise an exception onerror : `str`, optional what to do when the `gwdatafind` query itself fails, same options as for ``gaps`` Returns ------- cache : `list` of `str` a list of file paths pointing at GWF files matching the request """ vprint(' Finding %s-%s frames for [%d, %d)...' % (ifo[0], frametype, int(gpsstart), int(gpsend))) # find datafind host:port try: host = config.get('datafind', 'server') except (NoOptionError, NoSectionError): host = None port = None else: port = config.getint('datafind', 'port') # XXX HACK: LLO changed frame types on Dec 6 2013: LLOCHANGE = 1070291904 if re.match(r'L1_{CRMT}', frametype) and gpsstart < LLOCHANGE: frametype = frametype[-1] # query frames ifo = ifo[0].upper() gpsstart = int(floor(gpsstart)) gpsend = int(ceil(min(globalv.NOW, gpsend))) if gpsend <= gpsstart: return [] # parse match try: frametype, match = frametype.split('|', 1) except ValueError: match = None def _query(): return gwdatafind.find_urls(ifo[0].upper(), frametype, gpsstart, gpsend, urltype=urltype, on_gaps=gaps, match=match, host=host, port=port) try: cache = _query() except RuntimeError as e: sleep(1) try: cache = _query() except RuntimeError: if 'Invalid GPS times' in str(e): e.args = ('%s: %d ... %s' % (str(e), gpsstart, gpsend),) if onerror in ['ignore', None]: pass elif onerror in ['warn']: warnings.warn('Caught %s: %s' % (type(e).__name__, str(e))) else: raise cache = [] # XXX: if querying for day of LLO frame type change, do both if (ifo[0].upper() == 'L' and frametype in ['C', 'R', 'M', 'T'] and gpsstart < LLOCHANGE < gpsend): start = len(cache) and cache[-1].segment[1] or gpsstart if start < gpsend: cache.extend(gwdatafind.find_urls( ifo[0].upper(), 'L1_%s' % frametype, start, gpsend, urltype=urltype, on_gaps=gaps, host=host, port=port)[1:]) # extend cache beyond datafind's knowledge to reduce latency try: latest = cache[-1] ngps = len(re_gwf_gps_epoch.search( os.path.dirname(latest)).groupdict()['gpsepoch']) except (IndexError, AttributeError): pass else: while True: s, e = file_segment(latest) if s >= gpsend: break # replace GPS time of file basename new = latest.replace('-%d-' % s, '-%d-' % e) # replace GPS epoch in dirname new = new.replace('%s/' % str(s)[:ngps], '%s/' % str(e)[:ngps]) if os.path.isfile(new): cache.append(new) else: break # validate files existing and return cache = list(filter(os.path.exists, map(_urlpath, cache))) vprint(' %d found.\n' % len(cache)) return cache
def framecache_from_event(gid, observatories, frame_types, time_span=500, outdir=".", filename="frame.cache", verbose=False): """Get the frame cache for an event given the gracedb event id. Args: gid (str): The gracedb event id. observatories (list): See gwdatafind. frame_type (list): See gwdatafind. time_span (float): The time span before and after the trigger time. outdir (str, default="."): The output directory. filename (str, default="frame.cache"): The output filename. verbose (bool): Be verbose. Returns: Dictionary of instruments, trigger_times, gps_start_time, gps_end_time, channels_name. """ assert time_span >= 500., "Please use time_span larger or equal to 500." obs2ifo = {"H": "H1", "L": "L1", "V": "V1"} observatories = set(observatories) frame_types = set(frame_types) if len(observatories) != len(frame_types): raise ValueError("Must have as many frame_types as observatories.") # FIXME: This is not reliable, have a better way to map frame_type to observatory? obs_type_dict = dict([(obs, frame_type) for obs in observatories for frame_type in frame_types if obs == frame_type[0]]) gracedb_client = gracedb.GraceDb() coinc_xmldoc = lvalert_helper.get_coinc_xmldoc(gracedb_client, gid) eventid_trigger_dict = dict( (row.ifo, row) for row in lsctables.SnglInspiralTable.get_table(coinc_xmldoc)) channel_names_dict = dict([ (row.value.split("=")[0], row.value) for row in lsctables.ProcessParamsTable.get_table(coinc_xmldoc) if row.param == "--channel-name" ]) gwdata_metavar_headers = [ "instruments", "trigger_times", "gps_start_time", "gps_end_time", "channels_name" ] gwdata_metavar_values = [] urls = [] for observatory, frame_type in obs_type_dict.items(): trigger_time = eventid_trigger_dict[obs2ifo[observatory]].end gps_start_time = int(trigger_time - time_span) gps_end_time = int(trigger_time + time_span) gwdata_metavar_values.append( (obs2ifo[observatory], trigger_time, gps_start_time, gps_end_time, channel_names_dict[obs2ifo[observatory]])) urls += gwdatafind.find_urls(observatory, frame_type, gps_start_time, gps_end_time) with open(os.path.join(outdir, "frame.cache"), "w") as cache: for url in urls: filename = str(CacheEntry.from_T050017(url)) cache.write("%s\n" % filename) if verbose: sys.stderr.write( "writing %s to %s\n" % (filename, os.path.join(outdir, "frame.cache"))) if verbose: sys.stderr.write("Done.\n") return dict(zip(gwdata_metavar_headers, zip(*gwdata_metavar_values)))
def main(args=None): """Run the online Guardian node visualization tool """ parser = create_parser() args = parser.parse_args(args=args) fec_map = args.fec_map simulink = args.simulink daqsvn = args.daqsvn or ('https://daqsvn.ligo-la.caltech.edu/websvn/' 'listing.php?repname=daq_maps') if args.ifo == 'H1': if not fec_map: fec_map = 'https://lhocds.ligo-wa.caltech.edu/exports/detchar/fec/' if not simulink: simulink = 'https://lhocds.ligo-wa.caltech.edu/daq/simulink/' if args.ifo == 'L1': if not fec_map: fec_map = 'https://llocds.ligo-la.caltech.edu/exports/detchar/fec/' if not simulink: simulink = 'https://llocds.ligo-la.caltech.edu/daq/simulink/' span = Segment(args.gpsstart, args.gpsend) # let's go LOGGER.info('{} Overflows {}-{}'.format(args.ifo, int(args.gpsstart), int(args.gpsend))) # get segments if args.state_flag: state = DataQualityFlag.query(args.state_flag, int(args.gpsstart), int(args.gpsend), url=const.DEFAULT_SEGMENT_SERVER) tmp = type(state.active)() for i, seg in enumerate(state.active): if abs(seg) < args.segment_end_pad: continue tmp.append(type(seg)(seg[0], seg[1] - args.segment_end_pad)) state.active = tmp.coalesce() statea = state.active else: statea = SegmentList([span]) if not args.output_file: duration = abs(span) args.output_file = ('%s-OVERFLOWS-%d-%d.h5' % (args.ifo, int(args.gpsstart), duration)) LOGGER.debug("Set default output file as %s" % args.output_file) # set up container overflows = DataQualityDict() # prepare data access if args.nds: from gwpy.io import nds2 as io_nds2 host, port = args.nds.rsplit(':', 1) ndsconnection = io_nds2.connect(host, port=int(port)) if ndsconnection.get_protocol() == 1: cachesegs = SegmentList( [Segment(int(args.gpsstart), int(args.gpsend))]) else: cachesegs = io_nds2.get_availability( ['{0}:FEC-1_DAC_OVERFLOW_ACC_0_0'.format(args.ifo)], int(args.gpsstart), int(args.gpsend), ) else: # get frame cache cache = gwdatafind.find_urls(args.ifo[0], args.frametype, int(args.gpsstart), int(args.gpsend)) cachesegs = statea & cache_segments(cache) flag_desc = "ADC/DAC Overflow indicated by {0}" # get channel and find overflows for dcuid in args.dcuid: LOGGER.info("Processing DCUID %d" % dcuid) channel = daq.ligo_accum_overflow_channel(dcuid, args.ifo) overflows[channel] = DataQualityFlag(channel, known=cachesegs) if args.deep: LOGGER.debug(" -- Getting list of overflow channels") try: channels = daq.ligo_model_overflow_channels(dcuid, args.ifo, args.frametype, gpstime=span[0], nds=args.nds) except IndexError: # no frame found for GPS start, try GPS end channels = daq.ligo_model_overflow_channels(dcuid, args.ifo, args.frametype, gpstime=span[-1]) for chan in channels: # set up flags early overflows[chan] = DataQualityFlag( chan, known=cachesegs, description=flag_desc.format(chan), isgood=False, ) LOGGER.debug(" -- %d channels found" % len(channel)) for seg in cachesegs: LOGGER.debug(" -- Processing {}-{}".format(*seg)) if args.nds: read_kw = dict(connection=ndsconnection) else: read_kw = dict(source=cache, nproc=args.nproc) msg = "Reading ACCUM_OVERFLOW data:".rjust(30) data = get_data(channel, seg[0], seg[1], pad=0., verbose=msg, **read_kw) new = daq.find_overflow_segments( data, cumulative=True, ) overflows[channel] += new LOGGER.info(" -- {} overflows found".format(len(new.active))) if not new.active: continue # go deep! for s, e in tqdm.tqdm(new.active.protract(2), unit='ovfl', desc='Going deep'.rjust(30)): data = get_data(channels, s, e, **read_kw) for ch in channels: try: overflows[ch] += daq.find_overflow_segments( data[ch], cumulative=True, ) except KeyError: warnings.warn("Skipping {}".format(ch), UserWarning) continue LOGGER.debug(" -- Search complete") # write output LOGGER.info("Writing segments to %s" % args.output_file) table = table_from_segments( overflows, sngl_burst=args.output_file.endswith((".xml", ".xml.gz")), ) if args.integer_segments: for key in overflows: overflows[key] = overflows[key].round() if args.output_file.endswith((".h5", "hdf", ".hdf5")): with h5py.File(args.output_file, "w") as h5f: table.write(h5f, path="triggers") overflows.write(h5f, path="segments") else: table.write(args.output_file, overwrite=True) overflows.write(args.output_file, overwrite=True, append=True) # write HTML if args.html: # get base path base = os.path.dirname(args.html) os.chdir(base) if args.plot: args.plot = os.path.curdir if args.output_file: args.output_file = os.path.relpath(args.output_file, os.path.dirname(args.html)) if os.path.basename(args.html) == 'index.html': links = [ '%d-%d' % (int(args.gpsstart), int(args.gpsend)), ('Parameters', '#parameters'), ('Segments', [('Overflows', '#overflows')]), ('Results', '#results'), ] if args.state_flag: links[2][1].insert(0, ('State flag', '#state-flag')) (brand, class_) = htmlio.get_brand(args.ifo, 'Overflows', args.gpsstart) navbar = htmlio.navbar(links, class_=class_, brand=brand) page = htmlio.new_bootstrap_page( title='%s Overflows | %d-%d' % (args.ifo, int(args.gpsstart), int(args.gpsend)), navbar=navbar) else: page = htmlio.markup.page() page.div(class_='container') # -- header page.div(class_='pb-2 mt-3 mb-2 border-bottom') page.h1('%s ADC/DAC Overflows: %d-%d' % (args.ifo, int(args.gpsstart), int(args.gpsend))) page.div.close() # -- paramters content = [('DCUIDs', ' '.join(map(str, args.dcuid)))] if daqsvn: content.append(('FEC configuration', ( '<a href="{0}" target="_blank" title="{1} FEC configuration">' '{0}</a>').format(daqsvn, args.ifo))) if fec_map: content.append( ('FEC map', '<a href="{0}" target="_blank" title="{1} FEC ' 'map">{0}</a>'.format(fec_map, args.ifo))) if simulink: content.append( ('Simulink models', '<a href="{0}" target="_blank" title="{1} ' 'Simulink models">{0}</a>'.format(simulink, args.ifo))) page.h2('Parameters', class_='mt-4 mb-4', id_='parameters') page.div(class_='row') page.div(class_='col-md-9 col-sm-12') page.add( htmlio.parameter_table(content, start=args.gpsstart, end=args.gpsend, flag=args.state_flag)) page.div.close() # col-md-9 col-sm-12 # link to summary file if args.output_file: ext = ('HDF' if args.output_file.endswith( (".h5", "hdf", ".hdf5")) else 'XML') page.div(class_='col-md-3 col-sm-12') page.add( htmlio.download_btn( [('Segments ({})'.format(ext), args.output_file)], btnclass='btn btn-%s dropdown-toggle' % args.ifo.lower(), )) page.div.close() # col-md-3 col-sm-12 page.div.close() # row # -- command-line page.h5('Command-line:') page.add(htmlio.get_command_line(about=False, prog=PROG)) # -- segments page.h2('Segments', class_='mt-4', id_='segments') # give contextual information msg = ("This analysis searched for digital-to-analogue (DAC) or " "analogue-to-digital (ADC) conversion overflows in the {0} " "real-time controls system. ").format( SITE_MAP.get(args.ifo, 'LIGO')) if args.deep: msg += ( "A hierarchichal search was performed, with one cumulative " "overflow counter checked per front-end controller (FEC). " "For those models that indicated an overflow, the card- and " "slot-specific channels were then checked. ") msg += ( "Consant overflow is shown as yellow, while transient overflow " "is shown as red. If a data-quality flag was loaded for this " "analysis, it will be displayed in green.") page.add(htmlio.alert(msg, context=args.ifo.lower())) # record state segments if args.state_flag: page.h3('State flag', class_='mt-3', id_='state-flag') page.div(id_='accordion1') page.add( htmlio.write_flag_html(state, span, 'state', parent='accordion1', context='success', plotdir=args.plot, facecolor=(0.2, 0.8, 0.2), edgecolor='darkgreen', known={ 'facecolor': 'red', 'edgecolor': 'darkred', 'height': 0.4, })) page.div.close() # record overflow segments if sum(abs(s.active) for s in overflows.values()): page.h3('Overflows', class_='mt-3', id_='overflows') page.div(id_='accordion2') for i, (c, flag) in enumerate(list(overflows.items())): if abs(flag.active) == 0: continue if abs(flag.active) == abs(cachesegs): context = 'warning' else: context = 'danger' try: channel = cds.get_real_channel(flag.name) except Exception: title = '%s [%d]' % (flag.name, len(flag.active)) else: title = '%s (%s) [%d]' % (flag.name, channel, len(flag.active)) page.add( htmlio.write_flag_html(flag, span, i, parent='accordion2', title=title, context=context, plotdir=args.plot)) page.div.close() else: page.add( htmlio.alert('No overflows were found in this analysis', context=args.ifo.lower(), dismiss=False)) # -- results table page.h2('Results summary', class_='mt-4', id_='results') page.table(class_='table table-striped table-hover') # write table header page.thead() page.tr() for header in ['Channel', 'Connected signal', 'Num. overflows']: page.th(header) page.thead.close() # write body page.tbody() for c, seglist in overflows.items(): t = abs(seglist.active) if t == 0: page.tr() elif t == abs(cachesegs): page.tr(class_='table-warning') else: page.tr(class_='table-danger') page.td(c) try: page.td(cds.get_real_channel(str(c))) except Exception: page.td() page.td(len(seglist.active)) page.tr.close() page.tbody.close() page.table.close() # -- close and write htmlio.close_page(page, args.html) LOGGER.info("HTML written to %s" % args.html)