def test_properties(self): empty = DataQualityFlag() flag = DataQualityFlag(FLAG1, active=ACTIVE, known=KNOWN, padding=(-4, 8)) # name self.assertEqual(empty.name, None) self.assertEqual(flag.name, FLAG1) self.assertEqual(flag.ifo, FLAG1.split(':')[0]) self.assertEqual(flag.version, int(FLAG1.split(':')[-1])) # known self.assertIsInstance(empty.known, SegmentList) self.assertListEqual(empty.known, SegmentList()) self.assertListEqual(flag.known, KNOWN) # active self.assertIsInstance(empty.active, SegmentList) self.assertListEqual(empty.active, SegmentList()) self.assertListEqual(flag.active, ACTIVE) # padding self.assertTupleEqual(empty.padding, (0, 0)) self.assertTupleEqual(flag.padding, (-4, 8)) # texname self.assertEqual(flag.texname, FLAG1.replace('_', r'\_')) self.assertEqual(empty.texname, None) # livetime self.assertEqual(flag.livetime, 4)
def __init__(self, name, known=SegmentList(), active=SegmentList(), description=None, definition=None, hours=None, key=None, filename=None, url=None): """Initialise a new `SummaryState` """ # allow users to specify known as (start, end) if (isinstance(known, Segment) or (isinstance(known, tuple) and len(known) == 2 and not isinstance(known[0], tuple))): known = [known] super(SummaryState, self).__init__(name=name, known=known, active=active) self.description = description if definition: self.definition = re.sub('(\s|\n)', '', definition) else: self.definition = None self.key = key self.hours = hours self.url = url if known and active: self.ready = True else: self.ready = False self.filename = filename
def generate_all_state(start, end, register=True, **kwargs): """Build a new `SummaryState` for the given [start, end) interval. Parameters ---------- start : `~gwpy.time.LIGOTimeGPS`, float the GPS start time of the current analysis end : `~gwpy.time.LIGOTimeGPS`, float the GPS end time of the current analysis register : `bool`, optional should the new `SummaryState` be registered, default `True` **kwargs other keyword arguments passed to the `SummaryState` constructor Returns ------- allstate : `SummaryState` the newly created 'All' `SummaryState` """ now = min(end, NOW) all_ = SummaryState(ALLSTATE, known=SegmentList([Segment(start, end)]), active=SegmentList([Segment(start, now)]), **kwargs) all_.ready = True if register: register_state(all_) return all_
def test_query_dqsegdb(self, name, flag): result = query_dqsegdb(self.TEST_CLASS.query_dqsegdb, name, 0, 10) RESULT = QUERY_RESULTC[flag] assert isinstance(result, self.TEST_CLASS) utils.assert_segmentlist_equal(result.known, RESULT.known) utils.assert_segmentlist_equal(result.active, RESULT.active) result2 = query_dqsegdb(self.TEST_CLASS.query_dqsegdb, name, (0, 10)) utils.assert_flag_equal(result, result2) result2 = query_dqsegdb(self.TEST_CLASS.query_dqsegdb, name, SegmentList([(0, 10)])) utils.assert_flag_equal(result, result2) with pytest.raises(ValueError): query_dqsegdb(self.TEST_CLASS.query_dqsegdb, 'BAD-FLAG_NAME', SegmentList([(0, 10)])) with pytest.raises(HTTPError) as exc: query_dqsegdb(self.TEST_CLASS.query_dqsegdb, 'X1:GWPY-TEST:0', 0, 10) assert str(exc.value) == 'HTTP Error 404: Not found [X1:GWPY-TEST:0]' with pytest.raises(ValueError): self.TEST_CLASS.query_dqsegdb(QUERY_FLAGS[0], 1, 2, 3) with pytest.raises(ValueError): self.TEST_CLASS.query_dqsegdb(QUERY_FLAGS[0], (1, 2, 3))
def __init__(self, parent=None, tssb=None, conc=0.1): super(DataNode, self).__init__(parent=parent, tssb=tssb) # pi is a first-class citizen self.pi = 0.0 self.param = 0.0 self.param1 = 0.0 self.pi1 = 0.0 # used in MH to store old state self.path = None # set of nodes from root to this node self.ht = 0.0 if parent is None: self._conc = conc self.pi = 1.0 self.param = 1.0 else: self.pi = rand(1) * parent.pi parent.pi = parent.pi - self.pi self.param = self.pi # 此处初始化应该设置为SegmentList self.varphiR = SegmentList([Segment(0, 1)]) self.piR = SegmentList([Segment(0, 1)]) self.epsilon = "" # tssb node self.tNode = None
def test_get_frame_segments(find): assert segments.get_frame_segments("X", "X1_R", 0, 100) == SegmentList( [Segment(0, 10), Segment(20, 30)]) assert segments.get_frame_segments( "X", "X1_R", 25, 100, ) == SegmentList([Segment(25, 30)])
def _get_data_segments(self, channel): """Get data segments for this plot """ if self.state and not self.all_data: return self.state.active if channel.sample_rate is not None: return SegmentList( [self.span.protract(1 / channel.sample_rate.value)]) return SegmentList([self.span])
def plot_whitening(station, ts_list, start_time, end_time, seglist=None): """ Generate a spectrogram plot and normalized spectrogram norm: \sqrt{S(f,t)} / \sqrt{\overbar{S(f)}} """ stride, fftlength, overlap = 20, 6, 3 plot = SpectrogramPlot() ax = plot.gca() white_plot = SpectrogramPlot() wax = white_plot.gca() for ts in ts_list: if (len(ts) * ts.dt).value < stride: continue spec = ts.spectrogram(stride, fftlength=fftlength, overlap=overlap) ax.plot(spec, cmap='jet', norm=matplotlib.colors.LogNorm()) wspec = spec.ratio('median') wax.plot(wspec, vmin=0.1, vmax=100, cmap='jet', norm=matplotlib.colors.LogNorm()) ax.set_title('$\mathrm{' + station + '}$') ax.set_ylim(0.1, ts.sample_rate.value / 2.) ax.set_yscale('log') wax.set_title('$\mathrm{' + station + '}$') wax.set_ylim(0.1, ts.sample_rate.value / 2.) wax.set_yscale('log') plot.add_colorbar(label='Amplitude') white_plot.add_colorbar(label='Amplitude') if seglist != None: plot.add_state_segments(SegmentList(seglist[station].active), plotargs={ 'label': 'data present', 'facecolor': 'g', 'edgecolor': 'k' }) white_plot.add_state_segments(SegmentList(seglist[station].active), plotargs={ 'label': 'data present', 'facecolor': 'g', 'edgecolor': 'k' }) # Set limits plot.axes[0].set_epoch(start_time) plot.axes[2].set_epoch(start_time) #plot.axes[1].set_epoch(start_time) white_plot.axes[0].set_epoch(start_time) white_plot.axes[2].set_epoch(start_time) ax.set_xlim(start_time, end_time) wax.set_xlim(start_time, end_time) # Save figures plot.savefig("spectrogram.png", dpi=300) white_plot.savefig("whitened.png", dpi=300)
def get_spectrogram(channel, segments, config=ConfigParser(), cache=None, query=True, nds='guess', format='power', return_=True, frametype=None, multiprocess=True, datafind_error='raise', **fftparams): """Retrieve the time-series and generate a spectrogram of the given channel """ channel = get_channel(channel) # read data for all sub-channels specs = [] channels = re_channel.findall(channel.ndsname) for c in channels: specs.append(_get_spectrogram(c, segments, config=config, cache=cache, query=query, nds=nds, format=format, return_=return_, frametype=frametype, multiprocess=multiprocess, datafind_error=datafind_error, **fftparams)) if return_ and len(channels) == 1: return specs[0] elif return_: # get union of segments for all sub-channels datasegs = reduce(operator.and_, [sgl.segments for sgl in specs]) # build meta-spectrogram for all interseceted segments out = SpectrogramList() operators = [channel.name[m.span()[1]] for m in list(re_channel.finditer(channel.ndsname))[:-1]] for seg in datasegs: sg = _get_spectrogram(channels[0], SegmentList([seg]), config=config, query=False, format=format, return_=True)[0] sg.name = str(channel) for op, ch in zip(operators, channels[1:]): try: op = OPERATOR[op] except KeyError as e: e.args = ('Cannot parse math operator %r' % op,) raise data = _get_spectrogram(ch, SegmentList([seg]), config=config, query=False, format=format, return_=True) try: sg = op(sg, data[0]) except ValueError as e: if 'could not be broadcast together' in str(e): s = min(sg.shape[0], data[0].shape[0]) sg = op(sg[:s], data[0][:s]) else: raise out.append(sg) return out
def grab_time_triggers(wildcard, start, end): """Retrieve triggers from a given GPS time range """ time_segs = SegmentList([]) start_time_utc = tconvert(start) for filename in glob.glob(wildcard): data = SegmentList.read(filename) LOGGER.info(' '.join(['grabbing trigger file:', filename])) start_end_seg = Segment(start, end) c = data & SegmentList([start_end_seg]) time_segs += c start_time_utc += datetime.timedelta(days=1) return time_segs
def test_get_triggers(self): # test that trigfind raises a warning if the channel-level directory # doesn't exist with pytest.warns(UserWarning): out = triggers.get_triggers('X1:DOES_NOT_EXIST', 'omicron', SegmentList([Segment(0, 100)])) # check output type and columns self.assertIsInstance(out, numpy.ndarray) for col in ['time', 'frequency', 'snr']: self.assertIn(col, out.dtype.fields) # test that unknown ETG raises KeyError self.assertRaises(KeyError, triggers.get_triggers, 'X1:DOES_NOT_EXIST', 'fake-etg', SegmentList([Segment(0, 100)]))
def cache_overlaps(*caches): """Find segments of overlap in the given cache sets """ cache = [e for c in caches for e in c] cache.sort(key=lambda e: file_segment(e)[0]) overlap = SegmentList() segments = SegmentList() for e in cache: seg = file_segment(e) ol = SegmentList([seg]) & segments if abs(ol): overlap.extend(ol) segments.append(seg) return overlap
def test_cache_segments(self): # check empty input sl = cache_segments() self.assertIsInstance(sl, SegmentList) self.assertEquals(len(sl), 0) cache, segs = self.make_cache() try: # check good cache sl = cache_segments(cache) self.assertNotEquals(sl, segs) self.assertEquals(sl, type(segs)(segs).coalesce()) # check bad cache os.remove(cache[0].path) sl = cache_segments(cache) self.assertEquals(sl, segs[1:]) # check cache with no existing files sl = cache_segments(cache[:1]) self.assertEquals(sl, SegmentList()) # check errors self.assertRaises(TypeError, cache_segments, blah='blah') self.assertRaises(ValueError, cache_segments, cache, on_missing='error') self.assertRaises(ValueError, cache_segments, cache, on_missing='blah') # clean up finally: self.destroy_cache(cache)
def find_cache_segments(*caches): """Construct a :class:`~gwpy.segments.segments.SegmentList` describing the validity of a given :class:`~glue.lal.Cache`, or list of them. Parameters ---------- cache : :class:`~glue.lal.Cache` Cache of frame files to check Returns ------- segments : :class:`~gwpy.segments.segments.SegmentList` list of segments containing in cache """ out = SegmentList() nframes = sum(len(c) for c in caches) if nframes == 0: return out for cache in caches: # build segment for this cache if not len(cache): continue seg = cache[0].segment for e in cache: # if new segment doesn't overlap, append and start again if e.segment.disjoint(seg): out.append(seg) seg = e.segment # otherwise, append to current segment else: seg |= e.segment # append final segment and return out.append(seg) return out
def segments_from_array(array): """Convert a 2-dimensional `numpy.ndarray` to a `SegmentList` """ out = SegmentList() for row in array: out.append(Segment(*row)) return out
def divide_segmentlist(start,end,bins=4096,write=True,**kwargs): ''' Divide given period to segmenlist Parameters ---------- start : `int` GPS start time of given period end : `int` GPS end time of given period bins : `int`, optional The number of bins. Unit is second. Default value is 4096 =(2**12). Returns ------- segmentlist : `gwpy.segment.SegmentList` Divided segmentlist ''' if ((end-start) % bins) != 0: raise ValueError('Not divisible!') _start = range(start ,end ,bins) _end = range(start+bins,end+bins,bins) segmentlist = SegmentList([Segment(s,e) for s,e in zip(_start,_end)]) log.debug(segmentlist[0]) log.debug(segmentlist[-1]) if write: segmentlist.write('./segmentlist/total.txt') return segmentlist
def add_triggers(table, key, segments=None): """Add a `EventTable` to the global memory cache """ if segments is not None: table.meta['segments'] = segments try: old = globalv.TRIGGERS[key] except KeyError: new = globalv.TRIGGERS[key] = table new.meta.setdefault('segments', SegmentList()) else: new = globalv.TRIGGERS[key] = vstack_tables((old, table)) new.meta = old.meta new.meta['segments'] |= table.meta.get('segments', SegmentList()) new.meta['segments'].coalesce() return new
def read_cache(cache, segments, etg, nproc=1, timecolumn=None, **kwargs): """Read a table of events from a cache This function is mainly meant for use from the `get_triggers` method Parameters ---------- cache : :class:`glue.lal.Cache` the formatted list of files to read segments : `~gwpy.segments.SegmentList` the list of segments to read etg : `str` the name of the trigger generator that created the files nproc : `int`, optional the number of parallel processes to use when reading **kwargs other keyword arguments are passed to the `EventTable.read` or `{tableclass}.read` methods Returns ------- table : `~gwpy.table.EventTable`, `None` a table of events, or `None` if the cache has no overlap with the segments """ if isinstance(cache, Cache): cache = cache.sieve(segmentlist=segments) cache = cache.checkfilesexist()[0] cache.sort(key=lambda x: x.segment[0]) cache = cache.pfnlist() # some readers only like filenames else: cache = [urlparse(url).path for url in cache] if etg == 'pycbc_live': # remove empty HDF5 files cache = filter_pycbc_live_files(cache, ifo=kwargs['ifo']) if len(cache) == 0: return # read triggers table = EventTable.read(cache, **kwargs) # store read keywords in the meta table if timecolumn: table.meta['timecolumn'] = timecolumn # get back from cache entry if isinstance(cache, CacheEntry): cache = Cache([cache]) # append new events to existing table try: csegs = cache_segments(cache) & segments except (AttributeError, TypeError, ValueError): csegs = SegmentList() table.meta['segments'] = csegs if timecolumn: # already filtered on-the-fly return table # filter now return keep_in_segments(table, segments, etg)
def draw(self): """Read in all necessary data, and generate the figure. """ plot = self.init_plot() ax = plot.gca() # work out labels labels = self.pargs.pop('labels', self.channels) if isinstance(labels, str): labels = labels.split(',') labels = [str(s).strip('\n ') for s in labels] # add data for label, channel in zip(labels, self.channels): label = usetex_tex(label) if self.state and not self.all_data: valid = self.state.active else: valid = SegmentList([self.span]) data = get_timeseries(channel, valid, query=False) # handle no timeseries if not len(data): ax.plot([0], [0], visible=False, label=label) continue # plot time-series color = None for ts in data: # double-check log scales if self.logy: ts.value[ts.value == 0] = 1e-100 if color is None: line = ax.plot(ts, label=label)[0] color = line.get_color() else: ax.plot(ts, color=color, label=None) # allow channel data to set parameters if hasattr(data[0].channel, 'amplitude_range'): self.pargs.setdefault('ylim', data[0].channel.amplitude_range) # add horizontal lines to add for yval in self.pargs['hline']: try: yval = float(yval) except ValueError: continue else: ax.plot([self.start, self.end], [yval, yval], linestyle='--', color='red') # customise plot legendargs = self.parse_legend_kwargs() self.apply_parameters(ax, **self.pargs) if len(self.channels) > 1: ax.legend(**legendargs) # finalise self.add_state_segments(ax) return self.finalize()
def find_best_frames(ifo, frametype, start, end, **kwargs): """Find frames for the given type, replacing with a better type if needed """ # find cache for this frametype cache = find_frames(ifo, frametype, start, end, **kwargs) # check for gaps in current cache span = SegmentList([Segment(start, end)]) gaps = span - cache_segments(cache) # if gaps and using aggregated h(t), check short files if abs(gaps) and frametype in SHORT_HOFT_TYPES: f2 = SHORT_HOFT_TYPES[frametype] vprint(" Gaps discovered in aggregated h(t) type " "%s, checking %s\n" % (frametype, f2)) kwargs['gaps'] = 'ignore' cache.extend( filter(lambda e: file_segment(e) in gaps, find_frames(ifo, f2, start, end, **kwargs))) new = int(abs(gaps - cache_segments(cache))) if new: vprint(" %ss extra coverage with frametype %s\n" % (new, f2)) else: vprint(" No extra coverage with frametype %s\n" % f2) return cache, frametype
def fetch(self, config=GWSummConfigParser(), segdb_error='raise', datafind_error='raise', **kwargs): """Finalise this state by fetching its defining segments, either from global memory, or from the segment database """ # check we haven't done this before if self.ready: return self # fetch data if self.definition: match = re.search('(%s)' % '|'.join(MATHOPS.keys()), self.definition) else: match = None if self.filename: self._read_segments(self.filename) elif match: channel, thresh = self.definition.split(match.groups()[0]) channel = channel.rstrip() thresh = float(thresh.strip()) self._fetch_data(channel, thresh, match.groups()[0], config=config, datafind_error=datafind_error, **kwargs) # fetch segments elif self.definition: self._fetch_segments(config=config, segdb_error=segdb_error, **kwargs) # fetch null else: start = config.getfloat(DEFAULTSECT, 'gps-start-time') end = config.getfloat(DEFAULTSECT, 'gps-end-time') self.known = [(start, end)] self.active = self.known # restrict to given hours if self.hours: segs_ = SegmentList() # get start day d = Time(float(self.start), format='gps', scale='utc').datetime d.replace(hour=0, minute=0, second=0, microsecond=0) end_ = Time(float(self.end), format='gps', scale='utc').datetime while d < end_: # get GPS of day t = to_gps(d) # for each [start, end) hour pair, build a segment for h0, h1 in self.hours: segs_.append(Segment(t + h0 * 3600, t + h1 * 3600)) # increment and return d += datetime.timedelta(1) self.known &= segs_ self.active &= segs_ # FIXME self.ready = True return self
def draw(self): """Read in all necessary data and generate a figure """ keys = [] # generate data for i, channel in enumerate(self.channels): fftkwargs = dict((key, self.fftparams[key][i]) for key in self.fftparams if self.fftparams[key][i] is not None) rangekwargs = dict((key, self.rangeparams[key][i]) for key in self.rangeparams if self.rangeparams[key][i] is not None) if self.state and not self.all_data: valid = self.state.active else: valid = SegmentList([self.span]) rlist = self.range_func(channel, valid, query=self.read, **fftkwargs, **rangekwargs) try: keys.append(str(rlist[0].channel)) except IndexError: keys.append(get_range_channel(channel, **rangekwargs)) # reset channel lists and generate plot channels = self.channels self.channels = keys out = super(RangePlotMixin, self).draw() self.channels = channels return out
def find_cache_segments(*caches): """Return the segments covered by one or more data caches Parameters ---------- *cache : `~glue.lal.Cache` one or more file caches Returns ------- segments : `~gwpy.segments.SegmentList` list of segments containing in cache """ out = SegmentList() nframes = sum(len(c) for c in caches) if nframes == 0: return out for cache in caches: # build segment for this cache if not len(cache): continue seg = cache[0].segment for e in cache: # if new segment doesn't overlap, append and start again if e.segment.disjoint(seg): out.append(seg) seg = e.segment # otherwise, append to current segment else: seg |= e.segment # append final segment and return out.append(seg) return out
def draw(self): """Read in all necessary data, and generate the figure. """ # generate data keys = [] for i, channel in enumerate(self.channels): kwargs = dict((key, self.rangeparams[key][i]) for key in self.rangeparams if self.rangeparams[key][i] is not None) if self.state and not self.all_data: valid = self.state.active else: valid = SegmentList([self.span]) rlist = get_range(channel, valid, query=self.read, **kwargs) try: keys.append(rlist[0].channel) except IndexError: keys.append(get_range_channel(channel, **kwargs)) # reset channel lists and generate time-series plot channels = self.channels outputfile = self.outputfile self.channels = keys out = super(RangePlotMixin, self).draw(outputfile=outputfile) self.channels = channels return out
def combined_time_volume(self, allsegments, allranges): try: combined_range = TimeSeries(numpy.zeros(allranges[0].size), xindex=allranges[0].times, unit='Mpc') except IndexError: combined_range = TimeSeries(numpy.zeros(allranges[0].size), unit='Mpc', x0=allranges[0].x0, dx=allranges[0].dx) # get coincident observing segments pairs = list(combinations(allsegments, 2)) coincident = SegmentList() for pair in pairs: coincident.extend(pair[0] & pair[1]) coincident = coincident.coalesce() # get effective network range values = [r.value for r in allranges] values = [min(nlargest(2, x)) for x in zip(*values)] size = min([r.size for r in allranges]) combined_range[:size] = values * combined_range.unit # compute time-volume return self.calculate_time_volume(coincident, combined_range)
def test_add_state_segments(self): fig, ax = self.new() # mock up some segments and add them as 'state' segments segs = SegmentList([Segment(1, 2), Segment(4, 5)]) segax = fig.add_state_segments(segs) # check that the new axes aligns with the parent utils.assert_array_equal(segax.get_position().intervalx, ax.get_position().intervalx) coll = segax.collections[0] for seg, path in zip(segs, coll.get_paths()): utils.assert_array_equal(path.vertices, [(seg[0], -.4), (seg[1], -.4), (seg[1], .4), (seg[0], .4), (seg[0], -.4)]) with pytest.raises(ValueError): fig.add_state_segments(segs, location='left') # test that this doesn't work with non-timeseries axes fig = self.FIGURE_CLASS() ax = fig.gca(projection='rectilinear') with pytest.raises(ValueError) as exc: fig.add_state_segments(segs) assert str(exc.value) == ("No 'timeseries' Axes found, cannot anchor " "new segment Axes.")
def read_cache(cache, segments, etg, nproc=1, timecolumn=None, **kwargs): """Read a table of events from a cache This function is mainly meant for use from the `get_triggers` method Parameters ---------- cache : :class:`glue.lal.Cache` the formatted list of files to read segments : `~gwpy.segments.SegmentList` the list of segments to read etg : `str` the name of the trigger generator that created the files nproc : `int`, optional the number of parallel processes to use when reading **kwargs other keyword arguments are passed to the `EventTable.read` or `{tableclass}.read` methods Returns ------- table : `~gwpy.table.EventTable`, `None` a table of events, or `None` if the cache has no overlap with the segments """ if isinstance(cache, Cache): cache = cache.sieve(segmentlist=segments) cache = cache.checkfilesexist()[0] cache.sort(key=lambda x: x.segment[0]) if etg == 'pycbc_live': # remove empty HDF5 files cache = type(cache)( filter_pycbc_live_files(cache, ifo=kwargs['ifo'])) # if no files, skip if len(cache) == 0: return # use multiprocessing except for ascii reading # (since astropy doesn't allow it) if kwargs.get('format', 'none').startswith('ascii.'): cache = cache.pfnlist() else: kwargs['nproc'] = nproc if len(cache) == 1: cache = cache[0] # read triggers table = EventTable.read(cache, **kwargs) if timecolumn: table.meta['timecolumn'] = timecolumn # get back from cache entry if isinstance(cache, CacheEntry): cache = Cache([cache]) # append new events to existing table try: csegs = cache_segments(cache) except (AttributeError, TypeError): csegs = SegmentList() table.meta['segments'] = csegs return keep_in_segments(table, segments, etg)
def use_segmentlist(f, arg1, segments, *args, **kwargs): """Decorator a method to convert incoming segments into a `SegmentList` """ if isinstance(segments, DataQualityFlag): segments = segments.active elif not isinstance(segments, segmentlist): segments = SegmentList([Segment(*x) for x in segments]) return f(arg1, segments, *args, **kwargs)
def segmentlist_from_tree(tree, coalesce=False): """Read a `~ligo.segments.segmentlist` from a 'segments' `ROOT.Tree` """ segs = SegmentList() for i in range(tree.GetEntries()): tree.GetEntry(i) segs.append(Segment(tree.start, tree.end)) return segs
def test_query_dqsegdb_multi(self): segs = SegmentList([Segment(0, 2), Segment(8, 10)]) result = query_dqsegdb(self.TEST_CLASS.query_dqsegdb, QUERY_FLAGS[0], segs) RESULT = QUERY_RESULTC[QUERY_FLAGS[0]] assert isinstance(result, self.TEST_CLASS) utils.assert_segmentlist_equal(result.known, RESULT.known & segs) utils.assert_segmentlist_equal(result.active, RESULT.active & segs)