def _update_defaults_from_channels(self): c1, c2 = self.channels self.pargs.setdefault('xlabel', texify(str(c1))) self.pargs.setdefault('ylabel', texify(str(c2))) if hasattr(c1, 'amplitude_range'): self.pargs.setdefault('xlim', c1.amplitude_range) if hasattr(c2, 'amplitude_range'): self.pargs.setdefault('ylim', c2.amplitude_range)
def init_plot(self, data=[], FigureClass=Plot, geometry=(1, 1), projection='rectilinear', sharex=True, sharey=True, **kwargs): """Initialise the Figure and Axes objects for this `DataPlot`. """ # update plot defaults using channel data self._update_defaults_from_channels() # strip figure and axes params from pargs for key in NON_PLOT_PARAMS: try: kwargs.setdefault(key, self.pargs.pop(key)) except KeyError: continue # escape text for TeX if key in ('title', 'xlabel', 'ylabel'): kwargs[key] = texify(kwargs[key]) # create figure self.plot = FigureClass(*data, geometry=geometry, projection=projection, sharex=sharex, sharey=sharey, **kwargs) return self.plot
def draw(self): """Read in all necessary data, and generate the figure. """ plot = self.init_plot() ax = plot.gca() # work out labels labels = self.pargs.pop('labels', self.channels) if isinstance(labels, str): labels = labels.split(',') labels = [str(s).strip('\n ') for s in labels] # add data for label, channel in zip(labels, self.channels): label = texify(label) if self.state and not self.all_data: valid = self.state.active else: valid = SegmentList([self.span]) data = get_timeseries(channel, valid, query=False) # handle no timeseries if not len(data): ax.plot([0], [0], visible=False, label=label) continue # plot time-series color = None for ts in data: # double-check log scales if self.logy: ts.value[ts.value == 0] = 1e-100 if color is None: line = ax.plot(ts, label=label)[0] color = line.get_color() else: ax.plot(ts, color=color, label=None) # allow channel data to set parameters if hasattr(data[0].channel, 'amplitude_range'): self.pargs.setdefault('ylim', data[0].channel.amplitude_range) # add horizontal lines to add for yval in self.pargs['hline']: try: yval = float(yval) except ValueError: continue else: ax.plot([self.start, self.end], [yval, yval], linestyle='--', color='red') # customise plot legendargs = self.parse_legend_kwargs() self.apply_parameters(ax, **self.pargs) if len(self.channels) > 1: ax.legend(**legendargs) # finalise self.add_state_segments(ax) return self.finalize()
def get_channel_groups(self): """Find and group (mean, min, max) sets of channels for plotting. Returns ------- groups : `list` of `tuple` list of (channelname, channellist) tuples giving core channel name and an ordered list of channels. Ordering in preference of 'rms', 'mean', 'min', 'max'. Notes ----- This method used to return an `OrderedDict`, but was changed to return a `list` of `tuple` to enable plotting a channel multiple times on a plot, for whatever reason. """ all_ = self.channels out = [] for c in all_: if c.ifo == 'G1' and re.search(r'-(av|min|max)\Z', c.name): name = texify(c.name.rsplit('-', 1)[0]) else: name = texify(c.name.rsplit('.', 1)[0]) if ' ' in c.name: out.append((texify(c.name), [c])) else: try: id_ = list(zip(*out))[0].index(name) except (IndexError, ValueError): out.append((name, [c])) else: out[id_][1].append(c) order = ['rms', 'mean', 'av', 'min', 'max'] for channel, clist in out: clist.sort(key=lambda c: c.name.split('.')[-1] in order and order. index(c.name.split('.')[-1]) + 1 or 10) return out
def __init__(self, channels, start, end, state=None, outdir='.', etg=None, **kwargs): if (len(channels) == 1) and ('title' not in kwargs): kwargs['title'] = texify('%s (%s)' % (str(channels[0]), etg)) super(TriggerDataPlot, self).__init__(channels, start, end, state=state, outdir=outdir, **kwargs) self.etg = etg self.columns = [self.pargs.pop(c) for c in ('x', 'y', 'color')]
def draw(self, outputfile=None): """Get data and generate the figure. """ # get histogram parameters plot = self.init_plot() ax = plot.gca() if self.state: self.pargs.setdefault( 'suptitle', '[%s-%s, state: %s]' % (self.span[0], self.span[1], texify(str(self.state)))) suptitle = self.pargs.pop('suptitle', None) if suptitle: plot.suptitle(suptitle, y=0.993, va='top') # get data data = [] for channel in self.channels: if self.state and not self.all_data: valid = self.state.active else: valid = SegmentList([self.span]) data.append( get_timeseries(channel, valid, query=False).join(gap='ignore', pad=numpy.nan)) if len(data) == 1: data.append(data[0]) # histogram hist_kwargs = self.parse_hist_kwargs() h, xedges, yedges = numpy.histogram2d(data[0], data[1], **hist_kwargs) h = numpy.ma.masked_where(h == 0, h) x, y = numpy.meshgrid(xedges, yedges, copy=False, sparse=True) # plot pcmesh_kwargs = self.parse_pcmesh_kwargs() ax.pcolormesh(x, y, h.T, **pcmesh_kwargs) # customise plot self.apply_parameters(ax, **self.pargs) return self.finalize(outputfile=outputfile)
def main(args=None): """Run the hveto command-line interface """ # declare global variables # this is needed for multiprocessing utilities global acache, analysis, areadkw, atrigfindkw, auxiliary, auxetg global auxfreq, counter, livetime, minsnr, naux, pchannel, primary global rnd, snrs, windows # parse command-line parser = create_parser() args = parser.parse_args(args=args) ifo = args.ifo start = int(args.gpsstart) end = int(args.gpsend) duration = end - start # log startup LOGGER.info("-- Welcome to Hveto --") LOGGER.info("GPS start time: %d" % start) LOGGER.info("GPS end time: %d" % end) LOGGER.info("Interferometer: %s" % ifo) # -- initialisation ------------------------- # read configuration cp = config.HvetoConfigParser(ifo=ifo) cp.read(args.config_file) LOGGER.info("Parsed configuration file(s)") # format output directory outdir = _abs_path(args.output_directory) if not os.path.isdir(outdir): os.makedirs(outdir) os.chdir(outdir) LOGGER.info("Working directory: %s" % outdir) segdir = 'segments' plotdir = 'plots' trigdir = 'triggers' omegadir = 'scans' for d in [segdir, plotdir, trigdir, omegadir]: if not os.path.isdir(d): os.makedirs(d) # prepare html variables htmlv = { 'title': '%s Hveto | %d-%d' % (ifo, start, end), 'config': None, 'prog': PROG, 'context': ifo.lower(), } # get segments aflag = cp.get('segments', 'analysis-flag') url = cp.get('segments', 'url') padding = tuple(cp.getfloats('segments', 'padding')) if args.analysis_segments: segs_ = DataQualityDict.read(args.analysis_segments, gpstype=float) analysis = segs_[aflag] span = SegmentList([Segment(start, end)]) analysis.active &= span analysis.known &= span analysis.coalesce() LOGGER.debug("Segments read from disk") else: analysis = DataQualityFlag.query(aflag, start, end, url=url) LOGGER.debug("Segments recovered from %s" % url) if padding != (0, 0): mindur = padding[0] - padding[1] analysis.active = type(analysis.active)([s for s in analysis.active if abs(s) >= mindur]) analysis.pad(*padding, inplace=True) LOGGER.debug("Padding %s applied" % str(padding)) livetime = int(abs(analysis.active)) livetimepc = livetime / duration * 100. LOGGER.info("Retrieved %d segments for %s with %ss (%.2f%%) livetime" % (len(analysis.active), aflag, livetime, livetimepc)) # apply vetoes from veto-definer file try: vetofile = cp.get('segments', 'veto-definer-file') except configparser.NoOptionError: vetofile = None else: try: categories = cp.getfloats('segments', 'veto-definer-categories') except configparser.NoOptionError: categories = None # read file vdf = read_veto_definer_file(vetofile, start=start, end=end, ifo=ifo) LOGGER.debug("Read veto-definer file from %s" % vetofile) # get vetoes from segdb vdf.populate(source=url, segments=analysis.active, on_error='warn') # coalesce flags from chosen categories vetoes = DataQualityFlag('%s:VDF-VETOES:1' % ifo) nflags = 0 for flag in vdf: if not categories or vdf[flag].category in categories: vetoes += vdf[flag] nflags += 1 try: deadtime = int(abs(vetoes.active)) / int(abs(vetoes.known)) * 100 except ZeroDivisionError: deadtime = 0 LOGGER.debug("Coalesced %ss (%.2f%%) of deadtime from %d veto flags" % (abs(vetoes.active), deadtime, nflags)) # apply to analysis segments analysis -= vetoes LOGGER.debug("Applied vetoes from veto-definer file") livetime = int(abs(analysis.active)) livetimepc = livetime / duration * 100. LOGGER.info("%ss (%.2f%%) livetime remaining after vetoes" % (livetime, livetimepc)) snrs = cp.getfloats('hveto', 'snr-thresholds') minsnr = min(snrs) windows = cp.getfloats('hveto', 'time-windows') # record all segments segments = DataQualityDict() segments[analysis.name] = analysis # -- load channels -------------------------- # get primary channel name pchannel = cp.get('primary', 'channel') # read auxiliary cache if args.auxiliary_cache is not None: acache = read_cache(args.auxiliary_cache) else: acache = None # load auxiliary channels auxetg = cp.get('auxiliary', 'trigger-generator') auxfreq = cp.getfloats('auxiliary', 'frequency-range') try: auxchannels = cp.get('auxiliary', 'channels').strip('\n').split('\n') except config.configparser.NoOptionError: auxchannels = find_auxiliary_channels(auxetg, (start, end), ifo=ifo, cache=acache) cp.set('auxiliary', 'channels', '\n'.join(auxchannels)) LOGGER.debug("Auto-discovered %d " "auxiliary channels" % len(auxchannels)) else: auxchannels = sorted(set(auxchannels)) LOGGER.debug("Read list of %d auxiliary channels" % len(auxchannels)) # load unsafe channels list _unsafe = cp.get('safety', 'unsafe-channels') if os.path.isfile(_unsafe): # from file unsafe = set() with open(_unsafe, 'rb') as f: for c in f.read().rstrip('\n').split('\n'): if c.startswith('%(IFO)s'): unsafe.add(c.replace('%(IFO)s', ifo)) elif not c.startswith('%s:' % ifo): unsafe.add('%s:%s' % (ifo, c)) else: unsafe.add(c) else: # or from line-seprated list unsafe = set(_unsafe.strip('\n').split('\n')) unsafe.add(pchannel) cp.set('safety', 'unsafe-channels', '\n'.join(sorted(unsafe))) LOGGER.debug("Read list of %d unsafe channels" % len(unsafe)) # remove unsafe channels nunsafe = 0 for i in range(len(auxchannels) - 1, -1, -1): if auxchannels[i] in unsafe: LOGGER.warning("Auxiliary channel %r identified as unsafe and has " "been removed" % auxchannels[i]) auxchannels.pop(i) nunsafe += 1 LOGGER.debug("%d auxiliary channels identified as unsafe" % nunsafe) naux = len(auxchannels) LOGGER.info("Identified %d auxiliary channels to process" % naux) # record INI file in output HTML directory inifile = '%s-HVETO_CONFIGURATION-%d-%d.ini' % (ifo, start, duration) if os.path.isfile(inifile) and any( os.path.samefile(inifile, x) for x in args.config_file): LOGGER.debug("Cannot write INI file to %s, file was given as input") else: with open(inifile, 'w') as f: cp.write(f) LOGGER.info("Configuration recorded as %s" % inifile) htmlv['config'] = inifile # -- load primary triggers ------------------ # read primary cache if args.primary_cache is not None: pcache = read_cache(args.primary_cache) else: pcache = None # load primary triggers petg = cp.get('primary', 'trigger-generator') psnr = cp.getfloat('primary', 'snr-threshold') pfreq = cp.getfloats('primary', 'frequency-range') preadkw = cp.getparams('primary', 'read-') if pcache is not None: # auto-detect the file format LOGGER.debug('Unsetting the primary trigger file format') preadkw['format'] = None preadkw['path'] = 'triggers' ptrigfindkw = cp.getparams('primary', 'trigfind-') primary = get_triggers(pchannel, petg, analysis.active, snr=psnr, frange=pfreq, cache=pcache, nproc=args.nproc, trigfind_kwargs=ptrigfindkw, **preadkw) fcol, scol = primary.dtype.names[1:3] if len(primary): LOGGER.info("Read %d events for %s" % (len(primary), pchannel)) else: message = "No events found for %r in %d seconds of livetime" % ( pchannel, livetime) LOGGER.critical(message) # cluster primary triggers clusterkwargs = cp.getparams('primary', 'cluster-') if clusterkwargs: primary = primary.cluster(**clusterkwargs) LOGGER.info("%d primary events remain after clustering over %s" % (len(primary), clusterkwargs['rank'])) # -- bail out early ------------------------- # the bail out is done here so that we can at least generate the eventual # configuration file, mainly for HTML purposes # no segments if livetime == 0: message = ("No active segments found for analysis flag %r in interval " "[%d, %d)" % (aflag, start, end)) LOGGER.critical(message) htmlv['context'] = 'info' index = html.write_null_page(ifo, start, end, message, **htmlv) LOGGER.info("HTML report written to %s" % index) sys.exit(0) # no primary triggers if len(primary) == 0: htmlv['context'] = 'danger' index = html.write_null_page(ifo, start, end, message, **htmlv) LOGGER.info("HTML report written to %s" % index) sys.exit(0) # otherwise write all primary triggers to ASCII trigfile = os.path.join( trigdir, '%s-HVETO_RAW_TRIGS_ROUND_0-%d-%d.txt' % (ifo, start, duration), ) primary.write(trigfile, format='ascii', overwrite=True) # -- load auxiliary triggers ---------------- LOGGER.info("Reading triggers for aux channels...") counter = multiprocessing.Value('i', 0) areadkw = cp.getparams('auxiliary', 'read-') if acache is not None: # auto-detect the file format LOGGER.debug('Unsetting the auxiliary trigger file format') areadkw['format'] = None areadkw['path'] = 'triggers' atrigfindkw = cp.getparams('auxiliary', 'trigfind-') # map with multiprocessing if args.nproc > 1: pool = multiprocessing.Pool(processes=args.nproc) results = pool.map(_get_aux_triggers, auxchannels) pool.close() # map without multiprocessing else: results = map(_get_aux_triggers, auxchannels) LOGGER.info("All aux events loaded") auxiliary = dict(x for x in results if x is not None) auxchannels = sorted(auxiliary.keys()) chanfile = '%s-HVETO_CHANNEL_LIST-%d-%d.txt' % (ifo, start, duration) with open(chanfile, 'w') as f: for chan in auxchannels: print(chan, file=f) LOGGER.info("Recorded list of valid auxiliary channels in %s" % chanfile) # -- execute hveto analysis ----------------- minsig = cp.getfloat('hveto', 'minimum-significance') pevents = [primary] pvetoed = [] auxfcol, auxscol = auxiliary[auxchannels[0]].dtype.names[1:3] slabel = plot.get_column_label(scol) flabel = plot.get_column_label(fcol) auxslabel = plot.get_column_label(auxscol) auxflabel = plot.get_column_label(auxfcol) rounds = [] rnd = core.HvetoRound(1, pchannel, rank=scol) rnd.segments = analysis.active while True: LOGGER.info("-- Processing round %d --" % rnd.n) # write segments for this round segfile = os.path.join( segdir, '%s-HVETO_ANALYSIS_SEGS_ROUND_%d-%d-%d.txt' % (ifo, rnd.n, start, duration)) write_ascii_segments(segfile, rnd.segments) # calculate significances for this round if args.nproc > 1: # multiprocessing # separate channel list into chunks and process each chunk pool = multiprocessing.Pool( processes=min(args.nproc, len(auxiliary.keys()))) chunks = utils.channel_groups(list(auxiliary.keys()), args.nproc) results = pool.map(_find_max_significance, chunks) pool.close() winners, sigsets = zip(*results) # find winner of chunk winners winner = sorted(winners, key=lambda w: w.significance)[-1] # flatten sets of significances into one list newsignificances = sigsets[0] for subdict in sigsets[1:]: newsignificances.update(subdict) else: # single process winner, newsignificances = core.find_max_significance( primary, auxiliary, pchannel, snrs, windows, rnd.livetime) LOGGER.info("Round %d winner: %s" % (rnd.n, winner.name)) # plot significance drop here for the last round # only now do we actually have the new data to # calculate significance drop if rnd.n > 1: svg = (pngname % 'SIG_DROP').replace('.png', '.svg') # noqa: F821 plot.significance_drop( svg, oldsignificances, newsignificances, # noqa: F821 title=' | '.join([title, subtitle]), # noqa: F821 bbox_inches='tight') LOGGER.debug("Figure written to %s" % svg) svg = FancyPlot(svg, caption=plot.ROUND_CAPTION['SIG_DROP']) rounds[-1].plots.append(svg) oldsignificances = newsignificances # noqa: F841 # break out of the loop if the significance is below stopping point if winner.significance < minsig: LOGGER.info("Maximum signifiance below stopping point") LOGGER.debug(" (%.2f < %.2f)" % (winner.significance, minsig)) LOGGER.info("-- Rounds complete! --") break # work out the vetoes for this round allaux = auxiliary[winner.name][ auxiliary[winner.name][auxscol] >= winner.snr] winner.events = allaux coincs = allaux[core.find_coincidences(allaux['time'], primary['time'], dt=winner.window)] rnd.vetoes = winner.get_segments(allaux['time']) flag = DataQualityFlag( '%s:HVT-ROUND_%d:1' % (ifo, rnd.n), active=rnd.vetoes, known=rnd.segments, description="winner=%s, window=%s, snr=%s" % ( winner.name, winner.window, winner.snr)) segments[flag.name] = flag LOGGER.debug("Generated veto segments for round %d" % rnd.n) # link events before veto for plotting before = primary beforeaux = auxiliary[winner.name] # apply vetoes to primary primary, vetoed = core.veto(primary, rnd.vetoes) pevents.append(primary) pvetoed.append(vetoed) LOGGER.debug("Applied vetoes to primary") # record results rnd.winner = winner rnd.efficiency = (len(vetoed), len(primary) + len(vetoed)) rnd.use_percentage = (len(coincs), len(winner.events)) if rnd.n > 1: rnd.cum_efficiency = ( len(vetoed) + rounds[-1].cum_efficiency[0], rounds[0].efficiency[1]) rnd.cum_deadtime = ( rnd.deadtime[0] + rounds[-1].cum_deadtime[0], livetime) else: rnd.cum_efficiency = rnd.efficiency rnd.cum_deadtime = rnd.deadtime # apply vetoes to auxiliary if args.nproc > 1: # multiprocess # separate channel list into chunks and process each chunk pool = multiprocessing.Pool( processes=min(args.nproc, len(auxiliary.keys()))) chunks = utils.channel_groups(list(auxiliary.keys()), args.nproc) results = pool.map(_veto, chunks) pool.close() auxiliary = results[0] for subdict in results[1:]: auxiliary.update(subdict) else: # single process auxiliary = core.veto_all(auxiliary, rnd.vetoes) LOGGER.debug("Applied vetoes to auxiliary channels") # log results LOGGER.info("""Results for round %d:\n\n winner : %s significance : %s mu : %s snr : %s dt : %s use_percentage : %s efficiency : %s deadtime : %s cum. efficiency : %s cum. deadtime : %s\n\n""" % ( rnd.n, rnd.winner.name, rnd.winner.significance, rnd.winner.mu, rnd.winner.snr, rnd.winner.window, rnd.use_percentage, rnd.efficiency, rnd.deadtime, rnd.cum_efficiency, rnd.cum_deadtime)) # write segments segfile = os.path.join( segdir, '%s-HVETO_VETO_SEGS_ROUND_%d-%d-%d.txt' % ( ifo, rnd.n, start, duration)) write_ascii_segments(segfile, rnd.vetoes) LOGGER.debug("Round %d vetoes written to %s" % (rnd.n, segfile)) rnd.files['VETO_SEGS'] = (segfile,) # write triggers trigfile = os.path.join( trigdir, '%s-HVETO_%%s_TRIGS_ROUND_%d-%d-%d.txt' % ( ifo, rnd.n, start, duration)) for tag, arr in zip( ['WINNER', 'VETOED', 'RAW'], [winner.events, vetoed, primary]): f = trigfile % tag arr.write(f, format='ascii', overwrite=True) LOGGER.debug("Round %d %s events written to %s" % (rnd.n, tag.lower(), f)) rnd.files[tag] = f # record times to omega scan if args.omega_scans: N = len(vetoed) ind = random.sample(range(0, N), min(args.omega_scans, N)) rnd.scans = vetoed[ind] LOGGER.debug("Collected %d events to omega scan:\n\n%s\n\n" % (len(rnd.scans), rnd.scans)) # -- make some plots -- pngname = os.path.join(plotdir, '%s-HVETO_%%s_ROUND_%d-%d-%d.png' % ( ifo, rnd.n, start, duration)) wname = texify(rnd.winner.name) beforel = 'Before\n[%d]' % len(before) afterl = 'After\n[%d]' % len(primary) vetoedl = 'Vetoed\n(primary)\n[%d]' % len(vetoed) beforeauxl = 'All\n[%d]' % len(beforeaux) usedl = 'Used\n(aux)\n[%d]' % len(winner.events) coincl = 'Coinc.\n[%d]' % len(coincs) title = '%s Hveto round %d' % (ifo, rnd.n) ptitle = '%s: primary impact' % title atitle = '%s: auxiliary use' % title subtitle = 'winner: %s [%d-%d]' % (wname, start, end) # before/after histogram png = pngname % 'HISTOGRAM' plot.before_after_histogram( png, before[scol], primary[scol], label1=beforel, label2=afterl, xlabel=slabel, title=ptitle, subtitle=subtitle) LOGGER.debug("Figure written to %s" % png) png = FancyPlot(png, caption=plot.ROUND_CAPTION['HISTOGRAM']) rnd.plots.append(png) # snr versus time png = pngname % 'SNR_TIME' plot.veto_scatter( png, before, vetoed, x='time', y=scol, label1=beforel, label2=vetoedl, epoch=start, xlim=[start, end], ylabel=slabel, title=ptitle, subtitle=subtitle, legend_title="Primary:") LOGGER.debug("Figure written to %s" % png) png = FancyPlot(png, caption=plot.ROUND_CAPTION['SNR_TIME']) rnd.plots.append(png) # snr versus frequency png = pngname % 'SNR_%s' % fcol.upper() plot.veto_scatter( png, before, vetoed, x=fcol, y=scol, label1=beforel, label2=vetoedl, xlabel=flabel, ylabel=slabel, xlim=pfreq, title=ptitle, subtitle=subtitle, legend_title="Primary:") LOGGER.debug("Figure written to %s" % png) png = FancyPlot(png, caption=plot.ROUND_CAPTION['SNR']) rnd.plots.append(png) # frequency versus time coloured by SNR png = pngname % '%s_TIME' % fcol.upper() plot.veto_scatter( png, before, vetoed, x='time', y=fcol, color=scol, label1=None, label2=None, ylabel=flabel, clabel=slabel, clim=[3, 100], cmap='YlGnBu', epoch=start, xlim=[start, end], ylim=pfreq, title=ptitle, subtitle=subtitle) LOGGER.debug("Figure written to %s" % png) png = FancyPlot(png, caption=plot.ROUND_CAPTION['TIME']) rnd.plots.append(png) # aux used versus frequency png = pngname % 'USED_SNR_TIME' plot.veto_scatter( png, winner.events, vetoed, x='time', y=[auxscol, scol], label1=usedl, label2=vetoedl, ylabel=slabel, epoch=start, xlim=[start, end], title=atitle, subtitle=subtitle) LOGGER.debug("Figure written to %s" % png) png = FancyPlot(png, caption=plot.ROUND_CAPTION['USED_SNR_TIME']) rnd.plots.append(png) # snr versus time png = pngname % 'AUX_SNR_TIME' plot.veto_scatter( png, beforeaux, (winner.events, coincs), x='time', y=auxscol, label1=beforeauxl, label2=(usedl, coincl), epoch=start, xlim=[start, end], ylabel=auxslabel, title=atitle, subtitle=subtitle) LOGGER.debug("Figure written to %s" % png) png = FancyPlot(png, caption=plot.ROUND_CAPTION['AUX_SNR_TIME']) rnd.plots.append(png) # snr versus frequency png = pngname % 'AUX_SNR_FREQUENCY' plot.veto_scatter( png, beforeaux, (winner.events, coincs), x=auxfcol, y=auxscol, label1=beforeauxl, label2=(usedl, coincl), xlabel=auxflabel, ylabel=auxslabel, title=atitle, subtitle=subtitle, legend_title="Aux:") LOGGER.debug("Figure written to %s" % png) png = FancyPlot(png, caption=plot.ROUND_CAPTION['AUX_SNR_FREQUENCY']) rnd.plots.append(png) # frequency versus time coloured by SNR png = pngname % 'AUX_FREQUENCY_TIME' plot.veto_scatter( png, beforeaux, (winner.events, coincs), x='time', y=auxfcol, color=auxscol, label1=None, label2=[None, None], ylabel=auxflabel, clabel=auxslabel, clim=[3, 100], cmap='YlGnBu', epoch=start, xlim=[start, end], title=atitle, subtitle=subtitle) LOGGER.debug("Figure written to %s" % png) png = FancyPlot(png, caption=plot.ROUND_CAPTION['AUX_FREQUENCY_TIME']) rnd.plots.append(png) # move to the next round rounds.append(rnd) rnd = core.HvetoRound(rnd.n + 1, pchannel, rank=scol, segments=rnd.segments-rnd.vetoes) # write file with all segments segfile = os.path.join( segdir, '%s-HVETO_SEGMENTS-%d-%d.h5' % (ifo, start, duration)) segments.write(segfile, overwrite=True) LOGGER.debug("Segment summary written to %s" % segfile) LOGGER.debug("Making summary figures...") # -- exit early if no rounds above threshold if not rounds: message = ("No rounds completed above threshold. Analysis stopped " "with %s achieving significance of %.2f" % (winner.name, winner.significance)) LOGGER.critical(message) message = message.replace( winner.name, cis_link(winner.name, class_='alert-link')) message += '<br>[T<sub>win</sub>: %ss, SNR: %s]' % ( winner.window, winner.snr) htmlv['context'] = 'warning' index = html.write_null_page(ifo, start, end, message, **htmlv) LOGGER.info("HTML report written to %s" % index) sys.exit(0) # -- plot all rounds impact pngname = os.path.join(plotdir, '%s-HVETO_%%s_ALL_ROUNDS-%d-%d.png' % ( ifo, start, duration)) plots = [] title = '%s Hveto all rounds' % args.ifo subtitle = '%d rounds | %d-%d' % (len(rounds), start, end) # before/after histogram png = pngname % 'HISTOGRAM' beforel = 'Before analysis [%d events]' % len(pevents[0]) afterl = 'After %d rounds [%d]' % (len(pevents) - 1, len(pevents[-1])) plot.before_after_histogram( png, pevents[0][scol], pevents[-1][scol], label1=beforel, label2=afterl, xlabel=slabel, title=title, subtitle=subtitle) png = FancyPlot(png, caption=plot.HEADER_CAPTION['HISTOGRAM']) plots.append(png) LOGGER.debug("Figure written to %s" % png) # efficiency/deadtime curve png = pngname % 'ROC' plot.hveto_roc(png, rounds, title=title, subtitle=subtitle) png = FancyPlot(png, caption=plot.HEADER_CAPTION['ROC']) plots.append(png) LOGGER.debug("Figure written to %s" % png) # frequency versus time png = pngname % '%s_TIME' % fcol.upper() labels = [str(r.n) for r in rounds] legtitle = 'Vetoed at\nround' plot.veto_scatter( png, pevents[0], pvetoed, label1='', label2=labels, title=title, subtitle=subtitle, ylabel=flabel, x='time', y=fcol, epoch=start, xlim=[start, end], legend_title=legtitle) png = FancyPlot(png, caption=plot.HEADER_CAPTION['TIME']) plots.append(png) LOGGER.debug("Figure written to %s" % png) # snr versus time png = pngname % 'SNR_TIME' plot.veto_scatter( png, pevents[0], pvetoed, label1='', label2=labels, title=title, subtitle=subtitle, ylabel=slabel, x='time', y=scol, epoch=start, xlim=[start, end], legend_title=legtitle) png = FancyPlot(png, caption=plot.HEADER_CAPTION['SNR_TIME']) plots.append(png) LOGGER.debug("Figure written to %s" % png) # -- write summary states to ASCII table and JSON json_ = { 'user': getuser(), 'host': getfqdn(), 'date': str(datetime.datetime.now()), 'configuration': inifile, 'ifo': ifo, 'gpsstart': start, 'gpsend': end, 'call': ' '.join(sys.argv), 'rounds': [], } with open('summary-stats.txt', 'w') as f: # print header print('#N winner window SNR significance nveto use-percentage ' 'efficiency deadtime cumulative-efficiency cumulative-deadtime', file=f) for r in rounds: # extract relevant statistics results = [ ('round', r.n), ('name', r.winner.name), ('window', r.winner.window), ('snr', r.winner.snr), ('significance', r.winner.significance), ('nveto', r.efficiency[0]), ('use-percentage', r.use_percentage[0] / r.use_percentage[1] * 100.), ('efficiency', r.efficiency[0] / r.efficiency[1] * 100.), ('deadtime', r.deadtime[0] / r.deadtime[1] * 100.), ('cumulative-efficiency', r.cum_efficiency[0] / r.cum_efficiency[1] * 100.), ('cumulative-deadtime', r.cum_deadtime[0] / r.cum_deadtime[1] * 100.), ] # write to ASCII print(' '.join(map(str, list(zip(*results))[1])), file=f) # write to JSON results.append(('files', r.files)) json_['rounds'].append(dict(results)) LOGGER.debug("Summary table written to %s" % f.name) with open('summary-stats.json', 'w') as f: json.dump(json_, f, sort_keys=True) LOGGER.debug("Summary JSON written to %s" % f.name) # -- generate workflow for omega scans if args.omega_scans: omegatimes = list(map(str, sorted(numpy.unique( [t['time'] for r in rounds for t in r.scans])))) LOGGER.debug("Collected %d times to omega scan" % len(omegatimes)) newtimes = [t for t in omegatimes if not os.path.exists(os.path.join(omegadir, str(t)))] LOGGER.debug("%d scans already complete or in progress, %d remaining" % (len(omegatimes) - len(newtimes), len(newtimes))) if len(newtimes) > 0: LOGGER.info('Creating workflow for omega scans') flags = batch.get_command_line_flags( ifo=ifo, ignore_state_flags=True) condorcmds = batch.get_condor_arguments( timeout=4, extra_commands=["request_disk='1G'"], gps=start) batch.generate_dag( newtimes, flags=flags, submit=True, outdir=omegadir, condor_commands=condorcmds) LOGGER.info('Launched {} omega scans to condor'.format( len(newtimes))) else: LOGGER.debug('Skipping omega scans') # -- write HTML and finish index = html.write_hveto_page( ifo, start, end, rounds, plots, winners=[r.winner.name for r in rounds], **htmlv) LOGGER.debug("HTML written to %s" % index) LOGGER.debug("Analysis completed in %d seconds" % (time.time() - JOBSTART)) LOGGER.info("-- Hveto complete --")
def from_ini(cls, config, section, plotdir='plots', **kwargs): """Define a new `GuardianTab`. """ new = super(DataTab, cls).from_ini(config, section, **kwargs) new.plots = [] new.plotdir = plotdir new.ifo = config.get(section, 'IFO') # record node and states new.node = config.get(section, 'node') new.grdstates = OrderedDict() plot = [] for key, name in config.nditems(section): try: key = int(key) except ValueError: continue else: if name[0] == '*': plot.append(False) else: plot.append(True) new.grdstates[int(key)] = name.strip('*') try: new.transstates = list( map(int, config.get(section, 'transitions').split(','))) except NoOptionError: new.transstates = list(new.grdstates) # -- build plots ------------------------ new.set_layout([1, 2]) grdidxs = dict((state, idx) for idx, state in new.grdstates.items()) new.segmenttag = '%s:%s %%s' % (new.ifo, new.node) pstates = [ l for i, l in enumerate(list(new.grdstates.values())[::-1]) if plot[-i - 1] ] flags = [new.segmenttag % name for name in pstates] labels = ['[%d] %s' % (grdidxs[state], state) for state in pstates] # define colours cmap = get_cmap('brg')(Normalize(-1, 1)(numpy.linspace( -1, 1, len(pstates))))[::-1] th = len(flags) > 8 and (new.span[1] - new.span[0]) / 200. or 0 for state in new.states: # get common plot tag prefix tagprefix = 'GRD_%s' % re.sub(r'[-\s]', '_', new.node.upper()) if state.name.lower() != ALLSTATE: # include state name if not All tagprefix = '%s_%s' % (state.name, tagprefix) # segment plot new.plots.append( get_plot('guardian')(flags, new.span[0], new.span[1], state=state, labels=labels, outdir=plotdir, known={ 'hatch': 'x', 'alpha': 0.1, 'facecolor': 'none' }, tag='%s_SEGMENTS' % tagprefix, title='%s Guardian %s state' % (new.ifo, texify(new.node)), zorder=2)) # pie new.plots.append( get_plot('segment-pie')( flags, new.span[0], new.span[1], state=state, labels=pstates, colors=cmap, tag='%s_SEGMENT_PIE' % tagprefix, startangle=180, counterclock=False, wedge_linewidth=0.01, outdir=plotdir, title='%s Guardian %s state' % (new.ifo, texify(new.node)), legend_fontsize=16, legend_sorted=True, legend_threshold=th, )) # bar new.plots.append( get_plot('segment-bar')( flags, new.span[0], new.span[1], state=state, labels=pstates, sorted=True, tag='%s_SEGMENT_BAR' % tagprefix, outdir=plotdir, title='%s Guardian %s state' % (new.ifo, texify(new.node)), )) return new
def draw(self): from ..tabs.guardian import (REQUESTSTUB, NOMINALSTUB, re_guardian_index) plot = self.init_plot() ax = plot.gca() # get labels flags = [texify(str(f)) for f in self.flags] labels = self.pargs.pop('labels', self.pargs.pop('label', flags)) ax.set_insetlabels(self.pargs.pop('insetlabels', True)) if isinstance(labels, str): labels = labels.split(',') labels = [re_quote.sub('', str(s).strip('\n ')) for s in labels] # parse plotting arguments legendargs = self.parse_legend_kwargs() plotargs = self.parse_plot_kwargs()[0] plotargs.pop('label') height = plotargs.pop('height', .8) activecolor = plotargs.pop('facecolor') nominalcolor = self.pargs.pop('nominalcolor') requestcolor = self.pargs.pop('requestcolor') actargs = plotargs.copy() plotargs.update({ 'facecolor': nominalcolor, 'edgecolor': 'none', 'known': { 'alpha': 0.1, 'facecolor': 'lightgray' }, }) reqargs = plotargs.copy() reqargs.update({ 'facecolor': requestcolor, 'known': None, 'height': height, }) actargs.update({ 'facecolor': activecolor, 'known': None, }) if self.state and not self.all_data: valid = self.state.active else: valid = SegmentList([self.span]) # plot segments for y, (flag, label) in enumerate(list(zip(self.flags, labels))[::-1]): inreq = str(flag) + REQUESTSTUB nominal = str(flag) + NOMINALSTUB segs = get_segments([flag, inreq, nominal], validity=valid, query=False) # format label if self.fileformat != 'svg': try: idx, label = re_guardian_index.match(label).groups() except AttributeError: pass else: x = float(self.span[0]) - (float(abs(self.span)) * 0.005) ax.text(x, y, '[%s]' % idx, ha='right', va='center', fontsize=12) # plot segments ax.plot(segs[nominal], label=label, y=y, height=1., **plotargs) ax.plot(segs[inreq], label=label, y=y, collection='ignore', **reqargs) ax.plot(segs[flag], label=label, y=y, collection='ignore', height=.6, **actargs) # make custom legend seg = Segment(self.start - 10, self.start - 9) v = plotargs.pop('known', None) n = SegmentRectangle(seg, 0, facecolor=nominalcolor, edgecolor=nominalcolor) a = SegmentRectangle(seg, 0, facecolor=requestcolor, edgecolor=requestcolor) b = SegmentRectangle(seg, 0, facecolor=activecolor, edgecolor=actargs['edgecolor']) handles = [n, a, b] labels = ['Nominal', 'Request', 'Active'] if v: v.pop('collection', None) v['edgecolor'] = v.get('edgecolor') or tint(v['facecolor'], .5) handles.insert(0, SegmentRectangle(seg, 0, **v)) labels.insert(0, 'Alive') ax.legend(handles, labels, title='Node state', **legendargs) # customise plot for key, val in self.pargs.items(): try: getattr(ax, 'set_%s' % key)(val) except AttributeError: setattr(ax, key, val) if 'ylim' not in self.pargs: ax.set_ylim(-0.5, len(self.flags) - 0.5) epoch = ax.get_epoch() # add node MODE along the bottom sax = None seg_kw = {'y': 0, 'edgecolor': 'none', 'label': 'Mode'} legentry = OrderedDict() colors = iter(('#0066ff', '#8000bf', 'hotpink', 'yellow')) for ctag, mstate in [ # bit listing for channels (Nones are ignored) ('MODE', ('EXEC', 'MANAGED', 'MANUAL')), ('OP', (None, 'PAUSE', None)), ]: # get data data = get_timeseries('{0.ifo}:GRD-{0.node}_{1}'.format( self, ctag), valid, query=False).join(gap='pad', pad=-1) for i, m in filter(lambda x: x[1] is not None, enumerate(mstate)): x = (data == i).to_dqflag() fc = next(colors) if sax is None: sax = plot.add_segments_bar(x.active, facecolor=fc, **seg_kw) else: sax.plot_segmentlist(x.active, facecolor=fc, **seg_kw) legentry[m.title()] = SegmentRectangle(seg, 0, facecolor=fc, edgecolor=fc) seg_kw.update({'collection': 'ignore', 'label': None}) # add OK segments along the bottom ok = get_segments(flag.split(' ', 1)[0] + ' OK', validity=valid, query=False) seg_kw.pop('edgecolor', None) sax.plot_segmentlist(ok.active, facecolor=activecolor, edgecolor=actargs['edgecolor'], **seg_kw) legentry['`OK\''] = SegmentRectangle(seg, 0, facecolor=activecolor, edgecolor=actargs['edgecolor']) # make custom legend legendargs.update({ 'bbox_to_anchor': (1., 0.), 'loc': 'lower left', }) sax.legend(list(legentry.values()), list(legentry), title='Node mode', **legendargs) sax.tick_params(axis='y', which='major', labelsize=12) for ax_ in (ax, sax): ax_.set_epoch(epoch) return self.finalize()
def init_plots(self, plotdir=os.curdir): """Configure the default list of plots for this tab This method configures a veto-trigger glitchgram, histograms of before/after SNR and frequency/template duration, before and after glitchgrams, and a segment plot. This method is a mess, and should be re-written in a better way. """ namestr = self.title.split('/')[0] before = get_channel(str(self.channel)) for state in self.states: # -- configure segment plot segargs = { 'state': state, 'known': { 'alpha': 0.1, 'facecolor': 'lightgray' }, 'color': 'red', } if len(self.flags) == 1: sp = get_plot('segments')(self.flags, self.start, self.end, outdir=plotdir, labels=self.labels, title='Veto segments: %s' % (texify(namestr)), **segargs) else: sp = get_plot('segments')( [self.metaflag] + self.flags, self.start, self.end, labels=([self.intersection and 'Intersection' or 'Union'] + self.labels), outdir=plotdir, title='Veto segments: %s' % texify(namestr), **segargs) self.plots.append(sp) if self.channel: self.set_layout([ 2, ]) after = get_channel( veto_tag(before, self.metaflag, mode='after')) vetoed = get_channel( veto_tag(before, self.metaflag, mode='vetoed')) # -- configure trigger plots params = etg.get_etg_parameters(self.etg, IFO=before.ifo) glitchgramargs = { 'etg': self.etg, 'x': params['time'], 'y': params['frequency'], 'yscale': params.get('frequency-scale', 'log'), 'ylabel': params.get('frequency-label', get_column_label(params['frequency'])), 'ylim': params.get('ylim', [10, 5000]), 'edgecolor': 'none', 'legend-scatterpoints': 1, 'legend-borderaxespad': 0, 'legend-bbox_to_anchor': (1, 1), 'legend-loc': 'upper left', 'legend-frameon': False, } glitchgramargs.update(self.glitchgramargs) # plot before/after glitchgram self.plots.append( get_plot('triggers')([after, vetoed], self.start, self.end, state=state, title='Impact of %s (%s)' % (texify(namestr), self.etg), outdir=plotdir, labels=['After', 'Vetoed'], **glitchgramargs)) # plot histograms if params['det'] != params['snr']: statistics = ['snr', 'det'] xlims = [(5, 16384), (6, 15), (8, 8192)] else: statistics = ['snr'] xlims = [(5, 16384), (8, 8192)] self.layout.append(len(statistics) + 1) for column, xlim in zip(statistics + ['frequency'], xlims): self.plots.append( get_plot('trigger-histogram')( [before, after], self.start, self.end, state=state, column=params[column], etg=self.etg, outdir=plotdir, filterstr=self.filterstr, title='Impact of %s (%s)' % (texify(namestr), self.etg), labels=['Before', 'After'], xlim=xlim, xlabel=params.get('%s-label' % column, get_column_label( params[column])), color=['#ffa07a', '#1f77b4'], alpha=1, xscale=params.get('%s-scale' % column, 'log'), yscale='log', histtype='stepfilled', weights=1 / float(abs(self.span)), bins=100, ylim=(1 / float(abs(self.span)) * 0.5, 1), **{ 'legend-borderaxespad': 0, 'legend-bbox_to_anchor': (1., 1.), 'legend-loc': 'upper left', 'legend-frameon': False, })) # plot triggers before and after for stat in statistics: column = params[stat] glitchgramargs.update({ 'color': column, 'clim': params.get('%s-limits' % stat, [3, 100]), 'logcolor': params.get('%s-log' % stat, True), 'colorlabel': params.get('%s-label' % stat, get_column_label(column)), }) self.plots.append( get_plot('triggers')([before], self.start, self.end, state=state, outdir=plotdir, filterstr=self.filterstr, **glitchgramargs)) self.plots.append( get_plot('triggers')( [after], self.start, self.end, state=state, title='%s after %s (%s)' % (texify( str(self.channel)), texify(namestr), self.etg), outdir=plotdir, filterstr=self.filterstr, **glitchgramargs)) self.layout.append(2) else: self.set_layout([ 1, ])
def get_column_string(column): r""" Format the string columnName (e.g. xml table column) into latex format for an axis label. Parameters ---------- column : `str` string to format Examples -------- >>> get_column_string('snr') 'SNR' >>> get_column_string('bank_chisq_dof') r'Bank $\chi^2$ DOF' """ acro = ['snr', 'ra', 'dof', 'id', 'ms', 'far'] greek = [ 'alpha', 'beta', 'gamma', 'delta', 'epsilon', 'zeta', 'eta', 'theta', 'iota', 'kappa', 'lamda', 'mu', 'nu', 'xi', 'omicron', 'pi', 'rho', 'sigma', 'tau', 'upsilon', 'phi', 'chi', 'psi', 'omega' ] unit = ['ns'] sub = ['flow', 'fhigh', 'hrss', 'mtotal', 'mchirp'] tex = rcParams['text.usetex'] words = [] for word in re.split(r'\s', column): if word.isupper(): words.append(word) else: words.extend(re.split(r'_', word)) # parse words for i, word in enumerate(words): # get acronym in lower case if word in acro: words[i] = word.upper() # get numerical unit elif word in unit: words[i] = '$(%s)$' % word # get character with subscript text elif word in sub and tex: words[i] = r'%s$_{\mbox{\small %s}}$' % (word[0], word[1:]) # get greek word elif word in greek and tex: words[i] = r'$\%s$' % word # get starting with greek word elif re.match(r'(%s)' % '|'.join(greek), word) and tex: if word[-1].isdigit(): words[i] = r'$\%s_{%s}$' '' % tuple( re.findall(r"[a-zA-Z]+|\d+", word)) elif word.endswith('sq'): words[i] = r'$\%s^2$' % word.rstrip('sq') # get everything else else: if word.isupper(): words[i] = word else: words[i] = word.title() # escape underscore words[i] = texify(words[i]) return ' '.join(words)
def draw(self): # get columns xcolumn, ycolumn, ccolumn = self.columns # initialise figure plot = self.init_plot() ax = plot.gca() ax.grid(True, which='both') # work out labels labels = self.pargs.pop('labels', self.channels) if isinstance(labels, str): labels = labels.split(',') labels = [str(s).strip('\n ') for s in labels] # get colouring params cmap = self.pargs.pop('cmap') clim = self.pargs.pop('clim', self.pargs.pop('colorlim', None)) cnorm = 'log' if self.pargs.pop('logcolor', False) else None clabel = self.pargs.pop('colorlabel', None) no_loudest = self.pargs.pop('no-loudest', False) is not False loudest_by = self.pargs.pop('loudest-by', None) # get plot arguments plotargs = [] for i in range(len(self.channels)): plotargs.append(dict()) # get plot arguments for key in [ 'vmin', 'vmax', 'edgecolor', 'facecolor', 'cmap', 's', 'marker', 'rasterized' ]: try: val = self.pargs.pop(key) except KeyError: continue if key == 'facecolor' and len(self.channels) > 1 and val is None: val = color_cycle() if key == 'marker' and len(self.channels) > 1 and val is None: val = marker_cycle() elif isinstance(val, (list, tuple, cycle)): val = cycle(val) else: val = cycle([val] * len(self.channels)) for i in range(len(self.channels)): plotargs[i][key] = next(val) # add data valid = SegmentList([self.span]) if self.state and not self.all_data: valid &= self.state.active ntrigs = 0 for channel, label, pargs in zip(self.channels, labels, plotargs): try: channel = get_channel(channel) except ValueError: pass if '#' in str(channel) or '@' in str(channel): key = '%s,%s' % (str(channel), self.state and str(self.state) or 'All') else: key = str(channel) table = get_triggers(key, self.etg, valid, query=False) if self.filterstr is not None: table = table.filter(self.filterstr) ntrigs += len(table) # access channel parameters for limits for c, column in zip(('x', 'y', 'c'), (xcolumn, ycolumn, ccolumn)): if not column: continue # hack for SnglBurst frequency nonsense if column in ['peak_frequency', 'central_freq']: column = 'frequency' # set x and y in plotargs param = '%s_range' % column lim = '%slim' % c if (getattr(channel, param, None) is not None and c in ('x', 'y')): self.pargs.setdefault(lim, getattr(channel, param)) if isinstance(self.pargs[lim], Quantity): self.pargs[lim] = self.pargs[lim].value # set clim separately elif hasattr(channel, param): if not clim: clim = getattr(channel, param) ax.scatter(table[xcolumn], table[ycolumn], c=table[ccolumn] if ccolumn else None, label=label, **pargs) # customise plot legendargs = self.parse_legend_kwargs(markerscale=3) if len(self.channels) == 1: self.pargs.setdefault( 'title', texify('%s (%s)' % (str(self.channels[0]), self.etg))) for axis in ('x', 'y'): # prevent zeros on log scale scale = getattr(ax, 'get_{0}scale'.format(axis))() lim = getattr(ax, 'get_{0}lim'.format(axis))() if scale == 'log' and lim[0] <= 0 and not ntrigs: getattr(ax, 'set_{0}lim'.format(axis))(1, 10) self.apply_parameters(ax, **self.pargs) # correct log-scale empty axes if any(map(isinf, ax.get_ylim())): ax.set_ylim(0.1, 10) # add colorbar if ccolumn: if not ntrigs: ax.scatter([1], [1], c=[1], visible=False) ax.colorbar(cmap=cmap, clim=clim, norm=cnorm, label=clabel) if len(self.channels) == 1 and len(table) and not no_loudest: columns = [ x for x in (loudest_by or ccolumn or ycolumn, xcolumn, ycolumn, ccolumn) if x is not None ] self.add_loudest_event(ax, table, *columns, fontsize='large') if len(self.channels) > 1: ax.legend(**legendargs) # add state segments if isinstance(ax.xaxis.get_transform(), GPSTransform): self.add_state_segments(ax) self.add_future_shade() # finalise return self.finalize()
def significance_drop(outfile, old, new, show_channel_names=None, **kwargs): """Plot the signifiance drop for each channel """ channels = sorted(old.keys()) if show_channel_names is None: show_channel_names = len(channels) <= 50 plot = Plot(figsize=(20, 5)) plot.subplots_adjust(left=.07, right=.93) ax = plot.gca() if show_channel_names: plot.subplots_adjust(bottom=.4) winner = sorted(old.items(), key=lambda x: x[1])[-1][0] for i, c in enumerate(channels): if c == winner: color = 'orange' elif old[c] > new[c]: color = 'dodgerblue' else: color = 'red' ax.plot([i, i], [old[c], new[c]], color=color, linestyle='-', marker='o', markeredgecolor='k', markeredgewidth=.5, markersize=10, label=c, zorder=old[c]) ax.set_xlim(-1, len(channels)) ax.set_ybound(lower=0) # set xticks to show channel names if show_channel_names: ax.set_xticks(range(len(channels))) ax.set_xticklabels([texify(c) for c in channels]) for i, t in enumerate(ax.get_xticklabels()): t.set_rotation(270) t.set_verticalalignment('top') t.set_horizontalalignment('center') t.set_fontsize(8) # or just show systems of channels else: plot.canvas.draw() systems = {} for i, c in enumerate(channels): sys = c.split(':', 1)[1].split('-')[0].split('_')[0] try: systems[sys][1] += 1 except KeyError: systems[sys] = [i, 1] systems = sorted(systems.items(), key=lambda x: x[1][0]) labels, counts = zip(*systems) xticks, xmticks = zip(*[(a, a + b / 2.) for (a, b) in counts]) # show ticks at the edge of each group ax.set_xticks(xticks, minor=False) ax.set_xticklabels([], minor=False) # show label in the centre of each group ax.set_xticks(xmticks, minor=True) for t in ax.set_xticklabels(labels, minor=True): t.set_rotation(270) kwargs.setdefault('ylabel', 'Significance') # create interactivity if outfile.endswith('.svg'): _finalize_plot(plot, ax, outfile.replace('.svg', '.png'), close=False, **kwargs) tooltips = [] ylim = ax.get_ylim() yoffset = (ylim[1] - ylim[0]) * 0.061 bbox = {'fc': 'w', 'ec': '.5', 'alpha': .9, 'boxstyle': 'round'} xthresh = len(channels) / 10. for i, l in enumerate(ax.lines): x = l.get_xdata()[1] if x < xthresh: ha = 'left' elif x > (len(channels) - xthresh): ha = 'right' else: ha = 'center' y = l.get_ydata()[0] + yoffset c = l.get_label() tooltips.append( ax.annotate(texify(c), (x, y), ha=ha, zorder=ylim[1], bbox=bbox)) l.set_gid('line-%d' % i) tooltips[-1].set_gid('tooltip-%d' % i) f = BytesIO() plot.savefig(f, format='svg') tree, xmlid = etree.XMLID(f.getvalue()) tree.set('onload', 'init(evt)') for i in range(len(tooltips)): try: e = xmlid['tooltip-%d' % i] except KeyError: warnings.warn("Failed to recover tooltip %d" % i) continue e.set('visibility', 'hidden') for i, l in enumerate(ax.lines): e = xmlid['line-%d' % i] e.set('onmouseover', 'ShowTooltip(this)') e.set('onmouseout', 'HideTooltip(this)') tree.insert(0, etree.XML(SHOW_HIDE_JAVASCRIPT)) etree.ElementTree(tree).write(outfile) plot.close() else: _finalize_plot(plot, ax, outfile, **kwargs)
def format_label(label): label = str(label).strip('\n ') label = re_quote.sub('', label) return texify(label)
def _process_channel(input_): """Handle individual channels for multiprocessing """ if USETEX: gwplot.configure_mpl_tex() p4 = (.1, .1, .9, .95) chan = input_[1][0] ts = input_[1][1] lassocoef = nonzerocoef[chan] zeroed = lassocoef == 0 if zeroed: plot4 = None plot5 = None plot6 = None pcorr = None else: plot4 = None plot5 = None plot6 = None if trend_type == 'minute': pcorr = numpy.corrcoef(ts.value, primaryts.value)[0, 1] else: pcorr = 0.0 if abs(lassocoef) < threshold: with counter.get_lock(): counter.value += 1 pc = 100 * counter.value / len(nonzerodata) LOGGER.info( "Completed [%d/%d] %3d%% %-50s" % (counter.value, len(nonzerodata), pc, '(%s)' % str(chan))) sys.stdout.flush() return (chan, lassocoef, plot4, plot5, plot6, ts) # create time series subplots fig = Plot(figsize=(12, 8)) fig.subplots_adjust(*p4) ax1 = fig.add_subplot(2, 1, 1, xscale='auto-gps', epoch=start) ax1.plot(primaryts, label=texify(primary), color='black', linewidth=line_size_primary) ax1.set_xlabel(None) ax2 = fig.add_subplot(2, 1, 2, sharex=ax1, xlim=xlim) ax2.plot(ts, label=texify(chan), linewidth=line_size_aux) if range_is_primary: ax1.set_ylabel('Sensitive range [Mpc]') else: ax1.set_ylabel('Primary channel units') ax2.set_ylabel('Channel units') for ax in fig.axes: ax.legend(loc='best') channelstub = re_delim.sub('_', str(chan)).replace('_', '-', 1) plot4 = gwplot.save_figure(fig, '%s_TRENDS-%s.png' % (channelstub, gpsstub), bbox_inches='tight') # create scaled, sign-corrected, and overlayed timeseries tsscaled = scale(ts.value) if lassocoef < 0: tsscaled = numpy.negative(tsscaled) fig = Plot(figsize=(12, 4)) fig.subplots_adjust(*p1) ax = fig.gca(xscale='auto-gps', epoch=start, xlim=xlim) ax.plot(times, _descaler(target), label=texify(primary), color='black', linewidth=line_size_primary) ax.plot(times, _descaler(tsscaled), label=texify(chan), linewidth=line_size_aux) if range_is_primary: ax.set_ylabel('Sensitive range [Mpc]') else: ax.set_ylabel('Primary Channel Units') ax.legend(loc='best') plot5 = gwplot.save_figure(fig, '%s_COMPARISON-%s.png' % (channelstub, gpsstub), bbox_inches='tight') # scatter plot tsCopy = ts.value.reshape(-1, 1) primarytsCopy = primaryts.value.reshape(-1, 1) primaryReg = linear_model.LinearRegression() primaryReg.fit(tsCopy, primarytsCopy) primaryFit = primaryReg.predict(tsCopy) fig = Plot(figsize=(12, 4)) fig.subplots_adjust(*p1) ax = fig.gca() ax.set_xlabel(texify(chan) + ' [Channel units]') if range_is_primary: ax.set_ylabel('Sensitive range [Mpc]') else: ax.set_ylabel('Primary channel units') y_min = min(primaryts.value) y_max = max(primaryts.value) y_range = y_max - y_min ax.set_ylim(y_min - (y_range * 0.1), y_max + (y_range * 0.1)) ax.text(.9, .1, 'r = ' + str('{0:.2}'.format(pcorr)), verticalalignment='bottom', horizontalalignment='right', transform=ax.transAxes, color='black', size=20, bbox=dict(boxstyle='square', facecolor='white', alpha=.75, edgecolor='black')) ax.scatter(ts.value, primaryts.value, color='red') ax.plot(ts.value, primaryFit, color='black') ax.autoscale_view(tight=False, scalex=True, scaley=True) plot6 = gwplot.save_figure(fig, '%s_SCATTER-%s.png' % (channelstub, gpsstub), bbox_inches='tight') # increment counter and print status with counter.get_lock(): counter.value += 1 pc = 100 * counter.value / len(nonzerodata) LOGGER.info("Completed [%d/%d] %3d%% %-50s" % (counter.value, len(nonzerodata), pc, '(%s)' % str(chan))) sys.stdout.flush() return (chan, lassocoef, plot4, plot5, plot6, ts)
def _draw(self): """Load all data, and generate this `SpectrumDataPlot` """ plot = self.init_plot() ax = plot.gca() ax.grid(b=True, axis='both', which='both') if self.state: self.pargs.setdefault( 'suptitle', '[%s-%s, state: %s]' % (self.span[0], self.span[1], texify(str(self.state)))) suptitle = self.pargs.pop('suptitle', None) if suptitle: plot.suptitle(suptitle, y=0.993, va='top') # get spectrum format: 'amplitude' or 'power' sdform = self.pargs.pop('format') if sdform == 'rayleigh': method = 'rayleigh' else: method = None use_percentiles = str( self.pargs.pop('no-percentiles')).lower() == 'false' # parse plotting arguments plotargs = self.parse_plot_kwargs() legendargs = self.parse_legend_kwargs() use_legend = False # get reference arguments refs = self.parse_references() # add data if self.type == 'coherence-spectrum': iterator = list( zip(self.channels[0::2], self.channels[1::2], plotargs)) else: iterator = list(zip(self.channels, plotargs)) for chantuple in iterator: channel = chantuple[0] channel2 = chantuple[1] pargs = chantuple[-1] if self.state and not self.all_data: valid = self.state else: valid = SegmentList([self.span]) if self.type == 'coherence-spectrum': data = get_coherence_spectrum( [str(channel), str(channel2)], valid, query=False) elif self.type == 'range-spectrum': data = get_range_spectrum(str(channel), valid, query=False) elif self.type == 'cumulative-range-spectrum': data = get_range_spectrum(str(channel), valid, query=False, which='mean') if str(data.unit) == 'Mpc': data = (data**3).cumsum()**(1 / 3.) else: data = (data**2).cumsum()**(1 / 2.) try: data = (100 * data / data[-1], ) except IndexError: data = tuple() else: try: data = get_spectrum(str(channel), valid, query=False, format=sdform, method=method) except ValueError as exc: # math op failed beacuse one of the datasets is empty if ('could not be broadcast' in str(exc) and '(0,)' in str(exc)): data = [] else: raise # undo demodulation data = list(data) for i, spec in enumerate(data): data[i] = undo_demodulation(spec, channel, self.pargs.get('xlim', None)) # anticipate log problems if self.logx: data = [s[1:] for s in data] if self.logy: for sp in data: sp.value[sp.value == 0] = 1e-100 if 'label' in pargs: use_legend = True if data and self.type == 'cumulative-range-spectrum': pargs_reverse = pargs.copy() pargs_reverse.pop('label', None) pargs_reverse['linestyle'] = 'dashed' # plot cumulative spectrum and its reverse ax.plot(data[0], **pargs) ax.plot(100 - data[0], **pargs_reverse) elif data and use_percentiles: _, minline, maxline, _ = ax.plot_mmm(*data, **pargs) # make min, max lines lighter: minline.set_alpha(pargs.get('alpha', .1) * 2) maxline.set_alpha(pargs.get('alpha', .1) * 2) elif data: ax.plot(data[0], **pargs) # display references for source, refparams in refs.items(): refspec = io.read_frequencyseries(source) refparams.setdefault('zorder', -len(refs) + 1) if 'filter' in refparams: refspec = refspec.filter(*refparams.pop('filter')) if 'scale' in refparams: refspec *= refparams.pop('scale', 1) if 'label' in refparams: use_legend = True ax.plot(refspec, **refparams) # customise self.add_hvlines() self.apply_parameters(ax, **self.pargs) if use_legend or len(self.channels) > 1 or ax.legend_ is not None: ax.legend(**legendargs) return self.finalize()
def draw(self, outputfile=None): """Get data and generate the figure. """ # get plot and axes plot = self.init_plot() axes = plot.axes if self.state: self.pargs.setdefault( 'suptitle', '[%s-%s, state: %s]' % (self.span[0], self.span[1], texify(str(self.state)))) suptitle = self.pargs.pop('suptitle', None) if suptitle: plot.suptitle(suptitle, y=0.993, va='top') # extract histogram arguments histargs = self.parse_plot_kwargs() # get data data = [] for channel in self.channels: if self.state and not self.all_data: valid = self.state.active else: valid = SegmentList([self.span]) data.append( get_timeseries(channel, valid, query=False).join(gap='ignore', pad=numpy.nan)) # plot for ax, arr, pargs in zip(cycle(axes), data, histargs): # set range if not given if pargs.get('range') is None: pargs['range'] = self._get_range( data, # use range from first dataset if already calculated range=histargs[0].get('range'), # use xlim if manually set (user or INI) xlim=None if ax.get_autoscalex_on() else ax.get_xlim(), ) # plot histogram _, _, patches = ax.hist(arr, **pargs) # update edge color of histogram to be tinted version of face if pargs.get('histtype', None) == 'stepfilled': for p in patches: if not p.get_edgecolor()[3]: p.set_edgecolor(tint(p.get_facecolor(), .7)) # customise plot legendargs = self.parse_legend_kwargs() for i, ax in enumerate(axes): for key, val in self.pargs.items(): if key == 'title' and i > 0: continue if key == 'xlabel' and i < (len(axes) - 1): continue if key == 'ylabel' and ((len(axes) % 2 and i != len(axes) // 2) or (len(axes) % 2 == 0 and i > 0)): continue try: getattr(ax, 'set_%s' % key)(val) except AttributeError: setattr(ax, key, val) if len(self.channels) > 1: ax.legend(**legendargs) if len(axes) % 2 == 0 and axes[0].get_ylabel(): label = axes[0].yaxis.label ax = axes[int(len(axes) // 2) - 1] ax.set_ylabel(label.get_text()) ax.yaxis.label.set_position((0, -.2 / len(axes))) if len(axes) != 2: label.set_text('') # add extra axes and finalise return self.finalize(outputfile=outputfile)
def draw(self, outputfile=None): """Read in all necessary data, and generate the figure. """ plot = self.init_plot() ax = plot.gca() plotargs = self.parse_plot_kwargs() legendargs = self.parse_legend_kwargs() # add data channels, groups = list(zip(*self.get_channel_groups())) for clist, pargs in list(zip(groups, plotargs)): # get data valid = self._get_data_segments(clist[0]) data = [get_timeseries(c, valid, query=False) for c in clist] if len(clist) > 1: data = [tsl.join(gap='pad', pad=numpy.nan) for tsl in data] flatdata = [ts for tsl in data for ts in tsl] # validate parameters for ts in flatdata: # double-check empty if ts.x0 is None: ts.epoch = self.start # double-check log scales if self.logy: ts.value[ts.value == 0] = 1e-100 # set label try: label = pargs.pop('label') except KeyError: try: label = texify(flatdata[0].name) except IndexError: label = clist[0] else: if self.fileformat == 'svg' and not label.startswith( texify(str(flatdata[0].channel)).split('.')[0]): label += ' [%s]' % (texify(str(flatdata[0].channel))) # plot groups or single TimeSeries if len(clist) > 1: data[1].name = None # force no labels for shades data[2].name = None ax.plot_mmm(*data, label=label, **pargs) elif len(flatdata) == 0: ax.plot(data[0].EntryClass([], epoch=self.start, unit='s', name=label), label=label, **pargs) else: for ts in data[0]: line, = ax.plot(ts, label=label, **pargs) label = None pargs['color'] = line.get_color() # customise plot self.add_hvlines() self.apply_parameters(ax, **self.pargs) # add legend if ax.get_legend_handles_labels()[0]: ax.legend(**legendargs) self.add_state_segments(ax) self.add_future_shade() return self.finalize(outputfile=outputfile)
def from_ini(cls, config, section, plotdir='plots', **kwargs): new = super(ParentTab, cls).from_ini(config, section, **kwargs) # add information new.plots = [] new.channel = config.get(section, 'channel') new.ifo = config.get(section, 'IFO') new.modes = OrderedDict((int(idx), name) for (idx, name) in config.nditems(section) if idx.isdigit()) # ----------- # build plots new.set_layout([2, 2]) new.segmenttag = '%s:%%s' % (new.channel) tag = new.channel.split(':', 1)[-1].replace('-', '_') groups = type(new.modes)((idx, name.strip('*')) for (idx, name) in new.modes.items() if int(idx) % 10 == 0) pstates = OrderedDict((idx, name) for (idx, name) in new.modes.items() if not name.startswith('*')) # parse colors colors = dict((int(key[6:]), eval(color)) for (key, color) in config.nditems(section) if re.match(r'color-\d+', key)) for flag in pstates: group = int(flag // 10 * 10) if flag not in colors: colors[flag] = colors.get(group, None) # plot segments for flags, ptag in zip([groups, pstates], ['overview', 'details']): segcolors = [colors.get(flag, None) for flag in flags] if not any(segcolors): segcolors = None for state in new.states: new.plots.append( get_plot('segments')( [new.segmenttag % idx for idx in flags], new.span[0], new.span[1], labels=list(flags.values()), state=state, outdir=plotdir, pid='%s_SEGMENTS_%s' % (tag, ptag.upper()), active=segcolors, known={ 'alpha': 0.1, 'facecolor': 'lightgray' }, title='%s operating mode %s' % (new.ifo, ptag))) # plot pie charts try: explode = list( map(float, config.get(section, 'pie-explode').split(','))) except NoOptionError: explode = None ptag = 'overview' piecolors = [colors.get(flag, None) for flag in groups] if not any(piecolors): piecolors = None for state in new.states: labels = list(groups.values()) new.plots.append( get_plot('segment-pie')( [new.segmenttag % idx for idx in groups], new.span[0], new.span[1], state=state, labels=labels, outdir=plotdir, pid='%s_PIE_%s' % (tag, ptag.upper()), colors=piecolors, explode=explode, title='%s operating mode %s' % (new.ifo, ptag))) shortlabels = [ '\n'.join('{0:.9}.'.format(w) if len(w) > 9 else w for w in l.split()) for l in labels ] new.plots.append( get_plot('duty')( [new.segmenttag % idx for idx in groups], new.span[0], new.span[1], state=state, labels=shortlabels, outdir=plotdir, pid='%s_SEGMENT_BAR_%s' % (tag, ptag.upper()), colors=piecolors, stacked=True, ylim=[0, 100], ylabel=texify('Percentage [%] of available time'), legend_loc='upper left', legend_bbox_to_anchor=(1., 1), legend_fontsize=12, legend_borderaxespad=0, legend_frameon=False, legend_handlelength=1.0, legend_handletextpad=.5, title='%s operating mode %s' % (new.ifo, ptag))) return new
def main(args=None): """Run the lasso command-line interface """ # declare global variables # this is needed for multiprocessing utilities global auxdata, cluster_threshold, cmap, colors, counter, gpsstub global line_size_aux, line_size_primary, max_correlated_channels global nonzerocoef, nonzerodata, p1, primary, primary_mean, primary_std global primaryts, range_is_primary, re_delim, start, target, times global threshold, trend_type, xlim parser = create_parser() args = parser.parse_args(args=args) # get run params start = int(args.gpsstart) end = int(args.gpsend) pad = args.filter_padding # set pertinent global variables cluster_threshold = args.cluster_coefficient line_size_aux = args.line_size_aux line_size_primary = args.line_size_primary threshold = args.threshold trend_type = args.trend_type # let's go LOGGER.info('{} Lasso correlations {}-{}'.format(args.ifo, start, end)) # get primary channel frametype primary = args.primary_channel.format(ifo=args.ifo) range_is_primary = 'EFFECTIVE_RANGE_MPC' in args.primary_channel if args.primary_cache is not None: LOGGER.info("Using custom primary cache file") elif args.primary_frametype is None: try: args.primary_frametype = DEFAULT_FRAMETYPE[ args.primary_channel.split(':')[1]].format(ifo=args.ifo) except KeyError as exc: raise type(exc)("Could not determine primary channel's frametype, " "please specify with --primary-frametype") # create output directory if not os.path.isdir(args.output_dir): os.makedirs(args.output_dir) os.chdir(args.output_dir) # multiprocessing for plots nprocplot = (args.nproc_plot or args.nproc) if USETEX else 1 # bandpass primary if args.band_pass: try: flower, fupper = args.band_pass except TypeError: flower, fupper = None LOGGER.info("-- Loading primary channel data") bandts = get_data(primary, start - pad, end + pad, verbose='Reading primary:'.rjust(30), frametype=args.primary_frametype, source=args.primary_cache, nproc=args.nproc) if flower < 0 or fupper >= float((bandts.sample_rate / 2.).value): raise ValueError( "bandpass frequency is out of range for this " "channel, band (Hz): {0}, sample rate: {1}".format( args.band_pass, bandts.sample_rate)) # get darm BLRMS LOGGER.debug("-- Filtering data") if trend_type == 'minute': stride = 60 else: stride = 1 if flower: darmbl = (bandts.highpass( flower / 2., fstop=flower / 4., filtfilt=False, ftype='butter').notch(60, filtfilt=False).bandpass( flower, fupper, fstop=[flower / 2., fupper * 1.5], filtfilt=False, ftype='butter').crop(start, end)) darmblrms = darmbl.rms(stride) darmblrms.name = '%s %s-%s Hz BLRMS' % (primary, flower, fupper) else: darmbl = bandts.notch(60).crop(start, end) darmblrms = darmbl.rms(stride) darmblrms.name = '%s RMS' % primary primaryts = darmblrms bandts_asd = bandts.asd(4, 2, method='median') darmbl_asd = darmbl.asd(4, 2, method='median') spectrum_plots = gwplot.make_spectrum_plots(start, end, flower, fupper, args.primary_channel, bandts_asd, darmbl_asd) spectrum_plot_zoomed_out = spectrum_plots[0] spectrum_plot_zoomed_in = spectrum_plots[1] else: # load primary channel data LOGGER.info("-- Loading primary channel data") primaryts = get_data(primary, start, end, frametype=args.primary_frametype, source=args.primary_cache, verbose='Reading:'.rjust(30), nproc=args.nproc).crop(start, end) if args.remove_outliers: LOGGER.debug("-- Removing outliers above %f sigma" % args.remove_outliers) gwlasso.remove_outliers(primaryts, args.remove_outliers) elif args.remove_outliers_pf: LOGGER.debug("-- Removing outliers in the bottom {} percent " "of data".format(args.remove_outliers_pf)) gwlasso.remove_outliers(primaryts, args.remove_outliers_pf, method='pf') start = int(primaryts.span()[0]) end = int(primaryts.span()[1]) primary_mean = numpy.mean(primaryts.value) primary_std = numpy.std(primaryts.value) # get aux data LOGGER.info("-- Loading auxiliary channel data") if args.channel_file is None: host, port = io_nds2.host_resolution_order(args.ifo)[0] channels = ChannelList.query_nds2('*.mean', host=host, port=port, type='m-trend') else: with open(args.channel_file, 'r') as f: channels = [name.rstrip('\n') for name in f] nchan = len(channels) LOGGER.debug("Identified %d channels" % nchan) if trend_type == 'minute': frametype = '%s_M' % args.ifo # for minute trends else: frametype = '%s_T' % args.ifo # for second trends # read aux channels auxdata = get_data(channels, start, end, verbose='Reading:'.rjust(30), frametype=frametype, nproc=args.nproc, pad=0).crop(start, end) # -- removes flat data to be re-introdused later LOGGER.info('-- Pre-processing auxiliary channel data') auxdata = gwlasso.remove_flat(auxdata) flatable = Table(data=(list(set(channels) - set(auxdata.keys())), ), names=('Channels', )) LOGGER.debug('Removed {0} channels with flat data'.format(len(flatable))) LOGGER.debug('{0} channels remaining'.format(len(auxdata))) # -- remove bad data LOGGER.info("Removing any channels with bad data...") nbefore = len(auxdata) auxdata = gwlasso.remove_bad(auxdata) nafter = len(auxdata) LOGGER.debug('Removed {0} channels with bad data'.format(nbefore - nafter)) LOGGER.debug('{0} channels remaining'.format(nafter)) data = numpy.array([scale(ts.value) for ts in auxdata.values()]).T # -- perform lasso regression ------------------- # create model LOGGER.info('-- Fitting data to target') target = scale(primaryts.value) model = gwlasso.fit(data, target, alpha=args.alpha) LOGGER.info('Alpha: {}'.format(model.alpha)) # restructure results for convenience allresults = Table(data=(list(auxdata.keys()), model.coef_, numpy.abs(model.coef_)), names=('Channel', 'Lasso coefficient', 'rank')) allresults.sort('rank') allresults.reverse() useful = allresults['rank'] > 0 allresults.remove_column('rank') results = allresults[useful] # non-zero coefficient zeroed = allresults[numpy.invert(useful)] # zero coefficient # extract data for useful channels nonzerodata = {name: auxdata[name] for name in results['Channel']} nonzerocoef = {name: coeff for name, coeff in results.as_array()} # print results LOGGER.info('Found {} channels with |Lasso coefficient| >= {}:\n\n'.format( len(results), threshold)) print(results) print('\n\n') # convert to pandas set_option('max_colwidth', -1) df = results.to_pandas() df.index += 1 # write results to files gpsstub = '%d-%d' % (start, end - start) resultsfile = '%s-LASSO_RESULTS-%s.csv' % (args.ifo, gpsstub) results.write(resultsfile, format='csv', overwrite=True) zerofile = '%s-ZERO_COEFFICIENT_CHANNELS-%s.csv' % (args.ifo, gpsstub) zeroed.write(zerofile, format='csv', overwrite=True) flatfile = '%s-FLAT_CHANNELS-%s.csv' % (args.ifo, gpsstub) flatable.write(flatfile, format='csv', overwrite=True) # -- generate lasso plots modelFit = model.predict(data) re_delim = re.compile(r'[:_-]') p1 = (.1, .15, .9, .9) # global plot defaults for plot1, lasso model times = primaryts.times.value xlim = primaryts.span cmap = get_cmap('tab20') colors = [cmap(i) for i in numpy.linspace(0, 1, len(nonzerodata) + 1)] plot = Plot(figsize=(12, 4)) plot.subplots_adjust(*p1) ax = plot.gca(xscale='auto-gps', epoch=start, xlim=xlim) ax.plot(times, _descaler(target), label=texify(primary), color='black', linewidth=line_size_primary) ax.plot(times, _descaler(modelFit), label='Lasso model', linewidth=line_size_aux) if range_is_primary: ax.set_ylabel('Sensitive range [Mpc]') ax.set_title('Lasso Model of Range') else: ax.set_ylabel('Primary Channel Units') ax.set_title('Lasso Model of Primary Channel') ax.legend(loc='best') plot1 = gwplot.save_figure(plot, '%s-LASSO_MODEL-%s.png' % (args.ifo, gpsstub), bbox_inches='tight') # summed contributions plot = Plot(figsize=(12, 4)) plot.subplots_adjust(*p1) ax = plot.gca(xscale='auto-gps', epoch=start, xlim=xlim) ax.plot(times, _descaler(target), label=texify(primary), color='black', linewidth=line_size_primary) summed = 0 for i, name in enumerate(results['Channel']): summed += scale(nonzerodata[name].value) * nonzerocoef[name] if i: label = 'Channels 1-{0}'.format(i + 1) else: label = 'Channel 1' ax.plot(times, _descaler(summed), label=label, color=colors[i], linewidth=line_size_aux) if range_is_primary: ax.set_ylabel('Sensitive range [Mpc]') else: ax.set_ylabel('Primary Channel Units') ax.set_title('Summations of Channel Contributions to Model') ax.legend(loc='center left', bbox_to_anchor=(1.05, 0.5)) plot2 = gwplot.save_figure(plot, '%s-LASSO_CHANNEL_SUMMATION-%s.png' % (args.ifo, gpsstub), bbox_inches='tight') # individual contributions plot = Plot(figsize=(12, 4)) plot.subplots_adjust(*p1) ax = plot.gca(xscale='auto-gps', epoch=start, xlim=xlim) ax.plot(times, _descaler(target), label=texify(primary), color='black', linewidth=line_size_primary) for i, name in enumerate(results['Channel']): this = _descaler(scale(nonzerodata[name].value) * nonzerocoef[name]) if i: label = 'Channels 1-{0}'.format(i + 1) else: label = 'Channel 1' ax.plot(times, this, label=texify(name), color=colors[i], linewidth=line_size_aux) if range_is_primary: ax.set_ylabel('Sensitive range [Mpc]') else: ax.set_ylabel('Primary Channel Units') ax.set_title('Individual Channel Contributions to Model') ax.legend(loc='center left', bbox_to_anchor=(1.05, 0.5)) plot3 = gwplot.save_figure(plot, '%s-LASSO_CHANNEL_CONTRIBUTIONS-%s.png' % (args.ifo, gpsstub), bbox_inches='tight') # -- process aux channels, making plots LOGGER.info("-- Processing channels") counter = multiprocessing.Value('i', 0) # process channels pool = multiprocessing.Pool(nprocplot) results = pool.map(_process_channel, enumerate(list(nonzerodata.items()))) results = sorted(results, key=lambda x: abs(x[1]), reverse=True) # generate clustered time series plots counter = multiprocessing.Value('i', 0) max_correlated_channels = 20 if args.no_cluster is False: LOGGER.info("-- Generating clusters") pool = multiprocessing.Pool(nprocplot) clusters = pool.map(_generate_cluster, enumerate(results)) channelsfile = '%s-CHANNELS-%s.csv' % (args.ifo, gpsstub) numpy.savetxt(channelsfile, channels, delimiter=',', fmt='%s') # write html trange = '%d-%d' % (start, end) title = '%s Lasso Correlation: %s' % (args.ifo, trange) if args.band_pass: links = [trange ] + [(s, '#%s' % s.lower()) for s in ['Parameters', 'Spectra', 'Model', 'Results']] else: links = [trange] + [(s, '#%s' % s.lower()) for s in ['Parameters', 'Model', 'Results']] (brand, class_) = htmlio.get_brand(args.ifo, 'Lasso', start) navbar = htmlio.navbar(links, class_=class_, brand=brand) page = htmlio.new_bootstrap_page(title='%s Lasso | %s' % (args.ifo, trange), navbar=navbar) page.h1(title, class_='pb-2 mt-3 mb-2 border-bottom') # -- summary table content = [ ('Primary channel', markup.oneliner.code(primary)), ('Primary frametype', markup.oneliner.code(args.primary_frametype) or '-'), ('Primary cache file', markup.oneliner.code(args.primary_cache) or '-'), ('Outlier threshold', '%s sigma' % args.remove_outliers), ('Lasso coefficient threshold', str(threshold)), ('Cluster coefficient threshold', str(args.cluster_coefficient)), ('Non-zero coefficients', str(numpy.count_nonzero(model.coef_))), ('α (model)', '%.4f' % model.alpha) ] if args.band_pass: content.insert( 2, ('Primary bandpass', '{0}-{1} Hz'.format(flower, fupper))) page.h2('Parameters', class_='mt-4 mb-4', id_='parameters') page.div(class_='row') page.div(class_='col-md-9 col-sm-12') page.add(htmlio.parameter_table(content, start=start, end=end)) page.div.close() # col-md-9 col-sm-12 # -- download button files = [('%s analyzed channels (CSV)' % nchan, channelsfile), ('%s flat channels (CSV)' % len(flatable), flatfile), ('%s zeroed channels (CSV)' % len(zeroed), zerofile)] page.div(class_='col-md-3 col-sm-12') page.add( htmlio.download_btn(files, label='Channel information', btnclass='btn btn-%s dropdown-toggle' % args.ifo.lower())) page.div.close() # col-md-3 col-sm-12 page.div.close() # rowa # -- command-line page.h5('Command-line:') page.add(htmlio.get_command_line(about=False, prog=PROG)) if args.band_pass: page.h2('Primary channel spectra', class_='mt-4', id_='spectra') page.div(class_='card border-light card-body shadow-sm') page.div(class_='row') page.div(class_='col-md-6') spectra_img1 = htmlio.FancyPlot(spectrum_plot_zoomed_out) page.add(htmlio.fancybox_img(spectra_img1)) page.div.close() # col-md-6 page.div(class_='col-md-6') spectra_img2 = htmlio.FancyPlot(spectrum_plot_zoomed_in) page.add(htmlio.fancybox_img(spectra_img2)) page.div.close() # col-md-6 page.div.close() # row page.div.close() # card border-light card-body shadow-sm # -- model information page.h2('Model information', class_='mt-4', id_='model') page.div(class_='card card-%s card-body shadow-sm' % args.ifo.lower()) page.div(class_='row') page.div(class_='col-md-8 offset-md-2', id_='results-table') page.p('Below are the top {} mean minute-trend channels, ranked by ' 'Lasso correlation with the primary.'.format(df.shape[0])) page.add( df.to_html(classes=('table', 'table-sm', 'table-hover'), formatters={ 'Lasso coefficient': lambda x: "%.4f" % x, 'Channel': lambda x: str(htmlio.cis_link(x.split('.')[0])), '__index__': lambda x: str(x) }, escape=False, border=0).replace(' style="text-align: right;"', '')) page.div.close() # col-md-10 offset-md-1 page.div.close() # row page.div(class_='row', id_='primary-lasso') page.div(class_='col-md-8 offset-md-2') img1 = htmlio.FancyPlot(plot1) page.add(htmlio.fancybox_img(img1)) # primary lasso plot page.div.close() # col-md-8 offset-md-2 page.div.close() # primary-lasso page.div(class_='row', id_='channel-summation') img2 = htmlio.FancyPlot(plot2) page.div(class_='col-md-8 offset-md-2') page.add(htmlio.fancybox_img(img2)) page.div.close() # col-md-8 offset-md-2 page.div.close() # channel-summation page.div(class_='row', id_='channels-and-primary') img3 = htmlio.FancyPlot(plot3) page.div(class_='col-md-8 offset-md-2') page.add(htmlio.fancybox_img(img3)) page.div.close() # col-md-8 offset-md-2 page.div.close() # channels-and-primary page.div.close() # card card-<ifo> card-body shadow-sm # -- results page.h2('Top channels', class_='mt-4', id_='results') page.div(id_='results') # for each aux channel, create information container and put plots in it for i, (ch, lassocoef, plot4, plot5, plot6, ts) in enumerate(results): # set container color/context based on lasso coefficient if lassocoef == 0: break elif abs(lassocoef) < threshold: h = '%s [lasso coefficient = %.4f] (Below threshold)' % (ch, lassocoef) else: h = '%s [lasso coefficient = %.4f]' % (ch, lassocoef) if ((lassocoef is None) or (lassocoef == 0) or (abs(lassocoef) < threshold)): card = 'card border-light mb-1 shadow-sm' card_header = 'card-header bg-light' elif abs(lassocoef) >= .5: card = 'card border-danger mb-1 shadow-sm' card_header = 'card-header text-white bg-danger' elif abs(lassocoef) >= .2: card = 'card border-warning mb-1 shadow-sm' card_header = 'card-header text-white bg-warning' else: card = 'card border-info mb-1 shadow-sm' card_header = 'card-header text-white bg-info' page.div(class_=card) # heading page.div(class_=card_header) page.a(h, class_='collapsed card-link cis-link', href='#channel%d' % i, **{'data-toggle': 'collapse'}) page.div.close() # card-header # body page.div(id_='channel%d' % i, class_='collapse', **{'data-parent': '#results'}) page.div(class_='card-body') if lassocoef is None: page.p('The amplitude data for this channel is flat (does not ' 'change) within the chosen time period.') elif abs(lassocoef) < threshold: page.p('Lasso coefficient below the threshold of %g.' % (threshold)) else: for image in [plot4, plot5, plot6]: img = htmlio.FancyPlot(image) page.div(class_='row') page.div(class_='col-md-8 offset-md-2') page.add(htmlio.fancybox_img(img)) page.div.close() # col-md-8 offset-md-2 page.div.close() # row page.add('<hr class="row-divider">') if args.no_cluster is False: if clusters[i][0] is None: page.p("<font size='3'><br />No channels were highly " "correlated with this channel.</font>") else: page.div(class_='row', id_='clusters') page.div(class_='col-md-12') cimg = htmlio.FancyPlot(clusters[i][0]) page.add(htmlio.fancybox_img(cimg)) page.div.close() # col-md-12 page.div.close() # clusters if clusters[i][1] is not None: corr_link = markup.oneliner.a( 'Export {} channels (CSV)'.format( max_correlated_channels), href=clusters[i][1], download=clusters[i][1], ) page.button( corr_link, class_='btn btn-%s' % args.ifo.lower(), ) page.div.close() # card-body page.div.close() # collapse page.div.close() # card page.div.close() # results htmlio.close_page(page, 'index.html') # save and close LOGGER.info("-- Process Completed")
def draw(self, outputfile=None): """Generate the figure for this plot """ plot = self.init_plot() ax = plot.axes[0] # get plotting arguments cumulative = self.pargs.pop('cumulative', False) plotargs = self.parse_plot_kwargs() legendargs = self.parse_legend_kwargs() # set ylabel if cumulative: self.pargs.setdefault( 'ylabel', 'Cumulative time-volume [Mpc$^3$ kyr]', ) else: self.pargs.setdefault( 'ylabel', 'Time-volume [Mpc$^3$ kyr]', ) # get data allsegs, allranges = ([], []) for channel, flag, pargs in zip(self.channels, self.flags, plotargs): pad = 0 if self.state and not self.all_data: valid = self.state.active pad = numpy.nan elif channel.sample_rate.value: valid = SegmentList( [self.span.protract(1 / channel.sample_rate.value)]) else: valid = SegmentList([self.span]) data = get_timeseries(channel, valid, query=False).join(gap='pad', pad=pad) if not data.unit or data.unit.to_string() in ['', 'undef']: data.override_unit('Mpc') segments = get_segments(flag, valid, query=False) timevolume = self.calculate_time_volume(segments.active, data) if cumulative: ax.plot(timevolume.cumsum(), **pargs) else: ax.plot(timevolume, **pargs) allsegs.append(segments.active) allranges.append(data) # estimate combined time-volume if self.all_data and len(self.channels) > 1: pargs = plotargs[-1] pargs['color'] = '#000000' pargs['label'] = 'Combined' pargs['linestyle'] = '--' combined_timevolume = self.combined_time_volume(allsegs, allranges) if cumulative: ax.plot(combined_timevolume.cumsum(), **pargs) else: ax.plot(combined_timevolume, **pargs) # add horizontal lines to add for yval in self.pargs.get('hline', []): try: yval = float(yval) except ValueError: continue else: ax.plot([self.start, self.end], [yval, yval], linestyle='--', color='red') # customise plot self.apply_parameters(ax, **self.pargs) if (len(self.channels) > 1 or plotargs[0].get('label', None) in [texify(str(self.channels[0])), None]): ax.legend(**legendargs) # add extra axes and finalise self.add_state_segments(ax) self.add_future_shade() if ax.get_yscale() == 'log': ax.yaxis.set_major_locator(LogLocator()) else: ax.yaxis.set_major_locator(MaxNLocator(8)) ticks = ax.get_yticks() ax.yaxis.set_ticklabels(ticks) return self.finalize(outputfile=outputfile)
def _draw(self): """Load all data, and generate this `SpectrumDataPlot` """ plot = self.init_plot() ax = plot.gca() if self.state: self.pargs.setdefault( 'suptitle', '[%s-%s, state: %s]' % (self.span[0], self.span[1], texify(str(self.state)))) suptitle = self.pargs.pop('suptitle', None) if suptitle: plot.suptitle(suptitle, y=0.993, va='top') # parse plotting arguments cmap = self.pargs.pop('cmap', None) varargs = self.parse_variance_kwargs() plotargs = self.parse_plot_kwargs()[0] # get reference arguments refs = self.parse_references() # calculate spectral variance and plot # pad data request to over-fill plots (no gaps at the end) if self.state and not self.all_data: valid = self.state.active else: valid = SegmentList([self.span]) livetime = float(abs(valid)) if livetime: plotargs.setdefault('vmin', 1 / livetime) plotargs.setdefault('vmax', 1.) plotargs.pop('label') specgram = get_spectrogram(self.channels[0], valid, query=False, format='asd').join(gap='ignore') if specgram.size: asd = specgram.median(axis=0) asd.name = None variance = specgram.variance(**varargs) # normalize the variance variance /= livetime / specgram.dt.value # undo demodulation variance = undo_demodulation(variance, self.channels[0], ax.get_xlim()) # plot ax.plot(asd, color='grey', linewidth=0.3) ax.imshow(variance, cmap=cmap, **plotargs) # display references for source, refparams in refs.items(): refspec = io.read_frequencyseries(source) if 'filter' in refparams: refspec = refspec.filter(*refparams.pop('filter')) if 'scale' in refparams: refspec *= refparams.pop('scale', 1) ax.plot(refspec, **refparams) # customise self.add_hvlines() self.apply_parameters(ax, **self.pargs) ax.grid(b=True, axis='both', which='both') return self.finalize()
def _draw(self): """Load all data, and generate this `SpectrumDataPlot` """ plot = self.init_plot() ax = plot.gca() ax.grid(b=True, axis='both', which='both') if self.state: self.pargs.setdefault( 'suptitle', '[%s-%s, state: %s]' % (self.span[0], self.span[1], texify(str(self.state)))) suptitle = self.pargs.pop('suptitle', None) if suptitle: plot.suptitle(suptitle, y=0.993, va='top') # get spectrum format: 'amplitude' or 'power' sdform = self.pargs.pop('format') # parse plotting arguments plotargs = self.parse_plot_kwargs() legendargs = self.parse_legend_kwargs() # add data sumdata = [] for i, (channel, pargs) in enumerate(zip(self.channels, plotargs)): if self.state and not self.all_data: valid = self.state else: valid = SegmentList([self.span]) data = get_spectrum(str(channel), valid, query=False, format=sdform, method=None)[0] if i and data.size: sumdata.append(data) else: darmdata = data # anticipate log problems if self.logx: data = data[1:] if self.logy: data.value[data.value == 0] = 1e-100 pargs.setdefault('zorder', -i) ax.plot(data, **pargs) # assert all noise terms have the same resolution if any([x.dx != sumdata[0].dx for x in sumdata]): raise RuntimeError("Noise components have different resolutions, " "cannot construct sum of noises") # reshape noises if required n = max(x.size for x in sumdata) for i, d in enumerate(sumdata): if d.size < n: sumdata[i] = numpy.resize(numpy.require(d, requirements=['O']), (n, )) # plot sum of noises sumargs = self.parse_sum_params() sum_ = numpy.zeros_like(darmdata**2, shape=(n, )) for d in sumdata: d.override_unit(darmdata.unit) # enforce consistent units sum_ += d**2 ax.plot(sum_**(1 / 2.), zorder=1, **sumargs) ax.lines.insert(1, ax.lines.pop(-1)) # plot residual of noises if not self.pargs.pop('no-residual', False): resargs = self.parse_residual_params() try: residual = (darmdata**2 - sum_)**(1 / 2.) except ValueError: if not darmdata.size: # if no data, just copy nothing residual = darmdata else: # other error raise ax.plot(residual, zorder=-1000, **resargs) ax.lines.insert(1, ax.lines.pop(-1)) # finalize self.apply_parameters(ax, **self.pargs) ax.legend(**legendargs) return self.finalize()
def _generate_cluster(input_): """Generate cluster data for use below """ if USETEX: gwplot.configure_mpl_tex() currentchan = input_[1][0] currentts = input_[1][5] current = input_[0] p7 = (.135, .15, .95, .9) plot7 = None plot7_list = None if current < len(nonzerodata): cluster = [] for i, otheritem in enumerate(list(auxdata.items())): chan_, ts_ = otheritem if chan_ != currentchan: pcorr = numpy.corrcoef(currentts.value, ts_.value)[0, 1] if abs(pcorr) >= cluster_threshold: stub = re_delim.sub('_', chan_).replace('_', '-', 1) cluster.append([i, ts_, pcorr, chan_, stub]) if cluster: # write cluster table to file cluster = sorted(cluster, key=lambda x: abs(x[2]), reverse=True) clustertab = Table(data=list(zip(*cluster))[2:4], names=('Pearson Coefficient', 'Channel')) plot7_list = '%s_CLUSTER_LIST-%s.csv' % (re_delim.sub( '_', str(currentchan)).replace('_', '-', 1), gpsstub) clustertab.write(plot7_list, format='csv', overwrite=True) ncluster = min(len(cluster), max_correlated_channels) colors2 = [cmap(i) for i in numpy.linspace(0, 1, ncluster + 1)] # plot fig = Plot(figsize=(12, 4)) fig.subplots_adjust(*p7) ax = fig.gca(xscale='auto-gps') ax.plot(times, scale(currentts.value) * numpy.sign(input_[1][1]), label=texify(currentchan), linewidth=line_size_aux, color=colors[0]) for i in range(0, ncluster): this = cluster[i] ax.plot( times, scale(this[1].value) * numpy.sign(input_[1][1]) * numpy.sign(this[2]), color=colors2[i + 1], linewidth=line_size_aux, label=('{0}, r = {1:.2}'.format(texify(cluster[i][3]), cluster[i][2])), ) ax.margins(x=0) ax.set_ylabel('Scaled amplitude [arbitrary units]') ax.set_title('Highly Correlated Channels') ax.legend(loc='center left', bbox_to_anchor=(1.05, 0.5)) plot7 = gwplot.save_figure( fig, '%s_CLUSTER-%s.png' % (re_delim.sub( '_', str(currentchan)).replace('_', '-', 1), gpsstub), bbox_inches='tight') with counter.get_lock(): counter.value += 1 pc = 100 * counter.value / len(nonzerodata) LOGGER.info( "Completed [%d/%d] %3d%% %-50s" % (counter.value, len(nonzerodata), pc, '(%s)' % str(currentchan))) sys.stdout.flush() return (plot7, plot7_list)
def _draw(self): """Load all data, and generate this `SpectrumDataPlot` """ plot = self.init_plot() ax = plot.gca() ax.grid(b=True, axis='both', which='both') if self.state: self.pargs.setdefault( 'suptitle', '[%s-%s, state: %s]' % (self.span[0], self.span[1], texify(str(self.state)))) suptitle = self.pargs.pop('suptitle', None) if suptitle: plot.suptitle(suptitle, y=0.993, va='top') # get spectrum format: 'amplitude' or 'power' sdform = self.pargs.pop('format') # parse plotting arguments plotargs = self.parse_plot_kwargs()[0] # add data sumdata = [] for i, channel in enumerate(self.channels): if self.state and not self.all_data: valid = self.state else: valid = SegmentList([self.span]) data = get_spectrum(str(channel), valid, query=False, format=sdform, method=None)[0] if i and data.size: sumdata.append(data) else: target = data if target.size: # assert all noise terms have the same resolution if any([x.dx != target.dx for x in sumdata]): raise RuntimeError("Noise components have different " "resolutions, cannot construct " "sum of noises") # reshape noises if required n = target.size for i, d in enumerate(sumdata): if d.size < n: sumdata[i] = numpy.resize( numpy.require(d, requirements=['O']), (n, )) # calculate sum of noises sum_ = numpy.zeros_like(sumdata[0]**2, shape=(n, )) for d in sumdata: d.override_unit(sumdata[0].unit) # enforce consistent units sum_ += d**2 sum_ **= (1 / 2.) relative = sum_ / target else: # no data, so just use anything as a proxy relative = target # plot ratio of h(t) to sum of noises ax.plot(relative, **plotargs) # finalize plot self.apply_parameters(ax, **self.pargs) return self.finalize()
def draw(self): # data span start = self.gpstime - self.duration / 2. end = self.gpstime + self.duration / 2. # get data if self.use_nds: data = TimeSeriesDict.fetch(self.chanlist, start, end) else: from glue.datafind import GWDataFindHTTPConnection conn = GWDataFindHTTPConnection() cache = conn.find_frame_urls(self.ifo[0], '%s_R' % self.ifo, self.start, self.end, urltype='file') if len(cache) == 0: data = {} else: data = TimeSeriesDict.read(cache, self.chanlist, start=start, end=end) # make plot plot, axes = subplots(nrows=self.geometry[0], ncols=self.geometry[1], sharex=True, subplot_kw={'xscale': 'auto-gps'}, FigureClass=Plot, figsize=[12, 6]) axes[0, 0].set_xlim(start, end) for channel, ax in zip(self.chanlist, axes.flat): ax.set_epoch(self.gpstime) # plot data try: ax.plot(data[channel]) except KeyError: ax.text(self.gpstime, 0.5, "No data", va='center', ha='center', transform=ax.transData) # plot trip indicator ax.axvline(self.gpstime, linewidth=0.5, linestyle='--', color='red') ax.set_xlabel('') ax.set_ylabel('') ax.set_title(texify(channel.name), fontsize=10) ax.xaxis.set_minor_locator(NullLocator()) for tick in ax.yaxis.get_major_ticks(): tick.label.set_fontsize(10) for tick in ax.xaxis.get_major_ticks(): tick.label.set_fontsize(16) plot.text(0.5, 0.02, 'Time [seconds] from trip (%s)' % self.gpstime, ha='center', va='bottom', fontsize=24) plot.text(0.01, 0.5, 'Amplitude %s' % self.unit, ha='left', va='center', rotation='vertical', fontsize=24) plot.suptitle('%s %s %s watchdog trip: %s' % (self.ifo, self.chamber, self.sensor, self.gpstime), fontsize=24) plot.save(self.outputfile) plot.close() return self.outputfile