def plot_spectra(clusters, channel, unit='cts', xlog=True, legend=None, xlim=None, **kwargs): from glob import glob from gwpy.frequencyseries import FrequencySeries from gwpy.plot import Plot title = channel psds = {} for cluster in clusters: for filename in glob('*.hdf5'): try: psds[cluster] = FrequencySeries.read(filename, f'{cluster}-{channel}') print(f'found in {filename}') break except KeyError: continue else: raise KeyError(f'Could not find Nº{cluster}') if legend is None: legend = clusters # plot the group in one figure. plt = Plot(*(psds[cluster] for cluster in psds), separate=False, sharex=True, zorder=1, **kwargs) if xlim is not None: plt.gca().set_xlim(xlim) plt.gca().set_ylim((1e-48, 1e-37)) # modify the figure as a whole. # plt.add_segments_bar(dq, label='') # plt.gca().set_color_cycle(['red', 'green', 'blue', 'yellow']) if xlog: plt.gca().set_xscale('log') plt.gca().set_yscale('log') plt.gca().set_ylabel(f'Power Spectral Density [{unit}^2/Hz]') plt.suptitle(title) plt.legend(legend, prop={'size': 15}) # save to png. plt.save(f'{title}.png')
def test_save_legend(tmpdir): base = str(tmpdir) fig = Plot() ax = fig.gca() ax.plot(SERIES, label=SERIES.name) tsplot = plot.save_legend(ax, os.path.join(base, 'test.png')) assert tsplot == os.path.join(base, 'test.png') shutil.rmtree(base, ignore_errors=True)
def test_apply_parameters(self, plot): fig = Plot() ax = fig.gca() plot.apply_parameters(ax, **{ 'xlim': (10, 20), 'no-blah': 'anything', 'grid': False, }) assert ax.get_xlim() == (10, 20)
def hveto_roc(outfile, rounds, figsize=[9, 6], constants=[1, 5, 10, 20], **kwargs): efficiency = [] deadtime = [] for r in rounds: try: efficiency.append(r.cum_efficiency[0] / r.cum_efficiency[1]) except ZeroDivisionError: efficiency.append(0.) try: deadtime.append(r.cum_deadtime[0] / r.cum_deadtime[1]) except ZeroDivisionError: deadtime.append(0.) plot = Plot(figsize=figsize) ax = plot.gca() ax.plot(deadtime, efficiency, marker='o', linestyle='-') try: xbound = 10**floor(log10(deadtime[0])) except ValueError: xbound = 1e-4 try: ybound = 10**floor(log10(efficiency[0])) except ValueError: ybound = 1e-4 bound = min(xbound, ybound) axargs = { 'xlabel': 'Fractional deadtime', 'ylabel': 'Fractional efficiency', 'xscale': 'log', 'yscale': 'log', 'xlim': (bound, 1.), 'ylim': (bound, 1.), } axargs.update(kwargs) # draw some eff/dt contours if len(constants): for i, c in enumerate(constants): g = 1 - ((i + 1) / len(constants) * .5) x = axargs['xlim'] y = [a * c for a in x] ax.plot(x, y, linestyle='--', color=(g, g, g), label=str(c)) ax.legend(title='Eff/dt:', borderaxespad=0, bbox_to_anchor=(1.01, 1), handlelength=1, handletextpad=.5, loc='upper left') # save and close _finalize_plot(plot, ax, outfile, **axargs)
def plot_timeseries(*data, **kwargs): title = kwargs.pop('title', None) ylim = kwargs.pop('ylim', None) fname = kwargs.pop('fname', 'TimeSeries.png') plot = Plot(figsize=(15, 10)) ax0 = plot.gca() ax0.plot(*data) ax0.legend([text.to_string(_data.name) for _data in data], fontsize=20) ax0.set_xscale('auto-gps') ax0.set_ylabel(text.to_string(data[0].unit)) plot.add_state_segments(segments_daq_iy0_ok, label='IY0 DAQ State') plot.add_state_segments(segments_daq_ix1_ok, label='IX1 DAQ State') plot.suptitle(title, fontsize=40) if ylim: ax0.set_ylim(ylim[0], ylim[1]) plot.savefig(fname) plot.close()
def before_after_histogram(outfile, x, y, label1='Before', label2='After', bins=100, histtype='stepfilled', range=None, figsize=[9, 6], **kwargs): """Plot a histogram of SNR for two event distributions """ # format axis arguments axargs = { 'xscale': 'log', 'xlabel': 'Loudness', 'yscale': 'log', 'ylabel': 'Number of events', } axargs.update(kwargs) # create figure plot = Plot(figsize=figsize) ax = plot.gca() # make histogram if range is None: range = min(map(numpy.min, (x, y))), max(map(numpy.max, (x, y))) axargs.setdefault('xlim', range) histargs = { 'range': range, 'histtype': histtype, 'bins': bins, 'linewidth': 2, 'logbins': axargs['xscale'] == 'log', 'alpha': .8, } ax.hist(x, label=label1, facecolor='red', edgecolor='darkred', **histargs) ax.hist(y, label=label2, facecolor='dodgerblue', edgecolor='blue', **histargs) # add legend ax.legend(loc='upper right') # format axes axargs.setdefault('ylim', (.5, ax.yaxis.get_data_interval()[1] * 1.05)) _finalize_plot(plot, ax, outfile, **axargs)
def spectral_overlay(gps, qspecgram, fringe, output, multipliers=(1, 2, 4, 8), figsize=[12, 4]): """Overlay scattering fringe projections on top of a high-resolution spectrogram Parameters ---------- gps : `float` reference GPS time (in seconds) to serve as the origin qspecgram : `~gwpy.spectrogram.Spectrogram` an interpolated high-resolution spectrogram fringe : `~gwpy.timeseries.TimeSeries` projected fringe frequencies (in Hz) output : `str` name of the output file multipliers : `tuple`, optional collection of fringe harmonic numbers to plot, can be given in any order, default: `(1, 2, 4, 8)` figsize : `tuple`, optional size (width x height) of the final figure, default: `(12, 4)` """ plot = Plot(figsize=figsize) ax = plot.gca() # format spectrogram plot ax.set_title('Fringes: {0}, Spectrogram: {1}'.format( fringe.name, qspecgram.name)) _format_spectrogram(ax, qspecgram, colormap='binary') # overlay fringe frequencies _format_timeseries(ax, gps, fringe, multipliers=multipliers, linewidth=1.5) ax.set_ylim([ qspecgram.f0.to('Hz').value, qspecgram.frequencies.max().to('Hz').value ]) # save plot and close plot.savefig(output, bbox_inches='tight') plot.close()
def hveto_roc(outfile, rounds, figsize=[9, 6], constants=[1, 5, 10, 20], **kwargs): efficiency = [] deadtime = [] for r in rounds: try: efficiency.append(r.cum_efficiency[0] / r.cum_efficiency[1]) except ZeroDivisionError: efficiency.append(0.) try: deadtime.append(r.cum_deadtime[0] / r.cum_deadtime[1]) except ZeroDivisionError: deadtime.append(0.) plot = Plot(figsize=figsize) ax = plot.gca() ax.plot(deadtime, efficiency, marker='o', linestyle='-') try: xbound = 10 ** floor(log10(deadtime[0])) except ValueError: xbound = 1e-4 try: ybound = 10 ** floor(log10(efficiency[0])) except ValueError: ybound = 1e-4 bound = min(xbound, ybound) axargs = { 'xlabel': 'Fractional deadtime', 'ylabel': 'Fractional efficiency', 'xscale': 'log', 'yscale': 'log', 'xlim': (bound, 1.), 'ylim': (bound, 1.), } axargs.update(kwargs) # draw some eff/dt contours if len(constants): for i, c in enumerate(constants): g = 1 - ((i+1)/len(constants) * .5) x = axargs['xlim'] y = [a * c for a in x] ax.plot(x, y, linestyle='--', color=(g, g, g), label=str(c)) ax.legend(title='Eff/dt:', borderaxespad=0, bbox_to_anchor=(1.01, 1), handlelength=1, handletextpad=.5, loc='upper left') # save and close _finalize_plot(plot, ax, outfile, **axargs)
def spectral_overlay(gps, qspecgram, fringe, output, multipliers=(1, 2, 4, 8), figsize=[12, 4]): """Overlay scattering fringe projections on top of a high-resolution spectrogram Parameters ---------- gps : `float` reference GPS time (in seconds) to serve as the origin qspecgram : `~gwpy.spectrogram.Spectrogram` an interpolated high-resolution spectrogram fringe : `~gwpy.timeseries.TimeSeries` projected fringe frequencies (in Hz) output : `str` name of the output file multipliers : `tuple`, optional collection of fringe harmonic numbers to plot, can be given in any order, default: `(1, 2, 4, 8)` figsize : `tuple`, optional size (width x height) of the final figure, default: `(12, 4)` """ plot = Plot(figsize=figsize) ax = plot.gca() # format spectrogram plot ax.set_title('Fringes: {0}, Spectrogram: {1}'.format( fringe.name, qspecgram.name)) _format_spectrogram(ax, qspecgram, colormap='binary') # overlay fringe frequencies _format_timeseries(ax, gps, fringe, multipliers=multipliers, linewidth=1.5) ax.set_ylim([qspecgram.f0.to('Hz').value, qspecgram.frequencies.max().to('Hz').value]) # save plot and close plot.savefig(output, bbox_inches='tight') plot.close()
def plot_asd(data,replot=False,fftlength=2**7,show=False,**kwargs): if isinstance(data,TimeSeries): chname = data.name psd_specgram = data.spectrogram2(fftlength=fftlength, overlap=fftlength/2.0, window='hanning') elif isinstance(data,Spectrogram): chname = data.name specgram = data pngfname = to_pngfname(chname,ftype='ASD',**kwargs) if not replot and os.path.exists(pngfname): print('Skip plot {0}'.format(pngfname)) return None median = specgram.percentile(50) low = specgram.percentile(5) high = specgram.percentile(95) median = vel2vel(median) low = vel2vel(low) high = vel2vel(high) _f, _selfnoise = trillium.selfnoise(trillium='120QA',psd='ASD',unit='velo') _selfnoise = _selfnoise*1e6 plot = Plot() ax = plot.gca(xscale='log', xlim=(1e-3, 3e2), xlabel='Frequency [Hz]', #yscale='log', ylim=(1e-11, 3e-6), yscale='log', ylim=(1e-5, 3e-0), ylabel=r'Velocity [m/sec/\rtHz]') ax.plot(_f,_selfnoise,'-',linewidth=1,color='gray') ax.plot_mmm(median, low, high, color='gwpy:ligo-livingston') ax.set_title(chname.replace('_',' '),fontsize=16) ax.legend(labels=['Selfnoise','Measurement']) plot.savefig(pngfname) print('plot in ',pngfname) return plot
def before_after_histogram( outfile, x, y, label1='Before', label2='After', bins=100, histtype='stepfilled', range=None, figsize=[9, 6], **kwargs): """Plot a histogram of SNR for two event distributions """ # format axis arguments axargs = { 'xscale': 'log', 'xlabel': 'Loudness', 'yscale': 'log', 'ylabel': 'Number of events', } axargs.update(kwargs) # create figure plot = Plot(figsize=figsize) ax = plot.gca() # make histogram if range is None: range = min(map(numpy.min, (x, y))), max(map(numpy.max, (x, y))) axargs.setdefault('xlim', range) histargs = { 'range': range, 'histtype': histtype, 'bins': bins, 'linewidth': 2, 'logbins': axargs['xscale'] == 'log', 'alpha': .8, } ax.hist(x, label=label1, facecolor='red', edgecolor='darkred', **histargs) ax.hist(y, label=label2, facecolor='dodgerblue', edgecolor='blue', **histargs) # add legend ax.legend(loc='upper right') # format axes axargs.setdefault('ylim', (.5, ax.yaxis.get_data_interval()[1] * 1.05)) _finalize_plot(plot, ax, outfile, **axargs)
from gwpy.timeseries import TimeSeries from gwpy.plot import Plot h1 = TimeSeries.fetch_open_data('H1', 1126259457, 1126259467) h1b = h1.bandpass(50, 250).notch(60).notch(120) l1 = TimeSeries.fetch_open_data('L1', 1126259457, 1126259467) l1b = l1.bandpass(50, 250).notch(60).notch(120) plot = Plot(figsize=(12, 4.8)) ax = plot.gca(xscale='auto-gps') ax.plot(h1b, color='gwpy:ligo-hanford', label='LIGO-Hanford') ax.plot(l1b, color='gwpy:ligo-livingston', label='LIGO-Livingston') ax.set_epoch(1126259462.427) ax.set_xlim(1126259462, 1126259462.6) ax.set_ylim(-1e-21, 1e-21) ax.set_ylabel('Strain noise') ax.legend() plot.show()
def _process_channel(input_): """Handle individual channels for multiprocessing """ if USETEX: gwplot.configure_mpl_tex() p4 = (.1, .1, .9, .95) chan = input_[1][0] ts = input_[1][1] lassocoef = nonzerocoef[chan] zeroed = lassocoef == 0 if zeroed: plot4 = None plot5 = None plot6 = None pcorr = None else: plot4 = None plot5 = None plot6 = None if trend_type == 'minute': pcorr = numpy.corrcoef(ts.value, primaryts.value)[0, 1] else: pcorr = 0.0 if abs(lassocoef) < threshold: with counter.get_lock(): counter.value += 1 pc = 100 * counter.value / len(nonzerodata) LOGGER.info( "Completed [%d/%d] %3d%% %-50s" % (counter.value, len(nonzerodata), pc, '(%s)' % str(chan))) sys.stdout.flush() return (chan, lassocoef, plot4, plot5, plot6, ts) # create time series subplots fig = Plot(figsize=(12, 8)) fig.subplots_adjust(*p4) ax1 = fig.add_subplot(2, 1, 1, xscale='auto-gps', epoch=start) ax1.plot(primaryts, label=texify(primary), color='black', linewidth=line_size_primary) ax1.set_xlabel(None) ax2 = fig.add_subplot(2, 1, 2, sharex=ax1, xlim=xlim) ax2.plot(ts, label=texify(chan), linewidth=line_size_aux) if range_is_primary: ax1.set_ylabel('Sensitive range [Mpc]') else: ax1.set_ylabel('Primary channel units') ax2.set_ylabel('Channel units') for ax in fig.axes: ax.legend(loc='best') channelstub = re_delim.sub('_', str(chan)).replace('_', '-', 1) plot4 = gwplot.save_figure(fig, '%s_TRENDS-%s.png' % (channelstub, gpsstub), bbox_inches='tight') # create scaled, sign-corrected, and overlayed timeseries tsscaled = scale(ts.value) if lassocoef < 0: tsscaled = numpy.negative(tsscaled) fig = Plot(figsize=(12, 4)) fig.subplots_adjust(*p1) ax = fig.gca(xscale='auto-gps', epoch=start, xlim=xlim) ax.plot(times, _descaler(target), label=texify(primary), color='black', linewidth=line_size_primary) ax.plot(times, _descaler(tsscaled), label=texify(chan), linewidth=line_size_aux) if range_is_primary: ax.set_ylabel('Sensitive range [Mpc]') else: ax.set_ylabel('Primary Channel Units') ax.legend(loc='best') plot5 = gwplot.save_figure(fig, '%s_COMPARISON-%s.png' % (channelstub, gpsstub), bbox_inches='tight') # scatter plot tsCopy = ts.value.reshape(-1, 1) primarytsCopy = primaryts.value.reshape(-1, 1) primaryReg = linear_model.LinearRegression() primaryReg.fit(tsCopy, primarytsCopy) primaryFit = primaryReg.predict(tsCopy) fig = Plot(figsize=(12, 4)) fig.subplots_adjust(*p1) ax = fig.gca() ax.set_xlabel(texify(chan) + ' [Channel units]') if range_is_primary: ax.set_ylabel('Sensitive range [Mpc]') else: ax.set_ylabel('Primary channel units') y_min = min(primaryts.value) y_max = max(primaryts.value) y_range = y_max - y_min ax.set_ylim(y_min - (y_range * 0.1), y_max + (y_range * 0.1)) ax.text(.9, .1, 'r = ' + str('{0:.2}'.format(pcorr)), verticalalignment='bottom', horizontalalignment='right', transform=ax.transAxes, color='black', size=20, bbox=dict(boxstyle='square', facecolor='white', alpha=.75, edgecolor='black')) ax.scatter(ts.value, primaryts.value, color='red') ax.plot(ts.value, primaryFit, color='black') ax.autoscale_view(tight=False, scalex=True, scaley=True) plot6 = gwplot.save_figure(fig, '%s_SCATTER-%s.png' % (channelstub, gpsstub), bbox_inches='tight') # increment counter and print status with counter.get_lock(): counter.value += 1 pc = 100 * counter.value / len(nonzerodata) LOGGER.info("Completed [%d/%d] %3d%% %-50s" % (counter.value, len(nonzerodata), pc, '(%s)' % str(chan))) sys.stdout.flush() return (chan, lassocoef, plot4, plot5, plot6, ts)
def main(args=None): use('agg') rcParams.update({ 'figure.subplot.bottom': 0.15, 'figure.subplot.left': 0.1, 'figure.subplot.right': 0.83, 'figure.subplot.top': 0.93, 'figure.subplot.hspace': 0.25, 'axes.labelsize': 20, 'grid.color': 'gray', }) grid = GridSpec(2, 1) logger = log.Logger('omicron-status') try: omicronversion = str(get_omicron_version()) except KeyError: omicronversion = 'Unknown' logger.warning("Omicron version unknown") else: logger.info("Found omicron version: %s" % omicronversion) parser = create_parser() args = parser.parse_args(args=args) if args.ifo is None: parser.error("Cannot determine IFO prefix from sytem, " "please pass --ifo on the command line") group = args.group logger.info("Checking status for %r group" % group) archive = args.archive_directory proddir = args.production_directory.with_name( args.production_directory.name.format(group=args.group), ) outdir = args.output_directory outdir.mkdir(exist_ok=True, parents=True) tag = args.latency_archive_tag.format(group=args.group) filetypes = ['h5', 'xml.gz', 'root'] logger.debug("Set output directory to %s" % outdir) logger.debug( "Will process the following filetypes: {}".format( ", ".join(filetypes), ), ) # -- parse configuration file and get parameters -------------------------- cp = configparser.ConfigParser() ok = cp.read(args.config_file) if args.config_file not in ok: raise IOError( "Failed to read configuration file %r" % args.config_file, ) logger.info("Configuration read") # validate if not cp.has_section(group): raise configparser.NoSectionError(group) # get parameters obs = args.ifo[0] frametype = cp.get(group, 'frametype') padding = cp.getint(group, 'overlap-duration') / 2. mingap = cp.getint(group, 'chunk-duration') channels = args.channel if not channels: channels = [ c.split()[0] for c in cp.get(group, 'channels').strip('\n').split('\n') ] channels.sort() logger.debug("Found %d channels" % len(channels)) start = args.gps_start_time end = args.gps_end_time if end == NOW: end -= padding if args.state_flag: stateflag = args.state_flag statepad = tuple(map(float, args.state_pad.split(','))) else: try: stateflag = cp.get(group, 'state-flag') except configparser.NoOptionError: stateflag = None else: try: statepad = tuple( map( float, cp.get(group, 'state-padding').split(','), )) except configparser.NoOptionError: statepad = (0, 0) if stateflag: logger.debug("Parsed state flag: %r" % stateflag) logger.debug("Parsed state padding: %s" % repr(statepad)) logger.info("Processing %d-%d" % (start, end)) # -- define nagios JSON printer ------------------------------------------- def print_nagios_json(code, message, outfile, tag='status', **extras): out = { 'created_gps': NOW, 'status_intervals': [ { 'start_sec': 0, 'end_sec': args.unknown, 'num_status': code, 'txt_status': message }, { 'start_sec': args.unknown, 'num_status': 3, 'txt_status': 'Omicron %s check is not running' % tag }, ], 'author': { 'name': 'Duncan Macleod', 'email': '*****@*****.**', }, 'omicron': { 'version': omicronversion, 'group': group, 'channels': ' '.join(channels), 'frametype': frametype, 'state-flag': stateflag, }, 'pyomicron': { 'version': __version__, }, } out.update(extras) with open(outfile, 'w') as f: f.write(json.dumps(out)) logger.debug("nagios info written to %s" % outfile) # -- get condor status ------------------------------------------------ if not args.skip_condor: # connect to scheduler try: schedd = htcondor.Schedd() except RuntimeError as e: logger.warning("Caught %s: %s" % (type(e).__name__, e)) logger.info("Failed to connect to HTCondor scheduler, cannot " "determine condor status for %s" % group) schedd = None if not args.skip_condor and schedd: logger.info("-- Checking condor status --") # get DAG status jsonfp = outdir / "nagios-condor-{}.json".format(group) okstates = ['Running', 'Idle', 'Completed'] try: # check manager status qstr = 'OmicronManager == "{}" && Owner == "{}"'.format( group, args.user, ) try: jobs = schedd.query(qstr, ['JobStatus']) except IOError as e: warnings.warn("Caught IOError: %s [retrying...]" % str(e)) sleep(2) jobs = schedd.query(qstr, ['JobStatus']) logger.debug( "Found {} jobs for query {!r}".format(len(jobs), qstr), ) if len(jobs) > 1: raise RuntimeError( "Multiple OmicronManager jobs found for %r" % group) elif len(jobs) == 0: raise RuntimeError( "No OmicronManager job found for %r" % group, ) status = condor.JOB_STATUS[jobs[0]['JobStatus']] if status not in okstates: raise RuntimeError("OmicronManager status for %r: %r" % (group, status)) logger.debug("Manager status is %r" % status) # check node status jobs = schedd.query( 'OmicronProcess == "{}" && Owner == "{}"'.format( group, args.user, ), ['JobStatus', 'ClusterId'], ) logger.debug( "Found {} jobs for query {!r}".format(len(jobs), qstr), ) for job in jobs: status = condor.JOB_STATUS[job['JobStatus']] if status not in okstates: raise RuntimeError("Omicron node %s (%r) is %r" % (job['ClusterId'], group, status)) except RuntimeError as e: print_nagios_json(2, str(e), jsonfp, tag='condor') logger.warning("Failed to determine condor status: %r" % str(e)) except IOError as e: logger.warning("Caught %s: %s" % (type(e).__name__, e)) logger.info("Failed to connect to HTCondor scheduler, cannot " "determine condor status for %s" % group) else: print_nagios_json( 0, "Condor processing for %r is OK" % group, jsonfp, tag='condor', ) logger.info("Condor processing is OK") if not args.skip_job_duration: # get job duration history plot = Plot(figsize=[12, 3]) plot.subplots_adjust(bottom=.22, top=.87) ax = plot.gca(xscale="auto-gps") times, jobdur = condor.get_job_duration_history_shell('OmicronProcess', group, maxjobs=5000) logger.debug("Recovered duration history for %d omicron.exe jobs" % len(times)) line = ax.plot([0], [1], label='Omicron.exe')[0] ax.plot(times, jobdur, linestyle=' ', marker='.', color=line.get_color()) times, jobdur = condor.get_job_duration_history_shell( 'OmicronPostProcess', group, maxjobs=5000) logger.debug("Recovered duration history for %d post-processing jobs" % len(times)) line = ax.plot([0], [1], label='Post-processing')[0] ax.plot(times, jobdur, linestyle=' ', marker='.', color=line.get_color()) ax.legend(loc='upper left', borderaxespad=0, bbox_to_anchor=(1.01, 1), handlelength=1) ax.set_xlim(args.gps_start_time, args.gps_end_time) ax.set_epoch(ax.get_xlim()[1]) ax.set_yscale('log') ax.set_title('Omicron job durations for %r' % group) ax.set_ylabel('Job duration [seconds]') ax.xaxis.labelpad = 5 png = str(outdir / "nagios-condor-{}.png".format(group)) plot.save(png) plot.close() logger.debug("Saved condor plot to %s" % png) if args.skip_file_checks: sys.exit(0) # -- get file latency and archive completeness ---------------------------- logger.info("-- Checking file archive --") # get frame segments segs = segments.get_frame_segments(obs, frametype, start, end) # get state segments if stateflag is not None: segs &= segments.query_state_segments( stateflag, start, end, pad=statepad, ) try: end = segs[-1][1] except IndexError: pass # apply inwards padding to generate resolvable segments for i in range(len(segs) - 1, -1, -1): # if segment is shorter than padding, ignore it completely if abs(segs[i]) <= padding * 2: del segs[i] # otherwise apply padding to generate trigger segment else: segs[i] = segs[i].contract(padding) logger.debug("Found %d seconds of analysable time" % abs(segs)) # load archive latency latencyfile = outdir / "nagios-latency-{}.h5".format(tag) times = dict((c, dict((ft, None) for ft in filetypes)) for c in channels) ldata = dict((c, dict((ft, None) for ft in filetypes)) for c in channels) try: with h5py.File(latencyfile, 'r') as h5file: for c in channels: for ft in filetypes: try: times[c][ft] = h5file[c]['time'][ft][:] ldata[c][ft] = h5file[c]['latency'][ft][:] except KeyError: times[c][ft] = numpy.ndarray((0, )) ldata[c][ft] = numpy.ndarray((0, )) except OSError as exc: # file not found, or is corrupt warnings.warn("failed to load latency data from {}: {}".format( latencyfile, str(exc), )) for c in channels: for ft in filetypes: if not times[c].get(ft): times[c][ft] = numpy.ndarray((0, )) ldata[c][ft] = numpy.ndarray((0, )) else: logger.debug("Parsed latency data from %s" % latencyfile) # load acknowledged gaps acksegfile = str(outdir / "acknowledged-gaps-{}.txt".format(tag)) try: acknowledged = SegmentList.read(acksegfile, gpstype=float, format="segwizard") except IOError: # no file acknowledged = SegmentList() else: logger.debug( "Read %d segments from %s" % (len(acknowledged), acksegfile), ) acknowledged.coalesce() # build legend for segments leg = OrderedDict() leg['Analysable'] = SegmentRectangle( [0, 1], 0, facecolor='lightgray', edgecolor='gray', ) leg['Available'] = SegmentRectangle( [0, 1], 0, facecolor='lightgreen', edgecolor='green', ) leg['Missing'] = SegmentRectangle( [0, 1], 0, facecolor='red', edgecolor='darkred', ) leg['Unresolvable'] = SegmentRectangle( [0, 1], 0, facecolor='magenta', edgecolor='purple', ) leg['Overlapping'] = SegmentRectangle( [0, 1], 0, facecolor='yellow', edgecolor='orange', ) leg['Pending'] = SegmentRectangle( [0, 1], 0, facecolor='lightskyblue', edgecolor='blue', ) leg['Acknowledged'] = SegmentRectangle( [0, 1], 0, facecolor='sandybrown', edgecolor='brown', ) logger.debug("Checking archive for each channel...") # find files latency = {} gaps = {} overlap = {} pending = {} plots = {} for c in channels: # create data storate latency[c] = {} gaps[c] = {} overlap[c] = {} pending[c] = {} # create figure plot = Plot(figsize=[12, 5]) lax = plot.add_subplot(grid[0, 0], xscale="auto-gps") sax = plot.add_subplot(grid[1, 0], sharex=lax, projection='segments') colors = ['lightblue', 'dodgerblue', 'black'] for y, ft in enumerate(filetypes): # find files cache = io.find_omicron_files(c, start, end, archive, ext=ft) cpend = sieve_cache(io.find_pending_files(c, proddir, ext=ft), segment=Segment(start, end)) # get available segments avail = segments.cache_segments(cache) found = avail & segs pending[c][ft] = segments.cache_segments(cpend) & segs # remove gaps at the end that represent latency try: latency[c][ft] = abs(segs & type( segs)([type(segs[0])(found[-1][1], segs[-1][1])])) / 3600. except IndexError: latency[c][ft] = 0 processed = segs else: processed = segs & type(segs)( [type(segs[0])(start, found[-1][1])]) gaps[c][ft] = type(found)() lost = type(found)() for s in processed - found: if abs(s) < mingap and s in list(segs): lost.append(s) else: gaps[c][ft].append(s) # remove acknowledged gaps ack = gaps[c][ft] & acknowledged gaps[c][ft] -= acknowledged # print warnings if abs(gaps[c][ft]): warnings.warn("Gaps found in %s files for %s:\n%s" % (c, ft, gaps[c][ft])) overlap[c][ft] = segments.cache_overlaps(cache) if abs(overlap[c][ft]): warnings.warn("Overlap found in %s files for %s:\n%s" % (c, ft, overlap[c][ft])) # append archive times[c][ft] = numpy.concatenate((times[c][ft][-99999:], [NOW])) ldata[c][ft] = numpy.concatenate( (ldata[c][ft][-99999:], [latency[c][ft]])) # plot line = lax.plot( times[c][ft], ldata[c][ft], label=ft, color=colors[y], )[0] lax.plot(times[c][ft], ldata[c][ft], marker='.', linestyle=' ', color=line.get_color()) sax.plot_segmentlist(segs, y=y, label=ft, alpha=.5, facecolor=leg['Analysable'].get_facecolor(), edgecolor=leg['Analysable'].get_edgecolor()) sax.plot_segmentlist(pending[c][ft], y=y, facecolor=leg['Pending'].get_facecolor(), edgecolor=leg['Pending'].get_edgecolor()) sax.plot_segmentlist(avail, y=y, label=ft, alpha=.2, height=.1, facecolor=leg['Available'].get_facecolor(), edgecolor=leg['Available'].get_edgecolor()) sax.plot_segmentlist(found, y=y, label=ft, alpha=.5, facecolor=leg['Available'].get_facecolor(), edgecolor=leg['Available'].get_edgecolor()) sax.plot_segmentlist(lost, y=y, facecolor=leg['Unresolvable'].get_facecolor(), edgecolor=leg['Unresolvable'].get_edgecolor()) sax.plot_segmentlist(gaps[c][ft], y=y, facecolor=leg['Missing'].get_facecolor(), edgecolor=leg['Missing'].get_edgecolor()) sax.plot_segmentlist(overlap[c][ft], y=y, facecolor=leg['Overlapping'].get_facecolor(), edgecolor=leg['Overlapping'].get_edgecolor()) sax.plot_segmentlist(ack, y=y, facecolor=leg['Acknowledged'].get_facecolor(), edgecolor=leg['Acknowledged'].get_edgecolor()) # finalise plot lax.axhline(args.warning / 3600., color=(1.0, 0.7, 0.0), linestyle='--', linewidth=2, label='Warning', zorder=-1) lax.axhline(args.error / 3600., color='red', linestyle='--', linewidth=2, label='Critical', zorder=-1) lax.set_title('Omicron status: {}'.format(c)) lax.set_ylim(0, args.error / 1800.) lax.set_ylabel('Latency [hours]') lax.legend(loc='upper left', bbox_to_anchor=(1.01, 1), borderaxespad=0, handlelength=2, fontsize=12.4) lax.set_xlabel(' ') for ax in plot.axes: ax.set_xlim(args.gps_start_time, args.gps_end_time) ax.set_epoch(ax.get_xlim()[1]) sax.xaxis.labelpad = 5 sax.set_ylim(-.5, len(filetypes) - .5) sax.legend(leg.values(), leg.keys(), handlelength=1, fontsize=12.4, loc='lower left', bbox_to_anchor=(1.01, 0), borderaxespad=0) plots[c] = png = outdir / "nagios-latency-{}.png".format( c.replace(':', '-'), ) plot.save(png) plot.close() logger.debug(" %s" % c) # update latency and write archive h5file = h5py.File(latencyfile, 'w') for c in channels: g = h5file.create_group(c) for name, d in zip(['time', 'latency'], [times[c], ldata[c]]): g2 = g.create_group(name) for ft in filetypes: g2.create_dataset(ft, data=d[ft], compression='gzip') h5file.close() logger.debug("Stored latency data as HDF in %s" % latencyfile) # write nagios output for files status = [] for segset, tag in zip([gaps, overlap], ['gaps', 'overlap']): chans = [(c, segset[c]) for c in segset if abs(reduce(operator.or_, segset[c].values()))] jsonfp = outdir / "nagios-{}-{}.json".format(tag, group) status.append((tag, jsonfp)) if chans: gapstr = '\n'.join('%s: %s' % c for c in chans) code = 1 message = ("%s found in Omicron files for group %r\n%s" % (tag.title(), group, gapstr)) else: code = 0 message = ("No %s found in Omicron files for group %r" % (tag, group)) print_nagios_json(code, message, jsonfp, tag=tag, **{tag: dict(chans)}) # write group JSON jsonfp = outdir / "nagios-latency-{}.json".format(group) status.append(('latency', jsonfp)) code = 0 message = 'No channels have high latency for group %r' % group ldict = dict((c, max(latency[c].values())) for c in latency) for x, dt in zip([2, 1], [args.error, args.warning]): dh = dt / 3600. chans = [c for c in ldict if ldict[c] >= dh] if chans: code = x message = ( "%d channels found with high latency (above %s seconds)" % (len(chans), dt)) break print_nagios_json(code, message, jsonfp, tag='latency', latency=ldict) # auto-detect 'standard' JSON files for tag, name in zip( ['condor', 'omicron-online'], ['condor', 'processing'], ): f = outdir / "nagios-{}-{}.json".format(tag, group) if f.is_file(): status.insert(0, (name, f)) # write HTML summary if args.html: page = markup.page() page.init( title="%s Omicron Online status" % group, css=[ ('//maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/' 'bootstrap.min.css'), ('//cdnjs.cloudflare.com/ajax/libs/fancybox/2.1.5/' 'jquery.fancybox.min.css'), ], script=[ '//code.jquery.com/jquery-1.11.2.min.js', ('//maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/' 'bootstrap.min.js'), ('//cdnjs.cloudflare.com/ajax/libs/fancybox/2.1.5/' 'jquery.fancybox.min.js'), ], ) page.div(class_='container') # write header page.div(class_='page-header') page.h1('Omicron Online status: %s' % group) page.div.close() # page-header # write summary page.div(id_='json') page.h2("Processing status") for tag, f in status: jf = f.name page.a("%s status" % tag.title(), href=jf, role='button', target="_blank", id_="nagios-%s" % tag, class_='btn btn-default json-status') page.p(style="padding-top: 5px;") page.small( "Hover over button for explanation, click to open JSON file", ) page.p.close() page.div.close() # id=json # show plots page.div(id_='plots') page.h2("Channel details") page.div(class_='row') for channel in sorted(channels): png = plots[channel].name page.div(class_="col-sm-6 col-md-4") page.div(class_="panel panel-default") page.div(class_='panel-heading') page.h3(channel, class_='panel-title', style="font-size: 14px;") page.div.close() # panel-heading page.div(class_='panel-body') page.a(href=png, target="_blank", class_="fancybox", rel="channel-status-img") page.img(src=png, class_='img-responsive') page.a.close() page.div.close() # panel-body page.div.close() # panel page.div.close() # col page.div.close() # row page.div.close() # id=plots # dump parameters page.div(id_="parameters") page.h2("Parameters") for key, val in cp.items(group): page.p() page.strong("%s:" % key) page.add(val) page.p.close() page.div.close() # id=parameters # finish and close page.div.close() # container page.script(""" function setStatus(data, id) { var txt = data.status_intervals[0].txt_status.split("\\n")[0]; $("#"+id).attr("title", txt); var stat = data.status_intervals[0].num_status; if (stat == 0) { $("#"+id).addClass("btn-success"); } else if (stat == 1) { $("#"+id).addClass("btn-warning"); } else if (stat == 2){ $("#"+id).addClass("btn-danger"); } } $(document).ready(function() { $(".json-status").each(function() { var jsonf = $(this).attr("href"); var id = $(this).attr("id"); $.getJSON(jsonf, function(data) { setStatus(data, id); }); }); $(".fancybox").fancybox({nextEffect: 'none', prevEffect: 'none'}); });""", type="text/javascript") with (outdir / "index.html").open("w") as f: f.write(str(page)) logger.debug("HTML summary written to %s" % f.name)
from gwpy.plot import Plot plot = Plot() ax = plot.gca(xscale='log', xlim=(10, 1500), xlabel='Frequency [Hz]', yscale='log', ylim=(3e-24, 2e-20), ylabel=r'Strain noise [1/$\sqrt{\mathrm{Hz}}$]') ax.plot_mmm(median, low, high, color='gwpy:ligo-hanford') ax.set_title('LIGO-Hanford strain noise variation around GW170817', fontsize=16) plot.show()
# Calc Spectrum with percentile pdp = ppol.spectrogram2(fftlength=2, overlap=1, window='hanning')**(1 / 2.) pdp_m = pdp.percentile(50) pdp_l = pdp.percentile(5) pdp_h = pdp.percentile(95) pds = spol.spectrogram2(fftlength=2, overlap=1, window='hanning')**(1 / 2.) pds_m = pds.percentile(50) pds_l = pds.percentile(5) pds_h = pds.percentile(95) # Plot asd graph from gwpy.plot import Plot plot = Plot() ax = plot.gca( xscale='log', xlim=(1, 50000), xlabel='Frequency [Hz]', yscale='log', #ylim=(3e-24, 2e-20), ylabel=r'Voltage [V/\rtHz]') ax.plot_mmm(pdp_m, pdp_l, pdp_h, color='gwpy:ligo-hanford', label='P-polarized signal') ax.plot_mmm(pds_m, pds_l, pds_h, color='gwpy:ligo-livingston', label='S-polarized signal') ax.set_title('PD voltage', fontsize=16) ax.legend() plot.savefig('asd.png')
nproc=2, window='hanning')**(1 / 2.) print('fft done') # percentile median = sg.percentile(50) low = sg.percentile(5) high = sg.percentile(95) print('percentile done') # plot TimeSeries plot = data.plot() plot.savefig('./img_timeseries.png') plot.close() # plot Spectrum plot = Plot() ax = plot.gca( xscale='log', xlim=(1e-3, 200), xlabel='Frequency [Hz]', yscale='log', # ylim=(3e-24, 2e-20), ylabel=r'Ground Velocity [um/sec/\rtHz]') ax.plot_mmm(median, low, high, color='gwpy:ligo-hanford') ax.set_title('No title', fontsize=16) plot.savefig('./img_asd.png') # write data low.write('data1_exv_x_5pct.hdf5', format='hdf5', overwrite=True) median.write('data1_exv_x_50pct.hdf5', format='hdf5', overwrite=True) high.write('data1_exv_x_95pct.hdf5', format='hdf5', overwrite=True)
# Then we can download a simulation of the GW150914 signal from GWOSC: from astropy.utils.data import get_readable_fileobj url = ("https://www.gw-openscience.org/s/events/GW150914/P150914/" "fig2-unfiltered-waveform-H.txt") with get_readable_fileobj(url) as f: signal = TimeSeries.read(f, format='txt') signal.t0 = .5 # make sure this intersects with noise time samples # Note, since this simulation cuts off before a certain time, it is # important to taper its ends to zero to avoid ringing artifacts. # We can accomplish this using the # :meth:`~gwpy.timeseries.TimeSeries.taper` method. signal = signal.taper() # Since the time samples overlap, we can inject this into our noise data # using :meth:`~gwpy.types.series.Series.inject`: data = noise.inject(signal) # Finally, we can visualize the full process in the time domain: from gwpy.plot import Plot plot = Plot(noise, signal, data, separate=True, sharex=True, sharey=True) plot.gca().set_epoch(0) plot.show() # We can clearly see that the loud GW150914-like signal has been layered # on top of Gaussian noise with the correct amplitude and phase evolution.
def zoom(ts: [TimeSeries], t, dt=0.5 * u.s): print(t) plot2 = Plot(ts) ax = plot2.gca() ax.set_xlim(t - dt, t + dt) plot2.show()
def _generate_cluster(input_): """Generate cluster data for use below """ if USETEX: gwplot.configure_mpl_tex() currentchan = input_[1][0] currentts = input_[1][5] current = input_[0] p7 = (.135, .15, .95, .9) plot7 = None plot7_list = None if current < len(nonzerodata): cluster = [] for i, otheritem in enumerate(list(auxdata.items())): chan_, ts_ = otheritem if chan_ != currentchan: pcorr = numpy.corrcoef(currentts.value, ts_.value)[0, 1] if abs(pcorr) >= cluster_threshold: stub = re_delim.sub('_', chan_).replace('_', '-', 1) cluster.append([i, ts_, pcorr, chan_, stub]) if cluster: # write cluster table to file cluster = sorted(cluster, key=lambda x: abs(x[2]), reverse=True) clustertab = Table(data=list(zip(*cluster))[2:4], names=('Pearson Coefficient', 'Channel')) plot7_list = '%s_CLUSTER_LIST-%s.csv' % (re_delim.sub( '_', str(currentchan)).replace('_', '-', 1), gpsstub) clustertab.write(plot7_list, format='csv', overwrite=True) ncluster = min(len(cluster), max_correlated_channels) colors2 = [cmap(i) for i in numpy.linspace(0, 1, ncluster + 1)] # plot fig = Plot(figsize=(12, 4)) fig.subplots_adjust(*p7) ax = fig.gca(xscale='auto-gps') ax.plot(times, scale(currentts.value) * numpy.sign(input_[1][1]), label=texify(currentchan), linewidth=line_size_aux, color=colors[0]) for i in range(0, ncluster): this = cluster[i] ax.plot( times, scale(this[1].value) * numpy.sign(input_[1][1]) * numpy.sign(this[2]), color=colors2[i + 1], linewidth=line_size_aux, label=('{0}, r = {1:.2}'.format(texify(cluster[i][3]), cluster[i][2])), ) ax.margins(x=0) ax.set_ylabel('Scaled amplitude [arbitrary units]') ax.set_title('Highly Correlated Channels') ax.legend(loc='center left', bbox_to_anchor=(1.05, 0.5)) plot7 = gwplot.save_figure( fig, '%s_CLUSTER-%s.png' % (re_delim.sub( '_', str(currentchan)).replace('_', '-', 1), gpsstub), bbox_inches='tight') with counter.get_lock(): counter.value += 1 pc = 100 * counter.value / len(nonzerodata) LOGGER.info( "Completed [%d/%d] %3d%% %-50s" % (counter.value, len(nonzerodata), pc, '(%s)' % str(currentchan))) sys.stdout.flush() return (plot7, plot7_list)
def representative_spectra(channels, start, stop, rate, label='kmeans-labels', filename=DEFAULT_FILENAME, prefix='.', downloader=TimeSeriesDict.get, cluster_numbers=None, groups=None, **kwargs): """ Make representative spectra for each cluster based on the median psd for minutes in that cluster. Downloads only the raw minutes in the cluster to save. """ if groups is None: groups = channels # read the labels from the save file. labels = TimeSeries.read(filename, label, start=to_gps(start), end=to_gps(stop)) logger.info(f'Read labels {start} to {stop} from {filename}') if cluster_numbers is None: clusters = list(range(max(labels.value) + 1)) cluster_counts = list( len(labels.value[labels.value == c]) for c in clusters) largest_cluster = cluster_counts.index(max(cluster_counts)) clusters.remove(largest_cluster) logger.info( f'Largest cluster found to be Nº{largest_cluster} ({100 * max(cluster_counts) // len(labels.value)}%). Doing {clusters}.' ) cluster_counts.remove(max(cluster_counts)) else: clusters = cluster_numbers cluster_counts = list( len(labels.value[labels.value == c]) for c in clusters) t, v, d = labels.times, labels.value, diff(labels.value) pairs = list( zip([t[0]] + list(t[:-1][d != 0]), list(t[1:][d != 0]) + [t[-1]])) values = list(v[:-1][d != 0]) + [v[-1]] assert len(pairs) == len(values) # need to include start-| and |-end # l|r l|r l|r l|r # l,r l,r l,r l,r # l r,l r,l r,l r # zip(start + l[1:], r[:-1] + stop) print(pairs) for pair in pairs: print(int(pair[1].value) - int(pair[0].value)) print(values) # use h5py to make a mutable object pointing to a file on disk. save_file, filename = path2h5file( get_path(f'spectra-cache {start}', 'hdf5', prefix=prefix)) logger.debug(f'Initiated hdf5 stream to {filename}') logger.info(f'Patching {filename}...') for i, (dl_start, end) in enumerate(pairs): if values[i] in clusters: if not data_exists(channels, to_gps(end).seconds, save_file): logger.debug( f'Downloading Nº{values[i]} from {dl_start} to {end}...') try: dl = downloader(channels, start=to_gps(dl_start) - LIGOTimeGPS(60), end=to_gps(end) + LIGOTimeGPS(seconds=1)) out = TimeSeriesDict() for n in dl: out[n] = dl[n].resample(**better_aa_opts(dl[n], rate)) write_to_disk(out, to_gps(dl_start).seconds, save_file) except RuntimeError: # Cannot find all relevant data on any known server logger.warning( f"SKIPPING Nº{values[i]} from {dl_start} to {end} !!") logger.info('Reading data...') data = TimeSeriesDict.read(save_file, channels) logger.info('Starting PSD generation...') f = data[channels[0]].crop( start=to_gps(data[channels[0]].times[-1]) - LIGOTimeGPS(60), end=to_gps(data[channels[0]].times[-1])).psd().frequencies d = (to_gps(labels.times[-1]).seconds - to_gps(labels.times[1]).seconds) for i, cluster in enumerate(clusters): try: psds = { channel: FrequencySeries.read(filename, f'{cluster}-{channel}') for channel in channels } logger.info(f'Loaded Nº{cluster}.') except KeyError: logger.info( f'Doing Nº{cluster} ({100 * cluster_counts[i] / len(labels.value):.2f}% of data)...' ) with Progress(f'psd Nº{cluster} ({i + 1}/{len(clusters)})', len(channels) * d) as progress: psds = { channel: FrequencySeries(median(stack([ progress(data[channel].crop, pc * d + (to_gps(time).seconds - to_gps(labels.times[1]).seconds), start=to_gps(time) - LIGOTimeGPS(60), end=to_gps(time)).psd().value for c, time in zip(labels.value, labels.times) if c == cluster ]), axis=0), frequencies=f, name=f'{cluster}-{channel}') for pc, channel in enumerate(channels) } for name in psds.keys(): psds[name].write(filename, **writing_opts) # plotting is slow, so show a nice progress bar. logger.debug('Initiating plotting routine...') with Progress('plotting', len(groups)) as progress: for p, (group, lbls, title) in enumerate(groups): # plot the group in one figure. plt = Plot(*(psds[channel] for channel in group), separate=False, sharex=True, zorder=1, **kwargs) # plt.gca().set_xlim((30,60)) # modify the figure as a whole. # plt.add_segments_bar(dq, label='') plt.gca().set_xscale('log') plt.gca().set_yscale('log') plt.suptitle(title) plt.legend(lbls) # save to png. progress( plt.save, p, get_path(f'{cluster}-{title}', 'png', prefix=f'{prefix}/{cluster}'))
# a number of ASDs, using the :meth:`~gwpy.timeseries.TimeSeries.spectrogram2` # method: sg = hoft.spectrogram2(fftlength=4, overlap=2, window='hanning')**(1 / 2.) # From this we can trivially extract the median, 5th and 95th percentiles: median = sg.percentile(50) low = sg.percentile(5) high = sg.percentile(95) # Finally, we can make plot, using :meth:`~gwpy.plot.Axes.plot_mmm` to # display the 5th and 95th percentiles as a shaded region around the median: from gwpy.plot import Plot plot = Plot() ax = plot.gca(xscale='log', xlim=(10, 1500), xlabel='Frequency [Hz]', yscale='log', ylim=(3e-24, 2e-20), ylabel=r'Strain noise [1/\rtHz]') ax.plot_mmm(median, low, high, color='gwpy:ligo-hanford') ax.set_title('LIGO-Hanford strain noise variation around GW170817', fontsize=16) plot.show() # Now we can see that the ASD varies by factors of a few across most of the # frequency band, with notable exceptions, e.g. around the 60-Hz power line # harmonics (60 Hz, 120 Hz, 180 Hz, ...) where the noise is very stable.
def veto_scatter( outfile, a, b, label1='All', label2='Vetoed', x='time', y='snr', color=None, clim=None, clabel=None, cmap=None, clog=True, figsize=[9, 6],**kwargs): """Plot an x-y scatter of all/vetoed events """ # format axis arguments axargs = { 'yscale': 'log', 'ylabel': 'Loudness', } axargs['xscale'] = 'auto-gps' if x == 'time' else 'log' if isinstance(y, (list, tuple)): ya = y[0] yb = y[1] else: ya = yb = y axargs.update(kwargs) # create figure plot = Plot(figsize=figsize) ax = plot.gca() # add data if color is None: ax.scatter(a[x], a[ya], color='black', marker='o', label=label1, s=40) else: colorargs = {'edgecolor': 'none'} if clim: colorargs['vmin'] = clim[0] colorargs['vmax'] = clim[1] if clog: colorargs['norm'] = LogNorm(vmin=clim[0], vmax=clim[1]) a = a.copy() a.sort(color) m = ax.scatter(a[x], a[ya], c=a[color], label=label1, **colorargs) # add colorbar ax.colorbar(mappable=m, cmap=cmap, label=clabel) if isinstance(b, (list, tuple)) and len(b) == 2: # aux channel used/coinc (probably) colors = [{'color': c} for c in ( '#ffd200', # yellow '#d62728', # red )] elif isinstance(b, (list, tuple)): colors = list(rcParams['axes.prop_cycle']) else: b = [b] label2 = [label2] colors = [{'color': '#d62728'}] for i, data in enumerate(b): # setting the color here looks complicated, but is just a fancy # way of looping through the color cycle when scattering, but using # red if we only have one other data set ax.scatter(data[x], data[yb], marker='+', linewidth=1.5, label=label2[i], s=40, **colors[i % len(colors)]) # add legend if ax.get_legend_handles_labels()[0]: legargs = { 'loc': 'upper left', 'bbox_to_anchor': (1.01, 1), 'borderaxespad': 0, 'numpoints': 1, 'scatterpoints': 1, 'handlelength': 1, 'handletextpad': .5 } legargs.update(dict( (x[7:], axargs.pop(x)) for x in list(axargs.keys()) if x.startswith('legend_') )) ax.legend(**legargs) # finalize for axis in ['x', 'y']: # get data limits lim = list(getattr(ax, '%saxis' % axis).get_data_interval()) # use given ybound lim[0] = axargs.get('%sbound' % axis, lim[0]) # scale out for visual lim[0] *= 0.95 lim[1] *= 1.05 # handle logs if axargs.get("%sscale" % axis, "linear") == "log" and lim[0] <= 0.: lim[0] = None axargs.setdefault('%slim' % axis, lim) _finalize_plot(plot, ax, outfile, **axargs)
def significance_drop(outfile, old, new, show_channel_names=None, **kwargs): """Plot the signifiance drop for each channel """ channels = sorted(old.keys()) if show_channel_names is None: show_channel_names = len(channels) <= 50 plot = Plot(figsize=(20, 5)) plot.subplots_adjust(left=.07, right=.93) ax = plot.gca() if show_channel_names: plot.subplots_adjust(bottom=.4) winner = sorted(old.items(), key=lambda x: x[1])[-1][0] for i, c in enumerate(channels): if c == winner: color = 'orange' elif old[c] > new[c]: color = 'dodgerblue' else: color = 'red' ax.plot([i, i], [old[c], new[c]], color=color, linestyle='-', marker='o', markeredgecolor='k', markeredgewidth=.5, markersize=10, label=c, zorder=old[c]) ax.set_xlim(-1, len(channels)) ax.set_ybound(lower=0) # set xticks to show channel names if show_channel_names: ax.set_xticks(range(len(channels))) ax.set_xticklabels([texify(c) for c in channels]) for i, t in enumerate(ax.get_xticklabels()): t.set_rotation(270) t.set_verticalalignment('top') t.set_horizontalalignment('center') t.set_fontsize(8) # or just show systems of channels else: plot.canvas.draw() systems = {} for i, c in enumerate(channels): sys = c.split(':', 1)[1].split('-')[0].split('_')[0] try: systems[sys][1] += 1 except KeyError: systems[sys] = [i, 1] systems = sorted(systems.items(), key=lambda x: x[1][0]) labels, counts = zip(*systems) xticks, xmticks = zip(*[(a, a + b / 2.) for (a, b) in counts]) # show ticks at the edge of each group ax.set_xticks(xticks, minor=False) ax.set_xticklabels([], minor=False) # show label in the centre of each group ax.set_xticks(xmticks, minor=True) for t in ax.set_xticklabels(labels, minor=True): t.set_rotation(270) kwargs.setdefault('ylabel', 'Significance') # create interactivity if outfile.endswith('.svg'): _finalize_plot(plot, ax, outfile.replace('.svg', '.png'), close=False, **kwargs) tooltips = [] ylim = ax.get_ylim() yoffset = (ylim[1] - ylim[0]) * 0.061 bbox = {'fc': 'w', 'ec': '.5', 'alpha': .9, 'boxstyle': 'round'} xthresh = len(channels) / 10. for i, l in enumerate(ax.lines): x = l.get_xdata()[1] if x < xthresh: ha = 'left' elif x > (len(channels) - xthresh): ha = 'right' else: ha = 'center' y = l.get_ydata()[0] + yoffset c = l.get_label() tooltips.append( ax.annotate(texify(c), (x, y), ha=ha, zorder=ylim[1], bbox=bbox)) l.set_gid('line-%d' % i) tooltips[-1].set_gid('tooltip-%d' % i) f = BytesIO() plot.savefig(f, format='svg') tree, xmlid = etree.XMLID(f.getvalue()) tree.set('onload', 'init(evt)') for i in range(len(tooltips)): try: e = xmlid['tooltip-%d' % i] except KeyError: warnings.warn("Failed to recover tooltip %d" % i) continue e.set('visibility', 'hidden') for i, l in enumerate(ax.lines): e = xmlid['line-%d' % i] e.set('onmouseover', 'ShowTooltip(this)') e.set('onmouseout', 'HideTooltip(this)') tree.insert(0, etree.XML(SHOW_HIDE_JAVASCRIPT)) etree.ElementTree(tree).write(outfile) plot.close() else: _finalize_plot(plot, ax, outfile, **kwargs)
def veto_scatter(outfile, a, b, label1='All', label2='Vetoed', x='time', y='snr', color=None, clim=None, clabel=None, cmap=None, clog=True, figsize=[9, 6], **kwargs): """Plot an x-y scatter of all/vetoed events """ # format axis arguments axargs = { 'yscale': 'log', 'ylabel': 'Loudness', } axargs['xscale'] = 'auto-gps' if x == 'time' else 'log' if isinstance(y, (list, tuple)): ya = y[0] yb = y[1] else: ya = yb = y axargs.update(kwargs) # create figure plot = Plot(figsize=figsize) ax = plot.gca() # add data if color is None: ax.scatter(a[x], a[ya], color='black', marker='o', label=label1, s=40) else: colorargs = {'edgecolor': 'none'} if clim: colorargs['vmin'] = clim[0] colorargs['vmax'] = clim[1] if clog: colorargs['norm'] = LogNorm(vmin=clim[0], vmax=clim[1]) a = a.copy() a.sort(color) m = ax.scatter(a[x], a[ya], c=a[color], label=label1, **colorargs) # add colorbar ax.colorbar(mappable=m, cmap=cmap, label=clabel) if isinstance(b, (list, tuple)) and len(b) == 2: # aux channel used/coinc (probably) colors = [ { 'color': c } for c in ( '#ffd200', # yellow '#d62728', # red ) ] elif isinstance(b, (list, tuple)): colors = list(rcParams['axes.prop_cycle']) else: b = [b] label2 = [label2] colors = [{'color': '#d62728'}] for i, data in enumerate(b): # setting the color here looks complicated, but is just a fancy # way of looping through the color cycle when scattering, but using # red if we only have one other data set ax.scatter(data[x], data[yb], marker='+', linewidth=1.5, label=label2[i], s=40, **colors[i % len(colors)]) # add legend if ax.get_legend_handles_labels()[0]: legargs = { 'loc': 'upper left', 'bbox_to_anchor': (1.01, 1), 'borderaxespad': 0, 'numpoints': 1, 'scatterpoints': 1, 'handlelength': 1, 'handletextpad': .5 } legargs.update( dict((x[7:], axargs.pop(x)) for x in list(axargs.keys()) if x.startswith('legend_'))) ax.legend(**legargs) # finalize for axis in ['x', 'y']: # get data limits lim = list(getattr(ax, '%saxis' % axis).get_data_interval()) # use given ybound lim[0] = axargs.get('%sbound' % axis, lim[0]) # scale out for visual lim[0] *= 0.95 lim[1] *= 1.05 # handle logs if axargs.get("%sscale" % axis, "linear") == "log" and lim[0] <= 0.: lim[0] = None axargs.setdefault('%slim' % axis, lim) _finalize_plot(plot, ax, outfile, **axargs)
def cluster_plotter(channels, start, stop, prefix='.', label='kmeans-labels', groups=None, filename=DEFAULT_FILENAME, dqflag='L1:DMT-ANALYSIS_READY:1', xscale=None, unit=None, progressbar=True, **kwargs): """ Plots data with clusters labeled by color in the working directory, or a relative path given by prefix. Requires a .hdf5 file produced with a clustering function defined in this module to be in the working directory. **kwargs are forwarded to TimeSeries.plot(). :param prefix: relative path to output images. :param label: name attribute of labels TimeSeries saved in filename. :param groups: groups of channels to plot in the same figure. See the example. :param dqflag: data quality flag for segments bar. :param xscale: gps x-axis scale to use. :param unit: override y-axis unit. :param progressbar: show progress bar. >>> from gwpy.time import tconvert, from_gps >>> from datetime import timedelta >>> from cluster import cluster_plotter >>> >>> channels = [f'L1:ISI-GND_STS_ETMX_Z_BLRMS_1_3.mean,m-trend', 'L1:ISI-GND_STS_ETMY_Z_BLRMS_1_3.mean,m-trend'] >>> groups = [[channels, ('ETMX', 'ETMY'), 'L1:ISI-GND_STS_BLRMS_1_3 Z-axis']] # plot on the same figure. >>> >>> stop = from_gps(60 * (int(tconvert('now')) // 60)) # gets nearest minute to now >>> start = stop - timedelta(days=1) # cluster the past day >>> cluster_plotter(channels, start, stop, filename='my_kmeans.hdf5', groups=groups) """ # some defaults. if not kwargs: kwargs['color'] = 'k' kwargs['alpha'] = 0.3 if groups is None: groups = channels # read the data from the save file. data = TimeSeriesDict.read(filename, channels + [label], start=to_gps(start), end=to_gps(stop)) logger.info(f'Read {start} to {stop} from {filename}') # get segments for the duration specified. Note that this may require doing `ligo-proxy-init -p`. logger.debug(f'Getting segments for {dqflag} from {start} to {stop}...') dq = DataQualityFlag.query(dqflag, to_gps(start), to_gps(stop)) logger.info(f'Got segments for {dqflag} from {start} to {stop}.') # plotting is slow, so show a nice progress bar. logger.debug('Initiating plotting routine...') with Progress('plotting', len(channels), quiet=not progressbar) as progress: for p, (group, labels, title) in enumerate(groups): # plot the group in one figure. plt = Plot(*(data[channel] for channel in group), separate=True, sharex=True, zorder=1, **kwargs) # modify the axes one by one. axes = plt.get_axes() for i, ax in enumerate(axes): # namely, add a colored overlay that indicates clustering labels. ax.scatter(data[group[i]].times, data[group[i]].value, c=[colors[j] for j in data[label]], edgecolor='', s=4, zorder=2) ax.set_ylabel( f'{labels[i]} {data[group[i]].unit if unit is None else unit}' ) setp(ax.get_xticklabels(), visible=False) # modify the figure as a whole. plt.add_segments_bar(dq, label='') if xscale is not None: plt.gca().set_xscale(xscale) plt.suptitle(title) # save to png. progress(plt.save, p, get_path(title, 'png', prefix=prefix)) logger.info(f'Completed plotting for {start} to {stop} from {filename}')
def significance_drop(outfile, old, new, show_channel_names=None, **kwargs): """Plot the signifiance drop for each channel """ channels = sorted(old.keys()) if show_channel_names is None: show_channel_names = len(channels) <= 50 plot = Plot(figsize=(18, 6)) plot.subplots_adjust(left=.07, right=.93) ax = plot.gca() if show_channel_names: plot.subplots_adjust(bottom=.4) winner = sorted(old.items(), key=lambda x: x[1])[-1][0] for i, c in enumerate(channels): if c == winner: color = 'orange' elif old[c] > new[c]: color = 'dodgerblue' else: color = 'red' ax.plot([i, i], [old[c], new[c]], color=color, linestyle='-', marker='o', markeredgecolor='k', markeredgewidth=.5, markersize=10, label=c, zorder=old[c]) ax.set_xlim(-1, len(channels)) ax.set_ybound(lower=0) # set xticks to show channel names if show_channel_names: ax.set_xticks(range(len(channels))) ax.set_xticklabels([c.replace('_','\_') for c in channels]) for i, t in enumerate(ax.get_xticklabels()): t.set_rotation(270) t.set_verticalalignment('top') t.set_horizontalalignment('center') t.set_fontsize(8) # or just show systems of channels else: plot.canvas.draw() systems = {} for i, c in enumerate(channels): sys = c.split(':', 1)[1].split('-')[0].split('_')[0] try: systems[sys][1] += 1 except KeyError: systems[sys] = [i, 1] systems = sorted(systems.items(), key=lambda x: x[1][0]) labels, counts = zip(*systems) xticks, xmticks = zip(*[(a, a+b/2.) for (a, b) in counts]) # show ticks at the edge of each group ax.set_xticks(xticks, minor=False) ax.set_xticklabels([], minor=False) # show label in the centre of each group ax.set_xticks(xmticks, minor=True) for t in ax.set_xticklabels(labels, minor=True): t.set_rotation(270) kwargs.setdefault('ylabel', 'Significance') # create interactivity if outfile.endswith('.svg'): _finalize_plot(plot, ax, outfile.replace('.svg', '.png'), close=False, **kwargs) tooltips = [] ylim = ax.get_ylim() yoffset = (ylim[1] - ylim[0]) * 0.061 bbox = {'fc': 'w', 'ec': '.5', 'alpha': .9, 'boxstyle': 'round'} xthresh = len(channels) / 10. for i, l in enumerate(ax.lines): x = l.get_xdata()[1] if x < xthresh: ha = 'left' elif x > (len(channels) - xthresh): ha ='right' else: ha = 'center' y = l.get_ydata()[0] + yoffset c = l.get_label() tooltips.append(ax.annotate(c.replace('_', r'\_'), (x, y), ha=ha, zorder=ylim[1], bbox=bbox)) l.set_gid('line-%d' % i) tooltips[-1].set_gid('tooltip-%d' % i) f = BytesIO() plot.savefig(f, format='svg') tree, xmlid = etree.XMLID(f.getvalue()) tree.set('onload', 'init(evt)') for i in range(len(tooltips)): try: e = xmlid['tooltip-%d' % i] except KeyError: warnings.warn("Failed to recover tooltip %d" % i) continue e.set('visibility', 'hidden') for i, l in enumerate(ax.lines): e = xmlid['line-%d' % i] e.set('onmouseover', 'ShowTooltip(this)') e.set('onmouseout', 'HideTooltip(this)') tree.insert(0, etree.XML(SHOW_HIDE_JAVASCRIPT)) etree.ElementTree(tree).write(outfile) plot.close() else: _finalize_plot(plot, ax, outfile, **kwargs)
path='hoge') strain.write('./fs_gif.hdf5', format='hdf5', overwrite=True, path='hoge') # ----------------------------------------------- # Main # ----------------------------------------------- # ----------------------------------------------- # IXV1 DIFF12 # ----------------------------------------------- if comparison_ixv1_diff12: print('Comparison_IXV1_DIFF12') from gwpy.plot import Plot plot = Plot() ax = plot.gca(xscale='log', xlim=(1e-3, 100), yscale='log', ylim=(1e-7, 1e2)) ax.plot(ixv1, color='k', label='IXV1') ax.plot(diff12, color='r', label=r'(IXV1 - IXV2)/$\sqrt{2}$') ax.plot(tr120_selfnoise, color='k', label='Selfnoise', linestyle='--', zorder=0) ax.set_xlabel('Frequency [Hz]', fontsize=15) ax.set_ylabel(r'Displacement [um/$\sqrt{\mathrm{Hz}}$]', fontsize=15) ax.set_title('Trillium120QA Noise Investigation', fontsize=20) ax.text(110, 1e-7, 'START : {0}'.format(start), rotation=90,
_aanoise = 7e-8*u.V*np.ones(len(freq)) # [V/rtHz] _adcnoise = FrequencySeries(_adcnoise, frequencies=freq) _adcnoise2 = FrequencySeries(_adcnoise2, frequencies=freq) _ampnoise = FrequencySeries(_ampnoise, frequencies=freq) _aanoise = FrequencySeries(_aanoise, frequencies=freq) adcnoise = v2vel_120(_adcnoise)/amp ampnoise = v2vel_120(_ampnoise)/amp # aanoise = v2vel_120(_aanoise)/amp selfnoise_120q = tr120q.selfnoise(unit=unit) selfnoise_240 = tr240.selfnoise(unit=unit) # # Plot from gwpy.plot import Plot plot = Plot() ax = plot.gca(xscale='log', xlim=(1e-3, 10), xlabel='Frequency [Hz]', yscale='log', ylim=(1e-12, 1e-4), ylabel=r'Velocity [{0}/\rtHz]'.format(unit)) #ax.loglog(x1500,label='x1500 (ref. GIF data)') ax.loglog(selfnoise_120q,'k--',label='Self Noise 120Q',linewidth=1,alpha=0.7) #ax.loglog(selfnoise_240,'m--',label='Self Noise 240',linewidth=1,alpha=0.7) #ax.loglog(adcnoise,'r--',label='ADC Noise',linewidth=1,alpha=0.7) #ax.loglog(ampnoise,'g--',label='Amp Noise',linewidth=1,alpha=0.7) #ax.loglog(aanoise,'b--',label='AA Noise',linewidth=1,alpha=0.7) ax.loglog(exv,'k-',label='EXV') ax.loglog(ixv,label='IXV1') ax.set_xlim(1e-2,10) #ax.loglog(ixv2,label='IXV2') #ax.loglog(d12,label='IXV1-IXV2') #ax.loglog(strain,label='StrainMeter') ax.legend(fontsize=8,loc='lower left') ax.set_title('Seismometer, {dname}'.format(dname=dataname.replace('_','')),
def main(args=None): """Run the lasso command-line interface """ # declare global variables # this is needed for multiprocessing utilities global auxdata, cluster_threshold, cmap, colors, counter, gpsstub global line_size_aux, line_size_primary, max_correlated_channels global nonzerocoef, nonzerodata, p1, primary, primary_mean, primary_std global primaryts, range_is_primary, re_delim, start, target, times global threshold, trend_type, xlim parser = create_parser() args = parser.parse_args(args=args) # get run params start = int(args.gpsstart) end = int(args.gpsend) pad = args.filter_padding # set pertinent global variables cluster_threshold = args.cluster_coefficient line_size_aux = args.line_size_aux line_size_primary = args.line_size_primary threshold = args.threshold trend_type = args.trend_type # let's go LOGGER.info('{} Lasso correlations {}-{}'.format(args.ifo, start, end)) # get primary channel frametype primary = args.primary_channel.format(ifo=args.ifo) range_is_primary = 'EFFECTIVE_RANGE_MPC' in args.primary_channel if args.primary_cache is not None: LOGGER.info("Using custom primary cache file") elif args.primary_frametype is None: try: args.primary_frametype = DEFAULT_FRAMETYPE[ args.primary_channel.split(':')[1]].format(ifo=args.ifo) except KeyError as exc: raise type(exc)("Could not determine primary channel's frametype, " "please specify with --primary-frametype") # create output directory if not os.path.isdir(args.output_dir): os.makedirs(args.output_dir) os.chdir(args.output_dir) # multiprocessing for plots nprocplot = (args.nproc_plot or args.nproc) if USETEX else 1 # bandpass primary if args.band_pass: try: flower, fupper = args.band_pass except TypeError: flower, fupper = None LOGGER.info("-- Loading primary channel data") bandts = get_data(primary, start - pad, end + pad, verbose='Reading primary:'.rjust(30), frametype=args.primary_frametype, source=args.primary_cache, nproc=args.nproc) if flower < 0 or fupper >= float((bandts.sample_rate / 2.).value): raise ValueError( "bandpass frequency is out of range for this " "channel, band (Hz): {0}, sample rate: {1}".format( args.band_pass, bandts.sample_rate)) # get darm BLRMS LOGGER.debug("-- Filtering data") if trend_type == 'minute': stride = 60 else: stride = 1 if flower: darmbl = (bandts.highpass( flower / 2., fstop=flower / 4., filtfilt=False, ftype='butter').notch(60, filtfilt=False).bandpass( flower, fupper, fstop=[flower / 2., fupper * 1.5], filtfilt=False, ftype='butter').crop(start, end)) darmblrms = darmbl.rms(stride) darmblrms.name = '%s %s-%s Hz BLRMS' % (primary, flower, fupper) else: darmbl = bandts.notch(60).crop(start, end) darmblrms = darmbl.rms(stride) darmblrms.name = '%s RMS' % primary primaryts = darmblrms bandts_asd = bandts.asd(4, 2, method='median') darmbl_asd = darmbl.asd(4, 2, method='median') spectrum_plots = gwplot.make_spectrum_plots(start, end, flower, fupper, args.primary_channel, bandts_asd, darmbl_asd) spectrum_plot_zoomed_out = spectrum_plots[0] spectrum_plot_zoomed_in = spectrum_plots[1] else: # load primary channel data LOGGER.info("-- Loading primary channel data") primaryts = get_data(primary, start, end, frametype=args.primary_frametype, source=args.primary_cache, verbose='Reading:'.rjust(30), nproc=args.nproc).crop(start, end) if args.remove_outliers: LOGGER.debug("-- Removing outliers above %f sigma" % args.remove_outliers) gwlasso.remove_outliers(primaryts, args.remove_outliers) elif args.remove_outliers_pf: LOGGER.debug("-- Removing outliers in the bottom {} percent " "of data".format(args.remove_outliers_pf)) gwlasso.remove_outliers(primaryts, args.remove_outliers_pf, method='pf') start = int(primaryts.span()[0]) end = int(primaryts.span()[1]) primary_mean = numpy.mean(primaryts.value) primary_std = numpy.std(primaryts.value) # get aux data LOGGER.info("-- Loading auxiliary channel data") if args.channel_file is None: host, port = io_nds2.host_resolution_order(args.ifo)[0] channels = ChannelList.query_nds2('*.mean', host=host, port=port, type='m-trend') else: with open(args.channel_file, 'r') as f: channels = [name.rstrip('\n') for name in f] nchan = len(channels) LOGGER.debug("Identified %d channels" % nchan) if trend_type == 'minute': frametype = '%s_M' % args.ifo # for minute trends else: frametype = '%s_T' % args.ifo # for second trends # read aux channels auxdata = get_data(channels, start, end, verbose='Reading:'.rjust(30), frametype=frametype, nproc=args.nproc, pad=0).crop(start, end) # -- removes flat data to be re-introdused later LOGGER.info('-- Pre-processing auxiliary channel data') auxdata = gwlasso.remove_flat(auxdata) flatable = Table(data=(list(set(channels) - set(auxdata.keys())), ), names=('Channels', )) LOGGER.debug('Removed {0} channels with flat data'.format(len(flatable))) LOGGER.debug('{0} channels remaining'.format(len(auxdata))) # -- remove bad data LOGGER.info("Removing any channels with bad data...") nbefore = len(auxdata) auxdata = gwlasso.remove_bad(auxdata) nafter = len(auxdata) LOGGER.debug('Removed {0} channels with bad data'.format(nbefore - nafter)) LOGGER.debug('{0} channels remaining'.format(nafter)) data = numpy.array([scale(ts.value) for ts in auxdata.values()]).T # -- perform lasso regression ------------------- # create model LOGGER.info('-- Fitting data to target') target = scale(primaryts.value) model = gwlasso.fit(data, target, alpha=args.alpha) LOGGER.info('Alpha: {}'.format(model.alpha)) # restructure results for convenience allresults = Table(data=(list(auxdata.keys()), model.coef_, numpy.abs(model.coef_)), names=('Channel', 'Lasso coefficient', 'rank')) allresults.sort('rank') allresults.reverse() useful = allresults['rank'] > 0 allresults.remove_column('rank') results = allresults[useful] # non-zero coefficient zeroed = allresults[numpy.invert(useful)] # zero coefficient # extract data for useful channels nonzerodata = {name: auxdata[name] for name in results['Channel']} nonzerocoef = {name: coeff for name, coeff in results.as_array()} # print results LOGGER.info('Found {} channels with |Lasso coefficient| >= {}:\n\n'.format( len(results), threshold)) print(results) print('\n\n') # convert to pandas set_option('max_colwidth', -1) df = results.to_pandas() df.index += 1 # write results to files gpsstub = '%d-%d' % (start, end - start) resultsfile = '%s-LASSO_RESULTS-%s.csv' % (args.ifo, gpsstub) results.write(resultsfile, format='csv', overwrite=True) zerofile = '%s-ZERO_COEFFICIENT_CHANNELS-%s.csv' % (args.ifo, gpsstub) zeroed.write(zerofile, format='csv', overwrite=True) flatfile = '%s-FLAT_CHANNELS-%s.csv' % (args.ifo, gpsstub) flatable.write(flatfile, format='csv', overwrite=True) # -- generate lasso plots modelFit = model.predict(data) re_delim = re.compile(r'[:_-]') p1 = (.1, .15, .9, .9) # global plot defaults for plot1, lasso model times = primaryts.times.value xlim = primaryts.span cmap = get_cmap('tab20') colors = [cmap(i) for i in numpy.linspace(0, 1, len(nonzerodata) + 1)] plot = Plot(figsize=(12, 4)) plot.subplots_adjust(*p1) ax = plot.gca(xscale='auto-gps', epoch=start, xlim=xlim) ax.plot(times, _descaler(target), label=texify(primary), color='black', linewidth=line_size_primary) ax.plot(times, _descaler(modelFit), label='Lasso model', linewidth=line_size_aux) if range_is_primary: ax.set_ylabel('Sensitive range [Mpc]') ax.set_title('Lasso Model of Range') else: ax.set_ylabel('Primary Channel Units') ax.set_title('Lasso Model of Primary Channel') ax.legend(loc='best') plot1 = gwplot.save_figure(plot, '%s-LASSO_MODEL-%s.png' % (args.ifo, gpsstub), bbox_inches='tight') # summed contributions plot = Plot(figsize=(12, 4)) plot.subplots_adjust(*p1) ax = plot.gca(xscale='auto-gps', epoch=start, xlim=xlim) ax.plot(times, _descaler(target), label=texify(primary), color='black', linewidth=line_size_primary) summed = 0 for i, name in enumerate(results['Channel']): summed += scale(nonzerodata[name].value) * nonzerocoef[name] if i: label = 'Channels 1-{0}'.format(i + 1) else: label = 'Channel 1' ax.plot(times, _descaler(summed), label=label, color=colors[i], linewidth=line_size_aux) if range_is_primary: ax.set_ylabel('Sensitive range [Mpc]') else: ax.set_ylabel('Primary Channel Units') ax.set_title('Summations of Channel Contributions to Model') ax.legend(loc='center left', bbox_to_anchor=(1.05, 0.5)) plot2 = gwplot.save_figure(plot, '%s-LASSO_CHANNEL_SUMMATION-%s.png' % (args.ifo, gpsstub), bbox_inches='tight') # individual contributions plot = Plot(figsize=(12, 4)) plot.subplots_adjust(*p1) ax = plot.gca(xscale='auto-gps', epoch=start, xlim=xlim) ax.plot(times, _descaler(target), label=texify(primary), color='black', linewidth=line_size_primary) for i, name in enumerate(results['Channel']): this = _descaler(scale(nonzerodata[name].value) * nonzerocoef[name]) if i: label = 'Channels 1-{0}'.format(i + 1) else: label = 'Channel 1' ax.plot(times, this, label=texify(name), color=colors[i], linewidth=line_size_aux) if range_is_primary: ax.set_ylabel('Sensitive range [Mpc]') else: ax.set_ylabel('Primary Channel Units') ax.set_title('Individual Channel Contributions to Model') ax.legend(loc='center left', bbox_to_anchor=(1.05, 0.5)) plot3 = gwplot.save_figure(plot, '%s-LASSO_CHANNEL_CONTRIBUTIONS-%s.png' % (args.ifo, gpsstub), bbox_inches='tight') # -- process aux channels, making plots LOGGER.info("-- Processing channels") counter = multiprocessing.Value('i', 0) # process channels pool = multiprocessing.Pool(nprocplot) results = pool.map(_process_channel, enumerate(list(nonzerodata.items()))) results = sorted(results, key=lambda x: abs(x[1]), reverse=True) # generate clustered time series plots counter = multiprocessing.Value('i', 0) max_correlated_channels = 20 if args.no_cluster is False: LOGGER.info("-- Generating clusters") pool = multiprocessing.Pool(nprocplot) clusters = pool.map(_generate_cluster, enumerate(results)) channelsfile = '%s-CHANNELS-%s.csv' % (args.ifo, gpsstub) numpy.savetxt(channelsfile, channels, delimiter=',', fmt='%s') # write html trange = '%d-%d' % (start, end) title = '%s Lasso Correlation: %s' % (args.ifo, trange) if args.band_pass: links = [trange ] + [(s, '#%s' % s.lower()) for s in ['Parameters', 'Spectra', 'Model', 'Results']] else: links = [trange] + [(s, '#%s' % s.lower()) for s in ['Parameters', 'Model', 'Results']] (brand, class_) = htmlio.get_brand(args.ifo, 'Lasso', start) navbar = htmlio.navbar(links, class_=class_, brand=brand) page = htmlio.new_bootstrap_page(title='%s Lasso | %s' % (args.ifo, trange), navbar=navbar) page.h1(title, class_='pb-2 mt-3 mb-2 border-bottom') # -- summary table content = [ ('Primary channel', markup.oneliner.code(primary)), ('Primary frametype', markup.oneliner.code(args.primary_frametype) or '-'), ('Primary cache file', markup.oneliner.code(args.primary_cache) or '-'), ('Outlier threshold', '%s sigma' % args.remove_outliers), ('Lasso coefficient threshold', str(threshold)), ('Cluster coefficient threshold', str(args.cluster_coefficient)), ('Non-zero coefficients', str(numpy.count_nonzero(model.coef_))), ('α (model)', '%.4f' % model.alpha) ] if args.band_pass: content.insert( 2, ('Primary bandpass', '{0}-{1} Hz'.format(flower, fupper))) page.h2('Parameters', class_='mt-4 mb-4', id_='parameters') page.div(class_='row') page.div(class_='col-md-9 col-sm-12') page.add(htmlio.parameter_table(content, start=start, end=end)) page.div.close() # col-md-9 col-sm-12 # -- download button files = [('%s analyzed channels (CSV)' % nchan, channelsfile), ('%s flat channels (CSV)' % len(flatable), flatfile), ('%s zeroed channels (CSV)' % len(zeroed), zerofile)] page.div(class_='col-md-3 col-sm-12') page.add( htmlio.download_btn(files, label='Channel information', btnclass='btn btn-%s dropdown-toggle' % args.ifo.lower())) page.div.close() # col-md-3 col-sm-12 page.div.close() # rowa # -- command-line page.h5('Command-line:') page.add(htmlio.get_command_line(about=False, prog=PROG)) if args.band_pass: page.h2('Primary channel spectra', class_='mt-4', id_='spectra') page.div(class_='card border-light card-body shadow-sm') page.div(class_='row') page.div(class_='col-md-6') spectra_img1 = htmlio.FancyPlot(spectrum_plot_zoomed_out) page.add(htmlio.fancybox_img(spectra_img1)) page.div.close() # col-md-6 page.div(class_='col-md-6') spectra_img2 = htmlio.FancyPlot(spectrum_plot_zoomed_in) page.add(htmlio.fancybox_img(spectra_img2)) page.div.close() # col-md-6 page.div.close() # row page.div.close() # card border-light card-body shadow-sm # -- model information page.h2('Model information', class_='mt-4', id_='model') page.div(class_='card card-%s card-body shadow-sm' % args.ifo.lower()) page.div(class_='row') page.div(class_='col-md-8 offset-md-2', id_='results-table') page.p('Below are the top {} mean minute-trend channels, ranked by ' 'Lasso correlation with the primary.'.format(df.shape[0])) page.add( df.to_html(classes=('table', 'table-sm', 'table-hover'), formatters={ 'Lasso coefficient': lambda x: "%.4f" % x, 'Channel': lambda x: str(htmlio.cis_link(x.split('.')[0])), '__index__': lambda x: str(x) }, escape=False, border=0).replace(' style="text-align: right;"', '')) page.div.close() # col-md-10 offset-md-1 page.div.close() # row page.div(class_='row', id_='primary-lasso') page.div(class_='col-md-8 offset-md-2') img1 = htmlio.FancyPlot(plot1) page.add(htmlio.fancybox_img(img1)) # primary lasso plot page.div.close() # col-md-8 offset-md-2 page.div.close() # primary-lasso page.div(class_='row', id_='channel-summation') img2 = htmlio.FancyPlot(plot2) page.div(class_='col-md-8 offset-md-2') page.add(htmlio.fancybox_img(img2)) page.div.close() # col-md-8 offset-md-2 page.div.close() # channel-summation page.div(class_='row', id_='channels-and-primary') img3 = htmlio.FancyPlot(plot3) page.div(class_='col-md-8 offset-md-2') page.add(htmlio.fancybox_img(img3)) page.div.close() # col-md-8 offset-md-2 page.div.close() # channels-and-primary page.div.close() # card card-<ifo> card-body shadow-sm # -- results page.h2('Top channels', class_='mt-4', id_='results') page.div(id_='results') # for each aux channel, create information container and put plots in it for i, (ch, lassocoef, plot4, plot5, plot6, ts) in enumerate(results): # set container color/context based on lasso coefficient if lassocoef == 0: break elif abs(lassocoef) < threshold: h = '%s [lasso coefficient = %.4f] (Below threshold)' % (ch, lassocoef) else: h = '%s [lasso coefficient = %.4f]' % (ch, lassocoef) if ((lassocoef is None) or (lassocoef == 0) or (abs(lassocoef) < threshold)): card = 'card border-light mb-1 shadow-sm' card_header = 'card-header bg-light' elif abs(lassocoef) >= .5: card = 'card border-danger mb-1 shadow-sm' card_header = 'card-header text-white bg-danger' elif abs(lassocoef) >= .2: card = 'card border-warning mb-1 shadow-sm' card_header = 'card-header text-white bg-warning' else: card = 'card border-info mb-1 shadow-sm' card_header = 'card-header text-white bg-info' page.div(class_=card) # heading page.div(class_=card_header) page.a(h, class_='collapsed card-link cis-link', href='#channel%d' % i, **{'data-toggle': 'collapse'}) page.div.close() # card-header # body page.div(id_='channel%d' % i, class_='collapse', **{'data-parent': '#results'}) page.div(class_='card-body') if lassocoef is None: page.p('The amplitude data for this channel is flat (does not ' 'change) within the chosen time period.') elif abs(lassocoef) < threshold: page.p('Lasso coefficient below the threshold of %g.' % (threshold)) else: for image in [plot4, plot5, plot6]: img = htmlio.FancyPlot(image) page.div(class_='row') page.div(class_='col-md-8 offset-md-2') page.add(htmlio.fancybox_img(img)) page.div.close() # col-md-8 offset-md-2 page.div.close() # row page.add('<hr class="row-divider">') if args.no_cluster is False: if clusters[i][0] is None: page.p("<font size='3'><br />No channels were highly " "correlated with this channel.</font>") else: page.div(class_='row', id_='clusters') page.div(class_='col-md-12') cimg = htmlio.FancyPlot(clusters[i][0]) page.add(htmlio.fancybox_img(cimg)) page.div.close() # col-md-12 page.div.close() # clusters if clusters[i][1] is not None: corr_link = markup.oneliner.a( 'Export {} channels (CSV)'.format( max_correlated_channels), href=clusters[i][1], download=clusters[i][1], ) page.button( corr_link, class_='btn btn-%s' % args.ifo.lower(), ) page.div.close() # card-body page.div.close() # collapse page.div.close() # card page.div.close() # results htmlio.close_page(page, 'index.html') # save and close LOGGER.info("-- Process Completed")
# Then we can download a simulation of the GW150914 signal from LOSC: from astropy.utils.data import get_readable_fileobj source = 'https://losc.ligo.org/s/events/GW150914/P150914/' url = '%s/fig2-unfiltered-waveform-H.txt' % source with get_readable_fileobj(url) as f: signal = TimeSeries.read(f, format='txt') signal.t0 = .5 # make sure this intersects with noise time samples # Note, since this simulation cuts off before a certain time, it is # important to taper its ends to zero to avoid ringing artifacts. # We can accomplish this using the # :meth:`~gwpy.timeseries.TimeSeries.taper` method. signal = signal.taper() # Since the time samples overlap, we can inject this into our noise data # using :meth:`~gwpy.types.series.Series.inject`: data = noise.inject(signal) # Finally, we can visualize the full process in the time domain: from gwpy.plot import Plot plot = Plot(noise, signal, data, separate=True, sharex=True, sharey=True) plot.gca().set_epoch(0) plot.show() # We can clearly see that the loud GW150914-like signal has been layered # on top of Gaussian noise with the correct amplitude and phase evolution.
def whiten(data, ffttime, window, low_f, high_f, notch, rate): """ This function whitens the data and band-pass it in the range [low_f, high_f]. Parameters ---------- data: numpy array The signal to whiten as numpy array ffttime: int Portion of the strain to compute the psd window: str Type of function for the windowing low_f: int Lower bound of the band-pass filter high_f: int Upper bound of the band-pass filter notch: list Frequencies of the notch filters. Depends on the detector rate: int Resampling rate. Represents the sampling frequency Returns ------- whitened: numpy array The whitened and band-passed numpy array """ # Band-pass filter in [35, 250] bp = bandpass(float(low_f), float(high_f), data.sample_rate) #Notches for the 1st three harminics of the 60 Hz AC notches = [filter_design.notch(line, data.sample_rate) for line in notch] #Concatenate both filters zpk = filter_design.concatenate_zpks(bp, *notches) #Whiten and band-pass filter white = data.whiten(ffttime, int(ffttime / 2), window='hann') #whiten the data white_down = white.filter(zpk, filtfilt=True).resample( rate=rate, window='hann') #downsample to 2048Hz whitened = np.array(white_down) #Plot version with and without notches plot = Plot(figsize=(15, 6)) ax = plot.gca() ax.plot(white_down, label='Downsampled', alpha=0.7) ax.plot(white.filter(zpk, filtfilt=True), label='Not downsampled', alpha=0.7) ax.set_xscale('auto-gps') ax.set_ylabel('Frequency [Hz]') ax.set_title( 'LIGO-Livingston strain data whitened, band-passed in range [' + str(low_f) + '' + str(high_f) + '] $Hz$') plot.legend() plt.savefig('/home/melissa.lopez/Anomaly_Detection/Algorithm/dummy.png') plt.close() return whitened
color='gwpy:ligo-hanford') ax1, ax2 = plot.axes ax1.set_title('LIGO-Hanford strain data around GW150914') ax1.text(1.0, 1.01, 'Unfiltered data', transform=ax1.transAxes, ha='right') ax1.set_ylabel('Amplitude [strain]', y=-0.2) ax2.set_ylabel('') ax2.text(1.0, 1.01, '50-250\,Hz bandpass, notches at 60, 120, 180 Hz', transform=ax2.transAxes, ha='right') plot.show() plot.close() # hide # We see now a spike around 16 seconds into the data, so let's zoom into # that time (and prettify): plot = hfilt.plot(color='gwpy:ligo-hanford') ax = plot.gca() ax.set_title('LIGO-Hanford strain data around GW150914') ax.set_ylabel('Amplitude [strain]') ax.set_xlim(1126259462, 1126259462.6) ax.set_xscale('seconds', epoch=1126259462) plot.show() plot.close() # hide # Congratulations, you have succesfully filtered LIGO data to uncover the # first ever directly-detected gravitational wave signal, GW150914! # But wait, what about LIGO-Livingston? # We can easily add that to our figure by following the same procedure. # # First, we load the new data ldata = TimeSeries.fetch_open_data('L1', 1126259446, 1126259478)
from gwpy.plot import Plot plot = Plot() ax = plot.gca(xscale='log', xlim=(10, 1500), xlabel='Frequency [Hz]', yscale='log', ylim=(3e-24, 2e-20), ylabel=r'Strain noise [1/\rtHz]') ax.plot_mmm(median, low, high, color='gwpy:ligo-hanford') ax.set_title('LIGO-Hanford strain noise variation around GW170817', fontsize=16) plot.show()
def process_channel(input_,): chan, ts = input_ flat = ts.value.min() == ts.value.max() if flat: corr1 = None corr2 = None corr1s = None corr2s = None plot1 = None plot2 = None plot3 = None else: corr1 = numpy.corrcoef(ts.value, darmblrms.value)[0, 1] corr1s = spearmanr(ts.value, darmblrms.value)[0] if args.trend_type == 'minute': corr2 = numpy.corrcoef(ts.value, rangets.value)[0, 1] corr2s = spearmanr(ts.value, rangets.value)[0] else: corr2 = 0.0 corr2s = 0.0 # if all corralations are below threshold it does not plot if((abs(corr1) < args.threshold) and (abs(corr1s) < args.threshold) and (abs(corr2) < args.threshold) and (abs(corr2s) < args.threshold)): plot1 = None plot2 = None plot3 = None return (chan, corr1, corr2, plot1, plot2, plot3, corr1s, corr2s) plot = Plot(darmblrms, ts, rangets, xscale="auto-gps", separate=True, figsize=(12, 12)) plot.subplots_adjust(*p1) plot.axes[0].set_ylabel('$h(t)$ BLRMS [strain]') plot.axes[1].set_ylabel('Channel units') plot.axes[2].set_ylabel('Sensitive range [Mpc]') for ax in plot.axes: ax.legend(loc='best') ax.set_xlim(start, end) ax.set_epoch(start) channelstub = re_delim.sub('_', str(chan)).replace('_', '-', 1) plot1 = '%s_TRENDS-%s.png' % (channelstub, gpsstub) try: plot.save(plot1) except (IOError, IndexError): plot.save(plot1) except RuntimeError as e: if 'latex' in str(e).lower(): plot.save(plot1) else: raise plot.close() # plot auto-scaled verions tsscaled = ts.detrend() tsrms = numpy.sqrt(sum(tsscaled**2.0)/len(tsscaled)) if args.trend_type == 'minute': tsscaled *= (rangerms / tsrms) if corr1 > 0: tsscaled *= -1 else: tsscaled *= (darmrms / tsrms) if corr1 < 0: tsscaled *= -1 plot = Plot(darmscaled, rangescaled, tsscaled, xscale="auto-gps", figsize=[12, 6]) plot.subplots_adjust(*p2) ax = plot.gca() ax.set_xlim(start, end) ax.set_epoch(start) ax.set_ylabel('Scaled amplitude [arbitrary units]') ax.legend(loc='best') plot2 = '%s_COMPARISON-%s.png' % (channelstub, gpsstub) try: plot.save(plot2) except (IOError, IndexError): plot.save(plot2) except RuntimeError as e: if 'latex' in str(e).lower(): plot.save(plot2) else: raise plot.close() # plot scatter plots rangeColor = 'red' darmblrmsColor = 'blue' tsCopy = ts.reshape(-1, 1) rangetsCopy = rangets.reshape(-1, 1) darmblrmsCopy = darmblrms.reshape(-1, 1) darmblrmsReg = linear_model.LinearRegression() darmblrmsReg.fit(tsCopy, darmblrmsCopy) darmblrmsFit = darmblrmsReg.predict(tsCopy) rangeReg = linear_model.LinearRegression() rangeReg.fit(tsCopy, rangetsCopy) rangeFit = rangeReg.predict(tsCopy) fig = Plot(figsize=(12, 6)) fig.subplots_adjust(*p2) ax = fig.add_subplot(121) ax.set_xlabel('Channel units') ax.set_ylabel('Sensitive range [Mpc]') yrange = abs(max(darmblrms.value) - min(darmblrms.value)) upperLim = max(darmblrms.value) + .1 * yrange lowerLim = min(darmblrms.value) - .1 * yrange ax.set_ylim(lowerLim, upperLim) ax.text(.9, .1, 'r = ' + str('{0:.2}'.format(corr1)), verticalalignment='bottom', horizontalalignment='right', transform=ax.transAxes, color='black', size=20, bbox=dict(boxstyle='square', facecolor='white', alpha=.75, edgecolor='black')) fig.add_scatter(ts, darmblrms, color=darmblrmsColor) fig.add_line(ts, darmblrmsFit, color='black') ax = fig.add_subplot(122) ax.set_xlabel('Channel units') ax.set_ylabel('$h(t)$ BLRMS [strain]') ax.text(.9, .1, 'r = ' + str('{0:.2}'.format(corr2)), verticalalignment='bottom', horizontalalignment='right', transform=ax.transAxes, color='black', size=20, bbox=dict(boxstyle='square', facecolor='white', alpha=.75, edgecolor='black')) fig.add_scatter(ts, rangets, color=rangeColor) fig.add_line(ts, rangeFit, color='black') plot3 = '%s_SCATTER-%s.png' % (channelstub, gpsstub) try: fig.save(plot3) except (IOError, IndexError): fig.save(plot3) except RuntimeError as e: if 'latex' in str(e).lower(): fig.save(plot3) else: raise plt.close(fig) # increment counter and print status with counter.get_lock(): counter.value += 1 pc = 100 * counter.value / nchan LOGGER.debug("Completed [%d/%d] %3d%% %-50s" % (counter.value, nchan, pc, '(%s)' % str(chan))) sys.stdout.flush() return chan, corr1, corr2, plot1, plot2, plot3, corr1s, corr2s
tr240 = Trillium('240') v2vel = tr120q.v2vel v2vel = tr240.v2vel adcnoise = v2vel(_adcnoise) / amp ampnoise = v2vel(_ampnoise) / amp # aanoise = v2vel(_aanoise) / amp selfnoise_120q = tr120q.selfnoise() selfnoise_240 = tr240.selfnoise() # Plot if True: from gwpy.plot import Plot plot = Plot() ax = plot.gca(xscale='log', xlim=(1e-3, 10), xlabel='Frequency [Hz]', yscale='log', ylim=(1e-12, 1e-4), ylabel=r'Velocity [m/sec/\rtHz]') #ax.loglog(x1500,label='x1500 (ref. GIF data)') ax.loglog(selfnoise_120q, 'k--', label='Self Noise 120Q', linewidth=1, alpha=0.7) ax.loglog(selfnoise_240, 'm--', label='Self Noise 240', linewidth=1, alpha=0.7) ax.loglog(adcnoise, 'r--', label='ADC Noise', linewidth=1, alpha=0.7) ax.loglog(ampnoise, 'g--', label='Amp Noise', linewidth=1, alpha=0.7)
# convert with tf asd1 = vel2vel(asd1) asd2 = vel2vel(asd2) asd3 = vel2vel(asd3) adc = vel2vel(adc) noise12 = vel2vel(noise12) noise13 = vel2vel(noise13) signal12 = vel2vel(signal12) signal13 = vel2vel(signal13) print('convert with tf') # plot psd with noise plot = Plot() ax = plot.gca(xscale='log', xlim=(1e-3, 3e2), xlabel='Frequency [Hz]', yscale='log', ylim=(1e-5, 3e-0), ylabel=r'Velocity [um/sec/\rtHz]') ax.plot(_f, _selfnoise, '-', linewidth=1, color='gray') ax.plot(asd1, label='1', color='black', linewidth=3) #ax.plot(asd2,label='2',color='red',linewidth=1) #ax.plot(asd3,label='3',color='blue',linewidth=1) #ax.plot(adc,label='adc',color='green',linewidth=1) #ax.plot(noise13,'o',label='Noise13 : IXV*(1-coh13) ',markersize=1) #ax.plot(noise12,'o',label='Noise12 : IXV*(1-coh12) ',markersize=1) ax.plot(signal13, label='Signal13 : IXV*coh13 ', markersize=1) ax.plot(signal12, label='Signal12 : IXV*coh12 ', markersize=1) #ax.plot(signal_local,label='Local Signal') ax.legend() plot.savefig('ASD.png') print 'plot in ASD.png'
# Now, we can loop through the active segments of ``'H1_DATA'`` and fetch the # strain `TimeSeries` for each segment, calculating a # :class:`~gwpy.spectrogram.Spectrogram` for each segment. from gwpy.timeseries import TimeSeries spectrograms = [] for start, end in h1segs.active: h1strain = TimeSeries.fetch_open_data('H1', start, end, verbose=True) specgram = h1strain.spectrogram(30, fftlength=4)**(1 / 2.) spectrograms.append(specgram) # Finally, we can build a :meth:`~gwpy.spectrogram.Spectrogram.plot`: from gwpy.plot import Plot plot = Plot(figsize=(12, 6)) ax = plot.gca() for specgram in spectrograms: ax.imshow(specgram) ax.set_xscale('auto-gps', epoch='Sep 16 2010') ax.set_xlim('Sep 16 2010', 'Sep 17 2010') ax.set_ylim(40, 2000) ax.set_yscale('log') ax.set_ylabel('Frequency [Hz]') ax.set_title('LIGO-Hanford strain data') ax.colorbar(cmap='viridis', norm='log', clim=(1e-23, 1e-19), label=r'Strain noise [1/\rtHz]') plot.add_segments_bar(h1segs) plot.show()
def main(args=None): """Run the primary scattering command-line tool """ parser = create_parser() args = parser.parse_args(args=args) # set up logger logger = cli.logger( name=PROG.split('python -m ').pop(), level='DEBUG' if args.verbose else 'INFO', ) # useful variables fthresh = ( int(args.frequency_threshold) if args.frequency_threshold.is_integer() else args.frequency_threshold) multiplier = args.multiplier_for_threshold tstr = str(fthresh).replace('.', '_') gpsstr = '%s-%s' % (int(args.gpsstart), int(args.gpsend - args.gpsstart)) args.optic = args.optic or list(OPTIC_MOTION_CHANNELS.keys()) # go to working directory indir = os.getcwd() if not os.path.isdir(args.output_dir): os.makedirs(args.output_dir) os.chdir(args.output_dir) # set up output files summfile = '{}-SCATTERING_SUMMARY-{}.csv'.format( args.ifo, gpsstr) segfile = '{}-SCATTERING_SEGMENTS_{}_HZ-{}.h5'.format( args.ifo, tstr, gpsstr) # log start of process logger.info('{} Scattering {}-{}'.format( args.ifo, int(args.gpsstart), int(args.gpsend))) # -- get state segments ----------- span = Segment(args.gpsstart, args.gpsend) # get segments if args.state_flag is not None: state = DataQualityFlag.query( args.state_flag, int(args.gpsstart), int(args.gpsend), url=DEFAULT_SEGMENT_SERVER, ).coalesce() statea = [] padding = args.segment_start_pad + args.segment_end_pad for i, seg in enumerate(state.active): if abs(seg) > padding: statea.append(Segment( seg[0] + args.segment_start_pad, seg[1] - args.segment_end_pad, )) else: logger.debug( "Segment length {} shorter than padding length {}, " "skipping segment {}-{}".format(abs(seg), padding, *seg), ) statea = SegmentList(statea) logger.debug("Downloaded %d segments for %s" % (len(statea), args.state_flag)) else: statea = SegmentList([span]) livetime = float(abs(statea)) logger.debug("Processing %.2f s of livetime" % livetime) # -- load h(t) -------------------- args.main_channel = args.main_channel.format(IFO=args.ifo) logger.debug("Loading Omicron triggers for %s" % args.main_channel) if args.gpsstart >= 1230336018: # Jan 1 2019 ext = "h5" names = ["time", "frequency", "snr"] read_kw = { "columns": names, "selection": [ "{0} < frequency < {1}".format( args.fmin, multiplier * fthresh), ("time", in_segmentlist, statea), ], "format": "hdf5", "path": "triggers", } else: ext = "xml.gz" names = ['peak', 'peak_frequency', 'snr'] read_kw = { "columns": names, "selection": [ "{0} < peak_frequency < {1}".format( args.fmin, multiplier * fthresh), ('peak', in_segmentlist, statea), ], "format": 'ligolw', "tablename": "sngl_burst", } fullcache = [] for seg in statea: cache = gwtrigfind.find_trigger_files( args.main_channel, 'omicron', seg[0], seg[1], ext=ext, ) if len(cache) == 0: warnings.warn( "No Omicron triggers found for %s in segment [%d .. %d)" % (args.main_channel, seg[0], seg[1]), ) continue fullcache.extend(cache) # read triggers if fullcache: trigs = EventTable.read(fullcache, nproc=args.nproc, **read_kw) else: # no files (no livetime?) trigs = EventTable(names=names) highsnrtrigs = trigs[trigs['snr'] >= 8] logger.debug("%d read" % len(trigs)) # -- prepare HTML ----------------- links = [ '%d-%d' % (int(args.gpsstart), int(args.gpsend)), ('Parameters', '#parameters'), ('Segments', ( ('State flag', '#state-flag'), ('Optical sensors', '#osems'), ('Transmons', '#transmons'), )), ] if args.omega_scans: links.append(('Scans', '#omega-scans')) (brand, class_) = htmlio.get_brand(args.ifo, 'Scattering', args.gpsstart) navbar = htmlio.navbar(links, class_=class_, brand=brand) page = htmlio.new_bootstrap_page( title='%s Scattering | %d-%d' % ( args.ifo, int(args.gpsstart), int(args.gpsend)), navbar=navbar) page.div(class_='pb-2 mt-3 mb-2 border-bottom') page.h1('%s Scattering: %d-%d' % (args.ifo, int(args.gpsstart), int(args.gpsend))) page.div.close() # pb-2 mt-3 mb-2 border-bottom page.h2('Parameters', class_='mt-4 mb-4', id_='parameters') page.div(class_='row') page.div(class_='col-md-9 col-sm-12') page.add(htmlio.parameter_table( start=int(args.gpsstart), end=int(args.gpsend), flag=args.state_flag)) page.div.close() # col-md-9 col-sm-12 # link to summary files page.div(class_='col-md-3 col-sm-12') page.add(htmlio.download_btn( [('Segments (HDF)', segfile), ('Triggers (CSV)', summfile)], btnclass='btn btn-%s dropdown-toggle' % args.ifo.lower(), )) page.div.close() # col-md-3 col-sm-12 page.div.close() # row # command-line page.h5('Command-line:') page.add(htmlio.get_command_line(about=False, prog=PROG)) # section header page.h2('Segments', class_='mt-4', id_='segments') if statea: # contextual information paper = markup.oneliner.a( 'Accadia et al. (2010)', target='_blank', class_='alert-link', href='http://iopscience.iop.org/article/10.1088/0264-9381/27' '/19/194011') msg = ( "Segments marked \"optical sensors\" below show evidence of beam " "scattering between {0} and {1} Hz based on the velocity of optic " "motion, with fringe frequencies projected using equation (3) of " "{2}. Segments marked \"transmons\" are based on whitened, " "band-limited RMS trends of transmon sensors. In both cases, " "yellow panels denote weak evidence for scattering, while red " "panels denote strong evidence." ).format(args.fmin, multiplier * fthresh, str(paper)) page.add(htmlio.alert(msg, context=args.ifo.lower())) else: # null segments page.add(htmlio.alert('No active analysis segments were found', context='warning', dismiss=False)) # record state segments if args.state_flag is not None: page.h3('State flag', class_='mt-3', id_='state-flag') page.div(id_='accordion1') page.add(htmlio.write_flag_html( state, span, 'state', parent='accordion1', context='success', plotdir='', facecolor=(0.2, 0.8, 0.2), edgecolor='darkgreen', known={'facecolor': 'red', 'edgecolor': 'darkred', 'height': 0.4})) page.div.close() # -- find scattering evidence ----- # read data for OSEMs and transmons osems = ['%s:%s' % (args.ifo, c) for optic in args.optic for c in OPTIC_MOTION_CHANNELS[optic]] transmons = ['%s:%s' % (args.ifo, c) for c in TRANSMON_CHANNELS] allchannels = osems + transmons logger.info("Reading all timeseries data") alldata = [] n = len(statea) for i, seg in enumerate(statea): msg = "{0}/{1} {2}:".rjust(30).format( str(i + 1).rjust(len(str(n))), n, str(seg), ) if args.verbose else False alldata.append( get_data(allchannels, seg[0], seg[1], frametype=args.frametype.format(IFO=args.ifo), verbose=msg, nproc=args.nproc).resample(128)) try: # ensure that only available channels are analyzed osems = list( set(alldata[0].keys()) & set(alldata[-1].keys()) & set(osems)) transmons = list( set(alldata[0].keys()) & set(alldata[-1].keys()) & set(transmons)) except IndexError: osems = [] transmons = [] # initialize scattering segments scatter_segments = DataQualityDict() actives = SegmentList() # scattering based on OSEM velocity if statea: page.h3('Optical sensors (OSEMs)', class_='mt-3', id_='osems') page.div(id_='osems-group') logger.info('Searching for scatter based on OSEM velocity') for i, channel in enumerate(sorted(osems)): logger.info("-- Processing %s --" % channel) chanstr = re.sub('[:-]', '_', channel).replace('_', '-', 1) optic = channel.split('-')[1].split('_')[0] flag = '%s:DCH-%s_SCATTERING_GE_%s_HZ:1' % (args.ifo, optic, tstr) scatter_segments[channel] = DataQualityFlag( flag, isgood=False, description="Evidence for scattering above {0} Hz from {1} in " "{2}".format(fthresh, optic, channel), ) # set up plot(s) plot = Plot(figsize=[12, 12]) axes = {} axes['position'] = plot.add_subplot( 411, xscale='auto-gps', xlabel='') axes['fringef'] = plot.add_subplot( 412, sharex=axes['position'], xlabel='') axes['triggers'] = plot.add_subplot( 413, sharex=axes['position'], xlabel='') axes['segments'] = plot.add_subplot( 414, projection='segments', sharex=axes['position']) plot.subplots_adjust(bottom=.07, top=.95) fringecolors = [None] * len(FREQUENCY_MULTIPLIERS) histdata = dict((x, numpy.ndarray((0,))) for x in FREQUENCY_MULTIPLIERS) linecolor = None # loop over state segments and find scattering fringes for j, seg in enumerate(statea): logger.debug("Processing segment [%d .. %d)" % seg) ts = alldata[j][channel] # get raw data and plot line = axes['position'].plot(ts, color=linecolor)[0] linecolor = line.get_color() # get fringe frequency and plot fringef = get_fringe_frequency(ts, multiplier=1) for k, m in list(enumerate(FREQUENCY_MULTIPLIERS))[::-1]: fm = fringef * m line = axes['fringef'].plot( fm, color=fringecolors[k], label=(j == 0 and r'$f\times%d$' % m or None))[0] fringecolors[k] = line.get_color() histdata[m] = numpy.resize( histdata[m], (histdata[m].size + fm.size,)) histdata[m][-fm.size:] = fm.value # get segments and plot scatter = get_segments( fringef * multiplier, fthresh, name=flag, pad=args.segment_padding ) axes['segments'].plot( scatter, facecolor='red', edgecolor='darkred', known={'alpha': 0.6, 'facecolor': 'lightgray', 'edgecolor': 'gray', 'height': 0.4}, height=0.8, y=0, label=' ', ) scatter_segments[channel] += scatter logger.debug( " Found %d scattering segments" % (len(scatter.active))) logger.debug("Completed channel %s, found %d segments in total" % (channel, len(scatter_segments[channel].active))) # calculate efficiency and deadtime of veto deadtime = abs(scatter_segments[channel].active) try: deadtimepc = deadtime / livetime * 100 except ZeroDivisionError: deadtimepc = 0. logger.info("Deadtime: %.2f%% (%.2f/%ds)" % (deadtimepc, deadtime, livetime)) efficiency = in_segmentlist(highsnrtrigs[names[0]], scatter_segments[channel].active).sum() try: efficiencypc = efficiency / len(highsnrtrigs) * 100 except ZeroDivisionError: efficiencypc = 0. logger.info("Efficiency (SNR>=8): %.2f%% (%d/%d)" % (efficiencypc, efficiency, len(highsnrtrigs))) if deadtimepc == 0.: effdt = 0 else: effdt = efficiencypc/deadtimepc logger.info("Efficiency/Deadtime: %.2f" % effdt) if abs(scatter_segments[channel].active): actives.extend(scatter_segments[channel].active) # finalize plot logger.debug("Plotting") name = texify(channel) axes['position'].set_title("Scattering evidence in %s" % name) axes['position'].set_xlabel('') axes['position'].set_ylabel(r'Position [$\mu$m]') axes['position'].text( 0.01, 0.95, 'Optic position', transform=axes['position'].transAxes, va='top', ha='left', bbox={'edgecolor': 'none', 'facecolor': 'white', 'alpha': .5}) axes['fringef'].plot( span, [fthresh, fthresh], 'k--') axes['fringef'].set_xlabel('') axes['fringef'].set_ylabel(r'Frequency [Hz]') axes['fringef'].yaxis.tick_right() axes['fringef'].yaxis.set_label_position("right") axes['fringef'].set_ylim(0, multiplier * fthresh) axes['fringef'].text( 0.01, 0.95, 'Calculated fringe frequency', transform=axes['fringef'].transAxes, va='top', ha='left', bbox={'edgecolor': 'none', 'facecolor': 'white', 'alpha': .5}) handles, labels = axes['fringef'].get_legend_handles_labels() axes['fringef'].legend(handles[::-1], labels[::-1], loc='upper right', borderaxespad=0, bbox_to_anchor=(-0.01, 1.), handlelength=1) axes['triggers'].scatter( trigs[names[0]], trigs[names[1]], c=trigs[names[2]], edgecolor='none', ) name = texify(args.main_channel) axes['triggers'].text( 0.01, 0.95, '%s event triggers (Omicron)' % name, transform=axes['triggers'].transAxes, va='top', ha='left', bbox={'edgecolor': 'none', 'facecolor': 'white', 'alpha': .5}) axes['triggers'].set_ylabel('Frequency [Hz]') axes['triggers'].set_ylim(args.fmin, multiplier * fthresh) axes['triggers'].colorbar(cmap='YlGnBu', clim=(3, 100), norm='log', label='Signal-to-noise ratio') axes['segments'].set_ylim(-.55, .55) axes['segments'].text( 0.01, 0.95, r'Time segments with $f\times%d > %.2f$ Hz' % ( multiplier, fthresh), transform=axes['segments'].transAxes, va='top', ha='left', bbox={'edgecolor': 'none', 'facecolor': 'white', 'alpha': .5}) for ax in axes.values(): ax.set_epoch(int(args.gpsstart)) ax.set_xlim(*span) png = '%s_SCATTERING_%s_HZ-%s.png' % (chanstr, tstr, gpsstr) try: plot.save(png) except OverflowError as e: warnings.warn(str(e)) plot.axes[1].set_ylim(0, multiplier * fthresh) plot.refresh() plot.save(png) plot.close() logger.debug("%s written." % png) # make histogram histogram = Plot(figsize=[12, 6]) ax = histogram.gca() hrange = (0, multiplier * fthresh) for m, color in list(zip(histdata, fringecolors))[::-1]: if histdata[m].size: ax.hist( histdata[m], facecolor=color, alpha=.6, range=hrange, bins=50, histtype='stepfilled', label=r'$f\times%d$' % m, cumulative=-1, weights=ts.dx.value, bottom=1e-100, log=True) else: ax.plot(histdata[m], color=color, label=r'$f\times%d$' % m) ax.set_yscale('log') ax.set_ylim(.01, float(livetime)) ax.set_ylabel('Time with fringe above frequency [s]') ax.set_xlim(*hrange) ax.set_xlabel('Frequency [Hz]') ax.set_title(axes['position'].get_title()) handles, labels = ax.get_legend_handles_labels() ax.legend(handles[::-1], labels[::-1], loc='upper right') hpng = '%s_SCATTERING_HISTOGRAM-%s.png' % (chanstr, gpsstr) histogram.save(hpng) histogram.close() logger.debug("%s written." % hpng) # write HTML if deadtime != 0 and effdt > 2: context = 'danger' elif ((deadtime != 0 and effdt < 2) or (histdata[multiplier].size and histdata[multiplier].max() >= fthresh/2.)): context = 'warning' else: continue page.div(class_='card border-%s mb-1 shadow-sm' % context) page.div(class_='card-header text-white bg-%s' % context) page.a(channel, class_='collapsed card-link cis-link', href='#osem%s' % i, **{'data-toggle': 'collapse'}) page.div.close() # card-header page.div(id_='osem%s' % i, class_='collapse', **{'data-parent': '#osems-group'}) page.div(class_='card-body') page.div(class_='row') img = htmlio.FancyPlot( png, caption=SCATTER_CAPTION.format(CHANNEL=channel)) page.div(class_='col-md-10 offset-md-1') page.add(htmlio.fancybox_img(img)) page.div.close() # col-md-10 offset-md-1 himg = htmlio.FancyPlot( hpng, caption=HIST_CAPTION.format(CHANNEL=channel)) page.div(class_='col-md-10 offset-md-1') page.add(htmlio.fancybox_img(himg)) page.div.close() # col-md-10 offset-md-1 page.div.close() # row segs = StringIO() if deadtime: page.p("%d segments were found predicting a scattering fringe " "above %.2f Hz." % ( len(scatter_segments[channel].active), fthresh)) page.table(class_='table table-sm table-hover') page.tbody() page.tr() page.th('Deadtime') page.td('%.2f/%d seconds' % (deadtime, livetime)) page.td('%.2f%%' % deadtimepc) page.tr.close() page.tr() page.th('Efficiency<br><small>(SNR≥8 and ' '%.2f Hz</sub><f<sub>peak</sub><%.2f Hz)</small>' % (args.fmin, multiplier * fthresh)) page.td('%d/%d events' % (efficiency, len(highsnrtrigs))) page.td('%.2f%%' % efficiencypc) page.tr.close() page.tr() page.th('Efficiency/Deadtime') page.td() page.td('%.2f' % effdt) page.tr.close() page.tbody.close() page.table.close() scatter_segments[channel].active.write(segs, format='segwizard', coltype=float) page.pre(segs.getvalue()) else: page.p("No segments were found with scattering above %.2f Hz." % fthresh) page.div.close() # card-body page.div.close() # collapse page.div.close() # card if statea: # close accordion page.div.close() # osems-group # scattering based on transmon BLRMS if statea: page.h3('Transmons', class_='mt-3', id_='transmons') page.div(id_='transmons-group') logger.info('Searching for scatter based on band-limited RMS of transmons') for i, channel in enumerate(sorted(transmons)): logger.info("-- Processing %s --" % channel) optic = channel.split('-')[1][:6] flag = '%s:DCH-%s_SCATTERING_BLRMS:1' % (args.ifo, optic) scatter_segments[channel] = DataQualityFlag( flag, isgood=False, description="Evidence for scattering from whitened, band-limited " "RMS trends of {0}".format(channel), ) # loop over state segments and compute BLRMS for j, seg in enumerate(statea): logger.debug("Processing segment [%d .. %d)" % seg) wblrms = get_blrms( alldata[j][channel], flow=args.bandpass_flow, fhigh=args.bandpass_fhigh, ) scatter = get_segments( wblrms, numpy.mean(wblrms) + args.sigma * numpy.std(wblrms), name=flag, ) scatter_segments[channel] += scatter logger.debug( " Found %d scattering segments" % (len(scatter.active))) logger.debug("Completed channel %s, found %d segments in total" % (channel, len(scatter_segments[channel].active))) # calculate efficiency and deadtime of veto deadtime = abs(scatter_segments[channel].active) try: deadtimepc = deadtime / livetime * 100 except ZeroDivisionError: deadtimepc = 0. logger.info("Deadtime: %.2f%% (%.2f/%ds)" % (deadtimepc, deadtime, livetime)) highsnrtrigs = trigs[trigs['snr'] <= 200] efficiency = in_segmentlist(highsnrtrigs[names[0]], scatter_segments[channel].active).sum() try: efficiencypc = efficiency / len(highsnrtrigs) * 100 except ZeroDivisionError: efficiencypc = 0. logger.info("Efficiency (SNR>=8): %.2f%% (%d/%d)" % (efficiencypc, efficiency, len(highsnrtrigs))) if deadtimepc == 0.: effdt = 0 else: effdt = efficiencypc/deadtimepc logger.info("Efficiency/Deadtime: %.2f" % effdt) if abs(scatter_segments[channel].active): actives.extend(scatter_segments[channel].active) # write HTML if deadtime != 0 and effdt > 2: context = 'danger' elif deadtime != 0 and effdt < 2: context = 'warning' else: continue page.add(htmlio.write_flag_html( scatter_segments[channel], span, i, parent='transmons-group', title=channel, context=context, plotdir='')) if statea: # close accordion page.div.close() # transmons-group actives = actives.coalesce() # merge contiguous segments if statea and not actives: page.add(htmlio.alert( 'No evidence of scattering found in the channels analyzed', context=args.ifo.lower(), dismiss=False)) # identify triggers during active segments logger.debug('Writing a summary CSV record') ind = [i for i, trigtime in enumerate(highsnrtrigs[names[0]]) if trigtime in actives] gps = highsnrtrigs[names[0]][ind] freq = highsnrtrigs[names[1]][ind] snr = highsnrtrigs[names[2]][ind] segs = [y for x in gps for y in actives if x in y] table = EventTable( [gps, freq, snr, [seg[0] for seg in segs], [seg[1] for seg in segs]], names=('trigger_time', 'trigger_frequency', 'trigger_snr', 'segment_start', 'segment_end')) logger.info('The following {} triggers fell within active scattering ' 'segments:\n\n'.format(len(table))) print(table) print('\n\n') table.write(summfile, overwrite=True) # -- launch omega scans ----------- nscans = min(args.omega_scans, len(table)) if nscans > 0: # launch scans scandir = 'scans' ind = random.sample(range(0, len(table)), nscans) omegatimes = [str(t) for t in table['trigger_time'][ind]] logger.debug('Collected {} event times to omega scan: {}'.format( nscans, ', '.join(omegatimes))) logger.info('Creating workflow for omega scans') flags = batch.get_command_line_flags( ifo=args.ifo, ignore_state_flags=True) condorcmds = batch.get_condor_arguments(timeout=4, gps=args.gpsstart) batch.generate_dag(omegatimes, flags=flags, submit=True, outdir=scandir, condor_commands=condorcmds) logger.info('Launched {} omega scans to condor'.format(nscans)) # render HTML page.h2('Omega scans', class_='mt-4', id_='omega-scans') msg = ( 'The following event times correspond to significant Omicron ' 'triggers that occur during the scattering segments found above. ' 'To compare these against fringe frequency projections, please ' 'use the "simple scattering" module:', markup.oneliner.pre( '$ python -m gwdetchar.scattering.simple --help', ), ) page.add(htmlio.alert(msg, context=args.ifo.lower())) page.add(htmlio.scaffold_omega_scans( omegatimes, args.main_channel, scandir=scandir)) elif args.omega_scans: logger.info('No events found during active scattering segments') # -- finalize --------------------- # write segments scatter_segments.write(segfile, path="segments", overwrite=True) logger.debug("%s written" % segfile) # write HTML htmlio.close_page(page, 'index.html') logger.info("-- index.html written, all done --") # return to original directory os.chdir(indir)