def test_write_segments_ascii(self): for ncol in [2, 4]: with NamedTemporaryFile(suffix='.txt', delete=False) as f: segments.write_ascii(f.name, TEST_SEGMENTS, ncol=ncol) f.delete = True a = SegmentList.read(f.name, gpstype=float, strict=False) self.assertEqual(a, TEST_SEGMENTS_2)
def test_write_segments_ascii(self): for ncol in [2, 4]: with NamedTemporaryFile(suffix='.txt', delete=False) as f: segments.write_ascii(f.name, TEST_SEGMENTS, ncol=ncol) f.delete = True a = SegmentList.read(f.name, gpstype=float, strict=False) self.assertEqual(a, TEST_SEGMENTS_2)
def test_write_segments_ascii(ncol, tmpdir): outdir = str(tmpdir) out = os.path.join(outdir, 'test.txt') segments.write_ascii(out, TEST_SEGMENTS, ncol=ncol) a = SegmentList.read(out, gpstype=float, strict=False) assert a == TEST_SEGMENTS_2 # clean up shutil.rmtree(outdir, ignore_errors=True)
def read_segmentlist(total,skip=True,**kwargs): ''' ''' if skip: log.debug('Skip chekking segment') total = SegmentList.read('./segmentlist/total.txt') none = SegmentList.read('./segmentlist/nodata.txt') good = SegmentList.read('./segmentlist/available.txt') lack = SegmentList.read('./segmentlist/lackofdata.txt') glitch = SegmentList.read('./segmentlist/glitch.txt') else: good,none = check_nodata(total,skip=True,**kwargs) good,lack,glitch = check_baddata(good,**kwargs) log.debug('Checking done. Close.') exit() if not (len(total)-len(none)-len(lack)-len(glitch)==len(good)): log.debug('SegmentListError!') raise ValueError('Missmatch SegmentLists!') return good,none,lack,glitch
def grab_time_triggers(glob_wildcard): time_segs = SegmentList([]) start_time_utc = tconvert(args.gps_start_time) for filename in glob.glob(glob_wildcard): data = SegmentList.read(filename) print 'grabbing trigger file:' + filename time_segs += data # print time_segs start_time_utc += datetime.timedelta(days=1) return time_segs
def grab_time_triggers(glob_wildcard): time_segs = SegmentList([]) start_time_utc = tconvert(args.gps_start_time) for filename in glob.glob(glob_wildcard): data = SegmentList.read(filename) print 'grabbing trigger file:' + filename time_segs += data # print time_segs start_time_utc += datetime.timedelta(days=1) return time_segs
def grab_time_triggers(wildcard, start, end): """Retrieve triggers from a given GPS time range """ time_segs = SegmentList([]) start_time_utc = tconvert(start) for filename in glob.glob(wildcard): data = SegmentList.read(filename) LOGGER.info(' '.join(['grabbing trigger file:', filename])) start_end_seg = Segment(start, end) c = data & SegmentList([start_end_seg]) time_segs += c start_time_utc += datetime.timedelta(days=1) return time_segs
def check_nodata(segmentlist, prefix='./data', write=True, skip=False, **kwargs): ''' ''' from gwpy.segments import SegmentList if not skip: log.debug('Find segments') # find unchecked segments exists = iofunc.existance(segmentlist, ftype='gwf') not_checked = [ segmentlist[i] for i, exist in enumerate(exists) if not exist ] log.debug('{0}(/{1}) are not checked'.format(len(not_checked), len(segmentlist))) n = len(not_checked) #ans = [_check_nodata(segment,**kwargs)[1] for segment in not_checked] ans = [ _check_nodata(segment, headder='{0:04d}(/{1:04d})'.format(i, n), **kwargs)[1] for i, segment in enumerate(not_checked) ] # nodata segments nodata = SegmentList( [not_checked[i] for i, _ans in enumerate(ans) if 'NoData' in _ans]) else: nodata = SegmentList.read('./segmentlist/nodata.txt') # exist segments exist = diff(segmentlist, nodata) if len(exist) == 0: log.debug('No data are existed...') raise ValueError('No data Error.') if write: exist.write('./segmentlist/exist.txt') nodata.write('./segmentlist/nodata.txt') log.debug('./segmentlist/exist.txt Saved') log.debug('./segmentlist/nodata.txt Saved') log.debug('{0} segments are existed'.format(len(exist))) return exist, nodata
def main(args=None): """Run the hveto-trace command-line tool """ parser = create_parser() args = parser.parse_args(args=args) # initialize variables time = float(args.trigger_time) segment = None # initialize logger logger = cli.logger(name=PROG.split('python -m ').pop(), level=args.loglevel) logger.debug('Running in verbose mode') logger.debug('Search directory: {}'.format(args.directory)) try: # read veto segment statistics segment_stats = json.load( open(os.path.join(args.directory, 'summary-stats.json'))) except IOError: logger.critical("'summary-stats.json' was not found " "in the input directory") raise # loop over and log results to output for (i, cround) in enumerate(segment_stats['rounds']): seg_files = filter(lambda f_name: '.txt' in f_name, cround[u'files'][u'VETO_SEGS']) for f in seg_files: segments = SegmentList.read(os.path.join(args.directory, f)) if time in segments: segment = segments[segments.find(time)] logger.info('Trigger time {0} was vetoed in round {1} by ' 'segment {2}'.format(time, (i + 1), segment)) logger.debug('Round winner: {}'.format(cround['name'])) logger.debug('Significance: {}'.format(cround['significance'])) logger.debug('SNR: {}'.format(cround['snr'])) logger.debug('Window: {}'.format(cround['window'])) if segment is None: # if we got here, the signal was not vetoed logger.info('Trigger time {} was not vetoed'.format(time))
def main(args=None): """Run the trace tool """ parser = create_parser() args = parser.parse_args(args=args) directory = args.directory logger = cli.logger(name='hveto.trace', level=args.loglevel) logger.debug('Running in verbose mode') logger.debug('Search directory: %s' % directory) trigger_time = float(args.trigger_time) if directory[-1] != '/': directory += '/' try: segment_stats = json.load(open('%ssummary-stats.json' % directory)) except IOError: logger.error("'summary-stats.json' was not found " "in the input directory") sys.exit(0) for i, cround in enumerate(segment_stats['rounds']): seg_files = filter(lambda f_name: '.txt' in f_name, cround[u'files'][u'VETO_SEGS']) for f in seg_files: segments = SegmentList.read(os.path.join(directory, f)) for segment in segments: if segment[0] <= trigger_time <= segment[1]: logger.info('Signal was vetoed in round %d by ' 'segment %s' % ((i + 1), segment)) logger.debug('Winner: %s' % cround['name']) logger.debug('Significance: %s' % cround['significance']) logger.debug('SNR: %s' % cround['snr']) logger.debug('Window: %s' % cround['window']) sys.exit(0) logger.info('Signal was not vetoed.')
def test_write_segments_ascii(ncol): with NamedTemporaryFile(suffix='.txt', delete=False) as tmp: segments.write_ascii(tmp.name, TEST_SEGMENTS, ncol=ncol) tmp.delete = True a = SegmentList.read(tmp.name, gpstype=float, strict=False) assert a == TEST_SEGMENTS_2
#! /usr/bin/env python from __future__ import (division, print_function) import sys from gwpy.segments import SegmentList from gwpy.table.lsctables import SnglInspiralTable import matplotlib matplotlib.use('Agg') import matplotlib.pyplot as plt import numpy as np bbh_file = sys.argv[1] trigs = SnglInspiralTable.read(bbh_file) segs = SegmentList.read('L1_ER7_segments.txt') trigs = trigs.vetoed(segs) plot = trigs.plot('time', 'snr', edgecolor='none')#, epoch=1117378816) #plot.set_xlim(1117378816, 1117378816+(24*3600*11.0)) plot.set_ylabel('SNR') plot.set_yscale('log', nonposy='clip') plot.set_title('BBH triggers during the ER7 run') plot.savefig('H1_BBH_SNR.png')
def test_read_segwizard(self): active = SegmentList.read(SEGWIZ, coalesce=False) self.assertTrue(active == ACTIVE, 'SegmentList.read(segwizard) mismatch:\n\n%s\n\n%s' % (ACTIVE, active))
def process(self, config=GWSummConfigParser(), **kwargs): # set params self.rounds = None if not os.path.isdir(self.directory): self.rounds = None return # get some basic info ifo = config.get('DEFAULT', 'ifo') # read the configuration d = os.path.realpath(self.directory).rstrip('/') self.conf = dict() confs = glob(os.path.join(d, '%s-HVETO_CONF-*-*.txt' % ifo)) if len(confs) != 1: self.rounds = 'FAIL' return conffile = confs[0] try: with open(conffile) as f: self.conf = dict() lines = f.readlines()[3:] for line in lines: try: key, val = line.split(': ', 1) self.conf[key.strip()] = eval(val) except (ValueError, SyntaxError, NameError): pass except IOError: self.rounds = 'FAIL' return else: etg = self.conf.pop('AUXtype', None) if 'DEfnm' in self.conf: name = re_quote.sub('', self.conf['DEfnm']) self.primary = '%s:%s' % (ifo, name) if 'DEtype' in self.conf: hetg = re_quote.sub('', self.conf['DEtype']) if re.search('_%s\Z' % hetg, self.primary, re.I): self.primary = self.primary[:-len(hetg)-1] else: self.primary = None # find the segments try: ce = CacheEntry.from_T050017(conffile) except (ValueError): start = int(self.span[0]) duration = int(abs(self.span)) span = self.span else: start = int(ce.segment[0]) duration = int(abs(ce.segment)) span = ce.segment try: statefile = self.conf['dqfnm'] except KeyError: statefile = '%s-HVETO_DQ_SEGS-%d-%d.txt' % (ifo, start, duration) if not os.path.isfile(os.path.join(self.directory, statefile)): self.rounds = 'NOSEGMENTS' return # find the results table resultsfile = os.path.join(self.directory, 'summary_stats.txt') if not os.path.isfile(resultsfile): self.rounds = 'FAIL' return # determine the Hveto state cache = Cache([CacheEntry.from_T050017( os.path.join(self.directory, statefile))]) segments = SegmentList.read(cache) globalv.SEGMENTS[self.states[0].definition] = DataQualityFlag( self.states[0].definition, known=[span], active=segments) self.finalize_states(config=config, query=False) # read results file self.rounds = [] with open(resultsfile, 'r') as f: for line in f.readlines(): self.rounds.append(dict(zip(self.summaryrows, line.split(' ')[1:]))) # fix channel name c = '%s:%s' % (ifo, self.rounds[-1]['Winning channel']) if etg and re.search('_%s\Z' % etg, c, re.I): c = c.rsplit('_', 1)[0] self.rounds[-1]['Winning channel'] = c # read starting triggers rawfile = ('%s-HVETO_RAW_TRIGS_ROUND_0-%d-%d.txt' % (ifo, start, duration)) cache = Cache([CacheEntry.from_T050017( os.path.join(self.directory, rawfile))]) get_triggers('%s:hveto_start' % ifo, 'hveto', [self.span], config=config, cache=cache, return_=False) get_triggers('%s:hveto_vetoed_all' % ifo, 'hveto', [self.span], config=config, cache=Cache(), return_=False) for r in range(1, len(self.rounds) + 1): # read round veto triggers rawfile = ('%s-HVETO_VETOED_TRIGS_ROUND_%d-%d-%d.txt' % (ifo, r, start, duration)) cache = Cache([CacheEntry.from_T050017( os.path.join(self.directory, rawfile))]) trigs = get_triggers('%s:hveto_vetoed_round %d' % (ifo, r), 'hveto', [self.span], config=config, cache=cache) add_triggers(trigs, '%s:hveto_vetoed_all,hveto' % ifo, segments=SegmentList([self.span])) # read round veto segments segfile = ('%s-HVETO_VETO_SEGS_ROUND_%d-%d-%d.txt' % (ifo, r, start, duration)) cache = Cache([CacheEntry.from_T050017( os.path.join(self.directory, segfile))]) get_segments('%s:hveto_veto_segs_round_%d' % (ifo, r), [self.span], config=config, cache=cache, return_=False) for plot in self.plots: if isinstance(plot, HvetoSegmentSummaryPlot): plot.find_flags() kwargs['trigcache'] = Cache() kwargs['segmentcache'] = Cache() super(HvetoTab, self).process(config=config, **kwargs) # find some plots for plot in ['OVERAL_HISTOGRAM', 'OVERAL_EFF_DT'][::-1]: filename = ( '%s-HVETO_%s-%d-%d.png' % (ifo, plot, start, duration)) plotfile = os.path.join(self.directory, filename) if os.path.isfile(plotfile): p = SummaryPlot(os.path.join(self.url, filename), new=False) p.state = self.states[0] self.plots.insert(0, p) # delete data from archive del globalv.SEGMENTS[self.states[0].definition] for row in range(1, len(self.rounds) + 1): del globalv.SEGMENTS['%s:hveto_veto_segs_round_%s' % (ifo, row)]
def test_read_segwizard(self): active = SegmentList.read(SEGWIZ, coalesce=False) self.assertTrue( active == ACTIVE, 'SegmentList.read(segwizard) mismatch:\n\n%s\n\n%s' % (ACTIVE, active))
def test_write_segments_ascii(ncol): with NamedTemporaryFile(suffix='.txt', delete=False) as tmp: segments.write_ascii(tmp.name, TEST_SEGMENTS, ncol=ncol) tmp.delete = True a = SegmentList.read(tmp.name, gpstype=float, strict=False) assert a == TEST_SEGMENTS_2
def read_segments(source, coltype=int): return SegmentList.read( source, gpstype=coltype, format="segwizard", )
pattern_trigs_hveto = os.path.join(args.directory_path, '{}{:02}','{}{:02}{:02}', '*86400-DARM','*VETO_SEGS_ROUND*.txt') start_time_utc = tconvert(args.gps_start_time) end_time_utc = tconvert(args.gps_end_time) triggers = SegmentList([]) while start_time_utc < end_time_utc: day = start_time_utc.day month = start_time_utc.month year = start_time_utc.year wildcard_trigs_hveto = pattern_trigs_hveto.format(year, month, year, month, day) triggers = SegmentList([]) #grabbing the trigger files for filename in glob.glob(wildcard_trigs_hveto): #loading the triggers in data = SegmentList.read(filename) print data triggers += data start_time_utc += datetime.timedelta(days=1) #triggers.coalesce() start_end_seg = Segment(args.gps_start_time, args.gps_end_time) triggers = triggers & SegmentList([start_end_seg]) #print triggers triggers.write("total_hveto_trigs.txt") start_time_utc += datetime.timedelta(days=1) #SEGMENT HANDLING: begin for loop that loops over the range of all days/months/years
def read_segments(source, coltype=int): return SegmentList.read( source, gpstype=coltype, format="segwizard", )
#"IMC-REFL_DC_OUT_DQ", #"ALS-X_REFL_ERR_OUT_DQ", #"PEM-CS_MAG_EBAY_SUSRACK_Y_DQ", #"ASC-Y_TR_B_NSUM_OUT_DQ", #"ASC-AS_B_RF45_Q_PIT_OUT_DQ", #"SUS-OMC_M1_ISIWIT_T_DQ", #"PSL-ISS_AOM_DRIVER_MON_OUT_DQ", #"LSC-PRCL_OUT_DQ"] ifo = sys.argv[1] bbhdir = sys.argv[2] bbhfile= glob.glob(os.path.join(bbhdir, ifo+'*.xml.gz'))[0] omiccachedir = sys.argv[3] # Read in the segment file segments = SegmentList.read('/home/albert.wandui/detchar'+\ '/ER7/jul13/%s_ER7_segments.txt' %ifo) # Read in the BBH triggers bbh_trigs = SnglInspiralTable.read(bbhfile) # We only want the triggers in the given segments bbh_trigs = bbh_trigs.vetoed(segments) #bbh_trigs.sort(key=lambda x: x.end_time + x.end_time_ns * 1.0e-9) print "Read in all the BBH triggers!!!\n" print "Let's start working on the Omicron triggers...\n" # ---------------------------------------------------------------------------- # # Read in all the Omicron caches # Also get an idea of the speed of the code when reading from cache file vs # letting vet get the data itself
# #! coding:utf-8 import numpy as np import matplotlib.pyplot as plt from astropy import units as u from gwpy.timeseries import TimeSeriesDict, TimeSeries from gwpy.segments import SegmentList from gwpy.time import tconvert from gwpy.detector import Channel ok = SegmentList.read('segments_locked.txt') segnum = 0 start = ok[segnum][0] end = ok[segnum][1] chname = 'K1:GIF-X_STRAIN_OUT16' gif_x = TimeSeries.read('./segment_{0}/{1}.gwf'.format(segnum, chname), chname, verbose=True, nproc=2) * 1e6 * 3000 # disp chname = 'K1:PEM-SEIS_IXV_GND_X_OUT16' ixv_x = TimeSeries.read('./segment_{0}/{1}.gwf'.format(segnum, chname), chname, verbose=True, nproc=2) chname = 'K1:PEM-SEIS_EXV_GND_X_OUT16' exv_x = TimeSeries.read('./segment_{0}/{1}.gwf'.format(segnum, chname), chname, verbose=True,
def process(self, config=GWSummConfigParser(), **kwargs): # set params self.rounds = None if not os.path.isdir(self.directory): self.rounds = None return # get some basic info ifo = config.get('DEFAULT', 'ifo') # read the configuration d = os.path.realpath(self.directory).rstrip('/') self.conf = dict() confs = glob(os.path.join(d, '%s-HVETO_CONF-*-*.txt' % ifo)) if len(confs) != 1: self.rounds = 'FAIL' return conffile = confs[0] try: with open(conffile) as f: self.conf = dict() lines = f.readlines()[3:] for line in lines: try: key, val = line.split(': ', 1) self.conf[key.strip()] = eval(val) except (ValueError, SyntaxError, NameError): pass except IOError: self.rounds = 'FAIL' return else: etg = self.conf.pop('AUXtype', None) if 'DEfnm' in self.conf: name = re_quote.sub('', self.conf['DEfnm']) self.primary = '%s:%s' % (ifo, name) if 'DEtype' in self.conf: hetg = re_quote.sub('', self.conf['DEtype']) if re.search('_%s\Z' % hetg, self.primary, re.I): self.primary = self.primary[:-len(hetg) - 1] else: self.primary = None # find the segments try: ce = CacheEntry.from_T050017(conffile) except (ValueError): start = int(self.span[0]) duration = int(abs(self.span)) span = self.span else: start = int(ce.segment[0]) duration = int(abs(ce.segment)) span = ce.segment try: statefile = self.conf['dqfnm'] except KeyError: statefile = '%s-HVETO_DQ_SEGS-%d-%d.txt' % (ifo, start, duration) if not os.path.isfile(os.path.join(self.directory, statefile)): self.rounds = 'NOSEGMENTS' return # find the results table resultsfile = os.path.join(self.directory, 'summary_stats.txt') if not os.path.isfile(resultsfile): self.rounds = 'FAIL' return # determine the Hveto state cache = Cache( [CacheEntry.from_T050017(os.path.join(self.directory, statefile))]) segments = SegmentList.read(cache) globalv.SEGMENTS[self.states[0].definition] = DataQualityFlag( self.states[0].definition, known=[span], active=segments) self.finalize_states(config=config, query=False) # read results file self.rounds = [] with open(resultsfile, 'r') as f: for line in f.readlines(): self.rounds.append( dict(zip(self.summaryrows, line.split(' ')[1:]))) # fix channel name c = '%s:%s' % (ifo, self.rounds[-1]['Winning channel']) if etg and re.search('_%s\Z' % etg, c, re.I): c = c.rsplit('_', 1)[0] self.rounds[-1]['Winning channel'] = c # read starting triggers rawfile = ('%s-HVETO_RAW_TRIGS_ROUND_0-%d-%d.txt' % (ifo, start, duration)) cache = Cache( [CacheEntry.from_T050017(os.path.join(self.directory, rawfile))]) get_triggers('%s:hveto_start' % ifo, 'hveto', [self.span], config=config, cache=cache, tablename='sngl_burst', return_=False) get_triggers('%s:hveto_vetoed_all' % ifo, 'hveto', [self.span], config=config, cache=Cache(), tablename='sngl_burst') for r in range(1, len(self.rounds) + 1): # read round veto triggers rawfile = ('%s-HVETO_VETOED_TRIGS_ROUND_%d-%d-%d.txt' % (ifo, r, start, duration)) cache = Cache([ CacheEntry.from_T050017(os.path.join(self.directory, rawfile)) ]) trigs = get_triggers('%s:hveto_vetoed_round %d' % (ifo, r), 'hveto', [self.span], config=config, cache=cache, tablename='sngl_burst') globalv.TRIGGERS['%s:hveto_vetoed_all,hveto' % ifo].extend(trigs) # read round veto segments segfile = ('%s-HVETO_VETO_SEGS_ROUND_%d-%d-%d.txt' % (ifo, r, start, duration)) cache = Cache([ CacheEntry.from_T050017(os.path.join(self.directory, segfile)) ]) get_segments('%s:hveto_veto_segs_round_%d' % (ifo, r), [self.span], config=config, cache=cache, return_=False) for plot in self.plots: if isinstance(plot, HvetoSegmentSummaryPlot): plot.find_flags() kwargs['trigcache'] = Cache() kwargs['segmentcache'] = Cache() super(HvetoTab, self).process(config=config, **kwargs) # find some plots for plot in ['OVERAL_HISTOGRAM', 'OVERAL_EFF_DT'][::-1]: filename = ('%s-HVETO_%s-%d-%d.png' % (ifo, plot, start, duration)) plotfile = os.path.join(self.directory, filename) if os.path.isfile(plotfile): p = SummaryPlot(os.path.join(self.url, filename), new=False) p.state = self.states[0] self.plots.insert(0, p) # delete data from archive del globalv.SEGMENTS[self.states[0].definition] for row in range(1, len(self.rounds) + 1): del globalv.SEGMENTS['%s:hveto_veto_segs_round_%s' % (ifo, row)]
def load_segs(): segs = SegmentList.read('L1_ER7_segments.txt') return segs
def main(args=None): use('agg') rcParams.update({ 'figure.subplot.bottom': 0.15, 'figure.subplot.left': 0.1, 'figure.subplot.right': 0.83, 'figure.subplot.top': 0.93, 'figure.subplot.hspace': 0.25, 'axes.labelsize': 20, 'grid.color': 'gray', }) grid = GridSpec(2, 1) logger = log.Logger('omicron-status') try: omicronversion = str(get_omicron_version()) except KeyError: omicronversion = 'Unknown' logger.warning("Omicron version unknown") else: logger.info("Found omicron version: %s" % omicronversion) parser = create_parser() args = parser.parse_args(args=args) if args.ifo is None: parser.error("Cannot determine IFO prefix from sytem, " "please pass --ifo on the command line") group = args.group logger.info("Checking status for %r group" % group) archive = args.archive_directory proddir = args.production_directory.with_name( args.production_directory.name.format(group=args.group), ) outdir = args.output_directory outdir.mkdir(exist_ok=True, parents=True) tag = args.latency_archive_tag.format(group=args.group) filetypes = ['h5', 'xml.gz', 'root'] logger.debug("Set output directory to %s" % outdir) logger.debug( "Will process the following filetypes: {}".format( ", ".join(filetypes), ), ) # -- parse configuration file and get parameters -------------------------- cp = configparser.ConfigParser() ok = cp.read(args.config_file) if args.config_file not in ok: raise IOError( "Failed to read configuration file %r" % args.config_file, ) logger.info("Configuration read") # validate if not cp.has_section(group): raise configparser.NoSectionError(group) # get parameters obs = args.ifo[0] frametype = cp.get(group, 'frametype') padding = cp.getint(group, 'overlap-duration') / 2. mingap = cp.getint(group, 'chunk-duration') channels = args.channel if not channels: channels = [ c.split()[0] for c in cp.get(group, 'channels').strip('\n').split('\n') ] channels.sort() logger.debug("Found %d channels" % len(channels)) start = args.gps_start_time end = args.gps_end_time if end == NOW: end -= padding if args.state_flag: stateflag = args.state_flag statepad = tuple(map(float, args.state_pad.split(','))) else: try: stateflag = cp.get(group, 'state-flag') except configparser.NoOptionError: stateflag = None else: try: statepad = tuple( map( float, cp.get(group, 'state-padding').split(','), )) except configparser.NoOptionError: statepad = (0, 0) if stateflag: logger.debug("Parsed state flag: %r" % stateflag) logger.debug("Parsed state padding: %s" % repr(statepad)) logger.info("Processing %d-%d" % (start, end)) # -- define nagios JSON printer ------------------------------------------- def print_nagios_json(code, message, outfile, tag='status', **extras): out = { 'created_gps': NOW, 'status_intervals': [ { 'start_sec': 0, 'end_sec': args.unknown, 'num_status': code, 'txt_status': message }, { 'start_sec': args.unknown, 'num_status': 3, 'txt_status': 'Omicron %s check is not running' % tag }, ], 'author': { 'name': 'Duncan Macleod', 'email': '*****@*****.**', }, 'omicron': { 'version': omicronversion, 'group': group, 'channels': ' '.join(channels), 'frametype': frametype, 'state-flag': stateflag, }, 'pyomicron': { 'version': __version__, }, } out.update(extras) with open(outfile, 'w') as f: f.write(json.dumps(out)) logger.debug("nagios info written to %s" % outfile) # -- get condor status ------------------------------------------------ if not args.skip_condor: # connect to scheduler try: schedd = htcondor.Schedd() except RuntimeError as e: logger.warning("Caught %s: %s" % (type(e).__name__, e)) logger.info("Failed to connect to HTCondor scheduler, cannot " "determine condor status for %s" % group) schedd = None if not args.skip_condor and schedd: logger.info("-- Checking condor status --") # get DAG status jsonfp = outdir / "nagios-condor-{}.json".format(group) okstates = ['Running', 'Idle', 'Completed'] try: # check manager status qstr = 'OmicronManager == "{}" && Owner == "{}"'.format( group, args.user, ) try: jobs = schedd.query(qstr, ['JobStatus']) except IOError as e: warnings.warn("Caught IOError: %s [retrying...]" % str(e)) sleep(2) jobs = schedd.query(qstr, ['JobStatus']) logger.debug( "Found {} jobs for query {!r}".format(len(jobs), qstr), ) if len(jobs) > 1: raise RuntimeError( "Multiple OmicronManager jobs found for %r" % group) elif len(jobs) == 0: raise RuntimeError( "No OmicronManager job found for %r" % group, ) status = condor.JOB_STATUS[jobs[0]['JobStatus']] if status not in okstates: raise RuntimeError("OmicronManager status for %r: %r" % (group, status)) logger.debug("Manager status is %r" % status) # check node status jobs = schedd.query( 'OmicronProcess == "{}" && Owner == "{}"'.format( group, args.user, ), ['JobStatus', 'ClusterId'], ) logger.debug( "Found {} jobs for query {!r}".format(len(jobs), qstr), ) for job in jobs: status = condor.JOB_STATUS[job['JobStatus']] if status not in okstates: raise RuntimeError("Omicron node %s (%r) is %r" % (job['ClusterId'], group, status)) except RuntimeError as e: print_nagios_json(2, str(e), jsonfp, tag='condor') logger.warning("Failed to determine condor status: %r" % str(e)) except IOError as e: logger.warning("Caught %s: %s" % (type(e).__name__, e)) logger.info("Failed to connect to HTCondor scheduler, cannot " "determine condor status for %s" % group) else: print_nagios_json( 0, "Condor processing for %r is OK" % group, jsonfp, tag='condor', ) logger.info("Condor processing is OK") if not args.skip_job_duration: # get job duration history plot = Plot(figsize=[12, 3]) plot.subplots_adjust(bottom=.22, top=.87) ax = plot.gca(xscale="auto-gps") times, jobdur = condor.get_job_duration_history_shell('OmicronProcess', group, maxjobs=5000) logger.debug("Recovered duration history for %d omicron.exe jobs" % len(times)) line = ax.plot([0], [1], label='Omicron.exe')[0] ax.plot(times, jobdur, linestyle=' ', marker='.', color=line.get_color()) times, jobdur = condor.get_job_duration_history_shell( 'OmicronPostProcess', group, maxjobs=5000) logger.debug("Recovered duration history for %d post-processing jobs" % len(times)) line = ax.plot([0], [1], label='Post-processing')[0] ax.plot(times, jobdur, linestyle=' ', marker='.', color=line.get_color()) ax.legend(loc='upper left', borderaxespad=0, bbox_to_anchor=(1.01, 1), handlelength=1) ax.set_xlim(args.gps_start_time, args.gps_end_time) ax.set_epoch(ax.get_xlim()[1]) ax.set_yscale('log') ax.set_title('Omicron job durations for %r' % group) ax.set_ylabel('Job duration [seconds]') ax.xaxis.labelpad = 5 png = str(outdir / "nagios-condor-{}.png".format(group)) plot.save(png) plot.close() logger.debug("Saved condor plot to %s" % png) if args.skip_file_checks: sys.exit(0) # -- get file latency and archive completeness ---------------------------- logger.info("-- Checking file archive --") # get frame segments segs = segments.get_frame_segments(obs, frametype, start, end) # get state segments if stateflag is not None: segs &= segments.query_state_segments( stateflag, start, end, pad=statepad, ) try: end = segs[-1][1] except IndexError: pass # apply inwards padding to generate resolvable segments for i in range(len(segs) - 1, -1, -1): # if segment is shorter than padding, ignore it completely if abs(segs[i]) <= padding * 2: del segs[i] # otherwise apply padding to generate trigger segment else: segs[i] = segs[i].contract(padding) logger.debug("Found %d seconds of analysable time" % abs(segs)) # load archive latency latencyfile = outdir / "nagios-latency-{}.h5".format(tag) times = dict((c, dict((ft, None) for ft in filetypes)) for c in channels) ldata = dict((c, dict((ft, None) for ft in filetypes)) for c in channels) try: with h5py.File(latencyfile, 'r') as h5file: for c in channels: for ft in filetypes: try: times[c][ft] = h5file[c]['time'][ft][:] ldata[c][ft] = h5file[c]['latency'][ft][:] except KeyError: times[c][ft] = numpy.ndarray((0, )) ldata[c][ft] = numpy.ndarray((0, )) except OSError as exc: # file not found, or is corrupt warnings.warn("failed to load latency data from {}: {}".format( latencyfile, str(exc), )) for c in channels: for ft in filetypes: if not times[c].get(ft): times[c][ft] = numpy.ndarray((0, )) ldata[c][ft] = numpy.ndarray((0, )) else: logger.debug("Parsed latency data from %s" % latencyfile) # load acknowledged gaps acksegfile = str(outdir / "acknowledged-gaps-{}.txt".format(tag)) try: acknowledged = SegmentList.read(acksegfile, gpstype=float, format="segwizard") except IOError: # no file acknowledged = SegmentList() else: logger.debug( "Read %d segments from %s" % (len(acknowledged), acksegfile), ) acknowledged.coalesce() # build legend for segments leg = OrderedDict() leg['Analysable'] = SegmentRectangle( [0, 1], 0, facecolor='lightgray', edgecolor='gray', ) leg['Available'] = SegmentRectangle( [0, 1], 0, facecolor='lightgreen', edgecolor='green', ) leg['Missing'] = SegmentRectangle( [0, 1], 0, facecolor='red', edgecolor='darkred', ) leg['Unresolvable'] = SegmentRectangle( [0, 1], 0, facecolor='magenta', edgecolor='purple', ) leg['Overlapping'] = SegmentRectangle( [0, 1], 0, facecolor='yellow', edgecolor='orange', ) leg['Pending'] = SegmentRectangle( [0, 1], 0, facecolor='lightskyblue', edgecolor='blue', ) leg['Acknowledged'] = SegmentRectangle( [0, 1], 0, facecolor='sandybrown', edgecolor='brown', ) logger.debug("Checking archive for each channel...") # find files latency = {} gaps = {} overlap = {} pending = {} plots = {} for c in channels: # create data storate latency[c] = {} gaps[c] = {} overlap[c] = {} pending[c] = {} # create figure plot = Plot(figsize=[12, 5]) lax = plot.add_subplot(grid[0, 0], xscale="auto-gps") sax = plot.add_subplot(grid[1, 0], sharex=lax, projection='segments') colors = ['lightblue', 'dodgerblue', 'black'] for y, ft in enumerate(filetypes): # find files cache = io.find_omicron_files(c, start, end, archive, ext=ft) cpend = sieve_cache(io.find_pending_files(c, proddir, ext=ft), segment=Segment(start, end)) # get available segments avail = segments.cache_segments(cache) found = avail & segs pending[c][ft] = segments.cache_segments(cpend) & segs # remove gaps at the end that represent latency try: latency[c][ft] = abs(segs & type( segs)([type(segs[0])(found[-1][1], segs[-1][1])])) / 3600. except IndexError: latency[c][ft] = 0 processed = segs else: processed = segs & type(segs)( [type(segs[0])(start, found[-1][1])]) gaps[c][ft] = type(found)() lost = type(found)() for s in processed - found: if abs(s) < mingap and s in list(segs): lost.append(s) else: gaps[c][ft].append(s) # remove acknowledged gaps ack = gaps[c][ft] & acknowledged gaps[c][ft] -= acknowledged # print warnings if abs(gaps[c][ft]): warnings.warn("Gaps found in %s files for %s:\n%s" % (c, ft, gaps[c][ft])) overlap[c][ft] = segments.cache_overlaps(cache) if abs(overlap[c][ft]): warnings.warn("Overlap found in %s files for %s:\n%s" % (c, ft, overlap[c][ft])) # append archive times[c][ft] = numpy.concatenate((times[c][ft][-99999:], [NOW])) ldata[c][ft] = numpy.concatenate( (ldata[c][ft][-99999:], [latency[c][ft]])) # plot line = lax.plot( times[c][ft], ldata[c][ft], label=ft, color=colors[y], )[0] lax.plot(times[c][ft], ldata[c][ft], marker='.', linestyle=' ', color=line.get_color()) sax.plot_segmentlist(segs, y=y, label=ft, alpha=.5, facecolor=leg['Analysable'].get_facecolor(), edgecolor=leg['Analysable'].get_edgecolor()) sax.plot_segmentlist(pending[c][ft], y=y, facecolor=leg['Pending'].get_facecolor(), edgecolor=leg['Pending'].get_edgecolor()) sax.plot_segmentlist(avail, y=y, label=ft, alpha=.2, height=.1, facecolor=leg['Available'].get_facecolor(), edgecolor=leg['Available'].get_edgecolor()) sax.plot_segmentlist(found, y=y, label=ft, alpha=.5, facecolor=leg['Available'].get_facecolor(), edgecolor=leg['Available'].get_edgecolor()) sax.plot_segmentlist(lost, y=y, facecolor=leg['Unresolvable'].get_facecolor(), edgecolor=leg['Unresolvable'].get_edgecolor()) sax.plot_segmentlist(gaps[c][ft], y=y, facecolor=leg['Missing'].get_facecolor(), edgecolor=leg['Missing'].get_edgecolor()) sax.plot_segmentlist(overlap[c][ft], y=y, facecolor=leg['Overlapping'].get_facecolor(), edgecolor=leg['Overlapping'].get_edgecolor()) sax.plot_segmentlist(ack, y=y, facecolor=leg['Acknowledged'].get_facecolor(), edgecolor=leg['Acknowledged'].get_edgecolor()) # finalise plot lax.axhline(args.warning / 3600., color=(1.0, 0.7, 0.0), linestyle='--', linewidth=2, label='Warning', zorder=-1) lax.axhline(args.error / 3600., color='red', linestyle='--', linewidth=2, label='Critical', zorder=-1) lax.set_title('Omicron status: {}'.format(c)) lax.set_ylim(0, args.error / 1800.) lax.set_ylabel('Latency [hours]') lax.legend(loc='upper left', bbox_to_anchor=(1.01, 1), borderaxespad=0, handlelength=2, fontsize=12.4) lax.set_xlabel(' ') for ax in plot.axes: ax.set_xlim(args.gps_start_time, args.gps_end_time) ax.set_epoch(ax.get_xlim()[1]) sax.xaxis.labelpad = 5 sax.set_ylim(-.5, len(filetypes) - .5) sax.legend(leg.values(), leg.keys(), handlelength=1, fontsize=12.4, loc='lower left', bbox_to_anchor=(1.01, 0), borderaxespad=0) plots[c] = png = outdir / "nagios-latency-{}.png".format( c.replace(':', '-'), ) plot.save(png) plot.close() logger.debug(" %s" % c) # update latency and write archive h5file = h5py.File(latencyfile, 'w') for c in channels: g = h5file.create_group(c) for name, d in zip(['time', 'latency'], [times[c], ldata[c]]): g2 = g.create_group(name) for ft in filetypes: g2.create_dataset(ft, data=d[ft], compression='gzip') h5file.close() logger.debug("Stored latency data as HDF in %s" % latencyfile) # write nagios output for files status = [] for segset, tag in zip([gaps, overlap], ['gaps', 'overlap']): chans = [(c, segset[c]) for c in segset if abs(reduce(operator.or_, segset[c].values()))] jsonfp = outdir / "nagios-{}-{}.json".format(tag, group) status.append((tag, jsonfp)) if chans: gapstr = '\n'.join('%s: %s' % c for c in chans) code = 1 message = ("%s found in Omicron files for group %r\n%s" % (tag.title(), group, gapstr)) else: code = 0 message = ("No %s found in Omicron files for group %r" % (tag, group)) print_nagios_json(code, message, jsonfp, tag=tag, **{tag: dict(chans)}) # write group JSON jsonfp = outdir / "nagios-latency-{}.json".format(group) status.append(('latency', jsonfp)) code = 0 message = 'No channels have high latency for group %r' % group ldict = dict((c, max(latency[c].values())) for c in latency) for x, dt in zip([2, 1], [args.error, args.warning]): dh = dt / 3600. chans = [c for c in ldict if ldict[c] >= dh] if chans: code = x message = ( "%d channels found with high latency (above %s seconds)" % (len(chans), dt)) break print_nagios_json(code, message, jsonfp, tag='latency', latency=ldict) # auto-detect 'standard' JSON files for tag, name in zip( ['condor', 'omicron-online'], ['condor', 'processing'], ): f = outdir / "nagios-{}-{}.json".format(tag, group) if f.is_file(): status.insert(0, (name, f)) # write HTML summary if args.html: page = markup.page() page.init( title="%s Omicron Online status" % group, css=[ ('//maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/' 'bootstrap.min.css'), ('//cdnjs.cloudflare.com/ajax/libs/fancybox/2.1.5/' 'jquery.fancybox.min.css'), ], script=[ '//code.jquery.com/jquery-1.11.2.min.js', ('//maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/' 'bootstrap.min.js'), ('//cdnjs.cloudflare.com/ajax/libs/fancybox/2.1.5/' 'jquery.fancybox.min.js'), ], ) page.div(class_='container') # write header page.div(class_='page-header') page.h1('Omicron Online status: %s' % group) page.div.close() # page-header # write summary page.div(id_='json') page.h2("Processing status") for tag, f in status: jf = f.name page.a("%s status" % tag.title(), href=jf, role='button', target="_blank", id_="nagios-%s" % tag, class_='btn btn-default json-status') page.p(style="padding-top: 5px;") page.small( "Hover over button for explanation, click to open JSON file", ) page.p.close() page.div.close() # id=json # show plots page.div(id_='plots') page.h2("Channel details") page.div(class_='row') for channel in sorted(channels): png = plots[channel].name page.div(class_="col-sm-6 col-md-4") page.div(class_="panel panel-default") page.div(class_='panel-heading') page.h3(channel, class_='panel-title', style="font-size: 14px;") page.div.close() # panel-heading page.div(class_='panel-body') page.a(href=png, target="_blank", class_="fancybox", rel="channel-status-img") page.img(src=png, class_='img-responsive') page.a.close() page.div.close() # panel-body page.div.close() # panel page.div.close() # col page.div.close() # row page.div.close() # id=plots # dump parameters page.div(id_="parameters") page.h2("Parameters") for key, val in cp.items(group): page.p() page.strong("%s:" % key) page.add(val) page.p.close() page.div.close() # id=parameters # finish and close page.div.close() # container page.script(""" function setStatus(data, id) { var txt = data.status_intervals[0].txt_status.split("\\n")[0]; $("#"+id).attr("title", txt); var stat = data.status_intervals[0].num_status; if (stat == 0) { $("#"+id).addClass("btn-success"); } else if (stat == 1) { $("#"+id).addClass("btn-warning"); } else if (stat == 2){ $("#"+id).addClass("btn-danger"); } } $(document).ready(function() { $(".json-status").each(function() { var jsonf = $(this).attr("href"); var id = $(this).attr("id"); $.getJSON(jsonf, function(data) { setStatus(data, id); }); }); $(".fancybox").fancybox({nextEffect: 'none', prevEffect: 'none'}); });""", type="text/javascript") with (outdir / "index.html").open("w") as f: f.write(str(page)) logger.debug("HTML summary written to %s" % f.name)