def summarize(self): """ Return summary of all QC issues for network. Behaviour: Create and return a summary of all QC issues pertaining to network. Return Values: <str> containing summary """ import sys import os sys.path.append('%s/data/python' % os.environ['ANTELOPE']) from antelope.stock import epoch2str sys.path.remove('%s/data/python' % os.environ['ANTELOPE']) summary = 'QC Report for %s\n' % self.network summary = '%s%s - %s\n\n' % (summary, epoch2str(self.tstart, \ '%m/%d/%Y %H:%M:%S'), epoch2str(self.tend,'%m/%d/%Y %H:%M:%S')) if len(self.qc_station_reports) == 0: summary = "%s\n\nNo quality control issues.\n" % summary for qc_station_report in self.qc_station_reports: summary = '%s%s' % (summary, qc_station_report.summarize()) return summary
def _get_dlevents(self): self.logging.debug( "_get_dlevents()") self.cache = [] self.error_cache = [] steps = [ 'dbopen netperf'] fields = ['snet','time','npsta','perf'] for db in self.db: for v in extract_from_db( db, steps, fields, self.db_subset): try: v['perf'] = int( v['perf'] ) v['time'] = int( v['time'] ) v['year'] = int( stock.epoch2str(v['time'], '%Y', 'UTC') ) v['jday'] = int( stock.epoch2str(v['time'], '%j', 'UTC') ) v['id'] = '%s_%s_%s' % (v['snet'], v['jday'], v['year']) self.logging.debug('netperf() => %s, %s)' % (v['id'], v['perf']) ) self.cache.append( v ) except Exception, e: self.logging.complain('netperf() => parse exception)' ) v['exception'] = Exception v['error'] = e self.logging.complain( v ) self.error_cache.append( v )
def summarize(self): """ Return summary of all QC issues for network. Behaviour: Create and return a summary of all QC issues pertaining to network. Return Values: <str> containing summary """ import sys import os sys.path.append("%s/data/python" % os.environ["ANTELOPE"]) from antelope.stock import epoch2str sys.path.remove("%s/data/python" % os.environ["ANTELOPE"]) summary = "QC Report for %s\n" % self.network summary = "%s%s - %s\n\n" % ( summary, epoch2str(self.tstart, "%m/%d/%Y %H:%M:%S"), epoch2str(self.tend, "%m/%d/%Y %H:%M:%S"), ) if len(self.qc_station_reports) == 0: summary = "%s\n\nNo quality control issues.\n" % summary for qc_station_report in self.qc_station_reports: summary = "%s%s" % (summary, qc_station_report.summarize()) return summary
def _get_netperf(self): self.logging.debug("_get_netperf()") self.cache = [] self.error_cache = [] steps = ["dbopen netperf"] fields = ["snet", "time", "npsta", "perf"] for db in self.db: for v in extract_from_db(db, steps, fields, self.db_subset): try: v["perf"] = int(v["perf"]) v["time"] = int(v["time"]) v["year"] = int(stock.epoch2str(v["time"], "%Y", "UTC")) v["jday"] = int(stock.epoch2str(v["time"], "%j", "UTC")) v["id"] = "%s_%s_%s" % (v["snet"], v["jday"], v["year"]) self.logging.debug("netperf() => %s, %s)" % (v["id"], v["perf"])) self.cache.append(v) except Exception as e: self.logging.complain("netperf() => parse exception)") v["exception"] = Exception v["error"] = e self.logging.complain(v) self.error_cache.append(v)
def _get_detections(self): ''' Open detection table and get all data for current event. TODO: Deal with mopping up of detections not associated with events ''' if not self.table_present['detection']: return start = int(stock.now()) end = 0 self.logger.debug('Basing detection window on arrivals for evid [%d]' % self.evid) for orid in self.origins.keys(): arrivals = self.all_arrivals(orid=orid) for arrival in arrivals: if 'arrival.time' in arrival: time = arrival['arrival.time'] else: time = start if 'arrival.deltim' in arrival: deltim = arrival['arrival.deltim'] else: deltim = 0 if time - deltim < start: start = time - deltim if time + deltim > end: end = time + deltim if end > start: self.logger.debug('Windowing detections from %s to %s' % (stock.epoch2str(start, '%G %T')[:-4], stock.epoch2str(end, '%G %T')[:-4])) steps = ['dbopen detection'] steps += ['dbsubset time>%s && time<%s' % (start, end)] steps += ['dbsubset state=~/%s/' % state for state in self.detection_state_select] steps += ['dbsubset state!~/%s/' % state for state in self.detection_state_reject] steps += self._seed_channel_steps() self.detections.get_view(steps) else: self.logger.warning( 'Calculated time-window for detections is not valid: [%s,%s]' % (start, end))
def month_resolver(self, start_mth, start_yr, end_mth, end_yr, dbname_template): """Month list """ if self.include_times: months = {} else: months = [] vol_month = start_mth vol_year = start_yr while vol_year <= end_yr or (vol_year == end_yr and vol_month <= end_mth): voltime = antstock.str2epoch("%d/1/%d" % (vol_month, vol_year)) if vol_month < 12: vol_month += 1 else: vol_month = 1 vol_year += 1 volendtime = antstock.str2epoch("%d/1/%d" % (vol_month,vol_year)) - 1 dbname = antstock.epoch2str(int(voltime), dbname_template) if self.include_times: months[dbname] = (voltime,volendtime) else: months.append(dbname) #if os.path.exists(dbname) and os.path.isfile(dbname): # if self.include_times: # months[dbname] = (voltime,volendtime) # else: # months.append(dbname) #else: # antelog.notify("Dbpath '%s' does not exist." % dbname) return months
def new( self, rawpkt ): if not rawpkt[0] or int(float(rawpkt[0])) < 1: self.logging.info( 'Bad Packet: %s %s %s' % (rawpkt[0], rawpkt[1], rawpkt[2] ) ) return self._clean() self.rawpkt = rawpkt self.logging.debug( rawpkt ) self.id = rawpkt[0] self.time = float( rawpkt[2] ) self.strtime = stock.epoch2str( self.time, '%D %H:%M:%S %Z' ).strip() # Try to extract information from packet pkt = Pkt.Packet( rawpkt[1], rawpkt[2], rawpkt[3] ) self.srcname = pkt.srcname if pkt.srcname else rawpkt[1] self.logging.info( '%s %s %s' % (self.id, self.time, self.strtime) ) #self.logging.debug( pkt.pf ) if pkt.pf.has_key('dls'): self.dls = pkt.pf['dls'] self.valid = True self.__str__() else: self.dls = {} self.valid = False
def send(self): """Send network report to appropriate e-mail addresses. Behaviour: Send network report to appropriate e-mail addresses via SMTP server """ import sys import os sys.path.append("%s/data/python" % os.environ["ANTELOPE"]) from antelope.stock import epoch2str, now sys.path.remove("%s/data/python" % os.environ["ANTELOPE"]) import smtplib sender = "auto_qc-noreply@%s" % self.smtp_server from_line = "From: %s\n" % sender to_line = "To: %s" % self.email[0] for rec in self.email[1:]: to_line = "%s, %s " % (to_line, rec) to_line = "%s\n" % to_line subject_line = "Subject: auto_qc network report for %s %s\n" % ( self.network, epoch2str(now(), "%m/%d/%Y"), ) message = "%s%s%s%s" % (from_line, to_line, subject_line, self.summarize()) try: smtpObj = smtplib.SMTP(self.smtp_server) smtpObj.sendmail(sender, self.email, message) print("Network summary successfully sent.") except smtplib.SMTPException: print("Error: unable to send e-mail.\n\n") print(self.summarize())
def send(self): """Send network report to appropriate e-mail addresses. Behaviour: Send network report to appropriate e-mail addresses via SMTP server """ import sys import os sys.path.append('%s/data/python' % os.environ['ANTELOPE']) from antelope.stock import epoch2str, now sys.path.remove('%s/data/python' % os.environ['ANTELOPE']) import smtplib sender = 'auto_qc-noreply@%s' % self.smtp_server from_line = 'From: %s\n' % sender to_line = 'To: %s' % self.email[0] for rec in self.email[1:]: to_line = '%s, %s ' % (to_line, rec) to_line = '%s\n' % to_line subject_line = 'Subject: auto_qc network report for %s %s\n' % \ (self.network, epoch2str(now(),'%m/%d/%Y')) message = '%s%s%s%s' % (from_line, to_line, subject_line, self.summarize()) try: smtpObj = smtplib.SMTP(self.smtp_server) smtpObj.sendmail(sender, self.email,message) print 'Network summary successfully sent.' except smtplib.SMTPException: print 'Error: unable to send e-mail.\n\n' print self.summarize()
def readable_time(time, tformat="%D (%j) %H:%M:%S %z", tzone="UTC"): # Make want to save a readable copy # of the epoch times. try: if parse_sta_time(time) == "-": raise return stock.epoch2str(time, tformat, tzone) except Exception, e: return "-"
def readable_time(time, tformat="%D (%j) %H:%M:%S %z", tzone="UTC"): """Format epoch time in a human readable format.""" try: if parse_sta_time(time) == "-": raise return stock.epoch2str(time, tformat, tzone) except Exception: return "-"
def get3Ctr(wf_db, sta, chan3, tstart, tend): """ Input: db wfdisc pointer, station name, 3 channels, start,end Output: Obspy Stream with 3 waveform traces (segments) """ logger = logging.getLogger('dbarrival_params') st3c = Stream() if chan3 == None: return None for chan in chan3: cline = "{}:{} {} - {}".format(sta, chan, stock.epoch2str(tstart, "%D %H:%M:%S.%s"), stock.epoch2str(tend, "%D %H:%M:%S.%s")) logger.debug("get3Ctr: Getting data for {}".format(cline)) with ds.trdestroying(wf_db.trloadchan(tstart, tend, sta, chan)) as tr: if tr.record_count == 0: logger.warning("get3Ctr: Could not load data for {}".format(cline)) return None #tr.trfilter(pfile['filter']) tr.record = 0 try: time, samprate = tr.getv('time', 'samprate') except DbgetvError: logger.warning("get3Ctr: Could not get value 'samprate' for {}".format(cline)) return None data = [] for segment in tr.iter_record(): tmp_data = list(segment.trdata()) data += list(segment.trdata()) tr.trfree() data = array(data) stats = {'station': sta, 'channel': chan, 'sampling_rate': samprate} stats['starttime'] = UTCDateTime(time) otr = Trace(data=data, header=stats) st3c += otr ns = len(st3c[0].data) # TODO: write subroutine to check ns, if close (w/i a few samples) try again if len(st3c[1].data) != ns or len(st3c[2].data) != ns: logger.warning('get3Ctr: {0}'.format(cline)) logger.warning('get3Ctr: Length of data arrays not equal {0} {1} {2}'.format(len(st3c[0].data),len(st3c[1].data),len(st3c[2].data))) return None return st3c
def _get_dlevents(self): self.logging.debug("_get_dlevents()") self.cache = [] self.error_cache = [] steps = ["dbopen dlevent"] fields = ["dlname", "dlevtype", "dlcomment", "time"] for v in extract_from_db(self.db, steps, fields, self.db_subset): self.logging.debug("dlevent(%s)" % (v["dlname"])) snet, sta = v["dlname"].split("_", 1) v["snet"] = snet v["sta"] = sta v["year"] = stock.epoch2str(v["time"], "%Y", "UTC") v["month"] = stock.epoch2str(v["time"], "%L", "UTC") self.cache.append(v)
def _get_dlevents(self): self.logging.debug( "_get_dlevents()") self.cache = [] self.error_cache = [] steps = [ 'dbopen dlevent'] fields = ['dlname','dlevtype','dlcomment','time'] for v in extract_from_db(self.db, steps, fields, self.db_subset): self.logging.debug('dlevent(%s)' % (v['dlname']) ) snet,sta = v['dlname'].split('_',1) v['snet'] = snet v['sta'] = sta v['year'] = stock.epoch2str(v['time'], '%Y', 'UTC') v['month'] = stock.epoch2str(v['time'], '%L', 'UTC') self.cache.append( v )
def __init__(self, db, evid): #Mal, this shouldn't take evid as an argument with closing(dbopen(db, 'r')) as db: view = db.schema_tables['origin'] view = view.subset('evid == %s' % evid) view = view.subset('orid == prefor') view = view.join('netmag', outer=True) evid, time, lat, lon, depth, mag = view.getv('evid', 'time', 'lat', 'lon', 'depth', 'magnitude') year = int(epoch2str(time, '%Y')) month = int(epoch2str(time, '%m')) day = int(epoch2str(time, '%d')) hour = int(epoch2str(time, '%H')) minute = int(epoch2str(time, '%M')) second = float(epoch2str(time, '%S.%s')) self.append({'id': evid, 'year': year, 'month': month, 'day': day, 'hour': hour, 'min': minute, 'sec': second, 'lat': lat, 'lon': lon, 'depth': depth, 'mag': mag, 'arrivals': []}) view = view.join('assoc') view = view.join('arrival') for record in view.iter_record(): sta, arr_time, timeres, phase = record.getv('sta', 'arrival.time', 'timeres', 'phase') ttime = arr_time - time self[-1]['arrivals'].append({'staname': sta, 'ttime': ttime, 'qual': timeres, 'phase': phase})
def new(self, rawpkt): """Create a new Poc object, allowing reuse of an orb.Packet reference. This method is a hack around the memory leaks incurred by allocating a new orb.Packet structure repeatedly. Instead of allocating a new Pkt object and letting the garbage collector come around and free memory, we just repeatedly reuse the same object. """ self._clean() self.rawpkt = rawpkt if not rawpkt[0] or int(float(rawpkt[0])) < 1: return self.id = rawpkt[0] self.time = rawpkt[2] # Try to extract information from packet pkt = Pkt.Packet(rawpkt[1], rawpkt[2], rawpkt[3]) self.srcname = pkt.srcname if pkt.srcname else rawpkt[1] # Step around Antelope TypeError by explicitly calling # .keys() on the ParameterFile object. if "sn" in pkt.pf.keys(): self.sn = pkt.pf["sn"] else: return if "srcip" in pkt.pf.keys(): self.srcip = pkt.pf["srcip"] else: return if "time" in pkt.pf.keys(): self.poctime = float(pkt.pf["time"]) self.strtime = stock.epoch2str(self.poctime, "%D %H:%M:%S %Z").strip() else: return self.valid = True # Maybe we have some extra data... if "pocc2" in pkt.pf.keys(): self.pocc2 = pkt.pf["pocc2"] else: self.pocc2 = {}
def __init__( self, width, height, result, reference, ref_sta, ref_chan, sta, start, end, result_dir, debug_plot, orid=None, ): """Initialize Plot class.""" total = len(result) self.width = width self.height = height * total fig = plt.figure(figsize=(width, height)) axs = [ fig.add_subplot(3 * total, 3, j) for j in range(1, (3 * 3 * total) + 1) ] plt.tight_layout() fig.subplots_adjust(top=0.9, bottom=0.05) self.plot_data(axs, result, reference, ref_sta, ref_chan, sta, start, end) if debug_plot: plt.show() else: if not orid: filename = "%s_%s_%s.pdf" % ( ref_sta, sta, epoch2str(start, "%Y%j_%H_%M_%S.%s"), ) else: filename = "%s_%s_%s.pdf" % (ref_sta, sta, orid) path = "/".join([result_dir, filename]) if not os.path.exists(result_dir): os.makedirs(result_dir) fig.savefig(path, bbox_inches="tight", pad_inches=0.5, dpi=100)
def new( self, rawpkt ): self._clean() self.rawpkt = rawpkt if not rawpkt[0] or int(float(rawpkt[0])) < 1: return self.id = rawpkt[0] self.time = rawpkt[2] # Try to extract information from packet pkt = Pkt.Packet( rawpkt[1], rawpkt[2], rawpkt[3] ) self.srcname = pkt.srcname if pkt.srcname else rawpkt[1] if pkt.pf.has_key('sn'): self.sn = pkt.pf['sn'] else: return if pkt.pf.has_key('srcip'): self.srcip = pkt.pf['srcip'] else: return if pkt.pf.has_key('time'): self.poctime = float(pkt.pf['time']) self.strtime = stock.epoch2str( self.poctime, '%D %H:%M:%S %Z' ).strip() else: return self.valid = True # Maybe we have some extra data... if pkt.pf.has_key('pocc2'): self.pocc2 = pkt.pf['pocc2'] else: self.pocc2 = {}
def year_resolver(self, start_yr, end_yr, dbname_template): """Year list """ if self.include_times: years = {} else: years = [] for y in range(start_yr, end_yr + 1): voltime = antstock.str2epoch("1/1/%s 00:00:00" % y) volendtime = antstock.str2epoch("12/31/%s 23:59:59" % y) dbname = antstock.epoch2str(voltime, dbname_template) if self.include_times: years[dbname] = (voltime,volendtime) else: years.append(dbname) #if os.path.exists(dbname) and os.path.isfile(dbname): # if self.include_times: # years[dbname] = (voltime,volendtime) # else: # years.append(dbname) #else: # antelog.notify("Dbpath '%s' does not exist." % dbname) return years
def _init_from_db(self, db, evid): """ Initialize Event object using a CSS3.0 database as input. """ if evid == None: raise(Exception('No \'evid\' supplied. Could not ' 'initialize Event object from CSS3.0 database.')) with closing(dbopen(db, 'r')) as db: view = db.schema_tables['event'] view = view.join('origin') view = view.subset('evid == %s' % evid) view = view.subset('orid == prefor') #If for some reason this subset is empty, just take the first #solution as preferred. EG. prefor field is unitialized. if view.record_count == 0: view = db.schema_tables['origin'] view = db.schema_tables['event'] view = view.join('origin') view = view.subset('evid == %s' % evid) view = view.join('netmag', outer=True) view.record = 0 evid, time, lat, lon, depth, mag, magtype = view.getv('evid', 'time', 'lat', 'lon', 'depth', 'magnitude', 'magtype') self.evid = evid self.time = time self.lat = lat self.lon = lon self.depth = depth self.mag = mag self.magtype = magtype self.year = int(epoch2str(time, '%Y')) self.month = int(epoch2str(time, '%m')) self.day = int(epoch2str(time, '%d')) self.hour = int(epoch2str(time, '%H')) self.minute = int(epoch2str(time, '%M')) self.second = float(epoch2str(time, '%S.%s')) view = view.join('assoc') view = view.join('arrival') arrivals = [ record.getv('sta', 'arrival.time', 'phase') \ + (None, ) \ for record in view.iter_record() ] self.arrivals = [ Phase(sta, time, phase, qual) for sta, time, phase, qual in arrivals ]
def get_chan3(dbsite_chan, sta, chan, time): """ Return the appropriate 3 channel tuple for 3 component stations. """ logger = logging.getLogger('dbarrival_params') scsub = "sta =~ /{}/ && chan =~ /{}.{}/ "\ "&& ondate < _{}_ && (offdate > _{}_ || "\ "offdate == -1)".format(sta, chan[:2], chan[3:], time, time) logger.debug('get_chan3: scsub {}'.format(scsub)) view = dbsite_chan.subset("sta =~ /{}/ && chan =~ /{}.{}/ "\ "&& ondate < _{}_ && (offdate > _{}_ || "\ "offdate == -1)".format(sta, chan[:2], chan[3:], time, time)) if view.record_count != 3: logger.warning("get_chan3: Could not determine appropriate chan3 for "\ "{}:{} at {}".format(sta, chan, stock.epoch2str(time, "%Y%j %H:%M:%S.%s"))) view.free() return None chan3, ondates, offdates = [], [], [] for record in view.iter_record(): chan, ondate, offdate = record.getv('chan', 'ondate', 'offdate') chan3 += [chan] ondates += [ondate] offdates += [offdate] chan3 = tuple(sort_chans(chan3)) ondate = max(ondates) offdate = min(offdates) return chan3
def day_resolver(self, start_dy, end_dy, dbname_template): """Day list """ if self.include_times: days = {} else: days = [] while start_dy <= end_dy: voltime = antstock.epoch(start_day) volendtime = voltime + 86399 # one second less than a full day dbname = antstock.epoch2str(voltime, dbname_template) if self.include_times: days[dbname] = (voltime, volendtime) else: days.append(dbname) #if os.path.exists(dbname) and os.path.isfile(dbname): # if self.include_times: # days[dbname] = (voltime, volendtime) # else: # days.append(dbname) #else: # antelog.notify("Dbpath '%s' does not exist." % dbname) start_dy = yearday((antstock.epoch(start_dy) + 86400)) return days
def _init_from_aux(self, time, lat, lon, dpeth, mag, magtype=None, evid=None): """ Initialize Event object using an auxiliary read function. Auxiliary read function must parse flat file and pass parameters to Event contsructor. """ self.time = time self.lat = lat self.lon = lon self.depth = depth self.mag = mag self.magtype = magtype self.year = int(epoch2str(time, '%Y')) self.month = int(epoch2str(time, '%m')) self.day = int(epoch2str(time, '%d')) self.hour = int(epoch2str(time, '%H')) self.minute = int(epoch2str(time, '%M')) self.second = float(epoch2str(time, '%S.%s')) self.arrivals = phase_list
def __init__(self, db, evid): #Mal, this shouldn't take evid as an argument with closing(dbopen(db, 'r')) as db: view = db.schema_tables['origin'] view = view.subset('evid == %s' % evid) view = view.subset('orid == prefor') view = view.join('netmag', outer=True) evid, time, lat, lon, depth, mag = view.getv( 'evid', 'time', 'lat', 'lon', 'depth', 'magnitude') year = int(epoch2str(time, '%Y')) month = int(epoch2str(time, '%m')) day = int(epoch2str(time, '%d')) hour = int(epoch2str(time, '%H')) minute = int(epoch2str(time, '%M')) second = float(epoch2str(time, '%S.%s')) self.append({ 'id': evid, 'year': year, 'month': month, 'day': day, 'hour': hour, 'min': minute, 'sec': second, 'lat': lat, 'lon': lon, 'depth': depth, 'mag': mag, 'arrivals': [] }) view = view.join('assoc') view = view.join('arrival') for record in view.iter_record(): sta, arr_time, timeres, phase = record.getv( 'sta', 'arrival.time', 'timeres', 'phase') ttime = arr_time - time self[-1]['arrivals'].append({ 'staname': sta, 'ttime': ttime, 'qual': timeres, 'phase': phase })
def _get_list(self): try: db = datascope.dbopen(self.path, "r") except Exception as e: raise DbcentralException("Cannot open database %s (%s)" % (self.path, e)) try: db = db.lookup("", "clusters", "", "") except datascope.DblookupFieldError: self.type = "masquerade" self.nickname = None self.dbs[self.path] = {"times": [-10000000000.0, 10000000000.0]} self.logger.info("Not a dbcentral database. Set single database.") return else: self.type = "dbcentral" if self.nickname is None: raise ValueError("Need nickname for Dbcentral clustername regex.") try: db = db.lookup("", "clusters", "", "dbNULL") null_time, null_endtime = db.getv("time", "endtime") except Exception as e: raise DbcentralException( "Cannot look up null values in clusters table. (%s)" % e ) expr = "clustername =='%s'" % self.nickname try: db = db.subset(expr) except Exception as e: raise DbcentralException("Cannot subset on clustername. %s" % e) try: db = db.sort("time") nclusters = db.record_count except Exception as e: raise DbcentralException("Cannot sort on 'time' . %s" % e) if nclusters < 1: raise DbcentralException('No matches for nickname "%s".' % self.nickname) self.logger.debug("Records=%s" % nclusters) for i in range(nclusters): self.logger.debug("db.record=%s" % i) db.record = i try: dbname_template = db.extfile()[-1] except Exception as e: raise DbcentralException("Cannot run db.extfile(). %s" % e) self.logger.debug("dbname_template=%s" % dbname_template) try: volumes, net, time, endtime = db.getv( "volumes", "net", "time", "endtime" ) except Exception as e: raise DbcentralException( "Problems with db.getv('volumes','net'," + "'time','endtime'). (%s)\n" % e ) self.logger.debug("volumes=%s" % volumes) self.logger.debug("net=%s" % net) self.logger.debug("time=%s" % time) self.logger.debug("endtime=%s" % endtime) if endtime == null_endtime: # This will be problematic with realtime systems endtime = stock.now() self.logger.debug("endtime=%s" % endtime) start_year = int(stock.epoch2str(time, "%Y")) end_year = int(stock.epoch2str(endtime, "%Y")) start_month = int(stock.epoch2str(time, "%L")) end_month = int(stock.epoch2str(endtime, "%L")) if volumes == "single": dbname = stock.epoch2str(time, dbname_template) self._test_db(time, endtime, dbname) elif volumes == "year": for y in range(start_year, end_year + 1): voltime = stock.str2epoch("1/1/%s 00:00:00" % y) volendtime = stock.str2epoch("12/31/%s 23:59:59" % y) dbname = stock.epoch2str(voltime, dbname_template) self._test_db(voltime, volendtime, dbname) elif volumes == "month": vol_month = start_month vol_year = start_year vol_endmonth = end_month vol_endyear = end_year while vol_year < end_year or ( vol_year == end_year and vol_month <= end_month ): voltime = stock.str2epoch("%d/1/%d" % (vol_month, vol_year)) if vol_month < 12: vol_month = vol_month + 1 else: vol_year = vol_year + 1 vol_month = 1 volendtime = ( stock.str2epoch("%d/1/%d" % (vol_endmonth, vol_endyear)) - 1 ) dbname = stock.epoch2str(int(voltime), dbname_template) self._test_db(voltime, volendtime, dbname) elif volumes == "day": start_day = int(stock.yearday(time)) end_day = int(stock.yearday(endtime)) vol_day = start_day while vol_day <= end_day: voltime = stock.epoch(vol_day) volendtime = voltime + 86399 # full day -1 sec dbname = stock.epoch2str(voltime, dbname_template) if self._test_db(voltime, volendtime, dbname): self.dbs[dbname] = {"times": [time, endtime]} vol_day = stock.yearday((stock.epoch(vol_day) + 86400)) else: raise UnknownVolumeTypeException(volumes) self.logger.debug("DBS=%s" % self.dbs.keys())
def main(): ########## # Extract station list from master_stations ########## stations = [] with ds.closing(ds.dbopen("/aerun/sum/db/dbsum/dbsum", "r")) as db: steps = ["dbopen affiliation", "dbjoin site", "dbsubset net=~/AK/ && offdate==NULL", "dbsort sta"] with ds.freeing(db.process(steps)) as dbview: for record in dbview.iter_record(): stations.append(record.getv('sta')[0]) ########## # Extract waveform data into trace objects ########## twin = 600 tcurrent = float(stock.str2epoch('now')) tend = tcurrent - 60 tstart = tend - twin f = '%Y_%m_%d' date = stock.epoch2str(tstart,f) with ds.closing(ds.dbopen("/aerun/op/run/db/archive_%s" % date, "r")) as db: for sta in stations: data = {} samplerate = {} dbsta = db.lookup(table = 'wfdisc') dbsta = dbsta.subset('sta=~/%s/' % sta ) bband = dbsta.subset('chan=~/BH./') ####### # Extract Broadband seismic data ####### if bband.query('dbRECORD_COUNT') > 0: # print (tstart, tcurrent, "%s" % sta, "BHE") tr_bhe = dbsta.trloadchan(tstart, tend, "%s" % sta, "BHE") tr_bhe.trapply_calib() with ds.trfreeing(tr_bhe): if tr_bhe.query("dbRECORD_COUNT") > 0: tr_bhe.record = 0 data['BHE'] = tr_bhe.trdata() samplerate['BHE'] = tr_bhe.getv("samprate") tr_bhn = dbsta.trloadchan(tstart, tend, "%s" % sta, "BHN") tr_bhn.trapply_calib() with ds.trfreeing(tr_bhn): if tr_bhn.query("dbRECORD_COUNT") > 0: tr_bhn.record = 0 data['BHN'] = tr_bhn.trdata() samplerate['BHN'] = tr_bhn.getv("samprate") tr_bhz = dbsta.trloadchan(tstart, tend, "%s" % sta, "BHZ") tr_bhz.trapply_calib() with ds.trfreeing(tr_bhz): if tr_bhz.query("dbRECORD_COUNT") > 0: tr_bhz.record = 0 data['BHZ'] = tr_bhz.trdata() samplerate['BHZ'] = tr_bhz.getv("samprate") ####### # Extract moderate sample rate strong motion data ####### smot_b = dbsta.subset('chan=~/BN./') if smot_b.query('dbRECORD_COUNT') > 0: tr_bne = dbsta.trloadchan(tstart, tend, "%s" % sta, "BNE") tr_bne.trapply_calib() with ds.trfreeing(tr_bne): if tr_bne.query("dbRECORD_COUNT") > 0: tr_bne.record = 0 data['BNE'] = tr_bne.trdata() samplerate['BNE'] = tr_bne.getv("samprate") tr_bnn = dbsta.trloadchan(tstart, tend, "%s" % sta, "BNN") tr_bnn.trapply_calib() with ds.trfreeing(tr_bnn): if tr_bnn.query("dbRECORD_COUNT") > 0: tr_bnn.record = 0 data['BNN'] = tr_bnn.trdata() samplerate['BNN'] = tr_bnn.getv("samprate") tr_bnz = dbsta.trloadchan(tstart, tend, "%s" % sta, "BNZ") tr_bnz.trapply_calib() with ds.trfreeing(tr_bnz): if tr_bnz.query("dbRECORD_COUNT") > 0: tr_bnz.record = 0 data['BNZ'] = tr_bnz.trdata() samplerate['BNZ'] = tr_bnz.getv("samprate") ####### # Extract high sample rate strong motion data ####### smot_h = dbsta.subset('chan=~/HN./') if smot_h.query('dbRECORD_COUNT') > 0: tr_hne = dbsta.trloadchan(tstart, tend, "%s" % sta, "HNE") tr_hne.trapply_calib() with ds.trfreeing(tr_hne): if tr_hne.query("dbRECORD_COUNT") > 0: tr_hne.record = 0 data['HNE'] = tr_hne.trdata() samplerate['HNE'] = tr_hne.getv("samprate") tr_hnn = dbsta.trloadchan(tstart, tend, "%s" % sta, "HNN") tr_hnn.trapply_calib() with ds.trfreeing(tr_hnn): if tr_hnn.query("dbRECORD_COUNT") > 0: tr_hnn.record = 0 data['HNN'] = tr_hnn.trdata() samplerate['HNN'] = tr_hnn.getv("samprate") tr_hnz = dbsta.trloadchan(tstart, tend, "%s" % sta, "HNZ") tr_hnz.trapply_calib() with ds.trfreeing(tr_hnz): if tr_hnz.query("dbRECORD_COUNT") > 0: tr_hnz.record = 0 data['HNZ'] = tr_hnz.trdata() samplerate['HNZ'] = tr_hnz.getv("samprate") # if sta=="MLY": # plot_traces(sta, data, tstart, tend, samplerate) # shortperz = sbsta.subset('chan=~/EHZ/') # if smot.query('dbRECORD_COUNT') > 0: # tr_ehz = dbsta.trloadchan(tstart, tcurrent, "%s" % sta, "EHZ") print sta plot_traces(sta, data, tstart, tend, samplerate)
def new( self, rawpkt, name_type='pf/xi', select=False, reject=False, silent=False ): self.logging.debug( 'new packet' ) if not rawpkt['_id'] : if not silent: self.logging.warning( 'Bad Packet: %s' % rawpkt ) return self._clean() self.name_type = name_type self.rawpkt = self._convert_unicode( rawpkt ) if reject and re.search( reject, self.rawpkt['srcType'] ): self.logging.debug( 'type [%s] rejected by configuration' % self.rawpkt['srcType'] ) return if select and not re.search( select, self.rawpkt['srcType'] ): self.logging.debug( 'type [%s] missed selection by configuration' % self.rawpkt['srcType'] ) return self.logging.debug( self.rawpkt ) # Track IDs self.logSeqNo = self.rawpkt['messageLogSeqNo'] self.seqNo = self.rawpkt['seqNo'] self.id = "%s.%s" % ( self.logSeqNo, self.seqNo ) # Date object self.datetime = self.rawpkt[ 'timestamp' ] # Epoch string self.time = (self.datetime-datetime.datetime(1970,1,1)).total_seconds() # Time string self.strtime = stock.epoch2str( self.time, '%D %H:%M:%S %Z', tz='UTC' ).strip() self.q330 = self.rawpkt['q330Sn'] self.imei = self.rawpkt['deviceIMEI'] self.src = self.rawpkt['srcType'] self.srcname = self.src.lower() if not self.imei_buffer.add( imei=self.imei, serial=self.q330 ): self.logging.warning( 'Invalid Q330 serial [%s] for IMEI [%s]' % ( self.q330, self.imei ) ) self.q330 = self.imei_buffer( self.imei ) if not self.q330: if not silent: self.logging.warning( 'UNKNOWN IMEI [%s]: SKIP DATA PACKET!!!' % self.imei ) return else: if not silent: self.logging.warning( 'USING CACHED Q330 SERIAL [%s] FOR IMEI [%s]' % ( self.q330, self.imei ) ) for test in self.q330_serial_dlname: if test( self.q330 ): self.dlname = test( self.q330 ) self.logging.debug( '%s => %s' % (self.q330, self.dlname) ) if not self.dlname: if not silent: self.logging.warning( 'NO DLNAME FOR Q330 SERIAL: %s ' % self.q330 ) return self.logging.debug( self.src ) self.logging.debug( self.valueMap ) # Verify if we have data pairs if 'valueMap' in self.rawpkt: self.valueMap = self.rawpkt['valueMap'] # Extract each value to a new key:value on the dict for chan in self.valueMap: if chan in self.channel_mapping: if not self.channel_mapping[chan]: continue self.payload[ self.channel_mapping[chan] ] = self.valueMap[chan] self.logging.debug( '%s -> %s:%s' % (chan, self.channel_mapping[chan], self.valueMap[chan]) ) else: self.logging.warning( '[%s] NOT DEFINED IN PF FILE' % chan ) for test in [ 'xil1q', 'xil2q']: if test in self.payload and 'xisamp' in self.payload: try: self.payload[ test ] = float(self.payload[ test ]) / self.payload[ 'xisamp' ] except: pass self.pcktbuf = { 'dls': { self.dlname: self.payload }, 'q330' : self.q330, 'imei' : self.imei, 'src' : self.src, 'srcname' : self.srcname } self.logging.debug( self.payload ) # Try to build packet from info if self.name_type and self.time and self.payload: self.pkt.srcname = Pkt.SrcName( '%s/%s' % ( self.dlname,self.name_type ) ) self.pkt.time = self.time #self.logging.debug( self.pkt.type ) #self.logging.debug( self.pkt.srcname ) # Extract pf structure, update it and return it. temp = self.pkt.pf temp.update( self.pcktbuf ) self.pkt.pf = temp self.logging.debug( self.pkt.type ) self.logging.debug( self.pkt.srcname ) self.logging.debug( 'Pkt( %s, %s) => {%s}' % (self.pkt.srcname, self.pkt.time, self.pkt.pf.pf2string().replace('\n',', ').replace('\t',':') ) ) self.valid = True else: self.logging.warning( 'NO VALUABLE INFORMATION IN PACKET. dlname:%s time:%s' % (self.dlname, self.time ) ) return self.logging.info( str(self) )
def plot_data(self, axs, result, reference, ref_sta, ref_chan, sta, start, end): """Plot data.""" k = 0 for code in result: for i, chan in enumerate(result[code]): data = result[code][chan] if i == 0: ind = 0 + k if i == 1: ind = 1 + k if i == 2: ind = 2 + k axs[ind].plot(reference[chan], "b", label="%s_%s%s" % (ref_sta, ref_chan, chan)) axs[ind].plot(data.original, "r", label="%s_%s%s" % (sta, code, chan)) axs[ind + 3].plot(reference[chan], "b") axs[ind + 3].plot(data.rotated, "r") axs[ind].legend(loc="upper left", prop={"size": 6}) axs[ind + 6].xaxis.set_visible(False) axs[ind + 6].yaxis.set_visible(False) axs[ind + 6].patch.set_alpha(0.0) axs[ind + 6].axis("off") text = "Angle: %s\n" % data.azimuth text += "Xcorr: %s\n" % round(data.xcorr, 3) axs[ind + 6].annotate( six.ensure_text(text, "utf-8"), (0.5, 0.7), xycoords="axes fraction", va="top", ha="center", fontsize=6, bbox=dict(edgecolor="white", boxstyle="round, pad=0.5", fc="w"), size=12, ) # y-axis labels if i == 0: axs[ind].set_ylabel("original", fontsize=12) axs[ind + 3].set_ylabel("rotated", fontsize=12) axs[ind].set_yticks([]) axs[ind + 3].set_yticks([]) axs[ind].set_xticks([]) axs[ind + 3].set_xticks([]) # xticks and xtick labels tw = end - start dt = tw / len(reference[chan]) xticks = numpy.arange(0, len(reference[chan]), len(reference[chan]) / 4) xtick_labels = [ epoch2str(t, "%Y%j %H:%M:%S.%s") for t in [start + x * dt for x in xticks] ] xtick_labels = xticks * dt - 2 axs[ind + 3].set_xticks(xticks) axs[ind + 3].set_xticklabels(xtick_labels) axs[ind + 3].set_xlabel("time since predicated first-arrival (s)") if i == 1: axs[ind].set_title( "%s_%s compared to %s_%s" % (ref_sta, ref_chan, sta, code), fontsize=12, ) k += 9
]) #phsub = "iphase=='P'" #dbv = dbview.subset(phsub) dbv = dbview.subset(asub) #dbv2 = dbview.subset(phsub) #dbv = dbv2.subset("sta=~/EC14/") logger.info('{0} records in assoc-arrival join after subset {1}'.format(dbv.record_count, asub)) tj0 = time.time() for j in range(dbv.record_count): dbv.record = j (atime,sta,chan,iphs,phs,dist_deg,deltim,arid) = dbv.getv('arrival.time','sta', 'chan','iphase','assoc.phase', 'delta','deltim','arid') # TODO: check if -999.0, calculate if needed (seaz,esaz) = dbv.getv('seaz','esaz') pick_str = '{0} {1} {2} {3} '.format(sta, chan, iphs, stock.epoch2str(atime, "%D %H:%M:%S.%s")) logger.debug('Input pick: {0} {1} {2} {3} '.format(sta, chan, iphs, stock.epoch2str(atime, "%D %H:%M:%S.%s"))) chan3 = get_chan3(dbsc, sta, chan, atime) if chan3 is None: logger.warning('Could not determine chan3 for {0}, skipping.'.format(pick_str)) continue tstart = atime - pretime tend = atime + posttime # For some reason, doing this subset before calling # get3Ctr can SOMETIMES drastically speeds things up if args.presub: wsub = 'sta=~/{0}/'.format(sta) dbwfs = dbwf.subset(wsub) st3c = get3Ctr(dbwfs, sta, chan3, tstart, tend) else:
def parse_time(epoch_time): return stock.epoch2str(epoch_time, '%GT%TZ')
def run_gmt(stations, chans, db, i, filter, tstep, tpad, gmt_params, verbose): ######################## # Create the basemap plot for the given time window ######################## #os.system("grdimage -R-170/-130/52/72 -Jb-150/52/62/72/.25 /Users/mgardine/Development/GMT/alaska_5m.grd -I/Users/mgardine/Development/GMT/alaska_5m_int.grd -C/Users/mgardine/Development/GMT/gray_blue.cpt -Ba300f300/a300f300wsne:.\"\": -X1.5 -Y3 -P -K > ~/Development/scratch/realtime_gmv/regions_%s.ps" % i ) ## This is for the full state #os.system("grdimage -R-180/-127/50/72 -JCyl_stere/-153.5/61/6 /Users/mgardine/Development/GMT/alaska_3m.grd -I/Users/mgardine/Development/GMT/alaska_3m_int.grd -C/Users/mgardine/Development/GMT/gray_blue.cpt -Ba300f300g300/a300f300g300wsne:.\"\": -X1.5 -Y3 -P -K > ~/Development/scratch/realtime_gmv/regions_%s.ps" % i ) #os.system("pscoast -R -J -B -K -Di -N1/black -Wblack -A100/0/1 -P -O >> ~/Development/scratch/realtime_gmv/regions_%s.ps" % i) # This is for 3-d view os.system("grdview /Users/mgardine/Development/aec_public/GMT/alaska_3m.grd -C/Users/mgardine/Development/aec_public/GMT/gray_blue.cpt -JCyl_stere/-153.5/61/6 -I/Users/mgardine/Development/aec_public/GMT/alaska_3m_int.grd -R-180/-127/50/72 -Ba300f300g300/a300f300g300wsne:.\"\": -E200/40 -Qi -P -X0.5 -Y3 -K > ~/Development/scratch/realtime_gmv/regions_%s.ps" % i ) os.system("pscoast -R -J -B -K -Di -N1/black -Wblack -A100/0/1 -E200/40 -P -O >> ~/Development/scratch/realtime_gmv/regions_%s.ps" % i) ## This is for Anchorage SM network #os.system("grdimage -R-150.2146/-149.5274/61.0289/61.2956 -JCyl_stere/-149.871/61.1623/6 /Users/mgardine/Development/GMT/anchorage.grd -I/Users/mgardine/Development/GMT/anchorage_int.grd -C/Users/mgardine/Development/GMT/gray_blue_anch.cpt -Ba300f300g300/a300f300g300wsne:.\"\": -X1.5 -Y4 -P -K > ~/Development/scratch/realtime_gmv/regions_%s.ps" % i ) ### # Added to test plotting hypocenter as a red star (M6.2 Willow EQ) ### #os.system("echo -151.816 61.9449 | psxy -R -J -Sa0.2 -W7,red -P -O -K >> ~/Development/scratch/realtime_gmv/regions_%s.ps" %i ) #os.system("echo -151.95 61.66 | psxy -R -J -Sa0.2 -W7,red -P -O -K >> ~/Development/scratch/realtime_gmv/regions_%s.ps" %i ) # 3-d os.system("echo -153.15 59.91 0 | psxyz -R-180/-127/50/72/0/1 -J -Sa0.2 -E200/40 -W7,red -P -O -K >> ~/Development/scratch/realtime_gmv/regions_%s.ps" %i ) j = 0 data = [] for sta in stations: dbsta = db.lookup("","site","","") dbsta = dbsta.subset("sta=~/%s/ && offdate==NULL" % sta) ######################## # Check to ensure that station records exist ######################## nsta = dbsta.query("dbRECORD_COUNT") if (nsta < 1): print "Failed to load metadata for station %s, skipping " % sta j += 1 continue ######################## # Get lat, lon of station and extract waveform data for time window (plus # time pad for removal of filter transients) ######################## dbsta[3] = 0 lat = dbsta.getv('lat') lon = dbsta.getv('lon') tr = db.loadchan(i-tpad, i+tstep+tpad, "%s" % sta, "%s" % chans[j]) ######################## # Check to ensure that waveform data is complete ######################## nseg = tr.query("dbRECORD_COUNT") if (nseg < 1): f = '%D %H:%M:%S' if verbose: failed_time = stock.epoch2str(i, f) print "Failed to load data from %s:%s at %s" % (sta, chans[j], failed_time) tr.trdestroy() j += 1 continue ######################## # Apply instrument calibrations and filter ######################## tr.apply_calib() tr.filter("%s" % filter) tr[3] = 0 v = tr.data() ######################## # Calculate PGA in ug, ignoring the time pad value on each side of time window ######################## samprate = tr.getv("samprate") w = [] ### Not sure about the correctness of this:? #for k in range(int(samprate[0]),len(v)-int(samprate[0])): # w.append(abs(float(v[k]))) for k in range(0, len(v)-1): w.append(abs(float(v[k]))) if w: pga = max(w)/9810 #os.system("echo %s %s %s | psxy -R -J -C%s -Sc0.1 -W0.1 -P -O -K >> ~/Development/scratch/realtime_gmv/regions_%s.ps" % (lon[0], lat[0], pga, gmt_params['cptfile'], i) ) # 3-d os.system("echo %s %s %s | psxyz -R-180/-127/50/72/1/5000 -J -Jz3l -E200/40 -So0.1 -Gred -W0.1 -P -O -K >> ~/Development/scratch/realtime_gmv/regions_%s.ps" % (lon[0], lat[0], pga, i) ) else: f = '%D %H:%M:%S' if verbose: failed_time = stock.epoch2str(i, f) print "Station %s:%s has gap at time %s" % (sta, chans[j], failed_time) tr.trdestroy() j += 1
def new(self, rawpkt): """Create a new Packet object. This works a little differently than most python objects because the underlying Antelope C code is leaky. The recommended way to manipulate packets is to create a packet buffer in memory and continually write/rewrite to that, rather than reallocate new memory from the heap. Caveats: This whole class is not at all Pythonic, and really a gigantic hack. """ if not rawpkt[0] or int(float(rawpkt[0])) < 1: self.logger.info("Bad Packet: %s %s %s" % (rawpkt[0], rawpkt[1], rawpkt[2])) return self._clean() self.rawpkt = rawpkt self.logger.debug(rawpkt) self.id = rawpkt[0] self.time = float(rawpkt[2]) self.strtime = stock.epoch2str(self.time, "%D %H:%M:%S %Z").strip() # Try to extract information from packet pkt = Pkt.Packet(rawpkt[1], rawpkt[2], rawpkt[3]) self.srcname = pkt.srcname if pkt.srcname else rawpkt[1] self.logger.info("%s %s %s" % (self.id, self.time, self.strtime)) # self.logger.debug( pkt.pf ) # Antelope 5.7 stock.ParameterFile.__getitem__ doesn't like the "foo in # bar" format. # Just try retrieving the value and catch whatever exception we get. # Antelope throws warnings if the key isn't found. We don't care. # https://stackoverflow.com/questions/14463277/how-to-disable-python-warnings with warnings.catch_warnings(): warnings.simplefilter("ignore") try: self.dls = pkt.pf["dls"] self.valid = True except (KeyError, TypeError): self.dls = {} self.valid = False if self.valid: try: self.imei = pkt.pf["imei"] self.logger.info("Found imei: %s" % (pkt.pf["imei"])) except KeyError: pass try: self.q330 = pkt.pf["q330"] self.logger.info("Found q330: %s" % (pkt.pf["q330"])) except KeyError: pass
def main(): ######################## # Get information from execution flags ######################## (input, pfname) = configure() ######################## # Get information from parameter file ######################## (tstep, tpad, filter, sta, chan, animation_params, gmt_params) = get_pf(pfname) ######################## # Set start and end time for animation ######################## tstart = float(stock.str2epoch(input.ts)) if input.ts is "now": tstart = tstart - (tstart % (input.twin)) - input.twin tend = tstart + float(input.twin) ######################## # Open waveform database ######################## if not input.dbname: f = '%Y_%m_%d' date = stock.epoch2str(tstart,f) db = ds.dbopen("/aerun/op/run/db/archive_%s" % date, "r") else: db = ds.dbopen(input.dbname, "r") ######################## # Setup format for creating timestamp on GMT plots ######################## f = '%D %H:%M:%S %Z' timestamp = stock.epoch2str(tstart, f, "") ## For state-wide pstextstring = "-153 53 10 0 1 0 %s" % timestamp ## For Anchorage #pstextstring = "-150 53 10 0 1 0 %s" % timestamp current_time = tstart while current_time < tend: ######################## # Get waveform data, compute PGA, and make GMT plot ######################## run_gmt(sta, chan, db, current_time, filter, tstep, tpad, gmt_params, input.verbose) ######################## # Make a timestamp every 10 seconds on the GMT plot ######################## if (current_time%10 == 0): timestamp = stock.epoch2str(current_time, f, "") ## For state-wide pstextstring = "-160 51 10 0 1 0 %s" % timestamp # For Anchorage #pstextstring = "-150.1 61.04 10 0 1 0 %s" % timestamp print "Processing time %s" % timestamp #os.system("echo %s | pstext -R -J -Gblack -P -O -K >> ~/Development/scratch/realtime_gmv/regions_%s.ps" % (pstextstring,current_time)) # 3-d os.system("echo %s | pstext -R -J -E200/40 -Gblack -P -O -K >> ~/Development/scratch/realtime_gmv/regions_%s.ps" % (pstextstring,current_time)) else: #os.system("echo %s | pstext -R -J -Gblack -P -O -K >> ~/Development/scratch/realtime_gmv/regions_%s.ps" % (pstextstring,current_time)) # 3-d os.system("echo %s | pstext -R -J -E200/40 -Gblack -P -O -K >> ~/Development/scratch/realtime_gmv/regions_%s.ps" % (pstextstring,current_time)) ######################## # Plot scalebar ######################## ## For state-wide #os.system("psscale -D4.1i/0.9i/1.0i/.20i -C%s -B:\"PGA (ug)\": -L0.0 -P -O >> ~/Development/scratch/realtime_gmv/regions_%s.ps" % (gmt_params['cptfile'], current_time) ) ## For Anchorage #os.system("psxy /Users/mgardine/Development/GMT/alaska_roads.xy -J -R -m -W2,darkgrey -K -O >> ~/Development/scratch/realtime_gmv/regions_%s.ps" % current_time ) #os.system("psscale -D3.8i/0.7i/1.0i/.20i -C%s -B:\"PGA (mg)\": -L0.0 -P -O >> ~/Development/scratch/realtime_gmv/regions_%s.ps" % (gmt_params['cptfile'], current_time) ) current_time += tstep ######################## # Make animated .GIF, remove temporary GMT plots ######################## print "Making animation" ## For state-wide os.system("/opt/local/bin/gm convert -density 200% -delay 10 -loop 0 -resize 1280x1280 +map ~/Development/scratch/realtime_gmv/regions*.ps ~/Development/scratch/realtime_gmv/animation.gif") os.system("convert ~/Development/scratch/realtime_gmv/animation.gif -layers OptimizeTransparency -crop 720x720+163+300 +repage ~/Development/scratch/realtime_gmv/animation1.gif") ## For Anchorage #os.system("/opt/local/bin/gm convert -density 200% -delay 10 -loop 0 -resize 1280x1280 +map ~/Development/scratch/realtime_gmv/regions*.ps ~/Development/scratch/realtime_gmv/animation.gif") #os.system("convert ~/Development/scratch/realtime_gmv/animation.gif -layers OptimizeTransparency -crop 720x580+163+245 +repage ~/Development/scratch/realtime_gmv/animation1.gif") #os.system("rm -f ~/Development/scratch/realtime_gmv/regions_*.ps ~/Development/scratch/realtime_gmv/animation.gif") #os.system("mv ~/Development/scratch/realtime_gmv/animation1.gif %s/animation_rt.gif" % animation_params['webdir']) #os.system("ffmpeg -i ~/Development/scratch/realtime_gmv/animation1.gif -c:v libvpx -crf 12 ~/Development/scratch/realtime_gmv/animation.webm") db.close()
def run_gmt(stations, chans, db, i, filter, tstep, tpad, verbose): ######################## # Create the basemap plot for the given time window ######################## os.system("grdimage -R-170/-130/52/72 -Jb-150/52/62/72/.25 /Users/mgardine/Development/GMT/alaska_5m.grd -I/Users/mgardine/Development/GMT/alaska_5m_int.grd -C/Users/mgardine/Development/GMT/gray_blue.cpt -Ba300f300/a300f300wsne:.\"\": -X1.5 -Y3 -P -K > ~/Development/scratch/realtime_gmv/regions_%s.ps" %i) #os.system("pscoast -R-170/-130/52/72 -Jb-150/52/62/72/.25 -K -Di -N1 -Ba300f300/a300f300wsne:.\"\": -W -S224/255/255 -G250/250/210 -A100/0/1 -X1.5 -Y3 -P >> ~/Development/scratch/realtime_gmv/regions_%s.ps" % i) os.system("pscoast -R -Jb -B -K -Di -N1/black -Wblack -A100/0/1 -P -O >> ~/Development/scratch/realtime_gmv/regions_%s.ps" % i) j = 0 data = [] for sta in stations: dbsta = db.lookup("","site","","") dbsta = dbsta.subset("sta=~/%s/ && offdate==NULL" % sta) ######################## # Check to ensure that station records exist ######################## nsta = dbsta.query("dbRECORD_COUNT") if (nsta < 1): print "Failed to load metadata for station %s, skipping " % sta j += 1 continue ######################## # Get lat, lon of station and extract waveform data for time window (plus # time pad for removal of filter transients) ######################## dbsta[3] = 0 lat = dbsta.getv('lat') lon = dbsta.getv('lon') tr = db.loadchan(i-tpad, i+tstep+tpad, "%s" % sta, "%s" % chans[j]) ######################## # Check to ensure that waveform data is complete ######################## nseg = tr.query("dbRECORD_COUNT") if (nseg < 1): f = '%D %H:%M:%S' if verbose: failed_time = stock.epoch2str(i, f) print "Failed to load data from %s:%s at %s" % (sta, chans[j], failed_time) tr.trdestroy() j += 1 continue ######################## # Apply instrument calibrations and filter ######################## tr.apply_calib() tr.filter("%s" % filter) tr[3] = 0 v = tr.data() ######################## # Calculate PGA in ug, ignoring the time pad value on each side of time window ######################## samprate = tr.getv("samprate") w = [] for k in range(int(samprate[0]),len(v)-int(samprate[0])): w.append(abs(float(v[k]))) if w: pga = max(w)/9810 os.system("echo %s %s %s | psxy -R -Jb -Cwhitered1.cpt -Sc0.1 -W0.1 -P -O -K >> ~/Development/scratch/realtime_gmv/regions_%s.ps" % (lon[0], lat[0], pga, i)) else: f = '%D %H:%M:%S' if verbose: failed_time = stock.epoch2str(i, f) print "Station %s:%s has gap at time %s" % (sta, chans[j], failed_time) tr.trdestroy() j += 1
def main(): ######################## # Get information from execution flags ######################## (input, pfname) = configure() ######################## # Get information from parameter file ######################## (tstep, tpad, filter, sta, chan) = get_pf(pfname) ######################## # Set start and end time for animation ######################## tstart = float(stock.str2epoch(input.ts)) if input.ts is "now": tstart = tstart - (tstart % (input.twin)) - input.twin tend = tstart + input.twin #tstart = float(stock.str2epoch('8/27/2013 21:40:00')) #tend = float(stock.str2epoch('8/27/2013 21:50:00')) #tstart = float(stock.str2epoch('8/25/2013 17:47:00')) #tend = float(stock.str2epoch('8/25/2013 17:57:00')) #tstart = float(stock.str2epoch('8/21/2013 18:37:00')) #tend = float(stock.str2epoch('8/21/2013 18:44:00')) ######################## # Open waveform database ######################## if not input.dbname: f = '%Y_%m_%d' date = stock.epoch2str(tstart,f) db = ds.dbopen("/aerun/op/run/db/archive_%s" % date, "r") #db = ds.dbopen("/aerun/op/run/db/archive_2013_08_21", "r") #db = ds.dbopen("/aerun/op/run/db/archive_2013_08_25", "r") #db = ds.dbopen("/aerun/op/run/db/archive_2013_08_27", "r") else: db = ds.dbopen(input.dbname, "r") ######################## # Setup format for creating timestamp on GMT plots ######################## f = '%D %H:%M:%S' timestamp = stock.epoch2str(tstart, f) pstextstring = "-153 53 10 0 1 0 %s" % timestamp current_time = tstart while current_time < tend: ######################## # Get waveform data, compute PGA, and make GMT plot ######################## run_gmt(sta, chan, db, current_time, filter, tstep, tpad, input.verbose) ######################## # Make a timestamp every 10 seconds on the GMT plot ######################## if (current_time%10 == 0): timestamp = stock.epoch2str(current_time, f) pstextstring = "-153 53 10 0 1 0 %s" % timestamp print "Processing time %s" % timestamp os.system("echo %s | pstext -R -Jb -Gblack -P -O -K >> ~/Development/scratch/realtime_gmv/regions_%s.ps" % (pstextstring,current_time)) else: os.system("echo %s | pstext -R -Jb -Gblack -P -O -K >> ~/Development/scratch/realtime_gmv/regions_%s.ps" % (pstextstring,current_time)) ######################## # Plot scalebar ######################## os.system("psscale -D4.1i/0.9i/1.0i/.20i -Cwhitered1.cpt -B:\"PGA (ug)\": -L0.0 -P -O >> ~/Development/scratch/realtime_gmv/regions_%s.ps" % current_time) current_time += tstep ######################## # Make animated .GIF, remove temporary GMT plots ######################## print "Making animation" os.system("convert -delay 10 -loop 2 ~/Development/scratch/realtime_gmv/regions*.ps ~/Development/scratch/realtime_gmv/animation.gif") os.system("rm -f ~/Development/scratch/realtime_gmv/regions_*.ps") os.system("mv ~/Development/scratch/realtime_gmv/animation.gif /usr/local/mosaic/Input/mgardine/animation_rt.gif") db.close()
volumes,net,time,endtime = db.getv("volumes","net","time","endtime") except Exception,e: raise DbcentralException("Problems with db.getv('volumes','net','time','endtime'). (%s)\n" % e) self.logger.debug( "volumes=%s" % volumes ) self.logger.debug( "net=%s" % net ) self.logger.debug( "time=%s" % time ) self.logger.debug( "endtime=%s" % endtime ) if endtime == null_endtime: # This will be problematic with realtime systems endtime = stock.now() self.logger.debug( "endtime=%s" % endtime ) start_year = int(stock.epoch2str(time,"%Y")) end_year = int(stock.epoch2str(endtime,"%Y")) start_month = int(stock.epoch2str(time,"%L")) end_month = int(stock.epoch2str(endtime,"%L")) if volumes == 'single': self._test_db(voltime,volendtime,dbname) elif volumes == 'year': for y in range(start_year,end_year+1): voltime = stock.str2epoch("1/1/%s 00:00:00" % y) volendtime = stock.str2epoch("12/31/%s 23:59:59" % y) dbname = stock.epoch2str(voltime,dbname_template)
def generate_report(params): """ Initiate QC tests and report generation/distribution. Behaviour: The main function intended to be called by importer. Initiates QC tests and report generation/disribution. Arguments: params - All parametrs <dict> params['dbin'] - Input database <str> params['pf'] - Parameter file <str> params['network'] - Network <str> params['tstart'] - Epoch start time <float> params['tend'] - Epoch end time <float> """ import sys import os sys.path.append('%s/data/python' % os.environ['ANTELOPE']) from antelope.datascope import dbopen from antelope.stock import epoch2str sys.path.remove('%s/data/python' % os.environ['ANTELOPE']) params = _parse_pf(params) qc_network_report = _QC_Network_Report({'network': params['network'], \ 'tstart': params['tstart'], 'tend': params['tend'], \ 'send_email': params.pop('send_email'), 'email': params.pop('email'), \ 'smtp_server': params.pop('smtp_server')}) db = dbopen(params['dbin']) db = db.lookup(table='wfmeas') db = db.subset("time == _%f_ && endtime == _%f_" \ % (params['tstart'],params['tend'])) db = db.sort('sta') db = db.group('sta') for db.record in range(db.nrecs()): sta = db.getv('sta')[0] issue_params = {'sta': sta} db_sta = db.subset("sta =~ /%s/" % sta) db_sta = db_sta.ungroup() db_sta = db_sta.sort('chan') db_sta = db_sta.group('chan') for db_sta.record in range(db_sta.nrecs()): chan = db_sta.getv('chan')[0] issue_params['chan'] = chan db_chan = db_sta.subset("chan =~ /%s/" % chan) db_chan = db_chan.ungroup() db_chan = db_chan.sort('meastype') db_chan = db_chan.group('meastype') for db_chan.record in range(db_chan.nrecs()): db_meas = db_chan.subset("meastype =~ /%s/" \ % db_chan.getv('meastype')[0]) db_meas = db_meas.ungroup() db_meas = db_meas.sort('tmeas') count = db_meas.nrecs() db_meas.record = 0 meastype = db_meas.getv('meastype')[0] if count == 1: ts, twin = db_meas.getv('tmeas', 'twin') te = ts + twin message = "%s test failed once between %s - %s\t%s\n" \ % (meastype, epoch2str(ts, "%Y%j %H:%M:%S"), \ epoch2str(te, "%Y%j %H:%M:%S"), _build_wf_link(sta,\ chan, params['netcode'], ts, te)) else: message = "%s test failed %d times between:" \ % (meastype, count) for db_meas.record in range(count): ts, twin = db_meas.getv('tmeas', 'twin') te = ts + twin message = "%s\n\t\t\t%s - %s\t%s" \ % (message, epoch2str(ts, "%Y%j %H:%M:%S"), \ epoch2str(te, "%Y%j %H:%M:%S"), \ _build_wf_link(sta, chan, params['netcode'], \ ts, te)) message = "%s\n" % message issue_params['message'] = message qc_network_report.add_issue(_QC_issue(issue_params)) qc_network_report.report()
def get_stations(self, regex, time, reference=False, event_data=None): """Get site info for each station.""" yearday = stock.epoch2str(time, "%Y%j") steps = [ "dbsubset ondate <= %s && (offdate >= %s || offdate == NULL)" % (yearday, yearday) ] steps.extend(["dbsort sta"]) steps.extend(["dbsubset %s" % regex]) self.logger.info("Database query for stations:") self.logger.info(", ".join(steps)) with datascope.freeing(self.table.process(steps)) as dbview: self.logger.info("Extracting sites for origin from db") strings = [] for temp in dbview.iter_record(): (sta, lat, lon, chan) = temp.getv("sta", "lat", "lon", "chan") if len(chan) > 3: chan_code = chan[:2] + "._." else: chan_code = chan[:2] string = sta + chan_code if string not in strings: strings.append(string) try: self.stations[sta].append_chan(chan_code) except Exception: self.stations[sta] = Records(sta, lat, lon) self.stations[sta].append_chan(chan_code) if reference and sta != reference: ssaz = "%0.2f" % temp.ex_eval( "azimuth(%s,%s,%s,%s)" % ( self.stations[reference].lat, self.stations[reference].lon, lat, lon, )) ssdelta = "%0.4f" % temp.ex_eval( "distance(%s,%s,%s,%s)" % ( self.stations[reference].lat, self.stations[reference].lon, lat, lon, )) ssdistance = round( temp.ex_eval("deg2km(%s)" % ssdelta), 2) self.stations[sta].set_ss(ssaz, ssdelta, ssdistance) if event_data: seaz = "%0.2f" % temp.ex_eval( "azimuth(%s,%s,%s,%s)" % (lat, lon, event_data.lat, event_data.lon)) esaz = "%0.2f" % temp.ex_eval( "azimuth(%s,%s,%s,%s)" % (event_data.lat, event_data.lon, lat, lon)) delta = "%0.4f" % temp.ex_eval( "distance(%s,%s,%s,%s)" % (event_data.lat, event_data.lon, lat, lon)) realdistance = temp.ex_eval("deg2km(%s)" % delta) pdelay = int( temp.ex_eval("pphasetime(%s,%s)" % (delta, event_data.depth))) if pdelay > 0: pdelay -= 1 else: pdelay = 0 ptime = time + pdelay self.stations[sta].set_es(seaz, esaz, delta, realdistance, pdelay, ptime) return self.stations
def generate_report(params): """ Initiate QC tests and report generation/distribution. Behaviour: The main function intended to be called by importer. Initiates QC tests and report generation/disribution. Arguments: params - All parametrs <dict> params['dbin'] - Input database <str> params['pf'] - Parameter file <str> params['network'] - Network <str> params['tstart'] - Epoch start time <float> params['tend'] - Epoch end time <float> """ import sys import os sys.path.append("%s/data/python" % os.environ["ANTELOPE"]) from antelope.datascope import dbopen from antelope.stock import epoch2str sys.path.remove("%s/data/python" % os.environ["ANTELOPE"]) params = _parse_pf(params) qc_network_report = _QC_Network_Report( { "network": params["network"], "tstart": params["tstart"], "tend": params["tend"], "send_email": params.pop("send_email"), "email": params.pop("email"), "smtp_server": params.pop("smtp_server"), } ) db = dbopen(params["dbin"]) db = db.lookup(table="wfmeas") db = db.subset( "time == _%f_ && endtime == _%f_" % (params["tstart"], params["tend"]) ) db = db.sort("sta") db = db.group("sta") for db.record in range(db.nrecs()): sta = db.getv("sta")[0] issue_params = {"sta": sta} db_sta = db.subset("sta =~ /%s/" % sta) db_sta = db_sta.ungroup() db_sta = db_sta.sort("chan") db_sta = db_sta.group("chan") for db_sta.record in range(db_sta.nrecs()): chan = db_sta.getv("chan")[0] issue_params["chan"] = chan db_chan = db_sta.subset("chan =~ /%s/" % chan) db_chan = db_chan.ungroup() db_chan = db_chan.sort("meastype") db_chan = db_chan.group("meastype") for db_chan.record in range(db_chan.nrecs()): db_meas = db_chan.subset( "meastype =~ /%s/" % db_chan.getv("meastype")[0] ) db_meas = db_meas.ungroup() db_meas = db_meas.sort("tmeas") count = db_meas.nrecs() db_meas.record = 0 meastype = db_meas.getv("meastype")[0] if count == 1: ts, twin = db_meas.getv("tmeas", "twin") te = ts + twin message = "%s test failed once between %s - %s\t%s\n" % ( meastype, epoch2str(ts, "%Y%j %H:%M:%S"), epoch2str(te, "%Y%j %H:%M:%S"), _build_wf_link(sta, chan, params["netcode"], ts, te), ) else: message = "%s test failed %d times between:" % (meastype, count) for db_meas.record in range(count): ts, twin = db_meas.getv("tmeas", "twin") te = ts + twin message = "%s\n\t\t\t%s - %s\t%s" % ( message, epoch2str(ts, "%Y%j %H:%M:%S"), epoch2str(te, "%Y%j %H:%M:%S"), _build_wf_link(sta, chan, params["netcode"], ts, te), ) message = "%s\n" % message issue_params["message"] = message qc_network_report.add_issue(_QC_issue(issue_params)) qc_network_report.report()
def new(self, rawpkt, name_type="pf/xi", select=False, reject=False, silent=False): """Generate a new packet.""" self.logging.debug("new packet") if not rawpkt["_id"]: if not silent: self.logging.warning("Bad Packet: %s" % rawpkt) return self._clean() self.name_type = name_type # Disable _convert_unicod - it seems to have the opposite affect under # python 3 or the newer pymongo library. # self.rawpkt = self._convert_unicode(rawpkt) self.rawpkt = rawpkt if reject and re.search(reject, self.rawpkt["srcType"]): self.logging.debug("type [%s] rejected by configuration" % self.rawpkt["srcType"]) return if select and not re.search(select, self.rawpkt["srcType"]): self.logging.debug("type [%s] missed selection by configuration" % self.rawpkt["srcType"]) return self.logging.debug(self.rawpkt) # Track IDs self.logSeqNo = self.rawpkt["messageLogSeqNo"] self.seqNo = self.rawpkt["seqNo"] self.id = "%s.%s" % (self.logSeqNo, self.seqNo) # Date object self.datetime = self.rawpkt["timestamp"] # Epoch string self.time = (self.datetime - datetime.datetime(1970, 1, 1)).total_seconds() # Time string self.strtime = stock.epoch2str(self.time, "%D %H:%M:%S %Z", tz="UTC").strip() self.q330 = self.rawpkt["q330Sn"] self.imei = self.rawpkt["deviceIMEI"] self.src = self.rawpkt["srcType"] self.srcname = self.src.lower() if not self.imei_buffer.add(imei=self.imei, serial=self.q330): self.logging.warning("Invalid Q330 serial [%s] for IMEI [%s]" % (self.q330, self.imei)) self.q330 = self.imei_buffer(self.imei) if not self.q330: if not silent: self.logging.warning( "UNKNOWN IMEI [%s]: SKIP DATA PACKET!!!" % self.imei) return else: if not silent: self.logging.warning( "USING CACHED Q330 SERIAL [%s] FOR IMEI [%s]" % (self.q330, self.imei)) for test in self.q330_serial_dlname: if test(self.q330): self.dlname = test(self.q330) self.logging.debug("%s => %s" % (self.q330, self.dlname)) if not self.dlname: if not silent: self.logging.warning("NO DLNAME FOR Q330 SERIAL: %s " % self.q330) return self.logging.debug(self.src) self.logging.debug(self.valueMap) # Verify if we have data pairs if "valueMap" in self.rawpkt: self.valueMap = self.rawpkt["valueMap"] # Extract each value to a new key:value on the dict for chan in self.valueMap: if chan in self.channel_mapping: if not self.channel_mapping[chan]: continue self.payload[ self.channel_mapping[chan]] = self.valueMap[chan] self.logging.debug("%s -> %s:%s" % (chan, self.channel_mapping[chan], self.valueMap[chan])) else: self.logging.warning("[%s] NOT DEFINED IN PF FILE" % chan) for test in ["xil1q", "xil2q"]: if test in self.payload and "xisamp" in self.payload: try: self.payload[test] = (float(self.payload[test]) / self.payload["xisamp"]) except Exception: pass self.pcktbuf = { "dls": { self.dlname: self.payload }, "q330": self.q330, "imei": self.imei, "src": self.src, "srcname": self.srcname, } self.logging.debug(self.payload) # Try to build packet from info if self.name_type and self.time and self.payload: self.pkt.srcname = Pkt.SrcName("%s/%s" % (self.dlname, self.name_type)) self.pkt.time = self.time # self.logging.debug( self.pkt.type ) # self.logging.debug( self.pkt.srcname ) # Extract pf structure, update it and return it. temp = self.pkt.pf temp.update(self.pcktbuf) self.pkt.pf = temp self.logging.debug(self.pkt.type) self.logging.debug(self.pkt.srcname) self.logging.debug("Pkt( %s, %s) => {%s}" % ( self.pkt.srcname, self.pkt.time, self.pkt.pf.pf2string().replace("\n", ", ").replace("\t", ":"), )) self.valid = True else: self.logging.warning( "NO VALUABLE INFORMATION IN PACKET. dlname:%s time:%s" % (self.dlname, self.time)) return self.logging.info(str(self))
self.logger.debug("time=%s" % time) self.logger.debug("endtime=%s" % endtime) if endtime == null_endtime: # This will be problematic with realtime systems endtime = stock.now() self.logger.debug("endtime=%s" % endtime) if self.volumes == 'single': self._test_db(voltime, volendtime, dbname) elif self.volumes == 'year': start_year = int(stock.epoch2str(time, "%Y")) end_year = int(stock.epoch2str(endtime, "%Y")) for y in range(start_year, end_year + 1): voltime = stock.str2epoch("1/1/%s 00:00:00" % y) volendtime = stock.str2epoch("12/31/%s 23:59:59" % y) dbname = stock.epoch2str(voltime, dbname_template) self._test_db(voltime, volendtime, dbname) elif self.volumes == 'month': start_month = int(stock.epoch2str(time, "%L")) start_year = int(stock.epoch2str(time, "%Y")) end_month = int(stock.epoch2str(endtime, "%L"))