def get_seismic_station_metadata(self, start_time=0, end_time=None): """Retrieve station locations from db and yield as a generator. Args: start_time(numeric): unix timestamp. Stations with endtimes before this time will be excluded. end_time (numeric): unix timestamp. Stations with "time" after this time will be excluded. Yields: SeismicStationMetadata tuple: includes snet as one of the key/value pairs. Records are returned sorted by snet then by sta. """ stats = {"numsta": 0} # Construct a dbsubset format string dbsubset_cmd = self._onoff_subset(start_time, end_time) if len(dbsubset_cmd) > 0: dbsubset_cmd = "dbsubset " + dbsubset_cmd else: dbsubset_cmd = None # Define dbops process_list = ["dbopen site"] if dbsubset_cmd is not None: process_list.append(dbsubset_cmd) process_list += [ "dbjoin snetsta", "dbsort snet sta", ] self.logger.debug("Seismic process_list:\n%s", pformat(process_list)) with datascope.freeing(self.get_pointer()) as dbptr: dbptr = dbptr.process(process_list) for record in dbptr.iter_record(): stats["numsta"] += 1 vals = record.getv("snet", "sta", "lat", "lon", "ondate", "offdate") sta_data = SeismicStationMetadata( snet=vals[0], sta=vals[1], lat=vals[2], lon=vals[3], time=stock.epoch(vals[4]), endtime=stock.epoch(vals[5]), extra_channels=None, ) yield sta_data LOGGER.debug("Processed %d stations.", stats["numsta"])
def parse_sta_date(time, epoch=False, nullval="-"): """Verify that we have a valid ondate/offdate.""" try: if float(time) < 1.0: raise if stock.epoch(int(time)) > stock.now(): raise if epoch: return stock.epoch(int(time)) else: return int(time) except Exception: return nullval
def parse_sta_date(time, epoch=False, nullval="-"): """ Verify that we have a valid ondate/offdate. """ try: if float(time) < 1.0: raise if stock.epoch(int(time)) > stock.now(): raise if epoch: return stock.epoch(int(time)) else: return int(time) except Exception, e: return nullval
def day_resolver(self, start_dy, end_dy, dbname_template): """Day list """ if self.include_times: days = {} else: days = [] while start_dy <= end_dy: voltime = antstock.epoch(start_day) volendtime = voltime + 86399 # one second less than a full day dbname = antstock.epoch2str(voltime, dbname_template) if self.include_times: days[dbname] = (voltime, volendtime) else: days.append(dbname) #if os.path.exists(dbname) and os.path.isfile(dbname): # if self.include_times: # days[dbname] = (voltime, volendtime) # else: # days.append(dbname) #else: # antelog.notify("Dbpath '%s' does not exist." % dbname) start_dy = yearday((antstock.epoch(start_dy) + 86400)) return days
volendtime = stock.str2epoch("%d/1/%d" % (vol_endmonth,vol_endyear) ) - 1 dbname = stock.epoch2str(int(voltime), dbname_template) self._test_db(voltime,volendtime,dbname) elif volumes == 'day': start_day = int(stock.yearday(time)) end_day = int(stock.yearday(endtime)) vol_day = start_day while vol_day <= end_day: voltime = stock.epoch(vol_day) volendtime = voltime + 86399 # one second less than a full day dbname = stock.epoch2str(voltime, dbname_template) if self._test_db(voltime,volendtime,dbname): self.dbs[dbname] = {'times': [time,endtime]} vol_day = stock.yearday((stock.epoch(vol_day)+86400)) else: raise UnknownVolumeTypeException(volumes) self.logger.debug( "DBS=%s" % self.dbs.keys() ) def _test_db(self,time,endtime,dbname):
def _get_list(self): try: db = datascope.dbopen(self.path, "r") except Exception as e: raise DbcentralException("Cannot open database %s (%s)" % (self.path, e)) try: db = db.lookup("", "clusters", "", "") except datascope.DblookupFieldError: self.type = "masquerade" self.nickname = None self.dbs[self.path] = {"times": [-10000000000.0, 10000000000.0]} self.logger.info("Not a dbcentral database. Set single database.") return else: self.type = "dbcentral" if self.nickname is None: raise ValueError("Need nickname for Dbcentral clustername regex.") try: db = db.lookup("", "clusters", "", "dbNULL") null_time, null_endtime = db.getv("time", "endtime") except Exception as e: raise DbcentralException( "Cannot look up null values in clusters table. (%s)" % e ) expr = "clustername =='%s'" % self.nickname try: db = db.subset(expr) except Exception as e: raise DbcentralException("Cannot subset on clustername. %s" % e) try: db = db.sort("time") nclusters = db.record_count except Exception as e: raise DbcentralException("Cannot sort on 'time' . %s" % e) if nclusters < 1: raise DbcentralException('No matches for nickname "%s".' % self.nickname) self.logger.debug("Records=%s" % nclusters) for i in range(nclusters): self.logger.debug("db.record=%s" % i) db.record = i try: dbname_template = db.extfile()[-1] except Exception as e: raise DbcentralException("Cannot run db.extfile(). %s" % e) self.logger.debug("dbname_template=%s" % dbname_template) try: volumes, net, time, endtime = db.getv( "volumes", "net", "time", "endtime" ) except Exception as e: raise DbcentralException( "Problems with db.getv('volumes','net'," + "'time','endtime'). (%s)\n" % e ) self.logger.debug("volumes=%s" % volumes) self.logger.debug("net=%s" % net) self.logger.debug("time=%s" % time) self.logger.debug("endtime=%s" % endtime) if endtime == null_endtime: # This will be problematic with realtime systems endtime = stock.now() self.logger.debug("endtime=%s" % endtime) start_year = int(stock.epoch2str(time, "%Y")) end_year = int(stock.epoch2str(endtime, "%Y")) start_month = int(stock.epoch2str(time, "%L")) end_month = int(stock.epoch2str(endtime, "%L")) if volumes == "single": dbname = stock.epoch2str(time, dbname_template) self._test_db(time, endtime, dbname) elif volumes == "year": for y in range(start_year, end_year + 1): voltime = stock.str2epoch("1/1/%s 00:00:00" % y) volendtime = stock.str2epoch("12/31/%s 23:59:59" % y) dbname = stock.epoch2str(voltime, dbname_template) self._test_db(voltime, volendtime, dbname) elif volumes == "month": vol_month = start_month vol_year = start_year vol_endmonth = end_month vol_endyear = end_year while vol_year < end_year or ( vol_year == end_year and vol_month <= end_month ): voltime = stock.str2epoch("%d/1/%d" % (vol_month, vol_year)) if vol_month < 12: vol_month = vol_month + 1 else: vol_year = vol_year + 1 vol_month = 1 volendtime = ( stock.str2epoch("%d/1/%d" % (vol_endmonth, vol_endyear)) - 1 ) dbname = stock.epoch2str(int(voltime), dbname_template) self._test_db(voltime, volendtime, dbname) elif volumes == "day": start_day = int(stock.yearday(time)) end_day = int(stock.yearday(endtime)) vol_day = start_day while vol_day <= end_day: voltime = stock.epoch(vol_day) volendtime = voltime + 86399 # full day -1 sec dbname = stock.epoch2str(voltime, dbname_template) if self._test_db(voltime, volendtime, dbname): self.dbs[dbname] = {"times": [time, endtime]} vol_day = stock.yearday((stock.epoch(vol_day) + 86400)) else: raise UnknownVolumeTypeException(volumes) self.logger.debug("DBS=%s" % self.dbs.keys())
for i in range(antdb.dbquery(infraptr_grp, antdb.dbRECORD_COUNT)): infraptr_grp[3] = i sta, [db, view, end_rec, start_rec] = \ antdb.dbgetv(infraptr_grp, 'sta', 'bundle') all_stations[sta] = {'sensors': {'MEMS':False, 'NCPA':False, 'SETRA':False}, 'location': {'lat':0, 'lon':0}} for j in range(start_rec, end_rec): infraptr[3] = j # Cannot use time or endtime as that applies to the station, not to the inframet sensor ondate, offdate, chan, lat, lon = \ antdb.dbgetv(infraptr, 'ondate', 'offdate', 'chan', 'lat', 'lon') all_stations[sta]['location']['lat'] = lat all_stations[sta]['location']['lon'] = lon ondate = epoch(ondate) if offdate > 0: offdate = epoch(offdate) else: offdate = 'NULL' if chan == 'LDM_EP': if ondate <= end_time and (offdate == 'NULL' or offdate >= start_time): all_stations[sta]['sensors']['MEMS'] = True elif chan == 'BDF_EP' or chan == 'LDF_EP': if ondate <= end_time and (offdate == 'NULL' or offdate >= start_time): all_stations[sta]['sensors']['NCPA'] = True elif chan == 'BDO_EP' or chan == 'LDO_EP': if ondate <= end_time and (offdate == 'NULL' or offdate > start_time): all_stations[sta]['sensors']['SETRA'] = True
if vol_month < 12: vol_month = vol_month + 1 else: vol_year = vol_year + 1 vol_month = 1 elif self.volumes == 'day': start_day = int(stock.yearday(time)) end_day = int(stock.yearday(endtime)) vol_day = start_day while vol_day <= end_day: voltime = stock.epoch(vol_day) volendtime = voltime + 86399 # one second less than a full day dbname = stock.epoch2str(voltime, dbname_template) self._test_db(voltime, volendtime, dbname) vol_day = stock.yearday((stock.epoch(vol_day) + 86400)) else: self._problem( "Volumes type '%s' in cluster database not understood" % volumes) self.logger.debug("DBS=%s" % self.dbs.keys()) def _test_db(self, time, endtime, dbname):