def get_seismic_station_metadata(self, start_time=0, end_time=None):
"""Retrieve station locations from db and yield as a generator.
Args:
start_time(numeric): unix timestamp. Stations with endtimes before this time will be excluded.
end_time (numeric): unix timestamp. Stations with "time" after this time will be excluded.
Yields:
SeismicStationMetadata tuple: includes snet as one of the key/value pairs. Records are returned
sorted by snet then by sta.
"""
stats = {"numsta": 0}
# Construct a dbsubset format string
dbsubset_cmd = self._onoff_subset(start_time, end_time)
if len(dbsubset_cmd) > 0:
dbsubset_cmd = "dbsubset " + dbsubset_cmd
else:
dbsubset_cmd = None
# Define dbops
process_list = ["dbopen site"]
if dbsubset_cmd is not None:
process_list.append(dbsubset_cmd)
process_list += [
"dbjoin snetsta",
"dbsort snet sta",
]
self.logger.debug("Seismic process_list:\n%s", pformat(process_list))
with datascope.freeing(self.get_pointer()) as dbptr:
dbptr = dbptr.process(process_list)
for record in dbptr.iter_record():
stats["numsta"] += 1
vals = record.getv("snet", "sta", "lat", "lon", "ondate",
"offdate")
sta_data = SeismicStationMetadata(
snet=vals[0],
sta=vals[1],
lat=vals[2],
lon=vals[3],
time=stock.epoch(vals[4]),
endtime=stock.epoch(vals[5]),
extra_channels=None,
)
yield sta_data
LOGGER.debug("Processed %d stations.", stats["numsta"])
def parse_sta_date(time, epoch=False, nullval="-"):
"""Verify that we have a valid ondate/offdate."""
try:
if float(time) < 1.0:
raise
if stock.epoch(int(time)) > stock.now():
raise
if epoch:
return stock.epoch(int(time))
else:
return int(time)
except Exception:
return nullval
def parse_sta_date(time, epoch=False, nullval="-"):
"""
Verify that we have a valid ondate/offdate.
"""
try:
if float(time) < 1.0:
raise
if stock.epoch(int(time)) > stock.now():
raise
if epoch:
return stock.epoch(int(time))
else:
return int(time)
except Exception, e:
return nullval
def day_resolver(self, start_dy, end_dy, dbname_template):
"""Day list
"""
if self.include_times:
days = {}
else:
days = []
while start_dy <= end_dy:
voltime = antstock.epoch(start_day)
volendtime = voltime + 86399 # one second less than a full day
dbname = antstock.epoch2str(voltime, dbname_template)
if self.include_times:
days[dbname] = (voltime, volendtime)
else:
days.append(dbname)
#if os.path.exists(dbname) and os.path.isfile(dbname):
# if self.include_times:
# days[dbname] = (voltime, volendtime)
# else:
# days.append(dbname)
#else:
# antelog.notify("Dbpath '%s' does not exist." % dbname)
start_dy = yearday((antstock.epoch(start_dy) + 86400))
return days
volendtime = stock.str2epoch("%d/1/%d" % (vol_endmonth,vol_endyear) ) - 1
dbname = stock.epoch2str(int(voltime), dbname_template)
self._test_db(voltime,volendtime,dbname)
elif volumes == 'day':
start_day = int(stock.yearday(time))
end_day = int(stock.yearday(endtime))
vol_day = start_day
while vol_day <= end_day:
voltime = stock.epoch(vol_day)
volendtime = voltime + 86399 # one second less than a full day
dbname = stock.epoch2str(voltime, dbname_template)
if self._test_db(voltime,volendtime,dbname):
self.dbs[dbname] = {'times': [time,endtime]}
vol_day = stock.yearday((stock.epoch(vol_day)+86400))
else:
raise UnknownVolumeTypeException(volumes)
self.logger.debug( "DBS=%s" % self.dbs.keys() )
def _test_db(self,time,endtime,dbname):
def _get_list(self):
try:
db = datascope.dbopen(self.path, "r")
except Exception as e:
raise DbcentralException("Cannot open database %s (%s)" % (self.path, e))
try:
db = db.lookup("", "clusters", "", "")
except datascope.DblookupFieldError:
self.type = "masquerade"
self.nickname = None
self.dbs[self.path] = {"times": [-10000000000.0, 10000000000.0]}
self.logger.info("Not a dbcentral database. Set single database.")
return
else:
self.type = "dbcentral"
if self.nickname is None:
raise ValueError("Need nickname for Dbcentral clustername regex.")
try:
db = db.lookup("", "clusters", "", "dbNULL")
null_time, null_endtime = db.getv("time", "endtime")
except Exception as e:
raise DbcentralException(
"Cannot look up null values in clusters table. (%s)" % e
)
expr = "clustername =='%s'" % self.nickname
try:
db = db.subset(expr)
except Exception as e:
raise DbcentralException("Cannot subset on clustername. %s" % e)
try:
db = db.sort("time")
nclusters = db.record_count
except Exception as e:
raise DbcentralException("Cannot sort on 'time' . %s" % e)
if nclusters < 1:
raise DbcentralException('No matches for nickname "%s".' % self.nickname)
self.logger.debug("Records=%s" % nclusters)
for i in range(nclusters):
self.logger.debug("db.record=%s" % i)
db.record = i
try:
dbname_template = db.extfile()[-1]
except Exception as e:
raise DbcentralException("Cannot run db.extfile(). %s" % e)
self.logger.debug("dbname_template=%s" % dbname_template)
try:
volumes, net, time, endtime = db.getv(
"volumes", "net", "time", "endtime"
)
except Exception as e:
raise DbcentralException(
"Problems with db.getv('volumes','net',"
+ "'time','endtime'). (%s)\n" % e
)
self.logger.debug("volumes=%s" % volumes)
self.logger.debug("net=%s" % net)
self.logger.debug("time=%s" % time)
self.logger.debug("endtime=%s" % endtime)
if endtime == null_endtime:
# This will be problematic with realtime systems
endtime = stock.now()
self.logger.debug("endtime=%s" % endtime)
start_year = int(stock.epoch2str(time, "%Y"))
end_year = int(stock.epoch2str(endtime, "%Y"))
start_month = int(stock.epoch2str(time, "%L"))
end_month = int(stock.epoch2str(endtime, "%L"))
if volumes == "single":
dbname = stock.epoch2str(time, dbname_template)
self._test_db(time, endtime, dbname)
elif volumes == "year":
for y in range(start_year, end_year + 1):
voltime = stock.str2epoch("1/1/%s 00:00:00" % y)
volendtime = stock.str2epoch("12/31/%s 23:59:59" % y)
dbname = stock.epoch2str(voltime, dbname_template)
self._test_db(voltime, volendtime, dbname)
elif volumes == "month":
vol_month = start_month
vol_year = start_year
vol_endmonth = end_month
vol_endyear = end_year
while vol_year < end_year or (
vol_year == end_year and vol_month <= end_month
):
voltime = stock.str2epoch("%d/1/%d" % (vol_month, vol_year))
if vol_month < 12:
vol_month = vol_month + 1
else:
vol_year = vol_year + 1
vol_month = 1
volendtime = (
stock.str2epoch("%d/1/%d" % (vol_endmonth, vol_endyear)) - 1
)
dbname = stock.epoch2str(int(voltime), dbname_template)
self._test_db(voltime, volendtime, dbname)
elif volumes == "day":
start_day = int(stock.yearday(time))
end_day = int(stock.yearday(endtime))
vol_day = start_day
while vol_day <= end_day:
voltime = stock.epoch(vol_day)
volendtime = voltime + 86399 # full day -1 sec
dbname = stock.epoch2str(voltime, dbname_template)
if self._test_db(voltime, volendtime, dbname):
self.dbs[dbname] = {"times": [time, endtime]}
vol_day = stock.yearday((stock.epoch(vol_day) + 86400))
else:
raise UnknownVolumeTypeException(volumes)
self.logger.debug("DBS=%s" % self.dbs.keys())
for i in range(antdb.dbquery(infraptr_grp, antdb.dbRECORD_COUNT)):
infraptr_grp[3] = i
sta, [db, view, end_rec, start_rec] = \
antdb.dbgetv(infraptr_grp, 'sta', 'bundle')
all_stations[sta] = {'sensors': {'MEMS':False, 'NCPA':False,
'SETRA':False},
'location': {'lat':0, 'lon':0}}
for j in range(start_rec, end_rec):
infraptr[3] = j
# Cannot use time or endtime as that applies to the station, not to the inframet sensor
ondate, offdate, chan, lat, lon = \
antdb.dbgetv(infraptr, 'ondate', 'offdate', 'chan', 'lat', 'lon')
all_stations[sta]['location']['lat'] = lat
all_stations[sta]['location']['lon'] = lon
ondate = epoch(ondate)
if offdate > 0:
offdate = epoch(offdate)
else:
offdate = 'NULL'
if chan == 'LDM_EP':
if ondate <= end_time and (offdate == 'NULL' or offdate >= start_time):
all_stations[sta]['sensors']['MEMS'] = True
elif chan == 'BDF_EP' or chan == 'LDF_EP':
if ondate <= end_time and (offdate == 'NULL' or offdate >= start_time):
all_stations[sta]['sensors']['NCPA'] = True
elif chan == 'BDO_EP' or chan == 'LDO_EP':
if ondate <= end_time and (offdate == 'NULL' or offdate > start_time):
all_stations[sta]['sensors']['SETRA'] = True
if vol_month < 12:
vol_month = vol_month + 1
else:
vol_year = vol_year + 1
vol_month = 1
elif self.volumes == 'day':
start_day = int(stock.yearday(time))
end_day = int(stock.yearday(endtime))
vol_day = start_day
while vol_day <= end_day:
voltime = stock.epoch(vol_day)
volendtime = voltime + 86399 # one second less than a full day
dbname = stock.epoch2str(voltime, dbname_template)
self._test_db(voltime, volendtime, dbname)
vol_day = stock.yearday((stock.epoch(vol_day) + 86400))
else:
self._problem(
"Volumes type '%s' in cluster database not understood" %
volumes)
self.logger.debug("DBS=%s" % self.dbs.keys())
def _test_db(self, time, endtime, dbname):
