def __init__(self, app, da, validUntil):
self._da = da
self._validUntil = validUntil
self._extents = {}
self._extentsSorted = []
self._extentsOID = {}
for i in xrange(self._da.dataExtentCount()):
ext = self._da.dataExtent(i)
wid = ext.waveformID()
sid = "%s.%s.%s.%s" % (wid.networkCode(), wid.stationCode(),
wid.locationCode(), wid.channelCode())
restricted = app._openStreams is None or sid not in app._openStreams
if restricted and not app._allowRestricted:
continue
self._extents[sid] = (ext, restricted)
# Logging.debug("%s: %s ~ %s" % (sid, ext.start().iso(),
# ext.end().iso()))
if app._serveAvailability:
# load data attribute extents if availability is served
for i in xrange(da.dataExtentCount()):
extent = da.dataExtent(i)
app.query().loadDataAttributeExtents(extent)
# create a list of (extent, oid, restricted) tuples sorted by stream
self._extentsSorted = [(e, app.query().getCachedId(e), res)
for wid, (e, res) in sorted(self._extents.iteritems(),
key=lambda t: t[0])]
# create a dictionary of object ID to extents
self._extentsOID = dict((oid, (e, res))
for (e, oid, res) in self._extentsSorted)
Logging.info("loaded %i extents" % len(self._extents))
def _loadInventory(self):
Logging.debug("loading inventory")
dbr = DataModel.DatabaseReader(self.database())
self._inv = DataModel.Inventory()
# Load networks and stations
staCount = 0
for i in xrange(dbr.loadNetworks(self._inv)):
staCount += dbr.load(self._inv.network(i))
Logging.debug("loaded %i stations from %i networks" %
(staCount, self._inv.networkCount()))
# Load sensors, skip calibrations (not needed by StationXML exporter)
Logging.debug("loaded %i sensors" % dbr.loadSensors(self._inv))
# Load datalogger and its decimations, skip calibrations (not needed by
# StationXML exporter)
deciCount = 0
for i in xrange(dbr.loadDataloggers(self._inv)):
deciCount += dbr.loadDecimations(self._inv.datalogger(i))
Logging.debug("loaded %i decimations from %i dataloggers" %
(deciCount, self._inv.dataloggerCount()))
# Load responses
resPAZCount = dbr.loadResponsePAZs(self._inv)
resFIRCount = dbr.loadResponseFIRs(self._inv)
resPolCount = dbr.loadResponsePolynomials(self._inv)
resCount = resPAZCount + resFIRCount + resPolCount
Logging.debug("loaded %i responses (PAZ: %i, FIR: %i, Poly: %i)" %
(resCount, resPAZCount, resFIRCount, resPolCount))
Logging.info("inventory loaded")
def _loadInventory(self):
Logging.notice("loading inventory")
dbr = DataModel.DatabaseReader(self.database())
self._inv = DataModel.Inventory()
# Load networks and stations
staCount = 0
for i in xrange(dbr.loadNetworks(self._inv)):
staCount += dbr.load(self._inv.network(i))
Logging.debug("loaded %i stations from %i networks" % (
staCount, self._inv.networkCount()))
# Load sensors, skip calibrations (not needed by StationXML exporter)
Logging.debug("loaded %i sensors" % dbr.loadSensors(self._inv))
# Load datalogger and its decimations, skip calibrations (not needed by
# StationXML exporter)
deciCount = 0
for i in xrange(dbr.loadDataloggers(self._inv)):
deciCount += dbr.loadDecimations(self._inv.datalogger(i))
Logging.debug("loaded %i decimations from %i dataloggers" % (
deciCount, self._inv.dataloggerCount()))
# Load responses
resPAZCount = dbr.loadResponsePAZs(self._inv)
resFIRCount = dbr.loadResponseFIRs(self._inv)
resPolCount = dbr.loadResponsePolynomials(self._inv)
resCount = resPAZCount + resFIRCount + resPolCount
Logging.debug("loaded %i responses (PAZ: %i, FIR: %i, Poly: %i)" % (
resCount, resPAZCount, resFIRCount, resPolCount))
Logging.info("inventory loaded")
def __init__(self, config, name, group):
mediatorAddress = config.getString("connection.server")
dbDriverName = config.getString("database.type")
dbAddress = config.getString("database.parameters")
connection = Communication.Connection.Create(mediatorAddress, name, group)
if connection is None:
Logging.error("Cannot connect to Mediator")
raise ConnectionError, "connection could not be established"
else:
Logging.info("Connection has been established")
dbDriver = IO.DatabaseInterface.Create(dbDriverName)
if dbDriver is None:
Logging.error("Cannot find database driver " + dbDriverName)
raise DatabaseError, "driver not found"
if not dbDriver.connect(dbAddress):
Logging.error("Cannot connect to database at " + dbAddress)
raise DatabaseError, "connection could not be established"
self.__connection = connection
# This reference to dbDriver is essential, since dbQuery becomes
# invalid when dbDriver is deleted
self.__dbDriver = dbDriver
self.dbQuery = DatabaseQuery(dbDriver)
def run(self):
retn = False
try:
for user in self._authBlacklist:
self._userdb.blacklistUser(user)
site = self._site()
if not site:
return False
# start listen for incoming request
self.__tcpPort = reactor.listenTCP(self._port,
site,
self._connections,
self._listenAddress)
# setup signal handler
self.__sighup = False
signal.signal(signal.SIGHUP, self._sighupHandler)
task.LoopingCall(self._reloadTask).start(60, False)
# start processing
Logging.info("start listening")
log.addObserver(logSC3)
reactor.run()
retn = True
except Exception, e:
Logging.error(str(e))
def __init__(self, config, name, group):
mediatorAddress = config.getString("connection.server")
dbDriverName = config.getString("database.type")
dbAddress = config.getString("database.parameters")
connection = Communication.Connection.Create(mediatorAddress, name, group)
if connection is None:
Logging.error("Cannot connect to Mediator")
raise ConnectionError, "connection could not be established"
else:
Logging.info("Connection has been established")
dbDriver = IO.DatabaseInterface.Create(dbDriverName)
if dbDriver is None:
Logging.error("Cannot find database driver " + dbDriverName)
raise DatabaseError, "driver not found"
if not dbDriver.connect(dbAddress):
Logging.error("Cannot connect to database at " + dbAddress)
raise DatabaseError, "connection could not be established"
self.__connection = connection
# This reference to dbDriver is essential, since dbQuery becomes
# invalid when dbDriver is deleted
self.__dbDriver = dbDriver
self.dbQuery = DatabaseQuery(dbDriver)
def run(self):
retn = False
try:
for user in self._authBlacklist:
self._userdb.blacklistUser(user)
site = self._site()
if not site:
return False
# start listen for incoming request
self.__tcpPort = reactor.listenTCP(self._port,
site,
self._connections,
self._listenAddress)
# setup signal handler
self.__sighup = False
signal.signal(signal.SIGHUP, self._sighupHandler)
task.LoopingCall(self._reloadTask).start(60, False)
# start processing
Logging.info("start listening")
log.addObserver(logSC3)
reactor.run()
retn = True
except Exception, e:
Logging.error(str(e))
def emit(self, record):
msg = self.format(record)
if record.levelname == 'DEBUG':
Logging.debug(charstar(msg))
elif record.levelname == 'INFO':
Logging.info(charstar(msg))
elif record.levelname == 'WARNING':
Logging.warning(charstar(msg))
else:
Logging.error(charstar(msg))
def logSC3(entry):
try:
isError = entry['isError']
msg = entry['message']
if isError:
for l in msg:
Logging.error("[reactor] %s" % l)
else:
for l in msg:
Logging.info("[reactor] %s" % l)
except:
pass
def logSC3(entry):
try:
isError = entry['isError']
msg = entry['message']
if isError:
for l in msg:
Logging.error("[reactor] %s" % l)
else:
for l in msg:
Logging.info("[reactor] %s" % l)
except:
pass
def run(self):
rules = self.rules
iv = Client.Inventory.Instance().inventory()
if not rules:
return False
if not iv:
return False
Logging.debug("Loaded %d networks" % iv.networkCount())
if self.outputFile is None:
DataModel.Notifier.Enable()
self.setInterpretNotifierEnabled(True)
for net in self._loop(iv.network, iv.networkCount()):
(ncode, nstart, nend) = self._collect(net)
key = rules.findKey(ncode, nstart, nend)
if not key:
continue
att = rules.getNetworkAttributes(key)
self._modifyInventory("network", net, att)
Logging.info("%s %s" % (ncode, att))
for sta in self._loop(net.station, net.stationCount()):
(scode, sstart, send) = self._collect(sta)
att = rules.getStationAttributes(key, ncode, scode, None, None, sstart, send)
self._modifyInventory("station", sta, att)
if att:
Logging.info(" %s %s" % (scode, att))
for loc in self._loop(sta.sensorLocation, sta.sensorLocationCount()):
(lcode, lstart, lend) = self._collect(loc)
att = rules.getStationAttributes(key, ncode, scode, lcode, None, lstart, lend)
self._modifyInventory("location", loc, att)
if att:
Logging.info(" %s %s" % (lcode, att))
for cha in self._loop(loc.stream, loc.streamCount()):
(ccode, cstart, cend) = self._collect(cha)
att = rules.getStationAttributes(key, ncode, scode, lcode, ccode, cstart, cend)
self._modifyInventory("channel", cha, att)
if att:
Logging.info(" %s %s" % (ccode, att))
for sensor in self._loop(iv.sensor, iv.sensorCount()):
att = rules.getInstrumentsAttributes(sensor.name(), "Se")
self._modifyInventory("sensor", sensor, att)
for datalogger in self._loop(iv.datalogger, iv.dataloggerCount()):
att = rules.getInstrumentsAttributes(datalogger.name(), "Dl")
self._modifyInventory("datalogger", datalogger, att)
return True
def run(self):
rules = self.rules
iv = Client.Inventory.Instance().inventory()
if not rules:
return False
if not iv:
return False
Logging.debug("Loaded %d networks" % iv.networkCount())
if self.outputFile is None:
DataModel.Notifier.Enable()
self.setInterpretNotifierEnabled(True)
for net in self._loop(iv.network, iv.networkCount()):
(ncode, nstart, nend) = self._collect(net)
key = rules.findKey(ncode, nstart, nend)
if not key: continue
att = rules.getNetworkAttributes(key)
self._modifyInventory("network", net, att)
Logging.info("%s %s" % (ncode, att))
for sta in self._loop(net.station, net.stationCount()):
(scode, sstart, send) = self._collect(sta)
att = rules.getStationAttributes(key, ncode, scode, None, None,
sstart, send)
self._modifyInventory("station", sta, att)
if att: Logging.info(" %s %s" % (scode, att))
for loc in self._loop(sta.sensorLocation,
sta.sensorLocationCount()):
(lcode, lstart, lend) = self._collect(loc)
att = rules.getStationAttributes(key, ncode, scode, lcode,
None, lstart, lend)
self._modifyInventory("location", loc, att)
if att: Logging.info(" %s %s" % (lcode, att))
for cha in self._loop(loc.stream, loc.streamCount()):
(ccode, cstart, cend) = self._collect(cha)
att = rules.getStationAttributes(
key, ncode, scode, lcode, ccode, cstart, cend)
self._modifyInventory("channel", cha, att)
if att: Logging.info(" %s %s" % (ccode, att))
for sensor in self._loop(iv.sensor, iv.sensorCount()):
att = rules.getInstrumentsAttributes(sensor.name(), "Se")
self._modifyInventory("sensor", sensor, att)
for datalogger in self._loop(iv.datalogger, iv.dataloggerCount()):
att = rules.getInstrumentsAttributes(datalogger.name(), "Dl")
self._modifyInventory("datalogger", datalogger, att)
return True
def _reloadTask(self):
if not self.__sighup:
return
self.__sighup = False
Logging.info("reloading inventory")
self.reloadInventory()
site = self._site()
if site:
self.__tcpPort.factory = site
Logging.info("reload successful")
else:
Logging.info("reload failed")
self._userdb.dump()
def _reloadTask(self):
if not self.__sighup:
return
self.__sighup = False
Logging.info("reloading inventory")
self.reloadInventory()
site = self._site()
if site:
self.__tcpPort.factory = site
Logging.info("reload successful")
else:
Logging.info("reload failed")
self._userdb.dump()
def send_notifiers(self, group):
Nsize = DataModel.Notifier.Size()
if Nsize > 0:
Logging.info("trying to apply %d change%s" %
(Nsize, "s" if Nsize != 1 else ""))
else:
Logging.info("no changes to apply")
return 0
Nmsg = DataModel.Notifier.GetMessage(True)
it = Nmsg.iter()
msg = DataModel.NotifierMessage()
maxmsg = 100
sent = 0
mcount = 0
try:
try:
while it.get():
msg.attach(DataModel.Notifier_Cast(it.get()))
mcount += 1
if msg and mcount == maxmsg:
sent += mcount
Logging.debug("sending message (%5.1f %%)" %
(sent / float(Nsize) * 100.0))
self.send(group, msg)
msg.clear()
mcount = 0
self.sync()
it.next()
except:
pass
finally:
if msg.size():
Logging.debug("sending message (%5.1f %%)" % 100.0)
self.send(group, msg)
msg.clear()
self.sync()
Logging.info("done")
return mcount
def send_notifiers(self, group):
Nsize = DataModel.Notifier.Size()
if Nsize > 0:
Logging.info("trying to apply %d change%s" % (Nsize, "s" if Nsize != 1 else ""))
else:
Logging.info("no changes to apply")
return 0
Nmsg = DataModel.Notifier.GetMessage(True)
it = Nmsg.iter()
msg = DataModel.NotifierMessage()
maxmsg = 100
sent = 0
mcount = 0
try:
try:
while it.get():
msg.attach(DataModel.Notifier_Cast(it.get()))
mcount += 1
if msg and mcount == maxmsg:
sent += mcount
Logging.debug("sending message (%5.1f %%)" % (sent / float(Nsize) * 100.0))
self.send(group, msg)
msg.clear()
mcount = 0
self.sync()
it.next()
except:
pass
finally:
if msg.size():
Logging.debug("sending message (%5.1f %%)" % 100.0)
self.send(group, msg)
msg.clear()
self.sync()
Logging.info("done")
return mcount
def dump(self):
Logging.info("known users:")
for name, user in self.__users.items():
Logging.info(" %s %s %d" % (name, user[1], user[2]))
def loadStreams(self):
now = Core.Time.GMT()
inv = Client.Inventory.Instance()
self.streams = {}
# try to load streams by detecLocid and detecStream
mod = self.configModule()
if mod is not None and mod.configStationCount() > 0:
Logging.info('loading streams using detecLocid and detecStream')
for i in range(mod.configStationCount()):
cfg = mod.configStation(i)
net = cfg.networkCode()
sta = cfg.stationCode()
if sta in self.streams:
Logging.warning('ambiguous stream id found for station '
'%s.%s' % (net, sta))
continue
setup = DataModel.findSetup(cfg, self.name(), True)
if not setup:
Logging.warning('could not find station setup for %s.%s' % (
net, sta))
continue
params = DataModel.ParameterSet.Find(setup.parameterSetID())
if not params:
Logging.warning('could not find station parameters for '
'%s.%s' % (net, sta))
continue
detecLocid = ''
detecStream = None
for j in range(params.parameterCount()):
param = params.parameter(j)
if param.name() == 'detecStream':
detecStream = param.value()
elif param.name() == 'detecLocid':
detecLocid = param.value()
if detecStream is None:
Logging.warning('could not find detecStream for %s.%s' % (
net, sta))
continue
loc = inv.getSensorLocation(net, sta, detecLocid, now)
if loc is None:
Logging.warning('could not find preferred location for '
'%s.%s' % (net, sta))
continue
components = {}
tc = DataModel.ThreeComponents()
DataModel.getThreeComponents(tc, loc, detecStream[:2], now)
if tc.vertical():
cha = tc.vertical()
wfsID = DataModel.WaveformStreamID(net, sta, loc.code(),
cha.code(), '')
components[cha.code()[-1]] = wfsID
Logging.debug('add stream %s (vertical)' % wfs2Str(wfsID))
if tc.firstHorizontal():
cha = tc.firstHorizontal()
wfsID = DataModel.WaveformStreamID(net, sta, loc.code(),
cha.code(), '')
components[cha.code()[-1]] = wfsID
Logging.debug('add stream %s (first horizontal)' %
wfs2Str(wfsID))
if tc.secondHorizontal():
cha = tc.secondHorizontal()
wfsID = DataModel.WaveformStreamID(net, sta, loc.code(),
cha.code(), '')
components[cha.code()[-1]] = wfsID
Logging.debug(
'add stream %s (second horizontal)' % wfs2Str(wfsID))
if len(components) > 0:
self.streams[sta] = components
return
# fallback loading streams from inventory
Logging.warning('no configuration module available, loading streams '
'from inventory and selecting first available stream '
'matching epoch')
for iNet in range(inv.inventory().networkCount()):
net = inv.inventory().network(iNet)
Logging.debug('network %s: loaded %i stations' % (
net.code(), net.stationCount()))
for iSta in range(net.stationCount()):
sta = net.station(iSta)
try:
start = sta.start()
if not start <= now:
continue
except:
continue
try:
end = sta.end()
if not now <= end:
continue
except:
pass
for iLoc in range(sta.sensorLocationCount()):
loc = sta.sensorLocation(iLoc)
for iCha in range(loc.streamCount()):
cha = loc.stream(iCha)
wfsID = DataModel.WaveformStreamID(net.code(),
sta.code(), loc.code(), cha.code(), '')
comp = cha.code()[2]
if sta.code() not in self.streams:
components = {}
components[comp] = wfsID
self.streams[sta.code()] = components
else:
# Seismic Handler does not support network,
# location and channel code: make sure network and
# location codes match first item in station
# specific steam list
oldWfsID = list(self.streams[sta.code()].values())[0]
if net.code() != oldWfsID.networkCode() or \
loc.code() != oldWfsID.locationCode() or \
cha.code()[:2] != oldWfsID.channelCode()[:2]:
Logging.warning('ambiguous stream id found '
'for station %s, ignoring %s'
% (sta.code(), wfs2Str(wfsID)))
continue
self.streams[sta.code()][comp] = wfsID
Logging.debug('add stream %s' % wfs2Str(wfsID))
def run(self):
modeStr = None
if self._evaluationMode is not None:
modeStr = DataModel.EEvaluationModeNames.name(self._evaluationMode)
whitelistStr = "<None>"
if self._eventTypeWhitelist is not None:
whitelistStr = ", ".join(self._eventTypeWhitelist)
blacklistStr = "<None>"
if self._eventTypeBlacklist is not None:
blacklistStr = ", ".join(self._eventTypeBlacklist)
stationFilterStr = "<None>"
if self._stationFilter is not None:
stationFilterStr = self._stationFilter
dataSelectFilterStr = "<None>"
if self._dataSelectFilter is not None:
dataSelectFilterStr = self._dataSelectFilter
Logging.debug("\n" \
"configuration read:\n" \
" serve\n" \
" dataselect : %s\n" \
" event : %s\n" \
" station : %s\n" \
" listenAddress : %s\n" \
" port : %i\n" \
" connections : %i\n" \
" htpasswd : %s\n" \
" accessLog : %s\n" \
" queryObjects : %i\n" \
" realtimeGap : %s\n" \
" samples (M) : %s\n" \
" allowRestricted : %s\n" \
" useArclinkAccess: %s\n" \
" hideAuthor : %s\n" \
" evaluationMode : %s\n" \
" eventType\n" \
" whitelist : %s\n" \
" blacklist : %s\n" \
" inventory filter\n" \
" station : %s\n" \
" dataSelect : %s\n" \
" debug enabled : %s\n" \
" trackdb\n" \
" enabled : %s\n" \
" defaultUser : %s\n" \
" auth\n" \
" enabled : %s\n" \
" gnupgHome : %s\n" % (
self._serveDataSelect, self._serveEvent,
self._serveStation, self._listenAddress, self._port,
self._connections, self._htpasswd, self._accessLogFile,
self._queryObjects, self._realtimeGap, self._samplesM,
self._allowRestricted, self._useArclinkAccess,
self._hideAuthor, modeStr,
whitelistStr, blacklistStr, stationFilterStr,
dataSelectFilterStr, self._debugFilter,
self._trackdbEnabled, self._trackdbDefaultUser,
self._authEnabled, self._authGnupgHome))
if not self._serveDataSelect and not self._serveEvent and \
not self._serveStation:
Logging.error("all services disabled through configuration")
return False
# access logger if requested
if self._accessLogFile:
self._accessLog = Log(self._accessLogFile)
# load inventory needed by DataSelect and Station service
stationInv = dataSelectInv = None
if self._serveDataSelect or self._serveStation:
retn = False
stationInv = dataSelectInv = Inventory.Instance().inventory()
Logging.info("inventory loaded")
if self._serveDataSelect and self._serveStation:
# clone inventory if station and dataSelect filter are distinct
# else share inventory between both services
if self._stationFilter != self._dataSelectFilter:
dataSelectInv = self._cloneInventory(stationInv)
retn = self._filterInventory(stationInv, self._stationFilter, "station") and \
self._filterInventory(dataSelectInv, self._dataSelectFilter, "dataSelect")
else:
retn = self._filterInventory(stationInv,
self._stationFilter)
elif self._serveStation:
retn = self._filterInventory(stationInv, self._stationFilter)
else:
retn = self._filterInventory(dataSelectInv,
self._dataSelectFilter)
if not retn:
return False
if self._serveDataSelect:
self._access.initFromSC3Routing(self.query().loadRouting())
DataModel.PublicObject.SetRegistrationEnabled(False)
shareDir = os.path.join(Environment.Instance().shareDir(), 'fdsnws')
# Overwrite/set mime type of *.wadl and *.xml documents. Instead of
# using the official types defined in /etc/mime.types 'application/xml'
# is used as enforced by the FDSNWS spec.
static.File.contentTypes['.wadl'] = 'application/xml'
static.File.contentTypes['.xml'] = 'application/xml'
# create resource tree /fdsnws/...
root = ListingResource()
fileName = os.path.join(shareDir, 'favicon.ico')
fileRes = static.File(fileName, 'image/x-icon')
fileRes.childNotFound = NoResource()
fileRes.isLeaf = True
root.putChild('favicon.ico', fileRes)
prefix = ListingResource()
root.putChild('fdsnws', prefix)
# right now service version is shared by all services
serviceVersion = ServiceVersion()
# dataselect
if self._serveDataSelect:
dataselect = ListingResource()
prefix.putChild('dataselect', dataselect)
dataselect1 = DirectoryResource(
os.path.join(shareDir, 'dataselect.html'))
dataselect.putChild('1', dataselect1)
dataselect1.putChild('query', FDSNDataSelect(dataSelectInv))
msg = 'authorization for restricted time series data required'
authSession = self._getAuthSessionWrapper(dataSelectInv, msg)
dataselect1.putChild('queryauth', authSession)
dataselect1.putChild('version', serviceVersion)
fileRes = static.File(os.path.join(shareDir, 'dataselect.wadl'))
fileRes.childNotFound = NoResource()
dataselect1.putChild('application.wadl', fileRes)
fileRes = static.File(
os.path.join(shareDir, 'dataselect-builder.html'))
fileRes.childNotFound = NoResource()
dataselect1.putChild('builder', fileRes)
if self._authEnabled:
dataselect1.putChild(
'auth', AuthResource(self._authGnupgHome, self._userdb))
# event
if self._serveEvent:
event = ListingResource()
prefix.putChild('event', event)
event1 = DirectoryResource(os.path.join(shareDir, 'event.html'))
event.putChild('1', event1)
event1.putChild(
'query',
FDSNEvent(self._hideAuthor, self._evaluationMode,
self._eventTypeWhitelist, self._eventTypeBlacklist))
fileRes = static.File(os.path.join(shareDir, 'catalogs.xml'))
fileRes.childNotFound = NoResource()
event1.putChild('catalogs', fileRes)
fileRes = static.File(os.path.join(shareDir, 'contributors.xml'))
fileRes.childNotFound = NoResource()
event1.putChild('contributors', fileRes)
event1.putChild('version', serviceVersion)
fileRes = static.File(os.path.join(shareDir, 'event.wadl'))
fileRes.childNotFound = NoResource()
event1.putChild('application.wadl', fileRes)
fileRes = static.File(os.path.join(shareDir, 'event-builder.html'))
fileRes.childNotFound = NoResource()
event1.putChild('builder', fileRes)
# station
if self._serveStation:
station = ListingResource()
prefix.putChild('station', station)
station1 = DirectoryResource(os.path.join(shareDir,
'station.html'))
station.putChild('1', station1)
station1.putChild(
'query',
FDSNStation(stationInv, self._allowRestricted,
self._queryObjects))
station1.putChild('version', serviceVersion)
fileRes = static.File(os.path.join(shareDir, 'station.wadl'))
fileRes.childNotFound = NoResource()
station1.putChild('application.wadl', fileRes)
fileRes = static.File(
os.path.join(shareDir, 'station-builder.html'))
fileRes.childNotFound = NoResource()
station1.putChild('builder', fileRes)
# static files
fileRes = static.File(os.path.join(shareDir, 'js'))
fileRes.childNotFound = NoResource()
fileRes.hideInListing = True
prefix.putChild('js', fileRes)
fileRes = static.File(os.path.join(shareDir, 'css'))
fileRes.childNotFound = NoResource()
fileRes.hideInListing = True
prefix.putChild('css', fileRes)
retn = False
try:
# start listen for incoming request
reactor.listenTCP(self._port, Site(root), self._connections,
self._listenAddress)
# start processing
Logging.info("start listening")
log.addObserver(logSC3)
reactor.run()
retn = True
except Exception, e:
Logging.error(str(e))
def _sighupHandler(self, signum, frame):
Logging.info("SIGHUP received")
self.__sighup = True
def dump(self):
Logging.info("known users:")
for name, user in self.__users.items():
Logging.info(" %s %s %d" % (name, user[1], user[2]))
def _sighupHandler(self, signum, frame):
Logging.info("SIGHUP received")
self.__sighup = True
def run(self):
modeStr = None
if self._evaluationMode is not None:
modeStr = DataModel.EEvaluationModeNames.name(self._evaluationMode)
whitelistStr = "<None>"
if self._eventTypeWhitelist is not None:
whitelistStr = ", ".join(self._eventTypeWhitelist)
blacklistStr = "<None>"
if self._eventTypeBlacklist is not None:
blacklistStr = ", ".join(self._eventTypeBlacklist)
stationFilterStr = "<None>"
if self._stationFilter is not None:
stationFilterStr = self._stationFilter
dataSelectFilterStr = "<None>"
if self._dataSelectFilter is not None:
dataSelectFilterStr = self._dataSelectFilter
Logging.debug("\n" \
"configuration read:\n" \
" serve\n" \
" dataselect : %s\n" \
" event : %s\n" \
" station : %s\n" \
" listenAddress : %s\n" \
" port : %i\n" \
" connections : %i\n" \
" htpasswd : %s\n" \
" accessLog : %s\n" \
" queryObjects : %i\n" \
" realtimeGap : %s\n" \
" samples (M) : %s\n" \
" allowRestricted : %s\n" \
" useArclinkAccess: %s\n" \
" hideAuthor : %s\n" \
" evaluationMode : %s\n" \
" eventType\n" \
" whitelist : %s\n" \
" blacklist : %s\n" \
" inventory filter\n" \
" station : %s\n" \
" dataSelect : %s\n" \
" debug enabled : %s\n" \
" trackdb\n" \
" enabled : %s\n" \
" defaultUser : %s\n" \
" auth\n" \
" enabled : %s\n" \
" gnupgHome : %s\n" % (
self._serveDataSelect, self._serveEvent,
self._serveStation, self._listenAddress, self._port,
self._connections, self._htpasswd, self._accessLogFile,
self._queryObjects, self._realtimeGap, self._samplesM,
self._allowRestricted, self._useArclinkAccess,
self._hideAuthor, modeStr,
whitelistStr, blacklistStr, stationFilterStr,
dataSelectFilterStr, self._debugFilter,
self._trackdbEnabled, self._trackdbDefaultUser,
self._authEnabled, self._authGnupgHome))
if not self._serveDataSelect and not self._serveEvent and \
not self._serveStation:
Logging.error("all services disabled through configuration")
return False
# access logger if requested
if self._accessLogFile:
self._accessLog = Log(self._accessLogFile)
# load inventory needed by DataSelect and Station service
stationInv = dataSelectInv = None
if self._serveDataSelect or self._serveStation:
retn = False
stationInv = dataSelectInv = Inventory.Instance().inventory()
Logging.info("inventory loaded")
if self._serveDataSelect and self._serveStation:
# clone inventory if station and dataSelect filter are distinct
# else share inventory between both services
if self._stationFilter != self._dataSelectFilter:
dataSelectInv = self._cloneInventory(stationInv)
retn = self._filterInventory(stationInv, self._stationFilter, "station") and \
self._filterInventory(dataSelectInv, self._dataSelectFilter, "dataSelect")
else:
retn = self._filterInventory(stationInv, self._stationFilter)
elif self._serveStation:
retn = self._filterInventory(stationInv, self._stationFilter)
else:
retn = self._filterInventory(dataSelectInv, self._dataSelectFilter)
if not retn:
return False
if self._serveDataSelect:
self._access.initFromSC3Routing(self.query().loadRouting())
DataModel.PublicObject.SetRegistrationEnabled(False)
shareDir = os.path.join(Environment.Instance().shareDir(), 'fdsnws')
# Overwrite/set mime type of *.wadl and *.xml documents. Instead of
# using the official types defined in /etc/mime.types 'application/xml'
# is used as enforced by the FDSNWS spec.
static.File.contentTypes['.wadl'] = 'application/xml'
static.File.contentTypes['.xml'] = 'application/xml'
# create resource tree /fdsnws/...
root = ListingResource()
fileName = os.path.join(shareDir, 'favicon.ico')
fileRes = static.File(fileName, 'image/x-icon')
fileRes.childNotFound = NoResource()
fileRes.isLeaf = True
root.putChild('favicon.ico', fileRes)
prefix = ListingResource()
root.putChild('fdsnws', prefix)
# right now service version is shared by all services
serviceVersion = ServiceVersion()
# dataselect
if self._serveDataSelect:
dataselect = ListingResource()
prefix.putChild('dataselect', dataselect)
dataselect1 = DirectoryResource(os.path.join(shareDir, 'dataselect.html'))
dataselect.putChild('1', dataselect1)
dataselect1.putChild('query', FDSNDataSelect(dataSelectInv))
msg = 'authorization for restricted time series data required'
authSession = self._getAuthSessionWrapper(dataSelectInv, msg)
dataselect1.putChild('queryauth', authSession)
dataselect1.putChild('version', serviceVersion)
fileRes = static.File(os.path.join(shareDir, 'dataselect.wadl'))
fileRes.childNotFound = NoResource()
dataselect1.putChild('application.wadl', fileRes)
fileRes = static.File(os.path.join(shareDir, 'dataselect-builder.html'))
fileRes.childNotFound = NoResource()
dataselect1.putChild('builder', fileRes)
if self._authEnabled:
dataselect1.putChild('auth', AuthResource(self._authGnupgHome,
self._userdb))
# event
if self._serveEvent:
event = ListingResource()
prefix.putChild('event', event)
event1 = DirectoryResource(os.path.join(shareDir, 'event.html'))
event.putChild('1', event1)
event1.putChild('query', FDSNEvent(self._hideAuthor,
self._evaluationMode,
self._eventTypeWhitelist,
self._eventTypeBlacklist))
fileRes = static.File(os.path.join(shareDir, 'catalogs.xml'))
fileRes.childNotFound = NoResource()
event1.putChild('catalogs', fileRes)
fileRes = static.File(os.path.join(shareDir, 'contributors.xml'))
fileRes.childNotFound = NoResource()
event1.putChild('contributors', fileRes)
event1.putChild('version', serviceVersion)
fileRes = static.File(os.path.join(shareDir, 'event.wadl'))
fileRes.childNotFound = NoResource()
event1.putChild('application.wadl', fileRes)
fileRes = static.File(os.path.join(shareDir, 'event-builder.html'))
fileRes.childNotFound = NoResource()
event1.putChild('builder', fileRes)
# station
if self._serveStation:
station = ListingResource()
prefix.putChild('station', station)
station1 = DirectoryResource(os.path.join(shareDir, 'station.html'))
station.putChild('1', station1)
station1.putChild('query', FDSNStation(stationInv,
self._allowRestricted,
self._queryObjects))
station1.putChild('version', serviceVersion)
fileRes = static.File(os.path.join(shareDir, 'station.wadl'))
fileRes.childNotFound = NoResource()
station1.putChild('application.wadl', fileRes)
fileRes = static.File(os.path.join(shareDir, 'station-builder.html'))
fileRes.childNotFound = NoResource()
station1.putChild('builder', fileRes)
# static files
fileRes = static.File(os.path.join(shareDir, 'js'))
fileRes.childNotFound = NoResource()
fileRes.hideInListing = True
prefix.putChild('js', fileRes)
fileRes = static.File(os.path.join(shareDir, 'css'))
fileRes.childNotFound = NoResource()
fileRes.hideInListing = True
prefix.putChild('css', fileRes)
retn = False
try:
# start listen for incoming request
reactor.listenTCP(self._port, Site(root), self._connections,
self._listenAddress)
# start processing
Logging.info("start listening")
log.addObserver(logSC3)
reactor.run()
retn = True
except Exception, e:
Logging.error(str(e))
def blacklistUser(self, name):
Logging.info("blacklisting %s" % name)
self.__blacklist.add(name)
def __expireUsers(self):
for (name, (password, attributes, expires)) in self.__users.items():
if time.time() > expires:
Logging.info("de-registering %s" % name)
del self.__users[name]
def run(self):
modeStr = None
if self._evaluationMode is not None:
modeStr = DataModel.EEvaluationModeNames.name(self._evaluationMode)
whitelistStr = "<None>"
if self._eventTypeWhitelist is not None:
whitelistStr = ", ".join(self._eventTypeWhitelist)
blacklistStr = "<None>"
if self._eventTypeBlacklist is not None:
blacklistStr = ", ".join(self._eventTypeBlacklist)
Logging.notice("\n" \
"configuration read:\n" \
" serve\n" \
" dataselect : %s\n" \
" event : %s\n" \
" station : %s\n" \
" listenAddress : %s\n" \
" port : %i\n" \
" connections : %i\n" \
" htpasswd : %s\n" \
" accessLog : %s\n" \
" queryObjects : %i\n" \
" realtimeGap : %s\n" \
" samples (M) : %s\n" \
" allowRestricted : %s\n" \
" hideAuthor : %s\n" \
" evaluationMode : %s\n" \
" eventType\n" \
" whitelist : %s\n" \
" blacklist : %s\n" % (
self._serveDataSelect, self._serveEvent,
self._serveStation, self._listenAddress, self._port,
self._connections, self._htpasswd, self._accessLogFile,
self._queryObjects, self._realtimeGap, self._samplesM,
self._allowRestricted, self._hideAuthor, modeStr,
whitelistStr, blacklistStr))
if not self._serveDataSelect and not self._serveEvent and \
not self._serveStation:
Logging.error("all services disabled through configuration")
return False
# access logger if requested
if self._accessLogFile:
self._accessLog = Log(self._accessLogFile)
# load inventory needed by DataSelect and Station service
if self._serveDataSelect or self._serveStation:
self._loadInventory()
DataModel.PublicObject.SetRegistrationEnabled(False)
shareDir = os.path.join(Environment.Instance().shareDir(), 'fdsnws')
# Overwrite/set mime type of *.wadl and *.xml documents. Instead of
# using the official types defined in /etc/mime.types 'application/xml'
# is used as enforced by the FDSNWS spec.
static.File.contentTypes['.wadl'] = 'application/xml'
static.File.contentTypes['.xml'] = 'application/xml'
# create resource tree /fdsnws/...
root = ListingResource()
fileName = os.path.join(shareDir, 'favicon.ico')
fileRes = static.File(fileName, 'image/x-icon')
fileRes.childNotFound = NoResource()
fileRes.isLeaf = True
root.putChild('favicon.ico', fileRes)
prefix = ListingResource()
root.putChild('fdsnws', prefix)
# right now service version is shared by all services
serviceVersion = ServiceVersion()
# dataselect
if self._serveDataSelect:
dataselect = ListingResource()
prefix.putChild('dataselect', dataselect)
dataselect1 = DirectoryResource(os.path.join(shareDir, 'dataselect.html'))
dataselect.putChild('1', dataselect1)
dataselect1.putChild('query', FDSNDataSelect())
msg = 'authorization for restricted time series data required'
authSession = self._getAuthSessionWrapper(FDSNDataSelectRealm(), msg)
dataselect1.putChild('queryauth', authSession)
dataselect1.putChild('version', serviceVersion)
fileRes = static.File(os.path.join(shareDir, 'dataselect.wadl'))
fileRes.childNotFound = NoResource()
dataselect1.putChild('application.wadl', fileRes)
# event
if self._serveEvent:
event = ListingResource()
prefix.putChild('event', event)
event1 = DirectoryResource(os.path.join(shareDir, 'event.html'))
event.putChild('1', event1)
event1.putChild('query', FDSNEvent(self._hideAuthor,
self._evaluationMode,
self._eventTypeWhitelist,
self._eventTypeBlacklist))
fileRes = static.File(os.path.join(shareDir, 'catalogs.xml'))
fileRes.childNotFound = NoResource()
event1.putChild('catalogs', fileRes)
fileRes = static.File(os.path.join(shareDir, 'contributors.xml'))
fileRes.childNotFound = NoResource()
event1.putChild('contributors', fileRes)
event1.putChild('version', serviceVersion)
fileRes = static.File(os.path.join(shareDir, 'event.wadl'))
fileRes.childNotFound = NoResource()
event1.putChild('application.wadl', fileRes)
# station
if self._serveStation:
station = ListingResource()
prefix.putChild('station', station)
station1 = DirectoryResource(os.path.join(shareDir, 'station.html'))
station.putChild('1', station1)
station1.putChild('query', FDSNStation(self._inv, self._allowRestricted, self._queryObjects))
station1.putChild('version', serviceVersion)
fileRes = static.File(os.path.join(shareDir, 'station.wadl'))
fileRes.childNotFound = NoResource()
station1.putChild('application.wadl', fileRes)
retn = False
try:
# start listen for incoming request
reactor.listenTCP(self._port, Site(root), self._connections,
self._listenAddress)
# start processing
Logging.info("start listening")
log.addObserver(logSC3)
reactor.run()
retn = True
except Exception, e:
Logging.error(str(e))
def __expireUsers(self):
for (name, (password, attributes, expires)) in self.__users.items():
if time.time() > expires:
Logging.info("de-registering %s" % name)
del self.__users[name]
def _site(self):
modeStr = None
if self._evaluationMode is not None:
modeStr = DataModel.EEvaluationModeNames.name(self._evaluationMode)
whitelistStr = "<None>"
if self._eventTypeWhitelist is not None:
whitelistStr = ", ".join(self._eventTypeWhitelist)
blacklistStr = "<None>"
if self._eventTypeBlacklist is not None:
blacklistStr = ", ".join(self._eventTypeBlacklist)
stationFilterStr = "<None>"
if self._stationFilter is not None:
stationFilterStr = self._stationFilter
dataSelectFilterStr = "<None>"
if self._dataSelectFilter is not None:
dataSelectFilterStr = self._dataSelectFilter
Logging.debug("\n"
"configuration read:\n"
" serve\n"
" dataselect : %s\n"
" event : %s\n"
" station : %s\n"
" availability : %s\n"
" listenAddress : %s\n"
" port : %i\n"
" connections : %i\n"
" htpasswd : %s\n"
" accessLog : %s\n"
" queryObjects : %i\n"
" realtimeGap : %s\n"
" samples (M) : %s\n"
" recordBulkSize : %i\n"
" allowRestricted : %s\n"
" useArclinkAccess: %s\n"
" hideAuthor : %s\n"
" evaluationMode : %s\n"
" data availability\n"
" enabled : %s\n"
" cache duration: %i\n"
" repo name : %s\n"
" dcc name : %s\n"
" eventType\n"
" whitelist : %s\n"
" blacklist : %s\n"
" inventory filter\n"
" station : %s\n"
" dataSelect : %s\n"
" debug enabled : %s\n"
" trackdb\n"
" enabled : %s\n"
" defaultUser : %s\n"
" auth\n"
" enabled : %s\n"
" gnupgHome : %s\n"
" requestLog : %s\n" % (
self._serveDataSelect, self._serveEvent,
self._serveStation, self._serveAvailability,
self._listenAddress, self._port, self._connections,
self._htpasswd, self._accessLogFile, self._queryObjects,
self._realtimeGap, self._samplesM, self._recordBulkSize,
self._allowRestricted, self._useArclinkAccess,
self._hideAuthor, modeStr, self._daEnabled,
self._daCacheDuration, self._daRepositoryName,
self._daDCCName, whitelistStr, blacklistStr,
stationFilterStr, dataSelectFilterStr,
self._debugFilter, self._trackdbEnabled,
self._trackdbDefaultUser, self._authEnabled,
self._authGnupgHome, self._requestLogFile))
if not self._serveDataSelect and not self._serveEvent and \
not self._serveStation:
Logging.error("all services disabled through configuration")
return None
# access logger if requested
if self._accessLogFile:
self._accessLog = Log(self._accessLogFile)
# request logger if requested
if self._requestLogFile:
# import here, so we don't depend on GeoIP if request log is not needed
from seiscomp3.fdsnws.reqlog import RequestLog
self._requestLog = RequestLog(self._requestLogFile)
# load inventory needed by DataSelect and Station service
stationInv = dataSelectInv = None
if self._serveDataSelect or self._serveStation:
retn = False
stationInv = dataSelectInv = Inventory.Instance().inventory()
Logging.info("inventory loaded")
if self._serveDataSelect and self._serveStation:
# clone inventory if station and dataSelect filter are distinct
# else share inventory between both services
if self._stationFilter != self._dataSelectFilter:
dataSelectInv = self._cloneInventory(stationInv)
retn = self._filterInventory(stationInv, self._stationFilter, "station") and \
self._filterInventory(
dataSelectInv, self._dataSelectFilter, "dataSelect")
else:
retn = self._filterInventory(
stationInv, self._stationFilter)
elif self._serveStation:
retn = self._filterInventory(stationInv, self._stationFilter)
else:
retn = self._filterInventory(
dataSelectInv, self._dataSelectFilter)
if not retn:
return None
self._access = Access()
if self._serveDataSelect and self._useArclinkAccess:
self._access.initFromSC3Routing(self.query().loadRouting())
DataModel.PublicObject.SetRegistrationEnabled(False)
shareDir = os.path.join(Environment.Instance().shareDir(), 'fdsnws')
# Overwrite/set mime type of *.wadl and *.xml documents. Instead of
# using the official types defined in /etc/mime.types 'application/xml'
# is used as enforced by the FDSNWS spec.
static.File.contentTypes['.wadl'] = 'application/xml'
static.File.contentTypes['.xml'] = 'application/xml'
# create resource tree /fdsnws/...
root = ListingResource()
fileName = os.path.join(shareDir, 'favicon.ico')
fileRes = static.File(fileName, 'image/x-icon')
fileRes.childNotFound = NoResource()
fileRes.isLeaf = True
root.putChild('favicon.ico', fileRes)
prefix = ListingResource()
root.putChild('fdsnws', prefix)
# dataselect
if self._serveDataSelect:
dataselect = ListingResource(DataSelectVersion)
prefix.putChild('dataselect', dataselect)
lstFile = os.path.join(shareDir, 'dataselect.html')
dataselect1 = DirectoryResource(lstFile, DataSelectVersion)
dataselect.putChild('1', dataselect1)
dataselect1.putChild('query', FDSNDataSelect(
dataSelectInv, self._recordBulkSize))
msg = 'authorization for restricted time series data required'
authSession = self._getAuthSessionWrapper(dataSelectInv, msg)
dataselect1.putChild('queryauth', authSession)
dataselect1.putChild('version', ServiceVersion(DataSelectVersion))
fileRes = static.File(os.path.join(shareDir, 'dataselect.wadl'))
fileRes.childNotFound = NoResource(DataSelectVersion)
dataselect1.putChild('application.wadl', fileRes)
fileRes = static.File(os.path.join(
shareDir, 'dataselect-builder.html'))
fileRes.childNotFound = NoResource(DataSelectVersion)
dataselect1.putChild('builder', fileRes)
if self._authEnabled:
dataselect1.putChild('auth', AuthResource(
DataSelectVersion, self._authGnupgHome, self._userdb))
# event
if self._serveEvent:
event = ListingResource(EventVersion)
prefix.putChild('event', event)
lstFile = os.path.join(shareDir, 'event.html')
event1 = DirectoryResource(lstFile, EventVersion)
event.putChild('1', event1)
event1.putChild('query', FDSNEvent(self._hideAuthor,
self._evaluationMode,
self._eventTypeWhitelist,
self._eventTypeBlacklist,
self._eventFormats))
fileRes = static.File(os.path.join(shareDir, 'catalogs.xml'))
fileRes.childNotFound = NoResource(EventVersion)
event1.putChild('catalogs', fileRes)
fileRes = static.File(os.path.join(shareDir, 'contributors.xml'))
fileRes.childNotFound = NoResource(EventVersion)
event1.putChild('contributors', fileRes)
event1.putChild('version', ServiceVersion(EventVersion))
fileRes = static.File(os.path.join(shareDir, 'event.wadl'))
fileRes.childNotFound = NoResource(EventVersion)
event1.putChild('application.wadl', fileRes)
fileRes = static.File(os.path.join(shareDir, 'event-builder.html'))
fileRes.childNotFound = NoResource(EventVersion)
event1.putChild('builder', fileRes)
# station
if self._serveStation:
station = ListingResource(StationVersion)
prefix.putChild('station', station)
lstFile = os.path.join(shareDir, 'station.html')
station1 = DirectoryResource(lstFile, StationVersion)
station.putChild('1', station1)
station1.putChild('query', FDSNStation(stationInv,
self._allowRestricted,
self._queryObjects,
self._daEnabled))
station1.putChild('version', ServiceVersion(StationVersion))
# wadl, optionally filtered
filterList = [] if self._daEnabled else ['name="matchtimeseries"']
try:
fileRes = WADLFilter(os.path.join(shareDir, 'station.wadl'),
filterList)
except:
fileRes = NoResource(StationVersion)
station1.putChild('application.wadl', fileRes)
fileRes = static.File(os.path.join(
shareDir, 'station-builder.html'))
fileRes.childNotFound = NoResource(StationVersion)
station1.putChild('builder', fileRes)
# availability
if self._serveAvailability:
# create a set of waveformIDs which represent open channels
if self._serveDataSelect:
openStreams = set()
for iNet in xrange(dataSelectInv.networkCount()):
net = dataSelectInv.network(iNet)
if utils.isRestricted(net):
continue
for iSta in xrange(net.stationCount()):
sta = net.station(iSta)
if utils.isRestricted(sta):
continue
for iLoc in xrange(sta.sensorLocationCount()):
loc = sta.sensorLocation(iLoc)
for iCha in xrange(loc.streamCount()):
cha = loc.stream(iCha)
if utils.isRestricted(cha):
continue
openStreams.add("{0}.{1}.{2}.{3}".format(
net.code(), sta.code(),
loc.code(), cha.code()))
self._openStreams = openStreams
else:
self._openStreams = None
ext = ListingResource()
prefix.putChild('ext', ext)
availability = ListingResource(AvailabilityVersion)
ext.putChild('availability', availability)
lstFile = os.path.join(shareDir, 'availability.html')
availability1 = DirectoryResource(lstFile, AvailabilityVersion)
availability.putChild('1', availability1)
availability1.putChild('extent', AvailabilityExtent())
availability1.putChild('query', AvailabilityQuery())
availability1.putChild(
'version', ServiceVersion(AvailabilityVersion))
fileRes = static.File(os.path.join(shareDir, 'station.wadl'))
fileRes.childNotFound = NoResource(AvailabilityVersion)
availability1.putChild('availability.wadl', fileRes)
fileRes = static.File(os.path.join(
shareDir, 'availability-extent-builder.html'))
fileRes.childNotFound = NoResource(AvailabilityVersion)
availability1.putChild('builder-extent', fileRes)
fileRes = static.File(os.path.join(
shareDir, 'availability-builder.html'))
fileRes.childNotFound = NoResource(AvailabilityVersion)
availability1.putChild('builder', fileRes)
# static files
fileRes = static.File(os.path.join(shareDir, 'js'))
fileRes.childNotFound = NoResource()
fileRes.hideInListing = True
prefix.putChild('js', fileRes)
fileRes = static.File(os.path.join(shareDir, 'css'))
fileRes.childNotFound = NoResource()
fileRes.hideInListing = True
prefix.putChild('css', fileRes)
return Site(root)
def blacklistUser(self, name):
Logging.info("blacklisting %s" % name)
self.__blacklist.add(name)