def __init__(self): Application.__init__(self, len(sys.argv), sys.argv) self.setMessagingEnabled(False) self.setDatabaseEnabled(True, True) self.setRecordStreamEnabled(True) self._serverRoot = os.path.dirname(__file__) self._listenAddress = '0.0.0.0' # all interfaces self._port = 8080 self._connections = 5 self._queryObjects = 100000 # maximum number of objects per query self._realtimeGap = None # minimum data age: 5min self._samplesM = None # maximum number of samples per query self._htpasswd = '@CONFIGDIR@/fdsnws.htpasswd' self._accessLogFile = '' self._allowRestricted = True self._serveDataSelect = True self._serveEvent = True self._serveStation = True self._hideAuthor = False self._evaluationMode = None self._eventTypeWhitelist = None self._eventTypeBlacklist = None self._accessLog = None self._inv = None self._fileNamePrefix = 'fdsnws' # Leave signal handling to us Application.HandleSignals(False, False)
def __init__(self, argc, argv): # Log all messages to a file for S3 self._logfile_for_s3 = NamedTemporaryFile() self._logger_for_s3 = Logging.FileOutput(self._logfile_for_s3.name) for level in ('notice', 'error', 'warning', 'info', 'debug'): self._logger_for_s3.subscribe(Logging.getGlobalChannel(level)) Application.__init__(self, argc, argv) # default location to write outputs to self.output = settings.OUTPUT_DIR self.filename = None self.mag_type = None self.mag_value = None self.server = 'IRIS' self.fdsn_client = None self.networks = 'ALL' self.region = 'not specified' self.evid = None self.resultid = None self.notificationemail = None self.fromemail = None self.email_aws_region = None self.email_method = 'ses' self.email_subject_postfix = '' self.email_subject_prefix = '' self.smtp_server = None self.smtp_port = 25 self.smtp_ssl = False self.smtp_tls = False self.smtp_user = None self.smtp_password = None self.write_s3 = False self.bucket_name = None self.agency = 'GA' self.make_maps = True self.overwrite = False self.save_waveforms = None self.save_inventory = None self.waveforms = None self.inventory = None self.eqinfo: Optional[model.Event] = None # enable messaging support self.setMessagingEnabled(True) # disable database access self.setDatabaseEnabled(False, False) # default spread username self.setMessagingUsername("gawphase") # send all objects to the focal mechanism group self.setPrimaryMessagingGroup("FOCMECH")
def __init__(self): Application.__init__(self, len(sys.argv), sys.argv) self.setMessagingEnabled(True) self.setDatabaseEnabled(True, True) self.setRecordStreamEnabled(True) self.setLoadInventoryEnabled(True) self._serverRoot = os.path.dirname(__file__) self._listenAddress = '0.0.0.0' # all interfaces self._port = 8080 self._connections = 5 self._queryObjects = 100000 # maximum number of objects per query self._realtimeGap = None # minimum data age: 5min self._samplesM = None # maximum number of samples per query self._recordBulkSize = 102400 # desired record bulk size self._htpasswd = '@CONFIGDIR@/fdsnws.htpasswd' self._accessLogFile = '' self._requestLogFile = '' self._allowRestricted = True self._useArclinkAccess = False self._serveDataSelect = True self._serveEvent = True self._serveStation = True self._serveAvailability = False self._daEnabled = False self._daCacheDuration = 300 self._daCache = None self._openStreams = None self._daRepositoryName = 'primary' self._daDCCName = 'DCC' self._hideAuthor = False self._evaluationMode = None self._eventTypeWhitelist = None self._eventTypeBlacklist = None self._eventFormats = None self._stationFilter = None self._dataSelectFilter = None self._debugFilter = False self._accessLog = None self._fileNamePrefix = 'fdsnws' self._trackdbEnabled = False self._trackdbDefaultUser = '******' self._authEnabled = False self._authGnupgHome = '@ROOTDIR@/var/lib/gpg' self._authBlacklist = [] self._userdb = UserDB() self._access = Access() # Leave signal handling to us Application.HandleSignals(False, False)
def __init__(self): Application.__init__(self, len(sys.argv), sys.argv) self.setMessagingEnabled(True) self.setDatabaseEnabled(True, True) self.setRecordStreamEnabled(True) self.setLoadInventoryEnabled(True) self._serverRoot = os.path.dirname(__file__) self._listenAddress = '0.0.0.0' # all interfaces self._port = 8080 self._connections = 5 self._queryObjects = 100000 # maximum number of objects per query self._realtimeGap = None # minimum data age: 5min self._samplesM = None # maximum number of samples per query self._recordBulkSize = 102400 # desired record bulk size self._htpasswd = '@CONFIGDIR@/fdsnws.htpasswd' self._accessLogFile = '' self._allowRestricted = True self._useArclinkAccess = False self._serveDataSelect = True self._serveEvent = True self._serveStation = True self._hideAuthor = False self._evaluationMode = None self._eventTypeWhitelist = None self._eventTypeBlacklist = None self._eventFormats = None self._stationFilter = None self._dataSelectFilter = None self._debugFilter = False self._accessLog = None self._fileNamePrefix = 'fdsnws' self._trackdbEnabled = False self._trackdbDefaultUser = '******' self._authEnabled = False self._authGnupgHome = '@ROOTDIR@/var/lib/gpg' self._authBlacklist = [] self._userdb = UserDB() self._access = Access() # Leave signal handling to us Application.HandleSignals(False, False)
def __init__(self, inv, bufferSize, access=None, user=None): BaseResource.__init__(self, VERSION) self._rsURL = Application.Instance().recordStreamURL() self.__inv = inv self.__access = access self.__user = user self.__bufferSize = bufferSize
class FDSNEvent(resource.Resource): isLeaf = True #--------------------------------------------------------------------------- def __init__(self, hideAuthor=False, evaluationMode=None, eventTypeWhitelist=None, eventTypeBlacklist=None): self._hideAuthor = hideAuthor self._evaluationMode = evaluationMode self._eventTypeWhitelist = eventTypeWhitelist self._eventTypeBlacklist = eventTypeBlacklist #--------------------------------------------------------------------------- def render_GET(self, req): # Parse and validate GET parameters ro = _EventRequestOptions(req.args) try: ro.parse() except ValueError, e: Logging.warning(str(e)) return HTTP.renderErrorPage(req, http.BAD_REQUEST, str(e), ro) # Catalog filter is not supported, any filter value will result in 204 if ro.catalogs: msg = "no matching events found" return HTTP.renderErrorPage(req, http.NO_CONTENT, msg, ro) # updateafter not implemented if ro.updatedAfter: msg = "filtering based on update time not supported" return HTTP.renderErrorPage(req, http.SERVICE_UNAVAILABLE, msg, ro) # Exporter, 'None' is used for text output if ro.format in ro.VText: exp = None else: exp = Exporter.Create(ro.Exporters[ro.format]) if exp: exp.setFormattedOutput(bool(ro.formatted)) else: msg = "output format '%s' no available, export module '%s' could " \ "not be loaded." % (ro.format, ro.Exporters[ro.format]) return HTTP.renderErrorPage(req, http.SERVICE_UNAVAILABLE, msg, ro) # Create database query dbq = DataModel.DatabaseQuery(Application.Instance().database()) if dbq.hasError(): msg = "could not connect to database: %s" % dbq.errorMsg() return HTTP.renderErrorPage(req, http.SERVICE_UNAVAILABLE, msg, ro) # Process request in separate thread d = deferToThread(self._processRequest, req, ro, dbq, exp) req.notifyFinish().addErrback(utils.onCancel, d) d.addBoth(utils.onFinish, req) # The request is handled by the deferred object return server.NOT_DONE_YET
def __init__(self, argc, argv): Application.__init__(self, argc, argv) self.setMessagingEnabled(True) self.setDatabaseEnabled(True, True) self.addMessagingSubscription("EVENT") self.addMessagingSubscription("LOCATION") self.addMessagingSubscription("MAGNITUDE") self.addMessagingSubscription("FOCMECH") self.setAutoApplyNotifierEnabled(True) # object buffers self._state = {} self._origin = {} self._magnitude = {} self._focalmechanism = {} self._cleanupCounter = 0 self._xdebug = False self._cleanup_interval = 3600.
def render_GET(self, req): # Parse and validate GET parameters ro = _EventRequestOptions() try: ro.parseGET(req.args) ro.parse() except ValueError as e: Logging.warning(str(e)) return self.renderErrorPage(req, http.BAD_REQUEST, str(e), ro) # Catalog filter is not supported if ro.catalogs: msg = "catalog filter not supported" return self.renderErrorPage(req, http.BAD_REQUEST, msg, ro) if ro.comments and self._hideComments: msg = "including of comments not supported" return self.renderErrorPage(req, http.BAD_REQUEST, msg, ro) # updateafter not implemented if ro.updatedAfter: msg = "filtering based on update time not supported" return self.renderErrorPage(req, http.BAD_REQUEST, msg, ro) if self._formatList is not None and ro.format not in self._formatList: msg = "output format '%s' not available" % ro.format return self.renderErrorPage(req, http.BAD_REQUEST, msg, ro) # Exporter, 'None' is used for text output if ro.format in ro.VText: exp = None else: exp = Exporter.Create(ro.Exporters[ro.format]) if exp: exp.setFormattedOutput(bool(ro.formatted)) else: msg = "output format '%s' not available, export module '%s' could " \ "not be loaded." % (ro.format, ro.Exporters[ro.format]) return self.renderErrorPage(req, http.BAD_REQUEST, msg, ro) # Create database query db = DatabaseInterface.Open(Application.Instance().databaseURI()) if db is None: msg = "could not connect to database" return self.renderErrorPage(req, http.SERVICE_UNAVAILABLE, msg, ro) dbq = DataModel.DatabaseQuery(db) # Process request in separate thread d = deferToThread(self._processRequest, req, ro, dbq, exp) req.notifyFinish().addErrback(utils.onCancel, d) d.addBoth(utils.onFinish, req) # The request is handled by the deferred object return server.NOT_DONE_YET
def _prepareRequest(self, req, ro): if ro.availability and not self._daEnabled: msg = "including of availability information not supported" return self.renderErrorPage(req, http.BAD_REQUEST, msg, ro) if ro.updatedAfter: msg = "filtering based on update time not supported" return self.renderErrorPage(req, http.BAD_REQUEST, msg, ro) if ro.matchTimeSeries and not self._daEnabled: msg = "filtering based on available time series not supported" return self.renderErrorPage(req, http.BAD_REQUEST, msg, ro) # load data availability if requested dac = None if ro.availability or ro.matchTimeSeries: dac = Application.Instance().getDACache() if dac is None or len(dac.extents()) == 0: msg = "no data availabiltiy extent information found" return self.renderErrorPage(req, http.NO_CONTENT, msg, ro) # Exporter, 'None' is used for text output if ro.format in ro.VText: if ro.includeRes: msg = "response level output not available in text format" return self.renderErrorPage(req, http.BAD_REQUEST, msg, ro) req.setHeader('Content-Type', 'text/plain') d = deferToThread(self._processRequestText, req, ro, dac) else: exp = Exporter.Create(ro.Exporters[ro.format]) if exp is None: msg = "output format '%s' no available, export module '%s' " \ "could not be loaded." % ( ro.format, ro.Exporters[ro.format]) return self.renderErrorPage(req, http.BAD_REQUEST, msg, ro) req.setHeader('Content-Type', 'application/xml') exp.setFormattedOutput(bool(ro.formatted)) d = deferToThread(self._processRequestExp, req, ro, exp, dac) req.notifyFinish().addErrback(utils.onCancel, d) d.addBoth(utils.onFinish, req) # The request is handled by the deferred object return server.NOT_DONE_YET
def initConfiguration(self): if not Application.initConfiguration(self): return False cfg = self.configuration() # bind address and port try: self._listenAddress = cfg.getString('listenAddress') except ConfigException: pass try: self._port = cfg.getInt('port') except ConfigException: pass # maximum number of connections try: self._connections = cfg.getInt('connections') except ConfigException: pass # maximum number of objects per query, used in fdsnws-station and # fdsnws-event to limit main memory consumption try: self._queryObjects = cfg.getInt('queryObjects') except ConfigException: pass # restrict end time of request to now-realtimeGap seconds, used in # fdsnws-dataselect try: self._realtimeGap = cfg.getInt('realtimeGap') except ConfigException: pass # maximum number of samples (in units of million) per query, used in # fdsnws-dataselect to limit bandwidth try: self._samplesM = cfg.getDouble('samplesM') except ConfigException: pass # location of htpasswd file try: self._htpasswd = cfg.getString('htpasswd') except ConfigException: pass self._htpasswd = Environment.Instance().absolutePath(self._htpasswd) # location of access log file try: self._accessLogFile = Environment.Instance().absolutePath( cfg.getString('accessLog')) except ConfigException: pass # access to restricted inventory information try: self._allowRestricted = cfg.getBool('allowRestricted') except: pass # services to enable try: self._serveDataSelect = cfg.getBool('serveDataSelect') except: pass try: self._serveEvent = cfg.getBool('serveEvent') except: pass try: self._serveStation = cfg.getBool('serveStation') except: pass # event filter try: self._hideAuthor = cfg.getBool('hideAuthor') except: pass try: name = cfg.getString('evaluationMode') if name.lower() == DataModel.EEvaluationModeNames.name(DataModel.MANUAL): self._evaluationMode = DataModel.MANUAL elif name.lower() == DataModel.EEvaluationModeNames.name(DataModel.AUTOMATIC): self._evaluationMode = DataModel.AUTOMATIC else: print >> sys.stderr, "invalid evaluation mode string: %s" % name return False except: pass try: strings = cfg.getStrings('eventType.whitelist') if len(strings) > 1 or len(strings[0]): self._eventTypeWhitelist = [ s.lower() for s in strings ] except: pass try: strings = cfg.getStrings('eventType.blacklist') if len(strings) > 0 or len(strings[0]): self._eventTypeBlacklist = [ s.lower() for s in strings ] except: pass # prefix to be used as default for output filenames try: self._fileNamePrefix = cfg.getString('fileNamePrefix') except ConfigException: pass return True
def _processRequest(self, req, ro): if ro.quality != 'B' and ro.quality != 'M': msg = "quality other than 'B' or 'M' not supported" return HTTP.renderErrorPage(req, http.SERVICE_UNAVAILABLE, msg, ro) if ro.minimumLength: msg = "enforcing of minimum record length not supported" return HTTP.renderErrorPage(req, http.SERVICE_UNAVAILABLE, msg, ro) if ro.longestOnly: msg = "limitation to longest segment not supported" return HTTP.renderErrorPage(req, http.SERVICE_UNAVAILABLE, msg, ro) app = Application.Instance() ro._checkTimes(app._realtimeGap) maxSamples = None if app._samplesM is not None: maxSamples = app._samplesM * 1000000 samples = 0 app = Application.Instance() if app._trackdbEnabled: userid = ro.userName or app._trackdbDefaultUser reqid = 'ws' + str(int(round(time.time() * 1000) - 1420070400000)) xff = req.requestHeaders.getRawHeaders("x-forwarded-for") if xff: userIP = xff[0].split(",")[0].strip() else: userIP = req.getClientIP() tracker = RequestTrackerDB("fdsnws", app.connection(), reqid, "WAVEFORM", userid, "REQUEST WAVEFORM " + reqid, "fdsnws", userIP, req.getClientIP()) else: tracker = None # Open record stream rs = _MyRecordStream(self._rsURL, tracker, self.__bufferSize) # Add request streams # iterate over inventory networks for s in ro.streams: for net in self._networkIter(s): for sta in self._stationIter(net, s): for loc in self._locationIter(sta, s): for cha in self._streamIter(loc, s): try: start_time = max(cha.start(), s.time.start) except Exception: start_time = s.time.start try: end_time = min(cha.end(), s.time.end) except Exception: end_time = s.time.end if utils.isRestricted(cha) and \ (not self.__user or (self.__access and not self.__access.authorize(self.__user, net.code(), sta.code(), loc.code(), cha.code(), start_time, end_time))): continue # enforce maximum sample per request restriction if maxSamples is not None: try: n = cha.sampleRateNumerator() d = cha.sampleRateDenominator() except ValueError: msg = "skipping stream without sampling " \ "rate definition: %s.%s.%s.%s" % ( net.code(), sta.code(), loc.code(), cha.code()) Logging.warning(msg) continue # calculate number of samples for requested # time window diffSec = (end_time - start_time).length() samples += int(diffSec * n / d) if samples > maxSamples: msg = "maximum number of %sM samples " \ "exceeded" % str(app._samplesM) return HTTP.renderErrorPage( req, http.REQUEST_ENTITY_TOO_LARGE, msg, ro) Logging.debug("adding stream: %s.%s.%s.%s %s - %s" \ % (net.code(), sta.code(), loc.code(), cha.code(), start_time.iso(), end_time.iso())) rs.addStream(net.code(), sta.code(), loc.code(), cha.code(), start_time, end_time, utils.isRestricted(cha), sta.archiveNetworkCode()) # Build output filename fileName = Application.Instance()._fileNamePrefix.replace( "%time", time.strftime('%Y-%m-%dT%H:%M:%S')) + '.mseed' # Create producer for async IO prod = _WaveformProducer(req, ro, rs, fileName, tracker) req.registerProducer(prod, True) prod.resumeProducing() # The request is handled by the deferred object return server.NOT_DONE_YET
def initConfiguration(self): if not Application.initConfiguration(self): return False # bind address and port try: self._listenAddress = self.configGetString('listenAddress') except ConfigException: pass try: self._port = self.configGetInt('port') except ConfigException: pass # maximum number of connections try: self._connections = self.configGetInt('connections') except ConfigException: pass # maximum number of objects per query, used in fdsnws-station and # fdsnws-event to limit main memory consumption try: self._queryObjects = self.configGetInt('queryObjects') except ConfigException: pass # restrict end time of request to now-realtimeGap seconds, used in # fdsnws-dataselect try: self._realtimeGap = self.configGetInt('realtimeGap') except ConfigException: pass # maximum number of samples (in units of million) per query, used in # fdsnws-dataselect to limit bandwidth try: self._samplesM = self.configGetDouble('samplesM') except ConfigException: pass # location of htpasswd file try: self._htpasswd = self.configGetString('htpasswd') except ConfigException: pass self._htpasswd = Environment.Instance().absolutePath(self._htpasswd) # location of access log file try: self._accessLogFile = Environment.Instance().absolutePath( self.configGetString('accessLog')) except ConfigException: pass # access to restricted inventory information try: self._allowRestricted = self.configGetBool('allowRestricted') except: pass # use arclink-access bindings try: self._useArclinkAccess = self.configGetBool('useArclinkAccess') except: pass # services to enable try: self._serveDataSelect = self.configGetBool('serveDataSelect') except: pass try: self._serveEvent = self.configGetBool('serveEvent') except: pass try: self._serveStation = self.configGetBool('serveStation') except: pass # event filter try: self._hideAuthor = self.configGetBool('hideAuthor') except: pass try: name = self.configGetString('evaluationMode') if name.lower() == DataModel.EEvaluationModeNames.name( DataModel.MANUAL): self._evaluationMode = DataModel.MANUAL elif name.lower() == DataModel.EEvaluationModeNames.name( DataModel.AUTOMATIC): self._evaluationMode = DataModel.AUTOMATIC else: print >> sys.stderr, "invalid evaluation mode string: %s" % name return False except: pass try: strings = self.configGetStrings('eventType.whitelist') if len(strings) > 1 or len(strings[0]): self._eventTypeWhitelist = [s.lower() for s in strings] except: pass try: strings = self.configGetStrings('eventType.blacklist') if len(strings) > 0 or len(strings[0]): self._eventTypeBlacklist = [s.lower() for s in strings] except: pass # station filter try: self._stationFilter = Environment.Instance().absolutePath( self.configGetString('stationFilter')) except ConfigException: pass # dataSelect filter try: self._dataSelectFilter = Environment.Instance().absolutePath( self.configGetString('dataSelectFilter')) except ConfigException: pass # output filter debug information try: self._debugFilter = self.configGetBool('debugFilter') except ConfigException: pass # prefix to be used as default for output filenames try: self._fileNamePrefix = self.configGetString('fileNamePrefix') except ConfigException: pass # save request logs in database? try: self._trackdbEnabled = self.configGetBool('trackdb.enable') except ConfigException: pass # default user try: self._trackdbDefaultUser = self.configGetString( 'trackdb.defaultUser') except ConfigException: pass # enable authentication extension? try: self._authEnabled = self.configGetBool('auth.enable') except ConfigException: pass # GnuPG home directory try: self._authGnupgHome = self.configGetString('auth.gnupgHome') except ConfigException: pass self._authGnupgHome = Environment.Instance().absolutePath( self._authGnupgHome) return True
def _processRequestExp(self, req, ro, dbq, exp, ep): objCount = ep.eventCount() maxObj = Application.Instance()._queryObjects if not HTTP.checkObjects(req, objCount, maxObj): return False pickIDs = set() if ro.picks is None: ro.picks = True # add related information for iEvent in xrange(ep.eventCount()): if req._disconnected: return False e = ep.event(iEvent) if self._hideAuthor: self._removeAuthor(e) # eventDescriptions and comments objCount += dbq.loadEventDescriptions(e) if ro.comments: objCount += self._loadComments(dbq, e) if not HTTP.checkObjects(req, objCount, maxObj): return False # origin references: either all or preferred only dbIter = dbq.getObjects(e, DataModel.OriginReference.TypeInfo()) for obj in dbIter: oRef = DataModel.OriginReference.Cast(obj) if oRef is None: continue if ro.allOrigins: e.add(oRef) elif oRef.originID() == e.preferredOriginID(): e.add(oRef) dbIter.close() # TODO: if focal mechanisms are added make sure derived # origin is loaded objCount += e.originReferenceCount() if not HTTP.checkObjects(req, objCount, maxObj): return False # TODO: add focal mechanisms # origins for iORef in xrange(e.originReferenceCount()): if req._disconnected: return False oID = e.originReference(iORef).originID() obj = dbq.getObject(DataModel.Origin.TypeInfo(), oID) o = DataModel.Origin.Cast(obj) if o is None: continue ep.add(o) objCount += 1 if self._hideAuthor: self._removeAuthor(o) # comments if ro.comments: objCount += self._loadComments(dbq, o) if not HTTP.checkObjects(req, objCount, maxObj): return False # magnitudes dbIter = dbq.getObjects(oID, DataModel.Magnitude.TypeInfo()) for obj in dbIter: mag = DataModel.Magnitude.Cast(obj) if mag is None: continue if ro.allMags: o.add(mag) elif mag.publicID() == e.preferredMagnitudeID(): o.add(mag) dbIter.close() if self._hideAuthor: self._removeAuthor(mag) objCount += o.magnitudeCount() if ro.comments: for iMag in xrange(o.magnitudeCount()): objCount += self._loadComments(dbq, o.magnitude(iMag)) if not HTTP.checkObjects(req, objCount, maxObj): return False # TODO station magnitudes, amplitudes # - added pick id for each pick referenced by amplitude # arrivals if ro.arrivals: objCount += dbq.loadArrivals(o) if self._removeAuthor: for iArrival in xrange(o.arrivalCount()): self._removeAuthor(o.arrival(iArrival)) # collect pick IDs if requested if ro.picks: for iArrival in xrange(o.arrivalCount()): pickIDs.add(o.arrival(iArrival).pickID()) if not HTTP.checkObjects(req, objCount, maxObj): return False # picks if pickIDs: objCount += len(pickIDs) if not HTTP.checkObjects(req, objCount, maxObj): return False for pickID in pickIDs: obj = dbq.getObject(DataModel.Pick.TypeInfo(), pickID) pick = DataModel.Pick.Cast(obj) if pick is not None: if self._hideAuthor: self._removeAuthor(pick) if ro.comments: objCount += self._loadComments(dbq, pick) ep.add(pick) if not HTTP.checkObjects(req, objCount, maxObj): return False # write response sink = utils.Sink(req) if not exp.write(sink, ep): return False Logging.debug("%s: returned %i events and %i origins (total " \ "objects/bytes: %i/%i)" % (ro.service, ep.eventCount(), ep.originCount(), objCount, sink.written)) utils.accessLog(req, ro, http.OK, sink.written, None) return True
def _findEvents(self, ep, ro, dbq): db = Application.Instance().database() def _T(name): return db.convertColumnName(name) def _time(time): return db.timeToString(time) orderByMag = ro.orderBy and ro.orderBy.startswith('magnitude') reqMag = ro.mag or orderByMag reqDist = ro.geo and ro.geo.bCircle colPID = _T('publicID') colTime = _T('time_value') colMag = _T('magnitude_value') if orderByMag: colOrderBy = "m.%s" % colMag else: colOrderBy = "o.%s" % colTime bBox = None if ro.geo: colLat, colLon = _T('latitude_value'), _T('longitude_value') if ro.geo.bBox: bBox = ro.geo.bBox else: bBox = ro.geo.bCircle.calculateBBox() # SELECT -------------------------------- q = "SELECT DISTINCT pe.%s, e.*, %s" % (colPID, colOrderBy) if reqDist: # Great circle distance calculated by Haversine formula c = ro.geo.bCircle q += ", DEGREES(ACOS(" \ "COS(RADIANS(o.%s)) * COS(RADIANS(%s)) * " \ "COS(RADIANS(o.%s) - RADIANS(%s)) + SIN(RADIANS(o.%s)) * " \ "SIN(RADIANS(%s)))) AS distance" % ( colLat, c.lat, colLon, c.lon, colLat, c.lat) # FROM ---------------------------------- q += " FROM Event AS e, PublicObject AS pe" \ ", Origin AS o, PublicObject AS po" if reqMag: q += ", Magnitude AS m, PublicObject AS pm" # WHERE --------------------------------- q += " WHERE e._oid = pe._oid" # event type white list filter, defined via configuration and/or request # parameters types = None if self._eventTypeWhitelist and ro.eventTypes: types = self._eventTypeWhitelist.intersection(ro.eventTypes) if not types: Logging.debug('all requested event types filtered by ' 'configured event type white list') return elif self._eventTypeWhitelist: types = self._eventTypeWhitelist elif ro.eventTypes: types = ro.eventTypes if types is not None: allowNull = -1 in types types = [x for x in types if x >= 0] etqIn = "e.%s IN ('%s')" % (_T('type'), "', '".join( DataModel.EEventTypeNames.name(x) for x in types)) if allowNull: etqNull = "e.%s is NULL" % _T('type') if types: q += " AND (%s OR %s)" % (etqNull, etqIn) else: q += " AND %s" % etqNull else: q += " AND %s" % etqIn # event type black list filter, defined in configuration if self._eventTypeBlacklist: allowNull = -1 not in self._eventTypeBlacklist types = [x for x in self._eventTypeBlacklist if x >= 0] etqNotIn = "e.%s NOT IN ('%s')" % (_T('type'), "', '".join( DataModel.EEventTypeNames.name(x) for x in types)) if allowNull: etqNull = "e.%s is NULL" % _T('type') if types: q += " AND (%s OR %s)" % (etqNull, etqNotIn) else: q += " AND %s" % etqNull else: q += " AND %s" % etqNotIn # event agency id filter if ro.contributors: q += " AND e.%s AND upper(e.%s) IN('%s')" % ( _T('creationinfo_used'), _T('creationinfo_agencyid'), "', '".join(ro.contributors).upper()) # origin information filter q += " AND o._oid = po._oid AND po.%s = e.%s" % ( colPID, _T('preferredOriginID')) # evaluation mode config parameter if self._evaluationMode is not None: colEvalMode = _T('evaluationMode') q += " AND o.%s = '%s'" % (colEvalMode, DataModel.EEvaluationModeNames.name( self._evaluationMode)) # time if ro.time: colTimeMS = _T('time_value_ms') if ro.time.start is not None: t = _time(ro.time.start) ms = ro.time.start.microseconds() q += " AND (o.%s > '%s' OR (o.%s = '%s' AND o.%s >= %i))" % ( colTime, t, colTime, t, colTimeMS, ms) if ro.time.end is not None: t = _time(ro.time.end) ms = ro.time.end.microseconds() q += " AND (o.%s < '%s' OR (o.%s = '%s' AND o.%s <= %i))" % ( colTime, t, colTime, t, colTimeMS, ms) # bounding box if bBox: if bBox.minLat is not None: q += " AND o.%s >= %s" % (colLat, bBox.minLat) if bBox.maxLat is not None: q += " AND o.%s <= %s" % (colLat, bBox.maxLat) if bBox.dateLineCrossing(): q += " AND (o.%s >= %s OR o.%s <= %s)" % (colLon, bBox.minLon, colLon, bBox.maxLon) else: if bBox.minLon is not None: q += " AND o.%s >= %s" % (colLon, bBox.minLon) if bBox.maxLon is not None: q += " AND o.%s <= %s" % (colLon, bBox.maxLon) # depth if ro.depth: q += " AND o.%s" % _T("depth_used") colDepth = _T('depth_value') if ro.depth.min is not None: q += " AND o.%s >= %s" % (colDepth, ro.depth.min) if ro.depth.max is not None: q += " AND o.%s <= %s" % (colDepth, ro.depth.max) # updated after if ro.updatedAfter: t = _time(ro.updatedAfter) ms = ro.updatedAfter.microseconds() colCTime = _T('creationinfo_creationtime') colCTimeMS = _T('creationinfo_creationtime_ms') colMTime = _T('creationinfo_modificationtime') colMTimeMS = _T('creationinfo_modificationtime_ms') tFilter = "(o.%s > '%s' OR (o.%s = '%s' AND o.%s > %i))" q += " AND (" q += tFilter % (colCTime, t, colCTime, t, colCTimeMS, ms) + " OR " q += tFilter % (colMTime, t, colMTime, t, colMTimeMS, ms) + ")" # magnitude information filter if reqMag: q += " AND m._oid = pm._oid AND " if ro.mag and ro.mag.type: # join magnitude table on oID of origin and magnitude type q += "m._parent_oid = o._oid AND m.%s = '%s'" % ( _T('type'), dbq.toString(ro.mag.type)) else: # join magnitude table on preferred magnitude id of event q += "pm.%s = e.%s" % (colPID, _T('preferredMagnitudeID')) if ro.mag and ro.mag.min is not None: q += " AND m.%s >= %s" % (colMag, ro.mag.min) if ro.mag and ro.mag.max is not None: q += " AND m.%s <= %s" % (colMag, ro.mag.max) # ORDER BY ------------------------------ q += " ORDER BY %s" % colOrderBy if ro.orderBy and ro.orderBy.endswith('-asc'): q += " ASC" else: q += " DESC" # SUBQUERY distance (optional) ---------- if reqDist: q = "SELECT * FROM (%s) AS subquery WHERE distance " % q c = ro.geo.bCircle if c.minRad is not None: q += ">= %s" % c.minRad if c.maxRad is not None: if c.minRad is not None: q += " AND distance " q += "<= %s" % c.maxRad # LIMIT/OFFSET -------------------------- if ro.limit is not None or ro.offset is not None: # Postgres allows to omit the LIMIT parameter for offsets, MySQL # does not. According to the MySQL manual a very large number should # be used for this case. l = DBMaxUInt if ro.limit is not None: l = ro.limit q += " LIMIT %i" % l if ro.offset is not None: q += " OFFSET %i" % ro.offset Logging.debug("event query: %s" % q) for e in dbq.getObjectIterator(q, DataModel.Event.TypeInfo()): ep.add(DataModel.Event.Cast(e))
def __init__(self, userName=None): resource.Resource.__init__(self) self._rsURL = Application.Instance().recordStreamURL() self.userName = userName
def _processRequest(self, req, ro): if ro.quality != 'B' and ro.quality != 'M': msg = "quality other than 'B' or 'M' not supported" return self.renderErrorPage(req, http.BAD_REQUEST, msg, ro) if ro.minimumLength: msg = "enforcing of minimum record length not supported" return self.renderErrorPage(req, http.BAD_REQUEST, msg, ro) if ro.longestOnly: msg = "limitation to longest segment not supported" return self.renderErrorPage(req, http.BAD_REQUEST, msg, ro) app = Application.Instance() ro._checkTimes(app._realtimeGap) maxSamples = None if app._samplesM is not None: maxSamples = app._samplesM * 1000000 samples = 0 trackerList = [] if app._trackdbEnabled or app._requestLog: xff = req.requestHeaders.getRawHeaders("x-forwarded-for") if xff: userIP = xff[0].split(",")[0].strip() else: userIP = req.getClientIP() clientID = req.getHeader("User-Agent") if clientID: clientID = clientID[:80] else: clientID = "fdsnws" if app._trackdbEnabled: if ro.userName: userID = ro.userName else: userID = app._trackdbDefaultUser reqID = 'ws' + str(int(round(time.time() * 1000) - 1420070400000)) tracker = RequestTrackerDB(clientID, app.connection(), reqID, "WAVEFORM", userID, "REQUEST WAVEFORM " + reqID, "fdsnws", userIP, req.getClientIP()) trackerList.append(tracker) if app._requestLog: tracker = app._requestLog.tracker(ro.service, ro.userName, userIP, clientID) trackerList.append(tracker) # Open record stream rs = _MyRecordStream(self._rsURL, trackerList, self.__bufferSize) forbidden = None # Add request streams # iterate over inventory networks for s in ro.streams: for net in self._networkIter(s): netRestricted = utils.isRestricted(net) if not trackerList and netRestricted and not self.__user: forbidden = forbidden or (forbidden is None) continue for sta in self._stationIter(net, s): staRestricted = utils.isRestricted(sta) if not trackerList and staRestricted and not self.__user: forbidden = forbidden or (forbidden is None) continue for loc in self._locationIter(sta, s): for cha in self._streamIter(loc, s): start_time = max(cha.start(), s.time.start) try: end_time = min(cha.end(), s.time.end) except ValueError: end_time = s.time.end if (netRestricted or staRestricted or utils.isRestricted(cha) ) and (not self.__user or (self.__access and not self.__access.authorize( self.__user, net.code(), sta.code(), loc.code(), cha.code(), start_time, end_time))): for tracker in trackerList: net_class = 't' if net.code()[0] \ in "0123456789XYZ" else 'p' tracker.line_status( start_time, end_time, net.code(), sta.code(), cha.code(), loc.code(), True, net_class, True, [], "fdsnws", "DENIED", 0, "") forbidden = forbidden or (forbidden is None) continue forbidden = False # enforce maximum sample per request restriction if maxSamples is not None: try: n = cha.sampleRateNumerator() d = cha.sampleRateDenominator() except ValueError: msg = "skipping stream without sampling " \ "rate definition: %s.%s.%s.%s" % ( net.code(), sta.code(), loc.code(), cha.code()) Logging.warning(msg) continue # calculate number of samples for requested # time window diffSec = (end_time - start_time).length() samples += int(diffSec * n / d) if samples > maxSamples: msg = "maximum number of %sM samples " \ "exceeded" % str(app._samplesM) return self.renderErrorPage( req, http.REQUEST_ENTITY_TOO_LARGE, msg, ro) Logging.debug( "adding stream: %s.%s.%s.%s %s - %s" % (net.code(), sta.code(), loc.code(), cha.code(), start_time.iso(), end_time.iso())) rs.addStream(net.code(), sta.code(), loc.code(), cha.code(), start_time, end_time, utils.isRestricted(cha), sta.archiveNetworkCode()) if forbidden: for tracker in trackerList: tracker.volume_status("fdsnws", "DENIED", 0, "") tracker.request_status("END", "") msg = "access denied" return self.renderErrorPage(req, http.FORBIDDEN, msg, ro) elif forbidden is None: for tracker in trackerList: tracker.volume_status("fdsnws", "NODATA", 0, "") tracker.request_status("END", "") msg = "no metadata found" return self.renderErrorPage(req, http.NO_CONTENT, msg, ro) # Build output filename fileName = Application.Instance()._fileNamePrefix.replace( "%time", time.strftime('%Y-%m-%dT%H:%M:%S')) + '.mseed' # Create producer for async IO prod = _WaveformProducer(req, ro, rs, fileName, trackerList) req.registerProducer(prod, True) prod.resumeProducing() # The request is handled by the deferred object return server.NOT_DONE_YET
def _processRequest(self, req, ro): if ro.quality != 'B' and ro.quality != 'M': msg = "quality other than 'B' or 'M' not supported" return HTTP.renderErrorPage(req, http.SERVICE_UNAVAILABLE, msg, ro) if ro.minimumLength: msg = "enforcing of minimum record length not supported" return HTTP.renderErrorPage(req, http.SERVICE_UNAVAILABLE, msg, ro) if ro.longestOnly: msg = "limitation to longest segment not supported" return HTTP.renderErrorPage(req, http.SERVICE_UNAVAILABLE, msg, ro) app = Application.Instance() ro._checkTimes(app._realtimeGap) # Open record stream rs = RecordStream.Open(self._rsURL) if rs is None: msg = "could not open record stream" return HTTP.renderErrorPage(req, http.SERVICE_UNAVAILABLE, msg, ro) maxSamples = None if app._samplesM is not None: maxSamples = app._samplesM * 1000000 samples = 0 # Add request streams # iterate over inventory networks inv = Application.Instance()._inv for s in ro.streams: for net in self._networkIter(inv, s): if ro.userName is None and utils.isRestricted(net): continue for sta in self._stationIter(net, s): if ro.userName is None and utils.isRestricted(sta): continue for loc in self._locationIter(sta, s): for cha in self._streamIter(loc, s): # enforce maximum sample per request restriction if maxSamples is not None: try: n = cha.sampleRateNumerator() d = cha.sampleRateDenominator() except ValueException: msg = "skipping stream without sampling " \ "rate definition: %s.%s.%s.%s" % ( net.code(), sta.code(), loc.code(), cha.code()) Logging.warning(msg) continue # calculate number of samples for requested # time window diffSec = (s.time.end - s.time.start).length() samples += int(diffSec * n / d) if samples > maxSamples: msg = "maximum number of %sM samples " \ "exceeded" % str(app._samplesM) return HTTP.renderErrorPage( req, http.REQUEST_ENTITY_TOO_LARGE, msg, ro) Logging.debug("adding stream: %s.%s.%s.%s %s - %s" \ % (net.code(), sta.code(), loc.code(), cha.code(), s.time.start.iso(), s.time.end.iso())) rs.addStream(net.code(), sta.code(), loc.code(), cha.code(), s.time.start, s.time.end) # Build output filename fileName = Application.Instance()._fileNamePrefix + '.mseed' # Create producer for async IO req.registerProducer(_WaveformProducer(req, ro, rs, fileName), False) # The request is handled by the deferred object return server.NOT_DONE_YET
def init(self): """ SC3 specific method. Returning False means that we do not enter the SeisComP3 run loop. """ if Application.init(self) == False: return False result: Optional[model.WPhaseResult] = None Logging.enableConsoleLogging(Logging.getGlobalChannel("error")) wphase_failed = False if self.filename is not None: # Output JSON was provided, so we just send it to messaging. logger.info("Parsing W-Phase result from file.") result = model.WPhaseResult.parse_file(self.filename) else: # Run the inversion. try: logger.info("Starting W-Phase.") result = runwphase( output_dir=self.output, server=self.fdsn_client, eqinfo=self.eqinfo, networks=self.networks, make_maps=self.make_maps, output_dir_can_exist=self.overwrite, save_waveforms=self.save_waveforms, save_inventory=self.save_inventory, waveforms=self.waveforms, inventory=self.inventory, ) except Exception: logger.exception("W-Phase run failed.") wphase_failed = True if result is not None: try: objs = createAndSendObjects(result, self.connection(), evid=self.evid, agency=self.agency) except Exception: logger.exception("Failed to create objects for SC3.") else: filename = os.path.join(str(self.output), "sc3.xml") try: # write output to file writeSCML(filename, objs) logger.info("Stored results in SCML as %s", filename) except Exception: logger.exception("Failed to write SCML to %s.", filename) if self.write_s3: # We always want to write to S3, even in the event of failure as # the JSON output may explain the failure. We also want to do it # at the very end since we keep the sc3 log file. try: from wphase.aws import write_to_s3 write_to_s3(self.output, self.bucket_name, self.evid, self.resultid, [(self._logfile_for_s3.name, 'sc3.log')], logger.error) except Exception as e: logger.error('failed write to S3: %s', e) finally: self._logfile_for_s3.close() if self.notificationemail: # must be done after writing to S3 success = result is not None and result.MomentTensor is not None subject, body = self.createEmail( event_id=self.evid, result_id=self.resultid, result=result, call_succeeded=success, ) send_email( recipients=self.notificationemail, subject=self.email_subject_prefix + str(subject) + self.email_subject_postfix, message=body, from_email=self.fromemail, method=self.email_method, email_aws_region=self.email_aws_region, server=self.smtp_server, port=self.smtp_port, user=self.smtp_user, password=self.smtp_password, ssl=self.smtp_ssl, tls=self.smtp_tls, ) sys.exit(1 if wphase_failed else 0)
def validateParameters(self): """Called by the seiscomp client Application setup.""" if not Application.validateParameters(self): return False getter = self.processArg def getflag(name, to=None): getter(name, to=to, conv=lambda x: True) try: # If there is an unrecognized option it must be a JSON file # of wphase outputs. In this case, the file is parsed and pushed # to the messaging system and written to disk. self.filename = self.commandline().unrecognizedOptions()[0] except Exception: # Otherwise we expect a description of the location. Wphase is # then run and results pushed pushed to the messaging system # and written to disk. # depth only needed for BoM XML try: depth = float(self.commandline().optionDouble("depth")) except Exception: depth = 0. getter('save-waveforms', 'save_waveforms') getter('save-inventory', 'save_inventory') getter('waveforms', 'waveforms') getter('inventory', 'inventory') getter('server') if self.server is not None: self.fdsn_client = Client(self.server) elif not (self.waveforms and self.inventory): logger.error( 'Must provide either server or waveforms+inventory to run inversion.' ) return False getter('evid') try: self.eqinfo = model.Event( latitude=float(self.commandline().optionDouble("lat")), longitude=float(self.commandline().optionDouble("lon")), depth=depth, time=self.commandline().optionString("time"), ) except Exception: if not (self.fdsn_client and self.evid): logger.error( 'You must provide a JSON payload, lat/lon/time or an evid to fetch from FDSN' ) return False try: cat = self.fdsn_client.get_events(eventid=self.evid) origin = cat.events[0].preferred_origin() except Exception: logger.exception( "Could not retrieve event %s from FDSN server at %s", self.evid, self.server) return False self.eqinfo = model.Event( longitude=origin.longitude, latitude=origin.latitude, depth=origin.depth, time=origin.time, ) getter('sourcezone') getter('magtype', 'mag_type') getter('magvalue', 'mag_value', conv=float) getter('outputs', 'output') getter('networks') getter('region') getter('resultid') getter('notificationemail') getter('emailmethod', 'email_method') getter('fromemail') getter('email-subject-prefix', 'email_subject_prefix') getter('email-subject-postfix', 'email_subject_postfix') getter('emailawsregion', 'email_aws_region', 'us-west-2') getter('writeS3', 'write_s3', False, lambda x: True) getter('bucketname', 'bucket_name') getter('agency') getflag('make-maps', 'make_maps') getflag('overwrite', 'overwrite') getter('smtp-server', 'smtp_server') getter('smtp-port', 'smtp_port') getter('smtp-user', 'smtp_user') getter('smtp-password', 'smtp_password') getflag('smtp-ssl', 'smtp_ssl') getflag('smtp-tls', 'smtp_tls') if self.evid is not None: self.output = os.path.join(self.output, self.evid) self.eqinfo.id = self.evid if self.resultid is not None: self.output = os.path.join(self.output, self.resultid) if self.notificationemail is not None: self.notificationemail = str(self.notificationemail).split(',') if self.write_s3 and ( self.evid is None or \ self.bucket_name is None): logger.error( 'attempt to write to s3, but did not provide bucket name.') return False if self.notificationemail is not None and ( self.mag_type is None or \ self.mag_value is None or \ self.evid is None or \ self.resultid is None ): logger.error('cannot send email.') return False return True
class FDSNEvent(resource.Resource): isLeaf = True #--------------------------------------------------------------------------- def __init__(self, hideAuthor = False, evaluationMode = None, eventTypeWhitelist = None, eventTypeBlacklist = None, formatList = None): self._hideAuthor = hideAuthor self._evaluationMode = evaluationMode self._eventTypeWhitelist = eventTypeWhitelist self._eventTypeBlacklist = eventTypeBlacklist self._formatList = formatList #--------------------------------------------------------------------------- def render_OPTIONS(self, req): req.setHeader('Access-Control-Allow-Origin', '*') req.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS') req.setHeader('Access-Control-Allow-Headers', 'Accept, Content-Type, X-Requested-With, Origin') req.setHeader('Content-Type', 'text/plain') return "" #--------------------------------------------------------------------------- def render_GET(self, req): # Parse and validate GET parameters ro = _EventRequestOptions(req.args) try: ro.parse() except ValueError, e: Logging.warning(str(e)) return HTTP.renderErrorPage(req, http.BAD_REQUEST, str(e), ro) # Catalog filter is not supported if ro.catalogs: msg = "catalog filter not supported" return HTTP.renderErrorPage(req, http.BAD_REQUEST, msg, ro) # updateafter not implemented if ro.updatedAfter: msg = "filtering based on update time not supported" return HTTP.renderErrorPage(req, http.BAD_REQUEST, msg, ro) if self._formatList is not None and ro.format not in self._formatList: msg = "output format '%s' not available" % ro.format return HTTP.renderErrorPage(req, http.BAD_REQUEST, msg, ro) # Exporter, 'None' is used for text output if ro.format in ro.VText: exp = None else: exp = Exporter.Create(ro.Exporters[ro.format]) if exp: exp.setFormattedOutput(bool(ro.formatted)) else: msg = "output format '%s' not available, export module '%s' could " \ "not be loaded." % (ro.format, ro.Exporters[ro.format]) return HTTP.renderErrorPage(req, http.BAD_REQUEST, msg, ro) # Create database query db = DatabaseInterface.Open(Application.Instance().databaseURI()) if db is None: msg = "could not connect to database: %s" % dbq.errorMsg() return HTTP.renderErrorPage(req, http.SERVICE_UNAVAILABLE, msg, ro) dbq = DataModel.DatabaseQuery(db) # Process request in separate thread d = deferToThread(self._processRequest, req, ro, dbq, exp) req.notifyFinish().addErrback(utils.onCancel, d) d.addBoth(utils.onFinish, req) # The request is handled by the deferred object return server.NOT_DONE_YET
def _processRequestExp(self, req, ro, dbq, exp, ep): objCount = ep.eventCount() maxObj = Application.Instance()._queryObjects if not self.checkObjects(req, objCount, maxObj): return False pickIDs = set() if ro.picks is None: ro.picks = True # add related information for iEvent in range(ep.eventCount()): if req._disconnected: return False e = ep.event(iEvent) if self._hideAuthor: self._removeAuthor(e) originIDs = set() magIDs = set() magIDs.add(e.preferredMagnitudeID()) # eventDescriptions and comments objCount += dbq.loadEventDescriptions(e) if ro.comments: objCount += self._loadComments(dbq, e) if not self.checkObjects(req, objCount, maxObj): return False # origin references: either all or preferred only dbIter = dbq.getObjects(e, DataModel.OriginReference.TypeInfo()) for obj in dbIter: oRef = DataModel.OriginReference.Cast(obj) if oRef is None: continue if ro.allOrigins: e.add(oRef) originIDs.add(oRef.originID()) elif oRef.originID() == e.preferredOriginID(): e.add(oRef) originIDs.add(oRef.originID()) dbIter.close() objCount += e.originReferenceCount() # focalMechanism references: either none, preferred only or all if ro.fm or ro.allFMs: dbIter = dbq.getObjects( e, DataModel.FocalMechanismReference.TypeInfo()) for obj in dbIter: fmRef = DataModel.FocalMechanismReference.Cast(obj) if fmRef is None: continue if ro.allFMs: e.add(fmRef) elif fmRef.focalMechanismID( ) == e.preferredFocalMechanismID(): e.add(fmRef) dbIter.close() objCount += e.focalMechanismReferenceCount() if not self.checkObjects(req, objCount, maxObj): return False # focal mechanisms: process before origins to add derived origin to # originID list since it may be missing from origin reference list for iFMRef in range(e.focalMechanismReferenceCount()): if req._disconnected: return False fmID = e.focalMechanismReference(iFMRef).focalMechanismID() obj = dbq.getObject(DataModel.FocalMechanism.TypeInfo(), fmID) fm = DataModel.FocalMechanism.Cast(obj) if fm is None: continue ep.add(fm) objCount += 1 if self._hideAuthor: self._removeAuthor(fm) # comments if ro.comments: objCount += self._loadComments(dbq, fm) # momentTensors objCount += dbq.loadMomentTensors(fm) if not self.checkObjects(req, objCount, maxObj): return False for iMT in range(fm.momentTensorCount()): mt = fm.momentTensor(iMT) originIDs.add(mt.derivedOriginID()) magIDs.add(mt.momentMagnitudeID()) if self._hideAuthor: self._removeAuthor(mt) if ro.comments: for iMT in range(fm.momentTensorCount()): objCount += self._loadComments(dbq, mt) objCount += dbq.loadDataUseds(mt) objCount += dbq.loadMomentTensorPhaseSettings(mt) if ro.staMTs: objCount += dbq.loadMomentTensorStationContributions( mt) for iStaMT in range( mt.momentTensorStationContributionCount()): objCount += dbq.load( mt.momentTensorStationContribution(iStaMT)) if not self.checkObjects(req, objCount, maxObj): return False # find ID of origin containing preferred Magnitude if e.preferredMagnitudeID(): obj = dbq.getObject(DataModel.Magnitude.TypeInfo(), e.preferredMagnitudeID()) m = DataModel.Magnitude.Cast(obj) if m is not None: oID = dbq.parentPublicID(m) if oID: originIDs.add(oID) # origins for oID in sorted(originIDs): if req._disconnected: return False obj = dbq.getObject(DataModel.Origin.TypeInfo(), oID) o = DataModel.Origin.Cast(obj) if o is None: continue ep.add(o) objCount += 1 if self._hideAuthor: self._removeAuthor(o) # comments if ro.comments: objCount += self._loadComments(dbq, o) if not self.checkObjects(req, objCount, maxObj): return False # magnitudes dbIter = dbq.getObjects(oID, DataModel.Magnitude.TypeInfo()) for obj in dbIter: mag = DataModel.Magnitude.Cast(obj) if mag is None: continue if ro.allMags: o.add(mag) elif mag.publicID() in magIDs: o.add(mag) dbIter.close() if self._hideAuthor: self._removeAuthor(mag) objCount += o.magnitudeCount() if ro.comments: for iMag in range(o.magnitudeCount()): objCount += self._loadComments(dbq, o.magnitude(iMag)) if not self.checkObjects(req, objCount, maxObj): return False # TODO station magnitudes, amplitudes # - added pick id for each pick referenced by amplitude # arrivals if ro.arrivals: objCount += dbq.loadArrivals(o) if self._removeAuthor: for iArrival in range(o.arrivalCount()): self._removeAuthor(o.arrival(iArrival)) # collect pick IDs if requested if ro.picks: for iArrival in range(o.arrivalCount()): pickIDs.add(o.arrival(iArrival).pickID()) if not self.checkObjects(req, objCount, maxObj): return False # picks if pickIDs: objCount += len(pickIDs) if not self.checkObjects(req, objCount, maxObj): return False for pickID in sorted(pickIDs): obj = dbq.getObject(DataModel.Pick.TypeInfo(), pickID) pick = DataModel.Pick.Cast(obj) if pick is not None: if self._hideAuthor: self._removeAuthor(pick) if ro.comments: objCount += self._loadComments(dbq, pick) ep.add(pick) if not self.checkObjects(req, objCount, maxObj): return False # write response sink = utils.Sink(req) if not exp.write(sink, ep): return False Logging.debug("%s: returned %i events and %i origins (total " "objects/chars: %i/%i)" % (ro.service, ep.eventCount(), ep.originCount(), objCount, sink.written)) utils.accessLog(req, ro, http.OK, sink.written, None) return True
def accessLog(req, ro, code, length, err): logger = Application.Instance()._accessLog if logger is None: return logger.log(AccessLogEntry(req, ro, code, length, err))
def __init__(self, inv, access=None, user=None): resource.Resource.__init__(self) self._rsURL = Application.Instance().recordStreamURL() self.__inv = inv self.__access = access self.__user = user
def initConfiguration(self): if not Application.initConfiguration(self): return False # bind address and port try: self._listenAddress = self.configGetString('listenAddress') except ConfigException: pass try: self._port = self.configGetInt('port') except ConfigException: pass # maximum number of connections try: self._connections = self.configGetInt('connections') except ConfigException: pass # maximum number of objects per query, used in fdsnws-station and # fdsnws-event to limit main memory consumption try: self._queryObjects = self.configGetInt('queryObjects') except ConfigException: pass # restrict end time of request to now-realtimeGap seconds, used in # fdsnws-dataselect try: self._realtimeGap = self.configGetInt('realtimeGap') except ConfigException: pass # maximum number of samples (in units of million) per query, used in # fdsnws-dataselect to limit bandwidth try: self._samplesM = self.configGetDouble('samplesM') except ConfigException: pass try: self._recordBulkSize = self.configGetInt('recordBulkSize') except ConfigException: pass if self._recordBulkSize < 1: print >> sys.stderr, "Invalid recordBulkSize, must be larger than 0" return False # location of htpasswd file try: self._htpasswd = self.configGetString('htpasswd') except ConfigException: pass self._htpasswd = Environment.Instance().absolutePath(self._htpasswd) # location of access log file try: self._accessLogFile = Environment.Instance().absolutePath( self.configGetString('accessLog')) except ConfigException: pass # access to restricted inventory information try: self._allowRestricted = self.configGetBool('allowRestricted') except: pass # use arclink-access bindings try: self._useArclinkAccess = self.configGetBool('useArclinkAccess') except: pass # services to enable try: self._serveDataSelect = self.configGetBool('serveDataSelect') except: pass try: self._serveEvent = self.configGetBool('serveEvent') except: pass try: self._serveStation = self.configGetBool('serveStation') except: pass # event filter try: self._hideAuthor = self.configGetBool('hideAuthor') except: pass try: name = self.configGetString('evaluationMode') if name.lower() == DataModel.EEvaluationModeNames.name(DataModel.MANUAL): self._evaluationMode = DataModel.MANUAL elif name.lower() == DataModel.EEvaluationModeNames.name(DataModel.AUTOMATIC): self._evaluationMode = DataModel.AUTOMATIC else: print >> sys.stderr, "invalid evaluation mode string: %s" % name return False except: pass try: strings = self.configGetStrings('eventType.whitelist') if len(strings) > 1 or len(strings[0]): self._eventTypeWhitelist = [ s.lower() for s in strings ] except: pass try: strings = self.configGetStrings('eventType.blacklist') if len(strings) > 0 or len(strings[0]): self._eventTypeBlacklist = [ s.lower() for s in strings ] except: pass # station filter try: self._stationFilter = Environment.Instance().absolutePath(self.configGetString('stationFilter')) except ConfigException: pass # dataSelect filter try: self._dataSelectFilter = Environment.Instance().absolutePath(self.configGetString('dataSelectFilter')) except ConfigException: pass # output filter debug information try: self._debugFilter = self.configGetBool('debugFilter') except ConfigException: pass # prefix to be used as default for output filenames try: self._fileNamePrefix = self.configGetString('fileNamePrefix') except ConfigException: pass # save request logs in database? try: self._trackdbEnabled = self.configGetBool('trackdb.enable') except ConfigException: pass # default user try: self._trackdbDefaultUser = self.configGetString('trackdb.defaultUser') except ConfigException: pass # enable authentication extension? try: self._authEnabled = self.configGetBool('auth.enable') except ConfigException: pass # GnuPG home directory try: self._authGnupgHome = self.configGetString('auth.gnupgHome') except ConfigException: pass self._authGnupgHome = Environment.Instance().absolutePath(self._authGnupgHome) # If the database connection is passed via command line or configuration # file then messaging is disabled. Messaging is only used to get # the configured database connection URI. if self.databaseURI() != "": self.setMessagingEnabled(False) else: # Without the event service, event a database connection is not # required if the inventory is loaded from file if not self._serveEvent and not self._useArclinkAccess and not self.isInventoryDatabaseEnabled(): self.setMessagingEnabled(False) self.setDatabaseEnabled(False, False) return True
def initConfiguration(self): if not Application.initConfiguration(self): return False # bind address and port try: self._listenAddress = self.configGetString('listenAddress') except Exception: pass try: self._port = self.configGetInt('port') except Exception: pass # maximum number of connections try: self._connections = self.configGetInt('connections') except Exception: pass # maximum number of objects per query, used in fdsnws-station and # fdsnws-event to limit main memory consumption try: self._queryObjects = self.configGetInt('queryObjects') except Exception: pass # restrict end time of request to now-realtimeGap seconds, used in # fdsnws-dataselect try: self._realtimeGap = self.configGetInt('realtimeGap') except Exception: pass # maximum number of samples (in units of million) per query, used in # fdsnws-dataselect to limit bandwidth try: self._samplesM = self.configGetDouble('samplesM') except Exception: pass try: self._recordBulkSize = self.configGetInt('recordBulkSize') except Exception: pass if self._recordBulkSize < 1: print >> sys.stderr, "Invalid recordBulkSize, must be larger than 0" return False # location of htpasswd file try: self._htpasswd = self.configGetString('htpasswd') except Exception: pass self._htpasswd = Environment.Instance().absolutePath(self._htpasswd) # location of access log file try: self._accessLogFile = Environment.Instance().absolutePath( self.configGetString('accessLog')) except Exception: pass # location of request log file try: self._requestLogFile = Environment.Instance().absolutePath( self.configGetString('requestLog')) except Exception: pass # access to restricted inventory information try: self._allowRestricted = self.configGetBool('allowRestricted') except Exception: pass # use arclink-access bindings try: self._useArclinkAccess = self.configGetBool('useArclinkAccess') except Exception: pass # services to enable try: self._serveDataSelect = self.configGetBool('serveDataSelect') except Exception: pass try: self._serveEvent = self.configGetBool('serveEvent') except Exception: pass try: self._serveStation = self.configGetBool('serveStation') except Exception: pass try: self._serveAvailability = self.configGetBool('serveAvailability') except Exception: pass # data availability try: self._daEnabled = self.configGetBool('dataAvailability.enable') except Exception: pass try: self._daCacheDuration = self.configGetInt( 'dataAvailability.cacheDuration') except Exception: pass try: self._daRepositoryName = self.configGetString( 'dataAvailability.repositoryName') except Exception: pass try: self._daDCCName = self.configGetString('dataAvailability.dccName') except Exception: pass if self._serveAvailability and not self._daEnabled: print >> sys.stderr, "can't serve availabilty without " \ "dataAvailability.enable set to true" return False if not bool(re.match(r'^[a-zA-Z0-9_\ -]*$', self._daRepositoryName)): print >> sys.stderr, "invalid characters in dataAvailability.repositoryName" return False if not bool(re.match(r'^[a-zA-Z0-9_\ -]*$', self._daDCCName)): print >> sys.stderr, "invalid characters in dataAvailability.dccName" return False # event filter try: self._hideAuthor = self.configGetBool('hideAuthor') except Exception: pass try: name = self.configGetString('evaluationMode') if name.lower() == DataModel.EEvaluationModeNames.name(DataModel.MANUAL): self._evaluationMode = DataModel.MANUAL elif name.lower() == DataModel.EEvaluationModeNames.name(DataModel.AUTOMATIC): self._evaluationMode = DataModel.AUTOMATIC else: print >> sys.stderr, "invalid evaluation mode string: %s" % name return False except Exception: pass try: strings = self.configGetStrings('eventType.whitelist') if len(strings) > 1 or len(strings[0]): self._eventTypeWhitelist = [s.lower() for s in strings] except Exception: pass try: strings = self.configGetStrings('eventType.blacklist') if len(strings) > 0 or len(strings[0]): self._eventTypeBlacklist = [s.lower() for s in strings] except Exception: pass try: strings = self.configGetStrings('eventFormats') if len(strings) > 1 or len(strings[0]): self._eventFormats = [s.lower() for s in strings] except Exception: pass # station filter try: self._stationFilter = Environment.Instance().absolutePath( self.configGetString('stationFilter')) except Exception: pass # dataSelect filter try: self._dataSelectFilter = Environment.Instance().absolutePath( self.configGetString('dataSelectFilter')) except Exception: pass # output filter debug information try: self._debugFilter = self.configGetBool('debugFilter') except Exception: pass # prefix to be used as default for output filenames try: self._fileNamePrefix = self.configGetString('fileNamePrefix') except Exception: pass # save request logs in database? try: self._trackdbEnabled = self.configGetBool('trackdb.enable') except Exception: pass # default user try: self._trackdbDefaultUser = self.configGetString( 'trackdb.defaultUser') except Exception: pass # enable authentication extension? try: self._authEnabled = self.configGetBool('auth.enable') except Exception: pass # GnuPG home directory try: self._authGnupgHome = self.configGetString('auth.gnupgHome') except Exception: pass self._authGnupgHome = Environment.Instance().absolutePath(self._authGnupgHome) # blacklist of users/tokens try: strings = self.configGetStrings('auth.blacklist') if len(strings) > 1 or len(strings[0]): self._authBlacklist = strings except Exception: pass # If the database connection is passed via command line or configuration # file then messaging is disabled. Messaging is only used to get # the configured database connection URI. if self.databaseURI() != "": self.setMessagingEnabled(self._trackdbEnabled) else: # Without the event service, a database connection is not # required if the inventory is loaded from file and no data # availability information should be processed if not self._serveEvent and not self._useArclinkAccess and \ (not self._serveStation or ( not self.isInventoryDatabaseEnabled() and not self._daEnabled)): self.setMessagingEnabled(self._trackdbEnabled) self.setDatabaseEnabled(False, False) return True
def accessLog(req, ro, code, length, err): logger = Application.Instance()._accessLog # pylint: disable=W0212 if logger is None: return logger.log(AccessLogEntry(req, ro, code, length, err))