def __init__(self, appName):
# initialize SC3 environment
env = seiscomp3.System.Environment_Instance()
# set up logging
self.__syslog = seiscomp3.Logging.SyslogOutput()
self.__syslog.open(appName, syslog_facility)
for (v, c) in ((1, "error"), (2, "warning"), (2, "notice"),
(3, "info"), (4, "debug")):
if verbosity >= v:
self.__syslog.subscribe(seiscomp3.Logging.getGlobalChannel(c))
logs.debug = seiscomp3.Logging.debug
logs.info = seiscomp3.Logging.info
logs.notice = seiscomp3.Logging.notice
logs.warning = seiscomp3.Logging.warning
logs.error = seiscomp3.Logging.error
logs.notice("Starting webinterface")
# load SC3 config files from all standard locations (SEISCOMP_ROOT must be set)
self.__cfg = seiscomp3.Config.Config()
env.initConfig(self.__cfg, appName, env.CS_FIRST, env.CS_LAST, True)
self.__action_table = {}
self.__modules = {}
# Common config variables
self.server_folder = self.getConfigString('SERVER_FOLDER', None)
if not self.server_folder:
err = "%s: Cannot find server root, configuration not loaded" % (
appName)
raise Exception(err)
if not os.path.exists(self.server_folder):
err = "%s: Server root directory not found" % (appName)
raise Exception(err)
# Add inventory cache here, to be accessible to all modules
inventory = os.path.join(self.server_folder, 'data',
'Arclink-inventory.xml')
self.ic = InventoryCache(inventory)
# Load all modules in given directory.
# Modules must contain a class WI_Module, whose __init__() takes
# WebInterface object (our self) as an argument and calls our
# addAction().
#for f in glob.glob(os.path.join(env.shareDir(), "plugins", "webinterface", "*.py")):
for f in glob.glob(
os.path.join(self.server_folder, "wsgi", "modules", "*.py")):
self.__load_module(f)
logs.debug(str(self))
def __init__(self, appName):
# initialize SC3 environment
env = seiscomp3.System.Environment_Instance()
# set up logging
self.__syslog = seiscomp3.Logging.SyslogOutput()
self.__syslog.open(appName, syslog_facility)
for (v, c) in ((1, "error"), (2, "warning"), (2, "notice"), (3, "info"), (4, "debug")):
if verbosity >= v:
self.__syslog.subscribe(seiscomp3.Logging.getGlobalChannel(c))
logs.debug = seiscomp3.Logging.debug
logs.info = seiscomp3.Logging.info
logs.notice = seiscomp3.Logging.notice
logs.warning = seiscomp3.Logging.warning
logs.error = seiscomp3.Logging.error
logs.notice("Starting webinterface")
# load SC3 config files from all standard locations (SEISCOMP_ROOT must be set)
self.__cfg = seiscomp3.Config.Config()
env.initConfig(self.__cfg, appName, env.CS_FIRST, env.CS_LAST, True)
self.__action_table = {}
self.__modules = {}
# Common config variables
self.server_folder = self.getConfigString('SERVER_FOLDER', None)
if not self.server_folder:
err="%s: Cannot find server root, configuration not loaded" % (appName)
raise Exception(err)
if not os.path.exists(self.server_folder):
err="%s: Server root directory not found" % (appName)
raise Exception(err)
# Add inventory cache here, to be accessible to all modules
inventory = os.path.join(self.server_folder, 'data', 'Arclink-inventory.xml')
self.ic = InventoryCache(inventory)
# Load all modules in given directory.
# Modules must contain a class WI_Module, whose __init__() takes
# WebInterface object (our self) as an argument and calls our
# addAction().
#for f in glob.glob(os.path.join(env.shareDir(), "plugins", "webinterface", "*.py")):
for f in glob.glob(os.path.join(self.server_folder, "wsgi", "modules", "*.py")):
self.__load_module(f)
logs.debug(str(self))
def send_notifiers(self, group):
Nsize = DataModel.Notifier.Size()
if Nsize > 0:
logs.warning("trying to apply %d changes..." % Nsize)
else:
logs.notice("no changes to apply")
return 0
Nmsg = DataModel.Notifier.GetMessage(True)
it = Nmsg.iter()
msg = DataModel.NotifierMessage()
maxmsg = 100
sent = 0
mcount = 0
try:
try:
while it.get():
msg.attach(DataModel.Notifier_Cast(it.get()))
mcount += 1
if msg and mcount == maxmsg:
sent += mcount
logs.debug("sending message (%5.1f %%)" % (sent / float(Nsize) * 100.0))
self.send(group, msg)
msg.clear()
mcount = 0
self.sync("fill-db")
it.next()
except:
pass
finally:
if msg.size():
logs.debug("sending message (%5.1f %%)" % 100.0)
self.send(group, msg)
msg.clear()
self.sync("fill-db")
return mcount
def get_citation(nets, param, verbose):
postdata = ""
for (net, year) in nets:
postdata += "%s * * * %d-01-01T00:00:00Z %d-12-31T23:59:59Z\n" \
% (net, year, year)
if not isinstance(postdata, bytes):
postdata = postdata.encode('utf-8')
try:
proc = exec_fetch(param, postdata, verbose, True)
except OSError as e:
logs.error(str(e))
logs.error("error running fdsnws_fetch")
return 1
net_desc = {}
for line in proc.stdout:
try:
if isinstance(line, bytes):
line = line.decode('utf-8')
if not line or line.startswith('#'):
continue
(code, desc, start) = line.split('|')[:3]
year = dateutil.parser.parse(start).year
except (ValueError, UnicodeDecodeError) as e:
logs.error("error parsing text format: %s" % str(e))
continue
if code[0] in "0123456789XYZ":
net_desc["%s_%d" % (code, year)] = desc
else:
net_desc[code] = desc
logs.notice("You received seismic waveform data from the following "
"network(s):")
for code in sorted(net_desc):
logs.notice("%s %s" % (code, net_desc[code]))
logs.notice("\nAcknowledgment is extremely important for network operators\n"
"providing open data. When preparing publications, please\n"
"cite the data appropriately. The FDSN service at\n\n"
" http://www.fdsn.org/networks/citation/?networks=%s\n\n"
"provides a helpful guide based on available network\n"
"Digital Object Identifiers.\n"
% "+".join(sorted(net_desc)))
def get_citation(nets, param, verbose):
postdata = ""
for (net, year) in nets:
postdata += "%s * * * %d-01-01T00:00:00Z %d-12-31T23:59:59Z\n" \
% (net, year, year)
if not isinstance(postdata, bytes):
postdata = postdata.encode('utf-8')
try:
proc = exec_fetch(param, postdata, verbose, True)
except OSError as e:
logs.error(str(e))
logs.error("error running fdsnws_fetch")
return 1
net_desc = {}
for line in proc.stdout:
try:
if isinstance(line, bytes):
line = line.decode('utf-8')
if not line or line.startswith('#'):
continue
(code, desc, start) = line.split('|')[:3]
year = dateutil.parser.parse(start).year
except (ValueError, UnicodeDecodeError) as e:
logs.error("error parsing text format: %s" % str(e))
continue
if code[0] in "0123456789XYZ":
net_desc["%s_%d" % (code, year)] = desc
else:
net_desc[code] = desc
logs.notice("You received seismic waveform data from the following "
"network(s):")
for code in sorted(net_desc):
logs.notice("%s %s" % (code, net_desc[code]))
logs.notice("\nAcknowledgment is extremely important for network operators\n"
"providing open data. When preparing publications, please\n"
"cite the data appropriately. The FDSN service at\n\n"
" http://www.fdsn.org/networks/citation/?networks=%s\n\n"
"provides a helpful guide based on available network\n"
"Digital Object Identifiers.\n"
% "+".join(sorted(net_desc)))
except KeyError:
incNet = set()
incompleteResponse[netCode] = incNet
incNet.add(staCode)
except ValueError, e:
logs.error("%s: %s" % (f, str(e)))
for (netCode, restricted) in networkRestricted.iteritems():
inventory.setNetworkRestricted(netCode, restricted)
for (netCode, network) in inventory.networks.iteritems():
if netCode not in existingNetworks:
logs.notice("deleting network %s from inventory" %
(netCode,))
inventory.obj.remove(network.obj)
for ((netCode, staCode), station) in inventory.stations.iteritems():
if netCode in existingNetworks and (netCode, staCode) not in existingStations:
logs.notice("deleting station %s_%s from inventory" %
(netCode, staCode))
inventory.networks[netCode].obj.remove(station.obj)
if incompleteResponse:
logs.info(
"The following stations are missing full response data")
logs.info("Use dlsv2inv if needed")
# for netCode in sorted(incompleteResponse.keys()):
# logs.info("%s: %s" % (netCode, " ".join(sorted(list(incompleteResponse[netCode])))))
def run(self):
try:
seiscompRoot = self.commandline().unrecognizedOptions()[0]
sys.stderr.write("root directory: %s\n" % seiscompRoot)
try:
DCID = self.configGetString("datacenterID")
except:
logs.error("datacenterID not found in global.cfg")
return False
networkRestricted = {}
incompleteResponse = {}
global instdb
instdb = Instruments(DCID)
self.__load_file(loadGains,
os.path.join(seiscompRoot, "config", "gain.dlsv"))
# for backwards compatibility
self.__load_file(
loadGains, os.path.join(seiscompRoot, "config",
"gain.tab.out"))
self.__load_file(loadGains,
os.path.join(seiscompRoot, "config", "gain.tab"))
try:
self.__load_file(instdb.load_db,
os.path.join(seiscompRoot, "resp", "inst.db"))
self.__load_file(
instdb.load_sensor_attr,
os.path.join(seiscompRoot, "resp", "sensor_attr.csv"))
self.__load_file(
instdb.load_datalogger_attr,
os.path.join(seiscompRoot, "resp", "datalogger_attr.csv"))
except (IOError, NettabError) as e:
logs.error("fatal error: " + str(e))
return False
sc3Inv = seiscomp3.DataModel.Inventory()
inventory = InventoryWrapper(sc3Inv, DCID)
existingNetworks = set()
existingStations = set()
for f in glob.glob(os.path.join(seiscompRoot, "key", "network_*")):
try:
logs.debug("processing " + f)
netCode = f.split("/network_")[-1]
try:
kf = Keyfile(f)
except IOError as e:
logs.error(str(e))
continue
existingNetworks.add(netCode)
networkRestricted[netCode] = False
inventory.updateNetwork(netCode, kf)
except ValueError as e:
logs.error("%s: %s" % (f, str(e)))
for f in glob.glob(os.path.join(seiscompRoot, "key", "station_*")):
try:
logs.debug("processing " + f)
(netCode, staCode) = f.split("/station_")[-1].split('_', 1)
try:
kf = Keyfile(f)
except IOError as e:
logs.error(str(e))
continue
existingStations.add((netCode, staCode))
if netCode not in existingNetworks:
logs.warning(
"network %s does not exist, ignoring station %s" %
(netCode, staCode))
continue
if not hasattr(kf, "latitude") or not kf.latitude:
logs.warning("missing latitude for %s %s" %
(netCode, staCode))
continue
if not hasattr(kf, "longitude") or not kf.longitude:
logs.warning("missing longitude for %s %s" %
(netCode, staCode))
continue
if not hasattr(kf, "elevation") or not kf.elevation:
logs.warning("missing elevation for %s %s" %
(netCode, staCode))
continue
if not hasattr(kf, "depth1") or not kf.depth1:
logs.warning(
"missing depth of primary sensor for %s %s" %
(netCode, staCode))
continue
if decimal.Decimal(kf.latitude) == decimal.Decimal("0.0") and \
decimal.Decimal(kf.longitude) == decimal.Decimal("0.0"):
logs.warning("missing coordinates for %s %s" %
(netCode, staCode))
continue
if not hasattr(kf, "orientation1") or not kf.orientation1:
logs.warning(
"missing orientation of primary sensor for %s %s, using default"
% (netCode, staCode))
kf.orientation1 = "Z 0.0 -90.0; N 0.0 0.0; E 90.0 0.0"
if not hasattr(kf, "orientation2"):
kf.orientation2 = ""
if not hasattr(kf, "unit1") or not kf.unit1:
logs.warning(
"missing unit of primary sensor for %s %s, using M/S"
% (netCode, staCode))
kf.unit1 = "M/S"
if not hasattr(kf, "unit2"):
logs.warning(
"missing unit of secondary sensor for %s %s, using M/S**2"
% (netCode, staCode))
kf.unit2 = "M/S**2"
if not hasattr(kf, "type"):
kf.type = ""
restricted = False
# TODO: Make restricted part of the key file
if not inventory.updateStation(netCode, staCode,
restricted, kf):
try:
incNet = incompleteResponse[netCode]
except KeyError:
incNet = set()
incompleteResponse[netCode] = incNet
incNet.add(staCode)
except ValueError as e:
logs.error("%s: %s" % (f, str(e)))
for (netCode, restricted) in networkRestricted.items():
inventory.setNetworkRestricted(netCode, restricted)
for (netCode, network) in inventory.networks.items():
if netCode not in existingNetworks:
logs.notice("deleting network %s from inventory" %
(netCode, ))
inventory.obj.remove(network.obj)
for ((netCode, staCode), station) in inventory.stations.items():
if netCode in existingNetworks and (
netCode, staCode) not in existingStations:
logs.notice("deleting station %s_%s from inventory" %
(netCode, staCode))
inventory.networks[netCode].obj.remove(station.obj)
if incompleteResponse:
logs.info(
"The following stations are missing full response data")
logs.info("Use dlsv2inv if needed")
# for netCode in sorted(incompleteResponse.keys()):
# logs.info("%s: %s" % (netCode, " ".join(sorted(list(incompleteResponse[netCode])))))
tmpDict = sortDictionary(incompleteResponse)
for netCode in list(tmpDict.keys()):
tmpSortedList = list(tmpDict[netCode])
tmpSortedList.sort()
logs.info("%s: %s" % (netCode, " ".join(tmpSortedList)))
ar = seiscomp3.IO.XMLArchive()
if not self.output:
sys.stderr.write("Writing output to stdout\n")
if not ar.create("-"):
sys.stderr.write("Cannot open open stdout\n")
return False
else:
sys.stderr.write("Writing output to %s\n" % self.output)
if not ar.create(self.output):
sys.stderr.write("Cannot open open %s\n" % self.output)
return False
ar.setFormattedOutput(self.commandline().hasOption("formatted"))
ar.writeObject(sc3Inv)
except Exception:
logs.print_exc()
return True
except KeyError:
incNet = set()
incompleteResponse[netCode] = incNet
incNet.add(staCode)
except ValueError, e:
logs.error("%s: %s" % (f, str(e)))
for (netCode, restricted) in networkRestricted.iteritems():
inventory.setNetworkRestricted(netCode, restricted)
for (netCode, network) in inventory.networks.iteritems():
if netCode not in existingNetworks:
logs.notice("deleting network %s from inventory" % (netCode,))
inventory.obj.remove(network.obj)
for ((netCode, staCode), station) in inventory.stations.iteritems():
if netCode in existingNetworks and (netCode, staCode) not in existingStations:
logs.notice("deleting station %s_%s from inventory" % (netCode, staCode))
inventory.networks[netCode].obj.remove(station.obj)
if incompleteResponse:
logs.info("The following stations are missing full response data")
logs.info("Use dlsv2inv if needed")
#for netCode in sorted(incompleteResponse.keys()):
# logs.info("%s: %s" % (netCode, " ".join(sorted(list(incompleteResponse[netCode])))))
tmpDict = sortDictionary(incompleteResponse)
for netCode in tmpDict.keys():
if encrypted:
endung = endung + '.openssl'
return endung;
def main():
(SSLpasswordDict, addr, request_format, data_format, label, resp_dict, rebuild_volume, proxymode, user, timeout, retries, output_file, input_file, spfr) = process_options()
try:
ret = _main(SSLpasswordDict, addr, request_format, data_format, label, resp_dict, rebuild_volume, proxymode, user, timeout, retries, output_file, input_file, spfr)
except ArclinkError, e:
logs.error(str(e))
ret = 1
if addr.startswith("eida.gfz-potsdam.de:") or addr.startswith("webdc.eu:"):
logs.notice("\nin case of problems with your request, please contact [email protected]")
return ret
def _main(SSLpasswordDict, addr, request_format, data_format, label, resp_dict, rebuild_volume, proxymode, user, timeout, retries, output_file, input_file, spfr):
reblock_mseed = False
use_inventory = False
use_routing = not proxymode
req_args = {"compression": "bzip2"}
if data_format.upper() == "MSEED":
req_type = "WAVEFORM"
req_args["format"] = "MSEED"
elif data_format.upper() == "MSEED4K":
if encrypted:
endung = endung + '.openssl'
return endung;
def main():
(SSLpasswordDict, addr, request_format, data_format, label, resp_dict, rebuild_volume, proxymode, user, timeout, retries, output_file, input_file, spfr) = process_options()
try:
ret = _main(SSLpasswordDict, addr, request_format, data_format, label, resp_dict, rebuild_volume, proxymode, user, timeout, retries, output_file, input_file, spfr)
except ArclinkError, e:
logs.error(str(e))
ret = 1
if addr.startswith("eida.gfz-potsdam.de:") or addr.startswith("webdc.eu:"):
logs.notice("\nin case of problems with your request, please contact [email protected]")
return ret
def _main(SSLpasswordDict, addr, request_format, data_format, label, resp_dict, rebuild_volume, proxymode, user, timeout, retries, output_file, input_file, spfr):
reblock_mseed = False
use_inventory = False
use_routing = not proxymode
req_args = {"compression": "bzip2"}
if data_format.upper() == "MSEED":
req_type = "WAVEFORM"
req_args["format"] = "MSEED"
elif data_format.upper() == "MSEED4K":
# for key, value in environ.iteritems()])
# status = '200 OK'
# return send_plain_response(status, body, start_response)
logs.debug('Calling %s' % action)
try:
res_string = action(environ, parameters)
except PlsRedirect as redir:
return redirect_page(redir.url, start_response)
except WIError as error:
error_page = error.body
logs.notice('Error page %s: "%s"' % (error.status, error_page))
if error.verbosity > 1:
extra = 'VERBOSE verbose=%i' % error.verbosity
error_page += '\n' + extra
return send_plain_response(error.status, error_page, start_response)
if isinstance(res_string, basestring):
status = '200 OK'
body = res_string
return send_plain_response(status, body, start_response)
elif hasattr(res_string, 'filename'):
status = '200 OK'
body = res_string
return send_file_response(status, body, start_response)
# for key, value in environ.iteritems()])
# status = '200 OK'
# return send_plain_response(status, body, start_response)
logs.debug('Calling %s' % action)
try:
res_string = action(environ, parameters)
except PlsRedirect as redir:
return redirect_page(redir.url, start_response)
except WIError as error:
error_page = error.body
logs.notice('Error page %s: "%s"' % (error.status, error_page))
if error.verbosity > 1:
extra = 'VERBOSE verbose=%i' % error.verbosity
error_page += '\n' + extra
return send_plain_response(error.status, error_page, start_response)
if isinstance(res_string, basestring):
status = '200 OK'
body = res_string
return send_plain_response(status, body, start_response)
elif hasattr(res_string, 'filename'):
status = '200 OK'
body = res_string
return send_file_response(status, body, start_response)