def show_status(request): try: logs.info(" datacenter name: " + request.dcname) rqstat = request.status() except ArclinkError, e: logs.error(str(e)) return
def loadGains(fileName): try: fd = open(fileName) try: line = fd.readline() lineno = 0 while line: line = line.strip() lineno += 1 if not line or line[0] == '#': line = fd.readline() continue try: (deviceName, deviceIdPattern, streamPattern, gain) = line.split() if deviceName in gainTable: gainTable[deviceName].append((streamPattern, deviceIdPattern, float(gain))) else: gainTable[deviceName] = [ (streamPattern, deviceIdPattern, float(gain)) ] except (TypeError, ValueError): logs.error("%s:%d: parse error" % (fileName, lineno)) sys.exit(1) line = fd.readline() finally: fd.close() except IOError, e: logs.error("cannot open %s: %s" % (fileName, str(e))) sys.exit(1)
def __get_status(self, server, user, user_ip, req_id, start=0, count=100): try: (host, port) = server.split(':') port = int(port) except ValueError: logs.error("invalid server address in network XML: %s" % server) raise wsgicomm.WIInternalError, "invalid server address" try: arcl = Arclink() arcl.open_connection(host, port, user, user_ip=user_ip, timeout=self.status_timeout) try: #new version requires an arclink update to support pagination #status = arcl.get_status(req_id, start, count) status = arcl.get_status(req_id) status.request = status.request[:count] return status finally: arcl.close_connection() except (ArclinkError, socket.error) as e: raise wsgicomm.WIServiceError, str(e)
def parse_native(req, input_file): fd = open(input_file) try: rqline = fd.readline() while rqline: rqline = rqline.strip() if not rqline: rqline = fd.readline() logs.debug("skipping empty request line") continue rqsplit = rqline.split() if len(rqsplit) < 3: logs.error("invalid request line: '%s'" % (rqline, )) rqline = fd.readline() continue try: start_time = datetime.datetime( *map(int, rqsplit[0].split(","))) end_time = datetime.datetime(*map(int, rqsplit[1].split(","))) except ValueError, e: logs.error("syntax error (%s): '%s'" % (str(e), rqline)) rqline = fd.readline() continue network = rqsplit[2] station = "." channel = "." location = "." i = 3 if len(rqsplit) > 3 and rqsplit[3] != ".": station = rqsplit[3] i += 1 if len(rqsplit) > 4 and rqsplit[4] != ".": channel = rqsplit[4] i += 1 if len(rqsplit) > 5 and rqsplit[5] != ".": location = rqsplit[5] i += 1 while len(rqsplit) > i and rqsplit[i] == ".": i += 1 constraints = {} for arg in rqsplit[i:]: pv = arg.split('=', 1) if len(pv) != 2: raise ArclinkHandlerError, "invalid request syntax" constraints[pv[0]] = pv[1] req.add(network, station, channel, location, start_time, end_time, constraints) rqline = fd.readline() finally: fd.close()
def show_status(request): try: logs.info("datacenter name: " + request.dcname) rqstat = request.status() except ArclinkError, e: logs.error(str(e)) return
def parseDate(datestr): m = _rx_datetime.match(datestr) if not m: logs.error("invalid date: " + datestr) return (seiscomp3.Core.Time(1980, 1, 1, 0, 0, 0), "1980-01-01T00:00:00.0000Z") try: year = int(m.group("year")) (month, mday) = _dy2mdy(int(m.group("doy")), year) if m.group("hour"): hour = int(m.group("hour")) minute = int(m.group("minute")) else: hour = 0 minute = 0 if m.group("second"): second = int(m.group("second")) else: second = 0 coretime = seiscomp3.Core.Time(year, month, mday, hour, minute, second) except (TypeError, ValueError, IndexError): logs.error("invalid date: " + datestr) return (seiscomp3.Core.Time(1980, 1, 1, 0, 0, 0), "1980-01-01T00:00:00.0000Z") return (coretime, coretime.toString("%Y-%m-%dT%H:%M:%S.%fZ"))
def scan_cha(d): last_file = {} for f in os.listdir(d): try: (net, sta, loc, cha, ext, year, doy) = f.split('.') nets.add((net, int(year))) except ValueError: logs.error("invalid SDS file:" + p, True) continue if (net, sta, loc, cha) not in timespan: continue try: if doy > last_file[loc][0]: last_file[loc] = (doy, f) except KeyError: last_file[loc] = (doy, f) for (loc, (doy, f)) in last_file.items(): with open(d + '/' + f, 'rb') as fd: nslc = tuple(f.split('.')[:4]) rec = mseedlite.Record(fd) fd.seek(-rec.size, 2) rec = mseedlite.Record(fd) ts = timespan[nslc] if ts.start < rec.end_time < ts.end: ts.start = rec.end_time elif rec.end_time >= ts.end: del timespan[nslc]
def parseOrientation(orientation): for x in orientation.split(';'): try: (code, azimuth, dip) = x.split() yield (code, float(azimuth), float(dip)) except (TypeError, ValueError): logs.error("error parsing orientation %s at %s" % (orientation, x)) continue
def run(self): try: if self.dcid is None: print("Please specify datacenter/archive ID", file=sys.stderr) return False nettab = Nettab(self.dcid) instdb = Instruments(self.dcid) try: self.__load_file(instdb.load_db, self.inst_db_file) self.__load_file(nettab.load_statmap, self.stat_map_file) self.__load_file(nettab.load_access_net, self.access_net_file) self.__load_file(nettab.load_access_stat, self.access_stat_file) self.__load_file(instdb.load_sensor_attr, self.sensor_attr_file) self.__load_file(instdb.load_datalogger_attr, self.datalogger_attr_file) self.__load_file(nettab.load_network_attr, self.network_attr_file) self.__load_file(nettab.load_station_attr, self.station_attr_file) inv = SC3Inventory(seiscomp.datamodel.Inventory()) idx = 1 for tab in sorted(self.tab_files): print("Loading %s (%d/%d)" % (tab, idx, len(self.tab_files)), file=sys.stderr) self.__load_file(nettab.load_tab, tab) print("Generating data structures", file=sys.stderr) nettab.update_inventory(instdb, inv) idx = idx + 1 if self.isExitRequested(): print("Exit requested, abort", file=sys.stderr) return False print("Generating output", file=sys.stderr) ar = seiscomp.io.XMLArchive() ar.setFormattedOutput( self.commandline().hasOption("formatted")) ar.create(self.out_file) ar.writeObject(inv.obj) ar.close() print("Finished", file=sys.stderr) except (IOError, NettabError) as e: logs.error("fatal error: " + str(e)) return False except Exception: logs.print_exc() return False return True
def parse_breqfast(req, input_file): parser = BreqParser() parser.parse_email(input_file) req.content = parser.reqlist logs.debug("") if parser.failstr: logs.error(parser.failstr) else: logs.info("parsed %d lines from breqfast message" % len(req.content))
def main(): (SSLpasswordDict, addr, request_format, data_format, label, resp_dict, rebuild_volume, proxymode, user, timeout, retries, output_file, input_file, spfr) = process_options() try: ret = _main(SSLpasswordDict, addr, request_format, data_format, label, resp_dict, rebuild_volume, proxymode, user, timeout, retries, output_file, input_file, spfr) except ArclinkError, e: logs.error(str(e)) ret = 1
def parse_breqfast_from_handler(req, fh): parser = BreqParser() parser.parse_email_from_handler(fh) req.content = parser.reqlist logs.debug("") if parser.failstr: logs.error(parser.failstr) else: logs.info("parsed %d lines from breqfast message" % len(req.content))
def next(self): while True: try: return Record(self.__fd) except MSeedError, e: logs.error(str(e)) except MSeedNoData: pass
def next(self): while True: try: return Record(self.__fd) except MSeedNoData: pass except MSeedError, e: logs.error(str(e))
def getConfigFloat(self, name, default=None): try: return float(self.__cfg.getString(name)) except ValueError: logs.error("config parameter '%s' has invalid value" % name) return default #except seiscomp3.Config.OptionNotFoundException: except Exception, e: return default
def getConfigFloat(self, name, default = None): try: return float(self.__cfg.getString(name)) except ValueError: logs.error("config parameter '%s' has invalid value" % name) return default #except seiscomp3.Config.OptionNotFoundException: except Exception, e: return default
def parse_native_from_handler(req, fd): rqline = fd.readline() while rqline: rqline = rqline.strip() if not rqline: rqline = fd.readline() logs.debug("skipping empty request line") continue rqsplit = rqline.split() if len(rqsplit) < 3: logs.error("invalid request line: '%s'" % (rqline,)) rqline = fd.readline() continue try: start_time = datetime.datetime(*map(int, rqsplit[0].split(","))) end_time = datetime.datetime(*map(int, rqsplit[1].split(","))) except ValueError, e: logs.error("syntax error (%s): '%s'" % (str(e), rqline)) rqline = fd.readline() continue network = rqsplit[2] station = "." channel = "." location = "." i = 3 if len(rqsplit) > 3 and rqsplit[3] != ".": station = rqsplit[3] i += 1 if len(rqsplit) > 4 and rqsplit[4] != ".": channel = rqsplit[4] i += 1 if len(rqsplit) > 5 and rqsplit[5] != ".": location = rqsplit[5] i += 1 while len(rqsplit) > i and rqsplit[i] == ".": i += 1 constraints = {} for arg in rqsplit[i:]: pv = arg.split('=', 1) if len(pv) != 2: raise ArclinkError, "invalid request syntax" constraints[pv[0]] = pv[1] req.add(network, station, channel, location, start_time, end_time, constraints) rqline = fd.readline()
def __parse_arglist(self, arglist): d = {} for arg in arglist: pv = arg.split('=', 1) if len(pv) != 2: logs.error("invalid request args in status: " + args) continue d[pv[0]] = pv[1] return d
def request_purge(self, envir, params): """Delete one user request at a given server. Input: server server DCID user user ID request request ID Output: true """ dcid = params.get("server") user = params.get("user") req_id = params.get("request") if dcid is None: raise wsgicomm.WIClientError, "missing server" else: try: server = self.nodes[dcid]['address'] except KeyError: raise wsgicomm.WIClientError, "invalid server" if user is None: raise wsgicomm.WIClientError, "missing user ID" if req_id is None: raise wsgicomm.WIClientError, "missing request ID" try: (host, port) = server.split(':') port = int(port) except ValueError: logs.error("invalid server address in network XML: %s" % server) raise wsgicomm.WIInternalError, "invalid server address" try: arcl = Arclink() arcl.open_connection(host, port, user) except (ArclinkError, socket.error) as e: raise wsgicomm.WIServiceError, str(e) try: arcl.purge(req_id) return json.dumps(True) except (ArclinkError, socket.error) as e: arcl.close_connection() raise wsgicomm.WIServiceError, str(e)
def update(self, rate, rateDiv, orientation, datalogger, dataloggerId, seismometer, seismometerId, channel, depth, azimuth, dip, gainFreq, gainMult, gainUnit, format, restricted, inv): errmsg = None try: (smsn, smgain) = (seismometerId.split('/') + [None])[:2] (datalogger, dlgain, seismometer, smgain, gainFreq, gainUnit) = instdb.update_inventory(inv, datalogger, dataloggerId, gainMult, seismometer, smsn, smgain, rate, rateDiv) except NettabError as e: errmsg = str(e) dlgain = [] smgain = [] self.obj.setSampleRateNumerator(rate) self.obj.setSampleRateDenominator(rateDiv) self.obj.setDatalogger(datalogger) self.obj.setDataloggerSerialNumber(dataloggerId) self.obj.setDataloggerChannel(channel) self.obj.setSensor(seismometer) self.obj.setSensorSerialNumber(seismometerId) self.obj.setSensorChannel(channel) self.obj.setDepth(depth) self.obj.setAzimuth(azimuth) self.obj.setDip(dip) self.obj.setGainFrequency(gainFreq) self.obj.setGainUnit(gainUnit) self.obj.setFormat(format) self.obj.setFlags("GC") self.obj.setRestricted(restricted) self.obj.setShared(True) complete = True try: gain = smgain[channel] * dlgain[channel] except IndexError: complete = False gain = gainMult * getGain(datalogger, dataloggerId, seismometer, seismometerId, self.obj.code()) if gain == 0 and errmsg: logs.error(errmsg) self.obj.setGain(gain) return complete
def log_print(s): if logstream: logstream.write(s + "\n") else: try: fh = file(fname, "a") try: fh.write(s + "\n") finally: fh.close() except OSError: logs.error("Log file %s could not be opened!" % fname)
def parsePkgstr(pkgstr): result = {} for x in pkgstr.split(): m = _rx_pkg.match(x) if not m: logs.error("error parsing %s at %s" % (pkgstr, x)) continue result[m.group('pkg')] = m.group('profile') if 'trunk' not in result: result['trunk'] = None return result
def getStreamInfo(self, start_time, end_time, net, sta, cha, loc): try: stream_epochs = self.streamidx[(net, sta, cha, loc)] except KeyError: logs.error("%s,%s,%s,%s not found" % (net, sta, cha, loc)) return None for stream in stream_epochs: try: station = self.stations[self.sensorsLoc[stream[0]][0]] except IndexError: logs.error("cache inconsistency") return None # stream_start = datetime.datetime(station[8], 1, 1) # stream_end = datetime.datetime(station[9], 1, 1) if station[9] \ # else datetime.datetime(2030, 1, 1) stream_start = stream[6] stream_end = stream[7] if stream[7] is not None \ else (datetime.datetime.now() + datetime.timedelta(days=365)) if start_time >= stream_end or end_time <= stream_start: continue result = { 'latitude': station[5], 'longitude': station[6], 'elevation': station[10] } if stream[3] != 0: tdiff = end_time - start_time tdiff = tdiff.days * 86400 + tdiff.seconds samp = float(stream[4]) / float(stream[3]) # assuming approximately 1 byte per sample (compressed), # 512 bytes record size bytesper = 1 recsize = 512 result['size'] = int( recsize * math.ceil(float(tdiff * samp * bytesper) / recsize)) else: result['size'] = 0 return result return None
def run(self): try: if self.dcid is None: print >>sys.stderr, "Please specify datacenter/archive ID" return False nettab = Nettab(self.dcid) instdb = Instruments(self.dcid) try: self.__load_file(instdb.load_db, self.inst_db_file) self.__load_file(nettab.load_statmap, self.stat_map_file) self.__load_file(nettab.load_access_net, self.access_net_file) self.__load_file(nettab.load_access_stat, self.access_stat_file) self.__load_file(instdb.load_sensor_attr, self.sensor_attr_file) self.__load_file(instdb.load_datalogger_attr, self.datalogger_attr_file) self.__load_file(nettab.load_network_attr, self.network_attr_file) self.__load_file(nettab.load_station_attr, self.station_attr_file) inv = SC3Inventory(DataModel.Inventory()) idx = 1 for tab in sorted(self.tab_files): print >>sys.stderr, "Loading %s (%d/%d)" % (tab, idx, len(self.tab_files)) self.__load_file(nettab.load_tab, tab) print >>sys.stderr, "Generating data structures" nettab.update_inventory(instdb, inv) idx = idx + 1 if self.isExitRequested(): print >>sys.stderr, "Exit requested, abort" return False print >>sys.stderr, "Generating output" ar = IO.XMLArchive() ar.setFormattedOutput(self.commandline().hasOption("formatted")) ar.create(self.out_file) ar.writeObject(inv.obj) ar.close() print >>sys.stderr, "Finished" except (IOError, NettabError), e: logs.error("fatal error: " + str(e)) return False except Exception: logs.print_exc() return False return True
def __load_module(self, path): modname = os.path.splitext(os.path.basename(path))[0].replace('.', '_') if modname in self.__modules: logs.error("'%s' is already loaded!" % modname) return try: mod = imp.load_source('__wi_' + modname, path) except: logs.error("Error loading '%s'" % modname) logs.print_exc() return self.__modules[modname] = mod.WI_Module(self)
def getStreamInfo(self, start_time, end_time, net, sta, cha, loc): try: stream_epochs = self.streamidx[(net, sta, cha, loc)] except KeyError: logs.error("%s,%s,%s,%s not found" % (net, sta, cha, loc)) return None for stream in stream_epochs: try: station = self.stations[self.sensorsLoc[stream[0]][0]] except IndexError: logs.error("cache inconsistency") return None # stream_start = datetime.datetime(station[8], 1, 1) # stream_end = datetime.datetime(station[9], 1, 1) if station[9] \ # else datetime.datetime(2030, 1, 1) stream_start = stream[6] stream_end = stream[7] if stream[7] is not None \ else (datetime.datetime.now() + datetime.timedelta(days=365)) if start_time >= stream_end or end_time <= stream_start: continue result = {'latitude': station[5], 'longitude': station[6], 'elevation': station[10]} if stream[3] != 0: tdiff = end_time - start_time tdiff = tdiff.days * 86400 + tdiff.seconds samp = float(stream[4]) / float(stream[3]) # assuming approximately 1 byte per sample (compressed), # 512 bytes record size bytesper = 1 recsize = 512 result['size'] = int(recsize * math.ceil( float(tdiff * samp * bytesper) / recsize)) else: result['size'] = 0 return result return None
def __init__(self, source): if isinstance(source, basestring): fp = open(source) elif hasattr(source, "read"): fp = source else: raise TypeError, "invalid source" try: filename = fp.name except AttributeError: filename = '<???>' try: lineno = 0 while True: line = fp.readline() if not line: break lineno = lineno + 1 line = line.strip() if not line or line.startswith("#"): continue m = _rx_keyline.match(line.strip()) if m: s = m.group('p').replace('__', '.').split('_') if s[0] == '': s[0] = '_' k = reduce(lambda x, y: x + y[0] + y[1:].lower(), s[1:], s[0].lower()) if m.group('q') == '"': self.__dict__[k.replace('.', '_')] = m.group('v').replace( r'\"', r'"') else: self.__dict__[k.replace('.', '_')] = m.group('v') else: logs.error("%s:%d: parse error" % (filename, lineno)) finally: if fp is not source: fp.close()
def close(self): try: try: wfd = _WaveformData() self.__mseed_fd.seek(0) for rec in MSeedInput(self.__mseed_fd): wfd.add_data(rec) wfd.output_data(self.__fd, 0) except (MSeedError, SEEDError, DBError), e: logs.error("error reblocking Mini-SEED data: " + str(e)) finally: self.__mseed_fd.close() self.__fd.close()
def close(self): try: try: seed_volume = SEEDVolume(self.__inv, ORGANIZATION, LABEL, self.__resp_dict) self.__mseed_fd.seek(0) for rec in mseed.Input(self.__mseed_fd): seed_volume.add_data(rec) seed_volume.output(self.__fd) except (mseed.MSeedError, SEEDError, DBError), e: logs.error("error creating SEED volume: " + str(e)) finally: self.__mseed_fd.close() self.__fd.close()
def iterdata(self, time1, time2, net, sta, cha, loc): lasttime = None lastrecord = None recstream = [] if self.__is_iso(time1, time2, net, sta, os.path.exists): recstream.append(("isoarchive", self.isodir)) if self.__is_sds(time1, time2, net, sta, cha, loc, self.archdir, os.path.exists, os.listdir): recstream.append(("sdsarchive", self.archdir)) if self.__is_sds(time1, time2, net, sta, cha, loc, self.nrtdir, os.path.exists, os.listdir): recstream.append(("sdsarchive", self.nrtdir)) if not recstream and self.exists_db(time1, time2, net, sta, cha, loc): raise TemporaryUnavailabilityException for (service, source) in recstream: if lastrecord: try: etime = lastrecord.endTime() except Core.ValueException: logs.warning("SDS: record.endTime() raises Core.ValueException! Resulting SEED file maybe incorrect!") etime = lastrecord.startTime() timetuple = time.strptime(etime.toString("%Y-%m-%d %H:%M:%S"), "%Y-%m-%d %H:%M:%S") lasttime = datetime.datetime(*timetuple[:6])+datetime.timedelta(seconds=1) # avoids dublettes if lasttime >= time2: break time1 = lasttime lastrecord = None self._recstream = IO.RecordStream.Create(service) if not self._recstream: logs.error("Could not fetch recordstream service '%s'" % service) raise StopIteration if not self._recstream.setSource(source): logs.error("Could not set recordstream source '%s'" % source) self._recstream = None raise StopIteration logs.debug("%s %s: addStream for %s-%s" % (service, source, str(time1), str(time2))) self._recstream.addStream(net,sta,loc,cha,Core.Time.FromString(str(time1),"%Y-%m-%d %H:%M:%S"), Core.Time.FromString(str(time2),"%Y-%m-%d %H:%M:%S")) try: recinput = IO.RecordInput(self._recstream, Core.Array.DOUBLE, Core.Record.SAVE_RAW) record = recinput.next() while record: yield record.raw().str() lastrecord = record record = recinput.next() except Core.GeneralException, e: logs.error(e.what()) except Exception, e: logs.error("SDS: Unexpected exception occured: %s" % e)
def run(self): try: seiscompRoot = self.commandline().unrecognizedOptions()[0] sys.stderr.write("root directory: %s\n" % seiscompRoot) try: DCID = self.configGetString("datacenterID") except: logs.error("datacenterID not found in global.cfg") return False networkRestricted = {} incompleteResponse = {} global instdb instdb = Instruments(DCID) self.__load_file(loadGains, os.path.join(seiscompRoot, "config", "gain.dlsv")) # for backwards compatibility self.__load_file(loadGains, os.path.join(seiscompRoot, "config", "gain.tab.out")) self.__load_file(loadGains, os.path.join(seiscompRoot, "config", "gain.tab")) try: self.__load_file(instdb.load_db, os.path.join(seiscompRoot, "resp", "inst.db")) self.__load_file(instdb.load_sensor_attr, os.path.join(seiscompRoot, "resp", "sensor_attr.csv")) self.__load_file(instdb.load_datalogger_attr, os.path.join(seiscompRoot, "resp", "datalogger_attr.csv")) except (IOError, NettabError), e: logs.error("fatal error: " + str(e)) return False sc3Inv = seiscomp3.DataModel.Inventory() inventory = InventoryWrapper(sc3Inv, DCID) existingNetworks = set() existingStations = set() for f in glob.glob(os.path.join(seiscompRoot, "key", "network_*")): try: logs.debug("processing " + f) netCode = f.split("/network_")[-1] try: kf = Keyfile(f) except IOError, e: logs.error(str(e)) continue existingNetworks.add(netCode) networkRestricted[netCode] = False inventory.updateNetwork(netCode, kf) except ValueError, e: logs.error("%s: %s" % (f, str(e)))
def close(self): try: try: seed_volume = SEEDVolume(self.__inv, ORGANIZATION, LABEL, self.__resp_dict) self.__mseed_fd.seek(0) for rec in MSeedInput(self.__mseed_fd): seed_volume.add_data(rec) seed_volume.output(self.__fd) except (MSeedError, SEEDError, DBError), e: logs.error("error creating SEED volume: " + str(e)) finally: self.__mseed_fd.close() self.__fd.close()
def __load_nodelist(self, network_xml): for f in network_xml: try: tree = ET.parse(f) except Exception as e: logs.error("could not parse %s: %s" % (f, str(e))) continue root = tree.getroot() for e in root.findall('./node'): try: dcid = e.attrib['dcid'] name = e.attrib['name'] addr = e.attrib['address'] + ':' + e.attrib['port'] self.nodes[dcid] = {'name': name, 'address': addr} self.nodeaddr[addr] = dcid except KeyError: logs.error("invalid node element in %s" % (f, ))
def getGain(datalogger, dataloggerId, seismometer, seismometerId, streamCode): try: if datalogger == "DUMMY": dataloggerGain = 1.0 else: for (streamPattern, dataloggerIdPattern, dataloggerGain) in gainTable[datalogger]: if fnmatch.fnmatch(streamCode, streamPattern) and \ fnmatch.fnmatch(dataloggerId, dataloggerIdPattern): break else: logs.error("cannot find gain for %s, %s, %s" % (datalogger, dataloggerId, streamCode)) return 0 if seismometer == "DUMMY": seismometerGain = 1.0 else: for (streamPattern, seismometerIdPattern, seismometerGain) in gainTable[seismometer]: if fnmatch.fnmatch(streamCode, streamPattern) and \ fnmatch.fnmatch(seismometerId, seismometerIdPattern): break else: logs.error("cannot find gain for %s, %s, %s" % (seismometer, seismometerId, streamCode)) return 0 except KeyError, e: logs.error("cannot find gain for " + str(e)) return 0
def get_citation(nets, param, verbose): postdata = "" for (net, year) in nets: postdata += "%s * * * %d-01-01T00:00:00Z %d-12-31T23:59:59Z\n" \ % (net, year, year) if not isinstance(postdata, bytes): postdata = postdata.encode('utf-8') try: proc = exec_fetch(param, postdata, verbose, True) except OSError as e: logs.error(str(e)) logs.error("error running fdsnws_fetch") return 1 net_desc = {} for line in proc.stdout: try: if isinstance(line, bytes): line = line.decode('utf-8') if not line or line.startswith('#'): continue (code, desc, start) = line.split('|')[:3] year = dateutil.parser.parse(start).year except (ValueError, UnicodeDecodeError) as e: logs.error("error parsing text format: %s" % str(e)) continue if code[0] in "0123456789XYZ": net_desc["%s_%d" % (code, year)] = desc else: net_desc[code] = desc logs.notice("You received seismic waveform data from the following " "network(s):") for code in sorted(net_desc): logs.notice("%s %s" % (code, net_desc[code])) logs.notice("\nAcknowledgment is extremely important for network operators\n" "providing open data. When preparing publications, please\n" "cite the data appropriately. The FDSN service at\n\n" " http://www.fdsn.org/networks/citation/?networks=%s\n\n" "provides a helpful guide based on available network\n" "Digital Object Identifiers.\n" % "+".join(sorted(net_desc)))
def parseSampling(sampling): compressionLevel = "2" instrumentCode = "H" locationCode = "" endPreamble = sampling.find('_') if endPreamble > 0: for x in sampling[:endPreamble].split('/'): if x[0] == 'F': compressionLevel = x[1:] elif x[0] == 'L': locationCode = x[1:] elif x[0] == 'T': instrumentCode = x[1:] else: logs.warning("unknown code %s in %s" % (x[0], sampling)) if not sampling[endPreamble+1:]: return for x in sampling[endPreamble+1:].split('/'): m = _rx_samp.match(x) if not m: logs.error("error parsing sampling %s at %s" % (sampling, x)) continue try: sampleRate = decimal.Decimal(m.group('sampleRate')) except decimal.InvalidOperation: logs.error("error parsing sampling %s at %s" % (sampling, x)) continue bandCode = m.group('bandCode') if not bandCode: if sampleRate >= 80: bandCode = 'H' elif sampleRate >= 40: bandCode = 'S' elif sampleRate > 1: bandCode = 'B' elif sampleRate == 1: bandCode = 'L' elif sampleRate == decimal.Decimal("0.1"): bandCode = 'V' elif sampleRate == decimal.Decimal("0.01"): bandCode = 'U' else: logs.error( "could not determine band code for %s in %s" (x, sampling)) continue yield ((bandCode + instrumentCode, locationCode) + _rational(sampleRate) + ("Steim" + compressionLevel,))
def __parse_req_line(self, data): rqline = str(data).strip() if not rqline: logs.error("empty request line") return None rqsplit = rqline.split() if len(rqsplit) < 3: logs.error("invalid request line: %s" % (rqline, )) return None try: start_time = datetime.datetime(*map(int, rqsplit[0].split(","))) end_time = datetime.datetime(*map(int, rqsplit[1].split(","))) except ValueError as e: logs.error("syntax error (%s): '%s'" % (str(e), rqline)) return None network = rqsplit[2] station = "." channel = "." location = "." i = 3 if len(rqsplit) > 3 and rqsplit[3] != ".": station = rqsplit[3] i += 1 if len(rqsplit) > 4 and rqsplit[4] != ".": channel = rqsplit[4] i += 1 if len(rqsplit) > 5 and rqsplit[5] != ".": location = rqsplit[5] i += 1 while len(rqsplit) > i and rqsplit[i] == ".": i += 1 constraints = self.__parse_arglist(rqsplit[i:]) return RequestLine(start_time, end_time, network, station, channel, location)
def request_download(self, envir, params): """Download data. Input: server server DCID user user ID request request ID volume volume ID (optional) Output: iterable datastream. """ dcid = params.get("server") user = params.get("user") req_id = params.get("request") vol_id = params.get("volume") if dcid is None: raise wsgicomm.WIClientError, "missing server" else: try: server = self.nodes[dcid]['address'] except KeyError: raise wsgicomm.WIClientError, "invalid server" if user is None: raise wsgicomm.WIClientError, "missing user ID" if req_id is None: raise wsgicomm.WIClientError, "missing request ID" try: (host, port) = server.split(':') port = int(port) except ValueError: logs.error("invalid server address in network XML: %s" % server) raise wsgicomm.WIInternalError, "invalid server address" user_ip = envir.get('REMOTE_ADDR') try: arcl = Arclink() arcl.open_connection(host, port, user, user_ip=user_ip, timeout=self.download_timeout) except (ArclinkError, socket.error) as e: raise wsgicomm.WIServiceError, str(e) try: status = arcl.get_status(req_id) meta = self.__get_meta(status, dcid, req_id, vol_id) if meta is None: arcl.close_connection() raise wsgicomm.WIServiceError, "request is not downloadable" it = arcl.iterdownload(req_id, vol_id, raw=True) it.filename = meta[0] it.content_type = meta[1] return it except (ArclinkError, socket.error) as e: arcl.close_connection() raise wsgicomm.WIServiceError, str(e)