def show_lines(request): for (network, station, stream, loc_id, begin, end, constraints, blacklist) in request.content: logs.warning("%d,%d,%d,%d,%d,%d %d,%d,%d,%d,%d,%d %s %s %s %s %s" % \ (begin.year, begin.month, begin.day, begin.hour, begin.minute, begin.second, end.year, end.month, end.day, end.hour, end.minute, end.second, network, station, stream, loc_id, " ".join([ "%s=%s" % (a, v) for a, v in constraints.iteritems() ])))
def __iter_data(self, size, password, raw): bytes_read = 0 self.__errstate = False decompressor = None decryptor = None decStatus = None encStatus = None firstBlock = True try: exc = None try: while bytes_read < size: buf = self.__fd.read(min(BLOCKSIZE, size - bytes_read)) bytes_read += len(buf) if firstBlock: firstBlock = False (decryptor, encStatus) = self.__getDecryptor(buf, password) # Find out if data is encrypted if raw: decryptor = None if decryptor is not None: buf = decryptor.update(buf) (decompressor, decStatus) = self.__getDecompressor(buf) # Find out if data is compressed if raw: decompressor = None if decompressor is not None: buf = decompressor.decompress(buf) else: if decryptor is not None: buf = decryptor.update(buf) if decompressor is not None: buf = decompressor.decompress(buf) yield buf if decryptor is not None: buf = decryptor.final() if decompressor is not None and len(buf) > 0: buf = decompressor.decompress(buf) yield buf r = self.__fd.readline(LINESIZE).rstrip() if r != "END": raise ArclinkError, "END not found" except Exception, e: logs.warning("Download error: %s" % (str(e))) if decryptor is not None: logs.warning("Possible wrong password (%s)." % (password)) raise ArclinkError, "decrypt error." except Exception, e: exc = e
def iterdata(self, time1, time2, net, sta, cha, loc): lasttime = None lastrecord = None recstream = [] if self.__is_iso(time1, time2, net, sta, os.path.exists): recstream.append(("isoarchive", self.isodir)) if self.__is_sds(time1, time2, net, sta, cha, loc, self.archdir, os.path.exists, os.listdir): recstream.append(("sdsarchive", self.archdir)) if self.__is_sds(time1, time2, net, sta, cha, loc, self.nrtdir, os.path.exists, os.listdir): recstream.append(("sdsarchive", self.nrtdir)) if not recstream and self.exists_db(time1, time2, net, sta, cha, loc): raise TemporaryUnavailabilityException for (service, source) in recstream: if lastrecord: try: etime = lastrecord.endTime() except Core.ValueException: logs.warning("SDS: record.endTime() raises Core.ValueException! Resulting SEED file maybe incorrect!") etime = lastrecord.startTime() timetuple = time.strptime(etime.toString("%Y-%m-%d %H:%M:%S"), "%Y-%m-%d %H:%M:%S") lasttime = datetime.datetime(*timetuple[:6])+datetime.timedelta(seconds=1) # avoids dublettes if lasttime >= time2: break time1 = lasttime lastrecord = None self._recstream = IO.RecordStream.Create(service) if not self._recstream: logs.error("Could not fetch recordstream service '%s'" % service) raise StopIteration if not self._recstream.setSource(source): logs.error("Could not set recordstream source '%s'" % source) self._recstream = None raise StopIteration logs.debug("%s %s: addStream for %s-%s" % (service, source, str(time1), str(time2))) self._recstream.addStream(net,sta,loc,cha,Core.Time.FromString(str(time1),"%Y-%m-%d %H:%M:%S"), Core.Time.FromString(str(time2),"%Y-%m-%d %H:%M:%S")) try: recinput = IO.RecordInput(self._recstream, Core.Array.DOUBLE, Core.Record.SAVE_RAW) record = recinput.next() while record: yield record.raw().str() lastrecord = record record = recinput.next() except Core.GeneralException, e: logs.error(e.what()) except Exception, e: logs.error("SDS: Unexpected exception occured: %s" % e)
def parseSampling(sampling): compressionLevel = "2" instrumentCode = "H" locationCode = "" endPreamble = sampling.find('_') if endPreamble > 0: for x in sampling[:endPreamble].split('/'): if x[0] == 'F': compressionLevel = x[1:] elif x[0] == 'L': locationCode = x[1:] elif x[0] == 'T': instrumentCode = x[1:] else: logs.warning("unknown code %s in %s" % (x[0], sampling)) if not sampling[endPreamble+1:]: return for x in sampling[endPreamble+1:].split('/'): m = _rx_samp.match(x) if not m: logs.error("error parsing sampling %s at %s" % (sampling, x)) continue try: sampleRate = decimal.Decimal(m.group('sampleRate')) except decimal.InvalidOperation: logs.error("error parsing sampling %s at %s" % (sampling, x)) continue bandCode = m.group('bandCode') if not bandCode: if sampleRate >= 80: bandCode = 'H' elif sampleRate >= 40: bandCode = 'S' elif sampleRate > 1: bandCode = 'B' elif sampleRate == 1: bandCode = 'L' elif sampleRate == decimal.Decimal("0.1"): bandCode = 'V' elif sampleRate == decimal.Decimal("0.01"): bandCode = 'U' else: logs.error( "could not determine band code for %s in %s" (x, sampling)) continue yield ((bandCode + instrumentCode, locationCode) + _rational(sampleRate) + ("Steim" + compressionLevel,))
def parseSampling(sampling): compressionLevel = "2" instrumentCode = "H" locationCode = "" endPreamble = sampling.find('_') if endPreamble > 0: for x in sampling[:endPreamble].split('/'): if x[0] == 'F': compressionLevel = x[1:] elif x[0] == 'L': locationCode = x[1:] elif x[0] == 'T': instrumentCode = x[1:] else: logs.warning("unknown code %s in %s" % (x[0], sampling)) if not sampling[endPreamble+1:]: return for x in sampling[endPreamble+1:].split('/'): m = _rx_samp.match(x) if not m: logs.error("error parsing sampling %s at %s" % (sampling, x)) continue try: sampleRate = decimal.Decimal(m.group('sampleRate')) except decimal.InvalidOperation: logs.error("error parsing sampling %s at %s" % (sampling, x)) continue bandCode = m.group('bandCode') if not bandCode: if sampleRate >= 80: bandCode = 'H' elif sampleRate >= 40: bandCode = 'S' elif sampleRate > 1: bandCode = 'B' elif sampleRate == 1: bandCode = 'L' elif sampleRate == decimal.Decimal("0.1"): bandCode = 'V' elif sampleRate == decimal.Decimal("0.01"): bandCode = 'U' else: logs.error("could not determine band code for %s in %s" (x, sampling)) continue yield (( bandCode + instrumentCode, locationCode ) + _rational(sampleRate) + ("Steim" + compressionLevel,))
def end_request(self): try: if self.__request_size > self.__max_size: ex = "maximum request size exceeded" if self.__local_volume_fd is not None: if self.__local_volume_size == 0: self.__local_volume_fd.close(Arclink_ERROR(message=ex)) else: self.__local_volume_fd.close(Arclink_WARN(message=ex)) for (dcid, req) in self.__foreign_req.iteritems(): fd = self.__volume_factory.open(dcid) fd.close(Arclink_ERROR(message=ex)) return if self.__local_volume_fd is not None: if self.__local_volume_size == 0: self.__local_volume_fd.close(Arclink_NODATA) else: self.__local_volume_fd.close(Arclink_OK) dcid_req_pending = [] for (dcid, req) in self.__foreign_req.iteritems(): addr = self.__subnode_addr.get(dcid) if addr is None: logs.warning( "error submitting request to %s: routing failed" % (dcid, )) fd = self.__volume_factory.open(dcid) fd.close(Arclink_ERROR(message="routing failed")) continue thr = RequestThread(req, dcid, addr, self.__volume_factory) thr.start() dcid_req_pending.append(thr) for thr in dcid_req_pending: thr.join() finally: self.__volume_factory = None self.__local_volume_fd = None self.__local_volume_size = 0 self.__request_size = 0 self.__foreign_req = {}
def end_request(self): try: if self.__request_size > self.__max_size: ex = "maximum request size exceeded" if self.__local_volume_fd is not None: if self.__local_volume_size == 0: self.__local_volume_fd.close(Arclink_ERROR(message=ex)) else: self.__local_volume_fd.close(Arclink_WARN(message=ex)) for (dcid, req) in self.__foreign_req.iteritems(): fd = self.__volume_factory.open(dcid) fd.close(Arclink_ERROR(message=ex)) return if self.__local_volume_fd is not None: if self.__local_volume_size == 0: self.__local_volume_fd.close(Arclink_NODATA) else: self.__local_volume_fd.close(Arclink_OK) dcid_req_pending = [] for (dcid, req) in self.__foreign_req.iteritems(): addr = self.__subnode_addr.get(dcid) if addr is None: logs.warning("error submitting request to %s: routing failed" % (dcid,)) fd = self.__volume_factory.open(dcid) fd.close(Arclink_ERROR(message="routing failed")) continue thr = RequestThread(req, dcid, addr, self.__volume_factory) thr.start() dcid_req_pending.append(thr) for thr in dcid_req_pending: thr.join() finally: self.__volume_factory = None self.__local_volume_fd = None self.__local_volume_size = 0 self.__request_size = 0 self.__foreign_req = {}
def update(self, DCID, restricted, kf, inv): self.obj.setDescription(kf.statDesc) self.obj.setLatitude(float(kf.latitude)) self.obj.setLongitude(float(kf.longitude)) self.obj.setElevation(float(kf.elevation)) self.obj.setType(kf.type) self.obj.setRestricted(restricted) self.obj.setShared(True) self.obj.setArchive(DCID) self.__usedStreams = set() complete = self.__updateStreams(kf.sampling1, kf.orientation1, kf.datalogger, kf.dataloggerSn, kf.seismometer1, kf.seismometerSn1, float(kf.depth1), 0.02, float(kf.gainMult1), kf.unit1, kf.startDate, restricted, inv) if kf.sampling2: if not kf.depth2: logs.warning("missing depth of secondary sensor for %s %s" % (self.netCode, self.obj.code())) return if not hasattr(kf, "datalogger2") or not kf.datalogger2: kf.datalogger2 = kf.datalogger if not hasattr(kf, "dataloggerSn2") or not kf.dataloggerSn2: kf.dataloggerSn2 = kf.dataloggerSn complete = self.__updateStreams( kf.sampling2, kf.orientation2, kf.datalogger2, kf.dataloggerSn2, kf.seismometer2, kf.seismometerSn2, float(kf.depth2), 1.0, float(kf.gainMult2), kf.unit2, kf.startDate, restricted, inv) and complete for (locCode, streamCode) in set(self.streams.keys()) - self.__usedStreams: self.__loc[locCode].remove(self.streams[(locCode, streamCode)].obj) del self.streams[(locCode, streamCode)] inv.flush() return complete
def send_notifiers(self, group): Nsize = DataModel.Notifier.Size() if Nsize > 0: logs.warning("trying to apply %d changes..." % Nsize) else: logs.notice("no changes to apply") return 0 Nmsg = DataModel.Notifier.GetMessage(True) it = Nmsg.iter() msg = DataModel.NotifierMessage() maxmsg = 100 sent = 0 mcount = 0 try: try: while it.get(): msg.attach(DataModel.Notifier_Cast(it.get())) mcount += 1 if msg and mcount == maxmsg: sent += mcount logs.debug("sending message (%5.1f %%)" % (sent / float(Nsize) * 100.0)) self.send(group, msg) msg.clear() mcount = 0 self.sync("fill-db") it.next() except: pass finally: if msg.size(): logs.debug("sending message (%5.1f %%)" % 100.0) self.send(group, msg) msg.clear() self.sync("fill-db") return mcount
def run(self): fd = self.__volume_factory.open(self.__dcid) self.__req.submit(self.__addr, DEFAULT_USER, None) if self.__req.error: logs.warning("error submitting request to %s: %s" % (self.__dcid, self.__req.error)) fd.close(Arclink_ERROR(message=self.__req.error)) return try: self.__req.download_data(fd, True, False) logs.info("%s: request %s ready" % (self.__req.address, self.__req.id)) rqstat = self.__req.status() for vol in rqstat.volume: if vol.status != STATUS_OK: if vol.message: fd.close(Arclink_WARN(message=vol.message)) else: fd.close(Arclink_WARN) break else: if vol.message: fd.close(Arclink_OK(message=vol.message)) else: fd.close(Arclink_OK) self.__req.purge() except ArclinkTimeout, e: logs.warning("%s: %s" % (self.__req.address, str(e))) fd.close(Arclink_RETRY(message="timeout"))
def update(self, DCID, restricted, kf, inv): self.obj.setDescription(kf.statDesc) self.obj.setLatitude(float(kf.latitude)) self.obj.setLongitude(float(kf.longitude)) self.obj.setElevation(float(kf.elevation)) self.obj.setType(kf.type) self.obj.setRestricted(restricted) self.obj.setShared(True) self.obj.setArchive(DCID) self.__usedStreams = set() complete = self.__updateStreams(kf.sampling1, kf.orientation1, kf.datalogger, kf.dataloggerSn, kf.seismometer1, kf.seismometerSn1, float(kf.depth1), 0.02, float(kf.gainMult1), kf.unit1, kf.startDate, restricted, inv) if kf.sampling2: if not kf.depth2: logs.warning("missing depth of secondary sensor for %s %s" % (self.netCode, self.obj.code())) return if not hasattr(kf, "datalogger2") or not kf.datalogger2: kf.datalogger2 = kf.datalogger if not hasattr(kf, "dataloggerSn2") or not kf.dataloggerSn2: kf.dataloggerSn2 = kf.dataloggerSn complete = self.__updateStreams(kf.sampling2, kf.orientation2, kf.datalogger2, kf.dataloggerSn2, kf.seismometer2, kf.seismometerSn2, float(kf.depth2), 1.0, float(kf.gainMult2), kf.unit2, kf.startDate, restricted, inv) and complete for (locCode, streamCode) in set(self.streams.keys()) - self.__usedStreams: self.__loc[locCode].remove(self.streams[(locCode, streamCode)].obj) del self.streams[(locCode, streamCode)] inv.flush() return complete
for req in req_ok: for vol in req.status().volume: if vol.size == 0 or (arclink_status_string(vol.status) != "OK" and arclink_status_string(vol.status) != "WARNING"): continue try: req.download_data(fd_out, vol.id, block=True, purge=False, password=SSLpasswordDict.get(vol.dcid)) except ArclinkError, e: logs.error('error on downloading request: ' + str(e)) try: req.purge() except ArclinkError, e: logs.error('error on purging request: ' + str(e)) fd_out.close() logs.warning("saved file: %s" % filename) else: if rebuild_volume: logs.error('cannot rebuild volume, saving file as received MiniSeed') elif reblock_mseed: logs.error('cannot reblock MSEED, saving file as received MiniSeed') for req in req_ok: for vol in req.status().volume: if vol.size == 0 or (arclink_status_string(vol.status) != "OK" and arclink_status_string(vol.status) != "WARNING"): continue filename = None fd_out = None try:
logs.error("%s: %s" % (f, str(e))) for f in glob.glob(os.path.join(seiscompRoot, "key", "station_*")): try: logs.debug("processing " + f) (netCode, staCode) = f.split("/station_")[-1].split('_', 1) try: kf = Keyfile(f) except IOError, e: logs.error(str(e)) continue existingStations.add((netCode, staCode)) if netCode not in existingNetworks: logs.warning("network %s does not exist, ignoring station %s" % (netCode, staCode)) continue if not hasattr(kf, "latitude") or not kf.latitude: logs.warning("missing latitude for %s %s" % (netCode, staCode)) continue if not hasattr(kf, "longitude") or not kf.longitude: logs.warning("missing longitude for %s %s" % (netCode, staCode)) continue if not hasattr(kf, "elevation") or not kf.elevation: logs.warning("missing elevation for %s %s" % (netCode, staCode)) continue if not hasattr(kf, "depth1") or not kf.depth1:
def warning(self, s): logs.warning(str(self.__parser.getSystemId()) + ":" + \ str(self.__parser.getLineNumber()) + ":" + \ str(self.__parser.getColumnNumber()) + ": " + s)
def update(self): """Read the inventory file in XML format and store it in memory. All the information of the inventory is read into lists of networks, stations, sensor locations and streams. Only the necessary attributes are stored. This relies on the idea that some other agent should update the inventory file at a regular period of time. If the XML file have been already processed by other instance of this class, we could look for a temporary file containing a memory dump of the generated structures, avoiding the time invested in the construction. """ # Calculate when the next update should take place nextUpdate = self.lastUpdated + datetime.timedelta( seconds=self.time2refresh) # If the cache is still valid if nextUpdate > datetime.datetime.now(): return # Initialize lists self.networks = [] self.stations = [] self.sensorsLoc = [] self.streams = [] self.lastUpdated = datetime.datetime.now() # Just to shorten notation ptNets = self.networks ptStats = self.stations ptSens = self.sensorsLoc ptStre = self.streams start_time = datetime.datetime.now() # Look how old the two versions of inventory are. # First version: XML file try: xml_time = os.path.getmtime(self.inventory) except OSError as e: logs.error('No inventory file! Bye.') return ### NOT SURE WHAT WE SHOULD DO HERE. # Second version: A pickle dump of the processed structures in memory try: pic_time = os.path.getmtime(self.cachefile) except: pic_time = 0 lockfile = self.cachefile + '.lock' if pic_time > xml_time: try: if os.path.exists(lockfile): # Go to the whole processing of the XML file because the # pickle version is still being built. raise Exception with open(self.cachefile) as cache: (self.networks, self.stations, self.sensorsLoc, self.streams, self.streamidx) = pickle.load(cache) logs.info('Inventory loaded from pickle version') return except: pass logs.info('Processing XML: %s' % start_time) sensors = {} dataloggers = {} stationsDict = {} # Parse the inventory file # Two steps parser is defined. In the first one, a dictionary of # sensors and dataloggers is constructed. In the second one, the # networks/stations/sensors/streams tree structure is built. try: invfile = open(self.inventory) except IOError: msg = 'Error: Arclink-inventory.xml could not be opened.' logs.error(msg) raise wsgicomm.WIInternalError, msg for parsetype in ['SENSDAT', 'NET_STA']: # Traverse through the networks # get an iterable try: invfile.seek(0) context = ET.iterparse(invfile, events=("start", "end")) except IOError: msg = 'Error while trying to parse Arclink-inventory.xml.' logs.error(msg) raise wsgicomm.WIInternalError, msg # turn it into an iterator context = iter(context) # get the root element event, root = context.next() # Check that it is really an inventory if root.tag[-len('inventory'):] != 'inventory': msg = 'The file parsed seems not to be an inventory (XML).' logs.error(msg) raise wsgicomm.WIInternalError, msg # Extract the namespace from the root node namesp = root.tag[:-len('inventory')] for event, netw in context: # The tag of this node could actually be "network" or # "stationGroup". Now it is not being checked because # we need all the data, but if we need to filter, this # is the place. # if event == "end": if parsetype == 'NET_STA' and \ netw.tag == namesp + 'network': # Extract the year from start try: start_year = netw.get('start') start_year = int(start_year[:4]) except: start_year = None # Extract the year from end try: end_year = netw.get('end') end_year = int(end_year[:4]) except: end_year = None # Cast the attribute restricted try: if netw.get('restricted').lower() == 'true': restricted = 1 elif netw.get('restricted').lower() == 'false': restricted = 2 else: restricted = None except: restricted = None # Append the network to the list of networks ptNets.append([netw.get('code'), len(ptStats), None, None, start_year, end_year, netw.get('description'), restricted, netw.get('netClass'), netw.get('archive'), netw.get('institutions')]) last_child_station = len(ptStats) # Traverse through the stations for stat in netw.findall(namesp + 'station'): # Extract the year from start try: stat_start_string = stat.get('start') stat_start_date = datetime.datetime.strptime( stat_start_string, '%Y-%m-%dT%H:%M:%S.%fZ') except: stat_start_date = None # Extract the year from end try: stat_end_string = stat.get('end') stat_end_date = datetime.datetime.strptime( stat_end_string, '%Y-%m-%dT%H:%M:%S.%fZ') except: stat_end_date = None # Extract latitude try: lat = float(stat.get('latitude')) except: lat = None # Extract longitude try: lon = float(stat.get('longitude')) except: lon = None # Extract elevation try: elevation = float(stat.get('elevation')) except: elevation = None stationsDict[stat.get('publicID')] = len(ptStats) # Cast the attribute restricted try: if stat.get('restricted').lower() == 'true': restricted = 1 elif stat.get('restricted').lower() == 'false': restricted = 2 else: restricted = None except: restricted = None # Only store a reference to the network in the # first column ptStats.append([len(ptNets) - 1, len(ptSens), None, None, stat.get('code'), lat, lon, stat.get('description'), stat_start_date, stat_end_date, elevation, restricted]) last_child_station += 1 last_child_sensor = len(ptSens) sensXml = namesp + 'sensorLocation' for sensor in stat.findall(sensXml): # A reference to the containing station is # in the first column ptSens.append([len(ptStats) - 1, len(ptStre), None, None, sensor.get('code')]) last_child_sensor += 1 last_child_stream = len(ptStre) streXml = namesp + 'stream' for stream in sensor.findall(streXml): sens_type = sensors.get( stream.get('sensor')) try: d = stream.get('sampleRateDenominator') n = stream.get('sampleRateNumerator') denom = float(d) numer = float(n) except: denom = None numer = None try: startString = stream.get('start') startDate = datetime.datetime.strptime( startString, '%Y-%m-%dT%H:%M:%S.%fZ') except: startDate = None try: endString = stream.get('end') endDate = datetime.datetime.strptime( endString, '%Y-%m-%dT%H:%M:%S.%fZ') except: endDate = None # Cast the attribute restricted try: if stream.get('restricted').lower() \ == 'true': restricted = 1 elif stream.get('restricted').lower() \ == 'false': restricted = 2 else: restricted = None except: restricted = None auxCode = stream.get('code') auxDatLog = stream.get('datalogger') ptStre.append((len(ptSens) - 1, auxCode, sens_type, denom, numer, dataloggers.get(auxDatLog), startDate, endDate, restricted)) last_child_stream += 1 stream.clear() ptSens[-1][2] = last_child_stream sensor.clear() # Check if there is at least one stream. # Otherwise remove sensor. This case can happen # when there are only auxStreams instead of # streams if ptSens[-1][1] == ptSens[-1][2]: del ptSens[-1] last_child_sensor -= 1 self.stations[-1][2] = last_child_sensor stat.clear() # Check if there is at least one sensor. Otherwise # remove station. This case can happen when there # are only auxStreams instead of streams if ptStats[-1][1] == ptStats[-1][2]: del ptStats[-1] last_child_station -= 1 ptNets[-1][2] = last_child_station netw.clear() if((parsetype == 'SENSDAT') and (netw.tag == namesp + 'sensor')): pubId = netw.get('publicID') sensors[pubId] = netw.get('type') netw.clear() if((parsetype == 'SENSDAT') and (netw.tag == namesp + 'datalogger')): pubId = netw.get('publicID') dataloggers[pubId] = netw.get('description') netw.clear() if((parsetype == 'SENSDAT') and (netw.tag == namesp + 'stationGroup')): # Extract the year from start try: start_year = netw.get('start') start_year = int(start_year[:4]) except: start_year = None # Extract the year from end try: end_year = netw.get('end') end_year = int(end_year[:4]) except: end_year = None # Fill a list with station ID's. To be replaced later # with the index in self.stations virtualStations = [] statRefXml = namesp + 'stationReference' for statRef in netw.findall(statRefXml): virtualStations.append(statRef.get('stationID')) # Virtual networks are always permanent ptNets.append([netw.get('code'), None, None, virtualStations, start_year, end_year, netw.get('description'), False, 'p', 'GFZ', 'GFZ']) netw.clear() root.clear() invfile.close() # Resolving station references in virtual networks for netw in self.networks: if((netw[1] is None) and (netw[2] is None)): idxs = [] for stat in netw[3]: idxs.append(stationsDict[stat]) netw[3] = idxs end_time = datetime.datetime.now() logs.info('Done with XML: %s' % (end_time)) # Python 2.7: (end_time - start_time).total_seconds()) self.__indexStreams() if not os.path.exists(lockfile): try: lck = open(lockfile, 'w') os.chmod(lockfile, 0666) lck.close() except: logs.warning(('Error while attempting to create a lockfile' + ' (%s). Check whether the inventory is parsed' + ' every %d seconds. This could potentialy' + ' make some requests slower.') % (lockfile, self.time2refresh)) return with open(self.cachefile, 'wb') as cache: os.chmod(self.cachefile, 0666) pickle.dump((ptNets, ptStats, ptSens, ptStre, self.streamidx), cache) try: os.remove(lockfile) except: logs.error(('Error by removing lockfile (%s). Remove it' + ' manually or the pickle version will be always' + ' skipped.') % lockfile)
def send(self, *args): while not self.connection().send(*args): logs.warning("send failed, retrying") time.sleep(1)
def update(self): """Read the inventory file in XML format and store it in memory. All the information of the inventory is read into lists of networks, stations, sensor locations and streams. Only the necessary attributes are stored. This relies on the idea that some other agent should update the inventory file at a regular period of time. If the XML file have been already processed by other instance of this class, we could look for a temporary file containing a memory dump of the generated structures, avoiding the time invested in the construction. """ # Calculate when the next update should take place nextUpdate = self.lastUpdated + datetime.timedelta( seconds=self.time2refresh) # If the cache is still valid if nextUpdate > datetime.datetime.now(): return # Initialize lists self.networks = [] self.stations = [] self.sensorsLoc = [] self.streams = [] self.lastUpdated = datetime.datetime.now() # Just to shorten notation ptNets = self.networks ptStats = self.stations ptSens = self.sensorsLoc ptStre = self.streams start_time = datetime.datetime.now() # Look how old the two versions of inventory are. # First version: XML file try: xml_time = os.path.getmtime(self.inventory) except OSError as e: logs.error('No inventory file! Bye.') return ### NOT SURE WHAT WE SHOULD DO HERE. # Second version: A pickle dump of the processed structures in memory try: pic_time = os.path.getmtime(self.cachefile) except: pic_time = 0 lockfile = self.cachefile + '.lock' if pic_time > xml_time: try: if os.path.exists(lockfile): # Go to the whole processing of the XML file because the # pickle version is still being built. raise Exception with open(self.cachefile) as cache: (self.networks, self.stations, self.sensorsLoc, self.streams, self.streamidx) = pickle.load(cache) logs.info('Inventory loaded from pickle version') return except: pass logs.info('Processing XML: %s' % start_time) sensors = {} dataloggers = {} stationsDict = {} # Parse the inventory file. # There are two steps in parsing. In the first, a dictionary of # sensors and dataloggers is constructed. In the second step, the # networks/stations/sensors/streams tree structure is built. try: invfile = open(self.inventory) except IOError: msg = 'Error: could not open the inventory file ' + self.inventory logs.error(msg) raise wsgicomm.WIInternalError, msg for parsetype in ['SENSDAT', 'NET_STA']: # Traverse through the networks # get an iterable try: invfile.seek(0) context = ET.iterparse(invfile, events=("start", "end")) except IOError: msg = 'Error: could not parse the inventory file ' + self.inventory logs.error(msg) raise wsgicomm.WIInternalError, msg # turn it into an iterator context = iter(context) # get the root element event, root = context.next() # Check that it is really an inventory if root.tag[-len('inventory'):] != 'inventory': msg = 'The file parsed seems not to be an inventory (XML).' logs.error(msg) raise wsgicomm.WIInternalError, msg # Extract the namespace from the root node namesp = root.tag[:-len('inventory')] for event, netw in context: # The tag of this node could actually be "network" or # "stationGroup". Now it is not being checked because # we need all the data, but if we need to filter, this # is the place. # if event == "end": if parsetype == 'NET_STA' and \ netw.tag == namesp + 'network': # Extract the year from start try: start_year = netw.get('start') start_year = int(start_year[:4]) except: start_year = None # Extract the year from end try: end_year = netw.get('end') end_year = int(end_year[:4]) except: end_year = None # Cast the attribute restricted try: if netw.get('restricted').lower() == 'true': restricted = 1 elif netw.get('restricted').lower() == 'false': restricted = 2 else: restricted = None except: restricted = None # Append the network to the list of networks ptNets.append([ netw.get('code'), len(ptStats), None, None, start_year, end_year, netw.get('description'), restricted, netw.get('netClass'), netw.get('archive'), netw.get('institutions') ]) last_child_station = len(ptStats) # Traverse through the stations for stat in netw.findall(namesp + 'station'): # Extract the year from start try: stat_start_string = stat.get('start') stat_start_date = datetime.datetime.strptime( stat_start_string, '%Y-%m-%dT%H:%M:%S.%fZ') except: stat_start_date = None # Extract the year from end try: stat_end_string = stat.get('end') stat_end_date = datetime.datetime.strptime( stat_end_string, '%Y-%m-%dT%H:%M:%S.%fZ') except: stat_end_date = None # Extract latitude try: lat = float(stat.get('latitude')) except: lat = None # Extract longitude try: lon = float(stat.get('longitude')) except: lon = None # Extract elevation try: elevation = float(stat.get('elevation')) except: elevation = None stationsDict[stat.get('publicID')] = len(ptStats) # Cast the attribute restricted try: if stat.get('restricted').lower() == 'true': restricted = 1 elif stat.get('restricted').lower() == 'false': restricted = 2 else: restricted = None except: restricted = None # Only store a reference to the network in the # first column ptStats.append([ len(ptNets) - 1, len(ptSens), None, None, stat.get('code'), lat, lon, stat.get('description'), stat_start_date, stat_end_date, elevation, restricted ]) last_child_station += 1 last_child_sensor = len(ptSens) sensXml = namesp + 'sensorLocation' for sensor in stat.findall(sensXml): # A reference to the containing station is # in the first column ptSens.append([ len(ptStats) - 1, len(ptStre), None, None, sensor.get('code') ]) last_child_sensor += 1 last_child_stream = len(ptStre) streXml = namesp + 'stream' for stream in sensor.findall(streXml): sens_type = sensors.get( stream.get('sensor')) try: d = stream.get('sampleRateDenominator') n = stream.get('sampleRateNumerator') denom = float(d) numer = float(n) except: denom = None numer = None try: startString = stream.get('start') startDate = datetime.datetime.strptime( startString, '%Y-%m-%dT%H:%M:%S.%fZ') except: startDate = None try: endString = stream.get('end') endDate = datetime.datetime.strptime( endString, '%Y-%m-%dT%H:%M:%S.%fZ') except: endDate = None # Cast the attribute restricted try: if stream.get('restricted').lower() \ == 'true': restricted = 1 elif stream.get('restricted').lower() \ == 'false': restricted = 2 else: restricted = None except: restricted = None auxCode = stream.get('code') auxDatLog = stream.get('datalogger') ptStre.append( (len(ptSens) - 1, auxCode, sens_type, denom, numer, dataloggers.get(auxDatLog), startDate, endDate, restricted)) last_child_stream += 1 stream.clear() ptSens[-1][2] = last_child_stream sensor.clear() # Check if there is at least one stream. # Otherwise remove sensor. This case can happen # when there are only auxStreams instead of # streams if ptSens[-1][1] == ptSens[-1][2]: del ptSens[-1] last_child_sensor -= 1 self.stations[-1][2] = last_child_sensor stat.clear() # Check if there is at least one sensor. Otherwise # remove station. This case can happen when there # are only auxStreams instead of streams if ptStats[-1][1] == ptStats[-1][2]: del ptStats[-1] last_child_station -= 1 ptNets[-1][2] = last_child_station netw.clear() if ((parsetype == 'SENSDAT') and (netw.tag == namesp + 'sensor')): pubId = netw.get('publicID') sensors[pubId] = netw.get('type') netw.clear() if ((parsetype == 'SENSDAT') and (netw.tag == namesp + 'datalogger')): pubId = netw.get('publicID') dataloggers[pubId] = netw.get('description') netw.clear() if ((parsetype == 'SENSDAT') and (netw.tag == namesp + 'stationGroup')): # Extract the year from start try: start_year = netw.get('start') start_year = int(start_year[:4]) except: # March 2016: Quick workaround for virtual network with no start date; seems to break getStations() start_year = 1900 # None # Extract the year from end try: end_year = netw.get('end') end_year = int(end_year[:4]) except: end_year = None # Fill a list with station ID's. To be replaced later # with the index in self.stations virtualStations = [] statRefXml = namesp + 'stationReference' for statRef in netw.findall(statRefXml): virtualStations.append(statRef.get('stationID')) # Virtual networks are always permanent, # and have no archive DCID, since that just leads # to turf battles and much crying. netArchive = '' netInstitutes = netArchive # not used? ptNets.append([ netw.get('code'), None, None, virtualStations, start_year, end_year, netw.get('description'), False, 'p', netArchive, netInstitutes ]) netw.clear() root.clear() invfile.close() # Resolving station references in virtual networks for netw in self.networks: if ((netw[1] is None) and (netw[2] is None)): idxs = [] for stat in netw[3]: idxs.append(stationsDict[stat]) netw[3] = idxs end_time = datetime.datetime.now() logs.info( 'Done with XML: %s' % (end_time)) # Python 2.7: (end_time - start_time).total_seconds()) self.__indexStreams() if not os.path.exists(lockfile): try: lck = open(lockfile, 'w') os.chmod(lockfile, 0664) lck.close() except: logs.warning(('Error while attempting to create a lockfile' + ' (%s). Check whether the inventory is parsed' + ' every %d seconds. This could potentialy' + ' make some requests slower.') % (lockfile, self.time2refresh)) return with open(self.cachefile, 'wb') as cache: os.chmod(self.cachefile, 0664) pickle.dump((ptNets, ptStats, ptSens, ptStre, self.streamidx), cache) try: os.remove(lockfile) except: logs.error(('Error while removing lockfile (%s). Remove it' + ' manually or the pickle version will be always' + ' skipped.') % lockfile)
def main(): param0 = ["-y", "station", "-q", "format=text", "-q", "level=network"] param1 = ["-y", "station", "-q", "format=text", "-q", "level=channel"] param2 = ["-y", "dataselect", "-z"] times = {"starttime": datetime.datetime( 1900, 1, 1), "endtime": datetime.datetime(2100, 1, 1)} nets = set() def add_param0(option, opt_str, value, parser): param0.append(opt_str) param0.append(value) def add_param1(option, opt_str, value, parser): param1.append(opt_str) param1.append(value) def add_param2(option, opt_str, value, parser): param2.append(opt_str) param2.append(value) def add_param(option, opt_str, value, parser): add_param0(option, opt_str, value, parser) add_param1(option, opt_str, value, parser) add_param2(option, opt_str, value, parser) def add_time(option, opt_str, value, parser): add_param1(option, opt_str, value, parser) try: t = dateutil.parser.parse(value) except ValueError as e: raise optparse.OptionValueError( "option '%s': invalid time value: '%s'" % (opt_str, value)) if t.tzinfo is not None: t = t.astimezone(dateutil.tz.tzutc()).replace(tzinfo=None) times[option.dest] = t parser = optparse.OptionParser( usage="Usage: %prog [-h|--help] [OPTIONS] -o directory", version="%prog " + VERSION) parser.set_defaults( url="http://geofon.gfz-potsdam.de/eidaws/routing/1/", timeout=600, retries=10, retry_wait=60, threads=5, max_lines=1000, max_timespan=1440) parser.add_option("-v", "--verbose", action="store_true", default=False, help="verbose mode") parser.add_option("-u", "--url", type="string", action="callback", callback=add_param, help="URL of routing service (default %default)") parser.add_option("-N", "--network", type="string", action="callback", callback=add_param1, help="network code or pattern") parser.add_option("-S", "--station", type="string", action="callback", callback=add_param1, help="station code or pattern") parser.add_option("-L", "--location", type="string", action="callback", callback=add_param1, help="location code or pattern") parser.add_option("-C", "--channel", type="string", action="callback", callback=add_param1, help="channel code or pattern") parser.add_option("-s", "--starttime", type="string", action="callback", callback=add_time, help="start time") parser.add_option("-e", "--endtime", type="string", action="callback", callback=add_time, help="end time") parser.add_option("-t", "--timeout", type="int", action="callback", callback=add_param, help="request timeout in seconds (default %default)") parser.add_option("-r", "--retries", type="int", action="callback", callback=add_param, help="number of retries (default %default)") parser.add_option("-w", "--retry-wait", type="int", action="callback", callback=add_param, help="seconds to wait before each retry (default %default)") parser.add_option("-n", "--threads", type="int", action="callback", callback=add_param, help="maximum number of download threads (default %default)") parser.add_option("-c", "--credentials-file", type="string", action="callback", callback=add_param2, help="URL,user,password file (CSV format) for queryauth") parser.add_option("-a", "--auth-file", type="string", action="callback", callback=add_param2, help="file that contains the auth token") parser.add_option("-o", "--output-dir", type="string", help="SDS directory where downloaded data is written") parser.add_option("-l", "--max-lines", type="int", help="max lines per request (default %default)") parser.add_option("-m", "--max-timespan", type="int", help="max timespan per request in minutes (default %default)") parser.add_option("-z", "--no-citation", action="store_true", default=False, help="suppress network citation info") parser.add_option("-Z", "--no-check", action="store_true", default=False, help="suppress checking received routes and data") (options, args) = parser.parse_args() if args or not options.output_dir: parser.print_usage(sys.stderr) return 1 def log_alert(s): if sys.stderr.isatty(): s = "\033[31m" + s + "\033[m" sys.stderr.write(s + '\n') sys.stderr.flush() def log_notice(s): if sys.stderr.isatty(): s = "\033[32m" + s + "\033[m" sys.stderr.write(s + '\n') sys.stderr.flush() def log_verbose(s): sys.stderr.write(s + '\n') sys.stderr.flush() def log_silent(s): pass logs.error = log_alert logs.warning = log_alert logs.notice = log_notice logs.info = (log_silent, log_verbose)[options.verbose] logs.debug = log_silent try: try: proc = exec_fetch(param1, None, options.verbose, options.no_check) except OSError as e: logs.error(str(e)) logs.error("error running fdsnws_fetch") return 1 timespan = {} for line in proc.stdout: if isinstance(line, bytes): line = line.decode('utf-8') if not line or line.startswith('#'): continue starttime = max(dateutil.parser.parse( line.split('|')[15]), times['starttime']) endtime = min(dateutil.parser.parse( line.split('|')[16]), times['endtime']) if starttime.tzinfo is not None: starttime = starttime.astimezone( dateutil.tz.tzutc()).replace(tzinfo=None) if endtime.tzinfo is not None: endtime = endtime.astimezone( dateutil.tz.tzutc()).replace(tzinfo=None) try: ts = timespan[tuple(line.split('|')[:4])] if ts.start > starttime: ts.start = starttime ts.current = starttime if ts.end < endtime: ts.end = endtime except KeyError: timespan[tuple(line.split('|')[:4])] = Timespan( starttime, endtime) proc.stdout.close() proc.wait() if proc.returncode != 0: logs.error("error running fdsnws_fetch") return 1 if os.path.exists(options.output_dir): scan_sds(options.output_dir, timespan, nets) while len(timespan) > 0: postdata = "" ts_used = random.sample(timespan.items(), min( len(timespan), options.max_lines)) for ((net, sta, loc, cha), ts) in ts_used: te = min(ts.end, ts.start + datetime.timedelta(minutes=options.max_timespan)) if loc == '': loc = '--' postdata += "%s %s %s %s %sZ %sZ\n" \ % (net, sta, loc, cha, ts.start.isoformat(), te.isoformat()) if not isinstance(postdata, bytes): postdata = postdata.encode('utf-8') try: proc = exec_fetch(param2, postdata, options.verbose, options.no_check) except OSError as e: logs.error(str(e)) logs.error("error running fdsnws_fetch") return 1 got_data = False try: for rec in mseedlite.Input(proc.stdout): try: ts = timespan[(rec.net, rec.sta, rec.loc, rec.cha)] except KeyError: logs.warning("unexpected data: %s.%s.%s.%s" % (rec.net, rec.sta, rec.loc, rec.cha)) continue if rec.end_time <= ts.current: continue sds_dir = "%s/%d/%s/%s/%s.D" \ % (options.output_dir, rec.begin_time.year, rec.net, rec.sta, rec.cha) sds_file = "%s.%s.%s.%s.D.%s" \ % (rec.net, rec.sta, rec.loc, rec.cha, rec.begin_time.strftime('%Y.%j')) if not os.path.exists(sds_dir): os.makedirs(sds_dir) with open(sds_dir + '/' + sds_file, 'ab') as fd: fd.write(rec.header + rec.data) ts.current = rec.end_time nets.add((rec.net, rec.begin_time.year)) got_data = True except mseedlite.MSeedError as e: logs.error(str(e)) proc.stdout.close() proc.wait() if proc.returncode != 0: logs.error("error running fdsnws_fetch") return 1 for ((net, sta, loc, cha), ts) in ts_used: if not got_data: # no progress, skip to next segment ts.start += datetime.timedelta( minutes=options.max_timespan) else: # continue from current position ts.start = ts.current if ts.start >= ts.end: # timespan completed del timespan[(net, sta, loc, cha)] if nets and not options.no_citation: logs.info("retrieving network citation info") get_citation(nets, param0, options.verbose) except (IOError, Error) as e: logs.error(str(e)) return 1 return 0
def warning(s): logs.warning("%s:%d:%d: %s" % (filename, lineno, pos + 1, s))
def run(self): try: seiscompRoot = self.commandline().unrecognizedOptions()[0] sys.stderr.write("root directory: %s\n" % seiscompRoot) try: DCID = self.configGetString("datacenterID") except: logs.error("datacenterID not found in global.cfg") return False networkRestricted = {} incompleteResponse = {} global instdb instdb = Instruments(DCID) self.__load_file(loadGains, os.path.join(seiscompRoot, "config", "gain.dlsv")) # for backwards compatibility self.__load_file( loadGains, os.path.join(seiscompRoot, "config", "gain.tab.out")) self.__load_file(loadGains, os.path.join(seiscompRoot, "config", "gain.tab")) try: self.__load_file(instdb.load_db, os.path.join(seiscompRoot, "resp", "inst.db")) self.__load_file( instdb.load_sensor_attr, os.path.join(seiscompRoot, "resp", "sensor_attr.csv")) self.__load_file( instdb.load_datalogger_attr, os.path.join(seiscompRoot, "resp", "datalogger_attr.csv")) except (IOError, NettabError) as e: logs.error("fatal error: " + str(e)) return False sc3Inv = seiscomp3.DataModel.Inventory() inventory = InventoryWrapper(sc3Inv, DCID) existingNetworks = set() existingStations = set() for f in glob.glob(os.path.join(seiscompRoot, "key", "network_*")): try: logs.debug("processing " + f) netCode = f.split("/network_")[-1] try: kf = Keyfile(f) except IOError as e: logs.error(str(e)) continue existingNetworks.add(netCode) networkRestricted[netCode] = False inventory.updateNetwork(netCode, kf) except ValueError as e: logs.error("%s: %s" % (f, str(e))) for f in glob.glob(os.path.join(seiscompRoot, "key", "station_*")): try: logs.debug("processing " + f) (netCode, staCode) = f.split("/station_")[-1].split('_', 1) try: kf = Keyfile(f) except IOError as e: logs.error(str(e)) continue existingStations.add((netCode, staCode)) if netCode not in existingNetworks: logs.warning( "network %s does not exist, ignoring station %s" % (netCode, staCode)) continue if not hasattr(kf, "latitude") or not kf.latitude: logs.warning("missing latitude for %s %s" % (netCode, staCode)) continue if not hasattr(kf, "longitude") or not kf.longitude: logs.warning("missing longitude for %s %s" % (netCode, staCode)) continue if not hasattr(kf, "elevation") or not kf.elevation: logs.warning("missing elevation for %s %s" % (netCode, staCode)) continue if not hasattr(kf, "depth1") or not kf.depth1: logs.warning( "missing depth of primary sensor for %s %s" % (netCode, staCode)) continue if decimal.Decimal(kf.latitude) == decimal.Decimal("0.0") and \ decimal.Decimal(kf.longitude) == decimal.Decimal("0.0"): logs.warning("missing coordinates for %s %s" % (netCode, staCode)) continue if not hasattr(kf, "orientation1") or not kf.orientation1: logs.warning( "missing orientation of primary sensor for %s %s, using default" % (netCode, staCode)) kf.orientation1 = "Z 0.0 -90.0; N 0.0 0.0; E 90.0 0.0" if not hasattr(kf, "orientation2"): kf.orientation2 = "" if not hasattr(kf, "unit1") or not kf.unit1: logs.warning( "missing unit of primary sensor for %s %s, using M/S" % (netCode, staCode)) kf.unit1 = "M/S" if not hasattr(kf, "unit2"): logs.warning( "missing unit of secondary sensor for %s %s, using M/S**2" % (netCode, staCode)) kf.unit2 = "M/S**2" if not hasattr(kf, "type"): kf.type = "" restricted = False # TODO: Make restricted part of the key file if not inventory.updateStation(netCode, staCode, restricted, kf): try: incNet = incompleteResponse[netCode] except KeyError: incNet = set() incompleteResponse[netCode] = incNet incNet.add(staCode) except ValueError as e: logs.error("%s: %s" % (f, str(e))) for (netCode, restricted) in networkRestricted.items(): inventory.setNetworkRestricted(netCode, restricted) for (netCode, network) in inventory.networks.items(): if netCode not in existingNetworks: logs.notice("deleting network %s from inventory" % (netCode, )) inventory.obj.remove(network.obj) for ((netCode, staCode), station) in inventory.stations.items(): if netCode in existingNetworks and ( netCode, staCode) not in existingStations: logs.notice("deleting station %s_%s from inventory" % (netCode, staCode)) inventory.networks[netCode].obj.remove(station.obj) if incompleteResponse: logs.info( "The following stations are missing full response data") logs.info("Use dlsv2inv if needed") # for netCode in sorted(incompleteResponse.keys()): # logs.info("%s: %s" % (netCode, " ".join(sorted(list(incompleteResponse[netCode]))))) tmpDict = sortDictionary(incompleteResponse) for netCode in list(tmpDict.keys()): tmpSortedList = list(tmpDict[netCode]) tmpSortedList.sort() logs.info("%s: %s" % (netCode, " ".join(tmpSortedList))) ar = seiscomp3.IO.XMLArchive() if not self.output: sys.stderr.write("Writing output to stdout\n") if not ar.create("-"): sys.stderr.write("Cannot open open stdout\n") return False else: sys.stderr.write("Writing output to %s\n" % self.output) if not ar.create(self.output): sys.stderr.write("Cannot open open %s\n" % self.output) return False ar.setFormattedOutput(self.commandline().hasOption("formatted")) ar.writeObject(sc3Inv) except Exception: logs.print_exc() return True
def _main(SSLpasswordDict, addr, request_format, data_format, label, resp_dict, rebuild_volume, proxymode, user, timeout, retries, output_file, input_file, spfr): reblock_mseed = False use_inventory = False use_routing = not proxymode req_args = {"compression": "bzip2"} if data_format.upper() == "MSEED": req_type = "WAVEFORM" req_args["format"] = "MSEED" elif data_format.upper() == "MSEED4K": req_type = "WAVEFORM" req_args["format"] = "MSEED" reblock_mseed = True elif data_format.upper() == "FSEED": req_type = "WAVEFORM" if rebuild_volume: req_args["format"] = "MSEED" else: req_args["format"] = "FSEED" elif data_format.upper() == "DSEED": req_type = "RESPONSE" use_routing = False elif len(data_format) >= 3 and data_format.upper() == "INVENTORY"[:len(data_format)]: req_type = "INVENTORY" req_args["instruments"] = "true" use_routing = False else: logs.error("unsupported data format: %s" % (data_format,)) return 1 if resp_dict: req_args["resp_dict"] = "true" else: req_args["resp_dict"] = "false" mgr = ArclinkManager(addr, user, socket_timeout=timeout, download_retry=retries) req = mgr.new_request(req_type, req_args, label) if request_format == "native": if input_file: parse_native(req, input_file) else: parse_native_from_handler(req, sys.stdin) elif request_format == "breqfast": if input_file: parse_breqfast(req, input_file) else: parse_breqfast_from_handler(req, sys.stdin) else: logs.error("unsupported request format: %s" % (request_format,)) return 1 if not req.content: logs.error("empty request") return 1 wildcards = False for i in req.content: for j in i[:4]: if j.find("*") >= 0 or j.find("?") >= 0: wildcards = True break if (rebuild_volume or wildcards) and req_type != "INVENTORY": use_inventory = True (inv, req_ok, req_noroute, req_nodata) = mgr.execute(req, use_inventory, use_routing, spfr) ## Better report what was going on logs.info("\nthe following data requests were sent:") for req in req_ok: show_status(req) if req_nodata: logs.warning("\nthe following entries returned no data after trying all routes") show_lines(req_nodata) if req_noroute: logs.warning("\nthe following entries could not be routed:") show_lines(req_noroute) retry_lines = set() for req in req_ok: for vol in req.status().volume: for line in vol.line: if line.status == STATUS_RETRY: retry_lines.add(line.content) elif line.content in retry_lines: retry_lines.remove(line.content) if retry_lines: logs.warning("\nthe following data is temporarily off-line (try again later)") for ln in retry_lines: logs.warning(ln) ## Prepare to download canJoin = False volumecount = 0 if req_type == "WAVEFORM" and req_args.get("format") == "MSEED": canJoin = True for req in req_ok: for vol in req.status().volume: if (arclink_status_string(vol.status) == "OK" or arclink_status_string(vol.status) == "WARNING") and vol.size > 0: volumecount += 1 if vol.encrypted and (vol.dcid not in SSLpasswordDict): canJoin = False if arclink_status_string(vol.status) == "WARNING": logs.warning("some requests returned a Warning status") if volumecount == 0: logs.warning("\nnone of the requests returned data") return 1 if not canJoin and volumecount > 1: logs.warning('cannot merge volumes saving volumes as individual files') ## Download if canJoin: filename = output_file fd_out = open(filename, "wb") if rebuild_volume: logs.info("rebuilding SEED volume") fd_out = SeedOutput(fd_out, inv, label, resp_dict) elif reblock_mseed: logs.info("reblocking Mini-SEED data") fd_out = MSeed4KOutput(fd_out) for req in req_ok: for vol in req.status().volume: if vol.size == 0 or (arclink_status_string(vol.status) != "OK" and arclink_status_string(vol.status) != "WARNING"): continue try: req.download_data(fd_out, vol.id, block=True, purge=False, password=SSLpasswordDict.get(vol.dcid)) except ArclinkError, e: logs.error('error on downloading request: ' + str(e)) try: req.purge() except ArclinkError, e: logs.error('error on purging request: ' + str(e))
def iterdata(self, time1, time2, net, sta, cha, loc): lasttime = None lastrecord = None recstream = [] if self.__is_iso(time1, time2, net, sta, os.path.exists): recstream.append(("isoarchive", self.isodir)) if self.__is_sds(time1, time2, net, sta, cha, loc, self.archdir, os.path.exists, os.listdir): recstream.append(("sdsarchive", self.archdir)) if self.__is_sds(time1, time2, net, sta, cha, loc, self.nrtdir, os.path.exists, os.listdir): recstream.append(("sdsarchive", self.nrtdir)) if not recstream and self.exists_db(time1, time2, net, sta, cha, loc): raise TemporaryUnavailabilityException for (service, source) in recstream: if lastrecord: try: etime = lastrecord.endTime() except Core.ValueException: logs.warning( "SDS: record.endTime() raises Core.ValueException! Resulting SEED file maybe incorrect!" ) etime = lastrecord.startTime() timetuple = time.strptime(etime.toString("%Y-%m-%d %H:%M:%S"), "%Y-%m-%d %H:%M:%S") lasttime = datetime.datetime( *timetuple[:6]) + datetime.timedelta( seconds=1) # avoids dublettes if lasttime >= time2: break time1 = lasttime lastrecord = None self._recstream = IO.RecordStream.Create(service) if not self._recstream: logs.error("Could not fetch recordstream service '%s'" % service) raise StopIteration if not self._recstream.setSource(source): logs.error("Could not set recordstream source '%s'" % source) self._recstream = None raise StopIteration logs.debug("%s %s: addStream for %s-%s" % (service, source, str(time1), str(time2))) self._recstream.addStream( net, sta, loc, cha, Core.Time.FromString(str(time1), "%Y-%m-%d %H:%M:%S"), Core.Time.FromString(str(time2), "%Y-%m-%d %H:%M:%S")) try: recinput = IO.RecordInput(self._recstream, Core.Array.DOUBLE, Core.Record.SAVE_RAW) record = recinput.next() while record: yield record.raw().str() lastrecord = record record = recinput.next() except Core.GeneralException, e: logs.error(e.what()) except Exception, e: logs.error("SDS: Unexpected exception occured: %s" % e)
fd.close(Arclink_ERROR(message=rqstat.message)) else: logs.warning("%s: request %s returned no data (%s)" % (self.__req.address, str(rqstat.id), str(e))) fd.close(Arclink_NODATA) self.__req.purge() except (ArclinkError, socket.error), e: logs.warning("%s: error: %s" % (self.__req.address, str(e))) fd.close(Arclink_RETRY(message=str(e))) except socket.error, e: logs.warning("%s: error: %s" % (self.__req.address, str(e))) fd.close(Arclink_RETRY(message=str(e))) class WiggleFetcher(object): def __init__(self, nrtdir, archdir, isodir, filedb, max_size, dcid, subnode_addr, dcid_override): self.__volume_factory = None self.__local_volume_fd = None self.__local_volume_size = 0 self.__request_size = 0 self.__sds = sds.SDS(nrtdir, archdir, isodir, filedb) self.__arcl = ArclinkManager(None, socket_timeout=SOCKET_TIMEOUT, download_retry=DOWNLOAD_RETRY) self.__max_size = max_size
urllist.append(FTP_URL + '/' + prefix + addname + '.seed' + endung) try: req.purge() except ArclinkError, e: logs.error('error on purging request: ' + str(e)) if canJoin and fd_out is not None: fd_out.close() endung = build_filename(req.encStatus, req.decStatus, req.args) if endung: os.rename(filename, filename + endung) urllist.append(FTP_URL + '/' + prefix + '.seed' + endung) except (ArclinkError, socket.error), e: logs.warning("request failed: %s" % str(e)) failed_content[STATUS_ERROR] = wf_req.content emailmsg_hint = "" # The "size exceeded" message is misleading - it's the number # of different streams which is too large, not the amount of data. # Adjust request_size in arclink.ini to change this at the server. if (search("size exceeded", str(e))): emailmsg_hint = "(By default, only 1000 streams can be requested at once.)" emailmsg_extra = """ Your Arclink request failed with the following message: %s %s We hope that helps.\n\n""" % (str(e), emailmsg_hint)
class RequestThread(threading.Thread): lock = threading.RLock() class SafeVolume(object): def __init__(self, obj): self.__obj = obj def write(self, data): RequestThread.lock.acquire() try: return self.__obj.write(data) finally: RequestThread.lock.release() def close(self, status, error=False): RequestThread.lock.acquire() try: return self.__obj.close(status, error) finally: RequestThread.lock.release() class SafeVolumeFactory(object): def __init__(self, obj): self.__obj = obj def open(self, name): RequestThread.lock.acquire() try: return RequestThread.SafeVolume(self.__obj.open(name)) finally: RequestThread.lock.release() def __init__(self, req, dcid, addr, volume_factory): threading.Thread.__init__(self) self.__req = req self.__dcid = dcid self.__addr = addr self.__volume_factory = RequestThread.SafeVolumeFactory(volume_factory) def run(self): fd = self.__volume_factory.open(self.__dcid) self.__req.submit(self.__addr, DEFAULT_USER, None) if self.__req.error: logs.warning("error submitting request to %s: %s" % (self.__dcid, self.__req.error)) fd.close(Arclink_ERROR(message=self.__req.error)) return try: self.__req.download_data(fd, True, False) logs.info("%s: request %s ready" % (self.__req.address, self.__req.id)) rqstat = self.__req.status() for vol in rqstat.volume: if vol.status != STATUS_OK: if vol.message: fd.close(Arclink_WARN(message=vol.message)) else: fd.close(Arclink_WARN) break else: if vol.message: fd.close(Arclink_OK(message=vol.message)) else: fd.close(Arclink_OK) self.__req.purge() except ArclinkTimeout, e: logs.warning("%s: %s" % (self.__req.address, str(e))) fd.close(Arclink_RETRY(message="timeout")) except ArclinkError, e: try: rqstat = self.__req.status() if rqstat.error: logs.warning("%s: request %s failed: %s" % (self.__req.address, str( rqstat.id), str(rqstat.message))) fd.close(Arclink_ERROR(message=rqstat.message)) else: logs.warning("%s: request %s returned no data (%s)" % (self.__req.address, str(rqstat.id), str(e))) fd.close(Arclink_NODATA) self.__req.purge() except (ArclinkError, socket.error), e: logs.warning("%s: error: %s" % (self.__req.address, str(e))) fd.close(Arclink_RETRY(message=str(e)))
fd.close(Arclink_ERROR(message=rqstat.message)) else: logs.warning("%s: request %s returned no data (%s)" % (self.__req.address, str(rqstat.id), str(e))) fd.close(Arclink_NODATA) self.__req.purge() except (ArclinkError, socket.error), e: logs.warning("%s: error: %s" % (self.__req.address, str(e))) fd.close(Arclink_RETRY(message=str(e))) except socket.error, e: logs.warning("%s: error: %s" % (self.__req.address, str(e))) fd.close(Arclink_RETRY(message=str(e))) class WiggleFetcher(object): def __init__(self, nrtdir, archdir, isodir, filedb, max_size, dcid, subnode_addr, dcid_override): self.__volume_factory = None self.__local_volume_fd = None self.__local_volume_size = 0 self.__request_size = 0 self.__sds = sds.SDS(nrtdir, archdir, isodir, filedb) self.__arcl = ArclinkManager(None, socket_timeout = SOCKET_TIMEOUT, download_retry = DOWNLOAD_RETRY) self.__max_size = max_size self.__dcid = dcid self.__subnode_addr = subnode_addr
def submit_request(parser, req_name, breq_id): """ Routes the request and analyses its results. Creates the corresponding files in the Breq_fast processing directory. Returns an email message containing the processing status of the breqfast request. @arguments: parser, a BreqParser object req_name, a string defining the request name breq_id, a string specifying the internal Breq_fast request ID @return: a string, giving the processing status email message """ emailaddr = EMAIL_ADDR try: emailaddr = parser.tokendict["email"] except KeyError: pass label = LABEL try: label = parser.tokendict["label"] except KeyError: pass label = re.sub("[^\w]", "_", str(label)) arcl = ArclinkManager(DEFAULT_HOST + ":" + str(DEFAULT_PORT), emailaddr) # Default format is full SEED, however, we can request MSEED # and do the conversion here. In this case, we will end up # with a single SEED volume even if data comes from multiple # sources. wf_req = arcl.new_request("WAVEFORM", {"format": "FSEED"}, label) for x in parser.reqlist: wf_req.add(*x) # List of failed request lines associated to an error message. ok_content = [] failed_content = {} emailmsg = "" emailmsg_extra = "" reqlogmsg = "" try: global logstream logstream = cStringIO.StringIO() try: (inv, req_sent, req_noroute, req_nodata) = arcl.execute(wf_req, True, True) logs.info("the following data requests were sent:") for req in req_sent: logs.info(req.dcname) show_status(req.status()) if req_noroute: tmpstream = cStringIO.StringIO() req_noroute.dump(tmpstream) logs.info("the following entries could not be routed:") logs.info(tmpstream.getvalue()) if req_nodata: tmpstream = cStringIO.StringIO() req_nodata.dump(tmpstream) logs.info("the following entries returned no data:") logs.info(tmpstream.getvalue()) finally: reqlogmsg = logstream.getvalue() logstream = None if req_noroute: failed_content[STATUS_ROUTING] = req_noroute.content # This is necessary here because sometimes below we can't # catch full empty requests if req_nodata: failed_content[STATUS_NODATA] = req_nodata.content if not os.path.exists("%s/%s" % (FTP_DIR, req_name)): os.mkdir("%s/%s" % (FTP_DIR, req_name)) prefix = "%s/%s_%s" % (req_name, label, breq_id) urllist = [] canJoin = True volumecounts = 0 for req in req_sent: reqstatus = req.status() if reqstatus.encrypted: canJoin = False for vol in reqstatus.volume: if arclink_status_string(vol.status) == "OK" and vol.size > 0: volumecounts += 1 if vol.encrypted and vol.size > 0: canJoin = False addname = "" fd_out = None logs.warning("Can Join is set to: %s" % canJoin) logs.warning("We have %s volumes to download" % volumecounts) if canJoin and volumecounts > 0: filename = FTP_DIR + '/' + prefix + '.seed' fd_out = open(filename, "wb") fd_out = SeedOutput(fd_out, inv) cset = set() # process resent requests before original failed requests req_sent.reverse() for req in req_sent: for vol in req.status().volume: if vol.size == 0: continue if not canJoin: addname = str(".%s.%s" % (req.id, vol.id)) filename = FTP_DIR + '/' + prefix + addname + '.seed' fd_out = open(filename, "wb") vol_status = vol.status try: req.download_data(fd_out, vol.id, block=True, purge=False) except (ArclinkError, socket.error), e: logs.error('error on downloading request: ' + str(e)) if fd_out is not None: fd_out.close() raise except (IOError, OSError, DBError, SEEDError, mseed.MSeedError), e: logs.error("error creating SEED Volume: %s" % str(e)) vol_status = STATUS_ERROR
def _main(SSLpasswordDict, addr, request_format, data_format, label, resp_dict, rebuild_volume, proxymode, user, timeout, retries, output_file, input_file, spfr): reblock_mseed = False use_inventory = False use_routing = not proxymode req_args = {"compression": "bzip2"} if data_format.upper() == "MSEED": req_type = "WAVEFORM" req_args["format"] = "MSEED" elif data_format.upper() == "MSEED4K": req_type = "WAVEFORM" req_args["format"] = "MSEED" reblock_mseed = True elif data_format.upper() == "FSEED": req_type = "WAVEFORM" if rebuild_volume: req_args["format"] = "MSEED" else: req_args["format"] = "FSEED" elif data_format.upper() == "DSEED": req_type = "RESPONSE" use_routing = False elif len(data_format) >= 3 and data_format.upper() == "INVENTORY"[:len(data_format)]: req_type = "INVENTORY" req_args["instruments"] = "true" use_routing = False else: logs.error("unsupported data format: %s" % (data_format,)) return 1 if resp_dict: req_args["resp_dict"] = "true" else: req_args["resp_dict"] = "false" mgr = ArclinkManager(addr, user, socket_timeout=timeout, download_retry=retries) req = mgr.new_request(req_type, req_args, label) if request_format == "native": if input_file: parse_native(req, input_file) else: parse_native_from_handler(req, sys.stdin) elif request_format == "breqfast": if input_file: parse_breqfast(req, input_file) else: parse_breqfast_from_handler(req, sys.stdin) else: logs.error("unsupported request format: %s" % (request_format,)) return 1 if not req.content: logs.error("empty request") return 1 wildcards = False for i in req.content: for j in i[:4]: if j.find("*") >= 0 or j.find("?") >= 0: wildcards = True break if (rebuild_volume or wildcards) and req_type != "INVENTORY": use_inventory = True (inv, req_ok, req_noroute, req_nodata) = mgr.execute(req, use_inventory, use_routing, spfr) ## Better report what was going on logs.info("\nthe following data requests were sent:") for req in req_ok: show_status(req) if req_nodata: logs.warning("\nthe following entries returned no data after trying all routes") show_lines(req_nodata) if req_noroute: logs.warning("\nthe following entries could not be routed:") show_lines(req_noroute) retry_lines = set() for req in req_ok: for vol in req.status().volume: for line in vol.line: if line.status == STATUS_RETRY: retry_lines.add(line.content) elif line.content in retry_lines: retry_lines.remove(line.content) if retry_lines: logs.warning("\nthe following data is temporarily off-line (try again later)") for ln in retry_lines: logs.warning(ln) ## Prepare to download canJoin = True volumecount = 0 if req_type == "WAVEFORM" and req_args.get("format") != "MSEED": canJoin = False for req in req_ok: for vol in req.status().volume: if (arclink_status_string(vol.status) == "OK" or arclink_status_string(vol.status) == "WARNING") and vol.size > 0: volumecount += 1 if vol.encrypted and (vol.dcid not in SSLpasswordDict): canJoin = False if arclink_status_string(vol.status) == "WARNING": logs.warning("some requests returned a Warning status") if volumecount == 0: logs.warning("\nnone of the requests returned data") return 1 if not canJoin and volumecount > 1: logs.warning('cannot merge volumes saving volumes as individual files') ## Download if canJoin: filename = output_file fd_out = open(filename, "wb") if rebuild_volume: logs.info("rebuilding SEED volume") fd_out = SeedOutput(fd_out, inv, label, resp_dict) elif reblock_mseed: logs.info("reblocking Mini-SEED data") fd_out = MSeed4KOutput(fd_out) for req in req_ok: for vol in req.status().volume: if vol.size == 0 or (arclink_status_string(vol.status) != "OK" and arclink_status_string(vol.status) != "WARNING"): continue try: req.download_data(fd_out, vol.id, block=True, purge=False, password=SSLpasswordDict.get(vol.dcid)) except ArclinkError, e: logs.error('error on downloading request: ' + str(e)) try: req.purge() except ArclinkError, e: logs.error('error on purging request: ' + str(e))
os.rename(filename, filename + endung) urllist.append(FTP_URL + "/" + prefix + addname + ".seed" + endung) try: req.purge() except ArclinkError, e: logs.error("error on purging request: " + str(e)) if canJoin and fd_out is not None: fd_out.close() endung = build_filename(req.encStatus, req.decStatus, req.args) if endung: os.rename(filename, filename + endung) urllist.append(FTP_URL + "/" + prefix + ".seed" + endung) except (ArclinkError, socket.error), e: logs.warning("request failed: %s" % str(e)) failed_content[STATUS_ERROR] = wf_req.content emailmsg_hint = "" # The "size exceeded" message is misleading - it's the number # of different streams which is too large, not the amount of data. # Adjust request_size in arclink.ini to change this at the server. if search("size exceeded", str(e)): emailmsg_hint = "(By default, only 1000 streams can be requested at once.)" if search("size exceeded", str(e)) or search("empty request", str(e)): emailmsg_extra = """ Your Arclink request failed with the following message: %s %s We hope that helps.\n\n""" % (
def submit_request(parser, req_name, breq_id): """ Routes the request and analyses its results. Creates the corresponding files in the Breq_fast processing directory. Returns an email message containing the processing status of the breqfast request. @arguments: parser, a BreqParser object req_name, a string defining the request name breq_id, a string specifying the internal Breq_fast request ID @return: a string, giving the processing status email message """ emailaddr = EMAIL_ADDR try: emailaddr = parser.tokendict["email"] except KeyError: pass label = LABEL try: label = parser.tokendict["label"] except KeyError: pass label = re.sub("[^\w]", "_", str(label)) arcl = ArclinkManager(DEFAULT_HOST + ":" + str(DEFAULT_PORT), emailaddr) # Default format is full SEED, however, we can request MSEED # and do the conversion here. In this case, we will end up # with a single SEED volume even if data comes from multiple # sources. wf_req = arcl.new_request("WAVEFORM", {"format": "FSEED"}, label) for x in parser.reqlist: wf_req.add(*x) # List of failed request lines associated to an error message. ok_content = [] failed_content = {} emailmsg = "" emailmsg_extra = "" reqlogmsg = "" try: global logstream logstream = cStringIO.StringIO() try: (inv, req_sent, req_noroute, req_nodata) = arcl.execute(wf_req, True, True) logs.info("the following data requests were sent:") for req in req_sent: logs.info(req.dcname) show_status(req.status()) if req_noroute: tmpstream = cStringIO.StringIO() req_noroute.dump(tmpstream) logs.info("the following entries could not be routed:") logs.info(tmpstream.getvalue()) if req_nodata: tmpstream = cStringIO.StringIO() req_nodata.dump(tmpstream) logs.info("the following entries returned no data:") logs.info(tmpstream.getvalue()) finally: reqlogmsg = logstream.getvalue() logstream = None if req_noroute: failed_content[STATUS_ROUTING] = req_noroute.content # This is necessary here because sometimes below we can't catch full empty requests if req_nodata: failed_content[STATUS_NODATA] = req_nodata.content if not os.path.exists("%s/%s" % (FTP_DIR, req_name)): os.mkdir("%s/%s" % (FTP_DIR, req_name)) prefix = "%s/%s_%s" % (req_name, label, breq_id) urllist = [] canJoin = True volumecounts = 0 for req in req_sent: for vol in req.status().volume: if arclink_status_string(vol.status) == "OK" and vol.size > 0: volumecounts += 1 if vol.encrypted and vol.size > 0: canJoin = False sufix = "" addname = "" fd_out = None logs.warning("Can Join is set to: %s" % canJoin) logs.warning("We have %s volumes to download" % volumecounts) if canJoin and volumecounts > 0: filename = FTP_DIR + "/" + prefix + ".seed" fd_out = open(filename, "wb") fd_out = SeedOutput(fd_out, inv) cset = set() # process resent requests before original failed requests req_sent.reverse() for req in req_sent: for vol in req.status().volume: if vol.size == 0: continue if not canJoin: addname = str(".%s.%s" % (req.id, vol.id)) filename = FTP_DIR + "/" + prefix + addname + ".seed" fd_out = open(filename, "wb") vol_status = vol.status try: req.download_data(fd_out, vol.id, block=True, purge=False) except (ArclinkError, socket.error), e: logs.error("error on downloading request: " + str(e)) if fd_out is not None: fd_out.close() raise except (IOError, OSError, DBError, SEEDError, mseed.MSeedError), e: logs.error("error creating SEED Volume: %s" % str(e)) vol_status = STATUS_ERROR
logs.error("%s: %s" % (f, str(e))) for f in glob.glob(os.path.join(seiscompRoot, "key", "station_*")): try: logs.debug("processing " + f) (netCode, staCode) = f.split("/station_")[-1].split('_', 1) try: kf = Keyfile(f) except IOError, e: logs.error(str(e)) continue existingStations.add((netCode, staCode)) if netCode not in existingNetworks: logs.warning( "network %s does not exist, ignoring station %s" % (netCode, staCode)) continue if not hasattr(kf, "latitude") or not kf.latitude: logs.warning("missing latitude for %s %s" % (netCode, staCode)) continue if not hasattr(kf, "longitude") or not kf.longitude: logs.warning("missing longitude for %s %s" % (netCode, staCode)) continue if not hasattr(kf, "elevation") or not kf.elevation: logs.warning("missing elevation for %s %s" % (netCode, staCode))
def main(): param0 = ["-y", "station", "-q", "format=text", "-q", "level=network"] param1 = ["-y", "station", "-q", "format=text", "-q", "level=channel"] param2 = ["-y", "dataselect", "-z"] times = {"starttime": datetime.datetime(1900, 1, 1), "endtime": datetime.datetime(2100, 1, 1)} nets = set() def add_param0(option, opt_str, value, parser): param0.append(opt_str) param0.append(value) def add_param1(option, opt_str, value, parser): param1.append(opt_str) param1.append(value) def add_param2(option, opt_str, value, parser): param2.append(opt_str) param2.append(value) def add_param(option, opt_str, value, parser): add_param0(option, opt_str, value, parser) add_param1(option, opt_str, value, parser) add_param2(option, opt_str, value, parser) def add_time(option, opt_str, value, parser): add_param1(option, opt_str, value, parser) try: t = dateutil.parser.parse(value) except ValueError as e: raise optparse.OptionValueError("option '%s': invalid time value: '%s'" % (opt_str, value)) if t.tzinfo is not None: t = t.astimezone(dateutil.tz.tzutc()).replace(tzinfo=None) times[option.dest] = t parser = optparse.OptionParser( usage="Usage: %prog [-h|--help] [OPTIONS] -o directory", version="%prog " + VERSION) parser.set_defaults( url="http://geofon.gfz-potsdam.de/eidaws/routing/1/", timeout=600, retries=10, retry_wait=60, threads=5, max_lines=1000, max_timespan=1440) parser.add_option("-v", "--verbose", action="store_true", default=False, help="verbose mode") parser.add_option("-u", "--url", type="string", action="callback", callback=add_param, help="URL of routing service (default %default)") parser.add_option("-N", "--network", type="string", action="callback", callback=add_param1, help="network code or pattern") parser.add_option("-S", "--station", type="string", action="callback", callback=add_param1, help="station code or pattern") parser.add_option("-L", "--location", type="string", action="callback", callback=add_param1, help="location code or pattern") parser.add_option("-C", "--channel", type="string", action="callback", callback=add_param1, help="channel code or pattern") parser.add_option("-s", "--starttime", type="string", action="callback", callback=add_time, help="start time") parser.add_option("-e", "--endtime", type="string", action="callback", callback=add_time, help="end time") parser.add_option("-t", "--timeout", type="int", action="callback", callback=add_param, help="request timeout in seconds (default %default)") parser.add_option("-r", "--retries", type="int", action="callback", callback=add_param, help="number of retries (default %default)") parser.add_option("-w", "--retry-wait", type="int", action="callback", callback=add_param, help="seconds to wait before each retry (default %default)") parser.add_option("-n", "--threads", type="int", action="callback", callback=add_param, help="maximum number of download threads (default %default)") parser.add_option("-c", "--credentials-file", type="string", action="callback", callback=add_param2, help="URL,user,password file (CSV format) for queryauth") parser.add_option("-a", "--auth-file", type="string", action="callback", callback=add_param2, help="file that contains the auth token") parser.add_option("-o", "--output-dir", type="string", help="SDS directory where downloaded data is written") parser.add_option("-l", "--max-lines", type="int", help="max lines per request (default %default)") parser.add_option("-m", "--max-timespan", type="int", help="max timespan per request in minutes (default %default)") parser.add_option("-z", "--no-citation", action="store_true", default=False, help="suppress network citation info") parser.add_option("-Z", "--no-check", action="store_true", default=False, help="suppress checking received routes and data") (options, args) = parser.parse_args() if args or not options.output_dir: parser.print_usage(sys.stderr) return 1 def log_alert(s): if sys.stderr.isatty(): s = "\033[31m" + s + "\033[m" sys.stderr.write(s + '\n') sys.stderr.flush() def log_notice(s): if sys.stderr.isatty(): s = "\033[32m" + s + "\033[m" sys.stderr.write(s + '\n') sys.stderr.flush() def log_verbose(s): sys.stderr.write(s + '\n') sys.stderr.flush() def log_silent(s): pass logs.error = log_alert logs.warning = log_alert logs.notice = log_notice logs.info = (log_silent, log_verbose)[options.verbose] logs.debug = log_silent try: try: proc = exec_fetch(param1, None, options.verbose, options.no_check) except OSError as e: logs.error(str(e)) logs.error("error running fdsnws_fetch") return 1 timespan = {} for line in proc.stdout: if isinstance(line, bytes): line = line.decode('utf-8') if not line or line.startswith('#'): continue starttime = max(dateutil.parser.parse(line.split('|')[15]), times['starttime']) endtime = min(dateutil.parser.parse(line.split('|')[16]), times['endtime']) if starttime.tzinfo is not None: starttime = starttime.astimezone(dateutil.tz.tzutc()).replace(tzinfo=None) if endtime.tzinfo is not None: endtime = endtime.astimezone(dateutil.tz.tzutc()).replace(tzinfo=None) try: ts = timespan[tuple(line.split('|')[:4])] if ts.start > starttime: ts.start = starttime ts.current = starttime if ts.end < endtime: ts.end = endtime except KeyError: timespan[tuple(line.split('|')[:4])] = Timespan(starttime, endtime) proc.stdout.close() proc.wait() if proc.returncode != 0: logs.error("error running fdsnws_fetch") return 1 if os.path.exists(options.output_dir): scan_sds(options.output_dir, timespan, nets) while len(timespan) > 0: postdata = "" ts_used = random.sample(timespan.items(), min(len(timespan), options.max_lines)) for ((net, sta, loc, cha), ts) in ts_used: te = min(ts.end, ts.start + datetime.timedelta(minutes=options.max_timespan)) if loc == '': loc = '--' postdata += "%s %s %s %s %sZ %sZ\n" \ % (net, sta, loc, cha, ts.start.isoformat(), te.isoformat()) if not isinstance(postdata, bytes): postdata = postdata.encode('utf-8') try: proc = exec_fetch(param2, postdata, options.verbose, options.no_check) except OSError as e: logs.error(str(e)) logs.error("error running fdsnws_fetch") return 1 got_data = False try: for rec in mseedlite.Input(proc.stdout): try: ts = timespan[(rec.net, rec.sta, rec.loc, rec.cha)] except KeyError: logs.warning("unexpected data: %s.%s.%s.%s" % (rec.net, rec.sta, rec.loc, rec.cha)) continue if rec.end_time <= ts.current: continue sds_dir = "%s/%d/%s/%s/%s.D" \ % (options.output_dir, rec.begin_time.year, rec.net, rec.sta, rec.cha) sds_file = "%s.%s.%s.%s.D.%s" \ % (rec.net, rec.sta, rec.loc, rec.cha, rec.begin_time.strftime('%Y.%j')) if not os.path.exists(sds_dir): os.makedirs(sds_dir) with open(sds_dir + '/' + sds_file, 'ab') as fd: fd.write(rec.header + rec.data) ts.current = rec.end_time nets.add((rec.net, rec.begin_time.year)) got_data = True except mseedlite.MSeedError as e: logs.error(str(e)) proc.stdout.close() proc.wait() if proc.returncode != 0: logs.error("error running fdsnws_fetch") return 1 for ((net, sta, loc, cha), ts) in ts_used: if not got_data: # no progress, skip to next segment ts.start += datetime.timedelta(minutes=options.max_timespan) else: # continue from current position ts.start = ts.current if ts.start >= ts.end: # timespan completed del timespan[(net, sta, loc, cha)] if nets and not options.no_citation: logs.info("retrieving network citation info") get_citation(nets, param0, options.verbose) except (IOError, Error) as e: logs.error(str(e)) return 1 return 0
for req in req_ok: show_status(req) if req_nodata: logs.info("\nthe following entries returned no data:\n") req_nodata.dump(sys.stdout) if req_noroute: logs.info("\nthe following entries could not be routed:\n") req_noroute.dump(sys.stdout) else: if req_nodata: logs.info('some requests returned NODATA') if req_noroute: logs.warning('some requests could not be routed') warn = False for req in req_ok: for vol in req.status().volume: for line in vol.line: if (line.size == 0): warn = True if warn: logs.warning('some lines returned NODATA') ## Prepare to download canJoin = True volumecount = 0 if req_type == "WAVEFORM" and req_args.get("format") != "MSEED":
def __iter_data(self, size, password, raw): bytes_read = 0 self.__errstate = False decompressor = None decryptor = None decStatus = None encStatus = None firstBlock = True try: exc = None try: while bytes_read < size: buf = self.__fd.read(min(BLOCKSIZE, size - bytes_read)) bytes_read += len(buf) if firstBlock: firstBlock = False (decryptor, encStatus) = self.__getDecryptor( buf, password) # Find out if data is encrypted if raw: decryptor = None if decryptor is not None: buf = decryptor.update(buf) (decompressor, decStatus) = self.__getDecompressor( buf) # Find out if data is compressed if raw: decompressor = None if decompressor is not None: buf = decompressor.decompress(buf) else: if decryptor is not None: buf = decryptor.update(buf) if decompressor is not None: buf = decompressor.decompress(buf) yield buf if decryptor is not None: buf = decryptor.final() if decompressor is not None and len(buf) > 0: buf = decompressor.decompress(buf) yield buf r = self.__fd.readline(LINESIZE).rstrip() if r != "END": raise ArclinkError, "END not found" except Exception, e: logs.warning("Download error: %s" % (str(e))) if decryptor is not None: logs.warning("Possible wrong password (%s)." % (password)) raise ArclinkError, "decrypt error." except Exception, e: exc = e
def download_data(self, outfd, req_id, vol_id=None, pos=None, timeout=0, password=None, raw=False): if self.__wait: size = self.__wait_size self.__wait = False else: size = self.__init_download(req_id, vol_id, pos, timeout) bytes_read = 0 self.__errstate = False decompressor = None decryptor = None firstBlock = True try: try: while bytes_read < size: buf = self.__fd.read(min(BLOCKSIZE, size - bytes_read)) bytes_read += len(buf) if firstBlock: firstBlock = False (decryptor, encStatus) = self.__getDecryptor( buf, password) # Find out if data is encrypted if raw: decryptor = None if decryptor is not None: buf = decryptor.update(buf) (decompressor, decStatus) = self.__getDecompressor( buf) # Find out if data is compressed if raw: decompressor = None if decompressor is not None: buf = decompressor.decompress(buf) else: if decryptor is not None: buf = decryptor.update(buf) if decompressor is not None: buf = decompressor.decompress(buf) outfd.write(buf) if decryptor is not None: buf = decryptor.final() if decompressor is not None and len(buf) > 0: buf = decompressor.decompress(buf) outfd.write(buf) r = self.__fd.readline(LINESIZE).rstrip() if r != "END": raise ArclinkError, "END not found" except Exception, e: logs.warning("Download error: %s" % (str(e))) if decryptor is not None: logs.warning("Possible wrong password (%s)." % (password)) raise ArclinkError, "decrypt error." finally: encrypted = (encStatus is True) and (decryptor is None) if encrypted: compressed = None else: compressed = ((decStatus is True) and (decompressor is None)) if bytes_read and bytes_read != size: self.__errstate = True return (encrypted, compressed)