def _loadInventory(self): Logging.debug("loading inventory") dbr = DataModel.DatabaseReader(self.database()) self._inv = DataModel.Inventory() # Load networks and stations staCount = 0 for i in xrange(dbr.loadNetworks(self._inv)): staCount += dbr.load(self._inv.network(i)) Logging.debug("loaded %i stations from %i networks" % (staCount, self._inv.networkCount())) # Load sensors, skip calibrations (not needed by StationXML exporter) Logging.debug("loaded %i sensors" % dbr.loadSensors(self._inv)) # Load datalogger and its decimations, skip calibrations (not needed by # StationXML exporter) deciCount = 0 for i in xrange(dbr.loadDataloggers(self._inv)): deciCount += dbr.loadDecimations(self._inv.datalogger(i)) Logging.debug("loaded %i decimations from %i dataloggers" % (deciCount, self._inv.dataloggerCount())) # Load responses resPAZCount = dbr.loadResponsePAZs(self._inv) resFIRCount = dbr.loadResponseFIRs(self._inv) resPolCount = dbr.loadResponsePolynomials(self._inv) resCount = resPAZCount + resFIRCount + resPolCount Logging.debug("loaded %i responses (PAZ: %i, FIR: %i, Poly: %i)" % (resCount, resPAZCount, resFIRCount, resPolCount)) Logging.info("inventory loaded")
def run(self): try: if self.dcid is None: print("Please specify datacenter/archive ID", file=sys.stderr) return False nettab = Nettab(self.dcid) instdb = Instruments(self.dcid) try: self.__load_file(instdb.load_db, self.inst_db_file) self.__load_file(nettab.load_statmap, self.stat_map_file) self.__load_file(nettab.load_access_net, self.access_net_file) self.__load_file(nettab.load_access_stat, self.access_stat_file) self.__load_file(instdb.load_sensor_attr, self.sensor_attr_file) self.__load_file(instdb.load_datalogger_attr, self.datalogger_attr_file) self.__load_file(nettab.load_network_attr, self.network_attr_file) self.__load_file(nettab.load_station_attr, self.station_attr_file) inv = SC3Inventory(DataModel.Inventory()) idx = 1 for tab in sorted(self.tab_files): print("Loading %s (%d/%d)" % (tab, idx, len(self.tab_files)), file=sys.stderr) self.__load_file(nettab.load_tab, tab) print("Generating data structures", file=sys.stderr) nettab.update_inventory(instdb, inv) idx = idx + 1 if self.isExitRequested(): print("Exit requested, abort", file=sys.stderr) return False print("Generating output", file=sys.stderr) ar = IO.XMLArchive() ar.setFormattedOutput( self.commandline().hasOption("formatted")) ar.create(self.out_file) ar.writeObject(inv.obj) ar.close() print("Finished", file=sys.stderr) except (IOError, NettabError) as e: logs.error("fatal error: " + str(e)) return False except Exception: logs.print_exc() return False return True
def sc3Obj(self, sc3i=None): if not sc3i: sc3i = DataModel.Inventory() for network in list(self.n.values()): sc3n = network.sc3Obj(self.i) sc3i.add(sc3n) for sc3o in self.i.sc3Objs(): sc3i.add(sc3o) self.stationResolver.collectStations(sc3i) for stationGroup in list(self.g.values()): sc3g = stationGroup.sc3Obj(self.stationResolver) sc3i.add(sc3g) return sc3i
def _loadDatabase(self, dbUrl): m = re.match( "(?P<dbDriverName>^.*):\/\/(?P<dbAddress>.+?:.+?@.+?\/.+$)", dbUrl) if not m: raise Exception("error in parsing SC3 DB url") db = m.groupdict() try: registry = Client.PluginRegistry.Instance() registry.addPluginName("dbmysql") registry.loadPlugins() except Exception as e: raise "Cannot load database driver: %s" dbDriver = IO.DatabaseInterface.Create(db["dbDriverName"]) if dbDriver is None: raise Exception("Cannot find database driver " + db["dbDriverName"]) if not dbDriver.connect(db["dbAddress"]): raise Exception("Cannot connect to database at " + db["dbAddress"]) dbQuery = DataModel.DatabaseQuery(dbDriver) if dbQuery is None: raise Exception("Cannot get DB query object") print(" Loading inventory from database ... ", end=' ', file=sys.stderr) inventory = DataModel.Inventory() dbQuery.loadNetworks(inventory) for ni in range(inventory.networkCount()): dbQuery.loadStations(inventory.network(ni)) print("Done.", file=sys.stderr) if inventory: self.stationResolver.collectStations(inventory, True)
def _processRequestExp(self, req, ro, exp, dac): if req._disconnected: return False staCount, locCount, chaCount, extCount, objCount = 0, 0, 0, 0, 0 DataModel.PublicObject.SetRegistrationEnabled(False) newInv = DataModel.Inventory() dataloggers, sensors, extents = set(), set(), set() skipRestricted = not self._allowRestricted or \ (ro.restricted is not None and not ro.restricted) levelNet = not ro.includeSta levelSta = ro.includeSta and not ro.includeCha # iterate over inventory networks for net in ro.networkIter(self._inv, levelNet): if req._disconnected: return False if skipRestricted and utils.isRestricted(net): continue newNet = DataModel.Network(net) # Copy comments for i in xrange(net.commentCount()): newNet.add(DataModel.Comment(net.comment(i))) # iterate over inventory stations of current network for sta in ro.stationIter(net, levelSta): if req._disconnected: return False if skipRestricted and utils.isRestricted(sta): continue if not self.checkObjects(req, objCount, self._maxObj): return False if ro.includeCha: numCha, numLoc, d, s, e = \ self._processStation(newNet, net, sta, ro, dac, skipRestricted) if numCha > 0: locCount += numLoc chaCount += numCha extCount += len(e) objCount += numLoc + numCha + extCount if not self.checkObjects(req, objCount, self._maxObj): return False dataloggers |= d sensors |= s extents |= e elif self._matchStation(net, sta, ro, dac): if ro.includeSta: newSta = DataModel.Station(sta) # Copy comments for i in xrange(sta.commentCount()): newSta.add(DataModel.Comment(sta.comment(i))) newNet.add(newSta) else: # no station output requested: one matching station # is sufficient to include the network newInv.add(newNet) objCount += 1 break if newNet.stationCount() > 0: newInv.add(newNet) staCount += newNet.stationCount() objCount += staCount + 1 # Return 204 if no matching inventory was found if newInv.networkCount() == 0: msg = "no matching inventory found" data = self.renderErrorPage(req, http.NO_CONTENT, msg, ro) if data: utils.writeTS(req, data) return True # Copy references (dataloggers, responses, sensors) decCount, resCount = 0, 0 if ro.includeCha: decCount = self._copyReferences(newInv, req, objCount, self._inv, ro, dataloggers, sensors, self._maxObj) if decCount is None: return False else: resCount = newInv.responsePAZCount() + \ newInv.responseFIRCount() + \ newInv.responsePolynomialCount() + \ newInv.responseFAPCount() + \ newInv.responseIIRCount() objCount += resCount + decCount + newInv.dataloggerCount() + \ newInv.sensorCount() # Copy data extents objOut = newInv if len(extents) > 0: objCount += 1 da = DataModel.DataAvailability() for e in extents: da.add(DataModel.DataExtent(e)) objOut = ExportObjectList() objOut.append(newInv) objOut.append(da) sink = utils.Sink(req) if not exp.write(sink, objOut): return False Logging.debug( "%s: returned %iNet, %iSta, %iLoc, %iCha, " "%iDL, %iDec, %iSen, %iRes, %iDAExt (total objects/" "bytes: %i/%i) " % (ro.service, newInv.networkCount(), staCount, locCount, chaCount, newInv.dataloggerCount(), decCount, newInv.sensorCount(), resCount, extCount, objCount, sink.written)) utils.accessLog(req, ro, http.OK, sink.written, None) return True
raise ("Cannot load database driver: %s" % e) dbDriver = IO.DatabaseInterface.Create(db["dbDriverName"]) if dbDriver is None: raise Exception("Cannot find database driver " + db["dbDriverName"]) if not dbDriver.connect(db["dbAddress"]): raise Exception("Cannot connect to database at " + db["dbAddress"]) dbQuery = DataModel.DatabaseQuery(dbDriver) if dbQuery is None: raise Exception("Cannot get DB query object") print >> sys.stderr, " Loading inventory from database ... ", inventory = DataModel.Inventory() dbQuery.loadNetworks(inventory) for ni in xrange(inventory.networkCount()): dbQuery.loadStations(inventory.network(ni)) print >> sys.stderr, "Done." if inventory: self.stationResolver.collectStations(inventory, True) def _loadXml(self, folder): print >> sys.stderr, " Loading inventory from XML file ... ", for f in glob.glob(os.path.join(folder, "*.xml")): ar = IO.XMLArchive() ar.open(f) inventory = DataModel.Inventory_Cast(ar.readObject()) ar.close()