def _loadInventory(self):
Logging.debug("loading inventory")
dbr = DataModel.DatabaseReader(self.database())
self._inv = DataModel.Inventory()
# Load networks and stations
staCount = 0
for i in xrange(dbr.loadNetworks(self._inv)):
staCount += dbr.load(self._inv.network(i))
Logging.debug("loaded %i stations from %i networks" %
(staCount, self._inv.networkCount()))
# Load sensors, skip calibrations (not needed by StationXML exporter)
Logging.debug("loaded %i sensors" % dbr.loadSensors(self._inv))
# Load datalogger and its decimations, skip calibrations (not needed by
# StationXML exporter)
deciCount = 0
for i in xrange(dbr.loadDataloggers(self._inv)):
deciCount += dbr.loadDecimations(self._inv.datalogger(i))
Logging.debug("loaded %i decimations from %i dataloggers" %
(deciCount, self._inv.dataloggerCount()))
# Load responses
resPAZCount = dbr.loadResponsePAZs(self._inv)
resFIRCount = dbr.loadResponseFIRs(self._inv)
resPolCount = dbr.loadResponsePolynomials(self._inv)
resCount = resPAZCount + resFIRCount + resPolCount
Logging.debug("loaded %i responses (PAZ: %i, FIR: %i, Poly: %i)" %
(resCount, resPAZCount, resFIRCount, resPolCount))
Logging.info("inventory loaded")
def run(self):
try:
if self.dcid is None:
print("Please specify datacenter/archive ID", file=sys.stderr)
return False
nettab = Nettab(self.dcid)
instdb = Instruments(self.dcid)
try:
self.__load_file(instdb.load_db, self.inst_db_file)
self.__load_file(nettab.load_statmap, self.stat_map_file)
self.__load_file(nettab.load_access_net, self.access_net_file)
self.__load_file(nettab.load_access_stat,
self.access_stat_file)
self.__load_file(instdb.load_sensor_attr,
self.sensor_attr_file)
self.__load_file(instdb.load_datalogger_attr,
self.datalogger_attr_file)
self.__load_file(nettab.load_network_attr,
self.network_attr_file)
self.__load_file(nettab.load_station_attr,
self.station_attr_file)
inv = SC3Inventory(DataModel.Inventory())
idx = 1
for tab in sorted(self.tab_files):
print("Loading %s (%d/%d)" %
(tab, idx, len(self.tab_files)),
file=sys.stderr)
self.__load_file(nettab.load_tab, tab)
print("Generating data structures", file=sys.stderr)
nettab.update_inventory(instdb, inv)
idx = idx + 1
if self.isExitRequested():
print("Exit requested, abort", file=sys.stderr)
return False
print("Generating output", file=sys.stderr)
ar = IO.XMLArchive()
ar.setFormattedOutput(
self.commandline().hasOption("formatted"))
ar.create(self.out_file)
ar.writeObject(inv.obj)
ar.close()
print("Finished", file=sys.stderr)
except (IOError, NettabError) as e:
logs.error("fatal error: " + str(e))
return False
except Exception:
logs.print_exc()
return False
return True
def sc3Obj(self, sc3i=None):
if not sc3i:
sc3i = DataModel.Inventory()
for network in list(self.n.values()):
sc3n = network.sc3Obj(self.i)
sc3i.add(sc3n)
for sc3o in self.i.sc3Objs():
sc3i.add(sc3o)
self.stationResolver.collectStations(sc3i)
for stationGroup in list(self.g.values()):
sc3g = stationGroup.sc3Obj(self.stationResolver)
sc3i.add(sc3g)
return sc3i
def _loadDatabase(self, dbUrl):
m = re.match(
"(?P<dbDriverName>^.*):\/\/(?P<dbAddress>.+?:.+?@.+?\/.+$)", dbUrl)
if not m:
raise Exception("error in parsing SC3 DB url")
db = m.groupdict()
try:
registry = Client.PluginRegistry.Instance()
registry.addPluginName("dbmysql")
registry.loadPlugins()
except Exception as e:
raise "Cannot load database driver: %s"
dbDriver = IO.DatabaseInterface.Create(db["dbDriverName"])
if dbDriver is None:
raise Exception("Cannot find database driver " +
db["dbDriverName"])
if not dbDriver.connect(db["dbAddress"]):
raise Exception("Cannot connect to database at " + db["dbAddress"])
dbQuery = DataModel.DatabaseQuery(dbDriver)
if dbQuery is None:
raise Exception("Cannot get DB query object")
print(" Loading inventory from database ... ",
end=' ',
file=sys.stderr)
inventory = DataModel.Inventory()
dbQuery.loadNetworks(inventory)
for ni in range(inventory.networkCount()):
dbQuery.loadStations(inventory.network(ni))
print("Done.", file=sys.stderr)
if inventory:
self.stationResolver.collectStations(inventory, True)
def _processRequestExp(self, req, ro, exp, dac):
if req._disconnected:
return False
staCount, locCount, chaCount, extCount, objCount = 0, 0, 0, 0, 0
DataModel.PublicObject.SetRegistrationEnabled(False)
newInv = DataModel.Inventory()
dataloggers, sensors, extents = set(), set(), set()
skipRestricted = not self._allowRestricted or \
(ro.restricted is not None and not ro.restricted)
levelNet = not ro.includeSta
levelSta = ro.includeSta and not ro.includeCha
# iterate over inventory networks
for net in ro.networkIter(self._inv, levelNet):
if req._disconnected:
return False
if skipRestricted and utils.isRestricted(net):
continue
newNet = DataModel.Network(net)
# Copy comments
for i in xrange(net.commentCount()):
newNet.add(DataModel.Comment(net.comment(i)))
# iterate over inventory stations of current network
for sta in ro.stationIter(net, levelSta):
if req._disconnected:
return False
if skipRestricted and utils.isRestricted(sta):
continue
if not self.checkObjects(req, objCount, self._maxObj):
return False
if ro.includeCha:
numCha, numLoc, d, s, e = \
self._processStation(newNet, net, sta, ro, dac,
skipRestricted)
if numCha > 0:
locCount += numLoc
chaCount += numCha
extCount += len(e)
objCount += numLoc + numCha + extCount
if not self.checkObjects(req, objCount, self._maxObj):
return False
dataloggers |= d
sensors |= s
extents |= e
elif self._matchStation(net, sta, ro, dac):
if ro.includeSta:
newSta = DataModel.Station(sta)
# Copy comments
for i in xrange(sta.commentCount()):
newSta.add(DataModel.Comment(sta.comment(i)))
newNet.add(newSta)
else:
# no station output requested: one matching station
# is sufficient to include the network
newInv.add(newNet)
objCount += 1
break
if newNet.stationCount() > 0:
newInv.add(newNet)
staCount += newNet.stationCount()
objCount += staCount + 1
# Return 204 if no matching inventory was found
if newInv.networkCount() == 0:
msg = "no matching inventory found"
data = self.renderErrorPage(req, http.NO_CONTENT, msg, ro)
if data:
utils.writeTS(req, data)
return True
# Copy references (dataloggers, responses, sensors)
decCount, resCount = 0, 0
if ro.includeCha:
decCount = self._copyReferences(newInv, req, objCount, self._inv,
ro, dataloggers, sensors,
self._maxObj)
if decCount is None:
return False
else:
resCount = newInv.responsePAZCount() + \
newInv.responseFIRCount() + \
newInv.responsePolynomialCount() + \
newInv.responseFAPCount() + \
newInv.responseIIRCount()
objCount += resCount + decCount + newInv.dataloggerCount() + \
newInv.sensorCount()
# Copy data extents
objOut = newInv
if len(extents) > 0:
objCount += 1
da = DataModel.DataAvailability()
for e in extents:
da.add(DataModel.DataExtent(e))
objOut = ExportObjectList()
objOut.append(newInv)
objOut.append(da)
sink = utils.Sink(req)
if not exp.write(sink, objOut):
return False
Logging.debug(
"%s: returned %iNet, %iSta, %iLoc, %iCha, "
"%iDL, %iDec, %iSen, %iRes, %iDAExt (total objects/"
"bytes: %i/%i) " %
(ro.service, newInv.networkCount(), staCount, locCount, chaCount,
newInv.dataloggerCount(), decCount, newInv.sensorCount(),
resCount, extCount, objCount, sink.written))
utils.accessLog(req, ro, http.OK, sink.written, None)
return True
raise ("Cannot load database driver: %s" % e)
dbDriver = IO.DatabaseInterface.Create(db["dbDriverName"])
if dbDriver is None:
raise Exception("Cannot find database driver " +
db["dbDriverName"])
if not dbDriver.connect(db["dbAddress"]):
raise Exception("Cannot connect to database at " + db["dbAddress"])
dbQuery = DataModel.DatabaseQuery(dbDriver)
if dbQuery is None:
raise Exception("Cannot get DB query object")
print >> sys.stderr, " Loading inventory from database ... ",
inventory = DataModel.Inventory()
dbQuery.loadNetworks(inventory)
for ni in xrange(inventory.networkCount()):
dbQuery.loadStations(inventory.network(ni))
print >> sys.stderr, "Done."
if inventory:
self.stationResolver.collectStations(inventory, True)
def _loadXml(self, folder):
print >> sys.stderr, " Loading inventory from XML file ... ",
for f in glob.glob(os.path.join(folder, "*.xml")):
ar = IO.XMLArchive()
ar.open(f)
inventory = DataModel.Inventory_Cast(ar.readObject())
ar.close()
