def import_cluster_data(self, db, cluster_name):
"""Import from sources all data specific to a cluster."""
cluster = None
#
# Architecture Importer is both responsible for importing/updating data
# about cluster and nodes in database and creating the Cluster object
# for other importers.
#
logger.info("updating architecture for cluster %s", cluster_name)
self.arch = \
ArchitectureImporterFactory.factory(self, db, self.conf,
cluster_name)
self.arch.load()
self.arch.update()
cluster = self.arch.cluster
# check that cluster has been properly created and initialized
if cluster is None or cluster.cluster_id is None:
raise HPCStatsRuntimeError("problem in DB with cluster %s" %
(str(cluster)))
logger.info("updating users for cluster %s", cluster.name)
self.users = \
UserImporterFactory.factory(self, db, self.conf, cluster)
self.users.load()
self.users.update()
logger.info("updating filesystem usage for cluster %s", cluster.name)
self.fsusage = \
FSUsageImporterFactory.factory(self, db, self.conf, cluster)
self.fsusage.load()
self.fsusage.update()
logger.info("updating filesystem quota for cluster %s", cluster.name)
self.fsquota = \
FSQuotaImporterFactory.factory(self, db, self.conf, cluster)
self.fsquota.load()
self.fsquota.update()
logger.info("updating events for cluster %s", cluster.name)
self.events = \
EventImporterFactory.factory(self, db, self.conf, cluster)
self.events.load()
self.events.update()
logger.info("updating jobs for cluster %s", cluster.name)
self.jobs = JobImporterFactory.factory(self, db, self.conf, cluster)
self.jobs.load_update_window()
def import_cluster_data(self, db, cluster_name):
"""Import from sources all data specific to a cluster."""
cluster = None
#
# Architecture Importer is both responsible for importing/updating data
# about cluster and nodes in database and creating the Cluster object
# for other importers.
#
logger.info("updating architecture for cluster %s", cluster_name)
self.arch = \
ArchitectureImporterFactory.factory(self, db, self.conf,
cluster_name)
self.arch.load()
self.arch.update()
cluster = self.arch.cluster
# check that cluster has been properly created and initialized
if cluster is None or cluster.cluster_id is None:
raise HPCStatsRuntimeError("problem in DB with cluster %s"
% (str(cluster)))
logger.info("updating users for cluster %s", cluster.name)
self.users = \
UserImporterFactory.factory(self, db, self.conf, cluster)
self.users.load()
self.users.update()
logger.info("updating filesystem usage for cluster %s", cluster.name)
self.fsusage = \
FSUsageImporterFactory.factory(self, db, self.conf, cluster)
self.fsusage.load()
self.fsusage.update()
logger.info("updating events for cluster %s", cluster.name)
self.events = \
EventImporterFactory.factory(self, db, self.conf, cluster)
self.events.load()
self.events.update()
logger.info("updating jobs for cluster %s", cluster.name)
self.jobs = JobImporterFactory.factory(self, db, self.conf, cluster)
self.jobs.load_update_window()
def check_cluster_sources(self, db, cluster_name):
"""Check data sources for a cluster."""
cluster = None
logger.info("checking architecture source for cluster %s",
cluster_name)
self.arch = \
ArchitectureImporterFactory.factory(self, db, self.conf,
cluster_name)
self.arch.check()
cluster = Cluster(cluster_name)
logger.info("checking users source for cluster %s", cluster.name)
self.users = \
UserImporterFactory.factory(self, db, self.conf, cluster)
self.users.check()
logger.info("checking filesystem usage source for cluster %s",
cluster.name)
self.fsusage = \
FSUsageImporterFactory.factory(self, db, self.conf, cluster)
self.fsusage.check()
logger.info("checking filesystem quota source for cluster %s",
cluster.name)
self.fsquota = \
FSQuotaImporterFactory.factory(self, db, self.conf, cluster)
self.fsquota.check()
logger.info("checking events source for cluster %s", cluster.name)
self.events = \
EventImporterFactory.factory(self, db, self.conf, cluster)
self.events.check()
logger.info("checking jobs source for cluster %s", cluster.name)
self.jobs = \
JobImporterFactory.factory(self, db, self.conf, cluster)
self.jobs.check()
logger.info("every sources are properly available")
def check_cluster_sources(self, db, cluster_name):
"""Check data sources for a cluster."""
cluster = None
logger.info("checking architecture source for cluster %s",
cluster_name)
self.arch = \
ArchitectureImporterFactory.factory(self, db, self.conf,
cluster_name)
self.arch.check()
cluster = Cluster(cluster_name)
logger.info("checking users source for cluster %s",
cluster.name)
self.users = \
UserImporterFactory.factory(self, db, self.conf, cluster)
self.users.check()
logger.info("checking filesystem usage source for cluster %s",
cluster.name)
self.fsusage = \
FSUsageImporterFactory.factory(self, db, self.conf, cluster)
self.fsusage.check()
logger.info("checking events source for cluster %s",
cluster.name)
self.events = \
EventImporterFactory.factory(self, db, self.conf, cluster)
self.events.check()
logger.info("checking jobs source for cluster %s",
cluster.name)
self.jobs = \
JobImporterFactory.factory(self, db, self.conf, cluster)
self.jobs.check()
logger.info("every sources are properly available")
def main(args=sys.argv):
# Command line argument parser
usage = "%prog [options] command"
parser = StatsOptionParser(usage)
(options, args) = parser.parse_args(args[1:])
# validate options
parser.validate(options)
# configure logging
logging_level = logging.INFO
if options.debug:
logging_level = logging.DEBUG
logging.basicConfig(format = '%(levelname)s: %(filename)s: %(message)s',
level = logging_level,
stream = sys.stdout)
# Config file argument parser
config = HPCStatsConfig(options)
# dump entire config file
for section in config.sections():
logging.debug(section)
for option in config.options(section):
logging.debug(" %s = %s", option, config.get(section, option))
# Instantiate connexion to db
db_section = "hpcstatsdb"
dbhostname = config.get(db_section,"hostname")
dbport = config.get(db_section,"port")
dbname = config.get(db_section,"dbname")
dbuser = config.get(db_section,"user")
dbpass = config.get(db_section,"password")
db = HPCStatsdb(dbhostname, dbport, dbname, dbuser, dbpass)
db.bind()
logging.debug("db information %s %s %s %s %s" % db.infos())
cluster_finder = ClusterFinder(db)
cluster = cluster_finder.find(options.clustername)
if (options.arch):
logging.info("=> Updating architecture for cluster %s" % (options.clustername))
try:
arch_importer = ArchitectureImporterFactory().factory(db, config, cluster.get_name())
arch_importer.update_architecture()
db.commit()
except RuntimeError:
logging.error("error occured on architecture update.")
if (options.events):
logging.info("=> Updating events for cluster %s" % (options.clustername))
try:
event_importer = EventImporterFactory().factory(db, config, cluster.get_name())
event_importer.update_events()
db.commit()
except RuntimeError:
logging.error("error occured on events update.")
if (options.users):
logging.info("=> Updating users for cluster %s" % (options.clustername))
try:
user_importer = UserImporterFactory().factory(db, config, cluster.get_name())
user_importer.update_users()
db.commit()
except RuntimeError:
logging.error("error occured on users update.")
if (options.jobs):
logging.info("=> Update of jobs for cluster %s" % (options.clustername))
try:
job_importer = JobImporterFactory().factory(db, config, cluster.get_name())
# The last updated job in hpcstatsdb for this cluster
last_updated_id = job_importer.get_last_job_id()
# The unfinished jobs in hpcstatsdb for this cluster
ids = job_importer.get_unfinished_job_id()
jobs_to_update = ['not_empty']
new_jobs = ['not_empty']
nb_theads = 4
offset = 0
max_jobs = 100000
logging.debug("Get jobs to update")
jobs_to_update = job_importer.get_job_information_from_dbid_job_list(ids)
for job in jobs_to_update:
offset = offset + 1
if not offset % 10:
logging.debug("update job push %d" % offset)
job.update(db)
offset = 0
while new_jobs:
logging.debug("Get %d new jobs starting at offset %d" % (max_jobs, offset))
new_jobs = job_importer.get_job_for_id_above(last_updated_id, offset, max_jobs)
for job in new_jobs:
offset = offset + 1
if not offset % 10000:
logging.debug("create job push %d" % offset)
job.save(db)
db.commit()
except RuntimeError:
logging.error("error occured on jobs update.")
db.unbind()