def main(args=sys.argv): # Command line argument parser usage = "%prog [options] command" parser = StatsOptionParser(usage) (options, args) = parser.parse_args(args[1:]) # validate options parser.validate(options) # configure logging logging_level = logging.INFO if options.debug: logging_level = logging.DEBUG logging.basicConfig(format = '%(levelname)s: %(filename)s: %(message)s', level = logging_level, stream = sys.stdout) # Config file argument parser config = HPCStatsConfig(options) # dump entire config file for section in config.sections(): logging.debug(section) for option in config.options(section): logging.debug(" %s = %s", option, config.get(section, option)) # Instantiate connexion to db db_section = "hpcstatsdb" dbhostname = config.get(db_section,"hostname") dbport = config.get(db_section,"port") dbname = config.get(db_section,"dbname") dbuser = config.get(db_section,"user") dbpass = config.get(db_section,"password") db = HPCStatsdb(dbhostname, dbport, dbname, dbuser, dbpass) db.bind() logging.debug("db information %s %s %s %s %s" % db.infos()) cluster_finder = ClusterFinder(db) cluster = cluster_finder.find(options.clustername) if (options.arch): logging.info("=> Updating architecture for cluster %s" % (options.clustername)) try: arch_importer = ArchitectureImporterFactory().factory(db, config, cluster.get_name()) arch_importer.update_architecture() db.commit() except RuntimeError: logging.error("error occured on architecture update.") if (options.events): logging.info("=> Updating events for cluster %s" % (options.clustername)) try: event_importer = EventImporterFactory().factory(db, config, cluster.get_name()) event_importer.update_events() db.commit() except RuntimeError: logging.error("error occured on events update.") if (options.users): logging.info("=> Updating users for cluster %s" % (options.clustername)) try: user_importer = UserImporterFactory().factory(db, config, cluster.get_name()) user_importer.update_users() db.commit() except RuntimeError: logging.error("error occured on users update.") if (options.jobs): logging.info("=> Update of jobs for cluster %s" % (options.clustername)) try: job_importer = JobImporterFactory().factory(db, config, cluster.get_name()) # The last updated job in hpcstatsdb for this cluster last_updated_id = job_importer.get_last_job_id() # The unfinished jobs in hpcstatsdb for this cluster ids = job_importer.get_unfinished_job_id() jobs_to_update = ['not_empty'] new_jobs = ['not_empty'] nb_theads = 4 offset = 0 max_jobs = 100000 logging.debug("Get jobs to update") jobs_to_update = job_importer.get_job_information_from_dbid_job_list(ids) for job in jobs_to_update: offset = offset + 1 if not offset % 10: logging.debug("update job push %d" % offset) job.update(db) offset = 0 while new_jobs: logging.debug("Get %d new jobs starting at offset %d" % (max_jobs, offset)) new_jobs = job_importer.get_job_for_id_above(last_updated_id, offset, max_jobs) for job in new_jobs: offset = offset + 1 if not offset % 10000: logging.debug("create job push %d" % offset) job.save(db) db.commit() except RuntimeError: logging.error("error occured on jobs update.") db.unbind()