def setUp(self): self.filename = 'fake' self.cluster = Cluster('testcluster') HPCStatsConf.__bases__ = (MockConfigParser, object) self.conf = HPCStatsConf(self.filename, self.cluster.name) self.conf.conf = CONFIG self.app = None self.db = None self.importer = BusinessCodeImporterCSV(self.app, self.db, self.conf) init_reqs()
def setUp(self): self.filename = 'fake' self.cluster = Cluster('testcluster') self.cluster.cluster_id = 0 HPCStatsConf.__bases__ = (MockConfigParser, object) self.conf = HPCStatsConf(self.filename, self.cluster) self.conf.conf = CONFIG.copy() self.db = HPCStatsDB(self.conf) self.db.bind() self.app = MockApp(self.db, self.conf, self.cluster) self.importer = UserImporterLdap(self.app, self.db, self.conf, self.cluster) init_reqs()
def setUp(self): self.filename = 'fake' self.cluster = Cluster('testcluster') self.cluster.cluster_id = 0 HPCStatsConf.__bases__ = (MockConfigParser, object) self.conf = HPCStatsConf(self.filename, self.cluster) self.conf.conf = CONFIG.copy() self.db = HPCStatsDB(self.conf) self.db.bind() self.app = MockApp(self.db, self.conf, self.cluster) self.importer = UserImporterLdapSlurm(self.app, self.db, self.conf, self.cluster) # Disable strict_user_membership to avoid exception when user found # in Slurm and not in LDAP then. self.importer.strict_user_membership = False init_reqs()
def test_update_2(self): """ProjectImporterCSV.update() detect existing cluster and node """ cluster1 = Cluster('cluster1') node1 = Node('node1', cluster1, 'model1', 'test_partition', 12, 6 * 1024 ** 3, 1) MockPg2.PG_REQS['find_cluster'].set_assoc( params=( cluster1.name, ), result=[ [ 1 ] ] ) MockPg2.PG_REQS['find_node'].set_assoc( params=( node1.name, cluster1.cluster_id, ), result=[ [ 1 ] ] ) self.importer.cluster = cluster1 self.importer.nodes = [ node1 ] self.importer.update()
def check_cluster_sources(self, db, cluster_name): """Check data sources for a cluster.""" cluster = None logger.info("checking architecture source for cluster %s", cluster_name) self.arch = \ ArchitectureImporterFactory.factory(self, db, self.conf, cluster_name) self.arch.check() cluster = Cluster(cluster_name) logger.info("checking users source for cluster %s", cluster.name) self.users = \ UserImporterFactory.factory(self, db, self.conf, cluster) self.users.check() logger.info("checking filesystem usage source for cluster %s", cluster.name) self.fsusage = \ FSUsageImporterFactory.factory(self, db, self.conf, cluster) self.fsusage.check() logger.info("checking filesystem quota source for cluster %s", cluster.name) self.fsquota = \ FSQuotaImporterFactory.factory(self, db, self.conf, cluster) self.fsquota.check() logger.info("checking events source for cluster %s", cluster.name) self.events = \ EventImporterFactory.factory(self, db, self.conf, cluster) self.events.check() logger.info("checking jobs source for cluster %s", cluster.name) self.jobs = \ JobImporterFactory.factory(self, db, self.conf, cluster) self.jobs.check() logger.info("every sources are properly available")
def test_update(self): """ProjectImporterCSV.update() creates cluster and node if not existing """ cluster1 = Cluster('cluster1') node1 = Node('node1', cluster1, 'model1', 'test_partition', 12, 6 * 1024 ** 3, 1) MockPg2.PG_REQS['save_cluster'].set_assoc( params=( cluster1.name ), result=[ [ 1 ] ] ) MockPg2.PG_REQS['save_node'].set_assoc( params=( node1.name, cluster1.cluster_id, node1.partition, node1.cpu, node1.memory, node1.flops ), result=[ [ 1 ] ] ) self.importer.cluster = cluster1 self.importer.nodes = [ node1 ] self.importer.update()
def setUp(self): # setup conf self.filename = 'fake' self.cluster = Cluster('testcluster') HPCStatsConf.__bases__ = (MockConfigParser, object) self.conf = HPCStatsConf(self.filename, self.cluster) self.conf.conf = CONFIG.copy() # setup importer self.db = HPCStatsDB(self.conf) self.db.bind() self.app = MockApp(self.db, self.conf, self.cluster) self.importer = EventImporterSlurm(self.app, self.db, self.conf, self.cluster) init_reqs() # setup logger logging.setLoggerClass(HPCStatsLogger) self.logger = logging.getLogger(__name__) self.handler = MockLoggingHandler() self.logger.addHandler(self.handler) self.handler.reset() HPCStatsLogger.set_error_mgr(HPCStatsErrorMgr(self.conf))
def load(self): """Load Cluster, Nodes and partitions from Architecture files. Raises HPCStatsRuntimeError or HPCStatsSourceError if error is encountered while loading data from sources. It sets attributes cluster, nodes and partitions with loaded data. """ self.cluster = Cluster(self.cluster_name) self.nodes = [] self.partitions = {} self.read_arch() config_get = self.config_get partitions = config_get(self.cluster.name, "partitions").split(',') for partition in partitions: part_sect = self.cluster.name + "/" + partition nodegroups = config_get(part_sect, "nodegroups").split(',') job_partitions = config_get(part_sect, "job_partitions") \ .split(',') nodeset_part = NodeSet() # nodeset for the partitions attribute for nodegroup in nodegroups: nodegroup_sect = self.cluster.name + "/" + partition \ + "/" + nodegroup nodenames = config_get(nodegroup_sect, "names") nodeset_part.add(nodenames) sockets = config_get(nodegroup_sect, "sockets", isint=True) cores_per_socket = config_get(nodegroup_sect, "corespersocket", isint=True) cpu = sockets * cores_per_socket float_instructions = config_get(nodegroup_sect, "floatinstructions", isint=True) freq_str = config_get(nodegroup_sect, "frequency") freq = ArchitectureImporterArchfile.convert_freq(freq_str) if freq is None: raise HPCStatsSourceError( \ "format of frequency for nodeset %s/%s/%s (%s) " \ "'%s' is not valid" \ % ( self.cluster.name, partition, nodegroup, nodenames, freq_str )) flops = sockets * cores_per_socket * float_instructions * freq mem_str = config_get(nodegroup_sect, "memory") mem = ArchitectureImporterArchfile.convert_mem(mem_str) if mem is None: raise HPCStatsSourceError( \ "format of memory for nodeset %s/%s/%s (%s) " \ "'%s' is not valid" \ % ( self.cluster.name, partition, nodegroup, nodenames, mem_str )) model = config_get(nodegroup_sect, "model") nodeset_group = NodeSet(nodenames) for nodename in nodeset_group: # create and append node new_node = Node(name=nodename, cluster=self.cluster, model=model, partition=partition, cpu=cpu, memory=mem, flops=flops) self.nodes.append(new_node) self.partitions[str(nodeset_part)] = job_partitions
def run(self): """Run HPCStats Reporter application.""" self.run_check() logger.debug("running on cluster %s with interval %s" \ % (self.cluster_name, self.interval)) db = self.new_db() self.cluster = Cluster(self.cluster_name) # check if cluster really exists if not cluster.find(db): raise HPCStatsRuntimeError( \ "cluster %s does not exist in database. " \ "Available clusters are: %s." \ % (cluster, ",".join(available_clusters))) # get the total number of cpus inside the cluster logger.debug("main: getting nb cpus on cluster %s" % (cluster.name)) nb_cpus_cluster = cluster.get_nb_cpus(db) results = [] # get datetime of the first job min_datetime = cluster.get_min_datetime(db) #min_datetime = datetime(2011,5,1,0,0,0) max_datetime = datetime.now() tmp_datetime = min_datetime db.unbind() userstats_global = {} groupstats_global = {} processes_args = [] # construct intervals with process information mapping while tmp_datetime < max_datetime: # get the exacts beginning and end of the step sized interval # around the tmp datetime (begin,end) = self.get_interval_begin_end(tmp_datetime) # construct an array of args for each process/interval process_info = [] process_info.append(begin) process_info.append(end) # finally appends this append to the global array processes_args.append(process_info) # going to next interval interval = self.get_interval_timedelta() tmp_datetime += interval logger.debug(processes_args) # launch processes with their corresponding arguments parallel = True processes_results = [] if parallel: pool = Pool(4) processes_results = pool.map(self.run_interval, processes_args) else: for process_info in processes_args: process_results = run_interval(process_info) processes_results.append(process_results) # then get results for result in processes_results: str_date = result[0] groupstats = result.pop() userstats = result.pop() userstats_global[str_date] = userstats groupstats_global[str_date] = groupstats results.append(result) logger.debug("usersstats", userstats_global) logger.debug("groupsstats", groupstats_global) # print results using template mytemplate = Template( filename=self.get_template_filename(), input_encoding='utf-8', output_encoding='utf-8', default_filters=['decode.utf8'], encoding_errors='ignore' ) print mytemplate.render( cluster=cluster, step=self.interval, results=results, userstats_global=userstats_global, groupstats_global=groupstats_global)