def setUp(self): self.filename = 'fake' self.cluster = Cluster('testcluster') HPCStatsConf.__bases__ = (MockConfigParser, object) self.conf = HPCStatsConf(self.filename, self.cluster) self.conf.conf = CONFIG.copy() self.db = HPCStatsDB(self.conf) self.db.bind() self.app = MockApp(self.db, self.conf, self.cluster) self.importer = JobImporterSlurm(self.app, self.db, self.conf, self.cluster) init_reqs()
def factory(app, db, config, cluster): """This method returns the appropriate JobImporter object depending on what is specified in configuration. In case of configuration error, HPCStatsConfigurationException is raised. """ implem = config.get(cluster.name, 'jobs') if implem == "slurm": return JobImporterSlurm(app, db, config, cluster) else: raise HPCStatsConfigurationException( \ "JobImporter %s is not implemented" \ % (implem))
class TestsJobImporterSlurm(HPCStatsTestCase): @mock.patch("HPCStats.DB.HPCStatsDB.psycopg2", mock_psycopg2()) def setUp(self): self.filename = 'fake' self.cluster = Cluster('testcluster') HPCStatsConf.__bases__ = (MockConfigParser, object) self.conf = HPCStatsConf(self.filename, self.cluster) self.conf.conf = CONFIG.copy() self.db = HPCStatsDB(self.conf) self.db.bind() self.app = MockApp(self.db, self.conf, self.cluster) self.importer = JobImporterSlurm(self.app, self.db, self.conf, self.cluster) init_reqs() def test_init(self): """JobImporterSlurm.__init__() initializes object with attributes """ self.assertEquals(self.importer._dbhost, self.conf.conf[self.cluster.name + '/slurm']['host']) def load_app(self): """Load App objects for JobImporterSlurm.load() normal operation.""" j1_submit = datetime(2015, 3, 2, 16, 0, 1) j1_start = datetime(2015, 3, 2, 16, 0, 2) j1_end = datetime(2015, 3, 2, 16, 0, 3) j1_submit_ts = time.mktime(j1_submit.timetuple()) j1_start_ts = time.mktime(j1_start.timetuple()) j1_end_ts = time.mktime(j1_end.timetuple()) node1 = Node('node1', self.cluster, 'model1', 'partition1', 4, 4, 0) node2 = Node('node2', self.cluster, 'model1', 'partition1', 4, 4, 0) a1_create = datetime(2010, 1, 1, 12, 0, 0) user1 = User('user1', 'firstname1', 'lastname1', 'department1') account1 = Account(user1, self.cluster, 1000, 1000, a1_create, None) domain1 = Domain('domain1', 'domain 1') project1 = Project(domain1, 'project1', 'description project 1') business1 = Business('business1', 'business description 1') self.app.arch.nodes = [ node1, node2 ] self.app.users.accounts = [ account1 ] self.app.projects.projects = [ project1 ] self.app.business.businesses = [ business1 ] MockMySQLdb.MY_REQS['get_jobs_after_batchid']['res'] = \ [ [ 0, 0, 1000, 1000, j1_submit_ts, j1_start_ts, j1_end_ts, 2, '1=4', 'partition1', 'qos1', 1, 'node[1-2]', 'user1', 'job1', 'project1:business1' ], ] @mock.patch("%s.MySQLdb" % (module), mock_mysqldb()) def test_is_old_schema(self): """JobImporterSlurm._is_old_schema() should return True is SlurmDBD <15.08 is detected, False otherwise.""" self.load_app() self.importer.connect_db() MockMySQLdb.MY_REQS['job_table_cols']['res'] = \ [ [ 'cpus_alloc', ] , ] self.assertEquals(self.importer._is_old_schema(), True) MockMySQLdb.MY_REQS['job_table_cols']['res'] = [] self.assertEquals(self.importer._is_old_schema(), False) @mock.patch("%s.MySQLdb" % (module), mock_mysqldb()) def test_load(self): """JobImporterSlurm.load() works with simple data.""" self.load_app() # make sure new-schema is used here MockMySQLdb.MY_REQS['job_table_cols']['res'] = [ ] self.importer.load() self.assertEquals(len(self.importer.jobs), 1) self.assertEquals(len(self.importer.runs), 2) job = self.importer.jobs[0] self.assertEquals(job.nbcpu, 4) self.assertEquals(job.state, 'RUNNING') self.assertEquals(job.name, 'job1') self.assertEquals(job.queue, 'partition1-qos1') self.assertEquals(job.account, self.app.users.accounts[0]) self.assertEquals(job.project, self.app.projects.projects[0]) self.assertEquals(job.business, self.app.business.businesses[0]) @mock.patch("%s.MySQLdb" % (module), mock_mysqldb()) def test_load_old_schema(self): """JobImporterSlurm.load() works with simple data from old SlurmDBD <15.08 schema.""" self.load_app() MockMySQLdb.MY_REQS['job_table_cols']['res'] = \ [ [ 'cpus_alloc', ] , ] # replace TRES '1=4' by cpus_alloc 4 MockMySQLdb.MY_REQS['get_jobs_after_batchid']['res'][0][8] = 4 self.importer.load() self.assertEquals(len(self.importer.jobs), 1) self.assertEquals(len(self.importer.runs), 2) job = self.importer.jobs[0] self.assertEquals(job.nbcpu, 4) self.assertEquals(job.state, 'RUNNING') self.assertEquals(job.name, 'job1') self.assertEquals(job.queue, 'partition1-qos1') self.assertEquals(job.account, self.app.users.accounts[0]) self.assertEquals(job.project, self.app.projects.projects[0]) self.assertEquals(job.business, self.app.business.businesses[0]) @mock.patch("%s.MySQLdb" % (module), mock_mysqldb()) @mock.patch("%s.JobImporterSlurm.get_jobs_after_batchid" % (module)) def test_load_search_batchid(self, mock_get_jobs): """JobImporterSlurm.load() must search jobs after correct batch_id.""" MockPg2.PG_REQS['get_batchid_oldest_unfinished'].set_assoc( params=( self.cluster.cluster_id ), result=[ [ 2 ] ] ) MockPg2.PG_REQS['get_batchid_last'].set_assoc( params=( self.cluster.cluster_id ), result=[ [ 3 ] ] ) self.importer.load() mock_get_jobs.assert_called_with(2) # None unfinished job, search must be done with batch_id of lasti job. MockPg2.PG_REQS['get_batchid_oldest_unfinished'].set_assoc( params=( self.cluster.cluster_id ), result=[ ] ) MockPg2.PG_REQS['get_batchid_last'].set_assoc( params=( self.cluster.cluster_id ), result=[ [ 4 ] ] ) self.importer.load() mock_get_jobs.assert_called_with(4) # No job in DB: search starting -1. MockPg2.PG_REQS['get_batchid_oldest_unfinished'].set_assoc( params=( self.cluster.cluster_id ), result=[ ] ) MockPg2.PG_REQS['get_batchid_last'].set_assoc( params=( self.cluster.cluster_id ), result=[ ] ) self.importer.load() mock_get_jobs.assert_called_with(-1) @mock.patch("%s.MySQLdb" % (module), mock_mysqldb()) def test_load_account_not_found(self): """JobImporterSlurm.load() raises exception when account not found""" self.load_app() self.app.users.accounts = [ ] self.assertRaisesRegexp( HPCStatsSourceError, "account user1 not found in loaded account", self.importer.load) @mock.patch("%s.MySQLdb" % (module), mock_mysqldb()) def test_load_invalid_tres(self): """JobImporterSlurm.load() raises exception if invalid tres for a job is found""" self.load_app() MockMySQLdb.MY_REQS['get_jobs_after_batchid']['res'][0][8] = '0=0' self.assertRaisesRegexp( HPCStatsSourceError, "unable to extract cpus_alloc from job tres", self.importer.load) @mock.patch("%s.MySQLdb" % (module), mock_mysqldb()) def test_load_invalid_wckey(self): """JobImporterSlurm.load() raises exception when format of wckey is invalid. """ self.load_app() MockMySQLdb.MY_REQS['get_jobs_after_batchid']['res'][0][15] = 'fail' self.assertRaisesRegexp( HPCStatsSourceError, "format of wckey fail is not valid", self.importer.load) @mock.patch("%s.MySQLdb" % (module), mock_mysqldb()) def test_load_project_not_found(self): """JobImporterSlurm.load() raises exception when project not found.""" self.load_app() self.app.projects.projects = [ ] self.assertRaisesRegexp( HPCStatsSourceError, "project project1 not found in loaded projects", self.importer.load) @mock.patch("%s.MySQLdb" % (module), mock_mysqldb()) def test_load_business_not_found(self): """JobImporterSlurm.load() raises exception when business not found.""" self.load_app() self.app.business.businesses = [ ] self.assertRaisesRegexp( HPCStatsSourceError, "business code business1 not found in loaded business codes", self.importer.load) @mock.patch("%s.MySQLdb" % (module), mock_mysqldb()) def test_load_invalid_nodelist(self): """JobImporterSlurm.load() raises exception when format of nodelist is invalid. """ self.load_app() MockMySQLdb.MY_REQS['get_jobs_after_batchid']['res'][0][12] = \ 'nodelistfail[5-4]' self.assertRaisesRegexp( HPCStatsSourceError, "could not parse nodeset nodelistfail\[5\-4\] for job 0", self.importer.load) @mock.patch("%s.MySQLdb" % (module), mock_mysqldb()) def test_load_node_not_found(self): """JobImporterSlurm.load() raises exception when node not found.""" self.load_app() self.app.arch.nodes = [ ] self.assertRaisesRegexp( HPCStatsSourceError, "unable to find node node1 for job 0 in loaded nodes", self.importer.load) def test_job_partition(self): """JobImporterSlurm.job_partition() must return correct partition for based on job partition list and its nodelist. """ # Only one element in job partition list: it must be returned whatever # the nodelist and ArchitectureImporter job partitions self.app.arch.partitions = { } result = self.importer.job_partition(0, 'partition2', 'node[1-100]') self.assertEquals(result, 'partition2') # Multiple elements but None nodelist: it must return arbitrary the # first partition self.app.arch.partitions = { } result = self.importer.job_partition(0, 'partition1,partition2', None) self.assertEquals(result, 'partition1') # Multiple elements in partition and defined nodelist: it must return # a corresponding partition loaded by ArchitectureImporter and # associated to a nodelist that fully intersects self.app.arch.partitions = { 'node[1-100]': [ 'partitionX', 'partition2'] } result = self.importer.job_partition(0, 'partition1,partition2', 'node[1-100]') self.assertEquals(result, 'partition2') self.app.arch.partitions = { 'node[1-99]': [ 'partition1' ], 'node[1-100],bm[1-10]': [ 'partitionX', 'partition2' ] } result = self.importer.job_partition(0, 'partition1,partition2', 'node[1-100]') self.assertEquals(result, 'partition2')