def setUp(self):
self.filename = 'fake'
self.cluster = Cluster('testcluster')
HPCStatsConf.__bases__ = (MockConfigParser, object)
self.conf = HPCStatsConf(self.filename, self.cluster)
self.conf.conf = CONFIG.copy()
self.db = HPCStatsDB(self.conf)
self.db.bind()
self.app = MockApp(self.db, self.conf, self.cluster)
self.importer = JobImporterSlurm(self.app,
self.db,
self.conf,
self.cluster)
init_reqs()
def factory(app, db, config, cluster):
"""This method returns the appropriate JobImporter object depending
on what is specified in configuration. In case of configuration
error, HPCStatsConfigurationException is raised.
"""
implem = config.get(cluster.name, 'jobs')
if implem == "slurm":
return JobImporterSlurm(app, db, config, cluster)
else:
raise HPCStatsConfigurationException( \
"JobImporter %s is not implemented" \
% (implem))
class TestsJobImporterSlurm(HPCStatsTestCase):
@mock.patch("HPCStats.DB.HPCStatsDB.psycopg2", mock_psycopg2())
def setUp(self):
self.filename = 'fake'
self.cluster = Cluster('testcluster')
HPCStatsConf.__bases__ = (MockConfigParser, object)
self.conf = HPCStatsConf(self.filename, self.cluster)
self.conf.conf = CONFIG.copy()
self.db = HPCStatsDB(self.conf)
self.db.bind()
self.app = MockApp(self.db, self.conf, self.cluster)
self.importer = JobImporterSlurm(self.app,
self.db,
self.conf,
self.cluster)
init_reqs()
def test_init(self):
"""JobImporterSlurm.__init__() initializes object with attributes
"""
self.assertEquals(self.importer._dbhost,
self.conf.conf[self.cluster.name + '/slurm']['host'])
def load_app(self):
"""Load App objects for JobImporterSlurm.load() normal operation."""
j1_submit = datetime(2015, 3, 2, 16, 0, 1)
j1_start = datetime(2015, 3, 2, 16, 0, 2)
j1_end = datetime(2015, 3, 2, 16, 0, 3)
j1_submit_ts = time.mktime(j1_submit.timetuple())
j1_start_ts = time.mktime(j1_start.timetuple())
j1_end_ts = time.mktime(j1_end.timetuple())
node1 = Node('node1', self.cluster, 'model1', 'partition1', 4, 4, 0)
node2 = Node('node2', self.cluster, 'model1', 'partition1', 4, 4, 0)
a1_create = datetime(2010, 1, 1, 12, 0, 0)
user1 = User('user1', 'firstname1', 'lastname1', 'department1')
account1 = Account(user1, self.cluster, 1000, 1000, a1_create, None)
domain1 = Domain('domain1', 'domain 1')
project1 = Project(domain1, 'project1', 'description project 1')
business1 = Business('business1', 'business description 1')
self.app.arch.nodes = [ node1, node2 ]
self.app.users.accounts = [ account1 ]
self.app.projects.projects = [ project1 ]
self.app.business.businesses = [ business1 ]
MockMySQLdb.MY_REQS['get_jobs_after_batchid']['res'] = \
[
[ 0, 0, 1000, 1000, j1_submit_ts, j1_start_ts, j1_end_ts,
2, '1=4', 'partition1', 'qos1', 1, 'node[1-2]', 'user1',
'job1', 'project1:business1' ],
]
@mock.patch("%s.MySQLdb" % (module), mock_mysqldb())
def test_is_old_schema(self):
"""JobImporterSlurm._is_old_schema() should return True is SlurmDBD
<15.08 is detected, False otherwise."""
self.load_app()
self.importer.connect_db()
MockMySQLdb.MY_REQS['job_table_cols']['res'] = \
[ [ 'cpus_alloc', ] , ]
self.assertEquals(self.importer._is_old_schema(), True)
MockMySQLdb.MY_REQS['job_table_cols']['res'] = []
self.assertEquals(self.importer._is_old_schema(), False)
@mock.patch("%s.MySQLdb" % (module), mock_mysqldb())
def test_load(self):
"""JobImporterSlurm.load() works with simple data."""
self.load_app()
# make sure new-schema is used here
MockMySQLdb.MY_REQS['job_table_cols']['res'] = [ ]
self.importer.load()
self.assertEquals(len(self.importer.jobs), 1)
self.assertEquals(len(self.importer.runs), 2)
job = self.importer.jobs[0]
self.assertEquals(job.nbcpu, 4)
self.assertEquals(job.state, 'RUNNING')
self.assertEquals(job.name, 'job1')
self.assertEquals(job.queue, 'partition1-qos1')
self.assertEquals(job.account, self.app.users.accounts[0])
self.assertEquals(job.project, self.app.projects.projects[0])
self.assertEquals(job.business, self.app.business.businesses[0])
@mock.patch("%s.MySQLdb" % (module), mock_mysqldb())
def test_load_old_schema(self):
"""JobImporterSlurm.load() works with simple data from old SlurmDBD
<15.08 schema."""
self.load_app()
MockMySQLdb.MY_REQS['job_table_cols']['res'] = \
[ [ 'cpus_alloc', ] , ]
# replace TRES '1=4' by cpus_alloc 4
MockMySQLdb.MY_REQS['get_jobs_after_batchid']['res'][0][8] = 4
self.importer.load()
self.assertEquals(len(self.importer.jobs), 1)
self.assertEquals(len(self.importer.runs), 2)
job = self.importer.jobs[0]
self.assertEquals(job.nbcpu, 4)
self.assertEquals(job.state, 'RUNNING')
self.assertEquals(job.name, 'job1')
self.assertEquals(job.queue, 'partition1-qos1')
self.assertEquals(job.account, self.app.users.accounts[0])
self.assertEquals(job.project, self.app.projects.projects[0])
self.assertEquals(job.business, self.app.business.businesses[0])
@mock.patch("%s.MySQLdb" % (module), mock_mysqldb())
@mock.patch("%s.JobImporterSlurm.get_jobs_after_batchid" % (module))
def test_load_search_batchid(self, mock_get_jobs):
"""JobImporterSlurm.load() must search jobs after correct batch_id."""
MockPg2.PG_REQS['get_batchid_oldest_unfinished'].set_assoc(
params=( self.cluster.cluster_id ),
result=[ [ 2 ] ]
)
MockPg2.PG_REQS['get_batchid_last'].set_assoc(
params=( self.cluster.cluster_id ),
result=[ [ 3 ] ]
)
self.importer.load()
mock_get_jobs.assert_called_with(2)
# None unfinished job, search must be done with batch_id of lasti job.
MockPg2.PG_REQS['get_batchid_oldest_unfinished'].set_assoc(
params=( self.cluster.cluster_id ),
result=[ ]
)
MockPg2.PG_REQS['get_batchid_last'].set_assoc(
params=( self.cluster.cluster_id ),
result=[ [ 4 ] ]
)
self.importer.load()
mock_get_jobs.assert_called_with(4)
# No job in DB: search starting -1.
MockPg2.PG_REQS['get_batchid_oldest_unfinished'].set_assoc(
params=( self.cluster.cluster_id ),
result=[ ]
)
MockPg2.PG_REQS['get_batchid_last'].set_assoc(
params=( self.cluster.cluster_id ),
result=[ ]
)
self.importer.load()
mock_get_jobs.assert_called_with(-1)
@mock.patch("%s.MySQLdb" % (module), mock_mysqldb())
def test_load_account_not_found(self):
"""JobImporterSlurm.load() raises exception when account not found"""
self.load_app()
self.app.users.accounts = [ ]
self.assertRaisesRegexp(
HPCStatsSourceError,
"account user1 not found in loaded account",
self.importer.load)
@mock.patch("%s.MySQLdb" % (module), mock_mysqldb())
def test_load_invalid_tres(self):
"""JobImporterSlurm.load() raises exception if invalid tres for a job
is found"""
self.load_app()
MockMySQLdb.MY_REQS['get_jobs_after_batchid']['res'][0][8] = '0=0'
self.assertRaisesRegexp(
HPCStatsSourceError,
"unable to extract cpus_alloc from job tres",
self.importer.load)
@mock.patch("%s.MySQLdb" % (module), mock_mysqldb())
def test_load_invalid_wckey(self):
"""JobImporterSlurm.load() raises exception when format of wckey is
invalid.
"""
self.load_app()
MockMySQLdb.MY_REQS['get_jobs_after_batchid']['res'][0][15] = 'fail'
self.assertRaisesRegexp(
HPCStatsSourceError,
"format of wckey fail is not valid",
self.importer.load)
@mock.patch("%s.MySQLdb" % (module), mock_mysqldb())
def test_load_project_not_found(self):
"""JobImporterSlurm.load() raises exception when project not found."""
self.load_app()
self.app.projects.projects = [ ]
self.assertRaisesRegexp(
HPCStatsSourceError,
"project project1 not found in loaded projects",
self.importer.load)
@mock.patch("%s.MySQLdb" % (module), mock_mysqldb())
def test_load_business_not_found(self):
"""JobImporterSlurm.load() raises exception when business not found."""
self.load_app()
self.app.business.businesses = [ ]
self.assertRaisesRegexp(
HPCStatsSourceError,
"business code business1 not found in loaded business codes",
self.importer.load)
@mock.patch("%s.MySQLdb" % (module), mock_mysqldb())
def test_load_invalid_nodelist(self):
"""JobImporterSlurm.load() raises exception when format of nodelist
is invalid.
"""
self.load_app()
MockMySQLdb.MY_REQS['get_jobs_after_batchid']['res'][0][12] = \
'nodelistfail[5-4]'
self.assertRaisesRegexp(
HPCStatsSourceError,
"could not parse nodeset nodelistfail\[5\-4\] for job 0",
self.importer.load)
@mock.patch("%s.MySQLdb" % (module), mock_mysqldb())
def test_load_node_not_found(self):
"""JobImporterSlurm.load() raises exception when node not found."""
self.load_app()
self.app.arch.nodes = [ ]
self.assertRaisesRegexp(
HPCStatsSourceError,
"unable to find node node1 for job 0 in loaded nodes",
self.importer.load)
def test_job_partition(self):
"""JobImporterSlurm.job_partition() must return correct partition for
based on job partition list and its nodelist.
"""
# Only one element in job partition list: it must be returned whatever
# the nodelist and ArchitectureImporter job partitions
self.app.arch.partitions = { }
result = self.importer.job_partition(0, 'partition2', 'node[1-100]')
self.assertEquals(result, 'partition2')
# Multiple elements but None nodelist: it must return arbitrary the
# first partition
self.app.arch.partitions = { }
result = self.importer.job_partition(0, 'partition1,partition2', None)
self.assertEquals(result, 'partition1')
# Multiple elements in partition and defined nodelist: it must return
# a corresponding partition loaded by ArchitectureImporter and
# associated to a nodelist that fully intersects
self.app.arch.partitions = { 'node[1-100]': [ 'partitionX', 'partition2'] }
result = self.importer.job_partition(0, 'partition1,partition2', 'node[1-100]')
self.assertEquals(result, 'partition2')
self.app.arch.partitions = { 'node[1-99]': [ 'partition1' ],
'node[1-100],bm[1-10]': [ 'partitionX', 'partition2' ] }
result = self.importer.job_partition(0, 'partition1,partition2', 'node[1-100]')
self.assertEquals(result, 'partition2')
