def new_db(self):
"""Returns a new HPCStatsDB object."""
# Instantiate connexion to db
db = HPCStatsDB(self.conf)
db.bind()
return db
class TestsProjectImporterCSVUpdate(HPCStatsTestCase):
@mock.patch("HPCStats.DB.HPCStatsDB.psycopg2", mock_psycopg2())
def setUp(self):
self.filename = 'fake'
self.cluster = 'testcluster'
HPCStatsConf.__bases__ = (MockConfigParser, object)
self.conf = HPCStatsConf(self.filename, self.cluster)
self.conf.conf = CONFIG
self.app = None
self.db = HPCStatsDB(self.conf)
self.db.bind()
self.importer = ProjectImporterCSV(self.app, self.db, self.conf)
def test_update(self):
"""ProjectImporterCSV.update() works with simple data
"""
domain1 = Domain('dom1', 'domain name 1')
project1 = Project(domain1, 'code1', 'project description 1')
MockPg2.PG_REQS['save_project'].set_assoc(
params=( project1.code, project1.description, domain1.key ),
result=[ [ 1 ] ]
)
self.importer.projects = [ project1 ]
self.importer.domains = [ domain1 ]
self.importer.update()
class TestsProjectImporterCSVUpdate(HPCStatsTestCase):
@mock.patch("HPCStats.DB.HPCStatsDB.psycopg2", mock_psycopg2())
def setUp(self):
self.filename = 'fake'
self.cluster = 'testcluster'
HPCStatsConf.__bases__ = (MockConfigParser, object)
self.conf = HPCStatsConf(self.filename, self.cluster)
self.conf.conf = CONFIG
self.app = None
self.db = HPCStatsDB(self.conf)
self.db.bind()
self.importer = ProjectImporterCSV(self.app, self.db, self.conf)
def test_update(self):
"""ProjectImporterCSV.update() works with simple data
"""
domain1 = Domain('dom1', 'domain name 1')
project1 = Project(domain1, 'code1', 'project description 1')
MockPg2.PG_REQS['save_project'].set_assoc(params=(project1.code,
project1.description,
domain1.key),
result=[[1]])
self.importer.projects = [project1]
self.importer.domains = [domain1]
self.importer.update()
class TestsBusinessCodeImporterCSVUpdate(HPCStatsTestCase):
@mock.patch("HPCStats.DB.HPCStatsDB.psycopg2", mock_psycopg2())
def setUp(self):
self.filename = 'fake'
self.cluster = Cluster('testcluster')
HPCStatsConf.__bases__ = (MockConfigParser, object)
self.conf = HPCStatsConf(self.filename, self.cluster.name)
self.conf.conf = CONFIG
self.app = None
self.db = HPCStatsDB(self.conf)
self.db.bind()
self.importer = BusinessCodeImporterCSV(self.app, self.db, self.conf)
def test_update_not_exists(self):
"""ProjectImporterCSV.update() works when business code does not exist
"""
business1 = Business('code1', 'business description 1')
self.importer.businesses = [business1]
self.importer.update()
@mock.patch("%s.Business.save" % (module))
def test_update_not_exists_with_mock(self, mock_save):
"""ProjectImporterCSV.update() call Business.save() when business code
does not exist
"""
business1 = Business('code1', 'business description 1')
MockPg2.PG_REQS['existing_business'].set_assoc(
params=(business1.code, ), result=[])
self.importer.businesses = [business1]
self.importer.update()
mock_save.assert_called_with(self.db)
def test_update_exists(self):
"""ProjectImporterCSV.update() works when business code exists
"""
business1 = Business('code1', 'business description 1')
MockPg2.PG_REQS['existing_business'].set_assoc(
params=(business1.code, ), result=[['code1']])
self.importer.businesses = [business1]
self.importer.update()
@mock.patch("%s.Business.update" % (module))
def test_update_exists_with_mock(self, mock_update):
"""ProjectImporterCSV.update() call Business.update() when business
code exists
"""
business1 = Business('code1', 'business description 1')
MockPg2.PG_REQS['existing_business'].set_assoc(
params=(business1.code, ), result=[['code1']])
self.importer.businesses = [business1]
self.importer.update()
mock_update.assert_called_with(self.db)
class TestsArchitectureImporterArchfileUpdate(HPCStatsTestCase):
@mock.patch("HPCStats.DB.HPCStatsDB.psycopg2", mock_psycopg2())
def setUp(self):
self.filename = 'fake'
self.cluster = 'test_cluster'
HPCStatsConf.__bases__ = (MockConfigParser, object)
self.conf = HPCStatsConf(self.filename, self.cluster)
self.conf.conf = CONFIG.copy()
self.app = None
self.db = HPCStatsDB(self.conf)
self.db.bind()
self.importer = ArchitectureImporterArchfile(self.app,
self.db,
self.conf,
self.cluster)
init_reqs()
def test_update(self):
"""ProjectImporterCSV.update() creates cluster and node if not existing
"""
cluster1 = Cluster('cluster1')
node1 = Node('node1', cluster1, 'model1', 'test_partition', 12, 6 * 1024 ** 3, 1)
MockPg2.PG_REQS['save_cluster'].set_assoc(
params=( cluster1.name ),
result=[ [ 1 ] ]
)
MockPg2.PG_REQS['save_node'].set_assoc(
params=( node1.name, cluster1.cluster_id, node1.partition,
node1.cpu, node1.memory, node1.flops ),
result=[ [ 1 ] ]
)
self.importer.cluster = cluster1
self.importer.nodes = [ node1 ]
self.importer.update()
def test_update_2(self):
"""ProjectImporterCSV.update() detect existing cluster and node
"""
cluster1 = Cluster('cluster1')
node1 = Node('node1', cluster1, 'model1', 'test_partition', 12, 6 * 1024 ** 3, 1)
MockPg2.PG_REQS['find_cluster'].set_assoc(
params=( cluster1.name, ),
result=[ [ 1 ] ]
)
MockPg2.PG_REQS['find_node'].set_assoc(
params=( node1.name, cluster1.cluster_id, ),
result=[ [ 1 ] ]
)
self.importer.cluster = cluster1
self.importer.nodes = [ node1 ]
self.importer.update()
class TestsUserImporterLdapSlurm(HPCStatsTestCase):
@mock.patch("HPCStats.DB.HPCStatsDB.psycopg2", mock_psycopg2())
def setUp(self):
self.filename = 'fake'
self.cluster = Cluster('testcluster')
self.cluster.cluster_id = 0
HPCStatsConf.__bases__ = (MockConfigParser, object)
self.conf = HPCStatsConf(self.filename, self.cluster)
self.conf.conf = CONFIG.copy()
self.db = HPCStatsDB(self.conf)
self.db.bind()
self.app = MockApp(self.db, self.conf, self.cluster)
self.importer = UserImporterLdapSlurm(self.app,
self.db,
self.conf,
self.cluster)
# Disable strict_user_membership to avoid exception when user found
# in Slurm and not in LDAP then.
self.importer.strict_user_membership = False
init_reqs()
def test_init(self):
"""UserImporterLdap.__init__() initializes w/o error
"""
pass
@mock.patch("%s.ldap" % (p_module), mock_ldap())
@mock.patch("%s.MySQLdb" % (module), mock_mysqldb())
def test_load_user_no_ldap(self):
"""UserImporterLdapSlurm.load() should not user from Slurm if not found
in LDAP.
"""
users = [ ]
users_no_group = [ ]
fill_ldap_users(CONFIG['testcluster/ldap'], users, users_no_group)
MockMySQLdb.MY_REQS['get_users']['res'] = \
[ [ 'login1' ] ]
self.importer.load()
self.assertEquals(len(self.importer.users), 0)
@mock.patch("%s.ldap" % (p_module), mock_ldap())
@mock.patch("%s.MySQLdb" % (module), mock_mysqldb())
def test_load_user_ok_ldap(self):
"""UserImporterLdapSlurm.load() should load user from Slurm if also
found in LDAP.
"""
users = [ ]
users_no_group = [ 'login1' ]
fill_ldap_users(CONFIG['testcluster/ldap'], users, users_no_group)
MockMySQLdb.MY_REQS['get_users']['res'] = \
[ [ 'login1' ] ]
self.importer.load()
self.assertEquals(len(self.importer.users), 1)
self.assertEquals(len(self.importer.users_acct_ldap), 0)
self.assertEquals(len(self.importer.users_acct_slurm), 1)
@mock.patch("%s.ldap" % (p_module), mock_ldap())
@mock.patch("%s.MySQLdb" % (module), mock_mysqldb())
def test_load_user_in_group(self):
"""UserImporterLdapSlurm.load() should not load user from Slurm if
already loaded because it is member of cluster group.
"""
users = [ 'login1' ]
users_no_group = [ ]
fill_ldap_users(CONFIG['testcluster/ldap'], users, users_no_group)
MockMySQLdb.MY_REQS['get_users']['res'] = \
[ [ 'login1' ] ]
self.importer.load()
self.assertEquals(len(self.importer.users), 1)
self.assertEquals(len(self.importer.users_acct_ldap), 1)
self.assertEquals(len(self.importer.users_acct_slurm), 0)
@mock.patch("%s.ldap" % (p_module), mock_ldap())
@mock.patch("%s.MySQLdb" % (module), mock_mysqldb())
def test_load_no_redundancy(self):
"""UserImporterLdapSlurm.load() should manage redundancy with LDAP.
"""
users = [ 'login1', 'login2', 'login3' ]
fill_ldap_users(CONFIG['testcluster/ldap'], users)
MockMySQLdb.MY_REQS['get_users']['res'] = \
[ [ 'login1' ], [ 'login2' ], [ 'login4' ] ]
self.importer.load()
self.assertEquals(len(self.importer.users), 3)
self.assertEquals(len(self.importer.accounts), 3)
@mock.patch("%s.ldap" % (p_module), mock_ldap())
@mock.patch("%s.MySQLdb" % (module), mock_mysqldb())
@mock.patch("%s.User.save" % (module))
@mock.patch("%s.Account.save" % (module))
def test_update_new_user(self, m_account_save, m_user_save):
"""UserImporterLdapSlurm.slurm() should save the user and the account if
not existing in DB
"""
users = [ ]
users_no_group = [ 'login1' ]
fill_ldap_users(CONFIG['testcluster/ldap'], users, users_no_group)
MockMySQLdb.MY_REQS['get_users']['res'] = \
[ [ 'login1' ] ]
user1_id = 1
MockPg2.PG_REQS['find_user'].set_assoc(
params=( 'login1', ),
result=[ ]
)
MockPg2.PG_REQS['existing_account'].set_assoc(
params=( user1_id, self.cluster.cluster_id, ),
result=[ ]
)
self.importer.load()
self.importer.update()
self.assertEquals(self.importer.accounts[0].creation_date,
date(1970, 1, 1))
self.assertEquals(self.importer.accounts[0].deletion_date,
date(1970, 1, 1))
m_user_save.assert_called_with(self.db)
m_account_save.assert_called_with(self.db)
@mock.patch("%s.ldap" % (p_module), mock_ldap())
@mock.patch("%s.MySQLdb" % (module), mock_mysqldb())
@mock.patch("%s.User.update" % (module))
@mock.patch("%s.Account.update" % (module))
def test_update_user_account_exist(self, m_account_update, m_user_update):
"""UserImporterLdapSlurm.slurm() should update the user and do not
touch the account if they already exist in DB
"""
users = [ ]
users_no_group = [ 'login1' ]
fill_ldap_users(CONFIG['testcluster/ldap'], users, users_no_group)
MockMySQLdb.MY_REQS['get_users']['res'] = \
[ [ 'login1' ] ]
user1_id = 1
MockPg2.PG_REQS['find_user'].set_assoc(
params=( 'login1', ),
result=[ [ user1_id ] ]
)
MockPg2.PG_REQS['existing_account'].set_assoc(
params=( user1_id, self.cluster.cluster_id, ),
result=[ [ 0 ] ]
)
self.importer.load()
self.importer.update()
m_user_update.assert_called_with(self.db)
# ensure Account.update() is not called
self.assertRaises(AssertionError,
m_account_update.assert_called_with,
self.db,
None)
class TestsUserImporterLdap(HPCStatsTestCase):
@mock.patch("HPCStats.DB.HPCStatsDB.psycopg2", mock_psycopg2())
def setUp(self):
self.filename = 'fake'
self.cluster = Cluster('testcluster')
self.cluster.cluster_id = 0
HPCStatsConf.__bases__ = (MockConfigParser, object)
self.conf = HPCStatsConf(self.filename, self.cluster)
self.conf.conf = CONFIG.copy()
self.db = HPCStatsDB(self.conf)
self.db.bind()
self.app = MockApp(self.db, self.conf, self.cluster)
self.importer = UserImporterLdap(self.app,
self.db,
self.conf,
self.cluster)
init_reqs()
def test_init(self):
"""UserImporterLdap.__init__() initializes w/o error
"""
pass
@mock.patch("%s.ldap" % (module), mock_ldap())
def test_load_simple(self):
"""UserImporterLdap.load() should work with simple data from LDAP.
"""
users = [ 'login1', 'login2', 'login3' ]
fill_ldap_users(CONFIG['testcluster/ldap'], users)
MockPg2.PG_REQS['get_unclosed_accounts'].set_assoc (
params=( 0, ),
result=[ ]
)
self.importer.load()
self.assertEquals(len(self.importer.users), 3)
self.assertEquals(len(self.importer.accounts), 3)
@mock.patch("%s.ldap" % (module), mock_ldap())
def test_load_simple2(self):
"""UserImporterLdap.load() should work with simple data from LDAP and
HPCStatsDB.
"""
users = [ 'login1', 'login2', 'login3' ]
fill_ldap_users(CONFIG['testcluster/ldap'], users)
creation_user4 = datetime(2015, 3, 2, 16, 0, 1)
MockPg2.PG_REQS['get_unclosed_accounts'].set_assoc(
params=( 0, ),
result=[ [ 0, 'login4', 'name_user4', 'firstname_user4',
'department_user4', 0, 0, creation_user4 ] ]
)
self.importer.load()
self.assertEquals(len(self.importer.users), 4)
self.assertEquals(len(self.importer.accounts), 4)
@mock.patch("%s.ldap" % (module), mock_ldap())
@mock.patch("%s.User.save" % (module))
@mock.patch("%s.Account.save" % (module))
def test_load_update_new_user_other_accounts(self, m_account_save, m_user_save):
"""UserImporterLdap.update() create new user/account found in LDAP
and not found in HPCStatsDB with a creation date equals to today
because there are already existing accounts.
"""
users = [ 'login1' ]
fill_ldap_users(CONFIG['testcluster/ldap'], users)
MockPg2.PG_REQS['get_unclosed_accounts'].set_assoc(
params=( self.cluster.cluster_id, ),
result=[ ]
)
MockPg2.PG_REQS['nb_existing_accounts'].set_assoc(
params=( self.cluster.cluster_id, ),
result=[ [ 0 ], [ 1 ] ]
)
self.importer.load()
self.importer.update()
self.assertEquals(self.importer.accounts[0].creation_date, date.today())
m_user_save.assert_called_with(self.db)
m_account_save.assert_called_with(self.db)
@mock.patch("%s.ldap" % (module), mock_ldap())
@mock.patch("%s.User.save" % (module))
@mock.patch("%s.Account.save" % (module))
def test_load_update_new_user_no_account(self, m_account_save, m_user_save):
"""UserImporterLdap.update() save new user/account found in LDAP
and not found in HPCStatsDB with a creation date equals to epoch
because there is none existing accounts
"""
users = [ 'login1' ]
fill_ldap_users(CONFIG['testcluster/ldap'], users)
self.importer.load()
self.importer.update()
self.assertEquals(self.importer.accounts[0].creation_date, date(1970, 1, 1))
m_user_save.assert_called_with(self.db)
m_account_save.assert_called_with(self.db)
@mock.patch("%s.ldap" % (module), mock_ldap())
@mock.patch("%s.Account.update" % (module))
def test_load_update_close_account(self, m_account_update):
"""UserImporterLdap.update() close account found as unclosed in
HPCStatsDB and not found in LDAP.
"""
users = [ ]
fill_ldap_users(CONFIG['testcluster/ldap'], users)
creation_user2 = datetime(2015, 3, 2, 16, 0, 1)
MockPg2.PG_REQS['get_unclosed_accounts'].set_assoc(
params=( self.cluster.cluster_id, ),
result=[ [ 2, 'login2', 'name_user2', 'firstname_user2',
'department_user2', 0, 0, creation_user2 ] ]
)
self.importer.load()
self.importer.update()
self.assertEquals(self.importer.accounts[0].deletion_date, date.today())
m_account_update.assert_called_with(self.db)
@mock.patch("%s.ldap" % (module), mock_ldap())
@mock.patch("%s.User.update" % (module))
@mock.patch("%s.Account.save" % (module))
def test_load_update_user_wo_account(self, m_account_save, m_user_update):
"""UserImporterLdap.update() create account and update user found in
LDAP and in HPCStatsDB but w/o account on the cluster.
"""
users = [ 'login3' ]
fill_ldap_users(CONFIG['testcluster/ldap'], users)
user3_id = 3
MockPg2.PG_REQS['find_user'].set_assoc(
params=( 'login3', ),
result=[ [ user3_id ] ]
)
MockPg2.PG_REQS['existing_account'].set_assoc(
params=( user3_id, self.cluster.cluster_id, ),
result=[ ]
)
self.importer.load()
self.importer.update()
m_user_update.assert_called_with(self.db)
m_account_save.assert_called_with(self.db)
@mock.patch("%s.ldap" % (module), mock_ldap())
@mock.patch("%s.User.update" % (module))
@mock.patch("%s.Account.update" % (module))
def test_load_update_user_w_account(self, m_account_update, m_user_update):
"""UserImporterLdap.update() update user found in LDAP and in HPCStatsDB
with an unclosed account on the cluster.
"""
users = [ 'login4' ]
fill_ldap_users(CONFIG['testcluster/ldap'], users)
creation_user4 = datetime(2015, 3, 2, 16, 0, 1)
user4_id = 4
MockPg2.PG_REQS['find_user'].set_assoc(
params=( 'login4', ),
result=[ [ user4_id ] ]
)
MockPg2.PG_REQS['existing_account'].set_assoc(
params=( user4_id, self.cluster.cluster_id, ),
result=[ [ 0 ] ]
)
MockPg2.PG_REQS['load_account'].set_assoc(
params=( user4_id, self.cluster.cluster_id, ),
result=[ [ 0, 0, creation_user4, None ] ]
)
self.importer.load()
self.importer.update()
m_user_update.assert_called_with(self.db)
# ensure Account.update() is not called
self.assertRaises(AssertionError,
m_account_update.assert_called_with,
self.db,
None)
@mock.patch("%s.ldap" % (module), mock_ldap())
@mock.patch("%s.User.update" % (module))
@mock.patch("%s.Account.update" % (module))
def test_load_update_user_closed_account(self, m_account_update, m_user_update):
"""UserImporterLdap.update() update user found in LDAP and in HPCStatsDB
with a closed account on the cluster.
"""
users = [ 'login5' ]
fill_ldap_users(CONFIG['testcluster/ldap'], users)
user5_creation = datetime(2015, 3, 2, 16, 0, 1)
user5_deletion = datetime(2015, 3, 2, 16, 0, 1)
user5_id = 5
MockPg2.PG_REQS['find_user'].set_assoc(
params=( 'login5', ),
result=[ [ user5_id ] ]
)
MockPg2.PG_REQS['existing_account'].set_assoc(
params=( user5_id, self.cluster.cluster_id, ),
result=[ [ 0 ] ]
)
MockPg2.PG_REQS['load_account'].set_assoc(
params=( user5_id, self.cluster.cluster_id, ),
result=[ [ 0, 0, user5_creation, user5_deletion ] ]
)
self.importer.load()
self.importer.update()
self.assertEquals(self.importer.accounts[0].deletion_date, None)
m_user_update.assert_called_with(self.db)
m_account_update.assert_called_with(self.db)
class TestsJobImporterSlurm(HPCStatsTestCase):
@mock.patch("HPCStats.DB.HPCStatsDB.psycopg2", mock_psycopg2())
def setUp(self):
self.filename = 'fake'
self.cluster = Cluster('testcluster')
HPCStatsConf.__bases__ = (MockConfigParser, object)
self.conf = HPCStatsConf(self.filename, self.cluster)
self.conf.conf = CONFIG.copy()
self.db = HPCStatsDB(self.conf)
self.db.bind()
self.app = MockApp(self.db, self.conf, self.cluster)
self.importer = JobImporterSlurm(self.app,
self.db,
self.conf,
self.cluster)
init_reqs()
def test_init(self):
"""JobImporterSlurm.__init__() initializes object with attributes
"""
self.assertEquals(self.importer._dbhost,
self.conf.conf[self.cluster.name + '/slurm']['host'])
def load_app(self):
"""Load App objects for JobImporterSlurm.load() normal operation."""
j1_submit = datetime(2015, 3, 2, 16, 0, 1)
j1_start = datetime(2015, 3, 2, 16, 0, 2)
j1_end = datetime(2015, 3, 2, 16, 0, 3)
j1_submit_ts = time.mktime(j1_submit.timetuple())
j1_start_ts = time.mktime(j1_start.timetuple())
j1_end_ts = time.mktime(j1_end.timetuple())
node1 = Node('node1', self.cluster, 'model1', 'partition1', 4, 4, 0)
node2 = Node('node2', self.cluster, 'model1', 'partition1', 4, 4, 0)
a1_create = datetime(2010, 1, 1, 12, 0, 0)
user1 = User('user1', 'firstname1', 'lastname1', 'department1')
account1 = Account(user1, self.cluster, 1000, 1000, a1_create, None)
domain1 = Domain('domain1', 'domain 1')
project1 = Project(domain1, 'project1', 'description project 1')
business1 = Business('business1', 'business description 1')
self.app.arch.nodes = [ node1, node2 ]
self.app.users.accounts = [ account1 ]
self.app.projects.projects = [ project1 ]
self.app.business.businesses = [ business1 ]
MockMySQLdb.MY_REQS['get_jobs_after_batchid']['res'] = \
[
[ 0, 0, 1000, 1000, j1_submit_ts, j1_start_ts, j1_end_ts,
2, '1=4', 'partition1', 'qos1', 1, 'node[1-2]', 'user1',
'job1', 'project1:business1' ],
]
@mock.patch("%s.MySQLdb" % (module), mock_mysqldb())
def test_is_old_schema(self):
"""JobImporterSlurm._is_old_schema() should return True is SlurmDBD
<15.08 is detected, False otherwise."""
self.load_app()
self.importer.connect_db()
MockMySQLdb.MY_REQS['job_table_cols']['res'] = \
[ [ 'cpus_alloc', ] , ]
self.assertEquals(self.importer._is_old_schema(), True)
MockMySQLdb.MY_REQS['job_table_cols']['res'] = []
self.assertEquals(self.importer._is_old_schema(), False)
@mock.patch("%s.MySQLdb" % (module), mock_mysqldb())
def test_load(self):
"""JobImporterSlurm.load() works with simple data."""
self.load_app()
# make sure new-schema is used here
MockMySQLdb.MY_REQS['job_table_cols']['res'] = [ ]
self.importer.load()
self.assertEquals(len(self.importer.jobs), 1)
self.assertEquals(len(self.importer.runs), 2)
job = self.importer.jobs[0]
self.assertEquals(job.nbcpu, 4)
self.assertEquals(job.state, 'RUNNING')
self.assertEquals(job.name, 'job1')
self.assertEquals(job.queue, 'partition1-qos1')
self.assertEquals(job.account, self.app.users.accounts[0])
self.assertEquals(job.project, self.app.projects.projects[0])
self.assertEquals(job.business, self.app.business.businesses[0])
@mock.patch("%s.MySQLdb" % (module), mock_mysqldb())
def test_load_old_schema(self):
"""JobImporterSlurm.load() works with simple data from old SlurmDBD
<15.08 schema."""
self.load_app()
MockMySQLdb.MY_REQS['job_table_cols']['res'] = \
[ [ 'cpus_alloc', ] , ]
# replace TRES '1=4' by cpus_alloc 4
MockMySQLdb.MY_REQS['get_jobs_after_batchid']['res'][0][8] = 4
self.importer.load()
self.assertEquals(len(self.importer.jobs), 1)
self.assertEquals(len(self.importer.runs), 2)
job = self.importer.jobs[0]
self.assertEquals(job.nbcpu, 4)
self.assertEquals(job.state, 'RUNNING')
self.assertEquals(job.name, 'job1')
self.assertEquals(job.queue, 'partition1-qos1')
self.assertEquals(job.account, self.app.users.accounts[0])
self.assertEquals(job.project, self.app.projects.projects[0])
self.assertEquals(job.business, self.app.business.businesses[0])
@mock.patch("%s.MySQLdb" % (module), mock_mysqldb())
@mock.patch("%s.JobImporterSlurm.get_jobs_after_batchid" % (module))
def test_load_search_batchid(self, mock_get_jobs):
"""JobImporterSlurm.load() must search jobs after correct batch_id."""
MockPg2.PG_REQS['get_batchid_oldest_unfinished'].set_assoc(
params=( self.cluster.cluster_id ),
result=[ [ 2 ] ]
)
MockPg2.PG_REQS['get_batchid_last'].set_assoc(
params=( self.cluster.cluster_id ),
result=[ [ 3 ] ]
)
self.importer.load()
mock_get_jobs.assert_called_with(2)
# None unfinished job, search must be done with batch_id of lasti job.
MockPg2.PG_REQS['get_batchid_oldest_unfinished'].set_assoc(
params=( self.cluster.cluster_id ),
result=[ ]
)
MockPg2.PG_REQS['get_batchid_last'].set_assoc(
params=( self.cluster.cluster_id ),
result=[ [ 4 ] ]
)
self.importer.load()
mock_get_jobs.assert_called_with(4)
# No job in DB: search starting -1.
MockPg2.PG_REQS['get_batchid_oldest_unfinished'].set_assoc(
params=( self.cluster.cluster_id ),
result=[ ]
)
MockPg2.PG_REQS['get_batchid_last'].set_assoc(
params=( self.cluster.cluster_id ),
result=[ ]
)
self.importer.load()
mock_get_jobs.assert_called_with(-1)
@mock.patch("%s.MySQLdb" % (module), mock_mysqldb())
def test_load_account_not_found(self):
"""JobImporterSlurm.load() raises exception when account not found"""
self.load_app()
self.app.users.accounts = [ ]
self.assertRaisesRegexp(
HPCStatsSourceError,
"account user1 not found in loaded account",
self.importer.load)
@mock.patch("%s.MySQLdb" % (module), mock_mysqldb())
def test_load_invalid_tres(self):
"""JobImporterSlurm.load() raises exception if invalid tres for a job
is found"""
self.load_app()
MockMySQLdb.MY_REQS['get_jobs_after_batchid']['res'][0][8] = '0=0'
self.assertRaisesRegexp(
HPCStatsSourceError,
"unable to extract cpus_alloc from job tres",
self.importer.load)
@mock.patch("%s.MySQLdb" % (module), mock_mysqldb())
def test_load_invalid_wckey(self):
"""JobImporterSlurm.load() raises exception when format of wckey is
invalid.
"""
self.load_app()
MockMySQLdb.MY_REQS['get_jobs_after_batchid']['res'][0][15] = 'fail'
self.assertRaisesRegexp(
HPCStatsSourceError,
"format of wckey fail is not valid",
self.importer.load)
@mock.patch("%s.MySQLdb" % (module), mock_mysqldb())
def test_load_project_not_found(self):
"""JobImporterSlurm.load() raises exception when project not found."""
self.load_app()
self.app.projects.projects = [ ]
self.assertRaisesRegexp(
HPCStatsSourceError,
"project project1 not found in loaded projects",
self.importer.load)
@mock.patch("%s.MySQLdb" % (module), mock_mysqldb())
def test_load_business_not_found(self):
"""JobImporterSlurm.load() raises exception when business not found."""
self.load_app()
self.app.business.businesses = [ ]
self.assertRaisesRegexp(
HPCStatsSourceError,
"business code business1 not found in loaded business codes",
self.importer.load)
@mock.patch("%s.MySQLdb" % (module), mock_mysqldb())
def test_load_invalid_nodelist(self):
"""JobImporterSlurm.load() raises exception when format of nodelist
is invalid.
"""
self.load_app()
MockMySQLdb.MY_REQS['get_jobs_after_batchid']['res'][0][12] = \
'nodelistfail[5-4]'
self.assertRaisesRegexp(
HPCStatsSourceError,
"could not parse nodeset nodelistfail\[5\-4\] for job 0",
self.importer.load)
@mock.patch("%s.MySQLdb" % (module), mock_mysqldb())
def test_load_node_not_found(self):
"""JobImporterSlurm.load() raises exception when node not found."""
self.load_app()
self.app.arch.nodes = [ ]
self.assertRaisesRegexp(
HPCStatsSourceError,
"unable to find node node1 for job 0 in loaded nodes",
self.importer.load)
def test_job_partition(self):
"""JobImporterSlurm.job_partition() must return correct partition for
based on job partition list and its nodelist.
"""
# Only one element in job partition list: it must be returned whatever
# the nodelist and ArchitectureImporter job partitions
self.app.arch.partitions = { }
result = self.importer.job_partition(0, 'partition2', 'node[1-100]')
self.assertEquals(result, 'partition2')
# Multiple elements but None nodelist: it must return arbitrary the
# first partition
self.app.arch.partitions = { }
result = self.importer.job_partition(0, 'partition1,partition2', None)
self.assertEquals(result, 'partition1')
# Multiple elements in partition and defined nodelist: it must return
# a corresponding partition loaded by ArchitectureImporter and
# associated to a nodelist that fully intersects
self.app.arch.partitions = { 'node[1-100]': [ 'partitionX', 'partition2'] }
result = self.importer.job_partition(0, 'partition1,partition2', 'node[1-100]')
self.assertEquals(result, 'partition2')
self.app.arch.partitions = { 'node[1-99]': [ 'partition1' ],
'node[1-100],bm[1-10]': [ 'partitionX', 'partition2' ] }
result = self.importer.job_partition(0, 'partition1,partition2', 'node[1-100]')
self.assertEquals(result, 'partition2')
class TestsEventImporterSlurm(HPCStatsTestCase):
@mock.patch("HPCStats.DB.HPCStatsDB.psycopg2", mock_psycopg2())
def setUp(self):
# setup conf
self.filename = 'fake'
self.cluster = Cluster('testcluster')
HPCStatsConf.__bases__ = (MockConfigParser, object)
self.conf = HPCStatsConf(self.filename, self.cluster)
self.conf.conf = CONFIG.copy()
# setup importer
self.db = HPCStatsDB(self.conf)
self.db.bind()
self.app = MockApp(self.db, self.conf, self.cluster)
self.importer = EventImporterSlurm(self.app, self.db, self.conf,
self.cluster)
init_reqs()
# setup logger
logging.setLoggerClass(HPCStatsLogger)
self.logger = logging.getLogger(__name__)
self.handler = MockLoggingHandler()
self.logger.addHandler(self.handler)
self.handler.reset()
HPCStatsLogger.set_error_mgr(HPCStatsErrorMgr(self.conf))
def test_init(self):
"""EventImporterSlurm.__init__() initializes w/o error
"""
pass
def init_load_data(self):
"""Utility method to initialize data to make load() simply."""
self.e1_start = datetime(2015, 3, 2, 15, 59, 59)
self.e1_end = datetime(2015, 3, 2, 16, 0, 0)
self.node_name = 'node1'
e1_start_ts = time.mktime(self.e1_start.timetuple())
e1_end_ts = time.mktime(self.e1_end.timetuple())
MockMySQLdb.MY_REQS['get_events']['res'] = \
[
[ e1_start_ts, e1_end_ts, self.node_name, '1=16', 35, 'reason1' ],
]
MockMySQLdb.MY_REQS['event_table_cols']['res'] = []
self.app.arch.nodes = [
Node(self.node_name, self.cluster, 'model1', 'partition1', 16, 8,
0),
]
@mock.patch("%s.MySQLdb" % (module), mock_mysqldb())
def test_is_old_schema(self):
"""EventImporterSlurm._is_old_schema() should return True is SlurmDBD
<15.08 is detected, False otherwise."""
self.importer.connect_db()
MockMySQLdb.MY_REQS['event_table_cols']['res'] = \
[ [ 'cpu_count', ] , ]
self.assertEquals(self.importer._is_old_schema(), True)
MockMySQLdb.MY_REQS['event_table_cols']['res'] = []
self.assertEquals(self.importer._is_old_schema(), False)
@mock.patch("%s.MySQLdb" % (module), mock_mysqldb())
def test_load_simple(self):
"""EventImporterSlurm.load() works with simple data."""
self.init_load_data()
self.importer.load()
self.assertEquals(1, len(self.importer.events))
event = self.importer.events[0]
self.assertEquals(event.start_datetime, self.e1_start)
self.assertEquals(event.end_datetime, self.e1_end)
self.assertEquals(event.nb_cpu, 16)
self.assertEquals(event.event_type, 'ALLOCATED+RES')
self.assertEquals(event.reason, 'reason1')
@mock.patch("%s.MySQLdb" % (module), mock_mysqldb())
def test_load_old_schema(self):
"""EventImporterSlurm.load() works with simple data from old SlurmDBD
<15.08 schema."""
self.init_load_data()
MockMySQLdb.MY_REQS['event_table_cols']['res'] = \
[ [ 'cpu_count', ] , ]
MockMySQLdb.MY_REQS['get_events']['res'][0][3] = 16
self.importer.load()
event = self.importer.events[0]
self.assertEquals(event.nb_cpu, 16)
@mock.patch("%s.MySQLdb" % (module), mock_mysqldb())
@mock.patch("%s.EventImporterSlurm.get_new_events" % (module))
def test_load_search_datetime(self, mock_new_events):
"""EventImporterSlurm.load() must search new events starting from
correct datetime."""
# Both datetimes are defined, search must be done with start datetime
# of oldest unfinished event.
d1 = datetime(2015, 3, 2, 15, 59, 59)
d2 = datetime(2015, 3, 2, 16, 0, 0)
d1_ts = time.mktime(d1.timetuple())
d2_ts = time.mktime(d2.timetuple())
MockPg2.PG_REQS['get_end_last_event'].set_assoc(
params=(self.cluster.cluster_id), result=[[d1_ts]])
MockPg2.PG_REQS['get_start_oldest_unfinised_event'].set_assoc(
params=(self.cluster.cluster_id), result=[[d2_ts]])
self.importer.load()
mock_new_events.assert_called_with(d2_ts)
# None unfinished event, search must be done with end datetime of last
# event.
MockPg2.PG_REQS['get_end_last_event'].set_assoc(
params=(self.cluster.cluster_id), result=[[d1_ts]])
MockPg2.PG_REQS['get_start_oldest_unfinised_event'].set_assoc(
params=(self.cluster.cluster_id), result=[])
self.importer.load()
mock_new_events.assert_called_with(d1_ts)
default_datetime = datetime(1970, 1, 1, 0, 0)
# No event in DB: search starting from epoch.
MockPg2.PG_REQS['get_end_last_event'].set_assoc(
params=(self.cluster.cluster_id), result=[])
MockPg2.PG_REQS['get_start_oldest_unfinised_event'].set_assoc(
params=(self.cluster.cluster_id), result=[])
self.importer.load()
mock_new_events.assert_called_with(default_datetime)
@mock.patch("%s.MySQLdb" % (module), mock_mysqldb())
def test_load_unfound_node(self):
"""EventImporterSlurm.load() raises Exception if one event is linked to
a node not loaded by ArchitectureImporter."""
self.init_load_data()
self.app.arch.nodes = []
self.importer.log = self.logger
self.importer.load()
self.assertIn(
"EventImporterSlurm: ERROR E_E0001: event node %s is "
"unknown in cluster %s architecture, ignoring this event" %
(self.node_name, self.cluster.name),
self.handler.messages['warning'])
@mock.patch("%s.MySQLdb" % (module), mock_mysqldb())
def test_load_invalid_tres(self):
"""EventImporterSlurm.load() raises exception if invalid tres for an
event is found"""
self.init_load_data()
MockMySQLdb.MY_REQS['get_events']['res'][0][3] = '0=0'
self.assertRaisesRegexp(HPCStatsSourceError,
"unable to extract cpu_count from event tres",
self.importer.load)
@mock.patch("%s.MySQLdb" % (module), mock_mysqldb())
def test_merge_successive_events(self):
"""EventImporterSlurm.merge_successive_events() should merge successive
events in the list if they are on the same node w/ same type.
"""
e1_start = datetime(2015, 3, 2, 16, 0, 0)
e1_end = datetime(2015, 3, 2, 16, 10, 0)
e2_start = datetime(2015, 3, 2, 16, 10, 0)
e2_end = datetime(2015, 3, 2, 16, 20, 0)
e3_start = datetime(2015, 3, 2, 16, 20, 0)
e3_end = datetime(2015, 3, 2, 16, 30, 0)
node1 = [
Node('node1', self.cluster, 'model1', 'partition1', 16, 8, 0),
]
node2 = [
Node('node2', self.cluster, 'model1', 'partition1', 16, 8, 0),
]
# 3 successive events on one node with same type, they must be merged
# into one event.
events = [
Event(self.cluster, node1, 4, e1_start, e1_end, 'type1',
'reason1'),
Event(self.cluster, node1, 4, e2_start, e2_end, 'type1',
'reason1'),
Event(self.cluster, node1, 4, e3_start, e3_end, 'type1',
'reason1'),
]
merged = self.importer.merge_successive_events(events)
self.assertEquals(1, len(merged))
self.assertEquals(merged[0].start_datetime, e1_start)
self.assertEquals(merged[0].end_datetime, e3_end)
self.assertEquals(merged[0].event_type, 'type1')
self.assertEquals(merged[0].reason, 'reason1')
# 3 successive events on one node node1 with same type, with one event
# on another node node2 in the middle: all events on node1 must be
# merged while the other event on node2 must stay as is.
events = [
Event(self.cluster, node1, 4, e1_start, e1_end, 'type1',
'reason1'),
Event(self.cluster, node2, 4, e2_start, e2_end, 'type1',
'reason1'),
Event(self.cluster, node1, 4, e2_start, e2_end, 'type1',
'reason1'),
Event(self.cluster, node1, 4, e3_start, e3_end, 'type1',
'reason1'),
]
merged = self.importer.merge_successive_events(events)
self.assertEquals(2, len(merged))
self.assertEquals(merged[0].start_datetime, e1_start)
self.assertEquals(merged[0].end_datetime, e3_end)
self.assertEquals(merged[1].end_datetime, e2_end)
self.assertEquals(merged[0].node, node1)
self.assertEquals(merged[1].node, node2)
# 3 successive events on node1 but with different types, they must not
# be merged.
events = [
Event(self.cluster, node1, 4, e1_start, e1_end, 'type1',
'reason1'),
Event(self.cluster, node1, 4, e2_start, e2_end, 'type2',
'reason1'),
Event(self.cluster, node1, 4, e3_start, e3_end, 'type1',
'reason1'),
]
merged = self.importer.merge_successive_events(events)
self.assertEquals(3, len(merged))
@mock.patch("%s.MySQLdb" % (module), mock_mysqldb())
def test_txt_slurm_event_type(self):
"""EventImporterSlurm.txt_slurm_event_type() should give the
appropriate human readable string represation of an event type
according to its hex bitmap value.
"""
tests = [(0x0001, 'DOWN'), (0x0004, 'ERROR'), (0x0012, 'IDLE+NET'),
(0x8535, 'MIXED+NET+RES+RESUME+COMPLETING+MAINT')]
for value, expected in tests:
txt = EventImporterSlurm.txt_slurm_event_type(value)
self.assertEquals(txt, expected)
class TestsBusinessCodeImporterCSVUpdate(HPCStatsTestCase):
@mock.patch("HPCStats.DB.HPCStatsDB.psycopg2", mock_psycopg2())
def setUp(self):
self.filename = 'fake'
self.cluster = Cluster('testcluster')
HPCStatsConf.__bases__ = (MockConfigParser, object)
self.conf = HPCStatsConf(self.filename, self.cluster.name)
self.conf.conf = CONFIG
self.app = None
self.db = HPCStatsDB(self.conf)
self.db.bind()
self.importer = BusinessCodeImporterCSV(self.app, self.db, self.conf)
def test_update_not_exists(self):
"""ProjectImporterCSV.update() works when business code does not exist
"""
business1 = Business('code1', 'business description 1')
self.importer.businesses = [ business1 ]
self.importer.update()
@mock.patch("%s.Business.save" % (module))
def test_update_not_exists_with_mock(self, mock_save):
"""ProjectImporterCSV.update() call Business.save() when business code
does not exist
"""
business1 = Business('code1', 'business description 1')
MockPg2.PG_REQS['existing_business'].set_assoc(
params=( business1.code, ),
result=[ ]
)
self.importer.businesses = [ business1 ]
self.importer.update()
mock_save.assert_called_with(self.db)
def test_update_exists(self):
"""ProjectImporterCSV.update() works when business code exists
"""
business1 = Business('code1', 'business description 1')
MockPg2.PG_REQS['existing_business'].set_assoc(
params=( business1.code, ),
result=[ [ 'code1' ] ]
)
self.importer.businesses = [ business1 ]
self.importer.update()
@mock.patch("%s.Business.update" % (module))
def test_update_exists_with_mock(self, mock_update):
"""ProjectImporterCSV.update() call Business.update() when business
code exists
"""
business1 = Business('code1', 'business description 1')
MockPg2.PG_REQS['existing_business'].set_assoc(
params=( business1.code, ),
result=[ [ 'code1' ] ]
)
self.importer.businesses = [ business1 ]
self.importer.update()
mock_update.assert_called_with(self.db)
class TestsEventImporterSlurm(HPCStatsTestCase):
@mock.patch("HPCStats.DB.HPCStatsDB.psycopg2", mock_psycopg2())
def setUp(self):
self.filename = 'fake'
self.cluster = Cluster('testcluster')
HPCStatsConf.__bases__ = (MockConfigParser, object)
self.conf = HPCStatsConf(self.filename, self.cluster)
self.conf.conf = CONFIG.copy()
self.db = HPCStatsDB(self.conf)
self.db.bind()
self.app = MockApp(self.db, self.conf, self.cluster)
self.importer = EventImporterSlurm(self.app,
self.db,
self.conf,
self.cluster)
init_reqs()
def test_init(self):
"""EventImporterSlurm.__init__() initializes w/o error
"""
pass
def init_load_data(self):
"""Utility method to initialize data to make load() simply."""
self.e1_start = datetime(2015, 3, 2, 15, 59, 59)
self.e1_end = datetime(2015, 3, 2, 16, 0, 0)
self.node_name = 'node1'
e1_start_ts = time.mktime(self.e1_start.timetuple())
e1_end_ts = time.mktime(self.e1_end.timetuple())
MockMySQLdb.MY_REQS['get_events']['res'] = \
[
[ e1_start_ts, e1_end_ts, self.node_name, '1=16', 35, 'reason1' ],
]
MockMySQLdb.MY_REQS['event_table_cols']['res'] = [ ]
self.app.arch.nodes = [ Node(self.node_name, self.cluster, 'model1', 'partition1', 16, 8, 0), ]
@mock.patch("%s.MySQLdb" % (module), mock_mysqldb())
def test_is_old_schema(self):
"""EventImporterSlurm._is_old_schema() should return True is SlurmDBD
<15.08 is detected, False otherwise."""
self.importer.connect_db()
MockMySQLdb.MY_REQS['event_table_cols']['res'] = \
[ [ 'cpu_count', ] , ]
self.assertEquals(self.importer._is_old_schema(), True)
MockMySQLdb.MY_REQS['event_table_cols']['res'] = []
self.assertEquals(self.importer._is_old_schema(), False)
@mock.patch("%s.MySQLdb" % (module), mock_mysqldb())
def test_load_simple(self):
"""EventImporterSlurm.load() works with simple data."""
self.init_load_data()
self.importer.load()
self.assertEquals(1, len(self.importer.events))
event = self.importer.events[0]
self.assertEquals(event.start_datetime, self.e1_start)
self.assertEquals(event.end_datetime, self.e1_end)
self.assertEquals(event.nb_cpu, 16)
self.assertEquals(event.event_type, 'ALLOCATED+RES')
self.assertEquals(event.reason, 'reason1')
@mock.patch("%s.MySQLdb" % (module), mock_mysqldb())
def test_load_old_schema(self):
"""EventImporterSlurm.load() works with simple data from old SlurmDBD
<15.08 schema."""
self.init_load_data()
MockMySQLdb.MY_REQS['event_table_cols']['res'] = \
[ [ 'cpu_count', ] , ]
MockMySQLdb.MY_REQS['get_events']['res'][0][3] = 16
self.importer.load()
event = self.importer.events[0]
self.assertEquals(event.nb_cpu, 16)
@mock.patch("%s.MySQLdb" % (module), mock_mysqldb())
@mock.patch("%s.EventImporterSlurm.get_new_events" % (module))
def test_load_search_datetime(self, mock_new_events):
"""EventImporterSlurm.load() must search new events starting from
correct datetime."""
# Both datetimes are defined, search must be done with start datetime
# of oldest unfinished event.
d1 = datetime(2015, 3, 2, 15, 59, 59)
d2 = datetime(2015, 3, 2, 16, 0, 0)
d1_ts = time.mktime(d1.timetuple())
d2_ts = time.mktime(d2.timetuple())
MockPg2.PG_REQS['get_end_last_event'].set_assoc(
params=( self.cluster.cluster_id ),
result=[ [ d1_ts ] ]
)
MockPg2.PG_REQS['get_start_oldest_unfinised_event'].set_assoc(
params= ( self.cluster.cluster_id ),
result=[ [ d2_ts ] ]
)
self.importer.load()
mock_new_events.assert_called_with(d2_ts)
# None unfinished event, search must be done with end datetime of last
# event.
MockPg2.PG_REQS['get_end_last_event'].set_assoc(
params=( self.cluster.cluster_id ),
result=[ [ d1_ts ] ]
)
MockPg2.PG_REQS['get_start_oldest_unfinised_event'].set_assoc(
params= ( self.cluster.cluster_id ),
result=[ ]
)
self.importer.load()
mock_new_events.assert_called_with(d1_ts)
default_datetime = datetime(1970, 1, 1, 1, 0)
# No event in DB: search starting from epoch.
MockPg2.PG_REQS['get_end_last_event'].set_assoc(
params=( self.cluster.cluster_id ),
result=[ ]
)
MockPg2.PG_REQS['get_start_oldest_unfinised_event'].set_assoc(
params= ( self.cluster.cluster_id ),
result=[ ]
)
self.importer.load()
mock_new_events.assert_called_with(default_datetime)
@mock.patch("%s.MySQLdb" % (module), mock_mysqldb())
def test_load_unfound_node(self):
"""EventImporterSlurm.load() raises Exception if one event is linked to
a node not loaded by ArchitectureImporter."""
self.init_load_data()
self.app.arch.nodes = []
self.assertRaisesRegexp(
HPCStatsSourceError,
"event node %s not found in loaded nodes" % (self.node_name),
self.importer.load)
@mock.patch("%s.MySQLdb" % (module), mock_mysqldb())
def test_load_invalid_tres(self):
"""EventImporterSlurm.load() raises exception if invalid tres for an
event is found"""
self.init_load_data()
MockMySQLdb.MY_REQS['get_events']['res'][0][3] = '0=0'
self.assertRaisesRegexp(
HPCStatsSourceError,
"unable to extract cpu_count from event tres",
self.importer.load)
@mock.patch("%s.MySQLdb" % (module), mock_mysqldb())
def test_merge_successive_events(self):
"""EventImporterSlurm.merge_successive_events() should merge successive
events in the list if they are on the same node w/ same type.
"""
e1_start = datetime(2015, 3, 2, 16, 0, 0)
e1_end = datetime(2015, 3, 2, 16, 10, 0)
e2_start = datetime(2015, 3, 2, 16, 10, 0)
e2_end = datetime(2015, 3, 2, 16, 20, 0)
e3_start = datetime(2015, 3, 2, 16, 20, 0)
e3_end = datetime(2015, 3, 2, 16, 30, 0)
node1 = [ Node('node1', self.cluster, 'model1', 'partition1', 16, 8, 0), ]
node2 = [ Node('node2', self.cluster, 'model1', 'partition1', 16, 8, 0), ]
# 3 successive events on one node with same type, they must be merged
# into one event.
events = [
Event(self.cluster, node1, 4, e1_start, e1_end, 'type1', 'reason1'),
Event(self.cluster, node1, 4, e2_start, e2_end, 'type1', 'reason1'),
Event(self.cluster, node1, 4, e3_start, e3_end, 'type1', 'reason1'),
]
merged = self.importer.merge_successive_events(events)
self.assertEquals(1, len(merged))
self.assertEquals(merged[0].start_datetime, e1_start)
self.assertEquals(merged[0].end_datetime, e3_end)
self.assertEquals(merged[0].event_type, 'type1')
self.assertEquals(merged[0].reason, 'reason1')
# 3 successive events on one node node1 with same type, with one event
# on another node node2 in the middle: all events on node1 must be
# merged while the other event on node2 must stay as is.
events = [
Event(self.cluster, node1, 4, e1_start, e1_end, 'type1', 'reason1'),
Event(self.cluster, node2, 4, e2_start, e2_end, 'type1', 'reason1'),
Event(self.cluster, node1, 4, e2_start, e2_end, 'type1', 'reason1'),
Event(self.cluster, node1, 4, e3_start, e3_end, 'type1', 'reason1'),
]
merged = self.importer.merge_successive_events(events)
self.assertEquals(2, len(merged))
self.assertEquals(merged[0].start_datetime, e1_start)
self.assertEquals(merged[0].end_datetime, e3_end)
self.assertEquals(merged[1].end_datetime, e2_end)
self.assertEquals(merged[0].node, node1)
self.assertEquals(merged[1].node, node2)
# 3 successive events on node1 but with different types, they must not
# be merged.
events = [
Event(self.cluster, node1, 4, e1_start, e1_end, 'type1', 'reason1'),
Event(self.cluster, node1, 4, e2_start, e2_end, 'type2', 'reason1'),
Event(self.cluster, node1, 4, e3_start, e3_end, 'type1', 'reason1'),
]
merged = self.importer.merge_successive_events(events)
self.assertEquals(3, len(merged))
@mock.patch("%s.MySQLdb" % (module), mock_mysqldb())
def test_txt_slurm_event_type(self):
"""EventImporterSlurm.txt_slurm_event_type() should give the
appropriate human readable string represation of an event type
according to its hex bitmap value.
"""
tests = [ ( 0x0001, 'DOWN' ),
( 0x0004, 'ERROR' ),
( 0x0012, 'IDLE+NET' ),
( 0x8535, 'MIXED+NET+RES+RESUME+COMPLETING+MAINT' ) ]
for value, expected in tests:
txt = EventImporterSlurm.txt_slurm_event_type(value)
self.assertEquals(txt, expected)
class TestsUserImporterLdap(HPCStatsTestCase):
@mock.patch("HPCStats.DB.HPCStatsDB.psycopg2", mock_psycopg2())
def setUp(self):
self.filename = 'fake'
self.cluster = Cluster('testcluster')
self.cluster.cluster_id = 0
HPCStatsConf.__bases__ = (MockConfigParser, object)
self.conf = HPCStatsConf(self.filename, self.cluster)
self.conf.conf = CONFIG.copy()
self.db = HPCStatsDB(self.conf)
self.db.bind()
self.app = MockApp(self.db, self.conf, self.cluster)
self.importer = UserImporterLdap(self.app,
self.db,
self.conf,
self.cluster)
init_reqs()
def test_init(self):
"""UserImporterLdap.__init__() initializes w/o error
"""
pass
@mock.patch("%s.ldap" % (module), mock_ldap())
@mock.patch("%s.os.path.isfile" % module)
def test_check_groups_alias_file(self, m_isfile):
"""UserImporterLdap.check() should raise HPCStatsSourceError if
groups_alias_file does not exist
"""
# define file path to run the isfile check
self.importer.groups_alias_file = 'test'
# if file exist, everything is OK
m_isfile.return_value = True
self.importer.check()
# if file does not exist, must raise HPCStatsSourceError
m_isfile.return_value = False
self.assertRaisesRegexp(
HPCStatsSourceError,
"Groups alias file test does not exist",
self.importer.check)
def test_load_groups_alias(self):
"""UserImporterLdap.load_groups_alias() tests
"""
self.importer.groups_alias_file = 'test'
# tests valid content
aliases = "testAlong testA\n" \
"testBlong testB\n"
m_open = mock.mock_open(read_data=aliases)
with mock.patch("%s.open" % (module), m_open, create=True):
self.importer.load_groups_alias()
self.assertEquals(self.importer.groups_alias,
{'testAlong': 'testA',
'testBlong': 'testB'})
# tests various invalid content
wrong_aliases = ["testAlong", "testBlong testB fail\n",
"test:fail", "test;epic"]
for wrong_alias in wrong_aliases:
m_open = mock.mock_open(read_data=wrong_alias)
with mock.patch("%s.open" % (module), m_open, create=True):
self.assertRaisesRegexp(
HPCStatsSourceError,
"Malformed line in alias file test",
self.importer.load_groups_alias)
@mock.patch("%s.ldap" % (module), mock_ldap())
def test_load_simple(self):
"""UserImporterLdap.load() should work with simple data from LDAP.
"""
users = [ 'login1', 'login2', 'login3' ]
fill_ldap_users(CONFIG['testcluster/ldap'], users)
MockPg2.PG_REQS['get_unclosed_accounts'].set_assoc (
params=( 0, ),
result=[ ]
)
self.importer.load()
self.assertEquals(len(self.importer.users), 3)
self.assertEquals(len(self.importer.accounts), 3)
@mock.patch("%s.ldap" % (module), mock_ldap())
def test_load_simple2(self):
"""UserImporterLdap.load() should work with simple data from LDAP and
HPCStatsDB.
"""
users = [ 'login1', 'login2', 'login3' ]
fill_ldap_users(CONFIG['testcluster/ldap'], users)
creation_user4 = datetime(2015, 3, 2, 16, 0, 1)
MockPg2.PG_REQS['get_unclosed_accounts'].set_assoc(
params=( 0, ),
result=[ [ 0, 'login4', 'name_user4', 'firstname_user4',
'department_user4', 0, 0, creation_user4 ] ]
)
self.importer.load()
self.assertEquals(len(self.importer.users), 4)
self.assertEquals(len(self.importer.accounts), 4)
@mock.patch("%s.ldap" % (module), mock_ldap())
def test_load_user_no_dp_group(self):
"""UserImporterLdap.load() should work with user having no secondary group.
"""
user = '******'
fill_ldap_users(CONFIG['testcluster/ldap'], [user])
# remove secondary group result for login1
MockLdap.LDAP_REQS['secondary_groups_login1']['res'] = []
# set the cn result of the prim_group LDAP query
primary_group = 'primgrouptest'
MockLdap.LDAP_REQS['prim_group_0']['res'][0][1]['cn'][0] = primary_group
MockPg2.PG_REQS['get_unclosed_accounts'].set_assoc (
params=( 0, ),
result=[ ]
)
# test the user department has been computed based on its primary group
self.importer.load()
self.assertEquals(self.importer.users[0].department,
primary_group+'-unknown')
# test with alternative default subdir
subdir = 'testA'
self.importer.default_subdir = subdir
self.importer.load()
self.assertEquals(self.importer.users[0].department,
primary_group+'-'+subdir)
# test with alias
primary_group_complicated = 'primgrouptestcomplicated'
MockLdap.LDAP_REQS['prim_group_0']['res'][0][1]['cn'][0] = \
primary_group_complicated
self.importer.groups_alias = \
{ primary_group_complicated: primary_group }
self.importer.load()
self.assertEquals(self.importer.users[0].department,
primary_group+'-'+subdir)
# test without primary group -> department must be None
MockLdap.LDAP_REQS['prim_group_0']['res'] = {}
self.importer.load()
self.assertEquals(self.importer.users[0].department, None)
# test with multiple primary groups -> must raise HPCStatsSourceError
primary_group_complicated = 'primgrouptestcomplicated'
MockLdap.LDAP_REQS['prim_group_0']['res'] = [ 'result1', 'result2' ]
self.assertRaisesRegexp(
HPCStatsSourceError,
"too much results .%d. found for user %s primary group %d in " \
"base %s" % (2, user, 0, self.importer.ldap_dn_groups),
self.importer.load)
@mock.patch("%s.ldap" % (module), mock_ldap())
@mock.patch("%s.User.save" % (module))
@mock.patch("%s.Account.save" % (module))
def test_load_update_new_user_other_accounts(self, m_account_save, m_user_save):
"""UserImporterLdap.update() create new user/account found in LDAP
and not found in HPCStatsDB with a creation date equals to today
because there are already existing accounts.
"""
users = [ 'login1' ]
fill_ldap_users(CONFIG['testcluster/ldap'], users)
MockPg2.PG_REQS['get_unclosed_accounts'].set_assoc(
params=( self.cluster.cluster_id, ),
result=[ ]
)
MockPg2.PG_REQS['nb_existing_accounts'].set_assoc(
params=( self.cluster.cluster_id, ),
result=[ [ 0 ], [ 1 ] ]
)
self.importer.load()
self.importer.update()
self.assertEquals(self.importer.accounts[0].creation_date, date.today())
m_user_save.assert_called_with(self.db)
m_account_save.assert_called_with(self.db)
@mock.patch("%s.ldap" % (module), mock_ldap())
@mock.patch("%s.User.save" % (module))
@mock.patch("%s.Account.save" % (module))
def test_load_update_new_user_no_account(self, m_account_save, m_user_save):
"""UserImporterLdap.update() save new user/account found in LDAP
and not found in HPCStatsDB with a creation date equals to epoch
because there is none existing accounts
"""
users = [ 'login1' ]
fill_ldap_users(CONFIG['testcluster/ldap'], users)
self.importer.load()
self.importer.update()
self.assertEquals(self.importer.accounts[0].creation_date, date(1970, 1, 1))
m_user_save.assert_called_with(self.db)
m_account_save.assert_called_with(self.db)
@mock.patch("%s.ldap" % (module), mock_ldap())
@mock.patch("%s.Account.update" % (module))
def test_load_update_close_account(self, m_account_update):
"""UserImporterLdap.update() close account found as unclosed in
HPCStatsDB and not found in LDAP.
"""
users = [ ]
fill_ldap_users(CONFIG['testcluster/ldap'], users)
creation_user2 = datetime(2015, 3, 2, 16, 0, 1)
MockPg2.PG_REQS['get_unclosed_accounts'].set_assoc(
params=( self.cluster.cluster_id, ),
result=[ [ 2, 'login2', 'name_user2', 'firstname_user2',
'department_user2', 0, 0, creation_user2 ] ]
)
self.importer.load()
self.importer.update()
self.assertEquals(self.importer.accounts[0].deletion_date, date.today())
m_account_update.assert_called_with(self.db)
@mock.patch("%s.ldap" % (module), mock_ldap())
@mock.patch("%s.User.update" % (module))
@mock.patch("%s.Account.save" % (module))
def test_load_update_user_wo_account(self, m_account_save, m_user_update):
"""UserImporterLdap.update() create account and update user found in
LDAP and in HPCStatsDB but w/o account on the cluster.
"""
users = [ 'login3' ]
fill_ldap_users(CONFIG['testcluster/ldap'], users)
user3_id = 3
MockPg2.PG_REQS['find_user'].set_assoc(
params=( 'login3', ),
result=[ [ user3_id ] ]
)
MockPg2.PG_REQS['existing_account'].set_assoc(
params=( user3_id, self.cluster.cluster_id, ),
result=[ ]
)
self.importer.load()
self.importer.update()
m_user_update.assert_called_with(self.db)
m_account_save.assert_called_with(self.db)
@mock.patch("%s.ldap" % (module), mock_ldap())
@mock.patch("%s.User.update" % (module))
@mock.patch("%s.Account.update" % (module))
def test_load_update_user_w_account(self, m_account_update, m_user_update):
"""UserImporterLdap.update() update user found in LDAP and in HPCStatsDB
with an unclosed account on the cluster.
"""
users = [ 'login4' ]
fill_ldap_users(CONFIG['testcluster/ldap'], users)
creation_user4 = datetime(2015, 3, 2, 16, 0, 1)
user4_id = 4
MockPg2.PG_REQS['find_user'].set_assoc(
params=( 'login4', ),
result=[ [ user4_id ] ]
)
MockPg2.PG_REQS['existing_account'].set_assoc(
params=( user4_id, self.cluster.cluster_id, ),
result=[ [ 0 ] ]
)
MockPg2.PG_REQS['load_account'].set_assoc(
params=( user4_id, self.cluster.cluster_id, ),
result=[ [ 0, 0, creation_user4, None ] ]
)
self.importer.load()
self.importer.update()
m_user_update.assert_called_with(self.db)
# ensure Account.update() is not called
self.assertRaises(AssertionError,
m_account_update.assert_called_with,
self.db,
None)
@mock.patch("%s.ldap" % (module), mock_ldap())
@mock.patch("%s.User.update" % (module))
@mock.patch("%s.Account.update" % (module))
def test_load_update_user_closed_account(self, m_account_update, m_user_update):
"""UserImporterLdap.update() update user found in LDAP and in HPCStatsDB
with a closed account on the cluster.
"""
users = [ 'login5' ]
fill_ldap_users(CONFIG['testcluster/ldap'], users)
user5_creation = datetime(2015, 3, 2, 16, 0, 1)
user5_deletion = datetime(2015, 3, 2, 16, 0, 1)
user5_id = 5
MockPg2.PG_REQS['find_user'].set_assoc(
params=( 'login5', ),
result=[ [ user5_id ] ]
)
MockPg2.PG_REQS['existing_account'].set_assoc(
params=( user5_id, self.cluster.cluster_id, ),
result=[ [ 0 ] ]
)
MockPg2.PG_REQS['load_account'].set_assoc(
params=( user5_id, self.cluster.cluster_id, ),
result=[ [ 0, 0, user5_creation, user5_deletion ] ]
)
self.importer.load()
self.importer.update()
self.assertEquals(self.importer.accounts[0].deletion_date, None)
m_user_update.assert_called_with(self.db)
m_account_update.assert_called_with(self.db)
