section = 'constraints'
self.conf.conf = { section: {'ignored_errors': ''}}
mgr = HPCStatsErrorMgr(self.conf)
self.assertEquals(mgr.ignored_errors, set())
self.conf.conf = { section: {'ignored_errors': 'E_T0001'}}
mgr = HPCStatsErrorMgr(self.conf)
self.assertEquals(mgr.ignored_errors, set([Errors.E_T0001]))
self.conf.conf = { section: {'ignored_errors': 'E_T0001, fail'}}
mgr = HPCStatsErrorMgr(self.conf)
self.assertEquals(mgr.ignored_errors, set([Errors.E_T0001]))
self.conf.conf = { section: {'ignored_errors': ' fail '}}
mgr = HPCStatsErrorMgr(self.conf)
self.assertEquals(mgr.ignored_errors, set())
self.conf.conf = { section: {'ignored_errors': ' fail , E_T0001, fail,,, faild, E_T0002,E_J1111'}}
mgr = HPCStatsErrorMgr(self.conf)
self.assertEquals(mgr.ignored_errors, set([Errors.E_T0001, Errors.E_T0002]))
self.conf.conf = { section: {'ignored_errors': 'E_T0001,E_T0001'}}
mgr = HPCStatsErrorMgr(self.conf)
self.assertEquals(mgr.ignored_errors, set([Errors.E_T0001]))
if __name__ == '__main__':
loadtestcase(TestsHPCStatsErrorMgr)
self.filename = 'fake'
self.cluster = 'testcluster'
HPCStatsConf.__bases__ = (MockConfigParser, object)
self.conf = HPCStatsConf(self.filename, self.cluster)
self.conf.conf = CONFIG
self.app = None
self.db = HPCStatsDB(self.conf)
self.db.bind()
self.importer = ProjectImporterCSV(self.app, self.db, self.conf)
def test_update(self):
"""ProjectImporterCSV.update() works with simple data
"""
domain1 = Domain('dom1', 'domain name 1')
project1 = Project(domain1, 'code1', 'project description 1')
MockPg2.PG_REQS['save_project'].set_assoc(
params=( project1.code, project1.description, domain1.key ),
result=[ [ 1 ] ]
)
self.importer.projects = [ project1 ]
self.importer.domains = [ domain1 ]
self.importer.update()
if __name__ == '__main__':
loadtestcase(TestsProjectImporterCSVLoad)
loadtestcase(TestsProjectImporterCSVUpdate)
MockPg2.PG_REQS['exist_domain'].set_assoc(
params=( 'D1', ),
result=[ ]
)
self.modifier.run()
@mock.patch("HPCStats.DB.HPCStatsDB.psycopg2", mock_psycopg2())
def test_run_create_domain_already_existing(self):
"""HPCStatsModifier.run() raises exception when creating a domain that
already exists.
"""
params = { 'business': None,
'project': None,
'set_description': None,
'set_domain': None,
'new_domain': 'D1',
'domain_name': 'D1 description' }
self.modifier = HPCStatsModifier(self.conf, self.cluster, params)
MockPg2.PG_REQS['exist_domain'].set_assoc(
params=( 'D1', ),
result=[ [ 'D1' ] ]
)
self.assertRaisesRegexp(HPCStatsRuntimeError,
"domain D1 already exists in database",
self.modifier.run)
if __name__ == '__main__':
loadtestcase(TestsHPCStatsModifier)
fs_i = fs_m.return_value
event_i = event_m.return_value
job_i = job_m.return_value
self.importer.run()
arch_i.load.assert_called_once_with()
arch_i.update.assert_called_once_with()
user_i.load.assert_called_once_with()
user_i.update.assert_called_once_with()
fs_i.load.assert_called_once_with()
fs_i.update.assert_called_once_with()
event_i.load.assert_called_once_with()
event_i.update.assert_called_once_with()
job_i.load_update_window.assert_called_once_with()
def test_run_exception_no_hpcstatsdb(self):
"""HPCStatsImporter.run() raise exception when hpcstatsdb section is
missing.
"""
del self.conf.conf['hpcstatsdb']
self.assertRaisesRegexp(
HPCStatsConfigurationException,
"section hpcstatsdb not found",
self.importer.run)
if __name__ == '__main__':
loadtestcase(TestsHPCStatsImporter)
"--set-description or --set-domain parameters are required to modify a project",
self.parser.parse_args, argv)
def test_modify_project_conflict_params(self):
"""HPCStatsArgumentParser.parse_args() raise exception when
modify project w/ conflicting params
"""
argv = [
'modify', '--project-code', 'P1', '--set-description', 'desc P1',
'--set-domain', 'D1'
]
self.assertRaisesRegexp(
HPCStatsArgumentException,
"--set-description and --set-domain parameters are mutually exclusive to modify a project",
self.parser.parse_args, argv)
def test_modify_domain_missing_name(self):
"""HPCStatsArgumentParser.parse_args() raise exception when
modify domain w/o name
"""
argv = ['modify', '--new-domain', 'D1']
self.assertRaisesRegexp(
HPCStatsArgumentException,
"--domain-name parameter is required to create a new domain",
self.parser.parse_args, argv)
if __name__ == '__main__':
loadtestcase(TestsHPCStatsArgumentParser)
)
self.importer.cluster = cluster1
self.importer.nodes = [ node1 ]
self.importer.update()
def test_update_2(self):
"""ProjectImporterCSV.update() detect existing cluster and node
"""
cluster1 = Cluster('cluster1')
node1 = Node('node1', cluster1, 'model1', 'test_partition', 12, 6 * 1024 ** 3, 1)
MockPg2.PG_REQS['find_cluster'].set_assoc(
params=( cluster1.name, ),
result=[ [ 1 ] ]
)
MockPg2.PG_REQS['find_node'].set_assoc(
params=( node1.name, cluster1.cluster_id, ),
result=[ [ 1 ] ]
)
self.importer.cluster = cluster1
self.importer.nodes = [ node1 ]
self.importer.update()
if __name__ == '__main__':
loadtestcase(TestsArchitectureImporterArchfileLoad)
loadtestcase(TestsArchitectureImporterArchfileUpdate)
"""ProjectImporterCSV.update() works when business code exists
"""
business1 = Business('code1', 'business description 1')
MockPg2.PG_REQS['existing_business'].set_assoc(
params=(business1.code, ), result=[['code1']])
self.importer.businesses = [business1]
self.importer.update()
@mock.patch("%s.Business.update" % (module))
def test_update_exists_with_mock(self, mock_update):
"""ProjectImporterCSV.update() call Business.update() when business
code exists
"""
business1 = Business('code1', 'business description 1')
MockPg2.PG_REQS['existing_business'].set_assoc(
params=(business1.code, ), result=[['code1']])
self.importer.businesses = [business1]
self.importer.update()
mock_update.assert_called_with(self.db)
if __name__ == '__main__':
loadtestcase(TestsBusinessCodeImporterCSVLoad)
loadtestcase(TestsBusinessCodeImporterCSVUpdate)
self.assertEquals(merged[1].node, node2)
# 3 successive events on node1 but with different types, they must not
# be merged.
events = [
Event(self.cluster, node1, 4, e1_start, e1_end, 'type1', 'reason1'),
Event(self.cluster, node1, 4, e2_start, e2_end, 'type2', 'reason1'),
Event(self.cluster, node1, 4, e3_start, e3_end, 'type1', 'reason1'),
]
merged = self.importer.merge_successive_events(events)
self.assertEquals(3, len(merged))
@mock.patch("%s.MySQLdb" % (module), mock_mysqldb())
def test_txt_slurm_event_type(self):
"""EventImporterSlurm.txt_slurm_event_type() should give the
appropriate human readable string represation of an event type
according to its hex bitmap value.
"""
tests = [ ( 0x0001, 'DOWN' ),
( 0x0004, 'ERROR' ),
( 0x0012, 'IDLE+NET' ),
( 0x8535, 'MIXED+NET+RES+RESUME+COMPLETING+MAINT' ) ]
for value, expected in tests:
txt = EventImporterSlurm.txt_slurm_event_type(value)
self.assertEquals(txt, expected)
if __name__ == '__main__':
loadtestcase(TestsEventImporterSlurm)
business1 = Business('code1', 'business description 1')
MockPg2.PG_REQS['existing_business'].set_assoc(
params=( business1.code, ),
result=[ [ 'code1' ] ]
)
self.importer.businesses = [ business1 ]
self.importer.update()
@mock.patch("%s.Business.update" % (module))
def test_update_exists_with_mock(self, mock_update):
"""ProjectImporterCSV.update() call Business.update() when business
code exists
"""
business1 = Business('code1', 'business description 1')
MockPg2.PG_REQS['existing_business'].set_assoc(
params=( business1.code, ),
result=[ [ 'code1' ] ]
)
self.importer.businesses = [ business1 ]
self.importer.update()
mock_update.assert_called_with(self.db)
if __name__ == '__main__':
loadtestcase(TestsBusinessCodeImporterCSVLoad)
loadtestcase(TestsBusinessCodeImporterCSVUpdate)
self.logger.warn(Errors.E_T0001, "error 1 %s", 'test')
self.assertIn('TestsHPCStatsLogger: error 1 test',
self.handler.messages['warning'])
self.handler.reset()
error_mgr = MockHPCStatsErrorMgr()
HPCStatsLogger.set_error_mgr(error_mgr)
# error manager w/o ignored errors -> warning with ERROR <code> prefix
self.logger.warn(Errors.E_T0001, "error 2 %s", 'test')
self.assertIn('TestsHPCStatsLogger: ERROR E_T0001: error 2 test',
self.handler.messages['warning'])
print self.handler.messages
self.assertEquals(len(self.handler.messages['debug']), 0)
self.handler.reset()
# add E_T0001 to error manager ignored errors set -> debug with IGNORED
# ERROR <code> prefix
error_mgr.ignored_errors.add(Errors.E_T0001)
self.logger.warn(Errors.E_T0001, "error 3 %s", 'test')
self.assertIn(
'TestsHPCStatsLogger: (IGNORED) ERROR E_T0001: error 3 test',
self.handler.messages['debug'])
self.assertEquals(len(self.handler.messages['warning']), 0)
self.handler.reset()
if __name__ == '__main__':
loadtestcase(TestsHPCStatsLogger)
HPCStatsArgumentException,
"--set-description or --set-domain parameters are required to modify a project",
self.parser.parse_args,
argv)
def test_modify_project_conflict_params(self):
"""HPCStatsArgumentParser.parse_args() raise exception when
modify project w/ conflicting params
"""
argv = [ 'modify', '--project-code', 'P1', '--set-description', 'desc P1', '--set-domain', 'D1' ]
self.assertRaisesRegexp(
HPCStatsArgumentException,
"--set-description and --set-domain parameters are mutually exclusive to modify a project",
self.parser.parse_args,
argv)
def test_modify_domain_missing_name(self):
"""HPCStatsArgumentParser.parse_args() raise exception when
modify domain w/o name
"""
argv = [ 'modify', '--new-domain', 'D1' ]
self.assertRaisesRegexp(
HPCStatsArgumentException,
"--domain-name parameter is required to create a new domain",
self.parser.parse_args,
argv)
if __name__ == '__main__':
loadtestcase(TestsHPCStatsArgumentParser)
with a closed account on the cluster.
"""
users = [ 'login5' ]
fill_ldap_users(CONFIG['testcluster/ldap'], users)
user5_creation = datetime(2015, 3, 2, 16, 0, 1)
user5_deletion = datetime(2015, 3, 2, 16, 0, 1)
user5_id = 5
MockPg2.PG_REQS['find_user'].set_assoc(
params=( 'login5', ),
result=[ [ user5_id ] ]
)
MockPg2.PG_REQS['existing_account'].set_assoc(
params=( user5_id, self.cluster.cluster_id, ),
result=[ [ 0 ] ]
)
MockPg2.PG_REQS['load_account'].set_assoc(
params=( user5_id, self.cluster.cluster_id, ),
result=[ [ 0, 0, user5_creation, user5_deletion ] ]
)
self.importer.load()
self.importer.update()
self.assertEquals(self.importer.accounts[0].deletion_date, None)
m_user_update.assert_called_with(self.db)
m_account_update.assert_called_with(self.db)
if __name__ == '__main__':
loadtestcase(TestsUserImporterLdap)
self.modifier = HPCStatsModifier(self.conf, self.cluster, params)
MockPg2.PG_REQS['exist_domain'].set_assoc(params=('D1', ), result=[])
self.modifier.run()
@mock.patch("HPCStats.DB.HPCStatsDB.psycopg2", mock_psycopg2())
def test_run_create_domain_already_existing(self):
"""HPCStatsModifier.run() raises exception when creating a domain that
already exists.
"""
params = {
'business': None,
'project': None,
'set_description': None,
'set_domain': None,
'new_domain': 'D1',
'domain_name': 'D1 description'
}
self.modifier = HPCStatsModifier(self.conf, self.cluster, params)
MockPg2.PG_REQS['exist_domain'].set_assoc(params=('D1', ),
result=[['D1']])
self.assertRaisesRegexp(HPCStatsRuntimeError,
"domain D1 already exists in database",
self.modifier.run)
if __name__ == '__main__':
loadtestcase(TestsHPCStatsModifier)
self.cluster = 'testcluster'
HPCStatsConf.__bases__ = (MockConfigParser, object)
self.conf = HPCStatsConf(self.filename, self.cluster)
self.conf.conf = CONFIG
self.app = None
self.db = HPCStatsDB(self.conf)
self.db.bind()
self.importer = ProjectImporterCSV(self.app, self.db, self.conf)
def test_update(self):
"""ProjectImporterCSV.update() works with simple data
"""
domain1 = Domain('dom1', 'domain name 1')
project1 = Project(domain1, 'code1', 'project description 1')
MockPg2.PG_REQS['save_project'].set_assoc(params=(project1.code,
project1.description,
domain1.key),
result=[[1]])
self.importer.projects = [project1]
self.importer.domains = [domain1]
self.importer.update()
if __name__ == '__main__':
loadtestcase(TestsProjectImporterCSVLoad)
loadtestcase(TestsProjectImporterCSVUpdate)
"""
# none error manager
self.logger.warn(Errors.E_T0001, "error 1 %s", 'test')
self.assertIn('TestsHPCStatsLogger: error 1 test', self.handler.messages['warning'])
self.handler.reset()
error_mgr = MockHPCStatsErrorMgr()
HPCStatsLogger.set_error_mgr(error_mgr)
# error manager w/o ignored errors -> warning with ERROR <code> prefix
self.logger.warn(Errors.E_T0001, "error 2 %s", 'test')
self.assertIn('TestsHPCStatsLogger: ERROR E_T0001: error 2 test',
self.handler.messages['warning'])
print self.handler.messages
self.assertEquals(len(self.handler.messages['debug']), 0)
self.handler.reset()
# add E_T0001 to error manager ignored errors set -> debug with IGNORED
# ERROR <code> prefix
error_mgr.ignored_errors.add(Errors.E_T0001)
self.logger.warn(Errors.E_T0001, "error 3 %s", 'test')
self.assertIn('TestsHPCStatsLogger: (IGNORED) ERROR E_T0001: error 3 test',
self.handler.messages['debug'])
self.assertEquals(len(self.handler.messages['warning']), 0)
self.handler.reset()
if __name__ == '__main__':
loadtestcase(TestsHPCStatsLogger)
# 3 successive events on node1 but with different types, they must not
# be merged.
events = [
Event(self.cluster, node1, 4, e1_start, e1_end, 'type1',
'reason1'),
Event(self.cluster, node1, 4, e2_start, e2_end, 'type2',
'reason1'),
Event(self.cluster, node1, 4, e3_start, e3_end, 'type1',
'reason1'),
]
merged = self.importer.merge_successive_events(events)
self.assertEquals(3, len(merged))
@mock.patch("%s.MySQLdb" % (module), mock_mysqldb())
def test_txt_slurm_event_type(self):
"""EventImporterSlurm.txt_slurm_event_type() should give the
appropriate human readable string represation of an event type
according to its hex bitmap value.
"""
tests = [(0x0001, 'DOWN'), (0x0004, 'ERROR'), (0x0012, 'IDLE+NET'),
(0x8535, 'MIXED+NET+RES+RESUME+COMPLETING+MAINT')]
for value, expected in tests:
txt = EventImporterSlurm.txt_slurm_event_type(value)
self.assertEquals(txt, expected)
if __name__ == '__main__':
loadtestcase(TestsEventImporterSlurm)
with a closed account on the cluster.
"""
users = [ 'login5' ]
fill_ldap_users(CONFIG['testcluster/ldap'], users)
user5_creation = datetime(2015, 3, 2, 16, 0, 1)
user5_deletion = datetime(2015, 3, 2, 16, 0, 1)
user5_id = 5
MockPg2.PG_REQS['find_user'].set_assoc(
params=( 'login5', ),
result=[ [ user5_id ] ]
)
MockPg2.PG_REQS['existing_account'].set_assoc(
params=( user5_id, self.cluster.cluster_id, ),
result=[ [ 0 ] ]
)
MockPg2.PG_REQS['load_account'].set_assoc(
params=( user5_id, self.cluster.cluster_id, ),
result=[ [ 0, 0, user5_creation, user5_deletion ] ]
)
self.importer.load()
self.importer.update()
self.assertEquals(self.importer.accounts[0].deletion_date, None)
m_user_update.assert_called_with(self.db)
m_account_update.assert_called_with(self.db)
if __name__ == '__main__':
loadtestcase(TestsUserImporterLdap)
based on job partition list and its nodelist.
"""
# Only one element in job partition list: it must be returned whatever
# the nodelist and ArchitectureImporter job partitions
self.app.arch.partitions = { }
result = self.importer.job_partition(0, 'partition2', 'node[1-100]')
self.assertEquals(result, 'partition2')
# Multiple elements but None nodelist: it must return arbitrary the
# first partition
self.app.arch.partitions = { }
result = self.importer.job_partition(0, 'partition1,partition2', None)
self.assertEquals(result, 'partition1')
# Multiple elements in partition and defined nodelist: it must return
# a corresponding partition loaded by ArchitectureImporter and
# associated to a nodelist that fully intersects
self.app.arch.partitions = { 'node[1-100]': [ 'partitionX', 'partition2'] }
result = self.importer.job_partition(0, 'partition1,partition2', 'node[1-100]')
self.assertEquals(result, 'partition2')
self.app.arch.partitions = { 'node[1-99]': [ 'partition1' ],
'node[1-100],bm[1-10]': [ 'partitionX', 'partition2' ] }
result = self.importer.job_partition(0, 'partition1,partition2', 'node[1-100]')
self.assertEquals(result, 'partition2')
if __name__ == '__main__':
loadtestcase(TestsJobImporterSlurm)
users = [ ]
users_no_group = [ 'login1' ]
fill_ldap_users(CONFIG['testcluster/ldap'], users, users_no_group)
MockMySQLdb.MY_REQS['get_users']['res'] = \
[ [ 'login1' ] ]
user1_id = 1
MockPg2.PG_REQS['find_user'].set_assoc(
params=( 'login1', ),
result=[ [ user1_id ] ]
)
MockPg2.PG_REQS['existing_account'].set_assoc(
params=( user1_id, self.cluster.cluster_id, ),
result=[ [ 0 ] ]
)
self.importer.load()
self.importer.update()
m_user_update.assert_called_with(self.db)
# ensure Account.update() is not called
self.assertRaises(AssertionError,
m_account_update.assert_called_with,
self.db,
None)
if __name__ == '__main__':
loadtestcase(TestsUserImporterLdapSlurm)
self.assertEquals('default', self.conf.get_default('inexisting', 'param', 'default'))
self.assertEquals(3, self.conf.get_default('inexisting', 'param', 3))
self.assertEquals([], self.conf.get_default('inexisting', 'param', []))
self.assertEquals(None, self.conf.get_default('inexisting', 'param', None))
def test_get_clusters_list(self):
"""HPCStatsConf.get_clusters_list() tests
"""
self.conf.conf['clusters'] = {}
self.conf.conf['clusters']['clusters'] = 'clustera'
self.assertEquals(['clustera'], self.conf.get_clusters_list())
self.conf.conf['clusters']['clusters'] = 'clustera,clusterb'
self.assertEquals(['clustera','clusterb'], self.conf.get_clusters_list())
def test_get_list(self):
"""HPCStatsConf.get_list() tests
"""
self.assertEquals([], self.conf.get_list('inexisting', 'param'))
self.conf.conf['sectiona']['option_list'] = 'item1'
self.assertEquals(['item1'], self.conf.get_list('sectiona', 'option_list'))
self.conf.conf['sectiona']['option_list'] = 'item1,item2'
self.assertEquals(['item1', 'item2'], self.conf.get_list('sectiona', 'option_list'))
self.conf.conf['sectiona']['option_list'] = ' item1 ,item2 '
self.assertEquals(['item1', 'item2'], self.conf.get_list('sectiona', 'option_list'))
self.conf.conf['sectiona']['option_list'] = ' ,, item1 ,item2 ,, '
self.assertEquals(['item1', 'item2'], self.conf.get_list('sectiona', 'option_list'))
if __name__ == '__main__':
loadtestcase(TestsHPCStatsConf)