def test_correct_start_date_time(self):
file_name = 'tests/test/data/correct_start_datetime.pbs'
event_names = []
parser = PbsScriptParser(self._config, self._event_defs)
with open(file_name, 'r') as pbs_file:
parser.parse_file(pbs_file)
self.assertEqual(len(event_names), len(parser.events))
def test_missing_l_resource(self):
file_name = 'tests/test/data/missing_l_resource.pbs'
event_names = []
parser = PbsScriptParser(self._config, self._event_defs)
with open(file_name, 'r') as pbs_file:
parser.parse_file(pbs_file)
self.assertEqual(len(event_names), len(parser.events))
def test_size_in_bytes(self):
file_name = 'data/size_in_bytes.pbs'
size = 4096
parser = PbsScriptParser(self._config, self._event_defs)
with open(file_name, 'r') as pbs_file:
parser.parse_file(pbs_file)
job = parser.job
self.assertEquals(size, job.resource_specs['mem'])
def test_compute(self):
file_name = 'data/correct.pbs'
expected_cost = 685.99
parser = PbsScriptParser(self._config, self._event_defs)
with open(file_name, 'r') as pbs_file:
parser.parse_file(pbs_file)
quote_calc = QuoteCalculator(self._config)
credits = quote_calc.compute(parser.job)
self.assertAlmostEqual(expected_cost, credits, delta=0.1)
def test_mem_specs(self):
file_name = 'data/mem_specs.pbs'
event_names = ['invalid_mem_format']
parser = PbsScriptParser(self._config, self._event_defs)
with open(file_name, 'r') as pbs_file:
parser.parse_file(pbs_file)
self.assertEquals(len(event_names), len(parser.events))
for event in parser.events:
self.assertTrue(event['id'] in event_names)
def test_pbs_space_directive(self):
file_name = 'data/pbs_space_directive.pbs'
event_names = ['space_in_pbs_dir']
parser = PbsScriptParser(self._config, self._event_defs)
with open(file_name, 'r') as pbs_file:
parser.parse_file(pbs_file)
self.assertEquals(len(event_names), len(parser.events))
for event in parser.events:
self.assertTrue(event['id'] in event_names)
self.assertEquals(len(event_names), parser.nr_warnings)
def test_wrong_options(self):
file_name = 'data/wrong_options.pbs'
event_names = ['invalid_join', 'invalid_job_name']
parser = PbsScriptParser(self._config, self._event_defs)
with open(file_name, 'r') as pbs_file:
parser.parse_file(pbs_file)
self.assertEquals(len(event_names), len(parser.events))
for event in parser.events:
self.assertTrue(event['id'] in event_names)
self.assertEquals(len(event_names), parser.nr_errors)
def test_confused_mail_options(self):
file_name = 'tests/test/data/confused_mail_options.pbs'
event_names = ['invalid_mail_address']
parser = PbsScriptParser(self._config, self._event_defs)
with open(file_name, 'r') as pbs_file:
parser.parse_file(pbs_file)
self.assertEqual(len(event_names), len(parser.events))
for event in parser.events:
self.assertTrue(event['id'] in event_names)
self.assertEqual(len(event_names), parser.nr_warnings)
def test_misplaced_shebang(self):
file_name = 'data/misplaced_shebang.pbs'
event_names = ['missing_shebang', 'misplaced_shebang']
parser = PbsScriptParser(self._config, self._event_defs)
with open(file_name, 'r') as pbs_file:
parser.parse_file(pbs_file)
self.assertEquals(len(event_names), len(parser.events))
for event in parser.events:
self.assertTrue(event['id'] in event_names)
self.assertEquals(len(event_names), parser.nr_warnings)
def test_no_dash_options(self):
file_name = 'tests/test/data/no_dash_option.pbs'
event_names = ['malformed_pbs_dir']
parser = PbsScriptParser(self._config, self._event_defs)
with open(file_name, 'r') as pbs_file:
parser.parse_file(pbs_file)
self.assertEqual(len(event_names), len(parser.events))
for event in parser.events:
self.assertTrue(event['id'] in event_names)
self.assertEqual(len(event_names), parser.nr_errors)
def test_invalid_start_date_time(self):
file_name = 'tests/test/data/invalid_start_datetime.pbs'
event_names = ['invalid_datetime']
parser = PbsScriptParser(self._config, self._event_defs)
with open(file_name, 'r') as pbs_file:
parser.parse_file(pbs_file)
self.assertEqual(len(event_names), len(parser.events))
for event in parser.events:
self.assertTrue(event['id'] in event_names)
self.assertEqual(len(event_names), parser.nr_errors)
def test_correct(self):
file_name = 'data/correct.pbs'
nr_syntax_events = 0
nr_semantic_events = 0
parser = PbsScriptParser(self._config, self._event_defs)
with open(file_name, 'r') as pbs_file:
parser.parse_file(pbs_file)
self.assertEquals(nr_syntax_events, len(parser.events))
checker = JobChecker(self._config, self._event_defs)
checker.check(parser.job)
self.assertEquals(nr_semantic_events, len(checker.events))
def test_correct(self):
file_name = 'data/correct.pbs'
nr_syntax_events = 0
nr_semantic_events = 0
parser = PbsScriptParser(self._config, self._event_defs)
with open(file_name, 'r') as pbs_file:
parser.parse_file(pbs_file)
self.assertEquals(nr_syntax_events, len(parser.events))
checker = JobChecker(self._config, self._event_defs)
checker.check(parser.job)
self.assertEquals(nr_semantic_events, len(checker.events))
def test_dos(self):
file_name = 'data/dos.pbs'
nr_syntax_events = 4
event_names = ['dos_format']
parser = PbsScriptParser(self._config, self._event_defs)
with open(file_name, 'r') as pbs_file:
parser.parse_file(pbs_file)
self.assertEquals(nr_syntax_events, len(parser.events))
for event in parser.events:
self.assertTrue(event['id'] in event_names)
self.assertEquals(nr_syntax_events, parser.nr_errors)
def test_nodes_ppn_wrong_spec(self):
file_name = 'data/nodes_ppn_wrong_spec.pbs'
nr_syntax_events = 1
event_names = ['ppn_no_number']
parser = PbsScriptParser(self._config, self._event_defs)
with open(file_name, 'r') as pbs_file:
parser.parse_file(pbs_file)
self.assertEquals(nr_syntax_events, len(parser.events))
for event in parser.events:
self.assertTrue(event['id'] in event_names)
self.assertEquals(nr_syntax_events, parser.nr_errors)
def test_pmem_violation(self):
file_name = 'data/pmem_violation.pbs'
event_names = ['insufficient_nodes_mem']
parser = PbsScriptParser(self._config, self._event_defs)
with open(file_name, 'r') as pbs_file:
parser.parse_file(pbs_file)
checker = JobChecker(self._config, self._event_defs)
checker.check(parser.job)
self.assertEquals(len(event_names), len(checker.events))
for event in checker.events:
self.assertTrue(event['id'] in event_names)
self.assertEquals(len(event_names), checker.nr_errors)
def test_mem_pmem(self):
file_name = 'data/mem_pmem.pbs'
event_names = ['both_mem_pmem_specs']
parser = PbsScriptParser(self._config, self._event_defs)
with open(file_name, 'r') as pbs_file:
parser.parse_file(pbs_file)
checker = JobChecker(self._config, self._event_defs)
checker.check(parser.job)
self.assertEquals(len(event_names), len(checker.events))
for event in checker.events:
self.assertTrue(event['id'] in event_names)
self.assertEquals(len(event_names), checker.nr_warnings)
def test_mem_pmem(self):
file_name = 'data/mem_pmem.pbs'
event_names = ['both_mem_pmem_specs']
parser = PbsScriptParser(self._config, self._event_defs)
with open(file_name, 'r') as pbs_file:
parser.parse_file(pbs_file)
checker = JobChecker(self._config, self._event_defs)
checker.check(parser.job)
self.assertEquals(len(event_names), len(checker.events))
for event in checker.events:
self.assertTrue(event['id'] in event_names)
self.assertEquals(len(event_names), checker.nr_warnings)
def test_pmem_violation(self):
file_name = 'data/pmem_violation.pbs'
event_names = ['insufficient_nodes_mem']
parser = PbsScriptParser(self._config, self._event_defs)
with open(file_name, 'r') as pbs_file:
parser.parse_file(pbs_file)
checker = JobChecker(self._config, self._event_defs)
checker.check(parser.job)
self.assertEquals(len(event_names), len(checker.events))
for event in checker.events:
self.assertTrue(event['id'] in event_names)
self.assertEquals(len(event_names), checker.nr_errors)
def test_unknown_feature(self):
file_name = 'data/unknown_feature.pbs'
nr_syntax_events = 0
nr_semantic_events = 1
event_name = 'unknown_feature'
parser = PbsScriptParser(self._config, self._event_defs)
with open(file_name, 'r') as pbs_file:
parser.parse_file(pbs_file)
self.assertEquals(nr_syntax_events, len(parser.events))
checker = JobChecker(self._config, self._event_defs)
checker.check(parser.job)
self.assertEquals(nr_semantic_events, len(checker.events))
self.assertEquals(event_name, checker.events[0]['id'])
def test_too_large_ppn(self):
file_name = 'data/too_large_ppn.pbs'
nr_syntax_events = 0
nr_semantic_events = 1
event_name = 'insufficient_ppn_nodes'
parser = PbsScriptParser(self._config, self._event_defs)
with open(file_name, 'r') as pbs_file:
parser.parse_file(pbs_file)
self.assertEquals(nr_syntax_events, len(parser.events))
checker = JobChecker(self._config, self._event_defs)
checker.check(parser.job)
self.assertEquals(nr_semantic_events, len(checker.events))
self.assertEquals(event_name, checker.events[0]['id'])
def test_too_large_ppn(self):
file_name = 'data/too_large_ppn.pbs'
nr_syntax_events = 0
nr_semantic_events = 1
event_name = 'insufficient_ppn_nodes'
parser = PbsScriptParser(self._config, self._event_defs)
with open(file_name, 'r') as pbs_file:
parser.parse_file(pbs_file)
self.assertEquals(nr_syntax_events, len(parser.events))
checker = JobChecker(self._config, self._event_defs)
checker.check(parser.job)
self.assertEquals(nr_semantic_events, len(checker.events))
self.assertEquals(event_name, checker.events[0]['id'])
def test_unknown_feature(self):
file_name = 'data/unknown_feature.pbs'
nr_syntax_events = 0
nr_semantic_events = 1
event_name = 'unknown_feature'
parser = PbsScriptParser(self._config, self._event_defs)
with open(file_name, 'r') as pbs_file:
parser.parse_file(pbs_file)
self.assertEquals(nr_syntax_events, len(parser.events))
checker = JobChecker(self._config, self._event_defs)
checker.check(parser.job)
self.assertEquals(nr_semantic_events, len(checker.events))
self.assertEquals(event_name, checker.events[0]['id'])
def test_too_many_nodes(self):
file_name = 'data/too_many_nodes.pbs'
nr_syntax_events = 0
event_names = ['insufficient_nodes', 'insufficient_ppn_nodes',
'insufficient_nodes_mem']
parser = PbsScriptParser(self._config, self._event_defs)
with open(file_name, 'r') as pbs_file:
parser.parse_file(pbs_file)
self.assertEquals(nr_syntax_events, len(parser.events))
checker = JobChecker(self._config, self._event_defs)
checker.check(parser.job)
self.assertEquals(len(event_names), len(checker.events))
for event in checker.events:
self.assertTrue(event['id'] in event_names)
def test_too_many_nodes(self):
file_name = 'data/too_many_nodes.pbs'
nr_syntax_events = 0
event_names = ['insufficient_nodes', 'insufficient_ppn_nodes',
'insufficient_nodes_mem']
parser = PbsScriptParser(self._config, self._event_defs)
with open(file_name, 'r') as pbs_file:
parser.parse_file(pbs_file)
self.assertEquals(nr_syntax_events, len(parser.events))
checker = JobChecker(self._config, self._event_defs)
checker.check(parser.job)
self.assertEquals(len(event_names), len(checker.events))
for event in checker.events:
self.assertTrue(event['id'] in event_names)
def test_simple(self):
file_name = 'tests/test/data/simple.pbs'
nodes = 2
ppn = 4
walltime = 72*3600
partition = self._config['default_partition']
parser = PbsScriptParser(self._config, self._event_defs)
with open(file_name, 'r') as pbs_file:
parser.parse_file(pbs_file)
job = parser.job
nodes_specs = job.resource_specs['nodes'][0]
self.assertEqual(nodes, nodes_specs['nodes'])
self.assertEqual(ppn, nodes_specs['ppn'])
self.assertEqual(walltime, job.resource_specs['walltime'])
self.assertEqual(partition, job.resource_specs['partition'])
def test_working_dir_no_var(self):
file_name = 'tests/test/data/working_dir_no_var.pbs'
event_names = ['missspelled']
parser = PbsScriptParser(self._config, self._event_defs)
with open(file_name, 'r') as pbs_file:
parser.parse_file(pbs_file)
self.assertEqual(0, len(parser.events))
checker = JobChecker(self._config, self._event_defs)
checker.check(parser.job)
script_checker = ScriptChecker(self._config, self._event_defs)
script_checker.check(parser.job, parser.script_first_line_nr)
self.assertEqual(len(event_names), len(script_checker.events))
for event in script_checker.events:
self.assertTrue(event['id'] in event_names)
self.assertEqual(len(event_names), script_checker.nr_warnings)
def test_walltime_no_queue(self):
file_name = 'data/walltime_no_queue.pbs'
nr_syntax_events = 0
nr_semantic_events = 0
queue_name = 'qdef'
expected_walltime = 45*3600 + 15*60
parser = PbsScriptParser(self._config, self._event_defs)
with open(file_name, 'r') as pbs_file:
parser.parse_file(pbs_file)
self.assertEquals(nr_syntax_events, len(parser.events))
checker = JobChecker(self._config, self._event_defs)
checker.check(parser.job)
self.assertEquals(nr_semantic_events, len(checker.events))
job = parser.job
self.assertEquals(queue_name, job.queue)
self.assertEquals(expected_walltime, job.resource_spec('walltime'))
def test_walltime_no_queue(self):
file_name = 'data/walltime_no_queue.pbs'
nr_syntax_events = 0
nr_semantic_events = 0
queue_name = 'qdef'
expected_walltime = 45*3600 + 15*60
parser = PbsScriptParser(self._config, self._event_defs)
with open(file_name, 'r') as pbs_file:
parser.parse_file(pbs_file)
self.assertEquals(nr_syntax_events, len(parser.events))
checker = JobChecker(self._config, self._event_defs)
checker.check(parser.job)
self.assertEquals(nr_semantic_events, len(checker.events))
job = parser.job
self.assertEquals(queue_name, job.queue)
self.assertEquals(expected_walltime, job.resource_spec('walltime'))
def test_multiple_resources(self):
file_name = 'data/multiple_resources.pbs'
nr_specs = 2
nr_nodes = [3, 5]
features = ['mem128', 'mem64']
nr_features = 1
parser = PbsScriptParser(self._config, self._event_defs)
with open(file_name, 'r') as pbs_file:
parser.parse_file(pbs_file)
job = parser.job
node_specs = job.resource_specs['nodes']
self.assertEquals(nr_specs, len(node_specs))
for i, node_spec in enumerate(node_specs):
self.assertEquals(nr_nodes[i], node_spec['nodes'])
self.assertEquals(nr_features, len(node_spec['features']))
self.assertEquals(features[i], node_spec['features'][0])
def test_simple(self):
file_name = 'data/simple.pbs'
project = 'lp_qlint'
nodes = 2
ppn = 4
walltime = 72*3600
partition = self._config['default_partition']
parser = PbsScriptParser(self._config, self._event_defs)
with open(file_name, 'r') as pbs_file:
parser.parse_file(pbs_file)
job = parser.job
nodes_specs = job.resource_specs['nodes'][0]
self.assertEquals(nodes, nodes_specs['nodes'])
self.assertEquals(ppn, nodes_specs['ppn'])
self.assertEquals(walltime, job.resource_specs['walltime'])
self.assertEquals(partition, job.resource_specs['partition'])
def test_parsing(self):
file_name = 'data/correct.pbs'
name = 'my_job'
project = 'lp_qlint'
nodes = 2
ppn = 4
walltime = 72*3600
qos = 'debugging'
join = 'oe'
parser = PbsScriptParser(self._config, self._event_defs)
with open(file_name, 'r') as pbs_file:
parser.parse_file(pbs_file)
job = parser.job
self.assertEquals(name, job.name)
self.assertEquals(project, job.project)
nodes_specs = job.resource_specs['nodes'][0]
self.assertEquals(nodes, nodes_specs['nodes'])
self.assertEquals(ppn, nodes_specs['ppn'])
self.assertEquals(walltime, job.resource_specs['walltime'])
self.assertEquals(qos, job.resource_specs['qos'])
self.assertEquals(join, job.join)
if options.events:
conf['event_file'] = options.events
if not os.path.isfile(conf['cluster_db']):
msg = "### error: can not open cluster DB '{0}'\n"
sys.stderr.write(msg.format(conf['cluster_db']))
sys.exit(CAN_NOT_OPEN_CLUSTER_DB_FILE)
try:
with open(conf['event_file']) as event_file:
event_defs = json.load(event_file)
except EnvironmentError as error:
msg = "### error: can not open event file '{0}'\n"
sys.stderr.write(msg.format(conf['event_file']))
sys.exit(CAN_NOT_OPEN_EVENT_FILE)
indent = ' '*conf['report_indent']
try:
pbs_parser = PbsScriptParser(conf, event_defs)
try:
with open(options.pbs_file, 'r') as pbs_file:
pbs_parser.parse_file(pbs_file)
except EnvironmentError as error:
msg = "### error: can not open PBS file '{0}'\n"
sys.stderr.write(msg.format(options.events))
sys.exit(CAN_NOT_OPEN_PBS)
job_checker = JobChecker(conf, event_defs)
job_checker.check(pbs_parser.job)
pbs_parser.context = 'semantics'
pbs_parser.merge_events(job_checker.events)
script_checker = ScriptChecker(conf, event_defs)
script_checker.check(pbs_parser.job,
pbs_parser.script_first_line_nr)
pbs_parser.context = 'file'
