def test_non_mpi(self): """Test that node and cpu count are set to zero if MPI isn't used.""" parser = PBSParser('testSite', 'testHost', False) line = ('10/01/2011 01:05:30;E;21010040.lcgbatch01.gridpp.rl.ac.uk;user=plhcb010 group=prodlhcb jobname=cre09_655648918 ' 'queue=gridWN ctime=1317427361 qtime=1317427361 etime=1317427361 start=1317427530 [email protected] ' 'Resource_List.cput=800:00:00 Resource_List.neednodes=1:ppn=8 Resource_List.nodect=1 Resource_List.nodes=1:ppn=8 ' 'Resource_List.opsys=sl5 Resource_List.pcput=96:00:00 Resource_List.walltime=96:00:00 session=7428 end=1317427530 ' 'Exit_status=0 resources_used.cput=00:00:00 resources_used.mem=0kb resources_used.vmem=0kb resources_used.walltime=00:00:00') record = parser.parse(line) cont = record._record_content self.assertEquals(cont['NodeCount'], 0) self.assertEquals(cont['Processors'], 0)
def test_non_mpi(self): """Test that node and cpu count are set to zero if MPI isn't used.""" parser = PBSParser('testSite', 'testHost', False) line = ( '10/01/2011 01:05:30;E;21010040.lcgbatch01.gridpp.rl.ac.uk;user=plhcb010 group=prodlhcb jobname=cre09_655648918 ' 'queue=gridWN ctime=1317427361 qtime=1317427361 etime=1317427361 start=1317427530 [email protected] ' 'Resource_List.cput=800:00:00 Resource_List.neednodes=1:ppn=8 Resource_List.nodect=1 Resource_List.nodes=1:ppn=8 ' 'Resource_List.opsys=sl5 Resource_List.pcput=96:00:00 Resource_List.walltime=96:00:00 session=7428 end=1317427530 ' 'Exit_status=0 resources_used.cput=00:00:00 resources_used.mem=0kb resources_used.vmem=0kb resources_used.walltime=00:00:00' ) record = parser.parse(line) cont = record._record_content self.assertEquals(cont['NodeCount'], 0) self.assertEquals(cont['Processors'], 0)
class ParserPBSTest(TestCase): ''' Test case for PBS parser ''' def setUp(self): self.parser = PBSParser('testSite', 'testHost', True) def test_parse(self): line1 = ('10/02/2011 06:41:44;E;21048463.lcgbatch01.gridpp.rl.ac.uk;user=patls009 group=prodatls jobname=cre09_443343882 queue=grid4000M ' 'ctime=1317509574 qtime=1317509574 etime=1317509574 start=1317509945 [email protected] ' 'exec_host=lcg1277.gridpp.rl.ac.uk/5 Resource_List.cput=96:00:00 Resource_List.neednodes=lcg1277.gridpp.rl.ac.uk ' 'Resource_List.opsys=sl5 Resource_List.pcput=96:00:00 Resource_List.pmem=4000mb Resource_List.walltime=96:00:00 session=20374 ' 'end=1317534104 Exit_status=0 resources_used.cput=18:15:24 resources_used.mem=2031040kb resources_used.vmem=3335528kb resources_used.walltime=19:23:4') line1_values = {"JobName": "21048463.lcgbatch01.gridpp.rl.ac.uk", "LocalUserID":"patls009", "LocalUserGroup": "prodatls", "WallDuration":69784, "CpuDuration": 65724, "StartTime": datetime.datetime.utcfromtimestamp(1317509945), "StopTime": datetime.datetime.utcfromtimestamp(1317534104), "MemoryReal":2031040, "MemoryVirtual": 3335528, "NodeCount": 1, "Processors": 1 } line2 = ('10/01/2011 01:05:30;E;21010040.lcgbatch01.gridpp.rl.ac.uk;user=plhcb010 group=prodlhcb jobname=cre09_655648918 ' 'queue=gridWN ctime=1317427361 qtime=1317427361 etime=1317427361 start=1317427530 [email protected] ' 'exec_host=lcg0766.gridpp.rl.ac.uk/7+lcg0766.gridpp.rl.ac.uk/6+lcg0766.gridpp.rl.ac.uk/5+lcg0766.gridpp.rl.ac.uk/4+' 'lcg0766.gridpp.rl.ac.uk/3+lcg0766.gridpp.rl.ac.uk/2+lcg0766.gridpp.rl.ac.uk/1+lcg0766.gridpp.rl.ac.uk/0 ' 'Resource_List.cput=800:00:00 Resource_List.neednodes=1:ppn=8 Resource_List.nodect=1 Resource_List.nodes=1:ppn=8 ' 'Resource_List.opsys=sl5 Resource_List.pcput=96:00:00 Resource_List.walltime=96:00:00 session=7428 end=1317427530 ' 'Exit_status=0 resources_used.cput=00:00:00 resources_used.mem=0kb resources_used.vmem=0kb resources_used.walltime=00:00:00') line2_values = {"JobName": "21010040.lcgbatch01.gridpp.rl.ac.uk", "LocalUserID":"plhcb010", "LocalUserGroup": "prodlhcb", "WallDuration":0, "CpuDuration": 0, "StartTime": datetime.datetime.utcfromtimestamp(1317427530), "StopTime": datetime.datetime.utcfromtimestamp(1317427530), "MemoryReal":0, "MemoryVirtual": 0, "NodeCount": 1, "Processors": 8 } cases = {} cases[line1] = line1_values cases[line2] = line2_values for line in cases.keys(): record = self.parser.parse(line) cont = record._record_content self.assertTrue(cont.has_key("Site")) self.assertTrue(cont.has_key("JobName")) self.assertTrue(cont.has_key("LocalUserID")) self.assertTrue(cont.has_key("LocalUserGroup")) self.assertTrue(cont.has_key("WallDuration")) self.assertTrue(cont.has_key("CpuDuration")) self.assertTrue(cont.has_key("StartTime")) self.assertTrue(cont.has_key("StopTime")) self.assertTrue(cont.has_key("MemoryReal")) self.assertTrue(cont.has_key("MemoryVirtual")) for key in cases[line].keys(): self.assertEqual(cont[key], cases[line][key], "%s != %s for key %s" % (cont[key], cases[line][key], key)) def test_invalid_data(self): # missing separator between fields line = ('10/02/2011 06:41:44;E;21048463.lcgbatch01.gridpp.rl.ac.uk;user=patls009 group=prodatls jobname=cre09_443343882 queue=grid4000M ' 'ctime=1317509574 qtime=1317509574 etime=1317509574start=1317509945 [email protected] ' 'exec_host=lcg1277.gridpp.rl.ac.uk/5 Resource_List.cput=96:00:00 Resource_List.neednodes=lcg1277.gridpp.rl.ac.uk ' 'Resource_List.opsys=sl5 Resource_List.pcput=96:00:00 Resource_List.pmem=4000mb Resource_List.walltime=96:00:00 session=20374 ' 'end=1317534104 Exit_status=0 resources_used.cput=18:15:24 resources_used.mem=2031040kb resources_used.vmem=3335528kb resources_used.walltime=19:23:4') self.assertRaises(KeyError, self.parser.parse, line) def test_parse_mpi(self): exec_host = 'lcg0766.gridpp.rl.ac.uk/7+lcg0766.gridpp.rl.ac.uk/6+lcg0766.gridpp.rl.ac.uk/5+lcg0766.gridpp.rl.ac.uk/4+lcg0766.gridpp.rl.ac.uk/3+lcg0766.gridpp.rl.ac.uk/2+lcg0766.gridpp.rl.ac.uk/1+lcg0766.gridpp.rl.ac.uk/0' nodecount, processors = _parse_mpi(exec_host) assert (nodecount, processors) == (1, 8)
def setUp(self): self.parser = PBSParser('testSite', 'testHost', True)
class ParserPBSTest(unittest.TestCase): ''' Test case for PBS parser ''' def setUp(self): self.parser = PBSParser('testSite', 'testHost', True) def test_parse(self): lines = ( ('10/02/2011 06:41:44;E;21048463.lcgbatch01.gridpp.rl.ac.uk;user=patls009 group=prodatls jobname=cre09_443343882 queue=grid4000M ' 'ctime=1317509574 qtime=1317509574 etime=1317509574 start=1317509945 [email protected] ' 'exec_host=lcg1277.gridpp.rl.ac.uk/5 Resource_List.cput=96:00:00 Resource_List.neednodes=lcg1277.gridpp.rl.ac.uk ' 'Resource_List.opsys=sl5 Resource_List.pcput=96:00:00 Resource_List.pmem=4000mb Resource_List.walltime=96:00:00 session=20374 ' 'end=1317534104 Exit_status=0 resources_used.cput=18:15:24 resources_used.mem=2031040kb resources_used.vmem=3335528kb resources_used.walltime=19:23:4'), ('10/01/2011 01:05:30;E;21010040.lcgbatch01.gridpp.rl.ac.uk;user=plhcb010 group=prodlhcb jobname=cre09_655648918 ' 'queue=gridWN ctime=1317427361 qtime=1317427361 etime=1317427361 start=1317427530 [email protected] ' 'exec_host=lcg0766.gridpp.rl.ac.uk/7+lcg0766.gridpp.rl.ac.uk/6+lcg0766.gridpp.rl.ac.uk/5+lcg0766.gridpp.rl.ac.uk/4+' 'lcg0766.gridpp.rl.ac.uk/3+lcg0766.gridpp.rl.ac.uk/2+lcg0766.gridpp.rl.ac.uk/1+lcg0766.gridpp.rl.ac.uk/0 ' 'Resource_List.cput=800:00:00 Resource_List.neednodes=1:ppn=8 Resource_List.nodect=1 Resource_List.nodes=1:ppn=8 ' 'Resource_List.opsys=sl5 Resource_List.pcput=96:00:00 Resource_List.walltime=96:00:00 session=7428 end=1317427530 ' 'Exit_status=0 resources_used.cput=00:00:00 resources_used.mem=0kb resources_used.vmem=0kb resources_used.walltime=00:00:00'), ('11/30/2015 15:39:52;E;24940336.b0;user=atlaspt5 group=atlaspt jobname=cream_830820758 queue=atlas ctime=1448913907 qtime=1448913907 et' 'ime=1448913907 start=1448914025 [email protected] exec_host=b341/2 Resource_List.file=15gb Resource_List.neednode' 's=1 Resource_List.nodect=1 Resource_List.nodes=1 Resource_List.pmem=2000mb Resource_List.pvmem=4000mb Resource_List.vmem=4000mb Resour' 'ce_List.walltime=48:00:00 session=22256 total_execution_slots=1 unique_node_count=1 end=1448926792 Exit_status=271 resources_used.cput' '=03:32:02 resources_used.energy_used=0 resources_used.mem=2574308kb resources_used.vmem=4032516kb resources_used.walltime=03:32:47'), # This example is from Torque 5.1.2 where time is now in seconds ('11/30/2015 15:42:06;E;24940336.b0;user=atlaspt5 group=atlaspt jobname=cream_830820758 queue=atlas ctime=1448913907 qtime=1448913907 et' 'ime=1448913907 start=1448914025 [email protected] exec_host=b341/2 Resource_List.file=15gb Resource_List.neednode' 's=1 Resource_List.nodect=1 Resource_List.nodes=1 Resource_List.pmem=2000mb Resource_List.pvmem=4000mb Resource_List.vmem=4000mb Resour' 'ce_List.walltime=48:00:00 session=22256 total_execution_slots=1 unique_node_count=1 end=1448926926 Exit_status=0 resources_used.cput=1' '2722 resources_used.energy_used=0 resources_used.mem=2574308kb resources_used.vmem=4032516kb resources_used.walltime=12767'), # Torque-4.2.8 reports cores as exec_host=b372/4+b372/5+b372/6+b372/7 ('03/21/2015 12:10:19;E;21414868.b0;user=atprd008 group=atprd jobname=cream_944593744 queue=atlas ctime=1426952581 qtime=1426952581 etim' 'e=1426952581 start=1426952875 [email protected] exec_host=b372/4+b372/5+b372/6+b372/7 Resource_List.file=15gb Res' 'ource_List.nodect=1 Resource_List.nodes=1pn=4 Resource_List.pmem=2666mb Resource_List.vmem=16000mb Resource_List.walltime=16:00:00 ses' 'sion=5353 end=1426965019 Exit_status=0 resources_used.cput=13:01:39 resources_used.mem=7813056kb resources_used.vmem=12127680kb resour' 'ces_used.walltime=03:22:24'), # Torque 5.1.0 reports cores as exec_host=b391/0-3 and total_execution_slots=4 ('20150327:03/27/2015 00:06:27;E;21430205.b0;user=atprd006 group=atprd jobname=cream_238467179 queue=atlas ctime=1427422724 qtime=142742' '2724 etime=1427422724 start=1427422975 [email protected] exec_host=b391/0-3 Resource_List.file=15gb Resource_List' '.neednodes=1pn=4 Resource_List.nodect=1 Resource_List.nodes=1pn=4 Resource_List.pmem=2666mb Resource_List.vmem=16000mb Resource_List.w' 'alltime=16:00:00 session=26754 total_execution_slots=4 unique_node_count=1 end=1427439987 Exit_status=0 resources_used.cput=18:31:56 r' 'esources_used.energy_used=0 resources_used.mem=8212056kb resources_used.vmem=12526772kb resources_used.walltime=04:43:32'), # Torque 5.1.3 has cput in secondss but walltime in hh:mm:ss ('12/07/2016 01:08:31;E;37803854.b0;user=atcant2 group=atcant jobna' 'me=cream_105055799 queue=atlas ctime=1481101558 qtime=1481101558 ' 'etime=1481101558 start=1481101655 [email protected] ex' 'ec_host=b126/2 Resource_List.file=15gb Resource_List.neednodes=1 ' 'Resource_List.nodect=1 Resource_List.nodes=1 Resource_List.pmem=2' '000mb Resource_List.walltime=50:00:00 session=1664 total_executio' 'n_slots=1 unique_node_count=1 end=1481101711 Exit_status=0 resour' 'ces_used.cput=9 resources_used.energy_used=0 resources_used.mem=4' '1512kb resources_used.vmem=325428kb resources_used.walltime=00:00' ':56'), ) values = ( ("21048463.lcgbatch01.gridpp.rl.ac.uk", "patls009", "prodatls", 69784, 65724, datetime.datetime.utcfromtimestamp(1317509945), datetime.datetime.utcfromtimestamp(1317534104), 2031040, 3335528, 1, 1), ("21010040.lcgbatch01.gridpp.rl.ac.uk", "plhcb010", "prodlhcb", 0, 0, datetime.datetime.utcfromtimestamp(1317427530), datetime.datetime.utcfromtimestamp(1317427530), 0, 0, 1, 8), ("24940336.b0", "atlaspt5", "atlaspt", 3*3600+32*60+47, 3*3600+32*60+2, datetime.datetime.utcfromtimestamp(1448914025), datetime.datetime.utcfromtimestamp(1448926792), 2574308, 4032516, 1, 1), ("24940336.b0", "atlaspt5", "atlaspt", 12767, 12722, datetime.datetime.utcfromtimestamp(1448914025), datetime.datetime.utcfromtimestamp(1448926926), 2574308, 4032516, 1, 1), ("21414868.b0", "atprd008", "atprd", 3*3600+22*60+24, 13*3600+1*60+39, datetime.datetime.utcfromtimestamp(1426952875), datetime.datetime.utcfromtimestamp(1426965019), 7813056, 12127680, 1, 4), ("21430205.b0", "atprd006", "atprd", 4*3600+43*60+32, 18*3600+31*60+56, datetime.datetime.utcfromtimestamp(1427422975), datetime.datetime.utcfromtimestamp(1427439987), 8212056, 12526772, 1, 4), ("37803854.b0", "atcant2", "atcant", 56, 9, datetime.datetime.utcfromtimestamp(1481101655), datetime.datetime.utcfromtimestamp(1481101711), 41512, 325428, 1, 1), ) fields = ('JobName', 'LocalUserID', 'LocalUserGroup', 'WallDuration', 'CpuDuration', 'StartTime', 'StopTime', 'MemoryReal', 'MemoryVirtual', 'NodeCount', 'Processors') cases = {} for line, value in zip(lines, values): cases[line] = dict(zip(fields, value)) for line in cases.keys(): record = self.parser.parse(line) cont = record._record_content self.assertEqual(cont['Site'], 'testSite') self.assertEqual(cont['Infrastructure'], 'APEL-CREAM-PBS') self.assertEqual(cont['MachineName'], 'testHost') for key in cases[line].keys(): # Check all fields are present self.assertTrue(key in cont, "Key '%s' not in record." % key) # Check values are correct self.assertEqual(cont[key], cases[line][key], "%s != %s for key %s" % (cont[key], cases[line][key], key)) def test_invalid_data(self): # missing separator between fields line = ('10/02/2011 06:41:44;E;21048463.lcgbatch01.gridpp.rl.ac.uk;user=patls009 group=prodatls jobname=cre09_443343882 queue=grid4000M ' 'ctime=1317509574 qtime=1317509574 etime=1317509574start=1317509945 [email protected] ' 'exec_host=lcg1277.gridpp.rl.ac.uk/5 Resource_List.cput=96:00:00 Resource_List.neednodes=lcg1277.gridpp.rl.ac.uk ' 'Resource_List.opsys=sl5 Resource_List.pcput=96:00:00 Resource_List.pmem=4000mb Resource_List.walltime=96:00:00 session=20374 ' 'end=1317534104 Exit_status=0 resources_used.cput=18:15:24 resources_used.mem=2031040kb resources_used.vmem=3335528kb resources_used.walltime=19:23:4') self.assertRaises(KeyError, self.parser.parse, line) def test_parse_mpi(self): exec_hosts = ('lcg0766.gridpp.rl.ac.uk/7+lcg0766.gridpp.rl.ac.uk/6+lcg0766.gridpp.rl.ac.uk/5+lcg0766.gridpp.rl.ac.uk/4+lcg0766.gridpp.rl.ac.uk/3+lcg0766.gridpp.rl.ac.uk/2+lcg0766.gridpp.rl.ac.uk/1+lcg0766.gridpp.rl.ac.uk/0', 'b372/4+b372/5+b372/6+b372/7', 'b391/0-3', 'b391/0-1,5,11', ) # List of (node, cpu) counts for above exec_hosts counts = ((1, 8), (1, 4), (1, 4), (1, 4), ) for exec_host, count in zip(exec_hosts, counts): nodecount, processors = _parse_mpi(exec_host) self.assertEqual(nodecount, count[0]) self.assertEqual(processors, count[1]) def test_value_errors(self): """Test that the parser raises a ValueError for certain problems.""" lines = ( # End time less than start time ('11/30/2015 15:39:52;E;24940336.b0;user=atlaspt5 group=atlaspt jobname=cream_830820758 queue=atlas ctime=1448913907 qtime=1448913907 et' 'ime=1448913907 start=1448926792 [email protected] exec_host=b341/2 Resource_List.file=15gb Resource_List.neednode' 's=1 Resource_List.nodect=1 Resource_List.nodes=1 Resource_List.pmem=2000mb Resource_List.pvmem=4000mb Resource_List.vmem=4000mb Resour' 'ce_List.walltime=48:00:00 session=22256 total_execution_slots=1 unique_node_count=1 end=1448914025 Exit_status=271 resources_used.cput' '=03:32:02 resources_used.energy_used=0 resources_used.mem=2574308kb resources_used.vmem=4032516kb resources_used.walltime=03:32:47'), ) for line in lines: self.assertRaises(ValueError, self.parser.parse, line) def test_incomplete_job(self): """Test that the parser returns None for an incomplete job.""" line = ('11/30/2015 15:42:06;A;24940336.b0;user=atlaspt5 group=atlaspt jobname=cream_830820758 queue=atlas ctime=1448913907 qtime=1448913907 et' 'ime=1448913907 start=1448914025 [email protected] exec_host=b341/2 Resource_List.file=15gb Resource_List.neednode' 's=1 Resource_List.nodect=1 Resource_List.nodes=1 Resource_List.pmem=2000mb Resource_List.pvmem=4000mb Resource_List.vmem=4000mb Resour' 'ce_List.walltime=48:00:00 session=22256 total_execution_slots=1 unique_node_count=1 end=1448926926 Exit_status=0 resources_used.cput=1' '2722 resources_used.energy_used=0 resources_used.mem=2574308kb resources_used.vmem=4032516kb resources_used.walltime=12767') self.assertEqual(self.parser.parse(line), None) def test_non_mpi(self): """Test that node and cpu count are set to zero if MPI isn't used.""" parser = PBSParser('testSite', 'testHost', False) line = ('10/01/2011 01:05:30;E;21010040.lcgbatch01.gridpp.rl.ac.uk;user=plhcb010 group=prodlhcb jobname=cre09_655648918 ' 'queue=gridWN ctime=1317427361 qtime=1317427361 etime=1317427361 start=1317427530 [email protected] ' 'Resource_List.cput=800:00:00 Resource_List.neednodes=1:ppn=8 Resource_List.nodect=1 Resource_List.nodes=1:ppn=8 ' 'Resource_List.opsys=sl5 Resource_List.pcput=96:00:00 Resource_List.walltime=96:00:00 session=7428 end=1317427530 ' 'Exit_status=0 resources_used.cput=00:00:00 resources_used.mem=0kb resources_used.vmem=0kb resources_used.walltime=00:00:00') record = parser.parse(line) cont = record._record_content self.assertEquals(cont['NodeCount'], 0) self.assertEquals(cont['Processors'], 0)
class ParserPBSTest(unittest.TestCase): ''' Test case for PBS parser ''' def setUp(self): self.parser = PBSParser('testSite', 'testHost', True) def test_parse(self): lines = ( ('10/02/2011 06:41:44;E;21048463.lcgbatch01.gridpp.rl.ac.uk;user=patls009 group=prodatls jobname=cre09_443343882 queue=grid4000M ' 'ctime=1317509574 qtime=1317509574 etime=1317509574 start=1317509945 [email protected] ' 'exec_host=lcg1277.gridpp.rl.ac.uk/5 Resource_List.cput=96:00:00 Resource_List.neednodes=lcg1277.gridpp.rl.ac.uk ' 'Resource_List.opsys=sl5 Resource_List.pcput=96:00:00 Resource_List.pmem=4000mb Resource_List.walltime=96:00:00 session=20374 ' 'end=1317534104 Exit_status=0 resources_used.cput=18:15:24 resources_used.mem=2031040kb resources_used.vmem=3335528kb resources_used.walltime=19:23:4'), ('10/01/2011 01:05:30;E;21010040.lcgbatch01.gridpp.rl.ac.uk;user=plhcb010 group=prodlhcb jobname=cre09_655648918 ' 'queue=gridWN ctime=1317427361 qtime=1317427361 etime=1317427361 start=1317427530 [email protected] ' 'exec_host=lcg0766.gridpp.rl.ac.uk/7+lcg0766.gridpp.rl.ac.uk/6+lcg0766.gridpp.rl.ac.uk/5+lcg0766.gridpp.rl.ac.uk/4+' 'lcg0766.gridpp.rl.ac.uk/3+lcg0766.gridpp.rl.ac.uk/2+lcg0766.gridpp.rl.ac.uk/1+lcg0766.gridpp.rl.ac.uk/0 ' 'Resource_List.cput=800:00:00 Resource_List.neednodes=1:ppn=8 Resource_List.nodect=1 Resource_List.nodes=1:ppn=8 ' 'Resource_List.opsys=sl5 Resource_List.pcput=96:00:00 Resource_List.walltime=96:00:00 session=7428 end=1317427530 ' 'Exit_status=0 resources_used.cput=00:00:00 resources_used.mem=0kb resources_used.vmem=0kb resources_used.walltime=00:00:00'), ('11/30/2015 15:39:52;E;24940336.b0;user=atlaspt5 group=atlaspt jobname=cream_830820758 queue=atlas ctime=1448913907 qtime=1448913907 et' 'ime=1448913907 start=1448914025 [email protected] exec_host=b341/2 Resource_List.file=15gb Resource_List.neednode' 's=1 Resource_List.nodect=1 Resource_List.nodes=1 Resource_List.pmem=2000mb Resource_List.pvmem=4000mb Resource_List.vmem=4000mb Resour' 'ce_List.walltime=48:00:00 session=22256 total_execution_slots=1 unique_node_count=1 end=1448926792 Exit_status=271 resources_used.cput' '=03:32:02 resources_used.energy_used=0 resources_used.mem=2574308kb resources_used.vmem=4032516kb resources_used.walltime=03:32:47'), # This example is from Torque 5.1.2 where time is now in seconds ('11/30/2015 15:42:06;E;24940336.b0;user=atlaspt5 group=atlaspt jobname=cream_830820758 queue=atlas ctime=1448913907 qtime=1448913907 et' 'ime=1448913907 start=1448914025 [email protected] exec_host=b341/2 Resource_List.file=15gb Resource_List.neednode' 's=1 Resource_List.nodect=1 Resource_List.nodes=1 Resource_List.pmem=2000mb Resource_List.pvmem=4000mb Resource_List.vmem=4000mb Resour' 'ce_List.walltime=48:00:00 session=22256 total_execution_slots=1 unique_node_count=1 end=1448926926 Exit_status=0 resources_used.cput=1' '2722 resources_used.energy_used=0 resources_used.mem=2574308kb resources_used.vmem=4032516kb resources_used.walltime=12767'), ) values = ( ("21048463.lcgbatch01.gridpp.rl.ac.uk", "patls009", "prodatls", 69784, 65724, datetime.datetime.utcfromtimestamp(1317509945), datetime.datetime.utcfromtimestamp(1317534104), 2031040, 3335528, 1, 1), ("21010040.lcgbatch01.gridpp.rl.ac.uk", "plhcb010", "prodlhcb", 0, 0, datetime.datetime.utcfromtimestamp(1317427530), datetime.datetime.utcfromtimestamp(1317427530), 0, 0, 1, 8), ("24940336.b0", "atlaspt5", "atlaspt", 3*3600+32*60+47, 3*3600+32*60+2, datetime.datetime.utcfromtimestamp(1448914025), datetime.datetime.utcfromtimestamp(1448926792), 2574308, 4032516, 1, 1), ("24940336.b0", "atlaspt5", "atlaspt", 12767, 12722, datetime.datetime.utcfromtimestamp(1448914025), datetime.datetime.utcfromtimestamp(1448926926), 2574308, 4032516, 1, 1), ) fields = ('JobName', 'LocalUserID', 'LocalUserGroup', 'WallDuration', 'CpuDuration', 'StartTime', 'StopTime', 'MemoryReal', 'MemoryVirtual', 'NodeCount', 'Processors') cases = {} for line, value in zip(lines, values): cases[line] = dict(zip(fields, value)) for line in cases.keys(): record = self.parser.parse(line) cont = record._record_content self.assertEqual(cont['Site'], 'testSite') self.assertEqual(cont['Infrastructure'], 'APEL-CREAM-PBS') self.assertEqual(cont['MachineName'], 'testHost') for key in cases[line].keys(): # Check all fields are present self.assertTrue(key in cont, "Key '%s' not in record." % key) # Check values are correct self.assertEqual(cont[key], cases[line][key], "%s != %s for key %s" % (cont[key], cases[line][key], key)) def test_invalid_data(self): # missing separator between fields line = ('10/02/2011 06:41:44;E;21048463.lcgbatch01.gridpp.rl.ac.uk;user=patls009 group=prodatls jobname=cre09_443343882 queue=grid4000M ' 'ctime=1317509574 qtime=1317509574 etime=1317509574start=1317509945 [email protected] ' 'exec_host=lcg1277.gridpp.rl.ac.uk/5 Resource_List.cput=96:00:00 Resource_List.neednodes=lcg1277.gridpp.rl.ac.uk ' 'Resource_List.opsys=sl5 Resource_List.pcput=96:00:00 Resource_List.pmem=4000mb Resource_List.walltime=96:00:00 session=20374 ' 'end=1317534104 Exit_status=0 resources_used.cput=18:15:24 resources_used.mem=2031040kb resources_used.vmem=3335528kb resources_used.walltime=19:23:4') self.assertRaises(KeyError, self.parser.parse, line) def test_parse_mpi(self): exec_host = 'lcg0766.gridpp.rl.ac.uk/7+lcg0766.gridpp.rl.ac.uk/6+lcg0766.gridpp.rl.ac.uk/5+lcg0766.gridpp.rl.ac.uk/4+lcg0766.gridpp.rl.ac.uk/3+lcg0766.gridpp.rl.ac.uk/2+lcg0766.gridpp.rl.ac.uk/1+lcg0766.gridpp.rl.ac.uk/0' nodecount, processors = _parse_mpi(exec_host) self.assertEqual(nodecount, 1) self.assertEqual(processors, 8) def test_value_errors(self): """Test that the parser raises a ValueError for certain problems.""" lines = ( # End time less than start time ('11/30/2015 15:39:52;E;24940336.b0;user=atlaspt5 group=atlaspt jobname=cream_830820758 queue=atlas ctime=1448913907 qtime=1448913907 et' 'ime=1448913907 start=1448926792 [email protected] exec_host=b341/2 Resource_List.file=15gb Resource_List.neednode' 's=1 Resource_List.nodect=1 Resource_List.nodes=1 Resource_List.pmem=2000mb Resource_List.pvmem=4000mb Resource_List.vmem=4000mb Resour' 'ce_List.walltime=48:00:00 session=22256 total_execution_slots=1 unique_node_count=1 end=1448914025 Exit_status=271 resources_used.cput' '=03:32:02 resources_used.energy_used=0 resources_used.mem=2574308kb resources_used.vmem=4032516kb resources_used.walltime=03:32:47'), # Mixed duration formats (hh:mm:ss and s) ('11/30/2015 15:42:06;E;24940336.b0;user=atlaspt5 group=atlaspt jobname=cream_830820758 queue=atlas ctime=1448913907 qtime=1448913907 et' 'ime=1448913907 start=1448914025 [email protected] exec_host=b341/2 Resource_List.file=15gb Resource_List.neednode' 's=1 Resource_List.nodect=1 Resource_List.nodes=1 Resource_List.pmem=2000mb Resource_List.pvmem=4000mb Resource_List.vmem=4000mb Resour' 'ce_List.walltime=48:00:00 session=22256 total_execution_slots=1 unique_node_count=1 end=1448926926 Exit_status=0 resources_used.cput=1' '2722 resources_used.energy_used=0 resources_used.mem=2574308kb resources_used.vmem=4032516kb resources_used.walltime=03:32:47'), ) for line in lines: self.assertRaises(ValueError, self.parser.parse, line) def test_incomplete_job(self): """Test that the parser returns None for an incomplete job.""" line = ('11/30/2015 15:42:06;A;24940336.b0;user=atlaspt5 group=atlaspt jobname=cream_830820758 queue=atlas ctime=1448913907 qtime=1448913907 et' 'ime=1448913907 start=1448914025 [email protected] exec_host=b341/2 Resource_List.file=15gb Resource_List.neednode' 's=1 Resource_List.nodect=1 Resource_List.nodes=1 Resource_List.pmem=2000mb Resource_List.pvmem=4000mb Resource_List.vmem=4000mb Resour' 'ce_List.walltime=48:00:00 session=22256 total_execution_slots=1 unique_node_count=1 end=1448926926 Exit_status=0 resources_used.cput=1' '2722 resources_used.energy_used=0 resources_used.mem=2574308kb resources_used.vmem=4032516kb resources_used.walltime=12767') self.assertEqual(self.parser.parse(line), None) def test_non_mpi(self): """Test that node and cpu count are set to zero if MPI isn't used.""" parser = PBSParser('testSite', 'testHost', False) line = ('10/01/2011 01:05:30;E;21010040.lcgbatch01.gridpp.rl.ac.uk;user=plhcb010 group=prodlhcb jobname=cre09_655648918 ' 'queue=gridWN ctime=1317427361 qtime=1317427361 etime=1317427361 start=1317427530 [email protected] ' 'Resource_List.cput=800:00:00 Resource_List.neednodes=1:ppn=8 Resource_List.nodect=1 Resource_List.nodes=1:ppn=8 ' 'Resource_List.opsys=sl5 Resource_List.pcput=96:00:00 Resource_List.walltime=96:00:00 session=7428 end=1317427530 ' 'Exit_status=0 resources_used.cput=00:00:00 resources_used.mem=0kb resources_used.vmem=0kb resources_used.walltime=00:00:00') record = parser.parse(line) cont = record._record_content self.assertEquals(cont['NodeCount'], 0) self.assertEquals(cont['Processors'], 0)
class ParserPBSTest(unittest.TestCase): ''' Test case for PBS parser ''' def setUp(self): self.parser = PBSParser('testSite', 'testHost', True) def test_parse(self): lines = ( ('10/02/2011 06:41:44;E;21048463.lcgbatch01.gridpp.rl.ac.uk;user=patls009 group=prodatls jobname=cre09_443343882 queue=grid4000M ' 'ctime=1317509574 qtime=1317509574 etime=1317509574 start=1317509945 [email protected] ' 'exec_host=lcg1277.gridpp.rl.ac.uk/5 Resource_List.cput=96:00:00 Resource_List.neednodes=lcg1277.gridpp.rl.ac.uk ' 'Resource_List.opsys=sl5 Resource_List.pcput=96:00:00 Resource_List.pmem=4000mb Resource_List.walltime=96:00:00 session=20374 ' 'end=1317534104 Exit_status=0 resources_used.cput=18:15:24 resources_used.mem=2031040kb resources_used.vmem=3335528kb resources_used.walltime=19:23:4' ), ('10/01/2011 01:05:30;E;21010040.lcgbatch01.gridpp.rl.ac.uk;user=plhcb010 group=prodlhcb jobname=cre09_655648918 ' 'queue=gridWN ctime=1317427361 qtime=1317427361 etime=1317427361 start=1317427530 [email protected] ' 'exec_host=lcg0766.gridpp.rl.ac.uk/7+lcg0766.gridpp.rl.ac.uk/6+lcg0766.gridpp.rl.ac.uk/5+lcg0766.gridpp.rl.ac.uk/4+' 'lcg0766.gridpp.rl.ac.uk/3+lcg0766.gridpp.rl.ac.uk/2+lcg0766.gridpp.rl.ac.uk/1+lcg0766.gridpp.rl.ac.uk/0 ' 'Resource_List.cput=800:00:00 Resource_List.neednodes=1:ppn=8 Resource_List.nodect=1 Resource_List.nodes=1:ppn=8 ' 'Resource_List.opsys=sl5 Resource_List.pcput=96:00:00 Resource_List.walltime=96:00:00 session=7428 end=1317427530 ' 'Exit_status=0 resources_used.cput=00:00:00 resources_used.mem=0kb resources_used.vmem=0kb resources_used.walltime=00:00:00' ), ('11/30/2015 15:39:52;E;24940336.b0;user=atlaspt5 group=atlaspt jobname=cream_830820758 queue=atlas ctime=1448913907 qtime=1448913907 et' 'ime=1448913907 start=1448914025 [email protected] exec_host=b341/2 Resource_List.file=15gb Resource_List.neednode' 's=1 Resource_List.nodect=1 Resource_List.nodes=1 Resource_List.pmem=2000mb Resource_List.pvmem=4000mb Resource_List.vmem=4000mb Resour' 'ce_List.walltime=48:00:00 session=22256 total_execution_slots=1 unique_node_count=1 end=1448926792 Exit_status=271 resources_used.cput' '=03:32:02 resources_used.energy_used=0 resources_used.mem=2574308kb resources_used.vmem=4032516kb resources_used.walltime=03:32:47' ), # This example is from Torque 5.1.2 where time is now in seconds ('11/30/2015 15:42:06;E;24940336.b0;user=atlaspt5 group=atlaspt jobname=cream_830820758 queue=atlas ctime=1448913907 qtime=1448913907 et' 'ime=1448913907 start=1448914025 [email protected] exec_host=b341/2 Resource_List.file=15gb Resource_List.neednode' 's=1 Resource_List.nodect=1 Resource_List.nodes=1 Resource_List.pmem=2000mb Resource_List.pvmem=4000mb Resource_List.vmem=4000mb Resour' 'ce_List.walltime=48:00:00 session=22256 total_execution_slots=1 unique_node_count=1 end=1448926926 Exit_status=0 resources_used.cput=1' '2722 resources_used.energy_used=0 resources_used.mem=2574308kb resources_used.vmem=4032516kb resources_used.walltime=12767' ), # Torque-4.2.8 reports cores as exec_host=b372/4+b372/5+b372/6+b372/7 ('03/21/2015 12:10:19;E;21414868.b0;user=atprd008 group=atprd jobname=cream_944593744 queue=atlas ctime=1426952581 qtime=1426952581 etim' 'e=1426952581 start=1426952875 [email protected] exec_host=b372/4+b372/5+b372/6+b372/7 Resource_List.file=15gb Res' 'ource_List.nodect=1 Resource_List.nodes=1pn=4 Resource_List.pmem=2666mb Resource_List.vmem=16000mb Resource_List.walltime=16:00:00 ses' 'sion=5353 end=1426965019 Exit_status=0 resources_used.cput=13:01:39 resources_used.mem=7813056kb resources_used.vmem=12127680kb resour' 'ces_used.walltime=03:22:24'), # Torque 5.1.0 reports cores as exec_host=b391/0-3 and total_execution_slots=4 ('20150327:03/27/2015 00:06:27;E;21430205.b0;user=atprd006 group=atprd jobname=cream_238467179 queue=atlas ctime=1427422724 qtime=142742' '2724 etime=1427422724 start=1427422975 [email protected] exec_host=b391/0-3 Resource_List.file=15gb Resource_List' '.neednodes=1pn=4 Resource_List.nodect=1 Resource_List.nodes=1pn=4 Resource_List.pmem=2666mb Resource_List.vmem=16000mb Resource_List.w' 'alltime=16:00:00 session=26754 total_execution_slots=4 unique_node_count=1 end=1427439987 Exit_status=0 resources_used.cput=18:31:56 r' 'esources_used.energy_used=0 resources_used.mem=8212056kb resources_used.vmem=12526772kb resources_used.walltime=04:43:32' ), # Torque 5.1.3 has cput in secondss but walltime in hh:mm:ss ('12/07/2016 01:08:31;E;37803854.b0;user=atcant2 group=atcant jobna' 'me=cream_105055799 queue=atlas ctime=1481101558 qtime=1481101558 ' 'etime=1481101558 start=1481101655 [email protected] ex' 'ec_host=b126/2 Resource_List.file=15gb Resource_List.neednodes=1 ' 'Resource_List.nodect=1 Resource_List.nodes=1 Resource_List.pmem=2' '000mb Resource_List.walltime=50:00:00 session=1664 total_executio' 'n_slots=1 unique_node_count=1 end=1481101711 Exit_status=0 resour' 'ces_used.cput=9 resources_used.energy_used=0 resources_used.mem=4' '1512kb resources_used.vmem=325428kb resources_used.walltime=00:00' ':56'), ) values = ( ("21048463.lcgbatch01.gridpp.rl.ac.uk", "patls009", "prodatls", 69784, 65724, datetime.datetime.utcfromtimestamp(1317509945), datetime.datetime.utcfromtimestamp(1317534104), 2031040, 3335528, 1, 1), ("21010040.lcgbatch01.gridpp.rl.ac.uk", "plhcb010", "prodlhcb", 0, 0, datetime.datetime.utcfromtimestamp(1317427530), datetime.datetime.utcfromtimestamp(1317427530), 0, 0, 1, 8), ("24940336.b0", "atlaspt5", "atlaspt", 3 * 3600 + 32 * 60 + 47, 3 * 3600 + 32 * 60 + 2, datetime.datetime.utcfromtimestamp(1448914025), datetime.datetime.utcfromtimestamp(1448926792), 2574308, 4032516, 1, 1), ("24940336.b0", "atlaspt5", "atlaspt", 12767, 12722, datetime.datetime.utcfromtimestamp(1448914025), datetime.datetime.utcfromtimestamp(1448926926), 2574308, 4032516, 1, 1), ("21414868.b0", "atprd008", "atprd", 3 * 3600 + 22 * 60 + 24, 13 * 3600 + 1 * 60 + 39, datetime.datetime.utcfromtimestamp(1426952875), datetime.datetime.utcfromtimestamp(1426965019), 7813056, 12127680, 1, 4), ("21430205.b0", "atprd006", "atprd", 4 * 3600 + 43 * 60 + 32, 18 * 3600 + 31 * 60 + 56, datetime.datetime.utcfromtimestamp(1427422975), datetime.datetime.utcfromtimestamp(1427439987), 8212056, 12526772, 1, 4), ("37803854.b0", "atcant2", "atcant", 56, 9, datetime.datetime.utcfromtimestamp(1481101655), datetime.datetime.utcfromtimestamp(1481101711), 41512, 325428, 1, 1), ) fields = ('JobName', 'LocalUserID', 'LocalUserGroup', 'WallDuration', 'CpuDuration', 'StartTime', 'StopTime', 'MemoryReal', 'MemoryVirtual', 'NodeCount', 'Processors') cases = {} for line, value in zip(lines, values): cases[line] = dict(zip(fields, value)) for line in cases.keys(): record = self.parser.parse(line) cont = record._record_content self.assertEqual(cont['Site'], 'testSite') self.assertEqual(cont['Infrastructure'], 'APEL-CREAM-PBS') self.assertEqual(cont['MachineName'], 'testHost') for key in cases[line].keys(): # Check all fields are present self.assertTrue(key in cont, "Key '%s' not in record." % key) # Check values are correct self.assertEqual( cont[key], cases[line][key], "%s != %s for key %s" % (cont[key], cases[line][key], key)) def test_invalid_data(self): # missing separator between fields line = ( '10/02/2011 06:41:44;E;21048463.lcgbatch01.gridpp.rl.ac.uk;user=patls009 group=prodatls jobname=cre09_443343882 queue=grid4000M ' 'ctime=1317509574 qtime=1317509574 etime=1317509574start=1317509945 [email protected] ' 'exec_host=lcg1277.gridpp.rl.ac.uk/5 Resource_List.cput=96:00:00 Resource_List.neednodes=lcg1277.gridpp.rl.ac.uk ' 'Resource_List.opsys=sl5 Resource_List.pcput=96:00:00 Resource_List.pmem=4000mb Resource_List.walltime=96:00:00 session=20374 ' 'end=1317534104 Exit_status=0 resources_used.cput=18:15:24 resources_used.mem=2031040kb resources_used.vmem=3335528kb resources_used.walltime=19:23:4' ) self.assertRaises(KeyError, self.parser.parse, line) def test_parse_mpi(self): exec_hosts = ( 'lcg0766.gridpp.rl.ac.uk/7+lcg0766.gridpp.rl.ac.uk/6+lcg0766.gridpp.rl.ac.uk/5+lcg0766.gridpp.rl.ac.uk/4+lcg0766.gridpp.rl.ac.uk/3+lcg0766.gridpp.rl.ac.uk/2+lcg0766.gridpp.rl.ac.uk/1+lcg0766.gridpp.rl.ac.uk/0', 'b372/4+b372/5+b372/6+b372/7', 'b391/0-3', 'b391/0-1,5,11', ) # List of (node, cpu) counts for above exec_hosts counts = ( (1, 8), (1, 4), (1, 4), (1, 4), ) for exec_host, count in zip(exec_hosts, counts): nodecount, processors = _parse_mpi(exec_host) self.assertEqual(nodecount, count[0]) self.assertEqual(processors, count[1]) def test_value_errors(self): """Test that the parser raises a ValueError for certain problems.""" lines = ( # End time less than start time ('11/30/2015 15:39:52;E;24940336.b0;user=atlaspt5 group=atlaspt jobname=cream_830820758 queue=atlas ctime=1448913907 qtime=1448913907 et' 'ime=1448913907 start=1448926792 [email protected] exec_host=b341/2 Resource_List.file=15gb Resource_List.neednode' 's=1 Resource_List.nodect=1 Resource_List.nodes=1 Resource_List.pmem=2000mb Resource_List.pvmem=4000mb Resource_List.vmem=4000mb Resour' 'ce_List.walltime=48:00:00 session=22256 total_execution_slots=1 unique_node_count=1 end=1448914025 Exit_status=271 resources_used.cput' '=03:32:02 resources_used.energy_used=0 resources_used.mem=2574308kb resources_used.vmem=4032516kb resources_used.walltime=03:32:47' ), ) for line in lines: self.assertRaises(ValueError, self.parser.parse, line) def test_incomplete_job(self): """Test that the parser returns None for an incomplete job.""" line = ( '11/30/2015 15:42:06;A;24940336.b0;user=atlaspt5 group=atlaspt jobname=cream_830820758 queue=atlas ctime=1448913907 qtime=1448913907 et' 'ime=1448913907 start=1448914025 [email protected] exec_host=b341/2 Resource_List.file=15gb Resource_List.neednode' 's=1 Resource_List.nodect=1 Resource_List.nodes=1 Resource_List.pmem=2000mb Resource_List.pvmem=4000mb Resource_List.vmem=4000mb Resour' 'ce_List.walltime=48:00:00 session=22256 total_execution_slots=1 unique_node_count=1 end=1448926926 Exit_status=0 resources_used.cput=1' '2722 resources_used.energy_used=0 resources_used.mem=2574308kb resources_used.vmem=4032516kb resources_used.walltime=12767' ) self.assertEqual(self.parser.parse(line), None) def test_non_mpi(self): """Test that node and cpu count are set to zero if MPI isn't used.""" parser = PBSParser('testSite', 'testHost', False) line = ( '10/01/2011 01:05:30;E;21010040.lcgbatch01.gridpp.rl.ac.uk;user=plhcb010 group=prodlhcb jobname=cre09_655648918 ' 'queue=gridWN ctime=1317427361 qtime=1317427361 etime=1317427361 start=1317427530 [email protected] ' 'Resource_List.cput=800:00:00 Resource_List.neednodes=1:ppn=8 Resource_List.nodect=1 Resource_List.nodes=1:ppn=8 ' 'Resource_List.opsys=sl5 Resource_List.pcput=96:00:00 Resource_List.walltime=96:00:00 session=7428 end=1317427530 ' 'Exit_status=0 resources_used.cput=00:00:00 resources_used.mem=0kb resources_used.vmem=0kb resources_used.walltime=00:00:00' ) record = parser.parse(line) cont = record._record_content self.assertEquals(cont['NodeCount'], 0) self.assertEquals(cont['Processors'], 0)