class TestProcessFunctions(unittest.TestCase):
def setUp(self):
self.process = Process()
self.header = {
"tme": 0,
"pid": 1,
"ppid": 2,
"uid": 3,
"name": 4,
"cmd": 5,
"exit_code": 6,
"state": 7,
"gpid": 8
}
def test_setUp(self):
self.assertEqual(False, self.process.valid)
self.assertEqual(self.process.pid, 0)
self.assertEqual(self.process.ppid, 0)
self.assertEqual(self.process.gpid, 0)
self.assertEqual(self.process.uid, 0)
self.assertEqual(self.process.tme, 0)
self.assertEqual(self.process.traffic, [])
def test_exit_code(self):
process = Process(exit_code=1)
self.assertEqual(process.exit_code,
0) # TODO: why is here an output at all?
self.assertEqual(process.error_code, 0)
self.assertEqual(process.signal, 1)
process = Process(exit_code="256")
self.assertEqual(process.error_code, 1)
self.assertEqual(process.signal, 0)
process = Process(exit_code=257)
self.assertEqual(process.error_code, 1)
self.assertEqual(process.signal, 1)
def test_process_events(self):
row_start = [
"1406555483", "9939", "9881", "0", "(sge_shepherd)",
"sge_shepherd-5073566", "0", ".", "9939"
]
row_valid_start = [
"1406555483", "9939", "9881", "0", "(sge_shepherd)",
"sge_shepherd-5073566", "0", "fork", "9939"
]
row_end = [
"1406556210", "9940", "9881", "0", "(sge_shephrd)",
"sge_shepherd-5073567", "0", "exit", "9939"
]
row_end2 = [
"1406556210", "9940", "9881", "0", "(sge_shephrd)",
"sge_shepherd-5073566", "0", "exit", "9939"
]
row_valid_end = [
"1406556210", "9939", "9881", "0", "(sge_shepherd)",
"sge_shepherd-5073566", "0", "exit", "9939"
]
data_start_dict = {}
data_valid_start_dict = {}
data_end_dict = {}
data_end_dict2 = {}
data_valid_end_dict = {}
for key in self.header:
data_start_dict[key] = row_start[self.header[key]]
data_valid_start_dict[key] = row_valid_start[self.header[key]]
data_end_dict[key] = row_end[self.header[key]]
data_end_dict2[key] = row_end2[self.header[key]]
data_valid_end_dict[key] = row_valid_end[self.header[key]]
self.process.addProcessEvent(**data_start_dict)
self.process.addProcessEvent(**data_valid_end_dict)
self.assertEqual(False, self.process.valid)
process = Process()
process.addProcessEvent(**data_valid_end_dict)
self.assertEqual(process.name, data_valid_end_dict["name"])
process.addProcessEvent(**data_start_dict)
self.assertEqual(False, process.valid)
valid_process = Process()
valid_process.addProcessEvent(**data_valid_start_dict)
valid_process.addProcessEvent(**data_valid_end_dict)
self.assertEqual(True, valid_process.valid)
valid_process2 = Process()
valid_process2.addProcessEvent(**data_valid_end_dict)
valid_process2.addProcessEvent(**data_valid_start_dict)
self.assertEqual(True, valid_process2.valid)
process = Process()
process.addProcessEvent(**data_valid_start_dict)
self.assertRaises(ProcessMismatchException, process.addProcessEvent,
**data_end_dict)
process = Process()
process.addProcessEvent(**data_valid_start_dict)
self.assertRaises(ProcessMismatchException, process.addProcessEvent,
**data_end_dict2)
def test_event_conversion(self):
row_string = "1405011331,1405065581,30726,7733,30726,0,(sge_shepherd)," \
"sge_shepherd-4165419,0,0,1,,,,,0,,,exit"
row_header = "tme,exit_tme,pid,ppid,gpid,uid,name,cmd,error_code,signal,valid," \
"int_in_volume,int_out_volume,ext_in_volume,ext_out_volume,tree_depth," \
"process_type,color,state"
process = Process(
**dict(zip(row_header.split(","), row_string.split(","))))
self.assertEqual(
process.toProcessEvent(), {
"tme": 1405065581,
"name": "(sge_shepherd)",
"cmd": "sge_shepherd-4165419",
"pid": 30726,
"ppid": 7733,
"uid": 0,
"gpid": 30726,
"state": "exit",
"exit_code": 0
})
row_string = "1405011331,,30726,7733,30726,0,(sge_shepherd)," \
"sge_shepherd-4165419,0,0,1,,,,,0,,,."
process = Process(
**dict(zip(row_header.split(","), row_string.split(","))))
self.assertEqual(
process.toProcessEvent(), {
"tme": 1405011331,
"name": "(sge_shepherd)",
"cmd": "sge_shepherd-4165419",
"pid": 30726,
"ppid": 7733,
"uid": 0,
"gpid": 30726,
"state": ".",
})
def test_process_from_row(self):
row_string = "1405011331,1405065581,30726,7733,30726,0,(sge_shepherd)," \
"sge_shepherd-4165419,1,1,1,,,,,0,,,exit"
row_header = "tme,exit_tme,pid,ppid,gpid,uid,name,cmd,error_code,signal,valid," \
"int_in_volume,int_out_volume,ext_in_volume,ext_out_volume,tree_depth," \
"process_type,color,state"
process = Process.from_dict(
dict(zip(row_header.split(","), row_string.split(","))))
self.assertIsNotNone(process)
self.assertEqual(process.getDuration(), 54250)
self.assertEqual(process.getHeader(), row_header)
self.assertEqual(process.getRow(), row_string)
self.assertEqual(process.error_code, 1)
self.assertEqual(process.signal, 1)
self.assertEqual(process.exit_code, 257)
row_string = "1405011331,,30726,7733,30726,0,(sge_shepherd)," \
"sge_shepherd-4165419,0,0,1,,,,,0,,,exit"
process = Process.from_dict(
dict(zip(row_header.split(","), row_string.split(","))))
self.assertIsNotNone(process)
self.assertEqual(process.getDuration(), 0)
self.assertEqual(process.getHeader(), row_header)
self.assertEqual(
process.getRow(),
'1405011331,0,30726,7733,30726,0,(sge_shepherd),sge_shepherd-4165419,0,0,1,,,,,0,,,exit'
)
row_string = "1405011331,1405065581,30726,7733,30726,0,(sge_shepherd)," \
"sge_shepherd-4165419,,,1,,,,,0,,,exit"
process = Process.from_dict(
dict(zip(row_header.split(","), row_string.split(","))))
self.assertEqual(process.signal, 0)
self.assertEqual(process.error_code, 0)
row_string = "1405011331,1405065581,30726,7733,30726,0,(sge_shepherd)," \
"sge_shepherd-4165419,sth,,1,,,,,0,,,exit"
self.assertRaises(
ValueError, Process.from_dict,
dict(zip(row_header.split(","), row_string.split(","))))
row_string = "1405011331,1405065581,30726,7733,30726,0,(sge_shepherd)," \
"sge_shepherd-4165419,,,1,,,,,0,,,exit"
row_header = "tme,exit_tme,pid,ppid,gpid,uid,name,cmd,error_code,signal,valid," \
"int_in_volume,int_out_volume,ext_in_volume,ext_out_volume,tree_depth," \
"process_type,color,states"
self.assertRaises(
ArgumentNotDefinedException, Process.from_dict,
dict(zip(row_header.split(","), row_string.split(","))))
def test_traffic(self):
# TODO: test traffic stuff
pass
def test_batchsystem_id(self):
row_string = "1405011331,,30726,7733,30726,0,(sge_shepherd)," \
"sge_shepherd-4165419,0,0,1,,,,,0,,,exit"
row_header = "tme,exit_tme,pid,ppid,gpid,uid,name,cmd,error_code,signal,valid," \
"int_in_volume,int_out_volume,ext_in_volume,ext_out_volume,tree_depth," \
"process_type,color,state"
process = Process(
**dict(zip(row_header.split(","), row_string.split(","))))
self.assertEqual("sge_shepherd-4165419", process.cmd)
self.assertEqual(4165419, process.batchsystemId)
row_string = "1405011331,,30726,7733,30726,0,(sge_shepherd)," \
",0,0,1,,,,,0,,,exit"
process = Process(
**dict(zip(row_header.split(","), row_string.split(","))))
self.assertIsNone(process.batchsystemId)
def test_repr(self):
row_string = "1405011331,,30726,7733,30726,0,(sge_shepherd)," \
"sge_shepherd-4165419,0,0,1,,,,,0,,,exit"
row_header = "tme,exit_tme,pid,ppid,gpid,uid,name,cmd,error_code,signal,valid," \
"int_in_volume,int_out_volume,ext_in_volume,ext_out_volume,tree_depth," \
"process_type,color,state"
process = Process(
**dict(zip(row_header.split(","), row_string.split(","))))
self.assertEqual(
"Process: name ((sge_shepherd)), cmd (sge_shepherd-4165419), pid (30726), "
"ppid (7733), uid (0), gpid (30726), valid (1), tme (1405011331), "
"exit_tme (0), state (exit), error_code (0), signal (0), job_id (None), "
"tree_depth (0), process_type (), color (), int_in_volume (), "
"int_out_volume (), ext_in_volume (), ext_out_volume ()",
process.__repr__())
def parseRow(self, row=None, headerCache=None, tme=None):
if "state" in headerCache:
if "exit" in row[headerCache['state']]:
# load process and set exit arguments,
# afterwards remove it from cache
pid = int(row[headerCache['pid']])
process_index = self._process_cache.data_index(value=tme,
key=pid)
try:
process = self._process_cache.object_cache[pid][
process_index]
if (row[headerCache['name']] not in process.name
and row[headerCache['cmd']] not in process.cmd):
# wrong process selected!
logging.getLogger(self.__class__.__name__).warning(
"process %s has not been logged", row)
process = Process(name=row[headerCache['name']],
cmd=row[headerCache['cmd']],
pid=row[headerCache['pid']],
ppid=row[headerCache['ppid']],
uid=row[headerCache['uid']])
self._process_cache.add_data(data=process)
process_index = self._process_cache.data_index(
value=tme, key=pid)
except KeyError:
# exit event received firsts
process = Process()
process.addProcessEvent(
name=row[headerCache['name']],
cmd=row[headerCache['cmd']],
pid=row[headerCache['pid']],
ppid=row[headerCache['ppid']],
uid=row[headerCache['uid']],
tme=row[headerCache['tme']],
exit_code=row[headerCache['exit_code']],
gpid=row[headerCache['gpid']],
state=row[headerCache['state']])
self._process_cache.add_data(data=process)
process_index = self._process_cache.data_index(value=tme,
key=pid)
else:
process.addProcessEvent(
tme=row[headerCache['tme']],
exit_code=row[headerCache['exit_code']],
state=row[headerCache['state']])
try:
job = self._operator.getJob(tme=tme,
gpid=int(
row[headerCache['gpid']]))
except BasicException:
# the job is currently not known so remember as unknown
self._process_cache.unfound.add(process)
self._process_cache.remove_data(data=process, key=pid)
except Exception:
# the job is currently not known so remember as unknown
self._process_cache.unfound.add(process)
self._process_cache.remove_data(data=process, key=pid)
else:
# job has been found, so save current data
self._finish_process(job=job, process=process)
else:
# a new process is getting to know
# process has been started, so create and remember
process = self._create_process(row=row,
header_cache=headerCache)
if "sge_shepherd" in process.cmd:
# new pilot is starting
try:
job = self._operator.getJob(
tme=tme,
gpid=int(row[headerCache['gpid']]),
batchsystemId=process.batchsystemId)
if job.exit_tme and (int(job.exit_tme) < int(tme)):
self._operator.createJob(
tme=tme,
gpid=int(row[headerCache['gpid']]),
batchsystemId=process.batchsystemId)
else:
logging.getLogger(self.__class__.__name__).error(
"ATTENTION: job was not created as it already seems to be "
"existent - job_id from DB %d vs CSV %d",
job.job_id, process.batchsystemId)
except Exception:
self._operator.createJob(
tme=tme,
gpid=int(row[headerCache['gpid']]),
batchsystemId=process.batchsystemId)
else:
# load object
self._create_process(row=row, header_cache=headerCache)
def test_process_events(self):
row_start = [
"1406555483", "9939", "9881", "0", "(sge_shepherd)",
"sge_shepherd-5073566", "0", ".", "9939"
]
row_valid_start = [
"1406555483", "9939", "9881", "0", "(sge_shepherd)",
"sge_shepherd-5073566", "0", "fork", "9939"
]
row_end = [
"1406556210", "9940", "9881", "0", "(sge_shephrd)",
"sge_shepherd-5073567", "0", "exit", "9939"
]
row_end2 = [
"1406556210", "9940", "9881", "0", "(sge_shephrd)",
"sge_shepherd-5073566", "0", "exit", "9939"
]
row_valid_end = [
"1406556210", "9939", "9881", "0", "(sge_shepherd)",
"sge_shepherd-5073566", "0", "exit", "9939"
]
data_start_dict = {}
data_valid_start_dict = {}
data_end_dict = {}
data_end_dict2 = {}
data_valid_end_dict = {}
for key in self.header:
data_start_dict[key] = row_start[self.header[key]]
data_valid_start_dict[key] = row_valid_start[self.header[key]]
data_end_dict[key] = row_end[self.header[key]]
data_end_dict2[key] = row_end2[self.header[key]]
data_valid_end_dict[key] = row_valid_end[self.header[key]]
self.process.addProcessEvent(**data_start_dict)
self.process.addProcessEvent(**data_valid_end_dict)
self.assertEqual(False, self.process.valid)
process = Process()
process.addProcessEvent(**data_valid_end_dict)
self.assertEqual(process.name, data_valid_end_dict["name"])
process.addProcessEvent(**data_start_dict)
self.assertEqual(False, process.valid)
valid_process = Process()
valid_process.addProcessEvent(**data_valid_start_dict)
valid_process.addProcessEvent(**data_valid_end_dict)
self.assertEqual(True, valid_process.valid)
valid_process2 = Process()
valid_process2.addProcessEvent(**data_valid_end_dict)
valid_process2.addProcessEvent(**data_valid_start_dict)
self.assertEqual(True, valid_process2.valid)
process = Process()
process.addProcessEvent(**data_valid_start_dict)
self.assertRaises(ProcessMismatchException, process.addProcessEvent,
**data_end_dict)
process = Process()
process.addProcessEvent(**data_valid_start_dict)
self.assertRaises(ProcessMismatchException, process.addProcessEvent,
**data_end_dict2)
