class TestFileDataSource(unittest.TestCase):
def setUp(self):
self.dataSource = FileDataSource()
self.path = os.path.join(os.path.dirname(gnmutils_tests.__file__),
"data/c00-001-001")
def test_isAvailable(self):
self.assertTrue(self.dataSource.is_available())
def test_jobs(self):
index = -1
for index, job in enumerate(self.dataSource.jobs(path=self.path)):
self.assertIsNotNone(job)
self.assertEqual(index, 0)
def jobs(self, **kwargs):
"""
:param path:
:param data_path:
:param source:
:return:
"""
if "raw" in kwargs.get("source", "processed"):
for job in FileDataSource.jobs(self, **kwargs):
yield job
else:
with SQLCommand(dataSource=self._db_data_source) as sql_command:
path = kwargs.get("path", self.default_path)
level = directory_level(path)
job_object = DBJobObject(valid=True, completed=True)
if level == RUN_LEVEL:
_, workernode, run, _ = next(relevant_directories(path=path),
(None, None, None))
job_object.run = run
workernode_object = self._db_operator.load_or_create_workernode(data=workernode)
job_object.workernode_id = workernode_object.id_value
elif level == WORKERNODE_LEVEL:
workernode = os.path.split(path)[1]
workernode_object = self._db_operator.load_or_create_workernode(data=workernode)
job_object.workernode_id = workernode_object.id_value
elif level == FILE_LEVEL:
job_object = DBJobObject(
id=os.path.basename(path).split("-")[0], valid=True, completed=True)
for job_result in sql_command.find(job_object):
current_path = path
if level == BASE_LEVEL:
# join different workernodes and runs
workernode_object = self._db_operator.load_one(
data=DBWorkernodeObject(id=job_result.workernode_id)
)
current_path = os.path.join(os.path.join(path, workernode_object.name),
job_result.run)
elif level == WORKERNODE_LEVEL:
# join different runs
current_path = os.path.join(path, job_result.run)
elif level == FILE_LEVEL:
current_path = os.path.dirname(path)
for job in FileDataSource.read_job(
self,
path=current_path,
name=job_result.id_value):
yield job
def test_from_job(self):
file_path = os.path.join(
os.path.dirname(assess_tests.__file__),
"data/c01-007-102/1/1-process.csv"
)
data_source = FileDataSource()
for job in data_source.jobs(path=file_path):
prototype = Prototype.from_job(job)
self.assertIsNotNone(prototype)
self.assertEqual(prototype.node_count(), 9109)
last_tme = 0
for node in prototype.nodes(order_first=True):
self.assertTrue(last_tme <= node.tme)
last_tme = node.tme
def test_processes_in_order(self):
data_source = FileDataSource()
for job in data_source.jobs(path=self._file_path()):
last_tme = 0
last_pid = 0
for process in job.processes_in_order():
self.assertTrue(process.tme >= last_tme)
if last_tme == process.tme:
# also check for pid
self.assertTrue(
process.pid > last_pid
or ((last_pid + process.pid) % 32768 < 500),
"%d vs %d" % (last_pid, process.pid))
last_pid = process.pid
else:
last_pid = 0
last_tme = process.tme
def test_processes(self):
data_source = FileDataSource()
for job in data_source.jobs(path=self._file_path()):
tree = job.tree
count = 0
for node, depth in tree.walkDFS():
count += 1
# check pid order of children
initial = 0
last_tme = 0
for process in node.children:
self.assertTrue(
process.value.pid >= initial
or (process.value.pid < initial
and last_tme < process.value.tme),
"%d: initial %d differs %d (%s)" %
(count, initial, process.value.pid, [
(child.value.pid, child.value.tme)
for child in node.children
]))
initial = process.value.pid
last_tme = process.value.tme