def test_interpret_task_results_with_sorting(self):
""" Test results sorting in interpret method"""
task = self._get_core_task()
subtask_id = "xxyyzz"
files_dir = os.path.join(task.tmp_dir, subtask_id)
files = self.additional_dir_content([5], sub_dir=files_dir)
shutil.move(files[2], files[2] + ".log")
files[2] += ".log"
shutil.move(files[3], files[3] + "err.log")
files[3] += "err.log"
task.interpret_task_results(subtask_id, files, result_types["files"])
sorted_files = sorted([files[0], files[1], files[4]])
sorted_files[0] = outer_dir_path(sorted_files[0])
sorted_files[1] = outer_dir_path(sorted_files[1])
sorted_files[2] = outer_dir_path(sorted_files[2])
assert task.results[subtask_id] == [
sorted_files[0], sorted_files[1], sorted_files[2]
]
assert task.stderr[subtask_id] == files[3]
assert task.stdout[subtask_id] == files[2]
def filter_task_results(self,
task_results,
subtask_id,
log_ext=".log",
err_log_ext="err.log"):
""" From a list of files received in task_results, return only files that don't
have extension <log_ext> or <err_log_ext>. File with log_ext is saved as stdout
for this subtask (only one file is currently supported). File with err_log_ext is save
as stderr for this subtask (only one file is currently supported).
:param list task_results: list of files
:param str subtask_id: if of a given subtask
:param str log_ext: extension that stdout files have
:param str err_log_ext: extension that stderr files have
:return:
"""
filtered_task_results = []
for tr in task_results:
if tr.endswith(err_log_ext):
self.stderr[subtask_id] = tr
elif tr.endswith(log_ext):
self.stdout[subtask_id] = tr
else:
try:
new_tr = outer_dir_path(tr)
if os.path.isfile(new_tr):
os.remove(new_tr)
os.rename(tr, new_tr)
filtered_task_results.append(new_tr)
except (IOError, OSError) as err:
logger.warning("Cannot move file {} to new location: "
"{}".format(tr, err))
return filtered_task_results
def test_interpret_task_results_without_sorting(self):
task = self._get_core_task()
subtask_id = "xxyyzz"
files_dir = os.path.join(task.tmp_dir, subtask_id)
files = self.additional_dir_content([5], sub_dir=files_dir)
shutil.move(files[2], files[2] + ".log")
files[2] += ".log"
shutil.move(files[3], files[3] + "err.log")
files[3] += "err.log"
files_copy = copy(files)
task.interpret_task_results(subtask_id, files, result_types["files"],
False)
files[0] = outer_dir_path(files[0])
files[1] = outer_dir_path(files[1])
files[4] = outer_dir_path(files[4])
self.assertEqual(task.results[subtask_id],
[files[0], files[1], files[4]])
self.assertEqual(task.stderr[subtask_id], files[3])
self.assertEqual(task.stdout[subtask_id], files[2])
for f in files_copy:
with open(f, 'w'):
pass
task.interpret_task_results(subtask_id, files_copy,
result_types["files"], False)
self.assertEqual(task.results[subtask_id],
[files[0], files[1], files[4]])
for f in files_copy:
with open(f, 'w'):
pass
os.remove(files[0])
os.makedirs(files[0])
with self.assertLogs(logger, level="WARNING"):
task.interpret_task_results(subtask_id, files_copy,
result_types["files"], False)
assert task.results[subtask_id] == [files[1], files[4]]
os.removedirs(files[0])
for f in files + files_copy:
if os.path.isfile(f):
os.remove(f)
assert not os.path.isfile(f)
subtask_id = "aabbcc"
files_dir = os.path.join(task.tmp_dir, subtask_id)
files = self.additional_dir_content([5], sub_dir=files_dir)
shutil.move(files[2], files[2] + ".log")
files[2] += ".log"
shutil.move(files[3], files[3] + "err.log")
files[3] += "err.log"
res = [
self.__compress_and_dump_file(files[0], "abc" * 1000),
self.__compress_and_dump_file(files[1], "def" * 100),
self.__compress_and_dump_file(files[2], "outputlog"),
self.__compress_and_dump_file(files[3], "errlog"),
self.__compress_and_dump_file(files[4], "ghi")
]
task.interpret_task_results(subtask_id, res, result_types["data"],
False)
files[0] = outer_dir_path(files[0])
files[1] = outer_dir_path(files[1])
files[4] = outer_dir_path(files[4])
self.assertEqual(task.results[subtask_id],
[files[0], files[1], files[4]])
self.assertEqual(task.stderr[subtask_id], files[3])
self.assertEqual(task.stdout[subtask_id], files[2])
for f in [files[0], files[1], files[4]]:
self.assertTrue(
os.path.isfile(os.path.join(task.tmp_dir,
os.path.basename(f))))
for f in [files[2], files[3]]:
self.assertTrue(
os.path.isfile(
os.path.join(task.tmp_dir, subtask_id,
os.path.basename(f))))
subtask_id = "112233"
task.interpret_task_results(subtask_id, res, 58, False)
self.assertEqual(task.results[subtask_id], [])
self.assertEqual(task.stderr[subtask_id],
"[GOLEM] Task result 58 not supported")
self.assertEqual(task.stdout[subtask_id], "")
def testOuterInnerDir(self):
path = os.path.join('dir', 'subdir', 'file')
self.assertEqual(outer_dir_path(path), os.path.join('dir', 'file'))
self.assertEqual(outer_dir_path('file'), 'file')
self.assertEqual(outer_dir_path(''), '')
self.assertEqual(inner_dir_path(path, 'inner'), os.path.join('dir', 'subdir', 'inner', 'file'))