def merge_flm_files(self, new_flm, task, output):
computer = LocalComputer(task, self.root_path, self.__verify_flm_ready,
self.__verify_flm_failure,
lambda: self.query_extra_data_for_advanced_verification(new_flm),
use_task_resources=False,
additional_resources=[self.test_flm, new_flm])
computer.run()
if computer.tt is not None:
computer.tt.join()
else:
return False
if self.verification_error:
return False
commonprefix = common_dir(computer.tt.result['data'])
flm = find_file_with_ext(commonprefix, [".flm"])
stderr = filter(lambda x: os.path.basename(x) == "stderr.log", computer.tt.result['data'])
if flm is None or len(stderr) == 0:
return False
else:
try:
with open(stderr[0]) as f:
stderr_in = f.read()
if "ERROR" in stderr_in:
return False
except (IOError, OSError):
return False
shutil.copy(flm, os.path.join(self.tmp_dir, "test_result.flm"))
return True
def _run_docker_local_comp_task(self, render_task, timeout=60 * 5):
render_task.deadline = timeout_to_deadline(timeout)
local_computer = LocalComputer(
render_task, self.tempdir, Mock(), Mock(),
render_task.query_extra_data_for_test_task)
local_computer.run()
local_computer.tt.join(60)
return local_computer.tt
def test_dummy_real_task(self, mock_dtt):
mock_dtt.return_value = 1.0
task = self._get_test_task()
ctd = task.query_extra_data(1.0).ctd
print(ctd)
print(type(ctd))
from twisted.internet import reactor
d = Deferred()
computer = LocalComputer(
root_path=self.tempdir,
success_callback=Mock(),
error_callback=Mock(),
compute_task_def=ctd,
resources=task.task_resources,
)
computer.run()
computer.tt.join()
output = self._extract_results(computer, ctd['subtask_id'])
task.create_reference_data_for_task_validation()
def success(*args, **kwargs):
# pylint: disable=unused-argument
is_subtask_verified = task.verify_subtask(ctd['subtask_id'])
self.assertTrue(is_subtask_verified)
self.assertEqual(task.num_tasks_received, 1)
d.callback(True)
# assert good results - should pass
self.assertEqual(task.num_tasks_received, 0)
task.computation_finished(ctd['subtask_id'], [str(output)],
verification_finished=success)
reactor.iterate()
sync_wait(d, 40)
b = Deferred()
def failure(*args, **kwargs):
# pylint: disable=unused-argument
self.assertFalse(task.verify_subtask(ctd['subtask_id']))
self.assertEqual(task.num_tasks_received, 1)
b.callback(True)
# assert bad results - should fail
bad_output = output.parent / "badfile.result"
ctd = task.query_extra_data(10000.).ctd
task.computation_finished(ctd['subtask_id'], [str(bad_output)],
verification_finished=failure)
reactor.iterate()
sync_wait(b, 40)
def test_computer(self):
dm = DockerTaskThread.docker_manager = DockerManager.install()
dm.update_config(
status_callback=mock.Mock(),
done_callback=mock.Mock(),
work_dir=self.new_path,
in_background=True)
files = self.additional_dir_content([1])
lc = LocalComputer(root_path=self.path,
success_callback=self._success_callback,
error_callback=self._failure_callback,
get_compute_task_def=self._get_bad_task_def)
self.assertIsInstance(lc, LocalComputer)
lc.run()
assert self.last_error is not None
assert self.last_result is None
assert self.error_counter == 1
assert self.success_counter == 0
lc = LocalComputer(root_path=self.path,
success_callback=self._success_callback,
error_callback=self._failure_callback,
get_compute_task_def=self._get_better_task_def,
resources=[],
additional_resources=files)
lc.run()
lc.tt.join(60.0)
path_ = os.path.join(lc.test_task_res_path, os.path.basename(files[0]))
assert os.path.isfile(path_)
assert self.error_counter == 1
assert self.success_counter == 1
tt = self.TestTaskThread({'data': "some data"}, "some error")
lc.task_computed(tt)
assert self.last_result == {"data": "some data"}
assert self.last_result != "some error"
assert self.error_counter == 1
assert self.success_counter == 2
tt = self.TestTaskThread({}, "some error")
lc.task_computed(tt)
assert self.last_error == "some error"
assert self.error_counter == 2
assert self.success_counter == 2
tt = self.TestTaskThread({}, None)
lc.task_computed(tt)
assert self.last_error is None
assert self.error_counter == 3
assert self.success_counter == 2
tt = self.TestTaskThread({'data': "some data"}, None)
tt.error = True
lc.task_computed(tt)
assert self.last_error is None
assert self.error_counter == 4
assert self.success_counter == 2
def __generate_final_file(self, flm):
computer = LocalComputer(self,
self.root_path,
self.__final_img_ready,
self.__final_img_error,
self.query_extra_data_for_merge,
additional_resources=[flm])
computer.run()
computer.tt.join()
def _run_docker_local_comp_task(self, render_task, timeout=60 * 5):
render_task.deadline = timeout_to_deadline(timeout)
local_computer = LocalComputer(
root_path=self.tempdir,
success_callback=Mock(),
error_callback=Mock(),
get_compute_task_def=render_task.query_extra_data_for_test_task,
resources=render_task.task_resources)
local_computer.run()
local_computer.tt.join(60)
return local_computer.tt
def test_computer(self):
with self.assertRaises(TypeError):
LocalComputer(None, self.path, self._success_callback,
self._failure_callback, self._get_bad_task_def)
files = self.additional_dir_content([1])
task = Task(Mock(), Mock())
lc = LocalComputer(task, self.path, self._success_callback,
self._failure_callback, self._get_bad_task_def)
self.assertIsInstance(lc, LocalComputer)
lc.run()
assert self.last_error is not None
assert self.last_result is None
assert self.error_counter == 1
lc = LocalComputer(task,
self.path,
self._success_callback,
self._failure_callback,
self._get_better_task_def,
use_task_resources=False,
additional_resources=files)
lc.run()
lc.tt.join(60.0)
path_ = path.join(lc.test_task_res_path, path.basename(files[0]))
assert path.isfile(path_)
assert self.error_counter == 1
assert self.success_counter == 1
tt = self.TestTaskThread({'data': "some data"}, "some error")
lc.task_computed(tt)
assert self.last_result == {"data": "some data"}
assert self.last_result != "some error"
assert self.error_counter == 1
assert self.success_counter == 2
tt = self.TestTaskThread({}, "some error")
lc.task_computed(tt)
assert self.last_error == "some error"
assert self.error_counter == 2
assert self.success_counter == 2
tt = self.TestTaskThread({}, None)
lc.task_computed(tt)
assert self.last_error is None
assert self.error_counter == 3
assert self.success_counter == 2
tt = self.TestTaskThread({'data': "some data"}, None)
tt.error = True
lc.task_computed(tt)
assert self.last_error is None
assert self.error_counter == 4
assert self.success_counter == 2
def __generate_final_flm(self):
self.collected_file_names = OrderedDict(
sorted(self.collected_file_names.items()))
computer = LocalComputer(
self,
self.root_path,
self.__final_flm_ready,
self.__final_flm_failure,
self.query_extra_data_for_final_flm,
use_task_resources=False,
additional_resources=self.collected_file_names.values())
computer.run()
computer.tt.join()
def _run_task(self, extra_data, task):
computer = LocalComputer(
task,
self.root_path,
self.__box_rendered,
self.__box_render_error,
lambda: self.query_extra_data_for_advanced_verification(extra_data
),
additional_resources=[])
computer.run()
results = None
if computer.tt:
computer.tt.join()
results = computer.tt.result.get("data")
if results:
commonprefix = os.path.commonprefix(results)
img = find_file_with_ext(commonprefix,
["." + extra_data['output_format']])
if img is None:
logger.error("No image file created")
return img