def test_load_deleted_workflow_raises(self):
workflow = Workflow.create_and_init()
wf_module = workflow.tabs.first().wf_modules.create(order=0,
slug="step-1")
with self.assertRaises(Workflow.DoesNotExist):
self.run_with_async_db(
fetch.load_database_objects(workflow.id + 1, wf_module.id))
def test_load_deleted_step_raises(self):
workflow = Workflow.create_and_init()
step = workflow.tabs.first().steps.create(
order=0, slug="step-1", is_deleted=True
)
with self.assertRaises(Step.DoesNotExist):
self.run_with_async_db(fetch.load_database_objects(workflow.id, step.id))
def test_load_deleted_tab_raises(self):
workflow = Workflow.create_and_init()
tab2 = workflow.tabs.create(position=1, is_deleted=True)
wf_module = tab2.wf_modules.create(order=0, slug="step-1")
with self.assertRaises(WfModule.DoesNotExist):
self.run_with_async_db(
fetch.load_database_objects(workflow.id, wf_module.id))
def test_load_deleted_module_version_is_none(self):
workflow = Workflow.create_and_init()
wf_module = workflow.tabs.first().wf_modules.create(
order=0, slug="step-1", module_id_name="foodeleted"
)
result = self.run_with_async_db(
fetch.load_database_objects(workflow.id, wf_module.id)
)
self.assertIsNone(result.module_version)
def test_load_deleted_module_version_is_none(self):
workflow = Workflow.create_and_init()
step = workflow.tabs.first().steps.create(
order=0, slug="step-1", module_id_name="foodeleted", params={"a": "b"}
)
result = self.run_with_async_db(
fetch.load_database_objects(workflow.id, step.id)
)
self.assertIsNone(result.module_zipfile)
self.assertEqual(result.migrated_params_or_error, {})
def test_load_input_cached_render_result_is_none(self):
# Most of these tests assume the fetch is at step 0. This one tests
# step 1, when step 2 has no cached render result.
workflow = Workflow.create_and_init()
workflow.tabs.first().wf_modules.create(
order=0,
slug="step-1",
last_relevant_delta_id=workflow.last_delta_id)
step2 = workflow.tabs.first().wf_modules.create(order=1, slug="step-2")
result = self.run_with_async_db(
fetch.load_database_objects(workflow.id, step2.id))
self.assertEqual(result.input_cached_render_result, None)
def test_load_simple(self):
workflow = Workflow.create_and_init()
module_zipfile = create_module_zipfile("foo")
wf_module = workflow.tabs.first().wf_modules.create(
order=0, slug="step-1", module_id_name="foo")
with self.assertLogs("cjwstate.params", level=logging.INFO):
result = self.run_with_async_db(
fetch.load_database_objects(workflow.id, wf_module.id))
self.assertEqual(result.wf_module, wf_module)
self.assertEqual(result.module_zipfile, module_zipfile)
self.assertEqual(result.migrated_params_or_error, {})
self.assertIsNone(result.stored_object)
self.assertIsNone(result.input_cached_render_result)
def test_load_input_cached_render_result(self):
input_table = make_table(make_column("A", [1]))
workflow = Workflow.create_and_init()
step1 = workflow.tabs.first().steps.create(
order=0,
slug="step-1",
last_relevant_delta_id=workflow.last_delta_id)
write_to_rendercache(workflow, step1, workflow.last_delta_id,
input_table)
step2 = workflow.tabs.first().steps.create(order=1, slug="step-2")
result = self.run_with_async_db(
fetch.load_database_objects(workflow.id, step2.id))
self.assertEqual(result[4], step1.cached_render_result)
self.assertEqual(result.input_cached_render_result,
step1.cached_render_result)
def test_load_simple(self):
workflow = Workflow.create_and_init()
module_version = ModuleVersion.create_or_replace_from_spec(
{"id_name": "foo", "name": "Foo", "category": "Clean", "parameters": []}
)
wf_module = workflow.tabs.first().wf_modules.create(
order=0, slug="step-1", module_id_name="foo"
)
result = self.run_with_async_db(
fetch.load_database_objects(workflow.id, wf_module.id)
)
self.assertEqual(result[0], wf_module)
self.assertEqual(result.wf_module, wf_module)
self.assertEqual(result[1], module_version)
self.assertEqual(result.module_version, module_version)
self.assertIsNone(result[2])
self.assertIsNone(result[3])
def test_load_selected_stored_object(self):
workflow = Workflow.create_and_init()
step = workflow.tabs.first().steps.create(
order=0, slug="step-1", module_id_name="foodeleted"
)
with parquet_file({"A": [1]}) as path1:
storedobjects.create_stored_object(workflow.id, step.id, path1)
with parquet_file({"A": [2]}) as path2:
so2 = storedobjects.create_stored_object(workflow.id, step.id, path2)
with parquet_file({"A": [3]}) as path3:
storedobjects.create_stored_object(workflow.id, step.id, path3)
step.stored_data_version = so2.stored_at
step.save(update_fields=["stored_data_version"])
result = self.run_with_async_db(
fetch.load_database_objects(workflow.id, step.id)
)
self.assertEqual(result[3], so2)
self.assertEqual(result.stored_object, so2)
def test_load_migrate_params_even_when_invalid(self):
workflow = Workflow.create_and_init()
module_zipfile = create_module_zipfile(
"mod",
spec_kwargs={"parameters": [{
"id_name": "a",
"type": "string"
}]},
python_code=textwrap.dedent("""
def migrate_params(params):
return {"x": "y"} # does not validate
"""),
)
wf_module = workflow.tabs.first().wf_modules.create(
order=0, slug="step-1", module_id_name="mod", params={"a": "b"})
with self.assertLogs("cjwstate.params", level=logging.INFO):
result = self.run_with_async_db(
fetch.load_database_objects(workflow.id, wf_module.id))
self.assertEqual(result.migrated_params_or_error, {"x": "y"})
def test_load_input_cached_render_result(self):
with arrow_table_context({"A": [1]}) as atable:
input_render_result = RenderResult(atable)
workflow = Workflow.create_and_init()
step1 = workflow.tabs.first().steps.create(
order=0, slug="step-1", last_relevant_delta_id=workflow.last_delta_id
)
step2 = workflow.tabs.first().steps.create(order=1, slug="step-2")
rendercache.cache_render_result(
workflow, step1, workflow.last_delta_id, input_render_result
)
result = self.run_with_async_db(
fetch.load_database_objects(workflow.id, step2.id)
)
input_crr = step1.cached_render_result
assert input_crr is not None
self.assertEqual(result[4], input_crr)
self.assertEqual(result.input_cached_render_result, input_crr)
def test_load_migrate_params_raise_module_error(self):
workflow = Workflow.create_and_init()
module_zipfile = create_module_zipfile(
"mod",
spec_kwargs={"parameters": [{"id_name": "a", "type": "string"}]},
python_code=textwrap.dedent(
"""
def migrate_params(params):
raise RuntimeError("bad")
"""
),
)
step = workflow.tabs.first().steps.create(
order=0, slug="step-1", module_id_name="mod", params={"a": "b"}
)
with self.assertLogs("cjwstate.params", level=logging.INFO):
result = self.run_with_async_db(
fetch.load_database_objects(workflow.id, step.id)
)
self.assertIsInstance(result.migrated_params_or_error, ModuleExitedError)
self.assertRegex(result.migrated_params_or_error.log, ".*RuntimeError: bad")
