def test_input_crr_corrupt_cache_error_is_none(
self, downloaded_parquet_file, load_module
):
load_module.return_value.migrate_params.return_value = {}
load_module.return_value.fetch.return_value = FetchResult(self.output_path, [])
downloaded_parquet_file.side_effect = rendercache.CorruptCacheError(
"file not found"
)
input_metadata = TableMetadata(3, [Column("A", ColumnType.Text())])
input_crr = CachedRenderResult(1, 2, 3, "ok", [], {}, input_metadata)
fetch.fetch_or_wrap_error(
self.ctx,
self.chroot_context,
self.basedir,
WfModule(),
MockModuleVersion(),
{},
None,
input_crr,
self.output_path,
)
# fetch is still called, with `None` as argument.
self.assertIsNone(
load_module.return_value.fetch.call_args[1]["input_parquet_filename"]
)
def test_simple(self, load_module):
load_module.return_value.migrate_params.return_value = {"A": "B"}
load_module.return_value.fetch.return_value = FetchResult(self.output_path, [])
result = fetch.fetch_or_wrap_error(
self.ctx,
self.chroot_context,
self.basedir,
WfModule(params={"A": "input"}, secrets={"C": "wrong"}),
MockModuleVersion(
id_name="A", param_schema=ParamDType.Dict({"A": ParamDType.String()})
),
{"C": "D"},
None,
None,
self.output_path,
)
self.assertEqual(result, FetchResult(self.output_path, []))
load_module.return_value.migrate_params.assert_called_with({"A": "input"})
load_module.return_value.fetch.assert_called_with(
chroot_context=self.chroot_context,
basedir=self.basedir,
params=Params({"A": "B"}),
secrets={"C": "D"},
last_fetch_result=None,
input_parquet_filename=None,
output_filename=self.output_path.name,
)
def test_input_crr(self, downloaded_parquet_file, clean_value, load_module):
load_module.return_value.migrate_params.return_value = {}
load_module.return_value.fetch.return_value = FetchResult(self.output_path, [])
clean_value.return_value = {}
downloaded_parquet_file.return_value = Path("/path/to/x.parquet")
input_metadata = TableMetadata(3, [Column("A", ColumnType.Text())])
input_crr = CachedRenderResult(1, 2, 3, "ok", [], {}, input_metadata)
fetch.fetch_or_wrap_error(
self.ctx,
self.chroot_context,
self.basedir,
WfModule(),
MockModuleVersion(),
{},
None,
input_crr,
self.output_path,
)
# Passed file is downloaded from rendercache
downloaded_parquet_file.assert_called_with(input_crr, dir=self.basedir)
self.assertEqual(
load_module.return_value.fetch.call_args[1]["input_parquet_filename"],
"x.parquet",
)
# clean_value() is called with input metadata from CachedRenderResult
clean_value.assert_called()
self.assertEqual(clean_value.call_args[0][2], input_metadata)
def test_deleted_wf_module(self):
with self.assertLogs(level=logging.INFO):
result = fetch.fetch_or_wrap_error(
self.ctx,
self.basedir,
WfModule(),
None,
{},
None,
None,
self.output_path,
)
self.assertEqual(self.output_path.stat().st_size, 0)
self.assertEqual(result, self._err("Cannot fetch: module was deleted"))
def test_fetch_module_error(self, load_module):
load_module.return_value.migrate_params.return_value = {}
load_module.return_value.fetch.side_effect = ModuleExitedError(1, "bad")
with self.assertLogs(level=logging.ERROR):
result = fetch.fetch_or_wrap_error(
self.ctx,
self.basedir,
WfModule(),
MockModuleVersion(),
{},
None,
None,
self.output_path,
)
self.assertEqual(result, self._bug_err("exit code 1: bad"))
def test_load_module_compile_error(self, load_module):
load_module.side_effect = ModuleExitedError(1, "log")
with self.assertLogs(level=logging.ERROR):
result = fetch.fetch_or_wrap_error(
self.ctx,
self.basedir,
WfModule(),
MockModuleVersion("bad"),
{},
None,
None,
self.output_path,
)
self.assertEqual(self.output_path.stat().st_size, 0)
self.assertEqual(result, self._bug_err("exit code 1: log (during load)"))
def test_load_module_missing(self, load_module):
load_module.side_effect = FileNotFoundError
with self.assertLogs(level=logging.INFO):
result = fetch.fetch_or_wrap_error(
self.ctx,
self.basedir,
WfModule(),
MockModuleVersion("missing"),
{},
None,
None,
self.output_path,
)
self.assertEqual(self.output_path.stat().st_size, 0)
self.assertEqual(result, self._bug_err("FileNotFoundError"))
def test_pass_last_fetch_result(self, downloaded_file, load_module):
last_result_path = self.ctx.enter_context(
tempfile_context(prefix="last-result")
)
result_path = self.ctx.enter_context(tempfile_context(prefix="result"))
load_module.return_value.migrate_params.return_value = {}
load_module.return_value.fetch.return_value = FetchResult(result_path, [])
fetch.fetch_or_wrap_error(
self.ctx,
self.basedir,
WfModule(fetch_error=""),
MockModuleVersion(),
{},
FetchResult(last_result_path, []),
None,
self.output_path,
)
self.assertEqual(
load_module.return_value.fetch.call_args[1]["last_fetch_result"],
FetchResult(last_result_path, []),
)