def test_input_crr_corrupt_cache_error_is_none(
self, downloaded_parquet_file, load_module
):
load_module.return_value.migrate_params.return_value = {}
load_module.return_value.fetch.return_value = FetchResult(self.output_path, [])
downloaded_parquet_file.side_effect = rendercache.CorruptCacheError(
"file not found"
)
input_metadata = TableMetadata(3, [Column("A", ColumnType.Text())])
input_crr = CachedRenderResult(1, 2, 3, "ok", [], {}, input_metadata)
fetch.fetch_or_wrap_error(
self.ctx,
self.chroot_context,
self.basedir,
WfModule(),
MockModuleVersion(),
{},
None,
input_crr,
self.output_path,
)
# fetch is still called, with `None` as argument.
self.assertIsNone(
load_module.return_value.fetch.call_args[1]["input_parquet_filename"]
)
def test_pass_last_fetch_result(self, downloaded_file):
last_result_path = self.ctx.enter_context(
tempfile_context(prefix="last-result")
)
result_path = self.ctx.enter_context(tempfile_context(prefix="result"))
self.kernel.fetch.return_value = FetchResult(result_path, [])
with self.assertLogs("fetcher.fetch", level=logging.INFO):
fetch.fetch_or_wrap_error(
self.ctx,
self.chroot_context,
self.basedir,
"mod",
create_module_zipfile("mod"),
{},
{},
FetchResult(last_result_path, []),
None,
self.output_path,
)
self.assertEqual(
self.kernel.fetch.call_args[1]["last_fetch_result"],
FetchResult(last_result_path, []),
)
def test_input_crr(self, downloaded_parquet_file, clean_value, load_module):
load_module.return_value.migrate_params.return_value = {}
load_module.return_value.fetch.return_value = FetchResult(self.output_path, [])
clean_value.return_value = {}
downloaded_parquet_file.return_value = Path("/path/to/x.parquet")
input_metadata = TableMetadata(3, [Column("A", ColumnType.Text())])
input_crr = CachedRenderResult(1, 2, 3, "ok", [], {}, input_metadata)
fetch.fetch_or_wrap_error(
self.ctx,
self.chroot_context,
self.basedir,
WfModule(),
MockModuleVersion(),
{},
None,
input_crr,
self.output_path,
)
# Passed file is downloaded from rendercache
downloaded_parquet_file.assert_called_with(input_crr, dir=self.basedir)
self.assertEqual(
load_module.return_value.fetch.call_args[1]["input_parquet_filename"],
"x.parquet",
)
# clean_value() is called with input metadata from CachedRenderResult
clean_value.assert_called()
self.assertEqual(clean_value.call_args[0][2], input_metadata)
def test_migrated_params_is_invalid(self):
module_zipfile = create_module_zipfile(
"mod",
spec_kwargs={"parameters": [{
"id_name": "a",
"type": "string"
}]})
with self.assertLogs("fetcher.fetch", level=logging.ERROR):
result = fetch.fetch_or_wrap_error(
self.ctx,
self.chroot_context,
self.basedir,
"mod",
module_zipfile,
{"a": 2}, # invalid: should be string
{},
None,
None,
self.output_path,
)
self.assertEqual(
result,
self._bug_err("%s:migrate_params() output invalid params" %
module_zipfile.path.name),
)
def test_simple(self):
self.kernel.fetch.return_value = FetchResult(self.output_path)
module_zipfile = create_module_zipfile(
"mod",
spec_kwargs={"parameters": [{
"id_name": "A",
"type": "string"
}]})
with self.assertLogs("fetcher.fetch", level=logging.INFO):
result = fetch.fetch_or_wrap_error(
self.ctx,
self.chroot_context,
self.basedir,
"mod",
module_zipfile,
{"A": "B"},
{"C": "D"},
None,
None,
self.output_path,
)
self.assertEqual(result, FetchResult(self.output_path, []))
self.assertEqual(
self.kernel.fetch.call_args[1]["compiled_module"],
module_zipfile.compile_code_without_executing(),
)
self.assertEqual(self.kernel.fetch.call_args[1]["params"],
Params({"A": "B"}))
self.assertEqual(self.kernel.fetch.call_args[1]["secrets"], {"C": "D"})
self.assertIsNone(self.kernel.fetch.call_args[1]["last_fetch_result"])
self.assertIsNone(
self.kernel.fetch.call_args[1]["input_parquet_filename"])
def test_simple(self, load_module):
load_module.return_value.migrate_params.return_value = {"A": "B"}
load_module.return_value.fetch.return_value = FetchResult(self.output_path, [])
result = fetch.fetch_or_wrap_error(
self.ctx,
self.chroot_context,
self.basedir,
WfModule(params={"A": "input"}, secrets={"C": "wrong"}),
MockModuleVersion(
id_name="A", param_schema=ParamDType.Dict({"A": ParamDType.String()})
),
{"C": "D"},
None,
None,
self.output_path,
)
self.assertEqual(result, FetchResult(self.output_path, []))
load_module.return_value.migrate_params.assert_called_with({"A": "input"})
load_module.return_value.fetch.assert_called_with(
chroot_context=self.chroot_context,
basedir=self.basedir,
params=Params({"A": "B"}),
secrets={"C": "D"},
last_fetch_result=None,
input_parquet_filename=None,
output_filename=self.output_path.name,
)
def test_input_crr_corrupt_cache_error_is_none(self, downloaded_parquet_file):
self.kernel.fetch.return_value = FetchResult(self.output_path, [])
downloaded_parquet_file.side_effect = rendercache.CorruptCacheError(
"file not found"
)
input_metadata = TableMetadata(3, [Column("A", ColumnType.Text())])
input_crr = CachedRenderResult(1, 2, 3, "ok", [], {}, input_metadata)
with self.assertLogs("fetcher.fetch", level=logging.INFO):
fetch.fetch_or_wrap_error(
self.ctx,
self.chroot_context,
self.basedir,
"mod",
create_module_zipfile("mod"),
{},
{},
None,
input_crr,
self.output_path,
)
# fetch is still called, with `None` as argument.
self.assertIsNone(self.kernel.fetch.call_args[1]["input_parquet_filename"])
def test_pass_last_fetch_result(self, downloaded_file, load_module):
last_result_path = self.ctx.enter_context(
tempfile_context(prefix="last-result")
)
result_path = self.ctx.enter_context(tempfile_context(prefix="result"))
load_module.return_value.migrate_params.return_value = {}
load_module.return_value.fetch.return_value = FetchResult(result_path, [])
fetch.fetch_or_wrap_error(
self.ctx,
self.basedir,
WfModule(fetch_error=""),
MockModuleVersion(),
{},
FetchResult(last_result_path, []),
None,
self.output_path,
)
self.assertEqual(
load_module.return_value.fetch.call_args[1]["last_fetch_result"],
FetchResult(last_result_path, []),
)
def test_deleted_wf_module(self):
with self.assertLogs(level=logging.INFO):
result = fetch.fetch_or_wrap_error(
self.ctx,
self.basedir,
WfModule(),
None,
{},
None,
None,
self.output_path,
)
self.assertEqual(self.output_path.stat().st_size, 0)
self.assertEqual(result, self._err("Cannot fetch: module was deleted"))
def test_migrated_params_is_error(self):
with self.assertLogs("fetcher.fetch", level=logging.ERROR):
result = fetch.fetch_or_wrap_error(
self.ctx,
self.chroot_context,
self.basedir,
"mod",
create_module_zipfile("mod"),
ModuleExitedError("mod", 1, "Traceback:\n\n\nRuntimeError: bad"),
{},
None,
None,
self.output_path,
)
self.assertEqual(result, self._bug_err("exit code 1: RuntimeError: bad"))
def test_fetch_module_error(self, load_module):
load_module.return_value.migrate_params.return_value = {}
load_module.return_value.fetch.side_effect = ModuleExitedError(1, "bad")
with self.assertLogs(level=logging.ERROR):
result = fetch.fetch_or_wrap_error(
self.ctx,
self.basedir,
WfModule(),
MockModuleVersion(),
{},
None,
None,
self.output_path,
)
self.assertEqual(result, self._bug_err("exit code 1: bad"))
def test_load_module_compile_error(self, load_module):
load_module.side_effect = ModuleExitedError(1, "log")
with self.assertLogs(level=logging.ERROR):
result = fetch.fetch_or_wrap_error(
self.ctx,
self.basedir,
WfModule(),
MockModuleVersion("bad"),
{},
None,
None,
self.output_path,
)
self.assertEqual(self.output_path.stat().st_size, 0)
self.assertEqual(result, self._bug_err("exit code 1: log (during load)"))
def test_load_module_missing(self, load_module):
load_module.side_effect = FileNotFoundError
with self.assertLogs(level=logging.INFO):
result = fetch.fetch_or_wrap_error(
self.ctx,
self.basedir,
WfModule(),
MockModuleVersion("missing"),
{},
None,
None,
self.output_path,
)
self.assertEqual(self.output_path.stat().st_size, 0)
self.assertEqual(result, self._bug_err("FileNotFoundError"))
def test_fetch_module_error(self):
self.kernel.fetch.side_effect = ModuleExitedError("mod", 1, "RuntimeError: bad")
with self.assertLogs(level=logging.ERROR):
result = fetch.fetch_or_wrap_error(
self.ctx,
self.chroot_context,
self.basedir,
"mod",
create_module_zipfile("mod"),
{},
{},
None,
None,
self.output_path,
)
self.assertEqual(result, self._bug_err("exit code 1: RuntimeError: bad"))
def test_input_crr(self, downloaded_parquet_file, clean_value):
def do_fetch(
compiled_module,
chroot_context,
basedir,
params,
secrets,
last_fetch_result,
input_parquet_filename,
output_filename,
):
shutil.copy(basedir / input_parquet_filename,
basedir / output_filename)
return FetchResult(basedir / output_filename)
self.kernel.fetch.side_effect = do_fetch
clean_value.return_value = {}
with tempfile_context(dir=self.basedir,
suffix=".parquet") as parquet_path:
parquet_path.write_bytes(b"abc123")
downloaded_parquet_file.return_value = parquet_path
input_metadata = TableMetadata(3, [Column("A", ColumnType.Text())])
input_crr = CachedRenderResult(1, 2, 3, "ok", [], {},
input_metadata)
with self.assertLogs("fetcher.fetch", level=logging.INFO):
result = fetch.fetch_or_wrap_error(
self.ctx,
self.chroot_context,
self.basedir,
"mod",
create_module_zipfile("mod"),
{},
{},
None,
input_crr,
self.output_path,
)
# Passed file is downloaded from rendercache
self.assertEqual(result.path.read_bytes(), b"abc123")
# clean_value() is called with input metadata from CachedRenderResult
clean_value.assert_called()
self.assertEqual(clean_value.call_args[0][2], input_metadata)
def test_deleted_wf_module(self):
with self.assertLogs(level=logging.INFO):
result = fetch.fetch_or_wrap_error(
self.ctx,
self.chroot_context,
self.basedir,
"mod",
None,
{},
{},
None,
None,
self.output_path,
)
self.assertEqual(self.output_path.stat().st_size, 0)
self.assertEqual(
result,
self._err(I18nMessage("py.fetcher.fetch.no_loaded_module")))
