def test_clean_file_no_s3_file(self):
workflow = Workflow.create_and_init()
tab = workflow.tabs.first()
step = tab.steps.create(module_id_name="uploadfile",
order=0,
slug="step-1")
step2 = tab.steps.create(module_id_name="uploadfile",
order=1,
slug="step-2")
id = str(uuid.uuid4())
key = f"wf-${workflow.id}/wfm-${step.id}/${id}"
# Oops -- let's _not_ put the file!
# s3.put_bytes(s3.UserFilesBucket, key, b'1234')
UploadedFileModel.objects.create(step=step2,
name="x.csv.gz",
size=4,
uuid=id,
key=key)
result = self._call_clean_value(ParamSchema.File(),
id,
step_id=step.id)
self.assertIsNone(result)
# Assert that if a temporary file was created to house the download, it
# no longer exists.
self.assertListEqual(list(self.basedir.iterdir()), [])
def test_render_empty_file_param(self):
def render(arrow_table, params, output_path, *args, **kwargs):
self.assertIsNone(params["file"])
param_schema = ParamSchema.Dict({"file": ParamSchema.File()})
with ModuleTestEnv(param_schema=param_schema, render=render) as env:
env.call_render(make_table(), {"file": None})
def test_render_uploaded_files(self):
def render_arrow_v1(table, params, *, uploaded_files, **kwargs):
self.assertEqual(params["file"],
"406b5e37-f217-4e87-b6b2-eede3bec6492")
uploaded_file = uploaded_files[params["file"]]
self.assertEqual(uploaded_file.name, "x.data")
self.assertEqual(uploaded_file.uploaded_at,
datetime(2021, 4, 21, 12, 4, 5))
return ArrowRenderResult(make_table())
param_schema = ParamSchema.Dict({"file": ParamSchema.File()})
with ModuleTestEnv(param_schema=param_schema,
render_arrow_v1=render_arrow_v1) as env:
temp_path = env.basedir / "406b5e37-f217-4e87-b6b2-eede3bec6492_x.data"
temp_path.write_bytes(b"hello, world!")
env.call_render(
make_table(),
params={"file": "406b5e37-f217-4e87-b6b2-eede3bec6492"},
uploaded_files={
"406b5e37-f217-4e87-b6b2-eede3bec6492":
UploadedFile(
name="x.data",
filename=temp_path.name,
uploaded_at=datetime(2021, 4, 21, 12, 4, 5),
),
},
)
def test_clean_file_safe_filename(self):
workflow = Workflow.create_and_init()
tab = workflow.tabs.first()
step = tab.steps.create(module_id_name="uploadfile",
order=0,
slug="step-1")
key = f"wf-${workflow.id}/wfm-${step.id}/6e00511a-8ac4-4b72-9acc-9d069992b5cf"
s3.put_bytes(s3.UserFilesBucket, key, b"1234")
model = UploadedFileModel.objects.create(
step=step,
name="/etc/passwd.$/etc/passwd",
size=4,
uuid="6e00511a-8ac4-4b72-9acc-9d069992b5cf",
key=key,
)
with ExitStack() as inner_stack:
result = self._call_prep_params(
ParamSchema.Dict({"file": ParamSchema.File()}),
{"file": "6e00511a-8ac4-4b72-9acc-9d069992b5cf"},
step_id=step.id,
exit_stack=inner_stack,
)
self.assertEqual(
result.uploaded_files["6e00511a-8ac4-4b72-9acc-9d069992b5cf"],
UploadedFile(
"/etc/passwd.$/etc/passwd",
"6e00511a-8ac4-4b72-9acc-9d069992b5cf_-etc-passwd.--etc-passwd",
model.created_at,
),
)
def test_clean_file_no_uploaded_file(self):
workflow = Workflow.create_and_init()
tab = workflow.tabs.first()
step = tab.steps.create(module_id_name="uploadfile",
order=0,
slug="step-1")
result = self._call_clean_value(ParamSchema.File(),
str(uuid.uuid4()),
step_id=step.id)
self.assertIsNone(result)
# Assert that if a temporary file was created to house the download, it
# no longer exists.
self.assertListEqual(list(self.basedir.iterdir()), [])
def test_clean_file_happy_path(self):
workflow = Workflow.create_and_init()
tab = workflow.tabs.first()
step = tab.steps.create(module_id_name="uploadfile",
order=0,
slug="step-1")
key = f"wf-${workflow.id}/wfm-${step.id}/6e00511a-8ac4-4b72-9acc-9d069992b5cf"
s3.put_bytes(s3.UserFilesBucket, key, b"1234")
model = UploadedFileModel.objects.create(
step=step,
name="x.csv.gz",
size=4,
uuid="6e00511a-8ac4-4b72-9acc-9d069992b5cf",
key=key,
)
with ExitStack() as inner_stack:
result = self._call_prep_params(
ParamSchema.Dict({"file": ParamSchema.File()}),
{"file": "6e00511a-8ac4-4b72-9acc-9d069992b5cf"},
step_id=step.id,
exit_stack=inner_stack,
)
self.assertEqual(
result,
PrepParamsResult(
{"file": "6e00511a-8ac4-4b72-9acc-9d069992b5cf"},
tab_outputs={},
uploaded_files={
"6e00511a-8ac4-4b72-9acc-9d069992b5cf":
UploadedFile(
"x.csv.gz",
"6e00511a-8ac4-4b72-9acc-9d069992b5cf_x.csv.gz",
model.created_at,
)
},
),
)
self.assertEqual(
(self.basedir /
"6e00511a-8ac4-4b72-9acc-9d069992b5cf_x.csv.gz").read_bytes(),
b"1234",
)
# Assert that once `exit_stack` goes out of scope, file is deleted
self.assertFalse(
(self.basedir /
"6e00511a-8ac4-4b72-9acc-9d069992b5cf_x.csv.gz").exists())
def test_render_file_param(self):
def render(arrow_table, params, output_path, *args, **kwargs):
self.assertEqual(params["file"].read_bytes(), b"hi")
param_schema = ParamSchema.Dict({"file": ParamSchema.File()})
with ModuleTestEnv(param_schema=param_schema, render=render) as env:
filename = "839526fa-1adb-4eec-9d29-f5b4d2fbba30_x.tar.gz"
(env.basedir / filename).write_bytes(b"hi")
env.call_render(
make_table(),
{"file": "839526fa-1adb-4eec-9d29-f5b4d2fbba30"},
uploaded_files={
"839526fa-1adb-4eec-9d29-f5b4d2fbba30": UploadedFile(
"x.tar.gz", filename, datetime.now()
)
},
)
def test_clean_file_error(self):
with self.assertRaisesRegex(RuntimeError, "Unsupported: fetch file"):
clean_value(ParamSchema.File(), None, None)
def test_validate_non_str(self):
with pytest.raises(ValueError):
S.File().validate(0x13A5177)
def test_validate_invalid_str(self):
with pytest.raises(ValueError):
# one character too many
S.File().validate("f13aa5177-ee1a-4832-bfbb-6480b93984ab")
def test_validate_uuid(self):
S.File().validate("1e3a5177-ee1a-4832-bfbb-6480b93984ab") # do not raise
def test_validate_null(self):
S.File().validate(None) # do not raise
def test_default_null(self):
assert S.File().default is None
def test_clean_file_none(self):
self.assertIsNone(self._call_clean_value(ParamSchema.File(), None))
