def test_clean_file_no_s3_file(self): workflow = Workflow.create_and_init() tab = workflow.tabs.first() step = tab.steps.create(module_id_name="uploadfile", order=0, slug="step-1") step2 = tab.steps.create(module_id_name="uploadfile", order=1, slug="step-2") id = str(uuid.uuid4()) key = f"wf-${workflow.id}/wfm-${step.id}/${id}" # Oops -- let's _not_ put the file! # s3.put_bytes(s3.UserFilesBucket, key, b'1234') UploadedFileModel.objects.create(step=step2, name="x.csv.gz", size=4, uuid=id, key=key) result = self._call_clean_value(ParamSchema.File(), id, step_id=step.id) self.assertIsNone(result) # Assert that if a temporary file was created to house the download, it # no longer exists. self.assertListEqual(list(self.basedir.iterdir()), [])
def test_render_empty_file_param(self): def render(arrow_table, params, output_path, *args, **kwargs): self.assertIsNone(params["file"]) param_schema = ParamSchema.Dict({"file": ParamSchema.File()}) with ModuleTestEnv(param_schema=param_schema, render=render) as env: env.call_render(make_table(), {"file": None})
def test_render_uploaded_files(self): def render_arrow_v1(table, params, *, uploaded_files, **kwargs): self.assertEqual(params["file"], "406b5e37-f217-4e87-b6b2-eede3bec6492") uploaded_file = uploaded_files[params["file"]] self.assertEqual(uploaded_file.name, "x.data") self.assertEqual(uploaded_file.uploaded_at, datetime(2021, 4, 21, 12, 4, 5)) return ArrowRenderResult(make_table()) param_schema = ParamSchema.Dict({"file": ParamSchema.File()}) with ModuleTestEnv(param_schema=param_schema, render_arrow_v1=render_arrow_v1) as env: temp_path = env.basedir / "406b5e37-f217-4e87-b6b2-eede3bec6492_x.data" temp_path.write_bytes(b"hello, world!") env.call_render( make_table(), params={"file": "406b5e37-f217-4e87-b6b2-eede3bec6492"}, uploaded_files={ "406b5e37-f217-4e87-b6b2-eede3bec6492": UploadedFile( name="x.data", filename=temp_path.name, uploaded_at=datetime(2021, 4, 21, 12, 4, 5), ), }, )
def test_clean_file_safe_filename(self): workflow = Workflow.create_and_init() tab = workflow.tabs.first() step = tab.steps.create(module_id_name="uploadfile", order=0, slug="step-1") key = f"wf-${workflow.id}/wfm-${step.id}/6e00511a-8ac4-4b72-9acc-9d069992b5cf" s3.put_bytes(s3.UserFilesBucket, key, b"1234") model = UploadedFileModel.objects.create( step=step, name="/etc/passwd.$/etc/passwd", size=4, uuid="6e00511a-8ac4-4b72-9acc-9d069992b5cf", key=key, ) with ExitStack() as inner_stack: result = self._call_prep_params( ParamSchema.Dict({"file": ParamSchema.File()}), {"file": "6e00511a-8ac4-4b72-9acc-9d069992b5cf"}, step_id=step.id, exit_stack=inner_stack, ) self.assertEqual( result.uploaded_files["6e00511a-8ac4-4b72-9acc-9d069992b5cf"], UploadedFile( "/etc/passwd.$/etc/passwd", "6e00511a-8ac4-4b72-9acc-9d069992b5cf_-etc-passwd.--etc-passwd", model.created_at, ), )
def test_clean_file_no_uploaded_file(self): workflow = Workflow.create_and_init() tab = workflow.tabs.first() step = tab.steps.create(module_id_name="uploadfile", order=0, slug="step-1") result = self._call_clean_value(ParamSchema.File(), str(uuid.uuid4()), step_id=step.id) self.assertIsNone(result) # Assert that if a temporary file was created to house the download, it # no longer exists. self.assertListEqual(list(self.basedir.iterdir()), [])
def test_clean_file_happy_path(self): workflow = Workflow.create_and_init() tab = workflow.tabs.first() step = tab.steps.create(module_id_name="uploadfile", order=0, slug="step-1") key = f"wf-${workflow.id}/wfm-${step.id}/6e00511a-8ac4-4b72-9acc-9d069992b5cf" s3.put_bytes(s3.UserFilesBucket, key, b"1234") model = UploadedFileModel.objects.create( step=step, name="x.csv.gz", size=4, uuid="6e00511a-8ac4-4b72-9acc-9d069992b5cf", key=key, ) with ExitStack() as inner_stack: result = self._call_prep_params( ParamSchema.Dict({"file": ParamSchema.File()}), {"file": "6e00511a-8ac4-4b72-9acc-9d069992b5cf"}, step_id=step.id, exit_stack=inner_stack, ) self.assertEqual( result, PrepParamsResult( {"file": "6e00511a-8ac4-4b72-9acc-9d069992b5cf"}, tab_outputs={}, uploaded_files={ "6e00511a-8ac4-4b72-9acc-9d069992b5cf": UploadedFile( "x.csv.gz", "6e00511a-8ac4-4b72-9acc-9d069992b5cf_x.csv.gz", model.created_at, ) }, ), ) self.assertEqual( (self.basedir / "6e00511a-8ac4-4b72-9acc-9d069992b5cf_x.csv.gz").read_bytes(), b"1234", ) # Assert that once `exit_stack` goes out of scope, file is deleted self.assertFalse( (self.basedir / "6e00511a-8ac4-4b72-9acc-9d069992b5cf_x.csv.gz").exists())
def test_render_file_param(self): def render(arrow_table, params, output_path, *args, **kwargs): self.assertEqual(params["file"].read_bytes(), b"hi") param_schema = ParamSchema.Dict({"file": ParamSchema.File()}) with ModuleTestEnv(param_schema=param_schema, render=render) as env: filename = "839526fa-1adb-4eec-9d29-f5b4d2fbba30_x.tar.gz" (env.basedir / filename).write_bytes(b"hi") env.call_render( make_table(), {"file": "839526fa-1adb-4eec-9d29-f5b4d2fbba30"}, uploaded_files={ "839526fa-1adb-4eec-9d29-f5b4d2fbba30": UploadedFile( "x.tar.gz", filename, datetime.now() ) }, )
def test_clean_file_error(self): with self.assertRaisesRegex(RuntimeError, "Unsupported: fetch file"): clean_value(ParamSchema.File(), None, None)
def test_validate_non_str(self): with pytest.raises(ValueError): S.File().validate(0x13A5177)
def test_validate_invalid_str(self): with pytest.raises(ValueError): # one character too many S.File().validate("f13aa5177-ee1a-4832-bfbb-6480b93984ab")
def test_validate_uuid(self): S.File().validate("1e3a5177-ee1a-4832-bfbb-6480b93984ab") # do not raise
def test_validate_null(self): S.File().validate(None) # do not raise
def test_default_null(self): assert S.File().default is None
def test_clean_file_none(self): self.assertIsNone(self._call_clean_value(ParamSchema.File(), None))