def test_delete_deletes_from_s3(self):
s3.put_bytes(s3.StoredObjectsBucket, "test.dat", b"abcd")
workflow = Workflow.create_and_init()
step = workflow.tabs.first().steps.create(order=0, slug="step-1")
so = step.stored_objects.create(size=4, key="test.dat")
so.delete()
self.assertFalse(s3.exists(s3.StoredObjectsBucket, "test.dat"))
def test_delete_step(self):
result = RenderResult(arrow_table({"A": [1]}),
[RenderError(I18nMessage("X", {}, None), [])],
{})
cache_render_result(self.workflow, self.step, 1, result)
parquet_key = crr_parquet_key(self.step.cached_render_result)
self.step.delete()
self.assertFalse(s3.exists(BUCKET, parquet_key))
def test_delete_remove_uploaded_data_by_prefix_in_case_model_missing(self):
workflow = Workflow.create_and_init()
step = workflow.tabs.first().steps.create(order=0, slug="step-1")
uuid = str(uuidgen.uuid4())
key = step.uploaded_file_prefix + uuid
s3.put_bytes(s3.UserFilesBucket, key, b"A\n1")
# Don't create the UploadedFile. Simulates races during upload/delete
# that could write a file on S3 but not in our database.
# step.uploaded_files.create(name='t.csv', size=3, uuid=uuid, key=key)
step.delete() # do not crash
self.assertFalse(s3.exists(s3.UserFilesBucket, key))
def test_delete_step(self):
write_to_rendercache(
self.workflow,
self.step,
1,
table=make_table(make_column("A", [1])),
errors=[RenderError(I18nMessage("X", {}, None), [])],
json={"foo": "bar"},
)
parquet_key = crr_parquet_key(self.step.cached_render_result)
self.step.delete()
self.assertFalse(s3.exists(BUCKET, parquet_key))
def test_clear(self):
with arrow_table_context(make_column("A", [1])) as (path, table):
result = LoadedRenderResult(
path=path,
table=table,
columns=[Column("A", ColumnType.Number(format="{:,}"))],
errors=[],
json={},
)
cache_render_result(self.workflow, self.step, 1, result)
parquet_key = crr_parquet_key(self.step.cached_render_result)
clear_cached_render_result_for_step(self.step)
db_step = Step.objects.get(id=self.step.id)
self.assertIsNone(db_step.cached_render_result)
self.assertFalse(s3.exists(BUCKET, parquet_key))
def test_pre_finish_enforce_storage_limits(self, send_update):
send_update.side_effect = async_noop
_init_module("x")
self.kernel.migrate_params.side_effect = lambda m, p: p
workflow = Workflow.create_and_init()
step = workflow.tabs.first().steps.create(
order=0,
slug="step-123",
module_id_name="x",
file_upload_api_token="abc123",
params={"file": None},
)
s3.put_bytes(s3.UserFilesBucket, "foo/1.txt", b"1")
step.uploaded_files.create(
created_at=datetime.datetime(2020, 1, 1),
name="file1.txt",
size=1,
uuid="df46244d-268a-0001-9b47-360502dd9b32",
key="foo/1.txt",
)
s3.put_bytes(s3.UserFilesBucket, "foo/2.txt", b"22")
step.uploaded_files.create(
created_at=datetime.datetime(2020, 1, 2),
name="file2.txt",
size=2,
uuid="df46244d-268a-0002-9b47-360502dd9b32",
key="foo/2.txt",
)
s3.put_bytes(s3.UserFilesBucket, "foo/3.txt", b"333")
step.uploaded_files.create(
created_at=datetime.datetime(2020, 1, 3),
name="file3.txt",
size=3,
uuid="df46244d-268a-0003-9b47-360502dd9b32",
key="foo/3.txt",
)
# Upload the new file, "file4.txt"
s3.put_bytes(s3.TusUploadBucket, "new-key", b"4444")
with self.assertLogs(level=logging.INFO):
# Logs SetStepParams's migrate_params()
response = self.client.post(
f"/tusd-hooks",
{
"Upload": {
"MetaData": {
"filename": "file4.txt",
"workflowId": str(workflow.id),
"stepSlug": step.slug,
"apiToken": "abc123",
},
"Size": 7,
"Storage": {
"Bucket": s3.TusUploadBucket,
"Key": "new-key"
},
}
},
HTTP_HOOK_NAME="pre-finish",
content_type="application/json",
)
self.assertEqual(response.status_code, 200)
# Test excess uploaded files were deleted
self.assertEqual(
list(
step.uploaded_files.order_by("id").values_list("name",
flat=True)),
["file3.txt", "file4.txt"],
)
self.assertFalse(s3.exists(s3.UserFilesBucket, "foo/1.txt"))
self.assertFalse(s3.exists(s3.UserFilesBucket, "foo/2.txt"))
# Test delta nixes old files from clients' browsers
send_update.assert_called()
uploaded_file = step.uploaded_files.get(name="file4.txt")
self.assertEqual(
send_update.mock_calls[0][1][1].steps[step.id].files,
[
clientside.UploadedFile(
name="file4.txt",
uuid=uploaded_file.uuid,
size=7,
created_at=uploaded_file.created_at,
),
clientside.UploadedFile(
name="file3.txt",
uuid="df46244d-268a-0003-9b47-360502dd9b32",
size=3,
created_at=datetime.datetime(2020, 1, 3),
),
],
)
async def _assert_s3_ok():
"""Crash if S3 is not available."""
# The file doesn't need to be there; the request need only complete
s3.exists(s3.UserFilesBucket, "healthz")