def test_load_migrate_params_even_when_invalid(self):
workflow = Workflow.create_and_init()
module_zipfile = create_module_zipfile(
"mod",
spec_kwargs={"parameters": [{
"id_name": "a",
"type": "string"
}]},
python_code=textwrap.dedent("""
def migrate_params(params):
return {"x": "y"} # does not validate
"""),
)
wf_module = workflow.tabs.first().wf_modules.create(
order=0, slug="step-1", module_id_name="mod", params={"a": "b"})
with self.assertLogs("cjwstate.params", level=logging.INFO):
result = self.run_with_async_db(
fetch.load_database_objects(workflow.id, wf_module.id))
self.assertEqual(result.migrated_params_or_error, {"x": "y"})
def test_workflow_anonymous_user(self):
# Looking at example workflow as anonymous should create a new workflow
public_workflow = Workflow.create_and_init(
name="Other workflow public",
owner=self.otheruser,
public=True,
example=True,
)
# don't log in
response = self.client.get("/workflows/%d/" % public_workflow.id)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(
Workflow.objects.filter(owner=None).count(),
1) # should have duplicated the wf with this API call
# Ensure the anonymous users can't access the Python module
self.assertNotContains(response, '"pythoncode"')
def test_clear_file_upload_api_token(self):
# Currently, we don't restrict this API to just "upload" modules. We do
# restrict the actual _uploads_, so this oversight isn't a big deal.
user = User.objects.create()
workflow = Workflow.create_and_init(owner=user)
step = workflow.tabs.first().steps.create(
module_id_name="x",
order=0,
slug="step-1",
file_upload_api_token="abcd1234")
response = self.run_handler(
clear_file_upload_api_token,
user=user,
workflow=workflow,
stepSlug="step-1",
)
step.refresh_from_db()
self.assertResponse(response, data=None)
self.assertIsNone(step.file_upload_api_token)
def test_set_notifications(self, log_event):
user = User.objects.create(username="******", email="*****@*****.**")
workflow = Workflow.create_and_init(owner=user)
wf_module = workflow.tabs.first().wf_modules.create(
order=0, slug="step-1", notifications=False
)
response = self.run_handler(
set_notifications,
user=user,
workflow=workflow,
wfModuleId=wf_module.id,
notifications=True,
)
self.assertResponse(response, data=None)
wf_module.refresh_from_db()
self.assertEqual(wf_module.notifications, True)
log_event.assert_called()
def test_change_parameters_update_tab_delta_ids(self):
workflow = Workflow.create_and_init()
# Build the modules
create_module_zipfile(
"x", spec_kwargs={"parameters": [{"id_name": "x", "type": "integer"}]}
)
create_module_zipfile(
"tabby", spec_kwargs={"parameters": [{"id_name": "tab", "type": "tab"}]}
)
self.kernel.migrate_params.side_effect = lambda m, p: p
# tab1's step1 depends on tab2's step2
step1 = workflow.tabs.first().wf_modules.create(
order=0,
slug="step-1",
module_id_name="tabby",
last_relevant_delta_id=workflow.last_delta_id,
params={"tab": "tab-2"},
)
tab2 = workflow.tabs.create(position=1, slug="tab-2")
step2 = tab2.wf_modules.create(
order=0,
slug="step-1",
module_id_name="x",
last_relevant_delta_id=workflow.last_delta_id,
params={"x": 1},
)
with self.assertLogs(level=logging.INFO):
cmd = self.run_with_async_db(
commands.do(
ChangeParametersCommand,
workflow_id=workflow.id,
wf_module=step2,
new_values={"x": 2},
)
)
step1.refresh_from_db()
step2.refresh_from_db()
self.assertEqual(step1.last_relevant_delta_id, cmd.id)
self.assertEqual(step2.last_relevant_delta_id, cmd.id)
def test_set_secret_editor_access_denied(self):
user = User.objects.create(email="*****@*****.**")
workflow = Workflow.create_and_init(public=True)
workflow.acl.create(email=user.email, role=Role.EDITOR)
create_module_zipfile("g",
spec_kwargs={"parameters": [TestStringSecret]})
step = workflow.tabs.first().steps.create(module_id_name="g",
order=0,
slug="step-1")
response = self.run_handler(
set_secret,
user=user,
workflow=workflow,
stepId=step.id,
param="string_secret",
secret="foo",
)
self.assertResponse(response,
error="AuthError: no owner access to workflow")
def test_complete_happy_path(self, queue_render, send_update):
send_update.return_value = async_noop()
queue_render.return_value = async_noop()
_init_module("x")
workflow = Workflow.create_and_init()
wf_module = workflow.tabs.first().wf_modules.create(
order=0, slug="step-123", module_id_name="x", file_upload_api_token="abc123"
)
upload = wf_module.in_progress_uploads.create()
uuid = str(upload.id)
key = upload.get_upload_key()
minio.put_bytes(upload.Bucket, key, b"1234567")
response = self.client.post(
f"/api/v1/workflows/{workflow.id}/steps/step-123/uploads/{upload.id}",
{"filename": "test.csv"},
content_type="application/json",
HTTP_AUTHORIZATION="Bearer abc123",
)
self.assertEqual(response.status_code, 200)
# Upload and its S3 data were deleted
self.assertFalse(minio.exists(upload.Bucket, key))
upload.refresh_from_db()
self.assertTrue(upload.is_completed)
# Final upload was created
uploaded_file = wf_module.uploaded_files.first()
self.assertEqual(
uploaded_file.key, f"wf-{workflow.id}/wfm-{wf_module.id}/{uuid}.csv"
)
self.assertEqual(
minio.get_object_with_data(minio.UserFilesBucket, uploaded_file.key)[
"Body"
],
b"1234567",
)
self.assertEqual(uploaded_file.name, "test.csv")
# Return value includes uuid
data = json.loads(response.content)
self.assertEqual(data["uuid"], uuid)
self.assertEqual(data["name"], "test.csv")
self.assertEqual(data["size"], 7)
# Send deltas
send_update.assert_called()
queue_render.assert_called()
def test_report_module_error(self):
workflow = Workflow.create_and_init()
tab = workflow.tabs.first()
step = tab.steps.create(
order=0,
slug="step-1",
module_id_name="x",
last_relevant_delta_id=workflow.last_delta_id,
)
module_zipfile = create_module_zipfile(
"x",
spec_kwargs={"loads_data": True},
python_code="def render(table, params):\n undefined()",
)
with self.assertLogs(level=logging.INFO):
result = self.run_with_async_db(
execute_step(
self.chroot_context,
workflow,
step,
module_zipfile,
{},
Tab(tab.slug, tab.name),
RenderResult(),
{},
self.output_path,
))
assert_render_result_equals(
result,
RenderResult(errors=[
RenderError(
I18nMessage(
"py.renderer.execute.step.user_visible_bug_during_render",
{
"message":
"exit code 1: NameError: name 'undefined' is not defined"
},
None,
))
]),
)
def test_fetch_result_deleted_file_means_none(self):
workflow = Workflow.create_and_init()
tab = workflow.tabs.first()
wf_module = tab.wf_modules.create(
order=0,
slug="step-1",
module_id_name="x",
last_relevant_delta_id=workflow.last_delta_id,
)
with parquet_file({"A": [1]}) as path:
so = create_stored_object(workflow.id, wf_module.id, path)
wf_module.stored_data_version = so.stored_at
wf_module.save(update_fields=["stored_data_version"])
# Now delete the file on S3 -- but leave the DB pointing to it.
minio.remove(minio.StoredObjectsBucket, so.key)
def render(*args, fetch_result, **kwargs):
self.assertIsNone(fetch_result)
return RenderResult()
module_zipfile = create_module_zipfile(
"x",
python_code=textwrap.dedent("""
import pandas as pd
def render(table, params, *, fetch_result, **kwargs):
assert fetch_result is None
return pd.DataFrame()
"""),
)
with self.assertLogs(level=logging.INFO):
self.run_with_async_db(
execute_wfmodule(
self.chroot_context,
workflow,
wf_module,
module_zipfile,
{},
Tab(tab.slug, tab.name),
RenderResult(),
{},
self.output_path,
))
def test_email_delta_ignore_corrupt_cache_error(self, email_delta,
read_cache):
read_cache.side_effect = rendercache.CorruptCacheError
workflow = Workflow.create_and_init()
tab = workflow.tabs.first()
wf_module = tab.wf_modules.create(
order=0,
slug="step-1",
module_id_name="x",
last_relevant_delta_id=workflow.last_delta_id - 1,
notifications=True,
)
# We need to actually populate the cache to set up the test. The code
# under test will only try to open the render result if the database
# says there's something there.
rendercache.cache_render_result(
workflow,
wf_module,
workflow.last_delta_id - 1,
RenderResult(arrow_table({"A": [1]})),
)
wf_module.last_relevant_delta_id = workflow.last_delta_id
wf_module.save(update_fields=["last_relevant_delta_id"])
with arrow_table_context({"A": [2]}) as table2:
def render(*args, **kwargs):
return RenderResult(table2)
with self._stub_module(render):
with self.assertLogs(level=logging.ERROR):
self.run_with_async_db(
execute_wfmodule(
workflow,
wf_module,
{},
Tab(tab.slug, tab.name),
RenderResult(),
{},
Path("/unused"),
))
email_delta.assert_not_called()
def test_change_parameters_on_soft_deleted_wf_module(self):
workflow = Workflow.create_and_init()
wf_module = workflow.tabs.first().wf_modules.create(
order=0,
slug="step-1",
module_id_name="loadurl",
last_relevant_delta_id=workflow.last_delta_id,
is_deleted=True,
params={"url": ""},
)
cmd = self.run_with_async_db(
commands.do(
ChangeParametersCommand,
workflow_id=workflow.id,
wf_module=wf_module,
new_values={"url": "https://example.com"},
))
self.assertIsNone(cmd)
def test_clientside_update(self, send_update):
future_none = asyncio.Future()
future_none.set_result(None)
send_update.return_value = future_none
workflow = Workflow.create_and_init(selected_tab_position=0) # tab-1
tab2 = workflow.tabs.create(position=1, slug="tab-2")
cmd = self.run_with_async_db(
commands.do(DeleteTabCommand, workflow_id=workflow.id, tab=tab2))
delta1 = send_update.call_args[0][1]
self.assertEqual(delta1.workflow.tab_slugs, ["tab-1"])
self.assertFalse(delta1.tabs)
self.assertEqual(delta1.clear_tab_slugs, frozenset(["tab-2"]))
self.run_with_async_db(commands.undo(cmd))
delta2 = send_update.call_args[0][1]
self.assertEqual(delta2.workflow.tab_slugs, ["tab-1", "tab-2"])
self.assertEqual(list(delta2.tabs.keys()), ["tab-2"])
self.assertFalse(delta2.clear_tab_slugs)
def test_complete_upload_not_found(self):
_init_module("x")
workflow = Workflow.create_and_init()
workflow.tabs.first().wf_modules.create(
order=0,
slug="step-123",
module_id_name="x",
file_upload_api_token="abc123",
params={"file": None},
)
response = self.client.post(
f"/api/v1/workflows/{workflow.id}/steps/step-123/uploads/dcc00084-812d-4769-bf77-94518f18ff3d",
{"filename": "test.csv"},
content_type="application/json",
HTTP_AUTHORIZATION="Bearer abc123",
)
self.assertEqual(response.status_code, 404)
self.assertEqual(
json.loads(response.content)["error"]["code"], "upload-not-found")
def test_authorization_bearer_token_invalid(self):
_init_module("x")
workflow = Workflow.create_and_init()
workflow.tabs.first().wf_modules.create(
order=0,
slug="step-123",
module_id_name="x",
file_upload_api_token="abc123",
params={"file": None},
)
response = self.client.post(
f"/api/v1/workflows/{workflow.id}/steps/step-123/uploads",
HTTP_AUTHORIZATION="Bearer abc123XXX",
)
self.assertEqual(response.status_code, 403)
self.assertEqual(
json.loads(response.content)["error"]["code"],
"authorization-bearer-token-invalid",
)
def test_clean_file_happy_path(self):
workflow = Workflow.create_and_init()
tab = workflow.tabs.first()
step = tab.steps.create(module_id_name="uploadfile", order=0, slug="step-1")
id = str(uuid.uuid4())
key = f"wf-${workflow.id}/wfm-${step.id}/${id}"
s3.put_bytes(s3.UserFilesBucket, key, b"1234")
UploadedFile.objects.create(
step=step, name="x.csv.gz", size=4, uuid=id, key=key
)
with ExitStack() as inner_stack:
context = self._render_context(step_id=step.id, exit_stack=inner_stack)
result: Path = clean_value(ParamDType.File(), id, context)
self.assertIsInstance(result, Path)
self.assertEqual(result.read_bytes(), b"1234")
self.assertEqual(result.suffixes, [".csv", ".gz"])
# Assert that once `exit_stack` goes out of scope, file is deleted
self.assertFalse(result.exists())
def test_set_notifications_to_false(self, log_event):
user = User.objects.create(username="******", email="*****@*****.**")
workflow = Workflow.create_and_init(owner=user)
step = workflow.tabs.first().steps.create(order=0,
slug="step-1",
notifications=True)
response = self.run_handler(
set_notifications,
user=user,
workflow=workflow,
stepId=step.id,
notifications=False,
)
self.assertResponse(response, data=None)
step.refresh_from_db()
self.assertEqual(step.notifications, False)
log_event.assert_not_called() # only log if setting to true
def test_ignore_fresh_lesson(self):
workflow = Workflow.create_and_init(
last_viewed_at=(datetime.datetime.now() - FreshTimedelta),
lesson_slug="analyze-live-twitter",
)
tab = workflow.tabs.first()
step = tab.steps.create(
order=0,
slug="step-1",
module_id_name="fetcher",
auto_update_data=True,
next_update=datetime.datetime.now(),
is_deleted=True,
)
with self.assertLogs(lessonautoupdatedisabler.__name__, logging.INFO):
lessonautoupdatedisabler.disable_stale_auto_update()
step.refresh_from_db()
self.assertEqual(step.auto_update_data, True)
self.assertIsNotNone(step.next_update)
def setUp(self):
super().setUp()
self.queue_render_patcher = patch.object(rabbitmq, "queue_render")
self.queue_render = self.queue_render_patcher.start()
self.queue_render.side_effect = async_noop
self.queue_intercom_message_patcher = patch.object(
rabbitmq, "queue_intercom_message", async_noop)
self.queue_intercom_message_patcher.start()
self.user = create_user("user", "*****@*****.**")
self.workflow1 = Workflow.create_and_init(name="Workflow 1",
owner=self.user)
self.tab1 = self.workflow1.tabs.first()
# Add another user, with one public and one private workflow
self.otheruser = create_user("user2", "*****@*****.**")
def test_complete_json_form_error(self):
_init_module("x")
workflow = Workflow.create_and_init()
wf_module = workflow.tabs.first().wf_modules.create(
order=0, slug="step-123", module_id_name="x", file_upload_api_token="abc123"
)
upload = wf_module.in_progress_uploads.create()
key = upload.get_upload_key()
minio.put_bytes(upload.Bucket, key, b"1234567")
response = self.client.post(
f"/api/v1/workflows/{workflow.id}/steps/step-123/uploads/{upload.id}",
{"filename": None},
content_type="application/json",
HTTP_AUTHORIZATION="Bearer abc123",
)
self.assertEqual(response.status_code, 400)
error = json.loads(response.content)["error"]
self.assertEqual(error["code"], "body-has-errors")
self.assertIn("filename", error["errors"])
def test_delete_deletes_soft_deleted_step(self):
workflow = Workflow.create_and_init()
# Here's a soft-deleted module
step = workflow.tabs.first().steps.create(order=0,
slug="step-1",
module_id_name="foo",
is_deleted=True)
self.run_with_async_db(
commands.do(SetWorkflowTitle,
workflow_id=workflow.id,
new_value="1"))
self.run_with_async_db(commands.undo(
workflow.id)) # fix workflow.last_delta_id
workflow.deltas.all().delete()
workflow.delete_orphan_soft_deleted_models()
with self.assertRaises(Step.DoesNotExist):
step.refresh_from_db()
def test_set_stored_data_version_command_set_read(self):
version = "2018-12-12T21:30:00.000Z"
user = User.objects.create(username="******", email="*****@*****.**")
workflow = Workflow.create_and_init(owner=user)
wf_module = workflow.tabs.first().wf_modules.create(order=0, slug="step-1")
so = wf_module.stored_objects.create(
stored_at=isoparse(version), size=0, read=False
)
response = self.run_handler(
set_stored_data_version,
user=user,
workflow=workflow,
wfModuleId=wf_module.id,
version=version,
)
self.assertResponse(response, data=None)
so.refresh_from_db()
self.assertEqual(so.read, True)
def test_delete_protects_non_deleted_step(self):
workflow = Workflow.create_and_init()
# Here's a soft-deleted module
step = workflow.tabs.first().steps.create(order=0,
slug="step-1",
module_id_name="foo",
is_deleted=False)
# delete a delta
self.run_with_async_db(
commands.do(SetWorkflowTitle,
workflow_id=workflow.id,
new_value="1"))
self.run_with_async_db(commands.undo(
workflow.id)) # fix workflow.last_delta_id
workflow.deltas.all().delete()
workflow.delete_orphan_soft_deleted_models()
step.refresh_from_db() # no DoesNotExist: it's not deleted
def test_execute_tempdir_not_in_tmpfs(self):
# /tmp is RAM; /var/tmp is disk. Assert big files go on disk.
workflow = Workflow.create_and_init()
tab = workflow.tabs.first()
delta1 = workflow.last_delta
create_module_zipfile("mod")
tab.wf_modules.create(
order=0,
slug="step-1",
last_relevant_delta_id=delta1.id - 1,
module_id_name="mod",
)
with patch.object(Kernel,
"render",
side_effect=mock_render({"B": [2]})):
self._execute(workflow)
self.assertRegex(str(Kernel.render.call_args[1]["basedir"]),
r"/var/tmp/")
def test_redo_modify_last_applied_at(self, websockets_notify):
websockets_notify.side_effect = async_noop
date0 = datetime.datetime(2000, 1, 1)
date1 = datetime.datetime.now()
with freeze_time(date0):
workflow = Workflow.create_and_init()
delta = self.run_with_async_db(
commands.do(SetWorkflowTitle,
workflow_id=workflow.id,
new_value="1"))
self.run_with_async_db(commands.undo(workflow.id))
with freeze_time(date1):
self.run_with_async_db(commands.redo(workflow.id))
delta.refresh_from_db()
self.assertEqual(delta.last_applied_at, date1)
def test_delete_custom_report_chart(self, send_update):
future_none = asyncio.Future()
future_none.set_result(None)
send_update.return_value = future_none
workflow = Workflow.create_and_init(
selected_tab_position=0, has_custom_report=True
) # tab-1
tab2 = workflow.tabs.create(position=1, slug="tab-2")
step = tab2.steps.create(
order=0,
slug="step-x",
last_relevant_delta_id=workflow.last_delta_id,
params={},
)
block1 = workflow.blocks.create(
position=0, slug="block-1", block_type="Text", text_markdown="1"
)
block2 = workflow.blocks.create(
position=1, slug="block-2", block_type="Chart", step_id=step.id
)
block3 = workflow.blocks.create(
position=2, slug="block-3", block_type="Text", text_markdown="3"
)
self.run_with_async_db(
commands.do(DeleteTab, workflow_id=workflow.id, tab=tab2)
)
delta1 = send_update.call_args[0][1]
self.assertEqual(delta1.workflow.block_slugs, ["block-1", "block-3"])
self.assertEqual(delta1.blocks, {})
self.assertEqual(delta1.clear_block_slugs, frozenset(["block-2"]))
with self.assertRaises(Block.DoesNotExist):
block2.refresh_from_db()
block3.refresh_from_db()
self.assertEqual(block3.position, 1)
self.run_with_async_db(commands.undo(workflow.id))
delta2 = send_update.call_args[0][1]
self.assertEqual(delta2.workflow.block_slugs, ["block-1", "block-2", "block-3"])
self.assertEqual(delta2.blocks, {"block-2": clientside.ChartBlock("step-x")})
self.assertEqual(delta2.clear_block_slugs, frozenset())
def test_reorder_modules(self):
user = User.objects.create(username="******", email="*****@*****.**")
workflow = Workflow.create_and_init(owner=user) # tab-1
tab = workflow.tabs.first() # tab-1
step1 = tab.wf_modules.create(order=0, slug="step-1")
step2 = tab.wf_modules.create(order=1, slug="step-2")
response = self.run_handler(
reorder_modules,
user=user,
workflow=workflow,
tabSlug="tab-1",
wfModuleIds=[step2.id, step1.id],
)
self.assertResponse(response, data=None)
command = ReorderModulesCommand.objects.first()
self.assertEquals(command.tab_id, tab.id)
self.assertEquals(command.workflow_id, workflow.id)
def test_reset_file_upload_api_token(self):
# Currently, we don't restrict this API to just "upload" modules. We do
# restrict the actual _uploads_, so this oversight isn't a big deal.
user = User.objects.create()
workflow = Workflow.create_and_init(owner=user)
step = workflow.tabs.first().steps.create(module_id_name="x",
order=0,
slug="step-1")
response = self.run_handler(
reset_file_upload_api_token,
user=user,
workflow=workflow,
stepSlug="step-1",
)
step.refresh_from_db()
self.assertEqual(len(step.file_upload_api_token),
43) # 32 bytes, base64-encoded
self.assertResponse(response,
data={"apiToken": step.file_upload_api_token})
def test_reorder_blocks_on_custom_report(self, send_update):
future_none = asyncio.Future()
future_none.set_result(None)
send_update.return_value = future_none
workflow = Workflow.create_and_init(has_custom_report=True)
workflow.blocks.create(position=0,
slug="block-1",
block_type="Text",
text_markdown="1")
workflow.blocks.create(position=1,
slug="block-2",
block_type="Text",
text_markdown="1")
workflow.blocks.create(position=2,
slug="block-3",
block_type="Text",
text_markdown="1")
self.run_with_async_db(
commands.do(
ReorderBlocks,
workflow_id=workflow.id,
slugs=["block-2", "block-3", "block-1"],
))
self.assertEqual(
list(workflow.blocks.values_list("slug", "position")),
[("block-2", 0), ("block-3", 1), ("block-1", 2)],
)
delta1 = send_update.call_args[0][1]
self.assertIsNone(delta1.workflow.has_custom_report)
self.assertEqual(delta1.workflow.block_slugs,
["block-2", "block-3", "block-1"])
self.run_with_async_db(commands.undo(workflow.id))
self.assertEqual(
list(workflow.blocks.values_list("slug", "position")),
[("block-1", 0), ("block-2", 1), ("block-3", 2)],
)
delta2 = send_update.call_args[0][1]
self.assertIsNone(delta2.workflow.has_custom_report)
self.assertEqual(delta2.workflow.block_slugs,
["block-1", "block-2", "block-3"])
def test_set_collapsed_forces_bool(self):
user = User.objects.create(username="******", email="*****@*****.**")
workflow = Workflow.create_and_init(owner=user)
step = workflow.tabs.first().steps.create(order=0,
slug="step-1",
is_collapsed=False)
# bool('False') is true
response = self.run_handler(
set_collapsed,
user=user,
workflow=workflow,
stepId=step.id,
isCollapsed="False",
)
self.assertResponse(response, data=None)
step.refresh_from_db()
self.assertEqual(step.is_collapsed, True)
def test_fetch_integration_tempfiles_are_on_disk(self, create_result):
# /tmp is RAM; /var/tmp is disk. Assert big files go on disk.
workflow = Workflow.create_and_init()
create_module_zipfile(
"mod",
python_code=
("import pandas as pd\ndef fetch(params): return pd.DataFrame({'A': [1]})\ndef render(table, params): return table"
),
)
wf_module = workflow.tabs.first().wf_modules.create(
order=0, slug="step-1", module_id_name="mod")
with self.assertLogs(level=logging.INFO):
cjwstate.modules.init_module_system()
self.run_with_async_db(
fetch.fetch(workflow_id=workflow.id,
wf_module_id=wf_module.id))
create_result.assert_called()
saved_result: FetchResult = create_result.call_args[0][2]
self.assertRegex(str(saved_result.path), r"/var/tmp/")
