def test_execute_mark_unreachable(self, send_update):
future_none = asyncio.Future()
future_none.set_result(None)
send_update.return_value = future_none
workflow = Workflow.create_and_init()
tab = workflow.tabs.first()
create_module_zipfile(
"mod",
spec_kwargs={"loads_data": True},
python_code=
'def render(table, params): return "error, not warning"',
)
step1 = tab.steps.create(order=0, slug="step-1", module_id_name="mod")
step2 = tab.steps.create(order=1, slug="step-2", module_id_name="mod")
step3 = tab.steps.create(order=2, slug="step-3", module_id_name="mod")
self._execute(workflow)
# step1: error
step1.refresh_from_db()
with open_cached_render_result(step1.cached_render_result) as result:
self.assertEqual(result.path.read_bytes(), b"")
self.assertEqual(
step1.cached_render_result.errors,
[RenderError(TODO_i18n("error, not warning"))],
)
# step2, step3: unreachable (no errors, no table data)
step2.refresh_from_db()
self.assertEqual(step2.cached_render_result.status, "unreachable")
with open_cached_render_result(step2.cached_render_result) as result:
self.assertEqual(result.path.read_bytes(), b"")
self.assertEqual(step2.cached_render_result.errors, [])
step3.refresh_from_db()
with open_cached_render_result(step3.cached_render_result) as result:
self.assertEqual(result.path.read_bytes(), b"")
self.assertEqual(step3.cached_render_result.errors, [])
send_update.assert_called_with(
workflow.id,
clientside.Update(
steps={
step3.id:
clientside.StepUpdate(
render_result=step3.cached_render_result,
module_slug="mod")
}),
)
def test_execute_new_revision(self):
workflow = Workflow.create_and_init()
tab = workflow.tabs.first()
create_module_zipfile(
"mod",
spec_kwargs={"loads_data": True},
python_code=
'import pandas as pd\ndef render(table, params): return pd.DataFrame({"B": [2]})',
)
step = tab.steps.create(
order=0,
slug="step-1",
last_relevant_delta_id=1,
module_id_name="mod",
)
cache_render_result(workflow, step, 1,
RenderResult(arrow_table({"A": [1]})))
step.last_relevant_delta_id = 2
step.save(update_fields=["last_relevant_delta_id"])
self._execute(workflow)
step.refresh_from_db()
with open_cached_render_result(step.cached_render_result) as result:
assert_render_result_equals(result,
RenderResult(arrow_table({"B": [2]})))
def test_execute_new_revision(self):
workflow = Workflow.create_and_init()
tab = workflow.tabs.first()
delta1 = workflow.last_delta
create_module_zipfile(
"mod",
python_code=
'import pandas as pd\ndef render(table, params): return pd.DataFrame({"B": [2]})',
)
wf_module = tab.wf_modules.create(
order=0,
slug="step-1",
last_relevant_delta_id=delta1.id,
module_id_name="mod",
)
result1 = RenderResult(arrow_table({"A": [1]}))
cache_render_result(workflow, wf_module, delta1.id, result1)
delta2 = InitWorkflowCommand.create(workflow)
wf_module.last_relevant_delta_id = delta2.id
wf_module.save(update_fields=["last_relevant_delta_id"])
self._execute(workflow)
wf_module.refresh_from_db()
with open_cached_render_result(
wf_module.cached_render_result) as result:
assert_render_result_equals(result,
RenderResult(arrow_table({"B": [2]})))
def test_execute_new_revision(self):
workflow = Workflow.create_and_init()
tab = workflow.tabs.first()
create_module_zipfile(
"mod",
spec_kwargs={"loads_data": True},
python_code=
'import pandas as pd\ndef render(table, params): return pd.DataFrame({"B": [2]})',
)
step = tab.steps.create(
order=0,
slug="step-1",
last_relevant_delta_id=2,
module_id_name="mod",
)
# stale
write_to_rendercache(workflow, step, 1,
make_table(make_column("A", ["a"])))
self._execute(workflow)
step.refresh_from_db()
with open_cached_render_result(step.cached_render_result) as result:
assert_arrow_table_equals(result.table,
make_table(make_column("B", [2])))
def _execute_wfmodule_save(
workflow: Workflow, wf_module: WfModule, result: RenderResult
) -> SaveResult:
"""
Call rendercache.cache_render_result() and build notifications.OutputDelta.
All this runs synchronously within a database lock. (It's a separate
function so that when we're done awaiting it, we can continue executing in
a context that doesn't use a database thread.)
Raise UnneededExecution if the WfModule has changed in the interim.
"""
# raises UnneededExecution
with locked_wf_module(workflow, wf_module) as safe_wf_module:
if safe_wf_module.notifications:
stale_crr = safe_wf_module.get_stale_cached_render_result()
if stale_crr is None:
stale_result = None
else:
try:
# Read entire old Parquet file, blocking
with rendercache.open_cached_render_result(
stale_crr
) as stale_result:
pass # stale_result is deleted from disk but still mmapped
except rendercache.CorruptCacheError:
# No, let's not send an email. Corrupt cache probably means
# we've been messing with our codebase.
logger.exception(
"Ignoring CorruptCacheError on workflow %d, wf_module %d because we are about to overwrite it",
workflow.id,
wf_module.id,
)
stale_result = None
else:
stale_result = None
rendercache.cache_render_result(
workflow, safe_wf_module, wf_module.last_relevant_delta_id, result
)
if (
safe_wf_module.notifications
and stale_result is not None
and result != stale_result
):
safe_wf_module.has_unseen_notification = True
safe_wf_module.save(update_fields=["has_unseen_notification"])
maybe_delta = notifications.OutputDelta(
safe_wf_module.workflow.owner,
safe_wf_module.workflow,
safe_wf_module,
stale_result,
result,
)
else:
maybe_delta = None # nothing to email
return SaveResult(safe_wf_module.cached_render_result, maybe_delta)
def wfmodule_value_counts(request: HttpRequest, wf_module: WfModule):
try:
colname = request.GET["column"]
except KeyError:
return JsonResponse({"error": 'Missing a "column" parameter'}, status=400)
if not colname:
# User has not yet chosen a column. Empty response.
return JsonResponse({"values": {}})
cached_result = wf_module.cached_render_result
if cached_result is None:
# assume we'll get another request after execute finishes
return JsonResponse({"values": {}})
try:
column_index, column = next(
(i, c)
for i, c in enumerate(cached_result.table_metadata.columns)
if c.name == colname
)
except StopIteration:
return JsonResponse({"error": f'column "{colname}" not found'}, status=404)
# raise CorruptCacheError
try:
with open_cached_render_result(cached_result) as result:
arrow_table = result.table.table
# series may be of any type, not just str/categorical
series = arrow_table.column(column_index).to_pandas(
deduplicate_objects=True, ignore_metadata=True
)
except CorruptCacheError:
# We _could_ return an empty result set; but our only goal here is
# "don't crash" and this 404 seems to be the simplest implementation.
# (We assume that if the data is deleted, the user has moved elsewhere
# and this response is going to be ignored.)
return JsonResponse({"error": f'column "{colname}" not found'}, status=404)
# We only handle string. If it's not string, convert to string. (Rationale:
# this is used in Refine and Filter by Value, which are both solely
# String-based for now. Excel and Google Sheets only filter by String
# values, so we're in good company.) Remember: in JavaScript, Object keys
# must be String.
series = ptypes.ColumnType.from_arrow(column.type).format_series(series)
value_counts = series.value_counts().to_dict()
return JsonResponse({"values": value_counts})
def test_resume_without_rerunning_unneeded_renders(self, fake_load_module):
workflow = Workflow.create_and_init()
tab = workflow.tabs.first()
delta_id = workflow.last_delta_id
ModuleVersion.create_or_replace_from_spec({
"id_name": "mod",
"name": "Mod",
"category": "Clean",
"parameters": []
})
# wf_module1: has a valid, cached result
wf_module1 = tab.wf_modules.create(
order=0,
slug="step-1",
last_relevant_delta_id=delta_id,
module_id_name="mod",
)
cache_render_result(workflow, wf_module1, delta_id,
RenderResult(arrow_table({"A": [1]})))
# wf_module2: has no cached result (must be rendered)
wf_module2 = tab.wf_modules.create(
order=1,
slug="step-2",
last_relevant_delta_id=delta_id,
module_id_name="mod",
)
fake_loaded_module = Mock(LoadedModule)
fake_loaded_module.migrate_params.return_value = {}
fake_load_module.return_value = fake_loaded_module
result2 = RenderResult(arrow_table({"A": [2]}))
fake_loaded_module.render.return_value = result2
self._execute(workflow)
fake_loaded_module.render.assert_called_once() # only with module2
wf_module2.refresh_from_db()
with open_cached_render_result(
wf_module2.cached_render_result) as actual:
assert_render_result_equals(actual, result2)
def test_resume_without_rerunning_unneeded_renders(self):
workflow = Workflow.create_and_init()
tab = workflow.tabs.first()
delta_id = workflow.last_delta_id
create_module_zipfile(
# If this runs on step1, it'll return pd.DataFrame().
# If this runs on step2, it'll return step1-output * 2.
# ... step2's output depends on whether we run this on
# step1.
"mod",
spec_kwargs={"loads_data": True},
python_code="def render(table, params): return table * 2",
)
# step1: has a valid, cached result
step1 = tab.steps.create(
order=0,
slug="step-1",
last_relevant_delta_id=1,
module_id_name="mod",
)
cache_render_result(workflow, step1, 1,
RenderResult(arrow_table({"A": [1]})))
# step2: has no cached result (must be rendered)
step2 = tab.steps.create(
order=1,
slug="step-2",
last_relevant_delta_id=1,
module_id_name="mod",
)
self._execute(workflow)
step2.refresh_from_db()
with open_cached_render_result(step2.cached_render_result) as actual:
assert_render_result_equals(actual,
RenderResult(arrow_table({"A": [2]})))
def test_execute_new_revision(self, fake_load_module):
workflow = Workflow.create_and_init()
tab = workflow.tabs.first()
delta1 = workflow.last_delta
ModuleVersion.create_or_replace_from_spec({
"id_name": "mod",
"name": "Mod",
"category": "Clean",
"parameters": []
})
wf_module = tab.wf_modules.create(
order=0,
slug="step-1",
last_relevant_delta_id=delta1.id,
module_id_name="mod",
)
result1 = RenderResult(arrow_table({"A": [1]}))
cache_render_result(workflow, wf_module, delta1.id, result1)
delta2 = InitWorkflowCommand.create(workflow)
wf_module.last_relevant_delta_id = delta2.id
wf_module.save(update_fields=["last_relevant_delta_id"])
result2 = RenderResult(arrow_table({"B": [2]}))
fake_module = Mock(LoadedModule)
fake_module.migrate_params.return_value = {}
fake_load_module.return_value = fake_module
fake_module.render.return_value = result2
self._execute(workflow)
wf_module.refresh_from_db()
with open_cached_render_result(
wf_module.cached_render_result) as result:
assert_render_result_equals(result, result2)
def wfmodule_value_counts(request: HttpRequest, wf_module: WfModule):
try:
colname = request.GET["column"]
except KeyError:
return JsonResponse({"error": 'Missing a "column" parameter'},
status=400)
if not colname:
# User has not yet chosen a column. Empty response.
return JsonResponse({"values": {}})
cached_result = wf_module.cached_render_result
if cached_result is None:
# assume we'll get another request after execute finishes
return JsonResponse({"values": {}})
try:
column_index, column = next(
(i, c) for i, c in enumerate(cached_result.table_metadata.columns)
if c.name == colname)
except StopIteration:
return JsonResponse({"error": f'column "{colname}" not found'},
status=404)
if not isinstance(column.type, ColumnType.Text):
# We only return text values.
#
# Rationale: this is only used in Refine and Filter by Value. Both
# force text. The user can query a column before it's converted to
# text; but if he/she does, we shouldn't format as text unless we have
# a viable workflow that needs it. (Better would be to force the user
# to convert to text before doing anything else, no?)
return JsonResponse({"values": {}})
try:
# raise CorruptCacheError
with open_cached_render_result(cached_result) as result:
arrow_table = result.table.table
chunked_array = arrow_table.column(column_index)
except CorruptCacheError:
# We _could_ return an empty result set; but our only goal here is
# "don't crash" and this 404 seems to be the simplest implementation.
# (We assume that if the data is deleted, the user has moved elsewhere
# and this response is going to be ignored.)
return JsonResponse({"error": f'column "{colname}" not found'},
status=404)
if chunked_array.num_chunks == 0:
value_counts = {}
else:
assert chunked_array.num_chunks == 1
# Assume type is text. (We checked column.type is ColumnType.Text above.)
chunk = chunked_array.chunks[0]
if not hasattr(chunk, "dictionary"):
chunk = chunk.dictionary_encode()
try:
max_index = max(v.as_py() for v in chunk.indices
if v is not pa.NULL)
except ValueError:
# all nulls. Hack with "-1" makes the algorithm not-crash.
max_index = -1
counts = np.zeros(max_index + 1,
dtype=int) # if max_index = -1, counts is empty
for v in chunk.indices:
if v is not pa.NULL:
counts[v.as_py()] += 1
value_counts = {
value.as_py(): int(count)
for value, count in zip(chunk.dictionary, counts)
}
return JsonResponse({"values": value_counts})
async def result_column_value_counts(
request: HttpRequest,
workflow_id: int,
step_slug: str,
delta_id: int,
) -> JsonResponse:
try:
colname = request.GET["column"]
except KeyError:
return JsonResponse({"error": 'Missing a "column" parameter'},
status=status.BAD_REQUEST)
if not colname:
# User has not yet chosen a column. Empty response.
return JsonResponse({"values": {}})
# raise Http404, PermissionDenied
_, step = await _load_workflow_and_step(request, workflow_id, step_slug,
"all")
cached_result = step.cached_render_result
if cached_result is None or cached_result.delta_id != delta_id:
# assume we'll get another request after execute finishes
return JsonResponse({"values": {}})
try:
column_index, column = next(
(i, c) for i, c in enumerate(cached_result.table_metadata.columns)
if c.name == colname)
except StopIteration:
return JsonResponse({"error": f'column "{colname}" not found'},
status=status.NOT_FOUND)
if not isinstance(column.type, ColumnType.Text):
# We only return text values.
#
# Rationale: this is only used in Refine and Filter by Value. Both
# force text. The user can query a column before it's converted to
# text; but if he/she does, we shouldn't format as text unless we have
# a viable workflow that needs it. (Better would be to force the user
# to convert to text before doing anything else, no?)
return JsonResponse({"values": {}})
try:
# raise CorruptCacheError
with open_cached_render_result(cached_result) as result:
arrow_table = result.table.table
chunked_array = arrow_table.column(column_index)
except CorruptCacheError:
# We _could_ return an empty result set; but our only goal here is
# "don't crash" and this 404 seems to be the simplest implementation.
# (We assume that if the data is deleted, the user has moved elsewhere
# and this response is going to be ignored.)
return JsonResponse({"error": f'column "{colname}" not found'},
status=status.NOT_FOUND)
if chunked_array.num_chunks == 0:
value_counts = {}
else:
pyarrow_value_counts = chunked_array.value_counts()
# Assume type is text. (We checked column.type is ColumnType.Text above.)
#
# values can be either a StringArray or a DictionaryArray. In either case,
# .to_pylist() converts to a Python List[str].
values = pyarrow_value_counts.field("values").to_pylist()
counts = pyarrow_value_counts.field("counts").to_pylist()
value_counts = {v: c for v, c in zip(values, counts) if v is not None}
response = JsonResponse({"values": value_counts})
patch_response_headers(response, cache_timeout=600)
return response
def test_execute_mark_unreachable(self, send_update, fake_load_module):
future_none = asyncio.Future()
future_none.set_result(None)
send_update.return_value = future_none
workflow = Workflow.create_and_init()
tab = workflow.tabs.first()
delta_id = workflow.last_delta_id
ModuleVersion.create_or_replace_from_spec({
"id_name": "mod",
"name": "Mod",
"category": "Clean",
"parameters": []
})
wf_module1 = tab.wf_modules.create(
order=0,
slug="step-1",
last_relevant_delta_id=delta_id,
module_id_name="mod",
)
wf_module2 = tab.wf_modules.create(
order=1,
slug="step-2",
last_relevant_delta_id=delta_id,
module_id_name="mod",
)
wf_module3 = tab.wf_modules.create(
order=2,
slug="step-3",
last_relevant_delta_id=delta_id,
module_id_name="mod",
)
fake_module = Mock(LoadedModule)
fake_load_module.return_value = fake_module
fake_module.migrate_params.return_value = {}
error_result = RenderResult(
errors=[RenderError(I18nMessage.TODO_i18n("error, not warning"))])
fake_module.render.return_value = error_result
self._execute(workflow)
wf_module1.refresh_from_db()
self.assertEqual(wf_module1.cached_render_result.status, "error")
with open_cached_render_result(
wf_module1.cached_render_result) as result:
assert_render_result_equals(result, error_result)
wf_module2.refresh_from_db()
self.assertEqual(wf_module2.cached_render_result.status, "unreachable")
with open_cached_render_result(
wf_module2.cached_render_result) as result:
assert_render_result_equals(result, RenderResult())
wf_module3.refresh_from_db()
self.assertEqual(wf_module3.cached_render_result.status, "unreachable")
with open_cached_render_result(
wf_module3.cached_render_result) as result:
assert_render_result_equals(result, RenderResult())
send_update.assert_called_with(
workflow.id,
clientside.Update(
steps={
wf_module3.id:
clientside.StepUpdate(
render_result=wf_module3.cached_render_result)
}),
)
def test_execute_mark_unreachable(self, send_update):
future_none = asyncio.Future()
future_none.set_result(None)
send_update.return_value = future_none
workflow = Workflow.create_and_init()
tab = workflow.tabs.first()
delta_id = workflow.last_delta_id
create_module_zipfile(
"mod",
python_code='def render(table, params): return "error, not warning"'
)
wf_module1 = tab.wf_modules.create(
order=0,
slug="step-1",
last_relevant_delta_id=delta_id,
module_id_name="mod",
)
wf_module2 = tab.wf_modules.create(
order=1,
slug="step-2",
last_relevant_delta_id=delta_id,
module_id_name="mod",
)
wf_module3 = tab.wf_modules.create(
order=2,
slug="step-3",
last_relevant_delta_id=delta_id,
module_id_name="mod",
)
error_result = RenderResult(
errors=[RenderError(I18nMessage.TODO_i18n("error, not warning"))])
self._execute(workflow)
wf_module1.refresh_from_db()
self.assertEqual(wf_module1.cached_render_result.status, "error")
with open_cached_render_result(
wf_module1.cached_render_result) as result:
assert_render_result_equals(result, error_result)
wf_module2.refresh_from_db()
self.assertEqual(wf_module2.cached_render_result.status, "unreachable")
with open_cached_render_result(
wf_module2.cached_render_result) as result:
assert_render_result_equals(result, RenderResult())
wf_module3.refresh_from_db()
self.assertEqual(wf_module3.cached_render_result.status, "unreachable")
with open_cached_render_result(
wf_module3.cached_render_result) as result:
assert_render_result_equals(result, RenderResult())
send_update.assert_called_with(
workflow.id,
clientside.Update(
steps={
wf_module3.id:
clientside.StepUpdate(
render_result=wf_module3.cached_render_result,
module_slug="mod")
}),
)
def test_execute_mark_unreachable(self, send_delta_async,
fake_load_module):
send_delta_async.return_value = future_none
workflow = Workflow.create_and_init()
tab = workflow.tabs.first()
delta_id = workflow.last_delta_id
ModuleVersion.create_or_replace_from_spec({
"id_name": "mod",
"name": "Mod",
"category": "Clean",
"parameters": []
})
wf_module1 = tab.wf_modules.create(
order=0,
slug="step-1",
last_relevant_delta_id=delta_id,
module_id_name="mod",
)
wf_module2 = tab.wf_modules.create(
order=1,
slug="step-2",
last_relevant_delta_id=delta_id,
module_id_name="mod",
)
wf_module3 = tab.wf_modules.create(
order=2,
slug="step-3",
last_relevant_delta_id=delta_id,
module_id_name="mod",
)
fake_module = Mock(LoadedModule)
fake_load_module.return_value = fake_module
fake_module.migrate_params.return_value = {}
error_result = RenderResult(
errors=[RenderError(I18nMessage.TODO_i18n("error, not warning"))])
fake_module.render.return_value = error_result
self._execute(workflow)
wf_module1.refresh_from_db()
self.assertEqual(wf_module1.cached_render_result.status, "error")
with open_cached_render_result(
wf_module1.cached_render_result) as result:
assert_render_result_equals(result, error_result)
wf_module2.refresh_from_db()
self.assertEqual(wf_module2.cached_render_result.status, "unreachable")
with open_cached_render_result(
wf_module2.cached_render_result) as result:
assert_render_result_equals(result, RenderResult())
wf_module3.refresh_from_db()
self.assertEqual(wf_module3.cached_render_result.status, "unreachable")
with open_cached_render_result(
wf_module3.cached_render_result) as result:
assert_render_result_equals(result, RenderResult())
send_delta_async.assert_called_with(
workflow.id,
{
"updateWfModules": {
str(wf_module3.id): {
"output_status": "unreachable",
"quick_fixes": [],
"output_error": "",
"output_columns": [],
"output_n_rows": 0,
"cached_render_result_delta_id": delta_id,
}
}
},
)