def as_quick_fix(self):
"""Build a QuickFix that would resolve this error."""
# Even if there are multiple wanted_types, let's only show the user
# a single QuickFix. It's less noisy that way. (Revisit later if
# this becomes an issue.)
#
# ... ignore self.found_type for now because [2019-04-10] none of
# our converters are split by input type.
# 'names': user-visible colnames
names = ', '.join([f'"{c}"' for c in self.column_names])
# 'colnames': param value for the module
params = {'colnames': self.column_names}
if 'text' in self.wanted_types:
return QuickFix(f'Convert {names} to Text', 'prependModule',
['converttotext', params])
elif 'number' in self.wanted_types:
return QuickFix(f'Convert {names} to Numbers', 'prependModule',
['converttexttonumber', params])
elif 'datetime' in self.wanted_types:
return QuickFix(f'Convert {names} to Dates & Times',
'prependModule', ['convert-date', params])
else:
raise RuntimeError(
f'Unhandled wanted_types: {self.wanted_types}')
def test_coerce_dict_with_quickfix_dict(self):
dataframe = DataFrame({'A': [1, 2]})
quick_fix = QuickFix('Hi', 'prependModule',
['texttodate', {
'column': 'created_at'
}])
result = ProcessResult.coerce({
'dataframe':
dataframe,
'error':
'an error',
'json': {
'foo': 'bar'
},
'quick_fixes': [
{
'text': 'Hi',
'action': 'prependModule',
'args': ['texttodate', {
'column': 'created_at'
}],
},
]
})
expected = ProcessResult(dataframe,
'an error',
json={'foo': 'bar'},
quick_fixes=[quick_fix])
self.assertEqual(result, expected)
def test_assign_and_save(self):
result = ProcessResult(
dataframe=pandas.DataFrame({"a": [1]}),
error="err",
json={"foo": "bar"},
quick_fixes=[QuickFix("X", "prependModule", ["x"])],
columns=[Column("a", ColumnType.NUMBER("{:,d}"))],
)
self.wf_module.cache_render_result(self.delta.id, result)
cached = self.wf_module.cached_render_result
self.assertEqual(cached.wf_module_id, self.wf_module.id)
self.assertEqual(cached.delta_id, self.delta.id)
self.assertEqual(cached.result, result)
self.assertEqual(
cached.parquet_key,
(
f"wf-{self.workflow.id}/wfm-{self.wf_module.id}"
f"/delta-{self.delta.id}.dat"
),
)
db_wf_module = WfModule.objects.get(id=self.wf_module.id)
from_db = db_wf_module.cached_render_result
self.assertEqual(from_db.wf_module_id, self.wf_module.id)
self.assertEqual(cached.delta_id, self.delta.id)
self.assertEqual(from_db.result, result)
def test_assign_and_save(self):
result = ProcessResult(
dataframe=pandas.DataFrame({'a': [1]}),
error='err',
json={'foo': 'bar'},
quick_fixes=[QuickFix('X', 'prependModule', ['x'])],
columns=[Column('a', ColumnType.NUMBER('{:,d}'))]
)
self.wf_module.cache_render_result(self.delta.id, result)
cached = self.wf_module.cached_render_result
self.assertEqual(cached.wf_module_id, self.wf_module.id)
self.assertEqual(cached.delta_id, self.delta.id)
self.assertEqual(cached.result, result)
self.assertEqual(
cached.parquet_key,
(
f'wf-{self.workflow.id}/wfm-{self.wf_module.id}'
f'/delta-{self.delta.id}.dat'
)
)
db_wf_module = WfModule.objects.get(id=self.wf_module.id)
from_db = db_wf_module.cached_render_result
self.assertEqual(from_db.wf_module_id, self.wf_module.id)
self.assertEqual(cached.delta_id, self.delta.id)
self.assertEqual(from_db.result, result)
def test_coerce_dict_with_quickfix_dict(self):
dataframe = pd.DataFrame({"A": [1, 2]})
quick_fix = QuickFix("Hi", "prependModule",
["texttodate", {
"column": "created_at"
}])
result = ProcessResult.coerce({
"dataframe":
dataframe,
"error":
"an error",
"json": {
"foo": "bar"
},
"quick_fixes": [{
"text": "Hi",
"action": "prependModule",
"args": ["texttodate", {
"column": "created_at"
}],
}],
})
expected = ProcessResult(dataframe,
"an error",
json={"foo": "bar"},
quick_fixes=[quick_fix])
self.assertEqual(result, expected)
def from_wf_module(wf_module: "WfModule") -> "CachedRenderResult":
"""
Read the CachedRenderResult or None from a WfModule.
This does not read the _result_ from disk. If you want a "snapshot in
time" of the ProcessResult you need a lock, like this:
# Lock the workflow, making sure we don't overwrite data
with workflow.cooperative_lock():
# Read from database
cached_result = wf_module.get_cached_render_result()
# Read from disk
cached_result.result
(There's not much point in reading from disk within this method,
because a "snapshot in time" is impossible anyway: half the data is in
the database and the other half is on disk.)
"""
if wf_module.cached_render_result_delta_id is None:
return None
delta_id = wf_module.cached_render_result_delta_id
status = wf_module.cached_render_result_status
error = wf_module.cached_render_result_error
columns = wf_module.cached_render_result_columns
nrows = wf_module.cached_render_result_nrows
# TODO [2019-01-24] once we've deployed and wiped all caches, nix this
# 'columns' check and assume 'columns' is always set when we get here
if columns is None:
# this cached value is stale because _Workbench_ has been updated
# and doesn't support it any more
return None
# cached_render_result_json is sometimes a memoryview
json_bytes = bytes(wf_module.cached_render_result_json)
if json_bytes:
json_dict = json.loads(json_bytes)
else:
json_dict = None
quick_fixes = wf_module.cached_render_result_quick_fixes
if not quick_fixes:
quick_fixes = []
# Coerce from dict to QuickFixes
quick_fixes = [QuickFix(**qf) for qf in quick_fixes]
ret = CachedRenderResult(
workflow_id=wf_module.workflow_id,
wf_module_id=wf_module.id,
delta_id=delta_id,
status=status,
error=error,
json=json_dict,
quick_fixes=quick_fixes,
table_shape=TableShape(nrows, columns),
)
# Keep in mind: ret.result has not been loaded yet. It might not exist
# when we do try reading it.
return ret
def test_as_quick_fixes(self):
err = PromptingError([
PromptingError.WrongColumnType(['A'], 'text',
frozenset({'number'})),
PromptingError.WrongColumnType(['B', 'C'], 'datetime',
frozenset({'number'})),
])
result = err.as_quick_fixes()
self.assertEqual(result, [
QuickFix('Convert "A" to Numbers', 'prependModule',
['converttexttonumber', {
'colnames': ['A']
}]),
QuickFix('Convert "B", "C" to Numbers', 'prependModule',
['converttexttonumber', {
'colnames': ['B', 'C']
}]),
])
def test_quick_fixes_convert_to_text(self):
err = PromptingError([
PromptingError.WrongColumnType(["A", "B"], None,
frozenset({"text"}))
])
quick_fixes_result = err.as_quick_fixes()
self.assertEqual(
quick_fixes_result,
[
QuickFix(
"Convert to Text",
"prependModule",
["converttotext", {
"colnames": ["A", "B"]
}],
)
],
)
error_result = err.as_error_str()
self.assertEqual(error_result,
"The columns “A” and “B” must be converted to Text.")