def test_clean_tab_tab_delete_race_raises_unneededexecution(self):
"""
If a user deletes the tab during render, raise UnneededExecution.
It doesn't really matter _what_ the return value is, since the render()
result will never be saved if this WfModule's delta has changed.
UnneededExecution just seems like the quickest way out of this mess:
it's an error the caller is meant to raise anyway, unlike
`Tab.DoesNotExist`.
"""
# tab_output is what 'render' _thinks_ the output should be
tab_output = ProcessResult(pd.DataFrame({'A': [1, 2]}))
workflow = Workflow.create_and_init()
tab = workflow.tabs.first()
wfm = tab.wf_modules.create(
order=0, last_relevant_delta_id=workflow.last_delta_id)
wfm.cache_render_result(workflow.last_delta_id, tab_output)
tab.is_deleted = True
tab.save(update_fields=['is_deleted'])
# Simulate reality: wfm.last_relevant_delta_id will change
wfm.last_relevant_delta_id += 1
wfm.save(update_fields=['last_relevant_delta_id'])
context = RenderContext(
workflow.id, None, {
tab.slug: StepResultShape('ok', tab_output.table_shape),
}, None)
schema = ParamDType.Dict({'tab': ParamDType.Tab()})
with self.assertRaises(UnneededExecution):
clean_value(schema, {'tab': tab.slug}, context)
def test_clean_multicolumn_from_other_tab(self):
tab_output = ProcessResult(pd.DataFrame({'A-from-tab-2': [1, 2]}))
workflow = Workflow.create_and_init()
tab = workflow.tabs.first()
wfm = tab.wf_modules.create(
order=0, last_relevant_delta_id=workflow.last_delta_id)
wfm.cache_render_result(workflow.last_delta_id, tab_output)
schema = ParamDType.Dict({
'tab':
ParamDType.Tab(),
'columns':
ParamDType.Multicolumn(tab_parameter='tab'),
})
param_values = {
'tab': tab.slug,
'columns': 'A-from-tab-1,A-from-tab-2'
}
params = Params(schema, param_values, {})
context = RenderContext(
workflow.id,
TableShape(3, [
Column('A-from-tab-1', ColumnType.NUMBER),
]), {
tab.slug: StepResultShape('ok', tab_output.table_shape),
}, params)
result = clean_value(schema, param_values, context)
# result['tab'] is not what we're testing here
self.assertEqual(result['columns'], 'A-from-tab-2')
def test_clean_tab_wf_module_changed_raises_unneededexecution(self):
"""
If a user changes tabs' output during render, raise UnneededExecution.
It doesn't really matter _what_ the return value is, since the render()
result will never be saved if this WfModule's delta has changed.
UnneededExecution seems like the simplest contract to enforce.
"""
# tab_output is what 'render' _thinks_ the output should be
tab_output = ProcessResult(pd.DataFrame({'A': [1, 2]}))
workflow = Workflow.create_and_init()
tab = workflow.tabs.first()
wfm = tab.wf_modules.create(
order=0, last_relevant_delta_id=workflow.last_delta_id)
wfm.cache_render_result(workflow.last_delta_id, tab_output)
# Simulate reality: wfm.last_relevant_delta_id will change
wfm.last_relevant_delta_id += 1
wfm.save(update_fields=['last_relevant_delta_id'])
context = RenderContext(
workflow.id, None, {
tab.slug: StepResultShape('ok', tab_output.table_shape),
}, None)
schema = ParamDType.Dict({'tab': ParamDType.Tab()})
with self.assertRaises(UnneededExecution):
clean_value(schema, {'tab': tab.slug}, context)
def test_map_omit_missing_table_columns(self):
# Currently, "omit" means "set empty". There's a valid use case for
# actually _removing_ colnames here, but [adamhooper, 2019-01-04] we
# haven't defined that case yet.
dtype = ParamDType.Map(value_dtype=ParamDType.Column())
value = dtype.omit_missing_table_columns({'a': 'X', 'b': 'Y'}, {'X'})
self.assertEqual(value, {'a': 'X', 'b': ''})
def test_clean_normal_dict(self):
context = RenderContext(None, None, None, None)
schema = ParamDType.Dict({
'str': ParamDType.String(),
'int': ParamDType.Integer(),
})
value = {'str': 'foo', 'int': 3}
expected = dict(value) # no-op
result = clean_value(schema, value, context)
self.assertEqual(result, expected)
def test_clean_tab_missing_tab_selected_gives_none(self):
"""
If the user has selected a nonexistent tab, pretend tab is blank.
The JS side of things will see the nonexistent tab, but not render().
"""
context = RenderContext(None, None, {}, None)
schema = ParamDType.Dict({'tab': ParamDType.Tab()})
result = clean_value(schema, {'tab': 'tab-XXX'}, context)
self.assertEqual(result, {'tab': None})
def test_clean_column_missing_becomes_empty_string(self):
context = RenderContext(
None, TableShape(3, [
Column('A', ColumnType.NUMBER),
]), None, None)
schema = ParamDType.Dict({
'column': ParamDType.Column(),
})
value = {'column': 'B'}
result = clean_value(schema, value, context)
self.assertEqual(result, {'column': ''})
def test_clean_multicolumn_missing_is_removed(self):
context = RenderContext(
None,
TableShape(3, [
Column('A', ColumnType.NUMBER),
Column('B', ColumnType.NUMBER),
]), None, None)
schema = ParamDType.Dict({
'columns': ParamDType.Multicolumn(),
})
value = {'columns': 'A,X,B'}
result = clean_value(schema, value, context)
self.assertEqual(result, {'columns': 'A,B'})
def test_map_parse(self):
dtype = ParamDType.parse({
'type': 'map',
'value_dtype': {
'type': 'dict', # test nesting
'properties': {
'foo': {
'type': 'string'
},
},
},
})
self.assertEqual(
repr(dtype),
repr(
ParamDType.Map(value_dtype=ParamDType.Dict(
properties={
'foo': ParamDType.String(),
}))))
def test_clean_tabs_preserve_ordering(self):
tab2_output = ProcessResult(pd.DataFrame({'A': [1, 2]}))
tab3_output = ProcessResult(pd.DataFrame({'B': [2, 3]}))
workflow = Workflow.create_and_init()
tab1 = workflow.tabs.first()
tab2 = workflow.tabs.create(position=1, slug='tab-2', name='Tab 2')
tab3 = workflow.tabs.create(position=1, slug='tab-3', name='Tab 3')
wfm2 = tab2.wf_modules.create(
order=0, last_relevant_delta_id=workflow.last_delta_id)
wfm2.cache_render_result(workflow.last_delta_id, tab2_output)
wfm3 = tab3.wf_modules.create(
order=0, last_relevant_delta_id=workflow.last_delta_id)
wfm3.cache_render_result(workflow.last_delta_id, tab3_output)
# RenderContext's dict ordering determines desired tab order. (Python
# 3.7 spec: dict is ordered in insertion order. CPython 3.6 and PyPy 7
# do this, too.)
context = RenderContext(
workflow.id, None, {
tab1.slug: None,
tab2.slug: StepResultShape('ok', tab2_output.table_shape),
tab3.slug: StepResultShape('ok', tab3_output.table_shape),
}, None)
schema = ParamDType.Dict({'tabs': ParamDType.Multitab()})
# Supply wrongly-ordered tabs. Cleaned, they should be in order.
result = clean_value(schema, {'tabs': [tab3.slug, tab2.slug]}, context)
self.assertEqual(result['tabs'][0].slug, tab2.slug)
self.assertEqual(result['tabs'][0].name, tab2.name)
self.assertEqual(result['tabs'][0].columns, {
'A': RenderColumn('A', 'number'),
})
assert_frame_equal(result['tabs'][0].dataframe,
pd.DataFrame({'A': [1, 2]}))
self.assertEqual(result['tabs'][1].slug, tab3.slug)
self.assertEqual(result['tabs'][1].name, tab3.name)
self.assertEqual(result['tabs'][1].columns, {
'B': RenderColumn('B', 'number'),
})
assert_frame_equal(result['tabs'][1].dataframe,
pd.DataFrame({'B': [2, 3]}))
def test_clean_tab_happy_path(self):
tab_output = ProcessResult(pd.DataFrame({'A': [1, 2]}))
workflow = Workflow.create_and_init()
tab = workflow.tabs.first()
wfm = tab.wf_modules.create(
order=0, last_relevant_delta_id=workflow.last_delta_id)
wfm.cache_render_result(workflow.last_delta_id, tab_output)
context = RenderContext(
workflow.id, None, {
tab.slug: StepResultShape('ok', tab_output.table_shape),
}, None)
schema = ParamDType.Dict({'tab': ParamDType.Tab()})
result = clean_value(schema, {'tab': tab.slug}, context)
self.assertEqual(result['tab'].slug, tab.slug)
self.assertEqual(result['tab'].name, tab.name)
self.assertEqual(result['tab'].columns, {
'A': RenderColumn('A', 'number'),
})
assert_frame_equal(result['tab'].dataframe, pd.DataFrame({'A': [1,
2]}))
def test_multichartseries_omit_missing_table_columns(self):
dtype = ParamDType.Multichartseries()
value = dtype.omit_missing_table_columns([
{
'column': 'X',
'color': '#abcdef'
},
{
'column': 'Y',
'color': '#abc123'
},
], {'X', 'Z'})
self.assertEqual(value, [{'column': 'X', 'color': '#abcdef'}])
def test_param_schema_implicit(self):
mv = ModuleVersion.create_or_replace_from_spec(
{
'id_name':
'x',
'name':
'x',
'category':
'Clean',
'parameters': [
{
'id_name': 'foo',
'type': 'string',
'default': 'X'
},
{
'id_name': 'bar',
'type': 'secret',
'name': 'Secret'
},
{
'id_name': 'baz',
'type': 'menu',
'menu_items': 'a|b|c',
'default': 2
},
]
},
source_version_hash='1.0')
self.assertEqual(
repr(mv.param_schema),
repr(
ParamDType.Dict({
'foo':
ParamDType.String(default='X'),
'baz':
ParamDType.Enum(choices={0, 1, 2}, default=2),
})))
def test_param_schema_explicit(self):
mv = ModuleVersion.create_or_replace_from_spec(
{
'id_name': 'x',
'name': 'x',
'category': 'Clean',
'parameters': [{
'id_name': 'whee',
'type': 'custom'
}],
'param_schema': {
'id_name': {
'type': 'dict',
'properties': {
'x': {
'type': 'integer'
},
'y': {
'type': 'string',
'default': 'X'
},
},
},
},
},
source_version_hash='1.0')
self.assertEqual(
repr(mv.param_schema),
repr(
ParamDType.Dict({
'id_name':
ParamDType.Dict({
'x': ParamDType.Integer(),
'y': ParamDType.String(default='X'),
}),
})))
def test_clean_tab_no_tab_selected_gives_none(self):
context = RenderContext(None, None, {}, None)
schema = ParamDType.Dict({'tab': ParamDType.Tab()})
result = clean_value(schema, {'tab': ''}, context)
self.assertEqual(result, {'tab': None})
def test_clean_tab_no_tab_output_raises_cycle(self):
context = RenderContext(None, None, {'tab-1': None}, None)
schema = ParamDType.Dict({'tab': ParamDType.Tab()})
with self.assertRaises(TabCycleError):
clean_value(schema, {'tab': 'tab-1'}, context)
def test_map_coerce_dict_wrong_value_type(self):
dtype = ParamDType.Map(value_dtype=ParamDType.String())
value = dtype.coerce({'a': 1, 'b': None})
self.assertEqual(value, {'a': '1', 'b': ''})
def test_map_coerce_non_dict(self):
dtype = ParamDType.Map(value_dtype=ParamDType.String())
value = dtype.coerce([1, 2, 3])
self.assertEqual(value, {})
def test_map_coerce_none(self):
dtype = ParamDType.Map(value_dtype=ParamDType.String())
value = dtype.coerce(None)
self.assertEqual(value, {})
def test_clean_tab_tab_error_raises_cycle(self):
shape = StepResultShape('error', TableShape(0, []))
context = RenderContext(None, None, {'tab-1': shape}, None)
schema = ParamDType.Dict({'tab': ParamDType.Tab()})
with self.assertRaises(TabOutputUnreachableError):
clean_value(schema, {'tab': 'tab-1'}, context)
def test_map_validate_bad_value_dtype(self):
dtype = ParamDType.Map(value_dtype=ParamDType.String())
value = {'a': 1, 'c': 2}
with self.assertRaises(ValueError):
dtype.validate(value)
def test_map_validate_ok(self):
dtype = ParamDType.Map(value_dtype=ParamDType.String())
value = {'a': 'b', 'c': 'd'}
dtype.validate(value)
def test_clean_tabs_nix_missing_tab(self):
context = RenderContext(None, None, {}, None)
schema = ParamDType.Dict({'tabs': ParamDType.Multitab()})
result = clean_value(schema, {'tabs': ['tab-missing']}, context)
self.assertEqual(result['tabs'], [])
def test_clean_tabs_tab_error_raises_cycle(self):
context = RenderContext(None, None, {'tab-1': None}, None)
schema = ParamDType.Dict({'tabs': ParamDType.Multitab()})
with self.assertRaises(TabCycleError):
clean_value(schema, {'tabs': ['tab-1']}, context)