def test_error_if_using_undeclared_variable():
notebook_w_warning = """
# + tags=['parameters']
a = 1
b = 2
# +
# variable "c" is used but never declared!
a + b + c
"""
source = NotebookSource(notebook_w_warning,
ext_in='py',
kernelspec_name='python3',
static_analysis=True)
params = Params._from_dict({
'product': File('output.ipynb'),
'a': 1,
'b': 2
})
with pytest.raises(RenderError) as excinfo:
source.render(params)
assert "undefined name 'c'" in str(excinfo.value)
def test_cannot_init_with_upstream_key():
with pytest.raises(ValueError) as excinfo:
Params({'upstream': None})
msg = ('Task params cannot be initialized with an '
'"upstream" key as it automatically added upon rendering')
assert str(excinfo.value) == msg
def test_ignores_static_analysis_if_non_python_file():
source = NotebookSource(new_nb(fmt='r:light'),
ext_in='R',
static_analysis=True)
params = Params._from_dict({'product': File('output.ipynb')})
source.render(params)
def test_error_if_static_analysis_on_a_non_python_nb():
source = NotebookSource(new_nb(fmt='r:light'),
ext_in='R',
static_analysis=True)
params = Params._from_dict({'product': File('output.ipynb')})
with pytest.raises(NotImplementedError):
source.render(params)
def test_script_source():
source = SQLScriptSource("""
{% set product = PostgresRelation(["schema", "name", "table"]) %}
CREATE TABLE {{product}} AS
SELECT * FROM some_table
""")
params = Params._from_dict({'product': 'this should be ignored'})
assert source.render(params)
def test_removes_papermill_metadata():
source = NotebookSource(new_nb(fmt='ipynb'),
ext_in='ipynb',
kernelspec_name='python3')
params = Params._from_dict({'product': File('file.ipynb')})
source.render(params)
nb = source.nb_obj_rendered
assert all('papermill' not in cell['metadata'] for cell in nb.cells)
def _unserialize_params(params_original, unserializer):
params = params_original.to_dict()
params['upstream'] = {
k: unserializer(product=v)
for k, v in params['upstream'].items()
}
params = Params._from_dict(params, copy=False)
return params
def test_str_ignores_injected_cell(tmp_directory):
path = Path('nb.py')
path.write_text(notebook_ab)
source = NotebookSource(path)
source.render(Params._from_dict(dict(a=42, product=File('file.txt'))))
source.save_injected_cell()
source = NotebookSource(path)
# injected cell should not be considered part of the source code
assert 'a = 42' not in str(source)
def test_error_if_missing_params():
source = NotebookSource(notebook_ab,
ext_in='py',
kernelspec_name='python3',
static_analysis=True)
params = Params._from_dict({'product': File('output.ipynb'), 'a': 1})
with pytest.raises(TypeError) as excinfo:
source.render(params)
assert "Missing params: 'b'" in str(excinfo.value)
def test_error_message_when_initialized_from_str(tmp_nbs, method, kwargs):
source = NotebookSource("""
# + tags=["parameters"]
""", ext_in='py')
source.render(Params._from_dict({'product': File('file.ipynb')}))
with pytest.raises(ValueError) as excinfo:
getattr(source, method)(**kwargs)
expected = (f"Cannot use '{method}' if notebook was not "
"initialized from a file")
assert str(excinfo.value) == expected
def test_static_analysis(hot_reload, tmp_directory):
nb = jupytext.reads(notebook_ab, fmt='py:light')
path = Path('nb.ipynb')
path.write_text(jupytext.writes(nb, fmt='ipynb'))
source = NotebookSource(path, static_analysis=True, hot_reload=hot_reload)
params = Params._from_dict({
'product': File('output.ipynb'),
'a': 1,
'b': 2
})
source.render(params)
def test_warn_if_using_default_value():
source = NotebookSource(notebook_ab,
ext_in='py',
kernelspec_name='python3',
static_analysis=True)
params = Params._from_dict({'product': File('output.ipynb'), 'a': 1})
with pytest.warns(UserWarning) as record:
source.render(params)
assert "Missing parameters: {'b'}, will use default value" in [
str(warning.message) for warning in record
]
def _unserialize_params(params_original, unserializer):
"""
User the user-provided function to unserialize params['upstream']
"""
params = params_original.to_dict()
params['upstream'] = {
k: _unserializer(v, unserializer)
for k, v in params['upstream'].items()
}
params = Params._from_dict(params, copy=False)
return params
def test_no_error_if_missing_product_or_upstream():
code = """
# + tags=["parameters"]
# +
"""
source = NotebookSource(code,
ext_in='py',
kernelspec_name='python3',
static_analysis=True)
params = Params._from_dict({'product': File('output.ipynb')})
source.render(params)
def test_error_if_passing_undeclared_parameter():
source = NotebookSource(notebook_ab,
ext_in='py',
kernelspec_name='python3',
static_analysis=True)
params = Params._from_dict({
'product': File('output.ipynb'),
'a': 1,
'b': 2,
'c': 3
})
with pytest.raises(TypeError) as excinfo:
source.render(params)
assert "Unexpected params: 'c'" in str(excinfo.value)
def test_error_if_undefined_name():
notebook_w_error = """
# + tags=['parameters']
# +
df.head()
"""
source = NotebookSource(notebook_w_error,
ext_in='py',
kernelspec_name='python3',
static_analysis=True)
params = Params._from_dict({'product': File('output.ipynb')})
with pytest.raises(RenderError) as excinfo:
source.render(params)
assert "undefined name 'df'" in str(excinfo.value)
def test_injects_parameters_on_render(nb_str, ext):
s = NotebookSource(nb_str, ext_in=ext)
params = Params._from_dict({
'some_param': 1,
'product': File('output.ipynb')
})
s.render(params)
nb = nbformat.reads(s.nb_str_rendered, as_version=nbformat.NO_CONVERT)
# cell 0: parameters
# cell 1: injected-parameters
injected = nb.cells[1]
tags = injected['metadata']['tags']
assert len(tags) == 1
assert tags[0] == 'injected-parameters'
# py 3.5 does not gurantee order, so we check them separately
assert 'some_param = 1' in injected['source']
assert 'product = "output.ipynb"' in injected['source']
def test_error_if_syntax_error():
notebook_w_error = """
# + tags=['parameters']
a = 1
b = 2
# +
if
"""
source = NotebookSource(notebook_w_error,
ext_in='py',
kernelspec_name='python3',
static_analysis=True)
params = Params._from_dict({
'product': File('output.ipynb'),
'a': 1,
'b': 2
})
with pytest.raises(SyntaxError) as excinfo:
source.render(params)
assert 'invalid syntax\n\nif\n\n ^\n' in str(excinfo.value)
def test_inject_cell(tmp_directory):
nb = jupytext.reads('', fmt=None)
model = {'content': nb, 'name': 'script.py'}
inject_cell(model,
params=Params._from_dict({'product': File('output.ipynb')}))
def test_init_from_dict(copy, expected):
d = {'upstream': None, 'product': None}
params = Params._from_dict(d, copy=copy)
assert (params._dict is d) is expected
def test_str():
source = NotebookSource(notebook_ab, ext_in='py')
source.render(Params._from_dict({'product':
File('path/to/file/data.csv')}))
assert str(source) == ('\na = 1\nb = 2\nproduct = None\na + b')
def test_cannot_modify_param():
p = Params({'a': 1})
with pytest.raises(RuntimeError):
p['a'] = 1
def test_error_if_initialized_with_non_mapping(value):
with pytest.raises(TypeError):
Params(value)
def test_get_param():
p = Params({'a': 1})
assert p['a'] == 1
def test_params_only(params, expected):
p = Params._from_dict(params)
assert p.to_json_serializable(params_only=True) == expected
def __init__(self, product, dag, name=None, params=None):
self._params = Params(params)
if name is None:
# use name inferred from the source object
self._name = self._source.name
if self._name is None:
raise AttributeError('Task name can only be None if it '
'can be inferred from the source object. '
'For example, when the task receives a '
'pathlib.Path, when using SourceLoader '
'or in PythonCallable. Pass a value '
'explicitly')
else:
self._name = name
if not isinstance(dag, AbstractDAG):
raise TypeError(
f"'dag' must be an instance of DAG, got {type(dag)!r}")
# NOTE: we should get rid of this, maybe just add hooks that are
# called back on the dag object to avoid having a reference here
self.dag = dag
dag._add_task(self)
if not hasattr(self, '_source'):
raise RuntimeError(
'self._source must be initialized before calling '
'__init__ in Task')
if self._source is None:
raise TypeError(
'_init_source must return a source object, got None')
if isinstance(product, Product):
self._product = product
if self.PRODUCT_CLASSES_ALLOWED is not None:
if not isinstance(self._product, self.PRODUCT_CLASSES_ALLOWED):
raise TypeError('{} only supports the following product '
'classes: {}, got {}'.format(
type(self).__name__,
self.PRODUCT_CLASSES_ALLOWED,
type(self._product).__name__))
else:
# if assigned a tuple/list of products, create a MetaProduct
self._product = MetaProduct(product)
if self.PRODUCT_CLASSES_ALLOWED is not None:
if not all(
isinstance(p, self.PRODUCT_CLASSES_ALLOWED)
for p in self._product):
raise TypeError('{} only supports the following product '
'classes: {}, got {}'.format(
type(self).__name__,
self.PRODUCT_CLASSES_ALLOWED,
type(self._product).__name__))
self._logger = logging.getLogger('{}.{}'.format(
__name__,
type(self).__name__))
self.product.task = self
self._client = None
self.exec_status = TaskStatus.WaitingRender
self._on_finish = None
self._on_failure = None
self._on_render = None
def test_set_item():
params = Params._from_dict({'a': 1})
params._setitem('a', 2)
assert params['a'] == 2
