def test_error_if_using_undeclared_variable(): notebook_w_warning = """ # + tags=['parameters'] a = 1 b = 2 # + # variable "c" is used but never declared! a + b + c """ source = NotebookSource(notebook_w_warning, ext_in='py', kernelspec_name='python3', static_analysis=True) params = Params._from_dict({ 'product': File('output.ipynb'), 'a': 1, 'b': 2 }) with pytest.raises(RenderError) as excinfo: source.render(params) assert "undefined name 'c'" in str(excinfo.value)
def test_cannot_init_with_upstream_key(): with pytest.raises(ValueError) as excinfo: Params({'upstream': None}) msg = ('Task params cannot be initialized with an ' '"upstream" key as it automatically added upon rendering') assert str(excinfo.value) == msg
def test_ignores_static_analysis_if_non_python_file(): source = NotebookSource(new_nb(fmt='r:light'), ext_in='R', static_analysis=True) params = Params._from_dict({'product': File('output.ipynb')}) source.render(params)
def test_error_if_static_analysis_on_a_non_python_nb(): source = NotebookSource(new_nb(fmt='r:light'), ext_in='R', static_analysis=True) params = Params._from_dict({'product': File('output.ipynb')}) with pytest.raises(NotImplementedError): source.render(params)
def test_script_source(): source = SQLScriptSource(""" {% set product = PostgresRelation(["schema", "name", "table"]) %} CREATE TABLE {{product}} AS SELECT * FROM some_table """) params = Params._from_dict({'product': 'this should be ignored'}) assert source.render(params)
def test_removes_papermill_metadata(): source = NotebookSource(new_nb(fmt='ipynb'), ext_in='ipynb', kernelspec_name='python3') params = Params._from_dict({'product': File('file.ipynb')}) source.render(params) nb = source.nb_obj_rendered assert all('papermill' not in cell['metadata'] for cell in nb.cells)
def _unserialize_params(params_original, unserializer): params = params_original.to_dict() params['upstream'] = { k: unserializer(product=v) for k, v in params['upstream'].items() } params = Params._from_dict(params, copy=False) return params
def test_str_ignores_injected_cell(tmp_directory): path = Path('nb.py') path.write_text(notebook_ab) source = NotebookSource(path) source.render(Params._from_dict(dict(a=42, product=File('file.txt')))) source.save_injected_cell() source = NotebookSource(path) # injected cell should not be considered part of the source code assert 'a = 42' not in str(source)
def test_error_if_missing_params(): source = NotebookSource(notebook_ab, ext_in='py', kernelspec_name='python3', static_analysis=True) params = Params._from_dict({'product': File('output.ipynb'), 'a': 1}) with pytest.raises(TypeError) as excinfo: source.render(params) assert "Missing params: 'b'" in str(excinfo.value)
def test_error_message_when_initialized_from_str(tmp_nbs, method, kwargs): source = NotebookSource(""" # + tags=["parameters"] """, ext_in='py') source.render(Params._from_dict({'product': File('file.ipynb')})) with pytest.raises(ValueError) as excinfo: getattr(source, method)(**kwargs) expected = (f"Cannot use '{method}' if notebook was not " "initialized from a file") assert str(excinfo.value) == expected
def test_static_analysis(hot_reload, tmp_directory): nb = jupytext.reads(notebook_ab, fmt='py:light') path = Path('nb.ipynb') path.write_text(jupytext.writes(nb, fmt='ipynb')) source = NotebookSource(path, static_analysis=True, hot_reload=hot_reload) params = Params._from_dict({ 'product': File('output.ipynb'), 'a': 1, 'b': 2 }) source.render(params)
def test_warn_if_using_default_value(): source = NotebookSource(notebook_ab, ext_in='py', kernelspec_name='python3', static_analysis=True) params = Params._from_dict({'product': File('output.ipynb'), 'a': 1}) with pytest.warns(UserWarning) as record: source.render(params) assert "Missing parameters: {'b'}, will use default value" in [ str(warning.message) for warning in record ]
def _unserialize_params(params_original, unserializer): """ User the user-provided function to unserialize params['upstream'] """ params = params_original.to_dict() params['upstream'] = { k: _unserializer(v, unserializer) for k, v in params['upstream'].items() } params = Params._from_dict(params, copy=False) return params
def test_no_error_if_missing_product_or_upstream(): code = """ # + tags=["parameters"] # + """ source = NotebookSource(code, ext_in='py', kernelspec_name='python3', static_analysis=True) params = Params._from_dict({'product': File('output.ipynb')}) source.render(params)
def test_error_if_passing_undeclared_parameter(): source = NotebookSource(notebook_ab, ext_in='py', kernelspec_name='python3', static_analysis=True) params = Params._from_dict({ 'product': File('output.ipynb'), 'a': 1, 'b': 2, 'c': 3 }) with pytest.raises(TypeError) as excinfo: source.render(params) assert "Unexpected params: 'c'" in str(excinfo.value)
def test_error_if_undefined_name(): notebook_w_error = """ # + tags=['parameters'] # + df.head() """ source = NotebookSource(notebook_w_error, ext_in='py', kernelspec_name='python3', static_analysis=True) params = Params._from_dict({'product': File('output.ipynb')}) with pytest.raises(RenderError) as excinfo: source.render(params) assert "undefined name 'df'" in str(excinfo.value)
def test_injects_parameters_on_render(nb_str, ext): s = NotebookSource(nb_str, ext_in=ext) params = Params._from_dict({ 'some_param': 1, 'product': File('output.ipynb') }) s.render(params) nb = nbformat.reads(s.nb_str_rendered, as_version=nbformat.NO_CONVERT) # cell 0: parameters # cell 1: injected-parameters injected = nb.cells[1] tags = injected['metadata']['tags'] assert len(tags) == 1 assert tags[0] == 'injected-parameters' # py 3.5 does not gurantee order, so we check them separately assert 'some_param = 1' in injected['source'] assert 'product = "output.ipynb"' in injected['source']
def test_error_if_syntax_error(): notebook_w_error = """ # + tags=['parameters'] a = 1 b = 2 # + if """ source = NotebookSource(notebook_w_error, ext_in='py', kernelspec_name='python3', static_analysis=True) params = Params._from_dict({ 'product': File('output.ipynb'), 'a': 1, 'b': 2 }) with pytest.raises(SyntaxError) as excinfo: source.render(params) assert 'invalid syntax\n\nif\n\n ^\n' in str(excinfo.value)
def test_inject_cell(tmp_directory): nb = jupytext.reads('', fmt=None) model = {'content': nb, 'name': 'script.py'} inject_cell(model, params=Params._from_dict({'product': File('output.ipynb')}))
def test_init_from_dict(copy, expected): d = {'upstream': None, 'product': None} params = Params._from_dict(d, copy=copy) assert (params._dict is d) is expected
def test_str(): source = NotebookSource(notebook_ab, ext_in='py') source.render(Params._from_dict({'product': File('path/to/file/data.csv')})) assert str(source) == ('\na = 1\nb = 2\nproduct = None\na + b')
def test_cannot_modify_param(): p = Params({'a': 1}) with pytest.raises(RuntimeError): p['a'] = 1
def test_error_if_initialized_with_non_mapping(value): with pytest.raises(TypeError): Params(value)
def test_get_param(): p = Params({'a': 1}) assert p['a'] == 1
def test_params_only(params, expected): p = Params._from_dict(params) assert p.to_json_serializable(params_only=True) == expected
def __init__(self, product, dag, name=None, params=None): self._params = Params(params) if name is None: # use name inferred from the source object self._name = self._source.name if self._name is None: raise AttributeError('Task name can only be None if it ' 'can be inferred from the source object. ' 'For example, when the task receives a ' 'pathlib.Path, when using SourceLoader ' 'or in PythonCallable. Pass a value ' 'explicitly') else: self._name = name if not isinstance(dag, AbstractDAG): raise TypeError( f"'dag' must be an instance of DAG, got {type(dag)!r}") # NOTE: we should get rid of this, maybe just add hooks that are # called back on the dag object to avoid having a reference here self.dag = dag dag._add_task(self) if not hasattr(self, '_source'): raise RuntimeError( 'self._source must be initialized before calling ' '__init__ in Task') if self._source is None: raise TypeError( '_init_source must return a source object, got None') if isinstance(product, Product): self._product = product if self.PRODUCT_CLASSES_ALLOWED is not None: if not isinstance(self._product, self.PRODUCT_CLASSES_ALLOWED): raise TypeError('{} only supports the following product ' 'classes: {}, got {}'.format( type(self).__name__, self.PRODUCT_CLASSES_ALLOWED, type(self._product).__name__)) else: # if assigned a tuple/list of products, create a MetaProduct self._product = MetaProduct(product) if self.PRODUCT_CLASSES_ALLOWED is not None: if not all( isinstance(p, self.PRODUCT_CLASSES_ALLOWED) for p in self._product): raise TypeError('{} only supports the following product ' 'classes: {}, got {}'.format( type(self).__name__, self.PRODUCT_CLASSES_ALLOWED, type(self._product).__name__)) self._logger = logging.getLogger('{}.{}'.format( __name__, type(self).__name__)) self.product.task = self self._client = None self.exec_status = TaskStatus.WaitingRender self._on_finish = None self._on_failure = None self._on_render = None
def test_set_item(): params = Params._from_dict({'a': 1}) params._setitem('a', 2) assert params['a'] == 2