def save(self, model, path=""):
"""
This is called when a file is saved
"""
if self.manager and path in self.manager:
out = self.manager.overwrite(model, path)
return out
else:
check_metadata_filter(self.log, model)
# not sure what's the difference between model['path'] and path
# but path has leading "/", _model_in_dag strips it
key = self._model_in_dag(model, path)
if key:
content = model['content']
metadata = content.get('metadata', {}).get('ploomber', {})
if not metadata.get('injected_manually'):
self.log.info(
'[Ploomber] Cleaning up injected cell in {}...'.
format(model.get('name') or ''))
model['content'] = _cleanup_rendered_nb(content)
self.log.info("[Ploomber] Deleting product's metadata...")
self.dag_mapping.delete_metadata(key)
return super().save(model, path)
def test_cleanup_rendered_nb(nb, expected_n, expected_source):
out = _cleanup_rendered_nb(jupytext.reads(nb))
assert len(out['cells']) == expected_n
assert [c['source'] for c in out['cells']] == expected_source
def develop(self, app='notebook', args=None):
"""
Opens the rendered notebook (with injected parameters) and adds a
"debugging-settings" cell to the that changes directory to the current
active directory. This will reflect conditions when callign
`DAG.build()`. This modified notebook is saved in the same location as
the source with a "-tmp" added to the filename. Changes to this
notebook can be exported to the original notebook after the notebook
process is shut down. The "injected-parameters" and
"debugging-settings" cells are deleted before saving.
Parameters
----------
app : {'notebook', 'lab'}, default: 'notebook'
Which Jupyter application to use
args : str
Extra parameters passed to the jupyter application
Notes
-----
Be careful when developing tasks interacively. If the task has run
successfully, you overwrite products but don't save the
updated source code, your DAG will enter an inconsistent state where
the metadata won't match the overwritten product.
If you modify the source code and call develop again, the source
code will be updated only if the ``hot_reload option`` is turned on.
See :class:`ploomber.DAGConfigurator` for details.
"""
# TODO: this code needs refactoring, should be a context manager
# like the one we have for PythonCallable.develop that abstracts
# the handling of the temporary notebook while editing
apps = {'notebook', 'lab'}
if app not in apps:
raise ValueError('"app" must be one of {}, got: "{}"'.format(
apps, app))
if self.source.language != 'python':
raise NotImplementedError(
'develop is not implemented for "{}" '
'notebooks, only python is supported'.format(
self.source.language))
if self.source.loc is None:
raise ValueError('Can only use develop in notebooks loaded '
'from files, not from str')
nb = _read_rendered_notebook(self.source.nb_str_rendered)
name = self.source.loc.name
suffix = self.source.loc.suffix
name_new = name.replace(suffix, '-tmp.ipynb')
tmp = self.source.loc.with_name(name_new)
content = nbformat.writes(nb, version=nbformat.NO_CONVERT)
tmp.write_text(content)
# open notebook with injected debugging cell
try:
subprocess.run(['jupyter', app, str(tmp)] +
shlex.split(args or ''),
check=True)
except KeyboardInterrupt:
print(f'Jupyter {app} application closed...')
# read tmp file again, to see if the user made any changes
content_new = Path(tmp).read_text()
# maybe exclude changes in tmp cells?
if content == content_new:
print('No changes found...')
else:
# save changes
if _save():
nb = nbformat.reads(content_new,
as_version=nbformat.NO_CONVERT)
# remove injected-parameters and debugging-settings cells if
# they exist
_cleanup_rendered_nb(nb)
# write back in the same format and original location
ext_source = Path(self.source.loc).suffix[1:]
print('Saving notebook to: ', self.source.loc)
jupytext.write(nb, self.source.loc, fmt=ext_source)
else:
print('Not saving changes...')
# remove tmp file
Path(tmp).unlink()