def papermill(notebook_path, output_path, parameters, parameters_raw,
parameters_file, parameters_yaml, parameters_base64, kernel):
"""Utility for executing a single notebook on a container.
Take a source notebook, apply parameters to the source notebook,
execute the notebook with the kernel specified, and save the
output in the destination notebook.
"""
if parameters_base64:
parameters_final = yaml.load(base64.b64decode(parameters_base64))
elif parameters_yaml:
parameters_final = yaml.load(parameters_yaml)
elif parameters_file:
parameters_final = read_yaml_file(parameters_file)
else:
# Read in Parameters
parameters_final = {}
for name, value in parameters:
parameters_final[name] = _resolve_type(value)
for name, value in parameters_raw:
parameters_final[name] = value
execute_notebook(notebook_path,
output_path,
parameters_final,
kernel_name=kernel)
def _parameterize_notebook(nb, kernel_name, parameters):
# Load from a file if 'parameters' is a string.
if isinstance(parameters, string_types):
parameters = read_yaml_file(parameters)
# Generate parameter content based on the kernal_name
kernel_name = kernel_name or nb.metadata.kernelspec.name
param_content = _build_parameter_code(kernel_name, parameters)
# Remove the old cell and replace it with a new one containing parameter content.
param_cell_index = _find_parameters_index(nb)
old_parameters = nb.cells[param_cell_index]
before = nb.cells[:param_cell_index]
after = nb.cells[param_cell_index + 1:]
newcell = nbformat.v4.new_code_cell(source=param_content)
newcell.metadata['tags'] = old_parameters.metadata.tags
nb.cells = before + [newcell] + after
def papermill(notebook, output, parameters, raw_parameters, parameters_file,
parameters_yaml, parameters_base64, kernel):
"""Utility for executing a single notebook on a container."""
if parameters_base64:
parameters_final = yaml.load(base64.b64decode(parameters_base64))
elif parameters_yaml:
parameters_final = yaml.load(parameters_yaml)
elif parameters_file:
parameters_final = read_yaml_file(parameters_file)
else:
# Read in Parameters
parameters_final = {}
for name, value in parameters:
parameters_final[name] = _resolve_type(value)
for name, value in raw_parameters:
parameters_final[name] = value
# Notebook Execution
execute_notebook(notebook, output, parameters_final, kernel_name=kernel)
def generate_notebooks_from_yml(input_nb_path: str, yml_parameters_path: str):
"""
Generate a set of notebooks using the Papermill APIs. The input notebook
must contain a cell with a `parameters` tag. Papermill will generate a set of
notebooks based on the parameters defined in the input yaml
Args:
input_nb_path: string
Path to the source template notebook
yml_parameters_path: string
Path to the yaml spec with parameters
Returns: list
A list of paths to the generated notebooks
"""
y = read_yaml_file(yml_parameters_path)
input_nb = load_notebook_node(input_nb_path)
# Create the cartesian product of the parameters
hp_values = list(product(*y.values()))
# Now recreate a dictionary with the correct keys
hp_dicts = [dict(zip(y.keys(), x)) for x in hp_values]
# For each combination of parameters generate a notebook from the template
output_paths = list()
for params in hp_dicts:
params_str = print_dict_parametes(params)
output_path = input_nb_path.replace(".ipynb",
"") + params_str + ".ipynb"
output_nb = parameterize_notebook(input_nb, parameters=params)
# write the nb to file
write_ipynb(output_nb, path=output_path)
output_paths.append((output_path, params_str))
# Return list of generated notebook paths
return output_paths