def jupyter_run(config,
notebook=None,
prepare_only=False,
param=None,
param_raw=None):
"""
Executes the step in browserless mode using papermill
Parameters
----------
config : str
path to the configuration file
notebook : str
name of node to run, if None, then run the first one
parameters : list
List of parameters and their values
prepare_only : bool
Indicates if papermill should replace the parameters of the
notebook only or it should execute all the cells too
Returns
-------
string
the path to the output notebook resulting from the execution
of this step
"""
# TODO implements parameters passing
if notebook:
app_config = AiscalatorConfig(config=config, step_selection=notebook)
else:
app_config = AiscalatorConfig(config=config)
return command.jupyter_run(app_config,
prepare_only=prepare_only,
param=param,
param_raw=param_raw)
def push(conf, notebook):
"""Push a job into the DAGS folder to schedule in Airflow."""
if notebook:
for note in notebook:
app_config = AiscalatorConfig(config=conf,
dag_selection=note)
click.echo(command.airflow_push(app_config))
else:
app_config = AiscalatorConfig(config=conf)
click.echo(command.airflow_push(app_config))
def run(conf, notebook, param, param_raw):
"""Run the notebook from an aiscalate config without GUI."""
if notebook:
for note in notebook:
app_config = AiscalatorConfig(config=conf, step_selection=note)
click.echo(
command.jupyter_run(app_config,
param=param,
param_raw=param_raw))
else:
app_config = AiscalatorConfig(config=conf)
click.echo(
command.jupyter_run(app_config, param=param, param_raw=param_raw))
def start():
"""Start docker images to bring airflow services up."""
click.echo(command.airflow_up(AiscalatorConfig()))
click.echo("""
Airflow: http://localhost:8080
Flower: http://localhost:5555
""")
def edit(conf, notebook):
"""Edit DAG job"""
if len(notebook) < 2:
notebook = notebook[0] if notebook else None
app_config = AiscalatorConfig(config=conf,
dag_selection=notebook)
click.echo(command.airflow_edit(app_config))
else:
raise click.BadArgumentUsage("Expecting one or less notebook names")
def edit(conf, notebook, param, param_raw):
"""Edit the notebook from an aiscalate config with JupyterLab."""
if len(notebook) < 2:
notebook = notebook[0] if notebook else None
app_config = AiscalatorConfig(config=conf, step_selection=notebook)
click.echo(
command.jupyter_edit(app_config, param=param, param_raw=param_raw))
else:
raise click.BadArgumentUsage("Expecting one or less notebook names")
def prompt_edit(file):
"""
When creating a new step, if it is already defined,
ask to edit instead
Parameters
----------
file : str
existing configuration file
"""
msg = file + ' already exists. Did you mean to run:\n'
for i in sys.argv:
if i != "new":
msg += i + ' '
else:
break
msg += "edit " + file + " instead?"
if click.confirm(msg, abort=True):
conf = AiscalatorConfig(config=file)
click.echo(command.jupyter_edit(conf))
def jupyter_new(name, path, output_format="hocon"):
"""
Starts a Jupyter Lab environment configured to edit a brand new step
Parameters
----------
name : str
name of the new step
path : str
path to where the new step files should be created
output_format : str
the format of the new configuration file to produce
Returns
-------
string
Url of the running jupyter lab
"""
step_file = os.path.join(path, name, name) + '.conf'
if os.path.dirname(step_file):
makedirs(os.path.dirname(step_file), exist_ok=True)
copy_replace(data_file("../config/template/step.conf"),
step_file,
pattern="Untitled",
replace_value=name)
if output_format != 'hocon':
file = os.path.join(path, name, name) + '.' + output_format
step_file = convert_to_format(step_file,
output=file,
output_format=output_format)
notebook = os.path.join(path, name, 'notebook', name) + '.ipynb'
if os.path.dirname(notebook):
makedirs(os.path.dirname(notebook), exist_ok=True)
copy_replace(data_file("../config/template/notebook.json"), notebook)
open(os.path.join(path, name, "apt_repository.txt"), 'a').close()
open(os.path.join(path, name, "apt_packages.txt"), 'a').close()
open(os.path.join(path, name, "requirements.txt"), 'a').close()
open(os.path.join(path, name, "lab_extensions.txt"), 'a').close()
jupyter_edit(AiscalatorConfig(config=step_file, step_selection=name))
def test_prepare_docker_image_env_extra_options():
"""Test the _prepare_docker_image_env."""
options_list = command._prepare_docker_image_env(
AiscalatorConfig("tests/jupyter/sample_pyhocon.conf"))
'bridge' == options_list[-1]
'--network' == options_list[-2]
def run(service, subcommand):
"""Run sub-command in a running docker service."""
if not subcommand:
subcommand = None
click.echo(command.airflow_cmd(AiscalatorConfig(),
service=service, cmd=subcommand))
def stop():
"""Stop docker images to bring airflow services down."""
click.echo(command.airflow_down(AiscalatorConfig()))
def setup(config_home, append, workspace):
"""Setup interactively the Airflow home folder and configurations."""
click.echo(command.airflow_setup(AiscalatorConfig(),
config_home, workspace,
append=append))
