def cancel():
"""Cancels the current notebook build and schedule"""
validate_notebook_directory(treebeard_env, treebeard_config)
spinner: Any = Halo(text="cancelling", spinner="dots")
click.echo(f"🌲 Cancelling {notebook_id}")
spinner.start()
requests.delete(notebooks_endpoint, headers=treebeard_env.dict())
spinner.stop()
click.echo(f"🛑 Done!")
def cancel():
"""Cancels the current notebook build and schedule"""
validate_notebook_directory(treebeard_env, treebeard_config)
notif = f"🌲 Cancelling {notebook_id}"
spinner: Any = Halo(text=notif, spinner="dots")
spinner.start()
requests.delete(runner_endpoint,
headers=treebeard_env.dict()) # type: ignore
spinner.stop()
click.echo(f"{notif}...🛑 cancellation confirmed!")
def push(files: List[IO]):
"""Uploads files marked in treebeard.yaml as 'secret'"""
click.echo(f"🌲 Pushing Secrets for project {treebeard_env.project_id}")
validate_notebook_directory(treebeard_env, treebeard_config)
secrets_archive = get_secrets_archive(files)
response = requests.post(
secrets_endpoint,
files={"secrets": open(secrets_archive.name, "rb")},
headers=treebeard_env.dict(),
)
if response.status_code != 200:
click.echo(
f"Error: service failure pushing secrets, {response.status_code}: {response.text}"
)
return
click.echo("🔐 done!")
def status():
"""Show the status of the current notebook"""
validate_notebook_directory(treebeard_env, treebeard_config)
response = requests.get(runner_endpoint,
headers=treebeard_env.dict()) # type: ignore
if response.status_code != 200:
raise click.ClickException(f"Request failed: {response.text}")
json_data = json.loads(response.text)
if len(json_data) == 0:
fatal_exit(
"This notebook has not been run. Try running it with `treebeard run`"
)
click.echo("🌲 Recent runs:\n")
max_results = 5
status_emoji = {
"SUCCESS": "✅",
"QUEUED": "💤",
"WORKING": "⏳",
"FAILURE": "❌",
"TIMEOUT": "⏰",
"CANCELLED": "🛑",
}
runs: List[Run] = [
Run.parse_obj(run) for run in json_data["runs"][-max_results:]
] # type: ignore
for run in runs:
now = parser.isoparse(datetime.datetime.utcnow().isoformat() + "Z")
start_time = parser.isoparse(run.start_time)
time_string: str = timeago_format(start_time, now=now)
mechanism: str = run.trigger["mechanism"]
ran_via = "" if len(mechanism) == 0 else f"via {mechanism}"
try:
branch = f"🔀{run.trigger['branch']}"
except:
branch = ""
click.echo(
f" {status_emoji[run.status]} {time_string} {ran_via} {branch} -- {run.url}"
)
def push_secrets(files: List[IO[Any]], confirm: bool):
"""Uploads files marked in treebeard.yaml as 'secret'"""
validate_notebook_directory(treebeard_env, treebeard_config)
click.echo(
f"🌲 Pushing secrets for {treebeard_env.project_id}/{treebeard_env.notebook_id}"
)
secrets_archive = get_secrets_archive(files, confirm=confirm)
response = requests.post( # type: ignore
secrets_endpoint,
files={"secrets": open(secrets_archive.name, "rb")},
headers=treebeard_env.dict(),
)
if response.status_code != 200:
click.echo(
f"Error: service failure pushing secrets, {response.status_code}: {response.text}"
)
return
click.echo("🔐 secrets pushed\n")
def run(
cli_context: CliContext,
watch: bool,
notebooks: List[str],
ignore: List[str],
local: bool,
confirm: bool,
push_secrets: bool,
dockerless: bool,
upload: bool,
):
"""
Run a notebook and optionally schedule it to run periodically
"""
notebooks = list(notebooks)
ignore = list(ignore)
validate_notebook_directory(treebeard_env, treebeard_config)
# Apply cli config overrides
treebeard_yaml_path: str = tempfile.mktemp() # type: ignore
with open(treebeard_yaml_path, "w") as yaml_file:
if notebooks:
treebeard_config.notebooks = notebooks
yaml.dump(treebeard_config.dict(), yaml_file) # type: ignore
if dockerless:
click.echo(
f"🌲 Running locally without docker using your current python environment"
)
if not confirm and not click.confirm(
f"Warning: This will clear the outputs of your notebooks, continue?",
default=True,
):
sys.exit(0)
# Note: import runtime.run causes win/darwin devices missing magic to fail at start
import treebeard.runtime.run
treebeard.runtime.run.start(upload_outputs=upload) # will sys.exit
params = {}
if treebeard_config.schedule:
if confirm or click.confirm(
f"📅 treebeard.yaml contains schedule '{treebeard_config.schedule}'. Enable it?"
):
params["schedule"] = treebeard_config.schedule
if (not local and len(treebeard_config.secret) > 0 and not confirm
and not push_secrets):
push_secrets = click.confirm("Push secrets first?", default=True)
if push_secrets:
push_secrets_to_store([], confirm=confirm)
if treebeard_config:
ignore += (treebeard_config.ignore + treebeard_config.secret +
treebeard_config.output_dirs)
click.echo("🌲 Copying project to tempdir and stripping notebooks")
temp_dir = tempfile.mkdtemp()
copy_tree(os.getcwd(), str(temp_dir), preserve_symlinks=1)
notebooks_files = treebeard_config.get_deglobbed_notebooks()
for notebooks_file in notebooks_files:
try:
subprocess.check_output(["nbstripout"] + notebooks_file,
cwd=temp_dir)
except:
print(f"Failed to nbstripout {notebooks_file}! Is it valid?")
click.echo(notebooks_files)
click.echo("🌲 Compressing Repo")
with tempfile.NamedTemporaryFile("wb", suffix=".tar.gz",
delete=False) as src_archive:
with tarfile.open(fileobj=src_archive, mode="w:gz") as tar:
def zip_filter(info: tarfile.TarInfo):
if info.name.endswith("treebeard.yaml"):
return None
for ignored in ignore:
if info.name in glob.glob(ignored, recursive=True):
return None
# if len(git_files) > 0 and info.name not in git_files:
# return None
click.echo(f" Including {info.name}")
return info
tar.add(
str(temp_dir),
arcname=os.path.basename(os.path.sep),
filter=zip_filter,
)
tar.add(config_path, arcname=os.path.basename(config_path))
tar.add(treebeard_yaml_path, arcname="treebeard.yaml")
if not confirm and not click.confirm("Confirm source file set is correct?",
default=True):
click.echo("Exiting")
sys.exit()
if local:
build_tag = str(time.mktime(datetime.datetime.today().timetuple()))
repo_image_name = f"gcr.io/treebeard-259315/projects/{project_id}/{sanitise_notebook_id(str(notebook_id))}:{build_tag}"
click.echo(f"🌲 Building {repo_image_name} Locally\n")
secrets_archive = get_secrets_archive()
repo_url = f"file://{src_archive.name}"
secrets_url = f"file://{secrets_archive.name}"
status = run_repo(
str(project_id),
str(notebook_id),
treebeard_env.run_id,
build_tag,
repo_url,
secrets_url,
branch="cli",
local=True,
)
click.echo(f"Local build exited with status code {status}")
sys.exit(status)
size = os.path.getsize(src_archive.name)
max_upload_size = "100MB"
if size > parse_size(max_upload_size):
fatal_exit(
click.style(
(f"ERROR: Compressed notebook directory is {format_size(size)},"
f" max upload size is {max_upload_size}. \nPlease ensure you ignore any virtualenv subdirectory"
" using `treebeard run --ignore venv`"),
fg="red",
))
time_seconds = int(time.mktime(datetime.datetime.today().timetuple()))
build_tag = str(time_seconds)
upload_api = f"{api_url}/source_upload_url/{project_id}/{notebook_id}/{build_tag}"
resp = requests.get(upload_api) # type: ignore
signed_url: str = resp.text
put_resp = requests.put( # type: ignore
signed_url,
open(src_archive.name, "rb"),
headers={"Content-Type": "application/x-tar"},
)
assert put_resp.status_code == 200
if os.getenv("GITHUB_ACTIONS"):
params["event"] = os.getenv("GITHUB_EVENT_NAME")
params["sha"] = os.getenv("GITHUB_SHA")
params["branch"] = os.getenv("GITHUB_REF").split("/")[-1]
workflow = os.getenv("GITHUB_WORKFLOW")
params["workflow"] = (workflow.replace(".yml",
"").replace(".yaml",
"").split("/")[-1])
click.echo(f"🌲 submitting archive to runner ({format_size(size)})...")
submit_endpoint = f"{api_url}/runs/{treebeard_env.project_id}/{treebeard_env.notebook_id}/{build_tag}"
response = requests.post( # type: ignore
submit_endpoint,
params=params,
headers={
"api_key": treebeard_env.api_key,
"email": treebeard_env.email
},
)
shutil.rmtree(temp_dir)
if response.status_code != 200:
raise click.ClickException(f"Request failed: {response.text}")
try:
json_data = json.loads(response.text)
click.echo(f"✨ Run has been accepted! {json_data['admin_url']}")
except:
click.echo("❗ Request to run failed")
click.echo(sys.exc_info())
if watch:
build_result = None
while not build_result:
time.sleep(5)
response = requests.get(
runner_endpoint, headers=treebeard_env.dict()) # type: ignore
json_data = json.loads(response.text)
if len(json_data["runs"]) == 0:
status = "FAILURE"
else:
status = json_data["runs"][-1]["status"]
click.echo(f"{get_time()} Build status: {status}")
if status == "SUCCESS":
build_result = status
click.echo(f"Build result: {build_result}")
elif status in [
"FAILURE", "TIMEOUT", "INTERNAL_ERROR", "CANCELLED"
]:
fatal_exit(f"Build failed")
def run(cli_context: CliContext, t: str, watch: bool, ignore: List[str],
local: bool):
"""
Run a notebook and optionally schedule it to run periodically
"""
validate_notebook_directory(treebeard_env, treebeard_config)
params = {}
if t:
params["schedule"] = t
spinner: Any = Halo(text="🌲 Compressing Repo\n", spinner="dots")
spinner.start()
if treebeard_config:
ignore += (treebeard_config.ignore + treebeard_config.secret +
treebeard_config.output_dirs)
# Create a temporary file for the compressed directory
# compressed file accessible at f.name
# git_files: Set[str] = set(
# subprocess.check_output(
# "git ls-files || exit 0", shell=True, stderr=subprocess.DEVNULL
# )
# .decode()
# .splitlines()
# )
with tempfile.NamedTemporaryFile("wb", suffix=".tar.gz",
delete=False) as src_archive:
click.echo("\n")
with tarfile.open(fileobj=src_archive, mode="w:gz") as tar:
def zip_filter(info: tarfile.TarInfo):
for ignored in ignore:
if info.name in glob.glob(ignored):
return None
# if len(git_files) > 0 and info.name not in git_files:
# return None
click.echo(f" Including {info.name}")
return info
tar.add(os.getcwd(),
arcname=os.path.basename(os.path.sep),
filter=zip_filter)
tar.add(config_path, arcname=os.path.basename(config_path))
size = os.path.getsize(src_archive.name)
max_upload_size = "100MB"
if size > parse_size(max_upload_size):
fatal_exit(
click.style(
(f"ERROR: Compressed notebook directory is {format_size(size)},"
f" max upload size is {max_upload_size}. \nPlease ensure you ignore any virtualenv subdirectory"
" using `treebeard run --ignore venv`"),
fg="red",
))
if local:
spinner.stop()
build_tag = str(time.mktime(datetime.today().timetuple()))
repo_image_name = (
f"gcr.io/treebeard-259315/projects/{project_id}/{notebook_id}:{build_tag}"
)
click.echo(f"🌲 Building {repo_image_name} Locally\n")
secrets_archive = get_secrets_archive()
repo_url = f"file://{src_archive.name}"
secrets_url = f"file://{secrets_archive.name}"
run_repo(
str(project_id),
str(notebook_id),
treebeard_env.run_id,
build_tag,
repo_url,
secrets_url,
local=True,
)
sys.exit(0)
spinner.text = "🌲 submitting notebook to runner\n"
response = requests.post(
notebooks_endpoint,
files={"repo": open(src_archive.name, "rb")},
params=params,
headers=treebeard_env.dict(),
)
if response.status_code != 200:
raise click.ClickException(f"Request failed: {response.text}")
spinner.stop()
try:
json_data = json.loads(response.text)
click.echo(f"✨ Run has been accepted! {json_data['admin_url']}")
except:
click.echo("❗ Request to run failed")
click.echo(sys.exc_info())
if watch:
# spinner = Halo(text='watching build', spinner='dots')
# spinner.start()
build_result = None
while not build_result:
time.sleep(5)
response = requests.get(notebooks_endpoint,
headers=treebeard_env.dict())
json_data = json.loads(response.text)
status = json_data["runs"][-1]["status"]
click.echo(f"{get_time()} Build status: {status}")
if status == "SUCCESS":
build_result = status
# spinner.stop()
click.echo(f"Build result: {build_result}")
elif status in [
"FAILURE", "TIMEOUT", "INTERNAL_ERROR", "CANCELLED"
]:
fatal_exit(f"Build failed")
def run_repo(
notebooks: List[str] = [],
env: List[str] = [],
ignore: List[str] = [],
confirm: bool = True,
use_docker: bool = False,
upload: bool = False,
debug: bool = False,
req_file_path: Optional[str] = None,
usagelogging: bool = False,
github_details: Optional[GitHubDetails] = None,
treebeard_env: Optional[TreebeardEnv] = None,
treebeard_config: Optional[TreebeardConfig] = None,
) -> int:
"""
Run a notebook and optionally schedule it to run periodically
"""
notebooks = list(notebooks)
ignore = list(ignore)
if not treebeard_env:
treebeard_env = conf.get_treebeard_env(github_details)
if not treebeard_config:
treebeard_config = conf.get_treebeard_config()
treebeard_context = TreebeardContext(
treebeard_env=treebeard_env,
treebeard_config=treebeard_config,
config_path=conf.get_config_path(),
github_details=github_details,
)
if debug:
click.echo(
f"Treebeard context:\n{json.dumps(treebeard_context.dict(), sort_keys=True, indent=4)}"
)
setup_sentry(treebeard_context.treebeard_env)
treebeard_config = treebeard_context.treebeard_config
treebeard_config.debug = debug
validate_notebook_directory(treebeard_context.treebeard_env,
treebeard_context.treebeard_config, upload)
# Apply cli config overrides
if notebooks:
treebeard_config.notebooks = notebooks
if ignore:
treebeard_config.ignore = ignore + treebeard_config.output_dirs
if "TREEBEARD_START_TIME" not in os.environ:
os.environ["TREEBEARD_START_TIME"] = get_time()
if not use_docker:
if upload:
update(
treebeard_context,
status="WORKING",
update_url=
f"{api_url}/{treebeard_context.treebeard_env.run_path}/update",
)
click.echo(
f"🌲 Running locally without docker using your current python environment"
)
if not confirm and not click.confirm(
f"Warning: This will clear the outputs of your notebooks, continue?",
default=True,
):
sys.exit(1)
# Note: import runtime.run causes win/darwin devices missing magic to fail at start
import treebeard.runtime.run
nbrun = treebeard.runtime.run.NotebookRun(treebeard_context)
status = nbrun.start(upload=upload, logging=usagelogging)
return status
if upload:
update(
treebeard_context,
status="BUILDING",
update_url=
f"{api_url}/{treebeard_context.treebeard_env.run_path}/update",
)
click.echo("🌲 Creating Project bundle")
temp_dir = tempfile.mkdtemp()
copy_tree(os.getcwd(), str(temp_dir), preserve_symlinks=1)
if req_file_path:
try:
dest = basename(req_file_path)
if req_file_path.endswith("ml"):
dest = "environment.yml"
elif req_file_path.endswith("txt"):
dest = "requirements.txt"
copyfile(req_file_path, f"{temp_dir}/{dest}")
click.echo(f"req-file-path: Copied {req_file_path} to {dest}")
except Exception as ex:
click.echo(
f"Error occurred locating your req-file using req-file-path {req_file_path}:\n{ex}"
)
raise ex
# Overwrite config with in-memory-modified
with open(f"{temp_dir}/{get_config_file_name()}", "w") as yaml_file:
yaml.dump(treebeard_config.dict(), yaml_file) # type: ignore
return build.build(
treebeard_context,
temp_dir,
envs_to_forward=env,
upload=upload,
usagelogging=usagelogging,
)