def output(id, url, download):
"""
Shows the output url of the run.
By default opens the output page in your default browser.
"""
experiment = ExperimentClient().get(id)
task_instance = TaskInstanceClient().get(
get_module_task_instance_id(experiment.task_instances))
if "output" in task_instance.output_ids:
resource = ResourceClient().get(task_instance.output_ids["output"])
output_dir_url = "{}/viewer/{}".format(floyd.floyd_host, resource.uri)
if url:
floyd_logger.info(output_dir_url)
else:
if download:
output_dir_url = "{}&download=true".format(output_dir_url)
ExperimentClient().download_tar(url=output_dir_url,
untar=True,
delete_after_untar=True)
else:
floyd_logger.info(
"Opening output directory in your browser ...")
webbrowser.open(output_dir_url)
else:
floyd_logger.error("Output directory not available")
def delete(id, yes):
"""
Delete project run
"""
experiment = ExperimentClient().get(id)
task_instance = TaskInstanceClient().get(
get_module_task_instance_id(experiment.task_instances))
if experiment.state in ["queued", "running"]:
floyd_logger.info(
"Experiment in {} state cannot be deleted. Stop it first".format(
experiment.state))
return
if not yes:
click.confirm('Delete Run: {}?'.format(experiment.name),
abort=True,
default=False)
if task_instance.module_id:
ModuleClient().delete(task_instance.module_id)
if ExperimentClient().delete(id):
floyd_logger.info("Experiment deleted")
else:
floyd_logger.error("Failed to delete experiment")
def info(job_name_or_id):
"""
Prints detailed info for the run
"""
try:
experiment = ExperimentClient().get(normalize_job_name(job_name_or_id))
except FloydException:
experiment = ExperimentClient().get(job_name_or_id)
task_instance_id = get_module_task_instance_id(experiment.task_instances)
task_instance = TaskInstanceClient().get(
task_instance_id) if task_instance_id else None
normalized_job_name = normalize_job_name(experiment.name)
table = [["Job name", normalized_job_name],
[
"Output name",
normalized_job_name + '/output' if task_instance else None
], ["Created", experiment.created_pretty],
["Status", experiment.state],
["Duration(s)", experiment.duration_rounded],
["Instance", experiment.instance_type_trimmed],
["Description", experiment.description]]
if task_instance and task_instance.mode in ['jupyter', 'serving']:
table.append(["Mode", task_instance.mode])
table.append(["Url", experiment.service_url])
if experiment.tensorboard_url:
table.append(["Tensorboard", experiment.tensorboard_url])
floyd_logger.info(tabulate(table))
def info(id):
"""
Prints detailed info for the run
"""
experiment = ExperimentClient().get(id)
task_instance_id = get_module_task_instance_id(experiment.task_instances)
task_instance = TaskInstanceClient().get(
task_instance_id) if task_instance_id else None
mode = url = None
if experiment.state == "running":
if task_instance and task_instance.mode in ['jupyter', 'serving']:
mode = task_instance.mode
url = get_task_url(task_instance.id)
table = [["Run ID", experiment.id], ["Name", experiment.name],
["Created", experiment.created_pretty],
["Status", experiment.state],
["Duration(s)", experiment.duration_rounded],
["Output ID", task_instance.id if task_instance else None],
["Instance", experiment.instance_type_trimmed],
["Version", experiment.description]]
if mode:
table.append(["Mode", mode])
if url:
table.append(["Url", url])
floyd_logger.info(tabulate(table))
def logs(id, url, tail, sleep_duration=1):
"""
Print the logs of the run.
"""
experiment = ExperimentClient().get(id)
task_instance = TaskInstanceClient().get(
get_module_task_instance_id(experiment.task_instances))
log_url = "{}/api/v1/resources/{}?content=true".format(
floyd.floyd_host, task_instance.log_id)
if url:
floyd_logger.info(log_url)
return
if tail:
floyd_logger.info("Launching job ...")
current_shell_output = ""
while True:
# Get the logs in a loop and log the new lines
log_file_contents = get_url_contents(log_url)
print_output = log_file_contents[len(current_shell_output):]
if len(print_output.strip()):
floyd_logger.info(print_output)
current_shell_output = log_file_contents
sleep(sleep_duration)
else:
log_file_contents = get_url_contents(log_url)
if len(log_file_contents.strip()):
floyd_logger.info(log_file_contents)
else:
floyd_logger.info("Launching job now. Try after a few seconds.")
def clone(id):
"""
Download files from a job.
This will download the files that were originally uploaded at
the start of the job.
"""
try:
experiment = ExperimentClient().get(
normalize_job_name(id, use_config=False))
except FloydException:
experiment = ExperimentClient().get(id)
task_instance_id = get_module_task_instance_id(experiment.task_instances)
task_instance = TaskInstanceClient().get(
task_instance_id) if task_instance_id else None
if not task_instance:
sys.exit(
"Cannot clone this version of the job. Try a different version.")
module = ModuleClient().get(
task_instance.module_id) if task_instance else None
code_url = "{}/api/v1/resources/{}?content=true&download=true".format(
floyd.floyd_host, module.resource_id)
ExperimentClient().download_tar(url=code_url,
untar=True,
delete_after_untar=True)
def clone(id):
"""
Download the code for the experiment to the current path
"""
experiment = ExperimentClient().get(id)
task_instance_id = get_module_task_instance_id(experiment.task_instances)
task_instance = TaskInstanceClient().get(task_instance_id) if task_instance_id else None
if not task_instance:
sys.exit("Cannot clone this version of the job. Try a different version.")
module = ModuleClient().get(task_instance.module_id) if task_instance else None
code_url = "{}/api/v1/resources/{}?content=true&download=true".format(floyd.floyd_host,
module.resource_id)
ExperimentClient().download_tar(url=code_url,
untar=True,
delete_after_untar=True)
def output(id, url):
"""
Shows the output url of the run.
By default opens the output page in your default browser.
"""
experiment = ExperimentClient().get(id)
task_instance = TaskInstanceClient().get(get_module_task_instance_id(experiment.task_instances))
if "output" in task_instance.output_ids:
output_dir_url = "{}/api/v1/resources/{}?content=true".format(floyd.floyd_host,
task_instance.output_ids["output"])
if url:
floyd_logger.info(output_dir_url)
else:
floyd_logger.info("Opening output directory in your browser ...")
webbrowser.open(output_dir_url)
else:
floyd_logger.error("Output directory not available")
def clone(id, path):
"""
- Download all files from a job
Eg: alice/projects/mnist/1/
Note: This will download the files that were originally uploaded at
the start of the job.
- Download files in a specific path from a job
Specify the path to a directory and download all its files and subdirectories.
Eg: --path models/checkpoint1
"""
try:
experiment = ExperimentClient().get(
normalize_job_name(id, use_config=False))
except FloydException:
experiment = ExperimentClient().get(id)
task_instance_id = get_module_task_instance_id(experiment.task_instances)
task_instance = TaskInstanceClient().get(
task_instance_id) if task_instance_id else None
if not task_instance:
sys.exit(
"Cannot clone this version of the job. Try a different version.")
module = ModuleClient().get(
task_instance.module_id) if task_instance else None
if path:
# Download a directory from Code
code_url = "{}/api/v1/download/artifacts/code/{}?is_dir=true&path={}".format(
floyd.floyd_host, experiment.id, path)
else:
# Download the full Code
code_url = "{}/api/v1/resources/{}?content=true&download=true".format(
floyd.floyd_host, module.resource_id)
ExperimentClient().download_tar(url=code_url,
untar=True,
delete_after_untar=True)
def run(ctx, gpu, env, data, mode, command):
"""
Run a command on Floyd. Floyd will upload contents of the
current directory and run your command remotely.
This command will generate a run id for reference.
"""
command_str = ' '.join(command)
experiment_config = ExperimentConfigManager.get_config()
access_token = AuthConfigManager.get_access_token()
version = experiment_config.version
experiment_name = "{}/{}:{}".format(access_token.username,
experiment_config.name,
version)
# Create module
module = Module(name=experiment_name,
description=version,
command=command_str,
mode=get_mode_parameter(mode),
family_id=experiment_config.family_id,
default_container=get_docker_image(env, gpu),
version=version)
module_id = ModuleClient().create(module)
floyd_logger.debug("Created module with id : {}".format(module_id))
# Create experiment request
instance_type = GPU_INSTANCE_TYPE if gpu else CPU_INSTANCE_TYPE
experiment_request = ExperimentRequest(name=experiment_name,
description=version,
module_id=module_id,
data_id=data,
predecessor=experiment_config.experiment_predecessor,
family_id=experiment_config.family_id,
version=version,
instance_type=instance_type)
experiment_id = ExperimentClient().create(experiment_request)
floyd_logger.debug("Created experiment : {}".format(experiment_id))
# Update expt config including predecessor
experiment_config.increment_version()
experiment_config.set_module_predecessor(module_id)
experiment_config.set_experiment_predecessor(experiment_id)
ExperimentConfigManager.set_config(experiment_config)
table_output = [["RUN ID", "NAME", "VERSION"],
[experiment_id, experiment_name, version]]
floyd_logger.info(tabulate(table_output, headers="firstrow"))
floyd_logger.info("")
if mode != 'default':
while True:
# Wait for the experiment to become available
try:
experiment = ExperimentClient().get(experiment_id)
break
except Exception:
floyd_logger.debug("Experiment not available yet: {}".format(experiment_id))
sleep(1)
continue
# Print the path to jupyter notebook
if mode == 'jupyter':
jupyter_url = get_task_url(get_module_task_instance_id(experiment.task_instances))
floyd_logger.info("Waiting for Jupyter notebook to become available ...")
if wait_for_url(jupyter_url):
floyd_logger.info("\nPath to jupyter notebook: {}".format(jupyter_url))
else:
floyd_logger.info("Problem starting the notebook. View logs for more information")
# Print the path to serving endpoint
if mode == 'serve':
floyd_logger.info("Path to service endpoint: {}".format(
get_task_url(get_module_task_instance_id(experiment.task_instances))))
floyd_logger.info("""
To view logs enter:
floyd logs {}
""".format(experiment_id))
