def tensorboards(ctx, query, sort, page): """List tensorboard jobs for this project. Uses [Caching](/references/polyaxon-cli/#caching) """ user, project_name = get_project_or_local(ctx.obj.get('project')) page = page or 1 try: response = PolyaxonClient().project.list_tensorboards(username=user, project_name=project_name, query=query, sort=sort, page=page) except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error('Could not get tensorboards for project `{}`.'.format(project_name)) Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1) meta = get_meta_response(response) if meta: Printer.print_header('Tensorboards for project `{}/{}`.'.format(user, project_name)) Printer.print_header('Navigation:') dict_tabulate(meta) else: Printer.print_header('No tensorboards found for project `{}/{}`.'.format(user, project_name)) objects = [Printer.add_status_color(o.to_light_dict(humanize_values=True)) for o in response['results']] objects = list_dicts_to_tabulate(objects) if objects: Printer.print_header("Tensorboards:") objects.pop('project_name', None) dict_tabulate(objects, is_list_dict=True)
def delete(ctx): """Delete project. Uses [Caching](/references/polyaxon-cli/#caching) """ user, project_name = get_project_or_local(ctx.obj.get('project')) if not click.confirm("Are sure you want to delete project `{}/{}`".format( user, project_name)): click.echo('Existing without deleting project.') sys.exit(1) try: response = PolyaxonClient().project.delete_project(user, project_name) local_project = ProjectManager.get_config() if local_project and (user, project_name) == (local_project.user, local_project.name): # Purge caching ProjectManager.purge() except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error('Could not delete project `{}/{}`.'.format( user, project_name)) Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1) if response.status_code == 204: Printer.print_success("Project `{}/{}` was delete successfully".format( user, project_name))
def get(ctx): """Get info for current project, by project_name, or user/project_name. Uses [Caching](/references/polyaxon-cli/#caching) Examples: To get current project: \b ```bash $ polyaxon project get ``` To get a project by name \b ```bash $ polyaxon project get user/project ``` """ user, project_name = get_project_or_local(ctx.obj.get('project')) try: response = PolyaxonClient().project.get_project(user, project_name) except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error('Could not get project `{}`.'.format(project_name)) Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1) get_project_details(response)
def download(ctx, commit): """Download code of the current project.""" user, project_name = get_project_or_local(ctx.obj.get('project')) try: PolyaxonClient().project.download_repo(user, project_name, commit=commit) except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error('Could not download code for project `{}`.'.format(project_name)) Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1) Printer.print_success('Files downloaded.')
def update(ctx, name, description, tags, private): """Update project. Uses [Caching](/references/polyaxon-cli/#caching) Example: \b ```bash $ polyaxon update foobar --description="Image Classification with DL using TensorFlow" ``` \b ```bash $ polyaxon update mike1/foobar --description="Image Classification with DL using TensorFlow" ``` \b ```bash $ polyaxon update --tags="foo, bar" ``` """ user, project_name = get_project_or_local(ctx.obj.get('project')) update_dict = {} if name: update_dict['name'] = name if description: update_dict['description'] = description if private is not None: update_dict['is_public'] = not private tags = validate_tags(tags) if tags: update_dict['tags'] = tags if not update_dict: Printer.print_warning( 'No argument was provided to update the project.') sys.exit(1) try: response = PolyaxonClient().project.update_project( user, project_name, update_dict) except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error( 'Could not update project `{}`.'.format(project_name)) Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1) Printer.print_success("Project updated.") get_project_details(response)
def init(project, polyaxonfile): """Initialize a new polyaxonfile specification.""" user, project_name = get_project_or_local(project) try: project_config = PolyaxonClient().project.get_project( user, project_name) except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error( 'Make sure you have a project with this name `{}`'.format(project)) Printer.print_error( 'You can a create new project with this command: ' 'polyaxon project create ' '--name={} [--description=...] [--tags=...]'.format(project_name)) Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1) init_project = False if ProjectManager.is_initialized(): local_project = ProjectManager.get_config() click.echo( 'Warning! This project is already initialized with the following project:' ) with clint.textui.indent(4): clint.textui.puts('User: {}'.format(local_project.user)) clint.textui.puts('Project: {}'.format(local_project.name)) if click.confirm('Would you like to override this current config?', default=False): init_project = True else: init_project = True if init_project: ProjectManager.purge() ProjectManager.set_config(project_config, init=True) Printer.print_success('Project was initialized') else: Printer.print_header('Project config was not changed.') init_ignore = False if IgnoreManager.is_initialized(): click.echo('Warning! Found a .polyaxonignore file.') if click.confirm('Would you like to override it?', default=False): init_ignore = True else: init_ignore = True if init_ignore: IgnoreManager.init_config() Printer.print_success('New .polyaxonignore file was created.') else: Printer.print_header('.polyaxonignore file was not changed.') if polyaxonfile: create_polyaxonfile()
def git(ctx, url, private, sync): # pylint:disable=assign-to-new-keyword """Set/Sync git repo on this project. Uses [Caching](/references/polyaxon-cli/#caching) Example: \b ```bash $ polyaxon project git --url=https://github.com/polyaxon/polyaxon-quick-start ``` \b ```bash $ polyaxon project git --url=https://github.com/polyaxon/polyaxon-quick-start --private ``` """ user, project_name = get_project_or_local(ctx.obj.get('project')) def git_set_url(): if private: click.echo('\nSetting a private git repo "{}" on project: {} ...\n'.format( url, project_name)) else: click.echo('\nSetting a public git repo "{}" on project: {} ...\n'.format( url, project_name)) try: PolyaxonClient().project.set_repo(user, project_name, url, not private) except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error('Could not set git repo on project `{}`.'.format(project_name)) Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1) Printer.print_success('Project was successfully initialized with `{}`.'.format(url)) def git_sync_repo(): try: response = PolyaxonClient().project.sync_repo(user, project_name) except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error('Could not sync git repo on project `{}`.'.format(project_name)) Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1) click.echo(response.status_code) Printer.print_success('Project was successfully synced with latest changes.') if url: git_set_url() if sync: git_sync_repo()
def ci(ctx, enable, disable): # pylint:disable=assign-to-new-keyword """Enable/Disable CI on this project. Uses [Caching](/references/polyaxon-cli/#caching) Example: \b ```bash $ polyaxon project ci --enable ``` \b ```bash $ polyaxon project ci --disable ``` """ user, project_name = get_project_or_local(ctx.obj.get('project')) def enable_ci(): try: PolyaxonClient().project.enable_ci(user, project_name) except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error( 'Could not enable CI on project `{}`.'.format(project_name)) Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1) Printer.print_success( 'Polyaxon CI was successfully enabled on project: `{}`.'.format( project_name)) def disable_ci(): try: PolyaxonClient().project.disable_ci(user, project_name) except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error( 'Could not disable CI on project `{}`.'.format(project_name)) Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1) Printer.print_success( 'Polyaxon CI was successfully disabled on project: `{}`.'.format( project_name)) if enable: enable_ci() if disable: disable_ci()
def unbookmark(ctx): """Unbookmark project. Uses [Caching](/references/polyaxon-cli/#caching) """ user, project_name = get_project_or_local(ctx.obj.get('project')) try: PolyaxonClient().project.unbookmark(user, project_name) except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error('Could not unbookmark project `{}/{}`.'.format(user, project_name)) Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1) Printer.print_success("Project `{}/{}` is unbookmarked.".format(user, project_name))
def invalidate_builds(ctx): """Invalidate build job. Uses [Caching](/references/polyaxon-cli/#caching) Examples: \b ```bash $ polyaxon invalidate_builds ``` """ user, project_name = get_project_or_local(ctx.obj.get('project')) try: PolyaxonClient().project.invalidate_builds(user, project_name) except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error( 'Could not invalidate build jobs for project `{}`.'.format(project_name)) Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1) Printer.print_success( "Build jobs have being invalidated for project `{}`.".format(project_name))
def experiments(ctx, metrics, declarations, independent, group, query, sort, page): """List experiments for this project. Uses [Caching](/references/polyaxon-cli/#caching) Examples: Get all experiments: \b ```bash $ polyaxon project experiments ``` Get all experiments with with status {created or running}, and creation date between 2018-01-01 and 2018-01-02, and declarations activation equal to sigmoid and metric loss less or equal to 0.2 \b ```bash $ polyaxon project experiments \ -q "status:created|running, started_at:2018-01-01..2018-01-02, \ declarations.activation:sigmoid, metric.loss:<=0.2" ``` Get all experiments sorted by update date \b ```bash $ polyaxon project experiments -s "-updated_at" ``` """ user, project_name = get_project_or_local(ctx.obj.get('project')) page = page or 1 try: response = PolyaxonClient().project.list_experiments( username=user, project_name=project_name, independent=independent, group=group, metrics=metrics, declarations=declarations, query=query, sort=sort, page=page) except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error( 'Could not get experiments for project `{}`.'.format(project_name)) Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1) meta = get_meta_response(response) if meta: Printer.print_header('Experiments for project `{}/{}`.'.format( user, project_name)) Printer.print_header('Navigation:') dict_tabulate(meta) else: Printer.print_header( 'No experiments found for project `{}/{}`.'.format( user, project_name)) if metrics: objects = get_experiments_with_metrics(response) elif declarations: objects = get_experiments_with_declarations(response) else: objects = [ Printer.add_status_color(o.to_light_dict(humanize_values=True)) for o in response['results'] ] objects = list_dicts_to_tabulate(objects) if objects: Printer.print_header("Experiments:") objects.pop('project_name', None) dict_tabulate(objects, is_list_dict=True)
def groups(ctx, query, sort, page): """List experiment groups for this project. Uses [Caching](/references/polyaxon-cli/#caching) Examples: Get all groups: \b ```bash $ polyaxon project groups ``` Get all groups with with status {created or running}, and creation date between 2018-01-01 and 2018-01-02, and search algorithm not in {grid or random search} \b ```bash $ polyaxon project groups \ -q "status:created|running, started_at:2018-01-01..2018-01-02, search_algorithm:~grid|random" ``` Get all groups sorted by update date \b ```bash $ polyaxon project groups -s "-updated_at" ``` """ user, project_name = get_project_or_local(ctx.obj.get('project')) page = page or 1 try: response = PolyaxonClient().project.list_experiment_groups( username=user, project_name=project_name, query=query, sort=sort, page=page) except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error( 'Could not get experiment groups for project `{}`.'.format( project_name)) Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1) meta = get_meta_response(response) if meta: Printer.print_header('Experiment groups for project `{}/{}`.'.format( user, project_name)) Printer.print_header('Navigation:') dict_tabulate(meta) else: Printer.print_header( 'No experiment groups found for project `{}/{}`.'.format( user, project_name)) objects = [ Printer.add_status_color(o.to_light_dict(humanize_values=True)) for o in response['results'] ] objects = list_dicts_to_tabulate(objects) if objects: Printer.print_header("Experiment groups:") objects.pop('project', None) objects.pop('user', None) dict_tabulate(objects, is_list_dict=True)
def init(project, run, model): """Initialize a new polyaxonfile specification.""" user, project_name = get_project_or_local(project) try: project_config = PolyaxonClient().project.get_project( user, project_name) except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error( 'Make sure you have a project with this name `{}`'.format(project)) Printer.print_error( 'You can a create new project with this command: ' 'polyaxon project create ' '--name={} [--description=...] [--tags=...]'.format(project_name)) Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1) if not any([model, run]) and not all([model, run]): Printer.print_error("You must specify which init option, " "possible values: `--model` or `--run`.") sys.exit(1) result = False if model: result = create_init_file(constants.INIT_FILE_MODEL) elif run: result = create_init_file(constants.INIT_FILE_RUN) if result: ProjectManager.purge() ProjectManager.set_config(project_config, init=True) IgnoreManager.init_config() Printer.print_success( "Project `{}` was initialized and Polyaxonfile was created successfully `{}`" .format(project, constants.INIT_FILE)) sys.exit(1) # if we are here the file was not created if not os.path.isfile(constants.INIT_FILE): Printer.print_error( "Something went wrong, init command did not create a file.\n" "Possible reasons: you don't have the write to create the file.") sys.exit(1) # file was already there, let's check if the project passed correspond to this file try: PolyaxonFile(constants.INIT_FILE).specification except (PolyaxonfileError, ValidationError) as e: Printer.print_error( "Something went wrong, init command did not create a file.\n" "Another file already exist with.") Printer.print_error('Error message: `{}`.'.format(e)) sys.exit(1) # At this point we check if we need to re init configurations ProjectManager.purge() ProjectManager.set_config(project_config, init=True) IgnoreManager.init_config() Printer.print_success( "Project `{}` was initialized and Polyaxonfile was created successfully `{}`" .format(project, constants.INIT_FILE))
def get_build_or_local(project=None, build=None): user, project_name = get_project_or_local(project) build = build or BuildJobManager.get_config_or_raise().id return user, project_name, build
def run( ctx, project, file, # pylint:disable=redefined-builtin name, tags, description, ttl, debug, upload, log, local, conda_env, params): """Run polyaxonfile specification. Examples: \b ```bash $ polyaxon run -f file -f file_override ... ``` Upload before running \b ```bash $ polyaxon run -f file -u ``` Run and set description and tags for this run \b ```bash $ polyaxon run -f file -u --description="Description of the current run" --tags="foo, bar, moo" ``` Run and set a unique name for this run \b ```bash polyaxon run --name=foo ``` Run for a specific project \b ```bash $ polyaxon run -p project1 -f file.yaml ``` Run with updated params \b ```bash $ polyaxon run -p project1 -f file.yaml -P param1=234.2 -P param2=relu ``` """ if not file: file = PolyaxonFile.check_default_path(path='.') if not file: file = '' if debug and not ttl: Printer.print_error( 'Debug mode requires a ttl to properly set a sleep time.') sys.exit(1) debug_ttl = None if debug and ttl: debug_ttl = ttl ttl = None specification = check_polyaxonfile(file, params=params, debug_ttl=debug_ttl, log=False).specification spec_cond = (specification.is_experiment or specification.is_group or specification.is_job or specification.is_build) if not spec_cond: Printer.print_error( 'This command expects an experiment, a group, a job, or a build specification,' 'received instead a `{}` specification'.format(specification.kind)) if specification.is_notebook: click.echo( 'Please check "polyaxon notebook --help" to start a notebook.') elif specification.is_tensorboard: click.echo( 'Please check: "polyaxon tensorboard --help" to start a tensorboard.' ) sys.exit(1) user, project_name = get_project_or_local(project) tags = validate_tags(tags) if local: try: specification.apply_context() except PolyaxonSchemaError: Printer.print_error( 'Could not run this polyaxonfile locally, ' 'a context is required to resolve it dependencies.') if conda_env: conda_run(ctx=ctx, name=name, user=user, project_name=project_name, description=description, tags=tags, specification=specification, log=log, conda_env=conda_env) else: docker_run(ctx=ctx, name=name, user=user, project_name=project_name, description=description, tags=tags, specification=specification, log=log) else: platform_run(ctx=ctx, name=name, user=user, project_name=project_name, description=description, tags=tags, specification=specification, ttl=ttl, upload=upload, log=log, can_upload=all([upload, project]))
def get_project_group_or_local(project=None, group=None): user, project_name = get_project_or_local(project) group = get_group_or_local(group) return user, project_name, group
def builds(ctx, query, sort, page): """List build jobs for this project. Uses [Caching](/references/polyaxon-cli/#caching) Examples: Get all builds: \b ```bash $ polyaxon project builds ``` Get all builds with with status not in {created or running} \b ```bash $ polyaxon project builds -q "status:~created" ``` Get all builds with with status failed \b ```bash $ polyaxon project builds -q "status:failed" ``` Get all builds sorted by update date \b ```bash $ polyaxon project builds -s "-updated_at" ``` """ user, project_name = get_project_or_local(ctx.obj.get('project')) page = page or 1 try: response = PolyaxonClient().project.list_builds( username=user, project_name=project_name, query=query, sort=sort, page=page) except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error( 'Could not get builds for project `{}`.'.format(project_name)) Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1) meta = get_meta_response(response) if meta: Printer.print_header('Builds for project `{}/{}`.'.format( user, project_name)) Printer.print_header('Navigation:') dict_tabulate(meta) else: Printer.print_header('No builds found for project `{}/{}`.'.format( user, project_name)) objects = [ Printer.add_status_color(o.to_light_dict(humanize_values=True)) for o in response['results'] ] objects = list_dicts_to_tabulate(objects) if objects: Printer.print_header("Builds:") objects.pop('project_name', None) dict_tabulate(objects, is_list_dict=True)
def run(ctx, project, file, name, tags, description, ttl, u, l): # pylint:disable=redefined-builtin """Run polyaxonfile specification. Examples: \b ```bash $ polyaxon run -f file -f file_override ... ``` Upload before running \b ```bash $ polyaxon run -f file -u ``` Run and set description and tags for this run \b ```bash $ polyaxon run -f file -u --description="Description of the current run" --tags="foo, bar, moo" ``` Run and set a unique name for this run \b ```bash polyaxon run --name=foo ``` Run for a specific project \b ```bash $ polyaxon run -p project1 -f file.yaml ``` """ if not file: file = PolyaxonFile.check_default_path(path='.') if not file: file = '' specification = check_polyaxonfile(file, log=False).specification spec_cond = (specification.is_experiment or specification.is_group or specification.is_job or specification.is_build) if not spec_cond: Printer.print_error( 'This command expects an experiment, a group, a job, or a build specification,' 'received instead a `{}` specification'.format(specification.kind)) if specification.is_notebook: click.echo('Please check "polyaxon notebook --help" to start a notebook.') elif specification.is_tensorboard: click.echo('Please check: "polyaxon tensorboard --help" to start a tensorboard.') sys.exit(1) # Check if we need to upload if u: if project: Printer.print_error('Uploading is not supported when switching project context!') click.echo('Please, either omit the `-u` option or `-p` / `--project=` option.') sys.exit(1) ctx.invoke(upload, sync=False) user, project_name = get_project_or_local(project) project_client = PolyaxonClient().project tags = validate_tags(tags) def run_experiment(): click.echo('Creating an independent experiment.') experiment = ExperimentConfig( name=name, description=description, tags=tags, config=specification.parsed_data, ttl=ttl) try: response = PolyaxonClient().project.create_experiment(user, project_name, experiment) cache.cache(config_manager=ExperimentManager, response=response) Printer.print_success('Experiment `{}` was created'.format(response.id)) except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error('Could not create experiment.') Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1) def run_group(): click.echo('Creating an experiment group with the following definition:') experiments_def = specification.experiments_def get_group_experiments_info(**experiments_def) experiment_group = ExperimentGroupConfig( name=name, description=description, tags=tags, content=specification._data) # pylint:disable=protected-access try: response = project_client.create_experiment_group(user, project_name, experiment_group) cache.cache(config_manager=GroupManager, response=response) Printer.print_success('Experiment group {} was created'.format(response.id)) except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error('Could not create experiment group.') Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1) def run_job(): click.echo('Creating a job.') job = JobConfig( name=name, description=description, tags=tags, config=specification.parsed_data, ttl=ttl) try: response = project_client.create_job(user, project_name, job) cache.cache(config_manager=JobManager, response=response) Printer.print_success('Job {} was created'.format(response.id)) except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error('Could not create job.') Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1) def run_build(): click.echo('Creating a build.') job = JobConfig( name=name, description=description, tags=tags, config=specification.parsed_data, ttl=ttl) try: response = project_client.create_build(user, project_name, job) cache.cache(config_manager=BuildJobManager, response=response) Printer.print_success('Build {} was created'.format(response.id)) except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error('Could not create build.') Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1) logs = None if specification.is_experiment: run_experiment() logs = experiment_logs elif specification.is_group: run_group() elif specification.is_job: run_job() logs = job_logs elif specification.is_build: run_build() logs = build_logs # Check if we need to invoke logs if l and logs: ctx.obj = {'project': '{}/{}'.format(user, project_name)} ctx.invoke(logs)
def get_job_or_local(project=None, job=None): user, project_name = get_project_or_local(project) job = job or JobManager.get_config_or_raise().id return user, project_name, job