def list_imagesets(print_details: bool, explore_details: bool, filter_str: str): """List available image sets on S3. Args: print_details (bool): T/F print detailed view to console explore_details (bool): T/F bring up detailed view in pager filter_str (str): string to detailed view on. None if not provided by user. """ imageset_names = cli_spinner("Finding image sets on S3...", get_imageset_names) if explore_details or print_details: detailed_info = cli_spinner("Downloading imageset metadata from S3...", _get_detailed_imageset_info, imageset_names, filter_str=filter_str) if explore_details: pydoc.pager(detailed_info) else: click.echo(detailed_info) return for name in imageset_names: click.echo(name)
def process_result(ctx: click.Context, result: CreateOutput, config: str): """Processes output of dataset creation Args: ctx (Context): click context object result (CreateOutput): result of dataset creation plugin config (str): original config provided by user Returns: result (CreateOutput): result of dataset creation plugin """ if result is not None: # Gets dataset information from CreateInput ci = ctx.obj dataset_name = ci.metadata['dataset_name'] dataset_path = ci.dataset_path / dataset_name # Uploads dataset to S3 if (ci.upload): bucketConfig = get_config() bucket = bucketConfig["dataset_bucket_name"] cli_spinner("Uploading dataset to S3...", upload_directory, bucket_name=bucket, prefix=dataset_name, local_path=dataset_path) # Deletes local dataset if (ci.delete_local): cli_spinner("Deleting " + dataset_name + " dataset...", shutil.rmtree, dataset_path) return result
def inspect_dataset(dataset_name: str): """See detailed metadata about a dataset. Args: dataset_name (str): string name of the dataset to inspect """ try: metadata = cli_spinner("Downloading dataset metadata from S3...", get_dataset_metadata, dataset_name) click.echo(_stringify_metadata(metadata, colored=True)) # get_dataset_metadata will raise a value error if the given dataset name does not exist on S3 # OR if that dataset does not contain a metadata file. Currently these two situations are indistinguishible # TODO: separate errors for bad dataset name and dataset lacking metadata. This will need to be done in the # get_dataset_metadata function except ValueError as e: raise click.exceptions.BadParameter(dataset_name, param=dataset_name, param_hint='dataset name')
def inspect_imageset(imageset_name: str): """See detailed metadata about a dataset. Args: dataset_name (str): string name of the dataset to inspect """ try: metadata = cli_spinner("Downloading imageset metadata from S3...", get_imageset_metadata, imageset_name) # can check if the metadata returned is for an individual image if it has pose info if 'pose' in metadata.keys(): click.echo(Fore.RED + ('Set-wide metadata not found. ' 'Falling back to sample image metadata.')) click.echo(_stringify_metadata(metadata, colored=True)) # get_imageset_metadata will raise a value error if imageset name not found except ValueError as e: raise click.exceptions.BadParameter(imageset_name, param=imageset_name, param_hint='imageset name') # get imageset_metadata will raise a KeyError if the imageset does not contain any metadata files. except KeyError as e: raise click.exceptions.ClickException( f'Given imageset "{imageset_name}" does not contain any metadata files.' )
def process_result(ctx: click.Context, result: TrainOutput, config: str): """Processes the result of a training by analyzing the given TrainOutput object. This callback is called after ANY command originating from the train command group, hence the check to see if a result was actually returned - plugins simply do not return a TrainOutput from non-training commands. Args: ctx (Context): click context object ti (TrainInput): TrainInput object stored at ctx.obj, created at start of training. This object contains the metadata from the run and the path to the training artifacts (at ti.artifact_path) which is printed to the user after a local training. result (TrainOutput): training output object returned by training plugin config (str): config option from train command. Click requires that command callbacks accept the options from the original command. """ if result is not None: # only plugin training commands that return a TrainOutput will activate this block # thus ctx.obj will always be a TrainInput object # NOTE: you cannot use the @pass_train decorator on process_result, otherwise on # non-training plugin commands, the TrainInput __init__ will be called by Click # when process_result runs and no TrainInput is at ctx.obj ti = ctx.obj # store git info for plugin # NOTE: this will fail for plugins not installed via source plugin_repo_root = git.is_repo(result.plugin_dir) git_info = {} if plugin_repo_root: git_info["plugin_git_sha"] = git.git_sha(plugin_repo_root) # note running the patch commands in repo root will include patches for all plugins git_info["plugin_tracked_git_patch"] = git.git_patch_tracked(plugin_repo_root) git_info["plugin_untracked_git_patch"] = git.git_patch_untracked(plugin_repo_root) else: git_info = git.retrieve_from_pkg(result.plugin_dir) ti.metadata.update(git_info) # upload if not in local mode, determined by user defined artifact_path field in config if not ti.config.get('artifact_path'): uuid = cli_spinner('Uploading artifacts...', _upload_result, result, ti.metadata, ti.plugin_metadata) click.echo(f'Artifact UUID: {uuid}') else: with open(ti.artifact_path / 'metadata.json', 'w') as f: json.dump(ti.metadata, f, indent=2) click.echo(f'LOCAL MODE: Not uploading model to S3. Model is located at: {ti.artifact_path}') # stop, terminate, or do nothing to ec2 based on policy ec2_policy = ti.config.get('ec2_policy') # check if the policy is to stop or terminate if ec2_policy == None or ec2_policy == 'stop' or ec2_policy == 'terminate': policy_str = ec2_policy if ec2_policy else 'default' click.echo(f'Checking for EC2 instance and applying policy "{policy_str}"...') try: # grab ec2 id with urlopen(EC2_INSTANCE_ID_URL, timeout=5) as url: ec2_instance_id = url.read().decode('utf-8') click.echo(f'EC2 Runtime detected.') client = boto3.client('ec2') # default is stop if ec2_policy == None or ec2_policy == 'stop': click.echo("Stopping...") client.stop_instances(InstanceIds=[ec2_instance_id], DryRun=False) else: click.echo("Terminating...") client.terminate_instances(InstanceIds=[ec2_instance_id], DryRun=False) except URLError: click.echo('No EC2 runtime detected. Doing nothing.') else: click.echo('Not checking for EC2 runtime since policy is to keep running.') return result