def list_imagesets(print_details: bool, explore_details: bool,
filter_str: str):
"""List available image sets on S3.
Args:
print_details (bool): T/F print detailed view to console
explore_details (bool): T/F bring up detailed view in pager
filter_str (str): string to detailed view on. None if not provided by user.
"""
imageset_names = cli_spinner("Finding image sets on S3...",
get_imageset_names)
if explore_details or print_details:
detailed_info = cli_spinner("Downloading imageset metadata from S3...",
_get_detailed_imageset_info,
imageset_names,
filter_str=filter_str)
if explore_details:
pydoc.pager(detailed_info)
else:
click.echo(detailed_info)
return
for name in imageset_names:
click.echo(name)
def process_result(ctx: click.Context, result: CreateOutput, config: str):
"""Processes output of dataset creation
Args:
ctx (Context): click context object
result (CreateOutput): result of dataset creation plugin
config (str): original config provided by user
Returns:
result (CreateOutput): result of dataset creation plugin
"""
if result is not None:
# Gets dataset information from CreateInput
ci = ctx.obj
dataset_name = ci.metadata['dataset_name']
dataset_path = ci.dataset_path / dataset_name
# Uploads dataset to S3
if (ci.upload):
bucketConfig = get_config()
bucket = bucketConfig["dataset_bucket_name"]
cli_spinner("Uploading dataset to S3...",
upload_directory,
bucket_name=bucket,
prefix=dataset_name,
local_path=dataset_path)
# Deletes local dataset
if (ci.delete_local):
cli_spinner("Deleting " + dataset_name + " dataset...",
shutil.rmtree, dataset_path)
return result
def inspect_dataset(dataset_name: str):
"""See detailed metadata about a dataset.
Args:
dataset_name (str): string name of the dataset to inspect
"""
try:
metadata = cli_spinner("Downloading dataset metadata from S3...",
get_dataset_metadata, dataset_name)
click.echo(_stringify_metadata(metadata, colored=True))
# get_dataset_metadata will raise a value error if the given dataset name does not exist on S3
# OR if that dataset does not contain a metadata file. Currently these two situations are indistinguishible
# TODO: separate errors for bad dataset name and dataset lacking metadata. This will need to be done in the
# get_dataset_metadata function
except ValueError as e:
raise click.exceptions.BadParameter(dataset_name,
param=dataset_name,
param_hint='dataset name')
def inspect_imageset(imageset_name: str):
"""See detailed metadata about a dataset.
Args:
dataset_name (str): string name of the dataset to inspect
"""
try:
metadata = cli_spinner("Downloading imageset metadata from S3...",
get_imageset_metadata, imageset_name)
# can check if the metadata returned is for an individual image if it has pose info
if 'pose' in metadata.keys():
click.echo(Fore.RED + ('Set-wide metadata not found. '
'Falling back to sample image metadata.'))
click.echo(_stringify_metadata(metadata, colored=True))
# get_imageset_metadata will raise a value error if imageset name not found
except ValueError as e:
raise click.exceptions.BadParameter(imageset_name,
param=imageset_name,
param_hint='imageset name')
# get imageset_metadata will raise a KeyError if the imageset does not contain any metadata files.
except KeyError as e:
raise click.exceptions.ClickException(
f'Given imageset "{imageset_name}" does not contain any metadata files.'
)
def process_result(ctx: click.Context, result: TrainOutput, config: str):
"""Processes the result of a training by analyzing the given TrainOutput object.
This callback is called after ANY command originating from the train command
group, hence the check to see if a result was actually returned - plugins
simply do not return a TrainOutput from non-training commands.
Args:
ctx (Context): click context object
ti (TrainInput): TrainInput object stored at ctx.obj, created at start of training.
This object contains the metadata from the run and the path to the
training artifacts (at ti.artifact_path) which is printed to the user
after a local training.
result (TrainOutput): training output object returned by training plugin
config (str): config option from train command. Click requires that command
callbacks accept the options from the original command.
"""
if result is not None:
# only plugin training commands that return a TrainOutput will activate this block
# thus ctx.obj will always be a TrainInput object
# NOTE: you cannot use the @pass_train decorator on process_result, otherwise on
# non-training plugin commands, the TrainInput __init__ will be called by Click
# when process_result runs and no TrainInput is at ctx.obj
ti = ctx.obj
# store git info for plugin
# NOTE: this will fail for plugins not installed via source
plugin_repo_root = git.is_repo(result.plugin_dir)
git_info = {}
if plugin_repo_root:
git_info["plugin_git_sha"] = git.git_sha(plugin_repo_root)
# note running the patch commands in repo root will include patches for all plugins
git_info["plugin_tracked_git_patch"] = git.git_patch_tracked(plugin_repo_root)
git_info["plugin_untracked_git_patch"] = git.git_patch_untracked(plugin_repo_root)
else:
git_info = git.retrieve_from_pkg(result.plugin_dir)
ti.metadata.update(git_info)
# upload if not in local mode, determined by user defined artifact_path field in config
if not ti.config.get('artifact_path'):
uuid = cli_spinner('Uploading artifacts...', _upload_result, result, ti.metadata, ti.plugin_metadata)
click.echo(f'Artifact UUID: {uuid}')
else:
with open(ti.artifact_path / 'metadata.json', 'w') as f:
json.dump(ti.metadata, f, indent=2)
click.echo(f'LOCAL MODE: Not uploading model to S3. Model is located at: {ti.artifact_path}')
# stop, terminate, or do nothing to ec2 based on policy
ec2_policy = ti.config.get('ec2_policy')
# check if the policy is to stop or terminate
if ec2_policy == None or ec2_policy == 'stop' or ec2_policy == 'terminate':
policy_str = ec2_policy if ec2_policy else 'default'
click.echo(f'Checking for EC2 instance and applying policy "{policy_str}"...')
try:
# grab ec2 id
with urlopen(EC2_INSTANCE_ID_URL, timeout=5) as url:
ec2_instance_id = url.read().decode('utf-8')
click.echo(f'EC2 Runtime detected.')
client = boto3.client('ec2')
# default is stop
if ec2_policy == None or ec2_policy == 'stop':
click.echo("Stopping...")
client.stop_instances(InstanceIds=[ec2_instance_id], DryRun=False)
else:
click.echo("Terminating...")
client.terminate_instances(InstanceIds=[ec2_instance_id], DryRun=False)
except URLError:
click.echo('No EC2 runtime detected. Doing nothing.')
else:
click.echo('Not checking for EC2 runtime since policy is to keep running.')
return result