pass
DEPLOYMENT_TYPES_MAP = collections.OrderedDict(
(("TFSERVING", "Tensorflow Serving on K8s"),
# ("GRADIENT", "Gradient Jobs"),
))
DEPLOYMENT_MACHINE_TYPES = ("G1", "G6", "G12", "K80", "P100", "GV100")
@deployments.command("create", help="Create new deployment")
@click.option(
"--deploymentType",
"deploymentType",
type=ChoiceType(DEPLOYMENT_TYPES_MAP, case_sensitive=False),
required=True,
help=
"Model deployment type. Only TensorFlow models can currently be deployed",
)
@click.option(
"--modelId",
"modelId",
required=True,
help="ID of a trained model",
)
@click.option(
"--name",
"name",
required=True,
help="Human-friendly name for new model deployment",
def common_experiment_create_multi_node_options(f):
options = [
click.option(
"--experimentType",
"experimentTypeId",
type=ChoiceType(MULTI_NODE_EXPERIMENT_TYPES_MAP, case_sensitive=False),
required=True,
help="Experiment Type",
),
click.option(
"--workerContainer",
"workerContainer",
required=True,
help="Worker container",
),
click.option(
"--workerMachineType",
"workerMachineType",
required=True,
help="Worker machine type",
),
click.option(
"--workerCommand",
"workerCommand",
required=True,
help="Worker command",
),
click.option(
"--workerCount",
"workerCount",
type=int,
required=True,
help="Worker count",
),
click.option(
"--parameterServerContainer",
"parameterServerContainer",
required=True,
help="Parameter server container",
),
click.option(
"--parameterServerMachineType",
"parameterServerMachineType",
required=True,
help="Parameter server machine type",
),
click.option(
"--parameterServerCommand",
"parameterServerCommand",
required=True,
help="Parameter server command",
),
click.option(
"--parameterServerCount",
"parameterServerCount",
type=int,
required=True,
help="Parameter server count",
),
click.option(
"--workerContainerUser",
"workerContainerUser",
help="Worker container user",
),
click.option(
"--workerRegistryUsername",
"workerRegistryUsername",
help="Worker container registry username",
),
click.option(
"--workerRegistryPassword",
"workerRegistryPassword",
help="Worker registry password",
),
click.option(
"--parameterServerContainerUser",
"parameterServerContainerUser",
help="Parameter server container user",
),
click.option(
"--parameterServerRegistryContainerUser",
"parameterServerRegistryContainerUser",
help="Parameter server registry container user",
),
click.option(
"--parameterServerRegistryPassword",
"parameterServerRegistryPassword",
help="Parameter server registry password",
),
]
return functools.reduce(lambda x, opt: opt(x), reversed(options), f)
def deployments_tags():
pass
@deployments_group.group(name="metrics",
help="Read model deployment metrics",
cls=ClickGroup)
def deployments_metrics():
pass
@deployments_group.command("create", help="Create new deployment")
@click.option(
"--deploymentType",
"deployment_type",
type=ChoiceType(DEPLOYMENT_TYPES_MAP, case_sensitive=False),
required=True,
help="Model deployment type",
cls=common.GradientOption,
)
@click.option(
"--projectId",
"project_id",
help="Project ID",
cls=common.GradientOption,
)
@click.option(
"--modelId",
"model_id",
help="ID of a trained model",
cls=common.GradientOption,
"stream",
short_help="Watch live notebook metrics",
help="Watch live notebook metrics. Shows CPU and RAM usage by default",
)
@click.option(
"--id",
"notebook_id",
required=True,
cls=common.GradientOption,
help="ID of the notebook",
)
@click.option(
"--metric",
"metrics_list",
multiple=True,
type=ChoiceType(constants.METRICS_MAP, case_sensitive=False),
default=(constants.BuiltinMetrics.cpu_percentage,
constants.BuiltinMetrics.memory_usage),
help=
"One or more metrics that you want to read. Defaults to cpuPercentage and memoryUsage",
cls=common.GradientOption,
)
@click.option(
"--interval",
"interval",
default="30s",
help="Interval",
cls=common.GradientOption,
)
@api_key_option
@common.options_file
@cli.group("machines", help="Manage machines", cls=ClickGroup)
def machines_group():
pass
check_machine_availability_help = "Get machine availability for the given region and machine type. " \
"Note: availability is only provided for the dedicated GPU machine types. " \
"Also, not all machine types are available in all regions"
@machines_group.command("availability", help=check_machine_availability_help)
@click.option(
"--region",
"region",
type=ChoiceType(REGIONS_MAP, case_sensitive=False),
required=True,
help="Name of the region",
)
@click.option(
"--machineType",
"machine_type",
type=click.Choice(constants.MACHINE_TYPES),
required=True,
help="Machine type",
)
@api_key_option
def check_machine_availability(region, machine_type, api_key):
machines_api = http_client.API(config.CONFIG_HOST, api_key=api_key)
command = machines_commands.CheckAvailabilityCommand(api=machines_api)
command.execute(region, machine_type)
@models_group.command("create",
help="Create a model from an url or dataset id")
@click.option(
"--name",
"name",
required=True,
help="Model name",
cls=common.GradientOption,
)
@click.option(
"--modelType",
"model_type",
required=True,
type=ChoiceType(constants.MODEL_TYPES_MAP, case_sensitive=False),
help="Model type",
cls=common.GradientOption,
)
@click.option(
"--datasetRef",
"dataset_ref",
required=True,
help="Dataset ref to associate a model with",
cls=common.GradientOption,
)
@click.option(
"--projectId",
"project_id",
help="ID of a project",
cls=common.GradientOption,
def common_experiment_create_multi_node_options(f):
options = [
click.option(
"--experimentType",
"experiment_type_id",
type=ChoiceType(constants.MULTI_NODE_EXPERIMENT_TYPES_MAP,
case_sensitive=False),
required=True,
help="Experiment Type",
cls=common.GradientOption,
),
click.option(
"--workerContainer",
"worker_container",
metavar="<container>",
required=True,
help="Worker container",
cls=common.GradientOption,
),
click.option(
"--workerMachineType",
"worker_machine_type",
metavar="<machine type>",
required=True,
help="Worker machine type",
cls=common.GradientOption,
),
click.option(
"--workerCommand",
"worker_command",
metavar="<command>",
required=True,
help="Worker command",
cls=common.GradientOption,
),
click.option(
"--workerCount",
"worker_count",
type=int,
required=True,
help="Worker count",
cls=common.GradientOption,
),
click.option(
"--parameterServerContainer",
"parameter_server_container",
metavar="<container>",
help="Parameter server container (GRPC only)",
cls=common.GradientOption,
),
click.option(
"--parameterServerMachineType",
"parameter_server_machine_type",
metavar="<machine type>",
help="Parameter server machine type (GRPC only)",
cls=common.GradientOption,
),
click.option(
"--parameterServerCommand",
"parameter_server_command",
metavar="<command>",
help="Parameter server command (GRPC only)",
cls=common.GradientOption,
),
click.option(
"--parameterServerCount",
"parameter_server_count",
type=int,
help="Parameter server count (GRPC only)",
cls=common.GradientOption,
),
click.option(
"--masterContainer",
"master_container",
metavar="<container>",
help="Master container (MPI only)",
cls=common.GradientOption,
),
click.option(
"--masterMachineType",
"master_machine_type",
metavar="<machine type>",
help="Master machine type (MPI only)",
cls=common.GradientOption,
),
click.option(
"--masterCount",
"master_count",
help="Master count (MPI only)",
cls=common.GradientOption,
),
click.option(
"--masterCommand",
"master_command",
metavar="<command>",
help="Master command (MPI only)",
cls=common.GradientOption,
),
click.option(
"--workerContainerUser",
"worker_container_user",
help="Worker container user",
cls=common.GradientOption,
),
click.option(
"--workerRegistryUsername",
"worker_registry_username",
help="Worker container registry username",
cls=common.GradientOption,
),
click.option(
"--workerRegistryPassword",
"worker_registry_password",
metavar="<password>",
help="Worker registry password",
cls=common.GradientOption,
),
click.option(
"--workerRegistryUrl",
"worker_registry_url",
metavar="<registry url>",
help="Worker registry URL",
cls=common.GradientOption,
),
click.option(
"--parameterServerContainerUser",
"parameter_server_container_user",
help="Parameter server container user",
cls=common.GradientOption,
),
click.option(
"--parameterServerRegistryUsername",
"parameter_server_registry_username",
help="Parameter server registry username",
cls=common.GradientOption,
),
click.option(
"--parameterServerRegistryPassword",
"parameter_server_registry_password",
metavar="<password>",
help="Parameter server registry password",
cls=common.GradientOption,
),
click.option(
"--parameterServerRegistryUrl",
"parameter_server_registry_url",
metavar="<registry url>",
help="Parameter server registry URL",
cls=common.GradientOption,
),
click.option(
"--masterContainerUser",
"master_container_user",
help="Master container user (MPI only)",
cls=common.GradientOption,
),
click.option(
"--masterRegistryUsername",
"master_registry_username",
metavar="<username>",
help="Master registry username (MPI only)",
cls=common.GradientOption,
),
click.option(
"--masterRegistryPassword",
"master_registry_password",
metavar="<password>",
help="Master registry password (MPI only)",
cls=common.GradientOption,
),
click.option("--masterRegistryUrl",
"master_registry_url",
metavar="<registry url>",
help="Master registry URL (MPI only)",
cls=common.GradientOption),
]
return functools.reduce(lambda x, opt: opt(x), reversed(options), f)
def dataset_options(f):
options = [
click.option(
"--datasetId",
"dataset_id_list",
metavar="<dateset id>",
multiple=True,
help="Dataset ID",
cls=common.GradientDatasetOption,
),
click.option(
"--datasetUri",
"dataset_uri_list",
metavar="<dateset uri>",
multiple=True,
help="Url to S3 bucket with dataset",
cls=common.GradientDatasetOption,
),
click.option(
"--datasetName",
"dataset_name_list",
multiple=True,
metavar="<dateset name>",
help="Name of dataset",
cls=common.GradientDatasetOption,
),
click.option(
"--datasetAwsAccessKeyId",
"dataset_access_key_id_list",
multiple=True,
metavar="<AWS access key>",
help="S3 bucket's Access Key ID",
cls=common.GradientDatasetOption,
),
click.option(
"--datasetAwsSecretAccessKey",
"dataset_secret_access_key_list",
multiple=True,
help="S3 bucket's Secret Access Key",
cls=common.GradientDatasetOption,
),
click.option(
"--datasetAwsEndpoint",
"dataset_endpoint_list",
multiple=True,
help="S3 endpoint URL",
cls=common.GradientDatasetOption,
),
click.option(
"--datasetVersionId",
"dataset_version_id_list",
metavar="<version ID>",
multiple=True,
help="S3 dataset's version ID",
cls=common.GradientDatasetOption,
),
click.option(
"--datasetEtag",
"dataset_etag_list",
metavar="<etag>",
multiple=True,
help="S3 dataset's ETag",
cls=common.GradientDatasetOption,
),
click.option(
"--datasetVolumeKind",
"dataset_volume_kind_list",
multiple=True,
type=ChoiceType(constants.DATASET_VOLUME_KINDS,
case_sensitive=False),
help=
"S3 dataset's volume kind. If used, --datasetVolumeSize has to be set as well",
cls=common.GradientDatasetOption,
),
click.option(
"--datasetVolumeSize",
"dataset_volume_size_list",
multiple=True,
help="S3 dataset's volume size",
cls=common.GradientDatasetOption,
),
]
return functools.reduce(lambda x, opt: opt(x), reversed(options), f)