def get_pipeline_and_version_fields_from_yaml(yaml_content):
"""
Get pipeline and pipeline version fields from the given yaml file. Remove the pipeline_ and version_ prefix so that
the fields can be used to create/update pipelines and versions.
:param dict yaml_content: the content of the yaml
:return dict, dict, dict: a dictionary containing pipeline, pipeline input and pipeline version parameters
"""
pipeline_fields, input_fields, output_fields = define_pipeline(
yaml_content, pipeline_name=None)
version_fields = dict()
if yaml_content:
for p in PIPELINE_VERSION_FIELDS:
# Rename 'version_{p}' to '{p}'
input_field = PIPELINE_VERSION_FIELDS_RENAMED[
p] if p in PIPELINE_VERSION_FIELDS_RENAMED else p
version_fields[p] = set_dict_default(
None,
yaml_content,
input_field,
set_type=PIPELINE_VERSION_FIELD_TYPES[p]
if p in PIPELINE_VERSION_FIELD_TYPES else str)
return pipeline_fields, input_fields, output_fields, version_fields
def set_pipeline_version_defaults(fields, yaml_content, existing_version=None):
"""
Define pipeline version fields.
For each field i in PIPELINE_VERSION_FIELDS:
- Use value in fields if specified
- If not; Use value in yaml content if specified
- If not; Use existing version (this is only used for updating, not creating)
:param dict fields: the command options
:param dict yaml_content: the content of the yaml
:param ubiops.PipelineVersion existing_version: the current pipeline version if exists
:return dict: a dictionary containing all PipelineVersion parameters
"""
for k, v in PIPELINE_VERSION_FIELDS_RENAMED.items():
fields[k] = fields[v]
del fields[v]
for k in [k for k, v in PIPELINE_VERSION_FIELD_TYPES.items() if v == dict]:
if k in fields and fields[k] is not None:
fields[k] = strings_to_dict(fields[k])
if yaml_content:
for p in PIPELINE_VERSION_FIELDS:
# Rename 'version_{p}' to '{p}'
input_field = PIPELINE_VERSION_FIELDS_RENAMED[
p] if p in PIPELINE_VERSION_FIELDS_RENAMED else p
value = fields[input_field] if input_field in fields else None
fields[p] = set_dict_default(
value,
yaml_content,
input_field,
set_type=PIPELINE_VERSION_FIELD_TYPES[p]
if p in PIPELINE_VERSION_FIELD_TYPES else str)
if existing_version:
for p in PIPELINE_VERSION_FIELDS:
value = fields[p] if p in fields else None
fields[p] = set_object_default(value, existing_version, p)
return fields
def define_deployment_version(fields, yaml_content, extra_yaml_fields):
"""
Define deployment version fields
For each field i in [DEPLOYMENT_VERSION_FIELDS + extra_yaml_fields]:
- Use value in fields if specified
- If not; Use value in yaml content if specified
- If not; Use existing version (this is only used for updating, not creating)
Rename field-key if the key is in DEPLOYMENT_VERSION_FIELDS_RENAMED.values(). This is done to
solve inconsistencies between CLI options/yaml keys and API parameters.
:param dict fields: the command options
:param dict yaml_content: the content of the yaml
:param list(str) extra_yaml_fields: additional yaml fields that are not DeploymentVersion parameters,
e.g., deployment_file
:return dict: a dictionary containing all DeploymentVersion parameters (+extra yaml fields)
"""
for k, yaml_key in DEPLOYMENT_VERSION_FIELDS_RENAMED.items():
fields[k] = fields.pop(yaml_key, None)
for k in [
k for k, v in DEPLOYMENT_VERSION_FIELD_TYPES.items() if v == dict
]:
if k in fields and fields[k] is not None:
fields[k] = strings_to_dict(fields[k])
if yaml_content:
for p in [*DEPLOYMENT_VERSION_FIELDS, *extra_yaml_fields]:
yaml_key = DEPLOYMENT_VERSION_FIELDS_RENAMED[
p] if p in DEPLOYMENT_VERSION_FIELDS_RENAMED else p
value = fields[p] if p in fields else None
fields[p] = set_dict_default(
value,
yaml_content,
yaml_key,
set_type=DEPLOYMENT_VERSION_FIELD_TYPES[p]
if p in DEPLOYMENT_VERSION_FIELD_TYPES else str)
return fields
def pipeline_versions_create(pipeline_name, version_name, yaml_file, format_,
**kwargs):
"""
Create a version of a pipeline.
\b
It is possible to define the parameters using a yaml file.
For example:
```
pipeline_name: my-pipeline-name
version_name: my-pipeline-version
version_description: Version created via command line.
version_labels:
my-key-1: my-label-1
my-key-2: my-label-2
request_retention_mode: none
request_retention_time: 604800
objects:
- name: object1
reference_name: my-deployment-name
reference_version: my-deployment-version
attachments:
- destination_name: object1
sources:
- source_name: pipeline_start
mapping:
- source_field_name: my-pipeline-param1
destination_field_name: my-deployment-param1
```
Those parameters can also be provided as command options. If both a `<yaml_file>` is set and
options are given, the options defined by `<yaml_file>` will be overwritten by the specified command options.
The version name can either be passed as command argument or specified inside the yaml file using `<version_name>`.
"""
project_name = get_current_project(error=True)
yaml_content = read_yaml(yaml_file, required_fields=[])
client = init_client()
assert 'pipeline_name' in yaml_content or pipeline_name, \
'Please, specify the pipeline name in either the yaml file or as a command argument'
assert 'version_name' in yaml_content or version_name, \
'Please, specify the version name in either the yaml file or as a command argument'
pipeline_name = set_dict_default(pipeline_name, yaml_content,
'pipeline_name')
version_name = set_dict_default(version_name, yaml_content, 'version_name')
# Define the pipeline version
kwargs = set_pipeline_version_defaults(kwargs, yaml_content, None)
# Rename objects reference version
kwargs = rename_pipeline_object_reference_version(content=kwargs)
version = api.PipelineVersionCreate(
version=version_name,
**{k: kwargs[k]
for k in PIPELINE_VERSION_FIELDS})
response = client.pipeline_versions_create(project_name=project_name,
pipeline_name=pipeline_name,
data=version)
client.api_client.close()
print_item(item=response,
row_attrs=LIST_ITEMS,
rename={
'pipeline': 'pipeline_name',
'version': 'version_name',
**PIPELINE_VERSION_FIELDS_RENAMED
},
fmt=format_)
def define_pipeline(yaml_content, pipeline_name, current_pipeline_name=None):
"""
Define pipeline by api.PipelineCreate class.
- Pipeline name:
- Use pipeline_name if specified
- If not; Use pipeline_name in yaml content if specified
- If not; Use current_pipeline_name (this is only used for updating, not creating)
- Use yaml content for: description, input_type and input_fields
:param dict yaml_content: the content of the yaml
:param str/None pipeline_name: the name of the pipeline
:param str/None current_pipeline_name: the name of the current pipeline if it already exists
:return dict, dict, dict: the pipeline, and the pipeline input and output parameters
"""
pipeline_name = set_dict_default(pipeline_name, yaml_content,
'pipeline_name')
if pipeline_name is None and current_pipeline_name:
pipeline_name = current_pipeline_name
description = set_dict_default(None, yaml_content, 'pipeline_description')
if 'input_fields' in yaml_content and isinstance(
yaml_content['input_fields'], list):
input_fields = [
api.PipelineInputFieldCreate(name=item['name'],
data_type=item['data_type'])
for item in yaml_content['input_fields']
]
else:
input_fields = None
if 'output_fields' in yaml_content and isinstance(
yaml_content['output_fields'], list):
output_fields = [
api.PipelineOutputFieldCreate(name=item['name'],
data_type=item['data_type'])
for item in yaml_content['output_fields']
]
else:
output_fields = None
if 'pipeline_labels' in yaml_content:
labels = yaml_content['pipeline_labels']
else:
labels = {}
pipeline_data = {
'name': pipeline_name,
'description': description,
'labels': labels
}
input_data = {
'input_type': yaml_content['input_type'],
'input_fields': input_fields
}
output_data = {
'output_type': yaml_content['output_type'],
'output_fields': output_fields
}
return pipeline_data, input_data, output_data
def deployments_create(deployment_name, yaml_file, format_):
"""
Create a new deployment.
\b
Define the deployment parameters using a yaml file.
For example:
```
deployment_name: my-deployment-name
deployment_description: Deployment created via command line.
deployment_labels:
my-key-1: my-label-1
my-key-2: my-label-2
input_type: structured
input_fields:
- name: param1
data_type: int
- name: param2
data_type: string
output_type: plain
```
The deployment name can either be passed as argument or specified inside the yaml
file. If it is both passed as argument and specified inside the yaml file, the value
passed as argument is used.
Possible input/output types: [structured, plain]. Possible data_types: [blob, int,
string, double, bool, array_string, array_int, array_double].
"""
project_name = get_current_project(error=True)
yaml_content = read_yaml(yaml_file, required_fields=DEPLOYMENT_REQUIRED_FIELDS)
client = init_client()
assert 'deployment_name' in yaml_content or deployment_name, 'Please, specify the deployment name in either the ' \
'yaml file or as a command argument'
deployment_name = set_dict_default(deployment_name, yaml_content, 'deployment_name')
description = set_dict_default(None, yaml_content, 'deployment_description')
if 'input_fields' in yaml_content and isinstance(yaml_content['input_fields'], list):
input_fields = [api.DeploymentInputFieldCreate(name=item['name'], data_type=item['data_type'])
for item in yaml_content['input_fields']]
else:
input_fields = None
if 'output_fields' in yaml_content and isinstance(yaml_content['output_fields'], list):
output_fields = [api.DeploymentInputFieldCreate(name=item['name'], data_type=item['data_type'])
for item in yaml_content['output_fields']]
else:
output_fields = None
if 'deployment_labels' in yaml_content:
labels = yaml_content['deployment_labels']
else:
labels = {}
deployment = api.DeploymentCreate(
name=deployment_name,
description=description,
input_type=yaml_content['input_type'],
output_type=yaml_content['output_type'],
input_fields=input_fields,
output_fields=output_fields,
labels=labels
)
response = client.deployments_create(project_name=project_name, data=deployment)
client.api_client.close()
print_item(
item=response,
row_attrs=LIST_ITEMS,
required_front=['id', 'name', 'project', 'description', 'labels', 'input_type', 'output_type'],
optional=['input_fields name', 'input_fields data_type', 'output_fields name', 'output_fields data_type'],
required_end=['creation_date', 'last_updated'],
rename={'name': 'deployment_name', 'description': 'deployment_description', 'labels': 'deployment_labels'},
fmt=format_
)
def deployments_deploy(deployment_name, version_name, directory, output_path, yaml_file, overwrite, assume_yes, quiet,
**kwargs):
"""
Deploy a new version of a deployment.
Please, specify the code `<directory>` that should be deployed. The files in this directory
will be zipped and uploaded. Subdirectories and files that shouldn't be contained in the
ZIP can be specified in an ignore file, which is by default '.ubiops-ignore'. The structure of this
file is assumed to be equal to the wellknown '.gitignore' file.
If you want to store a local copy of the uploaded zip file, please use the `<output_path>` option.
The `<output_path>` option will be used as output location of the zip file. If the `<output_path>` is a
directory, the zip will be saved in `[deployment_name]_[deployment_version]_[datetime.now()].zip`. Use
the `<assume_yes>` option to overwrite without confirmation if file specified in `<output_path>` already exists.
Provide either deployment mode 'express' or 'batch', default is 'express'.
\b
It is possible to define the parameters using a yaml file.
For example:
```
deployment_name: my-deployment-name
version_name: my-deployment-version
version_description: Version created via command line.
version_labels:
my-key-1: my-label-1
my-key-2: my-label-2
language: python3.7
instance_type: 2048mb
minimum_instances: 0
maximum_instances: 1
maximum_idle_time: 300
request_retention_mode: none
request_retention_time: 604800
deployment_mode: express
```
Those parameters can also be provided as command options. If both a `<yaml_file>` is set and options are given,
the options defined by `<yaml_file>` will be overwritten by the specified command options. The deployment name can
either be passed as command argument or specified inside the yaml file using `<deployment_name>`.
It's not possible to update the programming language and deployment mode of an existing deployment version.
"""
if output_path is None:
store_zip = False
output_path = ''
else:
store_zip = True
project_name = get_current_project(error=True)
client = init_client()
yaml_content = read_yaml(yaml_file, required_fields=[])
assert 'deployment_name' in yaml_content or deployment_name, 'Please, specify the deployment name in either the ' \
'yaml file or as a command argument'
assert 'version_name' in yaml_content or version_name, 'Please, specify the version name in either the yaml ' \
'file or as a command option'
deployment_name = set_dict_default(deployment_name, yaml_content, 'deployment_name')
version_name = set_dict_default(version_name, yaml_content, 'version_name')
existing_version = None
if overwrite:
try:
existing_version = client.deployment_versions_get(
project_name=project_name, deployment_name=deployment_name, version=version_name
)
except api.exceptions.ApiException:
# Do nothing if version doesn't exist
pass
kwargs = define_deployment_version(
kwargs, yaml_content, extra_yaml_fields=['deployment_file', 'ignore_file']
)
kwargs['ignore_file'] = DEFAULT_IGNORE_FILE if kwargs['ignore_file'] is None else kwargs['ignore_file']
if not quiet and kwargs['memory_allocation'] and not kwargs['instance_type']:
click.secho(
"Deprecation warning: parameter 'memory_allocation' is deprecated, use 'instance_type' instead",
fg='red'
)
zip_path = zip_dir(
directory=directory,
output_path=output_path,
ignore_filename=kwargs['ignore_file'],
deployment_name=deployment_name,
version_name=version_name,
force=assume_yes
)
try:
has_uploaded_zips = False
has_changed_fields = False
if not (overwrite and existing_version):
version = api.DeploymentVersionCreate(
version=version_name, **{k: kwargs[k] for k in DEPLOYMENT_VERSION_FIELDS}
)
client.deployment_versions_create(project_name=project_name, deployment_name=deployment_name, data=version)
else:
revisions = client.revisions_list(
project_name=project_name, deployment_name=deployment_name, version=version_name
)
has_uploaded_zips = len(revisions) > 0
if overwrite and existing_version:
has_changed_fields = update_existing_deployment_version(
client, project_name, deployment_name, version_name, existing_version, kwargs
)
has_changed_env_vars = update_deployment_file(
client, project_name, deployment_name, version_name, kwargs['deployment_file']
)
if has_uploaded_zips and (has_changed_fields or has_changed_env_vars):
# Wait for changes being applied
click.echo("Waiting for changes to take effect... This takes %d seconds." % UPDATE_TIME)
sleep(UPDATE_TIME)
client.revisions_file_upload(
project_name=project_name, deployment_name=deployment_name, version=version_name, file=zip_path
)
client.api_client.close()
except Exception as e:
if os.path.isfile(zip_path) and not store_zip:
os.remove(zip_path)
client.api_client.close()
raise e
if os.path.isfile(zip_path):
if store_zip:
if not quiet:
click.echo("Created zip: %s" % zip_path)
else:
os.remove(zip_path)
if not quiet:
click.echo("Deployment was successfully deployed")
def versions_create(deployment_name, version_name, yaml_file, format_,
**kwargs):
"""Create a version of a deployment.
\b
It is possible to define the parameters using a yaml file.
For example:
```
deployment_name: my-deployment-name
version_name: my-deployment-version
version_description: Version created via command line.
version_labels:
my-key-1: my-label-1
my-key-2: my-label-2
language: python3.7
instance_type: 2048mb
minimum_instances: 0
maximum_instances: 1
maximum_idle_time: 300
request_retention_mode: none
request_retention_time: 604800
deployment_mode: express
```
Provide either deployment mode 'express' or 'batch', default is 'express'.
Those parameters can also be provided as command options. If both a `<yaml_file>` is set and
options are given, the options defined by `<yaml_file>` will be overwritten by the specified command options.
The version name can either be passed as command argument or specified inside the yaml file using `<version_name>`.
"""
project_name = get_current_project(error=True)
yaml_content = read_yaml(yaml_file, required_fields=[])
client = init_client()
assert 'deployment_name' in yaml_content or deployment_name, 'Please, specify the deployment name in either ' \
'the yaml file or as a command argument'
assert 'version_name' in yaml_content or version_name, 'Please, specify the version name in either ' \
'the yaml file or as a command argument'
kwargs = define_deployment_version(kwargs,
yaml_content,
extra_yaml_fields=['deployment_file'])
if format_ != 'json' and kwargs[
'memory_allocation'] and not kwargs['instance_type']:
click.secho(
"Deprecation warning: parameter 'memory_allocation' is deprecated, use 'instance_type' instead",
fg='red')
deployment_name = set_dict_default(deployment_name, yaml_content,
'deployment_name')
version_name = set_dict_default(version_name, yaml_content, 'version_name')
version = api.DeploymentVersionCreate(
version=version_name,
**{k: kwargs[k]
for k in DEPLOYMENT_VERSION_FIELDS})
response = client.deployment_versions_create(
project_name=project_name,
deployment_name=deployment_name,
data=version)
update_deployment_file(client, project_name, deployment_name, version_name,
kwargs['deployment_file'])
client.api_client.close()
print_item(item=response,
row_attrs=LIST_ITEMS,
rename={
'deployment': 'deployment_name',
'version': 'version_name',
**DEPLOYMENT_VERSION_FIELDS_RENAMED
},
fmt=format_)