def testSecurityPolicy(self):
security_policy_name = _UniqueName('my-policy')
with self._SecurityPolicy(security_policy_name):
# Update the security policy
self.result_file_path = os.path.join(self.temp_path, 'exported')
self.Run('compute security-policies export {0}'
' --file-name "{1}" --file-format yaml'.format(
security_policy_name, self.result_file_path))
security_policy = yaml.load_path(self.result_file_path)
self.assertEqual('', security_policy['description'])
self.assertEqual(1, len(security_policy['rules']))
default_rule = security_policy['rules'][0]
self.assertEqual('default rule', default_rule['description'])
self.assertEqual(2147483647, default_rule['priority'])
self.assertEqual('SRC_IPS_V1',
default_rule['match']['versionedExpr'])
self.assertEqual('*',
default_rule['match']['config']['srcIpRanges'][0])
self.assertEqual('allow', default_rule['action'])
self.assertEqual(False, default_rule['preview'])
security_policy['description'] = 'new description'
security_policy['rules'] = []
with open(self.result_file_path, 'w') as export_file:
security_policies_utils.WriteToFile(export_file,
security_policy, 'json')
self.Run('compute security-policies import {0}'
' --file-name "{1}" --file-format yaml'.format(
security_policy_name, self.result_file_path))
self.Run('compute security-policies export {0}'
' --file-name "{1}" --file-format json'.format(
security_policy_name, self.result_file_path))
security_policy = yaml.load_path(self.result_file_path)
self.assertEqual('new description', security_policy['description'])
self.assertEqual(1, len(security_policy['rules']))
default_rule = security_policy['rules'][0]
self.assertEqual('default rule', default_rule['description'])
self.assertEqual(2147483647, default_rule['priority'])
self.assertEqual('SRC_IPS_V1',
default_rule['match']['versionedExpr'])
self.assertEqual('*',
default_rule['match']['config']['srcIpRanges'][0])
self.assertEqual('allow', default_rule['action'])
self.assertEqual(False, default_rule['preview'])
def ParseBasicLevelConditions(path):
"""Parse a YAML representation of basic level conditions..
Args:
path: str, path to file containing basic level conditions
Returns:
list of Condition objects.
Raises:
ParseError: if the file could not be read into the proper object
"""
data = yaml.load_path(path)
if not data:
raise ParseError(path, 'File is empty')
messages = util.GetMessages()
message_class = messages.Condition
try:
conditions = [encoding.DictToMessage(c, message_class) for c in data]
except Exception as err:
raise InvalidFormatError(path, str(err), message_class)
_ValidateAllFieldsRecognized(path, conditions)
return conditions
def ParseBasicLevelConditions(path):
"""Parse a YAML representation of basic level conditions..
Args:
path: str, path to file containing basic level conditions
Returns:
list of Condition objects.
Raises:
ParseError: if the file could not be read into the proper object
"""
data = yaml.load_path(path)
if not data:
raise ParseError(path, 'File is empty')
messages = util.GetMessages()
message_class = messages.Condition
try:
conditions = [encoding.DictToMessage(c, message_class) for c in data]
except Exception as err:
raise InvalidFormatError(path, str(err), message_class)
_ValidateAllFieldsRecognized(path, conditions)
return conditions
def ParseYamlToRole(file_path, role_message_type):
"""Construct an IAM Role protorpc.Message from a Yaml formatted file.
Args:
file_path: Path to the Yaml IAM Role file.
role_message_type: Role message type to convert Yaml to.
Returns:
a protorpc.Message of type role_message_type filled in from the Yaml
role file.
Raises:
BadFileException if the Yaml file is malformed or does not exist.
"""
role_to_parse = yaml.load_path(file_path)
if 'stage' in role_to_parse:
role_to_parse['stage'] = role_to_parse['stage'].upper()
try:
role = encoding.PyValueToMessage(role_message_type, role_to_parse)
except (AttributeError) as e:
# Raised when the YAML file is not properly formatted YAML role file.
raise gcloud_exceptions.BadFileException(
'Role file {0} is not a properly formatted YAML role file. {1}'.
format(file_path, str(e)))
except (apitools_messages.DecodeError, binascii.Error) as e:
# DecodeError is raised when etag is badly formatted (not proper Base64)
raise IamEtagReadError(
'The etag of role file {0} is not properly formatted. {1}'.format(
file_path, str(e)))
return role
def ParseServicePerimetersBase(path, version=None):
"""Parse a YAML representation of a list of Service Perimeters.
Args:
path: str, path to file containing service perimeters
version: str, api version of ACM to use for proto messages
Returns:
list of Service Perimeters objects.
Raises:
ParseError: if the file could not be read into the proper object
"""
data = yaml.load_path(path)
if not data:
raise ParseError(path, 'File is empty')
messages = util.GetMessages(version=version)
message_class = messages.ServicePerimeter
try:
conditions = [encoding.DictToMessage(c, message_class) for c in data]
except Exception as err:
raise InvalidFormatError(path, six.text_type(err), message_class)
_ValidateAllFieldsRecognized(path, conditions)
return conditions
def ParseWhoisContact(path):
if path is None:
return None
raw_contact = yaml.load_path(path)
messages = registrations.GetMessagesModule()
parsed_contact = messages.WhoisContact(**raw_contact)
return parsed_contact
def GetCredentialsConfigFromFile(filename):
"""Returns the JSON content of a credentials config file.
This function is useful when the content of a file need to be inspected first
before determining how to handle it (how to initialize the underlying
credentials). Only UTF-8 JSON files are supported.
Args:
filename (str): The filepath to the ADC file representing credentials.
Returns:
Optional(Mapping): The JSON content.
Raises:
BadCredentialFileException: If JSON parsing of the file fails.
"""
try:
# YAML is a superset of JSON.
content = yaml.load_path(filename)
except UnicodeDecodeError as e:
raise BadCredentialFileException(
'File {0} is not utf-8 encoded: {1}'.format(filename, e))
except yaml.YAMLParseError as e:
raise BadCredentialFileException(
'Could not read json file {0}: {1}'.format(filename, e))
# Require the JSON content to be an object.
# Credentials and configs are always objects.
if not isinstance(content, dict):
raise BadCredentialFileException(
'Could not read json file {0}'.format(filename))
return content
def _ProcessTagFromFile(self, tag_template_ref, tag_file):
"""Processes a tag file into the request."""
try:
tag = yaml.load_path(tag_file)
if not isinstance(tag, dict):
raise InvalidTagFileError(
'Error parsing tag file: [invalid format]')
except yaml.YAMLParseError as e:
raise InvalidTagFileError('Error parsing tag file: [{}]'.format(e))
tag_template = self.template_service.Get(
self.messages.DatacatalogProjectsLocationsTagTemplatesGetRequest(
name=tag_template_ref.RelativeName(), ))
field_to_field_type = {}
for additional_property in tag_template.fields.additionalProperties:
message_type = additional_property.value.type
field_to_field_type[additional_property.key] = (
self._GetFieldType(message_type))
additional_properties = []
for field_id, field_value in six.iteritems(tag):
if field_id not in field_to_field_type:
raise InvalidTagError(
'Error parsing tag file: [{}] is not a valid field.'.
format(field_id))
additional_properties.append(
self.messages.GoogleCloudDatacatalogV1Tag.FieldsValue.
AdditionalProperty(
key=field_id,
value=self._MakeTagField(field_to_field_type[field_id],
field_value),
))
return self.messages.GoogleCloudDatacatalogV1Tag.FieldsValue(
additionalProperties=additional_properties, )
def VersionedParseCustomLevel(path):
"""Parse a YAML representation of custom level conditions.
Args:
path: str, path to file containing custom level expression
Returns:
string of CEL expression.
Raises:
ParseError: if the file could not be read into the proper object
"""
data = yaml.load_path(path)
if not data:
raise ParseError(path, 'File is empty')
messages = util.GetMessages(version=api_version)
message_class = messages.Expr
try:
expr = encoding.DictToMessage(data, message_class)
except Exception as err:
raise InvalidFormatError(path, six.text_type(err), message_class)
_ValidateAllCustomFieldsRecognized(path, expr)
return expr
def ParseMessageFromYamlFile(path, message_type, error_message):
"""Parse a Yaml file.
Args:
path: Yaml file path. If path is None returns None.
message_type: Message type to parse YAML into.
error_message: Error message to print in case of parsing error.
Returns:
parsed message of type message_type.
"""
if path is None:
return None
raw_message = yaml.load_path(path)
try:
parsed_message = encoding.PyValueToMessage(message_type, raw_message)
except Exception as e:
# This error may be slightly different in Py2 and Py3.
raise exceptions.Error('{}: {}'.format(error_message, e))
unknown_fields = []
for message in encoding.UnrecognizedFieldIter(parsed_message):
outer_message = ''.join([edge.field + '.' for edge in message[0]])
unknown_fields += [outer_message + field for field in message[1]]
unknown_fields.sort()
if unknown_fields:
raise exceptions.Error(
('{}.\nProblematic fields: \'{}\'').format(error_message,
', '.join(unknown_fields)))
return parsed_message
def VersionedParseAccessLevels(path):
"""Parse a YAML representation of a list of Access Levels with basic/custom level conditions.
Args:
path: str, path to file containing basic/custom access levels
Returns:
list of Access Level objects.
Raises:
ParseError: if the file could not be read into the proper object
"""
data = yaml.load_path(path)
if not data:
raise ParseError(path, 'File is empty')
messages = util.GetMessages(version=api_version)
message_class = messages.AccessLevel
try:
levels = [encoding.DictToMessage(c, message_class) for c in data]
except Exception as err:
raise InvalidFormatError(path, six.text_type(err), message_class)
_ValidateAllLevelFieldsRecognized(path, levels)
return levels
def ParseYamlorJsonPolicyFile(policy_file_path, policy_message_type):
"""Create an IAM Policy protorpc.Message from a YAML or JSON formatted file.
Args:
policy_file_path: Path to the YAML or JSON IAM policy file.
policy_message_type: Policy message type to convert YAML to.
Returns:
a protorpc.Message of type policy_message_type filled in from the input
policy file.
Raises:
BadFileException if the YAML or JSON file is malformed.
IamEtagReadError if the etag is badly formatted.
"""
policy_to_parse = yaml.load_path(policy_file_path)
try:
policy = encoding.PyValueToMessage(policy_message_type, policy_to_parse)
except (AttributeError) as e:
# Raised when the input file is not properly formatted YAML policy file.
raise gcloud_exceptions.BadFileException(
'Policy file [{0}] is not a properly formatted YAML or JSON '
'policy file. {1}'
.format(policy_file_path, str(e)))
except (apitools_messages.DecodeError) as e:
# DecodeError is raised when etag is badly formatted (not proper Base64)
raise IamEtagReadError(
'The etag of policy file [{0}] is not properly formatted. {1}'
.format(policy_file_path, str(e)))
return policy
def ParseYamlToRole(file_path, role_message_type):
"""Construct an IAM Role protorpc.Message from a Yaml formatted file.
Args:
file_path: Path to the Yaml IAM Role file.
role_message_type: Role message type to convert Yaml to.
Returns:
a protorpc.Message of type role_message_type filled in from the Yaml
role file.
Raises:
BadFileException if the Yaml file is malformed or does not exist.
"""
role_to_parse = yaml.load_path(file_path)
if 'stage' in role_to_parse:
role_to_parse['stage'] = role_to_parse['stage'].upper()
try:
role = encoding.PyValueToMessage(role_message_type, role_to_parse)
except (AttributeError) as e:
# Raised when the YAML file is not properly formatted YAML role file.
raise gcloud_exceptions.BadFileException(
'Role file {0} is not a properly formatted YAML role file. {1}'
.format(file_path, str(e)))
except (apitools_messages.DecodeError) as e:
# DecodeError is raised when etag is badly formatted (not proper Base64)
raise IamEtagReadError(
'The etag of role file {0} is not properly formatted. {1}'
.format(file_path, str(e)))
return role
def _SetImagePush(skaffold_file, shared_docker):
"""Set build.local.push value in skaffold file.
Args:
skaffold_file: Skaffold file handle.
shared_docker: Boolean that is true if docker instance is shared between the
kubernetes cluster and local docker builder.
Yields:
Path of skaffold file with build.local.push value set to the proper value.
"""
# TODO(b/149935260): This function can be removed when
# https://github.com/GoogleContainerTools/skaffold/issues/3668 is resolved.
if not shared_docker:
# If docker is not shared, use the default value (false). There is no need
# to rewrite the skaffold file.
yield skaffold_file
else:
skaffold_yaml = yaml.load_path(skaffold_file.name)
local_block = yaml_helper.GetOrCreate(skaffold_yaml,
('build', 'local'))
local_block['push'] = False
with cross_platform_temp_file.NamedTempFile(
yaml.dump(skaffold_yaml)) as patched_skaffold_file:
yield patched_skaffold_file
def WithServiceYaml(self, yaml_path):
"""Overrides settings with service.yaml and returns a new Settings object."""
yaml_dict = yaml.load_path(yaml_path)
message = messages_util.DictToMessageWithErrorCheck(
yaml_dict, RUN_MESSAGES_MODULE.Service)
knative_service = k8s_service.Service(message, RUN_MESSAGES_MODULE)
replacements = {}
# Planned attributes in
# http://doc/1ah6LB9we-FSEhcBZ7_4XQlnOPClTyyQW_O3Q5WNUuJc#bookmark=id.j3st2l8a3s19
try:
[container] = knative_service.spec.template.spec.containers
except ValueError:
raise exceptions.Error(
'knative Service must have exactly one container.')
for var in container.env:
replacements.setdefault('env_vars', {})[var.name] = var.value
service_account_name = knative_service.spec.template.spec.serviceAccountName
if service_account_name:
replacements['credential'] = ServiceAccountSetting(
name=service_account_name)
return self.replace(**replacements)
def GetResourceAndUpdateFieldsFromFile(file_path, resource_message_type):
"""Returns the resource message and update fields in file."""
try:
resource_to_parse = yaml.load_path(file_path)
except yaml.YAMLParseError as e:
raise exceptions.BadFileException(
'Policy config file [{0}] cannot be parsed. {1}'.format(
file_path, six.text_type(e)))
except yaml.FileLoadError as e:
raise exceptions.BadFileException(
'Policy config file [{0}] cannot be opened or read. {1}'.format(
file_path, six.text_type(e)))
if not isinstance(resource_to_parse, dict):
raise exceptions.BadFileException(
'Policy config file [{0}] is not a properly formatted YAML or JSON '
'file.'.format(file_path))
update_fields = list(resource_to_parse.keys())
try:
resource = encoding.PyValueToMessage(resource_message_type,
resource_to_parse)
except (AttributeError) as e:
raise exceptions.BadFileException(
'Policy config file [{0}] is not a properly formatted YAML or JSON '
'file. {1}'.format(file_path, six.text_type(e)))
return (resource, update_fields)
def ParseYamlorJsonPolicyFile(policy_file_path, policy_message_type):
"""Create an IAM Policy protorpc.Message from a YAML or JSON formatted file.
Args:
policy_file_path: Path to the YAML or JSON IAM policy file.
policy_message_type: Policy message type to convert YAML to.
Returns:
a protorpc.Message of type policy_message_type filled in from the input
policy file.
Raises:
BadFileException if the YAML or JSON file is malformed.
IamEtagReadError if the etag is badly formatted.
"""
policy_to_parse = yaml.load_path(policy_file_path)
try:
policy = encoding.PyValueToMessage(policy_message_type,
policy_to_parse)
except (AttributeError) as e:
# Raised when the input file is not properly formatted YAML policy file.
raise gcloud_exceptions.BadFileException(
'Policy file [{0}] is not a properly formatted YAML or JSON '
'policy file. {1}'.format(policy_file_path, str(e)))
except (apitools_messages.DecodeError) as e:
# DecodeError is raised when etag is badly formatted (not proper Base64)
raise IamEtagReadError(
'The etag of policy file [{0}] is not properly formatted. {1}'.
format(policy_file_path, str(e)))
return policy
def _LoadTestConfig():
"""Loads test config based on environment variable setting."""
env_config_file = encoding.GetEncodedValue(os.environ,
'CLOUD_SDK_TEST_CONFIG')
if not env_config_file:
env_config_file = cli_test_base.CliTestBase.Resource(
'tests', 'lib', 'e2e', 'integration_test_config.yaml')
if not os.path.isfile(env_config_file):
return None
elif not os.path.isfile(env_config_file):
if os.path.isabs(env_config_file):
resource_config_file = None
else:
# Relative path and does not exists wrt to current dir.
# Check wrt to project root.
resource_config_file = cli_test_base.CliTestBase.Resource(
env_config_file)
if os.path.isfile(resource_config_file):
env_config_file = resource_config_file
else:
resource_config_file = None
if resource_config_file is None:
raise ValueError(
'CLOUD_SDK_TEST_CONFIG env var set to non-existent '
'file {}'.format(env_config_file))
return yaml.load_path(env_config_file)
def _LoadData(path):
try:
return yaml.load_path(path)
except yaml.FileLoadError as err:
raise ParseError(path, 'Problem loading file: {}'.format(err))
except yaml.YAMLParseError as err:
raise ParseError(path, 'Problem parsing data as YAML: {}'.format(err))
def FromArgs(cls, args):
"""Build TrainingCustomInputServerConfig from argparse.Namespace."""
tier = args.scale_tier
if not tier:
if args.config:
data = yaml.load_path(args.config)
tier = data.get('trainingInput', {}).get('scaleTier', None)
parsed_tier = ScaleTierFlagMap().GetEnumForChoice(tier)
return cls(
scale_tier=parsed_tier,
runtime_version=args.runtime_version,
master_machine_type=args.master_machine_type,
master_image_uri=args.master_image_uri,
master_accelerator_type=(args.master_accelerator.get('type')
if args.master_accelerator else None),
master_accelerator_count=(args.master_accelerator.get('count')
if args.master_accelerator else None),
parameter_machine_type=args.parameter_server_machine_type,
parameter_machine_count=args.parameter_server_count,
parameter_image_uri=args.parameter_server_image_uri,
parameter_accelerator_type=args.parameter_server_accelerator.get(
'type') if args.parameter_server_accelerator else None,
parameter_accelerator_count=args.parameter_server_accelerator.get(
'count') if args.parameter_server_accelerator else None,
worker_machine_type=args.worker_machine_type,
worker_machine_count=args.worker_count,
worker_image_uri=args.worker_image_uri,
work_accelerator_type=(args.worker_accelerator.get('type')
if args.worker_accelerator else None),
work_accelerator_count=(args.worker_accelerator.get('count')
if args.worker_accelerator else None))
def ParseYamlOrJsonPolicyFile(policy_file_path, policy_message_type):
"""Create an IAM Policy protorpc.Message from a YAML or JSON formatted file.
Returns the parsed policy object and FieldMask derived from input dict.
Args:
policy_file_path: Path to the YAML or JSON IAM policy file.
policy_message_type: Policy message type to convert YAML to.
Returns:
a tuple of (policy, updateMask) where policy is a protorpc.Message of type
policy_message_type filled in from the JSON or YAML policy file and
updateMask is a FieldMask containing policy fields to be modified, based on
which fields are present in the input file.
Raises:
BadFileException if the YAML or JSON file is malformed.
IamEtagReadError if the etag is badly formatted.
"""
policy_to_parse = yaml.load_path(policy_file_path)
try:
policy = encoding.PyValueToMessage(policy_message_type,
policy_to_parse)
update_mask = ','.join(sorted(policy_to_parse.keys()))
except (AttributeError) as e:
# Raised when the input file is not properly formatted YAML policy file.
raise gcloud_exceptions.BadFileException(
'Policy file [{0}] is not a properly formatted YAML or JSON '
'policy file. {1}'.format(policy_file_path, str(e)))
except (apitools_messages.DecodeError, binascii.Error) as e:
# DecodeError is raised when etag is badly formatted (not proper Base64)
raise IamEtagReadError(
'The etag of policy file [{0}] is not properly formatted. {1}'.
format(policy_file_path, str(e)))
return (policy, update_mask)
def LoadFromFile(cls, filename):
try:
data = yaml.load_path(filename)
except yaml.Error as error:
raise Error('unable to load kubeconfig for {0}: {1}'.format(
filename, error.inner_error))
cls._Validate(data)
return cls(data, filename)
def ReadYaml(file_path, message_type):
parsed_yaml = yaml.load_path(file_path)
try:
message = encoding.PyValueToMessage(message_type, parsed_yaml)
except Exception as e:
raise exceptions.ParseError(
'Cannot parse YAML from file {0}: [{1}]'.format(file_path, e))
return message
def testWriteEnvYaml(self):
env = {'hello': 'world', 'foo': 'bar'}
output_dir = self.CreateTempDir()
util.WriteEnvYaml(env, output_dir)
env_file_path = os.path.join(output_dir, 'env.yaml')
yaml_env = yaml.load_path(env_file_path)
self.assertEqual(env, yaml_env)
def LoadFromFile(cls, filename):
try:
data = yaml.load_path(filename)
except yaml.Error as error:
raise Error('unable to load kubeconfig for {0}: {1}'.format(
filename, error.inner_error))
cls._Validate(data)
return cls(data, filename)
def LoadYamlFromPath(path):
try:
data = yaml.load_path(path)
except yaml.Error as e:
raise cloudbuild_exceptions.ParserError(path, e.inner_error)
if not yaml.dict_like(data):
raise cloudbuild_exceptions.ParserError(
path, "Could not parse as a dictionary.")
return data
def Create(self,
config_path,
display_name,
parent=None,
max_trial_count=None,
parallel_trial_count=None,
algorithm=None,
kms_key_name=None):
"""Creates a hyperparameter tuning job with given parameters.
Args:
config_path: str, the file path of the hyperparameter tuning job
configuration.
display_name: str, the display name of the created hyperparameter tuning
job.
parent: str, parent of the created hyperparameter tuning job. e.g.
/projects/xxx/locations/xxx/
max_trial_count: int, the desired total number of Trials. The default
value is 1.
parallel_trial_count: int, the desired number of Trials to run in
parallel. The default value is 1.
algorithm: AlgorithmValueValuesEnum, the search algorithm specified for
the Study.
kms_key_name: A customer-managed encryption key to use for the
hyperparameter tuning job.
Returns:
Created hyperparameter tuning job.
"""
job_spec = self.messages.GoogleCloudAiplatformV1beta1HyperparameterTuningJob(
)
if config_path:
data = yaml.load_path(config_path)
if data:
job_spec = messages_util.DictToMessageWithErrorCheck(
data, self.messages.
GoogleCloudAiplatformV1beta1HyperparameterTuningJob)
job_spec.maxTrialCount = max_trial_count
job_spec.parallelTrialCount = parallel_trial_count
if display_name:
job_spec.displayName = display_name
if algorithm and job_spec.studySpec:
job_spec.studySpec.algorithm = algorithm
if kms_key_name is not None:
job_spec.encryptionSpec = self.messages.GoogleCloudAiplatformV1beta1EncryptionSpec(
kmsKeyName=kms_key_name)
return self._service.Create(
self.messages.
AiplatformProjectsLocationsHyperparameterTuningJobsCreateRequest(
parent=parent,
googleCloudAiplatformV1beta1HyperparameterTuningJob=job_spec))
def Create(self, location_ref, args):
"""Creates a model deployment monitoring job."""
endpoint_ref = _ParseEndpoint(args.endpoint, location_ref)
job_spec = self.messages.GoogleCloudAiplatformV1beta1ModelDeploymentMonitoringJob(
)
if args.monitoring_config_from_file:
data = yaml.load_path(args.monitoring_config_from_file)
if data:
job_spec = messages_util.DictToMessageWithErrorCheck(
data, self.messages.
GoogleCloudAiplatformV1beta1ModelDeploymentMonitoringJob)
else:
job_spec.modelDeploymentMonitoringObjectiveConfigs = self._ConstructObjectiveConfigForCreate(
location_ref, endpoint_ref.RelativeName(),
args.feature_thresholds, args.dataset, args.bigquery_uri,
args.data_format, args.gcs_uris, args.target_field,
args.training_sampling_rate)
job_spec.endpoint = endpoint_ref.RelativeName()
job_spec.displayName = args.display_name
job_spec.modelMonitoringAlertConfig = self.messages.GoogleCloudAiplatformV1beta1ModelMonitoringAlertConfig(
emailAlertConfig=self.messages.
GoogleCloudAiplatformV1beta1ModelMonitoringAlertConfigEmailAlertConfig(
userEmails=args.emails))
job_spec.loggingSamplingStrategy = self.messages.GoogleCloudAiplatformV1beta1SamplingStrategy(
randomSampleConfig=self.messages.
GoogleCloudAiplatformV1beta1SamplingStrategyRandomSampleConfig(
sampleRate=args.prediction_sampling_rate))
job_spec.modelDeploymentMonitoringScheduleConfig = self.messages.GoogleCloudAiplatformV1beta1ModelDeploymentMonitoringScheduleConfig(
monitorInterval='{}s'.format(
six.text_type(3600 * int(args.monitoring_frequency))))
if args.predict_instance_schema:
job_spec.predictInstanceSchemaUri = args.predict_instance_schema
if args.analysis_instance_schema:
job_spec.analysisInstanceSchemaUri = args.analysis_instance_schema
if args.log_ttl:
job_spec.logTtl = '{}s'.format(
six.text_type(86400 * int(args.log_ttl)))
if args.sample_predict_request:
instance_json = model_monitoring_jobs_util.ReadInstanceFromArgs(
args.sample_predict_request)
job_spec.samplePredictInstance = encoding.PyValueToMessage(
extra_types.JsonValue, instance_json)
return self._service.Create(
self.messages.
AiplatformProjectsLocationsModelDeploymentMonitoringJobsCreateRequest(
parent=location_ref.RelativeName(),
googleCloudAiplatformV1beta1ModelDeploymentMonitoringJob=
job_spec))
def GetRuntimes(args):
"""Gets a list of unique runtimes that the user is about to run.
Args:
args: A list of arguments (typically sys.argv).
Returns:
A set of runtime strings. If python27 and libraries section is populated
in any of the yaml-files, 'python27-libs', a fake runtime id, will be part
of the set, in conjunction with the original 'python27'.
Raises:
MultipleAppYamlError: The supplied application configuration has duplicate
app yamls.
"""
runtimes = set()
for arg in args:
# Check all the arguments to see if they're application yaml files or
# directories that include yaml files.
yaml_candidate = None
if (os.path.isfile(arg)
and os.path.splitext(arg)[1] in _YAML_FILE_EXTENSIONS):
yaml_candidate = arg
elif os.path.isdir(arg):
for extension in _YAML_FILE_EXTENSIONS:
fullname = os.path.join(arg, 'app' + extension)
if os.path.isfile(fullname):
if yaml_candidate:
raise MultipleAppYamlError(
'Directory "{0}" contains conflicting files {1}'.
format(arg, ' and '.join(yaml_candidate)))
yaml_candidate = fullname
if yaml_candidate:
try:
info = yaml.load_path(yaml_candidate)
except yaml.Error:
continue
# safe_load can return arbitrary objects, we need a dict.
if not isinstance(info, dict):
continue
# Grab the runtime from the yaml, if it exists.
if 'runtime' in info:
runtime = info.get('runtime')
if type(runtime) == str:
if runtime == 'python27' and info.get('libraries'):
runtimes.add('python27-libs')
runtimes.add(runtime)
if runtime in _WARNING_RUNTIMES:
log.warn(_WARNING_RUNTIMES[runtime])
elif os.path.isfile(os.path.join(arg, 'WEB-INF', 'appengine-web.xml')):
# For unstanged Java App Engine apps, which may not have any yaml files.
runtimes.add('java')
return runtimes
def ParseFrom(cls, filename):
"""Parse local config from filename."""
config = yaml.load_path(filename)
# TODO(b/78124357): Implement validation in more general way.
for field in config.keys():
if field not in _VALID_FIELDS_SET:
raise ConfigError('Invalid field {} in {}'.format(field, filename))
return LocalConfig(config.get(_SERVICE, None), config.get(_REGION, None))
def Create(self,
parent,
specs=None,
config_path=None,
display_name=None,
python_package_uri=None):
"""Constructs a request and sends it to the endpoint to create a custom job instance."""
if not python_package_uri:
python_package_uri = []
job_spec = self.messages.GoogleCloudAiplatformV1beta1CustomJobSpec()
if config_path:
data = yaml.load_path(config_path)
if data:
job_spec = encoding.DictToMessage(
data, self.messages.GoogleCloudAiplatformV1beta1CustomJobSpec)
worker_pool_specs = []
for spec in specs:
machine_type = spec.get('machine-type')
if not spec.get('replica-count'):
replica_count = 1
else:
replica_count = int(spec.get('replica-count'))
container_image_uri = spec.get('container-image-uri')
python_image_uri = spec.get('python-image-uri')
python_module = spec.get('python-module')
machine_spec = self.messages.GoogleCloudAiplatformV1beta1MachineSpec(
machineType=machine_type)
worker_pool_spec = self.messages.GoogleCloudAiplatformV1beta1WorkerPoolSpec(
replicaCount=replica_count, machineSpec=machine_spec)
if container_image_uri:
worker_pool_spec.containerSpec = self.messages.GoogleCloudAiplatformV1beta1ContainerSpec(
imageUri=container_image_uri)
# TODO(b/161753810): Pass args and commands to the python package
# and container.
if python_package_uri or python_image_uri or python_module:
worker_pool_spec.pythonPackageSpec = self.messages.GoogleCloudAiplatformV1beta1PythonPackageSpec(
executorImageUri=python_image_uri,
packageUris=python_package_uri,
pythonModule=python_module)
worker_pool_specs.append(worker_pool_spec)
if worker_pool_specs:
job_spec.workerPoolSpecs = worker_pool_specs
validation.ValidateWorkerPoolSpec(job_spec.workerPoolSpecs)
custom_job = self.messages.GoogleCloudAiplatformV1beta1CustomJob(
displayName=display_name, jobSpec=job_spec)
return self._service.Create(
self.messages.AiplatformProjectsLocationsCustomJobsCreateRequest(
parent=parent, googleCloudAiplatformV1beta1CustomJob=custom_job))
def GetResourceAndUpdateFieldsFromFile(file_path, resource_message_type):
try:
resource_to_parse = yaml.load_path(file_path)
update_fields = list(resource_to_parse.keys())
resource = encoding.PyValueToMessage(resource_message_type,
resource_to_parse)
return (resource, update_fields)
except (AttributeError) as e:
raise exceptions.BadFileException(
'Policy config file [{0}] is not a properly formatted YAML or JSON '
'file. {1}'.format(file_path, str(e)))
def testEmptyDefaultKubeconfig(self):
path = kconfig.Kubeconfig.DefaultPath()
file_utils.MakeDir(os.path.dirname(path))
with open(path, 'w') as fp:
fp.write('')
self.assertTrue(os.path.isfile(path))
kconfig.Kubeconfig.Default()
data = yaml.load_path(path)
self.assertIsNotNone(data)
with open(path, 'r') as fp:
self.assertEqual(fp.read(), _EMPTY_KUBECONFIG)
def _ReadExplanationMetadata(self, explanation_metadata_file):
explanation_metadata = None
if not explanation_metadata_file:
raise gcloud_exceptions.BadArgumentException(
'--explanation-metadata-file',
'Explanation metadata file must be specified.')
# Yaml is a superset of json, so parse json file as yaml.
data = yaml.load_path(explanation_metadata_file)
if data:
explanation_metadata = messages_util.DictToMessageWithErrorCheck(
data, self.messages.GoogleCloudAiplatformV1beta1ExplanationMetadata)
return explanation_metadata
def ParseTemplate(template_file, params=None, params_from_file=None):
"""Parse and apply params into a template file.
Args:
template_file: The path to the file to open and parse.
params: a dict of param-name -> param-value
params_from_file: a dict of param-name -> param-file
Returns:
The parsed template dict
Raises:
yaml.Error: When the template file cannot be read or parsed.
ToolException: If any params are not provided or the YAML file is invalid.
"""
params = params or {}
params_from_file = params_from_file or {}
joined_params = dict(params)
for key, file_path in params_from_file.iteritems():
if key in joined_params:
raise exceptions.ToolException('Duplicate param key: ' + key)
try:
with open(file_path) as opened_file:
joined_params[key] = opened_file.read()
except IOError as e:
raise exceptions.ToolException(
'Could not load param key "{0}" from file "{1}": {2}'.format(
key, file_path, e.strerror))
template = yaml.load_path(template_file)
if not isinstance(template, dict) or 'template' not in template:
raise exceptions.ToolException(
'Invalid template format. Root must be a mapping with single '
'"template" value')
(template, missing_params, used_params) = ReplaceTemplateParams(
template, joined_params)
if missing_params:
raise exceptions.ToolException(
'Some parameters were present in the template but not provided on '
'the command line: ' + ', '.join(sorted(missing_params)))
unused_params = set(joined_params.keys()) - used_params
if unused_params:
raise exceptions.ToolException(
'Some parameters were specified on the command line but not referenced '
'in the template: ' + ', '.join(sorted(unused_params)))
return template
def AddOptions(messages, options_file, type_provider):
"""Parse api options from the file and add them to type_provider.
Args:
messages: The API message to use.
options_file: String path expression pointing to a type-provider options
file.
type_provider: A TypeProvider message on which the options will be set.
Returns:
The type_provider after applying changes.
Raises:
exceptions.ConfigError: the api options file couldn't be parsed as yaml
"""
if not options_file:
return type_provider
yaml_content = yaml.load_path(options_file)
if yaml_content:
if 'collectionOverrides' in yaml_content:
type_provider.collectionOverrides = []
for collection_override_data in yaml_content['collectionOverrides']:
collection_override = messages.CollectionOverride(
collection=collection_override_data['collection'])
if 'options' in collection_override_data:
collection_override.options = _OptionsFrom(
messages, collection_override_data['options'])
type_provider.collectionOverrides.append(collection_override)
if 'options' in yaml_content:
type_provider.options = _OptionsFrom(messages, yaml_content['options'])
if 'credential' in yaml_content:
type_provider.credential = _CredentialFrom(
messages, yaml_content['credential'])
return type_provider
def BuildVersion(self, name,
path=None,
deployment_uri=None,
runtime_version=None,
labels=None,
machine_type=None,
description=None,
framework=None):
"""Create a Version object.
The object is based on an optional YAML configuration file and the
parameters to this method; any provided method parameters override any
provided in-file configuration.
The file may only have the fields given in
VersionsClientBase._ALLOWED_YAML_FIELDS specified; the only parameters
allowed are those that can be specified on the command line.
Args:
name: str, the name of the version object to create.
path: str, the path to the YAML file.
deployment_uri: str, the deploymentUri to set for the Version
runtime_version: str, the runtimeVersion to set for the Version
labels: Version.LabelsValue, the labels to set for the version
machine_type: str, the machine type to serve the model version on.
description: str, the version description.
framework: FrameworkValueValuesEnum, the ML framework used to train this
version of the model.
Returns:
A Version object (for the corresponding API version).
Raises:
InvalidVersionConfigFile: If the file contains unexpected fields.
"""
version = self.version_class()
if path:
try:
data = yaml.load_path(path)
except (yaml.Error) as err:
raise InvalidVersionConfigFile(
'Could not read Version configuration file [{path}]:\n\n'
'{err}'.format(path=path, err=str(err.inner_error)))
if data:
version = encoding.DictToMessage(data, self.version_class)
specified_fields = set([f.name for f in version.all_fields() if
getattr(version, f.name)])
invalid_fields = (specified_fields - self._ALLOWED_YAML_FIELDS |
set(version.all_unrecognized_fields()))
if invalid_fields:
raise InvalidVersionConfigFile(
'Invalid {noun} [{fields}] in configuration file [{path}]. '
'Allowed fields: [{allowed}].'.format(
noun=text.Pluralize(len(invalid_fields), 'field'),
fields=', '.join(sorted(invalid_fields)),
path=path,
allowed=', '.join(sorted(self._ALLOWED_YAML_FIELDS))))
additional_fields = {
'name': name,
'deploymentUri': deployment_uri,
'runtimeVersion': runtime_version,
'labels': labels,
'machineType': machine_type,
'description': description,
'framework': framework
}
for field_name, value in additional_fields.items():
if value is not None:
setattr(version, field_name, value)
return version
def BuildTrainingJob(self,
path=None,
module_name=None,
job_name=None,
trainer_uri=None,
region=None,
job_dir=None,
scale_tier=None,
user_args=None,
runtime_version=None,
labels=None):
"""Builds a Cloud ML Engine Job from a config file and/or flag values.
Args:
path: path to a yaml configuration file
module_name: value to set for moduleName field (overrides yaml file)
job_name: value to set for jobName field (overrides yaml file)
trainer_uri: List of values to set for trainerUri field (overrides yaml
file)
region: compute region in which to run the job (overrides yaml file)
job_dir: Cloud Storage working directory for the job (overrides yaml
file)
scale_tier: ScaleTierValueValuesEnum the scale tier for the job
(overrides yaml file)
user_args: [str]. A list of arguments to pass through to the job.
(overrides yaml file)
runtime_version: the runtime version in which to run the job (overrides
yaml file)
labels: Job.LabelsValue, the Cloud labels for the job
Returns:
A constructed Job object.
"""
job = self.job_class()
if path:
data = yaml.load_path(path)
if data:
job = encoding.DictToMessage(data, self.job_class)
if job_name:
job.jobId = job_name
if labels is not None:
job.labels = labels
if not job.trainingInput:
job.trainingInput = self.training_input_class()
additional_fields = {
'pythonModule': module_name,
'args': user_args,
'packageUris': trainer_uri,
'region': region,
'jobDir': job_dir,
'scaleTier': scale_tier,
'runtimeVersion': runtime_version
}
for field_name, value in additional_fields.items():
if value is not None:
setattr(job.trainingInput, field_name, value)
return job
