def CreateDeployment(service_name,
image_name,
memory_limit=None,
cpu_limit=None,
cpu_request=None):
"""Create a deployment specification for a service.
Args:
service_name: Name of the service.
image_name: Image tag.
memory_limit: Container memory limit.
cpu_limit: Container cpu limit.
cpu_request: Container cpu request.
Returns:
Dictionary object representing the deployment yaml.
"""
deployment = yaml.load(_POD_TEMPLATE.format(service=service_name))
container = yaml.load(
_CONTAINER_TEMPLATE.format(service=service_name, image=image_name))
if memory_limit is not None:
limits = yaml_helper.GetOrCreate(container, ('resources', 'limits'))
limits['memory'] = memory_limit
if cpu_limit is not None:
limits = yaml_helper.GetOrCreate(container, ('resources', 'limits'))
limits['cpu'] = six.text_type(cpu_limit)
if cpu_request is not None:
requests = yaml_helper.GetOrCreate(container, ('resources', 'requests'))
requests['cpu'] = six.text_type(cpu_request)
containers = yaml_helper.GetOrCreate(
deployment, ('spec', 'template', 'spec', 'containers'), constructor=list)
containers.append(container)
return deployment
def testAddSecret(self):
yaml_text = textwrap.dedent("""\
apiVersion: v1
kind: Deployment
metadata:
name: my-service
labels:
service: my-service
spec:
template:
spec:
containers:
- name: my-service-container
image: image-name
env:
- name: PORT
value: "8080"
ports:
- containerPort: 8080
""")
deployment = yaml.load(yaml_text)
credential_generator = local.CredentialGenerator(lambda: None)
credential_generator.ModifyDeployment(deployment)
credential_generator.ModifyContainer(
deployment['spec']['template']['spec']['containers'][0])
expected_yaml_text = textwrap.dedent("""\
apiVersion: v1
kind: Deployment
metadata:
name: my-service
labels:
service: my-service
spec:
template:
spec:
containers:
- name: my-service-container
image: image-name
env:
- name: PORT
value: "8080"
- name: GOOGLE_APPLICATION_CREDENTIALS
value: /etc/local_development_credential/local_development_service_account.json
ports:
- containerPort: 8080
volumeMounts:
- mountPath: /etc/local_development_credential
name: local-development-credential
readOnly: true
volumes:
- name: local-development-credential
secret:
secretName: local-development-credential
""")
self.assertEqual(deployment, yaml.load(expected_yaml_text))
def ProcessScalingConfigsFile(scaling_configs_file):
"""Reads a JSON/YAML scaling_configs_file and returns collectiong of scaling configs object."""
try:
scaling_configs = json.loads(scaling_configs_file[0])
except ValueError as e:
try:
scaling_configs = yaml.load(scaling_configs_file[0])
except yaml.YAMLParseError as e:
raise InvalidSpecFileError(
'Error parsing scaling_configs file: [{}]'.format(e))
messages = utils.GetMessages()
message_class = messages.ScalingConfig
selector = messages.LabelSelector()
scaling_configs_message = []
try:
for sc in scaling_configs:
esc = encoding.DictToMessage(sc, message_class)
if not esc.selectors:
# Add default selector if not set
esc.selectors = [selector]
# Convert yaml to json
spec = yaml.load(esc.fleetAutoscalerSpec)
spec_as_json_str = json.dumps(spec)
esc.fleetAutoscalerSpec = spec_as_json_str
scaling_configs_message.append(esc)
except AttributeError:
raise InvalidSchemaError(
'Invalid schema: expected proper scaling configs')
except _messages.ValidationError as e:
# The most likely reason this is reaised is the file that is submitted is
# following new format (json/yaml without string blob) and we will parse
# with the new format
for sc in scaling_configs:
s = messages.ScalingConfig()
if 'selectors' in sc:
s.selectors = sc['selectors']
else:
# Add default selector if not set
s.selectors = [selector]
if 'name' in sc:
s.name = sc['name']
if 'schedules' in sc:
s.schedules = sc['schedules']
if 'fleetAutoscalerSpec' not in sc:
raise InvalidSchemaError(
'Invalid schema: expected proper scaling configs')
spec_as_json_str = json.dumps(sc['fleetAutoscalerSpec'])
s.fleetAutoscalerSpec = spec_as_json_str
scaling_configs_message.append(s)
return scaling_configs_message
def testIamCommands(self):
cryptokey = next(self.cryptokey_namer)
self.RunKms('keys', 'create', cryptokey, '--keyring', self.keyring,
'--location', self.glbl, '--purpose', 'encryption')
self.RunKms('keys', 'get-iam-policy', cryptokey, '--keyring',
self.keyring, '--location', self.glbl)
# default expected output for new cryptokey
self.AssertOutputContains('etag: ACAB')
self.ClearOutput()
policy_file = self.Touch(self.temp_path,
contents="""{{
"etag": "ACAB",
"bindings": [ {{ "members": ["serviceAccount:{0}"], "role": "roles/owner" }} ]
}}
""".format(self.Account()))
self.RunKms('keys', 'set-iam-policy', cryptokey, '--keyring',
self.keyring, '--location', self.glbl, policy_file)
etag = yaml.load(self.GetOutput())['etag']
self.ClearOutput()
files.WriteFileContents(policy_file, """{{
"etag": "{0}"
}}
""".format(etag))
self.RunKms('keys', 'set-iam-policy', cryptokey, '--keyring',
self.keyring, '--location', self.glbl, policy_file)
# "bindings" is not mentioned, so it should be unchanged.
self.AssertOutputContains("""bindings:
- members:
- serviceAccount:{0}
role: roles/owner
""".format(self.Account()))
etag = yaml.load(self.GetOutput())['etag']
self.ClearOutput()
files.WriteFileContents(
policy_file, """{{
"etag": "{0}",
"bindings": []
}}
""".format(etag))
self.RunKms('keys', 'set-iam-policy', cryptokey, '--keyring',
self.keyring, '--location', self.glbl, policy_file)
# "bindings" is set to [], so all entries should be removed.
self.AssertOutputNotContains('bindings:')
def ProcessFleetConfigsFile(fleet_configs_file):
"""Reads a JSON/YAML fleet_configs_file and returns collectiong of fleet configs object."""
try:
fleet_configs = json.loads(fleet_configs_file[0])
except ValueError as e:
try:
fleet_configs = yaml.load(fleet_configs_file[0])
except yaml.YAMLParseError as e:
raise InvalidSpecFileError(
'Error parsing fleet_configs file: [{}]'.format(e))
messages = utils.GetMessages()
message_class = messages.FleetConfig
fleet_configs_message = []
try:
for fc in fleet_configs:
f = encoding.DictToMessage(fc, message_class)
spec = yaml.load(f.fleetSpec)
spec_as_json_str = json.dumps(spec)
f.fleetSpec = spec_as_json_str
fleet_configs_message.append(f)
except AttributeError:
raise InvalidSchemaError(
'Invalid schema: expected proper fleet configs')
except _messages.ValidationError as e:
# The most likely reason this is reaised is the file that is submitted is
# following new format (json/yaml without string blob) and we will parse
# with the new format
for fc in fleet_configs:
f = messages.FleetConfig()
if 'name' in fc:
f.name = fc['name']
if 'fleetSpec' not in fc:
raise InvalidSchemaError(
'Invalid schema: expected proper fleet configs')
spec_as_json_str = json.dumps(fc['fleetSpec'])
f.fleetSpec = spec_as_json_str
fleet_configs_message.append(f)
unrecognized_field_paths = _GetUnrecognizedFieldPaths(
fleet_configs_message)
if unrecognized_field_paths:
error_msg_lines = [
'Invalid schema, the following fields are unrecognized:'
]
error_msg_lines += unrecognized_field_paths
raise InvalidSchemaError('\n'.join(error_msg_lines))
return fleet_configs_message
def ChangeFromYamlFile(yaml_file, api_version='v1'):
"""Returns the change contained in the given yaml file.
Args:
yaml_file: file, A yaml file with change.
api_version: [str], the api version to use for creating the change object.
Returns:
Change, the change contained in the given yaml file.
Raises:
CorruptedTransactionFileError: if the record_set_dictionaries are invalid
"""
messages = apis.GetMessagesModule('dns', api_version)
try:
change_dict = yaml.load(yaml_file) or {}
except yaml.YAMLParseError:
raise CorruptedTransactionFileError()
if (change_dict.get('additions') is None or
change_dict.get('deletions') is None):
raise CorruptedTransactionFileError()
change = messages.Change()
change.additions = _RecordSetsFromDictionaries(
messages, change_dict['additions'])
change.deletions = _RecordSetsFromDictionaries(
messages, change_dict['deletions'])
return change
def ProcessScalingConfigsFile(scaling_configs_file):
"""Reads a JSON/YAML scaling_configs_file and returns collectiong of scaling configs object."""
try:
scaling_configs = json.loads(scaling_configs_file[0])
except ValueError as e:
try:
scaling_configs = yaml.load(scaling_configs_file[0])
except yaml.YAMLParseError as e:
raise InvalidSpecFileError(
'Error parsing scaling_configs file: [{}]'.format(e))
messages = utils.GetMessages()
message_class = messages.ScalingConfig
try:
selector = messages.LabelSelector()
scaling_configs_message = []
for sc in scaling_configs:
esc = encoding.DictToMessage(sc, message_class)
if not esc.selectors:
# Add default selector if not set
esc.selectors = [selector]
scaling_configs_message.append(esc)
except AttributeError:
raise InvalidSchemaError(
'Invalid schema: expected proper scaling configs')
except _messages.ValidationError as e:
# Unfortunately apitools doesn't provide a way to get the path to the
# invalid field here.
raise InvalidSchemaError('Invalid schema: [{}]'.format(e))
return scaling_configs_message
def _GetYamlImports(import_object):
"""Extract the import section of a file.
Args:
import_object: The object in which to look for imports.
Returns:
A list of dictionary objects, containing the keys 'path' and 'name' for each
file to import. If no name was found, we populate it with the value of path.
Raises:
ConfigError: If we cannont read the file, the yaml is malformed, or
the import object does not contain a 'path' field.
"""
content = import_object.GetContent()
yaml_content = yaml.load(content)
imports = []
if yaml_content and IMPORTS in yaml_content:
imports = yaml_content[IMPORTS]
# Validate the yaml imports, and make sure the optional name is set.
for i in imports:
if PATH not in i:
raise exceptions.ConfigError(
'Missing required field %s in import in file %s.' %
(PATH, import_object.full_path))
# Populate the name field.
if NAME not in i:
i[NAME] = i[PATH]
return imports
def _GetYamlImports(import_object, globbing_enabled=False):
"""Extract the import section of a file.
If the glob_imports config is set to true, expand any globs (e.g. *.jinja).
Named imports cannot be used with globs that expand to more than one file.
If globbing is disabled or a glob pattern does not expand to match any files,
importer will use the literal string as the file path.
Args:
import_object: The object in which to look for imports.
globbing_enabled: If true, will resolved glob patterns dynamically.
Returns:
A list of dictionary objects, containing the keys 'path' and 'name' for each
file to import. If no name was found, we populate it with the value of path.
Raises:
ConfigError: If we cannont read the file, the yaml is malformed, or
the import object does not contain a 'path' field.
"""
parent_dir = None
if not _IsUrl(import_object.full_path):
parent_dir = os.path.dirname(os.path.abspath(import_object.full_path))
content = import_object.GetContent()
yaml_content = yaml.load(content)
imports = []
if yaml_content and IMPORTS in yaml_content:
raw_imports = yaml_content[IMPORTS]
# Validate the yaml imports, and make sure the optional name is set.
for i in raw_imports:
if PATH not in i:
raise exceptions.ConfigError(
'Missing required field %s in import in file %s.' %
(PATH, import_object.full_path))
glob_matches = []
# Only expand globs if config set and the path is a local fs reference.
if globbing_enabled and parent_dir and not _IsUrl(i[PATH]):
# Set our working dir to the import_object's for resolving globs.
with files.ChDir(parent_dir):
# TODO(b/111880973): Replace with gcloud glob supporting ** wildcards.
glob_matches = glob.glob(i[PATH])
glob_matches = _SanitizeWindowsPathsGlobs(glob_matches)
# Multiple file case.
if len(glob_matches) > 1:
if NAME in i:
raise exceptions.ConfigError((
'Cannot use import name %s for path glob in file %s that'
' matches multiple objects.') %
(i[NAME],
import_object.full_path))
imports.extend([{NAME: g, PATH: g} for g in glob_matches])
continue
# Single file case. (URL, discrete file, or single glob match)
if len(glob_matches) == 1:
i[PATH] = glob_matches[0]
# Populate the name field.
if NAME not in i:
i[NAME] = i[PATH]
imports.append(i)
return imports
def GetFileAsMessage(path, message):
"""Reads a YAML or JSON object of type message from local path.
Args:
path: A local path to an object specification in YAML or JSON format.
message: The message type to be parsed from the file.
Returns:
Object of type message, if successful.
Raises:
files.Error, exceptions.ResourceManagerInputFileError
"""
in_text = files.GetFileContents(path)
if not in_text:
raise exceptions.ResourceManagerInputFileError(
'Empty policy file [{0}]'.format(path))
# Parse it, first trying YAML then JSON.
try:
result = encoding.PyValueToMessage(message, yaml.load(in_text))
except (ValueError, AttributeError, yaml.YAMLParseError):
try:
result = encoding.JsonToMessage(message, in_text)
except (ValueError, DecodeError) as e:
# ValueError is raised when JSON is badly formatted
# DecodeError is raised when a tag is badly formatted (not Base64)
raise exceptions.ResourceManagerInputFileError(
'Policy file [{0}] is not properly formatted YAML or JSON '
'due to [{1}]'.format(path, str(e)))
return result
def LoadTPUResourceSpecs(custom_help=None):
"""Read Yaml resource file and return a dict mapping name to resource spec."""
resource_file_contents = pkg_resources.GetResource(TPU_YAML_RESOURCE_PATH,
'resources.yaml')
if not resource_file_contents:
raise calliope_exceptions.BadFileException(
'Resources not found in path [{}]'.format(TPU_YAML_RESOURCE_PATH))
resource_dict = yaml.load(resource_file_contents)
resource_specs = []
for resource_name in TPU_YAML_SPEC_TEMPLATE:
spec = resource_dict.get(resource_name, None)
if not spec:
raise ValueError(
'Resource spec [{}] not found in resource spec {}.yaml'.format(
resource_name, TPU_YAML_RESOURCE_PATH))
# Don't modify template
temp_spec = copy.deepcopy(TPU_YAML_SPEC_TEMPLATE[resource_name])
temp_spec['spec'] = spec
if custom_help and custom_help.get(resource_name):
temp_spec['help_text'] = custom_help[resource_name]
resource_specs.append(
resource_arg_schema.YAMLResourceArgument.FromData(temp_spec))
return resource_specs
def testLoadManifest(self):
self.Touch(self.temp_path,
'runtimes.yaml',
contents=ManifestTest.MANIFEST_FILE)
self.StartObjectPatch(storage_api.StorageClient,
'ReadObject',
side_effect=lambda x: io.BytesIO(
ManifestTest.MANIFEST_FILE.encode('utf-8')))
for builder_root in [_URLFromFile(self.temp_path), 'gs://mybucket']:
m = runtime_builders.Manifest.LoadFromURI(builder_root +
'/runtimes.yaml')
self.assertEqual(m._data, yaml.load(ManifestTest.MANIFEST_FILE))
self.assertEqual(
set(m.Runtimes()),
{'erlang', 'erlang-1', 'erlang-1.2', 'erlang-0', 'erlang-0.1'})
self.assertIsNone(m.GetBuilderReference('foo'))
self.assertEqual(
m.GetBuilderReference('erlang-1.2'),
runtime_builders.BuilderReference(
'erlang-1.2', builder_root + '/erlang-1.2-12345.yaml'))
self.assertEqual(
m.GetBuilderReference('erlang'),
runtime_builders.BuilderReference(
'erlang-1.2', builder_root + '/erlang-1.2-12345.yaml'))
self.assertEqual(
m.GetBuilderReference('erlang-0'),
runtime_builders.BuilderReference('erlang-0.1', None,
'erlang-0.1 is deprecated.'))
def testMakeProcess_Distributed(self):
package_dir = self.Resource('tests', 'unit', 'command_lib',
'ml_engine', 'test_data', 'package_root')
run_root = os.path.join(self.temp_path, 'run_dir')
shutil.copytree(package_dir, run_root)
module_name = 'test_package.test_task'
out = io.BytesIO()
args = ['foo']
cluster = {'distributed': ['address_1']}
stdout, _ = local_train.MakeProcess(
module_name,
run_root,
args=args,
task_type='distributed',
index=0,
cluster=cluster,
stdout=subprocess.PIPE).communicate()
out.write(stdout)
self.assertEqual(
yaml.load(out.getvalue()), {
'TF_CONFIG': {
'job': {
'job_name': module_name,
'args': args
},
'task': {
'type': 'distributed',
'index': 0
},
'cluster': cluster,
'environment': 'cloud',
},
'PWD': run_root,
'ARGS': ['foo']
})
def GetFileAsMessage(path, message):
"""Reads a YAML or JSON object of type message from local path.
Args:
path: A local path to an object specification in YAML or JSON format.
message: The message type to be parsed from the file.
Returns:
Object of type message, if successful.
Raises:
files.Error, exceptions.ResourceManagerInputFileError
"""
in_text = files.ReadFileContents(path)
if not in_text:
raise exceptions.ResourceManagerInputFileError(
'Empty policy file [{0}]'.format(path))
# Parse it, first trying YAML then JSON.
try:
result = encoding.PyValueToMessage(message, yaml.load(in_text))
except (ValueError, AttributeError, yaml.YAMLParseError):
try:
result = encoding.JsonToMessage(message, in_text)
except (ValueError, DecodeError) as e:
# ValueError is raised when JSON is badly formatted
# DecodeError is raised when a tag is badly formatted (not Base64)
raise exceptions.ResourceManagerInputFileError(
'Policy file [{0}] is not properly formatted YAML or JSON '
'due to [{1}]'.format(path, six.text_type(e)))
return result
def _ParseWeeklyCycleFromFile(args, messages):
"""Parses WeeklyCycle message from file contents specified in args."""
weekly_cycle_dict = yaml.load(args.weekly_cycle_from_file)
day_enum = messages.ResourcePolicyWeeklyCycleDayOfWeek.DayValueValuesEnum
days_of_week = []
for day_and_time in weekly_cycle_dict:
if 'day' not in day_and_time or 'startTime' not in day_and_time:
raise exceptions.InvalidArgumentException(
args.GetFlag('weekly_cycle_from_file'),
'Each JSON/YAML object in the list must have the following keys: '
'[day, startTime].')
day = day_and_time['day'].upper()
try:
weekday = times.Weekday.Get(day)
except KeyError:
raise exceptions.InvalidArgumentException(
args.GetFlag('weekly_cycle_from_file'),
'Invalid value for `day`: [{}].'.format(day))
start_time = arg_parsers.Datetime.ParseUtcTime(
day_and_time['startTime'])
day, start_time = _ParseWeeklyDayAndTime(start_time, weekday)
days_of_week.append(
messages.ResourcePolicyWeeklyCycleDayOfWeek(day=day_enum(day),
startTime=start_time))
return messages.ResourcePolicyWeeklyCycle(dayOfWeeks=days_of_week)
def ProcessFleetConfigsFile(fleet_configs_file):
"""Reads a JSON/YAML fleet_configs_file and returns collectiong of fleet configs object."""
try:
fleet_configs = json.loads(fleet_configs_file[0])
except ValueError as e:
try:
fleet_configs = yaml.load(fleet_configs_file[0])
except yaml.YAMLParseError as e:
raise InvalidSpecFileError(
'Error parsing fleet_configs file: [{}]'.format(e))
messages = utils.GetMessages()
message_class = messages.FleetConfig
try:
fleet_configs_message = [encoding.DictToMessage(fc, message_class)
for fc in fleet_configs]
except AttributeError:
raise InvalidSchemaError(
'Invalid schema: expected proper fleet configs')
except _messages.ValidationError as e:
# Unfortunately apitools doesn't provide a way to get the path to the
# invalid field here.
raise InvalidSchemaError('Invalid schema: [{}]'.format(e))
unrecognized_field_paths = _GetUnrecognizedFieldPaths(fleet_configs_message)
if unrecognized_field_paths:
error_msg_lines = ['Invalid schema, the following fields are unrecognized:']
error_msg_lines += unrecognized_field_paths
raise InvalidSchemaError('\n'.join(error_msg_lines))
return fleet_configs_message
def ListResources(self, project=None, organization=None, folder=None):
"""List all exportable resources.
If parent (e.g. project, organization or folder) is passed then only list
the exportable resources for that parent.
Args:
project: string, project to list exportable resources for.
organization: string, organization to list exportable resources for.
folder: string, folder to list exportable resources for.
Returns:
supported resources formatted output listing exportable resources.
"""
if not (project or organization or folder):
yaml_obj_list = yaml.load(
self._CallPrintResources(output_format='yaml'), round_trip=True)
return yaml_obj_list
if project:
msg_sfx = ' for project [{}]'.format(project)
elif organization:
msg_sfx = ' for organization [{}]'.format(organization)
else:
msg_sfx = ' for folder [{}]'.format(folder)
with progress_tracker.ProgressTracker(
message='Listing exportable resource types' + msg_sfx,
aborted_message='Aborted Export.'):
supported_kinds = self.ListSupportedResourcesForParent(
project=project, organization=organization, folder=folder)
supported_kinds = [x.AsDict() for x in supported_kinds]
return supported_kinds
def testBlockText(self):
data_string = """\
a:
b: "this\\nis\\nsomething\\nwith\\nnewlines"
c: d
"""
data = yaml.load(data_string, round_trip=True, version=yaml.VERSION_1_2)
self.assertEqual(
'a:\n b: "this\\nis\\nsomething\\nwith\\nnewlines"\n c: d\n',
yaml.dump(data, round_trip=True))
context = updates.Context(data, 'a', updates.Mode.RESULT)
context.ForKey('b').Update('this\nis\na\ndifferent\nthing\nwith\nnewlines',
[updates.Mode.RESULT])
self.assertEqual("""\
a:
b: |-
this
is
a
different
thing
with
newlines
c: d
""", yaml.dump(context.BackingData(), round_trip=True))
def ParseReplicationFileContents(file_contents):
"""Reads replication policy file contents and returns its data.
Reads the contents of a json or yaml replication policy file which conforms to
https://cloud.google.com/secret-manager/docs/reference/rest/v1/projects.secrets#replication
and returns data needed to create a Secret with that policy. If the file
doesn't conform to the expected format, a BadFileException is raised.
For Secrets with an automtic policy, locations is empty and keys has
either 0 or 1 entry depending on whether the policy includes CMEK. For Secrets
with a user managed policy, the number of keys returns is either 0 or is equal
to the number of locations returned with the Nth key corresponding to the Nth
location.
Args:
file_contents (str): The unvalidated contents fo the replication file.
Returns:
result (str): Either "user-managed" or "automatic".
locations (list): Locations that are part of the user-managed replication
keys (list): list of kms keys to be used for each replica.
"""
try:
replication_policy = json.loads(file_contents)
return _ParseReplicationDict(replication_policy)
except ValueError:
# Assume that this is yaml.
pass
try:
replication_policy = yaml.load(file_contents)
return _ParseReplicationDict(replication_policy)
except yaml.YAMLParseError:
raise exceptions.BadFileException(
'Failed to parse replication policy file as json or yaml.')
def ReadInstanceFromArgs(path):
"""Reads the instance from the given file path ('-' for stdin).
Args:
path: str or None, a path to a file ('-' for stdin) containing the JSON
body.
Returns:
A instance.
Raises:
InvalidInstancesFileError: If the input file is invalid (invalid format or
contains too many/zero instances), or an improper combination of input
files was given.
"""
data = console_io.ReadFromFileOrStdin(path, binary=True)
with io.BytesIO(data) as f:
try:
instance = yaml.load(f)
except ValueError:
raise errors.InvalidInstancesFileError(
'Input instance are not in JSON format. '
'See `gcloud ai model-monitoring-jobs create --help` for details.')
if not isinstance(instance, dict):
raise errors.InvalidInstancesFileError(
'Input instance are not in JSON format. '
'See `gcloud ai model-monitoring-jobs create --help` for details.')
return instance
def testMaybeWriteAppYaml_GeneratedAppInfo(self):
"""Tests that file exists message not printed if app.yaml doesn't exist."""
rt = ext_runtime.ExternalizedRuntime.Load(self.runtime_def_dir,
self.env)
runtime_config = yaml.load(
textwrap.dedent("""\
runtime: python
env: 2
entrypoint: run_me_some_python!
handlers:
- url: .*
script: request
"""))
params = ext_runtime.Params()
self.Touch(directory=self.temp_path,
name='exists',
contents='my contents')
configurator = rt.Detect(self.temp_path, params)
configurator.SetGeneratedAppInfo(runtime_config)
configurator.MaybeWriteAppYaml()
self.assertNotIn(
('print', ext_runtime.FILE_EXISTS_MESSAGE.format('app.yaml')),
self.log)
self.AssertFileExistsWithContents(yaml.dump(runtime_config),
self.temp_path, 'app.yaml')
def testRunStatus(self):
cluster_name = 'mock-cluster'
msg = self.features_api.messages
msg_fs = self.features_api.messages.FeatureState
lifecycle_state = (
msg_fs.LifecycleStateValueValuesEnum
.ENABLED)
details_by_membership = (msg_fs.DetailsByMembershipValue(
additionalProperties=[msg_fs.DetailsByMembershipValue.AdditionalProperty(
key='projects/mock-project/locations/global/memberships/mock-cluster',
value=msg.FeatureStateDetails(
configmanagementFeatureState=msg.ConfigManagementFeatureState(
clusterName=cluster_name)))]))
feature = self.features_api._MakeFeature(
configmanagementFeatureSpec=self.features_api.messages
.ConfigManagementFeatureSpec(),
featureState=self._MakeFeatureState(lifecycle_state,
details_by_membership))
self.features_api.ExpectGet(feature)
self.RunCommand(['status'])
out = yaml.load(self.GetOutput())
self.assertIsNotNone(out)
split_out = out.split()
log.warning(split_out)
header = ['Name', 'Status', 'Last_Synced_Token', 'Sync_Branch',
'Last_Synced_Time']
for i, _ in enumerate(header):
self.assertEqual(split_out[i], header[i])
self.assertEqual(split_out[len(header)], cluster_name)
def _CreateVersionResource(self, service_config, manifest, version_id, build,
extra_config_settings=None):
"""Constructs a Version resource for deployment.
Args:
service_config: ServiceYamlInfo, Service info parsed from a service yaml
file.
manifest: Dictionary mapping source files to Google Cloud Storage
locations.
version_id: str, The version of the service.
build: BuildArtifact, The build ID, image path, or build options.
extra_config_settings: dict, client config settings to pass to the server
as beta settings.
Returns:
A Version resource whose Deployment includes either a container pointing
to a completed image, or a build pointing to an in-progress build.
"""
parsed_yaml = service_config.parsed.ToYAML()
config_dict = yaml.load(parsed_yaml)
try:
# pylint: disable=protected-access
schema_parser = convert_yaml.GetSchemaParser(self.client._VERSION)
json_version_resource = schema_parser.ConvertValue(config_dict)
except ValueError, e:
raise exceptions.ConfigError(
'[{f}] could not be converted to the App Engine configuration '
'format for the following reason: {msg}'.format(
f=service_config.file, msg=e.message))
def ProcessConfigOverrideFile(config_override_file, api_version):
"""Reads a JSON/YAML config_override_file and returns collection of config override object."""
try:
overrides = json.loads(config_override_file)
except ValueError as e:
try:
overrides = yaml.load(config_override_file)
except yaml.YAMLParseError as e:
raise InvalidSpecFileError(
'Error parsing config_override file: [{}]'.format(e))
messages = GetMessages(api_version)
message_class = messages.GameServerConfigOverride
try:
overrides_message = [
encoding.DictToMessage(o, message_class) for o in overrides
]
except AttributeError:
raise InvalidSchemaError(
'Invalid schema: unexpected game server config override(s) format.')
except _messages.ValidationError as e:
# Unfortunately apitools doesn't provide a way to get the path to the
# invalid field here.
raise InvalidSchemaError('Invalid schema: [{}]'.format(e))
unrecognized_field_paths = _GetUnrecognizedFieldPaths(overrides_message)
if unrecognized_field_paths:
error_msg_lines = ['Invalid schema, the following fields are unrecognized:']
error_msg_lines += unrecognized_field_paths
raise InvalidSchemaError('\n'.join(error_msg_lines))
return overrides_message
def FromPath(cls, resource_path):
"""Constructs a ResourceYAMLData from a standard resource_path.
Args:
resource_path: string, the dotted path of the resources.yaml file, e.g.
iot.device or compute.instance.
Returns:
A ResourceYAMLData object.
Raises:
InvalidResourcePathError: invalid resource_path string.
"""
match = re.search(_RESOURCE_PATH_PATTERN, resource_path)
if not match:
raise InvalidResourcePathError(
'Invalid resource_path: [{}].'.format(resource_path))
surface_name = match.group('surface_name')
resource_name = match.group('resource_name')
# Gets the directory name of the targeted YAML file.
# Example: googlecloudsdk.command_lib.iot.
dir_name = _RESOURCE_FILE_PREFIX + surface_name + '.'
resource_file = pkg_resources.GetResource(dir_name,
_RESOURCE_FILE_NAME)
# Loads the data from YAML file.
resource_data = yaml.load(resource_file)[resource_name]
return cls(resource_data)
def ChangeFromYamlFile(yaml_file, api_version='v1'):
"""Returns the change contained in the given yaml file.
Args:
yaml_file: file, A yaml file with change.
api_version: [str], the api version to use for creating the change object.
Returns:
Change, the change contained in the given yaml file.
Raises:
CorruptedTransactionFileError: if the record_set_dictionaries are invalid
"""
messages = apis.GetMessagesModule('dns', api_version)
try:
change_dict = yaml.load(yaml_file) or {}
except yaml.YAMLParseError:
raise CorruptedTransactionFileError()
if (change_dict.get('additions') is None
or change_dict.get('deletions') is None):
raise CorruptedTransactionFileError()
change = messages.Change()
change.additions = _RecordSetsFromDictionaries(messages,
change_dict['additions'])
change.deletions = _RecordSetsFromDictionaries(messages,
change_dict['deletions'])
return change
def _ParseMysqlSourceConfig(self, mysql_source_config_file, release_track):
"""Parses a mysql_sorce_config into the MysqlSourceConfig message."""
data = console_io.ReadFromFileOrStdin(
mysql_source_config_file, binary=False)
try:
mysql_sorce_config_head_data = yaml.load(data)
except Exception as e:
raise ds_exceptions.ParseError('Cannot parse YAML:[{0}]'.format(e))
mysql_sorce_config_data_object = mysql_sorce_config_head_data.get(
'mysql_source_config')
mysql_rdbms_data = mysql_sorce_config_data_object if mysql_sorce_config_data_object else mysql_sorce_config_head_data
include_objects_raw = mysql_rdbms_data.get(
util.GetRDBMSV1alpha1ToV1FieldName('allowlist', release_track), {})
include_objects_data = util.ParseMysqlSchemasListToMysqlRdbmsMessage(
self._messages, include_objects_raw, release_track)
exclude_objects_raw = mysql_rdbms_data.get(
util.GetRDBMSV1alpha1ToV1FieldName('rejectlist', release_track), {})
exclude_objects_data = util.ParseMysqlSchemasListToMysqlRdbmsMessage(
self._messages, exclude_objects_raw, release_track)
mysql_sourec_config_msg = self._messages.MysqlSourceConfig(
includeObjects=include_objects_data,
excludeObjects=exclude_objects_data)
return mysql_sourec_config_msg
def ReadRequest(input_file):
"""Reads a JSON request from the specified input file.
Args:
input_file: An open file-like object for the input file.
Returns:
A json object from the input file
Raises:
InvalidInstancesFileError: If the input file is invalid.
"""
try:
request = yaml.load(input_file)
except ValueError:
raise errors.InvalidInstancesFileError(
'Input instances are not in JSON format. '
'See `gcloud ai endpoints predict --help` for details.')
if not isinstance(request, dict):
raise errors.InvalidInstancesFileError(
'Input instances are not in JSON format. '
'See `gcloud ai endpoints predict --help` for details.')
if 'instances' not in request:
raise errors.InvalidInstancesFileError(
'Invalid JSON request: missing "instances" attribute')
if not isinstance(request['instances'], list):
raise errors.InvalidInstancesFileError(
'Invalid JSON request: "instances" must be a list')
return request
def ReadYaml(message_type, stream, schema_path=None):
"""Read yaml from a stream as a message.
Args:
message_type: Type of message to interpret the yaml as.
stream: Stream from which yaml should be read.
schema_path: Path to schema used to validate yaml, relative to schemas dir.
Returns:
Message that was read.
Raises:
ParseError: if yaml could not be parsed as the given message type.
"""
parsed_yaml = yaml.load(stream)
if schema_path:
# If a schema is provided, validate against it.
try:
_ValidateYaml(parsed_yaml, schema_path)
except jsonschema_exceptions.ValidationError as e:
raise exceptions.ParseError('Validation Error: [{0}]'.format(
e.message))
try:
message = encoding.PyValueToMessage(message_type, parsed_yaml)
except Exception as e:
raise exceptions.ParseError('Cannot parse YAML: [{0}]'.format(e))
return message
def _ParseGcsDestinationConfig(self, gcs_destination_config_file):
"""Parses a gcs_destination_config into the GcsDestinationConfig message."""
data = console_io.ReadFromFileOrStdin(
gcs_destination_config_file, binary=False)
try:
gcs_destination_head_config_data = yaml.load(data)
except Exception as e:
raise ds_exceptions.ParseError('Cannot parse YAML:[{0}]'.format(e))
gcs_destination_config_data_object = gcs_destination_head_config_data.get(
'gcs_destination_config')
gcs_destination_config_data = gcs_destination_config_data_object if gcs_destination_config_data_object else gcs_destination_head_config_data
path = gcs_destination_config_data.get('path', '')
file_rotation_mb = gcs_destination_config_data.get('file_rotation_mb', {})
file_rotation_interval = gcs_destination_config_data.get(
'file_rotation_interval', {})
gcs_dest_config_msg = self._messages.GcsDestinationConfig(
path=path, fileRotationMb=file_rotation_mb,
fileRotationInterval=file_rotation_interval)
if 'avro_file_format' in gcs_destination_config_data:
gcs_dest_config_msg.avroFileFormat = self._messages.AvroFileFormat()
elif 'json_file_format' in gcs_destination_config_data:
json_file_format_data = gcs_destination_config_data.get(
'json_file_format')
gcs_dest_config_msg.jsonFileFormat = self._messages.JsonFileFormat(
compression=json_file_format_data.get('compression'),
schemaFileFormat=json_file_format_data.get('schema_file_format'))
else:
raise ds_exceptions.ParseError(
'Cannot parse YAML: missing file format.')
return gcs_dest_config_msg
def _PopulateDefaultAuthConfig(self, configmap_obj, cluster_defaults):
"""Populates the default eventing config and adds an annotation."""
auth_config = yaml.load(
configmap_obj.data.get('default-auth-config', '{}'))
auth_config['clusterDefaults'] = cluster_defaults
configmap_obj.data['default-auth-config'] = yaml.dump(auth_config)
configmap_obj.annotations[_CLUSTER_INITIALIZED_ANNOTATION] = 'true'
def ProcessSchemaFromFile(schema_file):
try:
schema = yaml.load(schema_file)
except yaml.YAMLParseError as e:
raise InvalidSchemaFileError(
'Error parsing schema file: [{}]'.format(e))
return _SchemaToMessage(schema)
def testRunDescribe(self):
config_membership = '{0}/memberships/golden-cluster'.format(
self.features_api.parent)
lifecycle_state = (self.features_api.messages.FeatureState.
LifecycleStateValueValuesEnum.ENABLED)
multiclusteringress_feature_spec = self._MakeMultiClusterIngressFeatureSpec(
config_membership=config_membership)
feature = self.features_api._MakeFeature(
multiclusteringressFeatureSpec=multiclusteringress_feature_spec,
featureState=self._MakeFeatureState(lifecycle_state))
self.features_api.ExpectGet(feature)
self.RunCommand(['describe'])
out = yaml.load(self.GetOutput())
self.assertIsNotNone(out)
kwargs = {
'multiclusteringressFeatureSpec': {
'configMembership': config_membership
},
'featureState': {
'lifecycleState': lifecycle_state.name
}
}
self.assertEqual(out, kwargs)
def TryYaml():
try:
return yaml.load(input_string)
except yaml.YAMLParseError as e:
if hasattr(e.inner_error, 'problem_mark'):
mark = e.inner_error.problem_mark
log.error('Service config YAML had an error at position (%s:%s)'
% (mark.line+1, mark.column+1))
def _LoadData(path):
try:
with open(path) as config_file:
return yaml.load(config_file)
except yaml.error.YAMLError as err:
raise ParseError(path, 'Problem parsing data as YAML: {}'.format(err))
except EnvironmentError as err:
raise ParseError(path, 'Problem loading file: {}'.format(err))
def LoadCloudbuildConfigFromStream(stream, messages, params=None,
path=None):
"""Load a cloudbuild config file into a Build message.
Args:
stream: file-like object containing the JSON or YAML data to be decoded
messages: module, The messages module that has a Build type.
params: dict, parameters to substitute into the Build spec.
path: str or None. Optional path to be used in error messages.
Raises:
NotFoundException: If the file does not exist.
ParserError: If there was a problem parsing the file.
BadConfigException: If the config file has illegal values.
Returns:
Build message, The build that got decoded.
"""
# Turn the data into a dict
try:
structured_data = yaml.load(stream, file_hint=path)
except yaml.Error as e:
raise ParserError(path, e.inner_error)
if not isinstance(structured_data, dict):
raise ParserError(path, 'Could not parse into a message.')
# Transform snake_case into camelCase.
structured_data = _SnakeToCamel(structured_data) # type: dict
# Then, turn the dict into a proto message.
try:
build = _UnpackCheckUnused(structured_data, messages.Build)
except Exception as e:
# Catch all exceptions here beacuse a valid YAML can sometimes not be a
# valid message, so we need to catch all errors in the dict to message
# conversion.
raise BadConfigException(path, '%s' % e)
subst = structured_data.get('substitutions', {})
# Validate the substitution keys in the message.
for key in subst.iterkeys():
if not _BUILTIN_SUBSTITUTION_REGEX.match(key):
raise BadConfigException(
path,
'config cannot specify built-in substitutions')
# Merge the substitutions passed in the flag.
if params:
subst.update(params)
build.substitutions = cloudbuild_util.EncodeSubstitutions(subst, messages)
# Some problems can be caught before talking to the cloudbuild service.
if build.source:
raise BadConfigException(path, 'config cannot specify source')
if not build.steps:
raise BadConfigException(path, 'config must list at least one step')
return build
def UpdateMetadata(metadata, args):
"""Update konlet metadata entry using user-supplied data."""
# precondition: metadata.key == GCE_CONTAINER_DECLARATION
manifest = yaml.load(metadata.value)
if args.IsSpecified('container_image'):
manifest['spec']['containers'][0]['image'] = args.container_image
if args.IsSpecified('container_command'):
manifest['spec']['containers'][0]['command'] = [args.container_command]
if args.IsSpecified('clear_container_command'):
manifest['spec']['containers'][0].pop('command', None)
if args.IsSpecified('container_arg'):
manifest['spec']['containers'][0]['args'] = args.container_arg
if args.IsSpecified('clear_container_args'):
manifest['spec']['containers'][0].pop('args', None)
if args.container_privileged is True:
manifest['spec']['containers'][0]['securityContext']['privileged'] = True
if args.container_privileged is False:
manifest['spec']['containers'][0]['securityContext']['privileged'] = False
_UpdateMounts(manifest, args.remove_container_mounts or [],
args.container_mount_host_path or [],
args.container_mount_tmpfs or [])
_UpdateEnv(manifest,
itertools.chain.from_iterable(args.remove_container_env or []),
args.container_env_file, args.container_env or [])
if args.container_stdin is True:
manifest['spec']['containers'][0]['stdin'] = True
if args.container_stdin is False:
manifest['spec']['containers'][0]['stdin'] = False
if args.container_tty is True:
manifest['spec']['containers'][0]['tty'] = True
if args.container_tty is False:
manifest['spec']['containers'][0]['tty'] = False
if args.IsSpecified('container_restart_policy'):
manifest['spec']['restartPolicy'] = RESTART_POLICY_API[
args.container_restart_policy]
metadata.value = yaml.dump(manifest)
def LoadFromURI(cls, uri):
"""Loads a manifest from a gs:// or file:// path.
Args:
uri: str, A gs:// or file:// URI
Returns:
Manifest, the loaded manifest.
"""
log.debug('Loading runtimes manifest from [%s]', uri)
with _Read(uri) as f:
data = yaml.load(f, file_hint=uri)
return cls(uri, data)
def GetFileAsMessage(path, message, client):
"""Reads a YAML or JSON object of type message from path (local or GCS).
Args:
path: A local or GCS path to an object specification in YAML or JSON format.
message: The message type to be parsed from the file.
client: The storage_v1 client to use.
Returns:
Object of type message, if successful.
Raises:
files.Error, genomics_exceptions.GenomicsInputFileError
"""
if path.startswith(GCS_PREFIX):
# Download remote file to a local temp file
tf = tempfile.NamedTemporaryFile(delete=False)
tf.close()
bucket, obj = _SplitBucketAndObject(path)
storage_messages = core_apis.GetMessagesModule('storage', 'v1')
get_request = storage_messages.StorageObjectsGetRequest(
bucket=bucket, object=obj)
try:
download = transfer.Download.FromFile(tf.name, overwrite=True)
client.objects.Get(get_request, download=download)
del download # Explicitly close the stream so the results are there
except apitools_exceptions.HttpError as e:
raise genomics_exceptions.GenomicsInputFileError(
'Unable to read remote file [{0}] due to [{1}]'.format(path, str(e)))
path = tf.name
# Read the file.
in_text = files.GetFileContents(path)
if not in_text:
raise genomics_exceptions.GenomicsInputFileError(
'Empty file [{0}]'.format(path))
# Parse it, first trying YAML then JSON.
try:
result = encoding.PyValueToMessage(message, yaml.load(in_text))
except (ValueError, AttributeError, yaml.YAMLParseError):
try:
result = encoding.JsonToMessage(message, in_text)
except (ValueError, DecodeError) as e:
# ValueError is raised when JSON is badly formatted
# DecodeError is raised when a tag is badly formatted (not Base64)
raise genomics_exceptions.GenomicsInputFileError(
'Pipeline file [{0}] is not properly formatted YAML or JSON '
'due to [{1}]'.format(path, str(e)))
return result
def ConstructUpdateMaskFromPolicy(policy_file_path):
"""Construct a FieldMask based on input policy.
Args:
policy_file_path: Path to the JSON or YAML IAM policy file.
Returns:
a FieldMask containing policy fields to be modified, based on which fields
are present in the input file.
"""
policy_file = files.GetFileContents(policy_file_path)
# Since json is a subset of yaml, parse file as yaml.
policy = yaml.load(policy_file)
# The IAM update mask should only contain top level fields. Sort the fields
# for testing purposes.
return ','.join(sorted(policy.keys()))
def ReadEnvYaml(output_dir):
"""Reads and returns the environment values in output_dir/env.yaml file.
Args:
output_dir: str, Path of directory containing the env.yaml to be read.
Returns:
env: {str: str}
"""
env_file_path = os.path.join(output_dir, 'env.yaml')
try:
with open(env_file_path, 'r') as env_file:
return yaml.load(env_file)
except IOError as err:
if err.errno == errno.ENOENT:
raise NoEnvYamlError(output_dir)
else:
raise err
def FlattenLayoutOutputs(manifest_layout):
"""Takes the layout from a manifest and returns the flattened outputs.
List output 'foo: [A,B]' becomes 'foo[0]: A, foo[1]: B'
Dict output 'bar: {a:1, b:2}' becomes 'bar[a]: 1, bar[b]: 2'
Lists and Dicts whose values are themselves lists or dicts are not expanded.
Args:
manifest_layout: The 'layout' field from the manifest.
Returns:
A list of {'name': X, 'finalValue': Y} dicts built out of the 'outputs'
section of the layout.
"""
layout = yaml.load(manifest_layout)
if not isinstance(layout, dict) or 'outputs' not in layout:
return [] # Empty list
outputs = []
basic_outputs = layout['outputs']
for basic_output in basic_outputs:
if 'finalValue' not in basic_output or 'name' not in basic_output:
continue # No value to process
name = basic_output['name']
value = basic_output['finalValue']
if isinstance(value, list):
for pos in range(len(value)):
final_name = '%s[%d]' % (name, pos)
outputs.append(_BuildOutput(final_name, value[pos]))
elif isinstance(value, dict):
for key in value:
final_name = '%s[%s]' % (name, key)
outputs.append(_BuildOutput(final_name, value[key]))
else:
outputs.append(_BuildOutput(name, value))
return outputs
def CreateLogMetric(metric_name, description=None, log_filter=None, data=None):
"""Returns a LogMetric message based on a data stream or a description/filter.
Args:
metric_name: str, the name of the metric.
description: str, a description.
log_filter: str, the filter for the metric's filter field.
data: str, a stream of data read from a config file.
Returns:
LogMetric, the message representing the new metric.
"""
messages = GetMessages()
if data:
contents = yaml.load(data)
metric_msg = encoding.DictToMessage(contents,
messages.LogMetric)
metric_msg.name = metric_name
else:
metric_msg = messages.LogMetric(name=metric_name,
description=description,
filter=log_filter)
return metric_msg
def UpdateLogMetric(metric, description=None, log_filter=None, data=None):
"""Updates a LogMetric message given description, filter, and/or data.
Args:
metric: LogMetric, the original metric.
description: str, updated description if any.
log_filter: str, updated filter for the metric's filter field if any.
data: str, a stream of data read from a config file if any.
Returns:
LogMetric, the message representing the updated metric.
"""
messages = GetMessages()
if description:
metric.description = description
if log_filter:
metric.filter = log_filter
if data:
# Update the top-level fields only.
update_data = yaml.load(data)
metric_diff = encoding.DictToMessage(update_data, messages.LogMetric)
for field_name in update_data:
setattr(metric, field_name, getattr(metric_diff, field_name))
return metric
def DeserializeValue(value, fmt):
"""Parses the given JSON or YAML value."""
if fmt == 'json':
return json.loads(value)
else:
return yaml.load(value)
def ParseAsYaml(value): return yaml.load(value)