def _GetYamlImports(import_object, globbing_enabled=False):
"""Extract the import section of a file.
If the glob_imports config is set to true, expand any globs (e.g. *.jinja).
Named imports cannot be used with globs that expand to more than one file.
If globbing is disabled or a glob pattern does not expand to match any files,
importer will use the literal string as the file path.
Args:
import_object: The object in which to look for imports.
globbing_enabled: If true, will resolved glob patterns dynamically.
Returns:
A list of dictionary objects, containing the keys 'path' and 'name' for each
file to import. If no name was found, we populate it with the value of path.
Raises:
ConfigError: If we cannont read the file, the yaml is malformed, or
the import object does not contain a 'path' field.
"""
parent_dir = None
if not _IsUrl(import_object.full_path):
parent_dir = os.path.dirname(os.path.abspath(import_object.full_path))
content = import_object.GetContent()
yaml_content = yaml.load(content)
imports = []
if yaml_content and IMPORTS in yaml_content:
raw_imports = yaml_content[IMPORTS]
# Validate the yaml imports, and make sure the optional name is set.
for i in raw_imports:
if PATH not in i:
raise exceptions.ConfigError(
'Missing required field %s in import in file %s.' %
(PATH, import_object.full_path))
glob_matches = []
# Only expand globs if config set and the path is a local fs reference.
if globbing_enabled and parent_dir and not _IsUrl(i[PATH]):
# Set our working dir to the import_object's for resolving globs.
with files.ChDir(parent_dir):
# TODO(b/111880973): Replace with gcloud glob supporting ** wildcards.
glob_matches = glob.glob(i[PATH])
glob_matches = _SanitizeWindowsPathsGlobs(glob_matches)
# Multiple file case.
if len(glob_matches) > 1:
if NAME in i:
raise exceptions.ConfigError((
'Cannot use import name %s for path glob in file %s that'
' matches multiple objects.') %
(i[NAME],
import_object.full_path))
imports.extend([{NAME: g, PATH: g} for g in glob_matches])
continue
# Single file case. (URL, discrete file, or single glob match)
if len(glob_matches) == 1:
i[PATH] = glob_matches[0]
# Populate the name field.
if NAME not in i:
i[NAME] = i[PATH]
imports.append(i)
return imports
def _GetYamlImports(import_object):
"""Extract the import section of a file.
Args:
import_object: The object in which to look for imports.
Returns:
A list of dictionary objects, containing the keys 'path' and 'name' for each
file to import. If no name was found, we populate it with the value of path.
Raises:
ConfigError: If we cannont read the file, the yaml is malformed, or
the import object does not contain a 'path' field.
"""
try:
content = import_object.GetContent()
yaml_content = yaml.safe_load(content)
imports = []
if yaml_content and IMPORTS in yaml_content:
imports = yaml_content[IMPORTS]
# Validate the yaml imports, and make sure the optional name is set.
for i in imports:
if PATH not in i:
raise exceptions.ConfigError(
'Missing required field %s in import in file %s.' %
(PATH, import_object.full_path))
# Populate the name field.
if NAME not in i:
i[NAME] = i[PATH]
return imports
except yaml.YAMLError as e:
raise exceptions.ConfigError('Invalid yaml file %s. %s' %
(import_object.full_path, str(e)))
def _BuildImportObject(config=None, template=None,
composite_type=None, properties=None):
"""Build an import object from the given config name."""
if composite_type:
if not _IsValidCompositeTypeSyntax(composite_type):
raise exceptions.ConfigError('Invalid composite type syntax.')
return _ImportSyntheticCompositeTypeFile(composite_type, properties)
if config:
return _BuildFileImportObject(config)
if template:
return _BuildFileImportObject(template)
raise exceptions.ConfigError('No path or name for a config, template, or '
'composite type was specified.')
def GetContent(self):
if self.content is None:
try:
self.content = files.ReadFileContents(self.full_path)
except files.Error as e:
raise exceptions.ConfigError("Unable to read file '%s'. %s" %
(self.full_path, str(e)))
return self.content
def _ValidateUrl(url):
"""Make sure the url fits the format we expect."""
parsed_url = six.moves.urllib.parse.urlparse(url)
if parsed_url.scheme not in ('http', 'https'):
raise exceptions.ConfigError(
"URL '%s' scheme was '%s'; it must be either 'https' or 'http'."
% (url, parsed_url.scheme))
if not parsed_url.path or parsed_url.path == '/':
raise exceptions.ConfigError("URL '%s' doesn't have a path." % url)
if parsed_url.params or parsed_url.query or parsed_url.fragment:
raise exceptions.ConfigError(
"URL '%s' should only have a path, no params, queries, or fragments."
% url)
return url
def GetContent(self):
if self.content is None:
try:
with open(self.full_path, 'r') as resource:
self.content = resource.read()
except IOError as e:
raise exceptions.ConfigError("Unable to read file '%s'. %s" %
(self.full_path, e.message))
return self.content
def AddOptions(messages, options_file, type_provider):
"""Parse api options from the file and add them to type_provider.
Args:
messages: The API message to use.
options_file: String path expression pointing to a type-provider options
file.
type_provider: A TypeProvider message on which the options will be set.
Returns:
The type_provider after applying changes.
Raises:
exceptions.ConfigError: the api options file couldn't be parsed as yaml
"""
if not options_file:
return type_provider
file_contents = files.GetFileContents(options_file)
yaml_content = None
try:
yaml_content = yaml.safe_load(file_contents)
except yaml.YAMLError as exc:
raise exceptions.ConfigError(
'Could not load yaml file {0}: {1}'.format(options_file, exc))
if yaml_content:
if 'collectionOverrides' in yaml_content:
type_provider.collectionOverrides = []
for collection_override_data in yaml_content[
'collectionOverrides']:
collection_override = messages.CollectionOverride(
collection=collection_override_data['collection'])
if 'options' in collection_override_data:
collection_override.options = _OptionsFrom(
messages, collection_override_data['options'])
type_provider.collectionOverrides.append(collection_override)
if 'options' in yaml_content:
type_provider.options = _OptionsFrom(messages,
yaml_content['options'])
if 'credential' in yaml_content:
type_provider.credential = _CredentialFrom(
messages, yaml_content['credential'])
return type_provider
def AddOptions(options_file, type_provider):
"""Parse api options from the file and add them to type_provider.
Args:
options_file: String path expression pointing to a type-provider options
file.
type_provider: A TypeProvider message on which the options will be set.
Returns:
The type_provider after applying changes.
Raises:
exceptions.ConfigError: the api options file couldn't be parsed as yaml
"""
file_contents = files.GetFileContents(options_file)
yaml_content = None
try:
yaml_content = yaml.safe_load(file_contents)
except yaml.YAMLError, exc:
raise exceptions.ConfigError(
'Could not load yaml file {0}: {1}'.format(options_file, exc))
def CreateImports(messages, config_object):
"""Constructs a list of ImportFiles from the provided import file names.
Args:
messages: Object with v2 API messages.
config_object: Parent file that contains files to import.
Returns:
List of ImportFiles containing the name and content of the imports.
Raises:
ConfigError: if the import files cannot be read from the specified
location, the import does not have a 'path' attribute, or the filename
has already been imported.
"""
# Make a stack of Import objects. We use a stack because we want to make sure
# errors are grouped by template.
import_objects = []
# Seed the stack with imports from the user's config.
import_objects.extend(_GetImportObjects(config_object))
# Map of imported resource names to their full path, used to check for
# duplicate imports.
import_resource_map = {}
# List of import resources to return.
import_resources = []
while import_objects:
import_object = import_objects.pop()
process_object = True
# Check to see if the same name is being used to refer to multiple imports.
if import_object.GetName() in import_resource_map:
if (import_object.GetFullPath() == import_resource_map[
import_object.GetName()]):
# If the full path for these two objects is the same, we don't need to
# process it again
process_object = False
else:
# If the full path is different, fail.
raise exceptions.ConfigError(
'Files %s and %s both being imported as %s.' %
(import_object.GetFullPath(),
import_resource_map[import_object.GetName()],
import_object.GetName()))
if process_object:
# If this file is a template, see if there is a corresponding schema
# and then add all of it's imports to be processed.
if import_object.IsTemplate():
import_objects.extend(_HandleTemplateImport(import_object))
import_resource = messages.ImportFile(
name=import_object.GetName(),
content=import_object.GetContent())
import_resource_map[
import_object.GetName()] = import_object.GetFullPath()
import_resources.append(import_resource)
return import_resources
def BuildConfig(config=None,
template=None,
composite_type=None,
properties=None):
"""Takes the path to a config and returns a processed config.
Args:
config: Path to the yaml config file.
template: Path to the template config file.
composite_type: name of the composite type config.
properties: Dictionary of properties, only used if
the file is a template or composite type.
Returns:
A tuple of base_path, config_contents, and a list of import objects.
Raises:
ArgumentError: If using the properties flag for a config file
instead of a template or composite type.
"""
config_obj = _BuildImportObject(config=config,
template=template,
composite_type=composite_type,
properties=properties)
if composite_type:
return config_obj
if config:
if config_obj.IsTemplate():
# TODO(b/62844648): when support for passing templates with the config
# flag is completely removed, simply change the warning to an exception
log.warn(
'Creating deployments from templates with the \'--config\' '
'flag has been deprecated. Support for this will be '
'removed 2017/11/08. Please use \'--template\'instead.')
elif properties:
raise exceptions.ArgumentError(
'The properties flag should only be used '
'when using a template or composite type as your config file.')
else:
return config_obj
if template:
if not config_obj.IsTemplate():
raise exceptions.ArgumentError(
'The template flag should only be used '
'when using a template as your config file.')
# Otherwise we should build the config from scratch.
base_name = config_obj.GetBaseName()
# Build the single template resource.
custom_resource = {'type': base_name, 'name': _SanitizeBaseName(base_name)}
# Attach properties if we were given any.
if properties:
custom_resource['properties'] = properties
# Add the import and single resource together into a config file.
custom_dict = {
'imports': [
{
'path': base_name
},
],
'resources': [
custom_resource,
]
}
custom_outputs = []
# Import the schema file and attach the outputs to config if there is any
schema_path = config_obj.GetFullPath() + '.schema'
schema_name = config_obj.GetName() + '.schema'
schema_object = _BuildFileImportObject(schema_path, schema_name)
if schema_object.Exists():
schema_content = schema_object.GetContent()
config_name = custom_resource['name']
try:
yaml_schema = yaml.safe_load(schema_content)
if yaml_schema and OUTPUTS in yaml_schema:
for output_name in yaml_schema[OUTPUTS].keys():
custom_outputs.append({
'name':
output_name,
'value':
'$(ref.' + config_name + '.' + output_name + ')'
})
except yaml.YAMLError as e:
raise exceptions.ConfigError('Invalid schema file %s. %s' %
(schema_path, str(e)))
if custom_outputs:
custom_dict['outputs'] = custom_outputs
# Dump using default_flow_style=False to use spacing instead of '{ }'
custom_content = yaml.dump(custom_dict, default_flow_style=False)
# Override the template_object with it's new config_content
return config_obj.SetContent(custom_content)
def _BuildConfig(full_path, properties):
"""Takes the argument from the --config flag, and returns a processed config.
Args:
full_path: Path to the config yaml file, with an optional list of imports.
properties: Dictionary of properties, only used if the file is a template.
Returns:
A tuple of base_path, config_contents, and a list of import objects.
Raises:
ArgumentError: If using the properties flag for a config file
instead of a template.
"""
config_obj = _BuildImportObject(full_path)
if not config_obj.IsTemplate():
if properties:
raise exceptions.ArgumentError(
'The properties flag should only be used '
'when passing in a template as your config file.')
return config_obj
# Otherwise we should build the config from scratch.
base_name = config_obj.GetBaseName()
# Build the single template resource.
custom_resource = {'type': base_name, 'name': _SanitizeBaseName(base_name)}
# Attach properties if we were given any.
if properties:
custom_resource['properties'] = properties
# Add the import and single resource together into a config file.
custom_dict = {
'imports': [
{
'path': base_name
},
],
'resources': [
custom_resource,
]
}
custom_outputs = []
# Import the schema file and attach the outputs to config if there is any
schema_path = config_obj.GetFullPath() + '.schema'
schema_name = config_obj.GetName() + '.schema'
schema_object = _BuildImportObject(schema_path, schema_name)
if schema_object.Exists():
schema_content = schema_object.GetContent()
config_name = custom_resource['name']
try:
yaml_schema = yaml.safe_load(schema_content)
if yaml_schema and OUTPUTS in yaml_schema:
for output_name in yaml_schema[OUTPUTS].keys():
custom_outputs.append({
'name':
output_name,
'value':
'$(ref.' + config_name + '.' + output_name + ')'
})
except yaml.YAMLError as e:
raise exceptions.ConfigError('Invalid schema file %s. %s' %
(schema_path, str(e)))
if custom_outputs:
custom_dict['outputs'] = custom_outputs
# Dump using default_flow_style=False to use spacing instead of '{ }'
custom_content = yaml.dump(custom_dict, default_flow_style=False)
# Override the template_object with it's new config_content
return config_obj.SetContent(custom_content)