def _GetYamlImports(import_object, globbing_enabled=False): """Extract the import section of a file. If the glob_imports config is set to true, expand any globs (e.g. *.jinja). Named imports cannot be used with globs that expand to more than one file. If globbing is disabled or a glob pattern does not expand to match any files, importer will use the literal string as the file path. Args: import_object: The object in which to look for imports. globbing_enabled: If true, will resolved glob patterns dynamically. Returns: A list of dictionary objects, containing the keys 'path' and 'name' for each file to import. If no name was found, we populate it with the value of path. Raises: ConfigError: If we cannont read the file, the yaml is malformed, or the import object does not contain a 'path' field. """ parent_dir = None if not _IsUrl(import_object.full_path): parent_dir = os.path.dirname(os.path.abspath(import_object.full_path)) content = import_object.GetContent() yaml_content = yaml.load(content) imports = [] if yaml_content and IMPORTS in yaml_content: raw_imports = yaml_content[IMPORTS] # Validate the yaml imports, and make sure the optional name is set. for i in raw_imports: if PATH not in i: raise exceptions.ConfigError( 'Missing required field %s in import in file %s.' % (PATH, import_object.full_path)) glob_matches = [] # Only expand globs if config set and the path is a local fs reference. if globbing_enabled and parent_dir and not _IsUrl(i[PATH]): # Set our working dir to the import_object's for resolving globs. with files.ChDir(parent_dir): # TODO(b/111880973): Replace with gcloud glob supporting ** wildcards. glob_matches = glob.glob(i[PATH]) glob_matches = _SanitizeWindowsPathsGlobs(glob_matches) # Multiple file case. if len(glob_matches) > 1: if NAME in i: raise exceptions.ConfigError(( 'Cannot use import name %s for path glob in file %s that' ' matches multiple objects.') % (i[NAME], import_object.full_path)) imports.extend([{NAME: g, PATH: g} for g in glob_matches]) continue # Single file case. (URL, discrete file, or single glob match) if len(glob_matches) == 1: i[PATH] = glob_matches[0] # Populate the name field. if NAME not in i: i[NAME] = i[PATH] imports.append(i) return imports
def _GetYamlImports(import_object): """Extract the import section of a file. Args: import_object: The object in which to look for imports. Returns: A list of dictionary objects, containing the keys 'path' and 'name' for each file to import. If no name was found, we populate it with the value of path. Raises: ConfigError: If we cannont read the file, the yaml is malformed, or the import object does not contain a 'path' field. """ try: content = import_object.GetContent() yaml_content = yaml.safe_load(content) imports = [] if yaml_content and IMPORTS in yaml_content: imports = yaml_content[IMPORTS] # Validate the yaml imports, and make sure the optional name is set. for i in imports: if PATH not in i: raise exceptions.ConfigError( 'Missing required field %s in import in file %s.' % (PATH, import_object.full_path)) # Populate the name field. if NAME not in i: i[NAME] = i[PATH] return imports except yaml.YAMLError as e: raise exceptions.ConfigError('Invalid yaml file %s. %s' % (import_object.full_path, str(e)))
def _BuildImportObject(config=None, template=None, composite_type=None, properties=None): """Build an import object from the given config name.""" if composite_type: if not _IsValidCompositeTypeSyntax(composite_type): raise exceptions.ConfigError('Invalid composite type syntax.') return _ImportSyntheticCompositeTypeFile(composite_type, properties) if config: return _BuildFileImportObject(config) if template: return _BuildFileImportObject(template) raise exceptions.ConfigError('No path or name for a config, template, or ' 'composite type was specified.')
def GetContent(self): if self.content is None: try: self.content = files.ReadFileContents(self.full_path) except files.Error as e: raise exceptions.ConfigError("Unable to read file '%s'. %s" % (self.full_path, str(e))) return self.content
def _ValidateUrl(url): """Make sure the url fits the format we expect.""" parsed_url = six.moves.urllib.parse.urlparse(url) if parsed_url.scheme not in ('http', 'https'): raise exceptions.ConfigError( "URL '%s' scheme was '%s'; it must be either 'https' or 'http'." % (url, parsed_url.scheme)) if not parsed_url.path or parsed_url.path == '/': raise exceptions.ConfigError("URL '%s' doesn't have a path." % url) if parsed_url.params or parsed_url.query or parsed_url.fragment: raise exceptions.ConfigError( "URL '%s' should only have a path, no params, queries, or fragments." % url) return url
def GetContent(self): if self.content is None: try: with open(self.full_path, 'r') as resource: self.content = resource.read() except IOError as e: raise exceptions.ConfigError("Unable to read file '%s'. %s" % (self.full_path, e.message)) return self.content
def AddOptions(messages, options_file, type_provider): """Parse api options from the file and add them to type_provider. Args: messages: The API message to use. options_file: String path expression pointing to a type-provider options file. type_provider: A TypeProvider message on which the options will be set. Returns: The type_provider after applying changes. Raises: exceptions.ConfigError: the api options file couldn't be parsed as yaml """ if not options_file: return type_provider file_contents = files.GetFileContents(options_file) yaml_content = None try: yaml_content = yaml.safe_load(file_contents) except yaml.YAMLError as exc: raise exceptions.ConfigError( 'Could not load yaml file {0}: {1}'.format(options_file, exc)) if yaml_content: if 'collectionOverrides' in yaml_content: type_provider.collectionOverrides = [] for collection_override_data in yaml_content[ 'collectionOverrides']: collection_override = messages.CollectionOverride( collection=collection_override_data['collection']) if 'options' in collection_override_data: collection_override.options = _OptionsFrom( messages, collection_override_data['options']) type_provider.collectionOverrides.append(collection_override) if 'options' in yaml_content: type_provider.options = _OptionsFrom(messages, yaml_content['options']) if 'credential' in yaml_content: type_provider.credential = _CredentialFrom( messages, yaml_content['credential']) return type_provider
def AddOptions(options_file, type_provider): """Parse api options from the file and add them to type_provider. Args: options_file: String path expression pointing to a type-provider options file. type_provider: A TypeProvider message on which the options will be set. Returns: The type_provider after applying changes. Raises: exceptions.ConfigError: the api options file couldn't be parsed as yaml """ file_contents = files.GetFileContents(options_file) yaml_content = None try: yaml_content = yaml.safe_load(file_contents) except yaml.YAMLError, exc: raise exceptions.ConfigError( 'Could not load yaml file {0}: {1}'.format(options_file, exc))
def CreateImports(messages, config_object): """Constructs a list of ImportFiles from the provided import file names. Args: messages: Object with v2 API messages. config_object: Parent file that contains files to import. Returns: List of ImportFiles containing the name and content of the imports. Raises: ConfigError: if the import files cannot be read from the specified location, the import does not have a 'path' attribute, or the filename has already been imported. """ # Make a stack of Import objects. We use a stack because we want to make sure # errors are grouped by template. import_objects = [] # Seed the stack with imports from the user's config. import_objects.extend(_GetImportObjects(config_object)) # Map of imported resource names to their full path, used to check for # duplicate imports. import_resource_map = {} # List of import resources to return. import_resources = [] while import_objects: import_object = import_objects.pop() process_object = True # Check to see if the same name is being used to refer to multiple imports. if import_object.GetName() in import_resource_map: if (import_object.GetFullPath() == import_resource_map[ import_object.GetName()]): # If the full path for these two objects is the same, we don't need to # process it again process_object = False else: # If the full path is different, fail. raise exceptions.ConfigError( 'Files %s and %s both being imported as %s.' % (import_object.GetFullPath(), import_resource_map[import_object.GetName()], import_object.GetName())) if process_object: # If this file is a template, see if there is a corresponding schema # and then add all of it's imports to be processed. if import_object.IsTemplate(): import_objects.extend(_HandleTemplateImport(import_object)) import_resource = messages.ImportFile( name=import_object.GetName(), content=import_object.GetContent()) import_resource_map[ import_object.GetName()] = import_object.GetFullPath() import_resources.append(import_resource) return import_resources
def BuildConfig(config=None, template=None, composite_type=None, properties=None): """Takes the path to a config and returns a processed config. Args: config: Path to the yaml config file. template: Path to the template config file. composite_type: name of the composite type config. properties: Dictionary of properties, only used if the file is a template or composite type. Returns: A tuple of base_path, config_contents, and a list of import objects. Raises: ArgumentError: If using the properties flag for a config file instead of a template or composite type. """ config_obj = _BuildImportObject(config=config, template=template, composite_type=composite_type, properties=properties) if composite_type: return config_obj if config: if config_obj.IsTemplate(): # TODO(b/62844648): when support for passing templates with the config # flag is completely removed, simply change the warning to an exception log.warn( 'Creating deployments from templates with the \'--config\' ' 'flag has been deprecated. Support for this will be ' 'removed 2017/11/08. Please use \'--template\'instead.') elif properties: raise exceptions.ArgumentError( 'The properties flag should only be used ' 'when using a template or composite type as your config file.') else: return config_obj if template: if not config_obj.IsTemplate(): raise exceptions.ArgumentError( 'The template flag should only be used ' 'when using a template as your config file.') # Otherwise we should build the config from scratch. base_name = config_obj.GetBaseName() # Build the single template resource. custom_resource = {'type': base_name, 'name': _SanitizeBaseName(base_name)} # Attach properties if we were given any. if properties: custom_resource['properties'] = properties # Add the import and single resource together into a config file. custom_dict = { 'imports': [ { 'path': base_name }, ], 'resources': [ custom_resource, ] } custom_outputs = [] # Import the schema file and attach the outputs to config if there is any schema_path = config_obj.GetFullPath() + '.schema' schema_name = config_obj.GetName() + '.schema' schema_object = _BuildFileImportObject(schema_path, schema_name) if schema_object.Exists(): schema_content = schema_object.GetContent() config_name = custom_resource['name'] try: yaml_schema = yaml.safe_load(schema_content) if yaml_schema and OUTPUTS in yaml_schema: for output_name in yaml_schema[OUTPUTS].keys(): custom_outputs.append({ 'name': output_name, 'value': '$(ref.' + config_name + '.' + output_name + ')' }) except yaml.YAMLError as e: raise exceptions.ConfigError('Invalid schema file %s. %s' % (schema_path, str(e))) if custom_outputs: custom_dict['outputs'] = custom_outputs # Dump using default_flow_style=False to use spacing instead of '{ }' custom_content = yaml.dump(custom_dict, default_flow_style=False) # Override the template_object with it's new config_content return config_obj.SetContent(custom_content)
def _BuildConfig(full_path, properties): """Takes the argument from the --config flag, and returns a processed config. Args: full_path: Path to the config yaml file, with an optional list of imports. properties: Dictionary of properties, only used if the file is a template. Returns: A tuple of base_path, config_contents, and a list of import objects. Raises: ArgumentError: If using the properties flag for a config file instead of a template. """ config_obj = _BuildImportObject(full_path) if not config_obj.IsTemplate(): if properties: raise exceptions.ArgumentError( 'The properties flag should only be used ' 'when passing in a template as your config file.') return config_obj # Otherwise we should build the config from scratch. base_name = config_obj.GetBaseName() # Build the single template resource. custom_resource = {'type': base_name, 'name': _SanitizeBaseName(base_name)} # Attach properties if we were given any. if properties: custom_resource['properties'] = properties # Add the import and single resource together into a config file. custom_dict = { 'imports': [ { 'path': base_name }, ], 'resources': [ custom_resource, ] } custom_outputs = [] # Import the schema file and attach the outputs to config if there is any schema_path = config_obj.GetFullPath() + '.schema' schema_name = config_obj.GetName() + '.schema' schema_object = _BuildImportObject(schema_path, schema_name) if schema_object.Exists(): schema_content = schema_object.GetContent() config_name = custom_resource['name'] try: yaml_schema = yaml.safe_load(schema_content) if yaml_schema and OUTPUTS in yaml_schema: for output_name in yaml_schema[OUTPUTS].keys(): custom_outputs.append({ 'name': output_name, 'value': '$(ref.' + config_name + '.' + output_name + ')' }) except yaml.YAMLError as e: raise exceptions.ConfigError('Invalid schema file %s. %s' % (schema_path, str(e))) if custom_outputs: custom_dict['outputs'] = custom_outputs # Dump using default_flow_style=False to use spacing instead of '{ }' custom_content = yaml.dump(custom_dict, default_flow_style=False) # Override the template_object with it's new config_content return config_obj.SetContent(custom_content)