Exemplo n.º 1
0
def _GetYamlImports(import_object, globbing_enabled=False):
    """Extract the import section of a file.

  If the glob_imports config is set to true, expand any globs (e.g. *.jinja).
  Named imports cannot be used with globs that expand to more than one file.
  If globbing is disabled or a glob pattern does not expand to match any files,
  importer will use the literal string as the file path.

  Args:
    import_object: The object in which to look for imports.
    globbing_enabled: If true, will resolved glob patterns dynamically.

  Returns:
    A list of dictionary objects, containing the keys 'path' and 'name' for each
    file to import. If no name was found, we populate it with the value of path.

  Raises:
   ConfigError: If we cannont read the file, the yaml is malformed, or
       the import object does not contain a 'path' field.
  """
    parent_dir = None
    if not _IsUrl(import_object.full_path):
        parent_dir = os.path.dirname(os.path.abspath(import_object.full_path))
    content = import_object.GetContent()
    yaml_content = yaml.load(content)
    imports = []
    if yaml_content and IMPORTS in yaml_content:
        raw_imports = yaml_content[IMPORTS]
        # Validate the yaml imports, and make sure the optional name is set.
        for i in raw_imports:
            if PATH not in i:
                raise exceptions.ConfigError(
                    'Missing required field %s in import in file %s.' %
                    (PATH, import_object.full_path))
            glob_matches = []
            # Only expand globs if config set and the path is a local fs reference.
            if globbing_enabled and parent_dir and not _IsUrl(i[PATH]):
                # Set our working dir to the import_object's for resolving globs.
                with files.ChDir(parent_dir):
                    # TODO(b/111880973): Replace with gcloud glob supporting ** wildcards.
                    glob_matches = glob.glob(i[PATH])
                    glob_matches = _SanitizeWindowsPathsGlobs(glob_matches)
                # Multiple file case.
                if len(glob_matches) > 1:
                    if NAME in i:
                        raise exceptions.ConfigError((
                            'Cannot use import name %s for path glob in file %s that'
                            ' matches multiple objects.') %
                                                     (i[NAME],
                                                      import_object.full_path))
                    imports.extend([{NAME: g, PATH: g} for g in glob_matches])
                    continue
            # Single file case. (URL, discrete file, or single glob match)
            if len(glob_matches) == 1:
                i[PATH] = glob_matches[0]
            # Populate the name field.
            if NAME not in i:
                i[NAME] = i[PATH]
            imports.append(i)
    return imports
Exemplo n.º 2
0
def _GetYamlImports(import_object):
    """Extract the import section of a file.

  Args:
    import_object: The object in which to look for imports.

  Returns:
    A list of dictionary objects, containing the keys 'path' and 'name' for each
    file to import. If no name was found, we populate it with the value of path.

  Raises:
   ConfigError: If we cannont read the file, the yaml is malformed, or
       the import object does not contain a 'path' field.
  """
    try:
        content = import_object.GetContent()
        yaml_content = yaml.safe_load(content)
        imports = []
        if yaml_content and IMPORTS in yaml_content:
            imports = yaml_content[IMPORTS]
            # Validate the yaml imports, and make sure the optional name is set.
            for i in imports:
                if PATH not in i:
                    raise exceptions.ConfigError(
                        'Missing required field %s in import in file %s.' %
                        (PATH, import_object.full_path))
                # Populate the name field.
                if NAME not in i:
                    i[NAME] = i[PATH]
        return imports
    except yaml.YAMLError as e:
        raise exceptions.ConfigError('Invalid yaml file %s. %s' %
                                     (import_object.full_path, str(e)))
Exemplo n.º 3
0
def _BuildImportObject(config=None, template=None,
                       composite_type=None, properties=None):
  """Build an import object from the given config name."""
  if composite_type:
    if not _IsValidCompositeTypeSyntax(composite_type):
      raise exceptions.ConfigError('Invalid composite type syntax.')
    return _ImportSyntheticCompositeTypeFile(composite_type, properties)
  if config:
    return _BuildFileImportObject(config)
  if template:
    return _BuildFileImportObject(template)
  raise exceptions.ConfigError('No path or name for a config, template, or '
                               'composite type was specified.')
Exemplo n.º 4
0
 def GetContent(self):
     if self.content is None:
         try:
             self.content = files.ReadFileContents(self.full_path)
         except files.Error as e:
             raise exceptions.ConfigError("Unable to read file '%s'. %s" %
                                          (self.full_path, str(e)))
     return self.content
Exemplo n.º 5
0
    def _ValidateUrl(url):
        """Make sure the url fits the format we expect."""
        parsed_url = six.moves.urllib.parse.urlparse(url)

        if parsed_url.scheme not in ('http', 'https'):
            raise exceptions.ConfigError(
                "URL '%s' scheme was '%s'; it must be either 'https' or 'http'."
                % (url, parsed_url.scheme))

        if not parsed_url.path or parsed_url.path == '/':
            raise exceptions.ConfigError("URL '%s' doesn't have a path." % url)

        if parsed_url.params or parsed_url.query or parsed_url.fragment:
            raise exceptions.ConfigError(
                "URL '%s' should only have a path, no params, queries, or fragments."
                % url)

        return url
Exemplo n.º 6
0
 def GetContent(self):
     if self.content is None:
         try:
             with open(self.full_path, 'r') as resource:
                 self.content = resource.read()
         except IOError as e:
             raise exceptions.ConfigError("Unable to read file '%s'. %s" %
                                          (self.full_path, e.message))
     return self.content
Exemplo n.º 7
0
def AddOptions(messages, options_file, type_provider):
    """Parse api options from the file and add them to type_provider.

  Args:
    messages: The API message to use.
    options_file: String path expression pointing to a type-provider options
        file.
    type_provider: A TypeProvider message on which the options will be set.
  Returns:
    The type_provider after applying changes.
  Raises:
    exceptions.ConfigError: the api options file couldn't be parsed as yaml
  """
    if not options_file:
        return type_provider

    file_contents = files.GetFileContents(options_file)
    yaml_content = None
    try:
        yaml_content = yaml.safe_load(file_contents)
    except yaml.YAMLError as exc:
        raise exceptions.ConfigError(
            'Could not load yaml file {0}: {1}'.format(options_file, exc))

    if yaml_content:
        if 'collectionOverrides' in yaml_content:
            type_provider.collectionOverrides = []

            for collection_override_data in yaml_content[
                    'collectionOverrides']:
                collection_override = messages.CollectionOverride(
                    collection=collection_override_data['collection'])

                if 'options' in collection_override_data:
                    collection_override.options = _OptionsFrom(
                        messages, collection_override_data['options'])

                type_provider.collectionOverrides.append(collection_override)

        if 'options' in yaml_content:
            type_provider.options = _OptionsFrom(messages,
                                                 yaml_content['options'])

        if 'credential' in yaml_content:
            type_provider.credential = _CredentialFrom(
                messages, yaml_content['credential'])

    return type_provider
Exemplo n.º 8
0
def AddOptions(options_file, type_provider):
    """Parse api options from the file and add them to type_provider.

  Args:
    options_file: String path expression pointing to a type-provider options
        file.
    type_provider: A TypeProvider message on which the options will be set.
  Returns:
    The type_provider after applying changes.
  Raises:
    exceptions.ConfigError: the api options file couldn't be parsed as yaml
  """
    file_contents = files.GetFileContents(options_file)
    yaml_content = None
    try:
        yaml_content = yaml.safe_load(file_contents)
    except yaml.YAMLError, exc:
        raise exceptions.ConfigError(
            'Could not load yaml file {0}: {1}'.format(options_file, exc))
Exemplo n.º 9
0
def CreateImports(messages, config_object):
    """Constructs a list of ImportFiles from the provided import file names.

  Args:
    messages: Object with v2 API messages.
    config_object: Parent file that contains files to import.

  Returns:
    List of ImportFiles containing the name and content of the imports.

  Raises:
    ConfigError: if the import files cannot be read from the specified
        location, the import does not have a 'path' attribute, or the filename
        has already been imported.
  """
    # Make a stack of Import objects. We use a stack because we want to make sure
    # errors are grouped by template.
    import_objects = []

    # Seed the stack with imports from the user's config.
    import_objects.extend(_GetImportObjects(config_object))

    # Map of imported resource names to their full path, used to check for
    # duplicate imports.
    import_resource_map = {}

    # List of import resources to return.
    import_resources = []

    while import_objects:
        import_object = import_objects.pop()

        process_object = True

        # Check to see if the same name is being used to refer to multiple imports.
        if import_object.GetName() in import_resource_map:
            if (import_object.GetFullPath() == import_resource_map[
                    import_object.GetName()]):
                # If the full path for these two objects is the same, we don't need to
                # process it again
                process_object = False
            else:
                # If the full path is different, fail.
                raise exceptions.ConfigError(
                    'Files %s and %s both being imported as %s.' %
                    (import_object.GetFullPath(),
                     import_resource_map[import_object.GetName()],
                     import_object.GetName()))

        if process_object:
            # If this file is a template, see if there is a corresponding schema
            # and then add all of it's imports to be processed.
            if import_object.IsTemplate():
                import_objects.extend(_HandleTemplateImport(import_object))

            import_resource = messages.ImportFile(
                name=import_object.GetName(),
                content=import_object.GetContent())

            import_resource_map[
                import_object.GetName()] = import_object.GetFullPath()
            import_resources.append(import_resource)

    return import_resources
Exemplo n.º 10
0
def BuildConfig(config=None,
                template=None,
                composite_type=None,
                properties=None):
    """Takes the path to a config and returns a processed config.

  Args:
    config: Path to the yaml config file.
    template: Path to the template config file.
    composite_type: name of the composite type config.
    properties: Dictionary of properties, only used if
                the file is a template or composite type.

  Returns:
    A tuple of base_path, config_contents, and a list of import objects.

  Raises:
    ArgumentError: If using the properties flag for a config file
        instead of a template or composite type.
  """
    config_obj = _BuildImportObject(config=config,
                                    template=template,
                                    composite_type=composite_type,
                                    properties=properties)

    if composite_type:
        return config_obj

    if config:
        if config_obj.IsTemplate():
            # TODO(b/62844648): when support for passing templates with the config
            # flag is completely removed, simply change the warning to an exception
            log.warn(
                'Creating deployments from templates with the \'--config\' '
                'flag has been deprecated.  Support for this will be '
                'removed 2017/11/08.  Please use \'--template\'instead.')
        elif properties:
            raise exceptions.ArgumentError(
                'The properties flag should only be used '
                'when using a template or composite type as your config file.')
        else:
            return config_obj

    if template:
        if not config_obj.IsTemplate():
            raise exceptions.ArgumentError(
                'The template flag should only be used '
                'when using a template as your config file.')

    # Otherwise we should build the config from scratch.
    base_name = config_obj.GetBaseName()

    # Build the single template resource.
    custom_resource = {'type': base_name, 'name': _SanitizeBaseName(base_name)}

    # Attach properties if we were given any.
    if properties:
        custom_resource['properties'] = properties

    # Add the import and single resource together into a config file.
    custom_dict = {
        'imports': [
            {
                'path': base_name
            },
        ],
        'resources': [
            custom_resource,
        ]
    }

    custom_outputs = []

    # Import the schema file and attach the outputs to config if there is any
    schema_path = config_obj.GetFullPath() + '.schema'
    schema_name = config_obj.GetName() + '.schema'

    schema_object = _BuildFileImportObject(schema_path, schema_name)

    if schema_object.Exists():
        schema_content = schema_object.GetContent()
        config_name = custom_resource['name']
        try:
            yaml_schema = yaml.safe_load(schema_content)
            if yaml_schema and OUTPUTS in yaml_schema:
                for output_name in yaml_schema[OUTPUTS].keys():
                    custom_outputs.append({
                        'name':
                        output_name,
                        'value':
                        '$(ref.' + config_name + '.' + output_name + ')'
                    })
        except yaml.YAMLError as e:
            raise exceptions.ConfigError('Invalid schema file %s. %s' %
                                         (schema_path, str(e)))

    if custom_outputs:
        custom_dict['outputs'] = custom_outputs

    # Dump using default_flow_style=False to use spacing instead of '{ }'
    custom_content = yaml.dump(custom_dict, default_flow_style=False)

    # Override the template_object with it's new config_content
    return config_obj.SetContent(custom_content)
Exemplo n.º 11
0
def _BuildConfig(full_path, properties):
    """Takes the argument from the --config flag, and returns a processed config.

  Args:
    full_path: Path to the config yaml file, with an optional list of imports.
    properties: Dictionary of properties, only used if the file is a template.

  Returns:
    A tuple of base_path, config_contents, and a list of import objects.

  Raises:
    ArgumentError: If using the properties flag for a config file
        instead of a template.
  """
    config_obj = _BuildImportObject(full_path)

    if not config_obj.IsTemplate():
        if properties:
            raise exceptions.ArgumentError(
                'The properties flag should only be used '
                'when passing in a template as your config file.')

        return config_obj

    # Otherwise we should build the config from scratch.
    base_name = config_obj.GetBaseName()

    # Build the single template resource.
    custom_resource = {'type': base_name, 'name': _SanitizeBaseName(base_name)}

    # Attach properties if we were given any.
    if properties:
        custom_resource['properties'] = properties

    # Add the import and single resource together into a config file.
    custom_dict = {
        'imports': [
            {
                'path': base_name
            },
        ],
        'resources': [
            custom_resource,
        ]
    }

    custom_outputs = []

    # Import the schema file and attach the outputs to config if there is any
    schema_path = config_obj.GetFullPath() + '.schema'
    schema_name = config_obj.GetName() + '.schema'

    schema_object = _BuildImportObject(schema_path, schema_name)

    if schema_object.Exists():
        schema_content = schema_object.GetContent()
        config_name = custom_resource['name']
        try:
            yaml_schema = yaml.safe_load(schema_content)
            if yaml_schema and OUTPUTS in yaml_schema:
                for output_name in yaml_schema[OUTPUTS].keys():
                    custom_outputs.append({
                        'name':
                        output_name,
                        'value':
                        '$(ref.' + config_name + '.' + output_name + ')'
                    })
        except yaml.YAMLError as e:
            raise exceptions.ConfigError('Invalid schema file %s. %s' %
                                         (schema_path, str(e)))

    if custom_outputs:
        custom_dict['outputs'] = custom_outputs

    # Dump using default_flow_style=False to use spacing instead of '{ }'
    custom_content = yaml.dump(custom_dict, default_flow_style=False)

    # Override the template_object with it's new config_content
    return config_obj.SetContent(custom_content)