def _GetRefData(self, path): """Loads the YAML data from the given reference. A YAML reference must refer to a YAML file and an attribute within that file to extract. Args: path: str, The path of the YAML file to import. It must be in the form of: package.module:attribute.attribute, where the module path is separated from the sub attributes within the YAML by a ':'. Raises: LayoutException: If the given module or attribute cannot be loaded. Returns: The referenced YAML data. """ root = os.path.dirname(os.path.dirname(googlecloudsdk.__file__)) parts = path.split(':') if len(parts) != 2: raise LayoutException( 'Invalid Yaml reference: [{}]. References must be in the format: ' 'path(.path)+:attribute(.attribute)*'.format(path)) yaml_path = os.path.join(root, *parts[0].split('.')) yaml_path += '.yaml' try: data = yaml.load(pkg_resources.GetData(yaml_path)) except IOError as e: raise LayoutException( 'Failed to load Yaml reference file [{}]: {}'.format( yaml_path, e)) return self._GetAttribute(data, parts[1], yaml_path)
def LoadCommonType(impl_paths, path, release_track, construction_id, is_command, yaml_command_translator=None): """Loads a calliope command or group from a file. Args: impl_paths: [str], A list of file paths to the command implementation for this group or command. path: [str], A list of group names that got us down to this command group with respect to the CLI itself. This path should be used for things like error reporting when a specific element in the tree needs to be referenced. release_track: ReleaseTrack, The release track that we should load. construction_id: str, A unique identifier for the CLILoader that is being constructed. is_command: bool, True if we are loading a command, False to load a group. yaml_command_translator: YamlCommandTranslator, An instance of a translator to use to load the yaml data. Raises: CommandLoadFailure: If the command is invalid and cannot be loaded. Returns: The base._Common class for the command or group. """ implementations = [] for impl_file in impl_paths: if impl_file.endswith('.yaml'): if not is_command: raise CommandLoadFailure( '.'.join(path), Exception('Command groups cannot be implemented in yaml')) data = yaml.load(pkg_resources.GetData(impl_file), Loader=CreateYamlLoader(impl_file)) implementations.extend( (_ImplementationsFromYaml(path, data, yaml_command_translator))) else: module = _GetModuleFromPath(impl_file, path, construction_id) implementations.extend( _ImplementationsFromModule(module.__file__, module.__dict__.values(), is_command=is_command)) return _ExtractReleaseTrackImplementation(impl_paths[0], release_track, implementations)()
def _GetAllImplementations(impl_paths, path, construction_id, is_command, yaml_command_translator): """Gets all the release track command implementations. Can load both python and yaml modules. Args: impl_paths: [str], A list of file paths to the command implementation for this group or command. path: [str], A list of group names that got us down to this command group with respect to the CLI itself. This path should be used for things like error reporting when a specific element in the tree needs to be referenced. construction_id: str, A unique identifier for the CLILoader that is being constructed. is_command: bool, True if we are loading a command, False to load a group. yaml_command_translator: YamlCommandTranslator, An instance of a translator to use to load the yaml data. Raises: CommandLoadFailure: If the command is invalid and cannot be loaded. Returns: [(func->base._Common, [base.ReleaseTrack])], A list of tuples that can be passed to _ExtractReleaseTrackImplementation. Each item in this list represents a command implementation. The first element is a function that returns the implementation, and the second element is a list of release tracks it is valid for. """ implementations = [] for impl_file in impl_paths: if impl_file.endswith('.yaml'): if not is_command: raise CommandLoadFailure( '.'.join(path), Exception('Command groups cannot be implemented in yaml')) data = yaml.load(pkg_resources.GetData(impl_file), Loader=CreateYamlLoader(impl_file)) implementations.extend( (_ImplementationsFromYaml(path, data, yaml_command_translator))) else: module = _GetModuleFromPath(impl_file, path, construction_id) implementations.extend( _ImplementationsFromModule(module.__file__, module.__dict__.values(), is_command=is_command)) return implementations
def _CopyDefaultDockerfile(runtime, dst, cleanup): """Copies default Dockerfile for a given runtime into destination directory. Args: runtime: str, Runtime that we're looking for the Dockerfile for. dst: str, Directory path where to copy the Dockerfile. cleanup: bool, If true, delete the file on gcloud exit. Raises: InternalError: if there is no directory with default Dockerfiles. NoDefaultDockerfileError: if there is no default Dockerfile for a given runtime. """ log.info('Looking for the default %s for runtime [%s]', config.DOCKERFILE, runtime) runtime = _GetCanonicalRuntime(runtime) default_dockerfiles_dir = _GetDefaultDockerfilesDir() src = os.path.join(default_dockerfiles_dir, '{runtime}_app'.format(runtime=runtime), config.DOCKERFILE) try: src_data = pkg_resources.GetData(src) except IOError: raise NoDefaultDockerfileError( 'No default {dockerfile} for runtime [{runtime}] in the SDK. ' 'Use one of the supported runtimes: [{supported}].'.format( dockerfile=config.DOCKERFILE, runtime=runtime, supported='|'.join( _ListSupportedRuntimes(default_dockerfiles_dir)))) log.info( '%s for runtime [%s] is found in %s. Copying it into application ' 'directory.', config.DOCKERFILE, runtime, default_dockerfiles_dir) with open(os.path.join(dst, os.path.basename(src)), 'w') as dst_file: dst_file.write(src_data) # Delete the file after we're done if necessary. if cleanup: atexit.register(Clean, os.path.join(dst, config.DOCKERFILE))
def FindOrCopyDockerfile(runtime, dst, cleanup=True): """Copies default Dockerfile for a given runtime into destination directory. Default Dockerfile for runtime is used if there is no user provided dockerfile in the destination directory. Args: runtime: str, Runtime that we're looking for the Dockerfile for. dst: str, Directory path where to check for and copy to the Dockerfile. cleanup: bool, If true, delete the file on gcloud exit. Raises: IOError: raised by pkg_resources.GetData if the Dockerfile doesn't exist in the expected location. Returns: callable(), A function to be called to clean up the generated Dockerfile. """ log.info('Looking for the %s in %s', config.DOCKERFILE, dst) if os.path.exists(os.path.join(dst, config.DOCKERFILE)): log.info('Using %s found in %s', config.DOCKERFILE, dst) return lambda: None log.info('Looking for the default %s for runtime [%s]', config.DOCKERFILE, runtime) runtime = _GetCanonicalRuntime(runtime) default_dockerfiles_dir = GetGCloudDockerfilesDir() src = os.path.join(default_dockerfiles_dir, '{runtime}_app'.format(runtime=runtime), config.DOCKERFILE) src_data = pkg_resources.GetData(src) log.info( '%s for runtime [%s] is found in %s. Copying it into application ' 'directory.', config.DOCKERFILE, runtime, default_dockerfiles_dir) with open(os.path.join(dst, os.path.basename(src)), 'w') as dst_file: dst_file.write(src_data) # Delete the file after we're done if necessary. if cleanup: full_name = os.path.join(dst, config.DOCKERFILE) atexit.register(Clean, full_name) return lambda: Clean(full_name) return lambda: None
def CreateYamlLoader(impl_path): """Creates a custom yaml loader that handles includes from common data. Args: impl_path: str, The path to the file we are loading data from. Returns: yaml.Loader, A yaml loader to use. """ # TODO(b/64147277) Allow for importing from other places. common_file_path = os.path.join(os.path.dirname(impl_path), '__init__.yaml') common_data = None if os.path.exists(common_file_path): common_data = yaml.load(pkg_resources.GetData(common_file_path)) class Loader(yaml.Loader): """A custom yaml loader. It adds 2 different import capabilities. Assuming __init__.yaml has the contents: foo: a: b c: d baz: - e: f - g: h The first uses a custom constructor to insert data into your current file, so: bar: !COMMON foo.a results in: bar: b The second mechanism overrides construct_mapping and construct_sequence to post process the data and replace the merge macro with keys from the other file. We can't use the custom constructor for this as well because the merge key type in yaml is processed before custom constructors which makes importing and merging not possible. So: bar: _COMMON_: foo i: j results in: bar: a: b c: d i: j This can also be used to merge list contexts, so: bar: - _COMMON_baz - i: j results in: bar: - e: f - g: h - i: j You may also use the !REF and _REF_ directives in the same way. Instead of pulling from the common file, they can pull from an arbitrary yaml file somewhere in the googlecloudsdk tree. The syntax looks like: bar: !REF googlecloudsdk.foo.bar:a.b.c This will load googlecloudsdk/foo/bar.yaml and from that file return the a.b.c nested attribute. """ INCLUDE_COMMON_MACRO = '!COMMON' MERGE_COMMON_MACRO = '_COMMON_' INCLUDE_REF_MACRO = '!REF' MERGE_REF_MACRO = '_REF_' def __init__(self, stream): super(Loader, self).__init__(stream) def construct_mapping(self, *args, **kwargs): data = super(Loader, self).construct_mapping(*args, **kwargs) data = self._ConstructMappingHelper(Loader.MERGE_COMMON_MACRO, self._GetCommonData, data) return self._ConstructMappingHelper(Loader.MERGE_REF_MACRO, self._GetRefData, data) def _ConstructMappingHelper(self, macro, source_func, data): attribute_path = data.pop(macro, None) if not attribute_path: return data modified_data = {} for path in attribute_path.split(','): modified_data.update(source_func(path)) # Add the explicit data last so it can override the imports. modified_data.update(data) return modified_data def construct_sequence(self, *args, **kwargs): data = super(Loader, self).construct_sequence(*args, **kwargs) data = self._ConstructSequenceHelper(Loader.MERGE_COMMON_MACRO, self._GetCommonData, data) return self._ConstructSequenceHelper(Loader.MERGE_REF_MACRO, self._GetRefData, data) def _ConstructSequenceHelper(self, macro, source_func, data): new_list = [] for i in data: if isinstance(i, basestring) and i.startswith(macro): attribute_path = i[len(macro):] for path in attribute_path.split(','): new_list.extend(source_func(path)) else: new_list.append(i) return new_list def IncludeCommon(self, node): attribute_path = self.construct_scalar(node) return self._GetCommonData(attribute_path) def IncludeRef(self, node): attribute_path = self.construct_scalar(node) return self._GetRefData(attribute_path) def _GetCommonData(self, attribute_path): if not common_data: raise LayoutException( 'Command [{}] references [common command] data but it does not ' 'exist.'.format(impl_path)) return self._GetAttribute(common_data, attribute_path, 'common command') def _GetRefData(self, path): """Loads the YAML data from the given reference. A YAML reference must refer to a YAML file and an attribute within that file to extract. Args: path: str, The path of the YAML file to import. It must be in the form of: package.module:attribute.attribute, where the module path is separated from the sub attributes within the YAML by a ':'. Raises: LayoutException: If the given module or attribute cannot be loaded. Returns: The referenced YAML data. """ root = os.path.dirname(os.path.dirname(googlecloudsdk.__file__)) parts = path.split(':') if len(parts) != 2: raise LayoutException( 'Invalid Yaml reference: [{}]. References must be in the format: ' 'path(.path)+:attribute(.attribute)*'.format(path)) yaml_path = os.path.join(root, *parts[0].split('.')) yaml_path += '.yaml' try: data = yaml.load(pkg_resources.GetData(yaml_path)) except IOError as e: raise LayoutException( 'Failed to load Yaml reference file [{}]: {}'.format( yaml_path, e)) return self._GetAttribute(data, parts[1], yaml_path) def _GetAttribute(self, data, attribute_path, location): value = data for attribute in attribute_path.split('.'): value = value.get(attribute, None) if not value: raise LayoutException( 'Command [{}] references [{}] data attribute [{}] in ' 'path [{}] but it does not exist.'.format( impl_path, location, attribute, attribute_path)) return value Loader.add_constructor(Loader.INCLUDE_COMMON_MACRO, Loader.IncludeCommon) Loader.add_constructor(Loader.INCLUDE_REF_MACRO, Loader.IncludeRef) return Loader
def CreateYamlLoader(impl_path): """Creates a custom yaml loader that handles includes from common data. Args: impl_path: str, The path to the file we are loading data from. Returns: yaml.Loader, A yaml loader to use. """ # TODO(b/64147277) Allow for importing from other places. common_file_path = os.path.join(os.path.dirname(impl_path), '__init__.yaml') common_data = None if os.path.exists(common_file_path): common_data = yaml.load(pkg_resources.GetData(common_file_path)) class Loader(yaml.Loader): """A custom yaml loader. It adds 2 different import capabilities. Assuming __init__.yaml has the contents: foo: a: b c: d baz: - e: f - g: h The first uses a custom constructor to insert data into your current file, so: bar: !COMMON foo.a results in: bar: b The second mechanism overrides construct_mapping and construct_sequence to post process the data and replace the merge macro with keys from the other file. We can't use the custom constructor for this as well because the merge key type in yaml is processed before custom constructors which makes importing and merging not possible. So: bar: _COMMON_: foo i: j results in: bar: a: b c: d i: j This can also be used to merge list contexts, so: bar: - _COMMON_baz - i: j results in: bar: - e: f - g: h - i: j """ INCLUDE_MACRO = '!COMMON' MERGE_MACRO = '_COMMON_' def __init__(self, stream): super(Loader, self).__init__(stream) def construct_mapping(self, *args, **kwargs): data = super(Loader, self).construct_mapping(*args, **kwargs) attribute_path = data.pop(Loader.MERGE_MACRO, None) if attribute_path: for path in attribute_path.split(','): data.update(self._GetData(path)) return data def construct_sequence(self, *args, **kwargs): data = super(Loader, self).construct_sequence(*args, **kwargs) new_list = [] for i in data: if isinstance(i, basestring) and i.startswith( Loader.MERGE_MACRO): attribute_path = i[len(Loader.MERGE_MACRO):] for path in attribute_path.split(','): new_list.extend(self._GetData(path)) else: new_list.append(i) return new_list def include(self, node): attribute_path = self.construct_scalar(node) return self._GetData(attribute_path) def _GetData(self, attribute_path): if not common_data: raise LayoutException( 'Command [{}] references common command data but it does not exist.' .format(impl_path)) value = common_data for attribute in attribute_path.split('.'): value = value.get(attribute, None) if not value: raise LayoutException( 'Command [{}] references common command data attribute [{}] in ' 'path [{}] but it does not exist.'.format( impl_path, attribute, attribute_path)) return value Loader.add_constructor(Loader.INCLUDE_MACRO, Loader.include) return Loader