def test_untag_params(self): """Untag params in fields.""" untag_func = untag.Untag.untag fields_to_test = { 'foo': 'base', '*****@*****.**': 'dev', '*****@*****.**': 'prod', } fields = copy.deepcopy(fields_to_test) self.assertDictEqual({ 'foo': 'base', }, untag_func(fields, locale_identifier=None, params={ 'env': untag.UntagParamRegex(None), })) self.assertDictEqual({ 'foo': 'dev', }, untag_func(fields, locale_identifier=None, params={ 'env': untag.UntagParamRegex('dev'), })) self.assertDictEqual({ 'foo': 'prod', }, untag_func(fields, locale_identifier=None, params={ 'env': untag.UntagParamRegex('prod'), }))
def test_untag_params_nested(self): """Untag params in nested fields.""" untag_func = untag.Untag.untag fields_to_test = { 'nested': { 'foo': 'nested-base', }, 'nested@de': { 'foo': 'nested-de-base', '*****@*****.**': 'nested-de-dev', '*****@*****.**': 'nested-de-prod', } } fields = copy.deepcopy(fields_to_test) self.assertDictEqual({ 'nested': { 'foo': 'nested-base', }, }, untag_func(fields, locale_identifier=None, params={ 'env': untag.UntagParamRegex(None), })) self.assertDictEqual({ 'nested': { 'foo': 'nested-base', }, }, untag_func(fields, locale_identifier=None, params={ 'env': untag.UntagParamRegex('dev'), })) self.assertDictEqual({ 'nested': { 'foo': 'nested-de-dev', }, }, untag_func(fields, locale_identifier='de', params={ 'env': untag.UntagParamRegex('dev'), })) self.assertDictEqual({ 'nested': { 'foo': 'nested-de-prod', }, }, untag_func(fields, locale_identifier='de', params={ 'env': untag.UntagParamRegex('prod'), }))
def read_yaml(self, path, locale=None): """Read, parse, and untag a yaml file.""" contents = self.podcache.file_cache.get(path, locale=locale) if contents is None: label = '{} ({})'.format(path, locale) meta = {'path': path, 'locale': locale} with self.profile.timer('Pod.read_yaml', label=label, meta=meta): fields = self.podcache.file_cache.get(path, locale='__raw__') if fields is None: fields = utils.parse_yaml(self.read_file(path), pod=self, locale=locale) self.podcache.file_cache.add(path, fields, locale='__raw__') try: contents = untag.Untag.untag(fields, locale_identifier=locale, params={ 'env': untag.UntagParamRegex( self.env.name), }) self.podcache.file_cache.add(path, contents, locale=locale) except Exception: logging.error('Error parsing -> {}'.format(path)) raise return contents
def make_yaml_loader(pod, doc=None, locale=None, untag_params=None): # A default set of params for nested yaml parsing. if not untag_params and pod: untag_params = { 'env': untag.UntagParamRegex(pod.env.name), } # Tracing function for dependency graph. tracking_func = lambda *args, **kwargs: None if pod and doc: # Add the path to the dependency graph in case it has no external refs. pod.podcache.dependency_graph.add(doc.pod_path, doc.pod_path) def _track_dep(path): pod.podcache.dependency_graph.add(doc.pod_path, path) tracking_func = _track_dep base_loader = make_base_yaml_loader( pod, locale=locale, untag_params=untag_params, tracking_func=tracking_func) class YamlLoader(base_loader): @staticmethod def loader_locale(): return str(doc.locale_safe) if doc else locale @staticmethod def pod_path(): if doc: return doc.pod_path return None def construct_doc(self, node): def func(path): constructed_doc = pod.get_doc(path, locale=self.loader_locale()) if not constructed_doc.exists: raise errors.DocumentDoesNotExistError( 'Referenced document does not exist: {}'.format(path)) tracking_func(constructed_doc.pod_path) return constructed_doc return self._construct_func(node, func) def construct_static(self, node): def func(path): tracking_func(path) return pod.get_static(path, locale=self.loader_locale()) return self._construct_func(node, func) def construct_url(self, node): def func(path): tracking_func(path) return pod.get_url(path, locale=self.loader_locale()) return self._construct_func(node, func) YamlLoader.add_constructor(u'!g.doc', YamlLoader.construct_doc) YamlLoader.add_constructor(u'!g.static', YamlLoader.construct_static) YamlLoader.add_constructor(u'!g.url', YamlLoader.construct_url) return YamlLoader
def fields(self): fields = untag.Untag.untag(self.tagged_fields, params={ 'env': untag.UntagParamRegex( self.pod.env.name), }) return {} if not fields else fields
def untag(self, contents, locale=None): """Untag data using the pod specific untagging params.""" return untag.Untag.untag(contents, locale_identifier=locale, params={ 'env': untag.UntagParamRegex(self.env.name), })
def fields(self): locale_identifier = str(self._locale_kwarg or self.default_locale) return document_fields.DocumentFields( self.format.front_matter.data, locale_identifier, params={ 'env': untag.UntagParamRegex(self.pod.env.name), 'locale': untag.UntagParamLocaleRegex.from_pod( self.pod, self.collection), })
def _load_yaml(self, raw_yaml): try: return utils.load_yaml( raw_yaml, doc=self._doc, pod=self._doc.pod, untag_params={ 'env': untag.UntagParamRegex(self._doc.pod.env.name), 'locale': untag.UntagParamLocaleRegex.from_pod( self._doc.pod, self._doc.collection), }) except (yaml.composer.ComposerError, yaml.parser.ParserError, yaml.reader.ReaderError, yaml.scanner.ScannerError) as error: message = 'Error parsing {}: {}'.format(self._doc.pod_path, error) raise BadFormatError(message)
def set_env(self, env): if not isinstance(env, environment.Env): env = environment.Env(env) if env and env.name: content = untag.Untag.untag(self._parse_yaml(), params={ 'env': untag.UntagParamRegex(env.name), }) self._yaml = content # Preprocessors may depend on env, reset cache. # pylint: disable=no-member self.list_preprocessors.reset() self.podcache.reset() self.env = env
def set_env(self, env): if not isinstance(env, environment.Env): env = environment.Env(env) if env and env.name: content = untag.Untag.untag(self._parse_yaml(), params={ 'env': untag.UntagParamRegex(env.name), }) self._yaml = content # Preprocessors may depend on env, reset cache. # pylint: disable=no-member self.list_preprocessors.reset() self.podcache.reset() # Need to reload the extension configs for changes based on env. self._extensions_controller.update_extension_configs( self.yaml.get('ext', [])) self.env = env
def make_base_yaml_loader(pod, locale=None, untag_params=None, tracking_func=None): """Make a base yaml loader that does not touch collections or docs.""" if not tracking_func: tracking_func = lambda *args, **kwargs: None # A default set of params for nested yaml parsing. if not untag_params and pod: untag_params = { 'env': untag.UntagParamRegex(pod.env.name), } class BaseYamlLoader(yaml_Loader): @staticmethod def loader_locale(): return locale @staticmethod def pod_path(): return None @staticmethod def read_csv(pod_path): """Reads a csv file using a cache.""" file_cache = pod.podcache.file_cache contents = file_cache.get(pod_path) if contents is None: contents = pod.read_csv(pod_path) file_cache.add(pod_path, contents) return contents @staticmethod def read_file(pod_path): """Reads a file using a cache.""" file_cache = pod.podcache.file_cache contents = file_cache.get(pod_path) if contents is None: contents = pod.read_file(pod_path) file_cache.add(pod_path, contents) return contents @staticmethod def read_json(pod_path): """Reads a json file using a cache.""" file_cache = pod.podcache.file_cache contents = file_cache.get(pod_path) if contents is None: contents = pod.read_json(pod_path) file_cache.add(pod_path, contents) return contents @classmethod def read_string(cls, path): if '.' not in path: return None main, reference = path.split('.', 1) path = '/content/strings/{}.yaml'.format(main) tracking_func(path) if reference: data = structures.DeepReferenceDict( cls.read_yaml(path, locale=cls.loader_locale())) try: allow_draft = pod.podspec.fields.get('strings', {}).get('allow_draft') if allow_draft is False and data.get(DRAFT_KEY): raise DraftStringError( 'Encountered string in draft -> {}?{}'.format( path, reference)) value = data[reference] if value is None: if cls.pod_path(): pod.logger.warning('Missing {}.{} in {}'.format( main, reference, cls.pod_path())) pod.logger.warning('Missing {}.{}'.format( main, reference)) return value except KeyError: return None return None @classmethod def read_yaml(cls, pod_path, locale): """Reads a yaml file using a cache.""" file_cache = pod.podcache.file_cache contents = file_cache.get(pod_path, locale=locale) if contents is None: # Cannot use the file cache to store the raw data with the # `yaml.load` since constructors in the yaml loading are already # completed with the provided locale so untagged data is lost # and cannot be stored as raw data. contents = yaml.load(pod.read_file(pod_path), Loader=cls) or {} contents = untag.Untag.untag(contents, locale_identifier=locale, params=untag_params) file_cache.add(pod_path, contents, locale=locale) return contents def _construct_func(self, node, func): if isinstance(node, yaml.SequenceNode): items = [] for i, each in enumerate(node.value): items.append(func(node.value[i].value)) return items return func(node.value) def _track_dep_func(self, func): """Wrap a function with a call to the tracking function.""" def _func(path, *args, **kwargs): tracking_func(path) return func(path, *args, **kwargs) return _func def construct_csv(self, node): return self._construct_func(node, self._track_dep_func(self.read_csv)) def construct_file(self, node): return self._construct_func(node, self._track_dep_func(self.read_file)) def construct_gettext(self, node): return self._construct_func(node, gettext.gettext) def construct_json(self, node): return self._construct_func(node, self._track_dep_func(self.read_json)) def construct_string(self, node): return self._construct_func(node, self.read_string) def construct_yaml(self, node): def func(path): if '?' in path: path, reference = path.split('?') tracking_func(path) data = structures.DeepReferenceDict( self.read_yaml(path, locale=self.loader_locale())) try: return data[reference] except KeyError: return None tracking_func(path) return self.read_yaml(path, locale=self.loader_locale()) return self._construct_func(node, func) BaseYamlLoader.add_constructor(u'!_', BaseYamlLoader.construct_gettext) BaseYamlLoader.add_constructor(u'!g.csv', BaseYamlLoader.construct_csv) BaseYamlLoader.add_constructor(u'!g.file', BaseYamlLoader.construct_file) BaseYamlLoader.add_constructor(u'!g.json', BaseYamlLoader.construct_json) BaseYamlLoader.add_constructor(u'!g.string', BaseYamlLoader.construct_string) BaseYamlLoader.add_constructor(u'!g.yaml', BaseYamlLoader.construct_yaml) return BaseYamlLoader
def make_yaml_loader(pod, doc=None, locale=None, untag_params=None): loader_locale = locale # A default set of params for nested yaml parsing. if not untag_params and pod: untag_params = { 'env': untag.UntagParamRegex(pod.env.name), } class YamlLoader(yaml_Loader): @staticmethod def read_csv(pod_path): """Reads a csv file using a cache.""" file_cache = pod.podcache.file_cache contents = file_cache.get(pod_path) if contents is None: contents = pod.read_csv(pod_path) file_cache.add(pod_path, contents) return contents @staticmethod def read_file(pod_path): """Reads a file using a cache.""" file_cache = pod.podcache.file_cache contents = file_cache.get(pod_path) if contents is None: contents = pod.read_file(pod_path) file_cache.add(pod_path, contents) return contents @staticmethod def read_json(pod_path): """Reads a json file using a cache.""" file_cache = pod.podcache.file_cache contents = file_cache.get(pod_path) if contents is None: contents = pod.read_json(pod_path) file_cache.add(pod_path, contents) return contents @classmethod def read_yaml(cls, pod_path, locale): """Reads a yaml file using a cache.""" file_cache = pod.podcache.file_cache contents = file_cache.get(pod_path, locale=locale) if contents is None: contents = yaml.load(pod.read_file(pod_path), Loader=cls) or {} contents = untag.Untag.untag(contents, locale_identifier=locale, params=untag_params) file_cache.add(pod_path, contents, locale=locale) return contents def _construct_func(self, node, func): if isinstance(node, yaml.SequenceNode): items = [] for i, each in enumerate(node.value): items.append(func(node.value[i].value)) return items return func(node.value) def construct_csv(self, node): def func(path): if doc: pod.podcache.dependency_graph.add(doc.pod_path, path) return self.read_csv(path) return self._construct_func(node, func) def construct_doc(self, node): locale = str(doc.locale_safe) if doc else loader_locale pod_path = doc.pod_path if doc else None def func(path): contructed_doc = pod.get_doc(path, locale=locale) if not contructed_doc.exists: raise errors.DocumentDoesNotExistError( 'Referenced document does not exist: {}'.format(path)) pod.podcache.dependency_graph.add(pod_path, contructed_doc.pod_path) return contructed_doc return self._construct_func(node, func) def construct_file(self, node): def func(path): if doc: pod.podcache.dependency_graph.add(doc.pod_path, path) return self.read_file(path) return self._construct_func(node, func) def construct_gettext(self, node): return self._construct_func(node, gettext.gettext) def construct_json(self, node): if doc: pod.podcache.dependency_graph.add(doc.pod_path, node.value) return self._construct_func(node, self.read_json) def construct_static(self, node): locale = str(doc.locale_safe) if doc else loader_locale def func(path): if doc: pod.podcache.dependency_graph.add(doc.pod_path, path) return pod.get_static(path, locale=locale) return self._construct_func(node, func) def construct_string(self, node): def func(path): if '.' not in path: return None main, reference = path.split('.', 1) path = '/content/strings/{}.yaml'.format(main) locale = str(doc.locale_safe) if doc else loader_locale if doc: pod.podcache.dependency_graph.add(doc.pod_path, path) if reference: data = structures.DeepReferenceDict( self.read_yaml(path, locale=locale)) try: value = data[reference] if value is None: if doc: pod.logger.warning( 'Missing {}.{} in {}'.format( main, reference, doc.pod_path)) else: pod.logger.warning('Missing {}.{}'.format( main, reference)) return value except KeyError: return None return None return self._construct_func(node, func) def construct_url(self, node): locale = str(doc.locale_safe) if doc else loader_locale def func(path): if doc: pod.podcache.dependency_graph.add(doc.pod_path, path) return pod.get_url(path, locale=locale) return self._construct_func(node, func) def construct_yaml(self, node): def func(path): locale = str(doc.locale_safe) if doc else loader_locale if '?' in path: path, reference = path.split('?') if doc: pod.podcache.dependency_graph.add(doc.pod_path, path) data = structures.DeepReferenceDict( self.read_yaml(path, locale=locale)) try: return data[reference] except KeyError: return None if doc: pod.podcache.dependency_graph.add(doc.pod_path, path) return self.read_yaml(path, locale=locale) return self._construct_func(node, func) YamlLoader.add_constructor(u'!_', YamlLoader.construct_gettext) YamlLoader.add_constructor(u'!g.csv', YamlLoader.construct_csv) YamlLoader.add_constructor(u'!g.doc', YamlLoader.construct_doc) YamlLoader.add_constructor(u'!g.file', YamlLoader.construct_file) YamlLoader.add_constructor(u'!g.json', YamlLoader.construct_json) YamlLoader.add_constructor(u'!g.static', YamlLoader.construct_static) YamlLoader.add_constructor(u'!g.string', YamlLoader.construct_string) YamlLoader.add_constructor(u'!g.url', YamlLoader.construct_url) YamlLoader.add_constructor(u'!g.yaml', YamlLoader.construct_yaml) return YamlLoader
def test_untag_none(self): """Untag when there is a none value for the tagged value.""" untag_func = untag.Untag.untag fields_to_test = { 'foo': 'base', '*****@*****.**': None, } fields = copy.deepcopy(fields_to_test) self.assertDictEqual({ 'foo': 'base', }, untag_func(fields, locale_identifier=None, params={ 'env': untag.UntagParamRegex(None), })) self.assertDictEqual({ 'foo': None, }, untag_func(fields, locale_identifier=None, params={ 'env': untag.UntagParamRegex('prod'), })) fields_to_test = { 'nested': { 'foo': 'nested-base', }, 'nested@de': { 'foo': 'nested-de-base', '*****@*****.**': None, } } fields = copy.deepcopy(fields_to_test) self.assertDictEqual({ 'nested': { 'foo': 'nested-base', }, }, untag_func(fields, locale_identifier=None, params={ 'env': untag.UntagParamRegex(None), })) self.assertDictEqual({ 'nested': { 'foo': 'nested-base', }, }, untag_func(fields, locale_identifier=None, params={ 'env': untag.UntagParamRegex('dev'), })) self.assertDictEqual({ 'nested': { 'foo': None, }, }, untag_func(fields, locale_identifier='de', params={ 'env': untag.UntagParamRegex('prod'), }))