def test_untag_params(self):
"""Untag params in fields."""
untag_func = untag.Untag.untag
fields_to_test = {
'foo': 'base',
'*****@*****.**': 'dev',
'*****@*****.**': 'prod',
}
fields = copy.deepcopy(fields_to_test)
self.assertDictEqual({
'foo': 'base',
},
untag_func(fields,
locale_identifier=None,
params={
'env': untag.UntagParamRegex(None),
}))
self.assertDictEqual({
'foo': 'dev',
},
untag_func(fields,
locale_identifier=None,
params={
'env':
untag.UntagParamRegex('dev'),
}))
self.assertDictEqual({
'foo': 'prod',
},
untag_func(fields,
locale_identifier=None,
params={
'env':
untag.UntagParamRegex('prod'),
}))
def test_untag_params_nested(self):
"""Untag params in nested fields."""
untag_func = untag.Untag.untag
fields_to_test = {
'nested': {
'foo': 'nested-base',
},
'nested@de': {
'foo': 'nested-de-base',
'*****@*****.**': 'nested-de-dev',
'*****@*****.**': 'nested-de-prod',
}
}
fields = copy.deepcopy(fields_to_test)
self.assertDictEqual({
'nested': {
'foo': 'nested-base',
},
},
untag_func(fields,
locale_identifier=None,
params={
'env': untag.UntagParamRegex(None),
}))
self.assertDictEqual({
'nested': {
'foo': 'nested-base',
},
},
untag_func(fields,
locale_identifier=None,
params={
'env':
untag.UntagParamRegex('dev'),
}))
self.assertDictEqual({
'nested': {
'foo': 'nested-de-dev',
},
},
untag_func(fields,
locale_identifier='de',
params={
'env':
untag.UntagParamRegex('dev'),
}))
self.assertDictEqual({
'nested': {
'foo': 'nested-de-prod',
},
},
untag_func(fields,
locale_identifier='de',
params={
'env':
untag.UntagParamRegex('prod'),
}))
def read_yaml(self, path, locale=None):
"""Read, parse, and untag a yaml file."""
contents = self.podcache.file_cache.get(path, locale=locale)
if contents is None:
label = '{} ({})'.format(path, locale)
meta = {'path': path, 'locale': locale}
with self.profile.timer('Pod.read_yaml', label=label, meta=meta):
fields = self.podcache.file_cache.get(path, locale='__raw__')
if fields is None:
fields = utils.parse_yaml(self.read_file(path),
pod=self,
locale=locale)
self.podcache.file_cache.add(path,
fields,
locale='__raw__')
try:
contents = untag.Untag.untag(fields,
locale_identifier=locale,
params={
'env':
untag.UntagParamRegex(
self.env.name),
})
self.podcache.file_cache.add(path, contents, locale=locale)
except Exception:
logging.error('Error parsing -> {}'.format(path))
raise
return contents
def make_yaml_loader(pod, doc=None, locale=None, untag_params=None):
# A default set of params for nested yaml parsing.
if not untag_params and pod:
untag_params = {
'env': untag.UntagParamRegex(pod.env.name),
}
# Tracing function for dependency graph.
tracking_func = lambda *args, **kwargs: None
if pod and doc:
# Add the path to the dependency graph in case it has no external refs.
pod.podcache.dependency_graph.add(doc.pod_path, doc.pod_path)
def _track_dep(path):
pod.podcache.dependency_graph.add(doc.pod_path, path)
tracking_func = _track_dep
base_loader = make_base_yaml_loader(
pod, locale=locale, untag_params=untag_params,
tracking_func=tracking_func)
class YamlLoader(base_loader):
@staticmethod
def loader_locale():
return str(doc.locale_safe) if doc else locale
@staticmethod
def pod_path():
if doc:
return doc.pod_path
return None
def construct_doc(self, node):
def func(path):
constructed_doc = pod.get_doc(path, locale=self.loader_locale())
if not constructed_doc.exists:
raise errors.DocumentDoesNotExistError(
'Referenced document does not exist: {}'.format(path))
tracking_func(constructed_doc.pod_path)
return constructed_doc
return self._construct_func(node, func)
def construct_static(self, node):
def func(path):
tracking_func(path)
return pod.get_static(path, locale=self.loader_locale())
return self._construct_func(node, func)
def construct_url(self, node):
def func(path):
tracking_func(path)
return pod.get_url(path, locale=self.loader_locale())
return self._construct_func(node, func)
YamlLoader.add_constructor(u'!g.doc', YamlLoader.construct_doc)
YamlLoader.add_constructor(u'!g.static', YamlLoader.construct_static)
YamlLoader.add_constructor(u'!g.url', YamlLoader.construct_url)
return YamlLoader
def fields(self):
fields = untag.Untag.untag(self.tagged_fields,
params={
'env':
untag.UntagParamRegex(
self.pod.env.name),
})
return {} if not fields else fields
def untag(self, contents, locale=None):
"""Untag data using the pod specific untagging params."""
return untag.Untag.untag(contents,
locale_identifier=locale,
params={
'env':
untag.UntagParamRegex(self.env.name),
})
def fields(self):
locale_identifier = str(self._locale_kwarg or self.default_locale)
return document_fields.DocumentFields(
self.format.front_matter.data, locale_identifier,
params={
'env': untag.UntagParamRegex(self.pod.env.name),
'locale': untag.UntagParamLocaleRegex.from_pod(
self.pod, self.collection),
})
def _load_yaml(self, raw_yaml):
try:
return utils.load_yaml(
raw_yaml, doc=self._doc, pod=self._doc.pod,
untag_params={
'env': untag.UntagParamRegex(self._doc.pod.env.name),
'locale': untag.UntagParamLocaleRegex.from_pod(
self._doc.pod, self._doc.collection),
})
except (yaml.composer.ComposerError,
yaml.parser.ParserError,
yaml.reader.ReaderError,
yaml.scanner.ScannerError) as error:
message = 'Error parsing {}: {}'.format(self._doc.pod_path, error)
raise BadFormatError(message)
def set_env(self, env):
if not isinstance(env, environment.Env):
env = environment.Env(env)
if env and env.name:
content = untag.Untag.untag(self._parse_yaml(),
params={
'env':
untag.UntagParamRegex(env.name),
})
self._yaml = content
# Preprocessors may depend on env, reset cache.
# pylint: disable=no-member
self.list_preprocessors.reset()
self.podcache.reset()
self.env = env
def set_env(self, env):
if not isinstance(env, environment.Env):
env = environment.Env(env)
if env and env.name:
content = untag.Untag.untag(self._parse_yaml(), params={
'env': untag.UntagParamRegex(env.name),
})
self._yaml = content
# Preprocessors may depend on env, reset cache.
# pylint: disable=no-member
self.list_preprocessors.reset()
self.podcache.reset()
# Need to reload the extension configs for changes based on env.
self._extensions_controller.update_extension_configs(
self.yaml.get('ext', []))
self.env = env
def make_base_yaml_loader(pod,
locale=None,
untag_params=None,
tracking_func=None):
"""Make a base yaml loader that does not touch collections or docs."""
if not tracking_func:
tracking_func = lambda *args, **kwargs: None
# A default set of params for nested yaml parsing.
if not untag_params and pod:
untag_params = {
'env': untag.UntagParamRegex(pod.env.name),
}
class BaseYamlLoader(yaml_Loader):
@staticmethod
def loader_locale():
return locale
@staticmethod
def pod_path():
return None
@staticmethod
def read_csv(pod_path):
"""Reads a csv file using a cache."""
file_cache = pod.podcache.file_cache
contents = file_cache.get(pod_path)
if contents is None:
contents = pod.read_csv(pod_path)
file_cache.add(pod_path, contents)
return contents
@staticmethod
def read_file(pod_path):
"""Reads a file using a cache."""
file_cache = pod.podcache.file_cache
contents = file_cache.get(pod_path)
if contents is None:
contents = pod.read_file(pod_path)
file_cache.add(pod_path, contents)
return contents
@staticmethod
def read_json(pod_path):
"""Reads a json file using a cache."""
file_cache = pod.podcache.file_cache
contents = file_cache.get(pod_path)
if contents is None:
contents = pod.read_json(pod_path)
file_cache.add(pod_path, contents)
return contents
@classmethod
def read_string(cls, path):
if '.' not in path:
return None
main, reference = path.split('.', 1)
path = '/content/strings/{}.yaml'.format(main)
tracking_func(path)
if reference:
data = structures.DeepReferenceDict(
cls.read_yaml(path, locale=cls.loader_locale()))
try:
allow_draft = pod.podspec.fields.get('strings',
{}).get('allow_draft')
if allow_draft is False and data.get(DRAFT_KEY):
raise DraftStringError(
'Encountered string in draft -> {}?{}'.format(
path, reference))
value = data[reference]
if value is None:
if cls.pod_path():
pod.logger.warning('Missing {}.{} in {}'.format(
main, reference, cls.pod_path()))
pod.logger.warning('Missing {}.{}'.format(
main, reference))
return value
except KeyError:
return None
return None
@classmethod
def read_yaml(cls, pod_path, locale):
"""Reads a yaml file using a cache."""
file_cache = pod.podcache.file_cache
contents = file_cache.get(pod_path, locale=locale)
if contents is None:
# Cannot use the file cache to store the raw data with the
# `yaml.load` since constructors in the yaml loading are already
# completed with the provided locale so untagged data is lost
# and cannot be stored as raw data.
contents = yaml.load(pod.read_file(pod_path), Loader=cls) or {}
contents = untag.Untag.untag(contents,
locale_identifier=locale,
params=untag_params)
file_cache.add(pod_path, contents, locale=locale)
return contents
def _construct_func(self, node, func):
if isinstance(node, yaml.SequenceNode):
items = []
for i, each in enumerate(node.value):
items.append(func(node.value[i].value))
return items
return func(node.value)
def _track_dep_func(self, func):
"""Wrap a function with a call to the tracking function."""
def _func(path, *args, **kwargs):
tracking_func(path)
return func(path, *args, **kwargs)
return _func
def construct_csv(self, node):
return self._construct_func(node,
self._track_dep_func(self.read_csv))
def construct_file(self, node):
return self._construct_func(node,
self._track_dep_func(self.read_file))
def construct_gettext(self, node):
return self._construct_func(node, gettext.gettext)
def construct_json(self, node):
return self._construct_func(node,
self._track_dep_func(self.read_json))
def construct_string(self, node):
return self._construct_func(node, self.read_string)
def construct_yaml(self, node):
def func(path):
if '?' in path:
path, reference = path.split('?')
tracking_func(path)
data = structures.DeepReferenceDict(
self.read_yaml(path, locale=self.loader_locale()))
try:
return data[reference]
except KeyError:
return None
tracking_func(path)
return self.read_yaml(path, locale=self.loader_locale())
return self._construct_func(node, func)
BaseYamlLoader.add_constructor(u'!_', BaseYamlLoader.construct_gettext)
BaseYamlLoader.add_constructor(u'!g.csv', BaseYamlLoader.construct_csv)
BaseYamlLoader.add_constructor(u'!g.file', BaseYamlLoader.construct_file)
BaseYamlLoader.add_constructor(u'!g.json', BaseYamlLoader.construct_json)
BaseYamlLoader.add_constructor(u'!g.string',
BaseYamlLoader.construct_string)
BaseYamlLoader.add_constructor(u'!g.yaml', BaseYamlLoader.construct_yaml)
return BaseYamlLoader
def make_yaml_loader(pod, doc=None, locale=None, untag_params=None):
loader_locale = locale
# A default set of params for nested yaml parsing.
if not untag_params and pod:
untag_params = {
'env': untag.UntagParamRegex(pod.env.name),
}
class YamlLoader(yaml_Loader):
@staticmethod
def read_csv(pod_path):
"""Reads a csv file using a cache."""
file_cache = pod.podcache.file_cache
contents = file_cache.get(pod_path)
if contents is None:
contents = pod.read_csv(pod_path)
file_cache.add(pod_path, contents)
return contents
@staticmethod
def read_file(pod_path):
"""Reads a file using a cache."""
file_cache = pod.podcache.file_cache
contents = file_cache.get(pod_path)
if contents is None:
contents = pod.read_file(pod_path)
file_cache.add(pod_path, contents)
return contents
@staticmethod
def read_json(pod_path):
"""Reads a json file using a cache."""
file_cache = pod.podcache.file_cache
contents = file_cache.get(pod_path)
if contents is None:
contents = pod.read_json(pod_path)
file_cache.add(pod_path, contents)
return contents
@classmethod
def read_yaml(cls, pod_path, locale):
"""Reads a yaml file using a cache."""
file_cache = pod.podcache.file_cache
contents = file_cache.get(pod_path, locale=locale)
if contents is None:
contents = yaml.load(pod.read_file(pod_path), Loader=cls) or {}
contents = untag.Untag.untag(contents,
locale_identifier=locale,
params=untag_params)
file_cache.add(pod_path, contents, locale=locale)
return contents
def _construct_func(self, node, func):
if isinstance(node, yaml.SequenceNode):
items = []
for i, each in enumerate(node.value):
items.append(func(node.value[i].value))
return items
return func(node.value)
def construct_csv(self, node):
def func(path):
if doc:
pod.podcache.dependency_graph.add(doc.pod_path, path)
return self.read_csv(path)
return self._construct_func(node, func)
def construct_doc(self, node):
locale = str(doc.locale_safe) if doc else loader_locale
pod_path = doc.pod_path if doc else None
def func(path):
contructed_doc = pod.get_doc(path, locale=locale)
if not contructed_doc.exists:
raise errors.DocumentDoesNotExistError(
'Referenced document does not exist: {}'.format(path))
pod.podcache.dependency_graph.add(pod_path,
contructed_doc.pod_path)
return contructed_doc
return self._construct_func(node, func)
def construct_file(self, node):
def func(path):
if doc:
pod.podcache.dependency_graph.add(doc.pod_path, path)
return self.read_file(path)
return self._construct_func(node, func)
def construct_gettext(self, node):
return self._construct_func(node, gettext.gettext)
def construct_json(self, node):
if doc:
pod.podcache.dependency_graph.add(doc.pod_path, node.value)
return self._construct_func(node, self.read_json)
def construct_static(self, node):
locale = str(doc.locale_safe) if doc else loader_locale
def func(path):
if doc:
pod.podcache.dependency_graph.add(doc.pod_path, path)
return pod.get_static(path, locale=locale)
return self._construct_func(node, func)
def construct_string(self, node):
def func(path):
if '.' not in path:
return None
main, reference = path.split('.', 1)
path = '/content/strings/{}.yaml'.format(main)
locale = str(doc.locale_safe) if doc else loader_locale
if doc:
pod.podcache.dependency_graph.add(doc.pod_path, path)
if reference:
data = structures.DeepReferenceDict(
self.read_yaml(path, locale=locale))
try:
value = data[reference]
if value is None:
if doc:
pod.logger.warning(
'Missing {}.{} in {}'.format(
main, reference, doc.pod_path))
else:
pod.logger.warning('Missing {}.{}'.format(
main, reference))
return value
except KeyError:
return None
return None
return self._construct_func(node, func)
def construct_url(self, node):
locale = str(doc.locale_safe) if doc else loader_locale
def func(path):
if doc:
pod.podcache.dependency_graph.add(doc.pod_path, path)
return pod.get_url(path, locale=locale)
return self._construct_func(node, func)
def construct_yaml(self, node):
def func(path):
locale = str(doc.locale_safe) if doc else loader_locale
if '?' in path:
path, reference = path.split('?')
if doc:
pod.podcache.dependency_graph.add(doc.pod_path, path)
data = structures.DeepReferenceDict(
self.read_yaml(path, locale=locale))
try:
return data[reference]
except KeyError:
return None
if doc:
pod.podcache.dependency_graph.add(doc.pod_path, path)
return self.read_yaml(path, locale=locale)
return self._construct_func(node, func)
YamlLoader.add_constructor(u'!_', YamlLoader.construct_gettext)
YamlLoader.add_constructor(u'!g.csv', YamlLoader.construct_csv)
YamlLoader.add_constructor(u'!g.doc', YamlLoader.construct_doc)
YamlLoader.add_constructor(u'!g.file', YamlLoader.construct_file)
YamlLoader.add_constructor(u'!g.json', YamlLoader.construct_json)
YamlLoader.add_constructor(u'!g.static', YamlLoader.construct_static)
YamlLoader.add_constructor(u'!g.string', YamlLoader.construct_string)
YamlLoader.add_constructor(u'!g.url', YamlLoader.construct_url)
YamlLoader.add_constructor(u'!g.yaml', YamlLoader.construct_yaml)
return YamlLoader
def test_untag_none(self):
"""Untag when there is a none value for the tagged value."""
untag_func = untag.Untag.untag
fields_to_test = {
'foo': 'base',
'*****@*****.**': None,
}
fields = copy.deepcopy(fields_to_test)
self.assertDictEqual({
'foo': 'base',
},
untag_func(fields,
locale_identifier=None,
params={
'env': untag.UntagParamRegex(None),
}))
self.assertDictEqual({
'foo': None,
},
untag_func(fields,
locale_identifier=None,
params={
'env':
untag.UntagParamRegex('prod'),
}))
fields_to_test = {
'nested': {
'foo': 'nested-base',
},
'nested@de': {
'foo': 'nested-de-base',
'*****@*****.**': None,
}
}
fields = copy.deepcopy(fields_to_test)
self.assertDictEqual({
'nested': {
'foo': 'nested-base',
},
},
untag_func(fields,
locale_identifier=None,
params={
'env': untag.UntagParamRegex(None),
}))
self.assertDictEqual({
'nested': {
'foo': 'nested-base',
},
},
untag_func(fields,
locale_identifier=None,
params={
'env':
untag.UntagParamRegex('dev'),
}))
self.assertDictEqual({
'nested': {
'foo': None,
},
},
untag_func(fields,
locale_identifier='de',
params={
'env':
untag.UntagParamRegex('prod'),
}))
