class DjangoCompatTest(SimpleTestCase):
classes = ['DjangoCsrf', 'DjangoI18n', 'DjangoStatic', 'DjangoUrl']
class CalledParse(Exception):
pass
@classmethod
def make_side_effect(cls, cls_name):
def parse(self, parser):
raise cls.CalledParse(cls_name)
return parse
def setUp(self):
for class_name in self.classes:
patcher = mock.patch(
'jdj_tags.extensions.{}.parse'.format(class_name),
side_effect=self.make_side_effect(class_name)
)
patcher.start()
self.addCleanup(patcher.stop)
self.env = Environment(extensions=[DjangoCompat])
def test_compat(self):
tags = [
('csrf_token', 'DjangoCsrf'),
('trans', 'DjangoI18n'),
('blocktrans', 'DjangoI18n'),
('static', 'DjangoStatic'),
('url', 'DjangoUrl'),
]
for tag, class_name in tags:
with self.assertRaisesMessage(self.CalledParse, class_name):
self.env.from_string('{% ' + tag + ' %}')
def test_with_extension():
env = Environment(extensions=[WithExtension])
t = env.from_string('{{ a }}{% with 2 as a %}{{ a }}{% endwith %}{{ a }}')
assert t.render(a=1) == '121'
t = env.from_string('{% with 2 as a %}{{ a }}{% endwith %}{{ a }}')
assert t.render(a=3) == '23'
def process(self, **context):
"""
Process handler request. Before executing requests render templates with context
:param context: Processing context
:returns: Requests response `<http://docs.python-requests.org/en/master/api/#requests.Response>` _.
"""
env = Environment(extensions=['jinja2_time.TimeExtension'])
url_template = env.from_string(self.url_template)
url = url_template.render(**context).replace(" ", "")
logger.debug("Request %s generates url %s" % (self, url))
params = self._url_params(**context)
logger.debug("Request %s generates params %s" % (self, params))
headers = self._header_params(**context)
logger.debug("Request %s generates header %s" % (self, headers))
if self.data_required():
data_template = env.from_string(self.data)
data = data_template.render(**context)
logger.debug("Request %s generates data %s" % (self, data))
r = self._get_method()(url, data=json.loads(data), headers=headers, params=params)
else:
r = self._get_method()(url, headers=headers, params=params)
return r
def test_replace(self):
env = Environment()
tmpl = env.from_string('{{ string|replace("o", 42) }}')
env = Environment(autoescape=True)
tmpl = env.from_string('{{ string|replace("o", 42) }}')
tmpl = env.from_string('{{ string|replace("<", 42) }}')
tmpl = env.from_string('{{ string|replace("o", ">x<") }}')
def test_spaceless():
from coffin.template.defaulttags import SpacelessExtension
env = Environment(extensions=[SpacelessExtension])
assert (
env.from_string(
"""{% spaceless %}
<p>
<a href="foo/">Foo</a>
</p>
{% endspaceless %}"""
).render()
== '<p><a href="foo/">Foo</a></p>'
)
assert (
env.from_string(
"""{% spaceless %}
<strong>
Hello
</strong>
{% endspaceless %}"""
).render()
== "<strong>\n Hello\n </strong>"
)
class MacrosTestCase(JinjaTestCase):
env = Environment(trim_blocks=True)
def test_simple(self):
tmpl = self.env.from_string("{% macro say_hello(name) %}Hello {{ name }}!{% endmacro %}\n{{ say_hello('Peter') }}")
def test_scoping(self):
tmpl = self.env.from_string("{% macro level1(data1) %}\n{% macro level2(data2) %}{{ data1 }}|{{ data2 }}{% endmacro %}\n{{ level2('bar') }}{% endmacro %}\n{{ level1('foo') }}")
def test_arguments(self):
tmpl = self.env.from_string("{% macro m(a, b, c='c', d='d') %}{{ a }}|{{ b }}|{{ c }}|{{ d }}{% endmacro %}\n{{ m() }}|{{ m('a') }}|{{ m('a', 'b') }}|{{ m(1, 2, 3) }}")
def test_varargs(self):
tmpl = self.env.from_string("{% macro test() %}{{ varargs|join('|') }}{% endmacro %}{{ test(1, 2, 3) }}")
def test_simple_call(self):
tmpl = self.env.from_string('{% macro test() %}[[{{ caller() }}]]{% endmacro %}{% call test() %}data{% endcall %}')
def test_complex_call(self):
tmpl = self.env.from_string("{% macro test() %}[[{{ caller('data') }}]]{% endmacro %}{% call(data) test() %}{{ data }}{% endcall %}")
def test_caller_undefined(self):
tmpl = self.env.from_string('{% set caller = 42 %}{% macro test() %}{{ caller is not defined }}{% endmacro %}{{ test() }}')
def test_include(self):
self.env = Environment(loader=DictLoader({'include': '{% macro test(foo) %}[{{ foo }}]{% endmacro %}'}))
tmpl = self.env.from_string('{% from "include" import test %}{{ test("foo") }}')
def test_macro_api(self):
tmpl = self.env.from_string('{% macro foo(a, b) %}{% endmacro %}{% macro bar() %}{{ varargs }}{{ kwargs }}{% endmacro %}{% macro baz() %}{{ caller() }}{% endmacro %}')
def test_callself(self):
tmpl = self.env.from_string('{% macro foo(x) %}{{ x }}{% if x > 1 %}|{{ foo(x - 1) }}{% endif %}{% endmacro %}{{ foo(5) }}')
def fill_template(player_data, lang, fundamental, yaku):
"""Build long text which shows the statistics of the target
player(s).
"""
target_games = player_data[0]['games'] if player_data else None
if not target_games:
return 'NO DATA\n'
env = Environment(autoescape=False)
env.filters.update(
format_float=format_float,
format_percentage=format_percentage,)
output_text = env.from_string(lang.tmpl_summary).render(
count_games=len(target_games),
started_at=target_games[0]['started_at'],
finished_at=target_games[-1]['finished_at'],
data=player_data)
if fundamental:
output_text += env.from_string(lang.tmpl_fundamental).render(
data=player_data)
if yaku:
yaku_name_map = {y:lang.yaku_names[i]
for i, y in enumerate(YakuTable)}
output_text += env.from_string(lang.tmpl_yaku_freq).render(
data=player_data,
YakuTable=YakuTable,
yaku_name_map=yaku_name_map)
return output_text
def test_cache(self):
from django_cofingo.extensions import CacheExtension
env = Environment(extensions=[CacheExtension])
x = 0
result = env.from_string(
'{%cache 500 "ab"%}{{x}}{%endcache%}').render({'x': x})
self.assertEqual(result, '0')
# cache is used; Jinja2 expressions work
x += 1
result = env.from_string(
'{%cache 50*10 "a"+"b"%}{{x}}{%endcache%}').render({'x': x})
self.assertEqual(result, '0')
# vary-arguments can be used
x += 1
result = env.from_string(
'{%cache 50*10 "ab" x "foo"%}{{x}}{%endcache%}').render({'x': x})
self.assertEqual(result, '2')
x += 1
result = env.from_string(
'{%cache 50*10 "ab" x "foo"%}{{x}}{%endcache%}').render({'x': x})
self.assertEqual(result, '3')
def main():
'''main function'''
args = get_parser().parse_args()
ec2 = boto3.resource('ec2')
print 'Instances:', list(i.id for i in ec2.instances.all())
env = Environment()
hosts = env.from_string(hosts_template)
ssh_config = env.from_string(ssh_config_template)
head_public_ip = get_head_public_ip(ec2)
worker_private_ips = get_work_private_ips(ec2)
hosts_rendered = hosts.render(head_public_ip=head_public_ip,
worker_private_ips=worker_private_ips)
ssh_config_rendered = ssh_config.render(head_public_ip=head_public_ip,
private_key=KEY_NAME)
if args.dry_run:
print '{:*^30}'.format(' Hosts ')
print hosts_rendered
print '{:*^30}'.format(' ssh_config ')
print ssh_config_rendered
else:
with open('hosts', 'w') as fd:
fd.write(hosts_rendered)
with open('amazon_ssh_config', 'w') as fd:
fd.write(ssh_config_rendered)
def test_nested_structures(self):
env = Environment(extensions=[SerializerExtension])
rendered = env.from_string('{{ data }}').render(data="foo")
self.assertEqual(rendered, u"foo")
data = OrderedDict([
('foo', OrderedDict([
('bar', 'baz'),
('qux', 42)
])
)
])
rendered = env.from_string('{{ data }}').render(data=data)
self.assertEqual(rendered, u"{'foo': {'bar': 'baz', 'qux': 42}}")
rendered = env.from_string('{{ data }}').render(data=[
OrderedDict(
foo='bar',
),
OrderedDict(
baz=42,
)
])
self.assertEqual(rendered, u"[{'foo': 'bar'}, {'baz': 42}]")
class ExtensionTest(TestCase):
def setUp(self):
self.env = Environment(extensions=[PipelineExtension], loader=
PackageLoader('pipeline', 'templates'))
def test_no_package(self):
template = self.env.from_string(u"""{% compressed_css "unknow" %}""")
self.assertEqual(u'', template.render())
template = self.env.from_string(u"""{% compressed_js "unknow" %}""")
self.assertEqual(u'', template.render())
def test_package_css(self):
template = self.env.from_string(u"""{% compressed_css "screen" %}""")
self.assertEqual(u'<link href="/static/screen.css" rel="stylesheet" type="text/css" />', template.render())
def test_package_css_debug(self):
with pipeline_settings(PIPELINE=False):
template = self.env.from_string(u"""{% compressed_css "screen" %}""")
self.assertEqual(u'''<link href="/static/pipeline/css/first.css" rel="stylesheet" type="text/css" />
<link href="/static/pipeline/css/second.css" rel="stylesheet" type="text/css" />
<link href="/static/pipeline/css/urls.css" rel="stylesheet" type="text/css" />''', template.render())
def test_package_js(self):
template = self.env.from_string(u"""{% compressed_js "scripts" %}""")
self.assertEqual(u'<script type="text/css" src="/static/scripts.css" charset="utf-8"></script>', template.render())
class DjangoStaticTest(SimpleTestCase):
@staticmethod
def _static(path):
return 'Static: {}'.format(path)
def setUp(self):
patcher = mock.patch('jdj_tags.extensions.django_static', side_effect=self._static)
self.static = patcher.start()
self.addCleanup(patcher.stop)
self.env = Environment(extensions=[DjangoStatic])
def test_simple(self):
template = self.env.from_string("{% static 'static.png' %}")
self.assertEqual('Static: static.png', template.render())
self.static.assert_called_with('static.png')
def test_as_var(self):
template = self.env.from_string(
"{% static 'static.png' as my_url %}My url is: {{ my_url }}!"
)
self.assertEqual('My url is: Static: static.png!', template.render())
self.static.assert_called_with('static.png')
def test():
from jinja2 import Environment
env = Environment(extensions=[HTMLCompress])
tmpl = env.from_string('''
<html>
<head>
<title>{{ title }}</title>
</head>
<script type=text/javascript>
if (foo < 42) {
document.write('Foo < Bar');
}
</script>
<body>
<li><a href="{{ href }}">{{ title }}</a><br>Test Foo
<li><a href="{{ href }}">{{ title }}</a><img src=test.png>
</body>
</html>
''')
print "-" * 30, "HTMLCompress"
print tmpl.render(title=42, href='index.html')
env = Environment(extensions=[SelectiveHTMLCompress])
tmpl = env.from_string('''
Normal <span> unchanged </span> stuff
{% strip %}Stripped <span class=foo > test </span>
<a href="foo"> test </a> {{ foo }}
Normal <stuff> again {{ foo }} </stuff>
<p>
Foo<br>Bar
Baz
<p>
Moep <span>Test</span> Moep
</p>
{% endstrip %}
''')
print "-" * 30, "SelectiveHTMLCompress"
print tmpl.render(foo=42)
env = Environment(extensions=[InvertedSelectiveHTMLCompress])
tmpl = env.from_string('''
{% unstrip %}
Normal <span> unchanged </span> stuff
{% endunstrip %}
Stripped <span class=foo > test </span>
<a href="foo"> test </a> {{ foo }}
Normal <stuff> again {{ foo }} </stuff>
<p>
Foo<br>Bar
Baz
<p>
Moep <span>Test</span> Moep
</p>
''')
print "-" * 30, "InvertedSelectiveHTMLCompress"
print tmpl.render(foo=42)
def test_strict_undefined(self):
env = Environment(undefined=StrictUndefined)
self.assert_raises(UndefinedError, env.from_string('{{ missing }}').render)
self.assert_raises(UndefinedError, env.from_string('{{ missing.attribute }}').render)
self.assert_raises(UndefinedError, env.from_string('{{ missing|list }}').render)
self.assert_equal(env.from_string('{{ missing is not defined }}').render(), 'True')
self.assert_raises(UndefinedError, env.from_string('{{ foo.missing }}').render, foo=42)
self.assert_raises(UndefinedError, env.from_string('{{ not missing }}').render)
def test_nonvolatile(self):
env = Environment(extensions=['jinja2.ext.autoescape'],
autoescape=True)
tmpl = env.from_string('{{ {"foo": "<test>"}|xmlattr|escape }}')
assert tmpl.render() == ' foo="<test>"'
tmpl = env.from_string('{% autoescape false %}{{ {"foo": "<test>"}'
'|xmlattr|escape }}{% endautoescape %}')
assert tmpl.render() == ' foo="&lt;test&gt;"'
def test_load_json_template(self):
loader = DictLoader({'foo': '{"bar": "my god is blue", "foo": [1, 2, 3]}'})
env = Environment(extensions=[SerializerExtension], loader=loader)
rendered = env.from_string('{% import_json "foo" as doc %}{{ doc.bar }}').render()
self.assertEqual(rendered, u"my god is blue")
with self.assertRaises(exceptions.TemplateNotFound):
env.from_string('{% import_json "does not exists" as doc %}').render()
def test_line_syntax_priority():
# XXX: why is the whitespace there in front of the newline?
env = Environment('{%', '%}', '${', '}', '/*', '*/', '##', '#')
tmpl = env.from_string(LINE_SYNTAX_PRIORITY1)
assert tmpl.render(seq=[1, 2]).strip() == '* 1\n* 2'
env = Environment('{%', '%}', '${', '}', '/*', '*/', '#', '##')
tmpl = env.from_string(LINE_SYNTAX_PRIORITY2)
assert tmpl.render(seq=[1, 2]).strip() == '* 1\n\n* 2'
def test_pluralize_as_filter(self):
""" Tests pluralize as a Jinja2 filter. """
env = Environment()
env.filters['pluralize'] = pluralize
tmpl = env.from_string("{{ 'vote'|pluralize }}")
assert tmpl.render() == 'votes'
tmpl = env.from_string("{{ 'goose'|pluralize }}")
assert tmpl.render() == 'geese'
def test_load_text_template(self):
loader = DictLoader({'foo': 'Foo!'})
env = Environment(extensions=[SerializerExtension], loader=loader)
rendered = env.from_string('{% import_text "foo" as doc %}{{ doc }}').render()
self.assertEqual(rendered, u"Foo!")
with self.assertRaises(exceptions.TemplateNotFound):
env.from_string('{% import_text "does not exists" as doc %}').render()
def generate_rst(self, input_dir, output_dir):
root = self.get_info()
i = 0
env = Environment(extensions=["jinja2.ext.do",])
env.filters['firstline'] = first_line
env.filters['rst_link'] = make_rst_link
env.filters['rst_table'] = render_rst_table
env.filters['rst_csvtable'] = render_rst_csv_table
env.filters['rst_heading'] = render_rst_heading
env.filters['extract_value'] = extract_value
env.filters['block_pad'] = block_pad
env.filters['common_head'] = calculate_common_head
env.filters['as_text'] = as_text
for (module,minfo) in root.plugins.iteritems():
out_base_path = '%s/reference/'%output_dir
sample_base_path = '%s/samples/'%output_dir
if minfo.namespace:
out_base_path = '%s/reference/%s/'%(output_dir, minfo.namespace)
hash = root.get_hash()
minfo.key = module
minfo.queries = {}
for (c,cinfo) in sorted(root.commands.iteritems()):
if module in cinfo.info.plugin:
more_info = self.fetch_command(c,cinfo)
if more_info:
cinfo = more_info
sfile = '%s%s_%s_samples.inc'%(sample_base_path, module, c)
if os.path.exists(sfile):
cinfo.sample = os.path.basename(sfile)
#all_samples.append((module, command, sfile))
cinfo.key = c
minfo.queries[c] = cinfo
minfo.aliases = {}
for (c,cinfo) in sorted(root.aliases.iteritems()):
if module in cinfo.info.plugin:
cinfo.key = c
minfo.aliases[c] = cinfo
minfo.paths = {}
for (c,cinfo) in sorted(root.paths.iteritems()):
if module in cinfo.info.plugin:
cinfo.key = c
minfo.paths[c] = cinfo
hash['module'] = minfo
i=i+1
log_debug('Processing module: %d of %d [%s]'%(i, len(root.plugins), module))
template = env.from_string(module_template)
render_template(hash, template, '%s/%s.rst'%(out_base_path, module))
log_debug('%s/samples/index.rst'%output_dir)
hash = root.get_hash()
template = env.from_string(samples_template)
render_template(hash, template, '%s/samples/index.rst'%output_dir)
def write_templated_output(output_dir, template_dir, title, data, extension):
"""Write out a templated version of the docs"""
from jinja2 import Environment
jinja = Environment(trim_blocks=True, lstrip_blocks=True)
# Make sure we have a valid output_dir
if not os.path.isdir(output_dir):
try:
os.makedirs(output_dir)
except Exception as error:
err("Output directory is not a directory, "
"and/or can't be created: %s" % error)
# Prepare for writing index.<extensions>
try:
outfile = open(output_dir + "/index." + extension, "wb")
except Exception as error:
err("Unable to create %s: %s" % (output_dir + "/index." + extension,
error))
# Prepare for reading index.j2.<extension>
try:
tmplfile = open(template_dir + "/index.j2." + extension, "r")
except Exception as error:
err("Unable to open index.j2.%s: %s" % (extension, error))
if extension == "html":
# Re-process the doc data to convert Markdown to HTML
data = process_markdown(data)
# Render and write index.<extension>
template = jinja.from_string(tmplfile.read().decode('utf-8'))
render = template.render(data=data, links=LINKS, title=title)
outfile.write(render.encode("utf-8"))
outfile.close()
tmplfile.close()
dbg("Wrote index." + extension)
# Render and write module docs
try:
tmplfile = open(template_dir + "/module.j2." + extension, "r")
template = jinja.from_string(tmplfile.read().decode('utf-8'))
except Exception as error:
err("Unable to open module.j2.%s: %s" % (extension, error))
for module in data:
with open("%s/%s.%s" % (output_dir,
module["name"],
extension), "wb") as docfile:
render = template.render(module=module,
type_order=TYPE_NAMES,
type_desc=TYPE_DESC)
docfile.write(render.encode("utf-8"))
dbg("Wrote %s.%s" % (module["name"], extension))
tmplfile.close()
def test_finalizer():
def finalize_none_empty(value):
if value is None:
value = u''
return value
env = Environment(finalize=finalize_none_empty)
tmpl = env.from_string('{% for item in seq %}|{{ item }}{% endfor %}')
assert tmpl.render(seq=(None, 1, "foo")) == '||1|foo'
tmpl = env.from_string('<{{ none }}>')
assert tmpl.render() == '<>'
def test_urlescape(self):
env = Environment(autoescape=True)
tmpl = env.from_string('{{ "Hello, world!"|urlescape }}')
assert tmpl.render() == 'Hello%2C%20world%21'
tmpl = env.from_string('{{ o|urlescape }}')
assert tmpl.render(o=u"Hello, world\u203d") == "Hello%2C%20world%E2%80%BD"
assert tmpl.render(o=(("f", 1),)) == "f=1"
assert tmpl.render(o=(('f', 1), ("z", 2))) == "f=1&z=2"
assert tmpl.render(o=((u"\u203d", 1),)) == "%E2%80%BD=1"
assert tmpl.render(o={u"\u203d": 1}) == "%E2%80%BD=1"
def test_finalizer(self):
def finalize_none_empty(value):
if value is None:
value = u''
return value
env = Environment(finalize=finalize_none_empty)
tmpl = env.from_string('{% for item in seq %}|{{ item }}{% endfor %}')
tmpl = env.from_string('<{{ none }}>')
def test_line_syntax():
env = Environment('<%', '%>', '${', '}', '<%#', '%>', '%')
tmpl = env.from_string(MAKO_SYNTAX)
assert [int(x.strip()) for x in tmpl.render(seq=range(5)).split()] == \
range(5)
env = Environment('<%', '%>', '${', '}', '<%#', '%>', '%', '##')
tmpl = env.from_string(MAKO_SYNTAX_LINECOMMENTS)
assert [int(x.strip()) for x in tmpl.render(seq=range(5)).split()] == \
range(5)
def test_replace(self):
env = Environment()
tmpl = env.from_string('{{ string|replace("o", 42) }}')
assert tmpl.render(string='<foo>') == '<f4242>'
env = Environment(autoescape=True)
tmpl = env.from_string('{{ string|replace("o", 42) }}')
assert tmpl.render(string='<foo>') == '<f4242>'
tmpl = env.from_string('{{ string|replace("<", 42) }}')
assert tmpl.render(string='<foo>') == '42foo>'
tmpl = env.from_string('{{ string|replace("o", ">x<") }}')
assert tmpl.render(string=Markup('foo')) == 'f>x<>x<'
def validate_template(value):
try:
env = Environment(extensions=['jinja2_time.TimeExtension'])
env.from_string(value)
except TemplateSyntaxError:
exctype, value = sys.exc_info()[:2]
raise ValidationError(_("Jinja error: %(error)s"), params={'error': value})
except:
exctype, value = sys.exc_info()[:2]
logger.error("Unexpected jinja validation: (%s, %s)" % (exctype, value))
raise ValidationError(_("Jinja template not valid"))
def test_replace(self):
env = Environment()
tmpl = env.from_string('{{ string|replace("o", 42) }}')
assert tmpl.render(string="<foo>") == "<f4242>"
env = Environment(autoescape=True)
tmpl = env.from_string('{{ string|replace("o", 42) }}')
assert tmpl.render(string="<foo>") == "<f4242>"
tmpl = env.from_string('{{ string|replace("<", 42) }}')
assert tmpl.render(string="<foo>") == "42foo>"
tmpl = env.from_string('{{ string|replace("o", ">x<") }}')
assert tmpl.render(string=Markup("foo")) == "f>x<>x<"
def test_load():
from coffin.template.defaulttags import LoadExtension
env = Environment(extensions=[LoadExtension])
# the load tag is a no-op
assert env.from_string('a{% load %}b').render() == 'ab'
assert env.from_string('a{% load news.photos %}b').render() == 'ab'
assert env.from_string('a{% load "news.photos" %}b').render() == 'ab'
# [bug] invalid code was generated under certain circumstances
assert env.from_string('{% set x=1 %}{% load "news.photos" %}').render() == ''
def test_finalizer(self):
def finalize_none_empty(value):
if value is None:
value = u""
return value
env = Environment(finalize=finalize_none_empty)
tmpl = env.from_string("{% for item in seq %}|{{ item }}{% endfor %}")
assert tmpl.render(seq=(None, 1, "foo")) == "||1|foo"
tmpl = env.from_string("<{{ none }}>")
assert tmpl.render() == "<>"
def test_normalizing(self, env):
for seq in '\r', '\r\n', '\n':
env = Environment(newline_sequence=seq)
tmpl = env.from_string('1\n2\r\n3\n4\n')
result = tmpl.render()
assert result.replace(seq, 'X') == '1X2X3X4'
def test_simple_reject(self, env):
env = Environment()
tmpl = env.from_string('{{ [1, 2, 3, 4, 5]|reject("odd")|join("|") }}')
assert tmpl.render() == '2|4'
class Jinja2TemplateHandler(TemplateHandler):
"""
This class implements the :ref:`Template <cement.core.template>` Handler
interface. It renders content as template, and supports copying entire
source template directories using the
`Jinja2 Templating Language <http://jinja.pocoo.org/>`_. Please
see the developer documentation on
:cement:`Template Handling <dev/template>`.
**Note** This extension has an external dependency on ``jinja2``. You
must include ``jinja2`` in your applications dependencies as Cement
explicitly does **not** include external dependencies for optional
extensions.
"""
class Meta:
"""Handler meta-data."""
label = 'jinja2'
def __init__(self, *args, **kw):
super(Jinja2TemplateHandler, self).__init__(*args, **kw)
# expose Jinja2 Environment instance so that we can manipulate it
# higher in application code if necessary
self.env = Environment(keep_trailing_newline=True)
def load(self, *args, **kw):
"""
Loads a template file first from ``self.app._meta.template_dirs`` and
secondly from ``self.app._meta.template_module``. The
``template_dirs`` have presedence.
Args:
template_path (str): The secondary path of the template **after**
either ``template_module`` or ``template_dirs`` prefix (set via
``App.Meta``)
Returns:
tuple: The content of the template (``str``), the type of template
(``str``: ``directory``, or ``module``), and the path (``str``) of
the directory or module)
Raises:
cement.core.exc.FrameworkError: If the template does not exist in
either the ``template_module`` or ``template_dirs``.
"""
content, _type, _path = super(Jinja2TemplateHandler,
self).load(*args, **kw)
if _type == 'directory':
self.env.loader = FileSystemLoader(self.app._meta.template_dirs)
elif _type == 'module':
parts = self.app._meta.template_module.rsplit('.', 1)
self.env.loader = PackageLoader(parts[0], package_path=parts[1])
return content, _type, _path
def render(self, content, data, *args, **kw):
"""
Render the given ``content`` as template with the ``data`` dictionary.
Args:
content (str): The template content to render.
data (dict): The data dictionary to render.
Returns:
str: The rendered template text
"""
LOG.debug("rendering output as text via %s" % self.__module__)
if not isinstance(content, str):
content = content.decode('utf-8')
tmpl = self.env.from_string(content)
return tmpl.render(**data)
def test_lstrip_trim(self, env):
env = Environment(lstrip_blocks=True, trim_blocks=True)
tmpl = env.from_string(''' {% if True %}\n {% endif %}''')
assert tmpl.render() == ""
def test_streamfilter_extension(self):
env = Environment(extensions=[StreamFilterExtension])
env.globals['gettext'] = lambda x: x.upper()
tmpl = env.from_string('Foo _(bar) Baz')
out = tmpl.render()
assert out == 'Foo BAR Baz'
def render_template_keep_undefined(cls, template_str, data):
env = Environment(loader=BaseLoader, undefined=DebugUndefined)
env.filters["jsonify"] = partial(json.dumps, default=str)
rtemplate = env.from_string(template_str)
return rtemplate.render(data)
def auto_report(dag, path, stylesheet=None):
try:
from jinja2 import Template, Environment, PackageLoader
except ImportError as e:
raise WorkflowError(
"Python package jinja2 must be installed to create reports.")
if not path.endswith(".html"):
raise WorkflowError("Report file does not end with .html")
custom_stylesheet = None
if stylesheet is not None:
try:
with open(stylesheet) as s:
custom_stylesheet = s.read()
except (Exception, BaseException) as e:
raise WorkflowError("Unable to read custom report stylesheet.", e)
logger.info("Creating report...")
env = Environment(
loader=PackageLoader("snakemake", "report"),
trim_blocks=True,
lstrip_blocks=True,
)
env.filters["get_resource_as_string"] = get_resource_as_string
persistence = dag.workflow.persistence
results = defaultdict(lambda: defaultdict(list))
records = defaultdict(JobRecord)
recorded_files = set()
for job in dag.jobs:
for f in itertools.chain(job.expanded_output, job.input):
if is_flagged(f, "report") and f not in recorded_files:
if not f.exists:
raise WorkflowError("File {} marked for report but does "
"not exist.".format(f))
report_obj = get_flag_value(f, "report")
def register_file(f, wildcards_overwrite=None):
wildcards = wildcards_overwrite or job.wildcards
category = Category(report_obj.category,
wildcards=wildcards,
job=job)
subcategory = Category(report_obj.subcategory,
wildcards=wildcards,
job=job)
results[category][subcategory].append(
FileRecord(
f,
job,
report_obj.caption,
env,
category,
wildcards_overwrite=wildcards_overwrite,
))
recorded_files.add(f)
if os.path.isfile(f):
register_file(f)
if os.path.isdir(f):
if not isinstance(report_obj.patterns, list):
raise WorkflowError(
"Invalid patterns given for report. Must be list.",
rule=job.rule,
)
if not report_obj.patterns:
raise WorkflowError(
"Directory marked for report but no file patterns given via patterns=[...]. "
"See report documentation.",
rule=job.rule,
)
for pattern in report_obj.patterns:
pattern = os.path.join(f, pattern)
wildcards = glob_wildcards(pattern)._asdict()
names = wildcards.keys()
for w in zip(*wildcards.values()):
w = dict(zip(names, w))
w.update(job.wildcards_dict)
w = Wildcards(fromdict=w)
f = apply_wildcards(pattern, w)
register_file(f, wildcards_overwrite=w)
for f in job.expanded_output:
meta = persistence.metadata(f)
if not meta:
logger.warning("Missing metadata for file {}. Maybe metadata "
"was deleted or it was created using an older "
"version of Snakemake. This is a non critical "
"warning.".format(f))
continue
try:
job_hash = meta["job_hash"]
rule = meta["rule"]
rec = records[(job_hash, rule)]
rec.rule = rule
rec.job = job
rec.starttime = min(rec.starttime, meta["starttime"])
rec.endtime = max(rec.endtime, meta["endtime"])
rec.conda_env_file = None
rec.conda_env = meta["conda_env"]
rec.container_img_url = meta["container_img_url"]
rec.output.append(f)
except KeyError as e:
print(e)
logger.warning("Metadata for file {} was created with a too "
"old Snakemake version.".format(f))
for subcats in results.values():
for catresults in subcats.values():
catresults.sort(key=lambda res: res.name)
# prepare runtimes
runtimes = [{
"rule": rec.rule,
"runtime": rec.endtime - rec.starttime
} for rec in sorted(records.values(), key=lambda rec: rec.rule)]
# prepare end times
timeline = [{
"rule":
rec.rule,
"starttime":
datetime.datetime.fromtimestamp(rec.starttime).isoformat(),
"endtime":
datetime.datetime.fromtimestamp(rec.endtime).isoformat(),
} for rec in sorted(records.values(), key=lambda rec: rec.rule)]
# prepare per-rule information
rules = defaultdict(list)
for rec in records.values():
rule = RuleRecord(rec.job, rec)
if rec.rule not in rules:
rules[rec.rule].append(rule)
else:
merged = False
for other in rules[rec.rule]:
if rule == other:
other.add(rec)
merged = True
break
if not merged:
rules[rec.rule].append(rule)
# rulegraph
rulegraph, xmax, ymax = rulegraph_d3_spec(dag)
# configfiles
configfiles = [ConfigfileRecord(f) for f in dag.workflow.configfiles]
seen = set()
files = [
seen.add(res.target) or res for cat in results.values()
for subcat in cat.values() for res in subcat if res.target not in seen
]
rst_links = textwrap.dedent("""
.. _Workflow: javascript:show_panel('workflow')
.. _Statistics: javascript:show_panel('statistics')
{% for cat, catresults in categories|dictsort %}
.. _{{ cat.name }}: javascript:show_panel("{{ cat.id }}")
{% endfor %}
{% for res in files %}
.. _{{ res.target }}: javascript:show_panel("{{ res.category.id }}")
{% endfor %}
""")
for cat, subcats in results.items():
for subcat, catresults in subcats.items():
for res in catresults:
res.render(env, rst_links, results, files)
# global description
text = ""
if dag.workflow.report_text:
with open(dag.workflow.report_text) as f:
class Snakemake:
config = dag.workflow.config
text = f.read() + rst_links
text = publish_parts(
env.from_string(text).render(snakemake=Snakemake,
categories=results,
files=files),
writer_name="html",
)["body"]
# record time
now = "{} {}".format(datetime.datetime.now().ctime(), time.tzname[0])
results_size = sum(res.size for cat in results.values()
for subcat in cat.values() for res in subcat)
try:
from pygments.formatters import HtmlFormatter
except ImportError:
raise WorkflowError(
"Python package pygments must be installed to create reports.")
# render HTML
template = env.get_template("report.html")
with open(path, "w", encoding="utf-8") as out:
out.write(
template.render(
results=results,
results_size=results_size,
configfiles=configfiles,
text=text,
rulegraph_nodes=rulegraph["nodes"],
rulegraph_links=rulegraph["links"],
rulegraph_width=xmax + 20,
rulegraph_height=ymax + 20,
runtimes=runtimes,
timeline=timeline,
rules=[rec for recs in rules.values() for rec in recs],
version=__version__,
now=now,
pygments_css=HtmlFormatter(
style="trac").get_style_defs(".source"),
custom_stylesheet=custom_stylesheet,
))
logger.info("Report created.")
def inline_render(file):
"""Take a file, look for all inline locations that have an associated template and render them inline"""
def starts_with(s, sub):
return s[0:len(sub)] == sub
ishtml = False
newcontents = []
env = Environment(loader=FileSystemLoader('.'),
line_statement_prefix=line_statement_prefix)
contents = read_file(file)
linenr = 0
laststart = 0
tpl = []
mode = 'init'
typ = '' # is this a build or autogen block?
should_render_template = True
for line in contents.split("\n"):
linenr += 1
isstart = re.search(r"^ *(// --) autogen", line)
isstart2 = re.search(r"^ *(// --) build=(.*)", line)
isend = re.search(r"^ *(// --) end (build|autogen)", line)
istpl = re.search(r"^ *(// --) (.*)", line)
ishtmlcomment = re.search(r"^<!--", line)
if mode == 'init':
if isstart is not None:
typ = "autogen"
should_render_template = True
newcontents.append(line)
laststart = linenr
mode = 'template'
elif isstart2 is not None:
typ = "build"
should_render_template = isstart2.group(
2) == get_context()['build']
newcontents.append(line)
laststart = linenr
mode = 'template'
else:
newcontents.append(line)
if ishtmlcomment is not None:
ishtml = True
else:
ishtml = False
elif mode == 'ignore':
if isend is not None:
if isend.group(2) != typ:
error("Started a '%s' block, but ended with '%s'" %
(typ, isend.group(2)))
newcontents.append(line)
mode = 'init'
elif mode == 'template':
if istpl is not None and isend is None:
tpl.append(istpl.group(2))
newcontents.append(line)
else:
tpl = "\n".join(tpl)
if starts_with(tpl, "c_to_js"):
newcontents.append(c_to_js(tpl[8:]))
else:
if should_render_template:
template = env.from_string(tpl)
if ishtml: newcontents.append('-->')
newcontents.append(
template.render(get_context()).strip("\n"))
if ishtml: newcontents.append('<!--')
tpl = []
mode = 'ignore'
if isend is not None:
if isend.group(2) != typ:
error("Started a '%s' block, but ended with '%s'" %
(typ, isend.group(2)))
newcontents.append(line)
mode = 'init'
if mode != 'init':
error(
"Unclosed autogen section started on line %d, stopped in mode %s."
% (laststart, mode))
newcontents = "\n".join(newcontents)
if contents != newcontents:
write_file(file, newcontents)
def test_macro_escaping(self):
env = Environment(autoescape=lambda x: False,
extensions=['jinja2.ext.autoescape'])
template = "{% macro m() %}<html>{% endmacro %}"
template += "{% autoescape true %}{{ m() }}{% endautoescape %}"
assert env.from_string(template).render()
def test_lstrip_nested(self, env):
env = Environment(lstrip_blocks=True, trim_blocks=False)
tmpl = env.from_string(
''' {% if True %}a {% if True %}b {% endif %}c {% endif %}''')
assert tmpl.render() == 'a b c '
def test_lstrip_inline(self, env):
env = Environment(lstrip_blocks=True, trim_blocks=False)
tmpl = env.from_string(''' {% if True %}hello {% endif %}''')
assert tmpl.render() == 'hello '
def test_lstrip_endline(self, env):
env = Environment(lstrip_blocks=True, trim_blocks=False)
tmpl = env.from_string(
''' hello{% if True %}\n goodbye{% endif %}''')
assert tmpl.render() == " hello\n goodbye"
def test_only_module(self):
env = Environment(extensions=['jinja2_module.ModuleExtension'])
template = env.from_string('''
No Fault in my stars
{% module "wait-stage.module" %}
''')
def test_comments():
from jinja2 import Environment
env = Environment('<!--', '-->', '{', '}')
tmpl = env.from_string(COMMENTS)
assert tmpl.render(seq=range(3)) == ("<ul>\n <li>0</li>\n "
"<li>1</li>\n <li>2</li>\n</ul>")
def test_balancing():
from jinja2 import Environment
env = Environment('{%', '%}', '${', '}')
tmpl = env.from_string(BALANCING)
assert tmpl.render(seq=range(3)) == "{'FOO': 0}{'FOO': 1}{'FOO': 2}"
def execute_all(meta_cnc, app_dir, context):
"""
Performs all REST operations defined in this meta-cnc file
Each 'snippet' in the 'snippets' stanza in the .meta-cnc file will be executed in turn
each entry in the 'snippets' stanza MUST have at least:
'name', 'rest_path', and 'rest_operation'
For a POST operation it must also include a 'payload' key
* The path can include jinja2 variables as well as the payload file. Both will be interpolated before executing
This allows things like the hostname, etc to be captured in the variables or target section
:param meta_cnc: a parsed .meta-cnc.yaml file (self.service in class based Views)
:param app_dir: which app_dir is this (panhandler, vistoq, etc) defined as self.app_dir on view classes
:param context: fully populated workflow from the calling view (self.get_workflow() on the view class)
:return: string suitable for presentation to the user
"""
if 'snippet_path' in meta_cnc:
snippets_dir = meta_cnc['snippet_path']
else:
# snippets_dir = Path(os.path.join(settings.BASE_DIR, app_dir, 'snippets', meta_cnc['name']))
raise CCFParserError('Could not locate .meta-cnc for REST execution')
response = dict()
response['status'] = 'success'
response['message'] = 'A-OK'
response['snippets'] = dict()
session = requests.Session()
# create our jinja env and load filters only once
environment = Environment(loader=BaseLoader())
for f in jinja_filters.defined_filters:
if hasattr(jinja_filters, f):
environment.filters[f] = getattr(jinja_filters, f)
try:
# execute our rest call for each item in the 'snippets' stanza of the meta-cnc file
for snippet in meta_cnc['snippets']:
if 'path' not in snippet:
print('Malformed meta-cnc error')
raise CCFParserError
name = snippet.get('name', '')
# allow snippets to be skipped using the 'when' attribute
if 'when' in snippet:
when_template = environment.from_string(snippet.get('when', ''))
when_result = str(when_template.render(context))
if when_result.lower() == 'false' or when_result.lower() == 'no':
print(f'Skipping snippet {name} due to when condition false')
continue
rest_path = snippet.get('path', '/api').strip()
rest_op = str(snippet.get('operation', 'get')).lower()
payload_name = snippet.get('payload', '')
header_dict = snippet.get('headers', dict())
# fix for issue #42
if type(header_dict) is not dict:
header_dict = dict()
# FIXME - implement this to give some control over what will be sent to rest server
content_type = snippet.get('content_type', '')
accepts_type = snippet.get('accepts_type', '')
headers = dict()
if content_type:
headers["Content-Type"] = content_type
if accepts_type:
headers['Accepts-Type'] = accepts_type
path_template = environment.from_string(rest_path)
# fix for #111 - ensure we strip and replace only after rendering to ensure inline conditionals work
# as expected.
url = path_template.render(context).replace('\n', '').replace(' ', '')
for k, v in header_dict.items():
v_template = environment.from_string(v)
v_interpolated = v_template.render(context)
print(f'adding {k} as {v_interpolated} to headers')
headers[k] = v_interpolated
# keep track of response text or json object
r = ''
if rest_op == 'post' and payload_name != '':
payload_path = os.path.join(snippets_dir, payload_name)
with open(payload_path, 'r') as payload_file:
payload_string = payload_file.read()
payload_template = environment.from_string(payload_string)
payload_interpolated = payload_template.render(context)
if 'Content-Type' in headers and 'form' in headers['Content-Type']:
print('Loading json data from payload')
try:
payload = json.loads(payload_interpolated)
except ValueError:
print('Could not load payload as json data!')
payload = payload_interpolated
else:
payload = payload_interpolated
res = session.post(url, data=payload, verify=False, headers=headers)
if res.status_code != 200:
print('Found a non-200 response status_code!')
print(res.status_code)
response['snippets'][name] = res.text
response['status'] = 'error'
response['message'] = res.status_code
break
if res.headers.get('content-type') == 'application/json':
try:
r = res.json()
except ValueError:
print('Could not parse JSON response from request')
r = res.text
else:
r = res.text
elif rest_op == 'get':
print(f'Performing REST get for snippet: {name}')
res = session.get(url, verify=False)
if res.headers.get('content-type') == 'application/json':
try:
r = res.json()
except ValueError:
r = res.text
else:
r = res.text
if res.status_code != 200:
response['status'] = 'error'
response['message'] = res.status_code
response['snippets'][name] = r
break
else:
print('Unknown REST operation found')
response['status'] = 'Error'
response['message'] = 'Unknown REST operation found'
return response
# collect the response text or json and continue
response['snippets'][name] = dict()
response['snippets'][name]['results'] = r
response['snippets'][name]['outputs'] = dict()
if 'outputs' in snippet:
outputs = output_utils.parse_outputs(meta_cnc, snippet, r)
response['snippets'][name]['outputs'] = outputs
context.update(outputs)
# return all the collected response
return response
except HTTPError as he:
response['status'] = 'error'
response['message'] = str(he)
return response
except requests.exceptions.ConnectionError as ce:
response['status'] = 'error'
response['message'] = str(ce)
return response
except MissingSchema as ms:
response['status'] = 'error'
response['message'] = ms
return response
except RequestException as re:
response['status'] = 'error'
response['message'] = re
return response
class DynamicInventory(object):
BASESCRIPT = '''#!/usr/bin/python
import json
data = """{{ data }}"""
data = json.loads(data)
print(json.dumps(data, indent=2, sort_keys=True))
'''
BASEINV = {
'_meta': {
'hostvars': {
'testhost': {}
}
}
}
def __init__(self, features):
self.ENV = Environment()
self.features = features
self.fpath = None
self.inventory = self.BASEINV.copy()
self.build()
def build(self):
xhost = 'testhost'
if 'script_host' in self.features:
self.inventory['_meta']['hostvars'][xhost]['findme'] = 'script_host'
else:
self.inventory['_meta']['hostvars'][xhost] = {}
if 'script_child' in self.features:
self.inventory['child'] = {
'hosts': [xhost],
'vars': {'findme': 'script_child'}
}
if 'script_parent' in self.features:
self.inventory['parent'] = {
'vars': {'findme': 'script_parent'}
}
if 'script_child' in self.features:
self.inventory['parent']['children'] = ['child']
else:
self.inventory['parent']['hosts'] = [xhost]
if 'script_all' in self.features:
self.inventory['all'] = {
'hosts': [xhost],
'vars': {
'findme': 'script_all'
},
}
else:
self.inventory['all'] = {
'hosts': [xhost],
}
def write_script(self):
fdir = os.path.join(TESTDIR, 'inventory')
if not os.path.isdir(fdir):
os.makedirs(fdir)
fpath = os.path.join(fdir, 'hosts')
# fpath = os.path.join(TESTDIR, 'inventory')
self.fpath = fpath
data = json.dumps(self.inventory)
t = self.ENV.from_string(self.BASESCRIPT)
fdata = t.render(data=data)
with open(fpath, 'w') as f:
f.write(fdata + '\n')
st = os.stat(fpath)
os.chmod(fpath, st.st_mode | stat.S_IEXEC)
def test_extension_nodes(self):
env = Environment(extensions=[TestExtension])
tmpl = env.from_string('{% test %}')
assert tmpl.render() == 'False|42|23|{}'
def test_scoping(self):
env = Environment(extensions=['jinja2.ext.autoescape'])
tmpl = env.from_string(
'{% autoescape true %}{% set x = "<x>" %}{{ x }}'
'{% endautoescape %}{{ x }}{{ "<y>" }}')
assert tmpl.render(x=1) == '<x>1<y>'
def test_safe(self):
env = Environment(autoescape=True)
tmpl = env.from_string('{{ "<div>foo</div>"|safe }}')
assert tmpl.render() == '<div>foo</div>'
tmpl = env.from_string('{{ "<div>foo</div>" }}')
assert tmpl.render() == '<div>foo</div>'
def test_preprocessor_extension(self):
env = Environment(extensions=[PreprocessorExtension])
tmpl = env.from_string('{[[TEST]]}')
assert tmpl.render(foo=42) == '{(42)}'
def test_call(self, env):
env = Environment()
env.globals['foo'] = lambda a, b, c, e, g: a + b + c + e + g
tmpl = env.from_string(
"{{ foo('a', c='d', e='f', *['b'], **{'g': 'h'}) }}")
assert tmpl.render() == 'abdfh'
def test_extend_late(self):
env = Environment()
env.add_extension('jinja2.ext.autoescape')
t = env.from_string(
'{% autoescape true %}{{ "<test>" }}{% endautoescape %}')
assert t.render() == '<test>'
def test_no_finalize_template_data(self):
e = Environment(finalize=lambda v: type(v).__name__)
t = e.from_string("<{{ value }}>")
# If template data was finalized, it would print "strintstr".
assert t.render(value=123) == "<int>"
class Renderer(object):
"""
Main engine to convert and ODT document into a jinja
compatible template.
Basic use example:
engine = Renderer('template')
result = engine.render()
Renderer provides an enviroment variable which can be used
to provide custom filters to the ODF render.
engine = Renderer('template.odt')
engine.environment.filters['custom_filer'] = filter_function
result = engine.render()
"""
def __init__(self, environment=None, **kwargs):
"""
Create a Renderer instance.
args:
environment: Use this jinja2 enviroment. If not specified, we
create a new environment for this class instance.
returns:
None
"""
self.log = logging.getLogger(__name__)
self.log.debug('Initing a Renderer instance\nTemplate')
if environment:
self.environment = environment
else:
self.environment = Environment(undefined=UndefinedSilently,
autoescape=True)
# Register filters
self.environment.filters['pad'] = pad_string
self.environment.filters['markdown'] = self.markdown_filter
self.environment.filters['image'] = self.image_filter
self.media_path = kwargs.pop('media_path', '')
self.media_callback = self.fs_loader
def media_loader(self, callback):
"""This sets the the media loader. A user defined function which
loads media. The function should take a template value, optionals
args and kwargs. Is media exists should return a tuple whose first
element if a file object type representing the media and its second
elements is the media mimetype.
See Renderer.fs_loader funcion for an example"""
self.media_callback = callback
return callback
def _unpack_template(self, template):
# And Open/libreOffice is just a ZIP file. Here we unarchive the file
# and return a dict with every file in the archive
self.log.debug('Unpacking template file')
archive_files = {}
archive = zipfile.ZipFile(template, 'r')
for zfile in archive.filelist:
archive_files[zfile.filename] = archive.read(zfile.filename)
return archive_files
self.log.debug('Unpack completed')
def _pack_document(self, files):
# Store to a zip files in files
self.log.debug('packing document')
zip_file = io.BytesIO()
zipdoc = zipfile.ZipFile(zip_file, 'a')
for fname, content in files.items():
if sys.version_info >= (2, 7):
zipdoc.writestr(fname, content, zipfile.ZIP_DEFLATED)
else:
zipdoc.writestr(fname, content)
self.log.debug('Document packing completed')
return zip_file
def _prepare_template_tags(self, xml_document):
""" Here we search for every field node present in xml_document.
For each field we found we do:
* if field is a print field ({{ field }}), we replace it with a
<text:span> node.
* if field is a control flow ({% %}), then we find immediate node of
type indicated in field's `text:description` attribute and replace
the whole node and its childrens with field's content.
If `text:description` attribute starts with `before::` or `after::`,
then we move field content before or after the node in description.
If no `text:description` is available, find the immediate common
parent of this and any other field and replace its child and
original parent of field with the field content.
e.g.: original
<table>
<table:row>
<field>{% for bar in bars %}</field>
</table:row>
<paragraph>
<field>{{ bar }}</field>
</paragraph>
<table:row>
<field>{% endfor %}</field>
</table:row>
</table>
After processing:
<table>
{% for bar in bars %}
<paragraph>
<text:span>{{ bar }}</text:span>
</paragraph>
{% endfor %}
</table>
"""
self.log.debug('Preparing template tags')
fields = xml_document.getElementsByTagName('text:text-input')
# First, count secretary fields
for field in fields:
if not field.hasChildNodes():
continue
field_content = field.childNodes[0].data.strip()
if not re.findall(r'(?is)^{[{|%].*[%|}]}$', field_content):
# Field does not contains jinja template tags
continue
is_block_tag = re.findall(r'(?is)^{%[^{}]*%}$', field_content)
self.inc_node_fields_count(field.parentNode,
'block' if is_block_tag else 'variable')
# Do field replacement and moving
for field in fields:
if not field.hasChildNodes():
continue
field_content = field.childNodes[0].data.strip()
if not re.findall(r'(?is)^{[{|%].*[%|}]}$', field_content):
# Field does not contains jinja template tags
continue
is_block_tag = re.findall(r'(?is)^{%[^{}]*%}$', field_content)
discard = field
field_reference = field.getAttribute('text:description').strip().lower()
if re.findall(r'\|markdown', field_content):
# a markdown field should take the whole paragraph
field_reference = 'text:p'
if field_reference:
# User especified a reference. Replace immediate parent node
# of type indicated in reference with this field's content.
node_type = FLOW_REFERENCES.get(field_reference, False)
if node_type:
discard = self._parent_of_type(field, node_type)
jinja_node = self.create_text_node(xml_document, field_content)
elif is_block_tag:
# Find the common immediate parent of this and any other field.
while discard.parentNode.secretary_field_count <= 1:
discard = discard.parentNode
if discard is not None:
jinja_node = self.create_text_node(xml_document,
field_content)
else:
jinja_node = self.create_text_span_node(xml_document,
field_content)
parent = discard.parentNode
if not field_reference.startswith('after::'):
parent.insertBefore(jinja_node, discard)
else:
if discard.isSameNode(parent.lastChild):
parent.appendChild(jinja_node)
else:
parent.insertBefore(jinja_node,
discard.nextSibling)
if field_reference.startswith(('after::', 'before::')):
# Do not remove whole field container. Just remove the
# <text:text-input> parent node if field has it.
discard = self._parent_of_type(field, 'text:p')
parent = discard.parentNode
parent.removeChild(discard)
@staticmethod
def _unescape_entities(xml_text):
"""
Strips tags of the form <text:span ...> from inside Jinja elements
and unescapes HTML codes for >, <, & and "
"""
unescape_rules = {
r'(?is)({([{%])[^%}]*?)(</?text:s.*?>)(.*?[%}]})': r'\1 \4',
r'(?is)({([{%])[^%}]*?)(>)(.*?[%}]})' : r'\1>\4',
r'(?is)({([{%])[^%}]*?)(<)(.*?[%}]})' : r'\1<\4',
r'(?is)({([{%])[^%}]*?)(&)(.*?[%}]})' : r'\1&\4',
r'(?is)({([{%])[^%}]*?)(")(.*?[%}]})' : r'\1"\4',
}
for regexp, replacement in unescape_rules.items():
subs_made = True
while subs_made:
xml_text, subs_made = re.subn(regexp, replacement, xml_text)
return xml_text
@staticmethod
def _encode_escape_chars(xml_text):
# Replace line feed and/or tabs within text span entities.
find_pattern = r'(?is)<text:([\S]+?)>([^>]*?([\n|\t])[^<]*?)</text:\1>'
for m in re.findall(find_pattern, xml_text):
replacement = m[1].replace('\n', '<text:line-break/>')
replacement = replacement.replace('\t', '<text:tab/>')
xml_text = xml_text.replace(m[1], replacement)
return xml_text
def add_media_to_archive(self, media, mime, name=''):
"""Adds to "Pictures" archive folder the file in `media` and register
it into manifest file."""
extension = None
if hasattr(media, 'name') and not name:
extension = path.splitext(media.name)
name = extension[0]
extension = extension[1]
if not extension:
extension = guess_extension(mime)
media_path = 'Pictures/%s%s' % (name, extension)
media.seek(0)
self.files[media_path] = media.read(-1)
if hasattr(media, 'close'):
media.close()
files_node = self.manifest.getElementsByTagName('manifest:manifest')[0]
node = self.create_node(self.manifest, 'manifest:file-entry', files_node)
node.setAttribute('manifest:full-path', media_path)
node.setAttribute('manifest:media-type', mime)
return media_path
def fs_loader(self, media, *args, **kwargs):
"""Loads a file from the file system.
:param media: A file object or a relative or absolute path of a file.
:type media: unicode
"""
if hasattr(media, 'seek') and hasattr(media, 'read'):
return (media, 'image/jpeg')
elif path.isfile(media):
filename = media
else:
if not self.media_path:
self.log.debug('media_path property not specified to load images from.')
return
filename = path.join(self.media_path, media)
if not path.isfile(filename):
self.log.debug('Media file "%s" does not exists.' % filename)
return
mime = guess_type(filename)
return (open(filename, 'rb'), mime[0] if mime else None)
def replace_images(self, xml_document):
"""Perform images replacements"""
self.log.debug('Inserting images')
frames = xml_document.getElementsByTagName('draw:frame')
for frame in frames:
if not frame.hasChildNodes():
continue
key = frame.getAttribute('draw:name')
if key not in self.template_images:
continue
# Get frame attributes
frame_attrs = dict()
for i in xrange(frame.attributes.length):
attr = frame.attributes.item(i)
frame_attrs[attr.name] = attr.value
# Get child draw:image node and its attrs
image_node = frame.childNodes[0]
image_attrs = dict()
for i in xrange(image_node.attributes.length):
attr = image_node.attributes.item(i)
image_attrs[attr.name] = attr.value
# Request to media loader the image to use
image = self.media_callback(self.template_images[key]['value'],
*self.template_images[key]['args'],
frame_attrs=frame_attrs,
image_attrs=image_attrs,
**self.template_images[key]['kwargs'])
# Update frame and image node attrs (if they where updated in
# media_callback call)
for k, v in frame_attrs.items():
frame.setAttribute(k, v)
for k, v in image_attrs.items():
image_node.setAttribute(k, v)
# Keep original image reference value
if isinstance(self.template_images[key]['value'], basestring):
frame.setAttribute('draw:name',
self.template_images[key]['value'])
# Does the madia loader returned something?
if not image:
continue
mname = self.add_media_to_archive(media=image[0], mime=image[1],
name=key)
if mname:
image_node.setAttribute('xlink:href', mname)
def _render_xml(self, xml_document, **kwargs):
# Prepare the xml object to be processed by jinja2
self.log.debug('Rendering XML object')
try:
self.template_images = dict()
self._prepare_template_tags(xml_document)
template_string = self._unescape_entities(xml_document.toxml())
jinja_template = self.environment.from_string(template_string)
result = jinja_template.render(**kwargs)
result = self._encode_escape_chars(result)
final_xml = parseString(result.encode('ascii', 'xmlcharrefreplace'))
if self.template_images:
self.replace_images(final_xml)
return final_xml
except ExpatError as e:
near = result.split('\n')[e.lineno -1][e.offset-50:e.offset+50]
raise ExpatError('ExpatError "%s" at line %d, column %d\nNear of: "[...]%s[...]"' % \
(ErrorString(e.code), e.lineno, e.offset, near))
except:
self.log.error('Error rendering template:\n%s',
xml_document.toprettyxml(), exc_info=True)
self.log.error('Unescaped template was:\n{}'.format(template_string))
raise
finally:
self.log.debug('Rendering xml object finished')
def render(self, template, **kwargs):
"""
Render a template
args:
template: A template file. Could be a string or a file instance
**kwargs: Template variables. Similar to jinja2
returns:
A binary stream which contains the rendered document.
"""
self.log.debug('Initing a template rendering')
self.files = self._unpack_template(template)
self.render_vars = {}
# Keep content and styles object since many functions or
# filters may work with then
self.content = parseString(self.files['content.xml'])
self.styles = parseString(self.files['styles.xml'])
self.manifest = parseString(self.files['META-INF/manifest.xml'])
# Render content.xml keeping just 'office:body' node.
rendered_content = self._render_xml(self.content, **kwargs)
self.content.getElementsByTagName('office:document-content')[0].replaceChild(
rendered_content.getElementsByTagName('office:body')[0],
self.content.getElementsByTagName('office:body')[0]
)
# Render styles.xml
self.styles = self._render_xml(self.styles, **kwargs)
self.log.debug('Template rendering finished')
self.files['content.xml'] = self.content.toxml().encode('ascii', 'xmlcharrefreplace')
self.files['styles.xml'] = self.styles.toxml().encode('ascii', 'xmlcharrefreplace')
self.files['META-INF/manifest.xml'] = self.manifest.toxml().encode('ascii', 'xmlcharrefreplace')
document = self._pack_document(self.files)
return document.getvalue()
def _parent_of_type(self, node, of_type):
# Returns the first immediate parent of type `of_type`.
# Returns None if nothing is found.
if hasattr(node, 'parentNode'):
if node.parentNode.nodeName.lower() == of_type:
return node.parentNode
else:
return self._parent_of_type(node.parentNode, of_type)
else:
return None
def create_node(self, xml_document, node_type, parent=None):
"""Creates a node in `xml_document` of type `node_type` and specified,
as child of `parent`."""
node = xml_document.createElement(node_type)
if parent:
parent.appendChild(node)
return node
def create_text_span_node(self, xml_document, content):
span = xml_document.createElement('text:span')
text_node = self.create_text_node(xml_document, content)
span.appendChild(text_node)
return span
def create_text_node(self, xml_document, text):
"""
Creates a text node
"""
return xml_document.createTextNode(text)
def inc_node_fields_count(self, node, field_type='variable'):
""" Increase field count of node and its parents """
if node is None:
return
if not hasattr(node, 'secretary_field_count'):
setattr(node, 'secretary_field_count', 0)
if not hasattr(node, 'secretary_variable_count'):
setattr(node, 'secretary_variable_count', 0)
if not hasattr(node, 'secretary_block_count'):
setattr(node, 'secretary_block_count', 0)
node.secretary_field_count += 1
if field_type == 'variable':
node.secretary_variable_count += 1
else:
node.secretary_block_count += 1
self.inc_node_fields_count(node.parentNode, field_type)
def get_style_by_name(self, style_name):
"""
Search in <office:automatic-styles> for style_name.
Return None if style_name is not found. Otherwise
return the style node
"""
auto_styles = self.content.getElementsByTagName(
'office:automatic-styles')[0]
if not auto_styles.hasChildNodes():
return None
for style_node in auto_styles.childNodes:
if style_node.hasAttribute('style:name') and \
(style_node.getAttribute('style:name') == style_name):
return style_node
return None
def insert_style_in_content(self, style_name, attributes=None,
**style_properties):
"""
Insert a new style into content.xml's <office:automatic-styles> node.
Returns a reference to the newly created node
"""
auto_styles = self.content.getElementsByTagName('office:automatic-styles')[0]
style_node = self.content.createElement('style:style')
style_node.setAttribute('style:name', style_name)
style_node.setAttribute('style:family', 'text')
style_node.setAttribute('style:parent-style-name', 'Standard')
if attributes:
for k, v in attributes.items():
style_node.setAttribute('style:%s' % k, v)
if style_properties:
style_prop = self.content.createElement('style:text-properties')
for k, v in style_properties.items():
style_prop.setAttribute('%s' % k, v)
style_node.appendChild(style_prop)
return auto_styles.appendChild(style_node)
def markdown_filter(self, markdown_text):
"""
Convert a markdown text into a ODT formated text
"""
if not isinstance(markdown_text, basestring):
return ''
from xml.dom import Node
from markdown_map import transform_map
try:
from markdown2 import markdown
except ImportError:
raise SecretaryError('Could not import markdown2 library. Install it using "pip install markdown2"')
styles_cache = {} # cache styles searching
html_text = markdown(markdown_text)
xml_object = parseString('<html>%s</html>' % html_text.encode('ascii', 'xmlcharrefreplace'))
# Transform HTML tags as specified in transform_map
# Some tags may require extra attributes in ODT.
# Additional attributes are indicated in the 'attributes' property
for tag in transform_map:
html_nodes = xml_object.getElementsByTagName(tag)
for html_node in html_nodes:
odt_node = xml_object.createElement(transform_map[tag]['replace_with'])
# Transfer child nodes
if html_node.hasChildNodes():
for child_node in html_node.childNodes:
odt_node.appendChild(child_node.cloneNode(True))
# Add style-attributes defined in transform_map
if 'style_attributes' in transform_map[tag]:
for k, v in transform_map[tag]['style_attributes'].items():
odt_node.setAttribute('text:%s' % k, v)
# Add defined attributes
if 'attributes' in transform_map[tag]:
for k, v in transform_map[tag]['attributes'].items():
odt_node.setAttribute(k, v)
# copy original href attribute in <a> tag
if tag == 'a':
if html_node.hasAttribute('href'):
odt_node.setAttribute('xlink:href',
html_node.getAttribute('href'))
# Does the node need to create an style?
if 'style' in transform_map[tag]:
name = transform_map[tag]['style']['name']
if not name in styles_cache:
style_node = self.get_style_by_name(name)
if style_node is None:
# Create and cache the style node
style_node = self.insert_style_in_content(
name, transform_map[tag]['style'].get('attributes', None),
**transform_map[tag]['style']['properties'])
styles_cache[name] = style_node
html_node.parentNode.replaceChild(odt_node, html_node)
def node_to_string(node):
result = node.toxml()
# linebreaks in preformated nodes should be converted to <text:line-break/>
if (node.__class__.__name__ != 'Text') and \
(node.getAttribute('text:style-name') == 'Preformatted_20_Text'):
result = result.replace('\n', '<text:line-break/>')
# All double linebreak should be replaced with an empty paragraph
return result.replace('\n\n', '<text:p text:style-name="Standard"/>')
return ''.join(node_as_str for node_as_str in map(node_to_string,
xml_object.getElementsByTagName('html')[0].childNodes))
def image_filter(self, value, *args, **kwargs):
"""Store value into template_images and return the key name where this
method stored it. The value returned it later used to load the image
from media loader and finally inserted into the final ODT document."""
key = uuid4().hex
self.template_images[key] = {
'value': value,
'args': args,
'kwargs': kwargs
}
return key
def test_finalize_constant_expression(self):
e = Environment(finalize=lambda v: "" if v is None else v)
t = e.from_string("<{{ none }}>")
assert t.render() == "<>"
def test_bool_reject(self, env):
env = Environment()
tmpl = env.from_string(
'{{ [none, false, 0, 1, 2, 3, 4, 5]|reject|join("|") }}')
assert tmpl.render() == 'None|False|0'
def test_finalize(self):
e = Environment(finalize=lambda v: "" if v is None else v)
t = e.from_string("{% for item in seq %}|{{ item }}{% endfor %}")
assert t.render(seq=(None, 1, "foo")) == "||1|foo"
def test_no_lstrip(self, env):
env = Environment(lstrip_blocks=True, trim_blocks=False)
tmpl = env.from_string(''' {%+ if True %}\n {%+ endif %}''')
assert tmpl.render() == " \n "
def test_keyword_folding():
env = Environment()
env.filters['testing'] = lambda value, some: value + some
assert env.from_string("{{ 'test'|testing(some='stuff') }}") \
.render() == 'teststuff'
