def create_topic(self) -> Topic:
"""Create and return an actual Topic for this scheduled topic."""
# if no user is specified, use the "generic"/automatic user (ID -1)
if self.user:
user = self.user
else:
user = (Session.object_session(self).query(User).filter(
User.user_id == -1).one())
# treat both the title and markdown as Jinja templates (sandboxed)
jinja_sandbox = SandboxedEnvironment()
jinja_variables = {"current_time_utc": utc_now()}
try:
title_template = jinja_sandbox.from_string(self.title)
title = title_template.render(jinja_variables)
except: # pylint: disable=bare-except
title = self.title
try:
markdown_template = jinja_sandbox.from_string(self.markdown)
markdown = markdown_template.render(jinja_variables)
except: # pylint: disable=bare-except
markdown = self.markdown
topic = Topic.create_text_topic(self.group, user, title, markdown)
topic.tags = self.tags
topic.schedule = self
return topic
def render_notification_template(notification_type,
context,
language_code=DEFAULT_LANGUAGE):
"""
Render a notification template with given context in given language
Returns a namedtuple containing all content fields (subject, html_body, text_body) of the template.
"""
template = NotificationTemplate.objects.get(type=notification_type)
env = SandboxedEnvironment(trim_blocks=True,
lstrip_blocks=True,
undefined=StrictUndefined)
with switch_language(template, language_code):
try:
subject = env.from_string(template.subject).render(context)
html_body = env.from_string(template.html_body).render(context)
if template.text_body:
text_body = env.from_string(template.text_body).render(context)
else:
text_body = strip_tags(html_body)
return RenderedTemplate(subject, html_body, text_body)
except TemplateError as e:
raise NotificationTemplateException(e) from e
def render_notification_template(template,
context,
language_code=DEFAULT_LANGUAGE):
"""
Render a notification template with given context in given language
Returns a namedtuple containing all content fields (subject, body_html, body_text) of the template.
"""
env = SandboxedEnvironment(trim_blocks=True,
lstrip_blocks=True,
undefined=StrictUndefined)
with switch_language(template, language_code):
try:
subject = env.from_string(template.subject).render(context)
body_html = env.from_string(template.body_html).render(context)
if template.body_text:
body_text = env.from_string(template.body_text).render(context)
else:
body_text = strip_tags(body_html)
return RenderedTemplate(subject, body_html, body_text)
except TemplateError as e:
raise NotificationTemplateException(e) from e
def render(self, context, language_code=DEFAULT_LANG):
"""
Render this notification template with given context and language
Returns a dict containing all content fields of the template. Example:
{'short_message': 'foo', 'subject': 'bar', 'body': 'baz', 'html_body': '<b>foobar</b>'}
"""
env = SandboxedEnvironment(trim_blocks=True,
lstrip_blocks=True,
undefined=StrictUndefined)
env.filters['reservation_time'] = reservation_time
env.filters['format_datetime'] = format_datetime
env.filters['format_datetime_tz'] = format_datetime_tz
logger.debug('Rendering template for notification %s' % self.type)
with switch_language(self, language_code):
try:
rendered_notification = {
attr: env.from_string(getattr(self, attr)).render(context)
for attr in ('short_message', 'subject', 'html_body')
}
if self.body:
rendered_notification['body'] = env.from_string(
self.body).render(context)
else:
# if text body is empty use html body without tags as text body
rendered_notification['body'] = strip_tags(
rendered_notification['html_body'])
return rendered_notification
except TemplateError as e:
raise NotificationTemplateException(e) from e
def render(self, context, language_code=DEFAULT_LANG):
"""
Render this notification template with given context and language
Returns a dict containing all content fields of the template. Example:
{'short_message': 'foo', 'subject': 'bar', 'body': 'baz', 'html_body': '<b>foobar</b>'}
"""
env = SandboxedEnvironment(trim_blocks=True, lstrip_blocks=True, undefined=StrictUndefined)
env.filters['reservation_time'] = reservation_time
env.filters['format_datetime'] = format_datetime
env.filters['format_datetime_tz'] = format_datetime_tz
logger.debug('Rendering template for notification %s' % self.type)
with switch_language(self, language_code):
try:
rendered_notification = {
attr: env.from_string(getattr(self, attr)).render(context)
for attr in ('short_message', 'subject', 'html_body')
}
if self.body:
rendered_notification['body'] = env.from_string(self.body).render(context)
else:
# if text body is empty use html body without tags as text body
rendered_notification['body'] = strip_tags(rendered_notification['html_body'])
return rendered_notification
except TemplateError as e:
raise NotificationTemplateException(e) from e
def on_post(self, req, resp):
form_body = uri.parse_query_string(req.context['body'])
try:
template_subject = form_body['templateSubject']
template_body = form_body['templateBody']
application = form_body['application']
except KeyError:
raise HTTPBadRequest('Missing keys from post body', '')
if not application:
resp.body = ujson.dumps({'error': 'No application found'})
resp.status = falcon.HTTP_400
return
app_json = get_local(req, 'applications/%s' % application)
sample_context_str = app_json.get('sample_context')
if not sample_context_str:
resp.body = ujson.dumps({'error': 'Missing sample_context from application config'})
resp.status = falcon.HTTP_400
logger.error('Missing sample context for app %s', application)
return
try:
sample_context = ujson.loads(sample_context_str)
except Exception:
resp.body = ujson.dumps({'error': 'Invalid application sample_context'})
resp.status = falcon.HTTP_400
logger.exception('Bad sample context for app %s', application)
return
# TODO: also move iris meta var to api
iris_sample_context = {
"message_id": 5456900,
"target": "user",
"priority": "Urgent",
"application": "Autoalerts",
"plan": "default plan",
"plan_id": 1843,
"incident_id": 178293332,
"template": "default template"
}
sample_context['iris'] = iris_sample_context
environment = SandboxedEnvironment()
try:
subject_template = environment.from_string(template_subject)
body_template = environment.from_string(template_body)
except Exception as e:
resp.body = ujson.dumps({'error': str(e), 'lineno': e.lineno})
resp.status = falcon.HTTP_400
else:
resp.body = ujson.dumps({
'template_subject': subject_template.render(sample_context),
'template_body': body_template.render(sample_context)
})
def test_binary_operator_intercepting(self, env):
def disable_op(left, right):
raise TemplateRuntimeError("that operator so does not work")
for expr, ctx, rv in ("1 + 2", {}, "3"), ("a + 2", {"a": 2}, "4"):
env = SandboxedEnvironment()
env.binop_table["+"] = disable_op
t = env.from_string("{{ %s }}" % expr)
assert t.render(ctx) == rv
env.intercepted_binops = frozenset(["+"])
t = env.from_string("{{ %s }}" % expr)
with pytest.raises(TemplateRuntimeError):
t.render(ctx)
def test_unary_operator_intercepting(self, env):
def disable_op(arg):
raise TemplateRuntimeError("that operator so does not work")
for expr, ctx, rv in ("-1", {}, "-1"), ("-a", {"a": 2}, "-2"):
env = SandboxedEnvironment()
env.unop_table["-"] = disable_op
t = env.from_string(f"{{{{ {expr} }}}}")
assert t.render(ctx) == rv
env.intercepted_unops = frozenset(["-"])
t = env.from_string(f"{{{{ {expr} }}}}")
with pytest.raises(TemplateRuntimeError):
t.render(ctx)
def test_unary_operator_intercepting(self, env):
def disable_op(arg):
raise TemplateRuntimeError('that operator so does not work')
for expr, ctx, rv in ('-1', {}, '-1'), ('-a', {'a': 2}, '-2'):
env = SandboxedEnvironment()
env.unop_table['-'] = disable_op
t = env.from_string('{{ %s }}' % expr)
assert t.render(ctx) == rv
env.intercepted_unops = frozenset(['-'])
t = env.from_string('{{ %s }}' % expr)
with pytest.raises(TemplateRuntimeError):
t.render(ctx)
def validate():
template_subject = request.form['templateSubject']
template_body = request.form['templateBody']
application = request.form['application']
if not application:
return json.dumps({'error': 'No application found'}), 400
app_data = client.get('applications/%s' % application).data
try:
app_json = json.loads(app_data)
except Exception:
return json.dumps({'error': 'Invalid application config in API'}), 400
sample_context_str = app_json.get('sample_context')
if not sample_context_str:
return json.dumps(
{'error': 'Missing sample_context from application config'}), 400
try:
sample_context = json.loads(sample_context_str)
except Exception:
return json.dumps({'error': 'Invalid application sample_context'}), 400
# TODO: also move iris meta var to api
iris_sample_context = {
"message_id": 5456900,
"target": "user",
"priority": "Urgent",
"application": "Autoalerts",
"plan": "default plan",
"plan_id": 1843,
"incident_id": 178293332,
"template": "default template"
}
sample_context['iris'] = iris_sample_context
environment = SandboxedEnvironment()
try:
subject_template = environment.from_string(template_subject)
body_template = environment.from_string(template_body)
except Exception as e:
return json.dumps({'error': str(e), 'lineno': e.lineno}), 500
else:
return json.dumps({
'template_subject':
subject_template.render(sample_context),
'template_body':
body_template.render(sample_context)
})
def test_binary_operator_intercepting(self):
def disable_op(left, right):
raise TemplateRuntimeError('that operator so does not work')
for expr, ctx, rv in ('1 + 2', {}, '3'), ('a + 2', {'a': 2}, '4'):
env = SandboxedEnvironment()
env.binop_table['+'] = disable_op
t = env.from_string('{{ %s }}' % expr)
assert t.render(ctx) == rv
env.intercepted_binops = frozenset(['+'])
t = env.from_string('{{ %s }}' % expr)
try:
t.render(ctx)
except TemplateRuntimeError as e:
pass
else:
self.fail('expected runtime error')
def render_notification_template(notification_type,
context,
language_code=DEFAULT_LANG):
"""
Render a notification template with given context
Returns a dict containing all content fields of the template. Example:
{'short_message': 'foo', 'subject': 'bar', 'body': 'baz'}
"""
try:
template = NotificationTemplate.objects.get(type=notification_type)
except NotificationTemplate.DoesNotExist as e:
raise NotificationTemplateException(e) from e
env = SandboxedEnvironment(trim_blocks=True,
lstrip_blocks=True,
undefined=StrictUndefined)
env.filters['reservation_time'] = reservation_time
env.filters['format_datetime'] = format_datetime
env.filters['format_datetime_tz'] = format_datetime_tz
logger.info('Rendering template for notification %s' % notification_type)
with switch_language(template, language_code):
try:
return {
attr: env.from_string(getattr(template, attr)).render(context)
for attr in ('short_message', 'subject', 'body')
}
except TemplateError as e:
raise NotificationTemplateException(e) from e
def test_unary_operator_intercepting(self):
def disable_op(arg):
raise TemplateRuntimeError('that operator so does not work')
for expr, ctx, rv in (('-1', {}, '-1'), ('-a', {'a': 2}, '-2')):
env = SandboxedEnvironment()
env.unop_table['-'] = disable_op
t = env.from_string('{{ %s }}' % expr)
env.intercepted_unops = frozenset(['-'])
t = env.from_string('{{ %s }}' % expr)
try:
t.render(ctx)
except TemplateRuntimeError as e:
pass
else:
self.fail('expected runtime error')
def test_binary_operator_intercepting(self, env):
def disable_op(left, right):
raise TemplateRuntimeError('that operator so does not work')
for expr, ctx, rv in ('1 + 2', {}, '3'), ('a + 2', {'a': 2}, '4'):
env = SandboxedEnvironment()
env.binop_table['+'] = disable_op
t = env.from_string('{{ %s }}' % expr)
assert t.render(ctx) == rv
env.intercepted_binops = frozenset(['+'])
t = env.from_string('{{ %s }}' % expr)
try:
t.render(ctx)
except TemplateRuntimeError as e:
pass
else:
assert False, 'expected runtime error'
def update_changelog(inplace_file: str, marker: str,
version_regex: str) -> None:
"""
Update the given changelog file in place.
Arguments:
inplace_file: The file to update in-place.
marker: The line after which to insert new contents.
version_regex: A regular expression to find currently documented versions in the file.
"""
env = SandboxedEnvironment(autoescape=True)
template = env.from_string(httpx.get(TEMPLATE_URL).text)
changelog = Changelog(".", style=COMMIT_STYLE)
if len(changelog.versions_list) == 1:
last_version = changelog.versions_list[0]
if last_version.planned_tag is None:
planned_tag = "v0.1.0"
last_version.tag = planned_tag
last_version.url += planned_tag
last_version.compare_url = last_version.compare_url.replace(
"HEAD", planned_tag)
lines = read_changelog(inplace_file)
last_released = latest(lines, re.compile(version_regex))
if last_released:
changelog.versions_list = unreleased(changelog.versions_list,
last_released)
rendered = template.render(changelog=changelog, inplace=True)
lines[lines.index(marker)] = rendered
write_changelog(inplace_file, lines)
def _flatten_variable(
var_name: str,
variable_defs: Dict[str, str],
variables_dag: Dict[str, Set[str]],
flattened_variables: Dict[str, str],
):
""" Helper function for flatten_recursive_variables.
Recurisvely resolve each variable definition
"""
var_deps = variables_dag[var_name]
for dep_var_name in var_deps:
# Resolve anything that is not defined
if dep_var_name not in flattened_variables:
_flatten_variable(
dep_var_name,
variable_defs,
variables_dag,
flattened_variables,
)
# Now all dependencies are solved
env = SandboxedEnvironment(autoescape=False)
template = env.from_string(variable_defs[var_name])
flattened_variables[var_name] = template.render(
**{
dep_var_name: flattened_variables[dep_var_name]
for dep_var_name in var_deps
})
def update_changelog(
inplace_file: str,
marker: str = MARKER,
version_regex: str = VERSION_REGEX,
template_filepath: str = CHANGELOG_TEMPLATE,
commit_style: str = "angular",
):
env = SandboxedEnvironment(autoescape=False)
changelog = Changelog(".", style=commit_style)
template = env.from_string(read_template(template_filepath))
if len(changelog.versions_list) == 1:
last_version = changelog.versions_list[0]
if last_version.planned_tag is None:
planned_tag = "0.1.0"
last_version.tag = planned_tag
last_version.url += planned_tag
last_version.compare_url = last_version.compare_url.replace(
"HEAD", planned_tag)
lines = read_changelog(inplace_file)
last_released = latest_version(lines, re.compile(version_regex))
if last_released:
changelog.versions_list = unreleased_versions(changelog.versions_list,
last_released)
rendered = template.render(changelog=changelog, inplace=True)
lines[lines.index(marker)] = rendered
write_changelog(inplace_file, lines)
def test_unary_operator_intercepting(self):
def disable_op(arg):
raise TemplateRuntimeError('that operator so does not work')
for expr, ctx, rv in ('-1', {}, '-1'), ('-a', {'a': 2}, '-2'):
env = SandboxedEnvironment()
env.unop_table['-'] = disable_op
t = env.from_string('{{{{ {0!s} }}}}'.format(expr))
assert t.render(ctx) == rv
env.intercepted_unops = frozenset(['-'])
t = env.from_string('{{{{ {0!s} }}}}'.format(expr))
try:
t.render(ctx)
except TemplateRuntimeError as e:
pass
else:
self.fail('expected runtime error')
def update_changelog(
inplace_file: str,
marker: str,
version_regex: str,
template_url: str,
commit_style: str,
) -> None:
"""Update the given changelog file in place.
Arguments:
inplace_file: The file to update in-place.
marker: The line after which to insert new contents.
version_regex: A regular expression to find currently documented versions in the file.
template_url: The URL to the Jinja template used to render contents.
commit_style: The style of commit messages to parse.
"""
env = SandboxedEnvironment(autoescape=False)
template = env.from_string(httpx.get(template_url).text)
changelog = Changelog(".", style=commit_style) # noqa: W0621 (shadowing changelog)
if len(changelog.versions_list) == 1:
last_version = changelog.versions_list[0]
if last_version.planned_tag is None:
planned_tag = "0.1.0"
last_version.tag = planned_tag
last_version.url += planned_tag
last_version.compare_url = last_version.compare_url.replace("HEAD", planned_tag)
lines = read_changelog(inplace_file)
last_released = latest(lines, re.compile(version_regex))
if last_released:
changelog.versions_list = unreleased(changelog.versions_list, last_released)
rendered = template.render(changelog=changelog, inplace=True)
lines[lines.index(marker)] = rendered
write_changelog(inplace_file, lines)
class BaseTemplateProcessor: # pylint: disable=too-few-public-methods
"""Base class for database-specific jinja context
There's this bit of magic in ``process_template`` that instantiates only
the database context for the active database as a ``models.Database``
object binds it to the context object, so that object methods
have access to
that context. This way, {{ hive.latest_partition('mytable') }} just
knows about the database it is operating in.
This means that object methods are only available for the active database
and are given access to the ``models.Database`` object and schema
name. For globally available methods use ``@classmethod``.
"""
engine: Optional[str] = None
def __init__(
self,
database: "Database",
query: Optional["Query"] = None,
table: Optional["SqlaTable"] = None,
extra_cache_keys: Optional[List[Any]] = None,
**kwargs: Any,
) -> None:
self._database = database
self._query = query
self._schema = None
if query and query.schema:
self._schema = query.schema
elif table:
self._schema = table.schema
extra_cache = ExtraCache(extra_cache_keys)
self._context = {
"url_param": extra_cache.url_param,
"current_user_id": extra_cache.current_user_id,
"current_username": extra_cache.current_username,
"cache_key_wrapper": extra_cache.cache_key_wrapper,
"filter_values": filter_values,
"form_data": {},
}
self._context.update(kwargs)
self._context.update(jinja_base_context)
if self.engine:
self._context[self.engine] = self
self._env = SandboxedEnvironment()
def process_template(self, sql: str, **kwargs: Any) -> str:
"""Processes a sql template
>>> sql = "SELECT '{{ datetime(2017, 1, 1).isoformat() }}'"
>>> process_template(sql)
"SELECT '2017-01-01T00:00:00'"
"""
template = self._env.from_string(sql)
kwargs.update(self._context)
return template.render(kwargs)
def test_sandbox_max_range(self, env):
from jinja2.sandbox import SandboxedEnvironment, MAX_RANGE
env = SandboxedEnvironment()
t = env.from_string("{% for item in range(total) %}{{ item }}{% endfor %}")
with pytest.raises(OverflowError):
t.render(total=MAX_RANGE + 1)
def process_template(content: str) -> str:
env = SandboxedEnvironment()
template = env.from_string(content)
context = {
"current_user_id": ExtraCache.current_user_id,
"current_username": ExtraCache.current_username,
}
return template.render(context)
def process_template(content):
env = SandboxedEnvironment()
template = env.from_string(content)
context = {
'current_user_id': current_user_id,
'current_username': current_username,
}
return template.render(context)
def process_template(content):
env = SandboxedEnvironment()
template = env.from_string(content)
context = {
"current_user_id": current_user_id,
"current_username": current_username
}
return template.render(context)
def serialize_html(self, request, data):
request.setHeader('content-type', 'text/html')
from jinja2.sandbox import SandboxedEnvironment
env = SandboxedEnvironment()
env.filters['urlencode'] = urllib.quote_plus
template = env.from_string(LIST_TEMPLATE)
return str(template.render(data))
def main(project_dir):
"""Load Jinja template file expand it write to packages.yml."""
env = SandboxedEnvironment()
with open(os.path.join(project_dir, "packages.yml.jinja2")) as f:
template_str = f.read()
template = env.from_string(
template_str, globals=dict(DBT_VERSION_TUPLE=DBT_VERSION_TUPLE))
expanded = template.render()
with open(os.path.join(project_dir, "packages.yml"), "w") as f:
f.write(expanded)
class BaseRenderer(object):
def __init__(self, loader=None):
self.env = SandboxedEnvironment(loader=loader)
self.env.filters['repr'] = repr
def render(self, template_name, context):
return self.env.get_template(template_name).render(context)
def render_string(self, source, context):
return self.env.from_string(source).render(context)
class BaseRenderer(object):
def __init__(self, loader=None):
self.env = SandboxedEnvironment(loader=loader)
self.env.filters['repr'] = repr
def render(self, template_name, context):
return self.env.get_template(template_name).render(context)
def render_string(self, source, context):
return self.env.from_string(source).render(context)
def main():
aaf = AAFClient('passerratings')
now = pendulum.now()
games = aaf.games_between(now - timedelta(days=7), now)
env = SandboxedEnvironment(loader=BaseLoader(), trim_blocks=True)
env.filters['passerrating'] = pr_filter
template = env.from_string(TEMPLATE)
print(template.render(games=games, subreddits=subreddits))
def preview(self, request, obj, **kwargs):
env = SandboxedEnvironment(trim_blocks=True,
lstrip_blocks=True,
undefined=DebugUndefined)
try:
body_html = env.from_string(obj.body_html).render(
dummy_context.context)
return HttpResponse(body_html)
except TemplateSyntaxError as e:
return HttpResponse(e)
def test_unsafe(self):
env = SandboxedEnvironment()
self.assert_raises(SecurityError, env.from_string('{{ foo.foo() }}').render, foo=PrivateStuff())
self.assert_equal(env.from_string('{{ foo.bar() }}').render(foo=PrivateStuff()), '23')
self.assert_raises(SecurityError, env.from_string('{{ foo._foo() }}').render, foo=PublicStuff())
self.assert_equal(env.from_string('{{ foo.bar() }}').render(foo=PublicStuff()), '23')
self.assert_equal(env.from_string('{{ foo.__class__ }}').render(foo=42), '')
self.assert_equal(env.from_string('{{ foo.func_code }}').render(foo=lambda : None), '')
self.assert_raises(SecurityError, env.from_string('{{ foo.__class__.__subclasses__() }}').render, foo=42)
def test_unsafe(self):
env = SandboxedEnvironment()
self.assert_raises(SecurityError, env.from_string('{{ foo.foo() }}').render, foo=PrivateStuff())
self.assert_equal(env.from_string('{{ foo.bar() }}').render(foo=PrivateStuff()), '23')
self.assert_raises(SecurityError, env.from_string('{{ foo._foo() }}').render, foo=PublicStuff())
self.assert_equal(env.from_string('{{ foo.bar() }}').render(foo=PublicStuff()), '23')
self.assert_equal(env.from_string('{{ foo.__class__ }}').render(foo=42), '')
self.assert_equal(env.from_string('{{ foo.func_code }}').render(foo=lambda : None), '')
self.assert_raises(SecurityError, env.from_string('{{ foo.__class__.__subclasses__() }}').render, foo=42)
class BaseRenderer:
def __init__(self, loader: BaseLoader = None) -> None:
self.env = SandboxedEnvironment(loader=loader, extensions=['jinja2.ext.i18n'])
self.env.filters['repr'] = repr
self.env.install_gettext_translations(get_translator())
def render(self, template_name: str, context: Dict) -> str:
return self.env.get_template(template_name).render(context)
def render_string(self, source: str, context: Dict) -> str:
return self.env.from_string(source).render(context)
class BaseTemplateProcessor(object):
"""Base class for database-specific jinja context
There's this bit of magic in ``process_template`` that instantiates only
the database context for the active database as a ``models.Database``
object binds it to the context object, so that object methods
have access to
that context. This way, {{ hive.latest_partition('mytable') }} just
knows about the database it is operating in.
This means that object methods are only available for the active database
and are given access to the ``models.Database`` object and schema
name. For globally available methods use ``@classmethod``.
"""
engine = None
def __init__(self, database=None, query=None, table=None, **kwargs):
self.database = database
self.query = query
self.schema = None
if query and query.schema:
self.schema = query.schema
elif table:
self.schema = table.schema
self.context = {
'url_param': url_param,
'current_user_id': current_user_id,
'current_username': current_username,
'form_data': {},
'user': g.user
,'filters_by_rol':get_filters_by_rol
}
self.context.update(kwargs)
self.context.update(BASE_CONTEXT)
if self.engine:
self.context[self.engine] = self
self.env = SandboxedEnvironment()
def process_template(self, sql, **kwargs):
"""Processes a sql template
>>> sql = "SELECT '{{ datetime(2017, 1, 1).isoformat() }}'"
>>> process_template(sql)
"SELECT '2017-01-01T00:00:00'"
"""
template = self.env.from_string(sql)
kwargs.update(self.context)
return template.render(kwargs)
def get_credits():
"""Return credits as Markdown.
Returns:
The credits page Markdown.
"""
jinja_env = SandboxedEnvironment(undefined=StrictUndefined)
commit = "166758a98d5e544aaa94fda698128e00733497f4"
template_url = f"https://raw.githubusercontent.com/pawamoy/jinja-templates/{commit}/credits.md"
template_data = get_credits_data()
template_text = httpx.get(template_url).text
return jinja_env.from_string(template_text).render(**template_data)
def get_credits():
"""Return credits as Markdown.
Returns:
The credits page Markdown.
"""
jinja_env = SandboxedEnvironment(undefined=StrictUndefined)
commit = "c78c29caa345b6ace19494a98b1544253cbaf8c1"
template_url = f"https://raw.githubusercontent.com/pawamoy/jinja-templates/{commit}/credits.md"
template_data = get_credits_data()
template_text = urlopen(template_url).read().decode("utf8") # noqa: S310
return jinja_env.from_string(template_text).render(**template_data)
def rend():
data = request.get_json()
if not isinstance(data, dict):
resp = jsonify({"error": "Invalid JSON"})
resp.status_code = 400
return resp
trim_blocks = bool(data.get("trim_blocks"))
lstrip_blocks = bool(data.get("lstrip_blocks"))
keep_trailing_newline = bool(data.get("keep_trailing_newline"))
newline_sequence = NEWLINE_SEQUENCES.get(data.get("newline_sequence"),
"\n")
extenstions = data.get("extensions", [])
extenstions = [extenstions] if isinstance(extenstions,
str) else extenstions
undefined = MAP_UNDEFINED.get(data.get("undefined"), jinja2.Undefined)
raw_template = data.get("template", "")
load_filter = data.get("filter", "default")
try:
yaml_data = yaml.safe_load(data.get("data")) or {}
if not isinstance(yaml_data, dict):
yaml_data = {"data": yaml_data}
jinja_env = SandboxedEnvironment(
trim_blocks=trim_blocks,
lstrip_blocks=lstrip_blocks,
keep_trailing_newline=keep_trailing_newline,
newline_sequence=newline_sequence,
extensions=extenstions,
undefined=undefined,
)
if load_filter == "ansible":
load_filter_ansible(jinja_env)
if load_filter == "salt":
load_filter_salt(jinja_env)
if load_filter == "st2":
load_filter_st2(jinja_env)
template = jinja_env.from_string(raw_template)
output = template.render(**yaml_data)
resp = jsonify({"template": output})
except yaml.error.YAMLError as e:
resp = jsonify({"error": f"YAML {type(e).__name__}", "msg": str(e)})
resp.status_code = 400
except jinja2.exceptions.TemplateError as e:
resp = jsonify({"error": f"Jinja {type(e).__name__}", "msg": str(e)})
resp.status_code = 400
except Exception as e:
resp = jsonify({"error": f"Error {type(e).__name__}", "msg": str(e)})
resp.status_code = 400
finally:
return resp
def __init__(self, text):
self.valid_parameters = []
self.valid_blocks = []
text = self._escape_text(text.encode('utf-8')).strip()
text = text.decode('utf-8')
template_environment = SandboxedEnvironment()
try:
self.template = template_environment.from_string(text)
self.error = None
except TemplateSyntaxError as e:
self.template = None
self.error = e
class BaseRenderer(object):
def __init__(self, loader=None):
# type: (BaseLoader) -> None
self.env = SandboxedEnvironment(loader=loader, extensions=['jinja2.ext.i18n'])
self.env.filters['repr'] = repr
self.env.install_gettext_translations(get_translator()) # type: ignore
def render(self, template_name, context):
# type: (unicode, Dict) -> unicode
return self.env.get_template(template_name).render(context)
def render_string(self, source, context):
# type: (unicode, Dict) -> unicode
return self.env.from_string(source).render(context)
class BaseTemplateProcessor(object):
"""Base class for database-specific jinja context
There's this bit of magic in ``process_template`` that instantiates only
the database context for the active database as a ``models.Database``
object binds it to the context object, so that object methods
have access to
that context. This way, {{ hive.latest_partition('mytable') }} just
knows about the database it is operating in.
This means that object methods are only available for the active database
and are given access to the ``models.Database`` object and schema
name. For globally available methods use ``@classmethod``.
"""
engine = None
def __init__(self, database=None, query=None, table=None, **kwargs):
self.database = database
self.query = query
self.schema = None
if query and query.schema:
self.schema = query.schema
elif table:
self.schema = table.schema
self.context = {
'url_param': url_param,
'current_user_id': current_user_id,
'current_username': current_username,
'filter_values': filter_values,
'form_data': {},
}
self.context.update(kwargs)
self.context.update(BASE_CONTEXT)
if self.engine:
self.context[self.engine] = self
self.env = SandboxedEnvironment()
def process_template(self, sql, **kwargs):
"""Processes a sql template
>>> sql = "SELECT '{{ datetime(2017, 1, 1).isoformat() }}'"
>>> process_template(sql)
"SELECT '2017-01-01T00:00:00'"
"""
template = self.env.from_string(sql)
kwargs.update(self.context)
return template.render(kwargs)
def test_unsafe(self, env):
env = SandboxedEnvironment()
pytest.raises(SecurityError, env.from_string("{{ foo.foo() }}").render,
foo=PrivateStuff())
assert env.from_string("{{ foo.bar() }}").render(foo=PrivateStuff()) == '23'
pytest.raises(SecurityError,
env.from_string("{{ foo._foo() }}").render,
foo=PublicStuff())
assert env.from_string("{{ foo.bar() }}").render(foo=PublicStuff()) == '23'
assert env.from_string("{{ foo.__class__ }}").render(foo=42) == ''
assert env.from_string("{{ foo.func_code }}").render(foo=lambda:None) == ''
# security error comes from __class__ already.
pytest.raises(SecurityError, env.from_string(
"{{ foo.__class__.__subclasses__() }}").render, foo=42)
def render_in_context(context, template_text, html_intent=False):
"""
Render the given Jinja2 template text in the script context.
:param context: Script context.
:type context: shuup.notify.script.Context
:param template_text: Jinja2 template text.
:type template_text: str
:param html_intent: Is the template text intended for HTML output?
This currently turns on autoescaping.
:type html_intent: bool
:return: Rendered template text
:rtype: str
:raises: Whatever Jinja2 might happen to raise
"""
# TODO: Add some filters/globals into this environment?
env = SandboxedEnvironment(autoescape=html_intent)
template = env.from_string(template_text)
return template.render(context.get_variables())
class JinjaProcessor(ComputableInputProcessor):
def __init__(self):
self.env = SandboxedEnvironment(trim_blocks=True,
lstrip_blocks=True)
self._globals = {'make_arr': make_arr}
def run(self, resource_name, computable_type, funct, data):
t = self.env.from_string(funct, globals=self._globals)
if computable_type == ComputablePassedTypes.full.name:
arr = make_arr(data)
my_inputs = arr[resource_name]
else:
my_inputs = {}
arr = {}
return t.render(resource_name=resource_name,
D=data,
R=arr,
**my_inputs).strip()
def _render_template(thread, template):
"""Render given iterable of templates in a sandboxed environment.
:param JobThread thread: JobThread instance the templates refer to
:param template: Template we need to render.
:returns: Rendered template(s)
"""
sandbox = SandboxedEnvironment()
rendered = (
sandbox.from_string(template).render(
stage=thread.stage,
substage=thread.substage,
distro=thread.distro,
arch=thread.arch
)
)
if not isinstance(rendered, str):
rendered = rendered.encode('ascii', 'ignore')
return rendered
def test_basic_format_safety(self):
env = SandboxedEnvironment()
t = env.from_string('{{ "a{0.__class__}b".format(42) }}')
assert t.render() == 'ab'
def test_attr_filter(self):
env = SandboxedEnvironment()
tmpl = env.from_string('{{ cls|attr("__subclasses__")() }}')
self.assert_raises(SecurityError, tmpl.render, cls=int)
def _jinja_renderer(self, file_content):
jinja_env = SandboxedEnvironment()
template = jinja_env.from_string(file_content.decode('utf-8'))
rendered = template.render(**self.data)
return rendered
def test_attr_filter():
env = SandboxedEnvironment()
tmpl = env.from_string('{{ 42|attr("__class__")|attr("__subclasses__")() }}')
raises(SecurityError, tmpl.render)
def test_safe_format_safety(self):
env = SandboxedEnvironment()
t = env.from_string('{{ ("a{0.__class__}b{1}"|safe).format(42, "<foo>") }}')
assert t.render() == 'ab<foo>'
def test_safe_format_all_okay(self):
env = SandboxedEnvironment()
t = env.from_string('{{ ("a{0.foo}b{1}"|safe).format({"foo": 42}, "<foo>") }}')
assert t.render() == 'a42b<foo>'
class BuiltinTemplateLoader(TemplateBridge, BaseLoader):
"""
Interfaces the rendering environment of jinja2 for use in Sphinx.
"""
# TemplateBridge interface
def init(self, builder, theme=None, dirs=None):
# type: (Builder, Theme, List[unicode]) -> None
# create a chain of paths to search
if theme:
# the theme's own dir and its bases' dirs
pathchain = theme.get_theme_dirs()
# the loader dirs: pathchain + the parent directories for all themes
loaderchain = pathchain + [path.join(p, '..') for p in pathchain]
elif dirs:
pathchain = list(dirs)
loaderchain = list(dirs)
else:
pathchain = []
loaderchain = []
# prepend explicit template paths
self.templatepathlen = len(builder.config.templates_path)
if builder.config.templates_path:
cfg_templates_path = [path.join(builder.confdir, tp)
for tp in builder.config.templates_path]
pathchain[0:0] = cfg_templates_path
loaderchain[0:0] = cfg_templates_path
# store it for use in newest_template_mtime
self.pathchain = pathchain
# make the paths into loaders
self.loaders = [SphinxFileSystemLoader(x) for x in loaderchain]
use_i18n = builder.app.translator is not None
extensions = use_i18n and ['jinja2.ext.i18n'] or []
self.environment = SandboxedEnvironment(loader=self,
extensions=extensions)
self.environment.filters['tobool'] = _tobool
self.environment.filters['toint'] = _toint
self.environment.filters['todim'] = _todim
self.environment.filters['slice_index'] = _slice_index
self.environment.globals['debug'] = contextfunction(pformat)
self.environment.globals['warning'] = warning
self.environment.globals['accesskey'] = contextfunction(accesskey)
self.environment.globals['idgen'] = idgen
if use_i18n:
self.environment.install_gettext_translations(builder.app.translator) # type: ignore # NOQA
def render(self, template, context): # type: ignore
# type: (unicode, Dict) -> unicode
return self.environment.get_template(template).render(context)
def render_string(self, source, context):
# type: (unicode, Dict) -> unicode
return self.environment.from_string(source).render(context)
def newest_template_mtime(self):
# type: () -> float
return max(mtimes_of_files(self.pathchain, '.html'))
# Loader interface
def get_source(self, environment, template):
# type: (Environment, unicode) -> Tuple[unicode, unicode, Callable]
loaders = self.loaders
# exclamation mark starts search from theme
if template.startswith('!'):
loaders = loaders[self.templatepathlen:]
template = template[1:]
for loader in loaders:
try:
return loader.get_source(environment, template)
except TemplateNotFound:
pass
raise TemplateNotFound(template)
class BuiltinTemplateLoader(TemplateBridge, BaseLoader):
"""
Interfaces the rendering environment of jinja2 for use in Sphinx.
"""
# TemplateBridge interface
def init(self, builder, theme=None, dirs=None):
# create a chain of paths to search
if theme:
# the theme's own dir and its bases' dirs
chain = theme.get_dirchain()
# then the theme parent paths
chain.extend(theme.themepath)
elif dirs:
chain = list(dirs)
else:
chain = []
# prepend explicit template paths
self.templatepathlen = len(builder.config.templates_path)
if builder.config.templates_path:
chain[0:0] = [path.join(builder.confdir, tp)
for tp in builder.config.templates_path]
# store it for use in newest_template_mtime
self.pathchain = chain
# make the paths into loaders
self.loaders = map(SphinxFileSystemLoader, chain)
use_i18n = builder.app.translator is not None
extensions = use_i18n and ['jinja2.ext.i18n'] or []
self.environment = SandboxedEnvironment(loader=self,
extensions=extensions)
self.environment.filters['tobool'] = _tobool
self.environment.filters['toint'] = _toint
self.environment.globals['debug'] = contextfunction(pformat)
self.environment.globals['accesskey'] = contextfunction(accesskey)
self.environment.globals['idgen'] = idgen
if use_i18n:
self.environment.install_gettext_translations(
builder.app.translator)
def render(self, template, context):
return self.environment.get_template(template).render(context)
def render_string(self, source, context):
return self.environment.from_string(source).render(context)
def newest_template_mtime(self):
return max(mtimes_of_files(self.pathchain, '.html'))
# Loader interface
def get_source(self, environment, template):
loaders = self.loaders
# exclamation mark starts search from theme
if template.startswith('!'):
loaders = loaders[self.templatepathlen:]
template = template[1:]
for loader in loaders:
try:
return loader.get_source(environment, template)
except TemplateNotFound:
pass
raise TemplateNotFound(template)
def test_basic_format_all_okay(self):
env = SandboxedEnvironment()
t = env.from_string('{{ "a{0.foo}b".format({"foo": 42}) }}')
assert t.render() == 'a42b'