def test_tokenize(self) -> None:
"""Test with simple rules"""
def word_token(m: typing.Match[str]) -> Token:
return m.end(), "w", m.group()
def blank_token(m: typing.Match[str]) -> Token:
return m.end(), "b", m.group()
def to_upper(s: str) -> str:
return s.upper()
def cleanup(tokens: typing.List[Token]) -> None:
for i in range(len(tokens)):
t = tokens[i]
if t[i] == "b":
tokens[i] = (t[0], "b", " ")
class Extension(object):
lexer_rules = {
100: (re.compile(r"\w+"), word_token),
200: (re.compile(r"\s+"), blank_token),
}
preprocessors = [to_upper]
postprocessors = [cleanup]
lexer = Lexer(**lexer_scan([Extension]))
assert [
(1, "w", "HELLO"),
(1, "b", " "),
(2, "w", "WORLD"),
] == lexer.tokenize("hello\n world")
def test_tokenize(self):
""" Test with simple rules
"""
import re
from wheezy.template.lexer import Lexer
from wheezy.template.lexer import lexer_scan
def word_token(m):
return m.end(), 'w', m.group()
def blank_token(m):
return m.end(), 'b', m.group()
def to_upper(s):
return s.upper()
def cleanup(tokens):
for i in range(len(tokens)):
t = tokens[i]
if t[i] == 'b':
tokens[i] = (t[0], 'b', ' ')
class Extension(object):
lexer_rules = {
100: (re.compile(r'\w+'), word_token),
200: (re.compile(r'\s+'), blank_token)
}
preprocessors = [to_upper]
postprocessors = [cleanup]
lexer = Lexer(**lexer_scan([Extension]))
assert [(1, 'w', 'HELLO'),
(1, 'b', ' '),
(2, 'w', 'WORLD')] == lexer.tokenize('hello\n world')
def __init__(self, loader, extensions, template_class=None):
self.lock = allocate_lock()
self.templates = {}
self.renders = {}
self.modules = {}
self.global_vars = {'_r': self.render, '_i': self.import_name}
self.loader = loader
self.template_class = template_class or Template
self.compiler = Compiler(self.global_vars, -2)
self.lexer = Lexer(**lexer_scan(extensions))
self.parser = Parser(**parser_scan(extensions))
self.builder = SourceBuilder(**builder_scan(extensions))
def __init__(
self,
loader: Loader,
extensions: typing.List[typing.Any],
template_class: typing.Optional[TemplateClass] = None,
) -> None:
self.lock = allocate_lock()
self.templates: typing.Dict[str, SupportsRender] = {}
self.renders: typing.Dict[str, RenderTemplate] = {}
self.modules: typing.Dict[str, ModuleType] = {}
self.global_vars = {"_r": self.render, "_i": self.import_name}
self.loader = loader
self.template_class = template_class or Template
self.compiler = Compiler(self.global_vars, -2)
self.lexer = Lexer(**lexer_scan(extensions))
self.parser = Parser(**parser_scan(extensions))
self.builder = SourceBuilder(**builder_scan(extensions))
def test_tokenize(self):
""" Test with simple rules
"""
import re
from wheezy.template.lexer import Lexer
def word_token(m):
return m.end(), 'w', m.group()
def blank_token(m):
return m.end(), 'b', m.group()
lexer = Lexer([
(re.compile(r'\w+'), word_token),
(re.compile(r'\s+'), blank_token),
])
assert [(1, 'w', 'hello'),
(1, 'b', '\n '),
(2, 'w', 'world')] == lexer.tokenize('hello\n world')
def __init__(self, loader, extensions, template_class=None):
self.lock = allocate_lock()
self.templates = {}
self.renders = {}
self.modules = {}
self.global_vars = {
'_r': self.render,
'_i': self.import_name
}
self.loader = loader
self.template_class = template_class or Template
self.compiler = Compiler(self.global_vars, -2)
self.lexer = Lexer(**lexer_scan(extensions))
self.parser = Parser(**parser_scan(extensions))
self.builder = SourceBuilder(**builder_scan(extensions))
def test_raises_error(self):
""" If there is no match it raises AssertionError.
"""
from wheezy.template.lexer import Lexer
lexer = Lexer([])
self.assertRaises(AssertionError, lambda: lexer.tokenize('test'))
def test_trivial(self):
""" Empty rules and source
"""
from wheezy.template.lexer import Lexer
lexer = Lexer([])
assert [] == lexer.tokenize('')
class Engine(object):
""" The core component of template engine.
"""
def __init__(self, loader, extensions, template_class=None):
self.lock = allocate_lock()
self.templates = {}
self.renders = {}
self.modules = {}
self.global_vars = {
'_r': self.render,
'_i': self.import_name
}
self.loader = loader
self.template_class = template_class or Template
self.compiler = Compiler(self.global_vars, -2)
self.lexer = Lexer(**lexer_scan(extensions))
self.parser = Parser(**parser_scan(extensions))
self.builder = SourceBuilder(**builder_scan(extensions))
def get_template(self, name):
""" Returns compiled template.
"""
try:
return self.templates[name]
except KeyError:
self.compile_template(name)
return self.templates[name]
def render(self, name, ctx, local_defs, super_defs):
""" Renders template by name in given context.
"""
try:
return self.renders[name](ctx, local_defs, super_defs)
except KeyError:
self.compile_template(name)
return self.renders[name](ctx, local_defs, super_defs)
def remove(self, name):
""" Removes given ``name`` from internal cache.
"""
self.lock.acquire(1)
try:
if name in self.renders:
del self.templates[name]
del self.renders[name]
if name in self.modules:
del self.modules[name]
finally:
self.lock.release()
# region: internal details
def import_name(self, name):
try:
return self.modules[name]
except KeyError:
self.compile_import(name)
return self.modules[name]
def compile_template(self, name):
self.lock.acquire(1)
try:
if name not in self.renders:
template_source = self.loader.load(name)
if template_source is None:
raise IOError('Template "%s" not found.' % name)
tokens = self.lexer.tokenize(template_source)
nodes = self.parser.parse(tokens)
source = self.builder.build_render(nodes)
#self.print_debug(name, tokens, nodes, source)
render_template = self.compiler.compile_source(
source, name)['render']
self.renders[name] = render_template
self.templates[name] = self.template_class(
name, render_template)
finally:
self.lock.release()
def compile_import(self, name):
self.lock.acquire(1)
try:
if name not in self.modules:
template_source = self.loader.load(name)
if template_source is None:
raise IOError('Import "%s" not found.' % name)
tokens = self.lexer.tokenize(template_source)
nodes = self.parser.parse(tokens)
source = self.builder.build_module(nodes)
#self.print_debug(name, tokens, nodes, source)
self.modules[name] = self.compiler.compile_module(
source, name)
finally:
self.lock.release()
def print_debug(self, name, tokens, nodes, source): # pragma: nocover
print(name.center(80, '-'))
from pprint import pprint
#pprint(tokens)
pprint(nodes)
from wheezy.template.utils import print_source
print_source(source, -1)
class Engine(object):
""" The core component of template engine.
"""
def __init__(self, loader, extensions, template_class=None):
self.lock = allocate_lock()
self.templates = {}
self.renders = {}
self.modules = {}
self.global_vars = {'_r': self.render, '_i': self.import_name}
self.loader = loader
self.template_class = template_class or Template
self.compiler = Compiler(self.global_vars, -2)
self.lexer = Lexer(**lexer_scan(extensions))
self.parser = Parser(**parser_scan(extensions))
self.builder = SourceBuilder(**builder_scan(extensions))
def get_template(self, name):
""" Returns compiled template.
"""
try:
return self.templates[name]
except KeyError:
self.compile_template(name)
return self.templates[name]
def render(self, name, ctx, local_defs, super_defs):
""" Renders template by name in given context.
"""
try:
return self.renders[name](ctx, local_defs, super_defs)
except KeyError:
self.compile_template(name)
return self.renders[name](ctx, local_defs, super_defs)
def remove(self, name):
""" Removes given ``name`` from internal cache.
"""
self.lock.acquire(1)
try:
if name in self.renders:
del self.templates[name]
del self.renders[name]
if name in self.modules:
del self.modules[name]
finally:
self.lock.release()
# region: internal details
def import_name(self, name):
try:
return self.modules[name]
except KeyError:
self.compile_import(name)
return self.modules[name]
def compile_template(self, name):
self.lock.acquire(1)
try:
if name not in self.renders:
template_source = self.loader.load(name)
if template_source is None:
raise IOError('Template "%s" not found.' % name)
tokens = self.lexer.tokenize(template_source)
nodes = self.parser.parse(tokens)
source = self.builder.build_render(nodes)
#self.print_debug(name, tokens, nodes, source)
render_template = self.compiler.compile_source(source,
name)['render']
self.renders[name] = render_template
self.templates[name] = self.template_class(
name, render_template)
finally:
self.lock.release()
def compile_import(self, name):
self.lock.acquire(1)
try:
if name not in self.modules:
template_source = self.loader.load(name)
if template_source is None:
raise IOError('Import "%s" not found.' % name)
tokens = self.lexer.tokenize(template_source)
nodes = self.parser.parse(tokens)
source = self.builder.build_module(nodes)
#self.print_debug(name, tokens, nodes, source)
self.modules[name] = self.compiler.compile_module(source, name)
finally:
self.lock.release()
def print_debug(self, name, tokens, nodes, source): # pragma: nocover
print(name.center(80, '-'))
from pprint import pprint
#pprint(tokens)
pprint(nodes)
from wheezy.template.utils import print_source
print_source(source, -1)
def test_raises_error(self) -> None:
"""If there is no match it raises AssertionError."""
lexer = Lexer([])
self.assertRaises(AssertionError, lambda: lexer.tokenize("test"))
def test_trivial(self) -> None:
"""Empty rules and source"""
lexer = Lexer([])
assert [] == lexer.tokenize("")
class Engine(object):
"""The core component of template engine."""
def __init__(
self,
loader: Loader,
extensions: typing.List[typing.Any],
template_class: typing.Optional[TemplateClass] = None,
) -> None:
self.lock = allocate_lock()
self.templates: typing.Dict[str, SupportsRender] = {}
self.renders: typing.Dict[str, RenderTemplate] = {}
self.modules: typing.Dict[str, ModuleType] = {}
self.global_vars = {"_r": self.render, "_i": self.import_name}
self.loader = loader
self.template_class = template_class or Template
self.compiler = Compiler(self.global_vars, -2)
self.lexer = Lexer(**lexer_scan(extensions))
self.parser = Parser(**parser_scan(extensions))
self.builder = SourceBuilder(**builder_scan(extensions))
def get_template(self, name: str) -> SupportsRender:
"""Returns compiled template."""
try:
return self.templates[name]
except KeyError:
self.compile_template(name)
return self.templates[name]
def render(
self,
name: str,
ctx: typing.Mapping[str, typing.Any],
local_defs: typing.Mapping[str, typing.Any],
super_defs: typing.Mapping[str, typing.Any],
) -> str:
"""Renders template by name in given context."""
try:
return self.renders[name](ctx, local_defs, super_defs)
except KeyError:
self.compile_template(name)
return self.renders[name](ctx, local_defs, super_defs)
def remove(self, name: str) -> None:
"""Removes given ``name`` from internal cache."""
self.lock.acquire(True)
try:
if name in self.renders:
del self.templates[name]
del self.renders[name]
if name in self.modules:
del self.modules[name]
finally:
self.lock.release()
# region: internal details
def import_name(self, name: str) -> ModuleType:
try:
return self.modules[name]
except KeyError:
self.compile_import(name)
return self.modules[name]
def compile_template(self, name: str) -> None:
self.lock.acquire(True)
try:
if name not in self.renders:
template_source = self.loader.load(name)
if template_source is None:
raise IOError('Template "%s" not found.' % name)
tokens = self.lexer.tokenize(template_source)
nodes = self.parser.parse(tokens)
source = self.builder.build_render(nodes)
# print_debug(name, tokens, nodes, source)
try:
render_template = self.compiler.compile_source(
source, name)["render"]
except SyntaxError as e:
raise complement_syntax_error(e, template_source, source)
self.renders[name] = render_template
self.templates[name] = self.template_class(
name, render_template)
finally:
self.lock.release()
def compile_import(self, name: str) -> None:
self.lock.acquire(True)
try:
if name not in self.modules:
template_source = self.loader.load(name)
if template_source is None:
raise IOError('Import "%s" not found.' % name)
tokens = self.lexer.tokenize(template_source)
nodes = self.parser.parse(tokens)
source = self.builder.build_module(nodes)
# print_debug(name, tokens, nodes, source)
try:
self.modules[name] = self.compiler.compile_module(
source, name)
except SyntaxError as e:
raise complement_syntax_error(e, template_source, source)
finally:
self.lock.release()
