def test_deferrable_handler_literal_continuation(self):
dispatcher = Dispatcher({}, None, {}, {})
node = Node()
node.value = '"foo\\\r\nbar"'
self.assertEqual(
'"foobar"',
deferrable_handler_literal_continuation(dispatcher, node))
def test_repeated_layouts(self):
class Block(Node):
pass
def space_layout(dispatcher, node, before, after, prev):
yield SimpleChunk(' ')
def newline_layout(dispatcher, node, before, after, prev):
yield SimpleChunk('n')
dispatcher = Dispatcher(
definitions={
'Node': (Space, ),
'Block': (JoinAttr(Iter(), value=(Newline, )), )
},
token_handler=None, # not actually produced
layout_handlers={
Space: space_layout,
Newline: newline_layout,
# drop the subsequent space
(Newline, Space): newline_layout,
},
deferrable_handlers={},
)
n0 = Block([])
self.assertEqual('', ''.join(c.text for c in walk(dispatcher, n0)))
n1 = Block([Node([])] * 1)
self.assertEqual(' ', ''.join(c.text for c in walk(dispatcher, n1)))
n2 = Block([Node([])] * 2)
self.assertEqual(' n', ''.join(c.text for c in walk(dispatcher, n2)))
n3 = Block([Node([])] * 3)
self.assertEqual(' nn', ''.join(c.text for c in walk(dispatcher, n3)))
def test_write_no_sourcemap(self):
root = mktemp()
definitions = {
'Node': (
Attr(attr='left'),
Attr(attr='op'),
Attr(attr='right'),
)
}
# the program node; attributes are assigned to mimic a real one
program = Node()
program.left, program.op, program.right = ('foo', '=', 'true')
program.sourcepath = join(root, 'original.js')
program._token_map = {
'foo': [(0, 1, 1)],
'=': [(4, 1, 5)],
'true': [(6, 1, 7)],
}
output_stream = StringIO()
output_stream.name = join(root, 'processed.js')
unparser = BaseUnparser(definitions)
io.write(unparser, program, output_stream)
self.assertEqual('foo=true', output_stream.getvalue())
def test_nested_layouts(self):
def simple_layout_space(dispatcher, node, before, after, prev):
yield SimpleChunk(' ')
def nested_layout(dispatcher, node, before, after, prev):
yield SimpleChunk('?')
def noop(*a, **kw):
return
yield # pragma: no cover
dispatcher = Dispatcher(
definitions={'Node': (
Space,
JoinAttr(Iter(), value=(Space, )),
)},
token_handler=noop,
layout_handlers={
Space: simple_layout_space,
(Space, Space): noop,
((Space, Space), Space): nested_layout,
},
deferrable_handlers={},
)
n0 = Node([])
self.assertEqual(' ', ''.join(c.text for c in walk(dispatcher, n0)))
n1 = Node([n0])
self.assertEqual('', ''.join(c.text for c in walk(dispatcher, n1)))
n2 = Node([n1, n0])
self.assertEqual('?', ''.join(c.text for c in walk(dispatcher, n2)))
def test_core_structures(self):
# initialise a barebone dispatcher.
node = Node()
dispatcher = Dispatcher({}, None, {}, {})
self.assertEqual([(
'{',
0,
0,
None,
None,
)], list(layout_handler_openbrace(dispatcher, node, None, None, None)))
self.assertEqual([(
'}',
0,
0,
None,
None,
)], list(layout_handler_closebrace(dispatcher, node, None, None,
None)))
self.assertEqual([(
';',
0,
0,
None,
None,
)], list(layout_handler_semicolon(dispatcher, node, None, None, None)))
# with token map
node._token_map = {
'{': [(0, 1, 1)],
'}': [(1, 1, 2)],
';': [(2, 1, 3)],
}
self.assertEqual([(
'{',
1,
1,
None,
None,
)], list(layout_handler_openbrace(dispatcher, node, None, None, None)))
self.assertEqual([(
'}',
1,
2,
None,
None,
)], list(layout_handler_closebrace(dispatcher, node, None, None,
None)))
self.assertEqual([(
';',
1,
3,
None,
None,
)], list(layout_handler_semicolon(dispatcher, node, None, None, None)))
def test_write_multiple(self):
root = mktemp()
definitions = {
'Node': (
Attr(attr='left'),
Text(value=' '),
Attr(attr='op'),
Text(value=' '),
Attr(attr='right'),
Text(value=';'),
)
}
# the program node; attributes are assigned to mimic a real one
program1 = Node()
program1.left, program1.op, program1.right = ('foo', '=', 'true')
program1.sourcepath = join(root, 'program1.js')
program1._token_map = {
'foo': [(0, 1, 1)],
'=': [(4, 1, 5)],
'true': [(6, 1, 7)],
}
program2 = Node()
program2.left, program2.op, program2.right = ('bar', '=', 'false')
program2.sourcepath = join(root, 'program2.js')
program2._token_map = {
'bar': [(0, 1, 1)],
'=': [(4, 1, 5)],
'false': [(6, 1, 7)],
}
# streams
output_stream = StringIO()
output_stream.name = join(root, 'packed.js')
sourcemap_stream = StringIO()
sourcemap_stream.name = join(root, 'packed.js.map')
unparser = BaseUnparser(definitions)
io.write(unparser, [program1, program2],
output_stream,
sourcemap_stream,
source_mapping_url=None)
self.assertEqual('foo = true;bar = false;', output_stream.getvalue())
sourcemap = json.loads(sourcemap_stream.getvalue())
self.assertEqual(
{
"version": 3,
"sources": ["program1.js", "program2.js"],
"names": [],
"mappings": "AAAA,WCAA",
"file": "packed.js"
}, sourcemap)
def test_write_same_stream_callable(self):
# streams
root = mktemp()
output_stream = StringIO()
output_stream.name = join(root, 'packed.js')
called = []
closed = []
def close():
closed.append(True)
output_stream.close = close
def f_output_stream():
called.append(True)
return output_stream
definitions = {
'Node': (
Attr(attr='text'),
Text(value=';'),
)
}
# the program node; attributes are assigned to mimic a real one
program = Node()
program.text = 'hello'
program.sourcepath = join(root, 'program.js')
program._token_map = {'hello': [(0, 1, 1)]}
unparser = BaseUnparser(definitions)
io.write(unparser, [program], f_output_stream, f_output_stream)
self.assertEqual(1, len(called))
self.assertEqual(1, len(closed))
output = output_stream.getvalue()
self.assertIn('hello', output)
self.assertNotIn('program.js', output)
# since output stream is a StringIO, default to utf8 encoding
self.assertIn('data:application/json;base64;charset=utf8', output)
# decode the base64 string
self.assertEqual(
{
"version": 3,
"sources": ["program.js"],
"names": [],
"mappings": "AAAA",
"file": "packed.js"
},
json.loads(
base64.b64decode(output.splitlines()[-1].split(',')[-1].encode(
'utf8')).decode('utf8')))
def test_write_sourcemap_omitted(self):
root = mktemp()
definitions = {
'Node': (
Attr(attr='left'),
Attr(attr='op'),
Attr(attr='right'),
)
}
# the program node; attributes are assigned to mimic a real one
program = Node()
program.left, program.op, program.right = ('foo', '=', 'true')
program.sourcepath = join(root, 'original.js')
program._token_map = {
'foo': [(0, 1, 1)],
'=': [(4, 1, 5)],
'true': [(6, 1, 7)],
}
# streams
output_stream = StringIO()
output_stream.name = join(root, 'processed.js')
sourcemap_stream = StringIO()
sourcemap_stream.name = join(root, 'processed.js.map')
unparser = BaseUnparser(definitions)
io.write(unparser,
program,
output_stream,
sourcemap_stream,
source_mapping_url=None)
sourcemap = json.loads(sourcemap_stream.getvalue())
self.assertEqual(
{
"version": 3,
"sources": ["original.js"],
"names": [],
"mappings": "AAAA,GAAI,CAAE",
"file": "processed.js"
}, sourcemap)
self.assertEqual('foo=true', output_stream.getvalue())
def test_write_error_handled_callable_closed(self):
# streams
root = mktemp()
output_stream = StringIO()
output_stream.name = join(root, 'packed.js')
closed = []
def close():
closed.append(True)
output_stream.close = close
def f_output_stream():
return output_stream
def f_error():
raise IOError('some error happened')
definitions = {
'Node': (
Attr(attr='text'),
Text(value=';'),
)
}
# the program node; attributes are assigned to mimic a real one
program = Node()
program.text = 'hello'
program.sourcepath = join(root, 'program.js')
program._token_map = {'hello': [(0, 1, 1)]}
unparser = BaseUnparser(definitions)
with self.assertRaises(IOError):
io.write(unparser, [program], f_output_stream, f_error)
self.assertEqual(1, len(closed))
self.assertEqual('hello;', output_stream.getvalue())
self.assertNotIn('program.js', output_stream.getvalue())
def test_prewalk_hooking(self):
results = {}
def prewalk_dummy(dispatcher, node):
results.update({'dispatcher': dispatcher, 'node': node})
return node
definitions = {'Node': ()}
unparser = BaseUnparser(definitions, prewalk_hooks=[prewalk_dummy])
self.assertEqual(results, {})
# invoke complete run to trigger prewalk hook.
root = Node()
self.assertEqual([], list(unparser(root)))
self.assertTrue(isinstance(results['dispatcher'], Dispatcher))
self.assertTrue(results['node'], root)
def test_called_prewalk_multicall(self):
prewalk = []
def rule():
prewalk.append(True)
return {}
root = Node()
definitions = {'Node': ()}
unparser = BaseUnparser(definitions, rules=(rule, ))
# invoke complete run to trigger prewalk hook.
self.assertEqual(len(prewalk), 0)
self.assertEqual([], list(unparser(root)))
self.assertEqual(len(prewalk), 1)
self.assertEqual([], list(unparser(root)))
self.assertEqual(len(prewalk), 2)
def test_called_prewalk_via_rules(self):
results = {}
def prewalk_dummy(dispatcher, node):
results.update({'dispatcher': dispatcher, 'node': node})
return node
def rule():
return {'prewalk_hooks': (prewalk_dummy, )}
definitions = {'Node': ()}
unparser = BaseUnparser(definitions, rules=(rule, ))
# invoke complete run to trigger prewalk hook.
root = Node()
self.assertEqual([], list(unparser(root)))
self.assertTrue(isinstance(results['dispatcher'], Dispatcher))
self.assertTrue(results['node'], root)
def test_catch_scope_basic(self):
with self.assertRaises(TypeError) as e:
CatchScope(None, None)
self.assertEqual(e.exception.args[0],
'CatchScopes must have a Scope as a parent')
parent = Scope(None)
with self.assertRaises(AttributeError):
# None isn't a node that can be resolved.
CatchScope(None, parent)
with self.assertRaises(AttributeError):
# doesn't work with generic node either
CatchScope(Node(), parent)
catcher = CatchScope(Catch(Identifier('e'), []), parent)
self.assertEqual(catcher.catch_symbol, 'e')
catcher.close()
with self.assertRaises(ValueError):
catcher.close()
def test_join_attr_issue_36(self):
# JoinAttr defined with parameter `value=None` resulted in
# infinite recursion due to the walker will assume that no rule
# is provided, triggering rule lookup which would repeat the
# work that was done.
class Block(Node):
pass
token_handler, layout_handlers, deferrable_handlers, declared_vars = (
setup_handlers(self))
dispatcher = Dispatcher(
definitions={
'Block': (JoinAttr(Iter(), value=None), ),
'Node': (Text(value='', ), ),
},
token_handler=token_handler,
layout_handlers={},
deferrable_handlers={},
)
nodes = Block([Node([])] * 3)
self.assertEqual(3, len(list(walk(dispatcher, nodes))))
def test_write_callables(self):
closed = []
class Stream(StringIO):
# don't actually close the stream so it can be read later by
# the tests
def close(self):
closed.append(self)
# streams
root = mktemp()
output_stream = Stream()
output_stream.name = join(root, 'packed.js')
sourcemap_stream = Stream()
sourcemap_stream.name = join(root, 'packed.js.map')
def f_output_stream():
return output_stream
def f_sourcemap_stream():
return sourcemap_stream
definitions = {
'Node': (
Attr(attr='left'),
Text(value=' '),
Attr(attr='op'),
Text(value=' '),
Attr(attr='right'),
Text(value=';'),
)
}
# the program node; attributes are assigned to mimic a real one
program1 = Node()
program1.left, program1.op, program1.right = ('foo', '=', 'true')
program1.sourcepath = join(root, 'program1.js')
program1._token_map = {
'foo': [(0, 1, 1)],
'=': [(4, 1, 5)],
'true': [(6, 1, 7)],
}
program2 = Node()
program2.left, program2.op, program2.right = ('bar', '=', 'false')
program2.sourcepath = join(root, 'program2.js')
program2._token_map = {
'bar': [(0, 1, 1)],
'=': [(4, 1, 5)],
'false': [(6, 1, 7)],
}
unparser = BaseUnparser(definitions)
io.write(unparser, [program1, program2],
f_output_stream,
f_sourcemap_stream,
source_mapping_url=None)
self.assertIn(output_stream, closed)
self.assertIn(sourcemap_stream, closed)
self.assertEqual('foo = true;bar = false;', output_stream.getvalue())
sourcemap = json.loads(sourcemap_stream.getvalue())
self.assertEqual(
{
"version": 3,
"sources": ["program1.js", "program2.js"],
"names": [],
"mappings": "AAAA,WCAA",
"file": "packed.js"
}, sourcemap)
def test_minimum_definition(self):
definitions = {'Node': ()}
unparser = BaseUnparser(definitions)
self.assertEqual([], list(unparser(Node())))
def test_prewalk_fail(self):
definitions = {}
unparser = BaseUnparser(definitions)
# can't lookup an empty definition
with self.assertRaises(KeyError):
self.assertEqual([], list(unparser(Node())))
def parser(text):
result = Node()
result.raw = text
return result
def test_scope_creation(self):
scope = Scope(None)
node = Node()
scope.nest(node)
self.assertEqual(scope.children[0].node, node)
self.assertEqual(scope.children[0].parent, scope)