def test_source_map_inferred_standard_newline(self):
# for cases where pretty printing happened
# e.g. (function() { console.log("hello world"); })()
err = setup_logger(self, sourcemap.logger)
stream = StringIO()
fragments = [
('(', 1, 1, None, None),
('function', 1, 2, None, None),
('(', 1, 10, None, None),
(') ', 1, 11, None, None),
('{\n', 1, 13, None, None),
# may be another special case here, to normalize the _first_
# fragment
(' ', None, None, None, None),
('console', 1, 15, None, 'demo.js'),
('.', 1, 22, None, None),
('log', 1, 23, None, None),
('(', 0, 0, None, None),
('"hello world"', 1, 27, None, None),
(')', 0, 0, None, None),
(';\n', 0, 0, None, None),
# note that the AST will need to record/provide the ending
# value to this if the usage on a newline is to be supported
# like so, otherwise all unmarked symbols will be clobbered
# in an indeterminate manner, given that there could be
# arbitrary amount of white spaces before the following
# ending characters in the original source text.
# In other words, if '}' is not tagged with (1,43), (or the
# position that it was from, the generated source map will
# guaranteed to be wrong as the starting column cannot be
# correctly inferred without the original text.
('}', 1, 43, None, None), # this one cannot be inferred.
(')', 0, 0, None, None), # this one can be.
('(', 1, 45, None, None), # next starting symbol
(')', 1, 46, None, None),
(';', 0, 0, None, None),
]
mapping, _, names = sourcemap.write(fragments, stream, normalize=False)
self.assertEqual(stream.getvalue(), textwrap.dedent("""
(function() {
console.log("hello world");
})();
""").strip())
self.assertEqual(names, [])
self.assertEqual([[
(0, 0, 0, 0), (1, 0, 0, 1), (8, 0, 0, 8), (1, 0, 0, 1),
(2, 0, 0, 2)
], [
(0,), (2, 0, 0, 2), (7, 0, 0, 7), (1, 0, 0, 1), (3, 0, 0, 3),
(1, 0, 0, 1), (13, 0, 0, 13), (1, 0, 0, 1)
], [
(0, 0, 0, 2), (1, 0, 0, 1), (1, 0, 0, 1), (1, 0, 0, 1),
(1, 0, 0, 1)
]], mapping)
self.assertNotIn("WARNING", err.getvalue())
# the normalized version should also have the correct offsets
mapping, sources, names = sourcemap.write(fragments, stream)
self.assertEqual(
[[(0, 0, 0, 0)], [(2, 0, 0, 14)], [(0, 0, 0, 28)]], mapping)
self.assertEqual(sources, ['demo.js'])
def test_source_map_wrongly_inferred_initial_indents(self):
# it should still be able to correct it
# this is why indentation should simply be omitted and replace
# both the line/col counter to None
stream = StringIO()
fragments = [
(' ', 0, 0, None, None), # was two spaces, now single space.
(' ', 0, 0, None, None),
('console', 1, 5, None, None),
('.', 1, 12, None, None),
('log', 1, 13, None, None),
('(', 0, 0, None, None),
('(', 0, 0, None, None),
('"hello world"', 1, 18, None, None),
(')', 0, 0, None, None),
(')', 0, 0, None, None),
(';', 0, 0, None, None),
]
mapping, _, names = sourcemap.write(fragments, stream, normalize=False)
self.assertEqual(stream.getvalue(), ' console.log(("hello world"));')
self.assertEqual(names, [])
self.assertEqual(mapping, [[
(0, 0, 0, 0), (1, 0, 0, 1),
(1, 0, 0, 3), (7, 0, 0, 7), (1, 0, 0, 1), (3, 0, 0, 3),
(1, 0, 0, 1), (1, 0, 0, 1), (13, 0, 0, 13), (1, 0, 0, 1),
(1, 0, 0, 1),
]])
mapping, _, names = sourcemap.write(fragments, stream)
self.assertEqual(mapping, [
[(0, 0, 0, 0), (2, 0, 0, 4)],
])
def test_multiple_call(self):
stream = StringIO()
fragments1 = [
('a', 1, 1, 'console', 'demo1.js'),
('.', 1, 8, None, 'demo1.js'),
('log', 1, 9, None, 'demo1.js'),
('(', 1, 12, None, 'demo1.js'),
('"hello world"', 1, 13, None, 'demo1.js'),
(')', 1, 26, None, 'demo1.js'),
(';', 1, 27, None, 'demo1.js'),
]
fragments2 = [
('a', 1, 1, 'console', 'demo2.js'),
('.', 1, 8, None, 'demo2.js'),
('log', 1, 9, None, 'demo2.js'),
('(', 1, 12, None, 'demo2.js'),
('"hello world"', 1, 13, None, 'demo2.js'),
(')', 1, 26, None, 'demo2.js'),
(';', 1, 27, None, 'demo2.js'),
]
book = sourcemap.default_book()
sources = sourcemap.Names()
names = sourcemap.Names()
mappings, _, _ = sourcemap.write(
fragments1,
stream,
book=book,
sources=sources,
names=names,
# Note that normalization is NOT supported until the last
# step.
normalize=False,
)
mappings, sources, namelist = sourcemap.write(fragments2,
stream,
book=book,
sources=sources,
names=names,
mappings=mappings)
self.assertEqual(stream.getvalue(),
'a.log("hello world");a.log("hello world");')
self.assertEqual(namelist, ['console'])
self.assertEqual(sources, ['demo1.js', 'demo2.js'])
self.assertEqual(mappings, [
[(0, 0, 0, 0, 0), (1, 0, 0, 7), (20, 1, 0, -7, 0), (1, 0, 0, 7)],
])
def test_source_map_known_standard_newline(self):
# for cases where pretty printing happened
# e.g. (function() { console.log("hello world"); })()
# with all values known.
err = setup_logger(self, sourcemap.logger)
stream = StringIO()
fragments = [
('(', 1, 1, None, None),
('function', 1, 2, None, None),
('(', 1, 10, None, None),
(') ', 1, 11, None, None),
('{\n', 1, 13, None, None),
# may be another special case here, to normalize the _first_
# fragment
(' ', None, None, None, None),
('console', 1, 15, None, None),
('.', 1, 22, None, None),
('log', 1, 23, None, None),
('(', 1, 26, None, None),
('"hello world"', 1, 27, None, None),
(')', 1, 40, None, None),
(';\n', 1, 41, None, None),
('}', 1, 43, None, None),
(')', 1, 44, None, None),
('(', 1, 45, None, None),
(')', 1, 46, None, None),
(';', 1, 47, None, None),
]
mapping, _, names = sourcemap.write(fragments, stream, normalize=False)
self.assertEqual(stream.getvalue(), textwrap.dedent("""
(function() {
console.log("hello world");
})();
""").strip())
self.assertEqual(names, [])
self.assertEqual([[
(0, 0, 0, 0), (1, 0, 0, 1), (8, 0, 0, 8), (1, 0, 0, 1),
(2, 0, 0, 2)
], [
(0,), (2, 0, 0, 2), (7, 0, 0, 7), (1, 0, 0, 1), (3, 0, 0, 3),
(1, 0, 0, 1), (13, 0, 0, 13), (1, 0, 0, 1)
], [
(0, 0, 0, 2), (1, 0, 0, 1), (1, 0, 0, 1), (1, 0, 0, 1),
(1, 0, 0, 1)
]], mapping)
self.assertNotIn("WARNING", err.getvalue())
# the normalized version should also have the correct offsets
mapping, _, names = sourcemap.write(fragments, stream)
self.assertEqual(
[[(0, 0, 0, 0)], [(2, 0, 0, 14)], [(0, 0, 0, 28)]], mapping)
def test_source_map_remapped_symbols_without_original(self):
# for cases where the program have been wrapped and transpiled
# e.g. (function() { $program })()
# using: console log "hello world"
stream = StringIO()
fragments = [
('(', None, None, None, None),
('function', None, None, None, None),
('(', None, None, None, None),
(') ', None, None, None, None),
('{\n', None, None, None, None),
# may be another special case here, to normalize the _first_
# fragment
(' ', None, None, None, None),
('console', 1, 1, None, None),
('.', None, None, None, None),
('log', 1, 9, None, None),
('(', None, None, None, None),
('"hello world"', 1, 13, None, None),
(')', None, None, None, None),
(';', None, None, None, None),
('\n', None, None, None, None),
('}', None, None, None, None),
(')', None, None, None, None),
('(', 0, None, None, None), # testing for alternative conds.
(')', None, 0, None, None),
(';', None, None, None, None),
]
mapping, _, names = sourcemap.write(fragments, stream, normalize=False)
self.assertEqual(stream.getvalue(), textwrap.dedent("""
(function() {
console.log("hello world");
})();
""").strip())
self.assertEqual(names, [])
self.assertEqual([[
(0,), (1,), (8,), (1,), (2,),
], [
(0,),
(2, 0, 0, 0), (7,), (1, 0, 0, 8), (3,), (1, 0, 0, 4), (13,),
(1,), (1,),
], [
(0,), (1,), (1,), (1,), (1,),
]], mapping)
# the normalized version should also have the correct offsets
mapping, _, names = sourcemap.write(fragments, stream)
self.assertEqual([[], [
(2, 0, 0, 0), (7,), (1, 0, 0, 8), (3,), (1, 0, 0, 4), (13,),
], []], mapping)
def test_source_map_inferred_trailing_newline_unknown(self):
err = setup_logger(self, sourcemap.logger)
stream = StringIO()
# Note the None values, as that signifies inferred elements.
fragments = [
('console.log(\n "hello world");', 1, 1, None, None),
('/* foo\nbar */', 0, 0, None, None),
(' /* foo\nbar */', None, None, None, 'some_script.js'),
]
mapping, _, names = sourcemap.write(fragments, stream)
self.assertEqual(
stream.getvalue(),
'console.log(\n "hello world");/* foo\nbar */ /* foo\nbar */',
)
self.assertEqual(names, [])
self.assertEqual(
[
[(0, 0, 0, 0)],
[(0, 0, 1, 0)],
# yeah the comments really came out wrong it looks like
[(0, 0, 0, 17), (6, )],
[],
],
mapping)
self.assertIn(
"WARNING text in the generated stream at line 3 may be mapped "
"incorrectly due to stream fragment containing a trailing newline "
"character provided without both lineno and colno defined; "
"text fragment originated from: <unknown>", err.getvalue())
self.assertIn(
"WARNING text in the generated stream at line 4 may be mapped "
"incorrectly due to stream fragment containing a trailing newline "
"character provided without both lineno and colno defined; "
"text fragment originated from: some_script.js", err.getvalue())
def test_multiple_unmapped_chunks(self):
stream = StringIO()
fragments = [
(' ', None, None, None, None),
(' ', None, None, None, None),
(' ', None, None, None, None),
('x', 1, 1, None, None),
('x', 1, 1, None, None),
('x', 1, 1, None, None),
(' ', None, None, None, None),
(' ', None, None, None, None),
(' ', None, None, None, None),
('y', 1, 3, None, None),
('y', 1, 3, None, None),
('y', 1, 3, None, None),
]
mapping, _, names = sourcemap.write(fragments, stream)
self.assertEqual(stream.getvalue(), ' xxx yyy')
self.assertEqual(names, [])
self.assertEqual(mapping, [[
(3, 0, 0, 0),
(1, 0, 0, 0),
(1, 0, 0, 0),
(1,),
(3, 0, 0, 2),
(1, 0, 0, 0),
(1, 0, 0, 0),
]])
def test_source_map_source_none_then_not_implemented_then_named(self):
stream = StringIO()
fragments = [
('console', 1, 1, None, None),
('.', 1, 8, None, NotImplemented),
('log', 1, 9, None, 'named.js'),
('(', 0, 0, None, None),
('"hello world"', 1, 13, None, None),
(')', 0, 0, None, None),
(';', 0, 0, None, NotImplemented),
]
mapping, sources, names = sourcemap.write(fragments, stream)
self.assertEqual(stream.getvalue(), 'console.log("hello world");')
self.assertEqual(sources, ['about:invalid', 'named.js'])
self.assertEqual(names, [])
self.assertEqual(mapping, [
# the first None implied value converted to invalid, then
# the named.js bumps it up 1, then backtracked back to 0 at
# the end.
# Yes, the example is a bit contrived, due to how the
# positions are interweaved between the two files.
[(0, 0, 0, 0), (8, 1, 0, 8), (18, -1, 0, 18)],
])
def test_source_map_inferred_trailing_newline_calculated(self):
err = setup_logger(self, sourcemap.logger)
stream = StringIO()
# Note the None values, as that signifies inferred elements.
fragments = [
('console.log(\n "hello world");', 1, 1, None, None),
]
mapping, _, names = sourcemap.write(fragments, stream)
self.assertEqual(stream.getvalue(), 'console.log(\n "hello world");')
self.assertEqual(names, [])
self.assertEqual([
[(0, 0, 0, 0)],
[(0, 0, 1, 0)],
], mapping)
self.assertNotIn("WARNING", err.getvalue())
def test_source_map_renamed(self):
stream = StringIO()
fragments = [
('a', 1, 1, 'console', None),
('.', 1, 8, None, None),
('b', 1, 9, 'log', None),
('(', 1, 12, None, None),
('"hello world"', 1, 13, None, None),
(')', 1, 26, None, None),
(';', 1, 27, None, None),
]
mapping, _, names = sourcemap.write(fragments, stream)
self.assertEqual(stream.getvalue(), 'a.b("hello world");')
self.assertEqual(names, ['console', 'log'])
self.assertEqual(mapping, [[(0, 0, 0, 0, 0), (1, 0, 0, 7),
(1, 0, 0, 1, 1), (1, 0, 0, 3)]])
def test_source_map_inferred_row_offset(self):
stream = StringIO()
# Note that the first row is 3.
fragments = [
('var', 3, 1, None, None),
(' ', 0, 0, None, None),
('main', 3, 5, None, None),
(';', 0, 0, None, None),
]
mapping, sources, names = sourcemap.write(
fragments, stream, normalize=False)
self.assertEqual(stream.getvalue(), 'var main;')
self.assertEqual(names, [])
self.assertEqual(sources, ['about:invalid'])
self.assertEqual(mapping, [
[(0, 0, 2, 0), (3, 0, 0, 3), (1, 0, 0, 1), (4, 0, 0, 4)],
])
def test_track_shrunk_name(self):
stream = StringIO()
# 123456789012345678901234567
# console.log("hello world");
fragments = [
('print', 1, 1, 'console.log', 'original.js'),
('(', 0, 0, None, None),
('"hello world"', 1, 13, None, None),
(')', 0, 0, None, None),
]
mapping, sources, names = sourcemap.write(fragments, stream)
self.assertEqual(stream.getvalue(), 'print("hello world")')
self.assertEqual(names, ['console.log'])
self.assertEqual(sources, ['original.js'])
self.assertEqual(mapping, [
[(0, 0, 0, 0, 0), (5, 0, 0, 11)],
])
def test_track_expanded_name(self):
stream = StringIO()
# 12345678901234567890
# print("hello world")
fragments = [
('console.log', 0, 0, 'print', 'original.py'),
('(', 0, 0, None, None),
('"hello world"', 1, 7, None, None),
(')', 0, 0, None, None),
(';', None, None, None, None),
]
mapping, sources, names = sourcemap.write(fragments, stream)
self.assertEqual(stream.getvalue(), 'console.log("hello world");')
self.assertEqual(names, ['print'])
self.assertEqual(sources, ['original.py'])
self.assertEqual(mapping, [
[(0, 0, 0, 0, 0), (11, 0, 0, 5), (15,)],
])
def test_source_map_mapped_initial_indent(self):
stream = StringIO()
fragments = [
(' ', 1, 1, None, None),
('console', 1, 3, None, None),
('.', 1, 10, None, None),
('log', 1, 11, None, None),
('(', 0, 0, None, None),
('"hello world"', 1, 15, None, None),
(')', 0, 0, None, None),
(';', 0, 0, None, None),
]
mapping, _, names = sourcemap.write(fragments, stream)
self.assertEqual(stream.getvalue(), ' console.log("hello world");')
self.assertEqual(names, [])
self.assertEqual(mapping, [
[(0, 0, 0, 0)],
])
def test_source_map_basic(self):
stream = StringIO()
fragments = [
('console', 1, 1, None, 'demo.js'),
('.', 1, 8, None, 'demo.js'),
('log', 1, 9, None, 'demo.js'),
('(', 1, 12, None, 'demo.js'),
('"hello world"', 1, 13, None, 'demo.js'),
(')', 1, 26, None, 'demo.js'),
(';', 1, 27, None, 'demo.js'),
]
mapping, sources, names = sourcemap.write(fragments, stream)
self.assertEqual(stream.getvalue(), 'console.log("hello world");')
self.assertEqual(sources, ['demo.js'])
self.assertEqual(names, [])
self.assertEqual(mapping, [
[(0, 0, 0, 0)],
])
def test_source_map_source_not_implemented(self):
stream = StringIO()
fragments = [
('console', 1, 1, None, NotImplemented),
('.', 1, 8, None, None),
('log', 1, 9, None, None),
('(', 0, 0, None, None),
('"hello world"', 1, 13, None, None),
(')', 0, 0, None, None),
(';', 0, 0, None, None),
]
mapping, sources, names = sourcemap.write(fragments, stream)
self.assertEqual(stream.getvalue(), 'console.log("hello world");')
self.assertEqual(sources, ['about:invalid'])
self.assertEqual(names, [])
self.assertEqual(mapping, [
[(0, 0, 0, 0)],
])
def test_donottrack_names_source_of_dropped(self):
stream = StringIO()
fragments = [
(' ', None, None, '\t', 'original.py'),
('console', None, None, 'print', 'nowhere.js'),
('.', None, None, 'dot', 'somewhere.js'),
('log', 1, 1, 'print', 'original.py'),
('(', 0, 0, None, None),
('"hello world"', 1, 7, None, None),
(')', 0, 0, None, None),
(';', None, None, None, None),
]
mapping, sources, names = sourcemap.write(fragments, stream)
self.assertEqual(stream.getvalue(), ' console.log("hello world");')
self.assertEqual(names, ['print'])
self.assertEqual(sources, ['original.py'])
self.assertEqual(mapping, [
[(10, 0, 0, 0, 0), (3, 0, 0, 5), (15,)],
])
def test_source_map_inferred(self):
stream = StringIO()
# Note the 0 values, as that signifies inferred elements for the
# row/col, and then None for source will be replaced with an
# 'about:invalid' source.
fragments = [
('console', 1, 1, None, None),
('.', 1, 8, None, None),
('log', 1, 9, None, None),
('(', 0, 0, None, None),
('"hello world"', 1, 13, None, None),
(')', 0, 0, None, None),
(';', 0, 0, None, None),
]
mapping, sources, names = sourcemap.write(fragments, stream)
self.assertEqual(stream.getvalue(), 'console.log("hello world");')
self.assertEqual(sources, ['about:invalid'])
self.assertEqual(names, [])
self.assertEqual(mapping, [
[(0, 0, 0, 0)],
])
def write(unparser,
nodes,
output_stream,
sourcemap_stream=None,
sourcemap_normalize_mappings=True,
sourcemap_normalize_paths=True,
source_mapping_url=NotImplemented):
"""
Write out the node using the unparser into an output stream, and
optionally the sourcemap using the sourcemap stream.
Ideally, file objects should be passed to the *_stream arguments, so
that the name resolution built into the sourcemap builder function
will be used. Also, if these file objects are opened using absolute
path arguments, enabling the sourcemap_normalize_paths flag will
have all paths normalized to their relative form.
If the provided streams are not anchored on the filesystem, or that
the provide node was generated from a string or in-memory stream,
the generation of the sourcemap should be done using the lower level
`write` function provided by the sourcemap module, which this method
wraps. Alternatively, the top level node should have its sourcepath
set to path that this node originated from.
Arguments
unparser
An unparser instance.
nodes
The Node or list of Nodes to stream to the output stream with
the unparser.
output_stream
Either a stream object or a callable that produces one. The
stream object to write to; its 'write' method will be invoked.
If a callable was provided, the 'close' method on its return
value will be called to close the stream.
sourcemap_stream
If one is provided, the sourcemap will be written out to it.
Like output_stream, it could also be a callable and be handled
in the same manner.
If this argument is the same as output_stream (note: the return
value of any callables are not compared), the stream object that
is the same as the output_stream will be used for writing out
the source map, and the source map will instead be encoded as a
'data:application/json;base64,' URL.
sourcemap_normalize_mappings
Flag for the normalization of the sourcemap mappings; Defaults
to True to enable a reduction in output size.
sourcemap_normalize_paths
If set to true, all absolute paths will be converted to the
relative form when the sourcemap is generated, if all paths
provided are in the absolute form.
Defaults to True to enable a reduction in output size.
source_mapping_url
If unspecified, the default derived path will be written as a
sourceMappingURL comment into the output stream. If explicitly
specified with a value, that will be written instead. Set to
None to disable this.
"""
closer = []
def get_stream(stream):
if callable(stream):
result = stream()
closer.append(result.close)
else:
result = stream
return result
def cleanup():
for close in reversed(closer):
close()
chunks = None
if isinstance(nodes, Node):
chunks = unparser(nodes)
elif isinstance(nodes, Iterable):
raw = [unparser(node) for node in nodes if isinstance(node, Node)]
if raw:
chunks = chain(*raw)
if not chunks:
raise TypeError('must either provide a Node or list containing Nodes')
try:
out_s = get_stream(output_stream)
sourcemap_stream = (out_s if sourcemap_stream is output_stream else
sourcemap_stream)
mappings, sources, names = sourcemap.write(
chunks, out_s, normalize=sourcemap_normalize_mappings)
if sourcemap_stream:
sourcemap_stream = get_stream(sourcemap_stream)
sourcemap.write_sourcemap(
mappings,
sources,
names,
out_s,
sourcemap_stream,
normalize_paths=sourcemap_normalize_paths,
source_mapping_url=source_mapping_url,
)
finally:
cleanup()
