def test_simple(self):
stream = BytesIO(simple_fast_import_stream)
outf = StringIO()
proc = info_processor.InfoProcessor(outf=outf)
p = parser.ImportParser(stream)
proc.process(p.iter_commands)
self.maxDiff = None
self.assertEqual(outf.getvalue(), """Command counts:
\t0\tblob
\t0\tcheckpoint
\t1\tcommit
\t0\tfeature
\t0\tprogress
\t0\treset
\t0\ttag
File command counts:
\t0\tfilemodify
\t0\tfiledelete
\t0\tfilecopy
\t0\tfilerename
\t0\tfiledeleteall
Parent counts:
\t1\tparents-0
\t0\ttotal revisions merged
Commit analysis:
\tno\tblobs referenced by SHA
\tno\texecutables
\tno\tseparate authors found
\tno\tsymlinks
Head analysis:
\t:1\trefs/heads/master
Merges:
""")
def test_done_without_feature(self):
s = io.BytesIO(b"""done
more data
""")
p = parser.ImportParser(s)
cmds = p.iter_commands()
self.assertEqual([], list(cmds))
def test_done_feature_missing_done(self):
s = io.BytesIO(b"""feature done
""")
p = parser.ImportParser(s)
cmds = p.iter_commands()
self.assertEqual(b"feature", next(cmds).name)
self.assertRaises(errors.PrematureEndOfStream, lambda: next(cmds))
def test_tagger_no_email_strict(self):
p = parser.ImportParser(
io.BytesIO(b"tag refs/tags/v1.0\n"
b"from :xxx\n"
b"tagger Joe Wong\n"
b"data 11\n"
b"create v1.0"))
self.assertRaises(errors.BadFormat, list, p.iter_commands())
def assertFiltering(self, input_stream, params, expected):
outf = BytesIO()
proc = filter_processor.FilterProcessor(params=params)
proc.outf = outf
s = BytesIO(input_stream)
p = parser.ImportParser(s)
proc.process(p.iter_commands)
out = outf.getvalue()
self.assertEqual(expected, out)
def test_done_with_feature(self):
s = io.BytesIO(b"""feature done
done
more data
""")
p = parser.ImportParser(s)
cmds = p.iter_commands()
self.assertEqual(b"feature", next(cmds).name)
self.assertRaises(StopIteration, lambda: next(cmds))
def test_tagger_no_email_not_strict(self):
p = parser.ImportParser(io.BytesIO(b"tag refs/tags/v1.0\n"
b"from :xxx\n"
b"tagger Joe Wong\n"
b"data 11\n"
b"create v1.0"),
strict=False)
cmds = list(p.iter_commands())
self.assertEqual(1, len(cmds))
self.assertTrue(isinstance(cmds[0], commands.TagCommand))
self.assertEqual(cmds[0].tagger[:2], (b'Joe Wong', None))
def assertHeads(self, input, expected):
s = StringIO(input)
p = parser.ImportParser(s)
reftracker = RefTracker()
for cmd in p.iter_commands():
if isinstance(cmd, commands.CommitCommand):
reftracker.track_heads(cmd)
# eat the file commands
list(cmd.iter_files())
elif isinstance(cmd, commands.ResetCommand):
if cmd.from_ is not None:
reftracker.track_heads_for_ref(cmd.ref, cmd.from_)
self.assertEqual(reftracker.heads, expected)
def test_tagger_with_email(self):
p = parser.ImportParser(
io.BytesIO(b"tag refs/tags/v1.0\n"
b"from :xxx\n"
b"tagger Joe Wong <*****@*****.**> 1234567890 -0600\n"
b"data 11\n"
b"create v1.0"))
cmds = list(p.iter_commands())
self.assertEqual(1, len(cmds))
self.assertTrue(isinstance(cmds[0], commands.TagCommand))
self.assertEqual(
cmds[0].tagger,
(b'Joe Wong', b'*****@*****.**', 1234567890.0, -21600))
def _parse(self):
if self.parsed:
return
for source in self.sources:
if source == "-":
infile = sys.stdin
else:
infile = open(source, 'rb')
try:
p = parser.ImportParser(infile)
self.processor.process(p.iter_commands)
finally:
if infile is not sys.stdin:
infile.close()
self.parsed = True
def _generate_info(self, source):
from io import StringIO
from fastimport import parser
from fastimport.errors import ParsingError
from ...errors import CommandError
from fastimport.processors import info_processor
stream = _get_source_stream(source)
output = StringIO()
try:
proc = info_processor.InfoProcessor(verbose=True, outf=output)
p = parser.ImportParser(stream)
try:
return_code = proc.process(p.iter_commands)
except ParsingError as e:
raise CommandError("%d: Parse error: %s" % (e.lineno, e))
lines = output.getvalue().splitlines()
finally:
output.close()
stream.seek(0)
return lines
def _run(source, processor_factory, verbose=False, user_map=None, **kwargs):
"""Create and run a processor.
:param source: a filename or '-' for standard input. If the
filename ends in .gz, it will be opened as a gzip file and
the stream will be implicitly uncompressed
:param processor_factory: a callable for creating a processor
:param user_map: if not None, the file containing the user map.
"""
from fastimport.errors import ParsingError
from ...errors import CommandError
from fastimport import parser
stream = _get_source_stream(source)
user_mapper = _get_user_mapper(user_map)
proc = processor_factory(verbose=verbose, **kwargs)
p = parser.ImportParser(stream, verbose=verbose, user_mapper=user_mapper)
try:
return proc.process(p.iter_commands)
except ParsingError as e:
raise CommandError("%d: Parse error: %s" % (e.lineno, e))
def import_stream(self, stream):
p = parser.ImportParser(stream)
self.process(p.iter_commands)
return self.markers
def test_iter_commands(self):
s = io.BytesIO(_sample_import_text)
p = parser.ImportParser(s)
result = []
for cmd in p.iter_commands():
result.append(cmd)
if cmd.name == b'commit':
for fc in cmd.iter_files():
result.append(fc)
self.assertEqual(len(result), 17)
cmd1 = result.pop(0)
self.assertEqual(b'progress', cmd1.name)
self.assertEqual(b'completed', cmd1.message)
cmd2 = result.pop(0)
self.assertEqual(b'blob', cmd2.name)
self.assertEqual(b'1', cmd2.mark)
self.assertEqual(b':1', cmd2.id)
self.assertEqual(b'aaaa', cmd2.data)
self.assertEqual(4, cmd2.lineno)
cmd3 = result.pop(0)
self.assertEqual(b'blob', cmd3.name)
self.assertEqual(b'@7', cmd3.id)
self.assertEqual(None, cmd3.mark)
self.assertEqual(b'bbbbb', cmd3.data)
self.assertEqual(7, cmd3.lineno)
cmd4 = result.pop(0)
self.assertEqual(b'commit', cmd4.name)
self.assertEqual(b'2', cmd4.mark)
self.assertEqual(b':2', cmd4.id)
self.assertEqual(b'initial import', cmd4.message)
self.assertEqual((b'bugs bunny', b'*****@*****.**', self.fake_time, 0),
cmd4.committer)
# namedtuple attributes
self.assertEqual(b'bugs bunny', cmd4.committer.name)
self.assertEqual(b'*****@*****.**', cmd4.committer.email)
self.assertEqual(self.fake_time, cmd4.committer.timestamp)
self.assertEqual(0, cmd4.committer.timezone)
self.assertEqual(None, cmd4.author)
self.assertEqual(11, cmd4.lineno)
self.assertEqual(b'refs/heads/master', cmd4.ref)
self.assertEqual(None, cmd4.from_)
self.assertEqual([], cmd4.merges)
file_cmd1 = result.pop(0)
self.assertEqual(b'filemodify', file_cmd1.name)
self.assertEqual(b'README', file_cmd1.path)
self.assertEqual(0o100644, file_cmd1.mode)
self.assertEqual(b'Welcome from bugs\n', file_cmd1.data)
cmd5 = result.pop(0)
self.assertEqual(b'commit', cmd5.name)
self.assertEqual(None, cmd5.mark)
self.assertEqual(b'@19', cmd5.id)
self.assertEqual(b'second commit', cmd5.message)
self.assertEqual((b'', b'*****@*****.**', self.fake_time, 0),
cmd5.committer)
self.assertEqual(None, cmd5.author)
self.assertEqual(19, cmd5.lineno)
self.assertEqual(b'refs/heads/master', cmd5.ref)
self.assertEqual(b':2', cmd5.from_)
self.assertEqual([], cmd5.merges)
file_cmd2 = result.pop(0)
self.assertEqual(b'filemodify', file_cmd2.name)
self.assertEqual(b'README', file_cmd2.path)
self.assertEqual(0o100644, file_cmd2.mode)
self.assertEqual(b'Welcome from bugs, etc.', file_cmd2.data)
cmd6 = result.pop(0)
self.assertEqual(cmd6.name, b'checkpoint')
cmd7 = result.pop(0)
self.assertEqual(b'progress', cmd7.name)
self.assertEqual(b'completed', cmd7.message)
cmd = result.pop(0)
self.assertEqual(b'commit', cmd.name)
self.assertEqual(b'3', cmd.mark)
self.assertEqual(None, cmd.from_)
cmd = result.pop(0)
self.assertEqual(b'commit', cmd.name)
self.assertEqual(b'4', cmd.mark)
self.assertEqual(b'Commit with heredoc-style message\n', cmd.message)
cmd = result.pop(0)
self.assertEqual(b'commit', cmd.name)
self.assertEqual(b'5', cmd.mark)
self.assertEqual(b'submodule test\n', cmd.message)
file_cmd1 = result.pop(0)
self.assertEqual(b'filemodify', file_cmd1.name)
self.assertEqual(b'tree-id', file_cmd1.path)
self.assertEqual(0o160000, file_cmd1.mode)
self.assertEqual(b"rev-id", file_cmd1.dataref)
cmd = result.pop(0)
self.assertEqual(b'feature', cmd.name)
self.assertEqual(b'whatever', cmd.feature_name)
self.assertEqual(None, cmd.value)
cmd = result.pop(0)
self.assertEqual(b'feature', cmd.name)
self.assertEqual(b'foo', cmd.feature_name)
self.assertEqual(b'bar', cmd.value)
cmd = result.pop(0)
self.assertEqual(b'commit', cmd.name)
self.assertEqual(b'6', cmd.mark)
self.assertEqual(b'test of properties', cmd.message)
self.assertEqual(
{
b'p1': None,
b'p2': b'hohum',
b'p3': b'alpha\nbeta\ngamma',
b'p4': b'whatever',
}, cmd.properties)
cmd = result.pop(0)
self.assertEqual(b'commit', cmd.name)
self.assertEqual(b'7', cmd.mark)
self.assertEqual(b'multi-author test', cmd.message)
self.assertEqual(b'', cmd.committer[0])
self.assertEqual(b'*****@*****.**', cmd.committer[1])
self.assertEqual(b'Fluffy', cmd.author[0])
self.assertEqual(b'*****@*****.**', cmd.author[1])
self.assertEqual(b'Daffy', cmd.more_authors[0][0])
self.assertEqual(b'*****@*****.**', cmd.more_authors[0][1])
self.assertEqual(b'Donald', cmd.more_authors[1][0])
self.assertEqual(b'*****@*****.**', cmd.more_authors[1][1])
def test_path_pair_simple(self):
p = parser.ImportParser(b'')
self.assertEqual([b'foo', b'bar'], p._path_pair(b'foo bar'))
def test_path_pair_spaces_in_first(self):
p = parser.ImportParser("")
self.assertEqual([b'foo bar', b'baz'], p._path_pair(b'"foo bar" baz'))
- def __init__(self, ui, repo, sources):
+ def __init__(self, ui, repo, sources, fallback_message_encoding, compress, blob_tree_depth, blob_tree):
self.ui = ui
self.sources = sources
- self.processor = HgImportProcessor(ui, repo)
+ self.processor = HgImportProcessor(ui, repo, compress, blob_tree_depth, blob_tree)
self.parsed = False
+ self.fallback_message_encoding = fallback_message_encoding
# converter_source methods
@@ -138,7 +147,7 @@ class fastimport_source(common.converter
else:
infile = open(source, 'rb')
try:
- p = parser.ImportParser(infile)
+ p = parser.ImportParser(infile, message_fallback_encoding = self.fallback_message_encoding)
self.processor.process(p.iter_commands)
finally:
if infile is not sys.stdin:
@@ -150,7 +159,7 @@ class HgImportProcessor(processor.Import
tagprefix = "refs/tags/"
- def __init__(self, ui, repo):
+ def __init__(self, ui, repo, compress, blob_tree_depth, blob_tree):
super(HgImportProcessor, self).__init__()
self.ui = ui
self.repo = repo
@@ -168,6 +177,10 @@ class HgImportProcessor(processor.Import
self.numblobs = 0 # for progress reporting
