def test_simple(self): stream = BytesIO(simple_fast_import_stream) outf = StringIO() proc = info_processor.InfoProcessor(outf=outf) p = parser.ImportParser(stream) proc.process(p.iter_commands) self.maxDiff = None self.assertEqual(outf.getvalue(), """Command counts: \t0\tblob \t0\tcheckpoint \t1\tcommit \t0\tfeature \t0\tprogress \t0\treset \t0\ttag File command counts: \t0\tfilemodify \t0\tfiledelete \t0\tfilecopy \t0\tfilerename \t0\tfiledeleteall Parent counts: \t1\tparents-0 \t0\ttotal revisions merged Commit analysis: \tno\tblobs referenced by SHA \tno\texecutables \tno\tseparate authors found \tno\tsymlinks Head analysis: \t:1\trefs/heads/master Merges: """)
def test_done_without_feature(self): s = io.BytesIO(b"""done more data """) p = parser.ImportParser(s) cmds = p.iter_commands() self.assertEqual([], list(cmds))
def test_done_feature_missing_done(self): s = io.BytesIO(b"""feature done """) p = parser.ImportParser(s) cmds = p.iter_commands() self.assertEqual(b"feature", next(cmds).name) self.assertRaises(errors.PrematureEndOfStream, lambda: next(cmds))
def test_tagger_no_email_strict(self): p = parser.ImportParser( io.BytesIO(b"tag refs/tags/v1.0\n" b"from :xxx\n" b"tagger Joe Wong\n" b"data 11\n" b"create v1.0")) self.assertRaises(errors.BadFormat, list, p.iter_commands())
def assertFiltering(self, input_stream, params, expected): outf = BytesIO() proc = filter_processor.FilterProcessor(params=params) proc.outf = outf s = BytesIO(input_stream) p = parser.ImportParser(s) proc.process(p.iter_commands) out = outf.getvalue() self.assertEqual(expected, out)
def test_done_with_feature(self): s = io.BytesIO(b"""feature done done more data """) p = parser.ImportParser(s) cmds = p.iter_commands() self.assertEqual(b"feature", next(cmds).name) self.assertRaises(StopIteration, lambda: next(cmds))
def test_tagger_no_email_not_strict(self): p = parser.ImportParser(io.BytesIO(b"tag refs/tags/v1.0\n" b"from :xxx\n" b"tagger Joe Wong\n" b"data 11\n" b"create v1.0"), strict=False) cmds = list(p.iter_commands()) self.assertEqual(1, len(cmds)) self.assertTrue(isinstance(cmds[0], commands.TagCommand)) self.assertEqual(cmds[0].tagger[:2], (b'Joe Wong', None))
def assertHeads(self, input, expected): s = StringIO(input) p = parser.ImportParser(s) reftracker = RefTracker() for cmd in p.iter_commands(): if isinstance(cmd, commands.CommitCommand): reftracker.track_heads(cmd) # eat the file commands list(cmd.iter_files()) elif isinstance(cmd, commands.ResetCommand): if cmd.from_ is not None: reftracker.track_heads_for_ref(cmd.ref, cmd.from_) self.assertEqual(reftracker.heads, expected)
def test_tagger_with_email(self): p = parser.ImportParser( io.BytesIO(b"tag refs/tags/v1.0\n" b"from :xxx\n" b"tagger Joe Wong <*****@*****.**> 1234567890 -0600\n" b"data 11\n" b"create v1.0")) cmds = list(p.iter_commands()) self.assertEqual(1, len(cmds)) self.assertTrue(isinstance(cmds[0], commands.TagCommand)) self.assertEqual( cmds[0].tagger, (b'Joe Wong', b'*****@*****.**', 1234567890.0, -21600))
def _parse(self): if self.parsed: return for source in self.sources: if source == "-": infile = sys.stdin else: infile = open(source, 'rb') try: p = parser.ImportParser(infile) self.processor.process(p.iter_commands) finally: if infile is not sys.stdin: infile.close() self.parsed = True
def _generate_info(self, source): from io import StringIO from fastimport import parser from fastimport.errors import ParsingError from ...errors import CommandError from fastimport.processors import info_processor stream = _get_source_stream(source) output = StringIO() try: proc = info_processor.InfoProcessor(verbose=True, outf=output) p = parser.ImportParser(stream) try: return_code = proc.process(p.iter_commands) except ParsingError as e: raise CommandError("%d: Parse error: %s" % (e.lineno, e)) lines = output.getvalue().splitlines() finally: output.close() stream.seek(0) return lines
def _run(source, processor_factory, verbose=False, user_map=None, **kwargs): """Create and run a processor. :param source: a filename or '-' for standard input. If the filename ends in .gz, it will be opened as a gzip file and the stream will be implicitly uncompressed :param processor_factory: a callable for creating a processor :param user_map: if not None, the file containing the user map. """ from fastimport.errors import ParsingError from ...errors import CommandError from fastimport import parser stream = _get_source_stream(source) user_mapper = _get_user_mapper(user_map) proc = processor_factory(verbose=verbose, **kwargs) p = parser.ImportParser(stream, verbose=verbose, user_mapper=user_mapper) try: return proc.process(p.iter_commands) except ParsingError as e: raise CommandError("%d: Parse error: %s" % (e.lineno, e))
def import_stream(self, stream): p = parser.ImportParser(stream) self.process(p.iter_commands) return self.markers
def test_iter_commands(self): s = io.BytesIO(_sample_import_text) p = parser.ImportParser(s) result = [] for cmd in p.iter_commands(): result.append(cmd) if cmd.name == b'commit': for fc in cmd.iter_files(): result.append(fc) self.assertEqual(len(result), 17) cmd1 = result.pop(0) self.assertEqual(b'progress', cmd1.name) self.assertEqual(b'completed', cmd1.message) cmd2 = result.pop(0) self.assertEqual(b'blob', cmd2.name) self.assertEqual(b'1', cmd2.mark) self.assertEqual(b':1', cmd2.id) self.assertEqual(b'aaaa', cmd2.data) self.assertEqual(4, cmd2.lineno) cmd3 = result.pop(0) self.assertEqual(b'blob', cmd3.name) self.assertEqual(b'@7', cmd3.id) self.assertEqual(None, cmd3.mark) self.assertEqual(b'bbbbb', cmd3.data) self.assertEqual(7, cmd3.lineno) cmd4 = result.pop(0) self.assertEqual(b'commit', cmd4.name) self.assertEqual(b'2', cmd4.mark) self.assertEqual(b':2', cmd4.id) self.assertEqual(b'initial import', cmd4.message) self.assertEqual((b'bugs bunny', b'*****@*****.**', self.fake_time, 0), cmd4.committer) # namedtuple attributes self.assertEqual(b'bugs bunny', cmd4.committer.name) self.assertEqual(b'*****@*****.**', cmd4.committer.email) self.assertEqual(self.fake_time, cmd4.committer.timestamp) self.assertEqual(0, cmd4.committer.timezone) self.assertEqual(None, cmd4.author) self.assertEqual(11, cmd4.lineno) self.assertEqual(b'refs/heads/master', cmd4.ref) self.assertEqual(None, cmd4.from_) self.assertEqual([], cmd4.merges) file_cmd1 = result.pop(0) self.assertEqual(b'filemodify', file_cmd1.name) self.assertEqual(b'README', file_cmd1.path) self.assertEqual(0o100644, file_cmd1.mode) self.assertEqual(b'Welcome from bugs\n', file_cmd1.data) cmd5 = result.pop(0) self.assertEqual(b'commit', cmd5.name) self.assertEqual(None, cmd5.mark) self.assertEqual(b'@19', cmd5.id) self.assertEqual(b'second commit', cmd5.message) self.assertEqual((b'', b'*****@*****.**', self.fake_time, 0), cmd5.committer) self.assertEqual(None, cmd5.author) self.assertEqual(19, cmd5.lineno) self.assertEqual(b'refs/heads/master', cmd5.ref) self.assertEqual(b':2', cmd5.from_) self.assertEqual([], cmd5.merges) file_cmd2 = result.pop(0) self.assertEqual(b'filemodify', file_cmd2.name) self.assertEqual(b'README', file_cmd2.path) self.assertEqual(0o100644, file_cmd2.mode) self.assertEqual(b'Welcome from bugs, etc.', file_cmd2.data) cmd6 = result.pop(0) self.assertEqual(cmd6.name, b'checkpoint') cmd7 = result.pop(0) self.assertEqual(b'progress', cmd7.name) self.assertEqual(b'completed', cmd7.message) cmd = result.pop(0) self.assertEqual(b'commit', cmd.name) self.assertEqual(b'3', cmd.mark) self.assertEqual(None, cmd.from_) cmd = result.pop(0) self.assertEqual(b'commit', cmd.name) self.assertEqual(b'4', cmd.mark) self.assertEqual(b'Commit with heredoc-style message\n', cmd.message) cmd = result.pop(0) self.assertEqual(b'commit', cmd.name) self.assertEqual(b'5', cmd.mark) self.assertEqual(b'submodule test\n', cmd.message) file_cmd1 = result.pop(0) self.assertEqual(b'filemodify', file_cmd1.name) self.assertEqual(b'tree-id', file_cmd1.path) self.assertEqual(0o160000, file_cmd1.mode) self.assertEqual(b"rev-id", file_cmd1.dataref) cmd = result.pop(0) self.assertEqual(b'feature', cmd.name) self.assertEqual(b'whatever', cmd.feature_name) self.assertEqual(None, cmd.value) cmd = result.pop(0) self.assertEqual(b'feature', cmd.name) self.assertEqual(b'foo', cmd.feature_name) self.assertEqual(b'bar', cmd.value) cmd = result.pop(0) self.assertEqual(b'commit', cmd.name) self.assertEqual(b'6', cmd.mark) self.assertEqual(b'test of properties', cmd.message) self.assertEqual( { b'p1': None, b'p2': b'hohum', b'p3': b'alpha\nbeta\ngamma', b'p4': b'whatever', }, cmd.properties) cmd = result.pop(0) self.assertEqual(b'commit', cmd.name) self.assertEqual(b'7', cmd.mark) self.assertEqual(b'multi-author test', cmd.message) self.assertEqual(b'', cmd.committer[0]) self.assertEqual(b'*****@*****.**', cmd.committer[1]) self.assertEqual(b'Fluffy', cmd.author[0]) self.assertEqual(b'*****@*****.**', cmd.author[1]) self.assertEqual(b'Daffy', cmd.more_authors[0][0]) self.assertEqual(b'*****@*****.**', cmd.more_authors[0][1]) self.assertEqual(b'Donald', cmd.more_authors[1][0]) self.assertEqual(b'*****@*****.**', cmd.more_authors[1][1])
def test_path_pair_simple(self): p = parser.ImportParser(b'') self.assertEqual([b'foo', b'bar'], p._path_pair(b'foo bar'))
def test_path_pair_spaces_in_first(self): p = parser.ImportParser("") self.assertEqual([b'foo bar', b'baz'], p._path_pair(b'"foo bar" baz'))
- def __init__(self, ui, repo, sources): + def __init__(self, ui, repo, sources, fallback_message_encoding, compress, blob_tree_depth, blob_tree): self.ui = ui self.sources = sources - self.processor = HgImportProcessor(ui, repo) + self.processor = HgImportProcessor(ui, repo, compress, blob_tree_depth, blob_tree) self.parsed = False + self.fallback_message_encoding = fallback_message_encoding # converter_source methods @@ -138,7 +147,7 @@ class fastimport_source(common.converter else: infile = open(source, 'rb') try: - p = parser.ImportParser(infile) + p = parser.ImportParser(infile, message_fallback_encoding = self.fallback_message_encoding) self.processor.process(p.iter_commands) finally: if infile is not sys.stdin: @@ -150,7 +159,7 @@ class HgImportProcessor(processor.Import tagprefix = "refs/tags/" - def __init__(self, ui, repo): + def __init__(self, ui, repo, compress, blob_tree_depth, blob_tree): super(HgImportProcessor, self).__init__() self.ui = ui self.repo = repo @@ -168,6 +177,10 @@ class HgImportProcessor(processor.Import self.numblobs = 0 # for progress reporting