def verify_heads(ui,repo,cache,force):
branches=repo.branchtags()
l=[(-repo.changelog.rev(n), n, t) for t, n in branches.items()]
l.sort()
# get list of hg's branches to verify, don't take all git has
for _,_,b in l:
b=get_branch(b)
sha1=get_git_sha1(b)
c=cache.get(b)
if sha1!=c:
sys.stderr.write('Error: Branch [%s] modified outside hg-fast-export:'
'\n%s (repo) != %s (cache)\n' % (b,sha1,c))
if not force: return False
# verify that branch has exactly one head
t={}
for h in repo.heads():
(_,_,_,_,_,_,branch,_)=get_changeset(ui,repo,h)
if t.get(branch,False):
sys.stderr.write('Error: repository has at least one unnamed head: hg r%s\n' %
repo.changelog.rev(h))
if not force: return False
t[branch]=True
return True
def verify_heads(ui, repo, cache, force, branchesmap):
branches = {}
for bn, heads in repo.branchmap().iteritems():
branches[bn] = branchtip(repo, heads)
l = [(-repo.changelog.rev(n), n, t) for t, n in branches.items()]
l.sort()
# get list of hg's branches to verify, don't take all git has
for _, _, b in l:
b = get_branch(b)
sanitized_name = sanitize_name(b, "branch", branchesmap)
sha1 = get_git_sha1(sanitized_name)
c = cache.get(sanitized_name)
if sha1 != c:
sys.stderr.write(
"Error: Branch [%s] modified outside hg-fast-export:" "\n%s (repo) != %s (cache)\n" % (b, sha1, c)
)
if not force:
return False
# verify that branch has exactly one head
t = {}
for h in repo.heads():
(_, _, _, _, _, _, branch, _) = get_changeset(ui, repo, h)
if t.get(branch, False):
sys.stderr.write("Error: repository has at least one unnamed head: hg r%s\n" % repo.changelog.rev(h))
if not force:
return False
t[branch] = True
return True
def verify_heads(ui, repo, cache, force, ignore_unnamed_heads, branchesmap):
branches = {}
for bn, heads in repo.branchmap().iteritems():
branches[bn] = branchtip(repo, heads)
l = [(-repo.changelog.rev(n), n, t) for t, n in branches.items()]
l.sort()
# get list of hg's branches to verify, don't take all git has
for _, _, b in l:
b = get_branch(b)
sanitized_name = sanitize_name(b, "branch", branchesmap)
sha1 = get_git_sha1(sanitized_name)
c = cache.get(sanitized_name)
if sha1 != c:
stderr_buffer.write(
b'Error: Branch [%s] modified outside hg-fast-export:'
b'\n%s (repo) != %s (cache)\n' %
(b, b'<None>' if sha1 is None else sha1, c))
if not force: return False
# verify that branch has exactly one head
t = {}
unnamed_heads = False
for h in repo.filtered(b'visible').heads():
(_, _, _, _, _, _, branch, _) = get_changeset(ui, repo, h)
if t.get(branch, False):
stderr_buffer.write(
b'Error: repository has an unnamed head: hg r%d\n' %
repo.changelog.rev(h))
unnamed_heads = True
if not force and not ignore_unnamed_heads: return False
t[branch] = True
if unnamed_heads and not force and not ignore_unnamed_heads: return False
return True
def verify_heads(ui,repo,cache,force,branchesmap, closed_branch_tips={}):
branches={}
for bn, heads in repo.branchmap().iteritems():
branches[bn] = branchtip(repo, heads)
l=[(-repo.changelog.rev(n), n, t) for t, n in branches.items()]
l.sort()
# get list of hg's branches to verify, don't take all git has
for _,_,b in l:
b=get_branch(b)
sanitized_name=sanitize_name(b,"branch",branchesmap)
sha1=get_git_sha1(sanitized_name)
c=cache.get(sanitized_name)
if sha1!=c:
sys.stderr.write('Error: Branch [%s] modified outside hg-fast-export:'
'\n%s (repo) != %s (cache)\n' % (b,sha1,c))
if not force: return False
# verify that branch has exactly one head
t={}
for h in repo.heads():
(node,_,_,_,_,desc,branch,_)=get_changeset(ui,repo,h)
if t.get(branch,False):
sys.stderr.write('Error: repository has at least one unnamed head: hg r%s\n' %
repo.changelog.rev(h))
# if not force: return False
sys.stderr.write(branch + '; ' + desc + '\n\n')
closed_branch_tips[node] = branch + '_closed'
t[branch]=True
return True
def hg2git(repourl,m,marksfile,mappingfile,headsfile,tipfile,
authors={},branchesmap={},tagsmap={},
sob=False,force=False,hgtags=False,notes=False,encoding='',fn_encoding=''):
def check_cache(filename, contents):
if len(contents) == 0:
sys.stderr.write('Warning: %s does not contain any data, this will probably make an incremental import fail\n' % filename)
_max=int(m)
old_marks=load_cache(marksfile,lambda s: int(s)-1)
mapping_cache=load_cache(mappingfile)
heads_cache=load_cache(headsfile)
state_cache=load_cache(tipfile)
if len(state_cache) != 0:
for (name, data) in [(marksfile, old_marks),
(mappingfile, mapping_cache),
(headsfile, state_cache)]:
check_cache(name, data)
ui,repo=setup_repo(repourl)
if not verify_heads(ui,repo,heads_cache,force,branchesmap):
return 1
try:
tip=repo.changelog.count()
except AttributeError:
tip=len(repo)
min=int(state_cache.get('tip',0))
max=_max
if _max<0 or max>tip:
max=tip
for rev in range(0,max):
(revnode,_,_,_,_,_,_,_)=get_changeset(ui,repo,rev,authors)
mapping_cache[revnode.encode('hex_codec')] = str(rev)
c=0
brmap={}
for rev in range(min,max):
c=export_commit(ui,repo,rev,old_marks,max,c,authors,branchesmap,
sob,brmap,hgtags,encoding,fn_encoding)
if notes:
for rev in range(min,max):
c=export_note(ui,repo,rev,c,authors, encoding, rev == min and min != 0)
state_cache['tip']=max
state_cache['repo']=repourl
save_cache(tipfile,state_cache)
save_cache(mappingfile,mapping_cache)
c=export_tags(ui,repo,old_marks,mapping_cache,c,authors,tagsmap)
sys.stderr.write('Issued %d commands\n' % c)
return 0
def hg2git(repourl,
m,
marksfile,
mappingfile,
headsfile,
tipfile,
authors={},
branchesmap={},
tagsmap={},
sob=False,
force=False,
hgtags=False,
notes=False,
encoding='',
fn_encoding=''):
_max = int(m)
old_marks = load_cache(marksfile, lambda s: int(s) - 1)
mapping_cache = load_cache(mappingfile)
heads_cache = load_cache(headsfile)
state_cache = load_cache(tipfile)
ui, repo = setup_repo(repourl)
if not verify_heads(ui, repo, heads_cache, force):
return 1
try:
tip = repo.changelog.count()
except AttributeError:
tip = len(repo)
min = int(state_cache.get('tip', 0))
max = _max
if _max < 0 or max > tip:
max = tip
for rev in range(0, max):
(revnode, _, _, _, _, _, _, _) = get_changeset(ui, repo, rev, authors)
mapping_cache[revnode.encode('hex_codec')] = str(rev)
c = 0
brmap = {}
for rev in range(min, max):
c = export_commit(ui, repo, rev, old_marks, max, c, authors,
branchesmap, sob, brmap, hgtags, notes, encoding,
fn_encoding)
state_cache['tip'] = max
state_cache['repo'] = repourl
save_cache(tipfile, state_cache)
save_cache(mappingfile, mapping_cache)
c = export_tags(ui, repo, old_marks, mapping_cache, c, authors, tagsmap)
sys.stderr.write('Issued %d commands\n' % c)
return 0
def hg2git(repourl,m,marksfile,mappingfile,headsfile,tipfile,authors={},sob=False,force=False,hgtags=False,notes=False,ignoreSub=False,fixBranch=False,encoding=''):
_max=int(m)
old_marks=load_cache(marksfile,lambda s: int(s)-1)
mapping_cache=load_cache(mappingfile)
heads_cache=load_cache(headsfile)
state_cache=load_cache(tipfile)
ui,repo=setup_repo(repourl)
if not verify_heads(ui,repo,heads_cache,force):
return 1
try:
tip=repo.changelog.count()
except AttributeError:
tip=len(repo)
min=int(state_cache.get('tip',0))
max=_max
if _max<0 or max>tip:
max=tip
subRepoWarnings = {}
for rev in range(0,max):
if not ignoreSub:
# check if repository uses unconverted subrepos
ctx=repo.changectx(str(rev))
if (ctx.substate):
subRepoWarnings=verify_subrepo(repourl, ctx, subRepoWarnings)
(revnode,_,_,_,_,_,_,_)=get_changeset(ui,repo,rev,authors)
mapping_cache[revnode.encode('hex_codec')] = str(rev)
if subRepoWarnings:
sys.stderr.write("\n")
for key in subRepoWarnings.keys():
sys.stderr.write(subRepoWarnings[key])
sys.stderr.write("\n")
return 1
c=0
brmap={}
for rev in range(min,max):
c=export_commit(ui,repo,rev,old_marks,max,c,authors,sob,brmap,hgtags,notes,repourl,ignoreSub,fixBranch,encoding)
state_cache['tip']=max
state_cache['repo']=repourl
save_cache(tipfile,state_cache)
save_cache(mappingfile,mapping_cache)
c=export_tags(ui,repo,old_marks,mapping_cache,c,authors)
sys.stderr.write('Issued %d commands\n' % c)
return 0
def hg2git(repourl,m,marksfile,mappingfile,headsfile,tipfile,
authors={},branchesmap={},tagsmap={},
sob=False,force=False,hgtags=False,notes=False,encoding='',fn_encoding=''):
_max=int(m)
old_marks=load_cache(marksfile,lambda s: int(s)-1)
mapping_cache=load_cache(mappingfile)
heads_cache=load_cache(headsfile)
state_cache=load_cache(tipfile)
ui,repo=setup_repo(repourl)
if not verify_heads(ui,repo,heads_cache,force):
return 1
try:
tip=repo.changelog.count()
except AttributeError:
tip=len(repo)
min=int(state_cache.get('tip',0))
max=_max
if _max<0 or max>tip:
max=tip
for rev in range(0,max):
(revnode,_,_,_,_,_,_,_)=get_changeset(ui,repo,rev,authors)
mapping_cache[revnode.encode('hex_codec')] = str(rev)
c=0
brmap={}
for rev in range(min,max):
c=export_commit(ui,repo,rev,old_marks,max,c,authors,branchesmap,
sob,brmap,hgtags,encoding,fn_encoding)
if notes:
for rev in range(min,max):
c=export_note(ui,repo,rev,c,authors, encoding, rev == min and min != 0)
state_cache['tip']=max
state_cache['repo']=repourl
save_cache(tipfile,state_cache)
save_cache(mappingfile,mapping_cache)
c=export_tags(ui,repo,old_marks,mapping_cache,c,authors,tagsmap)
sys.stderr.write('Issued %d commands\n' % c)
return 0
def get_tags(ui,repo,marks_cache,mapping_cache,max):
l=repo.tagslist()
good,bad=[],[]
for tag,node in l:
if tag=='tip': continue
rev=int(mapping_cache[node.encode('hex_codec')])
cache_sha1=marks_cache.get(str(int(rev)+1))
_,_,user,(_,_),_,desc,branch,_=get_changeset(ui,repo,rev)
if int(rev)>int(max):
bad.append([tag,branch,cache_sha1,rev,desc.split('\n')[0],user])
else:
good.append([tag,branch,cache_sha1,rev,desc.split('\n')[0],user])
good.sort()
bad.sort()
return good,bad
def get_tags(ui,repo,marks_cache,mapping_cache,max):
l=repo.tagslist()
good,bad=[],[]
for tag,node in l:
if tag=='tip': continue
rev=int(mapping_cache[node.encode('hex_codec')])
cache_sha1=marks_cache.get(str(int(rev)+1))
_,_,user,(_,_),_,desc,branch,_=get_changeset(ui,repo,rev)
if int(rev)>int(max):
bad.append([tag,branch,cache_sha1,rev,desc.split('\n')[0],user])
else:
good.append([tag,branch,cache_sha1,rev,desc.split('\n')[0],user])
good.sort()
bad.sort()
return good,bad
def get_branches(ui,repo,heads_cache,marks_cache,mapping_cache,max):
h=heads(ui,repo,max=max)
stale=dict.fromkeys(heads_cache)
changed=[]
unchanged=[]
for node,rev in h:
_,_,user,(_,_),_,desc,branch,_=get_changeset(ui,repo,rev)
del stale[branch]
git_sha1=get_git_sha1(branch)
cache_sha1=marks_cache.get(str(int(rev)+1))
if git_sha1!=None and git_sha1==cache_sha1:
unchanged.append([branch,cache_sha1,rev,desc.split('\n')[0],user])
else:
changed.append([branch,cache_sha1,rev,desc.split('\n')[0],user])
changed.sort()
unchanged.sort()
return stale,changed,unchanged
def get_branches(ui,repo,heads_cache,marks_cache,mapping_cache,max):
h=heads(ui,repo,max=max)
stale=dict.fromkeys(heads_cache)
changed=[]
unchanged=[]
for node,rev in h:
_,_,user,(_,_),_,desc,branch,_=get_changeset(ui,repo,rev)
del stale[branch]
git_sha1=get_git_sha1(branch)
cache_sha1=marks_cache.get(str(int(rev)+1))
if git_sha1!=None and git_sha1==cache_sha1:
unchanged.append([branch,cache_sha1,rev,desc.split('\n')[0],user])
else:
changed.append([branch,cache_sha1,rev,desc.split('\n')[0],user])
changed.sort()
unchanged.sort()
return stale,changed,unchanged
def export_note(ui,repo,revision,count,authors,encoding,is_first):
(revnode,_,user,(time,timezone),_,_,_,_)=get_changeset(ui,repo,revision,authors,encoding)
if repo[revnode].hidden():
return count
parents = [p for p in repo.changelog.parentrevs(revision) if p >= 0]
wr(b'commit refs/notes/hg')
wr(b'committer %s %d %s' % (user,time,timezone))
wr(b'data 0')
if is_first:
wr(b'from refs/notes/hg^0')
wr(b'N inline :%d' % (revision+1))
hg_hash=revsymbol(repo,b"%d" % revision).hex()
wr(b'data %d' % (len(hg_hash)))
wr_no_nl(hg_hash)
wr()
return checkpoint(count)
def hg2git(repourl,m,marksfile,mappingfile,headsfile,tipfile,authors={},sob=False,force=False):
_max=int(m)
marks_cache=load_cache(marksfile,mangle_mark)
mapping_cache=load_cache(mappingfile)
heads_cache=load_cache(headsfile)
state_cache=load_cache(tipfile)
ui,repo=setup_repo(repourl)
if not verify_heads(ui,repo,heads_cache,force):
return 1
try:
tip=repo.changelog.count()
except AttributeError:
tip=len(repo)
min=int(state_cache.get('tip',0))
max=_max
if _max<0 or max>tip:
max=tip
for rev in range(0,max):
(revnode,_,_,_,_,_,_,_)=get_changeset(ui,repo,rev,authors)
mapping_cache[revnode.encode('hex_codec')] = str(rev)
c=0
last={}
brmap={}
for rev in range(min,max):
c=export_commit(ui,repo,rev,marks_cache,mapping_cache,heads_cache,last,max,c,authors,sob,brmap)
state_cache['tip']=max
state_cache['repo']=repourl
save_cache(tipfile,state_cache)
save_cache(mappingfile,mapping_cache)
c=export_tags(ui,repo,marks_cache,mapping_cache,c,authors)
sys.stderr.write('Issued %d commands\n' % c)
return 0
def export_note(ui, repo, revision, count, authors, encoding, is_first):
(revnode, _, user, (time, timezone), _, _, _, _) = get_changeset(ui, repo, revision, authors, encoding)
parents = [p for p in repo.changelog.parentrevs(revision) if p >= 0]
wr("commit refs/notes/hg")
wr("committer %s %d %s" % (user, time, timezone))
wr("data 0")
if is_first:
wr("from refs/notes/hg^0")
wr("N inline :%d" % (revision + 1))
hg_hash = repo.changectx(str(revision)).hex()
wr("data %d" % (len(hg_hash)))
wr_no_nl(hg_hash)
wr()
return checkpoint(count)
wr("data %d" % (len(desc) + 1)) # wtf?
wr(desc)
wr()
def export_note(ui,repo,revision,count,authors,encoding,is_first):
(revnode,_,user,(time,timezone),_,_,_,_)=get_changeset(ui,repo,revision,authors,encoding)
parents = [p for p in repo.changelog.parentrevs(revision) if p >= 0]
wr('commit refs/notes/hg')
wr('committer %s %d %s' % (user,time,timezone))
wr('data 0')
if is_first:
wr('from refs/notes/hg^0')
wr('N inline :%d' % (revision+1))
hg_hash=repo.changectx(str(revision)).hex()
wr('data %d' % (len(hg_hash)))
wr_no_nl(hg_hash)
wr()
return checkpoint(count)
wr('data %d' % (len(desc)+1)) # wtf?
wr(desc)
wr()
def hg2git(repourl,
m,
marksfile,
mappingfile,
headsfile,
tipfile,
authors={},
branchesmap={},
tagsmap={},
sob=False,
force=False,
ignore_unnamed_heads=False,
hgtags=False,
notes=False,
encoding='',
fn_encoding='',
plugins={}):
def check_cache(filename, contents):
if len(contents) == 0:
sys.stderr.write(
'Warning: %s does not contain any data, this will probably make an incremental import fail\n'
% filename)
_max = int(m)
old_marks = load_cache(marksfile, lambda s: int(s) - 1)
mapping_cache = load_cache(mappingfile)
heads_cache = load_cache(headsfile)
state_cache = load_cache(tipfile)
if len(state_cache) != 0:
for (name, data) in [(marksfile, old_marks),
(mappingfile, mapping_cache),
(headsfile, state_cache)]:
check_cache(name, data)
ui, repo = setup_repo(repourl)
if not verify_heads(ui, repo, heads_cache, force, ignore_unnamed_heads,
branchesmap):
return 1
try:
tip = repo.changelog.count()
except AttributeError:
tip = len(repo)
min = int(state_cache.get('tip', 0))
max = _max
if _max < 0 or max > tip:
max = tip
for rev in range(0, max):
(revnode, _, _, _, _, _, _, _) = get_changeset(ui, repo, rev, authors)
if repo[revnode].hidden():
continue
mapping_cache[hexlify(revnode)] = b"%d" % rev
if submodule_mappings:
# Make sure that all mercurial submodules are registered in the submodule-mappings file
for rev in range(0, max):
ctx = revsymbol(repo, b"%d" % rev)
if ctx.hidden():
continue
if ctx.substate:
for key in ctx.substate:
if ctx.substate[key][
2] == 'hg' and key not in submodule_mappings:
sys.stderr.write(
"Error: %s not found in submodule-mappings\n" %
(key))
return 1
c = 0
brmap = {}
for rev in range(min, max):
c = export_commit(ui, repo, rev, old_marks, max, c, authors,
branchesmap, sob, brmap, hgtags, encoding,
fn_encoding, plugins)
if notes:
for rev in range(min, max):
c = export_note(ui, repo, rev, c, authors, encoding, rev == min
and min != 0)
state_cache['tip'] = max
state_cache['repo'] = repourl
save_cache(tipfile, state_cache)
save_cache(mappingfile, mapping_cache)
c = export_tags(ui, repo, old_marks, mapping_cache, c, authors, tagsmap)
sys.stderr.write('Issued %d commands\n' % c)
return 0
def export_commit(
ui, repo, revision, old_marks, max, count, authors, branchesmap, sob, brmap, hgtags, encoding="", fn_encoding=""
):
def get_branchname(name):
if brmap.has_key(name):
return brmap[name]
n = sanitize_name(name, "branch", branchesmap)
brmap[name] = n
return n
(revnode, _, user, (time, timezone), files, desc, branch, _) = get_changeset(ui, repo, revision, authors, encoding)
branch = get_branchname(branch)
parents = [p for p in repo.changelog.parentrevs(revision) if p >= 0]
if len(parents) == 0 and revision != 0:
wr("reset refs/heads/%s" % branch)
wr("commit refs/heads/%s" % branch)
wr("mark :%d" % (revision + 1))
if sob:
wr("author %s %d %s" % (get_author(desc, user, authors), time, timezone))
wr("committer %s %d %s" % (user, time, timezone))
wr("data %d" % (len(desc) + 1)) # wtf?
wr(desc)
wr()
ctx = repo.changectx(str(revision))
man = ctx.manifest()
added, changed, removed, type = [], [], [], ""
if len(parents) == 0:
# first revision: feed in full manifest
added = man.keys()
added.sort()
type = "full"
else:
wr("from %s" % revnum_to_revref(parents[0], old_marks))
if len(parents) == 1:
# later non-merge revision: feed in changed manifest
# if we have exactly one parent, just take the changes from the
# manifest without expensively comparing checksums
f = repo.status(repo.lookup(parents[0]), revnode)[:3]
added, changed, removed = f[1], f[0], f[2]
type = "simple delta"
else: # a merge with two parents
wr("merge %s" % revnum_to_revref(parents[1], old_marks))
# later merge revision: feed in changed manifest
# for many files comparing checksums is expensive so only do it for
# merges where we really need it due to hg's revlog logic
added, changed, removed = get_filechanges(repo, revision, parents, man)
type = "thorough delta"
sys.stderr.write(
"%s: Exporting %s revision %d/%d with %d/%d/%d added/changed/removed files\n"
% (branch, type, revision + 1, max, len(added), len(changed), len(removed))
)
if fn_encoding:
removed = [r.decode(fn_encoding).encode("utf8") for r in removed]
removed = [strip_leading_slash(x) for x in removed]
map(lambda r: wr("D %s" % r), removed)
export_file_contents(ctx, man, added, hgtags, fn_encoding)
export_file_contents(ctx, man, changed, hgtags, fn_encoding)
wr()
return checkpoint(count)
def export_commit(ui,
repo,
revision,
old_marks,
max,
count,
authors,
branchesmap,
sob,
brmap,
hgtags,
encoding='',
fn_encoding='',
plugins={}):
def get_branchname(name):
if name in brmap:
return brmap[name]
n = sanitize_name(name, "branch", branchesmap)
brmap[name] = n
return n
(revnode, _, user, (time, timezone), files, desc, branch,
_) = get_changeset(ui, repo, revision, authors, encoding)
if repo[revnode].hidden():
return count
branch = get_branchname(branch)
parents = [p for p in repo.changelog.parentrevs(revision) if p >= 0]
author = get_author(desc, user, authors)
hg_hash = revsymbol(repo, b"%d" % revision).hex()
if plugins and plugins['commit_message_filters']:
commit_data = {
'branch': branch,
'parents': parents,
'author': author,
'desc': desc,
'revision': revision,
'hg_hash': hg_hash
}
for filter in plugins['commit_message_filters']:
filter(commit_data)
branch = commit_data['branch']
parents = commit_data['parents']
author = commit_data['author']
desc = commit_data['desc']
if len(parents) == 0 and revision != 0:
wr(b'reset refs/heads/%s' % branch)
wr(b'commit refs/heads/%s' % branch)
wr(b'mark :%d' % (revision + 1))
if sob:
wr(b'author %s %d %s' % (author, time, timezone))
wr(b'committer %s %d %s' % (user, time, timezone))
wr(b'data %d' % (len(desc) + 1)) # wtf?
wr(desc)
wr()
ctx = revsymbol(repo, b"%d" % revision)
man = ctx.manifest()
added, changed, removed, type = [], [], [], ''
if len(parents) == 0:
# first revision: feed in full manifest
added = man.keys()
added.sort()
type = 'full'
else:
wr(b'from %s' % revnum_to_revref(parents[0], old_marks))
if len(parents) == 1:
# later non-merge revision: feed in changed manifest
# if we have exactly one parent, just take the changes from the
# manifest without expensively comparing checksums
f = repo.status(parents[0], revnode)
added, changed, removed = f.added, f.modified, f.removed
type = 'simple delta'
else: # a merge with two parents
wr(b'merge %s' % revnum_to_revref(parents[1], old_marks))
# later merge revision: feed in changed manifest
# for many files comparing checksums is expensive so only do it for
# merges where we really need it due to hg's revlog logic
added, changed, removed = get_filechanges(repo, revision, parents,
man)
type = 'thorough delta'
stderr_buffer.write(
b'%s: Exporting %s revision %d/%d with %d/%d/%d added/changed/removed files\n'
% (branch, type.encode(), revision + 1, max, len(added), len(changed),
len(removed)))
for filename in removed:
if fn_encoding:
filename = filename.decode(fn_encoding).encode('utf8')
filename = strip_leading_slash(filename)
if filename == b'.hgsub':
remove_gitmodules(ctx)
wr(b'D %s' % filename)
export_file_contents(ctx, man, added, hgtags, fn_encoding, plugins)
export_file_contents(ctx, man, changed, hgtags, fn_encoding, plugins)
wr()
return checkpoint(count)
def export_commit(ui,repo,revision,old_marks,max,count,authors,sob,brmap):
def get_branchname(name):
if brmap.has_key(name):
return brmap[name]
n=sanitize_name(name)
brmap[name]=n
return n
(revnode,_,user,(time,timezone),files,desc,branch,_)=get_changeset(ui,repo,revision,authors)
branch=get_branchname(branch)
parents = [p for p in repo.changelog.parentrevs(revision) if p >= 0]
if len(parents)==0 and revision != 0:
wr('reset refs/heads/%s' % branch)
wr('commit refs/heads/%s' % branch)
wr('mark :%d' % (revision+1))
if sob:
wr('author %s %d %s' % (get_author(desc,user,authors),time,timezone))
wr('committer %s %d %s' % (user,time,timezone))
wr('data %d' % (len(desc)+1)) # wtf?
wr(desc)
wr()
# Sort the parents based on revision ids so that we always get the
# same resulting git repo, no matter how the revisions were
# numbered.
parents.sort(key=repo.changelog.node, reverse=True)
ctx=repo.changectx(str(revision))
man=ctx.manifest()
added,changed,removed,type=[],[],[],''
if len(parents) == 0:
# first revision: feed in full manifest
added=man.keys()
added.sort()
type='full'
else:
wr('from %s' % revnum_to_revref(parents[0], old_marks))
if len(parents) == 1:
# later non-merge revision: feed in changed manifest
# if we have exactly one parent, just take the changes from the
# manifest without expensively comparing checksums
f=repo.status(repo.lookup(parents[0]),revnode)[:3]
added,changed,removed=f[1],f[0],f[2]
type='simple delta'
else: # a merge with two parents
wr('merge %s' % revnum_to_revref(parents[1], old_marks))
# later merge revision: feed in changed manifest
# for many files comparing checksums is expensive so only do it for
# merges where we really need it due to hg's revlog logic
added,changed,removed=get_filechanges(repo,revision,parents,man)
type='thorough delta'
sys.stderr.write('%s: Exporting %s revision %d/%d with %d/%d/%d added/changed/removed files\n' %
(branch,type,revision+1,max,len(added),len(changed),len(removed)))
map(lambda r: wr('D %s' % r),removed)
export_file_contents(ctx,man,added)
export_file_contents(ctx,man,changed)
wr()
return checkpoint(count)
def hg2git(repourl,
m,
marksfile,
mappingfile,
headsfile,
tipfile,
authors={},
sob=False,
force=False,
hgtags=False,
notes=False,
ignoreSub=False,
fixBranch=False,
encoding=''):
_max = int(m)
old_marks = load_cache(marksfile, lambda s: int(s) - 1)
mapping_cache = load_cache(mappingfile)
heads_cache = load_cache(headsfile)
state_cache = load_cache(tipfile)
ui, repo = setup_repo(repourl)
if not verify_heads(ui, repo, heads_cache, force):
return 1
try:
tip = repo.changelog.count()
except AttributeError:
tip = len(repo)
min = int(state_cache.get('tip', 0))
max = _max
if _max < 0 or max > tip:
max = tip
subRepoWarnings = {}
for rev in range(0, max):
if not ignoreSub:
# check if repository uses unconverted subrepos
ctx = repo.changectx(str(rev))
if (ctx.substate):
subRepoWarnings = verify_subrepo(repourl, ctx, subRepoWarnings)
(revnode, _, _, _, _, _, _, _) = get_changeset(ui, repo, rev, authors)
mapping_cache[revnode.encode('hex_codec')] = str(rev)
if subRepoWarnings:
sys.stderr.write("\n")
for key in subRepoWarnings.keys():
sys.stderr.write(subRepoWarnings[key])
sys.stderr.write("\n")
return 1
c = 0
brmap = {}
for rev in range(min, max):
c = export_commit(ui, repo, rev, old_marks, max, c, authors, sob,
brmap, hgtags, notes, repourl, ignoreSub, fixBranch,
encoding)
state_cache['tip'] = max
state_cache['repo'] = repourl
save_cache(tipfile, state_cache)
save_cache(mappingfile, mapping_cache)
c = export_tags(ui, repo, old_marks, mapping_cache, c, authors)
sys.stderr.write('Issued %d commands\n' % c)
return 0
def export_commit(ui,repo,revision,marks,mapping,heads,last,max,count,authors,sob,brmap):
def get_branchname(name):
if brmap.has_key(name):
return brmap[name]
n=sanitize_name(name)
brmap[name]=n
return n
(revnode,_,user,(time,timezone),files,desc,branch,_)=get_changeset(ui,repo,revision,authors)
parents=repo.changelog.parentrevs(revision)
branch=get_branchname(branch)
wr('commit refs/heads/%s' % branch)
wr('mark :%d' % (revision+1))
if sob:
wr('author %s %d %s' % (get_author(desc,user,authors),time,timezone))
wr('committer %s %d %s' % (user,time,timezone))
wr('data %d' % (len(desc)+1)) # wtf?
wr(desc)
wr()
pidx1, pidx2 = 0, 1
if parents[0] < parents[1]:
pidx1, pidx2 = 1, 0
src=heads.get(branch,'')
link=''
if src!='':
# if we have a cached head, this is an incremental import: initialize it
# and kill reference so we won't init it again
wr('from %s' % src)
heads[branch]=''
sys.stderr.write('%s: Initializing to parent [%s]\n' %
(branch,src))
link=src # avoid making a merge commit for incremental import
elif link=='' and not heads.has_key(branch) and revision>0:
# newly created branch and not the first one: connect to parent
tmp=get_parent_mark(parents[0],marks)
wr('from %s' % tmp)
sys.stderr.write('%s: Link new branch to parent [%s]\n' %
(branch,tmp))
link=tmp # avoid making a merge commit for branch fork
elif last.get(branch,revision) != parents[pidx1] and parents[pidx1] > 0 and revision > 0:
pm=get_parent_mark(parents[pidx1],marks)
sys.stderr.write('%s: Placing commit [r%d] in branch [%s] on top of [r%d]\n' %
(branch,revision,branch,parents[pidx1]));
wr('from %s' % pm)
if parents[pidx2] > 0:
pm=get_parent_mark(parents[pidx2],marks)
sys.stderr.write('%s: Merging with parent [%s] from [r%d]\n' %
(branch,pm,parents[pidx2]))
wr('merge %s' % pm)
last[branch]=revision
heads[branch]=''
# we need this later to write out tags
marks[str(revision)]=':%d'%(revision+1)
ctx=repo.changectx(str(revision))
man=ctx.manifest()
added,changed,removed,type=[],[],[],''
if revision==0:
# first revision: feed in full manifest
added=man.keys()
added.sort()
type='full'
elif is_merge(parents):
# later merge revision: feed in changed manifest
# for many files comparing checksums is expensive so only do it for
# merges where we really need it due to hg's revlog logic
added,changed,removed=get_filechanges(repo,revision,parents,man)
type='thorough delta'
else:
# later non-merge revision: feed in changed manifest
# if we have exactly one parent, just take the changes from the
# manifest without expensively comparing checksums
f=repo.status(repo.lookup(parents[0]),revnode)[:3]
added,changed,removed=f[1],f[0],f[2]
type='simple delta'
sys.stderr.write('%s: Exporting %s revision %d/%d with %d/%d/%d added/changed/removed files\n' %
(branch,type,revision+1,max,len(added),len(changed),len(removed)))
map(lambda r: wr('D %s' % r),removed)
export_file_contents(ctx,man,added)
export_file_contents(ctx,man,changed)
wr()
return checkpoint(count)
def export_commit(ui,
repo,
revision,
old_marks,
max,
count,
authors,
branchesmap,
sob,
brmap,
hgtags,
encoding='',
fn_encoding='',
filter_contents=None):
def get_branchname(name):
if brmap.has_key(name):
return brmap[name]
n = sanitize_name(name, "branch", branchesmap)
brmap[name] = n
return n
(revnode, _, user, (time, timezone), files, desc, branch,
_) = get_changeset(ui, repo, revision, authors, encoding)
branch = get_branchname(branch)
parents = [p for p in repo.changelog.parentrevs(revision) if p >= 0]
if len(parents) == 0 and revision != 0:
wr('reset refs/heads/%s' % branch)
wr('commit refs/heads/%s' % branch)
wr('mark :%d' % (revision + 1))
if sob:
wr('author %s %d %s' %
(get_author(desc, user, authors), time, timezone))
wr('committer %s %d %s' % (user, time, timezone))
wr('data %d' % (len(desc) + 1)) # wtf?
wr(desc)
wr()
ctx = revsymbol(repo, str(revision))
man = ctx.manifest()
added, changed, removed, type = [], [], [], ''
if len(parents) == 0:
# first revision: feed in full manifest
added = man.keys()
added.sort()
type = 'full'
else:
wr('from %s' % revnum_to_revref(parents[0], old_marks))
if len(parents) == 1:
# later non-merge revision: feed in changed manifest
# if we have exactly one parent, just take the changes from the
# manifest without expensively comparing checksums
f = repo.status(parents[0], revnode)[:3]
added, changed, removed = f[1], f[0], f[2]
type = 'simple delta'
else: # a merge with two parents
wr('merge %s' % revnum_to_revref(parents[1], old_marks))
# later merge revision: feed in changed manifest
# for many files comparing checksums is expensive so only do it for
# merges where we really need it due to hg's revlog logic
added, changed, removed = get_filechanges(repo, revision, parents,
man)
type = 'thorough delta'
# in case we have added a gitattribute generated file
# we need to copy the merged branch
merged_branch = get_branchname(repo.changectx(parents[1]).branch())
merge_lfs_attributes(branch, merged_branch)
sys.stderr.write(
'%s: Exporting %s revision %d/%d with %d/%d/%d added/changed/removed files\n'
% (branch, type, revision + 1, max, len(added), len(changed),
len(removed)))
if fn_encoding:
removed = [r.decode(fn_encoding).encode('utf8') for r in removed]
new_lfs_entry = False
for entry in added + changed:
if len(entry) > 5 and entry[:5] == ".hglf":
add_to_lfs(branch, strip_leading_slash(entry[5:]))
new_lfs_entry = True
for entry in removed:
if len(entry) > 5 and entry[:5] == ".hglf":
remove_from_lfs(branch, strip_leading_slash(entry[5:]))
new_lfs_entry = True
# if the pointer is removed, remove the git pointer
removed = [
x[5:] if len(x) > 5 and x[:5] == ".hglf" else x for x in removed
]
removed = [strip_leading_slash(x) for x in removed]
map(lambda r: wr('D %s' % r), removed)
export_file_contents(ctx, man, added, hgtags, fn_encoding, filter_contents)
export_file_contents(ctx, man, changed, hgtags, fn_encoding,
filter_contents)
if new_lfs_entry:
gitAttributes = build_lfs_attributes(branch)
if gitAttributes is not None:
wr('M %s inline %s' % ("100644", ".gitattributes"))
wr('data %d' % len(gitAttributes))
wr(gitAttributes)
sys.stderr.write('Exported .gitattributes\n')
wr()
return checkpoint(count)
# get list of hg's branches to verify, don't take all git has
for _,_,b in l:
b=get_branch(b)
sanitized_name=sanitize_name(b,"branch",branchesmap)
sha1=get_git_sha1(sanitized_name)
c=cache.get(sanitized_name)
if sha1!=c:
sys.stderr.write('Error: Branch [%s] modified outside hg-fast-export:'
'\n%s (repo) != %s (cache)\n' % (b,sha1,c))
if not force: return False
# verify that branch has exactly one head
t={}
for h in repo.heads():
(_,_,_,_,_,_,branch,_)=get_changeset(ui,repo,h)
if t.get(branch,False):
sys.stderr.write('Error: repository has at least one unnamed head: hg r%s\n' %
repo.changelog.rev(h))
if not force: return False
t[branch]=True
return True
def hg2git(repourl,m,marksfile,mappingfile,headsfile,tipfile,
authors={},branchesmap={},tagsmap={},
sob=False,force=False,hgtags=False,notes=False,encoding='',fn_encoding='',
plugins={}):
def check_cache(filename, contents):
if len(contents) == 0:
sys.stderr.write('Warning: %s does not contain any data, this will probably make an incremental import fail\n' % filename)
def export_commit(ui,
repo,
revision,
old_marks,
max,
count,
authors,
branchesmap,
sob,
brmap,
hgtags,
encoding='',
fn_encoding='',
filter_contents=None):
def get_branchname(name):
if brmap.has_key(name):
return brmap[name]
n = sanitize_name(name, "branch", branchesmap)
brmap[name] = n
return n
(revnode, _, user, (time, timezone), files, desc, branch,
_) = get_changeset(ui, repo, revision, authors, encoding)
branch = get_branchname(branch)
parents = [p for p in repo.changelog.parentrevs(revision) if p >= 0]
if len(parents) == 0 and revision != 0:
wr('reset refs/heads/%s' % branch)
wr('commit refs/heads/%s' % branch)
wr('mark :%d' % (revision + 1))
if sob:
wr('author %s %d %s' %
(get_author(desc, user, authors), time, timezone))
wr('committer %s %d %s' % (user, time, timezone))
wr('data %d' % (len(desc) + 1)) # wtf?
wr(desc)
wr()
ctx = revsymbol(repo, str(revision))
man = ctx.manifest()
added, changed, removed, type = [], [], [], ''
if len(parents) == 0:
# first revision: feed in full manifest
added = man.keys()
added.sort()
type = 'full'
else:
wr('from %s' % revnum_to_revref(parents[0], old_marks))
if len(parents) == 1:
# later non-merge revision: feed in changed manifest
# if we have exactly one parent, just take the changes from the
# manifest without expensively comparing checksums
f = repo.status(parents[0], revnode)[:3]
added, changed, removed = f[1], f[0], f[2]
type = 'simple delta'
else: # a merge with two parents
wr('merge %s' % revnum_to_revref(parents[1], old_marks))
# later merge revision: feed in changed manifest
# for many files comparing checksums is expensive so only do it for
# merges where we really need it due to hg's revlog logic
added, changed, removed = get_filechanges(repo, revision, parents,
man)
type = 'thorough delta'
sys.stderr.write(
'%s: Exporting %s revision %d/%d with %d/%d/%d added/changed/removed files\n'
% (branch, type, revision + 1, max, len(added), len(changed),
len(removed)))
if fn_encoding:
removed = [r.decode(fn_encoding).encode('utf8') for r in removed]
removed = [strip_leading_slash(x) for x in removed]
map(lambda r: wr('D %s' % r), removed)
export_file_contents(ctx, man, added, hgtags, fn_encoding, filter_contents)
export_file_contents(ctx, man, changed, hgtags, fn_encoding,
filter_contents)
wr()
return checkpoint(count)
def hg2git(
repourl,
m,
marksfile,
mappingfile,
headsfile,
tipfile,
authors={},
branchesmap={},
tagsmap={},
sob=False,
force=False,
hgtags=False,
notes=False,
encoding="",
fn_encoding="",
):
def check_cache(filename, contents):
if len(contents) == 0:
sys.stderr.write(
"Warning: %s does not contain any data, this will probably make an incremental import fail\n" % filename
)
_max = int(m)
old_marks = load_cache(marksfile, lambda s: int(s) - 1)
mapping_cache = load_cache(mappingfile)
heads_cache = load_cache(headsfile)
state_cache = load_cache(tipfile)
if len(state_cache) != 0:
for (name, data) in [(marksfile, old_marks), (mappingfile, mapping_cache), (headsfile, state_cache)]:
check_cache(name, data)
ui, repo = setup_repo(repourl)
if not verify_heads(ui, repo, heads_cache, force, branchesmap):
return 1
try:
tip = repo.changelog.count()
except AttributeError:
tip = len(repo)
min = int(state_cache.get("tip", 0))
max = _max
if _max < 0 or max > tip:
max = tip
for rev in range(0, max):
(revnode, _, _, _, _, _, _, _) = get_changeset(ui, repo, rev, authors)
mapping_cache[revnode.encode("hex_codec")] = str(rev)
c = 0
brmap = {}
for rev in range(min, max):
c = export_commit(
ui, repo, rev, old_marks, max, c, authors, branchesmap, sob, brmap, hgtags, encoding, fn_encoding
)
if notes:
for rev in range(min, max):
c = export_note(ui, repo, rev, c, authors, encoding, rev == min and min != 0)
state_cache["tip"] = max
state_cache["repo"] = repourl
save_cache(tipfile, state_cache)
save_cache(mappingfile, mapping_cache)
c = export_tags(ui, repo, old_marks, mapping_cache, c, authors, tagsmap)
sys.stderr.write("Issued %d commands\n" % c)
return 0
def export_commit(ui,repo,revision,old_marks,max,count,authors,
branchesmap,sob,brmap,hgtags,encoding='',fn_encoding='',
plugins={}):
def get_branchname(name):
if brmap.has_key(name):
return brmap[name]
n=sanitize_name(name, "branch", branchesmap)
brmap[name]=n
return n
(revnode,_,user,(time,timezone),files,desc,branch,_)=get_changeset(ui,repo,revision,authors,encoding)
branch=get_branchname(branch)
parents = [p for p in repo.changelog.parentrevs(revision) if p >= 0]
author = get_author(desc,user,authors)
if plugins and plugins['commit_message_filters']:
commit_data = {'branch': branch, 'parents': parents, 'author': author, 'desc': desc}
for filter in plugins['commit_message_filters']:
filter(commit_data)
branch = commit_data['branch']
parents = commit_data['parents']
author = commit_data['author']
desc = commit_data['desc']
if len(parents)==0 and revision != 0:
wr('reset refs/heads/%s' % branch)
wr('commit refs/heads/%s' % branch)
wr('mark :%d' % (revision+1))
if sob:
wr('author %s %d %s' % (author,time,timezone))
wr('committer %s %d %s' % (user,time,timezone))
wr('data %d' % (len(desc)+1)) # wtf?
wr(desc)
wr()
ctx=revsymbol(repo,str(revision))
man=ctx.manifest()
added,changed,removed,type=[],[],[],''
if len(parents) == 0:
# first revision: feed in full manifest
added=man.keys()
added.sort()
type='full'
else:
wr('from %s' % revnum_to_revref(parents[0], old_marks))
if len(parents) == 1:
# later non-merge revision: feed in changed manifest
# if we have exactly one parent, just take the changes from the
# manifest without expensively comparing checksums
f=repo.status(parents[0],revnode)[:3]
added,changed,removed=f[1],f[0],f[2]
type='simple delta'
else: # a merge with two parents
wr('merge %s' % revnum_to_revref(parents[1], old_marks))
# later merge revision: feed in changed manifest
# for many files comparing checksums is expensive so only do it for
# merges where we really need it due to hg's revlog logic
added,changed,removed=get_filechanges(repo,revision,parents,man)
type='thorough delta'
sys.stderr.write('%s: Exporting %s revision %d/%d with %d/%d/%d added/changed/removed files\n' %
(branch,type,revision+1,max,len(added),len(changed),len(removed)))
for filename in removed:
if fn_encoding:
filename=filename.decode(fn_encoding).encode('utf8')
filename=strip_leading_slash(filename)
if filename=='.hgsubstate':
remove_gitmodules(ctx)
wr('D %s' % filename)
export_file_contents(ctx,man,added,hgtags,fn_encoding,plugins)
export_file_contents(ctx,man,changed,hgtags,fn_encoding,plugins)
wr()
result = checkpoint(count)
# Ask for the git hash of the last commit
wr('get-mark :%d' % (revision+1))
sys.stdout.flush()
# Read the hash of the last commit
git_hash = cat_blob_fd.readline()
rev_number_to_git_hash[str(revision)] = git_hash
rev_hash_to_git_hash[ctx.hex()] = git_hash
write_hg2git_map (str(revision), ctx.hex(), git_hash)
return result
