def run(self):
from cinnabar import VERSION
from cinnabar.git import Git, GitProcess
from distutils.version import StrictVersion
parent_dir = os.path.dirname(os.path.dirname(__file__))
if not os.path.exists(os.path.join(parent_dir, '.git')) or \
check_enabled('no-version-check') or \
not interval_expired('version-check', 86400, globl=True):
return
REPO = 'https://github.com/glandium/git-cinnabar'
devnull = open(os.devnull, 'wb')
if VERSION.endswith('a'):
_, _, extra = StrictVersion(VERSION[:-1]).version
ref = 'refs/heads/next' if extra == 0 else 'refs/heads/master'
for line in Git.iter('ls-remote', REPO, ref, stderr=devnull):
sha1, head = line.split()
if head != ref:
continue
proc = GitProcess('-C',
parent_dir,
'merge-base',
'--is-ancestor',
sha1,
'HEAD',
stdout=devnull,
stderr=devnull)
if proc.wait() != 0:
self.message = (
'The `{}` branch of git-cinnabar was updated. '
'Please update your copy.\n'
'You can switch to the `release` branch if you want '
'to reduce these update notifications.'.format(
ref.partition('refs/heads/')[-1]))
break
else:
version = StrictVersion(VERSION)
newer_version = version
for line in Git.iter('ls-remote',
REPO,
'refs/tags/*',
stderr=devnull):
sha1, tag = line.split()
tag = tag.partition('refs/tags/')[-1]
try:
v = StrictVersion(tag)
except ValueError:
continue
if v > newer_version:
newer_version = v
if newer_version != version:
self.message = ('New git-cinnabar version available: {} '
'(current version: {})'.format(
newer_version, version))
def reclone(args):
'''reclone all mercurial remotes'''
from cinnabar.cmd.rollback import do_rollback
git_config = {}
metadata_commit = Git.resolve_ref('refs/cinnabar/metadata')
if metadata_commit:
git_config['cinnabar.previous-metadata'] = \
metadata_commit.decode('ascii')
# TODO: Avoid resetting at all, possibly leaving the repo with no metadata
# if this is interrupted somehow.
do_rollback(NULL_NODE_ID.decode('ascii'))
for line in Git.iter('config', '--get-regexp', 'remote\..*\.url'):
config, url = line.split()
name = config[len('remote.'):-len('.url')]
skip_pref = 'remote.%s.skipDefaultUpdate' % name.decode('ascii')
if (url.startswith((b'hg::', b'hg://'))
and Git.config(skip_pref) != 'true'):
Git.run('remote',
'update',
'--prune',
fsdecode(name),
config=git_config)
git_config = {}
print('Please note that reclone left your local branches untouched.')
print('They may be based on entirely different commits.')
def test_module_version(self):
module = one(
Git.iter('ls-tree',
'HEAD',
'cinnabar',
cwd=os.path.join(os.path.dirname(__file__), '..')))
self.assertEqual(CmdVersion.module_version(), split_ls_tree(module)[2])
def test_cinnabar_version(self):
desc = one(Git.iter('describe', '--tags', 'HEAD'))
version = Version(CmdVersion.cinnabar_version())
if '-' in desc:
last_tag, n, sha1 = desc.rsplit('-', 2)
self.assertGreater(version, Version(last_tag))
else:
self.assertEqual(version, Version(desc))
def run(self):
from cinnabar import VERSION
from cinnabar.git import Git, GitProcess
from distutils.version import StrictVersion
parent_dir = os.path.dirname(os.path.dirname(__file__))
if not os.path.exists(os.path.join(parent_dir, '.git')) or \
check_enabled('no-version-check') or \
not interval_expired('version-check', 86400):
return
REPO = 'https://github.com/glandium/git-cinnabar'
devnull = open(os.devnull, 'wb')
if VERSION.endswith('a'):
_, _, extra = StrictVersion(VERSION[:-1]).version
ref = 'refs/heads/next' if extra == 0 else 'refs/heads/master'
for line in Git.iter('ls-remote', REPO, ref, stderr=devnull):
sha1, head = line.split()
if head != ref:
continue
proc = GitProcess(
'-C', parent_dir, 'merge-base', '--is-ancestor', sha1,
'HEAD', stdout=devnull, stderr=devnull)
if proc.wait() != 0:
self.message = (
'The `{}` branch of git-cinnabar was updated. '
'Please update your copy.\n'
'You can switch to the `release` branch if you want '
'to reduce these update notifications.'
.format(ref.partition('refs/heads/')[-1]))
break
else:
version = StrictVersion(VERSION)
newer_version = version
for line in Git.iter('ls-remote', REPO, 'refs/tags/*',
stderr=devnull):
sha1, tag = line.split()
tag = tag.partition('refs/tags/')[-1]
try:
v = StrictVersion(tag)
except ValueError:
continue
if v > newer_version:
newer_version = v
if newer_version != version:
self.message = (
'New version available: {} (current version: {})'
.format(newer_version, version))
def old_helper_head():
from cinnabar.git import Git
from cinnabar.helper import GitHgHelper
version = GitHgHelper.VERSION
return list(Git.iter(
'log', 'HEAD', '--format=%H', '--pickaxe-regex',
'-S', '#define CMD_VERSION {}'.format(version),
cwd=os.path.join(os.path.dirname(__file__), '..')))[-1]
def test_helper_version(self):
helper = one(
Git.iter('ls-tree',
'HEAD',
'helper',
cwd=os.path.join(os.path.dirname(__file__), '..')))
self.assertEqual(CmdVersion.helper_version()[1],
split_ls_tree(helper)[2])
def test_cinnabar_version(self):
desc = one(Git.iter('describe', '--tags', 'HEAD'))
version = Version(CmdVersion.cinnabar_version())
if '-' in desc:
last_tag, n, sha1 = desc.rsplit('-', 2)
self.assertGreater(version, Version(last_tag))
else:
self.assertEqual(version, Version(desc))
def do_cinnabarclone(repo, manifest, store):
url = None
for line in manifest.splitlines():
line = line.strip()
spec, _, params = line.partition(' ')
params = {
k: v
for k, _, v in (p.partition('=') for p in params.split())
}
graft = params.pop('graft', None)
if params:
# Future proofing: ignore lines with unknown params, even if we
# support some that are present.
continue
if store._graft:
# When grafting, ignore lines without a graft revision.
if not graft:
continue
graft = graft.split(',')
revs = list(Git.iter('rev-parse', '--revs-only', *graft))
if len(revs) != len(graft):
continue
# We apparently have all the grafted revisions locally, ensure
# they're actually reachable.
if not any(Git.iter(
'rev-list', '--branches', '--tags', '--remotes',
'--max-count=1', '--ancestry-path', '--stdin',
stdin=('^{}^@'.format(c) for c in graft))):
continue
url, _, branch = spec.partition('#')
url, branch = (url.split('#', 1) + [None])[:2]
if url:
break
if not url:
logging.warn('Server advertizes cinnabarclone but didn\'t provide '
'a git repository url to fetch from.')
return False
parsed_url = urlparse(url)
if parsed_url.scheme not in ('http', 'https', 'git'):
logging.warn('Server advertizes cinnabarclone but provided a non '
'http/https git repository. Skipping.')
return False
sys.stderr.write('Fetching cinnabar metadata from %s\n' % url)
return store.merge(url, repo.url(), branch)
def old_helper_hash(head):
from cinnabar.git import Git, split_ls_tree
from cinnabar.util import one
return split_ls_tree(
one(
Git.iter('ls-tree',
head,
'helper',
cwd=os.path.join(os.path.dirname(__file__), '..'))))[2]
def local_bases():
for c in Git.iter('rev-list', '--stdin', '--topo-order',
'--full-history', '--boundary',
*(w for w in what if w), stdin=heads()):
if c[0] != '-':
continue
yield store.hg_changeset(c[1:])
for w in what:
rev = store.hg_changeset(w)
if rev:
yield rev
def reclone(args):
'''reclone all mercurial remotes'''
from cinnabar.cmd.rollback import do_rollback
do_rollback(NULL_NODE_ID)
for line in Git.iter('config', '--get-regexp', 'remote\..*\.url'):
config, url = line.split()
name = config[len('remote.'):-len('.url')]
skip_pref = 'remote.%s.skipDefaultUpdate' % name
if (url.startswith(('hg::', 'hg://'))
and Git.config(skip_pref) != 'true'):
Git.run('remote', 'update', '--prune', name)
print 'Please note that reclone left your local branches untouched.'
print 'They may be based on entirely different commits.'
def old_helper_head():
from cinnabar import VERSION
version = VERSION
if version.endswith('a'):
from distutils.version import StrictVersion
v = StrictVersion(VERSION[:-1]).version + (0, 0, 0)
if v[2] == 0:
from cinnabar.git import Git
from cinnabar.helper import GitHgHelper
version = GitHgHelper.VERSION
return list(Git.iter(
'log', 'HEAD', '--format=%H',
'-S', '#define CMD_VERSION {}'.format(version),
cwd=os.path.join(os.path.dirname(__file__), '..')))[-1]
version = '{}.{}.{}'.format(v[0], v[1], v[2] - 1)
return version
def find_user_password(self, realm, authuri):
try:
return url_passwordmgr.find_user_password(self, realm, authuri)
except error.Abort:
# Assume error.Abort is only thrown from the base class's
# find_user_password itself, which reflects that authentication
# information is missing and mercurial would want to get it
# from user input, but can't because the ui isn't interactive.
credentials = dict(
line.split(b'=', 1) for line in Git.iter(
'credential', 'fill', stdin=b'url=%s' % authuri))
username = credentials.get(b'username')
password = credentials.get(b'password')
if not username or not password:
raise
return username, password
def old_compatible_python():
'''Find the oldest version of the python code that is compatible with the
current helper'''
from cinnabar.git import Git
with open(os.path.join(os.path.dirname(__file__), '..', 'helper',
'cinnabar-helper.c')) as fh:
min_version = None
for l in fh:
if l.startswith('#define MIN_CMD_VERSION'):
min_version = l.rstrip().split()[-1][:2]
break
if not min_version:
raise Exception('Cannot find MIN_CMD_VERSION')
return list(Git.iter(
'log', 'HEAD', '--format=%H', '-S',
'class GitHgHelper(BaseHelper):\n VERSION = {}'.format(min_version),
cwd=os.path.join(os.path.dirname(__file__), '..')))[-1].decode()
def old_compatible_python():
'''Find the oldest version of the python code that is compatible with the
current helper'''
from cinnabar.git import Git
with open(os.path.join(os.path.dirname(__file__), '..', 'helper',
'cinnabar-helper.c')) as fh:
min_version = None
for l in fh:
if l.startswith('#define MIN_CMD_VERSION'):
min_version = l.rstrip().split()[-1][:2]
break
if not min_version:
raise Exception('Cannot find MIN_CMD_VERSION')
return list(Git.iter(
'log', 'HEAD', '--format=%H', '-S',
'class GitHgHelper(BaseHelper):\n VERSION = {}'.format(min_version),
cwd=os.path.join(os.path.dirname(__file__), '..')))[-1]
def old_helper_head():
from cinnabar import VERSION
from distutils.version import StrictVersion
version = VERSION
if version.endswith('a'):
v = StrictVersion(VERSION[:-1]).version
if v[2] == 0:
from cinnabar.git import Git
from cinnabar.helper import GitHgHelper
version = GitHgHelper.VERSION
return list(Git.iter(
'log', 'HEAD', '--format=%H',
'-S', '#define CMD_VERSION {}'.format(version),
cwd=os.path.join(os.path.dirname(__file__),
'..')))[-1].decode()
else:
v = StrictVersion(VERSION).version
return '{}.{}.{}'.format(v[0], v[1], max(v[2] - 1, 0))
def main(args):
cmd = args.pop(0)
if cmd == 'data':
store = GitHgStore()
if args[0] == '-c':
sys.stdout.write(store.changeset(args[1]).data)
elif args[0] == '-m':
sys.stdout.write(store.manifest(args[1]).data)
store.close()
elif cmd == 'fsck':
return fsck(args)
elif cmd == 'reclone':
for ref in Git.for_each_ref('refs/cinnabar',
'refs/remote-hg',
'refs/notes/cinnabar',
'refs/notes/remote-hg/git2hg',
format='%(refname)'):
Git.delete_ref(ref)
Git.close()
for line in Git.iter('config', '--get-regexp', 'remote\..*\.url'):
config, url = line.split()
name = config[len('remote.'):-len('.url')]
skip_pref = 'remote.%s.skipDefaultUpdate' % name
if (url.startswith('hg::')
and Git.config(skip_pref, 'bool') != 'true'):
Git.run('remote', 'update', '--prune', name)
print 'Please note that reclone left your local branches untouched.'
print 'They may be based on entirely different commits.'
elif cmd == 'hg2git':
for arg in args:
print GitHgHelper.hg2git(arg)
elif cmd == 'git2hg':
for arg in args:
data = GitHgHelper.git2hg(arg)
if data:
data = ChangesetData.parse(data)
print data.get('changeset', NULL_NODE_ID)
else:
print NULL_NODE_ID
else:
print >> sys.stderr, 'Unknown command:', cmd
return 1
def find_user_password(self, realm, authuri):
try:
return url_passwordmgr.find_user_password(self, realm,
authuri)
except error.Abort:
# Assume error.Abort is only thrown from the base class's
# find_user_password itself, which reflects that authentication
# information is missing and mercurial would want to get it
# from user input, but can't because the ui isn't interactive.
credentials = dict(
line.split('=', 1)
for line in Git.iter('credential', 'fill',
stdin='url=%s' % authuri)
)
username = credentials.get('username')
password = credentials.get('password')
if not username or not password:
raise
return username, password
def main(args):
cmd = args.pop(0)
if cmd == 'data':
store = GitHgStore()
if args[0] == '-c':
sys.stdout.write(store.changeset(args[1]).data)
elif args[0] == '-m':
sys.stdout.write(store.manifest(args[1]).data)
store.close()
elif cmd == 'fsck':
return fsck(args)
elif cmd == 'reclone':
for ref in Git.for_each_ref('refs/cinnabar', 'refs/remote-hg',
'refs/notes/cinnabar',
'refs/notes/remote-hg/git2hg',
format='%(refname)'):
Git.delete_ref(ref)
Git.close()
for line in Git.iter('config', '--get-regexp', 'remote\..*\.url'):
config, url = line.split()
name = config[len('remote.'):-len('.url')]
skip_pref = 'remote.%s.skipDefaultUpdate' % name
if (url.startswith('hg::') and
Git.config(skip_pref, 'bool') != 'true'):
Git.run('remote', 'update', '--prune', name)
print 'Please note that reclone left your local branches untouched.'
print 'They may be based on entirely different commits.'
elif cmd == 'hg2git':
for arg in args:
print GitHgHelper.hg2git(arg)
elif cmd == 'git2hg':
for arg in args:
data = GitHgHelper.git2hg(arg)
if data:
data = ChangesetData.parse(data)
print data.get('changeset', NULL_NODE_ID)
else:
print NULL_NODE_ID
else:
print >>sys.stderr, 'Unknown command:', cmd
return 1
def reclone(args):
'''reclone all mercurial remotes'''
from cinnabar.cmd.rollback import do_rollback
git_config = {}
metadata_commit = Git.resolve_ref('refs/cinnabar/metadata')
if metadata_commit:
git_config['cinnabar.previous-metadata'] = metadata_commit
# TODO: Avoid resetting at all, possibly leaving the repo with no metadata
# if this is interrupted somehow.
do_rollback(NULL_NODE_ID)
for line in Git.iter('config', '--get-regexp', 'remote\..*\.url'):
config, url = line.split()
name = config[len('remote.'):-len('.url')]
skip_pref = 'remote.%s.skipDefaultUpdate' % name
if (url.startswith(('hg::', 'hg://')) and
Git.config(skip_pref) != 'true'):
Git.run('remote', 'update', '--prune', name, config=git_config)
git_config = {}
print 'Please note that reclone left your local branches untouched.'
print 'They may be based on entirely different commits.'
def push(self, *refspecs):
try:
default = 'never' if self._graft else 'phase'
values = {
None: default,
'': default,
'never': 'never',
'phase': 'phase',
'always': 'always',
}
data = Git.config('cinnabar.data', self._remote.name,
values=values)
except InvalidConfig as e:
logging.error(e.message)
return 1
pushes = {s.lstrip('+'): (d, s.startswith('+'))
for s, d in (r.split(':', 1) for r in refspecs)}
if not self._repo.capable('unbundle'):
for source, (dest, force) in pushes.iteritems():
self._helper.write(
'error %s Remote does not support the "unbundle" '
'capability\n' % dest)
self._helper.write('\n')
self._helper.flush()
else:
repo_heads = self._branchmap.heads()
PushStore.adopt(self._store, self._graft)
pushed = push(self._repo, self._store, pushes, repo_heads,
self._branchmap.names())
status = {}
for source, (dest, _) in pushes.iteritems():
if dest.startswith('refs/tags/'):
if source:
status[dest] = 'Pushing tags is unsupported'
else:
status[dest] = \
'Deleting remote tags is unsupported'
continue
if not dest.startswith('refs/heads/bookmarks/'):
if source:
status[dest] = bool(len(pushed))
else:
status[dest] = \
'Deleting remote branches is unsupported'
continue
name = unquote(dest[21:])
if source:
source = self._store.hg_changeset(Git.resolve_ref(source))\
or ''
status[dest] = self._repo.pushkey(
'bookmarks', name, self._bookmarks.get(name, ''), source)
for source, (dest, force) in pushes.iteritems():
if status[dest] is True:
self._helper.write('ok %s\n' % dest)
elif status[dest]:
self._helper.write('error %s %s\n' % (dest, status[dest]))
else:
self._helper.write('error %s nothing changed on remote\n'
% dest)
self._helper.write('\n')
self._helper.flush()
if not pushed:
data = False
elif data == 'always':
data = True
elif data == 'phase':
phases = self._repo.listkeys('phases')
drafts = {}
if not phases.get('publishing', False):
drafts = set(p for p, is_draft in phases.iteritems()
if int(is_draft))
if not drafts:
data = True
else:
def draft_commits():
for d in drafts:
c = self._store.changeset_ref(d)
if c:
yield '^%s^@' % c
for h in pushed.heads():
yield h
args = ['rev-list', '--ancestry-path', '--topo-order',
'--stdin']
pushed_drafts = tuple(
Git.iter(*args, stdin=draft_commits()))
# Theoretically, we could have commits with no
# metadata that the remote declares are public, while
# the rest of our push is in a draft state. That is
# however so unlikely that it's not worth the effort
# to support partial metadata storage.
data = not bool(pushed_drafts)
elif data == 'never':
data = False
self._store.close(rollback=not data)
def fsck(args):
parser = argparse.ArgumentParser()
parser.add_argument(
'--manifests', action='store_true',
help='Validate manifests hashes')
parser.add_argument(
'--files', action='store_true',
help='Validate files hashes')
parser.add_argument(
'commit', nargs='*',
help='Specific commit or changeset to check')
args = parser.parse_args(args)
status = {
'broken': False,
'fixed': False,
}
def info(message):
sys.stderr.write('\r')
print message
def fix(message):
status['fixed'] = True
info(message)
def report(message):
status['broken'] = True
info(message)
store = GitHgStore()
store.init_fast_import(lambda: FastImport())
if args.commit:
all_hg2git = {}
all_notes = set()
commits = set()
all_git_commits = {}
for c in args.commit:
data = store.read_changeset_data(c)
if data:
all_notes.add(c)
commits.add(c)
c = data['changeset']
commit = GitHgHelper.hg2git(c)
if commit == NULL_NODE_ID and not data:
info('Unknown commit or changeset: %s' % c)
return 1
if commit != NULL_NODE_ID:
all_hg2git[c] = commit, 'commit'
if not data:
data = store.read_changeset_data(commit)
commits.add(commit)
if data:
all_notes.add(commit)
all_git_commits = Git.iter(
'log', '--no-walk=unsorted', '--stdin', '--format=%T %H',
stdin=commits)
else:
all_hg2git = {
path.replace('/', ''): (filesha1, intern(typ))
for mode, typ, filesha1, path in
progress_iter('Reading %d mercurial to git mappings',
Git.ls_tree('refs/cinnabar/hg2git', recursive=True))
}
all_notes = set(path.replace('/', '') for mode, typ, filesha1, path in
progress_iter(
'Reading %d commit to changeset mappings',
Git.ls_tree('refs/notes/cinnabar',
recursive=True)))
manifest_commits = OrderedDict((m, None) for m in progress_iter(
'Reading %d manifest trees',
Git.iter('rev-list', '--full-history',
'--topo-order', 'refs/cinnabar/manifest'))
)
all_git_heads = Git.for_each_ref('refs/cinnabar/branches',
format='%(refname)')
all_git_commits = Git.iter('log', '--topo-order', '--full-history',
'--reverse', '--stdin', '--format=%T %H',
stdin=all_git_heads)
store._hg2git_cache = {p: s for p, (s, t) in all_hg2git.iteritems()}
seen_changesets = set()
seen_manifests = set()
seen_manifest_refs = {}
seen_files = set()
seen_notes = set()
hg_manifest = None
dag = gitdag()
for line in progress_iter('Checking %d changesets', all_git_commits):
tree, node = line.split(' ')
if node not in all_notes:
report('Missing note for git commit: ' + node)
continue
seen_notes.add(node)
changeset_data = store.read_changeset_data(node)
changeset = changeset_data['changeset']
if 'extra' in changeset_data:
extra = changeset_data['extra']
header, message = GitHgHelper.cat_file(
'commit', node).split('\n\n', 1)
header = dict(l.split(' ', 1) for l in header.splitlines())
if 'committer' in extra:
committer_info = store.hg_author_info(header['committer'])
committer = '%s %d %d' % committer_info
if (committer != extra['committer'] and
header['committer'] != extra['committer'] and
committer_info[0] != extra['committer']):
report('Committer mismatch between commit and metadata for'
' changeset %s' % changeset)
if committer == extra['committer']:
fix('Fixing useless committer metadata for changeset %s'
% changeset)
del changeset_data['extra']['committer']
store._changesets[changeset] = LazyString(node)
if header['committer'] != header['author'] and not extra:
fix('Fixing useless empty extra metadata for changeset %s'
% changeset)
del changeset_data['extra']
store._changesets[changeset] = LazyString(node)
seen_changesets.add(changeset)
changeset_ref = store.changeset_ref(changeset)
if not changeset_ref:
report('Missing changeset in hg2git branch: %s' % changeset)
elif str(changeset_ref) != node:
report('Commit mismatch for changeset %s\n'
' hg2git: %s\n commit: %s'
% (changeset, changeset_ref, node))
hg_changeset = store.changeset(changeset, include_parents=True)
sha1 = hg_changeset.sha1
if hg_changeset.node != sha1:
try_fixup = False
if (changeset, sha1) in (
('8c557b7c03a4a753e5c163038f04862e9f65fce1',
'249b59139de8e08abeb6c4e261a137c756e7af0e'),
('ffdee4a4eb7fc7cae80dfc4cb2fe0c3178773dcf',
'415e9d2eac83d508bf58a4df585c5f6b2b0f44ed'),
):
header = hg_changeset.data.split('\n', 4)
start = sum(len(h) for h in header[:3]) + 1
changeset_data['patch'] = ((start, start + 1, '1'),)
try_fixup = True
# Some know cases of corruptions involve a whitespace after the
# timezone. Adding an empty extra metadata works around those.
elif 'extra' not in changeset_data:
changeset_data['extra'] = {}
try_fixup = True
if try_fixup:
hg_changeset = store.changeset(changeset, include_parents=True)
sha1 = hg_changeset.sha1
if hg_changeset.node == sha1:
fix('Fixing known sha1 mismatch for changeset %s' %
changeset)
store._changesets[changeset] = LazyString(node)
if hg_changeset.node != sha1:
report('Sha1 mismatch for changeset %s' % changeset)
dag.add(hg_changeset.node,
(hg_changeset.parent1, hg_changeset.parent2),
changeset_data.get('extra', {}).get('branch', 'default'))
manifest = changeset_data['manifest']
if manifest in seen_manifests:
continue
seen_manifests.add(manifest)
manifest_ref = store.manifest_ref(manifest)
if manifest_ref:
seen_manifest_refs[manifest_ref] = manifest
if not manifest_ref:
report('Missing manifest in hg2git branch: %s' % manifest)
elif not args.commit and manifest_ref not in manifest_commits:
report('Missing manifest commit in manifest branch: %s' %
manifest_ref)
if args.manifests or args.files:
parents = tuple(
store.read_changeset_data(store.changeset_ref(p))['manifest']
for p in (hg_changeset.parent1, hg_changeset.parent2)
if p != NULL_NODE_ID
)
if args.manifests:
try:
with GitHgHelper.query('check-manifest', manifest,
*parents) as stdout:
if stdout.readline().strip() != 'ok':
report('Sha1 mismatch for manifest %s' % manifest)
except NoHelperException:
hg_manifest = store.manifest(manifest)
hg_manifest.set_parents(*parents)
if hg_manifest.node != hg_manifest.sha1:
report('Sha1 mismatch for manifest %s' % manifest)
git_ls = one(Git.ls_tree(manifest_ref, 'git'))
if git_ls:
mode, typ, sha1, path = git_ls
else:
header, message = GitHgHelper.cat_file(
'commit', manifest_ref).split('\n\n', 1)
header = dict(l.split(' ', 1) for l in header.splitlines())
if header['tree'] == EMPTY_TREE:
sha1 = EMPTY_TREE
else:
report('Missing git tree in manifest commit %s' % manifest_ref)
sha1 = None
if sha1 and sha1 != tree:
report('Tree mismatch between manifest commit %s and commit %s'
% (manifest_ref, node))
if args.files:
changes = get_changes(
manifest_ref, tuple(store.manifest_ref(p) for p in parents),
'hg')
for path, hg_file, hg_fileparents in changes:
if hg_file != NULL_NODE_ID and hg_file not in seen_files:
file = store.file(hg_file)
file.set_parents(*hg_fileparents)
if file.node != file.sha1:
report('Sha1 mismatch for file %s in manifest %s'
% (hg_file, manifest_ref))
seen_files.add(hg_file)
if args.files:
all_hg2git = set(all_hg2git.iterkeys())
else:
all_hg2git = set(k for k, (s, t) in all_hg2git.iteritems()
if t == 'commit')
adjusted = {}
if not args.commit:
dangling = set(manifest_commits) - set(seen_manifest_refs)
if dangling:
def iter_manifests():
removed_one = False
yielded = False
previous = None
for obj in reversed(manifest_commits):
if obj in dangling:
fix('Removing metadata commit %s with no hg2git entry'
% obj)
removed_one = True
else:
if removed_one:
yield obj, previous
yielded = True
previous = obj
if removed_one and not yielded:
yield obj, False
for obj, parent in progress_iter('Adjusting %d metadata commits',
iter_manifests()):
mark = store._fast_import.new_mark()
if parent is False:
Git.update_ref('refs/cinnabar/manifest', obj)
continue
elif parent:
parents = (adjusted.get(parent, parent),)
with store._fast_import.commit(
ref='refs/cinnabar/manifest',
parents=parents, mark=mark) as commit:
mode, typ, tree, path = store._fast_import.ls(obj)
commit.filemodify('', tree, typ='tree')
adjusted[obj] = Mark(mark)
dangling = all_hg2git - seen_changesets - seen_manifests - seen_files
if dangling or adjusted:
with store._fast_import.commit(
ref='refs/cinnabar/hg2git',
parents=('refs/cinnabar/hg2git^0',)) as commit:
for obj in dangling:
fix('Removing dangling metadata for ' + obj)
commit.filedelete(sha1path(obj))
for obj, mark in progress_iter(
'Updating hg2git for %d metadata commits',
adjusted.iteritems()):
commit.filemodify(sha1path(seen_manifest_refs[obj]), mark,
typ='commit')
dangling = all_notes - seen_notes
if dangling:
with store._fast_import.commit(
ref='refs/notes/cinnabar',
parents=('refs/notes/cinnabar^0',)) as commit:
for c in dangling:
fix('Removing dangling note for commit ' + c)
# That's brute force, but meh.
for l in range(0, 10):
commit.filedelete(sha1path(c, l))
if status['broken']:
info('Your git-cinnabar repository appears to be corrupted. There\n'
'are known issues in older revisions that have been fixed.\n'
'Please try running the following command to reset:\n'
' git cinnabar reclone\n\n'
'Please note this command may change the commit sha1s. Your\n'
'local branches will however stay untouched.\n'
'Please report any corruption that fsck would detect after a\n'
'reclone.')
if not args.commit:
info('Checking head references...')
computed_heads = defaultdict(set)
for branch, head in dag.all_heads():
computed_heads[branch].add(head)
for branch in sorted(dag.tags()):
stored_heads = store.heads({branch})
for head in computed_heads[branch] - stored_heads:
fix('Adding missing head %s in branch %s' %
(head, branch))
store.add_head(head)
for head in stored_heads - computed_heads[branch]:
fix('Removing non-head reference to %s in branch %s' %
(head, branch))
store._hgheads.remove((branch, head))
store.close()
if status['broken']:
return 1
if status['fixed']:
return 2
return 0
def fsck(args):
parser = argparse.ArgumentParser()
parser.add_argument('--manifests',
action='store_true',
help='Validate manifests hashes')
parser.add_argument('--files',
action='store_true',
help='Validate files hashes')
parser.add_argument('commit',
nargs='*',
help='Specific commit or changeset to check')
args = parser.parse_args(args)
status = {
'broken': False,
'fixed': False,
}
def info(message):
sys.stderr.write('\r')
print message
def fix(message):
status['fixed'] = True
info(message)
def report(message):
status['broken'] = True
info(message)
store = GitHgStore()
store.init_fast_import(lambda: FastImport())
if args.commit:
all_hg2git = {}
all_notes = set()
commits = set()
all_git_commits = {}
for c in args.commit:
data = store.read_changeset_data(c)
if data:
all_notes.add(c)
commits.add(c)
c = data['changeset']
commit = GitHgHelper.hg2git(c)
if commit == NULL_NODE_ID and not data:
info('Unknown commit or changeset: %s' % c)
return 1
if commit != NULL_NODE_ID:
all_hg2git[c] = commit, 'commit'
if not data:
data = store.read_changeset_data(commit)
commits.add(commit)
if data:
all_notes.add(commit)
all_git_commits = Git.iter('log',
'--no-walk=unsorted',
'--stdin',
'--format=%T %H',
stdin=commits)
else:
all_hg2git = {
path.replace('/', ''): (filesha1, intern(typ))
for mode, typ, filesha1, path in progress_iter(
'Reading %d mercurial to git mappings',
Git.ls_tree('refs/cinnabar/hg2git', recursive=True))
}
all_notes = set(
path.replace('/', '')
for mode, typ, filesha1, path in progress_iter(
'Reading %d commit to changeset mappings',
Git.ls_tree('refs/notes/cinnabar', recursive=True)))
manifest_commits = OrderedDict((m, None) for m in progress_iter(
'Reading %d manifest trees',
Git.iter('rev-list', '--full-history', '--topo-order',
'refs/cinnabar/manifest')))
all_git_heads = Git.for_each_ref('refs/cinnabar/branches',
format='%(refname)')
all_git_commits = Git.iter('log',
'--topo-order',
'--full-history',
'--reverse',
'--stdin',
'--format=%T %H',
stdin=all_git_heads)
store._hg2git_cache = {p: s for p, (s, t) in all_hg2git.iteritems()}
seen_changesets = set()
seen_manifests = set()
seen_manifest_refs = {}
seen_files = set()
seen_notes = set()
hg_manifest = None
dag = gitdag()
for line in progress_iter('Checking %d changesets', all_git_commits):
tree, node = line.split(' ')
if node not in all_notes:
report('Missing note for git commit: ' + node)
continue
seen_notes.add(node)
changeset_data = store.read_changeset_data(node)
changeset = changeset_data['changeset']
if 'extra' in changeset_data:
extra = changeset_data['extra']
header, message = GitHgHelper.cat_file('commit',
node).split('\n\n', 1)
header = dict(l.split(' ', 1) for l in header.splitlines())
if 'committer' in extra:
committer_info = store.hg_author_info(header['committer'])
committer = '%s %d %d' % committer_info
if (committer != extra['committer']
and header['committer'] != extra['committer']
and committer_info[0] != extra['committer']):
report('Committer mismatch between commit and metadata for'
' changeset %s' % changeset)
if committer == extra['committer']:
fix('Fixing useless committer metadata for changeset %s' %
changeset)
del changeset_data['extra']['committer']
store._changesets[changeset] = LazyString(node)
if header['committer'] != header['author'] and not extra:
fix('Fixing useless empty extra metadata for changeset %s' %
changeset)
del changeset_data['extra']
store._changesets[changeset] = LazyString(node)
seen_changesets.add(changeset)
changeset_ref = store.changeset_ref(changeset)
if not changeset_ref:
report('Missing changeset in hg2git branch: %s' % changeset)
elif str(changeset_ref) != node:
report('Commit mismatch for changeset %s\n'
' hg2git: %s\n commit: %s' %
(changeset, changeset_ref, node))
hg_changeset = store.changeset(changeset, include_parents=True)
sha1 = hg_changeset.sha1
if hg_changeset.node != sha1:
try_fixup = False
if (changeset, sha1) in (
('8c557b7c03a4a753e5c163038f04862e9f65fce1',
'249b59139de8e08abeb6c4e261a137c756e7af0e'),
('ffdee4a4eb7fc7cae80dfc4cb2fe0c3178773dcf',
'415e9d2eac83d508bf58a4df585c5f6b2b0f44ed'),
):
header = hg_changeset.data.split('\n', 4)
start = sum(len(h) for h in header[:3]) + 1
changeset_data['patch'] = ((start, start + 1, '1'), )
try_fixup = True
# Some know cases of corruptions involve a whitespace after the
# timezone. Adding an empty extra metadata works around those.
elif 'extra' not in changeset_data:
changeset_data['extra'] = {}
try_fixup = True
if try_fixup:
hg_changeset = store.changeset(changeset, include_parents=True)
sha1 = hg_changeset.sha1
if hg_changeset.node == sha1:
fix('Fixing known sha1 mismatch for changeset %s' %
changeset)
store._changesets[changeset] = LazyString(node)
if hg_changeset.node != sha1:
report('Sha1 mismatch for changeset %s' % changeset)
dag.add(hg_changeset.node,
(hg_changeset.parent1, hg_changeset.parent2),
changeset_data.get('extra', {}).get('branch', 'default'))
manifest = changeset_data['manifest']
if manifest in seen_manifests:
continue
seen_manifests.add(manifest)
manifest_ref = store.manifest_ref(manifest)
if manifest_ref:
seen_manifest_refs[manifest_ref] = manifest
if not manifest_ref:
report('Missing manifest in hg2git branch: %s' % manifest)
elif not args.commit and manifest_ref not in manifest_commits:
report('Missing manifest commit in manifest branch: %s' %
manifest_ref)
if args.manifests or args.files:
parents = tuple(
store.read_changeset_data(store.changeset_ref(p))['manifest']
for p in (hg_changeset.parent1, hg_changeset.parent2)
if p != NULL_NODE_ID)
if args.manifests:
try:
with GitHgHelper.query('check-manifest', manifest,
*parents) as stdout:
if stdout.readline().strip() != 'ok':
report('Sha1 mismatch for manifest %s' % manifest)
except NoHelperException:
hg_manifest = store.manifest(manifest)
hg_manifest.set_parents(*parents)
if hg_manifest.node != hg_manifest.sha1:
report('Sha1 mismatch for manifest %s' % manifest)
git_ls = one(Git.ls_tree(manifest_ref, 'git'))
if git_ls:
mode, typ, sha1, path = git_ls
else:
header, message = GitHgHelper.cat_file('commit',
manifest_ref).split(
'\n\n', 1)
header = dict(l.split(' ', 1) for l in header.splitlines())
if header['tree'] == EMPTY_TREE:
sha1 = EMPTY_TREE
else:
report('Missing git tree in manifest commit %s' % manifest_ref)
sha1 = None
if sha1 and sha1 != tree:
report('Tree mismatch between manifest commit %s and commit %s' %
(manifest_ref, node))
if args.files:
changes = get_changes(
manifest_ref, tuple(store.manifest_ref(p) for p in parents),
'hg')
for path, hg_file, hg_fileparents in changes:
if hg_file != NULL_NODE_ID and hg_file not in seen_files:
file = store.file(hg_file)
file.set_parents(*hg_fileparents)
if file.node != file.sha1:
report('Sha1 mismatch for file %s in manifest %s' %
(hg_file, manifest_ref))
seen_files.add(hg_file)
if args.files:
all_hg2git = set(all_hg2git.iterkeys())
else:
all_hg2git = set(k for k, (s, t) in all_hg2git.iteritems()
if t == 'commit')
adjusted = {}
if not args.commit:
dangling = set(manifest_commits) - set(seen_manifest_refs)
if dangling:
def iter_manifests():
removed_one = False
yielded = False
previous = None
for obj in reversed(manifest_commits):
if obj in dangling:
fix('Removing metadata commit %s with no hg2git entry'
% obj)
removed_one = True
else:
if removed_one:
yield obj, previous
yielded = True
previous = obj
if removed_one and not yielded:
yield obj, False
for obj, parent in progress_iter('Adjusting %d metadata commits',
iter_manifests()):
mark = store._fast_import.new_mark()
if parent is False:
Git.update_ref('refs/cinnabar/manifest', obj)
continue
elif parent:
parents = (adjusted.get(parent, parent), )
with store._fast_import.commit(ref='refs/cinnabar/manifest',
parents=parents,
mark=mark) as commit:
mode, typ, tree, path = store._fast_import.ls(obj)
commit.filemodify('', tree, typ='tree')
adjusted[obj] = Mark(mark)
dangling = all_hg2git - seen_changesets - seen_manifests - seen_files
if dangling or adjusted:
with store._fast_import.commit(
ref='refs/cinnabar/hg2git',
parents=('refs/cinnabar/hg2git^0', )) as commit:
for obj in dangling:
fix('Removing dangling metadata for ' + obj)
commit.filedelete(sha1path(obj))
for obj, mark in progress_iter(
'Updating hg2git for %d metadata commits',
adjusted.iteritems()):
commit.filemodify(sha1path(seen_manifest_refs[obj]),
mark,
typ='commit')
dangling = all_notes - seen_notes
if dangling:
with store._fast_import.commit(
ref='refs/notes/cinnabar',
parents=('refs/notes/cinnabar^0', )) as commit:
for c in dangling:
fix('Removing dangling note for commit ' + c)
# That's brute force, but meh.
for l in range(0, 10):
commit.filedelete(sha1path(c, l))
if status['broken']:
info('Your git-cinnabar repository appears to be corrupted. There\n'
'are known issues in older revisions that have been fixed.\n'
'Please try running the following command to reset:\n'
' git cinnabar reclone\n\n'
'Please note this command may change the commit sha1s. Your\n'
'local branches will however stay untouched.\n'
'Please report any corruption that fsck would detect after a\n'
'reclone.')
if not args.commit:
info('Checking head references...')
computed_heads = defaultdict(set)
for branch, head in dag.all_heads():
computed_heads[branch].add(head)
for branch in sorted(dag.tags()):
stored_heads = store.heads({branch})
for head in computed_heads[branch] - stored_heads:
fix('Adding missing head %s in branch %s' % (head, branch))
store.add_head(head)
for head in stored_heads - computed_heads[branch]:
fix('Removing non-head reference to %s in branch %s' %
(head, branch))
store._hgheads.remove((branch, head))
store.close()
if status['broken']:
return 1
if status['fixed']:
return 2
return 0
def findcommon(repo, store, hgheads):
logger = logging.getLogger('findcommon')
logger.debug(hgheads)
if not hgheads:
return set()
sample_size = 100
sample = _sample(hgheads, sample_size)
known = repo.known(unhexlify(h) for h in sample)
known = set(h for h, k in izip(sample, known) if k)
logger.info('initial sample size: %d' % len(sample))
if len(known) == len(hgheads):
logger.info('all heads known')
return hgheads
git_heads = set(store.changeset_ref(h) for h in hgheads)
git_known = set(store.changeset_ref(h) for h in known)
logger.debug('known (sub)set: (%d) %s', len(known),
LazyCall(sorted, git_known))
args = ['rev-list', '--topo-order', '--full-history', '--parents',
'--stdin']
def revs():
for h in git_known:
yield '^%s' % h
for h in git_heads:
if h not in git_known:
yield h
dag = gitdag(chain(Git.iter(*args, stdin=revs()), git_known))
dag.tag_nodes_and_parents(git_known, 'known')
def log_dag(tag):
if not logger.isEnabledFor(logging.DEBUG):
return
logger.debug('%s dag size: %d' % (
tag, sum(1 for n in dag.iternodes(tag))))
heads = sorted(dag.heads(tag))
logger.debug('%s dag heads: (%d) %s' % (tag, len(heads), heads))
roots = sorted(dag.roots(tag))
logger.debug('%s dag roots: (%d) %s' % (tag, len(roots), roots))
log_dag('unknown')
log_dag('known')
while True:
unknown = set(chain(dag.heads(), dag.roots()))
if not unknown:
break
sample = set(_sample(unknown, sample_size))
if len(sample) < sample_size:
sample |= set(_sample(set(dag.iternodes()),
sample_size - len(sample)))
sample = list(sample)
hg_sample = [store.hg_changeset(h) for h in sample]
known = repo.known(unhexlify(h) for h in hg_sample)
unknown = set(h for h, k in izip(sample, known) if not k)
known = set(h for h, k in izip(sample, known) if k)
logger.info('next sample size: %d' % len(sample))
logger.debug('known (sub)set: (%d) %s', len(known),
LazyCall(sorted, known))
logger.debug('unknown (sub)set: (%d) %s', len(unknown),
LazyCall(sorted, unknown))
dag.tag_nodes_and_parents(known, 'known')
dag.tag_nodes_and_children(unknown, 'unknown')
log_dag('unknown')
log_dag('known')
return [store.hg_changeset(h) for h in dag.heads('known')]
def do_cinnabarclone(repo, manifest, store, limit_schemes=True):
GRAFT = {
None: None,
b'false': False,
b'true': True,
}
try:
enable_graft = Git.config('cinnabar.graft',
remote=repo.remote,
values=GRAFT)
except InvalidConfig:
enable_graft = None
url = None
candidates = []
for line in manifest.splitlines():
line = line.strip()
if not line:
continue
spec, _, params = line.partition(b' ')
params = {
k: v
for k, _, v in (p.partition(b'=') for p in params.split())
}
graft = params.pop(b'graft', None)
if params:
# Future proofing: ignore lines with unknown params, even if we
# support some that are present.
continue
# When grafting, ignore lines without a graft revision.
if store._graft and not graft:
continue
# When explicitly disabling graft, ignore lines with a graft revision.
if enable_graft is False and graft:
continue
graft = graft.split(b',') if graft else []
graft_u = []
for g in graft:
if SHA1_RE.match(g):
graft_u.append(g.decode('ascii'))
if len(graft) != len(graft_u):
continue
if graft:
revs = list(Git.iter('rev-parse', '--revs-only', *graft_u))
if len(revs) != len(graft):
continue
# We apparently have all the grafted revisions locally, ensure
# they're actually reachable.
if not any(
Git.iter('rev-list',
'--branches',
'--tags',
'--remotes',
'--max-count=1',
'--ancestry-path',
'--stdin',
stdin=(b'^%s^@' % c for c in graft),
stderr=open(os.devnull, 'wb'))):
continue
candidates.append((spec, len(graft) != 0))
if enable_graft is not False:
graft_filters = [True, False]
else:
graft_filters = [False]
for graft_filter in graft_filters:
for spec, graft in candidates:
if graft == graft_filter:
url, _, branch = spec.partition(b'#')
url, branch = (url.split(b'#', 1) + [None])[:2]
if url:
break
if url:
break
if not url:
logging.warn('Server advertizes cinnabarclone but didn\'t provide '
'a git repository url to fetch from.')
return False
parsed_url = urlparse(url)
if limit_schemes and parsed_url.scheme not in (b'http', b'https', b'git'):
logging.warn('Server advertizes cinnabarclone but provided a non '
'http/https git repository. Skipping.')
return False
sys.stderr.write('Fetching cinnabar metadata from %s\n' % fsdecode(url))
sys.stderr.flush()
return store.merge(url, repo.url(), branch)
def push(repo, store, what, repo_heads, repo_branches):
store.init_fast_import()
def heads():
for sha1 in store.heads(repo_branches):
yield '^%s' % store.changeset_ref(sha1)
def local_bases():
for c in Git.iter('rev-list', '--stdin', '--topo-order',
'--full-history', '--boundary',
*(w for w in what if w), stdin=heads()):
if c[0] != '-':
continue
yield store.hg_changeset(c[1:])
for w in what:
rev = store.hg_changeset(w)
if rev:
yield rev
common = findcommon(repo, store, set(local_bases()))
logging.info('common: %s' % common)
def revs():
for sha1 in common:
yield '^%s' % store.changeset_ref(sha1)
push_commits = list(Git.iter('rev-list', '--stdin', '--topo-order',
'--full-history', '--parents', '--reverse',
*(w for w in what if w), stdin=revs()))
pushed = False
if push_commits:
has_root = any(len(p) == 40 for p in push_commits)
force = all(v[1] for v in what.values())
if has_root and repo_heads:
if not force:
raise Exception('Cannot push a new root')
else:
logging.warn('Pushing a new root')
if force:
repo_heads = ['force']
else:
if not repo_heads:
repo_heads = [NULL_NODE_ID]
repo_heads = [unhexlify(h) for h in repo_heads]
if repo.local():
repo.local().ui.setconfig('server', 'validate', True)
b2caps = bundle2caps(repo) if unbundle20 else {}
if b2caps and (repo.url().startswith(('http://', 'https://')) or
not isinstance(repo, HelperRepo)):
b2caps['replycaps'] = True
cg = create_bundle(store, push_commits, b2caps)
if not isinstance(repo, HelperRepo):
cg = util.chunkbuffer(cg)
if not b2caps:
cg = cg1unpacker(cg, 'UN')
reply = repo.unbundle(cg, repo_heads, '')
if unbundle20 and isinstance(reply, unbundle20):
parts = iter(reply.iterparts())
for part in parts:
if part.type == 'output':
sys.stderr.write(part.read())
elif part.type == 'reply:changegroup':
# TODO: should check params['in-reply-to']
reply = int(part.params['return'])
else:
logging.getLogger('bundle2').warning(
'ignoring bundle2 part: %s', part.type)
pushed = reply != 0
return gitdag(push_commits) if pushed else ()
def main(args):
logger = logging.getLogger('-')
logger.info(args)
assert len(args) == 2
remote, url = args
git_dir = os.environ.get('GIT_DIR')
if Git.config('core.ignorecase', 'bool') == 'true':
sys.stderr.write(
'Your git configuration has core.ignorecase set to "true".\n'
'Usually, this means git detected the file system is case '
'insensitive.\n'
'Git-cinnabar does not support this setup.\n'
'Either use a case sensitive file system or set '
'core.ignorecase to "false".\n'
)
git_work_tree = os.path.dirname(git_dir)
if os.path.abspath(os.getcwd() + os.sep).startswith(
os.path.abspath(git_work_tree) + os.sep) or \
remote == 'hg::' + url or tuple(
Git.for_each_ref('refs/remotes/%s' % remote)):
sys.stderr.write(
'Use the following command to reclone:\n'
' git cinnabar reclone\n'
)
else:
sys.stderr.write(
'Use the following command to clone:\n'
' git -c core.ignorecase=false clone%(args)s hg::%(url)s '
'%(dir)s\n'
% {
'dir': git_work_tree,
'url': url,
'args': ' -o ' + remote if remote != 'origin' else ''
}
)
return 1
repo = get_repo(url)
store = GitHgStore()
logger.info(LazyString(lambda: '%s' % store.heads()))
helper = IOLogger(logging.getLogger('remote-helper'),
sys.stdin, sys.stdout)
branchmap = None
bookmarks = {}
HEAD = 'branches/default/tip'
while True:
cmd, args = read_cmd(helper)
if not cmd:
break
if cmd == 'capabilities':
assert not args
helper.write(
'option\n'
'import\n'
'bidi-import\n'
'push\n'
'refspec refs/heads/branches/*:'
'refs/cinnabar/refs/heads/branches/*\n'
'refspec refs/heads/bookmarks/*:'
'refs/cinnabar/refs/heads/bookmarks/*\n'
'refspec HEAD:refs/cinnabar/HEAD\n'
'\n'
)
helper.flush()
elif cmd == 'list':
assert not args or args == ['for-push']
if repo.capable('batch'):
batch = repo.batch()
branchmap = batch.branchmap()
heads = batch.heads()
bookmarks = batch.listkeys('bookmarks')
batch.submit()
branchmap = branchmap.value
heads = heads.value
bookmarks = bookmarks.value
else:
while True:
branchmap = repo.branchmap()
heads = repo.heads()
if heads == ['\0' * 20]:
heads = []
# Some branch heads can be non-heads topologically, but if
# some heads don't appear in the branchmap, then something
# was pushed to the repo between branchmap() and heads()
if set(heads).issubset(set(chain(*branchmap.values()))):
break
bookmarks = repo.listkeys('bookmarks')
branchmap = BranchMap(store, branchmap, heads)
unknowns = False
for branch in sorted(branchmap.names()):
branch_tip = branchmap.tip(branch)
for head in sorted(branchmap.heads(branch)):
sha1 = branchmap.git_sha1(head)
if sha1 == '?':
unknowns = True
if head == branch_tip:
continue
helper.write('%s refs/heads/branches/%s/%s\n' % (
sha1,
branch,
head,
))
if branch_tip:
helper.write('%s refs/heads/branches/%s/tip\n' % (
branchmap.git_sha1(branch_tip),
branch,
))
for name, sha1 in sorted(bookmarks.iteritems()):
ref = store.changeset_ref(sha1)
helper.write(
'%s refs/heads/bookmarks/%s\n'
% (ref if ref else '?', name)
)
if not unknowns:
for tag, ref in sorted(store.tags(branchmap.heads())):
helper.write('%s refs/tags/%s\n' % (ref, tag))
if '@' in bookmarks:
HEAD = 'bookmarks/@'
helper.write(
'@refs/heads/%s HEAD\n'
'\n'
% HEAD
)
helper.flush()
elif cmd == 'option':
assert len(args) == 2
name, value = args
if name == 'progress':
if value == 'true':
cinnabar.util.progress = True
helper.write('ok\n')
elif value == 'false':
cinnabar.util.progress = False
helper.write('ok\n')
else:
helper.write('unsupported\n')
else:
helper.write('unsupported\n')
helper.flush()
elif cmd == 'import':
try:
reflog = os.path.join(git_dir, 'logs', 'refs', 'cinnabar')
mkpath(reflog)
open(os.path.join(reflog, 'hg2git'), 'a').close()
open(os.path.join(reflog, 'manifest'), 'a').close()
assert len(args) == 1
refs = args
while cmd:
assert cmd == 'import'
cmd, args = read_cmd(helper)
assert args is None or len(args) == 1
if args:
refs.extend(args)
except:
# If anything wrong happens before we got all the import
# commands, we risk git picking the existing refs/cinnabar
# refs. Remove them.
for line in Git.for_each_ref('refs/cinnabar/refs/heads',
'refs/cinnabar/HEAD',
format='%(refname)'):
Git.delete_ref(ref)
raise
try:
def resolve_head(head):
if head.startswith('refs/heads/branches/'):
head = head[20:]
if head[-4:] == '/tip':
return branchmap.tip(head[:-4])
return head[-40:]
if head.startswith('refs/heads/bookmarks/'):
head = head[21:]
return bookmarks[head]
if head == 'HEAD':
return bookmarks.get('@') or branchmap.tip('default')
return None
wanted_refs = {k: v for k, v in (
(h, resolve_head(h)) for h in refs) if v}
heads = wanted_refs.values()
if not heads:
heads = branchmap.heads()
# Older versions would create a symbolic ref for
# refs/remote-hg/HEAD. Newer versions don't, and
# Git.update_ref doesn't remove the symbolic ref, so it needs
# to be removed first.
# Since git symbolic-ref only throws an error when the ref is
# not symbolic, just try to remove the symbolic ref every time
# and ignore errors.
tuple(Git.iter('symbolic-ref', '-d', 'refs/remote-hg/HEAD',
stderr=open(os.devnull, 'wb')))
refs_orig = {}
for line in Git.for_each_ref(
'refs/cinnabar/refs/heads', 'refs/cinnabar/HEAD',
format='%(objectname) %(refname)'):
sha1, ref = line.split(' ', 1)
refs_orig[ref] = sha1
except:
# If anything wrong happens before we actually pull, we risk
# git pucking the existing refs/cinnabar refs. Remove them.
# Unlike in the case above, we now have the list of refs git
# is expected, so we can just remove those.
for ref in refs:
Git.delete_ref('refs/cinnabar/' + ref)
raise
try:
store.init_fast_import(FastImport(sys.stdin, sys.stdout))
getbundle(repo, store, heads, branchmap)
except:
wanted_refs = {}
raise
finally:
for ref, value in wanted_refs.iteritems():
ref = 'refs/cinnabar/' + ref
if ref not in refs_orig or refs_orig[ref] != value:
Git.update_ref(ref, store.changeset_ref(value))
for ref in refs_orig:
if ref[14:] not in wanted_refs:
Git.delete_ref(ref)
store.close()
if not remote.startswith('hg::'):
prune = 'remote.%s.prune' % remote
if (Git.config(prune) != 'true' and
Git.config('fetch.prune') != 'true'):
sys.stderr.write(
'It is recommended that you set "%(conf)s" or '
'"fetch.prune" to "true".\n'
' git config %(conf)s true\n'
'or\n'
' git config fetch.prune true\n'
% {'conf': prune}
)
if store.tag_changes:
sys.stderr.write(
'\nRun the following command to update remote tags:\n')
if not remote.startswith('hg::'):
sys.stderr.write(' git remote update %s\n' % remote)
else:
sys.stderr.write(' git fetch --tags %s\n' % remote)
elif cmd == 'push':
if not remote.startswith('hg::'):
data_pref = 'remote.%s.cinnabar-data' % remote
data = Git.config(data_pref) or 'phase'
else:
data = 'phase'
if data not in ('never', 'phase', 'always'):
sys.stderr.write('Invalid value for %s: %s\n'
% (data_pref, data))
return 1
refspecs = []
refspecs.extend(args)
while True:
cmd, args = read_cmd(helper)
if not cmd:
break
assert cmd == 'push'
refspecs.extend(args)
pushes = {s.lstrip('+'): (d, s.startswith('+'))
for s, d in (r.split(':', 1) for r in refspecs)}
if isinstance(repo, bundlerepo):
for source, (dest, force) in pushes.iteritems():
helper.write('error %s Cannot push to a bundle file\n'
% dest)
helper.write('\n')
helper.flush()
else:
repo_heads = branchmap.heads()
PushStore.adopt(store)
pushed = push(repo, store, pushes, repo_heads,
branchmap.names())
status = {}
for source, (dest, _) in pushes.iteritems():
if dest.startswith('refs/tags/'):
if source:
status[dest] = 'Pushing tags is unsupported'
else:
status[dest] = \
'Deleting remote tags is unsupported'
continue
if not dest.startswith('refs/heads/bookmarks/'):
if source:
status[dest] = bool(len(pushed))
else:
status[dest] = \
'Deleting remote branches is unsupported'
continue
name = dest[21:]
if source:
source = store.hg_changeset(Git.resolve_ref(source)) \
or ''
status[dest] = repo.pushkey(
'bookmarks', name, bookmarks.get(name, ''), source)
for source, (dest, force) in pushes.iteritems():
if status[dest] is True:
helper.write('ok %s\n' % dest)
elif status[dest]:
helper.write('error %s %s\n' % (dest, status[dest]))
else:
helper.write('error %s nothing changed on remote\n'
% dest)
helper.write('\n')
helper.flush()
if not pushed:
data = False
elif data == 'always':
data = True
elif data == 'phase':
phases = repo.listkeys('phases')
drafts = {}
if not phases.get('publishing', False):
drafts = set(p for p, is_draft in phases.iteritems()
if int(is_draft))
if not drafts:
data = True
else:
def draft_commits():
for d in drafts:
c = store.changeset_ref(d)
if c:
yield '^%s^@' % c
for h in pushed.heads():
yield h
args = ['rev-list', '--ancestry-path', '--topo-order',
'--stdin']
pushed_drafts = tuple(
Git.iter(*args, stdin=draft_commits()))
# Theoretically, we could have commits with no
# metadata that the remote declares are public, while
# the rest of our push is in a draft state. That is
# however so unlikely that it's not worth the effort
# to support partial metadata storage.
data = not bool(pushed_drafts)
elif data == 'never':
data = False
store.close(rollback=not data)
store.close()
def main(args):
logger = logging.getLogger('-')
logger.info(args)
assert len(args) == 2
remote, url = args
git_dir = os.environ.get('GIT_DIR')
if Git.config('core.ignorecase', 'bool') == 'true':
sys.stderr.write(
'Your git configuration has core.ignorecase set to "true".\n'
'Usually, this means git detected the file system is case '
'insensitive.\n'
'Git-cinnabar does not support this setup.\n'
'Either use a case sensitive file system or set '
'core.ignorecase to "false".\n')
git_work_tree = os.path.dirname(git_dir)
if os.path.abspath(os.getcwd() + os.sep).startswith(
os.path.abspath(git_work_tree) + os.sep) or \
remote == 'hg::' + url or tuple(
Git.for_each_ref('refs/remotes/%s' % remote)):
sys.stderr.write('Use the following command to reclone:\n'
' git cinnabar reclone\n')
else:
sys.stderr.write(
'Use the following command to clone:\n'
' git -c core.ignorecase=false clone%(args)s hg::%(url)s '
'%(dir)s\n' % {
'dir': git_work_tree,
'url': url,
'args': ' -o ' + remote if remote != 'origin' else ''
})
return 1
repo = get_repo(url)
store = GitHgStore()
logger.info(LazyString(lambda: '%s' % store.heads()))
helper = IOLogger(logging.getLogger('remote-helper'), sys.stdin,
sys.stdout)
branchmap = None
bookmarks = {}
HEAD = 'branches/default/tip'
while True:
cmd, args = read_cmd(helper)
if not cmd:
break
if cmd == 'capabilities':
assert not args
helper.write('option\n'
'import\n'
'bidi-import\n'
'push\n'
'refspec refs/heads/branches/*:'
'refs/cinnabar/refs/heads/branches/*\n'
'refspec refs/heads/bookmarks/*:'
'refs/cinnabar/refs/heads/bookmarks/*\n'
'refspec HEAD:refs/cinnabar/HEAD\n'
'\n')
helper.flush()
elif cmd == 'list':
assert not args or args == ['for-push']
if repo.capable('batch'):
batch = repo.batch()
branchmap = batch.branchmap()
heads = batch.heads()
bookmarks = batch.listkeys('bookmarks')
batch.submit()
branchmap = branchmap.value
heads = heads.value
bookmarks = bookmarks.value
else:
while True:
branchmap = repo.branchmap()
heads = repo.heads()
if heads == ['\0' * 20]:
heads = []
# Some branch heads can be non-heads topologically, but if
# some heads don't appear in the branchmap, then something
# was pushed to the repo between branchmap() and heads()
if set(heads).issubset(set(chain(*branchmap.values()))):
break
bookmarks = repo.listkeys('bookmarks')
branchmap = BranchMap(store, branchmap, heads)
unknowns = False
for branch in sorted(branchmap.names()):
branch_tip = branchmap.tip(branch)
for head in sorted(branchmap.heads(branch)):
sha1 = branchmap.git_sha1(head)
if sha1 == '?':
unknowns = True
if head == branch_tip:
continue
helper.write('%s refs/heads/branches/%s/%s\n' % (
sha1,
branch,
head,
))
if branch_tip:
helper.write('%s refs/heads/branches/%s/tip\n' % (
branchmap.git_sha1(branch_tip),
branch,
))
for name, sha1 in sorted(bookmarks.iteritems()):
ref = store.changeset_ref(sha1)
helper.write('%s refs/heads/bookmarks/%s\n' %
(ref if ref else '?', name))
if not unknowns:
for tag, ref in sorted(store.tags(branchmap.heads())):
helper.write('%s refs/tags/%s\n' % (ref, tag))
if '@' in bookmarks:
HEAD = 'bookmarks/@'
helper.write('@refs/heads/%s HEAD\n' '\n' % HEAD)
helper.flush()
elif cmd == 'option':
assert len(args) == 2
name, value = args
if name == 'progress':
if value == 'true':
cinnabar.util.progress = True
helper.write('ok\n')
elif value == 'false':
cinnabar.util.progress = False
helper.write('ok\n')
else:
helper.write('unsupported\n')
else:
helper.write('unsupported\n')
helper.flush()
elif cmd == 'import':
try:
reflog = os.path.join(git_dir, 'logs', 'refs', 'cinnabar')
mkpath(reflog)
open(os.path.join(reflog, 'hg2git'), 'a').close()
open(os.path.join(reflog, 'manifest'), 'a').close()
assert len(args) == 1
refs = args
while cmd:
assert cmd == 'import'
cmd, args = read_cmd(helper)
assert args is None or len(args) == 1
if args:
refs.extend(args)
except:
# If anything wrong happens before we got all the import
# commands, we risk git picking the existing refs/cinnabar
# refs. Remove them.
for line in Git.for_each_ref('refs/cinnabar/refs/heads',
'refs/cinnabar/HEAD',
format='%(refname)'):
Git.delete_ref(ref)
raise
try:
def resolve_head(head):
if head.startswith('refs/heads/branches/'):
head = head[20:]
if head[-4:] == '/tip':
return branchmap.tip(head[:-4])
return head[-40:]
if head.startswith('refs/heads/bookmarks/'):
head = head[21:]
return bookmarks[head]
if head == 'HEAD':
return bookmarks.get('@') or branchmap.tip('default')
return None
wanted_refs = {
k: v
for k, v in ((h, resolve_head(h)) for h in refs) if v
}
heads = wanted_refs.values()
if not heads:
heads = branchmap.heads()
# Older versions would create a symbolic ref for
# refs/remote-hg/HEAD. Newer versions don't, and
# Git.update_ref doesn't remove the symbolic ref, so it needs
# to be removed first.
# Since git symbolic-ref only throws an error when the ref is
# not symbolic, just try to remove the symbolic ref every time
# and ignore errors.
tuple(
Git.iter('symbolic-ref',
'-d',
'refs/remote-hg/HEAD',
stderr=open(os.devnull, 'wb')))
refs_orig = {}
for line in Git.for_each_ref(
'refs/cinnabar/refs/heads',
'refs/cinnabar/HEAD',
format='%(objectname) %(refname)'):
sha1, ref = line.split(' ', 1)
refs_orig[ref] = sha1
except:
# If anything wrong happens before we actually pull, we risk
# git pucking the existing refs/cinnabar refs. Remove them.
# Unlike in the case above, we now have the list of refs git
# is expected, so we can just remove those.
for ref in refs:
Git.delete_ref('refs/cinnabar/' + ref)
raise
try:
store.init_fast_import(FastImport(sys.stdin, sys.stdout))
getbundle(repo, store, heads, branchmap)
except:
wanted_refs = {}
raise
finally:
for ref, value in wanted_refs.iteritems():
ref = 'refs/cinnabar/' + ref
if ref not in refs_orig or refs_orig[ref] != value:
Git.update_ref(ref, store.changeset_ref(value))
for ref in refs_orig:
if ref[14:] not in wanted_refs:
Git.delete_ref(ref)
store.close()
if not remote.startswith('hg::'):
prune = 'remote.%s.prune' % remote
if (Git.config(prune) != 'true'
and Git.config('fetch.prune') != 'true'):
sys.stderr.write(
'It is recommended that you set "%(conf)s" or '
'"fetch.prune" to "true".\n'
' git config %(conf)s true\n'
'or\n'
' git config fetch.prune true\n' % {'conf': prune})
if store.tag_changes:
sys.stderr.write(
'\nRun the following command to update remote tags:\n')
if not remote.startswith('hg::'):
sys.stderr.write(' git remote update %s\n' % remote)
else:
sys.stderr.write(' git fetch --tags %s\n' % remote)
elif cmd == 'push':
if not remote.startswith('hg::'):
data_pref = 'remote.%s.cinnabar-data' % remote
data = Git.config(data_pref) or 'phase'
else:
data = 'phase'
if data not in ('never', 'phase', 'always'):
sys.stderr.write('Invalid value for %s: %s\n' %
(data_pref, data))
return 1
refspecs = []
refspecs.extend(args)
while True:
cmd, args = read_cmd(helper)
if not cmd:
break
assert cmd == 'push'
refspecs.extend(args)
pushes = {
s.lstrip('+'): (d, s.startswith('+'))
for s, d in (r.split(':', 1) for r in refspecs)
}
if isinstance(repo, bundlerepo):
for source, (dest, force) in pushes.iteritems():
helper.write('error %s Cannot push to a bundle file\n' %
dest)
helper.write('\n')
helper.flush()
else:
repo_heads = branchmap.heads()
PushStore.adopt(store)
pushed = push(repo, store, pushes, repo_heads,
branchmap.names())
status = {}
for source, (dest, _) in pushes.iteritems():
if dest.startswith('refs/tags/'):
if source:
status[dest] = 'Pushing tags is unsupported'
else:
status[dest] = \
'Deleting remote tags is unsupported'
continue
if not dest.startswith('refs/heads/bookmarks/'):
if source:
status[dest] = bool(len(pushed))
else:
status[dest] = \
'Deleting remote branches is unsupported'
continue
name = dest[21:]
if source:
source = store.hg_changeset(Git.resolve_ref(source)) \
or ''
status[dest] = repo.pushkey('bookmarks', name,
bookmarks.get(name, ''),
source)
for source, (dest, force) in pushes.iteritems():
if status[dest] is True:
helper.write('ok %s\n' % dest)
elif status[dest]:
helper.write('error %s %s\n' % (dest, status[dest]))
else:
helper.write('error %s nothing changed on remote\n' %
dest)
helper.write('\n')
helper.flush()
if not pushed:
data = False
elif data == 'always':
data = True
elif data == 'phase':
phases = repo.listkeys('phases')
drafts = {}
if not phases.get('publishing', False):
drafts = set(p for p, is_draft in phases.iteritems()
if int(is_draft))
if not drafts:
data = True
else:
def draft_commits():
for d in drafts:
c = store.changeset_ref(d)
if c:
yield '^%s^@' % c
for h in pushed.heads():
yield h
args = [
'rev-list', '--ancestry-path', '--topo-order',
'--stdin'
]
pushed_drafts = tuple(
Git.iter(*args, stdin=draft_commits()))
# Theoretically, we could have commits with no
# metadata that the remote declares are public, while
# the rest of our push is in a draft state. That is
# however so unlikely that it's not worth the effort
# to support partial metadata storage.
data = not bool(pushed_drafts)
elif data == 'never':
data = False
store.close(rollback=not data)
store.close()
def test_module_version(self):
module = one(Git.iter(
'ls-tree', 'HEAD', 'cinnabar',
cwd=os.path.join(os.path.dirname(__file__), '..')))
self.assertEqual(CmdVersion.module_version(),
split_ls_tree(module)[2])
def test_helper_version(self):
helper = one(Git.iter(
'ls-tree', 'HEAD', 'helper',
cwd=os.path.join(os.path.dirname(__file__), '..')))
self.assertEqual(CmdVersion.helper_version()[1],
split_ls_tree(helper)[2])
def helper_hash(head='HEAD'):
from cinnabar.git import Git, split_ls_tree
from cinnabar.util import one
return split_ls_tree(one(Git.iter(
'ls-tree', head, 'helper',
cwd=os.path.join(os.path.dirname(__file__), '..'))))[2].decode()
