def reclone(args):
'''reclone all mercurial remotes'''
from cinnabar.cmd.rollback import do_rollback
git_config = {}
metadata_commit = Git.resolve_ref('refs/cinnabar/metadata')
if metadata_commit:
git_config['cinnabar.previous-metadata'] = \
metadata_commit.decode('ascii')
# TODO: Avoid resetting at all, possibly leaving the repo with no metadata
# if this is interrupted somehow.
do_rollback(NULL_NODE_ID.decode('ascii'))
for line in Git.iter('config', '--get-regexp', 'remote\..*\.url'):
config, url = line.split()
name = config[len('remote.'):-len('.url')]
skip_pref = 'remote.%s.skipDefaultUpdate' % name.decode('ascii')
if (url.startswith((b'hg::', b'hg://'))
and Git.config(skip_pref) != 'true'):
Git.run('remote',
'update',
'--prune',
fsdecode(name),
config=git_config)
git_config = {}
print('Please note that reclone left your local branches untouched.')
print('They may be based on entirely different commits.')
def setUp(self):
self.git_dir = os.environ.get('GIT_DIR')
tmpdir = tempfile.mkdtemp()
Git.run('init', '--bare', tmpdir, stdout=open(os.devnull, 'w'))
os.environ['GIT_DIR'] = tmpdir
os.environ['GIT_CINNABAR_EXPERIMENTS'] = \
'store' if self.NEW_STORE else ''
self.assertEquals(
GitHgHelper.supports((b'store', b'new')), self.NEW_STORE)
def reclone(args):
'''reclone all mercurial remotes'''
from cinnabar.cmd.rollback import do_rollback
do_rollback(NULL_NODE_ID)
for line in Git.iter('config', '--get-regexp', 'remote\..*\.url'):
config, url = line.split()
name = config[len('remote.'):-len('.url')]
skip_pref = 'remote.%s.skipDefaultUpdate' % name
if (url.startswith(('hg::', 'hg://'))
and Git.config(skip_pref) != 'true'):
Git.run('remote', 'update', '--prune', name)
print 'Please note that reclone left your local branches untouched.'
print 'They may be based on entirely different commits.'
def main(args):
cmd = args.pop(0)
if cmd == 'data':
store = GitHgStore()
if args[0] == '-c':
sys.stdout.write(store.changeset(args[1]).data)
elif args[0] == '-m':
sys.stdout.write(store.manifest(args[1]).data)
store.close()
elif cmd == 'fsck':
return fsck(args)
elif cmd == 'reclone':
for ref in Git.for_each_ref('refs/cinnabar',
'refs/remote-hg',
'refs/notes/cinnabar',
'refs/notes/remote-hg/git2hg',
format='%(refname)'):
Git.delete_ref(ref)
Git.close()
for line in Git.iter('config', '--get-regexp', 'remote\..*\.url'):
config, url = line.split()
name = config[len('remote.'):-len('.url')]
skip_pref = 'remote.%s.skipDefaultUpdate' % name
if (url.startswith('hg::')
and Git.config(skip_pref, 'bool') != 'true'):
Git.run('remote', 'update', '--prune', name)
print 'Please note that reclone left your local branches untouched.'
print 'They may be based on entirely different commits.'
elif cmd == 'hg2git':
for arg in args:
print GitHgHelper.hg2git(arg)
elif cmd == 'git2hg':
for arg in args:
data = GitHgHelper.git2hg(arg)
if data:
data = ChangesetData.parse(data)
print data.get('changeset', NULL_NODE_ID)
else:
print NULL_NODE_ID
else:
print >> sys.stderr, 'Unknown command:', cmd
return 1
def main(args):
cmd = args.pop(0)
if cmd == 'data':
store = GitHgStore()
if args[0] == '-c':
sys.stdout.write(store.changeset(args[1]).data)
elif args[0] == '-m':
sys.stdout.write(store.manifest(args[1]).data)
store.close()
elif cmd == 'fsck':
return fsck(args)
elif cmd == 'reclone':
for ref in Git.for_each_ref('refs/cinnabar', 'refs/remote-hg',
'refs/notes/cinnabar',
'refs/notes/remote-hg/git2hg',
format='%(refname)'):
Git.delete_ref(ref)
Git.close()
for line in Git.iter('config', '--get-regexp', 'remote\..*\.url'):
config, url = line.split()
name = config[len('remote.'):-len('.url')]
skip_pref = 'remote.%s.skipDefaultUpdate' % name
if (url.startswith('hg::') and
Git.config(skip_pref, 'bool') != 'true'):
Git.run('remote', 'update', '--prune', name)
print 'Please note that reclone left your local branches untouched.'
print 'They may be based on entirely different commits.'
elif cmd == 'hg2git':
for arg in args:
print GitHgHelper.hg2git(arg)
elif cmd == 'git2hg':
for arg in args:
data = GitHgHelper.git2hg(arg)
if data:
data = ChangesetData.parse(data)
print data.get('changeset', NULL_NODE_ID)
else:
print NULL_NODE_ID
else:
print >>sys.stderr, 'Unknown command:', cmd
return 1
def reclone(args):
'''reclone all mercurial remotes'''
from cinnabar.cmd.rollback import do_rollback
git_config = {}
metadata_commit = Git.resolve_ref('refs/cinnabar/metadata')
if metadata_commit:
git_config['cinnabar.previous-metadata'] = metadata_commit
# TODO: Avoid resetting at all, possibly leaving the repo with no metadata
# if this is interrupted somehow.
do_rollback(NULL_NODE_ID)
for line in Git.iter('config', '--get-regexp', 'remote\..*\.url'):
config, url = line.split()
name = config[len('remote.'):-len('.url')]
skip_pref = 'remote.%s.skipDefaultUpdate' % name
if (url.startswith(('hg::', 'hg://')) and
Git.config(skip_pref) != 'true'):
Git.run('remote', 'update', '--prune', name, config=git_config)
git_config = {}
print 'Please note that reclone left your local branches untouched.'
print 'They may be based on entirely different commits.'
def download(args):
'''download a prebuilt helper'''
helper = 'git-cinnabar-helper'
system = args.system
machine = args.machine
if system.startswith('MSYS_NT'):
system = 'Windows'
if system == 'Darwin':
system = 'macOS'
elif system == 'Windows':
helper += '.exe'
if machine == 'AMD64':
machine = 'x86_64'
available = (
('Linux', 'x86_64'),
('macOS', 'x86_64'),
('Windows', 'x86_64'),
('Windows', 'x86'),
)
if args.list:
for system, machine in available:
print "%s/%s" % (system, machine)
return 0
if (system, machine) not in available:
print >>sys.stderr, 'No download available for %s/%s' % (system,
machine)
return 1
if args.dev is False:
version = VERSION
if version.endswith('a'):
# For version x.y.za, download a development helper
args.dev = ''
script_path = os.path.dirname(os.path.abspath(sys.argv[0]))
if args.dev is not False:
sha1 = helper_hash()
if sha1 is None:
print >>sys.stderr, (
'Cannot find the right development helper for this '
'version of git cinnabar.')
return 1
url = 'https://index.taskcluster.net/v1/task/github'
url += '.glandium.git-cinnabar.helper.'
url += '{}.{}.{}.{}'.format(
sha1, system.lower(), machine,
args.dev.lower() if args.dev else '').rstrip('.')
url += '/artifacts/public/{}'.format(helper)
else:
if system in ('Windows', 'macOS'):
ext = 'zip'
else:
ext = 'tar.xz'
REPO_BASE = 'https://github.com/glandium'
url = '%s/git-cinnabar/releases/download/%s/git-cinnabar.%s.%s.%s' % (
REPO_BASE, version, system.lower(), machine.lower(), ext)
if args.url:
print url
return 0
if args.output:
d = os.path.dirname(args.output)
else:
d = script_path
if not os.access(d, os.W_OK):
d = os.path.join(os.path.expanduser('~'), '.git-cinnabar')
try:
os.makedirs(d)
except Exception:
pass
if not os.path.isdir(d):
print >>sys.stderr, (
'Cannot write to either %s or %s.' % (d, script_path))
return 1
print 'Downloading from %s...' % url
try:
reader = HTTPReader(url)
except HTTPError:
# Try again, just in case
try:
reader = HTTPReader(url)
except HTTPError as e:
print >>sys.stderr, (
'Download failed with status code %d\n' % e.code)
print >>sys.stderr, 'Error body was:\n\n%s' % e.read()
return 1
class ReaderProgress(object):
def __init__(self, reader, length=None):
self._reader = reader
self._length = length
self._read = 0
self._pos = 0
self._buf = ''
self._progress = Progress(' {}%' if self._length else ' {} bytes')
def read(self, length):
# See comment above tell
if self._pos < self._read:
assert self._read - self._pos <= 8
assert length <= len(self._buf)
data = self._buf[:length]
self._buf = self._buf[length:]
self._pos += length
else:
assert self._read == self._pos
data = self._reader.read(length)
self._read += len(data)
self._pos = self._read
# Keep the last 8 bytes we read for GzipFile
self._buf = data[-8:]
self.progress()
return data
def progress(self):
if self._length:
count = self._read * 100 / self._length
else:
count = self._read
self._progress.progress(count)
def finish(self):
self._progress.finish()
# GzipFile wants to seek to the end of the file and back, so we add
# enough tell/seek support to make it happy. It also rewinds 8 bytes
# for the CRC, so we also handle that.
def tell(self):
return self._pos
def seek(self, pos, how=os.SEEK_SET):
if how == os.SEEK_END:
self._pos = self._length + pos
elif how == os.SEEK_SET:
self._pos = pos
elif how == os.SEEK_CUR:
self._pos += pos
else:
raise NotImplementedError()
return self._pos
encoding = reader.fh.headers.get('Content-Encoding', 'identity')
helper_content = ReaderProgress(reader, reader.length)
if encoding == 'gzip':
class WrapGzipFile(GzipFile):
def finish(self):
self.fileobj.finish()
helper_content = WrapGzipFile(mode='rb', fileobj=helper_content)
if args.dev is False:
content = StringIO()
copyfileobj(helper_content, content)
if hasattr(helper_content, 'finish'):
helper_content.finish()
content.seek(0)
print 'Extracting %s...' % helper
if ext == 'zip':
zip = zipfile.ZipFile(content, 'r')
info = zip.getinfo('git-cinnabar/%s' % helper)
helper_content = ReaderProgress(zip.open(info), info.file_size)
elif ext == 'tar.xz':
class UntarProgress(ReaderProgress):
def __init__(self, content, helper):
self._proc = subprocess.Popen(
['tar', '-JxO', 'git-cinnabar/%s' % helper],
stdin=subprocess.PIPE, stdout=subprocess.PIPE)
super(UntarProgress, self).__init__(self._proc.stdout)
def send(stdin, content):
copyfileobj(content, stdin)
stdin.close()
self._thread = threading.Thread(
target=send, args=(self._proc.stdin, content))
self._thread.start()
def finish(self):
self._proc.wait()
self._thread.join()
super(UntarProgress, self).finish()
helper_content = UntarProgress(content, helper)
else:
assert False
fd, path = tempfile.mkstemp(prefix=helper, dir=d)
fh = os.fdopen(fd, 'wb')
success = False
try:
copyfileobj(helper_content, fh)
success = True
finally:
if hasattr(helper_content, 'finish'):
helper_content.finish()
fh.close()
if success:
mode = os.stat(path).st_mode
if args.output:
helper_path = args.output
else:
helper_path = os.path.join(d, helper)
try:
# on Windows it's necessary to remove the file first.
os.remove(helper_path)
except OSError as exc:
if exc.errno != errno.ENOENT:
raise
pass
os.rename(path, helper_path)
# Add executable bits wherever read bits are set
mode = mode | ((mode & 0444) >> 2)
os.chmod(helper_path, mode)
if not args.no_config:
Git.run('config', '--global', 'cinnabar.helper',
os.path.abspath(helper_path))
else:
os.unlink(path)
return 0
def download(args):
'''download a prebuilt helper'''
helper = 'git-cinnabar-helper'
system = args.system
machine = args.machine
if system.startswith('MSYS_NT'):
system = 'Windows'
if system == 'Darwin':
system = 'macOS'
elif system == 'Windows':
helper += '.exe'
if machine == 'AMD64':
machine = 'x86_64'
available = (
('Linux', 'x86_64'),
('macOS', 'x86_64'),
('macOS', 'arm64'),
('Windows', 'x86_64'),
('Windows', 'x86'),
)
if args.list:
for system, machine in available:
print("%s/%s" % (system, machine))
return 0
if (system, machine) not in available:
print('No download available for %s/%s' % (system, machine),
file=sys.stderr)
return 1
if args.dev is False:
version = VERSION
if version.endswith('a'):
# For version x.y.za, download a development helper
args.dev = ''
script_path = os.path.dirname(os.path.abspath(sys.argv[0]))
if args.dev is not False:
sha1 = helper_hash()
if sha1 is None:
print('Cannot find the right development helper for this '
'version of git cinnabar.',
file=sys.stderr)
return 1
url = 'https://community-tc.services.mozilla.com/api/index/v1/task/'
url += 'project.git-cinnabar.helper.'
url += '{}.{}.{}.{}'.format(
sha1.decode('ascii'), system.lower(), machine,
args.dev.lower() if args.dev else '').rstrip('.')
url += '/artifacts/public/{}'.format(helper)
else:
if system in ('Windows', 'macOS'):
ext = 'zip'
else:
ext = 'tar.xz'
REPO_BASE = 'https://github.com/glandium'
url = '%s/git-cinnabar/releases/download/%s/git-cinnabar.%s.%s.%s' % (
REPO_BASE, version, system.lower(), machine.lower(), ext)
if args.url:
print(url)
return 0
if args.output:
d = os.path.dirname(args.output)
else:
d = script_path
if not os.access(d, os.W_OK):
d = os.path.join(os.path.expanduser('~'), '.git-cinnabar')
try:
os.makedirs(d)
except Exception:
pass
if not os.path.isdir(d):
print('Cannot write to either %s or %s.' % (d, script_path),
file=sys.stderr)
return 1
print('Downloading from %s...' % url)
try:
reader = HTTPReader(url, disable_ssl=args.disable_ssl)
except HTTPError:
# Try again, just in case
try:
reader = HTTPReader(url, disable_ssl=args.disable_ssl)
except HTTPError as e:
print('Download failed with status code %d\n' % e.code,
file=sys.stderr)
print(
'Error body was:\n\n%s' % e.read().decode('utf-8', 'replace'),
file=sys.stderr)
return 1
class ReaderProgress(object):
def __init__(self, reader, length=None):
self._reader = reader
self._length = length
self._read = 0
self._pos = 0
self._buf = ''
self._progress = Progress(' {}%' if self._length else ' {} bytes')
def read(self, length):
# See comment above tell
if self._pos < self._read:
assert self._read - self._pos <= 8
assert length <= len(self._buf)
data = self._buf[:length]
self._buf = self._buf[length:]
self._pos += length
else:
assert self._read == self._pos
data = self._reader.read(length)
self._read += len(data)
self._pos = self._read
# Keep the last 8 bytes we read for GzipFile
self._buf = data[-8:]
self.progress()
return data
def progress(self):
if self._length:
count = self._read * 100 // self._length
else:
count = self._read
self._progress.progress(count)
def finish(self):
self._progress.finish()
# GzipFile wants to seek to the end of the file and back, so we add
# enough tell/seek support to make it happy. It also rewinds 8 bytes
# for the CRC, so we also handle that.
def tell(self):
return self._pos
def seek(self, pos, how=os.SEEK_SET):
if how == os.SEEK_END:
self._pos = self._length + pos
elif how == os.SEEK_SET:
self._pos = pos
elif how == os.SEEK_CUR:
self._pos += pos
else:
raise NotImplementedError()
return self._pos
encoding = reader.fh.headers.get('Content-Encoding', 'identity')
helper_content = ReaderProgress(reader, reader.length)
if encoding == 'gzip':
class WrapGzipFile(GzipFile):
def finish(self):
self.fileobj.finish()
helper_content = WrapGzipFile(mode='rb', fileobj=helper_content)
if args.dev is False:
content = BytesIO()
copyfileobj(helper_content, content)
if hasattr(helper_content, 'finish'):
helper_content.finish()
content.seek(0)
print('Extracting %s...' % helper)
if ext == 'zip':
zip = zipfile.ZipFile(content, 'r')
info = zip.getinfo('git-cinnabar/%s' % helper)
helper_content = ReaderProgress(zip.open(info), info.file_size)
elif ext == 'tar.xz':
class UntarProgress(ReaderProgress):
def __init__(self, content, helper):
self._proc = subprocess.Popen(
['tar', '-JxO', 'git-cinnabar/%s' % helper],
stdin=subprocess.PIPE, stdout=subprocess.PIPE)
super(UntarProgress, self).__init__(self._proc.stdout)
def send(stdin, content):
copyfileobj(content, stdin)
stdin.close()
self._thread = threading.Thread(
target=send, args=(self._proc.stdin, content))
self._thread.start()
def finish(self):
self._proc.wait()
self._thread.join()
super(UntarProgress, self).finish()
helper_content = UntarProgress(content, helper)
else:
assert False
fd, path = tempfile.mkstemp(prefix=helper, dir=d)
fh = os.fdopen(fd, 'wb')
success = False
try:
copyfileobj(helper_content, fh)
success = True
finally:
if hasattr(helper_content, 'finish'):
helper_content.finish()
fh.close()
if success:
mode = os.stat(path).st_mode
if args.output:
helper_path = args.output
else:
helper_path = os.path.join(d, helper)
try:
# on Windows it's necessary to remove the file first.
os.remove(helper_path)
except OSError as exc:
if exc.errno != errno.ENOENT:
raise
pass
os.rename(path, helper_path)
# Add executable bits wherever read bits are set
mode = mode | ((mode & 0o0444) >> 2)
os.chmod(helper_path, mode)
if not args.no_config:
Git.run('config', '--global', 'cinnabar.helper',
os.path.abspath(helper_path))
else:
os.unlink(path)
return 0
def download(args):
'''download a prebuilt helper'''
helper = 'git-cinnabar-helper'
system = args.system
machine = args.machine
if system.startswith('MSYS_NT'):
system = 'Windows'
if system == 'Darwin':
system = 'macOS'
elif system == 'Windows':
helper += '.exe'
if machine == 'AMD64':
machine = 'x86_64'
available = (
('Linux', 'x86_64'),
('macOS', 'x86_64'),
('Windows', 'x86_64'),
('Windows', 'x86'),
)
if args.list:
for system, machine in available:
print "%s/%s" % (system, machine)
return 0
if (system, machine) not in available:
print >>sys.stderr, 'No download available for %s/%s' % (system,
machine)
return 1
if args.dev is False and VERSION.endswith('a'):
args.dev = ''
script_path = os.path.dirname(os.path.abspath(sys.argv[0]))
if args.dev is not False:
sha1 = helper_hash()
if sha1 is None:
print >>sys.stderr, (
'Cannot find the right development helper for this '
'version of git cinnabar.')
return 1
url = 'https://index.taskcluster.net/v1/task/github'
url += '.glandium.git-cinnabar.helper.'
url += '{}.{}.{}.{}'.format(
sha1, system.lower(), machine,
args.dev.lower() if args.dev else '').rstrip('.')
url += '/artifacts/public/{}'.format(helper)
else:
if system in ('Windows', 'macOS'):
ext = 'zip'
else:
ext = 'tar.xz'
REPO_BASE = 'https://github.com/glandium'
url = '%s/git-cinnabar/releases/download/%s/git-cinnabar.%s.%s.%s' % (
REPO_BASE, VERSION, system.lower(), machine.lower(), ext)
if args.url:
print url
return 0
try:
import requests
except ImportError:
print >>sys.stderr, (
'Downloading the helper requires the `requests` python module.')
return 1
if args.output:
d = os.path.dirname(args.output)
else:
d = script_path
if not os.access(d, os.W_OK):
d = os.path.join(os.path.expanduser('~'), '.git-cinnabar')
try:
os.makedirs(d)
except Exception:
pass
if not os.path.isdir(d):
print >>sys.stderr, (
'Cannot write to either %s or %s.' % (d, script_path))
return 1
print 'Downloading from %s...' % url
req = requests.get(url, stream=True)
if req.status_code != 200:
# Try again, just in case
req = requests.get(url, stream=True)
if req.status_code != 200:
print >>sys.stderr, (
'Download failed with status code %d\n' % req.status_code)
print >>sys.stderr, 'Error body was:\n\n%s' % req.content
return 1
size = int(req.headers.get('Content-Length', '0'))
def progress(iter, size=0):
def _progress(iter, size):
read = 0
for chunk in iter:
read += len(chunk)
if size:
yield read * 100 / size, chunk
else:
yield read, chunk
fmt = ' {}%' if size else ' {} bytes'
return progress_enum(fmt, _progress(iter, size))
helper_content = progress(req.iter_content(chunk_size=4096), size)
if args.dev is False:
content = StringIO()
for chunk in helper_content:
content.write(chunk)
content.seek(0)
print 'Extracting %s...' % helper
if ext == 'zip':
zip = zipfile.ZipFile(content, 'r')
info = zip.getinfo('git-cinnabar/%s' % helper)
helper_content = progress(zip.open(info), info.file_size)
elif ext == 'tar.xz':
def tar_extract():
proc = subprocess.Popen(
['tar', '-JxO', 'git-cinnabar/%s' % helper],
stdin=subprocess.PIPE, stdout=subprocess.PIPE)
def send(stdin, content):
stdin.write(content)
stdin.close()
thread = threading.Thread(
target=send, args=(proc.stdin, content.getvalue()))
thread.start()
chunk = True
while chunk:
chunk = proc.stdout.read(4096)
yield chunk
proc.wait()
thread.join()
helper_content = progress(tar_extract())
else:
assert False
fd, path = tempfile.mkstemp(prefix=helper, dir=d)
success = False
try:
for chunk in helper_content:
os.write(fd, chunk)
success = True
finally:
os.close(fd)
if success:
mode = os.stat(path).st_mode
if args.output:
helper_path = args.output
else:
helper_path = os.path.join(d, helper)
try:
# on Windows it's necessary to remove the file first.
os.remove(helper_path)
except OSError as exc:
if exc.errno != errno.ENOENT:
raise
pass
os.rename(path, helper_path)
# Add executable bits wherever read bits are set
mode = mode | ((mode & 0444) >> 2)
os.chmod(helper_path, mode)
if not args.no_config:
Git.run('config', '--global', 'cinnabar.helper',
os.path.abspath(helper_path))
else:
os.unlink(path)
return 0
def setUp(self):
self.git_dir = os.environ.get('GIT_DIR')
tmpdir = tempfile.mkdtemp()
Git.run('init', '--bare', tmpdir, stdout=open(os.devnull, 'w'))
os.environ['GIT_DIR'] = tmpdir
