def _get_repo(remote):
if not changegroup or experiment('wire'):
if not changegroup and not check_enabled('no-mercurial'):
logging.warning('Mercurial libraries not found. Falling back to '
'experimental native access.')
stream = HgRepoHelper.connect(remote.url)
if stream:
return bundlerepo(remote.url, stream)
return HelperRepo(remote.url)
if remote.parsed_url.scheme == b'file':
# Make file://c:/... paths work by taking the netloc
path = remote.parsed_url.netloc + remote.parsed_url.path
if sys.platform == 'win32':
# TODO: This probably needs more thought.
path = path.lstrip(b'/')
if not os.path.isdir(path):
return bundlerepo(path)
ui = get_ui()
if changegroup and remote.parsed_url.scheme == b'file':
repo = localpeer(ui, path)
else:
try:
repo = hg.peer(ui, {}, remote.url)
except (error.RepoError, HTTPError, IOError):
if remote.parsed_url.scheme in ('http', 'https'):
return bundlerepo(remote.url, HTTPReader(remote.url))
raise
assert repo.capable(b'getbundle')
return repo
def get_repo(remote):
if not changegroup or experiment('wire'):
if not changegroup and not check_enabled('no-mercurial'):
logging.warning('Mercurial libraries not found. Falling back to '
'native access.')
logging.warning(
'Native access to mercurial repositories is experimental!')
stream = HgRepoHelper.connect(remote.url)
if stream:
return bundlerepo(remote.url, stream)
return HelperRepo(remote.url)
if remote.parsed_url.scheme == 'file':
path = remote.parsed_url.path
if sys.platform == 'win32':
# TODO: This probably needs more thought.
path = path.lstrip('/')
if not os.path.isdir(path):
return bundlerepo(path)
ui = get_ui()
if changegroup and remote.parsed_url.scheme == 'file':
repo = localpeer(ui, path)
else:
try:
repo = hg.peer(ui, {}, remote.url)
except (error.RepoError, urllib2.HTTPError, IOError):
return bundlerepo(remote.url, HTTPReader(remote.url))
assert repo.capable('getbundle')
return repo
def get_bundle(url):
reader = None
if not changegroup:
reader = BundleHelper.connect(url)
if not reader:
BundleHelper.close()
if not reader:
reader = HTTPReader(url)
return unbundle_fh(reader, url)
def get_bundle(url):
reader = None
if not changegroup:
reader = BundleHelper.connect(url)
if not reader:
BundleHelper.close()
if not reader:
disable_ssl = False
if sys.platform == 'win32':
disable_ssl = True
reader = HTTPReader(url, disable_ssl=disable_ssl)
return unbundle_fh(reader, url)
def get_clonebundle(repo):
url = Git.config('cinnabar.clonebundle')
if not url:
try:
if check_enabled('no-mercurial'):
raise ImportError('Do not use mercurial')
from mercurial.exchange import (
parseclonebundlesmanifest,
filterclonebundleentries,
)
except ImportError:
return None
bundles = repo._call('clonebundles')
class dummy(object):
pass
fakerepo = dummy()
fakerepo.requirements = set()
fakerepo.supportedformats = set()
fakerepo.ui = repo.ui
entries = parseclonebundlesmanifest(fakerepo, bundles)
if not entries:
return None
entries = filterclonebundleentries(fakerepo, entries)
if not entries:
return None
url = entries[0].get('URL')
if not url:
return None
sys.stderr.write('Getting clone bundle from %s\n' % url)
return unbundle_fh(HTTPReader(url), url)
def test_recovery(self):
sizes = {}
length = 0
for s in [162000, 64000, 57932]:
sizes[length] = s
length += s
the_test = self
# This HTTP server handler cuts responses before the full content is
# returned, according to the partial sizes defined above.
# It assumes the client will retry with a Range request starting from
# where it left, up to the end of the file.
class Handler(BaseHTTPRequestHandler):
def do_GET(self):
range_def = self.headers.getheader('Range')
if range_def:
start, end = range_def.partition('bytes=')[2].split('-')
start = int(start) if start else 0
end = int(end) if end else length - 1
the_test.assertIn(start, sizes)
the_test.assertEqual(end, length - 1)
self.send_response(206)
self.send_header('Content-Range',
'bytes %d-%d/%d' % (start, end, length))
else:
start = 0
self.send_response(200)
self.send_header('Content-Type', 'text/plain')
self.send_header('Content-Length', str(length))
self.send_header('Accept-Ranges', 'bytes')
self.end_headers()
buf = '-' * 4096
left = sizes[start]
while left:
if left < len(buf):
buf = buf[:left]
self.wfile.write(buf)
left -= len(buf)
def log_request(self, *args, **kwargs):
pass
server = HTTPServer(('', 0), Handler)
port = server.socket.getsockname()[1]
thread = Thread(target=server.serve_forever)
thread.start()
try:
reader = HTTPReader('http://localhost:%d/foo' % port)
read = 0
while True:
buf = reader.read(1250)
# If the read above is interrupted and the HTTPReader can
# recover with a range request, we still expect the right
# size.
self.assertEqual(len(buf), min(1250, length - read))
read += len(buf)
if not buf:
break
self.assertEqual(read, length)
finally:
server.shutdown()
thread.join()
def download(args):
'''download a prebuilt helper'''
helper = 'git-cinnabar-helper'
system = args.system
machine = args.machine
if system.startswith('MSYS_NT'):
system = 'Windows'
if system == 'Darwin':
system = 'macOS'
elif system == 'Windows':
helper += '.exe'
if machine == 'AMD64':
machine = 'x86_64'
available = (
('Linux', 'x86_64'),
('macOS', 'x86_64'),
('macOS', 'arm64'),
('Windows', 'x86_64'),
('Windows', 'x86'),
)
if args.list:
for system, machine in available:
print("%s/%s" % (system, machine))
return 0
if (system, machine) not in available:
print('No download available for %s/%s' % (system, machine),
file=sys.stderr)
return 1
if args.dev is False:
version = VERSION
if version.endswith('a'):
# For version x.y.za, download a development helper
args.dev = ''
script_path = os.path.dirname(os.path.abspath(sys.argv[0]))
if args.dev is not False:
sha1 = helper_hash()
if sha1 is None:
print('Cannot find the right development helper for this '
'version of git cinnabar.',
file=sys.stderr)
return 1
url = 'https://community-tc.services.mozilla.com/api/index/v1/task/'
url += 'project.git-cinnabar.helper.'
url += '{}.{}.{}.{}'.format(
sha1.decode('ascii'), system.lower(), machine,
args.dev.lower() if args.dev else '').rstrip('.')
url += '/artifacts/public/{}'.format(helper)
else:
if system in ('Windows', 'macOS'):
ext = 'zip'
else:
ext = 'tar.xz'
REPO_BASE = 'https://github.com/glandium'
url = '%s/git-cinnabar/releases/download/%s/git-cinnabar.%s.%s.%s' % (
REPO_BASE, version, system.lower(), machine.lower(), ext)
if args.url:
print(url)
return 0
if args.output:
d = os.path.dirname(args.output)
else:
d = script_path
if not os.access(d, os.W_OK):
d = os.path.join(os.path.expanduser('~'), '.git-cinnabar')
try:
os.makedirs(d)
except Exception:
pass
if not os.path.isdir(d):
print('Cannot write to either %s or %s.' % (d, script_path),
file=sys.stderr)
return 1
print('Downloading from %s...' % url)
try:
reader = HTTPReader(url, disable_ssl=args.disable_ssl)
except HTTPError:
# Try again, just in case
try:
reader = HTTPReader(url, disable_ssl=args.disable_ssl)
except HTTPError as e:
print('Download failed with status code %d\n' % e.code,
file=sys.stderr)
print(
'Error body was:\n\n%s' % e.read().decode('utf-8', 'replace'),
file=sys.stderr)
return 1
class ReaderProgress(object):
def __init__(self, reader, length=None):
self._reader = reader
self._length = length
self._read = 0
self._pos = 0
self._buf = ''
self._progress = Progress(' {}%' if self._length else ' {} bytes')
def read(self, length):
# See comment above tell
if self._pos < self._read:
assert self._read - self._pos <= 8
assert length <= len(self._buf)
data = self._buf[:length]
self._buf = self._buf[length:]
self._pos += length
else:
assert self._read == self._pos
data = self._reader.read(length)
self._read += len(data)
self._pos = self._read
# Keep the last 8 bytes we read for GzipFile
self._buf = data[-8:]
self.progress()
return data
def progress(self):
if self._length:
count = self._read * 100 // self._length
else:
count = self._read
self._progress.progress(count)
def finish(self):
self._progress.finish()
# GzipFile wants to seek to the end of the file and back, so we add
# enough tell/seek support to make it happy. It also rewinds 8 bytes
# for the CRC, so we also handle that.
def tell(self):
return self._pos
def seek(self, pos, how=os.SEEK_SET):
if how == os.SEEK_END:
self._pos = self._length + pos
elif how == os.SEEK_SET:
self._pos = pos
elif how == os.SEEK_CUR:
self._pos += pos
else:
raise NotImplementedError()
return self._pos
encoding = reader.fh.headers.get('Content-Encoding', 'identity')
helper_content = ReaderProgress(reader, reader.length)
if encoding == 'gzip':
class WrapGzipFile(GzipFile):
def finish(self):
self.fileobj.finish()
helper_content = WrapGzipFile(mode='rb', fileobj=helper_content)
if args.dev is False:
content = BytesIO()
copyfileobj(helper_content, content)
if hasattr(helper_content, 'finish'):
helper_content.finish()
content.seek(0)
print('Extracting %s...' % helper)
if ext == 'zip':
zip = zipfile.ZipFile(content, 'r')
info = zip.getinfo('git-cinnabar/%s' % helper)
helper_content = ReaderProgress(zip.open(info), info.file_size)
elif ext == 'tar.xz':
class UntarProgress(ReaderProgress):
def __init__(self, content, helper):
self._proc = subprocess.Popen(
['tar', '-JxO', 'git-cinnabar/%s' % helper],
stdin=subprocess.PIPE, stdout=subprocess.PIPE)
super(UntarProgress, self).__init__(self._proc.stdout)
def send(stdin, content):
copyfileobj(content, stdin)
stdin.close()
self._thread = threading.Thread(
target=send, args=(self._proc.stdin, content))
self._thread.start()
def finish(self):
self._proc.wait()
self._thread.join()
super(UntarProgress, self).finish()
helper_content = UntarProgress(content, helper)
else:
assert False
fd, path = tempfile.mkstemp(prefix=helper, dir=d)
fh = os.fdopen(fd, 'wb')
success = False
try:
copyfileobj(helper_content, fh)
success = True
finally:
if hasattr(helper_content, 'finish'):
helper_content.finish()
fh.close()
if success:
mode = os.stat(path).st_mode
if args.output:
helper_path = args.output
else:
helper_path = os.path.join(d, helper)
try:
# on Windows it's necessary to remove the file first.
os.remove(helper_path)
except OSError as exc:
if exc.errno != errno.ENOENT:
raise
pass
os.rename(path, helper_path)
# Add executable bits wherever read bits are set
mode = mode | ((mode & 0o0444) >> 2)
os.chmod(helper_path, mode)
if not args.no_config:
Git.run('config', '--global', 'cinnabar.helper',
os.path.abspath(helper_path))
else:
os.unlink(path)
return 0
def test_recovery(self):
sizes = {}
length = 0
for s in [162000, 64000, 57932]:
sizes[length] = s
length += s
the_test = self
# This HTTP server handler cuts responses before the full content is
# returned, according to the partial sizes defined above.
# It assumes the client will retry with a Range request starting from
# where it left, up to the end of the file.
class Handler(BaseHTTPRequestHandler):
redirected_once = False
errored_once = False
def do_GET(self):
if self.path == '/foo':
the_test.assertFalse(Handler.redirected_once)
Handler.redirected_once = True
self.send_response(301)
self.send_header('Location', '/bar')
self.end_headers()
return
elif self.path != '/bar':
self.send_response(404)
self.end_headers()
return
range_def = self.headers.get('Range')
if range_def:
if not Handler.errored_once:
Handler.errored_once = True
self.send_response(500)
self.end_headers()
return
start, end = range_def.partition('bytes=')[2].split('-')
start = int(start) if start else 0
end = int(end) if end else length - 1
the_test.assertIn(start, sizes)
the_test.assertEqual(end, length - 1)
self.send_response(206)
self.send_header('Content-Range',
'bytes %d-%d/%d' % (start, end, length))
else:
start = 0
self.send_response(200)
self.send_header('Content-Type', 'text/plain')
self.send_header('Content-Length', str(length))
self.send_header('Accept-Ranges', 'bytes')
self.end_headers()
buf = b'-' * 4096
left = sizes[start]
while left:
if left < len(buf):
buf = buf[:left]
self.wfile.write(buf)
left -= len(buf)
def log_request(self, *args, **kwargs):
pass
server = HTTPServer(('', 0), Handler)
port = server.socket.getsockname()[1]
thread = Thread(target=server.serve_forever)
thread.start()
try:
reader = HTTPReader('http://localhost:%d/foo' % port)
read = 0
while True:
buf = reader.read(1250)
# If the read above is interrupted and the HTTPReader can
# recover with a range request, we still expect the right
# size.
self.assertEqual(len(buf), min(1250, length - read))
read += len(buf)
if not buf:
break
self.assertEqual(read, length)
finally:
server.shutdown()
thread.join()