def run_command(self, host, command, username=None, port=None):
assert isinstance(command, str)
command = command.encode(SSHGitClient.DEFAULT_ENCODING)
sshcmd = ui.config(b"ui", b"ssh", b"ssh")
args = procutil.sshargs(
sshcmd, pycompat.bytesurl(host), username, port
)
cmd = b'%s %s %s' % (sshcmd, args, procutil.shellquote(command))
# consistent with mercurial
ui.debug(b'running %s\n' % cmd)
# we cannot use Mercurial's procutil.popen4() since it
# always redirects stderr into a pipe
proc = subprocess.Popen(
procutil.tonativestr(cmd),
shell=True,
bufsize=0,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
)
return SubprocessWrapper(proc)
def _basictransfer(self, obj, action, localstore):
"""Download or upload a single object using basic transfer protocol
obj: dict, an object description returned by batch API
action: string, one of ['upload', 'download']
localstore: blobstore.local
See https://github.com/git-lfs/git-lfs/blob/master/docs/api/\
basic-transfers.md
"""
oid = obj[b'oid']
href = obj[b'actions'][action].get(b'href')
headers = obj[b'actions'][action].get(b'header', {}).items()
request = util.urlreq.request(pycompat.strurl(href))
if action == b'upload':
# If uploading blobs, read data from local blobstore.
if not localstore.verify(oid):
raise error.Abort(
_(b'detected corrupt lfs object: %s') % oid,
hint=_(b'run hg verify'),
)
request.data = filewithprogress(localstore.open(oid), None)
request.get_method = lambda: r'PUT'
request.add_header(r'Content-Type', r'application/octet-stream')
request.add_header(r'Content-Length', len(request.data))
for k, v in headers:
request.add_header(pycompat.strurl(k), pycompat.strurl(v))
response = b''
try:
with contextlib.closing(self.urlopener.open(request)) as req:
ui = self.ui # Shorten debug lines
if self.ui.debugflag:
ui.debug(b'Status: %d\n' % req.status)
# lfs-test-server and hg serve return headers in different
# order
headers = pycompat.bytestr(req.info()).strip()
ui.debug(b'%s\n' % b'\n'.join(sorted(headers.splitlines())))
if action == b'download':
# If downloading blobs, store downloaded data to local
# blobstore
localstore.download(oid, req)
else:
while True:
data = req.read(1048576)
if not data:
break
response += data
if response:
ui.debug(b'lfs %s response: %s' % (action, response))
except util.urlerr.httperror as ex:
if self.ui.debugflag:
self.ui.debug(
b'%s: %s\n' % (oid, ex.read())
) # XXX: also bytes?
raise LfsRemoteError(
_(b'LFS HTTP error: %s (oid=%s, action=%s)')
% (stringutil.forcebytestr(ex), oid, action)
)
except util.urlerr.urlerror as ex:
hint = _(b'attempted connection to %s') % pycompat.bytesurl(
util.urllibcompat.getfullurl(request)
)
raise LfsRemoteError(
_(b'LFS error: %s') % _urlerrorreason(ex), hint=hint
)
def _batchrequest(self, pointers, action):
"""Get metadata about objects pointed by pointers for given action
Return decoded JSON object like {'objects': [{'oid': '', 'size': 1}]}
See https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md
"""
objects = [
{r'oid': pycompat.strurl(p.oid()), r'size': p.size()}
for p in pointers
]
requestdata = pycompat.bytesurl(
json.dumps(
{r'objects': objects, r'operation': pycompat.strurl(action),}
)
)
url = b'%s/objects/batch' % self.baseurl
batchreq = util.urlreq.request(pycompat.strurl(url), data=requestdata)
batchreq.add_header(r'Accept', r'application/vnd.git-lfs+json')
batchreq.add_header(r'Content-Type', r'application/vnd.git-lfs+json')
try:
with contextlib.closing(self.urlopener.open(batchreq)) as rsp:
rawjson = rsp.read()
except util.urlerr.httperror as ex:
hints = {
400: _(
b'check that lfs serving is enabled on %s and "%s" is '
b'supported'
)
% (self.baseurl, action),
404: _(b'the "lfs.url" config may be used to override %s')
% self.baseurl,
}
hint = hints.get(ex.code, _(b'api=%s, action=%s') % (url, action))
raise LfsRemoteError(
_(b'LFS HTTP error: %s') % stringutil.forcebytestr(ex),
hint=hint,
)
except util.urlerr.urlerror as ex:
hint = (
_(b'the "lfs.url" config may be used to override %s')
% self.baseurl
)
raise LfsRemoteError(
_(b'LFS error: %s') % _urlerrorreason(ex), hint=hint
)
try:
response = pycompat.json_loads(rawjson)
except ValueError:
raise LfsRemoteError(
_(b'LFS server returns invalid JSON: %s')
% rawjson.encode("utf-8")
)
if self.ui.debugflag:
self.ui.debug(b'Status: %d\n' % rsp.status)
# lfs-test-server and hg serve return headers in different order
headers = pycompat.bytestr(rsp.info()).strip()
self.ui.debug(b'%s\n' % b'\n'.join(sorted(headers.splitlines())))
if r'objects' in response:
response[r'objects'] = sorted(
response[r'objects'], key=lambda p: p[r'oid']
)
self.ui.debug(
b'%s\n'
% pycompat.bytesurl(
json.dumps(
response,
indent=2,
separators=(r'', r': '),
sort_keys=True,
)
)
)
def encodestr(x):
if isinstance(x, pycompat.unicode):
return x.encode('utf-8')
return x
return pycompat.rapply(encodestr, response)