def debuginstall(ui, fm): fm.write( b"fsmonitor-watchman", _(b"fsmonitor checking for watchman binary... (%s)\n"), ui.configpath(b"fsmonitor", b"watchman_exe"), ) root = tempfile.mkdtemp() c = watchmanclient.client(ui, root) err = None try: v = c.command(b"version") fm.write( b"fsmonitor-watchman-version", _(b" watchman binary version %s\n"), pycompat.bytestr(v["version"]), ) except watchmanclient.Unavailable as e: err = stringutil.forcebytestr(e) fm.condwrite( err, b"fsmonitor-watchman-error", _(b" watchman binary missing or broken: %s\n"), err, ) return 1 if err else 0
def transfer(chunk): for obj in chunk: objsize = obj.get(b'size', 0) if self.ui.verbose: if action == b'download': msg = _(b'lfs: downloading %s (%s)\n') elif action == b'upload': msg = _(b'lfs: uploading %s (%s)\n') self.ui.note( msg % (obj.get(b'oid'), util.bytecount(objsize)) ) retry = self.retry while True: try: self._basictransfer(obj, action, localstore) yield 1, obj.get(b'oid') break except socket.error as ex: if retry > 0: self.ui.note( _(b'lfs: failed: %r (remaining retry %d)\n') % (stringutil.forcebytestr(ex), retry) ) retry -= 1 continue raise
def _watchmantofsencoding(path): """Fix path to match watchman and local filesystem encoding watchman's paths encoding can differ from filesystem encoding. For example, on Windows, it's always utf-8. """ try: decoded = path.decode(_watchmanencoding) except UnicodeDecodeError as e: raise error.Abort(stringutil.forcebytestr(e), hint=b'watchman encoding error') try: encoded = decoded.encode(_fsencoding, 'strict') except UnicodeEncodeError as e: raise error.Abort(stringutil.forcebytestr(e)) return encoded
def sendfile(self, filename, hash): self.ui.debug(b'remotestore: sendfile(%s, %s)\n' % (filename, hash)) try: with lfutil.httpsendfile(self.ui, filename) as fd: return self._put(hash, fd) except IOError as e: raise error.Abort( _(b'remotestore: could not open file %s: %s') % (filename, stringutil.forcebytestr(e)))
def _getfile(self, tmpfile, filename, hash): try: chunks = self._get(hash) except urlerr.httperror as e: # 401s get converted to error.Aborts; everything else is fine being # turned into a StoreError raise basestore.StoreError(filename, hash, self.url, stringutil.forcebytestr(e)) except urlerr.urlerror as e: # This usually indicates a connection problem, so don't # keep trying with the other files... they will probably # all fail too. raise error.Abort(b'%s: %s' % (urlutil.hidepassword(self.url), e.reason)) except IOError as e: raise basestore.StoreError(filename, hash, self.url, stringutil.forcebytestr(e)) return lfutil.copyandhash(chunks, tmpfile)
def _command(self, *args): watchmanargs = (args[0], self._root) + args[1:] try: if self._watchmanclient is None: self._firsttime = False watchman_exe = self._ui.configpath(b'fsmonitor', b'watchman_exe') self._watchmanclient = pywatchman.client( timeout=self._timeout, useImmutableBser=True, binpath=procutil.tonativestr(watchman_exe), ) return self._watchmanclient.query(*watchmanargs) except pywatchman.CommandError as ex: if 'unable to resolve root' in ex.msg: raise WatchmanNoRoot(self._root, stringutil.forcebytestr(ex.msg)) raise Unavailable(stringutil.forcebytestr(ex.msg)) except pywatchman.WatchmanError as ex: raise Unavailable(stringutil.forcebytestr(ex))
def applyone(self, repo, node, cl, patchfile, merge=False, log=False, filter=None): '''apply the patch in patchfile to the repository as a transplant''' (manifest, user, (time, timezone), files, message) = cl[:5] date = "%d %d" % (time, timezone) extra = {'transplant_source': node} if filter: (user, date, message) = self.filter(filter, node, cl, patchfile) if log: # we don't translate messages inserted into commits message += '\n(transplanted from %s)' % nodemod.hex(node) self.ui.status(_('applying %s\n') % nodemod.short(node)) self.ui.note('%s %s\n%s\n' % (user, date, message)) if not patchfile and not merge: raise error.Abort(_('can only omit patchfile if merging')) if patchfile: try: files = set() patch.patch(self.ui, repo, patchfile, files=files, eolmode=None) files = list(files) except Exception as inst: seriespath = os.path.join(self.path, 'series') if os.path.exists(seriespath): os.unlink(seriespath) p1 = repo.dirstate.p1() p2 = node self.log(user, date, message, p1, p2, merge=merge) self.ui.write(stringutil.forcebytestr(inst) + '\n') raise TransplantError(_('fix up the working directory and run ' 'hg transplant --continue')) else: files = None if merge: p1, p2 = repo.dirstate.parents() repo.setparents(p1, node) m = match.always(repo.root, '') else: m = match.exact(repo.root, '', files) n = repo.commit(message, user, date, extra=extra, match=m, editor=self.getcommiteditor()) if not n: self.ui.warn(_('skipping emptied changeset %s\n') % nodemod.short(node)) return None if not merge: self.transplants.set(n, node) return n
def _handleunavailable(ui, state, ex): """Exception handler for Watchman interaction exceptions""" if isinstance(ex, watchmanclient.Unavailable): # experimental config: fsmonitor.verbose if ex.warn and ui.configbool(b'fsmonitor', b'verbose'): if b'illegal_fstypes' not in stringutil.forcebytestr(ex): ui.warn(stringutil.forcebytestr(ex) + b'\n') if ex.invalidate: state.invalidate() # experimental config: fsmonitor.verbose if ui.configbool(b'fsmonitor', b'verbose'): ui.log( b'fsmonitor', b'Watchman unavailable: %s\n', stringutil.forcebytestr(ex.msg), ) else: ui.log( b'fsmonitor', b'Watchman exception: %s\n', stringutil.forcebytestr(ex), )
def do_relink(src, dst, files, ui): def relinkfile(src, dst): bak = dst + b'.bak' os.rename(dst, bak) try: util.oslink(src, dst) except OSError: os.rename(bak, dst) raise os.remove(bak) CHUNKLEN = 65536 relinked = 0 savedbytes = 0 progress = ui.makeprogress(_(b'relinking'), unit=_(b'files'), total=len(files)) pos = 0 for f, sz in files: pos += 1 source = os.path.join(src, f) tgt = os.path.join(dst, f) # Binary mode, so that read() works correctly, especially on Windows sfp = open(source, b'rb') dfp = open(tgt, b'rb') sin = sfp.read(CHUNKLEN) while sin: din = dfp.read(CHUNKLEN) if sin != din: break sin = sfp.read(CHUNKLEN) sfp.close() dfp.close() if sin: ui.debug(b'not linkable: %s\n' % f) continue try: relinkfile(source, tgt) progress.update(pos, item=f) relinked += 1 savedbytes += sz except OSError as inst: ui.warn(b'%s: %s\n' % (tgt, stringutil.forcebytestr(inst))) progress.complete() ui.status( _(b'relinked %d files (%s reclaimed)\n') % (relinked, util.bytecount(savedbytes)))
def logservicecall(logger, service, **kwargs): start = time.time() logger(service, eventtype=b'start', **kwargs) try: yield logger(service, eventtype=b'success', elapsedms=(time.time() - start) * 1000, **kwargs) except Exception as e: logger(service, eventtype=b'failure', elapsedms=(time.time() - start) * 1000, errormsg=stringutil.forcebytestr(e), **kwargs) raise
def hook(ui, repo, hooktype, node=None, **kwargs): """add comment to bugzilla for each changeset that refers to a bugzilla bug id. only add a comment once per bug, so same change seen multiple times does not fill bug with duplicate data.""" if node is None: raise error.Abort( _(b'hook type %s does not pass a changeset id') % hooktype ) try: bz = bugzilla(ui, repo) ctx = repo[node] bugs = bz.find_bugs(ctx) if bugs: for bug in bugs: bz.update(bug, bugs[bug], ctx) bz.notify(bugs, stringutil.email(ctx.user())) except Exception as e: raise error.Abort(_(b'Bugzilla error: %s') % stringutil.forcebytestr(e))
def handlechangegroup_widen(op, inpart): """Changegroup exchange handler which restores temporarily-stripped nodes""" # We saved a bundle with stripped node data we must now restore. # This approach is based on mercurial/repair.py@6ee26a53c111. repo = op.repo ui = op.ui chgrpfile = op._widen_bundle del op._widen_bundle vfs = repo.vfs ui.note(_(b"adding branch\n")) f = vfs.open(chgrpfile, b"rb") try: gen = exchange.readbundle(ui, f, chgrpfile, vfs) # silence internal shuffling chatter override = {(b'ui', b'quiet'): True} if ui.verbose: override = {} with ui.configoverride(override): if isinstance(gen, bundle2.unbundle20): with repo.transaction(b'strip') as tr: bundle2.processbundle(repo, gen, lambda: tr) else: gen.apply( repo, b'strip', b'bundle:' + vfs.join(chgrpfile), True ) finally: f.close() # remove undo files for undovfs, undofile in repo.undofiles(): try: undovfs.unlink(undofile) except OSError as e: if e.errno != errno.ENOENT: ui.warn( _(b'error removing %s: %s\n') % (undovfs.join(undofile), stringutil.forcebytestr(e)) ) # Remove partial backup only if there were no exceptions op._widen_uninterr.__exit__(None, None, None) vfs.unlink(chgrpfile)
def _lookup(repo, proto, key): localkey = encoding.tolocal(key) if isinstance(localkey, str) and _scratchbranchmatcher(localkey): scratchnode = repo.bundlestore.index.getnode(localkey) if scratchnode: return "%d %s\n" % (1, scratchnode) else: return "%d %s\n" % (0, 'scratch branch %s not found' % localkey) else: try: r = hex(repo.lookup(localkey)) return "%d %s\n" % (1, r) except Exception as inst: if repo.bundlestore.index.getbundle(localkey): return "%d %s\n" % (1, localkey) else: r = stringutil.forcebytestr(inst) return "%d %s\n" % (0, r)
def _urlerrorreason(urlerror): '''Create a friendly message for the given URLError to be used in an LfsRemoteError message. ''' inst = urlerror if isinstance(urlerror.reason, Exception): inst = urlerror.reason if util.safehasattr(inst, b'reason'): try: # usually it is in the form (errno, strerror) reason = inst.reason.args[1] except (AttributeError, IndexError): # it might be anything, for example a string reason = inst.reason if isinstance(reason, pycompat.unicode): # SSLError of Python 2.7.9 contains a unicode reason = encoding.unitolocal(reason) return reason elif getattr(inst, "strerror", None): return encoding.strtolocal(inst.strerror) else: return stringutil.forcebytestr(urlerror)
def _batchrequest(self, pointers, action): """Get metadata about objects pointed by pointers for given action Return decoded JSON object like {'objects': [{'oid': '', 'size': 1}]} See https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md """ objects = [ {r'oid': pycompat.strurl(p.oid()), r'size': p.size()} for p in pointers ] requestdata = pycompat.bytesurl( json.dumps( {r'objects': objects, r'operation': pycompat.strurl(action),} ) ) url = b'%s/objects/batch' % self.baseurl batchreq = util.urlreq.request(pycompat.strurl(url), data=requestdata) batchreq.add_header(r'Accept', r'application/vnd.git-lfs+json') batchreq.add_header(r'Content-Type', r'application/vnd.git-lfs+json') try: with contextlib.closing(self.urlopener.open(batchreq)) as rsp: rawjson = rsp.read() except util.urlerr.httperror as ex: hints = { 400: _( b'check that lfs serving is enabled on %s and "%s" is ' b'supported' ) % (self.baseurl, action), 404: _(b'the "lfs.url" config may be used to override %s') % self.baseurl, } hint = hints.get(ex.code, _(b'api=%s, action=%s') % (url, action)) raise LfsRemoteError( _(b'LFS HTTP error: %s') % stringutil.forcebytestr(ex), hint=hint, ) except util.urlerr.urlerror as ex: hint = ( _(b'the "lfs.url" config may be used to override %s') % self.baseurl ) raise LfsRemoteError( _(b'LFS error: %s') % _urlerrorreason(ex), hint=hint ) try: response = pycompat.json_loads(rawjson) except ValueError: raise LfsRemoteError( _(b'LFS server returns invalid JSON: %s') % rawjson.encode("utf-8") ) if self.ui.debugflag: self.ui.debug(b'Status: %d\n' % rsp.status) # lfs-test-server and hg serve return headers in different order headers = pycompat.bytestr(rsp.info()).strip() self.ui.debug(b'%s\n' % b'\n'.join(sorted(headers.splitlines()))) if r'objects' in response: response[r'objects'] = sorted( response[r'objects'], key=lambda p: p[r'oid'] ) self.ui.debug( b'%s\n' % pycompat.bytesurl( json.dumps( response, indent=2, separators=(r'', r': '), sort_keys=True, ) ) ) def encodestr(x): if isinstance(x, pycompat.unicode): return x.encode('utf-8') return x return pycompat.rapply(encodestr, response)
def _basictransfer(self, obj, action, localstore): """Download or upload a single object using basic transfer protocol obj: dict, an object description returned by batch API action: string, one of ['upload', 'download'] localstore: blobstore.local See https://github.com/git-lfs/git-lfs/blob/master/docs/api/\ basic-transfers.md """ oid = obj[b'oid'] href = obj[b'actions'][action].get(b'href') headers = obj[b'actions'][action].get(b'header', {}).items() request = util.urlreq.request(pycompat.strurl(href)) if action == b'upload': # If uploading blobs, read data from local blobstore. if not localstore.verify(oid): raise error.Abort( _(b'detected corrupt lfs object: %s') % oid, hint=_(b'run hg verify'), ) request.data = filewithprogress(localstore.open(oid), None) request.get_method = lambda: r'PUT' request.add_header(r'Content-Type', r'application/octet-stream') request.add_header(r'Content-Length', len(request.data)) for k, v in headers: request.add_header(pycompat.strurl(k), pycompat.strurl(v)) response = b'' try: with contextlib.closing(self.urlopener.open(request)) as req: ui = self.ui # Shorten debug lines if self.ui.debugflag: ui.debug(b'Status: %d\n' % req.status) # lfs-test-server and hg serve return headers in different # order headers = pycompat.bytestr(req.info()).strip() ui.debug(b'%s\n' % b'\n'.join(sorted(headers.splitlines()))) if action == b'download': # If downloading blobs, store downloaded data to local # blobstore localstore.download(oid, req) else: while True: data = req.read(1048576) if not data: break response += data if response: ui.debug(b'lfs %s response: %s' % (action, response)) except util.urlerr.httperror as ex: if self.ui.debugflag: self.ui.debug( b'%s: %s\n' % (oid, ex.read()) ) # XXX: also bytes? raise LfsRemoteError( _(b'LFS HTTP error: %s (oid=%s, action=%s)') % (stringutil.forcebytestr(ex), oid, action) ) except util.urlerr.urlerror as ex: hint = _(b'attempted connection to %s') % pycompat.bytesurl( util.urllibcompat.getfullurl(request) ) raise LfsRemoteError( _(b'LFS error: %s') % _urlerrorreason(ex), hint=hint )
def storebundle(op, params, bundlefile): log = _getorcreateinfinitepushlogger(op) parthandlerstart = time.time() log(scratchbranchparttype, eventtype=b'start') index = op.repo.bundlestore.index store = op.repo.bundlestore.store op.records.add(scratchbranchparttype + b'_skippushkey', True) bundle = None try: # guards bundle bundlepath = b"bundle:%s+%s" % (op.repo.root, bundlefile) bundle = hg.repository(op.repo.ui, bundlepath) bookmark = params.get(b'bookmark') bookprevnode = params.get(b'bookprevnode', b'') force = params.get(b'force') if bookmark: oldnode = index.getnode(bookmark) else: oldnode = None bundleheads = bundle.revs(b'heads(bundle())') if bookmark and len(bundleheads) > 1: raise error.Abort( _(b'cannot push more than one head to a scratch branch')) revs = _getrevs(bundle, oldnode, force, bookmark) # Notify the user of what is being pushed plural = b's' if len(revs) > 1 else b'' op.repo.ui.warn(_(b"pushing %d commit%s:\n") % (len(revs), plural)) maxoutput = 10 for i in range(0, min(len(revs), maxoutput)): firstline = bundle[revs[i]].description().split(b'\n')[0][:50] op.repo.ui.warn(b" %s %s\n" % (revs[i], firstline)) if len(revs) > maxoutput + 1: op.repo.ui.warn(b" ...\n") firstline = bundle[revs[-1]].description().split(b'\n')[0][:50] op.repo.ui.warn(b" %s %s\n" % (revs[-1], firstline)) nodesctx = [bundle[rev] for rev in revs] inindex = lambda rev: bool(index.getbundle(bundle[rev].hex())) if bundleheads: newheadscount = sum(not inindex(rev) for rev in bundleheads) else: newheadscount = 0 # If there's a bookmark specified, there should be only one head, # so we choose the last node, which will be that head. # If a bug or malicious client allows there to be a bookmark # with multiple heads, we will place the bookmark on the last head. bookmarknode = nodesctx[-1].hex() if nodesctx else None key = None if newheadscount: with open(bundlefile, b'rb') as f: bundledata = f.read() with logservicecall(log, b'bundlestore', bundlesize=len(bundledata)): bundlesizelimit = 100 * 1024 * 1024 # 100 MB if len(bundledata) > bundlesizelimit: error_msg = (b'bundle is too big: %d bytes. ' + b'max allowed size is 100 MB') raise error.Abort(error_msg % (len(bundledata), )) key = store.write(bundledata) with logservicecall(log, b'index', newheadscount=newheadscount), index: if key: index.addbundle(key, nodesctx) if bookmark: index.addbookmark(bookmark, bookmarknode) _maybeaddpushbackpart(op, bookmark, bookmarknode, bookprevnode, params) log( scratchbranchparttype, eventtype=b'success', elapsedms=(time.time() - parthandlerstart) * 1000, ) except Exception as e: log( scratchbranchparttype, eventtype=b'failure', elapsedms=(time.time() - parthandlerstart) * 1000, errormsg=stringutil.forcebytestr(e), ) raise finally: if bundle: bundle.close()