def _getfile(self, tmpfile, filename, hash): path = lfutil.findfile(self.remote, hash) if not path: raise basestore.StoreError(filename, hash, self.url, _(b"can't get file locally")) with open(path, b'rb') as fd: return lfutil.copyandhash(util.filechunkiter(fd), tmpfile)
def drain(self): ''' Need to read all data from request, httplib is half-duplex ''' length = int(self.env.get('CONTENT_LENGTH', 0)) for s in util.filechunkiter(self.inp, limit=length): pass
def download(self, oid, src, content_length): """Read the blob from the remote source in chunks, verify the content, and write to this local blobstore.""" sha256 = hashlib.sha256() size = 0 with self.vfs(oid, b'wb', atomictemp=True) as fp: for chunk in util.filechunkiter(src, size=1048576): fp.write(chunk) sha256.update(chunk) size += len(chunk) # If the server advertised a length longer than what we actually # received, then we should expect that the server crashed while # producing the response (but the server has no way of telling us # that), and we really don't need to try to write the response to # the localstore, because it's not going to match the expected. if content_length is not None and int(content_length) != size: msg = (b"Response length (%s) does not match Content-Length " b"header (%d): likely server-side crash") raise LfsRemoteError(_(msg) % (size, int(content_length))) realoid = hex(sha256.digest()) if realoid != oid: raise LfsCorruptionError( _(b'corrupt remote lfs object: %s') % oid) self._linktousercache(oid)
def snapshot_wdir(ui, repo, files, tmproot): '''snapshot files from working directory. if not using snapshot, -I/-X does not work and recursive diff in tools like kdiff3 and meld displays too many files.''' repo_root = repo.root dirname = os.path.basename(repo_root) if dirname == "": dirname = "root" base = os.path.join(tmproot, dirname) os.mkdir(base) ui.note(_('making snapshot of %d files from working dir\n') % (len(files))) fns_and_mtime = [] for fn in files: wfn = util.pconvert(fn) ui.note(' %s\n' % wfn) dest = os.path.join(base, wfn) destdir = os.path.dirname(dest) if not os.path.isdir(destdir): os.makedirs(destdir) fp = open(dest, 'wb') for chunk in util.filechunkiter(repo.wopener(wfn)): fp.write(chunk) fp.close() fns_and_mtime.append((dest, os.path.join(repo_root, fn), os.path.getmtime(dest))) return dirname, fns_and_mtime
def hexsha1(data): """hexsha1 returns the hex-encoded sha1 sum of the data in the file-like object data""" h = util.sha1() for chunk in util.filechunkiter(data): h.update(chunk) return h.hexdigest()
def hexsha1(fileobj): """hexsha1 returns the hex-encoded sha1 sum of the data in the file-like object data""" h = hashlib.sha1() for chunk in util.filechunkiter(fileobj): h.update(chunk) return hex(h.digest())
def consumestreamclone(repo, fp): """Apply the contents from a streaming clone file. This code is copied from Mercurial. Until Mercurial 3.5, this code was a closure in wireproto.py and not consumeable by extensions. """ lock = repo.lock() try: repo.ui.status(_('streaming all changes\n')) l = fp.readline() try: total_files, total_bytes = map(int, l.split(' ', 1)) except (ValueError, TypeError): raise error.ResponseError( _('unexpected response from remote server:'), l) repo.ui.status(_('%d files to transfer, %s of data\n') % (total_files, util.bytecount(total_bytes))) handled_bytes = 0 repo.ui.progress(_('clone'), 0, total=total_bytes) start = time.time() tr = repo.transaction(_('clone')) try: for i in xrange(total_files): # XXX doesn't support '\n' or '\r' in filenames l = fp.readline() try: name, size = l.split('\0', 1) size = int(size) except (ValueError, TypeError): raise error.ResponseError( _('unexpected response from remote server:'), l) if repo.ui.debugflag: repo.ui.debug('adding %s (%s)\n' % (name, util.bytecount(size))) # for backwards compat, name was partially encoded ofp = repo.svfs(store.decodedir(name), 'w') for chunk in util.filechunkiter(fp, limit=size): handled_bytes += len(chunk) repo.ui.progress(_('clone'), handled_bytes, total=total_bytes) ofp.write(chunk) ofp.close() tr.close() finally: tr.release() # Writing straight to files circumvented the inmemory caches repo.invalidate() elapsed = time.time() - start if elapsed <= 0: elapsed = 0.001 repo.ui.progress(_('clone'), None) repo.ui.status(_('transferred %s in %.1f seconds (%s/sec)\n') % (util.bytecount(total_bytes), elapsed, util.bytecount(total_bytes / elapsed))) finally: lock.release()
def _getfile(self, tmpfile, filename, hash): path = lfutil.findfile(self.remote, hash) if not path: raise basestore.StoreError(filename, hash, self.url, _("can't get file locally")) with open(path, 'rb') as fd: return lfutil.copyandhash( util.filechunkiter(fd), tmpfile)
def hashfile(file): if not os.path.exists(file): return '' hasher = hashlib.sha1('') with open(file, 'rb') as fd: for data in util.filechunkiter(fd): hasher.update(data) return hasher.hexdigest()
def consumev1(repo, fp, filecount, bytecount): """Apply the contents from version 1 of a streaming clone file handle. This takes the output from "streamout" and applies it to the specified repository. Like "streamout," the status line added by the wire protocol is not handled by this function. """ lock = repo.lock() try: repo.ui.status( _('%d files to transfer, %s of data\n') % (filecount, util.bytecount(bytecount))) handled_bytes = 0 repo.ui.progress(_('clone'), 0, total=bytecount) start = time.time() tr = repo.transaction(_('clone')) try: for i in xrange(filecount): # XXX doesn't support '\n' or '\r' in filenames l = fp.readline() try: name, size = l.split('\0', 1) size = int(size) except (ValueError, TypeError): raise error.ResponseError( _('unexpected response from remote server:'), l) if repo.ui.debugflag: repo.ui.debug('adding %s (%s)\n' % (name, util.bytecount(size))) # for backwards compat, name was partially encoded ofp = repo.svfs(store.decodedir(name), 'w') for chunk in util.filechunkiter(fp, limit=size): handled_bytes += len(chunk) repo.ui.progress(_('clone'), handled_bytes, total=bytecount) ofp.write(chunk) ofp.close() tr.close() finally: tr.release() # Writing straight to files circumvented the inmemory caches repo.invalidate() elapsed = time.time() - start if elapsed <= 0: elapsed = 0.001 repo.ui.progress(_('clone'), None) repo.ui.status( _('transferred %s in %.1f seconds (%s/sec)\n') % (util.bytecount(bytecount), elapsed, util.bytecount(bytecount / elapsed))) finally: lock.release()
def hashfile(file): if not os.path.exists(file): return '' hasher = util.sha1('') fd = open(file, 'rb') for data in util.filechunkiter(fd, 128 * 1024): hasher.update(data) fd.close() return hasher.hexdigest()
def link(src, dest): try: util.oslink(src, dest) except OSError: # if hardlinks fail, fallback on atomic copy dst = util.atomictempfile(dest) for chunk in util.filechunkiter(open(src, 'rb')): dst.write(chunk) dst.close() os.chmod(dest, os.stat(src).st_mode)
def verify(self, oid): """Indicate whether or not the hash of the underlying file matches its name.""" sha256 = hashlib.sha256() with self.open(oid) as fp: for chunk in util.filechunkiter(fp, size=1048576): sha256.update(chunk) return oid == sha256.hexdigest()
def link(src, dest): """Try to create hardlink - if that fails, efficiently make a copy.""" util.makedirs(os.path.dirname(dest)) try: util.oslink(src, dest) except OSError: # if hardlinks fail, fallback on atomic copy with open(src, 'rb') as srcf, util.atomictempfile(dest) as dstf: for chunk in util.filechunkiter(srcf): dstf.write(chunk) os.chmod(dest, os.stat(src).st_mode)
def copytostoreabsolute(repo, file, hash): if inusercache(repo.ui, hash): link(usercachepath(repo.ui, hash), storepath(repo, hash)) else: util.makedirs(os.path.dirname(storepath(repo, hash))) with open(file, 'rb') as srcf: with util.atomictempfile(storepath(repo, hash), createmode=repo.store.createmode) as dstf: for chunk in util.filechunkiter(srcf): dstf.write(chunk) linktousercache(repo, hash)
def copytostoreabsolute(repo, file, hash): util.makedirs(os.path.dirname(storepath(repo, hash))) if inusercache(repo.ui, hash): link(usercachepath(repo.ui, hash), storepath(repo, hash)) else: dst = util.atomictempfile(storepath(repo, hash)) for chunk in util.filechunkiter(open(file, 'rb')): dst.write(chunk) dst.close() util.copymode(file, storepath(repo, hash)) linktousercache(repo, hash)
def consumev1(repo, fp, filecount, bytecount): """Apply the contents from version 1 of a streaming clone file handle. This takes the output from "streamout" and applies it to the specified repository. Like "streamout," the status line added by the wire protocol is not handled by this function. """ lock = repo.lock() try: repo.ui.status(_('%d files to transfer, %s of data\n') % (filecount, util.bytecount(bytecount))) handled_bytes = 0 repo.ui.progress(_('clone'), 0, total=bytecount) start = time.time() tr = repo.transaction(_('clone')) try: for i in xrange(filecount): # XXX doesn't support '\n' or '\r' in filenames l = fp.readline() try: name, size = l.split('\0', 1) size = int(size) except (ValueError, TypeError): raise error.ResponseError( _('unexpected response from remote server:'), l) if repo.ui.debugflag: repo.ui.debug('adding %s (%s)\n' % (name, util.bytecount(size))) # for backwards compat, name was partially encoded ofp = repo.svfs(store.decodedir(name), 'w') for chunk in util.filechunkiter(fp, limit=size): handled_bytes += len(chunk) repo.ui.progress(_('clone'), handled_bytes, total=bytecount) ofp.write(chunk) ofp.close() tr.close() finally: tr.release() # Writing straight to files circumvented the inmemory caches repo.invalidate() elapsed = time.time() - start if elapsed <= 0: elapsed = 0.001 repo.ui.progress(_('clone'), None) repo.ui.status(_('transferred %s in %.1f seconds (%s/sec)\n') % (util.bytecount(bytecount), elapsed, util.bytecount(bytecount / elapsed))) finally: lock.release()
def copytostoreabsolute(repo, file, hash): if inusercache(repo.ui, hash): link(usercachepath(repo.ui, hash), storepath(repo, hash)) elif not getattr(repo, "_isconverting", False): util.makedirs(os.path.dirname(storepath(repo, hash))) dst = util.atomictempfile(storepath(repo, hash), createmode=repo.store.createmode) for chunk in util.filechunkiter(open(file, 'rb')): dst.write(chunk) dst.close() linktousercache(repo, hash)
def copytostoreabsolute(repo, file, hash): if inusercache(repo.ui, hash): link(usercachepath(repo.ui, hash), storepath(repo, hash)) else: util.makedirs(os.path.dirname(storepath(repo, hash))) dst = util.atomictempfile(storepath(repo, hash), createmode=repo.store.createmode) for chunk in util.filechunkiter(open(file, 'rb')): dst.write(chunk) dst.close() linktousercache(repo, hash)
def link(src, dest): """Try to create hardlink - if that fails, efficiently make a copy.""" util.makedirs(os.path.dirname(dest)) try: util.oslink(src, dest) except OSError: # if hardlinks fail, fallback on atomic copy dst = util.atomictempfile(dest) for chunk in util.filechunkiter(open(src, 'rb')): dst.write(chunk) dst.close() os.chmod(dest, os.stat(src).st_mode)
def bail(response, headers={}): length = int(req.env.get('CONTENT_LENGTH', 0)) for s in util.filechunkiter(req, limit=length): # drain incoming bundle, else client will not see # response when run outside cgi script pass status = headers.pop('status', HTTP_OK) req.header(headers.items()) req.respond(status, HGTYPE) req.write('0\n') req.write(response)
def overridecat(orig, ui, repo, file1, *pats, **opts): ctx = scmutil.revsingle(repo, opts.get('rev')) err = 1 notbad = set() m = scmutil.match(ctx, (file1, ) + pats, opts) origmatchfn = m.matchfn def lfmatchfn(f): if origmatchfn(f): return True lf = lfutil.splitstandin(f) if lf is None: return False notbad.add(lf) return origmatchfn(lf) m.matchfn = lfmatchfn origbadfn = m.bad def lfbadfn(f, msg): if not f in notbad: origbadfn(f, msg) m.bad = lfbadfn for f in ctx.walk(m): fp = cmdutil.makefileobj(repo, opts.get('output'), ctx.node(), pathname=f) lf = lfutil.splitstandin(f) if lf is None or origmatchfn(f): # duplicating unreachable code from commands.cat data = ctx[f].data() if opts.get('decode'): data = repo.wwritedata(f, data) fp.write(data) else: hash = lfutil.readstandin(repo, lf, ctx.rev()) if not lfutil.inusercache(repo.ui, hash): store = basestore._openstore(repo) success, missing = store.get([(lf, hash)]) if len(success) != 1: raise util.Abort( _('largefile %s is not in cache and could not be ' 'downloaded') % lf) path = lfutil.usercachepath(repo.ui, hash) fpin = open(path, "rb") for chunk in util.filechunkiter(fpin, 128 * 1024): fp.write(chunk) fpin.close() fp.close() err = 0 return err
def generatestreamclone(repo): """Emit content for a streaming clone. This is a generator of raw chunks that constitute a streaming clone. This code is copied from Mercurial. Until Mercurial 3.5, this code was a closure in wireproto.py and not consumeable by extensions. """ entries = [] total_bytes = 0 # Get consistent snapshot of repo, lock during scan. lock = repo.lock() try: repo.ui.debug('scanning\n') for name, ename, size in repo.store.walk(): if size: entries.append((name, size)) total_bytes += size finally: lock.release() repo.ui.debug('%d files, %d bytes to transfer\n' % (len(entries), total_bytes)) yield '%d %d\n' % (len(entries), total_bytes) sopener = repo.svfs oldaudit = sopener.mustaudit debugflag = repo.ui.debugflag sopener.mustaudit = False try: for name, size in entries: if debugflag: repo.ui.debug('sending %s (%d bytes)\n' % (name, size)) # partially encode name over the wire for backwards compat yield '%s\0%d\n' % (store.encodedir(name), size) if size <= 65536: fp = sopener(name) try: data = fp.read(size) finally: fp.close() yield data else: for chunk in util.filechunkiter(sopener(name), limit=size): yield chunk finally: sopener.mustaudit = oldaudit
def download(self, oid, src): """Read the blob from the remote source in chunks, verify the content, and write to this local blobstore.""" sha256 = hashlib.sha256() with self.vfs(oid, 'wb', atomictemp=True) as fp: for chunk in util.filechunkiter(src, size=1048576): fp.write(chunk) sha256.update(chunk) realoid = sha256.hexdigest() if realoid != oid: raise LfsCorruptionError(_('corrupt remote lfs object: %s') % oid) self._linktousercache(oid)
def overridecat(orig, ui, repo, file1, *pats, **opts): ctx = scmutil.revsingle(repo, opts.get('rev')) err = 1 notbad = set() m = scmutil.match(ctx, (file1,) + pats, opts) origmatchfn = m.matchfn def lfmatchfn(f): if origmatchfn(f): return True lf = lfutil.splitstandin(f) if lf is None: return False notbad.add(lf) return origmatchfn(lf) m.matchfn = lfmatchfn origbadfn = m.bad def lfbadfn(f, msg): if not f in notbad: origbadfn(f, msg) m.bad = lfbadfn for f in ctx.walk(m): fp = cmdutil.makefileobj(repo, opts.get('output'), ctx.node(), pathname=f) lf = lfutil.splitstandin(f) if lf is None or origmatchfn(f): # duplicating unreachable code from commands.cat data = ctx[f].data() if opts.get('decode'): data = repo.wwritedata(f, data) fp.write(data) else: hash = lfutil.readstandin(repo, lf, ctx.rev()) if not lfutil.inusercache(repo.ui, hash): store = basestore._openstore(repo) success, missing = store.get([(lf, hash)]) if len(success) != 1: raise util.Abort( _('largefile %s is not in cache and could not be ' 'downloaded') % lf) path = lfutil.usercachepath(repo.ui, hash) fpin = open(path, "rb") for chunk in util.filechunkiter(fpin, 128 * 1024): fp.write(chunk) fpin.close() fp.close() err = 0 return err
def getlfile(self, sha): """returns an iterable with the chunks of the file with sha sha""" stream = self._callstream("getlfile", sha=sha) length = stream.readline() try: length = int(length) except ValueError: self._abort(error.ResponseError(_("unexpected response:"), length)) # SSH streams will block if reading more than length for chunk in util.filechunkiter(stream, 128 * 1024, length): yield chunk # HTTP streams must hit the end to process the last empty # chunk of Chunked-Encoding so the connection can be reused. if issubclass(self.__class__, httppeer.httppeer): chunk = stream.read(1) if chunk: self._abort(error.ResponseError(_("unexpected response:"), chunk))
def emitrevlogdata(): try: for name, size in entries: if debugflag: repo.ui.debug('sending %s (%d bytes)\n' % (name, size)) # partially encode name over the wire for backwards compat yield '%s\0%d\n' % (store.encodedir(name), size) if size <= 65536: fp = svfs(name) try: data = fp.read(size) finally: fp.close() yield data else: for chunk in util.filechunkiter(svfs(name), limit=size): yield chunk finally: svfs.mustaudit = oldaudit
def download(self, oid, src): """Read the blob from the remote source in chunks, verify the content, and write to this local blobstore.""" sha256 = hashlib.sha256() with self.vfs(oid, 'wb', atomictemp=True) as fp: for chunk in util.filechunkiter(src, size=1048576): fp.write(chunk) sha256.update(chunk) realoid = sha256.hexdigest() if realoid != oid: raise error.Abort(_('corrupt remote lfs object: %s') % oid) # XXX: should we verify the content of the cache, and hardlink back to # the local store on success, but truncate, write and link on failure? if not self.cachevfs.exists(oid): self.ui.note(_('lfs: adding %s to the usercache\n') % oid) lfutil.link(self.vfs.join(oid), self.cachevfs.join(oid))
def getlfile(self, sha): """returns an iterable with the chunks of the file with sha sha""" stream = self._callstream("getlfile", sha=sha) length = stream.readline() try: length = int(length) except ValueError: self._abort( error.ResponseError(_("unexpected response:"), length)) # SSH streams will block if reading more than length for chunk in util.filechunkiter(stream, limit=length): yield chunk # HTTP streams must hit the end to process the last empty # chunk of Chunked-Encoding so the connection can be reused. if issubclass(self.__class__, httppeer.httppeer): chunk = stream.read(1) if chunk: self._abort( error.ResponseError(_("unexpected response:"), chunk))
def copyfromcache(repo, hash, filename): '''Copy the specified largefile from the repo or system cache to filename in the repository. Return true on success or false if the file was not found in either cache (which should not happened: this is meant to be called only after ensuring that the needed largefile exists in the cache).''' wvfs = repo.wvfs path = findfile(repo, hash) if path is None: return False wvfs.makedirs(wvfs.dirname(wvfs.join(filename))) # The write may fail before the file is fully written, but we # don't use atomic writes in the working copy. with open(path, 'rb') as srcfd, wvfs(filename, 'wb') as destfd: gothash = copyandhash( util.filechunkiter(srcfd), destfd) if gothash != hash: repo.ui.warn(_('%s: data corruption in %s with hash %s\n') % (filename, path, gothash)) wvfs.unlink(filename) return False return True
def copyfromcache(repo, hash, filename): '''Copy the specified largefile from the repo or system cache to filename in the repository. Return true on success or false if the file was not found in either cache (which should not happened: this is meant to be called only after ensuring that the needed largefile exists in the cache).''' wvfs = repo.wvfs path = findfile(repo, hash) if path is None: return False wvfs.makedirs(wvfs.dirname(wvfs.join(filename))) # The write may fail before the file is fully written, but we # don't use atomic writes in the working copy. with open(path, 'rb') as srcfd: with wvfs(filename, 'wb') as destfd: gothash = copyandhash( util.filechunkiter(srcfd), destfd) if gothash != hash: repo.ui.warn(_('%s: data corruption in %s with hash %s\n') % (filename, path, gothash)) wvfs.unlink(filename) return False return True
def _docensor(ui, repo, path, rev='', tombstone='', **opts): if not path: raise error.Abort(_('must specify file path to censor')) if not rev: raise error.Abort(_('must specify revision to censor')) wctx = repo[None] m = scmutil.match(wctx, (path,)) if m.anypats() or len(m.files()) != 1: raise error.Abort(_('can only specify an explicit filename')) path = m.files()[0] flog = repo.file(path) if not len(flog): raise error.Abort(_('cannot censor file with no history')) rev = scmutil.revsingle(repo, rev, rev).rev() try: ctx = repo[rev] except KeyError: raise error.Abort(_('invalid revision identifier %s') % rev) try: fctx = ctx.filectx(path) except error.LookupError: raise error.Abort(_('file does not exist at revision %s') % rev) fnode = fctx.filenode() headctxs = [repo[c] for c in repo.heads()] heads = [c for c in headctxs if path in c and c.filenode(path) == fnode] if heads: headlist = ', '.join([short(c.node()) for c in heads]) raise error.Abort(_('cannot censor file in heads (%s)') % headlist, hint=_('clean/delete and commit first')) wp = wctx.parents() if ctx.node() in [p.node() for p in wp]: raise error.Abort(_('cannot censor working directory'), hint=_('clean/delete/update first')) flogv = flog.version & 0xFFFF if flogv != revlog.REVLOGNG: raise error.Abort( _('censor does not support revlog version %d') % (flogv,)) tombstone = filelog.packmeta({"censored": tombstone}, "") crev = fctx.filerev() if len(tombstone) > flog.rawsize(crev): raise error.Abort(_( 'censor tombstone must be no longer than censored data')) # Using two files instead of one makes it easy to rewrite entry-by-entry idxread = repo.svfs(flog.indexfile, 'r') idxwrite = repo.svfs(flog.indexfile, 'wb', atomictemp=True) if flog.version & revlog.REVLOGNGINLINEDATA: dataread, datawrite = idxread, idxwrite else: dataread = repo.svfs(flog.datafile, 'r') datawrite = repo.svfs(flog.datafile, 'wb', atomictemp=True) # Copy all revlog data up to the entry to be censored. rio = revlog.revlogio() offset = flog.start(crev) for chunk in util.filechunkiter(idxread, limit=crev * rio.size): idxwrite.write(chunk) for chunk in util.filechunkiter(dataread, limit=offset): datawrite.write(chunk) def rewriteindex(r, newoffs, newdata=None): """Rewrite the index entry with a new data offset and optional new data. The newdata argument, if given, is a tuple of three positive integers: (new compressed, new uncompressed, added flag bits). """ offlags, comp, uncomp, base, link, p1, p2, nodeid = flog.index[r] flags = revlog.gettype(offlags) if newdata: comp, uncomp, nflags = newdata flags |= nflags offlags = revlog.offset_type(newoffs, flags) e = (offlags, comp, uncomp, r, link, p1, p2, nodeid) idxwrite.write(rio.packentry(e, None, flog.version, r)) idxread.seek(rio.size, 1) def rewrite(r, offs, data, nflags=revlog.REVIDX_DEFAULT_FLAGS): """Write the given full text to the filelog with the given data offset. Returns: The integer number of data bytes written, for tracking data offsets. """ flag, compdata = flog.compress(data) newcomp = len(flag) + len(compdata) rewriteindex(r, offs, (newcomp, len(data), nflags)) datawrite.write(flag) datawrite.write(compdata) dataread.seek(flog.length(r), 1) return newcomp # Rewrite censored revlog entry with (padded) tombstone data. pad = ' ' * (flog.rawsize(crev) - len(tombstone)) offset += rewrite(crev, offset, tombstone + pad, revlog.REVIDX_ISCENSORED) # Rewrite all following filelog revisions fixing up offsets and deltas. for srev in xrange(crev + 1, len(flog)): if crev in flog.parentrevs(srev): # Immediate children of censored node must be re-added as fulltext. try: revdata = flog.revision(srev) except error.CensoredNodeError as e: revdata = e.tombstone dlen = rewrite(srev, offset, revdata) else: # Copy any other revision data verbatim after fixing up the offset. rewriteindex(srev, offset) dlen = flog.length(srev) for chunk in util.filechunkiter(dataread, limit=dlen): datawrite.write(chunk) offset += dlen idxread.close() idxwrite.close() if dataread is not idxread: dataread.close() datawrite.close()
def unbundle(repo, req): """ Because protocol.unbundle uses tempfile, we cannot use it in AppEngine environment. Here is an alternative implementation which does not use tempfiles. """ proto = os.environ.get("wsgi.url_scheme") or "http" their_heads = req.form["heads"][0].split(" ") def check_heads(): heads = map(hex, repo.heads()) return their_heads == [hex("force")] or their_heads == heads # fail early if possible if not check_heads(): length = int(req.headers.get("Content-Length", 0)) for s in util.filechunkiter(req.body_file, limit=length): # drain incoming bundle, else client will not see # response when run outside cgi script pass raise ErrorResponse(HTTP_OK, "unsynced changes") # do not lock repo until all changegroup data is # streamed. save to temporary file. fp = StringIO() length = int(req.headers["Content-Length"]) if int(length) > 50 * 1024: # Content is too long: raise ErrorResponse(HTTP_OK, "Bundled change size is greater than 50k." " Split changes using -r flag.") for s in util.filechunkiter(req.body_file, limit=length): fp.write(s) try: if not check_heads(): raise ErrorResponse(HTTP_OK, "unsynced changes") fp.seek(0) header = fp.read(6) if header.startswith("HG") and not header.startswith("HG10"): raise ValueError("unknown bundle version") elif header not in changegroup.bundletypes: raise ValueError("unknown bundle compression type") gen = changegroup.unbundle(header, fp) # send addchangegroup output to client oldio = sys.stdout, sys.stderr sys.stderr = sys.stdout = StringIO() try: url = "remote:%s:%s" % (proto, req.remote_addr) try: ret = repo.addchangegroup(gen, "serve", url) except util.Abort, inst: sys.stdout.write("abort: %s\n" % inst) ret = 0 finally: val = sys.stdout.getvalue() sys.stdout, sys.stderr = oldio req.respond(HTTP_OK, protocol.HGTYPE) return ("%d\n%s" % (ret, val),) except ValueError, inst: raise ErrorResponse(HTTP_OK, inst)
def getfile(self, fp): length = int(self.req.env["CONTENT_LENGTH"]) for s in util.filechunkiter(self.req, limit=length): fp.write(s)
def generator(): yield '%d\n' % length for chunk in util.filechunkiter(f): yield chunk
def unbundle(repo, req): proto = req.env.get('wsgi.url_scheme') or 'http' their_heads = req.form['heads'][0].split(' ') def check_heads(): heads = map(hex, repo.heads()) return their_heads == [hex('force')] or their_heads == heads # fail early if possible if not check_heads(): req.drain() raise ErrorResponse(HTTP_OK, 'unsynced changes') # do not lock repo until all changegroup data is # streamed. save to temporary file. fd, tempname = tempfile.mkstemp(prefix='hg-unbundle-') fp = os.fdopen(fd, 'wb+') try: length = int(req.env['CONTENT_LENGTH']) for s in util.filechunkiter(req, limit=length): fp.write(s) try: lock = repo.lock() try: if not check_heads(): raise ErrorResponse(HTTP_OK, 'unsynced changes') fp.seek(0) header = fp.read(6) if header.startswith('HG') and not header.startswith('HG10'): raise ValueError('unknown bundle version') elif header not in changegroupmod.bundletypes: raise ValueError('unknown bundle compression type') gen = changegroupmod.unbundle(header, fp) # send addchangegroup output to client oldio = sys.stdout, sys.stderr sys.stderr = sys.stdout = cStringIO.StringIO() try: url = 'remote:%s:%s:%s' % ( proto, urllib.quote(req.env.get('REMOTE_HOST', '')), urllib.quote(req.env.get('REMOTE_USER', ''))) try: ret = repo.addchangegroup(gen, 'serve', url) except util.Abort, inst: sys.stdout.write("abort: %s\n" % inst) ret = 0 finally: val = sys.stdout.getvalue() sys.stdout, sys.stderr = oldio req.respond(HTTP_OK, HGTYPE) return '%d\n%s' % (ret, val), finally: lock.release() except ValueError, inst: raise ErrorResponse(HTTP_OK, inst) except (OSError, IOError), inst: filename = getattr(inst, 'filename', '') # Don't send our filesystem layout to the client if filename.startswith(repo.root): filename = filename[len(repo.root) + 1:] else: filename = '' error = getattr(inst, 'strerror', 'Unknown error') if inst.errno == errno.ENOENT: code = HTTP_NOT_FOUND else: code = HTTP_SERVER_ERROR raise ErrorResponse(code, '%s: %s' % (error, filename))
def unbundle(web, req): def bail(response, headers={}): length = int(req.env.get('CONTENT_LENGTH', 0)) for s in util.filechunkiter(req, limit=length): # drain incoming bundle, else client will not see # response when run outside cgi script pass status = headers.pop('status', HTTP_OK) req.header(headers.items()) req.respond(status, HGTYPE) req.write('0\n') req.write(response) # enforce that you can only unbundle with POST requests if req.env['REQUEST_METHOD'] != 'POST': headers = {'status': '405 Method Not Allowed'} bail('unbundle requires POST request\n', headers) return # require ssl by default, auth info cannot be sniffed and # replayed ssl_req = web.configbool('web', 'push_ssl', True) if ssl_req: if req.env.get('wsgi.url_scheme') != 'https': bail('ssl required\n') return proto = 'https' else: proto = 'http' # do not allow push unless explicitly allowed if not web.check_perm(req, 'push', False): bail('push not authorized\n', headers={'status': '401 Unauthorized'}) return their_heads = req.form['heads'][0].split(' ') def check_heads(): heads = map(hex, web.repo.heads()) return their_heads == [hex('force')] or their_heads == heads # fail early if possible if not check_heads(): bail('unsynced changes\n') return req.respond(HTTP_OK, HGTYPE) # do not lock repo until all changegroup data is # streamed. save to temporary file. fd, tempname = tempfile.mkstemp(prefix='hg-unbundle-') fp = os.fdopen(fd, 'wb+') try: length = int(req.env['CONTENT_LENGTH']) for s in util.filechunkiter(req, limit=length): fp.write(s) try: lock = web.repo.lock() try: if not check_heads(): req.write('0\n') req.write('unsynced changes\n') return fp.seek(0) header = fp.read(6) if header.startswith('HG') and not header.startswith('HG10'): raise ValueError('unknown bundle version') elif header not in changegroupmod.bundletypes: raise ValueError('unknown bundle compression type') gen = changegroupmod.unbundle(header, fp) # send addchangegroup output to client oldio = sys.stdout, sys.stderr sys.stderr = sys.stdout = cStringIO.StringIO() try: url = 'remote:%s:%s' % (proto, req.env.get('REMOTE_HOST', '')) try: ret = web.repo.addchangegroup(gen, 'serve', url) except util.Abort, inst: sys.stdout.write("abort: %s\n" % inst) ret = 0 finally: val = sys.stdout.getvalue() sys.stdout, sys.stderr = oldio req.write('%d\n' % ret) req.write(val) finally: del lock except ValueError, inst: req.write('0\n') req.write(str(inst) + '\n') except (OSError, IOError), inst: req.write('0\n') filename = getattr(inst, 'filename', '') # Don't send our filesystem layout to the client if filename.startswith(web.repo.root): filename = filename[len(web.repo.root)+1:] else: filename = '' error = getattr(inst, 'strerror', 'Unknown error') if inst.errno == errno.ENOENT: code = HTTP_NOT_FOUND else: code = HTTP_SERVER_ERROR req.respond(code) req.write('%s: %s\n' % (error, filename))
def unbundle(repo, req): proto = req.env.get('wsgi.url_scheme') or 'http' their_heads = req.form['heads'][0].split(' ') def check_heads(): heads = map(hex, repo.heads()) return their_heads == [hex('force')] or their_heads == heads # fail early if possible if not check_heads(): req.drain() raise ErrorResponse(HTTP_OK, 'unsynced changes') # do not lock repo until all changegroup data is # streamed. save to temporary file. fd, tempname = tempfile.mkstemp(prefix='hg-unbundle-') fp = os.fdopen(fd, 'wb+') try: length = int(req.env['CONTENT_LENGTH']) for s in util.filechunkiter(req, limit=length): fp.write(s) try: lock = repo.lock() try: if not check_heads(): raise ErrorResponse(HTTP_OK, 'unsynced changes') fp.seek(0) header = fp.read(6) if header.startswith('HG') and not header.startswith('HG10'): raise ValueError('unknown bundle version') elif header not in changegroupmod.bundletypes: raise ValueError('unknown bundle compression type') gen = changegroupmod.unbundle(header, fp) # send addchangegroup output to client oldio = sys.stdout, sys.stderr sys.stderr = sys.stdout = cStringIO.StringIO() try: url = 'remote:%s:%s:%s' % ( proto, urllib.quote(req.env.get('REMOTE_HOST', '')), urllib.quote(req.env.get('REMOTE_USER', ''))) try: ret = repo.addchangegroup(gen, 'serve', url) except util.Abort, inst: sys.stdout.write("abort: %s\n" % inst) ret = 0 finally: val = sys.stdout.getvalue() sys.stdout, sys.stderr = oldio req.respond(HTTP_OK, HGTYPE) return '%d\n%s' % (ret, val), finally: lock.release() except ValueError, inst: raise ErrorResponse(HTTP_OK, inst) except (OSError, IOError), inst: filename = getattr(inst, 'filename', '') # Don't send our filesystem layout to the client if filename.startswith(repo.root): filename = filename[len(repo.root)+1:] else: filename = '' error = getattr(inst, 'strerror', 'Unknown error') if inst.errno == errno.ENOENT: code = HTTP_NOT_FOUND else: code = HTTP_SERVER_ERROR raise ErrorResponse(code, '%s: %s' % (error, filename))
def censor(ui, repo, path, rev='', tombstone='', **opts): if not path: raise util.Abort(_('must specify file path to censor')) if not rev: raise util.Abort(_('must specify revision to censor')) wctx = repo[None] m = scmutil.match(wctx, (path,)) if m.anypats() or len(m.files()) != 1: raise util.Abort(_('can only specify an explicit filename')) path = m.files()[0] flog = repo.file(path) if not len(flog): raise util.Abort(_('cannot censor file with no history')) rev = scmutil.revsingle(repo, rev, rev).rev() try: ctx = repo[rev] except KeyError: raise util.Abort(_('invalid revision identifier %s') % rev) try: fctx = ctx.filectx(path) except error.LookupError: raise util.Abort(_('file does not exist at revision %s') % rev) fnode = fctx.filenode() headctxs = [repo[c] for c in repo.heads()] heads = [c for c in headctxs if path in c and c.filenode(path) == fnode] if heads: headlist = ', '.join([short(c.node()) for c in heads]) raise util.Abort(_('cannot censor file in heads (%s)') % headlist, hint=_('clean/delete and commit first')) wp = wctx.parents() if ctx.node() in [p.node() for p in wp]: raise util.Abort(_('cannot censor working directory'), hint=_('clean/delete/update first')) flogv = flog.version & 0xFFFF if flogv != revlog.REVLOGNG: raise util.Abort( _('censor does not support revlog version %d') % (flogv,)) tombstone = filelog.packmeta({"censored": tombstone}, "") crev = fctx.filerev() if len(tombstone) > flog.rawsize(crev): raise util.Abort(_( 'censor tombstone must be no longer than censored data')) # Using two files instead of one makes it easy to rewrite entry-by-entry idxread = repo.svfs(flog.indexfile, 'r') idxwrite = repo.svfs(flog.indexfile, 'wb', atomictemp=True) if flog.version & revlog.REVLOGNGINLINEDATA: dataread, datawrite = idxread, idxwrite else: dataread = repo.svfs(flog.datafile, 'r') datawrite = repo.svfs(flog.datafile, 'wb', atomictemp=True) # Copy all revlog data up to the entry to be censored. rio = revlog.revlogio() offset = flog.start(crev) for chunk in util.filechunkiter(idxread, limit=crev * rio.size): idxwrite.write(chunk) for chunk in util.filechunkiter(dataread, limit=offset): datawrite.write(chunk) def rewriteindex(r, newoffs, newdata=None): """Rewrite the index entry with a new data offset and optional new data. The newdata argument, if given, is a tuple of three positive integers: (new compressed, new uncompressed, added flag bits). """ offlags, comp, uncomp, base, link, p1, p2, nodeid = flog.index[r] flags = revlog.gettype(offlags) if newdata: comp, uncomp, nflags = newdata flags |= nflags offlags = revlog.offset_type(newoffs, flags) e = (offlags, comp, uncomp, r, link, p1, p2, nodeid) idxwrite.write(rio.packentry(e, None, flog.version, r)) idxread.seek(rio.size, 1) def rewrite(r, offs, data, nflags=revlog.REVIDX_DEFAULT_FLAGS): """Write the given full text to the filelog with the given data offset. Returns: The integer number of data bytes written, for tracking data offsets. """ flag, compdata = flog.compress(data) newcomp = len(flag) + len(compdata) rewriteindex(r, offs, (newcomp, len(data), nflags)) datawrite.write(flag) datawrite.write(compdata) dataread.seek(flog.length(r), 1) return newcomp # Rewrite censored revlog entry with (padded) tombstone data. pad = ' ' * (flog.rawsize(crev) - len(tombstone)) offset += rewrite(crev, offset, tombstone + pad, revlog.REVIDX_ISCENSORED) # Rewrite all following filelog revisions fixing up offsets and deltas. for srev in xrange(crev + 1, len(flog)): if crev in flog.parentrevs(srev): # Immediate children of censored node must be re-added as fulltext. try: revdata = flog.revision(srev) except error.CensoredNodeError as e: revdata = e.tombstone dlen = rewrite(srev, offset, revdata) else: # Copy any other revision data verbatim after fixing up the offset. rewriteindex(srev, offset) dlen = flog.length(srev) for chunk in util.filechunkiter(dataread, limit=dlen): datawrite.write(chunk) offset += dlen idxread.close() idxwrite.close() if dataread is not idxread: dataread.close() datawrite.close()
def getfile(self, fp): length = int(self.req.env['CONTENT_LENGTH']) for s in util.filechunkiter(self.req, limit=length): fp.write(s)