def consumestreamclone(repo, fp): """Apply the contents from a streaming clone file. This code is copied from Mercurial. Until Mercurial 3.5, this code was a closure in wireproto.py and not consumeable by extensions. """ lock = repo.lock() try: repo.ui.status(_('streaming all changes\n')) l = fp.readline() try: total_files, total_bytes = map(int, l.split(' ', 1)) except (ValueError, TypeError): raise error.ResponseError( _('unexpected response from remote server:'), l) repo.ui.status(_('%d files to transfer, %s of data\n') % (total_files, util.bytecount(total_bytes))) handled_bytes = 0 repo.ui.progress(_('clone'), 0, total=total_bytes) start = time.time() tr = repo.transaction(_('clone')) try: for i in xrange(total_files): # XXX doesn't support '\n' or '\r' in filenames l = fp.readline() try: name, size = l.split('\0', 1) size = int(size) except (ValueError, TypeError): raise error.ResponseError( _('unexpected response from remote server:'), l) if repo.ui.debugflag: repo.ui.debug('adding %s (%s)\n' % (name, util.bytecount(size))) # for backwards compat, name was partially encoded ofp = repo.svfs(store.decodedir(name), 'w') for chunk in util.filechunkiter(fp, limit=size): handled_bytes += len(chunk) repo.ui.progress(_('clone'), handled_bytes, total=total_bytes) ofp.write(chunk) ofp.close() tr.close() finally: tr.release() # Writing straight to files circumvented the inmemory caches repo.invalidate() elapsed = time.time() - start if elapsed <= 0: elapsed = 0.001 repo.ui.progress(_('clone'), None) repo.ui.status(_('transferred %s in %.1f seconds (%s/sec)\n') % (util.bytecount(total_bytes), elapsed, util.bytecount(total_bytes / elapsed))) finally: lock.release()
def consumev1(repo, fp, filecount, bytecount): """Apply the contents from version 1 of a streaming clone file handle. This takes the output from "streamout" and applies it to the specified repository. Like "streamout," the status line added by the wire protocol is not handled by this function. """ lock = repo.lock() try: repo.ui.status( _('%d files to transfer, %s of data\n') % (filecount, util.bytecount(bytecount))) handled_bytes = 0 repo.ui.progress(_('clone'), 0, total=bytecount) start = time.time() tr = repo.transaction(_('clone')) try: for i in xrange(filecount): # XXX doesn't support '\n' or '\r' in filenames l = fp.readline() try: name, size = l.split('\0', 1) size = int(size) except (ValueError, TypeError): raise error.ResponseError( _('unexpected response from remote server:'), l) if repo.ui.debugflag: repo.ui.debug('adding %s (%s)\n' % (name, util.bytecount(size))) # for backwards compat, name was partially encoded ofp = repo.svfs(store.decodedir(name), 'w') for chunk in util.filechunkiter(fp, limit=size): handled_bytes += len(chunk) repo.ui.progress(_('clone'), handled_bytes, total=bytecount) ofp.write(chunk) ofp.close() tr.close() finally: tr.release() # Writing straight to files circumvented the inmemory caches repo.invalidate() elapsed = time.time() - start if elapsed <= 0: elapsed = 0.001 repo.ui.progress(_('clone'), None) repo.ui.status( _('transferred %s in %.1f seconds (%s/sec)\n') % (util.bytecount(bytecount), elapsed, util.bytecount(bytecount / elapsed))) finally: lock.release()
def consumev1(repo, fp, filecount, bytecount): """Apply the contents from version 1 of a streaming clone file handle. This takes the output from "streamout" and applies it to the specified repository. Like "streamout," the status line added by the wire protocol is not handled by this function. """ lock = repo.lock() try: repo.ui.status(_('%d files to transfer, %s of data\n') % (filecount, util.bytecount(bytecount))) handled_bytes = 0 repo.ui.progress(_('clone'), 0, total=bytecount) start = time.time() tr = repo.transaction(_('clone')) try: for i in xrange(filecount): # XXX doesn't support '\n' or '\r' in filenames l = fp.readline() try: name, size = l.split('\0', 1) size = int(size) except (ValueError, TypeError): raise error.ResponseError( _('unexpected response from remote server:'), l) if repo.ui.debugflag: repo.ui.debug('adding %s (%s)\n' % (name, util.bytecount(size))) # for backwards compat, name was partially encoded ofp = repo.svfs(store.decodedir(name), 'w') for chunk in util.filechunkiter(fp, limit=size): handled_bytes += len(chunk) repo.ui.progress(_('clone'), handled_bytes, total=bytecount) ofp.write(chunk) ofp.close() tr.close() finally: tr.release() # Writing straight to files circumvented the inmemory caches repo.invalidate() elapsed = time.time() - start if elapsed <= 0: elapsed = 0.001 repo.ui.progress(_('clone'), None) repo.ui.status(_('transferred %s in %.1f seconds (%s/sec)\n') % (util.bytecount(bytecount), elapsed, util.bytecount(bytecount / elapsed))) finally: lock.release()
def _walkstreamfiles(orig, repo): if state.shallowremote: # if we are shallow ourselves, stream our local commits if shallowrepo.requirement in repo.requirements: striplen = len(repo.store.path) + 1 readdir = repo.store.rawvfs.readdir visit = [os.path.join(repo.store.path, 'data')] while visit: p = visit.pop() for f, kind, st in readdir(p, stat=True): fp = p + '/' + f if kind == stat.S_IFREG: if not fp.endswith('.i') and not fp.endswith('.d'): n = util.pconvert(fp[striplen:]) yield (store.decodedir(n), n, st.st_size) if kind == stat.S_IFDIR: visit.append(fp) shallowtrees = repo.ui.configbool('remotefilelog', 'shallowtrees', False) if 'treemanifest' in repo.requirements and not shallowtrees: for (u, e, s) in repo.store.datafiles(): if (u.startswith('meta/') and (u.endswith('.i') or u.endswith('.d'))): yield (u, e, s) # Return .d and .i files that do not match the shallow pattern match = state.match if match and not match.always(): for (u, e, s) in repo.store.datafiles(): f = u[5:-2] # trim data/... and .i/.d if not state.match(f): yield (u, e, s) for x in repo.store.topfiles(): if shallowtrees and x[0][:15] == '00manifesttree.': continue if state.noflatmf and x[0][:11] == '00manifest.': continue yield x elif shallowrepo.requirement in repo.requirements: # don't allow cloning from a shallow repo to a full repo # since it would require fetching every version of every # file in order to create the revlogs. raise error.Abort(_("Cannot clone from a shallow repo " "to a full repo.")) else: for x in orig(repo): yield x
def _walkstreamfiles(orig, repo, matcher=None): if state.shallowremote: # if we are shallow ourselves, stream our local commits if shallowutil.isenabled(repo): striplen = len(repo.store.path) + 1 readdir = repo.store.rawvfs.readdir visit = [os.path.join(repo.store.path, b'data')] while visit: p = visit.pop() for f, kind, st in readdir(p, stat=True): fp = p + b'/' + f if kind == stat.S_IFREG: if not fp.endswith(b'.i') and not fp.endswith( b'.d'): n = util.pconvert(fp[striplen:]) d = store.decodedir(n) t = store.FILETYPE_OTHER yield (t, d, n, st.st_size) if kind == stat.S_IFDIR: visit.append(fp) if scmutil.istreemanifest(repo): for (t, u, e, s) in repo.store.datafiles(): if u.startswith(b'meta/') and (u.endswith(b'.i') or u.endswith(b'.d')): yield (t, u, e, s) # Return .d and .i files that do not match the shallow pattern match = state.match if match and not match.always(): for (t, u, e, s) in repo.store.datafiles(): f = u[5:-2] # trim data/... and .i/.d if not state.match(f): yield (t, u, e, s) for x in repo.store.topfiles(): if state.noflatmf and x[1][:11] == b'00manifest.': continue yield x elif shallowutil.isenabled(repo): # don't allow cloning from a shallow repo to a full repo # since it would require fetching every version of every # file in order to create the revlogs. raise error.Abort( _(b"Cannot clone from a shallow repo to a full repo.")) else: for x in orig(repo, matcher): yield x
def _walk(self, relpath, recurse, allfiles=False): """modifies walk to return non .i/.d files so streaming clones can send remotefilelog store/data files """ path = self.path if relpath: path += '/' + relpath striplen = len(self.path) + 1 l = [] if self.rawvfs.isdir(path): visit = [path] readdir = self.rawvfs.readdir while visit: p = visit.pop() for f, kind, st in readdir(p, stat=True): fp = p + '/' + f if (kind == stat.S_IFREG and (allfiles or f[-2:] in ('.d', '.i'))): n = util.pconvert(fp[striplen:]) l.append((storemod.decodedir(n), n, st.st_size)) elif kind == stat.S_IFDIR and recurse: visit.append(fp) l.sort() return l