def _verifyfile(self, cctx, cset, contents, standin, verified): filename = lfutil.splitstandin(standin) if not filename: return False fctx = cctx[standin] key = (filename, fctx.filenode()) if key in verified: return False expecthash = fctx.data()[0:40] verified.add(key) if not lfutil.instore(self.remote, expecthash): self.ui.warn( _('changeset %s: %s missing\n' ' (looked for hash %s)\n') % (cset, filename, expecthash)) return True # failed if contents: storepath = lfutil.storepath(self.remote, expecthash) actualhash = lfutil.hashfile(storepath) if actualhash != expecthash: self.ui.warn( _('changeset %s: %s: contents differ\n' ' (%s:\n' ' expected hash %s,\n' ' but got %s)\n') % (cset, filename, storepath, expecthash, actualhash)) return True # failed return False
def cachelfiles(ui, repo, node): '''cachelfiles ensures that all largefiles needed by the specified revision are present in the repository's largefile cache. returns a tuple (cached, missing). cached is the list of files downloaded by this operation; missing is the list of files that were needed but could not be found.''' lfiles = lfutil.listlfiles(repo, node) toget = [] for lfile in lfiles: expectedhash = repo[node][lfutil.standin(lfile)].data().strip() # if it exists and its hash matches, it might have been locally # modified before updating and the user chose 'local'. in this case, # it will not be in any store, so don't look for it. if ((not os.path.exists(repo.wjoin(lfile)) or expectedhash != lfutil.hashfile(repo.wjoin(lfile))) and not lfutil.findfile(repo, expectedhash)): toget.append((lfile, expectedhash)) if toget: store = basestore._openstore(repo) ret = store.get(toget) return ret return ([], [])
def overrideupdate(orig, ui, repo, *pats, **opts): lfdirstate = lfutil.openlfdirstate(ui, repo) s = lfdirstate.status(match_.always(repo.root, repo.getcwd()), [], False, False, False) (unsure, modified, added, removed, missing, unknown, ignored, clean) = s # Need to lock between the standins getting updated and their # largefiles getting updated wlock = repo.wlock() try: if opts['check']: mod = len(modified) > 0 for lfile in unsure: standin = lfutil.standin(lfile) if repo['.'][standin].data().strip() != \ lfutil.hashfile(repo.wjoin(lfile)): mod = True else: lfdirstate.normal(lfile) lfdirstate.write() if mod: raise util.Abort(_('uncommitted changes')) # XXX handle removed differently if not opts['clean']: for lfile in unsure + modified + added: lfutil.updatestandin(repo, lfutil.standin(lfile)) finally: wlock.release() return orig(ui, repo, *pats, **opts)
def overrideupdate(orig, ui, repo, *pats, **opts): lfdirstate = lfutil.openlfdirstate(ui, repo) s = lfdirstate.status(match_.always(repo.root, repo.getcwd()), [], False, False, False) (unsure, modified, added, removed, missing, unknown, ignored, clean) = s # Need to lock between the standins getting updated and their # largefiles getting updated wlock = repo.wlock() try: if opts['check']: mod = len(modified) > 0 for lfile in unsure: standin = lfutil.standin(lfile) if repo['.'][standin].data().strip() != \ lfutil.hashfile(repo.wjoin(lfile)): mod = True else: lfdirstate.normal(lfile) lfdirstate.write() if mod: raise util.Abort(_('uncommitted local changes')) # XXX handle removed differently if not opts['clean']: for lfile in unsure + modified + added: lfutil.updatestandin(repo, lfutil.standin(lfile)) finally: wlock.release() return orig(ui, repo, *pats, **opts)
def cachelfiles(ui, repo, node): '''cachelfiles ensures that all largefiles needed by the specified revision are present in the repository's largefile cache. returns a tuple (cached, missing). cached is the list of files downloaded by this operation; missing is the list of files that were needed but could not be found.''' lfiles = lfutil.listlfiles(repo, node) toget = [] for lfile in lfiles: # If we are mid-merge, then we have to trust the standin that is in the # working copy to have the correct hashvalue. This is because the # original hg.merge() already updated the standin as part of the normal # merge process -- we just have to udpate the largefile to match. if (getattr(repo, "_ismerging", False) and os.path.exists(repo.wjoin(lfutil.standin(lfile)))): expectedhash = lfutil.readstandin(repo, lfile) else: expectedhash = repo[node][lfutil.standin(lfile)].data().strip() # if it exists and its hash matches, it might have been locally # modified before updating and the user chose 'local'. in this case, # it will not be in any store, so don't look for it. if ((not os.path.exists(repo.wjoin(lfile)) or expectedhash != lfutil.hashfile(repo.wjoin(lfile))) and not lfutil.findfile(repo, expectedhash)): toget.append((lfile, expectedhash)) if toget: store = basestore._openstore(repo) ret = store.get(toget) return ret return ([], [])
def cachelfiles(ui, repo, node): """cachelfiles ensures that all largefiles needed by the specified revision are present in the repository's largefile cache. returns a tuple (cached, missing). cached is the list of files downloaded by this operation; missing is the list of files that were needed but could not be found.""" lfiles = lfutil.listlfiles(repo, node) toget = [] for lfile in lfiles: # If we are mid-merge, then we have to trust the standin that is in the # working copy to have the correct hashvalue. This is because the # original hg.merge() already updated the standin as part of the normal # merge process -- we just have to udpate the largefile to match. if getattr(repo, "_ismerging", False) and os.path.exists(repo.wjoin(lfutil.standin(lfile))): expectedhash = lfutil.readstandin(repo, lfile) else: expectedhash = repo[node][lfutil.standin(lfile)].data().strip() # if it exists and its hash matches, it might have been locally # modified before updating and the user chose 'local'. in this case, # it will not be in any store, so don't look for it. if ( not os.path.exists(repo.wjoin(lfile)) or expectedhash != lfutil.hashfile(repo.wjoin(lfile)) ) and not lfutil.findfile(repo, expectedhash): toget.append((lfile, expectedhash)) if toget: store = basestore._openstore(repo) ret = store.get(toget) return ret return ([], [])
def _verifyfile(self, cctx, cset, contents, standin, verified): filename = lfutil.splitstandin(standin) if not filename: return False fctx = cctx[standin] key = (filename, fctx.filenode()) if key in verified: return False expecthash = fctx.data()[0:40] storepath, exists = lfutil.findstorepath(self.remote, expecthash) verified.add(key) if not exists: self.ui.warn( _('changeset %s: %s references missing %s\n') % (cset, filename, storepath)) return True # failed if contents: actualhash = lfutil.hashfile(storepath) if actualhash != expecthash: self.ui.warn( _('changeset %s: %s references corrupted %s\n') % (cset, filename, storepath)) return True # failed return False
def _updatelfile(repo, lfdirstate, lfile): '''updates a single largefile and copies the state of its standin from the repository's dirstate to its state in the lfdirstate. returns 1 if the file was modified, -1 if the file was removed, 0 if the file was unchanged, and None if the needed largefile was missing from the cache.''' ret = 0 abslfile = repo.wjoin(lfile) absstandin = repo.wjoin(lfutil.standin(lfile)) if os.path.exists(absstandin): if os.path.exists(absstandin + '.orig') and os.path.exists(abslfile): shutil.copyfile(abslfile, abslfile + '.orig') expecthash = lfutil.readstandin(repo, lfile) if (expecthash != '' and (not os.path.exists(abslfile) or expecthash != lfutil.hashfile(abslfile))): if not lfutil.copyfromcache(repo, expecthash, lfile): # use normallookup() to allocate entry in largefiles dirstate, # because lack of it misleads lfilesrepo.status() into # recognition that such cache missing files are REMOVED. if lfile not in repo[None]: # not switched to normal file util.unlinkpath(abslfile, ignoremissing=True) lfdirstate.normallookup(lfile) return None # don't try to set the mode else: # Synchronize largefile dirstate to the last modified time of # the file lfdirstate.normal(lfile) ret = 1 mode = os.stat(absstandin).st_mode if mode != os.stat(abslfile).st_mode: os.chmod(abslfile, mode) ret = 1 else: # Remove lfiles for which the standin is deleted, unless the # lfile is added to the repository again. This happens when a # largefile is converted back to a normal file: the standin # disappears, but a new (normal) file appears as the lfile. if (os.path.exists(abslfile) and repo.dirstate.normalize(lfile) not in repo[None]): util.unlinkpath(abslfile) ret = -1 state = repo.dirstate[lfutil.standin(lfile)] if state == 'n': # When rebasing, we need to synchronize the standin and the largefile, # because otherwise the largefile will get reverted. But for commit's # sake, we have to mark the file as unclean. if getattr(repo, "_isrebasing", False): lfdirstate.normallookup(lfile) else: lfdirstate.normal(lfile) elif state == 'r': lfdirstate.remove(lfile) elif state == 'a': lfdirstate.add(lfile) elif state == '?': lfdirstate.drop(lfile) return ret
def _verifyfiles(self, contents, filestocheck): failed = False for cset, filename, expectedhash in filestocheck: storepath, exists = lfutil.findstorepath(self.remote, expectedhash) if not exists: self.ui.warn( _('changeset %s: %s references missing %s\n') % (cset, filename, storepath)) failed = True elif contents: actualhash = lfutil.hashfile(storepath) if actualhash != expectedhash: self.ui.warn( _('changeset %s: %s references corrupted %s\n') % (cset, filename, storepath)) failed = True return failed
def _updatelfile(repo, lfdirstate, lfile): '''updates a single largefile and copies the state of its standin from the repository's dirstate to its state in the lfdirstate. returns 1 if the file was modified, -1 if the file was removed, 0 if the file was unchanged, and None if the needed largefile was missing from the cache.''' ret = 0 abslfile = repo.wjoin(lfile) absstandin = repo.wjoin(lfutil.standin(lfile)) if os.path.exists(absstandin): if os.path.exists(absstandin+'.orig'): shutil.copyfile(abslfile, abslfile+'.orig') expecthash = lfutil.readstandin(repo, lfile) if (expecthash != '' and (not os.path.exists(abslfile) or expecthash != lfutil.hashfile(abslfile))): if not lfutil.copyfromcache(repo, expecthash, lfile): return None # don't try to set the mode or update the dirstate ret = 1 mode = os.stat(absstandin).st_mode if mode != os.stat(abslfile).st_mode: os.chmod(abslfile, mode) ret = 1 else: if os.path.exists(abslfile): os.unlink(abslfile) ret = -1 state = repo.dirstate[lfutil.standin(lfile)] if state == 'n': lfdirstate.normal(lfile) elif state == 'r': lfdirstate.remove(lfile) elif state == 'a': lfdirstate.add(lfile) elif state == '?': lfdirstate.drop(lfile) return ret
def status(self, node1='.', node2=None, match=None, ignored=False, clean=False, unknown=False, listsubrepos=False): listignored, listclean, listunknown = ignored, clean, unknown if not self.lfstatus: return super(lfilesrepo, self).status(node1, node2, match, listignored, listclean, listunknown, listsubrepos) else: # some calls in this function rely on the old version of status self.lfstatus = False if isinstance(node1, context.changectx): ctx1 = node1 else: ctx1 = self[node1] if isinstance(node2, context.changectx): ctx2 = node2 else: ctx2 = self[node2] working = ctx2.rev() is None parentworking = working and ctx1 == self['.'] def inctx(file, ctx): try: if ctx.rev() is None: return file in ctx.manifest() ctx[file] return True except KeyError: return False if match is None: match = match_.always(self.root, self.getcwd()) wlock = None try: try: # updating the dirstate is optional # so we don't wait on the lock wlock = self.wlock(False) except error.LockError: pass # First check if there were files specified on the # command line. If there were, and none of them were # largefiles, we should just bail here and let super # handle it -- thus gaining a big performance boost. lfdirstate = lfutil.openlfdirstate(ui, self) if match.files() and not match.anypats(): for f in lfdirstate: if match(f): break else: return super(lfilesrepo, self).status(node1, node2, match, listignored, listclean, listunknown, listsubrepos) # Create a copy of match that matches standins instead # of largefiles. def tostandins(files): if not working: return files newfiles = [] dirstate = self.dirstate for f in files: sf = lfutil.standin(f) if sf in dirstate: newfiles.append(sf) elif sf in dirstate.dirs(): # Directory entries could be regular or # standin, check both newfiles.extend((f, sf)) else: newfiles.append(f) return newfiles m = copy.copy(match) m._files = tostandins(m._files) result = super(lfilesrepo, self).status(node1, node2, m, ignored, clean, unknown, listsubrepos) if working: def sfindirstate(f): sf = lfutil.standin(f) dirstate = self.dirstate return sf in dirstate or sf in dirstate.dirs() match._files = [f for f in match._files if sfindirstate(f)] # Don't waste time getting the ignored and unknown # files from lfdirstate s = lfdirstate.status(match, [], False, listclean, False) (unsure, modified, added, removed, missing, _unknown, _ignored, clean) = s if parentworking: for lfile in unsure: standin = lfutil.standin(lfile) if standin not in ctx1: # from second parent modified.append(lfile) elif ctx1[standin].data().strip() \ != lfutil.hashfile(self.wjoin(lfile)): modified.append(lfile) else: clean.append(lfile) lfdirstate.normal(lfile) else: tocheck = unsure + modified + added + clean modified, added, clean = [], [], [] for lfile in tocheck: standin = lfutil.standin(lfile) if inctx(standin, ctx1): if ctx1[standin].data().strip() != \ lfutil.hashfile(self.wjoin(lfile)): modified.append(lfile) else: clean.append(lfile) else: added.append(lfile) # Standins no longer found in lfdirstate has been # removed for standin in ctx1.manifest(): if not lfutil.isstandin(standin): continue lfile = lfutil.splitstandin(standin) if not match(lfile): continue if lfile not in lfdirstate: removed.append(lfile) # Filter result lists result = list(result) # Largefiles are not really removed when they're # still in the normal dirstate. Likewise, normal # files are not really removed if they are still in # lfdirstate. This happens in merges where files # change type. removed = [f for f in removed if f not in self.dirstate] result[2] = [f for f in result[2] if f not in lfdirstate] lfiles = set(lfdirstate._map) # Unknown files result[4] = set(result[4]).difference(lfiles) # Ignored files result[5] = set(result[5]).difference(lfiles) # combine normal files and largefiles normals = [[fn for fn in filelist if not lfutil.isstandin(fn)] for filelist in result] lfiles = (modified, added, removed, missing, [], [], clean) result = [sorted(list1 + list2) for (list1, list2) in zip(normals, lfiles)] else: def toname(f): if lfutil.isstandin(f): return lfutil.splitstandin(f) return f result = [[toname(f) for f in items] for items in result] if wlock: lfdirstate.write() finally: if wlock: wlock.release() if not listunknown: result[4] = [] if not listignored: result[5] = [] if not listclean: result[6] = [] self.lfstatus = True return result
def updatelfiles(ui, repo, filelist=None, printmessage=None, normallookup=False, checked=False): """Update largefiles according to standins in the working directory If ``printmessage`` is other than ``None``, it means "print (or ignore, for false) message forcibly". """ statuswriter = lfutil.getstatuswriter(ui, repo, printmessage) wlock = repo.wlock() try: lfdirstate = lfutil.openlfdirstate(ui, repo) lfiles = set(lfutil.listlfiles(repo)) | set(lfdirstate) if filelist is not None: filelist = set(filelist) lfiles = [f for f in lfiles if f in filelist] update = {} updated, removed = 0, 0 for lfile in lfiles: abslfile = repo.wjoin(lfile) absstandin = repo.wjoin(lfutil.standin(lfile)) if os.path.exists(absstandin): if os.path.exists(absstandin + ".orig") and os.path.exists(abslfile): shutil.copyfile(abslfile, abslfile + ".orig") util.unlinkpath(absstandin + ".orig") expecthash = lfutil.readstandin(repo, lfile) if expecthash != "" and ( checked or not os.path.exists(abslfile) or expecthash != lfutil.hashfile(abslfile) ): if lfile not in repo[None]: # not switched to normal file util.unlinkpath(abslfile, ignoremissing=True) # use normallookup() to allocate an entry in largefiles # dirstate, because lack of it misleads # lfilesrepo.status() into recognition that such cache # missing files are removed. lfdirstate.normallookup(lfile) update[lfile] = expecthash else: # Remove lfiles for which the standin is deleted, unless the # lfile is added to the repository again. This happens when a # largefile is converted back to a normal file: the standin # disappears, but a new (normal) file appears as the lfile. if os.path.exists(abslfile) and repo.dirstate.normalize(lfile) not in repo[None]: util.unlinkpath(abslfile) removed += 1 # largefile processing might be slow and be interrupted - be prepared lfdirstate.write() if lfiles: statuswriter(_("getting changed largefiles\n")) cachelfiles(ui, repo, None, lfiles) for lfile in lfiles: update1 = 0 expecthash = update.get(lfile) if expecthash: if not lfutil.copyfromcache(repo, expecthash, lfile): # failed ... but already removed and set to normallookup continue # Synchronize largefile dirstate to the last modified # time of the file lfdirstate.normal(lfile) update1 = 1 # copy the state of largefile standin from the repository's # dirstate to its state in the lfdirstate. abslfile = repo.wjoin(lfile) absstandin = repo.wjoin(lfutil.standin(lfile)) if os.path.exists(absstandin): mode = os.stat(absstandin).st_mode if mode != os.stat(abslfile).st_mode: os.chmod(abslfile, mode) update1 = 1 updated += update1 lfutil.synclfdirstate(repo, lfdirstate, lfile, normallookup) lfdirstate.write() if lfiles: statuswriter(_("%d largefiles updated, %d removed\n") % (updated, removed)) finally: wlock.release()
def status(self, node1='.', node2=None, match=None, ignored=False, clean=False, unknown=False, listsubrepos=False): listignored, listclean, listunknown = ignored, clean, unknown if not self.lfstatus: return super(lfilesrepo, self).status(node1, node2, match, listignored, listclean, listunknown, listsubrepos) else: # some calls in this function rely on the old version of status self.lfstatus = False if isinstance(node1, context.changectx): ctx1 = node1 else: ctx1 = repo[node1] if isinstance(node2, context.changectx): ctx2 = node2 else: ctx2 = repo[node2] working = ctx2.rev() is None parentworking = working and ctx1 == self['.'] def inctx(file, ctx): try: if ctx.rev() is None: return file in ctx.manifest() ctx[file] return True except KeyError: return False if match is None: match = match_.always(self.root, self.getcwd()) # First check if there were files specified on the # command line. If there were, and none of them were # largefiles, we should just bail here and let super # handle it -- thus gaining a big performance boost. lfdirstate = lfutil.openlfdirstate(ui, self) if match.files() and not match.anypats(): for f in lfdirstate: if match(f): break else: return super(lfilesrepo, self).status(node1, node2, match, listignored, listclean, listunknown, listsubrepos) # Create a copy of match that matches standins instead # of largefiles. def tostandins(files): if not working: return files newfiles = [] dirstate = repo.dirstate for f in files: sf = lfutil.standin(f) if sf in dirstate: newfiles.append(sf) elif sf in dirstate.dirs(): # Directory entries could be regular or # standin, check both newfiles.extend((f, sf)) else: newfiles.append(f) return newfiles # Create a function that we can use to override what is # normally the ignore matcher. We've already checked # for ignored files on the first dirstate walk, and # unecessarily re-checking here causes a huge performance # hit because lfdirstate only knows about largefiles def _ignoreoverride(self): return False m = copy.copy(match) m._files = tostandins(m._files) # Get ignored files here even if we weren't asked for them; we # must use the result here for filtering later result = super(lfilesrepo, self).status(node1, node2, m, True, clean, unknown, listsubrepos) if working: try: # Any non-largefiles that were explicitly listed must be # taken out or lfdirstate.status will report an error. # The status of these files was already computed using # super's status. # Override lfdirstate's ignore matcher to not do # anything origignore = lfdirstate._ignore lfdirstate._ignore = _ignoreoverride def sfindirstate(f): sf = lfutil.standin(f) dirstate = repo.dirstate return sf in dirstate or sf in dirstate.dirs() match._files = [f for f in match._files if sfindirstate(f)] # Don't waste time getting the ignored and unknown # files again; we already have them s = lfdirstate.status(match, [], False, listclean, False) (unsure, modified, added, removed, missing, unknown, ignored, clean) = s # Replace the list of ignored and unknown files with # the previously caclulated lists, and strip out the # largefiles lfiles = set(lfdirstate._map) ignored = set(result[5]).difference(lfiles) unknown = set(result[4]).difference(lfiles) if parentworking: for lfile in unsure: standin = lfutil.standin(lfile) if standin not in ctx1: # from second parent modified.append(lfile) elif ctx1[standin].data().strip() \ != lfutil.hashfile(self.wjoin(lfile)): modified.append(lfile) else: clean.append(lfile) lfdirstate.normal(lfile) else: tocheck = unsure + modified + added + clean modified, added, clean = [], [], [] for lfile in tocheck: standin = lfutil.standin(lfile) if inctx(standin, ctx1): if ctx1[standin].data().strip() != \ lfutil.hashfile(self.wjoin(lfile)): modified.append(lfile) else: clean.append(lfile) else: added.append(lfile) finally: # Replace the original ignore function lfdirstate._ignore = origignore for standin in ctx1.manifest(): if not lfutil.isstandin(standin): continue lfile = lfutil.splitstandin(standin) if not match(lfile): continue if lfile not in lfdirstate: removed.append(lfile) # Filter result lists result = list(result) # Largefiles are not really removed when they're # still in the normal dirstate. Likewise, normal # files are not really removed if it's still in # lfdirstate. This happens in merges where files # change type. removed = [f for f in removed if f not in repo.dirstate] result[2] = [f for f in result[2] if f not in lfdirstate] # Unknown files unknown = set(unknown).difference(ignored) result[4] = [f for f in unknown if (repo.dirstate[f] == '?' and not lfutil.isstandin(f))] # Ignored files were calculated earlier by the dirstate, # and we already stripped out the largefiles from the list result[5] = ignored # combine normal files and largefiles normals = [[fn for fn in filelist if not lfutil.isstandin(fn)] for filelist in result] lfiles = (modified, added, removed, missing, [], [], clean) result = [sorted(list1 + list2) for (list1, list2) in zip(normals, lfiles)] else: def toname(f): if lfutil.isstandin(f): return lfutil.splitstandin(f) return f result = [[toname(f) for f in items] for items in result] if not listunknown: result[4] = [] if not listignored: result[5] = [] if not listclean: result[6] = [] self.lfstatus = True return result
def status( self, node1=".", node2=None, match=None, ignored=False, clean=False, unknown=False, listsubrepos=False ): listignored, listclean, listunknown = ignored, clean, unknown if not self.lfstatus: return super(lfilesrepo, self).status( node1, node2, match, listignored, listclean, listunknown, listsubrepos ) else: # some calls in this function rely on the old version of status self.lfstatus = False if isinstance(node1, context.changectx): ctx1 = node1 else: ctx1 = repo[node1] if isinstance(node2, context.changectx): ctx2 = node2 else: ctx2 = repo[node2] working = ctx2.rev() is None parentworking = working and ctx1 == self["."] def inctx(file, ctx): try: if ctx.rev() is None: return file in ctx.manifest() ctx[file] return True except KeyError: return False if match is None: match = match_.always(self.root, self.getcwd()) # First check if there were files specified on the # command line. If there were, and none of them were # largefiles, we should just bail here and let super # handle it -- thus gaining a big performance boost. lfdirstate = lfutil.openlfdirstate(ui, self) if match.files() and not match.anypats(): for f in lfdirstate: if match(f): break else: return super(lfilesrepo, self).status( node1, node2, match, listignored, listclean, listunknown, listsubrepos ) # Create a copy of match that matches standins instead # of largefiles. def tostandins(files): if not working: return files newfiles = [] dirstate = repo.dirstate for f in files: sf = lfutil.standin(f) if sf in dirstate: newfiles.append(sf) elif sf in dirstate.dirs(): # Directory entries could be regular or # standin, check both newfiles.extend((f, sf)) else: newfiles.append(f) return newfiles # Create a function that we can use to override what is # normally the ignore matcher. We've already checked # for ignored files on the first dirstate walk, and # unecessarily re-checking here causes a huge performance # hit because lfdirstate only knows about largefiles def _ignoreoverride(self): return False m = copy.copy(match) m._files = tostandins(m._files) # Get ignored files here even if we weren't asked for them; we # must use the result here for filtering later result = super(lfilesrepo, self).status(node1, node2, m, True, clean, unknown, listsubrepos) if working: try: # Any non-largefiles that were explicitly listed must be # taken out or lfdirstate.status will report an error. # The status of these files was already computed using # super's status. # Override lfdirstate's ignore matcher to not do # anything origignore = lfdirstate._ignore lfdirstate._ignore = _ignoreoverride def sfindirstate(f): sf = lfutil.standin(f) dirstate = repo.dirstate return sf in dirstate or sf in dirstate.dirs() match._files = [f for f in match._files if sfindirstate(f)] # Don't waste time getting the ignored and unknown # files again; we already have them s = lfdirstate.status(match, [], False, listclean, False) (unsure, modified, added, removed, missing, unknown, ignored, clean) = s # Replace the list of ignored and unknown files with # the previously caclulated lists, and strip out the # largefiles lfiles = set(lfdirstate._map) ignored = set(result[5]).difference(lfiles) unknown = set(result[4]).difference(lfiles) if parentworking: for lfile in unsure: standin = lfutil.standin(lfile) if standin not in ctx1: # from second parent modified.append(lfile) elif ctx1[standin].data().strip() != lfutil.hashfile(self.wjoin(lfile)): modified.append(lfile) else: clean.append(lfile) lfdirstate.normal(lfile) else: tocheck = unsure + modified + added + clean modified, added, clean = [], [], [] for lfile in tocheck: standin = lfutil.standin(lfile) if inctx(standin, ctx1): if ctx1[standin].data().strip() != lfutil.hashfile(self.wjoin(lfile)): modified.append(lfile) else: clean.append(lfile) else: added.append(lfile) finally: # Replace the original ignore function lfdirstate._ignore = origignore for standin in ctx1.manifest(): if not lfutil.isstandin(standin): continue lfile = lfutil.splitstandin(standin) if not match(lfile): continue if lfile not in lfdirstate: removed.append(lfile) # Filter result lists result = list(result) # Largefiles are not really removed when they're # still in the normal dirstate. Likewise, normal # files are not really removed if it's still in # lfdirstate. This happens in merges where files # change type. removed = [f for f in removed if f not in repo.dirstate] result[2] = [f for f in result[2] if f not in lfdirstate] # Unknown files unknown = set(unknown).difference(ignored) result[4] = [f for f in unknown if (repo.dirstate[f] == "?" and not lfutil.isstandin(f))] # Ignored files were calculated earlier by the dirstate, # and we already stripped out the largefiles from the list result[5] = ignored # combine normal files and largefiles normals = [[fn for fn in filelist if not lfutil.isstandin(fn)] for filelist in result] lfiles = (modified, added, removed, missing, [], [], clean) result = [sorted(list1 + list2) for (list1, list2) in zip(normals, lfiles)] else: def toname(f): if lfutil.isstandin(f): return lfutil.splitstandin(f) return f result = [[toname(f) for f in items] for items in result] if not listunknown: result[4] = [] if not listignored: result[5] = [] if not listclean: result[6] = [] self.lfstatus = True return result
def updatelfiles(ui, repo, filelist=None, printmessage=True, normallookup=False): wlock = repo.wlock() try: lfdirstate = lfutil.openlfdirstate(ui, repo) lfiles = set(lfutil.listlfiles(repo)) | set(lfdirstate) if filelist is not None: lfiles = [f for f in lfiles if f in filelist] update = {} updated, removed = 0, 0 for lfile in lfiles: abslfile = repo.wjoin(lfile) absstandin = repo.wjoin(lfutil.standin(lfile)) if os.path.exists(absstandin): if (os.path.exists(absstandin + '.orig') and os.path.exists(abslfile)): shutil.copyfile(abslfile, abslfile + '.orig') util.unlinkpath(absstandin + '.orig') expecthash = lfutil.readstandin(repo, lfile) if (expecthash != '' and (not os.path.exists(abslfile) or expecthash != lfutil.hashfile(abslfile))): if lfile not in repo[None]: # not switched to normal file util.unlinkpath(abslfile, ignoremissing=True) # use normallookup() to allocate entry in largefiles # dirstate, because lack of it misleads # lfilesrepo.status() into recognition that such cache # missing files are REMOVED. lfdirstate.normallookup(lfile) update[lfile] = expecthash else: # Remove lfiles for which the standin is deleted, unless the # lfile is added to the repository again. This happens when a # largefile is converted back to a normal file: the standin # disappears, but a new (normal) file appears as the lfile. if (os.path.exists(abslfile) and repo.dirstate.normalize(lfile) not in repo[None]): util.unlinkpath(abslfile) removed += 1 # largefile processing might be slow and be interrupted - be prepared lfdirstate.write() if lfiles: if printmessage: ui.status(_('getting changed largefiles\n')) cachelfiles(ui, repo, None, lfiles) for lfile in lfiles: update1 = 0 expecthash = update.get(lfile) if expecthash: if not lfutil.copyfromcache(repo, expecthash, lfile): # failed ... but already removed and set to normallookup continue # Synchronize largefile dirstate to the last modified # time of the file lfdirstate.normal(lfile) update1 = 1 # copy the state of largefile standin from the repository's # dirstate to its state in the lfdirstate. abslfile = repo.wjoin(lfile) absstandin = repo.wjoin(lfutil.standin(lfile)) if os.path.exists(absstandin): mode = os.stat(absstandin).st_mode if mode != os.stat(abslfile).st_mode: os.chmod(abslfile, mode) update1 = 1 updated += update1 standin = lfutil.standin(lfile) if standin in repo.dirstate: stat = repo.dirstate._map[standin] state, mtime = stat[0], stat[3] else: state, mtime = '?', -1 if state == 'n': if normallookup or mtime < 0: # state 'n' doesn't ensure 'clean' in this case lfdirstate.normallookup(lfile) else: lfdirstate.normal(lfile) elif state == 'm': lfdirstate.normallookup(lfile) elif state == 'r': lfdirstate.remove(lfile) elif state == 'a': lfdirstate.add(lfile) elif state == '?': lfdirstate.drop(lfile) lfdirstate.write() if printmessage and lfiles: ui.status( _('%d largefiles updated, %d removed\n') % (updated, removed)) finally: wlock.release()
def status(self, node1='.', node2=None, match=None, ignored=False, clean=False, unknown=False, listsubrepos=False): listignored, listclean, listunknown = ignored, clean, unknown orig = super(lfilesrepo, self).status if not self.lfstatus: return orig(node1, node2, match, listignored, listclean, listunknown, listsubrepos) # some calls in this function rely on the old version of status self.lfstatus = False ctx1 = self[node1] ctx2 = self[node2] working = ctx2.rev() is None parentworking = working and ctx1 == self['.'] if match is None: match = match_.always(self.root, self.getcwd()) wlock = None try: try: # updating the dirstate is optional # so we don't wait on the lock wlock = self.wlock(False) except error.LockError: pass # First check if paths or patterns were specified on the # command line. If there were, and they don't match any # largefiles, we should just bail here and let super # handle it -- thus gaining a big performance boost. lfdirstate = lfutil.openlfdirstate(ui, self) if not match.always(): for f in lfdirstate: if match(f): break else: return orig(node1, node2, match, listignored, listclean, listunknown, listsubrepos) # Create a copy of match that matches standins instead # of largefiles. def tostandins(files): if not working: return files newfiles = [] dirstate = self.dirstate for f in files: sf = lfutil.standin(f) if sf in dirstate: newfiles.append(sf) elif sf in dirstate.dirs(): # Directory entries could be regular or # standin, check both newfiles.extend((f, sf)) else: newfiles.append(f) return newfiles m = copy.copy(match) m._files = tostandins(m._files) result = orig(node1, node2, m, ignored, clean, unknown, listsubrepos) if working: def sfindirstate(f): sf = lfutil.standin(f) dirstate = self.dirstate return sf in dirstate or sf in dirstate.dirs() match._files = [f for f in match._files if sfindirstate(f)] # Don't waste time getting the ignored and unknown # files from lfdirstate unsure, s = lfdirstate.status(match, [], False, listclean, False) (modified, added, removed, clean) = (s.modified, s.added, s.removed, s.clean) if parentworking: for lfile in unsure: standin = lfutil.standin(lfile) if standin not in ctx1: # from second parent modified.append(lfile) elif ctx1[standin].data().strip() \ != lfutil.hashfile(self.wjoin(lfile)): modified.append(lfile) else: if listclean: clean.append(lfile) lfdirstate.normal(lfile) else: tocheck = unsure + modified + added + clean modified, added, clean = [], [], [] checkexec = self.dirstate._checkexec for lfile in tocheck: standin = lfutil.standin(lfile) if standin in ctx1: abslfile = self.wjoin(lfile) if ((ctx1[standin].data().strip() != lfutil.hashfile(abslfile)) or (checkexec and ('x' in ctx1.flags(standin)) != bool(lfutil.getexecutable(abslfile)))): modified.append(lfile) elif listclean: clean.append(lfile) else: added.append(lfile) # at this point, 'removed' contains largefiles # marked as 'R' in the working context. # then, largefiles not managed also in the target # context should be excluded from 'removed'. removed = [ lfile for lfile in removed if lfutil.standin(lfile) in ctx1 ] # Standins no longer found in lfdirstate has been # removed for standin in ctx1.walk(lfutil.getstandinmatcher(self)): lfile = lfutil.splitstandin(standin) if not match(lfile): continue if lfile not in lfdirstate: removed.append(lfile) # Filter result lists result = list(result) # Largefiles are not really removed when they're # still in the normal dirstate. Likewise, normal # files are not really removed if they are still in # lfdirstate. This happens in merges where files # change type. removed = [f for f in removed if f not in self.dirstate] result[2] = [f for f in result[2] if f not in lfdirstate] lfiles = set(lfdirstate._map) # Unknown files result[4] = set(result[4]).difference(lfiles) # Ignored files result[5] = set(result[5]).difference(lfiles) # combine normal files and largefiles normals = [[ fn for fn in filelist if not lfutil.isstandin(fn) ] for filelist in result] lfstatus = (modified, added, removed, s.deleted, [], [], clean) result = [ sorted(list1 + list2) for (list1, list2) in zip(normals, lfstatus) ] else: # not against working directory result = [[lfutil.splitstandin(f) or f for f in items] for items in result] if wlock: lfdirstate.write() finally: if wlock: wlock.release() self.lfstatus = True return scmutil.status(*result)
def status(self, node1='.', node2=None, match=None, ignored=False, clean=False, unknown=False, listsubrepos=False): listignored, listclean, listunknown = ignored, clean, unknown orig = super(lfilesrepo, self).status if not self.lfstatus: return orig(node1, node2, match, listignored, listclean, listunknown, listsubrepos) # some calls in this function rely on the old version of status self.lfstatus = False ctx1 = self[node1] ctx2 = self[node2] working = ctx2.rev() is None parentworking = working and ctx1 == self['.'] if match is None: match = match_.always(self.root, self.getcwd()) wlock = None try: try: # updating the dirstate is optional # so we don't wait on the lock wlock = self.wlock(False) except error.LockError: pass # First check if paths or patterns were specified on the # command line. If there were, and they don't match any # largefiles, we should just bail here and let super # handle it -- thus gaining a big performance boost. lfdirstate = lfutil.openlfdirstate(ui, self) if not match.always(): for f in lfdirstate: if match(f): break else: return orig(node1, node2, match, listignored, listclean, listunknown, listsubrepos) # Create a copy of match that matches standins instead # of largefiles. def tostandins(files): if not working: return files newfiles = [] dirstate = self.dirstate for f in files: sf = lfutil.standin(f) if sf in dirstate: newfiles.append(sf) elif sf in dirstate.dirs(): # Directory entries could be regular or # standin, check both newfiles.extend((f, sf)) else: newfiles.append(f) return newfiles m = copy.copy(match) m._files = tostandins(m._files) result = orig(node1, node2, m, ignored, clean, unknown, listsubrepos) if working: def sfindirstate(f): sf = lfutil.standin(f) dirstate = self.dirstate return sf in dirstate or sf in dirstate.dirs() match._files = [f for f in match._files if sfindirstate(f)] # Don't waste time getting the ignored and unknown # files from lfdirstate unsure, s = lfdirstate.status(match, [], False, listclean, False) (modified, added, removed, clean) = (s.modified, s.added, s.removed, s.clean) if parentworking: for lfile in unsure: standin = lfutil.standin(lfile) if standin not in ctx1: # from second parent modified.append(lfile) elif ctx1[standin].data().strip() \ != lfutil.hashfile(self.wjoin(lfile)): modified.append(lfile) else: if listclean: clean.append(lfile) lfdirstate.normal(lfile) else: tocheck = unsure + modified + added + clean modified, added, clean = [], [], [] checkexec = self.dirstate._checkexec for lfile in tocheck: standin = lfutil.standin(lfile) if standin in ctx1: abslfile = self.wjoin(lfile) if ((ctx1[standin].data().strip() != lfutil.hashfile(abslfile)) or (checkexec and ('x' in ctx1.flags(standin)) != bool(lfutil.getexecutable(abslfile)))): modified.append(lfile) elif listclean: clean.append(lfile) else: added.append(lfile) # at this point, 'removed' contains largefiles # marked as 'R' in the working context. # then, largefiles not managed also in the target # context should be excluded from 'removed'. removed = [lfile for lfile in removed if lfutil.standin(lfile) in ctx1] # Standins no longer found in lfdirstate has been # removed for standin in ctx1.walk(lfutil.getstandinmatcher(self)): lfile = lfutil.splitstandin(standin) if not match(lfile): continue if lfile not in lfdirstate: removed.append(lfile) # Filter result lists result = list(result) # Largefiles are not really removed when they're # still in the normal dirstate. Likewise, normal # files are not really removed if they are still in # lfdirstate. This happens in merges where files # change type. removed = [f for f in removed if f not in self.dirstate] result[2] = [f for f in result[2] if f not in lfdirstate] lfiles = set(lfdirstate._map) # Unknown files result[4] = set(result[4]).difference(lfiles) # Ignored files result[5] = set(result[5]).difference(lfiles) # combine normal files and largefiles normals = [[fn for fn in filelist if not lfutil.isstandin(fn)] for filelist in result] lfstatus = (modified, added, removed, s.deleted, [], [], clean) result = [sorted(list1 + list2) for (list1, list2) in zip(normals, lfstatus)] else: # not against working directory result = [[lfutil.splitstandin(f) or f for f in items] for items in result] if wlock: lfdirstate.write() finally: if wlock: wlock.release() self.lfstatus = True return scmutil.status(*result)
def updatelfiles(ui, repo, filelist=None, printmessage=None, normallookup=False, checked=False): '''Update largefiles according to standins in the working directory If ``printmessage`` is other than ``None``, it means "print (or ignore, for false) message forcibly". ''' statuswriter = lfutil.getstatuswriter(ui, repo, printmessage) wlock = repo.wlock() try: lfdirstate = lfutil.openlfdirstate(ui, repo) lfiles = set(lfutil.listlfiles(repo)) | set(lfdirstate) if filelist is not None: filelist = set(filelist) lfiles = [f for f in lfiles if f in filelist] update = {} updated, removed = 0, 0 for lfile in lfiles: abslfile = repo.wjoin(lfile) absstandin = repo.wjoin(lfutil.standin(lfile)) if os.path.exists(absstandin): if (os.path.exists(absstandin + '.orig') and os.path.exists(abslfile)): shutil.copyfile(abslfile, abslfile + '.orig') util.unlinkpath(absstandin + '.orig') expecthash = lfutil.readstandin(repo, lfile) if (expecthash != '' and (checked or not os.path.exists(abslfile) or expecthash != lfutil.hashfile(abslfile))): if lfile not in repo[None]: # not switched to normal file util.unlinkpath(abslfile, ignoremissing=True) # use normallookup() to allocate an entry in largefiles # dirstate, because lack of it misleads # lfilesrepo.status() into recognition that such cache # missing files are removed. lfdirstate.normallookup(lfile) update[lfile] = expecthash else: # Remove lfiles for which the standin is deleted, unless the # lfile is added to the repository again. This happens when a # largefile is converted back to a normal file: the standin # disappears, but a new (normal) file appears as the lfile. if (os.path.exists(abslfile) and repo.dirstate.normalize(lfile) not in repo[None]): util.unlinkpath(abslfile) removed += 1 # largefile processing might be slow and be interrupted - be prepared lfdirstate.write() if lfiles: statuswriter(_('getting changed largefiles\n')) cachelfiles(ui, repo, None, lfiles) for lfile in lfiles: update1 = 0 expecthash = update.get(lfile) if expecthash: if not lfutil.copyfromcache(repo, expecthash, lfile): # failed ... but already removed and set to normallookup continue # Synchronize largefile dirstate to the last modified # time of the file lfdirstate.normal(lfile) update1 = 1 # copy the state of largefile standin from the repository's # dirstate to its state in the lfdirstate. abslfile = repo.wjoin(lfile) absstandin = repo.wjoin(lfutil.standin(lfile)) if os.path.exists(absstandin): mode = os.stat(absstandin).st_mode if mode != os.stat(abslfile).st_mode: os.chmod(abslfile, mode) update1 = 1 updated += update1 lfutil.synclfdirstate(repo, lfdirstate, lfile, normallookup) lfdirstate.write() if lfiles: statuswriter( _('%d largefiles updated, %d removed\n') % (updated, removed)) finally: wlock.release()
def status(self, node1='.', node2=None, match=None, ignored=False, clean=False, unknown=False, listsubrepos=False): listignored, listclean, listunknown = ignored, clean, unknown if not self.lfstatus: try: return super(lfiles_repo, self).status(node1, node2, match, listignored, listclean, listunknown, listsubrepos) except TypeError: return super(lfiles_repo, self).status(node1, node2, match, listignored, listclean, listunknown) else: # some calls in this function rely on the old version of status self.lfstatus = False if isinstance(node1, context.changectx): ctx1 = node1 else: ctx1 = repo[node1] if isinstance(node2, context.changectx): ctx2 = node2 else: ctx2 = repo[node2] working = ctx2.rev() is None parentworking = working and ctx1 == self['.'] def inctx(file, ctx): try: if ctx.rev() is None: return file in ctx.manifest() ctx[file] return True except KeyError: return False if match is None: match = match_.always(self.root, self.getcwd()) # Create a copy of match that matches standins instead # of largefiles. def tostandin(file): if inctx(lfutil.standin(file), ctx2): return lfutil.standin(file) return file m = copy.copy(match) m._files = [tostandin(f) for f in m._files] # get ignored, clean, and unknown but remove them # later if they were not asked for try: result = super(lfiles_repo, self).status(node1, node2, m, True, True, True, listsubrepos) except TypeError: result = super(lfiles_repo, self).status(node1, node2, m, True, True, True) if working: # hold the wlock while we read largefiles and # update the lfdirstate wlock = repo.wlock() try: # Any non-largefiles that were explicitly listed must be # taken out or lfdirstate.status will report an error. # The status of these files was already computed using # super's status. lfdirstate = lfutil.openlfdirstate(ui, self) match._files = [f for f in match._files if f in lfdirstate] s = lfdirstate.status(match, [], listignored, listclean, listunknown) (unsure, modified, added, removed, missing, unknown, ignored, clean) = s if parentworking: for lfile in unsure: if ctx1[lfutil.standin(lfile)].data().strip() \ != lfutil.hashfile(self.wjoin(lfile)): modified.append(lfile) else: clean.append(lfile) lfdirstate.normal(lfile) lfdirstate.write() else: tocheck = unsure + modified + added + clean modified, added, clean = [], [], [] for lfile in tocheck: standin = lfutil.standin(lfile) if inctx(standin, ctx1): if ctx1[standin].data().strip() != \ lfutil.hashfile(self.wjoin(lfile)): modified.append(lfile) else: clean.append(lfile) else: added.append(lfile) finally: wlock.release() for standin in ctx1.manifest(): if not lfutil.isstandin(standin): continue lfile = lfutil.splitstandin(standin) if not match(lfile): continue if lfile not in lfdirstate: removed.append(lfile) # Handle unknown and ignored differently lfiles = (modified, added, removed, missing, [], [], clean) result = list(result) # Unknown files result[4] = [f for f in unknown if (repo.dirstate[f] == '?' and not lfutil.isstandin(f))] # Ignored files must be ignored by both the dirstate and # lfdirstate result[5] = set(ignored).intersection(set(result[5])) # combine normal files and largefiles normals = [[fn for fn in filelist if not lfutil.isstandin(fn)] for filelist in result] result = [sorted(list1 + list2) for (list1, list2) in zip(normals, lfiles)] else: def toname(f): if lfutil.isstandin(f): return lfutil.splitstandin(f) return f result = [[toname(f) for f in items] for items in result] if not listunknown: result[4] = [] if not listignored: result[5] = [] if not listclean: result[6] = [] self.lfstatus = True return result
def status(self, node1='.', node2=None, match=None, ignored=False, clean=False, unknown=False, listsubrepos=False): listignored, listclean, listunknown = ignored, clean, unknown if not self.lfstatus: return super(lfilesrepo, self).status(node1, node2, match, listignored, listclean, listunknown, listsubrepos) else: # some calls in this function rely on the old version of status self.lfstatus = False ctx1 = self[node1] ctx2 = self[node2] working = ctx2.rev() is None parentworking = working and ctx1 == self['.'] def inctx(file, ctx): try: if ctx.rev() is None: return file in ctx.manifest() ctx[file] return True except KeyError: return False if match is None: match = match_.always(self.root, self.getcwd()) wlock = None try: try: # updating the dirstate is optional # so we don't wait on the lock wlock = self.wlock(False) except error.LockError: pass # First check if there were files specified on the # command line. If there were, and none of them were # largefiles, we should just bail here and let super # handle it -- thus gaining a big performance boost. lfdirstate = lfutil.openlfdirstate(ui, self) if match.files() and not match.anypats(): for f in lfdirstate: if match(f): break else: return super(lfilesrepo, self).status(node1, node2, match, listignored, listclean, listunknown, listsubrepos) # Create a copy of match that matches standins instead # of largefiles. def tostandins(files): if not working: return files newfiles = [] dirstate = self.dirstate for f in files: sf = lfutil.standin(f) if sf in dirstate: newfiles.append(sf) elif sf in dirstate.dirs(): # Directory entries could be regular or # standin, check both newfiles.extend((f, sf)) else: newfiles.append(f) return newfiles m = copy.copy(match) m._files = tostandins(m._files) result = super(lfilesrepo, self).status(node1, node2, m, ignored, clean, unknown, listsubrepos) if working: def sfindirstate(f): sf = lfutil.standin(f) dirstate = self.dirstate return sf in dirstate or sf in dirstate.dirs() match._files = [ f for f in match._files if sfindirstate(f) ] # Don't waste time getting the ignored and unknown # files from lfdirstate s = lfdirstate.status(match, [], False, listclean, False) (unsure, modified, added, removed, missing, _unknown, _ignored, clean) = s if parentworking: for lfile in unsure: standin = lfutil.standin(lfile) if standin not in ctx1: # from second parent modified.append(lfile) elif ctx1[standin].data().strip() \ != lfutil.hashfile(self.wjoin(lfile)): modified.append(lfile) else: clean.append(lfile) lfdirstate.normal(lfile) else: tocheck = unsure + modified + added + clean modified, added, clean = [], [], [] for lfile in tocheck: standin = lfutil.standin(lfile) if inctx(standin, ctx1): if ctx1[standin].data().strip() != \ lfutil.hashfile(self.wjoin(lfile)): modified.append(lfile) else: clean.append(lfile) else: added.append(lfile) # Standins no longer found in lfdirstate has been # removed for standin in ctx1.manifest(): if not lfutil.isstandin(standin): continue lfile = lfutil.splitstandin(standin) if not match(lfile): continue if lfile not in lfdirstate: removed.append(lfile) # Filter result lists result = list(result) # Largefiles are not really removed when they're # still in the normal dirstate. Likewise, normal # files are not really removed if they are still in # lfdirstate. This happens in merges where files # change type. removed = [ f for f in removed if f not in self.dirstate ] result[2] = [ f for f in result[2] if f not in lfdirstate ] lfiles = set(lfdirstate._map) # Unknown files result[4] = set(result[4]).difference(lfiles) # Ignored files result[5] = set(result[5]).difference(lfiles) # combine normal files and largefiles normals = [[ fn for fn in filelist if not lfutil.isstandin(fn) ] for filelist in result] lfiles = (modified, added, removed, missing, [], [], clean) result = [ sorted(list1 + list2) for (list1, list2) in zip(normals, lfiles) ] else: def toname(f): if lfutil.isstandin(f): return lfutil.splitstandin(f) return f result = [[toname(f) for f in items] for items in result] if wlock: lfdirstate.write() finally: if wlock: wlock.release() if not listunknown: result[4] = [] if not listignored: result[5] = [] if not listclean: result[6] = [] self.lfstatus = True return result
def updatelfiles(ui, repo, filelist=None, printmessage=True, normallookup=False): wlock = repo.wlock() try: lfdirstate = lfutil.openlfdirstate(ui, repo) lfiles = set(lfutil.listlfiles(repo)) | set(lfdirstate) if filelist is not None: filelist = set(filelist) lfiles = [f for f in lfiles if f in filelist] update = {} updated, removed = 0, 0 for lfile in lfiles: abslfile = repo.wjoin(lfile) absstandin = repo.wjoin(lfutil.standin(lfile)) if os.path.exists(absstandin): if (os.path.exists(absstandin + '.orig') and os.path.exists(abslfile)): shutil.copyfile(abslfile, abslfile + '.orig') util.unlinkpath(absstandin + '.orig') expecthash = lfutil.readstandin(repo, lfile) if (expecthash != '' and (not os.path.exists(abslfile) or expecthash != lfutil.hashfile(abslfile))): if lfile not in repo[None]: # not switched to normal file util.unlinkpath(abslfile, ignoremissing=True) # use normallookup() to allocate entry in largefiles # dirstate, because lack of it misleads # lfilesrepo.status() into recognition that such cache # missing files are REMOVED. lfdirstate.normallookup(lfile) update[lfile] = expecthash else: # Remove lfiles for which the standin is deleted, unless the # lfile is added to the repository again. This happens when a # largefile is converted back to a normal file: the standin # disappears, but a new (normal) file appears as the lfile. if (os.path.exists(abslfile) and repo.dirstate.normalize(lfile) not in repo[None]): util.unlinkpath(abslfile) removed += 1 # largefile processing might be slow and be interrupted - be prepared lfdirstate.write() if lfiles: if printmessage: ui.status(_('getting changed largefiles\n')) cachelfiles(ui, repo, None, lfiles) for lfile in lfiles: update1 = 0 expecthash = update.get(lfile) if expecthash: if not lfutil.copyfromcache(repo, expecthash, lfile): # failed ... but already removed and set to normallookup continue # Synchronize largefile dirstate to the last modified # time of the file lfdirstate.normal(lfile) update1 = 1 # copy the state of largefile standin from the repository's # dirstate to its state in the lfdirstate. abslfile = repo.wjoin(lfile) absstandin = repo.wjoin(lfutil.standin(lfile)) if os.path.exists(absstandin): mode = os.stat(absstandin).st_mode if mode != os.stat(abslfile).st_mode: os.chmod(abslfile, mode) update1 = 1 updated += update1 lfutil.synclfdirstate(repo, lfdirstate, lfile, normallookup) if filelist is not None: # If "local largefile" is chosen at file merging, it is # not listed in "filelist" (= dirstate syncing is # omitted), because the standin file is not changed before and # after merging. # But the status of such files may have to be changed by # merging. For example, locally modified ("M") largefile # has to become re-added("A"), if it is "normal" file in # the target revision of linear-merging. for lfile in lfdirstate: if lfile not in filelist: lfutil.synclfdirstate(repo, lfdirstate, lfile, True) lfdirstate.write() if printmessage and lfiles: ui.status( _('%d largefiles updated, %d removed\n') % (updated, removed)) finally: wlock.release()
def status(self, node1='.', node2=None, match=None, ignored=False, clean=False, unknown=False, listsubrepos=False): listignored, listclean, listunknown = ignored, clean, unknown if not self.lfstatus: try: return super(lfiles_repo, self).status(node1, node2, match, listignored, listclean, listunknown, listsubrepos) except TypeError: return super(lfiles_repo, self).status(node1, node2, match, listignored, listclean, listunknown) else: # some calls in this function rely on the old version of status self.lfstatus = False if isinstance(node1, context.changectx): ctx1 = node1 else: ctx1 = repo[node1] if isinstance(node2, context.changectx): ctx2 = node2 else: ctx2 = repo[node2] working = ctx2.rev() is None parentworking = working and ctx1 == self['.'] def inctx(file, ctx): try: if ctx.rev() is None: return file in ctx.manifest() ctx[file] return True except KeyError: return False if match is None: match = match_.always(self.root, self.getcwd()) # Create a copy of match that matches standins instead # of largefiles. def tostandin(file): if inctx(lfutil.standin(file), ctx2): return lfutil.standin(file) return file m = copy.copy(match) m._files = [tostandin(f) for f in m._files] # get ignored, clean, and unknown but remove them # later if they were not asked for try: result = super(lfiles_repo, self).status(node1, node2, m, True, True, True, listsubrepos) except TypeError: result = super(lfiles_repo, self).status(node1, node2, m, True, True, True) if working: # hold the wlock while we read largefiles and # update the lfdirstate wlock = repo.wlock() try: # Any non-largefiles that were explicitly listed must be # taken out or lfdirstate.status will report an error. # The status of these files was already computed using # super's status. lfdirstate = lfutil.openlfdirstate(ui, self) match._files = [ f for f in match._files if f in lfdirstate ] s = lfdirstate.status(match, [], listignored, listclean, listunknown) (unsure, modified, added, removed, missing, unknown, ignored, clean) = s if parentworking: for lfile in unsure: if ctx1[lfutil.standin(lfile)].data().strip() \ != lfutil.hashfile(self.wjoin(lfile)): modified.append(lfile) else: clean.append(lfile) lfdirstate.normal(lfile) lfdirstate.write() else: tocheck = unsure + modified + added + clean modified, added, clean = [], [], [] for lfile in tocheck: standin = lfutil.standin(lfile) if inctx(standin, ctx1): if ctx1[standin].data().strip() != \ lfutil.hashfile(self.wjoin(lfile)): modified.append(lfile) else: clean.append(lfile) else: added.append(lfile) finally: wlock.release() for standin in ctx1.manifest(): if not lfutil.isstandin(standin): continue lfile = lfutil.splitstandin(standin) if not match(lfile): continue if lfile not in lfdirstate: removed.append(lfile) # Handle unknown and ignored differently lfiles = (modified, added, removed, missing, [], [], clean) result = list(result) # Unknown files result[4] = [ f for f in unknown if (repo.dirstate[f] == '?' and not lfutil.isstandin(f) ) ] # Ignored files must be ignored by both the dirstate and # lfdirstate result[5] = set(ignored).intersection(set(result[5])) # combine normal files and largefiles normals = [[ fn for fn in filelist if not lfutil.isstandin(fn) ] for filelist in result] result = [ sorted(list1 + list2) for (list1, list2) in zip(normals, lfiles) ] else: def toname(f): if lfutil.isstandin(f): return lfutil.splitstandin(f) return f result = [[toname(f) for f in items] for items in result] if not listunknown: result[4] = [] if not listignored: result[5] = [] if not listclean: result[6] = [] self.lfstatus = True return result
def updatelfiles(ui, repo, filelist=None, printmessage=True, normallookup=False): wlock = repo.wlock() try: lfdirstate = lfutil.openlfdirstate(ui, repo) lfiles = set(lfutil.listlfiles(repo)) | set(lfdirstate) if filelist is not None: filelist = set(filelist) lfiles = [f for f in lfiles if f in filelist] update = {} updated, removed = 0, 0 for lfile in lfiles: abslfile = repo.wjoin(lfile) absstandin = repo.wjoin(lfutil.standin(lfile)) if os.path.exists(absstandin): if (os.path.exists(absstandin + '.orig') and os.path.exists(abslfile)): shutil.copyfile(abslfile, abslfile + '.orig') util.unlinkpath(absstandin + '.orig') expecthash = lfutil.readstandin(repo, lfile) if (expecthash != '' and (not os.path.exists(abslfile) or expecthash != lfutil.hashfile(abslfile))): if lfile not in repo[None]: # not switched to normal file util.unlinkpath(abslfile, ignoremissing=True) # use normallookup() to allocate entry in largefiles # dirstate, because lack of it misleads # lfilesrepo.status() into recognition that such cache # missing files are REMOVED. lfdirstate.normallookup(lfile) update[lfile] = expecthash else: # Remove lfiles for which the standin is deleted, unless the # lfile is added to the repository again. This happens when a # largefile is converted back to a normal file: the standin # disappears, but a new (normal) file appears as the lfile. if (os.path.exists(abslfile) and repo.dirstate.normalize(lfile) not in repo[None]): util.unlinkpath(abslfile) removed += 1 # largefile processing might be slow and be interrupted - be prepared lfdirstate.write() if lfiles: if printmessage: ui.status(_('getting changed largefiles\n')) cachelfiles(ui, repo, None, lfiles) for lfile in lfiles: update1 = 0 expecthash = update.get(lfile) if expecthash: if not lfutil.copyfromcache(repo, expecthash, lfile): # failed ... but already removed and set to normallookup continue # Synchronize largefile dirstate to the last modified # time of the file lfdirstate.normal(lfile) update1 = 1 # copy the state of largefile standin from the repository's # dirstate to its state in the lfdirstate. abslfile = repo.wjoin(lfile) absstandin = repo.wjoin(lfutil.standin(lfile)) if os.path.exists(absstandin): mode = os.stat(absstandin).st_mode if mode != os.stat(abslfile).st_mode: os.chmod(abslfile, mode) update1 = 1 updated += update1 lfutil.synclfdirstate(repo, lfdirstate, lfile, normallookup) if filelist is not None: # If "local largefile" is chosen at file merging, it is # not listed in "filelist" (= dirstate syncing is # omitted), because the standin file is not changed before and # after merging. # But the status of such files may have to be changed by # merging. For example, locally modified ("M") largefile # has to become re-added("A"), if it is "normal" file in # the target revision of linear-merging. for lfile in lfdirstate: if lfile not in filelist: lfutil.synclfdirstate(repo, lfdirstate, lfile, True) lfdirstate.write() if printmessage and lfiles: ui.status(_('%d largefiles updated, %d removed\n') % (updated, removed)) finally: wlock.release()
def updatelfiles(ui, repo, filelist=None, printmessage=True): wlock = repo.wlock() try: lfdirstate = lfutil.openlfdirstate(ui, repo) lfiles = set(lfutil.listlfiles(repo)) | set(lfdirstate) if filelist is not None: lfiles = [f for f in lfiles if f in filelist] update = {} updated, removed = 0, 0 for lfile in lfiles: abslfile = repo.wjoin(lfile) absstandin = repo.wjoin(lfutil.standin(lfile)) if os.path.exists(absstandin): if (os.path.exists(absstandin + '.orig') and os.path.exists(abslfile)): shutil.copyfile(abslfile, abslfile + '.orig') expecthash = lfutil.readstandin(repo, lfile) if (expecthash != '' and (not os.path.exists(abslfile) or expecthash != lfutil.hashfile(abslfile))): if lfile not in repo[None]: # not switched to normal file util.unlinkpath(abslfile, ignoremissing=True) # use normallookup() to allocate entry in largefiles # dirstate, because lack of it misleads # lfilesrepo.status() into recognition that such cache # missing files are REMOVED. lfdirstate.normallookup(lfile) update[lfile] = expecthash else: # Remove lfiles for which the standin is deleted, unless the # lfile is added to the repository again. This happens when a # largefile is converted back to a normal file: the standin # disappears, but a new (normal) file appears as the lfile. if (os.path.exists(abslfile) and repo.dirstate.normalize(lfile) not in repo[None]): util.unlinkpath(abslfile) removed += 1 # largefile processing might be slow and be interrupted - be prepared lfdirstate.write() if lfiles: if printmessage: ui.status(_('getting changed largefiles\n')) cachelfiles(ui, repo, None, lfiles) for lfile in lfiles: update1 = 0 expecthash = update.get(lfile) if expecthash: if not lfutil.copyfromcache(repo, expecthash, lfile): # failed ... but already removed and set to normallookup continue # Synchronize largefile dirstate to the last modified # time of the file lfdirstate.normal(lfile) update1 = 1 # copy the state of largefile standin from the repository's # dirstate to its state in the lfdirstate. abslfile = repo.wjoin(lfile) absstandin = repo.wjoin(lfutil.standin(lfile)) if os.path.exists(absstandin): mode = os.stat(absstandin).st_mode if mode != os.stat(abslfile).st_mode: os.chmod(abslfile, mode) update1 = 1 updated += update1 state = repo.dirstate[lfutil.standin(lfile)] if state == 'n': # When rebasing, we need to synchronize the standin and the # largefile, because otherwise the largefile will get reverted. # But for commit's sake, we have to mark the file as unclean. if getattr(repo, "_isrebasing", False): lfdirstate.normallookup(lfile) else: lfdirstate.normal(lfile) elif state == 'r': lfdirstate.remove(lfile) elif state == 'a': lfdirstate.add(lfile) elif state == '?': lfdirstate.drop(lfile) lfdirstate.write() if printmessage and lfiles: ui.status(_('%d largefiles updated, %d removed\n') % (updated, removed)) finally: wlock.release()