def updatelfiles(ui, repo, filelist=None, printmessage=True): wlock = repo.wlock() try: lfdirstate = lfutil.openlfdirstate(ui, repo) lfiles = set(lfutil.listlfiles(repo)) | set(lfdirstate) if filelist is not None: lfiles = [f for f in lfiles if f in filelist] printed = False if printmessage and lfiles: ui.status(_('getting changed largefiles\n')) printed = True cachelfiles(ui, repo, None, lfiles) updated, removed = 0, 0 for f in lfiles: i = _updatelfile(repo, lfdirstate, f) if i: if i > 0: updated += i else: removed -= i if printmessage and (removed or updated) and not printed: ui.status(_('getting changed largefiles\n')) printed = True lfdirstate.write() if printed and printmessage: ui.status(_('%d largefiles updated, %d removed\n') % (updated, removed)) finally: wlock.release()
def updatelfiles(ui, repo, filelist=None, printmessage=True): wlock = repo.wlock() try: lfdirstate = lfutil.openlfdirstate(ui, repo) lfiles = set(lfutil.listlfiles(repo)) | set(lfdirstate) if filelist is not None: lfiles = [f for f in lfiles if f in filelist] printed = False if printmessage and lfiles: ui.status(_('getting changed largefiles\n')) printed = True cachelfiles(ui, repo, None, lfiles) updated, removed = 0, 0 for f in lfiles: i = _updatelfile(repo, lfdirstate, f) if i: if i > 0: updated += i else: removed -= i if printmessage and (removed or updated) and not printed: ui.status(_('getting changed largefiles\n')) printed = True lfdirstate.write() if printed and printmessage: ui.status( _('%d largefiles updated, %d removed\n') % (updated, removed)) finally: wlock.release()
def scmutiladdremove(orig, repo, pats=[], opts={}, dry_run=None, similarity=None): if not lfutil.islfilesrepo(repo): return orig(repo, pats, opts, dry_run, similarity) # Get the list of missing largefiles so we can remove them lfdirstate = lfutil.openlfdirstate(repo.ui, repo) s = lfdirstate.status(match_.always(repo.root, repo.getcwd()), [], False, False, False) (unsure, modified, added, removed, missing, unknown, ignored, clean) = s # Call into the normal remove code, but the removing of the standin, we want # to have handled by original addremove. Monkey patching here makes sure # we don't remove the standin in the largefiles code, preventing a very # confused state later. if missing: m = [repo.wjoin(f) for f in missing] repo._isaddremove = True removelargefiles(repo.ui, repo, *m, **opts) repo._isaddremove = False # Call into the normal add code, and any files that *should* be added as # largefiles will be addlargefiles(repo.ui, repo, *pats, **opts) # Now that we've handled largefiles, hand off to the original addremove # function to take care of the rest. Make sure it doesn't do anything with # largefiles by installing a matcher that will ignore them. installnormalfilesmatchfn(repo[None].manifest()) result = orig(repo, pats, opts, dry_run, similarity) restorematchfn() return result
def updatelfiles(ui, repo, filelist=None, printmessage=True): wlock = repo.wlock() try: lfdirstate = lfutil.openlfdirstate(ui, repo) lfiles = set(lfutil.listlfiles(repo)) | set(lfdirstate) if filelist is not None: lfiles = [f for f in lfiles if f in filelist] printed = False if printmessage and lfiles: ui.status(_("getting changed largefiles\n")) printed = True cachelfiles(ui, repo, ".") updated, removed = 0, 0 for i in map(lambda f: _updatelfile(repo, lfdirstate, f), lfiles): # increment the appropriate counter according to _updatelfile's # return value updated += i > 0 and i or 0 removed -= i < 0 and i or 0 if printmessage and (removed or updated) and not printed: ui.status(_("getting changed largefiles\n")) printed = True lfdirstate.write() if printed and printmessage: ui.status(_("%d largefiles updated, %d removed\n") % (updated, removed)) finally: wlock.release()
def updatelfiles(ui, repo, filelist=None, printmessage=True): wlock = repo.wlock() try: lfdirstate = lfutil.openlfdirstate(ui, repo) lfiles = set(lfutil.listlfiles(repo)) | set(lfdirstate) if filelist is not None: lfiles = [f for f in lfiles if f in filelist] printed = False if printmessage and lfiles: ui.status(_('getting changed largefiles\n')) printed = True cachelfiles(ui, repo, '.') updated, removed = 0, 0 for i in map(lambda f: _updatelfile(repo, lfdirstate, f), lfiles): # increment the appropriate counter according to _updatelfile's # return value updated += i > 0 and i or 0 removed -= i < 0 and i or 0 if printmessage and (removed or updated) and not printed: ui.status(_('getting changed largefiles\n')) printed = True lfdirstate.write() if printed and printmessage: ui.status(_('%d largefiles updated, %d removed\n') % (updated, removed)) finally: wlock.release()
def overrideupdate(orig, ui, repo, *pats, **opts): lfdirstate = lfutil.openlfdirstate(ui, repo) s = lfdirstate.status(match_.always(repo.root, repo.getcwd()), [], False, False, False) (unsure, modified, added, removed, missing, unknown, ignored, clean) = s # Need to lock between the standins getting updated and their # largefiles getting updated wlock = repo.wlock() try: if opts['check']: mod = len(modified) > 0 for lfile in unsure: standin = lfutil.standin(lfile) if repo['.'][standin].data().strip() != \ lfutil.hashfile(repo.wjoin(lfile)): mod = True else: lfdirstate.normal(lfile) lfdirstate.write() if mod: raise util.Abort(_('uncommitted changes')) # XXX handle removed differently if not opts['clean']: for lfile in unsure + modified + added: lfutil.updatestandin(repo, lfutil.standin(lfile)) finally: wlock.release() return orig(ui, repo, *pats, **opts)
def overrideupdate(orig, ui, repo, *pats, **opts): lfdirstate = lfutil.openlfdirstate(ui, repo) s = lfdirstate.status(match_.always(repo.root, repo.getcwd()), [], False, False, False) (unsure, modified, added, removed, missing, unknown, ignored, clean) = s # Need to lock between the standins getting updated and their # largefiles getting updated wlock = repo.wlock() try: if opts['check']: mod = len(modified) > 0 for lfile in unsure: standin = lfutil.standin(lfile) if repo['.'][standin].data().strip() != \ lfutil.hashfile(repo.wjoin(lfile)): mod = True else: lfdirstate.normal(lfile) lfdirstate.write() if mod: raise util.Abort(_('uncommitted local changes')) # XXX handle removed differently if not opts['clean']: for lfile in unsure + modified + added: lfutil.updatestandin(repo, lfutil.standin(lfile)) finally: wlock.release() return orig(ui, repo, *pats, **opts)
def override_remove(orig, ui, repo, *pats, **opts): manifest = repo[None].manifest() installnormalfilesmatchfn(manifest) orig(ui, repo, *pats, **opts) restorematchfn() after, force = opts.get('after'), opts.get('force') if not pats and not after: raise util.Abort(_('no files specified')) m = scmutil.match(repo[None], pats, opts) try: repo.lfstatus = True s = repo.status(match=m, clean=True) finally: repo.lfstatus = False modified, added, deleted, clean = [[ f for f in list if lfutil.standin(f) in manifest ] for list in [s[0], s[1], s[3], s[6]]] def warn(files, reason): for f in files: ui.warn( _('not removing %s: %s (use -f to force removal)\n') % (m.rel(f), reason)) if force: remove, forget = modified + deleted + clean, added elif after: remove, forget = deleted, [] warn(modified + added + clean, _('file still exists')) else: remove, forget = deleted + clean, [] warn(modified, _('file is modified')) warn(added, _('file has been marked for add')) for f in sorted(remove + forget): if ui.verbose or not m.exact(f): ui.status(_('removing %s\n') % m.rel(f)) # Need to lock because standin files are deleted then removed from the # repository and we could race inbetween. wlock = repo.wlock() try: lfdirstate = lfutil.openlfdirstate(ui, repo) for f in remove: if not after: os.unlink(repo.wjoin(f)) currentdir = os.path.split(f)[0] while currentdir and not os.listdir(repo.wjoin(currentdir)): os.rmdir(repo.wjoin(currentdir)) currentdir = os.path.split(currentdir)[0] lfdirstate.remove(f) lfdirstate.write() forget = [lfutil.standin(f) for f in forget] remove = [lfutil.standin(f) for f in remove] lfutil.repo_forget(repo, forget) lfutil.repo_remove(repo, remove, unlink=True) finally: wlock.release()
def overriderevert(orig, ui, repo, *pats, **opts): # Because we put the standins in a bad state (by updating them) # and then return them to a correct state we need to lock to # prevent others from changing them in their incorrect state. wlock = repo.wlock() try: lfdirstate = lfutil.openlfdirstate(ui, repo) (modified, added, removed, missing, unknown, ignored, clean) = \ lfutil.lfdirstatestatus(lfdirstate, repo, repo['.'].rev()) lfdirstate.write() for lfile in modified: lfutil.updatestandin(repo, lfutil.standin(lfile)) for lfile in missing: if (os.path.exists(repo.wjoin(lfutil.standin(lfile)))): os.unlink(repo.wjoin(lfutil.standin(lfile))) oldstandins = lfutil.getstandinsstate(repo) def overridematch(ctx, pats=[], opts={}, globbed=False, default='relpath'): match = oldmatch(ctx, pats, opts, globbed, default) m = copy.copy(match) def tostandin(f): if lfutil.standin(f) in ctx: return lfutil.standin(f) elif lfutil.standin(f) in repo[None]: return None return f m._files = [tostandin(f) for f in m._files] m._files = [f for f in m._files if f is not None] m._fmap = set(m._files) m._always = False origmatchfn = m.matchfn def matchfn(f): if lfutil.isstandin(f): return (origmatchfn(lfutil.splitstandin(f)) and (f in repo[None] or f in ctx)) return origmatchfn(f) m.matchfn = matchfn return m oldmatch = installmatchfn(overridematch) try: orig(ui, repo, *pats, **opts) finally: restorematchfn() newstandins = lfutil.getstandinsstate(repo) filelist = lfutil.getlfilestoupdate(oldstandins, newstandins) lfcommands.updatelfiles(ui, repo, filelist, printmessage=False) finally: wlock.release()
def override_remove(orig, ui, repo, *pats, **opts): manifest = repo[None].manifest() installnormalfilesmatchfn(manifest) orig(ui, repo, *pats, **opts) restorematchfn() after, force = opts.get('after'), opts.get('force') if not pats and not after: raise util.Abort(_('no files specified')) m = scmutil.match(repo[None], pats, opts) try: repo.lfstatus = True s = repo.status(match=m, clean=True) finally: repo.lfstatus = False modified, added, deleted, clean = [[f for f in list if lfutil.standin(f) in manifest] for list in [s[0], s[1], s[3], s[6]]] def warn(files, reason): for f in files: ui.warn(_('not removing %s: %s (use -f to force removal)\n') % (m.rel(f), reason)) if force: remove, forget = modified + deleted + clean, added elif after: remove, forget = deleted, [] warn(modified + added + clean, _('file still exists')) else: remove, forget = deleted + clean, [] warn(modified, _('file is modified')) warn(added, _('file has been marked for add')) for f in sorted(remove + forget): if ui.verbose or not m.exact(f): ui.status(_('removing %s\n') % m.rel(f)) # Need to lock because standin files are deleted then removed from the # repository and we could race inbetween. wlock = repo.wlock() try: lfdirstate = lfutil.openlfdirstate(ui, repo) for f in remove: if not after: os.unlink(repo.wjoin(f)) currentdir = os.path.split(f)[0] while currentdir and not os.listdir(repo.wjoin(currentdir)): os.rmdir(repo.wjoin(currentdir)) currentdir = os.path.split(currentdir)[0] lfdirstate.remove(f) lfdirstate.write() forget = [lfutil.standin(f) for f in forget] remove = [lfutil.standin(f) for f in remove] lfutil.repo_forget(repo, forget) lfutil.repo_remove(repo, remove, unlink=True) finally: wlock.release()
def overriderevert(orig, ui, repo, *pats, **opts): # Because we put the standins in a bad state (by updating them) # and then return them to a correct state we need to lock to # prevent others from changing them in their incorrect state. wlock = repo.wlock() try: lfdirstate = lfutil.openlfdirstate(ui, repo) (modified, added, removed, missing, unknown, ignored, clean) = \ lfutil.lfdirstatestatus(lfdirstate, repo, repo['.'].rev()) lfdirstate.write() for lfile in modified: lfutil.updatestandin(repo, lfutil.standin(lfile)) for lfile in missing: if (os.path.exists(repo.wjoin(lfutil.standin(lfile)))): os.unlink(repo.wjoin(lfutil.standin(lfile))) oldstandins = lfutil.getstandinsstate(repo) def overridematch(ctx, pats=[], opts={}, globbed=False, default='relpath'): match = oldmatch(ctx, pats, opts, globbed, default) m = copy.copy(match) def tostandin(f): if lfutil.standin(f) in ctx: return lfutil.standin(f) elif lfutil.standin(f) in repo[None]: return None return f m._files = [tostandin(f) for f in m._files] m._files = [f for f in m._files if f is not None] m._fmap = set(m._files) m._always = False origmatchfn = m.matchfn def matchfn(f): if lfutil.isstandin(f): return (origmatchfn(lfutil.splitstandin(f)) and (f in repo[None] or f in ctx)) return origmatchfn(f) m.matchfn = matchfn return m oldmatch = installmatchfn(overridematch) try: orig(ui, repo, *pats, **opts) finally: restorematchfn() newstandins = lfutil.getstandinsstate(repo) filelist = lfutil.getlfilestoupdate(oldstandins, newstandins) # lfdirstate should be 'normallookup'-ed for updated files, # because reverting doesn't touch dirstate for 'normal' files # when target revision is explicitly specified: in such case, # 'n' and valid timestamp in dirstate doesn't ensure 'clean' # of target (standin) file. lfcommands.updatelfiles(ui, repo, filelist, printmessage=False, normallookup=True) finally: wlock.release()
def overridestatus(node1='.', node2=None, match=None, ignored=False, clean=False, unknown=False, listsubrepos=False): r = oldstatus(node1, node2, match, ignored, clean, unknown, listsubrepos) lfdirstate = lfutil.openlfdirstate(ui, repo) modified, added, removed, deleted, unknown, ignored, clean = r unknown = [f for f in unknown if lfdirstate[f] == '?'] ignored = [f for f in ignored if lfdirstate[f] == '?'] return modified, added, removed, deleted, unknown, ignored, clean
def remove_largefiles(ui, repo, *pats, **opts): after = opts.get('after') if not pats and not after: raise util.Abort(_('no files specified')) m = scmutil.match(repo[None], pats, opts) try: repo.lfstatus = True s = repo.status(match=m, clean=True) finally: repo.lfstatus = False manifest = repo[None].manifest() modified, added, deleted, clean = [[ f for f in list if lfutil.standin(f) in manifest ] for list in [s[0], s[1], s[3], s[6]]] def warn(files, reason): for f in files: ui.warn( _('not removing %s: %s (use forget to undo)\n') % (m.rel(f), reason)) if after: remove, forget = deleted, [] warn(modified + added + clean, _('file still exists')) else: remove, forget = deleted + clean, [] warn(modified, _('file is modified')) warn(added, _('file has been marked for add')) for f in sorted(remove + forget): if ui.verbose or not m.exact(f): ui.status(_('removing %s\n') % m.rel(f)) # Need to lock because standin files are deleted then removed from the # repository and we could race inbetween. wlock = repo.wlock() try: lfdirstate = lfutil.openlfdirstate(ui, repo) for f in remove: if not after: # If this is being called by addremove, notify the user that we # are removing the file. if getattr(repo, "_isaddremove", False): ui.status(_('removing %s\n') % f) if os.path.exists(repo.wjoin(f)): util.unlinkpath(repo.wjoin(f)) lfdirstate.remove(f) lfdirstate.write() forget = [lfutil.standin(f) for f in forget] remove = [lfutil.standin(f) for f in remove] lfutil.repo_forget(repo, forget) # If this is being called by addremove, let the original addremove # function handle this. if not getattr(repo, "_isaddremove", False): lfutil.repo_remove(repo, remove, unlink=True) finally: wlock.release()
def add_largefiles(ui, repo, *pats, **opts): large = opts.pop("large", None) lfsize = lfutil.getminsize(ui, lfutil.islfilesrepo(repo), opts.pop("lfsize", None)) lfmatcher = None if lfutil.islfilesrepo(repo): lfpats = ui.configlist(lfutil.longname, "patterns", default=[]) if lfpats: lfmatcher = match_.match(repo.root, "", list(lfpats)) lfnames = [] m = scmutil.match(repo[None], pats, opts) m.bad = lambda x, y: None wctx = repo[None] for f in repo.walk(m): exact = m.exact(f) lfile = lfutil.standin(f) in wctx nfile = f in wctx exists = lfile or nfile # Don't warn the user when they attempt to add a normal tracked file. # The normal add code will do that for us. if exact and exists: if lfile: ui.warn(_("%s already a largefile\n") % f) continue if exact or not exists: abovemin = lfsize and os.lstat(repo.wjoin(f)).st_size >= lfsize * 1024 * 1024 if large or abovemin or (lfmatcher and lfmatcher(f)): lfnames.append(f) if ui.verbose or not exact: ui.status(_("adding %s as a largefile\n") % m.rel(f)) bad = [] standins = [] # Need to lock, otherwise there could be a race condition between # when standins are created and added to the repo. wlock = repo.wlock() try: if not opts.get("dry_run"): lfdirstate = lfutil.openlfdirstate(ui, repo) for f in lfnames: standinname = lfutil.standin(f) lfutil.writestandin(repo, standinname, hash="", executable=lfutil.getexecutable(repo.wjoin(f))) standins.append(standinname) if lfdirstate[f] == "r": lfdirstate.normallookup(f) else: lfdirstate.add(f) lfdirstate.write() bad += [lfutil.splitstandin(f) for f in lfutil.repo_add(repo, standins) if f in m.files()] finally: wlock.release() return bad
def remove_largefiles(ui, repo, *pats, **opts): after = opts.get('after') if not pats and not after: raise util.Abort(_('no files specified')) m = scmutil.match(repo[None], pats, opts) try: repo.lfstatus = True s = repo.status(match=m, clean=True) finally: repo.lfstatus = False manifest = repo[None].manifest() modified, added, deleted, clean = [[f for f in list if lfutil.standin(f) in manifest] for list in [s[0], s[1], s[3], s[6]]] def warn(files, reason): for f in files: ui.warn(_('not removing %s: %s (use forget to undo)\n') % (m.rel(f), reason)) if after: remove, forget = deleted, [] warn(modified + added + clean, _('file still exists')) else: remove, forget = deleted + clean, [] warn(modified, _('file is modified')) warn(added, _('file has been marked for add')) for f in sorted(remove + forget): if ui.verbose or not m.exact(f): ui.status(_('removing %s\n') % m.rel(f)) # Need to lock because standin files are deleted then removed from the # repository and we could race inbetween. wlock = repo.wlock() try: lfdirstate = lfutil.openlfdirstate(ui, repo) for f in remove: if not after: # If this is being called by addremove, notify the user that we # are removing the file. if getattr(repo, "_isaddremove", False): ui.status(_('removing %s\n' % f)) if os.path.exists(repo.wjoin(f)): util.unlinkpath(repo.wjoin(f)) lfdirstate.remove(f) lfdirstate.write() forget = [lfutil.standin(f) for f in forget] remove = [lfutil.standin(f) for f in remove] lfutil.repo_forget(repo, forget) # If this is being called by addremove, let the original addremove # function handle this. if not getattr(repo, "_isaddremove", False): lfutil.repo_remove(repo, remove, unlink=True) finally: wlock.release()
def override_rollback(orig, ui, repo, **opts): result = orig(ui, repo, **opts) merge.update(repo, node=None, branchmerge=False, force=True, partial=lfutil.isstandin) lfdirstate = lfutil.openlfdirstate(ui, repo) lfiles = lfutil.listlfiles(repo) oldlfiles = lfutil.listlfiles(repo, repo[None].parents()[0].rev()) for file in lfiles: if file in oldlfiles: lfdirstate.normallookup(file) else: lfdirstate.add(file) lfdirstate.write() return result
def overrideforget(orig, ui, repo, *pats, **opts): installnormalfilesmatchfn(repo[None].manifest()) result = orig(ui, repo, *pats, **opts) restorematchfn() m = scmutil.match(repo[None], pats, opts) try: repo.lfstatus = True s = repo.status(match=m, clean=True) finally: repo.lfstatus = False forget = sorted(s[0] + s[1] + s[3] + s[6]) forget = [f for f in forget if lfutil.standin(f) in repo[None].manifest()] for f in forget: if lfutil.standin(f) not in repo.dirstate and not \ os.path.isdir(m.rel(lfutil.standin(f))): ui.warn(_('not removing %s: file is already untracked\n') % m.rel(f)) result = 1 for f in forget: if ui.verbose or not m.exact(f): ui.status(_('removing %s\n') % m.rel(f)) # Need to lock because standin files are deleted then removed from the # repository and we could race in-between. wlock = repo.wlock() try: lfdirstate = lfutil.openlfdirstate(ui, repo) for f in forget: if lfdirstate[f] == 'a': lfdirstate.drop(f) else: lfdirstate.remove(f) lfdirstate.write() standins = [lfutil.standin(f) for f in forget] for f in standins: util.unlinkpath(repo.wjoin(f), ignoremissing=True) repo[None].forget(standins) finally: wlock.release() return result
def overrideforget(orig, ui, repo, *pats, **opts): installnormalfilesmatchfn(repo[None].manifest()) result = orig(ui, repo, *pats, **opts) restorematchfn() m = scmutil.match(repo[None], pats, opts) try: repo.lfstatus = True s = repo.status(match=m, clean=True) finally: repo.lfstatus = False forget = sorted(s[0] + s[1] + s[3] + s[6]) forget = [f for f in forget if lfutil.standin(f) in repo[None].manifest()] for f in forget: if lfutil.standin(f) not in repo.dirstate and not \ os.path.isdir(m.rel(lfutil.standin(f))): ui.warn( _('not removing %s: file is already untracked\n') % m.rel(f)) result = 1 for f in forget: if ui.verbose or not m.exact(f): ui.status(_('removing %s\n') % m.rel(f)) # Need to lock because standin files are deleted then removed from the # repository and we could race in-between. wlock = repo.wlock() try: lfdirstate = lfutil.openlfdirstate(ui, repo) for f in forget: if lfdirstate[f] == 'a': lfdirstate.drop(f) else: lfdirstate.remove(f) lfdirstate.write() standins = [lfutil.standin(f) for f in forget] for f in standins: util.unlinkpath(repo.wjoin(f), ignoremissing=True) repo[None].forget(standins) finally: wlock.release() return result
def overridecopy(orig, ui, repo, pats, opts, rename=False): # doesn't remove largefile on rename if len(pats) < 2: # this isn't legal, let the original function deal with it return orig(ui, repo, pats, opts, rename) def makestandin(relpath): path = scmutil.canonpath(repo.root, repo.getcwd(), relpath) return os.path.join(repo.wjoin(lfutil.standin(path))) fullpats = scmutil.expandpats(pats) dest = fullpats[-1] if os.path.isdir(dest): if not os.path.isdir(makestandin(dest)): os.makedirs(makestandin(dest)) # This could copy both lfiles and normal files in one command, # but we don't want to do that. First replace their matcher to # only match normal files and run it, then replace it to just # match largefiles and run it again. nonormalfiles = False nolfiles = False try: try: installnormalfilesmatchfn(repo[None].manifest()) result = orig(ui, repo, pats, opts, rename) except util.Abort, e: if str(e) != _('no files to copy'): raise e else: nonormalfiles = True result = 0 finally: restorematchfn() # The first rename can cause our current working directory to be removed. # In that case there is nothing left to copy/rename so just quit. try: repo.getcwd() except OSError: return result try: try: # When we call orig below it creates the standins but we don't add # them to the dir state until later so lock during that time. wlock = repo.wlock() manifest = repo[None].manifest() oldmatch = None # for the closure def overridematch(ctx, pats=[], opts={}, globbed=False, default='relpath'): newpats = [] # The patterns were previously mangled to add the standin # directory; we need to remove that now for pat in pats: if match_.patkind(pat) is None and lfutil.shortname in pat: newpats.append(pat.replace(lfutil.shortname, '')) else: newpats.append(pat) match = oldmatch(ctx, newpats, opts, globbed, default) m = copy.copy(match) lfile = lambda f: lfutil.standin(f) in manifest m._files = [lfutil.standin(f) for f in m._files if lfile(f)] m._fmap = set(m._files) m._always = False origmatchfn = m.matchfn m.matchfn = lambda f: (lfutil.isstandin(f) and (f in manifest) and origmatchfn(lfutil.splitstandin(f)) or None) return m oldmatch = installmatchfn(overridematch) listpats = [] for pat in pats: if match_.patkind(pat) is not None: listpats.append(pat) else: listpats.append(makestandin(pat)) try: origcopyfile = util.copyfile copiedfiles = [] def overridecopyfile(src, dest): if (lfutil.shortname in src and dest.startswith(repo.wjoin(lfutil.shortname))): destlfile = dest.replace(lfutil.shortname, '') if not opts['force'] and os.path.exists(destlfile): raise IOError('', _('destination largefile already exists')) copiedfiles.append((src, dest)) origcopyfile(src, dest) util.copyfile = overridecopyfile result += orig(ui, repo, listpats, opts, rename) finally: util.copyfile = origcopyfile lfdirstate = lfutil.openlfdirstate(ui, repo) for (src, dest) in copiedfiles: if (lfutil.shortname in src and dest.startswith(repo.wjoin(lfutil.shortname))): srclfile = src.replace(repo.wjoin(lfutil.standin('')), '') destlfile = dest.replace(repo.wjoin(lfutil.standin('')), '') destlfiledir = os.path.dirname(repo.wjoin(destlfile)) or '.' if not os.path.isdir(destlfiledir): os.makedirs(destlfiledir) if rename: os.rename(repo.wjoin(srclfile), repo.wjoin(destlfile)) lfdirstate.remove(srclfile) else: util.copyfile(repo.wjoin(srclfile), repo.wjoin(destlfile)) lfdirstate.add(destlfile) lfdirstate.write() except util.Abort, e: if str(e) != _('no files to copy'): raise e else: nolfiles = True finally: restorematchfn() wlock.release() if nolfiles and nonormalfiles: raise util.Abort(_('no files to copy')) return result
def addlargefiles(ui, repo, *pats, **opts): large = opts.pop('large', None) lfsize = lfutil.getminsize( ui, lfutil.islfilesrepo(repo), opts.pop('lfsize', None)) lfmatcher = None if lfutil.islfilesrepo(repo): lfpats = ui.configlist(lfutil.longname, 'patterns', default=[]) if lfpats: lfmatcher = match_.match(repo.root, '', list(lfpats)) lfnames = [] m = scmutil.match(repo[None], pats, opts) m.bad = lambda x, y: None wctx = repo[None] for f in repo.walk(m): exact = m.exact(f) lfile = lfutil.standin(f) in wctx nfile = f in wctx exists = lfile or nfile # Don't warn the user when they attempt to add a normal tracked file. # The normal add code will do that for us. if exact and exists: if lfile: ui.warn(_('%s already a largefile\n') % f) continue if (exact or not exists) and not lfutil.isstandin(f): wfile = repo.wjoin(f) # In case the file was removed previously, but not committed # (issue3507) if not os.path.exists(wfile): continue abovemin = (lfsize and os.lstat(wfile).st_size >= lfsize * 1024 * 1024) if large or abovemin or (lfmatcher and lfmatcher(f)): lfnames.append(f) if ui.verbose or not exact: ui.status(_('adding %s as a largefile\n') % m.rel(f)) bad = [] standins = [] # Need to lock, otherwise there could be a race condition between # when standins are created and added to the repo. wlock = repo.wlock() try: if not opts.get('dry_run'): lfdirstate = lfutil.openlfdirstate(ui, repo) for f in lfnames: standinname = lfutil.standin(f) lfutil.writestandin(repo, standinname, hash='', executable=lfutil.getexecutable(repo.wjoin(f))) standins.append(standinname) if lfdirstate[f] == 'r': lfdirstate.normallookup(f) else: lfdirstate.add(f) lfdirstate.write() bad += [lfutil.splitstandin(f) for f in repo[None].add(standins) if f in m.files()] finally: wlock.release() return bad
def commit(self, text="", user=None, date=None, match=None, force=False, editor=False, extra={}): orig = super(lfilesrepo, self).commit wlock = self.wlock() try: # Case 0: Automated committing # # While automated committing (like rebase, transplant # and so on), this code path is used to avoid: # (1) updating standins, because standins should # be already updated at this point # (2) aborting when stadnins are matched by "match", # because automated committing may specify them directly # if getattr(self, "_isrebasing", False) or \ getattr(self, "_istransplanting", False): result = orig(text=text, user=user, date=date, match=match, force=force, editor=editor, extra=extra) if result: lfdirstate = lfutil.openlfdirstate(ui, self) for f in self[result].files(): if lfutil.isstandin(f): lfile = lfutil.splitstandin(f) lfutil.synclfdirstate(self, lfdirstate, lfile, False) lfdirstate.write() return result # Case 1: user calls commit with no specific files or # include/exclude patterns: refresh and commit all files that # are "dirty". if ((match is None) or (not match.anypats() and not match.files())): # Spend a bit of time here to get a list of files we know # are modified so we can compare only against those. # It can cost a lot of time (several seconds) # otherwise to update all standins if the largefiles are # large. lfdirstate = lfutil.openlfdirstate(ui, self) dirtymatch = match_.always(self.root, self.getcwd()) unsure, s = lfdirstate.status(dirtymatch, [], False, False, False) modifiedfiles = unsure + s.modified + s.added + s.removed lfiles = lfutil.listlfiles(self) # this only loops through largefiles that exist (not # removed/renamed) for lfile in lfiles: if lfile in modifiedfiles: if os.path.exists(self.wjoin( lfutil.standin(lfile))): # this handles the case where a rebase is being # performed and the working copy is not updated # yet. if os.path.exists(self.wjoin(lfile)): lfutil.updatestandin( self, lfutil.standin(lfile)) lfdirstate.normal(lfile) result = orig(text=text, user=user, date=date, match=match, force=force, editor=editor, extra=extra) if result is not None: for lfile in lfdirstate: if lfile in modifiedfiles: if (not os.path.exists(self.wjoin( lfutil.standin(lfile)))) or \ (not os.path.exists(self.wjoin(lfile))): lfdirstate.drop(lfile) # This needs to be after commit; otherwise precommit hooks # get the wrong status lfdirstate.write() return result lfiles = lfutil.listlfiles(self) match._files = self._subdirlfs(match.files(), lfiles) # Case 2: user calls commit with specified patterns: refresh # any matching big files. smatcher = lfutil.composestandinmatcher(self, match) standins = self.dirstate.walk(smatcher, [], False, False) # No matching big files: get out of the way and pass control to # the usual commit() method. if not standins: return orig(text=text, user=user, date=date, match=match, force=force, editor=editor, extra=extra) # Refresh all matching big files. It's possible that the # commit will end up failing, in which case the big files will # stay refreshed. No harm done: the user modified them and # asked to commit them, so sooner or later we're going to # refresh the standins. Might as well leave them refreshed. lfdirstate = lfutil.openlfdirstate(ui, self) for standin in standins: lfile = lfutil.splitstandin(standin) if lfdirstate[lfile] != 'r': lfutil.updatestandin(self, standin) lfdirstate.normal(lfile) else: lfdirstate.drop(lfile) # Cook up a new matcher that only matches regular files or # standins corresponding to the big files requested by the # user. Have to modify _files to prevent commit() from # complaining "not tracked" for big files. match = copy.copy(match) origmatchfn = match.matchfn # Check both the list of largefiles and the list of # standins because if a largefile was removed, it # won't be in the list of largefiles at this point match._files += sorted(standins) actualfiles = [] for f in match._files: fstandin = lfutil.standin(f) # ignore known largefiles and standins if f in lfiles or fstandin in standins: continue actualfiles.append(f) match._files = actualfiles def matchfn(f): if origmatchfn(f): return f not in lfiles else: return f in standins match.matchfn = matchfn result = orig(text=text, user=user, date=date, match=match, force=force, editor=editor, extra=extra) # This needs to be after commit; otherwise precommit hooks # get the wrong status lfdirstate.write() return result finally: wlock.release()
def commit(self, text="", user=None, date=None, match=None, force=False, editor=False, extra={}): orig = super(lfilesrepo, self).commit wlock = self.wlock() try: # Case 0: Automated committing # # While automated committing (like rebase, transplant # and so on), this code path is used to avoid: # (1) updating standins, because standins should # be already updated at this point # (2) aborting when stadnins are matched by "match", # because automated committing may specify them directly # if getattr(self, "_isrebasing", False) or getattr(self, "_istransplanting", False): result = orig(text=text, user=user, date=date, match=match, force=force, editor=editor, extra=extra) if result: lfdirstate = lfutil.openlfdirstate(ui, self) for f in self[result].files(): if lfutil.isstandin(f): lfile = lfutil.splitstandin(f) lfutil.synclfdirstate(self, lfdirstate, lfile, False) lfdirstate.write() return result # Case 1: user calls commit with no specific files or # include/exclude patterns: refresh and commit all files that # are "dirty". if (match is None) or (not match.anypats() and not match.files()): # Spend a bit of time here to get a list of files we know # are modified so we can compare only against those. # It can cost a lot of time (several seconds) # otherwise to update all standins if the largefiles are # large. lfdirstate = lfutil.openlfdirstate(ui, self) dirtymatch = match_.always(self.root, self.getcwd()) unsure, s = lfdirstate.status(dirtymatch, [], False, False, False) modifiedfiles = unsure + s.modified + s.added + s.removed lfiles = lfutil.listlfiles(self) # this only loops through largefiles that exist (not # removed/renamed) for lfile in lfiles: if lfile in modifiedfiles: if os.path.exists(self.wjoin(lfutil.standin(lfile))): # this handles the case where a rebase is being # performed and the working copy is not updated # yet. if os.path.exists(self.wjoin(lfile)): lfutil.updatestandin(self, lfutil.standin(lfile)) lfdirstate.normal(lfile) result = orig(text=text, user=user, date=date, match=match, force=force, editor=editor, extra=extra) if result is not None: for lfile in lfdirstate: if lfile in modifiedfiles: if (not os.path.exists(self.wjoin(lfutil.standin(lfile)))) or ( not os.path.exists(self.wjoin(lfile)) ): lfdirstate.drop(lfile) # This needs to be after commit; otherwise precommit hooks # get the wrong status lfdirstate.write() return result lfiles = lfutil.listlfiles(self) match._files = self._subdirlfs(match.files(), lfiles) # Case 2: user calls commit with specified patterns: refresh # any matching big files. smatcher = lfutil.composestandinmatcher(self, match) standins = self.dirstate.walk(smatcher, [], False, False) # No matching big files: get out of the way and pass control to # the usual commit() method. if not standins: return orig(text=text, user=user, date=date, match=match, force=force, editor=editor, extra=extra) # Refresh all matching big files. It's possible that the # commit will end up failing, in which case the big files will # stay refreshed. No harm done: the user modified them and # asked to commit them, so sooner or later we're going to # refresh the standins. Might as well leave them refreshed. lfdirstate = lfutil.openlfdirstate(ui, self) for standin in standins: lfile = lfutil.splitstandin(standin) if lfdirstate[lfile] != "r": lfutil.updatestandin(self, standin) lfdirstate.normal(lfile) else: lfdirstate.drop(lfile) # Cook up a new matcher that only matches regular files or # standins corresponding to the big files requested by the # user. Have to modify _files to prevent commit() from # complaining "not tracked" for big files. match = copy.copy(match) origmatchfn = match.matchfn # Check both the list of largefiles and the list of # standins because if a largefile was removed, it # won't be in the list of largefiles at this point match._files += sorted(standins) actualfiles = [] for f in match._files: fstandin = lfutil.standin(f) # ignore known largefiles and standins if f in lfiles or fstandin in standins: continue actualfiles.append(f) match._files = actualfiles def matchfn(f): if origmatchfn(f): return f not in lfiles else: return f in standins match.matchfn = matchfn result = orig(text=text, user=user, date=date, match=match, force=force, editor=editor, extra=extra) # This needs to be after commit; otherwise precommit hooks # get the wrong status lfdirstate.write() return result finally: wlock.release()
def status(self, node1='.', node2=None, match=None, ignored=False, clean=False, unknown=False, listsubrepos=False): listignored, listclean, listunknown = ignored, clean, unknown if not self.lfstatus: try: return super(lfiles_repo, self).status(node1, node2, match, listignored, listclean, listunknown, listsubrepos) except TypeError: return super(lfiles_repo, self).status(node1, node2, match, listignored, listclean, listunknown) else: # some calls in this function rely on the old version of status self.lfstatus = False if isinstance(node1, context.changectx): ctx1 = node1 else: ctx1 = repo[node1] if isinstance(node2, context.changectx): ctx2 = node2 else: ctx2 = repo[node2] working = ctx2.rev() is None parentworking = working and ctx1 == self['.'] def inctx(file, ctx): try: if ctx.rev() is None: return file in ctx.manifest() ctx[file] return True except KeyError: return False if match is None: match = match_.always(self.root, self.getcwd()) # Create a copy of match that matches standins instead # of largefiles. def tostandin(file): if inctx(lfutil.standin(file), ctx2): return lfutil.standin(file) return file m = copy.copy(match) m._files = [tostandin(f) for f in m._files] # get ignored, clean, and unknown but remove them # later if they were not asked for try: result = super(lfiles_repo, self).status(node1, node2, m, True, True, True, listsubrepos) except TypeError: result = super(lfiles_repo, self).status(node1, node2, m, True, True, True) if working: # hold the wlock while we read largefiles and # update the lfdirstate wlock = repo.wlock() try: # Any non-largefiles that were explicitly listed must be # taken out or lfdirstate.status will report an error. # The status of these files was already computed using # super's status. lfdirstate = lfutil.openlfdirstate(ui, self) match._files = [ f for f in match._files if f in lfdirstate ] s = lfdirstate.status(match, [], listignored, listclean, listunknown) (unsure, modified, added, removed, missing, unknown, ignored, clean) = s if parentworking: for lfile in unsure: if ctx1[lfutil.standin(lfile)].data().strip() \ != lfutil.hashfile(self.wjoin(lfile)): modified.append(lfile) else: clean.append(lfile) lfdirstate.normal(lfile) lfdirstate.write() else: tocheck = unsure + modified + added + clean modified, added, clean = [], [], [] for lfile in tocheck: standin = lfutil.standin(lfile) if inctx(standin, ctx1): if ctx1[standin].data().strip() != \ lfutil.hashfile(self.wjoin(lfile)): modified.append(lfile) else: clean.append(lfile) else: added.append(lfile) finally: wlock.release() for standin in ctx1.manifest(): if not lfutil.isstandin(standin): continue lfile = lfutil.splitstandin(standin) if not match(lfile): continue if lfile not in lfdirstate: removed.append(lfile) # Handle unknown and ignored differently lfiles = (modified, added, removed, missing, [], [], clean) result = list(result) # Unknown files result[4] = [ f for f in unknown if (repo.dirstate[f] == '?' and not lfutil.isstandin(f) ) ] # Ignored files must be ignored by both the dirstate and # lfdirstate result[5] = set(ignored).intersection(set(result[5])) # combine normal files and largefiles normals = [[ fn for fn in filelist if not lfutil.isstandin(fn) ] for filelist in result] result = [ sorted(list1 + list2) for (list1, list2) in zip(normals, lfiles) ] else: def toname(f): if lfutil.isstandin(f): return lfutil.splitstandin(f) return f result = [[toname(f) for f in items] for items in result] if not listunknown: result[4] = [] if not listignored: result[5] = [] if not listclean: result[6] = [] self.lfstatus = True return result
def updatelfiles(ui, repo, filelist=None, printmessage=True, normallookup=False): wlock = repo.wlock() try: lfdirstate = lfutil.openlfdirstate(ui, repo) lfiles = set(lfutil.listlfiles(repo)) | set(lfdirstate) if filelist is not None: filelist = set(filelist) lfiles = [f for f in lfiles if f in filelist] update = {} updated, removed = 0, 0 for lfile in lfiles: abslfile = repo.wjoin(lfile) absstandin = repo.wjoin(lfutil.standin(lfile)) if os.path.exists(absstandin): if (os.path.exists(absstandin + '.orig') and os.path.exists(abslfile)): shutil.copyfile(abslfile, abslfile + '.orig') util.unlinkpath(absstandin + '.orig') expecthash = lfutil.readstandin(repo, lfile) if (expecthash != '' and (not os.path.exists(abslfile) or expecthash != lfutil.hashfile(abslfile))): if lfile not in repo[None]: # not switched to normal file util.unlinkpath(abslfile, ignoremissing=True) # use normallookup() to allocate entry in largefiles # dirstate, because lack of it misleads # lfilesrepo.status() into recognition that such cache # missing files are REMOVED. lfdirstate.normallookup(lfile) update[lfile] = expecthash else: # Remove lfiles for which the standin is deleted, unless the # lfile is added to the repository again. This happens when a # largefile is converted back to a normal file: the standin # disappears, but a new (normal) file appears as the lfile. if (os.path.exists(abslfile) and repo.dirstate.normalize(lfile) not in repo[None]): util.unlinkpath(abslfile) removed += 1 # largefile processing might be slow and be interrupted - be prepared lfdirstate.write() if lfiles: if printmessage: ui.status(_('getting changed largefiles\n')) cachelfiles(ui, repo, None, lfiles) for lfile in lfiles: update1 = 0 expecthash = update.get(lfile) if expecthash: if not lfutil.copyfromcache(repo, expecthash, lfile): # failed ... but already removed and set to normallookup continue # Synchronize largefile dirstate to the last modified # time of the file lfdirstate.normal(lfile) update1 = 1 # copy the state of largefile standin from the repository's # dirstate to its state in the lfdirstate. abslfile = repo.wjoin(lfile) absstandin = repo.wjoin(lfutil.standin(lfile)) if os.path.exists(absstandin): mode = os.stat(absstandin).st_mode if mode != os.stat(abslfile).st_mode: os.chmod(abslfile, mode) update1 = 1 updated += update1 lfutil.synclfdirstate(repo, lfdirstate, lfile, normallookup) if filelist is not None: # If "local largefile" is chosen at file merging, it is # not listed in "filelist" (= dirstate syncing is # omitted), because the standin file is not changed before and # after merging. # But the status of such files may have to be changed by # merging. For example, locally modified ("M") largefile # has to become re-added("A"), if it is "normal" file in # the target revision of linear-merging. for lfile in lfdirstate: if lfile not in filelist: lfutil.synclfdirstate(repo, lfdirstate, lfile, True) lfdirstate.write() if printmessage and lfiles: ui.status(_('%d largefiles updated, %d removed\n') % (updated, removed)) finally: wlock.release()
def commit(self, text="", user=None, date=None, match=None, force=False, editor=False, extra={}): orig = super(lfilesrepo, self).commit wlock = repo.wlock() try: # Case 0: Rebase or Transplant # We have to take the time to pull down the new largefiles now. # Otherwise, any largefiles that were modified in the # destination changesets get overwritten, either by the rebase # or in the first commit after the rebase or transplant. # updatelfiles will update the dirstate to mark any pulled # largefiles as modified if getattr(repo, "_isrebasing", False) or \ getattr(repo, "_istransplanting", False): lfcommands.updatelfiles(repo.ui, repo, filelist=None, printmessage=False) result = orig(text=text, user=user, date=date, match=match, force=force, editor=editor, extra=extra) return result # Case 1: user calls commit with no specific files or # include/exclude patterns: refresh and commit all files that # are "dirty". if ((match is None) or (not match.anypats() and not match.files())): # Spend a bit of time here to get a list of files we know # are modified so we can compare only against those. # It can cost a lot of time (several seconds) # otherwise to update all standins if the largefiles are # large. lfdirstate = lfutil.openlfdirstate(ui, self) dirtymatch = match_.always(repo.root, repo.getcwd()) s = lfdirstate.status(dirtymatch, [], False, False, False) modifiedfiles = [] for i in s: modifiedfiles.extend(i) lfiles = lfutil.listlfiles(self) # this only loops through largefiles that exist (not # removed/renamed) for lfile in lfiles: if lfile in modifiedfiles: if os.path.exists( self.wjoin(lfutil.standin(lfile))): # this handles the case where a rebase is being # performed and the working copy is not updated # yet. if os.path.exists(self.wjoin(lfile)): lfutil.updatestandin(self, lfutil.standin(lfile)) lfdirstate.normal(lfile) for lfile in lfdirstate: if lfile in modifiedfiles: if not os.path.exists( repo.wjoin(lfutil.standin(lfile))): lfdirstate.drop(lfile) result = orig(text=text, user=user, date=date, match=match, force=force, editor=editor, extra=extra) # This needs to be after commit; otherwise precommit hooks # get the wrong status lfdirstate.write() return result for f in match.files(): if lfutil.isstandin(f): raise util.Abort( _('file "%s" is a largefile standin') % f, hint=('commit the largefile itself instead')) # Case 2: user calls commit with specified patterns: refresh # any matching big files. smatcher = lfutil.composestandinmatcher(self, match) standins = lfutil.dirstatewalk(self.dirstate, smatcher) # No matching big files: get out of the way and pass control to # the usual commit() method. if not standins: return orig(text=text, user=user, date=date, match=match, force=force, editor=editor, extra=extra) # Refresh all matching big files. It's possible that the # commit will end up failing, in which case the big files will # stay refreshed. No harm done: the user modified them and # asked to commit them, so sooner or later we're going to # refresh the standins. Might as well leave them refreshed. lfdirstate = lfutil.openlfdirstate(ui, self) for standin in standins: lfile = lfutil.splitstandin(standin) if lfdirstate[lfile] <> 'r': lfutil.updatestandin(self, standin) lfdirstate.normal(lfile) else: lfdirstate.drop(lfile) # Cook up a new matcher that only matches regular files or # standins corresponding to the big files requested by the # user. Have to modify _files to prevent commit() from # complaining "not tracked" for big files. lfiles = lfutil.listlfiles(repo) match = copy.copy(match) origmatchfn = match.matchfn # Check both the list of largefiles and the list of # standins because if a largefile was removed, it # won't be in the list of largefiles at this point match._files += sorted(standins) actualfiles = [] for f in match._files: fstandin = lfutil.standin(f) # ignore known largefiles and standins if f in lfiles or fstandin in standins: continue # append directory separator to avoid collisions if not fstandin.endswith(os.sep): fstandin += os.sep actualfiles.append(f) match._files = actualfiles def matchfn(f): if origmatchfn(f): return f not in lfiles else: return f in standins match.matchfn = matchfn result = orig(text=text, user=user, date=date, match=match, force=force, editor=editor, extra=extra) # This needs to be after commit; otherwise precommit hooks # get the wrong status lfdirstate.write() return result finally: wlock.release()
def status(self, node1='.', node2=None, match=None, ignored=False, clean=False, unknown=False, listsubrepos=False): listignored, listclean, listunknown = ignored, clean, unknown if not self.lfstatus: return super(lfilesrepo, self).status(node1, node2, match, listignored, listclean, listunknown, listsubrepos) else: # some calls in this function rely on the old version of status self.lfstatus = False ctx1 = self[node1] ctx2 = self[node2] working = ctx2.rev() is None parentworking = working and ctx1 == self['.'] def inctx(file, ctx): try: if ctx.rev() is None: return file in ctx.manifest() ctx[file] return True except KeyError: return False if match is None: match = match_.always(self.root, self.getcwd()) wlock = None try: try: # updating the dirstate is optional # so we don't wait on the lock wlock = self.wlock(False) except error.LockError: pass # First check if there were files specified on the # command line. If there were, and none of them were # largefiles, we should just bail here and let super # handle it -- thus gaining a big performance boost. lfdirstate = lfutil.openlfdirstate(ui, self) if match.files() and not match.anypats(): for f in lfdirstate: if match(f): break else: return super(lfilesrepo, self).status(node1, node2, match, listignored, listclean, listunknown, listsubrepos) # Create a copy of match that matches standins instead # of largefiles. def tostandins(files): if not working: return files newfiles = [] dirstate = self.dirstate for f in files: sf = lfutil.standin(f) if sf in dirstate: newfiles.append(sf) elif sf in dirstate.dirs(): # Directory entries could be regular or # standin, check both newfiles.extend((f, sf)) else: newfiles.append(f) return newfiles m = copy.copy(match) m._files = tostandins(m._files) result = super(lfilesrepo, self).status(node1, node2, m, ignored, clean, unknown, listsubrepos) if working: def sfindirstate(f): sf = lfutil.standin(f) dirstate = self.dirstate return sf in dirstate or sf in dirstate.dirs() match._files = [ f for f in match._files if sfindirstate(f) ] # Don't waste time getting the ignored and unknown # files from lfdirstate s = lfdirstate.status(match, [], False, listclean, False) (unsure, modified, added, removed, missing, _unknown, _ignored, clean) = s if parentworking: for lfile in unsure: standin = lfutil.standin(lfile) if standin not in ctx1: # from second parent modified.append(lfile) elif ctx1[standin].data().strip() \ != lfutil.hashfile(self.wjoin(lfile)): modified.append(lfile) else: clean.append(lfile) lfdirstate.normal(lfile) else: tocheck = unsure + modified + added + clean modified, added, clean = [], [], [] for lfile in tocheck: standin = lfutil.standin(lfile) if inctx(standin, ctx1): if ctx1[standin].data().strip() != \ lfutil.hashfile(self.wjoin(lfile)): modified.append(lfile) else: clean.append(lfile) else: added.append(lfile) # Standins no longer found in lfdirstate has been # removed for standin in ctx1.manifest(): if not lfutil.isstandin(standin): continue lfile = lfutil.splitstandin(standin) if not match(lfile): continue if lfile not in lfdirstate: removed.append(lfile) # Filter result lists result = list(result) # Largefiles are not really removed when they're # still in the normal dirstate. Likewise, normal # files are not really removed if they are still in # lfdirstate. This happens in merges where files # change type. removed = [ f for f in removed if f not in self.dirstate ] result[2] = [ f for f in result[2] if f not in lfdirstate ] lfiles = set(lfdirstate._map) # Unknown files result[4] = set(result[4]).difference(lfiles) # Ignored files result[5] = set(result[5]).difference(lfiles) # combine normal files and largefiles normals = [[ fn for fn in filelist if not lfutil.isstandin(fn) ] for filelist in result] lfiles = (modified, added, removed, missing, [], [], clean) result = [ sorted(list1 + list2) for (list1, list2) in zip(normals, lfiles) ] else: def toname(f): if lfutil.isstandin(f): return lfutil.splitstandin(f) return f result = [[toname(f) for f in items] for items in result] if wlock: lfdirstate.write() finally: if wlock: wlock.release() if not listunknown: result[4] = [] if not listignored: result[5] = [] if not listclean: result[6] = [] self.lfstatus = True return result
def status(self, node1='.', node2=None, match=None, ignored=False, clean=False, unknown=False, listsubrepos=False): listignored, listclean, listunknown = ignored, clean, unknown if not self.lfstatus: try: return super(lfiles_repo, self).status(node1, node2, match, listignored, listclean, listunknown, listsubrepos) except TypeError: return super(lfiles_repo, self).status(node1, node2, match, listignored, listclean, listunknown) else: # some calls in this function rely on the old version of status self.lfstatus = False if isinstance(node1, context.changectx): ctx1 = node1 else: ctx1 = repo[node1] if isinstance(node2, context.changectx): ctx2 = node2 else: ctx2 = repo[node2] working = ctx2.rev() is None parentworking = working and ctx1 == self['.'] def inctx(file, ctx): try: if ctx.rev() is None: return file in ctx.manifest() ctx[file] return True except KeyError: return False if match is None: match = match_.always(self.root, self.getcwd()) # Create a copy of match that matches standins instead # of largefiles. def tostandin(file): if inctx(lfutil.standin(file), ctx2): return lfutil.standin(file) return file m = copy.copy(match) m._files = [tostandin(f) for f in m._files] # get ignored, clean, and unknown but remove them # later if they were not asked for try: result = super(lfiles_repo, self).status(node1, node2, m, True, True, True, listsubrepos) except TypeError: result = super(lfiles_repo, self).status(node1, node2, m, True, True, True) if working: # hold the wlock while we read largefiles and # update the lfdirstate wlock = repo.wlock() try: # Any non-largefiles that were explicitly listed must be # taken out or lfdirstate.status will report an error. # The status of these files was already computed using # super's status. lfdirstate = lfutil.openlfdirstate(ui, self) match._files = [f for f in match._files if f in lfdirstate] s = lfdirstate.status(match, [], listignored, listclean, listunknown) (unsure, modified, added, removed, missing, unknown, ignored, clean) = s if parentworking: for lfile in unsure: if ctx1[lfutil.standin(lfile)].data().strip() \ != lfutil.hashfile(self.wjoin(lfile)): modified.append(lfile) else: clean.append(lfile) lfdirstate.normal(lfile) lfdirstate.write() else: tocheck = unsure + modified + added + clean modified, added, clean = [], [], [] for lfile in tocheck: standin = lfutil.standin(lfile) if inctx(standin, ctx1): if ctx1[standin].data().strip() != \ lfutil.hashfile(self.wjoin(lfile)): modified.append(lfile) else: clean.append(lfile) else: added.append(lfile) finally: wlock.release() for standin in ctx1.manifest(): if not lfutil.isstandin(standin): continue lfile = lfutil.splitstandin(standin) if not match(lfile): continue if lfile not in lfdirstate: removed.append(lfile) # Handle unknown and ignored differently lfiles = (modified, added, removed, missing, [], [], clean) result = list(result) # Unknown files result[4] = [f for f in unknown if (repo.dirstate[f] == '?' and not lfutil.isstandin(f))] # Ignored files must be ignored by both the dirstate and # lfdirstate result[5] = set(ignored).intersection(set(result[5])) # combine normal files and largefiles normals = [[fn for fn in filelist if not lfutil.isstandin(fn)] for filelist in result] result = [sorted(list1 + list2) for (list1, list2) in zip(normals, lfiles)] else: def toname(f): if lfutil.isstandin(f): return lfutil.splitstandin(f) return f result = [[toname(f) for f in items] for items in result] if not listunknown: result[4] = [] if not listignored: result[5] = [] if not listclean: result[6] = [] self.lfstatus = True return result
def updatelfiles(ui, repo, filelist=None, printmessage=None, normallookup=False, checked=False): '''Update largefiles according to standins in the working directory If ``printmessage`` is other than ``None``, it means "print (or ignore, for false) message forcibly". ''' statuswriter = lfutil.getstatuswriter(ui, repo, printmessage) wlock = repo.wlock() try: lfdirstate = lfutil.openlfdirstate(ui, repo) lfiles = set(lfutil.listlfiles(repo)) | set(lfdirstate) if filelist is not None: filelist = set(filelist) lfiles = [f for f in lfiles if f in filelist] update = {} updated, removed = 0, 0 for lfile in lfiles: abslfile = repo.wjoin(lfile) absstandin = repo.wjoin(lfutil.standin(lfile)) if os.path.exists(absstandin): if (os.path.exists(absstandin + '.orig') and os.path.exists(abslfile)): shutil.copyfile(abslfile, abslfile + '.orig') util.unlinkpath(absstandin + '.orig') expecthash = lfutil.readstandin(repo, lfile) if (expecthash != '' and (checked or not os.path.exists(abslfile) or expecthash != lfutil.hashfile(abslfile))): if lfile not in repo[None]: # not switched to normal file util.unlinkpath(abslfile, ignoremissing=True) # use normallookup() to allocate an entry in largefiles # dirstate, because lack of it misleads # lfilesrepo.status() into recognition that such cache # missing files are removed. lfdirstate.normallookup(lfile) update[lfile] = expecthash else: # Remove lfiles for which the standin is deleted, unless the # lfile is added to the repository again. This happens when a # largefile is converted back to a normal file: the standin # disappears, but a new (normal) file appears as the lfile. if (os.path.exists(abslfile) and repo.dirstate.normalize(lfile) not in repo[None]): util.unlinkpath(abslfile) removed += 1 # largefile processing might be slow and be interrupted - be prepared lfdirstate.write() if lfiles: statuswriter(_('getting changed largefiles\n')) cachelfiles(ui, repo, None, lfiles) for lfile in lfiles: update1 = 0 expecthash = update.get(lfile) if expecthash: if not lfutil.copyfromcache(repo, expecthash, lfile): # failed ... but already removed and set to normallookup continue # Synchronize largefile dirstate to the last modified # time of the file lfdirstate.normal(lfile) update1 = 1 # copy the state of largefile standin from the repository's # dirstate to its state in the lfdirstate. abslfile = repo.wjoin(lfile) absstandin = repo.wjoin(lfutil.standin(lfile)) if os.path.exists(absstandin): mode = os.stat(absstandin).st_mode if mode != os.stat(abslfile).st_mode: os.chmod(abslfile, mode) update1 = 1 updated += update1 lfutil.synclfdirstate(repo, lfdirstate, lfile, normallookup) lfdirstate.write() if lfiles: statuswriter( _('%d largefiles updated, %d removed\n') % (updated, removed)) finally: wlock.release()
def updatelfiles(ui, repo, filelist=None, printmessage=None, normallookup=False): '''Update largefiles according to standins in the working directory If ``printmessage`` is other than ``None``, it means "print (or ignore, for false) message forcibly". ''' statuswriter = lfutil.getstatuswriter(ui, repo, printmessage) with repo.wlock(): lfdirstate = lfutil.openlfdirstate(ui, repo) lfiles = set(lfutil.listlfiles(repo)) | set(lfdirstate) if filelist is not None: filelist = set(filelist) lfiles = [f for f in lfiles if f in filelist] update = {} updated, removed = 0, 0 wvfs = repo.wvfs for lfile in lfiles: rellfile = lfile rellfileorig = os.path.relpath(scmutil.origpath( ui, repo, wvfs.join(rellfile)), start=repo.root) relstandin = lfutil.standin(lfile) relstandinorig = os.path.relpath(scmutil.origpath( ui, repo, wvfs.join(relstandin)), start=repo.root) if wvfs.exists(relstandin): if (wvfs.exists(relstandinorig) and wvfs.exists(rellfile)): shutil.copyfile(wvfs.join(rellfile), wvfs.join(rellfileorig)) wvfs.unlinkpath(relstandinorig) expecthash = lfutil.readstandin(repo, lfile) if expecthash != '': if lfile not in repo[None]: # not switched to normal file wvfs.unlinkpath(rellfile, ignoremissing=True) # use normallookup() to allocate an entry in largefiles # dirstate to prevent lfilesrepo.status() from reporting # missing files as removed. lfdirstate.normallookup(lfile) update[lfile] = expecthash else: # Remove lfiles for which the standin is deleted, unless the # lfile is added to the repository again. This happens when a # largefile is converted back to a normal file: the standin # disappears, but a new (normal) file appears as the lfile. if (wvfs.exists(rellfile) and repo.dirstate.normalize(lfile) not in repo[None]): wvfs.unlinkpath(rellfile) removed += 1 # largefile processing might be slow and be interrupted - be prepared lfdirstate.write() if lfiles: statuswriter(_('getting changed largefiles\n')) cachelfiles(ui, repo, None, lfiles) for lfile in lfiles: update1 = 0 expecthash = update.get(lfile) if expecthash: if not lfutil.copyfromcache(repo, expecthash, lfile): # failed ... but already removed and set to normallookup continue # Synchronize largefile dirstate to the last modified # time of the file lfdirstate.normal(lfile) update1 = 1 # copy the state of largefile standin from the repository's # dirstate to its state in the lfdirstate. rellfile = lfile relstandin = lfutil.standin(lfile) if wvfs.exists(relstandin): mode = wvfs.stat(relstandin).st_mode if mode != wvfs.stat(rellfile).st_mode: wvfs.chmod(rellfile, mode) update1 = 1 updated += update1 lfutil.synclfdirstate(repo, lfdirstate, lfile, normallookup) lfdirstate.write() if lfiles: statuswriter( _('%d largefiles updated, %d removed\n') % (updated, removed))
def commit(self, text="", user=None, date=None, match=None, force=False, editor=False, extra={}): orig = super(lfilesrepo, self).commit wlock = self.wlock() try: # Case 0: Rebase or Transplant # We have to take the time to pull down the new largefiles now. # Otherwise, any largefiles that were modified in the # destination changesets get overwritten, either by the rebase # or in the first commit after the rebase or transplant. # updatelfiles will update the dirstate to mark any pulled # largefiles as modified if getattr(self, "_isrebasing", False) or \ getattr(self, "_istransplanting", False): lfcommands.updatelfiles(self.ui, self, filelist=None, printmessage=False) result = orig(text=text, user=user, date=date, match=match, force=force, editor=editor, extra=extra) return result # Case 1: user calls commit with no specific files or # include/exclude patterns: refresh and commit all files that # are "dirty". if ((match is None) or (not match.anypats() and not match.files())): # Spend a bit of time here to get a list of files we know # are modified so we can compare only against those. # It can cost a lot of time (several seconds) # otherwise to update all standins if the largefiles are # large. lfdirstate = lfutil.openlfdirstate(ui, self) dirtymatch = match_.always(self.root, self.getcwd()) s = lfdirstate.status(dirtymatch, [], False, False, False) (unsure, modified, added, removed, _missing, _unknown, _ignored, _clean) = s modifiedfiles = unsure + modified + added + removed lfiles = lfutil.listlfiles(self) # this only loops through largefiles that exist (not # removed/renamed) for lfile in lfiles: if lfile in modifiedfiles: if os.path.exists( self.wjoin(lfutil.standin(lfile))): # this handles the case where a rebase is being # performed and the working copy is not updated # yet. if os.path.exists(self.wjoin(lfile)): lfutil.updatestandin(self, lfutil.standin(lfile)) lfdirstate.normal(lfile) result = orig(text=text, user=user, date=date, match=match, force=force, editor=editor, extra=extra) if result is not None: for lfile in lfdirstate: if lfile in modifiedfiles: if (not os.path.exists(self.wjoin( lfutil.standin(lfile)))) or \ (not os.path.exists(self.wjoin(lfile))): lfdirstate.drop(lfile) # This needs to be after commit; otherwise precommit hooks # get the wrong status lfdirstate.write() return result lfiles = lfutil.listlfiles(self) match._files = self._subdirlfs(match.files(), lfiles) # Case 2: user calls commit with specified patterns: refresh # any matching big files. smatcher = lfutil.composestandinmatcher(self, match) standins = self.dirstate.walk(smatcher, [], False, False) # No matching big files: get out of the way and pass control to # the usual commit() method. if not standins: return orig(text=text, user=user, date=date, match=match, force=force, editor=editor, extra=extra) # Refresh all matching big files. It's possible that the # commit will end up failing, in which case the big files will # stay refreshed. No harm done: the user modified them and # asked to commit them, so sooner or later we're going to # refresh the standins. Might as well leave them refreshed. lfdirstate = lfutil.openlfdirstate(ui, self) for standin in standins: lfile = lfutil.splitstandin(standin) if lfdirstate[lfile] != 'r': lfutil.updatestandin(self, standin) lfdirstate.normal(lfile) else: lfdirstate.drop(lfile) # Cook up a new matcher that only matches regular files or # standins corresponding to the big files requested by the # user. Have to modify _files to prevent commit() from # complaining "not tracked" for big files. match = copy.copy(match) origmatchfn = match.matchfn # Check both the list of largefiles and the list of # standins because if a largefile was removed, it # won't be in the list of largefiles at this point match._files += sorted(standins) actualfiles = [] for f in match._files: fstandin = lfutil.standin(f) # ignore known largefiles and standins if f in lfiles or fstandin in standins: continue # append directory separator to avoid collisions if not fstandin.endswith(os.sep): fstandin += os.sep actualfiles.append(f) match._files = actualfiles def matchfn(f): if origmatchfn(f): return f not in lfiles else: return f in standins match.matchfn = matchfn result = orig(text=text, user=user, date=date, match=match, force=force, editor=editor, extra=extra) # This needs to be after commit; otherwise precommit hooks # get the wrong status lfdirstate.write() return result finally: wlock.release()
class fakerepo(object): dirstate = lfutil.openlfdirstate(ui, repo)
def overriderevert(orig, ui, repo, *pats, **opts): # Because we put the standins in a bad state (by updating them) # and then return them to a correct state we need to lock to # prevent others from changing them in their incorrect state. wlock = repo.wlock() try: lfdirstate = lfutil.openlfdirstate(ui, repo) (modified, added, removed, missing, unknown, ignored, clean) = \ lfutil.lfdirstatestatus(lfdirstate, repo, repo['.'].rev()) lfdirstate.write() for lfile in modified: lfutil.updatestandin(repo, lfutil.standin(lfile)) for lfile in missing: if (os.path.exists(repo.wjoin(lfutil.standin(lfile)))): os.unlink(repo.wjoin(lfutil.standin(lfile))) try: ctx = scmutil.revsingle(repo, opts.get('rev')) oldmatch = None # for the closure def overridematch(ctx, pats=[], opts={}, globbed=False, default='relpath'): match = oldmatch(ctx, pats, opts, globbed, default) m = copy.copy(match) def tostandin(f): if lfutil.standin(f) in ctx: return lfutil.standin(f) elif lfutil.standin(f) in repo[None]: return None return f m._files = [tostandin(f) for f in m._files] m._files = [f for f in m._files if f is not None] m._fmap = set(m._files) m._always = False origmatchfn = m.matchfn def matchfn(f): if lfutil.isstandin(f): # We need to keep track of what largefiles are being # matched so we know which ones to update later -- # otherwise we accidentally revert changes to other # largefiles. This is repo-specific, so duckpunch the # repo object to keep the list of largefiles for us # later. if origmatchfn(lfutil.splitstandin(f)) and \ (f in repo[None] or f in ctx): lfileslist = getattr(repo, '_lfilestoupdate', []) lfileslist.append(lfutil.splitstandin(f)) repo._lfilestoupdate = lfileslist return True else: return False return origmatchfn(f) m.matchfn = matchfn return m oldmatch = installmatchfn(overridematch) scmutil.match matches = overridematch(repo[None], pats, opts) orig(ui, repo, *pats, **opts) finally: restorematchfn() lfileslist = getattr(repo, '_lfilestoupdate', []) lfcommands.updatelfiles(ui, repo, filelist=lfileslist, printmessage=False) # empty out the largefiles list so we start fresh next time repo._lfilestoupdate = [] for lfile in modified: if lfile in lfileslist: if os.path.exists(repo.wjoin(lfutil.standin(lfile))) and lfile\ in repo['.']: lfutil.writestandin(repo, lfutil.standin(lfile), repo['.'][lfile].data().strip(), 'x' in repo['.'][lfile].flags()) lfdirstate = lfutil.openlfdirstate(ui, repo) for lfile in added: standin = lfutil.standin(lfile) if standin not in ctx and (standin in matches or opts.get('all')): if lfile in lfdirstate: lfdirstate.drop(lfile) util.unlinkpath(repo.wjoin(standin)) lfdirstate.write() finally: wlock.release()
def updatelfiles(ui, repo, filelist=None, printmessage=None, normallookup=False): '''Update largefiles according to standins in the working directory If ``printmessage`` is other than ``None``, it means "print (or ignore, for false) message forcibly". ''' statuswriter = lfutil.getstatuswriter(ui, repo, printmessage) wlock = repo.wlock() try: lfdirstate = lfutil.openlfdirstate(ui, repo) lfiles = set(lfutil.listlfiles(repo)) | set(lfdirstate) if filelist is not None: filelist = set(filelist) lfiles = [f for f in lfiles if f in filelist] update = {} updated, removed = 0, 0 for lfile in lfiles: abslfile = repo.wjoin(lfile) absstandin = repo.wjoin(lfutil.standin(lfile)) if os.path.exists(absstandin): if (os.path.exists(absstandin + '.orig') and os.path.exists(abslfile)): shutil.copyfile(abslfile, abslfile + '.orig') util.unlinkpath(absstandin + '.orig') expecthash = lfutil.readstandin(repo, lfile) if expecthash != '': if lfile not in repo[None]: # not switched to normal file util.unlinkpath(abslfile, ignoremissing=True) # use normallookup() to allocate an entry in largefiles # dirstate to prevent lfilesrepo.status() from reporting # missing files as removed. lfdirstate.normallookup(lfile) update[lfile] = expecthash else: # Remove lfiles for which the standin is deleted, unless the # lfile is added to the repository again. This happens when a # largefile is converted back to a normal file: the standin # disappears, but a new (normal) file appears as the lfile. if (os.path.exists(abslfile) and repo.dirstate.normalize(lfile) not in repo[None]): util.unlinkpath(abslfile) removed += 1 # largefile processing might be slow and be interrupted - be prepared lfdirstate.write() if lfiles: statuswriter(_('getting changed largefiles\n')) cachelfiles(ui, repo, None, lfiles) for lfile in lfiles: update1 = 0 expecthash = update.get(lfile) if expecthash: if not lfutil.copyfromcache(repo, expecthash, lfile): # failed ... but already removed and set to normallookup continue # Synchronize largefile dirstate to the last modified # time of the file lfdirstate.normal(lfile) update1 = 1 # copy the state of largefile standin from the repository's # dirstate to its state in the lfdirstate. abslfile = repo.wjoin(lfile) absstandin = repo.wjoin(lfutil.standin(lfile)) if os.path.exists(absstandin): mode = os.stat(absstandin).st_mode if mode != os.stat(abslfile).st_mode: os.chmod(abslfile, mode) update1 = 1 updated += update1 lfutil.synclfdirstate(repo, lfdirstate, lfile, normallookup) lfdirstate.write() if lfiles: statuswriter(_('%d largefiles updated, %d removed\n') % (updated, removed)) finally: wlock.release()
def updatelfiles(ui, repo, filelist=None, printmessage=True, normallookup=False): wlock = repo.wlock() try: lfdirstate = lfutil.openlfdirstate(ui, repo) lfiles = set(lfutil.listlfiles(repo)) | set(lfdirstate) if filelist is not None: filelist = set(filelist) lfiles = [f for f in lfiles if f in filelist] update = {} updated, removed = 0, 0 for lfile in lfiles: abslfile = repo.wjoin(lfile) absstandin = repo.wjoin(lfutil.standin(lfile)) if os.path.exists(absstandin): if (os.path.exists(absstandin + '.orig') and os.path.exists(abslfile)): shutil.copyfile(abslfile, abslfile + '.orig') util.unlinkpath(absstandin + '.orig') expecthash = lfutil.readstandin(repo, lfile) if (expecthash != '' and (not os.path.exists(abslfile) or expecthash != lfutil.hashfile(abslfile))): if lfile not in repo[None]: # not switched to normal file util.unlinkpath(abslfile, ignoremissing=True) # use normallookup() to allocate entry in largefiles # dirstate, because lack of it misleads # lfilesrepo.status() into recognition that such cache # missing files are REMOVED. lfdirstate.normallookup(lfile) update[lfile] = expecthash else: # Remove lfiles for which the standin is deleted, unless the # lfile is added to the repository again. This happens when a # largefile is converted back to a normal file: the standin # disappears, but a new (normal) file appears as the lfile. if (os.path.exists(abslfile) and repo.dirstate.normalize(lfile) not in repo[None]): util.unlinkpath(abslfile) removed += 1 # largefile processing might be slow and be interrupted - be prepared lfdirstate.write() if lfiles: if printmessage: ui.status(_('getting changed largefiles\n')) cachelfiles(ui, repo, None, lfiles) for lfile in lfiles: update1 = 0 expecthash = update.get(lfile) if expecthash: if not lfutil.copyfromcache(repo, expecthash, lfile): # failed ... but already removed and set to normallookup continue # Synchronize largefile dirstate to the last modified # time of the file lfdirstate.normal(lfile) update1 = 1 # copy the state of largefile standin from the repository's # dirstate to its state in the lfdirstate. abslfile = repo.wjoin(lfile) absstandin = repo.wjoin(lfutil.standin(lfile)) if os.path.exists(absstandin): mode = os.stat(absstandin).st_mode if mode != os.stat(abslfile).st_mode: os.chmod(abslfile, mode) update1 = 1 updated += update1 lfutil.synclfdirstate(repo, lfdirstate, lfile, normallookup) if filelist is not None: # If "local largefile" is chosen at file merging, it is # not listed in "filelist" (= dirstate syncing is # omitted), because the standin file is not changed before and # after merging. # But the status of such files may have to be changed by # merging. For example, locally modified ("M") largefile # has to become re-added("A"), if it is "normal" file in # the target revision of linear-merging. for lfile in lfdirstate: if lfile not in filelist: lfutil.synclfdirstate(repo, lfdirstate, lfile, True) lfdirstate.write() if printmessage and lfiles: ui.status( _('%d largefiles updated, %d removed\n') % (updated, removed)) finally: wlock.release()
def debugdirstate(ui, repo): '''Show basic information for the largefiles dirstate''' lfdirstate = lfutil.openlfdirstate(ui, repo) for file_, ent in sorted(lfdirstate._map.iteritems()): mode = '%3o' % (ent[1] & 0777 & ~util.umask) ui.write("%c %s %10d %s\n" % (ent[0], mode, ent[2], file_))
def status(self, node1='.', node2=None, match=None, ignored=False, clean=False, unknown=False, listsubrepos=False): listignored, listclean, listunknown = ignored, clean, unknown if not self.lfstatus: return super(lfilesrepo, self).status(node1, node2, match, listignored, listclean, listunknown, listsubrepos) else: # some calls in this function rely on the old version of status self.lfstatus = False if isinstance(node1, context.changectx): ctx1 = node1 else: ctx1 = repo[node1] if isinstance(node2, context.changectx): ctx2 = node2 else: ctx2 = repo[node2] working = ctx2.rev() is None parentworking = working and ctx1 == self['.'] def inctx(file, ctx): try: if ctx.rev() is None: return file in ctx.manifest() ctx[file] return True except KeyError: return False if match is None: match = match_.always(self.root, self.getcwd()) # First check if there were files specified on the # command line. If there were, and none of them were # largefiles, we should just bail here and let super # handle it -- thus gaining a big performance boost. lfdirstate = lfutil.openlfdirstate(ui, self) if match.files() and not match.anypats(): for f in lfdirstate: if match(f): break else: return super(lfilesrepo, self).status(node1, node2, match, listignored, listclean, listunknown, listsubrepos) # Create a copy of match that matches standins instead # of largefiles. def tostandins(files): if not working: return files newfiles = [] dirstate = repo.dirstate for f in files: sf = lfutil.standin(f) if sf in dirstate: newfiles.append(sf) elif sf in dirstate.dirs(): # Directory entries could be regular or # standin, check both newfiles.extend((f, sf)) else: newfiles.append(f) return newfiles # Create a function that we can use to override what is # normally the ignore matcher. We've already checked # for ignored files on the first dirstate walk, and # unecessarily re-checking here causes a huge performance # hit because lfdirstate only knows about largefiles def _ignoreoverride(self): return False m = copy.copy(match) m._files = tostandins(m._files) # Get ignored files here even if we weren't asked for them; we # must use the result here for filtering later result = super(lfilesrepo, self).status(node1, node2, m, True, clean, unknown, listsubrepos) if working: try: # Any non-largefiles that were explicitly listed must be # taken out or lfdirstate.status will report an error. # The status of these files was already computed using # super's status. # Override lfdirstate's ignore matcher to not do # anything origignore = lfdirstate._ignore lfdirstate._ignore = _ignoreoverride def sfindirstate(f): sf = lfutil.standin(f) dirstate = repo.dirstate return sf in dirstate or sf in dirstate.dirs() match._files = [f for f in match._files if sfindirstate(f)] # Don't waste time getting the ignored and unknown # files again; we already have them s = lfdirstate.status(match, [], False, listclean, False) (unsure, modified, added, removed, missing, unknown, ignored, clean) = s # Replace the list of ignored and unknown files with # the previously caclulated lists, and strip out the # largefiles lfiles = set(lfdirstate._map) ignored = set(result[5]).difference(lfiles) unknown = set(result[4]).difference(lfiles) if parentworking: for lfile in unsure: standin = lfutil.standin(lfile) if standin not in ctx1: # from second parent modified.append(lfile) elif ctx1[standin].data().strip() \ != lfutil.hashfile(self.wjoin(lfile)): modified.append(lfile) else: clean.append(lfile) lfdirstate.normal(lfile) else: tocheck = unsure + modified + added + clean modified, added, clean = [], [], [] for lfile in tocheck: standin = lfutil.standin(lfile) if inctx(standin, ctx1): if ctx1[standin].data().strip() != \ lfutil.hashfile(self.wjoin(lfile)): modified.append(lfile) else: clean.append(lfile) else: added.append(lfile) finally: # Replace the original ignore function lfdirstate._ignore = origignore for standin in ctx1.manifest(): if not lfutil.isstandin(standin): continue lfile = lfutil.splitstandin(standin) if not match(lfile): continue if lfile not in lfdirstate: removed.append(lfile) # Filter result lists result = list(result) # Largefiles are not really removed when they're # still in the normal dirstate. Likewise, normal # files are not really removed if it's still in # lfdirstate. This happens in merges where files # change type. removed = [f for f in removed if f not in repo.dirstate] result[2] = [f for f in result[2] if f not in lfdirstate] # Unknown files unknown = set(unknown).difference(ignored) result[4] = [f for f in unknown if (repo.dirstate[f] == '?' and not lfutil.isstandin(f))] # Ignored files were calculated earlier by the dirstate, # and we already stripped out the largefiles from the list result[5] = ignored # combine normal files and largefiles normals = [[fn for fn in filelist if not lfutil.isstandin(fn)] for filelist in result] lfiles = (modified, added, removed, missing, [], [], clean) result = [sorted(list1 + list2) for (list1, list2) in zip(normals, lfiles)] else: def toname(f): if lfutil.isstandin(f): return lfutil.splitstandin(f) return f result = [[toname(f) for f in items] for items in result] if not listunknown: result[4] = [] if not listignored: result[5] = [] if not listclean: result[6] = [] self.lfstatus = True return result
def addlargefiles(ui, repo, *pats, **opts): large = opts.pop('large', None) lfsize = lfutil.getminsize(ui, lfutil.islfilesrepo(repo), opts.pop('lfsize', None)) lfmatcher = None if lfutil.islfilesrepo(repo): lfpats = ui.configlist(lfutil.longname, 'patterns', default=[]) if lfpats: lfmatcher = match_.match(repo.root, '', list(lfpats)) lfnames = [] m = scmutil.match(repo[None], pats, opts) m.bad = lambda x, y: None wctx = repo[None] for f in repo.walk(m): exact = m.exact(f) lfile = lfutil.standin(f) in wctx nfile = f in wctx exists = lfile or nfile # Don't warn the user when they attempt to add a normal tracked file. # The normal add code will do that for us. if exact and exists: if lfile: ui.warn(_('%s already a largefile\n') % f) continue if (exact or not exists) and not lfutil.isstandin(f): wfile = repo.wjoin(f) # In case the file was removed previously, but not committed # (issue3507) if not os.path.exists(wfile): continue abovemin = (lfsize and os.lstat(wfile).st_size >= lfsize * 1024 * 1024) if large or abovemin or (lfmatcher and lfmatcher(f)): lfnames.append(f) if ui.verbose or not exact: ui.status(_('adding %s as a largefile\n') % m.rel(f)) bad = [] standins = [] # Need to lock, otherwise there could be a race condition between # when standins are created and added to the repo. wlock = repo.wlock() try: if not opts.get('dry_run'): lfdirstate = lfutil.openlfdirstate(ui, repo) for f in lfnames: standinname = lfutil.standin(f) lfutil.writestandin(repo, standinname, hash='', executable=lfutil.getexecutable( repo.wjoin(f))) standins.append(standinname) if lfdirstate[f] == 'r': lfdirstate.normallookup(f) else: lfdirstate.add(f) lfdirstate.write() bad += [ lfutil.splitstandin(f) for f in repo[None].add(standins) if f in m.files() ] finally: wlock.release() return bad
def status( self, node1=".", node2=None, match=None, ignored=False, clean=False, unknown=False, listsubrepos=False ): listignored, listclean, listunknown = ignored, clean, unknown if not self.lfstatus: return super(lfilesrepo, self).status( node1, node2, match, listignored, listclean, listunknown, listsubrepos ) else: # some calls in this function rely on the old version of status self.lfstatus = False if isinstance(node1, context.changectx): ctx1 = node1 else: ctx1 = repo[node1] if isinstance(node2, context.changectx): ctx2 = node2 else: ctx2 = repo[node2] working = ctx2.rev() is None parentworking = working and ctx1 == self["."] def inctx(file, ctx): try: if ctx.rev() is None: return file in ctx.manifest() ctx[file] return True except KeyError: return False if match is None: match = match_.always(self.root, self.getcwd()) # First check if there were files specified on the # command line. If there were, and none of them were # largefiles, we should just bail here and let super # handle it -- thus gaining a big performance boost. lfdirstate = lfutil.openlfdirstate(ui, self) if match.files() and not match.anypats(): for f in lfdirstate: if match(f): break else: return super(lfilesrepo, self).status( node1, node2, match, listignored, listclean, listunknown, listsubrepos ) # Create a copy of match that matches standins instead # of largefiles. def tostandins(files): if not working: return files newfiles = [] dirstate = repo.dirstate for f in files: sf = lfutil.standin(f) if sf in dirstate: newfiles.append(sf) elif sf in dirstate.dirs(): # Directory entries could be regular or # standin, check both newfiles.extend((f, sf)) else: newfiles.append(f) return newfiles # Create a function that we can use to override what is # normally the ignore matcher. We've already checked # for ignored files on the first dirstate walk, and # unecessarily re-checking here causes a huge performance # hit because lfdirstate only knows about largefiles def _ignoreoverride(self): return False m = copy.copy(match) m._files = tostandins(m._files) # Get ignored files here even if we weren't asked for them; we # must use the result here for filtering later result = super(lfilesrepo, self).status(node1, node2, m, True, clean, unknown, listsubrepos) if working: try: # Any non-largefiles that were explicitly listed must be # taken out or lfdirstate.status will report an error. # The status of these files was already computed using # super's status. # Override lfdirstate's ignore matcher to not do # anything origignore = lfdirstate._ignore lfdirstate._ignore = _ignoreoverride def sfindirstate(f): sf = lfutil.standin(f) dirstate = repo.dirstate return sf in dirstate or sf in dirstate.dirs() match._files = [f for f in match._files if sfindirstate(f)] # Don't waste time getting the ignored and unknown # files again; we already have them s = lfdirstate.status(match, [], False, listclean, False) (unsure, modified, added, removed, missing, unknown, ignored, clean) = s # Replace the list of ignored and unknown files with # the previously caclulated lists, and strip out the # largefiles lfiles = set(lfdirstate._map) ignored = set(result[5]).difference(lfiles) unknown = set(result[4]).difference(lfiles) if parentworking: for lfile in unsure: standin = lfutil.standin(lfile) if standin not in ctx1: # from second parent modified.append(lfile) elif ctx1[standin].data().strip() != lfutil.hashfile(self.wjoin(lfile)): modified.append(lfile) else: clean.append(lfile) lfdirstate.normal(lfile) else: tocheck = unsure + modified + added + clean modified, added, clean = [], [], [] for lfile in tocheck: standin = lfutil.standin(lfile) if inctx(standin, ctx1): if ctx1[standin].data().strip() != lfutil.hashfile(self.wjoin(lfile)): modified.append(lfile) else: clean.append(lfile) else: added.append(lfile) finally: # Replace the original ignore function lfdirstate._ignore = origignore for standin in ctx1.manifest(): if not lfutil.isstandin(standin): continue lfile = lfutil.splitstandin(standin) if not match(lfile): continue if lfile not in lfdirstate: removed.append(lfile) # Filter result lists result = list(result) # Largefiles are not really removed when they're # still in the normal dirstate. Likewise, normal # files are not really removed if it's still in # lfdirstate. This happens in merges where files # change type. removed = [f for f in removed if f not in repo.dirstate] result[2] = [f for f in result[2] if f not in lfdirstate] # Unknown files unknown = set(unknown).difference(ignored) result[4] = [f for f in unknown if (repo.dirstate[f] == "?" and not lfutil.isstandin(f))] # Ignored files were calculated earlier by the dirstate, # and we already stripped out the largefiles from the list result[5] = ignored # combine normal files and largefiles normals = [[fn for fn in filelist if not lfutil.isstandin(fn)] for filelist in result] lfiles = (modified, added, removed, missing, [], [], clean) result = [sorted(list1 + list2) for (list1, list2) in zip(normals, lfiles)] else: def toname(f): if lfutil.isstandin(f): return lfutil.splitstandin(f) return f result = [[toname(f) for f in items] for items in result] if not listunknown: result[4] = [] if not listignored: result[5] = [] if not listclean: result[6] = [] self.lfstatus = True return result
def status(self, node1='.', node2=None, match=None, ignored=False, clean=False, unknown=False, listsubrepos=False): listignored, listclean, listunknown = ignored, clean, unknown orig = super(lfilesrepo, self).status if not self.lfstatus: return orig(node1, node2, match, listignored, listclean, listunknown, listsubrepos) # some calls in this function rely on the old version of status self.lfstatus = False ctx1 = self[node1] ctx2 = self[node2] working = ctx2.rev() is None parentworking = working and ctx1 == self['.'] if match is None: match = match_.always(self.root, self.getcwd()) wlock = None try: try: # updating the dirstate is optional # so we don't wait on the lock wlock = self.wlock(False) except error.LockError: pass # First check if paths or patterns were specified on the # command line. If there were, and they don't match any # largefiles, we should just bail here and let super # handle it -- thus gaining a big performance boost. lfdirstate = lfutil.openlfdirstate(ui, self) if not match.always(): for f in lfdirstate: if match(f): break else: return orig(node1, node2, match, listignored, listclean, listunknown, listsubrepos) # Create a copy of match that matches standins instead # of largefiles. def tostandins(files): if not working: return files newfiles = [] dirstate = self.dirstate for f in files: sf = lfutil.standin(f) if sf in dirstate: newfiles.append(sf) elif sf in dirstate.dirs(): # Directory entries could be regular or # standin, check both newfiles.extend((f, sf)) else: newfiles.append(f) return newfiles m = copy.copy(match) m._files = tostandins(m._files) result = orig(node1, node2, m, ignored, clean, unknown, listsubrepos) if working: def sfindirstate(f): sf = lfutil.standin(f) dirstate = self.dirstate return sf in dirstate or sf in dirstate.dirs() match._files = [f for f in match._files if sfindirstate(f)] # Don't waste time getting the ignored and unknown # files from lfdirstate unsure, s = lfdirstate.status(match, [], False, listclean, False) (modified, added, removed, clean) = (s.modified, s.added, s.removed, s.clean) if parentworking: for lfile in unsure: standin = lfutil.standin(lfile) if standin not in ctx1: # from second parent modified.append(lfile) elif ctx1[standin].data().strip() \ != lfutil.hashfile(self.wjoin(lfile)): modified.append(lfile) else: if listclean: clean.append(lfile) lfdirstate.normal(lfile) else: tocheck = unsure + modified + added + clean modified, added, clean = [], [], [] checkexec = self.dirstate._checkexec for lfile in tocheck: standin = lfutil.standin(lfile) if standin in ctx1: abslfile = self.wjoin(lfile) if ((ctx1[standin].data().strip() != lfutil.hashfile(abslfile)) or (checkexec and ('x' in ctx1.flags(standin)) != bool(lfutil.getexecutable(abslfile)))): modified.append(lfile) elif listclean: clean.append(lfile) else: added.append(lfile) # at this point, 'removed' contains largefiles # marked as 'R' in the working context. # then, largefiles not managed also in the target # context should be excluded from 'removed'. removed = [lfile for lfile in removed if lfutil.standin(lfile) in ctx1] # Standins no longer found in lfdirstate has been # removed for standin in ctx1.walk(lfutil.getstandinmatcher(self)): lfile = lfutil.splitstandin(standin) if not match(lfile): continue if lfile not in lfdirstate: removed.append(lfile) # Filter result lists result = list(result) # Largefiles are not really removed when they're # still in the normal dirstate. Likewise, normal # files are not really removed if they are still in # lfdirstate. This happens in merges where files # change type. removed = [f for f in removed if f not in self.dirstate] result[2] = [f for f in result[2] if f not in lfdirstate] lfiles = set(lfdirstate._map) # Unknown files result[4] = set(result[4]).difference(lfiles) # Ignored files result[5] = set(result[5]).difference(lfiles) # combine normal files and largefiles normals = [[fn for fn in filelist if not lfutil.isstandin(fn)] for filelist in result] lfstatus = (modified, added, removed, s.deleted, [], [], clean) result = [sorted(list1 + list2) for (list1, list2) in zip(normals, lfstatus)] else: # not against working directory result = [[lfutil.splitstandin(f) or f for f in items] for items in result] if wlock: lfdirstate.write() finally: if wlock: wlock.release() self.lfstatus = True return scmutil.status(*result)
def updatelfiles(ui, repo, filelist=None, printmessage=True): wlock = repo.wlock() try: lfdirstate = lfutil.openlfdirstate(ui, repo) lfiles = set(lfutil.listlfiles(repo)) | set(lfdirstate) if filelist is not None: lfiles = [f for f in lfiles if f in filelist] update = {} updated, removed = 0, 0 for lfile in lfiles: abslfile = repo.wjoin(lfile) absstandin = repo.wjoin(lfutil.standin(lfile)) if os.path.exists(absstandin): if (os.path.exists(absstandin + '.orig') and os.path.exists(abslfile)): shutil.copyfile(abslfile, abslfile + '.orig') expecthash = lfutil.readstandin(repo, lfile) if (expecthash != '' and (not os.path.exists(abslfile) or expecthash != lfutil.hashfile(abslfile))): if lfile not in repo[None]: # not switched to normal file util.unlinkpath(abslfile, ignoremissing=True) # use normallookup() to allocate entry in largefiles # dirstate, because lack of it misleads # lfilesrepo.status() into recognition that such cache # missing files are REMOVED. lfdirstate.normallookup(lfile) update[lfile] = expecthash else: # Remove lfiles for which the standin is deleted, unless the # lfile is added to the repository again. This happens when a # largefile is converted back to a normal file: the standin # disappears, but a new (normal) file appears as the lfile. if (os.path.exists(abslfile) and repo.dirstate.normalize(lfile) not in repo[None]): util.unlinkpath(abslfile) removed += 1 # largefile processing might be slow and be interrupted - be prepared lfdirstate.write() if lfiles: if printmessage: ui.status(_('getting changed largefiles\n')) cachelfiles(ui, repo, None, lfiles) for lfile in lfiles: update1 = 0 expecthash = update.get(lfile) if expecthash: if not lfutil.copyfromcache(repo, expecthash, lfile): # failed ... but already removed and set to normallookup continue # Synchronize largefile dirstate to the last modified # time of the file lfdirstate.normal(lfile) update1 = 1 # copy the state of largefile standin from the repository's # dirstate to its state in the lfdirstate. abslfile = repo.wjoin(lfile) absstandin = repo.wjoin(lfutil.standin(lfile)) if os.path.exists(absstandin): mode = os.stat(absstandin).st_mode if mode != os.stat(abslfile).st_mode: os.chmod(abslfile, mode) update1 = 1 updated += update1 state = repo.dirstate[lfutil.standin(lfile)] if state == 'n': # When rebasing, we need to synchronize the standin and the # largefile, because otherwise the largefile will get reverted. # But for commit's sake, we have to mark the file as unclean. if getattr(repo, "_isrebasing", False): lfdirstate.normallookup(lfile) else: lfdirstate.normal(lfile) elif state == 'r': lfdirstate.remove(lfile) elif state == 'a': lfdirstate.add(lfile) elif state == '?': lfdirstate.drop(lfile) lfdirstate.write() if printmessage and lfiles: ui.status(_('%d largefiles updated, %d removed\n') % (updated, removed)) finally: wlock.release()
def overridecopy(orig, ui, repo, pats, opts, rename=False): # doesn't remove largefile on rename if len(pats) < 2: # this isn't legal, let the original function deal with it return orig(ui, repo, pats, opts, rename) def makestandin(relpath): path = scmutil.canonpath(repo.root, repo.getcwd(), relpath) return os.path.join(repo.wjoin(lfutil.standin(path))) fullpats = scmutil.expandpats(pats) dest = fullpats[-1] if os.path.isdir(dest): if not os.path.isdir(makestandin(dest)): os.makedirs(makestandin(dest)) # This could copy both lfiles and normal files in one command, # but we don't want to do that. First replace their matcher to # only match normal files and run it, then replace it to just # match largefiles and run it again. nonormalfiles = False nolfiles = False try: try: installnormalfilesmatchfn(repo[None].manifest()) result = orig(ui, repo, pats, opts, rename) except util.Abort, e: if str(e) != _('no files to copy'): raise e else: nonormalfiles = True result = 0 finally: restorematchfn() # The first rename can cause our current working directory to be removed. # In that case there is nothing left to copy/rename so just quit. try: repo.getcwd() except OSError: return result try: try: # When we call orig below it creates the standins but we don't add # them to the dir state until later so lock during that time. wlock = repo.wlock() manifest = repo[None].manifest() oldmatch = None # for the closure def overridematch(ctx, pats=[], opts={}, globbed=False, default='relpath'): newpats = [] # The patterns were previously mangled to add the standin # directory; we need to remove that now for pat in pats: if match_.patkind(pat) is None and lfutil.shortname in pat: newpats.append(pat.replace(lfutil.shortname, '')) else: newpats.append(pat) match = oldmatch(ctx, newpats, opts, globbed, default) m = copy.copy(match) lfile = lambda f: lfutil.standin(f) in manifest m._files = [lfutil.standin(f) for f in m._files if lfile(f)] m._fmap = set(m._files) m._always = False origmatchfn = m.matchfn m.matchfn = lambda f: (lfutil.isstandin(f) and (f in manifest) and origmatchfn( lfutil.splitstandin(f)) or None) return m oldmatch = installmatchfn(overridematch) listpats = [] for pat in pats: if match_.patkind(pat) is not None: listpats.append(pat) else: listpats.append(makestandin(pat)) try: origcopyfile = util.copyfile copiedfiles = [] def overridecopyfile(src, dest): if (lfutil.shortname in src and dest.startswith(repo.wjoin(lfutil.shortname))): destlfile = dest.replace(lfutil.shortname, '') if not opts['force'] and os.path.exists(destlfile): raise IOError( '', _('destination largefile already exists')) copiedfiles.append((src, dest)) origcopyfile(src, dest) util.copyfile = overridecopyfile result += orig(ui, repo, listpats, opts, rename) finally: util.copyfile = origcopyfile lfdirstate = lfutil.openlfdirstate(ui, repo) for (src, dest) in copiedfiles: if (lfutil.shortname in src and dest.startswith(repo.wjoin(lfutil.shortname))): srclfile = src.replace(repo.wjoin(lfutil.standin('')), '') destlfile = dest.replace(repo.wjoin(lfutil.standin('')), '') destlfiledir = os.path.dirname( repo.wjoin(destlfile)) or '.' if not os.path.isdir(destlfiledir): os.makedirs(destlfiledir) if rename: os.rename(repo.wjoin(srclfile), repo.wjoin(destlfile)) lfdirstate.remove(srclfile) else: util.copyfile(repo.wjoin(srclfile), repo.wjoin(destlfile)) lfdirstate.add(destlfile) lfdirstate.write() except util.Abort, e: if str(e) != _('no files to copy'): raise e else: nolfiles = True finally: restorematchfn() wlock.release() if nolfiles and nonormalfiles: raise util.Abort(_('no files to copy')) return result
def status(self, node1='.', node2=None, match=None, ignored=False, clean=False, unknown=False, listsubrepos=False): listignored, listclean, listunknown = ignored, clean, unknown orig = super(lfilesrepo, self).status if not self.lfstatus: return orig(node1, node2, match, listignored, listclean, listunknown, listsubrepos) # some calls in this function rely on the old version of status self.lfstatus = False ctx1 = self[node1] ctx2 = self[node2] working = ctx2.rev() is None parentworking = working and ctx1 == self['.'] if match is None: match = match_.always(self.root, self.getcwd()) wlock = None try: try: # updating the dirstate is optional # so we don't wait on the lock wlock = self.wlock(False) except error.LockError: pass # First check if paths or patterns were specified on the # command line. If there were, and they don't match any # largefiles, we should just bail here and let super # handle it -- thus gaining a big performance boost. lfdirstate = lfutil.openlfdirstate(ui, self) if not match.always(): for f in lfdirstate: if match(f): break else: return orig(node1, node2, match, listignored, listclean, listunknown, listsubrepos) # Create a copy of match that matches standins instead # of largefiles. def tostandins(files): if not working: return files newfiles = [] dirstate = self.dirstate for f in files: sf = lfutil.standin(f) if sf in dirstate: newfiles.append(sf) elif sf in dirstate.dirs(): # Directory entries could be regular or # standin, check both newfiles.extend((f, sf)) else: newfiles.append(f) return newfiles m = copy.copy(match) m._files = tostandins(m._files) result = orig(node1, node2, m, ignored, clean, unknown, listsubrepos) if working: def sfindirstate(f): sf = lfutil.standin(f) dirstate = self.dirstate return sf in dirstate or sf in dirstate.dirs() match._files = [f for f in match._files if sfindirstate(f)] # Don't waste time getting the ignored and unknown # files from lfdirstate unsure, s = lfdirstate.status(match, [], False, listclean, False) (modified, added, removed, clean) = (s.modified, s.added, s.removed, s.clean) if parentworking: for lfile in unsure: standin = lfutil.standin(lfile) if standin not in ctx1: # from second parent modified.append(lfile) elif ctx1[standin].data().strip() \ != lfutil.hashfile(self.wjoin(lfile)): modified.append(lfile) else: if listclean: clean.append(lfile) lfdirstate.normal(lfile) else: tocheck = unsure + modified + added + clean modified, added, clean = [], [], [] checkexec = self.dirstate._checkexec for lfile in tocheck: standin = lfutil.standin(lfile) if standin in ctx1: abslfile = self.wjoin(lfile) if ((ctx1[standin].data().strip() != lfutil.hashfile(abslfile)) or (checkexec and ('x' in ctx1.flags(standin)) != bool(lfutil.getexecutable(abslfile)))): modified.append(lfile) elif listclean: clean.append(lfile) else: added.append(lfile) # at this point, 'removed' contains largefiles # marked as 'R' in the working context. # then, largefiles not managed also in the target # context should be excluded from 'removed'. removed = [ lfile for lfile in removed if lfutil.standin(lfile) in ctx1 ] # Standins no longer found in lfdirstate has been # removed for standin in ctx1.walk(lfutil.getstandinmatcher(self)): lfile = lfutil.splitstandin(standin) if not match(lfile): continue if lfile not in lfdirstate: removed.append(lfile) # Filter result lists result = list(result) # Largefiles are not really removed when they're # still in the normal dirstate. Likewise, normal # files are not really removed if they are still in # lfdirstate. This happens in merges where files # change type. removed = [f for f in removed if f not in self.dirstate] result[2] = [f for f in result[2] if f not in lfdirstate] lfiles = set(lfdirstate._map) # Unknown files result[4] = set(result[4]).difference(lfiles) # Ignored files result[5] = set(result[5]).difference(lfiles) # combine normal files and largefiles normals = [[ fn for fn in filelist if not lfutil.isstandin(fn) ] for filelist in result] lfstatus = (modified, added, removed, s.deleted, [], [], clean) result = [ sorted(list1 + list2) for (list1, list2) in zip(normals, lfstatus) ] else: # not against working directory result = [[lfutil.splitstandin(f) or f for f in items] for items in result] if wlock: lfdirstate.write() finally: if wlock: wlock.release() self.lfstatus = True return scmutil.status(*result)
def updatelfiles(ui, repo, filelist=None, printmessage=True, normallookup=False): wlock = repo.wlock() try: lfdirstate = lfutil.openlfdirstate(ui, repo) lfiles = set(lfutil.listlfiles(repo)) | set(lfdirstate) if filelist is not None: lfiles = [f for f in lfiles if f in filelist] update = {} updated, removed = 0, 0 for lfile in lfiles: abslfile = repo.wjoin(lfile) absstandin = repo.wjoin(lfutil.standin(lfile)) if os.path.exists(absstandin): if (os.path.exists(absstandin + '.orig') and os.path.exists(abslfile)): shutil.copyfile(abslfile, abslfile + '.orig') util.unlinkpath(absstandin + '.orig') expecthash = lfutil.readstandin(repo, lfile) if (expecthash != '' and (not os.path.exists(abslfile) or expecthash != lfutil.hashfile(abslfile))): if lfile not in repo[None]: # not switched to normal file util.unlinkpath(abslfile, ignoremissing=True) # use normallookup() to allocate entry in largefiles # dirstate, because lack of it misleads # lfilesrepo.status() into recognition that such cache # missing files are REMOVED. lfdirstate.normallookup(lfile) update[lfile] = expecthash else: # Remove lfiles for which the standin is deleted, unless the # lfile is added to the repository again. This happens when a # largefile is converted back to a normal file: the standin # disappears, but a new (normal) file appears as the lfile. if (os.path.exists(abslfile) and repo.dirstate.normalize(lfile) not in repo[None]): util.unlinkpath(abslfile) removed += 1 # largefile processing might be slow and be interrupted - be prepared lfdirstate.write() if lfiles: if printmessage: ui.status(_('getting changed largefiles\n')) cachelfiles(ui, repo, None, lfiles) for lfile in lfiles: update1 = 0 expecthash = update.get(lfile) if expecthash: if not lfutil.copyfromcache(repo, expecthash, lfile): # failed ... but already removed and set to normallookup continue # Synchronize largefile dirstate to the last modified # time of the file lfdirstate.normal(lfile) update1 = 1 # copy the state of largefile standin from the repository's # dirstate to its state in the lfdirstate. abslfile = repo.wjoin(lfile) absstandin = repo.wjoin(lfutil.standin(lfile)) if os.path.exists(absstandin): mode = os.stat(absstandin).st_mode if mode != os.stat(abslfile).st_mode: os.chmod(abslfile, mode) update1 = 1 updated += update1 standin = lfutil.standin(lfile) if standin in repo.dirstate: stat = repo.dirstate._map[standin] state, mtime = stat[0], stat[3] else: state, mtime = '?', -1 if state == 'n': if normallookup or mtime < 0: # state 'n' doesn't ensure 'clean' in this case lfdirstate.normallookup(lfile) else: lfdirstate.normal(lfile) elif state == 'm': lfdirstate.normallookup(lfile) elif state == 'r': lfdirstate.remove(lfile) elif state == 'a': lfdirstate.add(lfile) elif state == '?': lfdirstate.drop(lfile) lfdirstate.write() if printmessage and lfiles: ui.status( _('%d largefiles updated, %d removed\n') % (updated, removed)) finally: wlock.release()
def status(self, node1='.', node2=None, match=None, ignored=False, clean=False, unknown=False, listsubrepos=False): listignored, listclean, listunknown = ignored, clean, unknown if not self.lfstatus: return super(lfilesrepo, self).status(node1, node2, match, listignored, listclean, listunknown, listsubrepos) else: # some calls in this function rely on the old version of status self.lfstatus = False if isinstance(node1, context.changectx): ctx1 = node1 else: ctx1 = self[node1] if isinstance(node2, context.changectx): ctx2 = node2 else: ctx2 = self[node2] working = ctx2.rev() is None parentworking = working and ctx1 == self['.'] def inctx(file, ctx): try: if ctx.rev() is None: return file in ctx.manifest() ctx[file] return True except KeyError: return False if match is None: match = match_.always(self.root, self.getcwd()) wlock = None try: try: # updating the dirstate is optional # so we don't wait on the lock wlock = self.wlock(False) except error.LockError: pass # First check if there were files specified on the # command line. If there were, and none of them were # largefiles, we should just bail here and let super # handle it -- thus gaining a big performance boost. lfdirstate = lfutil.openlfdirstate(ui, self) if match.files() and not match.anypats(): for f in lfdirstate: if match(f): break else: return super(lfilesrepo, self).status(node1, node2, match, listignored, listclean, listunknown, listsubrepos) # Create a copy of match that matches standins instead # of largefiles. def tostandins(files): if not working: return files newfiles = [] dirstate = self.dirstate for f in files: sf = lfutil.standin(f) if sf in dirstate: newfiles.append(sf) elif sf in dirstate.dirs(): # Directory entries could be regular or # standin, check both newfiles.extend((f, sf)) else: newfiles.append(f) return newfiles m = copy.copy(match) m._files = tostandins(m._files) result = super(lfilesrepo, self).status(node1, node2, m, ignored, clean, unknown, listsubrepos) if working: def sfindirstate(f): sf = lfutil.standin(f) dirstate = self.dirstate return sf in dirstate or sf in dirstate.dirs() match._files = [f for f in match._files if sfindirstate(f)] # Don't waste time getting the ignored and unknown # files from lfdirstate s = lfdirstate.status(match, [], False, listclean, False) (unsure, modified, added, removed, missing, _unknown, _ignored, clean) = s if parentworking: for lfile in unsure: standin = lfutil.standin(lfile) if standin not in ctx1: # from second parent modified.append(lfile) elif ctx1[standin].data().strip() \ != lfutil.hashfile(self.wjoin(lfile)): modified.append(lfile) else: clean.append(lfile) lfdirstate.normal(lfile) else: tocheck = unsure + modified + added + clean modified, added, clean = [], [], [] for lfile in tocheck: standin = lfutil.standin(lfile) if inctx(standin, ctx1): if ctx1[standin].data().strip() != \ lfutil.hashfile(self.wjoin(lfile)): modified.append(lfile) else: clean.append(lfile) else: added.append(lfile) # Standins no longer found in lfdirstate has been # removed for standin in ctx1.manifest(): if not lfutil.isstandin(standin): continue lfile = lfutil.splitstandin(standin) if not match(lfile): continue if lfile not in lfdirstate: removed.append(lfile) # Filter result lists result = list(result) # Largefiles are not really removed when they're # still in the normal dirstate. Likewise, normal # files are not really removed if they are still in # lfdirstate. This happens in merges where files # change type. removed = [f for f in removed if f not in self.dirstate] result[2] = [f for f in result[2] if f not in lfdirstate] lfiles = set(lfdirstate._map) # Unknown files result[4] = set(result[4]).difference(lfiles) # Ignored files result[5] = set(result[5]).difference(lfiles) # combine normal files and largefiles normals = [[fn for fn in filelist if not lfutil.isstandin(fn)] for filelist in result] lfiles = (modified, added, removed, missing, [], [], clean) result = [sorted(list1 + list2) for (list1, list2) in zip(normals, lfiles)] else: def toname(f): if lfutil.isstandin(f): return lfutil.splitstandin(f) return f result = [[toname(f) for f in items] for items in result] if wlock: lfdirstate.write() finally: if wlock: wlock.release() if not listunknown: result[4] = [] if not listignored: result[5] = [] if not listclean: result[6] = [] self.lfstatus = True return result
def commit(self, text="", user=None, date=None, match=None, force=False, editor=False, extra={}): orig = super(lfiles_repo, self).commit wlock = repo.wlock() try: if getattr(repo, "_isrebasing", False): # We have to take the time to pull down the new # largefiles now. Otherwise if we are rebasing, # any largefiles that were modified in the # destination changesets get overwritten, either # by the rebase or in the first commit after the # rebase. lfcommands.updatelfiles(repo.ui, repo) # Case 1: user calls commit with no specific files or # include/exclude patterns: refresh and commit all files that # are "dirty". if ((match is None) or (not match.anypats() and not match.files())): # Spend a bit of time here to get a list of files we know # are modified so we can compare only against those. # It can cost a lot of time (several seconds) # otherwise to update all standins if the largefiles are # large. lfdirstate = lfutil.openlfdirstate(ui, self) dirtymatch = match_.always(repo.root, repo.getcwd()) s = lfdirstate.status(dirtymatch, [], False, False, False) modifiedfiles = [] for i in s: modifiedfiles.extend(i) lfiles = lfutil.listlfiles(self) # this only loops through largefiles that exist (not # removed/renamed) for lfile in lfiles: if lfile in modifiedfiles: if os.path.exists(self.wjoin(lfutil.standin(lfile))): # this handles the case where a rebase is being # performed and the working copy is not updated # yet. if os.path.exists(self.wjoin(lfile)): lfutil.updatestandin(self, lfutil.standin(lfile)) lfdirstate.normal(lfile) for lfile in lfdirstate: if lfile in modifiedfiles: if not os.path.exists( repo.wjoin(lfutil.standin(lfile))): lfdirstate.drop(lfile) lfdirstate.write() return orig(text=text, user=user, date=date, match=match, force=force, editor=editor, extra=extra) for f in match.files(): if lfutil.isstandin(f): raise util.Abort( _('file "%s" is a largefile standin') % f, hint=('commit the largefile itself instead')) # Case 2: user calls commit with specified patterns: refresh # any matching big files. smatcher = lfutil.composestandinmatcher(self, match) standins = lfutil.dirstate_walk(self.dirstate, smatcher) # No matching big files: get out of the way and pass control to # the usual commit() method. if not standins: return orig(text=text, user=user, date=date, match=match, force=force, editor=editor, extra=extra) # Refresh all matching big files. It's possible that the # commit will end up failing, in which case the big files will # stay refreshed. No harm done: the user modified them and # asked to commit them, so sooner or later we're going to # refresh the standins. Might as well leave them refreshed. lfdirstate = lfutil.openlfdirstate(ui, self) for standin in standins: lfile = lfutil.splitstandin(standin) if lfdirstate[lfile] <> 'r': lfutil.updatestandin(self, standin) lfdirstate.normal(lfile) else: lfdirstate.drop(lfile) lfdirstate.write() # Cook up a new matcher that only matches regular files or # standins corresponding to the big files requested by the # user. Have to modify _files to prevent commit() from # complaining "not tracked" for big files. lfiles = lfutil.listlfiles(repo) match = copy.copy(match) orig_matchfn = match.matchfn # Check both the list of largefiles and the list of # standins because if a largefile was removed, it # won't be in the list of largefiles at this point match._files += sorted(standins) actualfiles = [] for f in match._files: fstandin = lfutil.standin(f) # ignore known largefiles and standins if f in lfiles or fstandin in standins: continue # append directory separator to avoid collisions if not fstandin.endswith(os.sep): fstandin += os.sep # prevalidate matching standin directories if util.any(st for st in match._files if st.startswith(fstandin)): continue actualfiles.append(f) match._files = actualfiles def matchfn(f): if orig_matchfn(f): return f not in lfiles else: return f in standins match.matchfn = matchfn return orig(text=text, user=user, date=date, match=match, force=force, editor=editor, extra=extra) finally: wlock.release()