def cachelfiles(ui, repo, node, filelist=None):
'''cachelfiles ensures that all largefiles needed by the specified revision
are present in the repository's largefile cache.
returns a tuple (cached, missing). cached is the list of files downloaded
by this operation; missing is the list of files that were needed but could
not be found.'''
lfiles = lfutil.listlfiles(repo, node)
if filelist:
lfiles = set(lfiles) & set(filelist)
toget = []
for lfile in lfiles:
try:
expectedhash = repo[node][lfutil.standin(lfile)].data().strip()
except IOError as err:
if err.errno == errno.ENOENT:
continue # node must be None and standin wasn't found in wctx
raise
if not lfutil.findfile(repo, expectedhash):
toget.append((lfile, expectedhash))
if toget:
store = basestore._openstore(repo)
ret = store.get(toget)
return ret
return ([], [])
def cachelfiles(ui, repo, node, filelist=None):
'''cachelfiles ensures that all largefiles needed by the specified revision
are present in the repository's largefile cache.
returns a tuple (cached, missing). cached is the list of files downloaded
by this operation; missing is the list of files that were needed but could
not be found.'''
lfiles = lfutil.listlfiles(repo, node)
if filelist:
lfiles = set(lfiles) & set(filelist)
toget = []
for lfile in lfiles:
try:
expectedhash = repo[node][lfutil.standin(lfile)].data().strip()
except IOError as err:
if err.errno == errno.ENOENT:
continue # node must be None and standin wasn't found in wctx
raise
if not lfutil.findfile(repo, expectedhash):
toget.append((lfile, expectedhash))
if toget:
store = basestore._openstore(repo)
ret = store.get(toget)
return ret
return ([], [])
def cachelfiles(ui, repo, node):
'''cachelfiles ensures that all largefiles needed by the specified revision
are present in the repository's largefile cache.
returns a tuple (cached, missing). cached is the list of files downloaded
by this operation; missing is the list of files that were needed but could
not be found.'''
lfiles = lfutil.listlfiles(repo, node)
toget = []
for lfile in lfiles:
# If we are mid-merge, then we have to trust the standin that is in the
# working copy to have the correct hashvalue. This is because the
# original hg.merge() already updated the standin as part of the normal
# merge process -- we just have to udpate the largefile to match.
if (getattr(repo, "_ismerging", False) and
os.path.exists(repo.wjoin(lfutil.standin(lfile)))):
expectedhash = lfutil.readstandin(repo, lfile)
else:
expectedhash = repo[node][lfutil.standin(lfile)].data().strip()
# if it exists and its hash matches, it might have been locally
# modified before updating and the user chose 'local'. in this case,
# it will not be in any store, so don't look for it.
if ((not os.path.exists(repo.wjoin(lfile)) or
expectedhash != lfutil.hashfile(repo.wjoin(lfile))) and
not lfutil.findfile(repo, expectedhash)):
toget.append((lfile, expectedhash))
if toget:
store = basestore._openstore(repo)
ret = store.get(toget)
return ret
return ([], [])
def uploadlfiles(ui, rsrc, rdst, files):
'''upload largefiles to the central store'''
if not files:
return
store = basestore._openstore(rsrc, rdst, put=True)
at = 0
ui.debug("sending statlfile command for %d largefiles\n" % len(files))
retval = store.exists(files)
files = filter(lambda h: not retval[h], files)
ui.debug("%d largefiles need to be uploaded\n" % len(files))
for hash in files:
ui.progress(_('uploading largefiles'),
at,
unit='largefile',
total=len(files))
source = lfutil.findfile(rsrc, hash)
if not source:
raise util.Abort(
_('largefile %s missing from store'
' (needs to be uploaded)') % hash)
# XXX check for errors here
store.put(source, hash)
at += 1
ui.progress(_('uploading largefiles'), None)
def uploadlfiles(ui, rsrc, rdst, files):
'''upload largefiles to the central store'''
if not files:
return
store = basestore._openstore(rsrc, rdst, put=True)
at = 0
ui.debug("sending statlfile command for %d largefiles\n" % len(files))
retval = store.exists(files)
files = filter(lambda h: not retval[h], files)
ui.debug("%d largefiles need to be uploaded\n" % len(files))
for hash in files:
ui.progress(_('uploading largefiles'), at, unit='largefile',
total=len(files))
source = lfutil.findfile(rsrc, hash)
if not source:
raise util.Abort(_('largefile %s missing from store'
' (needs to be uploaded)') % hash)
# XXX check for errors here
store.put(source, hash)
at += 1
ui.progress(_('uploading largefiles'), None)
def cachelfiles(ui, repo, node):
"""cachelfiles ensures that all largefiles needed by the specified revision
are present in the repository's largefile cache.
returns a tuple (cached, missing). cached is the list of files downloaded
by this operation; missing is the list of files that were needed but could
not be found."""
lfiles = lfutil.listlfiles(repo, node)
toget = []
for lfile in lfiles:
# If we are mid-merge, then we have to trust the standin that is in the
# working copy to have the correct hashvalue. This is because the
# original hg.merge() already updated the standin as part of the normal
# merge process -- we just have to udpate the largefile to match.
if getattr(repo, "_ismerging", False) and os.path.exists(repo.wjoin(lfutil.standin(lfile))):
expectedhash = lfutil.readstandin(repo, lfile)
else:
expectedhash = repo[node][lfutil.standin(lfile)].data().strip()
# if it exists and its hash matches, it might have been locally
# modified before updating and the user chose 'local'. in this case,
# it will not be in any store, so don't look for it.
if (
not os.path.exists(repo.wjoin(lfile)) or expectedhash != lfutil.hashfile(repo.wjoin(lfile))
) and not lfutil.findfile(repo, expectedhash):
toget.append((lfile, expectedhash))
if toget:
store = basestore._openstore(repo)
ret = store.get(toget)
return ret
return ([], [])
def cachelfiles(ui, repo, node):
'''cachelfiles ensures that all largefiles needed by the specified revision
are present in the repository's largefile cache.
returns a tuple (cached, missing). cached is the list of files downloaded
by this operation; missing is the list of files that were needed but could
not be found.'''
lfiles = lfutil.listlfiles(repo, node)
toget = []
for lfile in lfiles:
expectedhash = repo[node][lfutil.standin(lfile)].data().strip()
# if it exists and its hash matches, it might have been locally
# modified before updating and the user chose 'local'. in this case,
# it will not be in any store, so don't look for it.
if ((not os.path.exists(repo.wjoin(lfile)) or
expectedhash != lfutil.hashfile(repo.wjoin(lfile))) and
not lfutil.findfile(repo, expectedhash)):
toget.append((lfile, expectedhash))
if toget:
store = basestore._openstore(repo)
ret = store.get(toget)
return ret
return ([], [])
def overridecat(orig, ui, repo, file1, *pats, **opts):
ctx = scmutil.revsingle(repo, opts.get('rev'))
err = 1
notbad = set()
m = scmutil.match(ctx, (file1, ) + pats, opts)
origmatchfn = m.matchfn
def lfmatchfn(f):
if origmatchfn(f):
return True
lf = lfutil.splitstandin(f)
if lf is None:
return False
notbad.add(lf)
return origmatchfn(lf)
m.matchfn = lfmatchfn
origbadfn = m.bad
def lfbadfn(f, msg):
if not f in notbad:
origbadfn(f, msg)
m.bad = lfbadfn
for f in ctx.walk(m):
fp = cmdutil.makefileobj(repo,
opts.get('output'),
ctx.node(),
pathname=f)
lf = lfutil.splitstandin(f)
if lf is None or origmatchfn(f):
# duplicating unreachable code from commands.cat
data = ctx[f].data()
if opts.get('decode'):
data = repo.wwritedata(f, data)
fp.write(data)
else:
hash = lfutil.readstandin(repo, lf, ctx.rev())
if not lfutil.inusercache(repo.ui, hash):
store = basestore._openstore(repo)
success, missing = store.get([(lf, hash)])
if len(success) != 1:
raise util.Abort(
_('largefile %s is not in cache and could not be '
'downloaded') % lf)
path = lfutil.usercachepath(repo.ui, hash)
fpin = open(path, "rb")
for chunk in util.filechunkiter(fpin, 128 * 1024):
fp.write(chunk)
fpin.close()
fp.close()
err = 0
return err
def verifylfiles(ui, repo, all=False, contents=False):
'''Verify that every largefile revision in the current changeset
exists in the central store. With --contents, also verify that
the contents of each local largefile file revision are correct (SHA-1 hash
matches the revision ID). With --all, check every changeset in
this repository.'''
if all:
revs = repo.revs('all()')
else:
revs = ['.']
store = basestore._openstore(repo)
return store.verify(revs, contents=contents)
def verifylfiles(ui, repo, all=False, contents=False):
'''Verify that every largefile revision in the current changeset
exists in the central store. With --contents, also verify that
the contents of each local largefile file revision are correct (SHA-1 hash
matches the revision ID). With --all, check every changeset in
this repository.'''
if all:
revs = repo.revs('all()')
else:
revs = ['.']
store = basestore._openstore(repo)
return store.verify(revs, contents=contents)
def catlfile(repo, lfile, rev, filename):
hash = lfutil.readstandin(repo, lfile, rev)
if not lfutil.inusercache(repo.ui, hash):
store = basestore._openstore(repo)
success, missing = store.get([(lfile, hash)])
if len(success) != 1:
raise util.Abort(_("largefile %s is not in cache and could not be downloaded") % lfile)
path = lfutil.usercachepath(repo.ui, hash)
fpout = cmdutil.makefileobj(repo, filename)
fpin = open(path, "rb")
fpout.write(fpin.read())
fpout.close()
fpin.close()
return 0
def verifylfiles(ui, repo, all=False, contents=False):
'''Verify that every largefile revision in the current changeset
exists in the central store. With --contents, also verify that
the contents of each local largefile file revision are correct (SHA-1 hash
matches the revision ID). With --all, check every changeset in
this repository.'''
if all:
# Pass a list to the function rather than an iterator because we know a
# list will work.
revs = range(len(repo))
else:
revs = ['.']
store = basestore._openstore(repo)
return store.verify(revs, contents=contents)
def verifylfiles(ui, repo, all=False, contents=False):
'''Verify that every largefile revision in the current changeset
exists in the central store. With --contents, also verify that
the contents of each local largefile file revision are correct (SHA-1 hash
matches the revision ID). With --all, check every changeset in
this repository.'''
if all:
# Pass a list to the function rather than an iterator because we know a
# list will work.
revs = range(len(repo))
else:
revs = ['.']
store = basestore._openstore(repo)
return store.verify(revs, contents=contents)
def catlfile(repo, lfile, rev, filename):
hash = lfutil.readstandin(repo, lfile, rev)
if not lfutil.inusercache(repo.ui, hash):
store = basestore._openstore(repo)
success, missing = store.get([(lfile, hash)])
if len(success) != 1:
raise util.Abort(
_('largefile %s is not in cache and could not be downloaded')
% lfile)
path = lfutil.usercachepath(repo.ui, hash)
fpout = cmdutil.makefileobj(repo, filename)
fpin = open(path, "rb")
fpout.write(fpin.read())
fpout.close()
fpin.close()
return 0
def overridecat(orig, ui, repo, file1, *pats, **opts):
ctx = scmutil.revsingle(repo, opts.get('rev'))
err = 1
notbad = set()
m = scmutil.match(ctx, (file1,) + pats, opts)
origmatchfn = m.matchfn
def lfmatchfn(f):
if origmatchfn(f):
return True
lf = lfutil.splitstandin(f)
if lf is None:
return False
notbad.add(lf)
return origmatchfn(lf)
m.matchfn = lfmatchfn
origbadfn = m.bad
def lfbadfn(f, msg):
if not f in notbad:
origbadfn(f, msg)
m.bad = lfbadfn
for f in ctx.walk(m):
fp = cmdutil.makefileobj(repo, opts.get('output'), ctx.node(),
pathname=f)
lf = lfutil.splitstandin(f)
if lf is None or origmatchfn(f):
# duplicating unreachable code from commands.cat
data = ctx[f].data()
if opts.get('decode'):
data = repo.wwritedata(f, data)
fp.write(data)
else:
hash = lfutil.readstandin(repo, lf, ctx.rev())
if not lfutil.inusercache(repo.ui, hash):
store = basestore._openstore(repo)
success, missing = store.get([(lf, hash)])
if len(success) != 1:
raise util.Abort(
_('largefile %s is not in cache and could not be '
'downloaded') % lf)
path = lfutil.usercachepath(repo.ui, hash)
fpin = open(path, "rb")
for chunk in util.filechunkiter(fpin, 128 * 1024):
fp.write(chunk)
fpin.close()
fp.close()
err = 0
return err
def uploadlfiles(ui, rsrc, rdst, files):
"""upload largefiles to the central store"""
if not files:
return
store = basestore._openstore(rsrc, rdst, put=True)
at = 0
files = filter(lambda h: not store.exists(h), files)
for hash in files:
ui.progress(_("uploading largefiles"), at, unit="largefile", total=len(files))
source = lfutil.findfile(rsrc, hash)
if not source:
raise util.Abort(_("largefile %s missing from store" " (needs to be uploaded)") % hash)
# XXX check for errors here
store.put(source, hash)
at += 1
ui.progress(_("uploading largefiles"), None)
def _getoutgoings(repo, other, missing, addfunc):
"""get pairs of filename and largefile hash in outgoing revisions
in 'missing'.
largefiles already existing on 'other' repository are ignored.
'addfunc' is invoked with each unique pairs of filename and
largefile hash value.
"""
knowns = set()
lfhashes = set()
def dedup(fn, lfhash):
k = (fn, lfhash)
if k not in knowns:
knowns.add(k)
lfhashes.add(lfhash)
lfutil.getlfilestoupload(repo, missing, dedup)
if lfhashes:
lfexists = basestore._openstore(repo, other).exists(lfhashes)
for fn, lfhash in knowns:
if not lfexists[lfhash]: # lfhash doesn't exist on "other"
addfunc(fn, lfhash)
def uploadbfiles(ui, rsrc, rdst, files):
'''upload big files to the central store'''
# Don't upload locally. All bfiles are in the system wide cache
# so the other repo can just get them from there.
if not files or rdst.local():
return
store = basestore._openstore(rsrc, rdst, put=True)
at = 0
for hash in files:
ui.progress(_('uploading bfiles'), at, unit='bfile', total=len(files))
if store.exists(hash):
at += 1
continue
source = bfutil.findfile(rsrc, hash)
if not source:
raise util.Abort(_('Missing bfile %s needs to be uploaded') % hash)
# XXX check for errors here
store.put(source, hash)
at += 1
ui.progress('uploading bfiles', None)
if filelist:
lfiles = set(lfiles) & set(filelist)
toget = []
for lfile in lfiles:
try:
expectedhash = repo[node][lfutil.standin(lfile)].data().strip()
except IOError, err:
if err.errno == errno.ENOENT:
continue # node must be None and standin wasn't found in wctx
raise
if not lfutil.findfile(repo, expectedhash):
toget.append((lfile, expectedhash))
if toget:
store = basestore._openstore(repo)
ret = store.get(toget)
return ret
return ([], [])
def downloadlfiles(ui, repo, rev=None):
matchfn = scmutil.match(repo[None],
[repo.wjoin(lfutil.shortname)], {})
def prepare(ctx, fns):
pass
totalsuccess = 0
totalmissing = 0
if rev != []: # walkchangerevs on empty list would return all revs
for ctx in cmdutil.walkchangerevs(repo, matchfn, {'rev' : rev},
prepare):
if filelist:
lfiles = set(lfiles) & set(filelist)
toget = []
for lfile in lfiles:
try:
expectedhash = repo[node][lfutil.standin(lfile)].data().strip()
except IOError, err:
if err.errno == errno.ENOENT:
continue # node must be None and standin wasn't found in wctx
raise
if not lfutil.findfile(repo, expectedhash):
toget.append((lfile, expectedhash))
if toget:
store = basestore._openstore(repo)
ret = store.get(toget)
return ret
return ([], [])
def downloadlfiles(ui, repo, rev=None):
matchfn = scmutil.match(repo[None], [repo.wjoin(lfutil.shortname)], {})
def prepare(ctx, fns):
pass
totalsuccess = 0
totalmissing = 0
if rev != []: # walkchangerevs on empty list would return all revs
def revertbfiles(ui, repo, filelist=None):
wlock = repo.wlock()
try:
bfdirstate = bfutil.openbfdirstate(ui, repo)
s = bfdirstate.status(match_.always(repo.root, repo.getcwd()), [],
False, False, False)
unsure, modified, added, removed, missing, unknown, ignored, clean = s
bfiles = bfutil.listbfiles(repo)
toget = []
at = 0
updated = 0
for bfile in bfiles:
if filelist is None or bfile in filelist:
if not os.path.exists(repo.wjoin(bfutil.standin(bfile))):
bfdirstate.remove(bfile)
continue
if os.path.exists(repo.wjoin(bfutil.standin(os.path.join(bfile\
+ '.orig')))):
shutil.copyfile(repo.wjoin(bfile), repo.wjoin(bfile + \
'.orig'))
at += 1
expectedhash = repo[None][bfutil.standin(bfile)].data().strip()
mode = os.stat(repo.wjoin(bfutil.standin(bfile))).st_mode
if not os.path.exists(repo.wjoin(bfile)) or expectedhash != \
bfutil.hashfile(repo.wjoin(bfile)):
path = bfutil.findfile(repo, expectedhash)
if path is None:
toget.append((bfile, expectedhash))
else:
util.makedirs(os.path.dirname(repo.wjoin(bfile)))
shutil.copy(path, repo.wjoin(bfile))
os.chmod(repo.wjoin(bfile), mode)
updated += 1
if bfutil.standin(bfile) not in repo['.']:
bfdirstate.add(bfutil.unixpath(bfile))
elif expectedhash == repo['.'][bfutil.standin(bfile)] \
.data().strip():
bfdirstate.normal(bfutil.unixpath(bfile))
else:
bfutil.dirstate_normaldirty(bfdirstate,
bfutil.unixpath(bfile))
elif os.path.exists(repo.wjoin(bfile)) and mode != \
os.stat(repo.wjoin(bfile)).st_mode:
os.chmod(repo.wjoin(bfile), mode)
updated += 1
if bfutil.standin(bfile) not in repo['.']:
bfdirstate.add(bfutil.unixpath(bfile))
elif expectedhash == \
repo['.'][bfutil.standin(bfile)].data().strip():
bfdirstate.normal(bfutil.unixpath(bfile))
else:
bfutil.dirstate_normaldirty(bfdirstate,
bfutil.unixpath(bfile))
if toget:
store = basestore._openstore(repo)
success, missing = store.get(toget)
else:
success, missing = [], []
for (filename, hash) in success:
mode = os.stat(repo.wjoin(bfutil.standin(filename))).st_mode
os.chmod(repo.wjoin(filename), mode)
updated += 1
if bfutil.standin(filename) not in repo['.']:
bfdirstate.add(bfutil.unixpath(filename))
elif hash == repo['.'][bfutil.standin(filename)].data().strip():
bfdirstate.normal(bfutil.unixpath(filename))
else:
bfutil.dirstate_normaldirty(bfdirstate,
bfutil.unixpath(filename))
removed = 0
for bfile in bfdirstate:
if filelist is None or bfile in filelist:
if not os.path.exists(repo.wjoin(bfutil.standin(bfile))):
if os.path.exists(repo.wjoin(bfile)):
os.unlink(repo.wjoin(bfile))
removed += 1
if bfutil.standin(bfile) in repo['.']:
bfdirstate.remove(bfutil.unixpath(bfile))
else:
bfdirstate.forget(bfutil.unixpath(bfile))
else:
state = repo.dirstate[bfutil.standin(bfile)]
if state == 'n':
bfdirstate.normal(bfile)
elif state == 'r':
bfdirstate.remove(bfile)
elif state == 'a':
bfdirstate.add(bfile)
elif state == '?':
try:
# Mercurial >= 1.9
bfdirstate.drop(bfile)
except AttributeError:
# Mercurial <= 1.8
bfdirstate.forget(bfile)
bfdirstate.write()
finally:
wlock.release()
def updatebfiles(ui, repo):
wlock = repo.wlock()
try:
bfdirstate = bfutil.openbfdirstate(ui, repo)
s = bfdirstate.status(match_.always(repo.root, repo.getcwd()), [],
False, False, False)
unsure, modified, added, removed, missing, unknown, ignored, clean = s
bfiles = bfutil.listbfiles(repo)
toget = []
at = 0
updated = 0
removed = 0
printed = False
if bfiles:
ui.status(_('getting changed bfiles\n'))
printed = True
for bfile in bfiles:
at += 1
if os.path.exists(repo.wjoin(bfile)) and not \
os.path.exists(repo.wjoin(bfutil.standin(bfile))):
os.unlink(repo.wjoin(bfile))
removed += 1
bfdirstate.forget(bfutil.unixpath(bfile))
continue
expectedhash = repo[None][bfutil.standin(bfile)].data().strip()
mode = os.stat(repo.wjoin(bfutil.standin(bfile))).st_mode
if not os.path.exists(repo.wjoin(bfile)) or expectedhash != \
bfutil.hashfile(repo.wjoin(bfile)):
path = bfutil.findfile(repo, expectedhash)
if not path:
toget.append((bfile, expectedhash))
else:
util.makedirs(os.path.dirname(repo.wjoin(bfile)))
shutil.copy(path, repo.wjoin(bfile))
os.chmod(repo.wjoin(bfile), mode)
updated += 1
bfdirstate.normal(bfutil.unixpath(bfile))
elif os.path.exists(repo.wjoin(bfile)) and mode != \
os.stat(repo.wjoin(bfile)).st_mode:
os.chmod(repo.wjoin(bfile), mode)
updated += 1
bfdirstate.normal(bfutil.unixpath(bfile))
if toget:
store = basestore._openstore(repo)
(success, missing) = store.get(toget)
else:
success, missing = [],[]
for (filename, hash) in success:
mode = os.stat(repo.wjoin(bfutil.standin(filename))).st_mode
os.chmod(repo.wjoin(filename), mode)
updated += 1
bfdirstate.normal(bfutil.unixpath(filename))
for bfile in bfdirstate:
if bfile not in bfiles:
if os.path.exists(repo.wjoin(bfile)):
if not printed:
ui.status(_('getting changed bfiles\n'))
printed = True
os.unlink(repo.wjoin(bfile))
removed += 1
path = bfutil.unixpath(bfile)
try:
# Mercurial >= 1.9
bfdirstate.drop(path)
except AttributeError:
# Mercurial <= 1.8
bfdirstate.forget(path)
bfdirstate.write()
if printed:
ui.status(_('%d big files updated, %d removed\n') % (updated,
removed))
finally:
wlock.release()
