def upload_bfiles(ui, rsrc, rdst, files):
'''upload big files to the central store'''
if not files:
return
# Don't upload locally. All bfiles are in the system wide cache
# so the other repo can just get them from there.
if not rdst.path.startswith('http'):
return
store = basestore._open_store(rsrc, rdst.path, put=True)
at = 0
for hash in files:
ui.progress(_('Uploading bfiles'), at, unit='bfile', total=len(files))
if store.exists(hash):
at += 1
continue
source = bfutil.find_file(rsrc, hash)
if not source:
raise util.Abort(_('Missing bfile %s needs to be uploaded') % hash)
# XXX check for errors here
store.put(source, hash)
at += 1
ui.progress('Uploading bfiles', None)
def bfput(ui, repo, *files, **opts):
'''upload big files to the central store'''
# XXX should be able to upload uncommitted revisions too! maybe need
# --committed/--uncommitted options?
store = basestore._open_store(ui, repo, put=True)
pending = bfutil._find_pending_files(
repo, files, opts, pending=False, committed=True)
numfiles = numrevs = 0
for (dirname, filename, revs) in sorted(pending):
# upload files in order by mtime, then hash, mainly to avoid
# spurious test failures -- don't want to depend on filesystem
# order
sortable = []
for rev in revs:
revfile = os.path.join(dirname, filename, rev)
mtime = os.stat(revfile).st_mtime
sortable.append((mtime, rev, revfile))
sortable.sort() # by mtime
for (mtime, rev, revfile) in sortable:
ui.note('putting %s (rev %s)\n' % (filename, rev))
path = os.path.join(dirname, filename, rev)
store.put(path, filename, rev)
bfutil.unlinkpath(path)
numrevs += 1
numfiles += 1
if numfiles == 0:
ui.status(_('no pending files to put in store\n'))
return 1
else:
ui.status(_('uploaded %d big files (%d revs total)\n')
% (numfiles, numrevs))
return 0
def upload_bfiles(ui, rsrc, rdst, files):
'''upload big files to the central store'''
if not files:
return
# Don't upload locally. All bfiles are in the system wide cache
# so the other repo can just get them from there.
if not rdst.path.startswith('http'):
return
store = basestore._open_store(rsrc, rdst.path, put=True)
at = 0
for hash in files:
ui.progress(_('Uploading bfiles'), at, unit='bfile', total=len(files))
if store.exists(hash):
at += 1
continue
source = bfutil.find_file(rsrc, hash)
if not source:
raise util.Abort(_('Missing bfile %s needs to be uploaded') % hash)
# XXX check for errors here
store.put(source, hash)
at += 1
ui.progress('Uploading bfiles', None)
def bfverify(ui, repo, **opts):
'''Verify that every big file revision in the current changeset
exists in the central store. With --contents, also verify that
the contents of each big file revision are correct (SHA-1 hash
matches the revision ID). With --all, check every changeset in
this repository.'''
if opts.get('all'):
revs = xrange(len(repo))
else:
revs = ['.']
store = basestore._open_store(ui, repo)
return store.verify(revs, contents=opts.get('contents'))
def verify_bfiles(ui, repo, all=False, contents=False):
'''Verify that every big file revision in the current changeset
exists in the central store. With --contents, also verify that
the contents of each big file revision are correct (SHA-1 hash
matches the revision ID). With --all, check every changeset in
this repository.'''
if all:
# Pass a list to the function rather than an iterator because we know a list will work.
revs = range(len(repo))
else:
revs = ['.']
store = basestore._open_store(repo)
return store.verify(revs, contents=contents)
def verify_bfiles(ui, repo, all=False, contents=False):
'''Verify that every big file revision in the current changeset
exists in the central store. With --contents, also verify that
the contents of each big file revision are correct (SHA-1 hash
matches the revision ID). With --all, check every changeset in
this repository.'''
if all:
# Pass a list to the function rather than an iterator because we know a list will work.
revs = range(len(repo))
else:
revs = ['.']
store = basestore._open_store(repo)
return store.verify(revs, contents=contents)
def update_bfiles(ui, repo):
wlock = repo.wlock()
try:
bfdirstate = bfutil.open_bfdirstate(ui, repo)
s = bfdirstate.status(match_.always(repo.root, repo.getcwd()), [],
False, False, False)
(unsure, modified, added, removed, missing, unknown, ignored,
clean) = s
bfiles = bfutil.list_bfiles(repo)
toget = []
at = 0
updated = 0
removed = 0
printed = False
if bfiles:
ui.status(_('Getting changed bfiles\n'))
printed = True
for bfile in bfiles:
at += 1
if os.path.exists(repo.wjoin(bfile)) and not os.path.exists(
repo.wjoin(bfutil.standin(bfile))):
os.unlink(repo.wjoin(bfile))
removed += 1
bfdirstate.forget(bfutil.unixpath(bfile))
continue
expectedhash = repo[None][bfutil.standin(bfile)].data().strip()
mode = os.stat(repo.wjoin(bfutil.standin(bfile))).st_mode
if not os.path.exists(
repo.wjoin(bfile)) or expectedhash != bfutil.hashfile(
repo.wjoin(bfile)):
path = bfutil.find_file(repo, expectedhash)
if not path:
toget.append((bfile, expectedhash))
else:
util.makedirs(os.path.dirname(repo.wjoin(bfile)))
shutil.copy(path, repo.wjoin(bfile))
os.chmod(repo.wjoin(bfile), mode)
updated += 1
bfdirstate.normal(bfutil.unixpath(bfile))
elif os.path.exists(repo.wjoin(bfile)) and mode != os.stat(
repo.wjoin(bfile)).st_mode:
os.chmod(repo.wjoin(bfile), mode)
updated += 1
bfdirstate.normal(bfutil.unixpath(bfile))
if toget:
store = basestore._open_store(repo)
(success, missing) = store.get(toget)
else:
success, missing = [], []
for (filename, hash) in success:
mode = os.stat(repo.wjoin(bfutil.standin(filename))).st_mode
os.chmod(repo.wjoin(filename), mode)
updated += 1
bfdirstate.normal(bfutil.unixpath(filename))
for bfile in bfdirstate:
if bfile not in bfiles:
if os.path.exists(repo.wjoin(bfile)):
if not printed:
ui.status(_('Getting changed bfiles\n'))
printed = True
os.unlink(repo.wjoin(bfile))
removed += 1
bfdirstate.forget(bfutil.unixpath(bfile))
bfdirstate.write()
if printed:
ui.status(
_('%d big files updated, %d removed\n') % (updated, removed))
finally:
wlock.release()
def revert_bfiles(ui, repo):
wlock = repo.wlock()
try:
bfdirstate = bfutil.open_bfdirstate(ui, repo)
s = bfdirstate.status(match_.always(repo.root, repo.getcwd()), [],
False, False, False)
(unsure, modified, added, removed, missing, unknown, ignored,
clean) = s
bfiles = bfutil.list_bfiles(repo)
toget = []
at = 0
updated = 0
for bfile in bfiles:
if not os.path.exists(repo.wjoin(bfutil.standin(bfile))):
bfdirstate.remove(bfile)
continue
if os.path.exists(
repo.wjoin(bfutil.standin(os.path.join(bfile + '.orig')))):
shutil.copyfile(repo.wjoin(bfile), repo.wjoin(bfile + '.orig'))
at += 1
expectedhash = repo[None][bfutil.standin(bfile)].data().strip()
mode = os.stat(repo.wjoin(bfutil.standin(bfile))).st_mode
if not os.path.exists(
repo.wjoin(bfile)) or expectedhash != bfutil.hashfile(
repo.wjoin(bfile)):
path = bfutil.find_file(repo, expectedhash)
if path is None:
toget.append((bfile, expectedhash))
else:
util.makedirs(os.path.dirname(repo.wjoin(bfile)))
shutil.copy(path, repo.wjoin(bfile))
os.chmod(repo.wjoin(bfile), mode)
updated += 1
if bfutil.standin(bfile) not in repo['.']:
bfdirstate.add(bfutil.unixpath(bfile))
elif expectedhash == repo['.'][bfutil.standin(
bfile)].data().strip():
bfdirstate.normal(bfutil.unixpath(bfile))
else:
bfutil.dirstate_normaldirty(bfdirstate,
bfutil.unixpath(bfile))
elif os.path.exists(repo.wjoin(bfile)) and mode != os.stat(
repo.wjoin(bfile)).st_mode:
os.chmod(repo.wjoin(bfile), mode)
updated += 1
if bfutil.standin(bfile) not in repo['.']:
bfdirstate.add(bfutil.unixpath(bfile))
elif expectedhash == repo['.'][bfutil.standin(
bfile)].data().strip():
bfdirstate.normal(bfutil.unixpath(bfile))
else:
bfutil.dirstate_normaldirty(bfdirstate,
bfutil.unixpath(bfile))
if toget:
store = basestore._open_store(repo)
(success, missing) = store.get(toget)
else:
success, missing = [], []
for (filename, hash) in success:
mode = os.stat(repo.wjoin(bfutil.standin(filename))).st_mode
os.chmod(repo.wjoin(filename), mode)
updated += 1
if bfutil.standin(filename) not in repo['.']:
bfdirstate.add(bfutil.unixpath(filename))
elif hash == repo['.'][bfutil.standin(filename)].data().strip():
bfdirstate.normal(bfutil.unixpath(filename))
else:
bfutil.dirstate_normaldirty(bfdirstate,
bfutil.unixpath(filename))
removed = 0
for bfile in bfdirstate:
if not os.path.exists(repo.wjoin(bfutil.standin(bfile))):
if os.path.exists(repo.wjoin(bfile)):
os.unlink(repo.wjoin(bfile))
removed += 1
if bfutil.standin(bfile) in repo['.']:
bfdirstate.remove(bfutil.unixpath(bfile))
else:
bfdirstate.forget(bfutil.unixpath(bfile))
else:
state = repo.dirstate[bfutil.standin(bfile)]
if state == 'n':
bfdirstate.normal(bfile)
elif state == 'r':
bfdirstate.remove(bfile)
elif state == 'a':
bfdirstate.add(bfile)
elif state == '?':
bfdirstate.forget(bfile)
bfdirstate.write()
finally:
wlock.release()
def bfupdate(ui, repo, *pats, **opts):
'''get/delete big files for current changeset'''
# prefix user-supplied paths with '.hgbfiles/'
matcher = bfutil._get_standin_matcher(repo, pats, opts)
# Check status of standins (.hgbfiles/...) relative to main dirstate
# (i.e. detect changes to big files that have been bfadd'ed or
# bfrefresh'ed, but not committed).
bfutil._check_standins_status(ui, repo, matcher)
# Check status of actual big files relative to bfiles dirstate
# (changes that have not been bfrefresh'ed).
bfdirstate = bfutil._open_bfdirstate(ui, repo)
checkexec = bfdirstate._checkexec
bfmatcher = bfutil._get_matcher(repo, pats, opts, showbad=False)
bfutil._check_bfiles_status(ui, repo, bfdirstate, matcher=bfmatcher)
store = basestore._open_store(ui, repo)
updated = removed = 0
# Use dirstate.walk() rather than repo.walk() for performance: we do
# not care about unknown files at all, so don't waste time iterating
# over them. Should be safe since users should not create files in
# .hgbfiles; if they do, we'll ignore them here and that's just
# fine.
want_files = [] # hash has changed
update_mode = [] # mode may have changed
for standin in sorted(bfutil.dirstate_walk(repo.dirstate, matcher)):
want_hash = bfutil._read_standin(repo, standin)
filename = bfutil._split_standin(standin)
latest_hash = bfutil._read_latest(repo, filename)
if (not latest_hash or
latest_hash != want_hash or
not os.path.isfile(repo.wjoin(filename))):
# big file missing or hash has changed
want_files.append((filename, want_hash))
else:
# file content is up-to-date: need to update mode in case
# only it changed
if checkexec:
oldmode = os.stat(repo.wjoin(filename)).st_mode
newmode = os.stat(repo.wjoin(standin)).st_mode
if not bfutil._modes_equal(oldmode, newmode):
update_mode.append(filename)
# Some revisions might be stashed in .hg/bfiles/{pending,committed}
# -- e.g. anything that has been bfadded, bfrefreshed, or committed,
# but not yet bfput. Go there first: should be faster than hitting
# the central store.
success1 = []
still_want = []
for (filename, hash) in want_files:
found = False
for dir in ('pending', 'committed'):
pfilename = repo.join(os.path.join(
'bfiles', dir, filename, hash))
if os.path.isfile(pfilename):
ui.note(_('getting %s\n') % filename)
destfile = repo.wjoin(filename)
util.makedirs(os.path.dirname(destfile))
shutil.copyfile(pfilename, destfile)
found = True
break
if found:
success1.append((filename, hash))
else:
still_want.append((filename, hash))
# Now go to the central store for whatever we didn't find locally.
(success2, missing) = store.get(still_want)
success = success1 + success2
del success1, success2
assert len(success) + len(missing) == len(want_files), \
('some requested files not accounted for: '
'wanted %d files, got %d, missing %d'
% (len(want_files), len(success), len(missing)))
# Fix permissions of successfully downloaded files plus any files
# whose mode changed.
umask = os.umask(0777)
os.umask(umask)
update_mode = [fn for (fn, hash) in success] + update_mode
for filename in update_mode:
mode = os.stat(repo.wjoin(bfutil._standin(filename))).st_mode
mode = mode & ~umask
os.chmod(repo.wjoin(filename), mode)
bfdirstate.normal(filename)
for (filename, hash) in success:
bfutil._write_latest(repo, filename, hash)
ui.debug('bfdirstate.normal(%r)\n' % filename)
updated += 1
# Iterate over bfdirstate to find any big files from the
# previous changeset that are no longer in .hgbfiles: they can be
# deleted (unless they are now normal files).
ui.debug('searching for big files to remove\n')
for filename in bfdirstate:
standin = repo.wjoin(bfutil._standin(filename).replace('/', os.path.sep))
ui.debug('maybe removing %s\n' % filename)
if os.path.exists(standin):
continue
if bfutil._is_normal(repo, filename):
# big file has become a normal file
continue
ui.note('removing %s\n' % filename)
lfilename = filename.replace('/', os.path.sep)
try:
bfutil.unlinkpath(repo.wjoin(lfilename))
bfutil.unlinkpath(bfutil._get_latest(repo, lfilename))
except OSError, err:
# if the file is already deleted, who cares?
if err.errno != errno.ENOENT:
raise
bfutil.dirstate_drop(bfdirstate, filename)
removed += 1
def update_bfiles(ui, repo):
wlock = repo.wlock()
try:
bfdirstate = bfutil.open_bfdirstate(ui, repo)
s = bfdirstate.status(match_.always(repo.root, repo.getcwd()), [], False, False, False)
(unsure, modified, added, removed, missing, unknown, ignored, clean) = s
bfiles = bfutil.list_bfiles(repo)
toget = []
at = 0
updated = 0
removed = 0
printed = False
if bfiles:
ui.status(_('Getting changed bfiles\n'))
printed = True
for bfile in bfiles:
at += 1
if os.path.exists(repo.wjoin(bfile)) and not os.path.exists(repo.wjoin(bfutil.standin(bfile))):
os.unlink(repo.wjoin(bfile))
removed += 1
bfdirstate.forget(bfutil.unixpath(bfile))
continue
expectedhash = repo[None][bfutil.standin(bfile)].data().strip()
mode = os.stat(repo.wjoin(bfutil.standin(bfile))).st_mode
if not os.path.exists(repo.wjoin(bfile)) or expectedhash != bfutil.hashfile(repo.wjoin(bfile)):
path = bfutil.find_file(repo, expectedhash)
if not path:
toget.append((bfile, expectedhash))
else:
util.makedirs(os.path.dirname(repo.wjoin(bfile)))
shutil.copy(path, repo.wjoin(bfile))
os.chmod(repo.wjoin(bfile), mode)
updated += 1
bfdirstate.normal(bfutil.unixpath(bfile))
elif os.path.exists(repo.wjoin(bfile)) and mode != os.stat(repo.wjoin(bfile)).st_mode:
os.chmod(repo.wjoin(bfile), mode)
updated += 1
bfdirstate.normal(bfutil.unixpath(bfile))
if toget:
store = basestore._open_store(repo)
(success, missing) = store.get(toget)
else:
success, missing = [],[]
for (filename, hash) in success:
mode = os.stat(repo.wjoin(bfutil.standin(filename))).st_mode
os.chmod(repo.wjoin(filename), mode)
updated += 1
bfdirstate.normal(bfutil.unixpath(filename))
for bfile in bfdirstate:
if bfile not in bfiles:
if os.path.exists(repo.wjoin(bfile)):
if not printed:
ui.status(_('Getting changed bfiles\n'))
printed = True
os.unlink(repo.wjoin(bfile))
removed += 1
bfdirstate.forget(bfutil.unixpath(bfile))
bfdirstate.write()
if printed:
ui.status(_('%d big files updated, %d removed\n') % (updated, removed))
finally:
wlock.release()
def revert_bfiles(ui, repo):
wlock = repo.wlock()
try:
bfdirstate = bfutil.open_bfdirstate(ui, repo)
s = bfdirstate.status(match_.always(repo.root, repo.getcwd()), [], False, False, False)
(unsure, modified, added, removed, missing, unknown, ignored, clean) = s
bfiles = bfutil.list_bfiles(repo)
toget = []
at = 0
updated = 0
for bfile in bfiles:
if not os.path.exists(repo.wjoin(bfutil.standin(bfile))):
bfdirstate.remove(bfile)
continue
if os.path.exists(repo.wjoin(bfutil.standin(os.path.join(bfile + '.orig')))):
shutil.copyfile(repo.wjoin(bfile), repo.wjoin(bfile + '.orig'))
at += 1
expectedhash = repo[None][bfutil.standin(bfile)].data().strip()
mode = os.stat(repo.wjoin(bfutil.standin(bfile))).st_mode
if not os.path.exists(repo.wjoin(bfile)) or expectedhash != bfutil.hashfile(repo.wjoin(bfile)):
path = bfutil.find_file(repo, expectedhash)
if path is None:
toget.append((bfile, expectedhash))
else:
util.makedirs(os.path.dirname(repo.wjoin(bfile)))
shutil.copy(path, repo.wjoin(bfile))
os.chmod(repo.wjoin(bfile), mode)
updated += 1
if bfutil.standin(bfile) not in repo['.']:
bfdirstate.add(bfutil.unixpath(bfile))
elif expectedhash == repo['.'][bfutil.standin(bfile)].data().strip():
bfdirstate.normal(bfutil.unixpath(bfile))
else:
bfutil.dirstate_normaldirty(bfdirstate, bfutil.unixpath(bfile))
elif os.path.exists(repo.wjoin(bfile)) and mode != os.stat(repo.wjoin(bfile)).st_mode:
os.chmod(repo.wjoin(bfile), mode)
updated += 1
if bfutil.standin(bfile) not in repo['.']:
bfdirstate.add(bfutil.unixpath(bfile))
elif expectedhash == repo['.'][bfutil.standin(bfile)].data().strip():
bfdirstate.normal(bfutil.unixpath(bfile))
else:
bfutil.dirstate_normaldirty(bfdirstate, bfutil.unixpath(bfile))
if toget:
store = basestore._open_store(repo)
(success, missing) = store.get(toget)
else:
success, missing = [], []
for (filename, hash) in success:
mode = os.stat(repo.wjoin(bfutil.standin(filename))).st_mode
os.chmod(repo.wjoin(filename), mode)
updated += 1
if bfutil.standin(filename) not in repo['.']:
bfdirstate.add(bfutil.unixpath(filename))
elif hash == repo['.'][bfutil.standin(filename)].data().strip():
bfdirstate.normal(bfutil.unixpath(filename))
else:
bfutil.dirstate_normaldirty(bfdirstate, bfutil.unixpath(filename))
removed = 0
for bfile in bfdirstate:
if not os.path.exists(repo.wjoin(bfutil.standin(bfile))):
if os.path.exists(repo.wjoin(bfile)):
os.unlink(repo.wjoin(bfile))
removed += 1
if bfutil.standin(bfile) in repo['.']:
bfdirstate.remove(bfutil.unixpath(bfile))
else:
bfdirstate.forget(bfutil.unixpath(bfile))
else:
state = repo.dirstate[bfutil.standin(bfile)]
if state == 'n':
bfdirstate.normal(bfile)
elif state == 'r':
bfdirstate.remove(bfile)
elif state == 'a':
bfdirstate.add(bfile)
elif state == '?':
bfdirstate.forget(bfile)
bfdirstate.write()
finally:
wlock.release()