def _gethash(self, filename, hash):
"""Get file with the provided hash and store it in the local repo's
store and in the usercache.
filename is for informational messages only.
"""
util.makedirs(lfutil.storepath(self.repo, ''))
storefilename = lfutil.storepath(self.repo, hash)
tmpname = storefilename + '.tmp'
tmpfile = util.atomictempfile(tmpname,
createmode=self.repo.store.createmode)
try:
gothash = self._getfile(tmpfile, filename, hash)
except StoreError as err:
self.ui.warn(err.longmessage())
gothash = ""
tmpfile.close()
if gothash != hash:
if gothash != "":
self.ui.warn(
_('%s: data corruption (expected %s, got %s)\n') %
(filename, hash, gothash))
util.unlink(tmpname)
return False
util.rename(tmpname, storefilename)
lfutil.linktousercache(self.repo, hash)
return True
def _gethash(self, filename, hash):
"""Get file with the provided hash and store it in the local repo's
store and in the usercache.
filename is for informational messages only.
"""
util.makedirs(lfutil.storepath(self.repo, ''))
storefilename = lfutil.storepath(self.repo, hash)
tmpname = storefilename + '.tmp'
tmpfile = util.atomictempfile(
tmpname, createmode=self.repo.store.createmode)
try:
gothash = self._getfile(tmpfile, filename, hash)
except StoreError as err:
self.ui.warn(err.longmessage())
gothash = ""
tmpfile.close()
if gothash != hash:
if gothash != "":
self.ui.warn(
_('%s: data corruption (expected %s, got %s)\n') %
(filename, hash, gothash))
util.unlink(tmpname)
return False
util.rename(tmpname, storefilename)
lfutil.linktousercache(self.repo, hash)
return True
def get(self, files):
'''Get the specified largefiles from the store and write to local
files under repo.root. files is a list of (filename, hash)
tuples. Return (success, missing), lists of files successfully
downloaded and those not found in the store. success is a list
of (filename, hash) tuples; missing is a list of filenames that
we could not get. (The detailed error message will already have
been presented to the user, so missing is just supplied as a
summary.)'''
success = []
missing = []
ui = self.ui
util.makedirs(lfutil.storepath(self.repo, ''))
at = 0
available = self.exists(set(hash for (_filename, hash) in files))
for filename, hash in files:
ui.progress(_('getting largefiles'),
at,
unit='lfile',
total=len(files))
at += 1
ui.note(_('getting %s:%s\n') % (filename, hash))
if not available.get(hash):
ui.warn(
_('%s: largefile %s not available from %s\n') %
(filename, hash, self.url))
missing.append(filename)
continue
storefilename = lfutil.storepath(self.repo, hash)
tmpfile = util.atomictempfile(
storefilename + '.tmp', createmode=self.repo.store.createmode)
try:
hhash = self._getfile(tmpfile, filename, hash)
except StoreError, err:
ui.warn(err.longmessage())
hhash = ""
tmpfile.close()
if hhash != hash:
if hhash != "":
ui.warn(
_('%s: data corruption (expected %s, got %s)\n') %
(filename, hash, hhash))
util.unlink(storefilename + '.tmp')
missing.append(filename)
continue
util.rename(storefilename + '.tmp', storefilename)
lfutil.linktousercache(self.repo, hash)
success.append((filename, hhash))
def get(self, files):
'''Get the specified largefiles from the store and write to local
files under repo.root. files is a list of (filename, hash)
tuples. Return (success, missing), lists of files successfully
downloaded and those not found in the store. success is a list
of (filename, hash) tuples; missing is a list of filenames that
we could not get. (The detailed error message will already have
been presented to the user, so missing is just supplied as a
summary.)'''
success = []
missing = []
ui = self.ui
util.makedirs(lfutil.storepath(self.repo, ''))
at = 0
available = self.exists(set(hash for (_filename, hash) in files))
for filename, hash in files:
ui.progress(_('getting largefiles'), at, unit='lfile',
total=len(files))
at += 1
ui.note(_('getting %s:%s\n') % (filename, hash))
if not available.get(hash):
ui.warn(_('%s: largefile %s not available from %s\n')
% (filename, hash, self.url))
missing.append(filename)
continue
storefilename = lfutil.storepath(self.repo, hash)
tmpfile = util.atomictempfile(storefilename + '.tmp',
createmode=self.repo.store.createmode)
try:
hhash = self._getfile(tmpfile, filename, hash)
except StoreError, err:
ui.warn(err.longmessage())
hhash = ""
tmpfile.close()
if hhash != hash:
if hhash != "":
ui.warn(_('%s: data corruption (expected %s, got %s)\n')
% (filename, hash, hhash))
util.unlink(storefilename + '.tmp')
missing.append(filename)
continue
util.rename(storefilename + '.tmp', storefilename)
lfutil.linktousercache(self.repo, hash)
success.append((filename, hhash))
def putlfile(repo, proto, sha):
'''Server command for putting a largefile into a repository's local store
and into the user cache.'''
proto.redirect()
path = lfutil.storepath(repo, sha)
util.makedirs(os.path.dirname(path))
tmpfp = util.atomictempfile(path, createmode=repo.store.createmode)
try:
proto.getfile(tmpfp)
tmpfp._fp.seek(0)
if sha != lfutil.hexsha1(tmpfp._fp):
raise IOError(0, _('largefile contents do not match hash'))
tmpfp.close()
lfutil.linktousercache(repo, sha)
except IOError as e:
repo.ui.warn(
_('largefiles: failed to put %s into store: %s\n') %
(sha, e.strerror))
return wireproto.pushres(1)
finally:
tmpfp.discard()
return wireproto.pushres(0)
def putlfile(repo, proto, sha):
'''Put a largefile into a repository's local store and into the
user cache.'''
proto.redirect()
path = lfutil.storepath(repo, sha)
util.makedirs(os.path.dirname(path))
tmpfp = util.atomictempfile(path, createmode=repo.store.createmode)
try:
try:
proto.getfile(tmpfp)
tmpfp._fp.seek(0)
if sha != lfutil.hexsha1(tmpfp._fp):
raise IOError(0, _('largefile contents do not match hash'))
tmpfp.close()
lfutil.linktousercache(repo, sha)
except IOError, e:
repo.ui.warn(_('largefiles: failed to put %s into store: %s\n') %
(sha, e.strerror))
return wireproto.pushres(1)
finally:
tmpfp.discard()
return wireproto.pushres(0)
def _verifyfile(self, cctx, cset, contents, standin, verified):
filename = lfutil.splitstandin(standin)
if not filename:
return False
fctx = cctx[standin]
key = (filename, fctx.filenode())
if key in verified:
return False
expecthash = fctx.data()[0:40]
storepath = lfutil.storepath(self.remote, expecthash)
verified.add(key)
if not lfutil.instore(self.remote, expecthash):
self.ui.warn(
_('changeset %s: %s references missing %s\n')
% (cset, filename, storepath))
return True # failed
if contents:
actualhash = lfutil.hashfile(storepath)
if actualhash != expecthash:
self.ui.warn(
_('changeset %s: %s references corrupted %s\n')
% (cset, filename, storepath))
return True # failed
return False
def _gethash(self, filename, hash):
"""Get file with the provided hash and store it in the local repo's
store and in the usercache.
filename is for informational messages only.
"""
util.makedirs(lfutil.storepath(self.repo, ""))
storefilename = lfutil.storepath(self.repo, hash)
tmpname = storefilename + ".tmp"
tmpfile = util.atomictempfile(tmpname, createmode=self.repo.store.createmode)
try:
gothash = self._getfile(tmpfile, filename, hash)
except StoreError, err:
self.ui.warn(err.longmessage())
gothash = ""
def _verifyfile(self, cctx, cset, contents, standin, verified):
filename = lfutil.splitstandin(standin)
if not filename:
return False
fctx = cctx[standin]
key = (filename, fctx.filenode())
if key in verified:
return False
expecthash = fctx.data()[0:40]
verified.add(key)
if not lfutil.instore(self.remote, expecthash):
self.ui.warn(
_('changeset %s: %s missing\n'
' (looked for hash %s)\n') % (cset, filename, expecthash))
return True # failed
if contents:
storepath = lfutil.storepath(self.remote, expecthash)
actualhash = lfutil.hashfile(storepath)
if actualhash != expecthash:
self.ui.warn(
_('changeset %s: %s: contents differ\n'
' (%s:\n'
' expected hash %s,\n'
' but got %s)\n') %
(cset, filename, storepath, expecthash, actualhash))
return True # failed
return False
def _verifyfile(self, cctx, cset, contents, standin, verified):
filename = lfutil.splitstandin(standin)
if not filename:
return False
fctx = cctx[standin]
key = (filename, fctx.filenode())
if key in verified:
return False
expecthash = fctx.data()[0:40]
storepath = lfutil.storepath(self.remote, expecthash)
verified.add(key)
if not lfutil.instore(self.remote, expecthash):
self.ui.warn(
_('changeset %s: %s references missing %s\n') %
(cset, filename, storepath))
return True # failed
if contents:
actualhash = lfutil.hashfile(storepath)
if actualhash != expecthash:
self.ui.warn(
_('changeset %s: %s references corrupted %s\n') %
(cset, filename, storepath))
return True # failed
return False
def _verifyfile(self, cctx, cset, contents, standin, verified):
filename = lfutil.splitstandin(standin)
if not filename:
return False
fctx = cctx[standin]
key = (filename, fctx.filenode())
if key in verified:
return False
expecthash = fctx.data()[0:40]
verified.add(key)
if not lfutil.instore(self.remote, expecthash):
self.ui.warn(
_('changeset %s: %s missing\n'
' (looked for hash %s)\n')
% (cset, filename, expecthash))
return True # failed
if contents:
storepath = lfutil.storepath(self.remote, expecthash)
actualhash = lfutil.hashfile(storepath)
if actualhash != expecthash:
self.ui.warn(
_('changeset %s: %s: contents differ\n'
' (%s:\n'
' expected hash %s,\n'
' but got %s)\n')
% (cset, filename, storepath, expecthash, actualhash))
return True # failed
return False
def _gethash(self, filename, hash):
"""Get file with the provided hash and store it in the local repo's
store and in the usercache.
filename is for informational messages only.
"""
util.makedirs(lfutil.storepath(self.repo, ''))
storefilename = lfutil.storepath(self.repo, hash)
tmpname = storefilename + '.tmp'
tmpfile = util.atomictempfile(tmpname,
createmode=self.repo.store.createmode)
try:
gothash = self._getfile(tmpfile, filename, hash)
except StoreError, err:
self.ui.warn(err.longmessage())
gothash = ""
def _getfile(self, tmpfile, filename, hash):
if lfutil.instore(self.remote, hash):
path = lfutil.storepath(self.remote, hash)
elif lfutil.inusercache(self.ui, hash):
path = lfutil.usercachepath(self.ui, hash)
else:
raise basestore.StoreError(filename, hash, '',
_("Can't get file locally"))
fd = open(path, 'rb')
try:
return lfutil.copyandhash(fd, tmpfile)
finally:
fd.close()
def _getfile(self, tmpfile, filename, hash):
if lfutil.instore(self.remote, hash):
path = lfutil.storepath(self.remote, hash)
elif lfutil.inusercache(self.ui, hash):
path = lfutil.usercachepath(self.ui, hash)
else:
raise basestore.StoreError(filename, hash, '',
_("Can't get file locally"))
fd = open(path, 'rb')
try:
return lfutil.copyandhash(fd, tmpfile)
finally:
fd.close()
def get(self, files):
'''Get the specified largefiles from the store and write to local
files under repo.root. files is a list of (filename, hash)
tuples. Return (success, missing), lists of files successfuly
downloaded and those not found in the store. success is a list
of (filename, hash) tuples; missing is a list of filenames that
we could not get. (The detailed error message will already have
been presented to the user, so missing is just supplied as a
summary.)'''
success = []
missing = []
ui = self.ui
at = 0
for filename, hash in files:
ui.progress(_('getting largefiles'), at, unit='lfile',
total=len(files))
at += 1
ui.note(_('getting %s:%s\n') % (filename, hash))
storefilename = lfutil.storepath(self.repo, hash)
storedir = os.path.dirname(storefilename)
# No need to pass mode='wb' to fdopen(), since mkstemp() already
# opened the file in binary mode.
(tmpfd, tmpfilename) = tempfile.mkstemp(
dir=storedir, prefix=os.path.basename(filename))
tmpfile = os.fdopen(tmpfd, 'w')
try:
hhash = binascii.hexlify(self._getfile(tmpfile, filename, hash))
except StoreError, err:
ui.warn(err.longmessage())
hhash = ""
if hhash != hash:
if hhash != "":
ui.warn(_('%s: data corruption (expected %s, got %s)\n')
% (filename, hash, hhash))
tmpfile.close() # no-op if it's already closed
os.remove(tmpfilename)
missing.append(filename)
continue
if os.path.exists(storefilename): # Windows
os.remove(storefilename)
os.rename(tmpfilename, storefilename)
lfutil.linktousercache(self.repo, hash)
success.append((filename, hhash))
def get(self, files):
'''Get the specified largefiles from the store and write to local
files under repo.root. files is a list of (filename, hash)
tuples. Return (success, missing), lists of files successfuly
downloaded and those not found in the store. success is a list
of (filename, hash) tuples; missing is a list of filenames that
we could not get. (The detailed error message will already have
been presented to the user, so missing is just supplied as a
summary.)'''
success = []
missing = []
ui = self.ui
at = 0
for filename, hash in files:
ui.progress(_('getting largefiles'),
at,
unit='lfile',
total=len(files))
at += 1
ui.note(_('getting %s:%s\n') % (filename, hash))
storefilename = lfutil.storepath(self.repo, hash)
tmpfile = util.atomictempfile(
storefilename, createmode=self.repo.store.createmode)
try:
hhash = binascii.hexlify(self._getfile(tmpfile, filename,
hash))
except StoreError, err:
ui.warn(err.longmessage())
hhash = ""
if hhash != hash:
if hhash != "":
ui.warn(
_('%s: data corruption (expected %s, got %s)\n') %
(filename, hash, hhash))
tmpfile.discard() # no-op if it's already closed
missing.append(filename)
continue
tmpfile.close()
lfutil.linktousercache(self.repo, hash)
success.append((filename, hhash))
def get(self, files):
'''Get the specified largefiles from the store and write to local
files under repo.root. files is a list of (filename, hash)
tuples. Return (success, missing), lists of files successfuly
downloaded and those not found in the store. success is a list
of (filename, hash) tuples; missing is a list of filenames that
we could not get. (The detailed error message will already have
been presented to the user, so missing is just supplied as a
summary.)'''
success = []
missing = []
ui = self.ui
at = 0
for filename, hash in files:
ui.progress(_('getting largefiles'), at, unit='lfile',
total=len(files))
at += 1
ui.note(_('getting %s:%s\n') % (filename, hash))
storefilename = lfutil.storepath(self.repo, hash)
tmpfile = util.atomictempfile(storefilename,
createmode=self.repo.store.createmode)
try:
hhash = binascii.hexlify(self._getfile(tmpfile, filename, hash))
except StoreError, err:
ui.warn(err.longmessage())
hhash = ""
if hhash != hash:
if hhash != "":
ui.warn(_('%s: data corruption (expected %s, got %s)\n')
% (filename, hash, hhash))
tmpfile.discard() # no-op if it's already closed
missing.append(filename)
continue
tmpfile.close()
lfutil.linktousercache(self.repo, hash)
success.append((filename, hhash))
def put(self, source, hash):
if lfutil.instore(self.remote, hash):
return
lfutil.link(source, lfutil.storepath(self.remote, hash))
def put(self, source, hash):
util.makedirs(os.path.dirname(lfutil.storepath(self.remote, hash)))
if lfutil.instore(self.remote, hash):
return
lfutil.link(lfutil.storepath(self.repo, hash),
lfutil.storepath(self.remote, hash))
def put(self, source, hash):
util.makedirs(os.path.dirname(lfutil.storepath(self.remote, hash)))
if lfutil.instore(self.remote, hash):
return
lfutil.link(lfutil.storepath(self.repo, hash),
lfutil.storepath(self.remote, hash))