def _gethash(self, filename, hash):
"""Get file with the provided hash and store it in the local repo's
store and in the usercache.
filename is for informational messages only.
"""
util.makedirs(lfutil.storepath(self.repo, ''))
storefilename = lfutil.storepath(self.repo, hash)
tmpname = storefilename + '.tmp'
tmpfile = util.atomictempfile(
tmpname, createmode=self.repo.store.createmode)
try:
gothash = self._getfile(tmpfile, filename, hash)
except StoreError as err:
self.ui.warn(err.longmessage())
gothash = ""
tmpfile.close()
if gothash != hash:
if gothash != "":
self.ui.warn(
_('%s: data corruption (expected %s, got %s)\n') %
(filename, hash, gothash))
util.unlink(tmpname)
return False
util.rename(tmpname, storefilename)
lfutil.linktousercache(self.repo, hash)
return True
def putlfile(repo, proto, sha):
'''Put a largefile into a repository's local store and into the
user cache.'''
proto.redirect()
path = lfutil.storepath(repo, sha)
util.makedirs(os.path.dirname(path))
tmpfp = util.atomictempfile(path, createmode=repo.store.createmode)
try:
try:
proto.getfile(tmpfp)
tmpfp._fp.seek(0)
if sha != lfutil.hexsha1(tmpfp._fp):
raise IOError(0, _('largefile contents do not match hash'))
tmpfp.close()
lfutil.linktousercache(repo, sha)
except IOError, e:
repo.ui.warn(_('largefiles: failed to put %s into store: %s\n') %
(sha, e.strerror))
return wireproto.pushres(1)
finally:
tmpfp.discard()
return wireproto.pushres(0)
def putlfile(repo, proto, sha):
'''Server command for putting a largefile into a repository's local store
and into the user cache.'''
proto.redirect()
path = lfutil.storepath(repo, sha)
util.makedirs(os.path.dirname(path))
tmpfp = util.atomictempfile(path, createmode=repo.store.createmode)
try:
proto.getfile(tmpfp)
tmpfp._fp.seek(0)
if sha != lfutil.hexsha1(tmpfp._fp):
raise IOError(0, _('largefile contents do not match hash'))
tmpfp.close()
lfutil.linktousercache(repo, sha)
except IOError as e:
repo.ui.warn(
_('largefiles: failed to put %s into store: %s\n') %
(sha, e.strerror))
return wireproto.pushres(1)
finally:
tmpfp.discard()
return wireproto.pushres(0)
def _gethash(self, filename, hash):
"""Get file with the provided hash and store it in the local repo's
store and in the usercache.
filename is for informational messages only.
"""
util.makedirs(lfutil.storepath(self.repo, ''))
storefilename = lfutil.storepath(self.repo, hash)
tmpname = storefilename + '.tmp'
tmpfile = util.atomictempfile(tmpname,
createmode=self.repo.store.createmode)
try:
gothash = self._getfile(tmpfile, filename, hash)
except StoreError as err:
self.ui.warn(err.longmessage())
gothash = ""
tmpfile.close()
if gothash != hash:
if gothash != "":
self.ui.warn(
_('%s: data corruption (expected %s, got %s)\n') %
(filename, hash, gothash))
util.unlink(tmpname)
return False
util.rename(tmpname, storefilename)
lfutil.linktousercache(self.repo, hash)
return True
def get(self, files):
'''Get the specified largefiles from the store and write to local
files under repo.root. files is a list of (filename, hash)
tuples. Return (success, missing), lists of files successfully
downloaded and those not found in the store. success is a list
of (filename, hash) tuples; missing is a list of filenames that
we could not get. (The detailed error message will already have
been presented to the user, so missing is just supplied as a
summary.)'''
success = []
missing = []
ui = self.ui
util.makedirs(lfutil.storepath(self.repo, ''))
at = 0
available = self.exists(set(hash for (_filename, hash) in files))
for filename, hash in files:
ui.progress(_('getting largefiles'),
at,
unit='lfile',
total=len(files))
at += 1
ui.note(_('getting %s:%s\n') % (filename, hash))
if not available.get(hash):
ui.warn(
_('%s: largefile %s not available from %s\n') %
(filename, hash, self.url))
missing.append(filename)
continue
storefilename = lfutil.storepath(self.repo, hash)
tmpfile = util.atomictempfile(
storefilename + '.tmp', createmode=self.repo.store.createmode)
try:
hhash = self._getfile(tmpfile, filename, hash)
except StoreError, err:
ui.warn(err.longmessage())
hhash = ""
tmpfile.close()
if hhash != hash:
if hhash != "":
ui.warn(
_('%s: data corruption (expected %s, got %s)\n') %
(filename, hash, hhash))
util.unlink(storefilename + '.tmp')
missing.append(filename)
continue
util.rename(storefilename + '.tmp', storefilename)
lfutil.linktousercache(self.repo, hash)
success.append((filename, hhash))
def get(self, files):
'''Get the specified largefiles from the store and write to local
files under repo.root. files is a list of (filename, hash)
tuples. Return (success, missing), lists of files successfully
downloaded and those not found in the store. success is a list
of (filename, hash) tuples; missing is a list of filenames that
we could not get. (The detailed error message will already have
been presented to the user, so missing is just supplied as a
summary.)'''
success = []
missing = []
ui = self.ui
util.makedirs(lfutil.storepath(self.repo, ''))
at = 0
available = self.exists(set(hash for (_filename, hash) in files))
for filename, hash in files:
ui.progress(_('getting largefiles'), at, unit='lfile',
total=len(files))
at += 1
ui.note(_('getting %s:%s\n') % (filename, hash))
if not available.get(hash):
ui.warn(_('%s: largefile %s not available from %s\n')
% (filename, hash, self.url))
missing.append(filename)
continue
storefilename = lfutil.storepath(self.repo, hash)
tmpfile = util.atomictempfile(storefilename + '.tmp',
createmode=self.repo.store.createmode)
try:
hhash = self._getfile(tmpfile, filename, hash)
except StoreError, err:
ui.warn(err.longmessage())
hhash = ""
tmpfile.close()
if hhash != hash:
if hhash != "":
ui.warn(_('%s: data corruption (expected %s, got %s)\n')
% (filename, hash, hhash))
util.unlink(storefilename + '.tmp')
missing.append(filename)
continue
util.rename(storefilename + '.tmp', storefilename)
lfutil.linktousercache(self.repo, hash)
success.append((filename, hhash))
def get(self, files):
'''Get the specified largefiles from the store and write to local
files under repo.root. files is a list of (filename, hash)
tuples. Return (success, missing), lists of files successfuly
downloaded and those not found in the store. success is a list
of (filename, hash) tuples; missing is a list of filenames that
we could not get. (The detailed error message will already have
been presented to the user, so missing is just supplied as a
summary.)'''
success = []
missing = []
ui = self.ui
at = 0
for filename, hash in files:
ui.progress(_('getting largefiles'), at, unit='lfile',
total=len(files))
at += 1
ui.note(_('getting %s:%s\n') % (filename, hash))
storefilename = lfutil.storepath(self.repo, hash)
storedir = os.path.dirname(storefilename)
# No need to pass mode='wb' to fdopen(), since mkstemp() already
# opened the file in binary mode.
(tmpfd, tmpfilename) = tempfile.mkstemp(
dir=storedir, prefix=os.path.basename(filename))
tmpfile = os.fdopen(tmpfd, 'w')
try:
hhash = binascii.hexlify(self._getfile(tmpfile, filename, hash))
except StoreError, err:
ui.warn(err.longmessage())
hhash = ""
if hhash != hash:
if hhash != "":
ui.warn(_('%s: data corruption (expected %s, got %s)\n')
% (filename, hash, hhash))
tmpfile.close() # no-op if it's already closed
os.remove(tmpfilename)
missing.append(filename)
continue
if os.path.exists(storefilename): # Windows
os.remove(storefilename)
os.rename(tmpfilename, storefilename)
lfutil.linktousercache(self.repo, hash)
success.append((filename, hhash))
def get(self, files):
'''Get the specified largefiles from the store and write to local
files under repo.root. files is a list of (filename, hash)
tuples. Return (success, missing), lists of files successfuly
downloaded and those not found in the store. success is a list
of (filename, hash) tuples; missing is a list of filenames that
we could not get. (The detailed error message will already have
been presented to the user, so missing is just supplied as a
summary.)'''
success = []
missing = []
ui = self.ui
at = 0
for filename, hash in files:
ui.progress(_('getting largefiles'),
at,
unit='lfile',
total=len(files))
at += 1
ui.note(_('getting %s:%s\n') % (filename, hash))
storefilename = lfutil.storepath(self.repo, hash)
tmpfile = util.atomictempfile(
storefilename, createmode=self.repo.store.createmode)
try:
hhash = binascii.hexlify(self._getfile(tmpfile, filename,
hash))
except StoreError, err:
ui.warn(err.longmessage())
hhash = ""
if hhash != hash:
if hhash != "":
ui.warn(
_('%s: data corruption (expected %s, got %s)\n') %
(filename, hash, hhash))
tmpfile.discard() # no-op if it's already closed
missing.append(filename)
continue
tmpfile.close()
lfutil.linktousercache(self.repo, hash)
success.append((filename, hhash))
def get(self, files):
'''Get the specified largefiles from the store and write to local
files under repo.root. files is a list of (filename, hash)
tuples. Return (success, missing), lists of files successfuly
downloaded and those not found in the store. success is a list
of (filename, hash) tuples; missing is a list of filenames that
we could not get. (The detailed error message will already have
been presented to the user, so missing is just supplied as a
summary.)'''
success = []
missing = []
ui = self.ui
at = 0
for filename, hash in files:
ui.progress(_('getting largefiles'), at, unit='lfile',
total=len(files))
at += 1
ui.note(_('getting %s:%s\n') % (filename, hash))
storefilename = lfutil.storepath(self.repo, hash)
tmpfile = util.atomictempfile(storefilename,
createmode=self.repo.store.createmode)
try:
hhash = binascii.hexlify(self._getfile(tmpfile, filename, hash))
except StoreError, err:
ui.warn(err.longmessage())
hhash = ""
if hhash != hash:
if hhash != "":
ui.warn(_('%s: data corruption (expected %s, got %s)\n')
% (filename, hash, hhash))
tmpfile.discard() # no-op if it's already closed
missing.append(filename)
continue
tmpfile.close()
lfutil.linktousercache(self.repo, hash)
success.append((filename, hhash))
class basestore(object):
def __init__(self, ui, repo, url):
self.ui = ui
self.repo = repo
self.url = url
def put(self, source, hash):
'''Put source file into the store so it can be retrieved by hash.'''
raise NotImplementedError('abstract method')
def exists(self, hashes):
'''Check to see if the store contains the given hashes. Given an
iterable of hashes it returns a mapping from hash to bool.'''
raise NotImplementedError('abstract method')
def get(self, files):
'''Get the specified largefiles from the store and write to local
files under repo.root. files is a list of (filename, hash)
tuples. Return (success, missing), lists of files successfully
downloaded and those not found in the store. success is a list
of (filename, hash) tuples; missing is a list of filenames that
we could not get. (The detailed error message will already have
been presented to the user, so missing is just supplied as a
summary.)'''
success = []
missing = []
ui = self.ui
at = 0
available = self.exists(set(hash for (_filename, hash) in files))
for filename, hash in files:
ui.progress(_('getting largefiles'),
at,
unit='lfile',
total=len(files))
at += 1
ui.note(_('getting %s:%s\n') % (filename, hash))
if not available.get(hash):
ui.warn(
_('%s: largefile %s not available from %s\n') %
(filename, hash, util.hidepassword(self.url)))
missing.append(filename)
continue
if self._gethash(filename, hash):
success.append((filename, hash))
else:
missing.append(filename)
ui.progress(_('getting largefiles'), None)
return (success, missing)
def _gethash(self, filename, hash):
"""Get file with the provided hash and store it in the local repo's
store and in the usercache.
filename is for informational messages only.
"""
util.makedirs(lfutil.storepath(self.repo, ''))
storefilename = lfutil.storepath(self.repo, hash)
tmpname = storefilename + '.tmp'
tmpfile = util.atomictempfile(tmpname,
createmode=self.repo.store.createmode)
try:
gothash = self._getfile(tmpfile, filename, hash)
except StoreError, err:
self.ui.warn(err.longmessage())
gothash = ""
tmpfile.close()
if gothash != hash:
if gothash != "":
self.ui.warn(
_('%s: data corruption (expected %s, got %s)\n') %
(filename, hash, gothash))
util.unlink(tmpname)
return False
util.rename(tmpname, storefilename)
lfutil.linktousercache(self.repo, hash)
return True