def push(self, fp, storeurl, uniq=None):
"Push a local file (file pointer) to the store"
log.debug('local.push: url=%s', storeurl)
origpath = localpath = url2localpath(storeurl)
fpath,ext = os.path.splitext(origpath)
_mkdir (os.path.dirname(localpath))
uniq = uniq or make_uniq_code()
for x in xrange(len(uniq)-7):
#for x in range(100):
if not os.path.exists (localpath):
log.debug('local.write: %s -> %s' , tounicode(storeurl), tounicode(localpath))
#patch for no copy file uploads - check for regular file or file like object
try:
move_file (fp, localpath)
except OSError as e:
if not os.path.exists (localpath):
log.exception ("Problem moving file to%s ", localpath)
else:
log.error ("Problem moving file, but it seems to be there.. check permissions on store")
#log.debug ("local.push: top = %s path= %s",self.top_path, localpath )
ident = localpath[len(self.top_path):]
#if ident[0] == '/':
# ident = ident[1:]
ident = localpath2url(ident)
log.info('local push blob_id: %s -> %s', tounicode(ident), tounicode(localpath))
return ident, localpath
localpath = "%s-%s%s" % (fpath , uniq[3:7+x] , ext)
#localpath = "%s-%04d%s" % (fpath , x , ext)
log.warn ("local.write: File exists... trying %s", tounicode(localpath))
raise DuplicateFile(localpath)
def valid(self, storeurl):
#log.debug('valid ident %s top %s', ident, self.top)
#log.debug('valid local ident %s local top %s', url2localpath(ident), url2localpath(self.top))
if store_compare(storeurl, self.mount_url):
return storeurl
# It might be a shorted
storeurl,_ = split_subpath(storeurl)
scheme = urlparse.urlparse(storeurl).scheme
if not scheme and storeurl[0] != '/':
storeurl = urlparse.urljoin (self.top, storeurl)
# OLD STYLE : may have written %encoded values to file system
path = posixpath.normpath(urlparse.urlparse(storeurl).path)
path = force_filesys (path)
log.debug ("checking unquoted %s", tounicode(path))
if os.path.exists (path):
return path # not returning an actual URL ..
elif storeurl.startswith('file:///'):
#should have matched earlier
return None
elif storeurl.startswith('file://'):
storeurl = urlparse.urljoin(self.top, storeurl[7:])
else:
return None
localpath = url2localpath (storeurl)
log.debug ("checking %s", tounicode(localpath))
return os.path.exists(localpath) and localpath2url(localpath)
def _save_store(self,
store,
storepath,
resource,
fileobj=None,
rooturl=None):
'store the file to the named store'
resource_name, sub = split_subpath(resource.get('name'))
# Force a move into the store
if fileobj:
with self._get_driver(store) as driver:
if driver.readonly:
raise IllegalOperation('readonly store')
storeurl = posixpath.join(driver.mount_url, storepath)
log.debug('_save_store: %s from %s %s', storeurl,
driver.mount_url, storepath)
storeurl, localpath = driver.push(
fileobj, storeurl, resource.get('resource_uniq'))
# Store URL may be changed during driver push by disambiguation code.
# revert change from changeset:2297 to always use storeurl
#name = os.path.basename(resource_name) or tounicode(url2localpath(os.path.basename(storeurl)))
name = tounicode(url2localpath(os.path.basename(storeurl)))
resource.set('name', join_subpath(name, sub))
resource.set('value', join_subpath(storeurl, sub))
log.debug('_save_store: %s', etree.tostring(resource))
else:
# Try to reference the data in place (if on safe store)
storeurl, localpath = self._save_storerefs(store, storepath,
resource, rooturl)
#Store url may have changed due to conflict
if resource.get('value') is None: # This is multifile
name = os.path.basename(resource_name)
else:
name = tounicode(url2localpath(os.path.basename(storeurl)))
resource.set('name', join_subpath(name, sub))
# Update the store path reference to similar to the storeurl
storepath = storepath.split('/')
storepath[-1] = os.path.basename(storeurl)
log.debug('_save_store: %s %s %s', storeurl, localpath,
etree.tostring(resource))
return storeurl, storepath, localpath
def _force_storeurl_local(self, storeurl):
"""User available drivers to fetch a local copy of the storeurl and return path
@param storeurl: a valid storeurl
@return : (A local fileyststem path, rootportion) or None
"""
# KGK: temporary simplification
if storeurl.startswith('file://'):
return url2localpath(storeurl)
return None
def _local(self, storeurl):
"Make local path (converting local relative to full path)"
path,sub = split_subpath(storeurl)
if not path.startswith('file:///'):
if path.startswith('file://'):
path = os.path.join(self.top, path.replace('file://', ''))
else:
path = os.path.join(self.top, path)
path = url2localpath(path.replace('\\', '/'))
return path, sub
def __init__(self, mount_url=None, top = None, readonly=False, **kw):
"""Create a local storage driver:
:param path: format_path for how to store files
:param top: allow old style (relatave path file paths)
:param readonly: set repo readonly
"""
# posixpath.join '' force ending with /
self.mount_url = posixpath.join(mount_url,'')
self.mount_path = posixpath.join (url2localpath (self.mount_url),'')
datadir = data_url_path()
for key, value in kw.items():
setattr(self, key, string.Template(value).safe_substitute(datadir=datadir))
#self.top = posixpath.join(top or self.mount_url, '')
self.readonly = asbool(readonly)
if top:
self.top = posixpath.join(string.Template(top).safe_substitute(datadir=datadir), '')
self.top_path = posixpath.join(url2localpath(self.top), '')
else:
self.top = None
self.top_path = ''
self.options = kw
def delete(self, storeurl):
#ident,_ = split_subpath(ident) # reference counting required?
path,_sub = split_subpath(storeurl)
if not path.startswith('file:///'):
if path.startswith('file://'):
path = os.path.join(self.top, path.replace('file://', ''))
else:
path = os.path.join(self.top, path)
path = url2localpath(path.replace('\\', '/'))
log.info("local deleting %s", path)
if os.path.isfile (path):
try:
os.remove (path)
except OSError:
log.exception("Could not delete %s", path)
def pull (self, storeurl, localpath=None, blocking=True):
"Pull a store file to a local location"
#log.debug('local_store localpath: %s', path)
path,sub = split_subpath(storeurl)
if not path.startswith('file:///'):
if path.startswith('file://'):
path = os.path.join(self.top, path.replace('file://', ''))
else:
path = os.path.join(self.top, path)
path = url2localpath(path.replace('\\', '/'))
#log.debug('local_store localpath path: %s', path)
# if path is a directory, list contents
files = None
if os.path.isdir(path):
files = walk_deep(path)
files = sorted(files, key=blocked_alpha_num_sort) # use alpha-numeric block sort
elif not os.path.exists(path):
# No file at location .. fail
return None
# local storage can't extract sub paths, pass it along
return Blobs(path=path, sub=sub, files=files)
def url2local(path):
"decode url into a local path"
return url2localpath(path)
def relative(self, storeurl):
path = url2localpath (self.valid (storeurl))
log.debug ("MOUNT %s PATH %s",self.mount_path, path)
return path.replace (self.mount_path, '')
