class SwiftClient(object):
"""Client for Swift object/blob store of Openstack
See http://swift.openstack.org
Swift requires environment variables (OS_*) for the authentication and configuration"""
def __init__(self, container, prefix=''):
self.container = container
self.prefix = prefix
self.client = SwiftService()
def download(self, source, target):
objects = [self.prefix + '/' + source]
options = {'out_file': target}
return list(self.client.download(self.container, objects, options))
def upload(self, source, target):
object_name = self.prefix + '/' + target
objects = [SwiftUploadObject(source, object_name=object_name)]
return list(self.client.upload(self.container, objects))
def ls(self, path):
fpath = self.prefix + '/' + path + '/'
clisting = self.client.list(self.container, {'prefix': fpath})
listing = list(clisting)[0]['listing']
result = [d['name'].replace(fpath, '') for d in listing]
return result
def url(self, path=''):
return self.container + '/' + self.prefix + '/' + path
class SwiftStorage(Storage):
"""Storage on OpenStack swift service."""
def __init__(self, storage_id, container_name, auth_config=None, transfer_config=None):
super(SwiftStorage, self).__init__(storage_id)
opts = transfer_config or {}
opts["auth_version"] = "2.0"
if auth_config:
for k, v in six.iteritems(auth_config):
opts[k] = v
self._client = SwiftService(opts)
self._container = container_name
def _get_file_safe(self, remote_path, local_path):
tmpdir = tempfile.mkdtemp()
results = self._client.download(container=self._container,
objects=[remote_path],
options={"out_directory": tmpdir})
has_results = False
for r in results:
has_results = True
if not r["success"]:
raise RuntimeError("Cannot download [%s]: %s" % (remote_path, r["error"]))
timestamp = float(r["response_dict"]["headers"]["x-timestamp"])
os.utime(os.path.join(tmpdir, remote_path), (timestamp, timestamp))
if not has_results:
raise RuntimeError("Cannot copy download [%s]" % (remote_path, "NO RESULT"))
shutil.move(os.path.join(tmpdir, remote_path), local_path)
shutil.rmtree(tmpdir, ignore_errors=True)
def _check_existing_file(self, remote_path, local_path):
(local_dir, basename) = os.path.split(local_path)
if os.path.exists(local_path):
results = self._client.stat(self._container, objects=[remote_path])
local_stat = os.stat(local_path)
for r in results:
if r['success']:
if int(r['headers']['content-length']) != local_stat.st_size:
return False
timestamp = float(r["headers"]["x-timestamp"])
if int(local_stat.st_mtime) == int(timestamp):
return True
else:
LOGGER.debug('Cannot find %s or %s', local_path)
return False
def stat(self, remote_path):
if not remote_path.endswith('/'):
results = self._client.stat(self._container, objects=[remote_path])
for r in results:
if r['success']:
return {'is_dir': False,
'size': r['headers']['content-length'],
'last_modified': r['headers']['x-timestamp']}
remote_path += '/'
results = self._client.list(container=self._container, options={"prefix": remote_path,
"delimiter": "/"})
for r in results:
if r['success']:
return {'is_dir': True}
return False
def push_file(self, local_path, remote_path):
(local_dir, basename) = os.path.split(local_path)
obj = SwiftUploadObject(local_path, object_name=remote_path)
results = self._client.upload(self._container, [obj])
has_results = False
for r in results:
has_results = True
if not r["success"]:
raise RuntimeError("Cannot push file [%s]>[%s]: %s" % (local_path, remote_path, r["error"]))
if not has_results:
raise RuntimeError("Cannot push file [%s]>[%s]: %s" % (local_path, remote_path, "NO RESULTS"))
def stream(self, remote_path, buffer_size=1024):
def generate():
tmpdir = tempfile.mkdtemp()
results = self._client.download(container=self._container,
objects=[remote_path],
options={"out_directory": tmpdir})
has_results = False
for r in results:
has_results = True
if not r["success"]:
raise RuntimeError("Cannot download file [%s]: %s", (remote_path, r["error"]))
if not has_results:
raise RuntimeError("Cannot download file [%s]: NO RESULTS", (remote_path))
with open(os.path.join(tmpdir, remote_path), "rb") as f:
for chunk in iter(lambda: f.read(buffer_size), b''):
yield chunk
shutil.rmtree(tmpdir, ignore_errors=True)
return generate()
def listdir(self, remote_path, recursive=False):
options = {"prefix": remote_path}
if not recursive:
options["delimiter"] = "/"
list_parts_gen = self._client.list(container=self._container,
options=options)
lsdir = {}
for page in list_parts_gen:
if page["success"]:
for item in page["listing"]:
if "subdir" in item:
lsdir[item["subdir"]] = {'is_dir': True}
else:
path = item["name"]
last_modified = datetime.strptime(item["last_modified"], '%Y-%m-%dT%H:%M:%S.%f')
lsdir[path] = {'size': item["bytes"],
'last_modified': datetime.timestamp(last_modified)}
return lsdir
def mkdir(self, remote_path):
pass
def _delete_single(self, remote_path, isdir):
if not isdir:
results = self._client.delete(container=self._container, objects=[remote_path])
has_results = False
for r in results:
has_results = True
if not r["success"]:
raise RuntimeError("Cannot delete file [%s]: %s" % (remote_path, r["error"]))
if not has_results:
raise RuntimeError("Cannot delete file [%s]: NO RESULT" % (remote_path))
def rename(self, old_remote_path, new_remote_path):
listfiles = self.listdir(old_remote_path, True)
for f in listfiles:
assert f[:len(old_remote_path)] == old_remote_path, "inconsistent listdir result"
obj = SwiftCopyObject(f, {"destination": "/%s/%s%s" % (
self._container,
new_remote_path,
f[len(old_remote_path):])})
results = self._client.copy(self._container, [obj])
has_results = False
for r in results:
has_results = True
if not r["success"]:
raise RuntimeError("Cannot copy file [%s]: %s" % (old_remote_path, r["error"]))
if not has_results:
raise RuntimeError("Cannot copy file [%s]: NO RESULT" % (old_remote_path))
self._delete_single(f, False)
def exists(self, remote_path):
result = self._client.list(container=self._container, options={"prefix": remote_path,
"delimiter": "/"})
for page in result:
if page["success"]:
for item in page["listing"]:
if "subdir" in item:
return True
if (item["name"] == remote_path or
remote_path == '' or
remote_path.endswith('/') or
item["name"].startswith(remote_path + '/')):
return True
return False
def isdir(self, remote_path):
if not remote_path.endswith('/'):
return self.exists(remote_path+'/')
return self.exists(remote_path)
def _internal_path(self, path):
# OpenStack does not work with paths but keys. This function possibly adapts a
# path-like representation to a OpenStack key.
if path.startswith('/'):
return path[1:]
return path
class Swift(object):
def __init__(self, settings):
options = get_os_settings(settings)
self.swift = SwiftService(options)
options = get_swift_settings(settings)
self.temp_url_key = options['temp_url_key']
# TODO: hard coded template
self.name_template = options['container'] + '_{name}'
# TODO: this should be a module level method
def build_object_name(self, user_id, path='', name=None):
# - if name is None, return full folder path with trailing slash
# - the returned does not have a leading slash
if not user_id or '/' in user_id:
raise ValueError('Invalid userid', user_id)
container = self.name_template.format(name=user_id)
parts = []
if path:
# disallow '..'
if '..' in path:
raise ValueError('Invalid path', path)
# strip all leading trailing slashes from path
# deduplicate double slashes
path = '/'.join(x for x in path.split('/') if x)
if path:
parts.append(path)
if name:
# build a file path
if '/' in name or name in ('..', '.'):
raise ValueError('Invalid name', name)
parts.append(name)
else:
# ensure we get a trailing slash if there is no name
# -> it is a folder
parts.append('')
return container, '/'.join(parts)
def _create_container(self, container):
return self.swift.post(
container=container,
options={
# swiftservice converts this to X-Container-Meta-Temp-Url-Key
'meta': {
'temp-url-key': self.temp_url_key,
# TODO: hard coded 10G quota
'quota-bytes': str(int(10e9)),
'access-control-allow-origin': '*',
}
})
def stat(self, user_id, path=''):
container, object_prefix = self.build_object_name(user_id, path)
if path:
# object stat requested
pass
else:
# container stat requested
try:
stat = self.swift.stat(container=container)
except SwiftError as e:
if e.exception.http_status == 404:
# container does not exists
res = self._create_container(container)
stat = self.swift.stat(container=container)
else:
raise
headers = stat['headers']
return {
'used':
safe_int(headers.get('x-container-bytes-used', None)),
'quota':
safe_int(headers.get('x-container-meta-quota-bytes', None)),
'count':
safe_int(headers.get('x-container-object-count', None)),
'created':
safe_isodate(headers.get('x-timestamp', None)),
}
return {stat.items}
def list(self, user_id, path=''):
container, object_prefix = self.build_object_name(user_id, path)
for data in self.swift.list(container=container,
options={
'delimiter': '/',
'prefix': object_prefix
}):
if data['action'] == ['list_container_part'
] and not data['success']:
data = self._create_container(container)
if data['success']:
for item in data['listing']:
# filter current folder
if item.get('subdir', None) == object_prefix:
# ignore current directory
continue
elif item.get('name', None) == object_prefix:
# ignore the current directory
continue
else:
if item.get('subdir', None):
# it is a pseudo dir
yield {
'name':
item.get('subdir')[len(object_prefix):].strip(
'/'),
'bytes':
0,
'content_type':
'application/directory',
}
else:
item['name'] = item['name'][len(object_prefix):]
yield item
# skip error handling below
continue
# TODO: we are raising an exception here... jumping out fo the
# generator.... should be fine for this method, but
# does this have the potential to leak threads?
# SwiftService uses threads to generate results
ex = data['error']
if isinstance(ex, SwiftError):
if not path and ex.exception.http_status == 404:
# ex.exception should be a ClientException, not found
# if path is empty, we ignore it, it means, the
# user container does not exist yet.
break
raise ex
def create_folder(self, user_id, path='', description=None):
container, object_path = self.build_object_name(user_id, path)
# create upload object
object_path = SwiftUploadObject(None,
object_name=object_path,
options={
'dir_marker': True,
'meta': {
'description': description
or '',
},
})
folders = []
for res in self.swift.upload(container, [object_path]):
if not res['success']:
raise res['error']
if res['action'] == 'create_container':
# if res['response_dict']['reason'] == 'Created'
# status will be 202 if container already existed
if res['response_dict']['status'] == 201:
# set up metadata for user container
res = self._create_container(container)
# TODO: project only:
if res['action'] == 'create_dir_marker':
meta = {}
if description:
meta['description'] = description
folder = SwiftPostObject(object_name=res['object'],
options={
'header': res['headers'],
'meta': meta,
})
folders.append(folder)
# TODO: check whether we should use post above instead of upload
# maybe we can avoid calling swift twice?
# also woke sure container get's created in case of post
ret = []
for res in self.swift.post(container, folders):
if not res['success']:
raise res['error']
ret.append(res)
return ret
def delete_folder(self, user_id, path=''):
container, object_path = self.build_object_name(user_id, path)
# don't use delimiter here, otherwise swift.delete will only see
# one level of subfolders and won't be able to delete everything
# TODO: can this delete the container as well?
for res in self.swift.delete(container=container,
options={'prefix': object_path}):
yield res['object'][len(object_path):]
def upload_file(self,
user_id,
path,
name,
file,
content_type='application/octet-stream',
content_length=-1):
container, object_name = self.build_object_name(user_id, path, name)
# prepend account and container to path
headers = {'Content-Type': content_type or 'application/octet-stream'}
# if content_length >= 0:
# headers['Content-Length'] = str(content_length)
upload_obj = SwiftUploadObject(source=LengthWrapper(
file, content_length, True),
object_name=object_name,
options={'header': headers})
log = logging.getLogger(__name__)
log.info('Tool Upload %s', upload_obj)
for res in self.swift.upload(container, [upload_obj]):
if res['action'] == 'create_container':
res = self._create_container(container)
# Getting a funny response iterator here
# 1. action: create_container
# 2. action: upload_object
log.info('Tool Result %s', res)
if res.get('error', None):
# res['error'].http_status == 413:
# -> Request Entity Too Large
# res['error'].http_resonse_content == 'Upload exceeds quota'
raise res['error']
def delete_file(self, user_id, path, name):
container, object_name = self.build_object_name(user_id, path, name)
# TODO: could set options['prefix'] to make sure we don't delete
# anything outside project/folder
# TODO: coould this delete the container?
res = self.swift.delete(container=container, objects=[object_name])
for res in self.swift.delete(container=container,
objects=[object_name]):
if res.get('error', None):
raise res['error']
def generate_temp_url(self, user_id, path, name):
container, object_name = self.build_object_name(user_id, path, name)
# discover swift endpoint urls
conn = get_conn(self.swift._options)
url, token = conn.get_auth()
urlparts = urlparse(url)
# generate swift path /v1/<account>/<container>/<userid>/path
path = '/'.join((urlparts.path, container, object_name))
# TODO: valid for 5 minutes
temp_url = generate_temp_url(path,
300,
self.temp_url_key,
method='GET')
return urljoin(url, temp_url)
class Swift(object):
def __init__(self, config):
self.config = config
options = self._init_swift_options()
options['object_uu_threads'] = 20
self.swift = SwiftService(options=options)
container_names = []
try:
list_account_part = self.swift.list()
for page in list_account_part:
if page["success"]:
for item in page["listing"]:
i_name = item["name"]
container_names.append(i_name)
if i_name == self.config["swift_container"]:
print("using SWIFT",
self.config["swift_container"], "container:",
item)
else:
logging.error(
"error listing SWIFT object storage containers")
except SwiftError as e:
logging.exception("error listing containers")
if self.config["swift_container"] not in container_names:
# create the container
try:
self.swift.post(container=self.config["swift_container"])
except SwiftError:
logging.exception(
"error creating SWIFT object storage container " +
self.config["swift_container"])
else:
logging.debug(
"container already exists on SWIFT object storage: " +
self.config["swift_container"])
def _init_swift_options(self):
options = {}
for key in self.config["swift"]:
if len(self.config["swift"][key].strip()) > 0:
options[key] = self.config["swift"][key]
return options
def upload_file_to_swift(self, file_path, dest_path=None):
"""
Upload the given file to current SWIFT object storage container
"""
objs = []
# file object
file_name = os.path.basename(file_path)
object_name = file_name
if dest_path != None:
object_name = dest_path + "/" + file_name
obj = SwiftUploadObject(file_path, object_name=object_name)
objs.append(obj)
try:
for result in self.swift.upload(self.config["swift_container"],
objs):
if not result['success']:
error = result['error']
if result['action'] == "upload_object":
logging.error(
"Failed to upload object %s to container %s: %s" %
(self.config["swift_container"], result['object'],
error))
else:
logging.error("%s" % error)
except SwiftError:
logging.exception("error uploading file to SWIFT container")
def upload_files_to_swift(self, file_paths, dest_path=None):
"""
Bulk upload of a list of files to current SWIFT object storage container under the same destination path
"""
objs = []
# file object
for file_path in file_paths:
file_name = os.path.basename(file_path)
object_name = file_name
if dest_path != None:
object_name = dest_path + "/" + file_name
obj = SwiftUploadObject(file_path, object_name=object_name)
objs.append(obj)
try:
for result in self.swift.upload(self.config["swift_container"],
objs):
if not result['success']:
error = result['error']
if result['action'] == "upload_object":
logging.error(
"Failed to upload object %s to container %s: %s" %
(self.config["swift_container"], result['object'],
error))
else:
logging.error("%s" % error)
except SwiftError:
logging.exception("error uploading file to SWIFT container")
def download_file(self, file_path, dest_path):
"""
Download a file given a path and returns the download destination file path.
"""
objs = [file_path]
try:
for down_res in self.swift.download(
container=self.config["swift_container"], objects=objs):
if down_res['success']:
#print("'%s' downloaded" % down_res['object'])
local_path = down_res['path']
#print(local_path)
shutil.move(local_path, dest_path)
else:
logging.error("'%s' download failed" % down_res['object'])
except SwiftError:
logging.exception("error downloading file from SWIFT container")
def get_swift_list(self, dir_name=None):
"""
Return all contents of a given dir in SWIFT object storage.
Goes through the pagination to obtain all file names.
afaik, this is terribly inefficient, as we have to go through all the objects of the storage.
"""
result = []
try:
list_parts_gen = self.swift.list(
container=self.config["swift_container"])
for page in list_parts_gen:
if page["success"]:
for item in page["listing"]:
if dir_name == None or item["name"].startswith(
dir_name):
result.append(item["name"])
else:
logging.error(page["error"])
except SwiftError as e:
logger.error(e.value)
return result
def remove_file(self, file_path):
"""
Remove an existing file on the SWIFT object storage
"""
try:
objs = [file_path]
for result in self.swift.delete(self.config["swift_container"],
objs):
if not result['success']:
error = result['error']
if result['action'] == "delete_object":
logging.error(
"Failed to delete object %s from container %s: %s"
% (self.config["swift_container"],
result['object'], error))
else:
logging.error("%s" % error)
except SwiftError:
logging.exception("error removing file from SWIFT container")
def remove_all_files(self):
"""
Remove all the existing files on the SWIFT object storage
"""
try:
list_parts_gen = self.swift.list(
container=self.config["swift_container"])
for page in list_parts_gen:
if page["success"]:
to_delete = []
for item in page["listing"]:
to_delete.append(item["name"])
for del_res in self.swift.delete(
container=self.config["swift_container"],
objects=to_delete):
if not del_res['success']:
error = del_res['error']
if del_res['action'] == "delete_object":
logging.error(
"Failed to delete object %s from container %s: %s"
% (self.config["swift_container"],
del_res['object'], error))
else:
logging.error("%s" % error)
except SwiftError:
logging.exception("error removing all files from SWIFT container")
class SwiftFS(HasTraits):
container = Unicode(os.environ.get('CONTAINER', 'demo'))
storage_url = Unicode(help="The base URL for containers",
default_value='http://example.com',
config=True)
delimiter = Unicode("/", help="Path delimiter", config=True)
root_dir = Unicode("/", config=True)
log = logging.getLogger('SwiftFS')
def __init__(self, **kwargs):
super(self.__class__, self).__init__(**kwargs)
# With the python swift client, the connection is automagically
# created using environment variables (I know... horrible or what?)
self.log.info("using swift container `%s`", self.container)
# open connection to swift container
self.swift = SwiftService()
# make sure container exists
try:
result = self.swift.post(container=self.container)
except SwiftError as e:
self.log.error("creating container %s", e.value)
raise HTTPError(404, e.value)
if not result["success"]:
msg = "could not create container %s" % self.container
self.log.error(msg)
raise HTTPError(404, msg)
# see 'list' at https://docs.openstack.org/developer/python-swiftclient/service-api.html
# Returns a list of all objects that start with the prefix given
# Of course, in a proper heirarchical file-system, list-dir only returns the files
# in that dir, so we need to filter the list to me ONLY those objects where the
# 'heirarchical' bit of the name stops at the path given
# The method has 2 modes: 1 when the list of names is returned with the full
# path-name, and one where the name is just the "file name"
@LogMethodResults()
def listdir(self, path="", with_prefix=False, this_dir_only=True):
"""
list all the "files" in the "directory" for the given path.
If the 'this_dir_only' is False (it is True by default), then
the full list of all objects in that path are returned (needed for a
rename, for example)
returns a list of dictionaries for each object:
{'bytes': 11,
'hash': '3e25960a79dbc69b674cd4ec67a72c62',
'last_modified': '2017-06-06T08:55:36.473Z',
'name': 'foo/bar/thingamy.bob'}
"""
files = []
# Get all objects that match the known path
path = self.clean_path(path)
_opts = {'prefix': path}
try:
dir_listing = self.swift.list(container=self.container,
options=_opts)
for page in dir_listing: # each page is up to 10,000 items
if page["success"]:
files.extend(page["listing"]) # page is returning a list
else:
raise page["error"]
except SwiftError as e:
self.log.error("SwiftFS.listdir %s", e.value)
if this_dir_only:
# make up the pattern to compile into our regex engine
regex_delim = re.escape(self.delimiter)
if len(path) > 0:
regex_path = re.escape(path.rstrip(self.delimiter))
pattern = '^({0}{1}[^{1}]+{1}?|{0})$'.format(
regex_path, regex_delim)
else:
pattern = '^[^{0}]+{0}?$'.format(regex_delim)
self.log.debug("restrict directory pattern is: `%s`", pattern)
regex = re.compile(pattern, re.UNICODE)
new_files = []
for f in files:
if regex.match(f['name']):
new_files.append(f)
files = new_files
return files
# We can 'stat' files, but not directories
@LogMethodResults()
def isfile(self, path):
if path is None or path == '':
self.log.debug("SwiftFS.isfile has no path, returning False")
return False
_isfile = False
if not path.endswith(self.delimiter):
path = self.clean_path(path)
try:
response = self.swift.stat(container=self.container,
objects=[path])
except Exception as e:
self.log.error("SwiftFS.isfile %s", e.value)
for r in response:
if r['success']:
_isfile = True
else:
self.log.error('Failed to retrieve stats for %s' %
r['object'])
break
return _isfile
# We can 'list' direcotries, but not 'stat' them
@LogMethodResults()
def isdir(self, path):
# directories mush have a trailing slash on them.
# The core code seems to remove any trailing slash, so lets add it back
# on
if not path.endswith(self.delimiter):
path = path + self.delimiter
# Root directory checks
if path == self.delimiter: # effectively root directory
self.log.debug("SwiftFS.isdir found root dir - returning True")
return True
_isdir = False
path = self.clean_path(path)
_opts = {}
if re.search('\w', path):
_opts = {'prefix': path}
try:
self.log.debug("SwiftFS.isdir setting prefix to '%s'", path)
response = self.swift.list(container=self.container, options=_opts)
except SwiftError as e:
self.log.error("SwiftFS.isdir %s", e.value)
for r in response:
if r['success']:
_isdir = True
else:
self.log.error('Failed to retrieve stats for %s' % path)
break
return _isdir
@LogMethod()
def cp(self, old_path, new_path):
self._copymove(old_path, new_path, with_delete=False)
@LogMethod()
def mv(self, old_path, new_path):
self._copymove(old_path, new_path, with_delete=True)
@LogMethod()
def remove_container(self):
response = {}
try:
response = self.swift.stat(container=self.container)
except SwiftError as e:
self.log.error("SwiftFS.remove_container %s", e.value)
if 'success' in response and response['success'] == True:
try:
response = self.swift.delete(container=self.container)
except SwiftError as e:
self.log.error("SwiftFS.remove_container %s", e.value)
for r in response:
self.log.debug("SwiftFS.rm action: `%s` success: `%s`",
r['action'], r['success'])
@LogMethod()
def rm(self, path, recursive=False):
if path in ["", self.delimiter]:
self.do_error('Cannot delete root directory', code=400)
return False
if not (self.isdir(path) or self.isfile(path)):
return False
if recursive:
for f in self._walk_path(path, dir_first=True):
self.log.debug("SwiftFS.rm recurse into `%s`", f)
self.rm(f)
self.log.info("SwiftFS.rm and now remove `%s`", path)
self.rm(path)
else:
self.log.info("SwiftFS.rm not recursing for `%s`", path)
files = self.listdir(path)
isEmpty = True
if len(files) > 1:
isEmpty = False
if len(files) == 1 and files[0]['name'] != path:
isEmpty = False
if not isEmpty:
self.do_error("directory %s not empty" % path, code=400)
path = self.clean_path(path)
try:
response = self.swift.delete(container=self.container,
objects=[path])
except SwiftError as e:
self.log.error("SwiftFS.rm %s", e.value)
return False
for r in response:
self.log.debug("SwiftFS.rm action: `%s` success: `%s`",
r['action'], r['success'])
return True
@LogMethod()
def _walk_path(self, path, dir_first=False):
if not dir_first:
yield path
for f in self.listdir(path):
if not dir_first:
yield f['name']
if self.guess_type(f['name']) == 'directory':
for ff in self._walk_path(f['name'], dir_first=dir_first):
yield ff
if dir_first:
yield f['name']
if dir_first:
yield path
# core function to copy or move file-objects
# does clever recursive stuff for directory trees
@LogMethod()
def _copymove(self, old_path, new_path, with_delete=False):
# check parent directory exists
self.checkParentDirExists(new_path)
for f in self._walk_path(old_path):
new_f = f.replace(old_path, new_path, 1)
if self.guess_type(f) == 'directory':
self.mkdir(new_f)
else:
old_path = self.clean_path(old_path)
new_path = self.clean_path(new_path)
try:
response = self.swift.copy(
self.container, [f], {
'destination':
self.delimiter + self.container + self.delimiter +
new_f
})
except SwiftError as e:
self.log.error(e.value)
raise
for r in response:
if r["success"]:
if r["action"] == "copy_object":
self.log.debug("object %s copied from /%s/%s" %
(r["destination"], r["container"],
r["object"]))
if r["action"] == "create_container":
self.log.debug("container %s created" %
r["container"])
else:
if "error" in r and isinstance(r["error"], Exception):
raise r["error"]
# we always test for delete: file or directory...
if with_delete:
self.rm(old_path, recursive=True)
# Directories are just objects that have a trailing '/'
@LogMethod()
def mkdir(self, path):
path = path.rstrip(self.delimiter)
path = path + self.delimiter
self._do_write(path, None)
# This works by downloading the file to disk then reading the contents of
# that file into memory, before deleting the file
# NOTE this is reading text files!
# NOTE this really only works with files in the local direcotry, but given
# local filestore will disappear when the docker ends, I'm not too bothered.
@LogMethod()
def read(self, path):
if self.guess_type(path) == "directory":
msg = "cannot read from path %s: it is a directory" % path
self.do_error(msg, code=400)
content = ''
fhandle, localFile = tempfile.mkstemp(prefix="swiftfs_")
os.close(fhandle)
path = self.clean_path(path)
try:
response = self.swift.download(container=self.container,
objects=[path],
options={"out_file": localFile})
except SwiftError as e:
self.log.error("SwiftFS.read %s", e.value)
return ''
for r in response:
if r['success']:
self.log.debug("SwiftFS.read: using local file %s", localFile)
with open(localFile) as lf:
content = lf.read()
os.remove(localFile)
return content
# Write is 'upload' and 'upload' needs a "file" it can read from
# We use io.StringIO for this
@LogMethod()
def write(self, path, content):
if self.guess_type(path) == "directory":
msg = "cannot write to path %s: it is a directory" % path
self.do_error(msg, code=400)
#path = self.clean_path(path)
# If we can't make the directory path, then we can't make the file!
#success = self._make_intermedate_dirs(path)
self._do_write(path, content)
@LogMethod()
def _make_intermedate_dirs(self, path):
# we loop over the path, checking for an object at every level
# of the hierachy, except the last item (which may be a file,
# or a directory itself
path_parts = re.split(self.delimiter, path)
current_path = ''
for p in path_parts[:-1]:
this_path = current_path + p + self.delimiter
if self.isfile(this_path):
self.log.error(
"SwiftFS._make_intermedate_dirs failure: dir exists at path `%s`"
% this_path)
return False
if not self.isdir(this_path):
self.log.debug(
"SwiftFS._make_intermedate_dirs making directory")
self._do_write(this_path, None)
current_path = this_path
return True
@LogMethod()
def _do_write(self, path, content):
# check parent directory exists
self.checkParentDirExists(path)
type = self.guess_type(path)
things = []
if type == "directory":
self.log.debug("SwiftFS._do_write create directory")
things.append(SwiftUploadObject(None, object_name=path))
else:
self.log.debug("SwiftFS._do_write create file/notebook from '%s'",
content)
output = io.BytesIO(content.encode('utf-8'))
things.append(SwiftUploadObject(output, object_name=path))
# Now do the upload
path = self.clean_path(path)
try:
response = self.swift.upload(self.container, things)
except SwiftError as e:
self.log.error("SwiftFS._do_write swift-error: %s", e.value)
raise
except ClientException as e:
self.log.error("SwiftFS._do_write client-error: %s", e.value)
raise
for r in response:
self.log.debug("SwiftFS._do_write action: '%s', response: '%s'",
r['action'], r['success'])
@LogMethodResults()
def guess_type(self, path, allow_directory=True):
"""
Guess the type of a file.
If allow_directory is False, don't consider the possibility that the
file is a directory.
Parameters
----------
path: string
"""
_type = ''
if path.endswith(".ipynb"):
_type = "notebook"
elif allow_directory and path.endswith(self.delimiter):
_type = "directory"
elif allow_directory and self.isdir(path):
_type = "directory"
else:
_type = "file"
return _type
@LogMethod()
def clean_path(self, path):
# strip of any leading '/'
path = path.lstrip(self.delimiter)
if self.guess_type(path) == 'directory':
# ensure we have a / at the end of directory paths
path = path.rstrip(self.delimiter) + self.delimiter
if path == self.delimiter:
path = ''
return path
@LogMethodResults()
def checkParentDirExists(self, path):
"""checks if the parent directory of a path exists"""
p = path.strip(self.delimiter)
p = p.split(self.delimiter)[:-1]
p = self.delimiter.join(p)
self.log.debug("SwiftFS.checkDirExists: directory name %s", p)
if not self.isdir(p):
self.do_error('parent directory does not exist %s' % p, code=400)
@LogMethod()
def do_error(self, msg, code=500):
self.log.error(msg)
raise HTTPError(code, msg)
## SwiftService interface
# SwiftService API will actually use your environment auth variables if they are not specified in the script,
# but here is how to specify them explicitly:
options = {
"auth_version": '2.0',
"os_username": username,
"os_password": password,
"os_tenant_name": project_name,
"os_auth_url": auth_url,
}
swift_service = SwiftService(options=options)
result = swift_service.list()
for item in result:
for container in item['listing']:
print container
## Connection interface
swift_conn = Connection(authurl=auth_url,
user=username,
key=password,
tenant_name=project_name,
auth_version=2)
headers, containers = swift_conn.get_account()
for con in containers:
print con
