class SwiftClient(object):
"""Client for Swift object/blob store of Openstack
See http://swift.openstack.org
Swift requires environment variables (OS_*) for the authentication and configuration"""
def __init__(self, container, prefix=''):
self.container = container
self.prefix = prefix
self.client = SwiftService()
def download(self, source, target):
objects = [self.prefix + '/' + source]
options = {'out_file': target}
return list(self.client.download(self.container, objects, options))
def upload(self, source, target):
object_name = self.prefix + '/' + target
objects = [SwiftUploadObject(source, object_name=object_name)]
return list(self.client.upload(self.container, objects))
def ls(self, path):
fpath = self.prefix + '/' + path + '/'
clisting = self.client.list(self.container, {'prefix': fpath})
listing = list(clisting)[0]['listing']
result = [d['name'].replace(fpath, '') for d in listing]
return result
def url(self, path=''):
return self.container + '/' + self.prefix + '/' + path
def getObjsBackend(objs, backend, config):
if(backend == 'hdfs'):
client = Client(socket.gethostname(), config['HADOOP_RPC_PORT'], use_trash=False)
for obj in objs:
try:
copy_gen = client.copyToLocal([obj[0]], obj[1])
for copy_item in copy_gen:
pass
except Exception as e:
print(e)
elif(backend == 'swift'):
options = {'os_auth_url': os.environ['OS_AUTH_URL'], 'os_username': os.environ['OS_USERNAME'], 'os_password': os.environ['OS_PASSWORD'], 'os_tenant_id': os.environ['OS_TENANT_ID'], 'os_tenant_name': os.environ['OS_TENANT_NAME']}
swiftService = SwiftService(options=options)
for obj in objs:
# Create the containers which are used in this application for Object Storage
if(obj[0] == 'sqlite.db'):
swiftService.post(container='containerFiles')
swiftService.post(container='containerFeatures')
swiftService.post(container='containerModules')
out_file = obj[1] # Get the output file location from runner
localoptions = {'out_file': out_file}
objects = []
objects.append(obj[0])
swiftDownload = swiftService.download(container='containerModules', objects=objects, options=localoptions)
for downloaded in swiftDownload:
if("error" in downloaded.keys()):
raise RuntimeError(downloaded['error'])
# print(downloaded)
elif(backend == 'nfs'): # Every file is already in respective local dirs
pass
class SwiftStorage(Storage):
"""Storage on OpenStack swift service."""
def __init__(self, storage_id, container_name, auth_config=None, transfer_config=None):
super(SwiftStorage, self).__init__(storage_id)
opts = transfer_config or {}
opts["auth_version"] = "2.0"
if auth_config:
for k, v in six.iteritems(auth_config):
opts[k] = v
self._client = SwiftService(opts)
self._container = container_name
def _get_file_safe(self, remote_path, local_path):
tmpdir = tempfile.mkdtemp()
results = self._client.download(container=self._container,
objects=[remote_path],
options={"out_directory": tmpdir})
has_results = False
for r in results:
has_results = True
if not r["success"]:
raise RuntimeError("Cannot download [%s]: %s" % (remote_path, r["error"]))
timestamp = float(r["response_dict"]["headers"]["x-timestamp"])
os.utime(os.path.join(tmpdir, remote_path), (timestamp, timestamp))
if not has_results:
raise RuntimeError("Cannot copy download [%s]" % (remote_path, "NO RESULT"))
shutil.move(os.path.join(tmpdir, remote_path), local_path)
shutil.rmtree(tmpdir, ignore_errors=True)
def _check_existing_file(self, remote_path, local_path):
(local_dir, basename) = os.path.split(local_path)
if os.path.exists(local_path):
results = self._client.stat(self._container, objects=[remote_path])
local_stat = os.stat(local_path)
for r in results:
if r['success']:
if int(r['headers']['content-length']) != local_stat.st_size:
return False
timestamp = float(r["headers"]["x-timestamp"])
if int(local_stat.st_mtime) == int(timestamp):
return True
else:
LOGGER.debug('Cannot find %s or %s', local_path)
return False
def stat(self, remote_path):
if not remote_path.endswith('/'):
results = self._client.stat(self._container, objects=[remote_path])
for r in results:
if r['success']:
return {'is_dir': False,
'size': r['headers']['content-length'],
'last_modified': r['headers']['x-timestamp']}
remote_path += '/'
results = self._client.list(container=self._container, options={"prefix": remote_path,
"delimiter": "/"})
for r in results:
if r['success']:
return {'is_dir': True}
return False
def push_file(self, local_path, remote_path):
(local_dir, basename) = os.path.split(local_path)
obj = SwiftUploadObject(local_path, object_name=remote_path)
results = self._client.upload(self._container, [obj])
has_results = False
for r in results:
has_results = True
if not r["success"]:
raise RuntimeError("Cannot push file [%s]>[%s]: %s" % (local_path, remote_path, r["error"]))
if not has_results:
raise RuntimeError("Cannot push file [%s]>[%s]: %s" % (local_path, remote_path, "NO RESULTS"))
def stream(self, remote_path, buffer_size=1024):
def generate():
tmpdir = tempfile.mkdtemp()
results = self._client.download(container=self._container,
objects=[remote_path],
options={"out_directory": tmpdir})
has_results = False
for r in results:
has_results = True
if not r["success"]:
raise RuntimeError("Cannot download file [%s]: %s", (remote_path, r["error"]))
if not has_results:
raise RuntimeError("Cannot download file [%s]: NO RESULTS", (remote_path))
with open(os.path.join(tmpdir, remote_path), "rb") as f:
for chunk in iter(lambda: f.read(buffer_size), b''):
yield chunk
shutil.rmtree(tmpdir, ignore_errors=True)
return generate()
def listdir(self, remote_path, recursive=False):
options = {"prefix": remote_path}
if not recursive:
options["delimiter"] = "/"
list_parts_gen = self._client.list(container=self._container,
options=options)
lsdir = {}
for page in list_parts_gen:
if page["success"]:
for item in page["listing"]:
if "subdir" in item:
lsdir[item["subdir"]] = {'is_dir': True}
else:
path = item["name"]
last_modified = datetime.strptime(item["last_modified"], '%Y-%m-%dT%H:%M:%S.%f')
lsdir[path] = {'size': item["bytes"],
'last_modified': datetime.timestamp(last_modified)}
return lsdir
def mkdir(self, remote_path):
pass
def _delete_single(self, remote_path, isdir):
if not isdir:
results = self._client.delete(container=self._container, objects=[remote_path])
has_results = False
for r in results:
has_results = True
if not r["success"]:
raise RuntimeError("Cannot delete file [%s]: %s" % (remote_path, r["error"]))
if not has_results:
raise RuntimeError("Cannot delete file [%s]: NO RESULT" % (remote_path))
def rename(self, old_remote_path, new_remote_path):
listfiles = self.listdir(old_remote_path, True)
for f in listfiles:
assert f[:len(old_remote_path)] == old_remote_path, "inconsistent listdir result"
obj = SwiftCopyObject(f, {"destination": "/%s/%s%s" % (
self._container,
new_remote_path,
f[len(old_remote_path):])})
results = self._client.copy(self._container, [obj])
has_results = False
for r in results:
has_results = True
if not r["success"]:
raise RuntimeError("Cannot copy file [%s]: %s" % (old_remote_path, r["error"]))
if not has_results:
raise RuntimeError("Cannot copy file [%s]: NO RESULT" % (old_remote_path))
self._delete_single(f, False)
def exists(self, remote_path):
result = self._client.list(container=self._container, options={"prefix": remote_path,
"delimiter": "/"})
for page in result:
if page["success"]:
for item in page["listing"]:
if "subdir" in item:
return True
if (item["name"] == remote_path or
remote_path == '' or
remote_path.endswith('/') or
item["name"].startswith(remote_path + '/')):
return True
return False
def isdir(self, remote_path):
if not remote_path.endswith('/'):
return self.exists(remote_path+'/')
return self.exists(remote_path)
def _internal_path(self, path):
# OpenStack does not work with paths but keys. This function possibly adapts a
# path-like representation to a OpenStack key.
if path.startswith('/'):
return path[1:]
return path
class SwiftObjectStorageDriver(ObjectStorageDriver):
"""
Archive driver using swift-api as backing store.
Buckets presented as part of object lookup in this API are mapped to object key prefixes in the backing S3 store so that a single bucket (or set of buckets)
can be used since namespaces are limited.
"""
__config_name__ = 'swift'
__driver_version__ = '1'
__uri_scheme__ = 'swift'
_key_format = '{prefix}{userid}/{container}/{key}'
DEFAULT_AUTH_TIMEOUT = 10
def __init__(self, config):
super(SwiftObjectStorageDriver, self).__init__(config)
# Initialize the client
self.client_config = config
self.container_name = self.config.get('container')
self.can_create_container = self.config.get('create_container', False)
self.auth_options = copy.copy(self.client_config)
if 'container' in self.auth_options:
self.auth_options.pop('container')
if 'create_container' in self.auth_options:
self.auth_options.pop('create_container')
self.client = SwiftService(options=self.auth_options)
if not self.container_name:
raise ValueError(
'Cannot configure swift driver with out a provided container to use'
)
self.prefix = self.config.get('anchore_key_prefix', '')
self._check_creds()
self._check_container()
def _check_creds(self):
"""
Simple operation to verify creds work without state change
:return: True on success
"""
try:
resp = self.client.stat()
if resp['success']:
return True
elif resp.get('error') and resp.get('error').http_status in [
401, 403
]:
raise BadCredentialsError(self.auth_options,
endpoint=None,
cause=resp.get('error'))
elif resp.get('error'):
raise DriverConfigurationError(cause=resp.get('error'))
else:
raise DriverConfigurationError(
Exception(
'Got unsuccessful response from stat operation against service: {}'
.format(resp)))
except SwiftError as e:
raise DriverConfigurationError(e)
def _check_container(self):
try:
resp = self.client.stat(container=self.container_name)
except SwiftError as e:
if e.exception.http_status == 404 and self.can_create_container:
try:
self.client.post(container=self.container_name)
except Exception as e:
logger.exception(e)
raise e
else:
raise DriverConfigurationError(e)
def _build_key(self, userId, usrBucket, key):
return self._key_format.format(prefix=self.prefix,
userid=userId,
container=usrBucket,
key=key)
def _parse_uri(self, uri):
parsed = urllib.parse.urlparse(uri, scheme=self.__uri_scheme__)
container = parsed.hostname
key = parsed.path[1:] # Strip leading '/'
return container, key
def get_by_uri(self, uri):
try:
container, key = self._parse_uri(uri)
if container != self.container_name:
logger.warn(
'Container mismatch between content_uri and configured cotnainer name: {} in db record, but {} in config'
.format(container, self.container_name))
resp = self.client.download(container=container,
objects=[key],
options={'out_file': '-'})
for obj in resp:
if 'contents' in obj and obj['action'] == 'download_object':
content = b''.join([x for x in obj['contents']])
ret = utils.ensure_bytes(content)
return (ret)
elif obj['action'] == 'download_object' and not obj['success']:
raise ObjectKeyNotFoundError(bucket='',
key='',
userId='',
caused_by=None)
raise Exception(
'Unexpected operation/action from swift: {}'.format(
obj['action']))
except SwiftError as e:
raise ObjectStorageDriverError(cause=e)
def delete_by_uri(self, uri):
try:
container, key = self._parse_uri(uri)
if container != self.container_name:
logger.warn(
'Container mismatch between content_uri and configured bucket name: {} in db record, but {} in config'
.format(container, self.container_name))
resp = self.client.delete(container=container, objects=[key])
for r in resp:
if r['success'] and r['action'] == 'delete_object':
return True
except Exception as e:
raise e
def exists(self, uri):
try:
container, key = self._parse_uri(uri)
if container != self.container_name:
logger.warn(
'Bucket mismatch between content_uri and configured bucket name: {} in db record, but {} in config'
.format(container, self.container_name))
resp = self.client.download(container=container,
objects=[key],
options={
'out_file': '-',
'no_download': True
})
for obj in resp:
if 'success' in obj and obj['success'] and obj[
'action'] == 'download_object':
return True
elif obj['action'] == 'download_object' and not obj['success']:
return False
raise Exception(
'Unexpected operation/action from swift: {}'.format(
obj['action']))
except SwiftError as e:
raise ObjectStorageDriverError(cause=e)
def get(self, userId, bucket, key):
return self.get_by_uri(self.uri_for(userId, bucket, key))
def put(self, userId, bucket, key, data):
try:
uri = self.uri_for(userId, bucket, key)
swift_bucket, swift_key = self._parse_uri(uri)
obj = SwiftUploadObject(object_name=swift_key,
source=io.BytesIO(data))
resp = self.client.upload(container=swift_bucket, objects=[obj])
for upload in resp:
if upload['action'] == 'upload_object' and upload['success']:
return uri
else:
raise Exception('Failed uploading object to swift')
except Exception as e:
raise e
def delete(self, userId, bucket, key):
return self.delete_by_uri(self.uri_for(userId, bucket, key))
def uri_for(self, userId, bucket, key):
return '{}://{}/{}'.format(self.__uri_scheme__, self.container_name,
self._build_key(userId, bucket, key))
def download(source, dest=None):
"""
Download files from a SWIFT object store
@param source_info: Source information such as the source to download from.
@type source_info: a dictionary
@param local_dest_dir: The local directory to store file
@type local_dest_dir: str
@return: True and a list of file downloaded if successful. Otherwise False.
"""
if not dest:
dest = tempfile.mkstemp()
url = urlsplit(source['url'])
_, ver, account, container, object_name = url.path.split('/', 4)
swift_opts = {
'os_storage_url': '{scheme}://{netloc}/{ver}/{account}'.format(
scheme=re.sub(r'^swift\+', '', url.scheme),
netloc=url.netloc,
ver=ver,
account=account)
}
# SwiftService knows about environment variables
for opt in ('os_auth_url', 'os_username', 'os_password', 'os_tenant_name', 'os_storage_url'):
if opt in source:
swift_opts[opt] = source[opt]
try:
swift = SwiftService(swift_opts)
filelist = []
if os.path.exists(dest) and os.path.isdir(dest):
outfilename = os.path.join(dest, os.path.basename(object_name))
else:
outfilename = dest
retries = 5 # number of retries left
backoff = 30 # wait time between retries
backoff_inc = 30 # increase in wait time per retry
while retries:
filelist = []
retries -= 1
try:
for result in swift.download(container, [object_name], {'out_file': outfilename}):
# result dict: success
# action: 'download_object'
# success: True
# container: ...
# object: ...
# path: ....
# pseudodir: ...
# start_time, finish_time, headers_receipt, auth_end_time,
# read_length, attempts, response_dict
# result dict: error
# action: 'download_object'
# success: False
# error: ...
# traceback: ...
# container, object, error_timestamp, response_dict, path
# psudodir, attempts
if not result['success']:
raise Exception(
'Download from selfelfwift {container}/{object} to {out_file} failed with {error}'.format(out_file=outfilename, **result))
outfile = {'url': outfilename,
'name': os.path.basename(object_name),
'content_type': result['response_dict']['headers'].get('content-type', 'application/octet-stream')}
filelist.append(outfile)
# no exception we can continue
retries = 0
except Exception as e:
if not retries:
# reraise if no retries left
raise
LOG.warn("Download from Swift failed: %s - %d retries left", e, retries)
time.sleep(backoff)
backoff += backoff_inc
backoff_inc += 30
return filelist
except Exception as e:
LOG.error("Download from Swift failed: %s", e, exc_info=True)
raise
class Swift(object):
def __init__(self, config):
self.config = config
options = self._init_swift_options()
options['object_uu_threads'] = 20
self.swift = SwiftService(options=options)
container_names = []
try:
list_account_part = self.swift.list()
for page in list_account_part:
if page["success"]:
for item in page["listing"]:
i_name = item["name"]
container_names.append(i_name)
if i_name == self.config["swift_container"]:
print("using SWIFT",
self.config["swift_container"], "container:",
item)
else:
logging.error(
"error listing SWIFT object storage containers")
except SwiftError as e:
logging.exception("error listing containers")
if self.config["swift_container"] not in container_names:
# create the container
try:
self.swift.post(container=self.config["swift_container"])
except SwiftError:
logging.exception(
"error creating SWIFT object storage container " +
self.config["swift_container"])
else:
logging.debug(
"container already exists on SWIFT object storage: " +
self.config["swift_container"])
def _init_swift_options(self):
options = {}
for key in self.config["swift"]:
if len(self.config["swift"][key].strip()) > 0:
options[key] = self.config["swift"][key]
return options
def upload_file_to_swift(self, file_path, dest_path=None):
"""
Upload the given file to current SWIFT object storage container
"""
objs = []
# file object
file_name = os.path.basename(file_path)
object_name = file_name
if dest_path != None:
object_name = dest_path + "/" + file_name
obj = SwiftUploadObject(file_path, object_name=object_name)
objs.append(obj)
try:
for result in self.swift.upload(self.config["swift_container"],
objs):
if not result['success']:
error = result['error']
if result['action'] == "upload_object":
logging.error(
"Failed to upload object %s to container %s: %s" %
(self.config["swift_container"], result['object'],
error))
else:
logging.error("%s" % error)
except SwiftError:
logging.exception("error uploading file to SWIFT container")
def upload_files_to_swift(self, file_paths, dest_path=None):
"""
Bulk upload of a list of files to current SWIFT object storage container under the same destination path
"""
objs = []
# file object
for file_path in file_paths:
file_name = os.path.basename(file_path)
object_name = file_name
if dest_path != None:
object_name = dest_path + "/" + file_name
obj = SwiftUploadObject(file_path, object_name=object_name)
objs.append(obj)
try:
for result in self.swift.upload(self.config["swift_container"],
objs):
if not result['success']:
error = result['error']
if result['action'] == "upload_object":
logging.error(
"Failed to upload object %s to container %s: %s" %
(self.config["swift_container"], result['object'],
error))
else:
logging.error("%s" % error)
except SwiftError:
logging.exception("error uploading file to SWIFT container")
def download_file(self, file_path, dest_path):
"""
Download a file given a path and returns the download destination file path.
"""
objs = [file_path]
try:
for down_res in self.swift.download(
container=self.config["swift_container"], objects=objs):
if down_res['success']:
#print("'%s' downloaded" % down_res['object'])
local_path = down_res['path']
#print(local_path)
shutil.move(local_path, dest_path)
else:
logging.error("'%s' download failed" % down_res['object'])
except SwiftError:
logging.exception("error downloading file from SWIFT container")
def get_swift_list(self, dir_name=None):
"""
Return all contents of a given dir in SWIFT object storage.
Goes through the pagination to obtain all file names.
afaik, this is terribly inefficient, as we have to go through all the objects of the storage.
"""
result = []
try:
list_parts_gen = self.swift.list(
container=self.config["swift_container"])
for page in list_parts_gen:
if page["success"]:
for item in page["listing"]:
if dir_name == None or item["name"].startswith(
dir_name):
result.append(item["name"])
else:
logging.error(page["error"])
except SwiftError as e:
logger.error(e.value)
return result
def remove_file(self, file_path):
"""
Remove an existing file on the SWIFT object storage
"""
try:
objs = [file_path]
for result in self.swift.delete(self.config["swift_container"],
objs):
if not result['success']:
error = result['error']
if result['action'] == "delete_object":
logging.error(
"Failed to delete object %s from container %s: %s"
% (self.config["swift_container"],
result['object'], error))
else:
logging.error("%s" % error)
except SwiftError:
logging.exception("error removing file from SWIFT container")
def remove_all_files(self):
"""
Remove all the existing files on the SWIFT object storage
"""
try:
list_parts_gen = self.swift.list(
container=self.config["swift_container"])
for page in list_parts_gen:
if page["success"]:
to_delete = []
for item in page["listing"]:
to_delete.append(item["name"])
for del_res in self.swift.delete(
container=self.config["swift_container"],
objects=to_delete):
if not del_res['success']:
error = del_res['error']
if del_res['action'] == "delete_object":
logging.error(
"Failed to delete object %s from container %s: %s"
% (self.config["swift_container"],
del_res['object'], error))
else:
logging.error("%s" % error)
except SwiftError:
logging.exception("error removing all files from SWIFT container")
class SwiftFS(HasTraits):
container = Unicode(os.environ.get('CONTAINER', 'demo'))
storage_url = Unicode(help="The base URL for containers",
default_value='http://example.com',
config=True)
delimiter = Unicode("/", help="Path delimiter", config=True)
root_dir = Unicode("/", config=True)
log = logging.getLogger('SwiftFS')
def __init__(self, **kwargs):
super(self.__class__, self).__init__(**kwargs)
# With the python swift client, the connection is automagically
# created using environment variables (I know... horrible or what?)
self.log.info("using swift container `%s`", self.container)
# open connection to swift container
self.swift = SwiftService()
# make sure container exists
try:
result = self.swift.post(container=self.container)
except SwiftError as e:
self.log.error("creating container %s", e.value)
raise HTTPError(404, e.value)
if not result["success"]:
msg = "could not create container %s" % self.container
self.log.error(msg)
raise HTTPError(404, msg)
# see 'list' at https://docs.openstack.org/developer/python-swiftclient/service-api.html
# Returns a list of all objects that start with the prefix given
# Of course, in a proper heirarchical file-system, list-dir only returns the files
# in that dir, so we need to filter the list to me ONLY those objects where the
# 'heirarchical' bit of the name stops at the path given
# The method has 2 modes: 1 when the list of names is returned with the full
# path-name, and one where the name is just the "file name"
@LogMethodResults()
def listdir(self, path="", with_prefix=False, this_dir_only=True):
"""
list all the "files" in the "directory" for the given path.
If the 'this_dir_only' is False (it is True by default), then
the full list of all objects in that path are returned (needed for a
rename, for example)
returns a list of dictionaries for each object:
{'bytes': 11,
'hash': '3e25960a79dbc69b674cd4ec67a72c62',
'last_modified': '2017-06-06T08:55:36.473Z',
'name': 'foo/bar/thingamy.bob'}
"""
files = []
# Get all objects that match the known path
path = self.clean_path(path)
_opts = {'prefix': path}
try:
dir_listing = self.swift.list(container=self.container,
options=_opts)
for page in dir_listing: # each page is up to 10,000 items
if page["success"]:
files.extend(page["listing"]) # page is returning a list
else:
raise page["error"]
except SwiftError as e:
self.log.error("SwiftFS.listdir %s", e.value)
if this_dir_only:
# make up the pattern to compile into our regex engine
regex_delim = re.escape(self.delimiter)
if len(path) > 0:
regex_path = re.escape(path.rstrip(self.delimiter))
pattern = '^({0}{1}[^{1}]+{1}?|{0})$'.format(
regex_path, regex_delim)
else:
pattern = '^[^{0}]+{0}?$'.format(regex_delim)
self.log.debug("restrict directory pattern is: `%s`", pattern)
regex = re.compile(pattern, re.UNICODE)
new_files = []
for f in files:
if regex.match(f['name']):
new_files.append(f)
files = new_files
return files
# We can 'stat' files, but not directories
@LogMethodResults()
def isfile(self, path):
if path is None or path == '':
self.log.debug("SwiftFS.isfile has no path, returning False")
return False
_isfile = False
if not path.endswith(self.delimiter):
path = self.clean_path(path)
try:
response = self.swift.stat(container=self.container,
objects=[path])
except Exception as e:
self.log.error("SwiftFS.isfile %s", e.value)
for r in response:
if r['success']:
_isfile = True
else:
self.log.error('Failed to retrieve stats for %s' %
r['object'])
break
return _isfile
# We can 'list' direcotries, but not 'stat' them
@LogMethodResults()
def isdir(self, path):
# directories mush have a trailing slash on them.
# The core code seems to remove any trailing slash, so lets add it back
# on
if not path.endswith(self.delimiter):
path = path + self.delimiter
# Root directory checks
if path == self.delimiter: # effectively root directory
self.log.debug("SwiftFS.isdir found root dir - returning True")
return True
_isdir = False
path = self.clean_path(path)
_opts = {}
if re.search('\w', path):
_opts = {'prefix': path}
try:
self.log.debug("SwiftFS.isdir setting prefix to '%s'", path)
response = self.swift.list(container=self.container, options=_opts)
except SwiftError as e:
self.log.error("SwiftFS.isdir %s", e.value)
for r in response:
if r['success']:
_isdir = True
else:
self.log.error('Failed to retrieve stats for %s' % path)
break
return _isdir
@LogMethod()
def cp(self, old_path, new_path):
self._copymove(old_path, new_path, with_delete=False)
@LogMethod()
def mv(self, old_path, new_path):
self._copymove(old_path, new_path, with_delete=True)
@LogMethod()
def remove_container(self):
response = {}
try:
response = self.swift.stat(container=self.container)
except SwiftError as e:
self.log.error("SwiftFS.remove_container %s", e.value)
if 'success' in response and response['success'] == True:
try:
response = self.swift.delete(container=self.container)
except SwiftError as e:
self.log.error("SwiftFS.remove_container %s", e.value)
for r in response:
self.log.debug("SwiftFS.rm action: `%s` success: `%s`",
r['action'], r['success'])
@LogMethod()
def rm(self, path, recursive=False):
if path in ["", self.delimiter]:
self.do_error('Cannot delete root directory', code=400)
return False
if not (self.isdir(path) or self.isfile(path)):
return False
if recursive:
for f in self._walk_path(path, dir_first=True):
self.log.debug("SwiftFS.rm recurse into `%s`", f)
self.rm(f)
self.log.info("SwiftFS.rm and now remove `%s`", path)
self.rm(path)
else:
self.log.info("SwiftFS.rm not recursing for `%s`", path)
files = self.listdir(path)
isEmpty = True
if len(files) > 1:
isEmpty = False
if len(files) == 1 and files[0]['name'] != path:
isEmpty = False
if not isEmpty:
self.do_error("directory %s not empty" % path, code=400)
path = self.clean_path(path)
try:
response = self.swift.delete(container=self.container,
objects=[path])
except SwiftError as e:
self.log.error("SwiftFS.rm %s", e.value)
return False
for r in response:
self.log.debug("SwiftFS.rm action: `%s` success: `%s`",
r['action'], r['success'])
return True
@LogMethod()
def _walk_path(self, path, dir_first=False):
if not dir_first:
yield path
for f in self.listdir(path):
if not dir_first:
yield f['name']
if self.guess_type(f['name']) == 'directory':
for ff in self._walk_path(f['name'], dir_first=dir_first):
yield ff
if dir_first:
yield f['name']
if dir_first:
yield path
# core function to copy or move file-objects
# does clever recursive stuff for directory trees
@LogMethod()
def _copymove(self, old_path, new_path, with_delete=False):
# check parent directory exists
self.checkParentDirExists(new_path)
for f in self._walk_path(old_path):
new_f = f.replace(old_path, new_path, 1)
if self.guess_type(f) == 'directory':
self.mkdir(new_f)
else:
old_path = self.clean_path(old_path)
new_path = self.clean_path(new_path)
try:
response = self.swift.copy(
self.container, [f], {
'destination':
self.delimiter + self.container + self.delimiter +
new_f
})
except SwiftError as e:
self.log.error(e.value)
raise
for r in response:
if r["success"]:
if r["action"] == "copy_object":
self.log.debug("object %s copied from /%s/%s" %
(r["destination"], r["container"],
r["object"]))
if r["action"] == "create_container":
self.log.debug("container %s created" %
r["container"])
else:
if "error" in r and isinstance(r["error"], Exception):
raise r["error"]
# we always test for delete: file or directory...
if with_delete:
self.rm(old_path, recursive=True)
# Directories are just objects that have a trailing '/'
@LogMethod()
def mkdir(self, path):
path = path.rstrip(self.delimiter)
path = path + self.delimiter
self._do_write(path, None)
# This works by downloading the file to disk then reading the contents of
# that file into memory, before deleting the file
# NOTE this is reading text files!
# NOTE this really only works with files in the local direcotry, but given
# local filestore will disappear when the docker ends, I'm not too bothered.
@LogMethod()
def read(self, path):
if self.guess_type(path) == "directory":
msg = "cannot read from path %s: it is a directory" % path
self.do_error(msg, code=400)
content = ''
fhandle, localFile = tempfile.mkstemp(prefix="swiftfs_")
os.close(fhandle)
path = self.clean_path(path)
try:
response = self.swift.download(container=self.container,
objects=[path],
options={"out_file": localFile})
except SwiftError as e:
self.log.error("SwiftFS.read %s", e.value)
return ''
for r in response:
if r['success']:
self.log.debug("SwiftFS.read: using local file %s", localFile)
with open(localFile) as lf:
content = lf.read()
os.remove(localFile)
return content
# Write is 'upload' and 'upload' needs a "file" it can read from
# We use io.StringIO for this
@LogMethod()
def write(self, path, content):
if self.guess_type(path) == "directory":
msg = "cannot write to path %s: it is a directory" % path
self.do_error(msg, code=400)
#path = self.clean_path(path)
# If we can't make the directory path, then we can't make the file!
#success = self._make_intermedate_dirs(path)
self._do_write(path, content)
@LogMethod()
def _make_intermedate_dirs(self, path):
# we loop over the path, checking for an object at every level
# of the hierachy, except the last item (which may be a file,
# or a directory itself
path_parts = re.split(self.delimiter, path)
current_path = ''
for p in path_parts[:-1]:
this_path = current_path + p + self.delimiter
if self.isfile(this_path):
self.log.error(
"SwiftFS._make_intermedate_dirs failure: dir exists at path `%s`"
% this_path)
return False
if not self.isdir(this_path):
self.log.debug(
"SwiftFS._make_intermedate_dirs making directory")
self._do_write(this_path, None)
current_path = this_path
return True
@LogMethod()
def _do_write(self, path, content):
# check parent directory exists
self.checkParentDirExists(path)
type = self.guess_type(path)
things = []
if type == "directory":
self.log.debug("SwiftFS._do_write create directory")
things.append(SwiftUploadObject(None, object_name=path))
else:
self.log.debug("SwiftFS._do_write create file/notebook from '%s'",
content)
output = io.BytesIO(content.encode('utf-8'))
things.append(SwiftUploadObject(output, object_name=path))
# Now do the upload
path = self.clean_path(path)
try:
response = self.swift.upload(self.container, things)
except SwiftError as e:
self.log.error("SwiftFS._do_write swift-error: %s", e.value)
raise
except ClientException as e:
self.log.error("SwiftFS._do_write client-error: %s", e.value)
raise
for r in response:
self.log.debug("SwiftFS._do_write action: '%s', response: '%s'",
r['action'], r['success'])
@LogMethodResults()
def guess_type(self, path, allow_directory=True):
"""
Guess the type of a file.
If allow_directory is False, don't consider the possibility that the
file is a directory.
Parameters
----------
path: string
"""
_type = ''
if path.endswith(".ipynb"):
_type = "notebook"
elif allow_directory and path.endswith(self.delimiter):
_type = "directory"
elif allow_directory and self.isdir(path):
_type = "directory"
else:
_type = "file"
return _type
@LogMethod()
def clean_path(self, path):
# strip of any leading '/'
path = path.lstrip(self.delimiter)
if self.guess_type(path) == 'directory':
# ensure we have a / at the end of directory paths
path = path.rstrip(self.delimiter) + self.delimiter
if path == self.delimiter:
path = ''
return path
@LogMethodResults()
def checkParentDirExists(self, path):
"""checks if the parent directory of a path exists"""
p = path.strip(self.delimiter)
p = p.split(self.delimiter)[:-1]
p = self.delimiter.join(p)
self.log.debug("SwiftFS.checkDirExists: directory name %s", p)
if not self.isdir(p):
self.do_error('parent directory does not exist %s' % p, code=400)
@LogMethod()
def do_error(self, msg, code=500):
self.log.error(msg)
raise HTTPError(code, msg)
