def swift_container(): from swiftclient.service import SwiftService, SwiftError container_name = test_swift_container # Initialize the client v1_client_config = { 'user': test_swift_user, 'key': test_swift_key, 'auth': test_swift_auth_url, } client = SwiftService(options=v1_client_config) try: logger.info('Initializing a swift container: {}'.format(test_swift_container)) client.post(container=container_name) yield container_name, client finally: logger.info('Deleting/cleanup a swift container: {}'.format(test_swift_container)) try: client.delete(container=container_name) except Exception as e: logger.exception('Container cleanup exception') logger.warn('Error cleaning up swift container {}, may result in other test failures'.format(container_name))
def main(global_config, **settings): """ This function returns a Pyramid WSGI application. """ config = Configurator(settings=settings) # app specific stuff config.add_route(name='wps', pattern='/wps') config.add_route(name='outputs', pattern='/outputs/*filename') config.add_route(name='status', pattern='/status/*filename') # web routes config.add_static_view('static', 'static', cache_max_age=3600) config.add_route('home', '/') config.scan('.views') # ensure paths exist for name in ('workdir', 'statuspath', 'outputpath'): dirname = os.path.abspath(wpsconfig.get_config_value('server', name)) if not os.path.exists(dirname): os.makedirs(dirname) # TODO: init swift container here? # initialize swift storage container if active if wpsconfig.get_config_value('server', 'storage') == 'SwiftStorage': from swiftclient.service import SwiftService, SwiftError swift = SwiftService() container = wpsconfig.get_config_value('SwiftStorage', 'container') try: stat = swift.stat(container) except SwiftError as e: # e.exception.http_status should be 404 # create container res = swift.post(container) # res['success'] sholud be True stat = swift.stat(container) # we sholud have stat for container now from ecocloud_wps_demo.pywps.swiftstorage import get_temp_url_key cur_key = stat['headers'].get('x-container-meta-temp-url-key') temp_url_key = get_temp_url_key() if cur_key != temp_url_key: # setting temp_url_key res = swift.post(container, options={'meta': {'temp-url-key': temp_url_key}}) # res['success'] == True return config.make_wsgi_app()
def getObjsBackend(objs, backend, config): if(backend == 'hdfs'): client = Client(socket.gethostname(), config['HADOOP_RPC_PORT'], use_trash=False) for obj in objs: try: copy_gen = client.copyToLocal([obj[0]], obj[1]) for copy_item in copy_gen: pass except Exception as e: print(e) elif(backend == 'swift'): options = {'os_auth_url': os.environ['OS_AUTH_URL'], 'os_username': os.environ['OS_USERNAME'], 'os_password': os.environ['OS_PASSWORD'], 'os_tenant_id': os.environ['OS_TENANT_ID'], 'os_tenant_name': os.environ['OS_TENANT_NAME']} swiftService = SwiftService(options=options) for obj in objs: # Create the containers which are used in this application for Object Storage if(obj[0] == 'sqlite.db'): swiftService.post(container='containerFiles') swiftService.post(container='containerFeatures') swiftService.post(container='containerModules') out_file = obj[1] # Get the output file location from runner localoptions = {'out_file': out_file} objects = [] objects.append(obj[0]) swiftDownload = swiftService.download(container='containerModules', objects=objects, options=localoptions) for downloaded in swiftDownload: if("error" in downloaded.keys()): raise RuntimeError(downloaded['error']) # print(downloaded) elif(backend == 'nfs'): # Every file is already in respective local dirs pass
class Swift(object): def __init__(self, settings): options = get_os_settings(settings) self.swift = SwiftService(options) options = get_swift_settings(settings) self.temp_url_key = options['temp_url_key'] # TODO: hard coded template self.name_template = options['container'] + '_{name}' # TODO: this should be a module level method def build_object_name(self, user_id, path='', name=None): # - if name is None, return full folder path with trailing slash # - the returned does not have a leading slash if not user_id or '/' in user_id: raise ValueError('Invalid userid', user_id) container = self.name_template.format(name=user_id) parts = [] if path: # disallow '..' if '..' in path: raise ValueError('Invalid path', path) # strip all leading trailing slashes from path # deduplicate double slashes path = '/'.join(x for x in path.split('/') if x) if path: parts.append(path) if name: # build a file path if '/' in name or name in ('..', '.'): raise ValueError('Invalid name', name) parts.append(name) else: # ensure we get a trailing slash if there is no name # -> it is a folder parts.append('') return container, '/'.join(parts) def _create_container(self, container): return self.swift.post( container=container, options={ # swiftservice converts this to X-Container-Meta-Temp-Url-Key 'meta': { 'temp-url-key': self.temp_url_key, # TODO: hard coded 10G quota 'quota-bytes': str(int(10e9)), 'access-control-allow-origin': '*', } }) def stat(self, user_id, path=''): container, object_prefix = self.build_object_name(user_id, path) if path: # object stat requested pass else: # container stat requested try: stat = self.swift.stat(container=container) except SwiftError as e: if e.exception.http_status == 404: # container does not exists res = self._create_container(container) stat = self.swift.stat(container=container) else: raise headers = stat['headers'] return { 'used': safe_int(headers.get('x-container-bytes-used', None)), 'quota': safe_int(headers.get('x-container-meta-quota-bytes', None)), 'count': safe_int(headers.get('x-container-object-count', None)), 'created': safe_isodate(headers.get('x-timestamp', None)), } return {stat.items} def list(self, user_id, path=''): container, object_prefix = self.build_object_name(user_id, path) for data in self.swift.list(container=container, options={ 'delimiter': '/', 'prefix': object_prefix }): if data['action'] == ['list_container_part' ] and not data['success']: data = self._create_container(container) if data['success']: for item in data['listing']: # filter current folder if item.get('subdir', None) == object_prefix: # ignore current directory continue elif item.get('name', None) == object_prefix: # ignore the current directory continue else: if item.get('subdir', None): # it is a pseudo dir yield { 'name': item.get('subdir')[len(object_prefix):].strip( '/'), 'bytes': 0, 'content_type': 'application/directory', } else: item['name'] = item['name'][len(object_prefix):] yield item # skip error handling below continue # TODO: we are raising an exception here... jumping out fo the # generator.... should be fine for this method, but # does this have the potential to leak threads? # SwiftService uses threads to generate results ex = data['error'] if isinstance(ex, SwiftError): if not path and ex.exception.http_status == 404: # ex.exception should be a ClientException, not found # if path is empty, we ignore it, it means, the # user container does not exist yet. break raise ex def create_folder(self, user_id, path='', description=None): container, object_path = self.build_object_name(user_id, path) # create upload object object_path = SwiftUploadObject(None, object_name=object_path, options={ 'dir_marker': True, 'meta': { 'description': description or '', }, }) folders = [] for res in self.swift.upload(container, [object_path]): if not res['success']: raise res['error'] if res['action'] == 'create_container': # if res['response_dict']['reason'] == 'Created' # status will be 202 if container already existed if res['response_dict']['status'] == 201: # set up metadata for user container res = self._create_container(container) # TODO: project only: if res['action'] == 'create_dir_marker': meta = {} if description: meta['description'] = description folder = SwiftPostObject(object_name=res['object'], options={ 'header': res['headers'], 'meta': meta, }) folders.append(folder) # TODO: check whether we should use post above instead of upload # maybe we can avoid calling swift twice? # also woke sure container get's created in case of post ret = [] for res in self.swift.post(container, folders): if not res['success']: raise res['error'] ret.append(res) return ret def delete_folder(self, user_id, path=''): container, object_path = self.build_object_name(user_id, path) # don't use delimiter here, otherwise swift.delete will only see # one level of subfolders and won't be able to delete everything # TODO: can this delete the container as well? for res in self.swift.delete(container=container, options={'prefix': object_path}): yield res['object'][len(object_path):] def upload_file(self, user_id, path, name, file, content_type='application/octet-stream', content_length=-1): container, object_name = self.build_object_name(user_id, path, name) # prepend account and container to path headers = {'Content-Type': content_type or 'application/octet-stream'} # if content_length >= 0: # headers['Content-Length'] = str(content_length) upload_obj = SwiftUploadObject(source=LengthWrapper( file, content_length, True), object_name=object_name, options={'header': headers}) log = logging.getLogger(__name__) log.info('Tool Upload %s', upload_obj) for res in self.swift.upload(container, [upload_obj]): if res['action'] == 'create_container': res = self._create_container(container) # Getting a funny response iterator here # 1. action: create_container # 2. action: upload_object log.info('Tool Result %s', res) if res.get('error', None): # res['error'].http_status == 413: # -> Request Entity Too Large # res['error'].http_resonse_content == 'Upload exceeds quota' raise res['error'] def delete_file(self, user_id, path, name): container, object_name = self.build_object_name(user_id, path, name) # TODO: could set options['prefix'] to make sure we don't delete # anything outside project/folder # TODO: coould this delete the container? res = self.swift.delete(container=container, objects=[object_name]) for res in self.swift.delete(container=container, objects=[object_name]): if res.get('error', None): raise res['error'] def generate_temp_url(self, user_id, path, name): container, object_name = self.build_object_name(user_id, path, name) # discover swift endpoint urls conn = get_conn(self.swift._options) url, token = conn.get_auth() urlparts = urlparse(url) # generate swift path /v1/<account>/<container>/<userid>/path path = '/'.join((urlparts.path, container, object_name)) # TODO: valid for 5 minutes temp_url = generate_temp_url(path, 300, self.temp_url_key, method='GET') return urljoin(url, temp_url)
class SwiftObjectStorageDriver(ObjectStorageDriver): """ Archive driver using swift-api as backing store. Buckets presented as part of object lookup in this API are mapped to object key prefixes in the backing S3 store so that a single bucket (or set of buckets) can be used since namespaces are limited. """ __config_name__ = 'swift' __driver_version__ = '1' __uri_scheme__ = 'swift' _key_format = '{prefix}{userid}/{container}/{key}' DEFAULT_AUTH_TIMEOUT = 10 def __init__(self, config): super(SwiftObjectStorageDriver, self).__init__(config) # Initialize the client self.client_config = config self.container_name = self.config.get('container') self.can_create_container = self.config.get('create_container', False) self.auth_options = copy.copy(self.client_config) if 'container' in self.auth_options: self.auth_options.pop('container') if 'create_container' in self.auth_options: self.auth_options.pop('create_container') self.client = SwiftService(options=self.auth_options) if not self.container_name: raise ValueError( 'Cannot configure swift driver with out a provided container to use' ) self.prefix = self.config.get('anchore_key_prefix', '') self._check_creds() self._check_container() def _check_creds(self): """ Simple operation to verify creds work without state change :return: True on success """ try: resp = self.client.stat() if resp['success']: return True elif resp.get('error') and resp.get('error').http_status in [ 401, 403 ]: raise BadCredentialsError(self.auth_options, endpoint=None, cause=resp.get('error')) elif resp.get('error'): raise DriverConfigurationError(cause=resp.get('error')) else: raise DriverConfigurationError( Exception( 'Got unsuccessful response from stat operation against service: {}' .format(resp))) except SwiftError as e: raise DriverConfigurationError(e) def _check_container(self): try: resp = self.client.stat(container=self.container_name) except SwiftError as e: if e.exception.http_status == 404 and self.can_create_container: try: self.client.post(container=self.container_name) except Exception as e: logger.exception(e) raise e else: raise DriverConfigurationError(e) def _build_key(self, userId, usrBucket, key): return self._key_format.format(prefix=self.prefix, userid=userId, container=usrBucket, key=key) def _parse_uri(self, uri): parsed = urllib.parse.urlparse(uri, scheme=self.__uri_scheme__) container = parsed.hostname key = parsed.path[1:] # Strip leading '/' return container, key def get_by_uri(self, uri): try: container, key = self._parse_uri(uri) if container != self.container_name: logger.warn( 'Container mismatch between content_uri and configured cotnainer name: {} in db record, but {} in config' .format(container, self.container_name)) resp = self.client.download(container=container, objects=[key], options={'out_file': '-'}) for obj in resp: if 'contents' in obj and obj['action'] == 'download_object': content = b''.join([x for x in obj['contents']]) ret = utils.ensure_bytes(content) return (ret) elif obj['action'] == 'download_object' and not obj['success']: raise ObjectKeyNotFoundError(bucket='', key='', userId='', caused_by=None) raise Exception( 'Unexpected operation/action from swift: {}'.format( obj['action'])) except SwiftError as e: raise ObjectStorageDriverError(cause=e) def delete_by_uri(self, uri): try: container, key = self._parse_uri(uri) if container != self.container_name: logger.warn( 'Container mismatch between content_uri and configured bucket name: {} in db record, but {} in config' .format(container, self.container_name)) resp = self.client.delete(container=container, objects=[key]) for r in resp: if r['success'] and r['action'] == 'delete_object': return True except Exception as e: raise e def exists(self, uri): try: container, key = self._parse_uri(uri) if container != self.container_name: logger.warn( 'Bucket mismatch between content_uri and configured bucket name: {} in db record, but {} in config' .format(container, self.container_name)) resp = self.client.download(container=container, objects=[key], options={ 'out_file': '-', 'no_download': True }) for obj in resp: if 'success' in obj and obj['success'] and obj[ 'action'] == 'download_object': return True elif obj['action'] == 'download_object' and not obj['success']: return False raise Exception( 'Unexpected operation/action from swift: {}'.format( obj['action'])) except SwiftError as e: raise ObjectStorageDriverError(cause=e) def get(self, userId, bucket, key): return self.get_by_uri(self.uri_for(userId, bucket, key)) def put(self, userId, bucket, key, data): try: uri = self.uri_for(userId, bucket, key) swift_bucket, swift_key = self._parse_uri(uri) obj = SwiftUploadObject(object_name=swift_key, source=io.BytesIO(data)) resp = self.client.upload(container=swift_bucket, objects=[obj]) for upload in resp: if upload['action'] == 'upload_object' and upload['success']: return uri else: raise Exception('Failed uploading object to swift') except Exception as e: raise e def delete(self, userId, bucket, key): return self.delete_by_uri(self.uri_for(userId, bucket, key)) def uri_for(self, userId, bucket, key): return '{}://{}/{}'.format(self.__uri_scheme__, self.container_name, self._build_key(userId, bucket, key))
class Swift(object): def __init__(self, config): self.config = config options = self._init_swift_options() options['object_uu_threads'] = 20 self.swift = SwiftService(options=options) container_names = [] try: list_account_part = self.swift.list() for page in list_account_part: if page["success"]: for item in page["listing"]: i_name = item["name"] container_names.append(i_name) if i_name == self.config["swift_container"]: print("using SWIFT", self.config["swift_container"], "container:", item) else: logging.error( "error listing SWIFT object storage containers") except SwiftError as e: logging.exception("error listing containers") if self.config["swift_container"] not in container_names: # create the container try: self.swift.post(container=self.config["swift_container"]) except SwiftError: logging.exception( "error creating SWIFT object storage container " + self.config["swift_container"]) else: logging.debug( "container already exists on SWIFT object storage: " + self.config["swift_container"]) def _init_swift_options(self): options = {} for key in self.config["swift"]: if len(self.config["swift"][key].strip()) > 0: options[key] = self.config["swift"][key] return options def upload_file_to_swift(self, file_path, dest_path=None): """ Upload the given file to current SWIFT object storage container """ objs = [] # file object file_name = os.path.basename(file_path) object_name = file_name if dest_path != None: object_name = dest_path + "/" + file_name obj = SwiftUploadObject(file_path, object_name=object_name) objs.append(obj) try: for result in self.swift.upload(self.config["swift_container"], objs): if not result['success']: error = result['error'] if result['action'] == "upload_object": logging.error( "Failed to upload object %s to container %s: %s" % (self.config["swift_container"], result['object'], error)) else: logging.error("%s" % error) except SwiftError: logging.exception("error uploading file to SWIFT container") def upload_files_to_swift(self, file_paths, dest_path=None): """ Bulk upload of a list of files to current SWIFT object storage container under the same destination path """ objs = [] # file object for file_path in file_paths: file_name = os.path.basename(file_path) object_name = file_name if dest_path != None: object_name = dest_path + "/" + file_name obj = SwiftUploadObject(file_path, object_name=object_name) objs.append(obj) try: for result in self.swift.upload(self.config["swift_container"], objs): if not result['success']: error = result['error'] if result['action'] == "upload_object": logging.error( "Failed to upload object %s to container %s: %s" % (self.config["swift_container"], result['object'], error)) else: logging.error("%s" % error) except SwiftError: logging.exception("error uploading file to SWIFT container") def download_file(self, file_path, dest_path): """ Download a file given a path and returns the download destination file path. """ objs = [file_path] try: for down_res in self.swift.download( container=self.config["swift_container"], objects=objs): if down_res['success']: #print("'%s' downloaded" % down_res['object']) local_path = down_res['path'] #print(local_path) shutil.move(local_path, dest_path) else: logging.error("'%s' download failed" % down_res['object']) except SwiftError: logging.exception("error downloading file from SWIFT container") def get_swift_list(self, dir_name=None): """ Return all contents of a given dir in SWIFT object storage. Goes through the pagination to obtain all file names. afaik, this is terribly inefficient, as we have to go through all the objects of the storage. """ result = [] try: list_parts_gen = self.swift.list( container=self.config["swift_container"]) for page in list_parts_gen: if page["success"]: for item in page["listing"]: if dir_name == None or item["name"].startswith( dir_name): result.append(item["name"]) else: logging.error(page["error"]) except SwiftError as e: logger.error(e.value) return result def remove_file(self, file_path): """ Remove an existing file on the SWIFT object storage """ try: objs = [file_path] for result in self.swift.delete(self.config["swift_container"], objs): if not result['success']: error = result['error'] if result['action'] == "delete_object": logging.error( "Failed to delete object %s from container %s: %s" % (self.config["swift_container"], result['object'], error)) else: logging.error("%s" % error) except SwiftError: logging.exception("error removing file from SWIFT container") def remove_all_files(self): """ Remove all the existing files on the SWIFT object storage """ try: list_parts_gen = self.swift.list( container=self.config["swift_container"]) for page in list_parts_gen: if page["success"]: to_delete = [] for item in page["listing"]: to_delete.append(item["name"]) for del_res in self.swift.delete( container=self.config["swift_container"], objects=to_delete): if not del_res['success']: error = del_res['error'] if del_res['action'] == "delete_object": logging.error( "Failed to delete object %s from container %s: %s" % (self.config["swift_container"], del_res['object'], error)) else: logging.error("%s" % error) except SwiftError: logging.exception("error removing all files from SWIFT container")
class SwiftFS(HasTraits): container = Unicode(os.environ.get('CONTAINER', 'demo')) storage_url = Unicode(help="The base URL for containers", default_value='http://example.com', config=True) delimiter = Unicode("/", help="Path delimiter", config=True) root_dir = Unicode("/", config=True) log = logging.getLogger('SwiftFS') def __init__(self, **kwargs): super(self.__class__, self).__init__(**kwargs) # With the python swift client, the connection is automagically # created using environment variables (I know... horrible or what?) self.log.info("using swift container `%s`", self.container) # open connection to swift container self.swift = SwiftService() # make sure container exists try: result = self.swift.post(container=self.container) except SwiftError as e: self.log.error("creating container %s", e.value) raise HTTPError(404, e.value) if not result["success"]: msg = "could not create container %s" % self.container self.log.error(msg) raise HTTPError(404, msg) # see 'list' at https://docs.openstack.org/developer/python-swiftclient/service-api.html # Returns a list of all objects that start with the prefix given # Of course, in a proper heirarchical file-system, list-dir only returns the files # in that dir, so we need to filter the list to me ONLY those objects where the # 'heirarchical' bit of the name stops at the path given # The method has 2 modes: 1 when the list of names is returned with the full # path-name, and one where the name is just the "file name" @LogMethodResults() def listdir(self, path="", with_prefix=False, this_dir_only=True): """ list all the "files" in the "directory" for the given path. If the 'this_dir_only' is False (it is True by default), then the full list of all objects in that path are returned (needed for a rename, for example) returns a list of dictionaries for each object: {'bytes': 11, 'hash': '3e25960a79dbc69b674cd4ec67a72c62', 'last_modified': '2017-06-06T08:55:36.473Z', 'name': 'foo/bar/thingamy.bob'} """ files = [] # Get all objects that match the known path path = self.clean_path(path) _opts = {'prefix': path} try: dir_listing = self.swift.list(container=self.container, options=_opts) for page in dir_listing: # each page is up to 10,000 items if page["success"]: files.extend(page["listing"]) # page is returning a list else: raise page["error"] except SwiftError as e: self.log.error("SwiftFS.listdir %s", e.value) if this_dir_only: # make up the pattern to compile into our regex engine regex_delim = re.escape(self.delimiter) if len(path) > 0: regex_path = re.escape(path.rstrip(self.delimiter)) pattern = '^({0}{1}[^{1}]+{1}?|{0})$'.format( regex_path, regex_delim) else: pattern = '^[^{0}]+{0}?$'.format(regex_delim) self.log.debug("restrict directory pattern is: `%s`", pattern) regex = re.compile(pattern, re.UNICODE) new_files = [] for f in files: if regex.match(f['name']): new_files.append(f) files = new_files return files # We can 'stat' files, but not directories @LogMethodResults() def isfile(self, path): if path is None or path == '': self.log.debug("SwiftFS.isfile has no path, returning False") return False _isfile = False if not path.endswith(self.delimiter): path = self.clean_path(path) try: response = self.swift.stat(container=self.container, objects=[path]) except Exception as e: self.log.error("SwiftFS.isfile %s", e.value) for r in response: if r['success']: _isfile = True else: self.log.error('Failed to retrieve stats for %s' % r['object']) break return _isfile # We can 'list' direcotries, but not 'stat' them @LogMethodResults() def isdir(self, path): # directories mush have a trailing slash on them. # The core code seems to remove any trailing slash, so lets add it back # on if not path.endswith(self.delimiter): path = path + self.delimiter # Root directory checks if path == self.delimiter: # effectively root directory self.log.debug("SwiftFS.isdir found root dir - returning True") return True _isdir = False path = self.clean_path(path) _opts = {} if re.search('\w', path): _opts = {'prefix': path} try: self.log.debug("SwiftFS.isdir setting prefix to '%s'", path) response = self.swift.list(container=self.container, options=_opts) except SwiftError as e: self.log.error("SwiftFS.isdir %s", e.value) for r in response: if r['success']: _isdir = True else: self.log.error('Failed to retrieve stats for %s' % path) break return _isdir @LogMethod() def cp(self, old_path, new_path): self._copymove(old_path, new_path, with_delete=False) @LogMethod() def mv(self, old_path, new_path): self._copymove(old_path, new_path, with_delete=True) @LogMethod() def remove_container(self): response = {} try: response = self.swift.stat(container=self.container) except SwiftError as e: self.log.error("SwiftFS.remove_container %s", e.value) if 'success' in response and response['success'] == True: try: response = self.swift.delete(container=self.container) except SwiftError as e: self.log.error("SwiftFS.remove_container %s", e.value) for r in response: self.log.debug("SwiftFS.rm action: `%s` success: `%s`", r['action'], r['success']) @LogMethod() def rm(self, path, recursive=False): if path in ["", self.delimiter]: self.do_error('Cannot delete root directory', code=400) return False if not (self.isdir(path) or self.isfile(path)): return False if recursive: for f in self._walk_path(path, dir_first=True): self.log.debug("SwiftFS.rm recurse into `%s`", f) self.rm(f) self.log.info("SwiftFS.rm and now remove `%s`", path) self.rm(path) else: self.log.info("SwiftFS.rm not recursing for `%s`", path) files = self.listdir(path) isEmpty = True if len(files) > 1: isEmpty = False if len(files) == 1 and files[0]['name'] != path: isEmpty = False if not isEmpty: self.do_error("directory %s not empty" % path, code=400) path = self.clean_path(path) try: response = self.swift.delete(container=self.container, objects=[path]) except SwiftError as e: self.log.error("SwiftFS.rm %s", e.value) return False for r in response: self.log.debug("SwiftFS.rm action: `%s` success: `%s`", r['action'], r['success']) return True @LogMethod() def _walk_path(self, path, dir_first=False): if not dir_first: yield path for f in self.listdir(path): if not dir_first: yield f['name'] if self.guess_type(f['name']) == 'directory': for ff in self._walk_path(f['name'], dir_first=dir_first): yield ff if dir_first: yield f['name'] if dir_first: yield path # core function to copy or move file-objects # does clever recursive stuff for directory trees @LogMethod() def _copymove(self, old_path, new_path, with_delete=False): # check parent directory exists self.checkParentDirExists(new_path) for f in self._walk_path(old_path): new_f = f.replace(old_path, new_path, 1) if self.guess_type(f) == 'directory': self.mkdir(new_f) else: old_path = self.clean_path(old_path) new_path = self.clean_path(new_path) try: response = self.swift.copy( self.container, [f], { 'destination': self.delimiter + self.container + self.delimiter + new_f }) except SwiftError as e: self.log.error(e.value) raise for r in response: if r["success"]: if r["action"] == "copy_object": self.log.debug("object %s copied from /%s/%s" % (r["destination"], r["container"], r["object"])) if r["action"] == "create_container": self.log.debug("container %s created" % r["container"]) else: if "error" in r and isinstance(r["error"], Exception): raise r["error"] # we always test for delete: file or directory... if with_delete: self.rm(old_path, recursive=True) # Directories are just objects that have a trailing '/' @LogMethod() def mkdir(self, path): path = path.rstrip(self.delimiter) path = path + self.delimiter self._do_write(path, None) # This works by downloading the file to disk then reading the contents of # that file into memory, before deleting the file # NOTE this is reading text files! # NOTE this really only works with files in the local direcotry, but given # local filestore will disappear when the docker ends, I'm not too bothered. @LogMethod() def read(self, path): if self.guess_type(path) == "directory": msg = "cannot read from path %s: it is a directory" % path self.do_error(msg, code=400) content = '' fhandle, localFile = tempfile.mkstemp(prefix="swiftfs_") os.close(fhandle) path = self.clean_path(path) try: response = self.swift.download(container=self.container, objects=[path], options={"out_file": localFile}) except SwiftError as e: self.log.error("SwiftFS.read %s", e.value) return '' for r in response: if r['success']: self.log.debug("SwiftFS.read: using local file %s", localFile) with open(localFile) as lf: content = lf.read() os.remove(localFile) return content # Write is 'upload' and 'upload' needs a "file" it can read from # We use io.StringIO for this @LogMethod() def write(self, path, content): if self.guess_type(path) == "directory": msg = "cannot write to path %s: it is a directory" % path self.do_error(msg, code=400) #path = self.clean_path(path) # If we can't make the directory path, then we can't make the file! #success = self._make_intermedate_dirs(path) self._do_write(path, content) @LogMethod() def _make_intermedate_dirs(self, path): # we loop over the path, checking for an object at every level # of the hierachy, except the last item (which may be a file, # or a directory itself path_parts = re.split(self.delimiter, path) current_path = '' for p in path_parts[:-1]: this_path = current_path + p + self.delimiter if self.isfile(this_path): self.log.error( "SwiftFS._make_intermedate_dirs failure: dir exists at path `%s`" % this_path) return False if not self.isdir(this_path): self.log.debug( "SwiftFS._make_intermedate_dirs making directory") self._do_write(this_path, None) current_path = this_path return True @LogMethod() def _do_write(self, path, content): # check parent directory exists self.checkParentDirExists(path) type = self.guess_type(path) things = [] if type == "directory": self.log.debug("SwiftFS._do_write create directory") things.append(SwiftUploadObject(None, object_name=path)) else: self.log.debug("SwiftFS._do_write create file/notebook from '%s'", content) output = io.BytesIO(content.encode('utf-8')) things.append(SwiftUploadObject(output, object_name=path)) # Now do the upload path = self.clean_path(path) try: response = self.swift.upload(self.container, things) except SwiftError as e: self.log.error("SwiftFS._do_write swift-error: %s", e.value) raise except ClientException as e: self.log.error("SwiftFS._do_write client-error: %s", e.value) raise for r in response: self.log.debug("SwiftFS._do_write action: '%s', response: '%s'", r['action'], r['success']) @LogMethodResults() def guess_type(self, path, allow_directory=True): """ Guess the type of a file. If allow_directory is False, don't consider the possibility that the file is a directory. Parameters ---------- path: string """ _type = '' if path.endswith(".ipynb"): _type = "notebook" elif allow_directory and path.endswith(self.delimiter): _type = "directory" elif allow_directory and self.isdir(path): _type = "directory" else: _type = "file" return _type @LogMethod() def clean_path(self, path): # strip of any leading '/' path = path.lstrip(self.delimiter) if self.guess_type(path) == 'directory': # ensure we have a / at the end of directory paths path = path.rstrip(self.delimiter) + self.delimiter if path == self.delimiter: path = '' return path @LogMethodResults() def checkParentDirExists(self, path): """checks if the parent directory of a path exists""" p = path.strip(self.delimiter) p = p.split(self.delimiter)[:-1] p = self.delimiter.join(p) self.log.debug("SwiftFS.checkDirExists: directory name %s", p) if not self.isdir(p): self.do_error('parent directory does not exist %s' % p, code=400) @LogMethod() def do_error(self, msg, code=500): self.log.error(msg) raise HTTPError(code, msg)