class SwiftBackend(BakthatBackend): """Backend to handle OpenStack Swift upload/download.""" def __init__(self, conf={}, profile="default"): BakthatBackend.__init__(self, conf, profile) from swiftclient import Connection, ClientException self.con = Connection(self.conf["auth_url"], self.conf["access_key"], self.conf["secret_key"], auth_version=self.conf["auth_version"], tenant_name=self.conf["tenant_name"], insecure=self.conf["insecure"]) region_name = self.conf["region_name"] if region_name == DEFAULT_LOCATION: region_name = "" try: self.con.head_container(self.conf["s3_bucket"]) except ClientException, e: self.con.put_container(self.conf["s3_bucket"]) self.container = self.conf["s3_bucket"] self.container_key = "s3_bucket"
class SwiftStorage(object): def __init__(self, *args, **kwargs): super(SwiftStorage, self).__init__() self.__dict__.update(kwargs) config_data = get_config_json() # Load config variables. self.auth_url = config_data.get('swift').get('auth_url') self.access_key = config_data.get('swift').get('access_key') self.secret_key = config_data.get('swift').get('secret_key') self.auth_version = config_data.get('swift').get('auth_version') self.tenant_name = config_data.get('swift').get('tenant_name') self.insecure = True self.container = config_data.get('swift').get('container') self.name_backup = kwargs.get('name_backup', None) self.conn = Connection( authurl=self.auth_url, user=self.access_key, key=self.secret_key, auth_version=self.auth_version, tenant_name=self.tenant_name, insecure=self.insecure) try: self.conn.head_container(self.container) except: self.conn.put_container(self.container) if not self.name_backup: raise SystemExit(_error_codes.get(103)) def send_file(self, filename, **kwargs): try: backup_file = open(filename, 'rb') response = self.conn.put_object( self.container, filename, backup_file) print('Uploading file {0} to container "{1}" on swift'. format(filename, self.container)) except Exception, e: raise SystemExit(_error_codes.get(115).format( filename, 'Swift', e))
class SwiftStorage(object): def __init__(self, *args, **kwargs): super(SwiftStorage, self).__init__() self.__dict__.update(kwargs) config_data = get_config_json() # Load config variables. self.auth_url = config_data.get('swift').get('auth_url') self.access_key = config_data.get('swift').get('access_key') self.secret_key = config_data.get('swift').get('secret_key') self.auth_version = config_data.get('swift').get('auth_version') self.tenant_name = config_data.get('swift').get('tenant_name') self.insecure = True self.container = config_data.get('swift').get('container') self.name_backup = kwargs.get('name_backup', None) self.conn = Connection(authurl=self.auth_url, user=self.access_key, key=self.secret_key, auth_version=self.auth_version, tenant_name=self.tenant_name, insecure=self.insecure) try: self.conn.head_container(self.container) except: self.conn.put_container(self.container) if not self.name_backup: raise SystemExit(_error_codes.get(103)) def send_file(self, filename, **kwargs): try: backup_file = open(filename, 'rb') response = self.conn.put_object(self.container, filename, backup_file) print('Uploading file {0} to container "{1}" on swift'.format( filename, self.container)) except Exception, e: raise SystemExit( _error_codes.get(115).format(filename, 'Swift', e))
class PCABackend(duplicity.backend.Backend): """ Backend for OVH PCA """ def __init__(self, parsed_url): duplicity.backend.Backend.__init__(self, parsed_url) try: from swiftclient import Connection from swiftclient import ClientException except ImportError as e: raise BackendException("""\ PCA backend requires the python-swiftclient library. Exception: %s""" % str(e)) self.resp_exc = ClientException self.conn_cls = Connection conn_kwargs = {} # if the user has already authenticated if 'PCA_PREAUTHURL' in os.environ and 'PCA_PREAUTHTOKEN' in os.environ: conn_kwargs['preauthurl'] = os.environ['PCA_PREAUTHURL'] conn_kwargs['preauthtoken'] = os.environ['PCA_PREAUTHTOKEN'] else: if 'PCA_USERNAME' not in os.environ: raise BackendException('PCA_USERNAME environment variable ' 'not set.') if 'PCA_PASSWORD' not in os.environ: raise BackendException('PCA_PASSWORD environment variable ' 'not set.') if 'PCA_AUTHURL' not in os.environ: raise BackendException('PCA_AUTHURL environment variable ' 'not set.') conn_kwargs['user'] = os.environ['PCA_USERNAME'] conn_kwargs['key'] = os.environ['PCA_PASSWORD'] conn_kwargs['authurl'] = os.environ['PCA_AUTHURL'] os_options = {} if 'PCA_AUTHVERSION' in os.environ: conn_kwargs['auth_version'] = os.environ['PCA_AUTHVERSION'] if os.environ['PCA_AUTHVERSION'] == '3': if 'PCA_USER_DOMAIN_NAME' in os.environ: os_options.update({ 'user_domain_name': os.environ['PCA_USER_DOMAIN_NAME'] }) if 'PCA_USER_DOMAIN_ID' in os.environ: os_options.update( {'user_domain_id': os.environ['PCA_USER_DOMAIN_ID']}) if 'PCA_PROJECT_DOMAIN_NAME' in os.environ: os_options.update({ 'project_domain_name': os.environ['PCA_PROJECT_DOMAIN_NAME'] }) if 'PCA_PROJECT_DOMAIN_ID' in os.environ: os_options.update({ 'project_domain_id': os.environ['PCA_PROJECT_DOMAIN_ID'] }) if 'PCA_TENANTNAME' in os.environ: os_options.update( {'tenant_name': os.environ['PCA_TENANTNAME']}) if 'PCA_ENDPOINT_TYPE' in os.environ: os_options.update( {'endpoint_type': os.environ['PCA_ENDPOINT_TYPE']}) if 'PCA_USERID' in os.environ: os_options.update({'user_id': os.environ['PCA_USERID']}) if 'PCA_TENANTID' in os.environ: os_options.update( {'tenant_id': os.environ['PCA_TENANTID']}) if 'PCA_REGIONNAME' in os.environ: os_options.update( {'region_name': os.environ['PCA_REGIONNAME']}) else: conn_kwargs['auth_version'] = '2' if 'PCA_TENANTNAME' in os.environ: conn_kwargs['tenant_name'] = os.environ['PCA_TENANTNAME'] if 'PCA_REGIONNAME' in os.environ: os_options.update({'region_name': os.environ['PCA_REGIONNAME']}) conn_kwargs['os_options'] = os_options conn_kwargs['retries'] = 0 self.conn_kwargs = conn_kwargs # This folds the null prefix and all null parts, which means that: # //MyContainer/ and //MyContainer are equivalent. # //MyContainer//My/Prefix/ and //MyContainer/My/Prefix are equivalent. url_parts = [x for x in parsed_url.path.split('/') if x != ''] self.container = url_parts.pop(0) if url_parts: self.prefix = '%s/' % '/'.join(url_parts) else: self.prefix = '' policy = 'PCA' policy_header = 'X-Storage-Policy' container_metadata = None try: self.conn = Connection(**self.conn_kwargs) container_metadata = self.conn.head_container(self.container) except ClientException: pass except Exception as e: log.FatalError( "Connection failed: %s %s" % (e.__class__.__name__, str(e)), log.ErrorCode.connection_failed) if container_metadata is None: log.Info("Creating container %s" % self.container) try: headers = dict([[policy_header, policy]]) self.conn.put_container(self.container, headers=headers) except Exception as e: log.FatalError( "Container creation failed: %s %s" % (e.__class__.__name__, str(e)), log.ErrorCode.connection_failed) elif policy and container_metadata[policy_header.lower()] != policy: log.FatalError( "Container '%s' exists but its storage policy is '%s' not '%s'." % (self.container, container_metadata[policy_header.lower()], policy)) def _error_code(self, operation, e): if isinstance(e, self.resp_exc): if e.http_status == 404: return log.ErrorCode.backend_not_found def _put(self, source_path, remote_filename): self.conn.put_object(self.container, self.prefix + remote_filename, file(source_path.name)) def _get(self, remote_filename, local_path): body = self.preprocess_download(remote_filename, 60) if body: with open(local_path.name, 'wb') as f: for chunk in body: f.write(chunk) def _list(self): headers, objs = self.conn.get_container(self.container, full_listing=True, path=self.prefix) # removes prefix from return values. should check for the prefix ? return [o['name'][len(self.prefix):] for o in objs] def _delete(self, filename): self.conn.delete_object(self.container, self.prefix + filename) def _query(self, filename): sobject = self.conn.head_object(self.container, self.prefix + filename) return {'size': int(sobject['content-length'])} def preprocess_download(self, remote_filename, retry_period, wait=True): body = self.unseal(remote_filename) try: if wait: while not body: time.sleep(retry_period) self.conn = self.conn_cls(**self.conn_kwargs) body = self.unseal(remote_filename) self.conn.close() except Exception as e: log.FatalError( "Connection failed: %s %s" % (e.__class__.__name__, str(e)), log.ErrorCode.connection_failed) return body def unseal(self, remote_filename): try: _, body = self.conn.get_object(self.container, self.prefix + remote_filename, resp_chunk_size=1024) log.Info("File %s was successfully unsealed." % remote_filename) return body except self.resp_exc as e: # The object is sealed but being released. if e.http_status == 429: # The retry-after header contains the remaining duration before # the unsealing operation completes. duration = int(e.http_response_headers['Retry-After']) m, s = divmod(duration, 60) h, m = divmod(m, 60) eta = "%dh%02dm%02ds" % (h, m, s) log.Info("File %s is being unsealed, operation ETA is %s." % (remote_filename, eta)) else: raise
class SwiftBackend(duplicity.backend.Backend): """ Backend for Swift """ def __init__(self, parsed_url): duplicity.backend.Backend.__init__(self, parsed_url) try: from swiftclient import Connection from swiftclient import ClientException except ImportError as e: raise BackendException("""\ Swift backend requires the python-swiftclient library. Exception: %s""" % str(e)) self.resp_exc = ClientException conn_kwargs = {} # if the user has already authenticated if 'SWIFT_PREAUTHURL' in os.environ and 'SWIFT_PREAUTHTOKEN' in os.environ: conn_kwargs['preauthurl'] = os.environ['SWIFT_PREAUTHURL'] conn_kwargs['preauthtoken'] = os.environ['SWIFT_PREAUTHTOKEN'] else: if 'SWIFT_USERNAME' not in os.environ: raise BackendException('SWIFT_USERNAME environment variable ' 'not set.') if 'SWIFT_PASSWORD' not in os.environ: raise BackendException('SWIFT_PASSWORD environment variable ' 'not set.') if 'SWIFT_AUTHURL' not in os.environ: raise BackendException('SWIFT_AUTHURL environment variable ' 'not set.') conn_kwargs['user'] = os.environ['SWIFT_USERNAME'] conn_kwargs['key'] = os.environ['SWIFT_PASSWORD'] conn_kwargs['authurl'] = os.environ['SWIFT_AUTHURL'] os_options = {} if 'SWIFT_AUTHVERSION' in os.environ: conn_kwargs['auth_version'] = os.environ['SWIFT_AUTHVERSION'] if os.environ['SWIFT_AUTHVERSION'] == '3': if 'SWIFT_USER_DOMAIN_NAME' in os.environ: os_options.update({ 'user_domain_name': os.environ['SWIFT_USER_DOMAIN_NAME'] }) if 'SWIFT_USER_DOMAIN_ID' in os.environ: os_options.update( {'user_domain_id': os.environ['SWIFT_USER_DOMAIN_ID']}) if 'SWIFT_PROJECT_DOMAIN_NAME' in os.environ: os_options.update({ 'project_domain_name': os.environ['SWIFT_PROJECT_DOMAIN_NAME'] }) if 'SWIFT_PROJECT_DOMAIN_ID' in os.environ: os_options.update({ 'project_domain_id': os.environ['SWIFT_PROJECT_DOMAIN_ID'] }) if 'SWIFT_TENANTNAME' in os.environ: os_options.update( {'tenant_name': os.environ['SWIFT_TENANTNAME']}) if 'SWIFT_ENDPOINT_TYPE' in os.environ: os_options.update( {'endpoint_type': os.environ['SWIFT_ENDPOINT_TYPE']}) if 'SWIFT_USERID' in os.environ: os_options.update({'user_id': os.environ['SWIFT_USERID']}) if 'SWIFT_TENANTID' in os.environ: os_options.update( {'tenant_id': os.environ['SWIFT_TENANTID']}) if 'SWIFT_REGIONNAME' in os.environ: os_options.update( {'region_name': os.environ['SWIFT_REGIONNAME']}) else: conn_kwargs['auth_version'] = '1' if 'SWIFT_TENANTNAME' in os.environ: conn_kwargs['tenant_name'] = os.environ['SWIFT_TENANTNAME'] if 'SWIFT_REGIONNAME' in os.environ: os_options.update({'region_name': os.environ['SWIFT_REGIONNAME']}) conn_kwargs['os_options'] = os_options # This folds the null prefix and all null parts, which means that: # //MyContainer/ and //MyContainer are equivalent. # //MyContainer//My/Prefix/ and //MyContainer/My/Prefix are equivalent. url_parts = [x for x in parsed_url.path.split('/') if x != ''] self.container = url_parts.pop(0) if url_parts: self.prefix = '%s/' % '/'.join(url_parts) else: self.prefix = '' policy = globals.swift_storage_policy policy_header = 'X-Storage-Policy' container_metadata = None try: self.conn = Connection(**conn_kwargs) container_metadata = self.conn.head_container(self.container) except ClientException: pass except Exception as e: log.FatalError( "Connection failed: %s %s" % (e.__class__.__name__, str(e)), log.ErrorCode.connection_failed) if container_metadata is None: log.Info("Creating container %s" % self.container) try: headers = dict([[policy_header, policy]]) if policy else None self.conn.put_container(self.container, headers=headers) except Exception as e: log.FatalError( "Container creation failed: %s %s" % (e.__class__.__name__, str(e)), log.ErrorCode.connection_failed) elif policy and container_metadata[policy_header.lower()] != policy: log.FatalError( "Container '%s' exists but its storage policy is '%s' not '%s'." % (self.container, container_metadata[policy_header.lower()], policy)) def _error_code(self, operation, e): if isinstance(e, self.resp_exc): if e.http_status == 404: return log.ErrorCode.backend_not_found def _put(self, source_path, remote_filename): self.conn.put_object(self.container, self.prefix + remote_filename, file(source_path.name)) def _get(self, remote_filename, local_path): headers, body = self.conn.get_object(self.container, self.prefix + remote_filename, resp_chunk_size=1024) with open(local_path.name, 'wb') as f: for chunk in body: f.write(chunk) def _list(self): headers, objs = self.conn.get_container(self.container, full_listing=True, path=self.prefix) # removes prefix from return values. should check for the prefix ? return [o['name'][len(self.prefix):] for o in objs] def _delete(self, filename): self.conn.delete_object(self.container, self.prefix + filename) def _query(self, filename): sobject = self.conn.head_object(self.container, self.prefix + filename) return {'size': int(sobject['content-length'])}
class SwiftBackend(duplicity.backend.Backend): """ Backend for Swift """ def __init__(self, parsed_url): try: from swiftclient import Connection from swiftclient import ClientException except ImportError: raise BackendException("This backend requires " "the python-swiftclient library.") self.resp_exc = ClientException conn_kwargs = {} # if the user has already authenticated if 'SWIFT_PREAUTHURL' in os.environ and 'SWIFT_PREAUTHTOKEN' in os.environ: conn_kwargs['preauthurl'] = os.environ['SWIFT_PREAUTHURL'] conn_kwargs['preauthtoken'] = os.environ['SWIFT_PREAUTHTOKEN'] else: if 'SWIFT_USERNAME' not in os.environ: raise BackendException('SWIFT_USERNAME environment variable ' 'not set.') if 'SWIFT_PASSWORD' not in os.environ: raise BackendException('SWIFT_PASSWORD environment variable ' 'not set.') if 'SWIFT_AUTHURL' not in os.environ: raise BackendException('SWIFT_AUTHURL environment variable ' 'not set.') conn_kwargs['user'] = os.environ['SWIFT_USERNAME'] conn_kwargs['key'] = os.environ['SWIFT_PASSWORD'] conn_kwargs['authurl'] = os.environ['SWIFT_AUTHURL'] os_options = {} if 'SWIFT_AUTHVERSION' in os.environ: conn_kwargs['auth_version'] = os.environ['SWIFT_AUTHVERSION'] if os.environ['SWIFT_AUTHVERSION'] == '3': if 'SWIFT_USER_DOMAIN_NAME' in os.environ: os_options.update({'user_domain_name': os.environ['SWIFT_USER_DOMAIN_NAME']}) if 'SWIFT_USER_DOMAIN_ID' in os.environ: os_options.update({'user_domain_id': os.environ['SWIFT_USER_DOMAIN_ID']}) if 'SWIFT_PROJECT_DOMAIN_NAME' in os.environ: os_options.update({'project_domain_name': os.environ['SWIFT_PROJECT_DOMAIN_NAME']}) if 'SWIFT_PROJECT_DOMAIN_ID' in os.environ: os_options.update({'project_domain_id': os.environ['SWIFT_PROJECT_DOMAIN_ID']}) if 'SWIFT_TENANTNAME' in os.environ: os_options.update({'tenant_name': os.environ['SWIFT_TENANTNAME']}) if 'SWIFT_ENDPOINT_TYPE' in os.environ: os_options.update({'endpoint_type': os.environ['SWIFT_ENDPOINT_TYPE']}) if 'SWIFT_USERID' in os.environ: os_options.update({'user_id': os.environ['SWIFT_USERID']}) if 'SWIFT_TENANTID' in os.environ: os_options.update({'tenant_id': os.environ['SWIFT_TENANTID']}) if 'SWIFT_REGIONNAME' in os.environ: os_options.update({'region_name': os.environ['SWIFT_REGIONNAME']}) else: conn_kwargs['auth_version'] = '1' if 'SWIFT_TENANTNAME' in os.environ: conn_kwargs['tenant_name'] = os.environ['SWIFT_TENANTNAME'] if 'SWIFT_REGIONNAME' in os.environ: os_options.update({'region_name': os.environ['SWIFT_REGIONNAME']}) conn_kwargs['os_options'] = os_options # This folds the null prefix and all null parts, which means that: # //MyContainer/ and //MyContainer are equivalent. # //MyContainer//My/Prefix/ and //MyContainer/My/Prefix are equivalent. url_parts = [x for x in parsed_url.path.split('/') if x != ''] self.container = url_parts.pop(0) if url_parts: self.prefix = '%s/' % '/'.join(url_parts) else: self.prefix = '' container_metadata = None try: self.conn = Connection(**conn_kwargs) container_metadata = self.conn.head_container(self.container) except ClientException: pass except Exception as e: log.FatalError("Connection failed: %s %s" % (e.__class__.__name__, str(e)), log.ErrorCode.connection_failed) if container_metadata is None: log.Info("Creating container %s" % self.container) try: self.conn.put_container(self.container) except Exception as e: log.FatalError("Container creation failed: %s %s" % (e.__class__.__name__, str(e)), log.ErrorCode.connection_failed) def _error_code(self, operation, e): if isinstance(e, self.resp_exc): if e.http_status == 404: return log.ErrorCode.backend_not_found def _put(self, source_path, remote_filename): self.conn.put_object(self.container, self.prefix + remote_filename, file(source_path.name)) def _get(self, remote_filename, local_path): headers, body = self.conn.get_object(self.container, self.prefix + remote_filename) with open(local_path.name, 'wb') as f: for chunk in body: f.write(chunk) def _list(self): headers, objs = self.conn.get_container(self.container, full_listing=True, path=self.prefix) # removes prefix from return values. should check for the prefix ? return [o['name'][len(self.prefix):] for o in objs] def _delete(self, filename): self.conn.delete_object(self.container, self.prefix + filename) def _query(self, filename): sobject = self.conn.head_object(self.container, self.prefix + filename) return {'size': int(sobject['content-length'])}
class SwiftBackend(duplicity.backend.Backend): """ Backend for Swift """ def __init__(self, parsed_url): try: from swiftclient import Connection from swiftclient import ClientException except ImportError: raise BackendException("This backend requires " "the python-swiftclient library.") self.resp_exc = ClientException conn_kwargs = {} # if the user has already authenticated if 'SWIFT_PREAUTHURL' in os.environ and 'SWIFT_PREAUTHTOKEN' in os.environ: conn_kwargs['preauthurl'] = os.environ['SWIFT_PREAUTHURL'] conn_kwargs['preauthtoken'] = os.environ['SWIFT_PREAUTHTOKEN'] else: if 'SWIFT_USERNAME' not in os.environ: raise BackendException('SWIFT_USERNAME environment variable ' 'not set.') if 'SWIFT_PASSWORD' not in os.environ: raise BackendException('SWIFT_PASSWORD environment variable ' 'not set.') if 'SWIFT_AUTHURL' not in os.environ: raise BackendException('SWIFT_AUTHURL environment variable ' 'not set.') conn_kwargs['user'] = os.environ['SWIFT_USERNAME'] conn_kwargs['key'] = os.environ['SWIFT_PASSWORD'] conn_kwargs['authurl'] = os.environ['SWIFT_AUTHURL'] os_options = {} if 'SWIFT_AUTHVERSION' in os.environ: conn_kwargs['auth_version'] = os.environ['SWIFT_AUTHVERSION'] if os.environ['SWIFT_AUTHVERSION'] == '3': if 'SWIFT_USER_DOMAIN_NAME' in os.environ: os_options.update({ 'user_domain_name': os.environ['SWIFT_USER_DOMAIN_NAME'] }) if 'SWIFT_USER_DOMAIN_ID' in os.environ: os_options.update( {'user_domain_id': os.environ['SWIFT_USER_DOMAIN_ID']}) if 'SWIFT_PROJECT_DOMAIN_NAME' in os.environ: os_options.update({ 'project_domain_name': os.environ['SWIFT_PROJECT_DOMAIN_NAME'] }) if 'SWIFT_PROJECT_DOMAIN_ID' in os.environ: os_options.update({ 'project_domain_id': os.environ['SWIFT_PROJECT_DOMAIN_ID'] }) if 'SWIFT_TENANTNAME' in os.environ: os_options.update( {'tenant_name': os.environ['SWIFT_TENANTNAME']}) if 'SWIFT_ENDPOINT_TYPE' in os.environ: os_options.update( {'endpoint_type': os.environ['SWIFT_ENDPOINT_TYPE']}) else: conn_kwargs['auth_version'] = '1' if 'SWIFT_TENANTNAME' in os.environ: conn_kwargs['tenant_name'] = os.environ['SWIFT_TENANTNAME'] if 'SWIFT_REGIONNAME' in os.environ: os_options.update({'region_name': os.environ['SWIFT_REGIONNAME']}) conn_kwargs['os_options'] = os_options self.container = parsed_url.path.lstrip('/') container_metadata = None try: self.conn = Connection(**conn_kwargs) container_metadata = self.conn.head_container(self.container) except ClientException: pass except Exception as e: log.FatalError( "Connection failed: %s %s" % (e.__class__.__name__, str(e)), log.ErrorCode.connection_failed) if container_metadata is None: log.Info("Creating container %s" % self.container) try: self.conn.put_container(self.container) except Exception as e: log.FatalError( "Container creation failed: %s %s" % (e.__class__.__name__, str(e)), log.ErrorCode.connection_failed) def _error_code(self, operation, e): if isinstance(e, self.resp_exc): if e.http_status == 404: return log.ErrorCode.backend_not_found def _put(self, source_path, remote_filename): self.conn.put_object(self.container, remote_filename, file(source_path.name)) def _get(self, remote_filename, local_path): headers, body = self.conn.get_object(self.container, remote_filename) with open(local_path.name, 'wb') as f: for chunk in body: f.write(chunk) def _list(self): headers, objs = self.conn.get_container(self.container, full_listing=True) return [o['name'] for o in objs] def _delete(self, filename): self.conn.delete_object(self.container, filename) def _query(self, filename): sobject = self.conn.head_object(self.container, filename) return {'size': int(sobject['content-length'])}
class SwiftBackend(duplicity.backend.Backend): u""" Backend for Swift """ def __init__(self, parsed_url): duplicity.backend.Backend.__init__(self, parsed_url) try: from swiftclient.service import SwiftService from swiftclient import Connection from swiftclient import ClientException except ImportError as e: raise BackendException(u"""\ Swift backend requires the python-swiftclient library. Exception: %s""" % str(e)) self.resp_exc = ClientException conn_kwargs = {} os_options = {} svc_options = {} # if the user has already authenticated if u'SWIFT_PREAUTHURL' in os.environ and u'SWIFT_PREAUTHTOKEN' in os.environ: conn_kwargs[u'preauthurl'] = os.environ[u'SWIFT_PREAUTHURL'] conn_kwargs[u'preauthtoken'] = os.environ[u'SWIFT_PREAUTHTOKEN'] else: if u'SWIFT_USERNAME' not in os.environ: raise BackendException(u'SWIFT_USERNAME environment variable ' u'not set.') if u'SWIFT_PASSWORD' not in os.environ: raise BackendException(u'SWIFT_PASSWORD environment variable ' u'not set.') if u'SWIFT_AUTHURL' not in os.environ: raise BackendException(u'SWIFT_AUTHURL environment variable ' u'not set.') svc_options[u'os_username'] = conn_kwargs[u'user'] = os.environ[ u'SWIFT_USERNAME'] svc_options[u'os_password'] = conn_kwargs[u'key'] = os.environ[ u'SWIFT_PASSWORD'] svc_options[u'os_auth_url'] = conn_kwargs[u'authurl'] = os.environ[ u'SWIFT_AUTHURL'] if u'SWIFT_AUTHVERSION' in os.environ: svc_options[u'auth_version'] = conn_kwargs[ u'auth_version'] = os.environ[u'SWIFT_AUTHVERSION'] if os.environ[u'SWIFT_AUTHVERSION'] == u'3': if u'SWIFT_USER_DOMAIN_NAME' in os.environ: os_options.update({ u'user_domain_name': os.environ[u'SWIFT_USER_DOMAIN_NAME'] }) if u'SWIFT_USER_DOMAIN_ID' in os.environ: os_options.update({ u'user_domain_id': os.environ[u'SWIFT_USER_DOMAIN_ID'] }) if u'SWIFT_PROJECT_DOMAIN_NAME' in os.environ: os_options.update({ u'project_domain_name': os.environ[u'SWIFT_PROJECT_DOMAIN_NAME'] }) if u'SWIFT_PROJECT_DOMAIN_ID' in os.environ: os_options.update({ u'project_domain_id': os.environ[u'SWIFT_PROJECT_DOMAIN_ID'] }) if u'SWIFT_TENANTNAME' in os.environ: os_options.update( {u'tenant_name': os.environ[u'SWIFT_TENANTNAME']}) if u'SWIFT_ENDPOINT_TYPE' in os.environ: os_options.update( {u'endpoint_type': os.environ[u'SWIFT_ENDPOINT_TYPE']}) if u'SWIFT_USERID' in os.environ: os_options.update( {u'user_id': os.environ[u'SWIFT_USERID']}) if u'SWIFT_TENANTID' in os.environ: os_options.update( {u'tenant_id': os.environ[u'SWIFT_TENANTID']}) if u'SWIFT_REGIONNAME' in os.environ: os_options.update( {u'region_name': os.environ[u'SWIFT_REGIONNAME']}) else: conn_kwargs[u'auth_version'] = u'1' if u'SWIFT_TENANTNAME' in os.environ: conn_kwargs[u'tenant_name'] = os.environ[u'SWIFT_TENANTNAME'] if u'SWIFT_REGIONNAME' in os.environ: os_options.update( {u'region_name': os.environ[u'SWIFT_REGIONNAME']}) # formatting options for swiftclient.SwiftService for key in os_options.keys(): svc_options[u'os_' + key] = os_options[key] conn_kwargs[u'os_options'] = os_options # This folds the null prefix and all null parts, which means that: # //MyContainer/ and //MyContainer are equivalent. # //MyContainer//My/Prefix/ and //MyContainer/My/Prefix are equivalent. url_parts = [x for x in parsed_url.path.split(u'/') if x != u''] self.container = url_parts.pop(0) if url_parts: self.prefix = u'%s/' % u'/'.join(url_parts) else: self.prefix = u'' policy = config.swift_storage_policy policy_header = u'X-Storage-Policy' container_metadata = None try: log.Debug(u"Starting connection with arguments:'%s'" % conn_kwargs) self.conn = Connection(**conn_kwargs) container_metadata = self.conn.head_container(self.container) except ClientException as e: log.Debug(u"Connection failed: %s %s" % (e.__class__.__name__, str(e))) pass except Exception as e: log.FatalError( u"Connection failed: %s %s" % (e.__class__.__name__, str(e)), log.ErrorCode.connection_failed) if container_metadata is None: log.Info(u"Creating container %s" % self.container) try: headers = dict([[policy_header, policy]]) if policy else None self.conn.put_container(self.container, headers=headers) except Exception as e: log.FatalError( u"Container creation failed: %s %s" % (e.__class__.__name__, str(e)), log.ErrorCode.connection_failed) elif policy and container_metadata[policy_header.lower()] != policy: log.FatalError( u"Container '%s' exists but its storage policy is '%s' not '%s'." % (self.container, container_metadata[policy_header.lower()], policy)) else: log.Debug(u"Container already created: %s" % container_metadata) # checking service connection try: log.Debug(u"Starting Swiftservice: '%s'" % svc_options) self.svc = SwiftService(options=svc_options) container_stat = self.svc.stat(self.container) except ClientException as e: log.FatalError( u"Connection failed: %s %s" % (e.__class__.__name__, str(e)), log.ErrorCode.connection_failed) log.Debug(u"Container stats: %s" % container_stat) def _error_code(self, operation, e): # pylint: disable=unused-argument if isinstance(e, self.resp_exc): if e.http_status == 404: return log.ErrorCode.backend_not_found def _put(self, source_path, remote_filename): lp = util.fsdecode(source_path.name) if config.mp_segment_size > 0: from swiftclient.service import SwiftUploadObject st = os.stat(lp) # only upload using Dynamic Large Object if mpvolsize is triggered if st.st_size >= config.mp_segment_size: log.Debug(u"Uploading Dynamic Large Object") mp = self.svc.upload( self.container, [ SwiftUploadObject(lp, object_name=self.prefix + util.fsdecode(remote_filename)) ], options={u'segment_size': config.mp_segment_size}) uploads = [a for a in mp if u'container' not in a[u'action']] for upload in uploads: if not upload[u'success']: raise BackendException(upload[u'traceback']) return rp = self.prefix + util.fsdecode(remote_filename) log.Debug(u"Uploading '%s' to '%s' in remote container '%s'" % (lp, rp, self.container)) self.conn.put_object(container=self.container, obj=self.prefix + util.fsdecode(remote_filename), contents=open(lp, u'rb')) def _get(self, remote_filename, local_path): headers, body = self.conn.get_object(self.container, self.prefix + util.fsdecode(remote_filename), resp_chunk_size=1024) with open(local_path.name, u'wb') as f: for chunk in body: f.write(chunk) def _list(self): headers, objs = self.conn.get_container(self.container, full_listing=True, path=self.prefix) # removes prefix from return values. should check for the prefix ? return [o[u'name'][len(self.prefix):] for o in objs] def _delete(self, filename): # use swiftservice to correctly delete all segments in case of multipart uploads deleted = [ a for a in self.svc.delete(self.container, [self.prefix + util.fsdecode(filename)]) ] def _query(self, filename): # use swiftservice to correctly report filesize in case of multipart uploads sobject = [ a for a in self.svc.stat(self.container, [self.prefix + util.fsdecode(filename)]) ][0] sobj = {u'size': int(sobject[u'headers'][u'content-length'])} log.Debug(u"Objectquery: '%s' has size %s." % (util.fsdecode(filename), sobj[u'size'])) return sobj
class SwiftCheck: """ Functional test for Swift - basic workflow as follows: List containers List objects in those containers Update Metadata of the account Retrieve Metadata of the account Create a Container Select that container Update Metadata of the container Retrieve Metadata of the container Create an object in the container Retrieve the object in the container Update metadata of the object Retrieve metadata of the object Delete object Delete container. """ def __init__(self, logger, exec_time, **kwargs): self.swift_client = Connection( user=kwargs['os_username'], key=kwargs['os_password'], tenant_name=kwargs['os_tenant_name'], authurl=kwargs['os_auth_url'], auth_version="2.0", os_options={'region_name': kwargs['os_region']} ) self.service = 'Object Storage' self.logger = logger self.exec_time = exec_time self.zone = kwargs['os_zone'] self.region = kwargs['os_region'] self.failure = None self.overall_success = True self.tenant_name = kwargs['os_tenant_name'] @monitoring.timeit def list_containers(self): """ List all the existing containers """ try: self.success = True self.containers = self.swift_client.get_account()[1] self.logger.warning("Listing Containers:") for self.container in self.containers: self.logger.warning(self.container['name']) except Exception as e: self.success, self.overall_success = False, False self.failure = e self.logger.error( "Listing Swift containers failed %s", sys.exc_info()[1]) @monitoring.timeit def list_objects(self): """ List all the objects contained in each container """ try: self.success = True self.containers = self.swift_client.get_account()[1] self.logger.warning("Listing Objects:") for self.container in self.containers: try: for swift_container in self.swift_client.get_container( self.container['name'])[1]: self.logger.warning( self.container['name'] + " " + swift_container['name']) except: pass except Exception as e: self.success, self.overall_success = False, False self.failure = e self.logger.error( "Listing objects failed %s", sys.exc_info()[1]) @monitoring.timeit def create_container(self): """ Create a new container """ try: self.success = True self.container = 'swiftcheck' + str(time.time()) self.swift_client.put_container(self.container) self.logger.warning("Created Container %s", self.container) except Exception as e: self.success, self.overall_success = False, False self.failure = e self.logger.error( "Create Container failed %s", sys.exc_info()[1]) @monitoring.timeit def create_object(self): """ Create a new object in the newly created container """ try: self.success = True self.object = 'swiftcheckob' + str(time.time()) contents = 'yeah this is a pretty small object' self.swift_client.put_object(self.container, self.object, contents) self.logger.warning("Created Object %s", self.object) except Exception as e: self.success, self.overall_success = False, False self.failure = e self.logger.error( "Create Object Failed %s", sys.exc_info()[1]) @monitoring.timeit def get_container(self): """ Select the newly created container """ try: self.success = True self.containers = self.swift_client.get_account()[1] self.flag = 0 for container in self.containers: if container['name'] == self.container: self.logger.warning( "Getting Container Succeeded: %s", container['name']) self.flag = self.flag + 1 if self.flag == 0: raise cdn_exceptions.ClientException("Container not found") except cdn_exceptions.ClientException: self.success, self.overall_success = False, False self.failure = "Not found" self.logger.error("Container not found") except Exception as e: self.success, self.overall_success = False, False self.failure = e self.logger.error( "Selecting a container failed %s", sys.exc_info()[1]) @monitoring.timeit def get_object(self): """ Get the newly created object from the newly created container """ try: self.success = True object = self.swift_client.get_object(self.container, self.object) self.logger.warning("Getting Object Succeded: %s", object) except Exception as e: self.success, self.overall_success = False, False self.failure = e self.logger.error( "Getting object failed %s", sys.exc_info()[1]) @monitoring.timeit def retrieve_metadata_object(self): """ Retrieve the metadata of the selected object """ try: self.success = True object_info = self.swift_client.head_object( self.container, self.object) self.logger.warning( "Getting Object Head succeeded: %s", object_info) except Exception as e: self.success, self.overall_success = False, False self.failure = e self.logger.error( "Retrieving Object Head failed %s", sys.exc_info()[1]) @monitoring.timeit def retrieve_metadata_account(self): """ Retrieve the metadata of the account """ try: self.success = True account_info = self.swift_client.head_account() self.logger.warning( "Getting Account Head succeeded: %s", account_info) except Exception as e: self.success, self.overall_success = False, False self.failure = e self.logger.error( "Retrieving Account Head failed %s", sys.exc_info()[1]) @monitoring.timeit def retrieve_metadata_container(self): """ Retrieve the metadata of the container """ try: self.success = True container_info = self.swift_client.head_container(self.container) self.logger.warning( "Getting Container Head succeeded: %s", container_info) except Exception as e: self.success, self.overall_success = False, False self.failure = e self.logger.error( "Getting Container Head failed: %s", sys.exc_info()[1]) @monitoring.timeit def update_metadata_account(self): """ Update the metadata of the account """ try: self.success = True headers = {"X-Account-Meta-Temp-User": "******"} self.swift_client.post_account(headers) self.logger.warning( "Posting Metadata to Account succeeded: %s", headers) except Exception as e: self.success, self.overall_success = False, False self.failure = e self.logger.error( "Posting Metadata to Account failed %s", sys.exc_info()[1]) @monitoring.timeit def update_metadata_container(self): """ Update the metadata of the container """ try: self.success = True headers = {"X-Container-Meta-Temp-User": "******"} self.swift_client.post_container(self.container, headers) self.logger.warning( "Posting Metadata to Container succeeded: %s", headers) except Exception as e: self.success, self.overall_success = False, False self.failure = e self.logger.error( "Posting Metadata to Container failed %s", sys.exc_info()[1]) @monitoring.timeit def update_metadata_object(self): """ Update the metadata of the object """ try: self.success = True headers = {"X-Object-Meta-Temp-User": "******"} self.swift_client.post_object(self.container, self.object, headers) self.logger.warning( "Posting Metadata to Object succeeded: %s", headers) except Exception as e: self.success, self.overall_success = False, False self.failure = e self.logger.error( "Posting Metadata to Object failed %s", sys.exc_info()[1]) @monitoring.timeit def delete_object(self): """ Delete the object created earlier in the process flow """ try: self.swift_client.delete_object(self.container, self.object) self.logger.warning("Object Deleted") except Exception as e: self.success, self.overall_success = False, False self.failure = e self.logger.error( "Object Delete failed %s", sys.exc_info()[1]) @monitoring.timeit def delete_container(self): """ Delete the container that was created earlier in the process flow """ try: self.swift_client.delete_container(self.container) self.logger.warning("Container %s Deleted", self.container) except Exception as e: self.success, self.overall_success = False, False self.failure = e self.logger.error( "Container Delete Failed %s", sys.exc_info()[1]) def run(self): """ The driver method that runs all the steps in the process flow """ self.list_containers() self.list_objects() self.update_metadata_account() self.retrieve_metadata_account() self.create_container() self.get_container() self.update_metadata_container() self.retrieve_metadata_container() self.create_object() self.get_object() self.update_metadata_object() self.retrieve_metadata_object() if hasattr(self, 'object'): self.delete_object() if hasattr(self, 'container'): self.delete_container() if self.overall_success is True: exit(0) else: exit(1)
class SwiftBackend(duplicity.backend.Backend): """ Backend for Swift """ def __init__(self, parsed_url): try: from swiftclient import Connection from swiftclient import ClientException except ImportError: raise BackendException("This backend requires " "the python-swiftclient library.") self.resp_exc = ClientException conn_kwargs = {} # if the user has already authenticated if 'SWIFT_PREAUTHURL' in os.environ and 'SWIFT_PREAUTHTOKEN' in os.environ: conn_kwargs['preauthurl'] = os.environ['SWIFT_PREAUTHURL'] conn_kwargs['preauthtoken'] = os.environ['SWIFT_PREAUTHTOKEN'] else: if 'SWIFT_USERNAME' not in os.environ: raise BackendException('SWIFT_USERNAME environment variable ' 'not set.') if 'SWIFT_PASSWORD' not in os.environ: raise BackendException('SWIFT_PASSWORD environment variable ' 'not set.') if 'SWIFT_AUTHURL' not in os.environ: raise BackendException('SWIFT_AUTHURL environment variable ' 'not set.') conn_kwargs['user'] = os.environ['SWIFT_USERNAME'] conn_kwargs['key'] = os.environ['SWIFT_PASSWORD'] conn_kwargs['authurl'] = os.environ['SWIFT_AUTHURL'] if 'SWIFT_AUTHVERSION' in os.environ: conn_kwargs['auth_version'] = os.environ['SWIFT_AUTHVERSION'] else: conn_kwargs['auth_version'] = '1' if 'SWIFT_TENANTNAME' in os.environ: conn_kwargs['tenant_name'] = os.environ['SWIFT_TENANTNAME'] if 'SWIFT_REGIONNAME' in os.environ: conn_kwargs['os_options'] = {'region_name': os.environ['SWIFT_REGIONNAME']} self.container = parsed_url.path.lstrip('/') container_metadata = None try: self.conn = Connection(**conn_kwargs) container_metadata = self.conn.head_container(self.container) except ClientException: pass except Exception as e: log.FatalError("Connection failed: %s %s" % (e.__class__.__name__, str(e)), log.ErrorCode.connection_failed) if container_metadata is None: log.Info("Creating container %s" % self.container) try: self.conn.put_container(self.container) except Exception as e: log.FatalError("Container creation failed: %s %s" % (e.__class__.__name__, str(e)), log.ErrorCode.connection_failed) def _error_code(self, operation, e): if isinstance(e, self.resp_exc): if e.http_status == 404: return log.ErrorCode.backend_not_found def _put(self, source_path, remote_filename): self.conn.put_object(self.container, remote_filename, file(source_path.name)) def _get(self, remote_filename, local_path): headers, body = self.conn.get_object(self.container, remote_filename) with open(local_path.name, 'wb') as f: for chunk in body: f.write(chunk) def _list(self): headers, objs = self.conn.get_container(self.container, full_listing=True) return [o['name'] for o in objs] def _delete(self, filename): self.conn.delete_object(self.container, filename) def _query(self, filename): sobject = self.conn.head_object(self.container, filename) return {'size': int(sobject['content-length'])}
class PCABackend(duplicity.backend.Backend): """ Backend for OVH PCA """ def __init__(self, parsed_url): duplicity.backend.Backend.__init__(self, parsed_url) try: from swiftclient import Connection from swiftclient import ClientException except ImportError as e: raise BackendException("""\ PCA backend requires the python-swiftclient library. Exception: %s""" % str(e)) self.resp_exc = ClientException self.conn_cls = Connection conn_kwargs = {} # if the user has already authenticated if 'PCA_PREAUTHURL' in os.environ and 'PCA_PREAUTHTOKEN' in os.environ: conn_kwargs['preauthurl'] = os.environ['PCA_PREAUTHURL'] conn_kwargs['preauthtoken'] = os.environ['PCA_PREAUTHTOKEN'] else: if 'PCA_USERNAME' not in os.environ: raise BackendException('PCA_USERNAME environment variable ' 'not set.') if 'PCA_PASSWORD' not in os.environ: raise BackendException('PCA_PASSWORD environment variable ' 'not set.') if 'PCA_AUTHURL' not in os.environ: raise BackendException('PCA_AUTHURL environment variable ' 'not set.') conn_kwargs['user'] = os.environ['PCA_USERNAME'] conn_kwargs['key'] = os.environ['PCA_PASSWORD'] conn_kwargs['authurl'] = os.environ['PCA_AUTHURL'] os_options = {} if 'PCA_AUTHVERSION' in os.environ: conn_kwargs['auth_version'] = os.environ['PCA_AUTHVERSION'] if os.environ['PCA_AUTHVERSION'] == '3': if 'PCA_USER_DOMAIN_NAME' in os.environ: os_options.update({'user_domain_name': os.environ['PCA_USER_DOMAIN_NAME']}) if 'PCA_USER_DOMAIN_ID' in os.environ: os_options.update({'user_domain_id': os.environ['PCA_USER_DOMAIN_ID']}) if 'PCA_PROJECT_DOMAIN_NAME' in os.environ: os_options.update({'project_domain_name': os.environ['PCA_PROJECT_DOMAIN_NAME']}) if 'PCA_PROJECT_DOMAIN_ID' in os.environ: os_options.update({'project_domain_id': os.environ['PCA_PROJECT_DOMAIN_ID']}) if 'PCA_TENANTNAME' in os.environ: os_options.update({'tenant_name': os.environ['PCA_TENANTNAME']}) if 'PCA_ENDPOINT_TYPE' in os.environ: os_options.update({'endpoint_type': os.environ['PCA_ENDPOINT_TYPE']}) if 'PCA_USERID' in os.environ: os_options.update({'user_id': os.environ['PCA_USERID']}) if 'PCA_TENANTID' in os.environ: os_options.update({'tenant_id': os.environ['PCA_TENANTID']}) if 'PCA_REGIONNAME' in os.environ: os_options.update({'region_name': os.environ['PCA_REGIONNAME']}) else: conn_kwargs['auth_version'] = '2' if 'PCA_TENANTNAME' in os.environ: conn_kwargs['tenant_name'] = os.environ['PCA_TENANTNAME'] if 'PCA_REGIONNAME' in os.environ: os_options.update({'region_name': os.environ['PCA_REGIONNAME']}) conn_kwargs['os_options'] = os_options conn_kwargs['retries'] = 0 self.conn_kwargs = conn_kwargs # This folds the null prefix and all null parts, which means that: # //MyContainer/ and //MyContainer are equivalent. # //MyContainer//My/Prefix/ and //MyContainer/My/Prefix are equivalent. url_parts = [x for x in parsed_url.path.split('/') if x != ''] self.container = url_parts.pop(0) if url_parts: self.prefix = '%s/' % '/'.join(url_parts) else: self.prefix = '' policy = 'PCA' policy_header = 'X-Storage-Policy' container_metadata = None try: self.conn = Connection(**self.conn_kwargs) container_metadata = self.conn.head_container(self.container) except ClientException: pass except Exception as e: log.FatalError("Connection failed: %s %s" % (e.__class__.__name__, str(e)), log.ErrorCode.connection_failed) if container_metadata is None: log.Info("Creating container %s" % self.container) try: headers = dict([[policy_header, policy]]) self.conn.put_container(self.container, headers=headers) except Exception as e: log.FatalError("Container creation failed: %s %s" % (e.__class__.__name__, str(e)), log.ErrorCode.connection_failed) elif policy and container_metadata[policy_header.lower()] != policy: log.FatalError("Container '%s' exists but its storage policy is '%s' not '%s'." % (self.container, container_metadata[policy_header.lower()], policy)) def _error_code(self, operation, e): if isinstance(e, self.resp_exc): if e.http_status == 404: return log.ErrorCode.backend_not_found def _put(self, source_path, remote_filename): self.conn.put_object(self.container, self.prefix + remote_filename, file(source_path.name)) def _get(self, remote_filename, local_path): body = self.preprocess_download(remote_filename, 60) if body: with open(local_path.name, 'wb') as f: for chunk in body: f.write(chunk) def _list(self): headers, objs = self.conn.get_container(self.container, full_listing=True, path=self.prefix) # removes prefix from return values. should check for the prefix ? return [o['name'][len(self.prefix):] for o in objs] def _delete(self, filename): self.conn.delete_object(self.container, self.prefix + filename) def _query(self, filename): sobject = self.conn.head_object(self.container, self.prefix + filename) return {'size': int(sobject['content-length'])} def preprocess_download(self, remote_filename, retry_period, wait=True): body = self.unseal(remote_filename) try: if wait: while not body: time.sleep(retry_period) self.conn = self.conn_cls(**self.conn_kwargs) body = self.unseal(remote_filename) self.conn.close() except Exception as e: log.FatalError("Connection failed: %s %s" % (e.__class__.__name__, str(e)), log.ErrorCode.connection_failed) return body def unseal(self, remote_filename): try: _, body = self.conn.get_object(self.container, self.prefix + remote_filename, resp_chunk_size=1024) log.Info("File %s was successfully unsealed." % remote_filename) return body except self.resp_exc as e: # The object is sealed but being released. if e.http_status == 429: # The retry-after header contains the remaining duration before # the unsealing operation completes. duration = int(e.http_response_headers['Retry-After']) m, s = divmod(duration, 60) h, m = divmod(m, 60) eta = "%dh%02dm%02ds" % (h, m, s) log.Info("File %s is being unsealed, operation ETA is %s." % (remote_filename, eta)) else: raise
def configure(cls, settings): kwargs = super(OpenStackSwiftStorage, cls).configure(settings) # noinspection PyTypeChecker config = get_settings(settings, "storage.", auth_url=str, auth_version=str, password=str, username=str, user_id=str, tenant_name=str, tenant_id=str, project_name=str, project_id=str, user_domain_name=str, user_domain_id=str, project_domain_name=str, project_domain_id=str, endpoint_type=str, region_name=str, auth_token=str, storage_url=str, storage_policy=str, container=str) options = { 'authurl': config.get('auth_url'), 'auth_version': config.get('auth_version', None), 'user': config.get('username'), 'key': config.get('password'), 'preauthtoken': config.get('auth_token', None), 'preauthurl': config.get('storage_url', None), 'os_options': { 'username': config.get('username', None), 'user_id': config.get('user_id', None), 'user_domain_name': config.get('user_domain_name', None), 'user_domain_id': config.get('user_domain_id', None), 'project_domain_name': config.get('project_domain_name', None), 'project_domain_id': config.get('project_domain_id', None), 'tenant_id': config.get('tenant_id', None), 'tenant_name': config.get('tenant_name', None), 'project_id': config.get('project_id', None), 'project_name': config.get('project_name', None), 'endpoint_type': config.get('endpoint_type', None), 'region_name': config.get('region_name', None), }, 'force_auth_retry': True } client = Connection(**options) container = config.get('container') storage_policy = config.get('storage_policy', None) if storage_policy: try: caps = client.get_capabilities() LOG.debug('Swift capabilities: %s', caps) except ClientException as e: LOG.warning("Can't get swift capabilities: %s", e) else: policies = set() for policy in caps.get('swift', {}).get('policies', []): policies.add(policy.get('name', '').lower()) for alias in policy.get('aliases', '').split(','): policies.add(alias.strip().lower()) if policies and storage_policy.lower() not in policies: kwargs['storage_policy'] = storage_policy try: headers = client.head_container(container, headers={'user-agent': USER_AGENT}) LOG.info('Container exist: object_count = %s, bytes_used = %s', headers['x-container-object-count'], headers['x-container-bytes-used']) except ClientException as e: if e.http_status != 404: LOG.error('Failed to check container existence "%s": %s', container, e) raise create_container(client, container, storage_policy) kwargs['client'] = client kwargs['container'] = container return kwargs
del log_config['swift_user'] return log_config if __name__ == '__main__': parser = OptionParser(version='%prog 1.0', usage='Usage: %prog -c CONFIG_FILE') parser.add_option('-c', '--config', dest='config', default='/etc/swilog/swilog.conf', help='Configuration file') (options, args) = parser.parse_args(sys.argv[1:]) (config, logfiles) = read_configuration(options.config) # Star the clock here start_time = time.time() logger.warn('Initiating log processing.') conn = Connection(config['swift_auth'], config['swift_user'], config['swift_password']) cont_name = config['container'] try: container = conn.head_container(cont_name) except ClientException as e: if config['create_container']: logger.warn('No container by that name - Creating %s.' % cont_name) container = conn.put_container(cont_name) else: logger.fatal('Container %s does not exist. Exiting.' % cont_name) sys.exit(-1) # Loop through each declared log file and process for logfile in logfiles: log_conf = build_log_config(config, logfile) processor = LogProcessor(logger, log_conf) processor.process(conn) elapsed_time = time.time() - start_time
class PCABackend(duplicity.backend.Backend): u""" Backend for OVH PCA """ def __init__(self, parsed_url): duplicity.backend.Backend.__init__(self, parsed_url) try: from swiftclient import Connection from swiftclient import ClientException except ImportError as e: raise BackendException(u"""\ PCA backend requires the python-swiftclient library. Exception: %s""" % str(e)) self.resp_exc = ClientException self.conn_cls = Connection conn_kwargs = {} # if the user has already authenticated if u'PCA_PREAUTHURL' in os.environ and u'PCA_PREAUTHTOKEN' in os.environ: conn_kwargs[u'preauthurl'] = os.environ[u'PCA_PREAUTHURL'] conn_kwargs[u'preauthtoken'] = os.environ[u'PCA_PREAUTHTOKEN'] else: if u'PCA_USERNAME' not in os.environ: raise BackendException(u'PCA_USERNAME environment variable ' u'not set.') if u'PCA_PASSWORD' not in os.environ: raise BackendException(u'PCA_PASSWORD environment variable ' u'not set.') if u'PCA_AUTHURL' not in os.environ: raise BackendException(u'PCA_AUTHURL environment variable ' u'not set.') conn_kwargs[u'user'] = os.environ[u'PCA_USERNAME'] conn_kwargs[u'key'] = os.environ[u'PCA_PASSWORD'] conn_kwargs[u'authurl'] = os.environ[u'PCA_AUTHURL'] os_options = {} if u'PCA_AUTHVERSION' in os.environ: conn_kwargs[u'auth_version'] = os.environ[u'PCA_AUTHVERSION'] if os.environ[u'PCA_AUTHVERSION'] == u'3': if u'PCA_USER_DOMAIN_NAME' in os.environ: os_options.update({u'user_domain_name': os.environ[u'PCA_USER_DOMAIN_NAME']}) if u'PCA_USER_DOMAIN_ID' in os.environ: os_options.update({u'user_domain_id': os.environ[u'PCA_USER_DOMAIN_ID']}) if u'PCA_PROJECT_DOMAIN_NAME' in os.environ: os_options.update({u'project_domain_name': os.environ[u'PCA_PROJECT_DOMAIN_NAME']}) if u'PCA_PROJECT_DOMAIN_ID' in os.environ: os_options.update({u'project_domain_id': os.environ[u'PCA_PROJECT_DOMAIN_ID']}) if u'PCA_TENANTNAME' in os.environ: os_options.update({u'tenant_name': os.environ[u'PCA_TENANTNAME']}) if u'PCA_ENDPOINT_TYPE' in os.environ: os_options.update({u'endpoint_type': os.environ[u'PCA_ENDPOINT_TYPE']}) if u'PCA_USERID' in os.environ: os_options.update({u'user_id': os.environ[u'PCA_USERID']}) if u'PCA_TENANTID' in os.environ: os_options.update({u'tenant_id': os.environ[u'PCA_TENANTID']}) if u'PCA_REGIONNAME' in os.environ: os_options.update({u'region_name': os.environ[u'PCA_REGIONNAME']}) else: conn_kwargs[u'auth_version'] = u'2' if u'PCA_TENANTNAME' in os.environ: conn_kwargs[u'tenant_name'] = os.environ[u'PCA_TENANTNAME'] if u'PCA_REGIONNAME' in os.environ: os_options.update({u'region_name': os.environ[u'PCA_REGIONNAME']}) conn_kwargs[u'os_options'] = os_options conn_kwargs[u'retries'] = 0 self.conn_kwargs = conn_kwargs # This folds the null prefix and all null parts, which means that: # //MyContainer/ and //MyContainer are equivalent. # //MyContainer//My/Prefix/ and //MyContainer/My/Prefix are equivalent. url_parts = [x for x in parsed_url.path.split(u'/') if x != u''] self.container = url_parts.pop(0) if url_parts: self.prefix = u'%s/' % u'/'.join(url_parts) else: self.prefix = u'' policy = u'PCA' policy_header = u'X-Storage-Policy' container_metadata = None try: self.conn = Connection(**self.conn_kwargs) container_metadata = self.conn.head_container(self.container) except ClientException: pass except Exception as e: log.FatalError(u"Connection failed: %s %s" % (e.__class__.__name__, str(e)), log.ErrorCode.connection_failed) if container_metadata is None: log.Info(u"Creating container %s" % self.container) try: headers = dict([[policy_header, policy]]) self.conn.put_container(self.container, headers=headers) except Exception as e: log.FatalError(u"Container creation failed: %s %s" % (e.__class__.__name__, str(e)), log.ErrorCode.connection_failed) elif policy and container_metadata[policy_header.lower()] != policy: log.FatalError(u"Container '%s' exists but its storage policy is '%s' not '%s'." % (self.container, container_metadata[policy_header.lower()], policy)) def _error_code(self, operation, e): # pylint: disable= unused-argument if isinstance(e, self.resp_exc): if e.http_status == 404: return log.ErrorCode.backend_not_found def _put(self, source_path, remote_filename): self.conn.put_object(self.container, self.prefix + util.fsdecode(remote_filename), open(util.fsdecode(source_path.name), u'rb')) def _get(self, remote_filename, local_path): body = self.unseal(util.fsdecode(remote_filename)) if body: with open(util.fsdecode(local_path.name), u'wb') as f: for chunk in body: f.write(chunk) def __list_objs(self, ffilter=None): # full_listing should be set to True but a bug in python-swiftclient # doesn't forward query_string in this case... # bypass here using a patched copy (with query_string) from swiftclient code rv = self.conn.get_container(self.container, full_listing=False, path=self.prefix, query_string=u'policy_extra=true') listing = rv[1] while listing: marker = listing[-1][u'name'] version_marker = listing[-1].get(u'version_id') listing = self.conn.get_container(self.container, marker=marker, version_marker=version_marker, full_listing=False, path=self.prefix, query_string=u'policy_extra=true')[1] if listing: rv[1].extend(listing) if ffilter is not None: return filter(ffilter, rv[1]) return rv[1] def _list(self): return [util.fsencode(o[u'name'][len(self.prefix):]) for o in self.__list_objs()] def _delete(self, filename): self.conn.delete_object(self.container, self.prefix + util.fsdecode(filename)) def _query(self, filename): sobject = self.conn.head_object(self.container, self.prefix + util.fsdecode(filename)) return {u'size': int(sobject[u'content-length'])} def unseal(self, remote_filename): try: _, body = self.conn.get_object(self.container, self.prefix + remote_filename, resp_chunk_size=1024) log.Info(u"File %s was successfully unsealed." % remote_filename) return body except self.resp_exc as e: # The object is sealed but being released. if e.http_status == 429: # The retry-after header contains the remaining duration before # the unsealing operation completes. duration = int(e.http_response_headers[u'Retry-After']) m, s = divmod(duration, 60) h, m = divmod(m, 60) eta = u"%dh%02dm%02ds" % (h, m, s) log.Info(u"File %s is being unsealed, operation ETA is %s." % (remote_filename, eta)) else: log.FatalError(u"Connection failed: %s %s" % (e.__class__.__name__, str(e)), log.ErrorCode.connection_failed) return None def pre_process_download_batch(self, remote_filenames): u""" This is called before downloading volumes from this backend by main engine. For PCA, volumes passed as argument need to be unsealed. This method is blocking, showing a status at regular interval """ retry_interval = 60 # status will be shown every 60s # remote_filenames are bytes string u_remote_filenames = list(map(util.fsdecode, remote_filenames)) objs = self.__list_objs(ffilter=lambda x: x[u'name'] in u_remote_filenames) # first step: retrieve pca seal status for all required volumes # and launch unseal for all sealed files one_object_not_unsealed = False for o in objs: filename = util.fsdecode(o[u'name']) # see ovh documentation for policy_retrieval_state definition policy_retrieval_state = o[u'policy_retrieval_state'] log.Info(u"Volume %s. State : %s. " % (filename, policy_retrieval_state)) if policy_retrieval_state == u'sealed': log.Notice(u"Launching unseal of volume %s." % (filename)) self.unseal(o[u'name']) one_object_not_unsealed = True elif policy_retrieval_state == u"unsealing": one_object_not_unsealed = True # second step: display estimated time for last volume unseal # and loop until all volumes are unsealed while one_object_not_unsealed: one_object_not_unsealed = self.unseal_status(u_remote_filenames) time.sleep(retry_interval) # might be a good idea to show a progress bar here... else: log.Notice(u"All volumes to download are unsealed.") def unseal_status(self, u_remote_filenames): u""" Shows unsealing status for input volumes """ one_object_not_unsealed = False objs = self.__list_objs(ffilter=lambda x: util.fsdecode(x[u'name']) in u_remote_filenames) max_duration = 0 for o in objs: policy_retrieval_state = o[u'policy_retrieval_state'] filename = util.fsdecode(o[u'name']) if policy_retrieval_state == u'sealed': log.Notice(u"Error: volume is still in sealed state : %s." % (filename)) log.Notice(u"Launching unseal of volume %s." % (filename)) self.unseal(o[u'name']) one_object_not_unsealed = True elif policy_retrieval_state == u"unsealing": duration = int(o[u'policy_retrieval_delay']) log.Info(u"%s available in %d seconds." % (filename, duration)) if duration > max_duration: max_duration = duration one_object_not_unsealed = True m, s = divmod(max_duration, 60) h, m = divmod(m, 60) max_duration_eta = u"%dh%02dm%02ds" % (h, m, s) log.Notice(u"Need to wait %s before all volumes are unsealed." % (max_duration_eta)) return one_object_not_unsealed