Esempio n. 1
0
class SwiftConnection(object):
    order = 2

    def __init__(self):
        # Read configuration from environment variables (openstack.rc)
        auth_url = environ["OPENSTACK_AUTH_URL"]
        user_domain_name = environ["OPENSTACK_USER_DOMAIN_NAME"]
        user_name = environ["OPENSTACK_USERNAME"]
        password = environ["OPENSTACK_PASSWORD"]
        project_domain_name = environ["OPENSTACK_PROJECT_DOMAIN_NAME"]
        project_name = environ["OPENSTACK_PROJECT_NAME"]
        container_name = environ["OPENSTACK_CONTAINER_NAME"]
        ca_certificate = environ["OPENSTACK_CA_CERTIFICATE"]
        self.container_name = container_name
        options = {
            'user_domain_name': user_domain_name,
            'project_domain_name': project_domain_name,
            'project_name': project_name,
        }
        # Establish the connection with the object storage API cacert=ca_certificate,
        self.conn = Connection(user=user_name,
                               key=password,
                               authurl=auth_url,
                               auth_version=3,
                               os_options=options,
                               cacert=ca_certificate,
                               timeout=120)
        found = False
        for container in self.conn.get_account()[1]:
            cname = container['name']
            if cname == container_name:
                found = True
        if found is not True:
            # Create a new container
            container_name = container_name
            self.conn.put_container(container_name)

    def put(self, file_name, file_content):
        """Upload file object to swift bucket."""
        container_name = self.container_name
        self.conn.put_object(
            container_name,
            file_name,
            contents=file_content,
        )

    def get_file(self, file_name):
        """Download file object from swift bucket."""
        container_name = self.container_name
        self.conn.get_object(container_name, file_name)
Esempio n. 2
0
class swift_backend(object):
    '''
    Swift client to store object to and retrieve object from SWIFT 
    '''
    
    def __init__(self, username, password, tenant_name, authurl):
        self.username = username
        self.password = password
        self.tenant_name = tenant_name
        self.authurl = authurl
        
        self.os_options = {
            'tenant_id': None,
            'tenant_name': tenant_name,
            'service_type': None,
            'endpoint_type': None,
            'auth_token': None,
            'object_storage_url': None,
            'region_name': None,
        }
        
        self.conn = Connection(
                                self.authurl,
                                self.username,
                                self.password,
                                1, 
                                auth_version=2.0, 
                                os_options=self.os_options)
                               
         
    
    def put(self, container_name, filename, path):
        
        #put_headers = {'x-object-meta-mtime': "%f" % getmtime(path)}

        '''resps = self.conn.get_container(container_name)
     
        for resp in resps:
            print "resp is " + str(resp)'''
        
        self.conn.put_object(
                        container_name, filename, open(path, 'rb'))
     
        return
    
    
    def get(self, container_name, filename, output_name):
     
        headers, body = \
                self.conn.get_object(container_name, filename, resp_chunk_size=65536)
                
        fp = open(output_name, 'wb')
        
        for chunk in body:
            fp.write(chunk)
            
        fp.close()
             
        return 
Esempio n. 3
0
class SwiftStorage(Storage):

    def __init__(self, config):
        self._config = config
        self._root_path = self._config.storage_path
        self._connection = Connection(self._config.storage_auth_url, self._config.storage_account + ':' + self._config.storage_username, self._config.storage_password)
        self._container = self._config.storage_container
        self._index_lastupdate = None
    
    def _update_index(self):
        logger.info("_update_index")
        try:
            headers = self._connection.head_object(self._container, 'docker-registry.index')  
            lastupdatestr = headers['last-modified']
            if not lastupdatestr:
                return
            lastupdate = datetime.strptime(lastupdatestr, "%a, %d %b %Y %H:%M:%S %Z")
            logger.info('lastupdate: ' + str(lastupdate))
            refresh_index = False
            
            if not self._index_lastupdate:
                refresh_index = True
            elif self._index_lastupdate < lastupdate:
                refresh_index = True
            
            if not refresh_index:
                return
            logger.info("refresh index...")
            
            (headers, data) = self._connection.get_object(self._container, 'docker-registry.index')
            (level, temppath) = mkstemp()
            tempfile = open(temppath, 'w')
            tempfile.write(data)
            tempfile.close()
            
            with open(temppath, 'r') as f:
                for line in f:
                    line = line.strip()
                    if line.startswith("#"):
                        continue
                    tokens = line.split('\t')
                    if len(tokens) < 2:
                        continue
                    logger.info("Index: " + str(tokens))
                    realpath = os.path.join(self._root_path, tokens[0])
                    if not os.path.exists(realpath):
                        os.makedirs(realpath)
                    metafile = open(os.path.join(realpath, '.object'), 'w')
                    metafile.write(tokens[1])
                    metafile.close()
            
            os.remove(temppath)

            self._index_lastupdate = lastupdate
        except ClientException, inst:
            if inst.http_status != 404:
                raise inst
Esempio n. 4
0
class PCABackend(duplicity.backend.Backend):
    """
    Backend for OVH PCA
    """
    def __init__(self, parsed_url):
        duplicity.backend.Backend.__init__(self, parsed_url)

        try:
            from swiftclient import Connection
            from swiftclient import ClientException
        except ImportError as e:
            raise BackendException("""\
PCA backend requires the python-swiftclient library.
Exception: %s""" % str(e))

        self.resp_exc = ClientException
        self.conn_cls = Connection
        conn_kwargs = {}

        # if the user has already authenticated
        if 'PCA_PREAUTHURL' in os.environ and 'PCA_PREAUTHTOKEN' in os.environ:
            conn_kwargs['preauthurl'] = os.environ['PCA_PREAUTHURL']
            conn_kwargs['preauthtoken'] = os.environ['PCA_PREAUTHTOKEN']

        else:
            if 'PCA_USERNAME' not in os.environ:
                raise BackendException('PCA_USERNAME environment variable '
                                       'not set.')

            if 'PCA_PASSWORD' not in os.environ:
                raise BackendException('PCA_PASSWORD environment variable '
                                       'not set.')

            if 'PCA_AUTHURL' not in os.environ:
                raise BackendException('PCA_AUTHURL environment variable '
                                       'not set.')

            conn_kwargs['user'] = os.environ['PCA_USERNAME']
            conn_kwargs['key'] = os.environ['PCA_PASSWORD']
            conn_kwargs['authurl'] = os.environ['PCA_AUTHURL']

        os_options = {}

        if 'PCA_AUTHVERSION' in os.environ:
            conn_kwargs['auth_version'] = os.environ['PCA_AUTHVERSION']
            if os.environ['PCA_AUTHVERSION'] == '3':
                if 'PCA_USER_DOMAIN_NAME' in os.environ:
                    os_options.update({
                        'user_domain_name':
                        os.environ['PCA_USER_DOMAIN_NAME']
                    })
                if 'PCA_USER_DOMAIN_ID' in os.environ:
                    os_options.update(
                        {'user_domain_id': os.environ['PCA_USER_DOMAIN_ID']})
                if 'PCA_PROJECT_DOMAIN_NAME' in os.environ:
                    os_options.update({
                        'project_domain_name':
                        os.environ['PCA_PROJECT_DOMAIN_NAME']
                    })
                if 'PCA_PROJECT_DOMAIN_ID' in os.environ:
                    os_options.update({
                        'project_domain_id':
                        os.environ['PCA_PROJECT_DOMAIN_ID']
                    })
                if 'PCA_TENANTNAME' in os.environ:
                    os_options.update(
                        {'tenant_name': os.environ['PCA_TENANTNAME']})
                if 'PCA_ENDPOINT_TYPE' in os.environ:
                    os_options.update(
                        {'endpoint_type': os.environ['PCA_ENDPOINT_TYPE']})
                if 'PCA_USERID' in os.environ:
                    os_options.update({'user_id': os.environ['PCA_USERID']})
                if 'PCA_TENANTID' in os.environ:
                    os_options.update(
                        {'tenant_id': os.environ['PCA_TENANTID']})
                if 'PCA_REGIONNAME' in os.environ:
                    os_options.update(
                        {'region_name': os.environ['PCA_REGIONNAME']})

        else:
            conn_kwargs['auth_version'] = '2'
        if 'PCA_TENANTNAME' in os.environ:
            conn_kwargs['tenant_name'] = os.environ['PCA_TENANTNAME']
        if 'PCA_REGIONNAME' in os.environ:
            os_options.update({'region_name': os.environ['PCA_REGIONNAME']})

        conn_kwargs['os_options'] = os_options
        conn_kwargs['retries'] = 0

        self.conn_kwargs = conn_kwargs

        # This folds the null prefix and all null parts, which means that:
        #  //MyContainer/ and //MyContainer are equivalent.
        #  //MyContainer//My/Prefix/ and //MyContainer/My/Prefix are equivalent.
        url_parts = [x for x in parsed_url.path.split('/') if x != '']

        self.container = url_parts.pop(0)
        if url_parts:
            self.prefix = '%s/' % '/'.join(url_parts)
        else:
            self.prefix = ''

        policy = 'PCA'
        policy_header = 'X-Storage-Policy'

        container_metadata = None
        try:
            self.conn = Connection(**self.conn_kwargs)
            container_metadata = self.conn.head_container(self.container)
        except ClientException:
            pass
        except Exception as e:
            log.FatalError(
                "Connection failed: %s %s" % (e.__class__.__name__, str(e)),
                log.ErrorCode.connection_failed)

        if container_metadata is None:
            log.Info("Creating container %s" % self.container)
            try:
                headers = dict([[policy_header, policy]])
                self.conn.put_container(self.container, headers=headers)
            except Exception as e:
                log.FatalError(
                    "Container creation failed: %s %s" %
                    (e.__class__.__name__, str(e)),
                    log.ErrorCode.connection_failed)
        elif policy and container_metadata[policy_header.lower()] != policy:
            log.FatalError(
                "Container '%s' exists but its storage policy is '%s' not '%s'."
                % (self.container, container_metadata[policy_header.lower()],
                   policy))

    def _error_code(self, operation, e):
        if isinstance(e, self.resp_exc):
            if e.http_status == 404:
                return log.ErrorCode.backend_not_found

    def _put(self, source_path, remote_filename):
        self.conn.put_object(self.container, self.prefix + remote_filename,
                             file(source_path.name))

    def _get(self, remote_filename, local_path):
        body = self.preprocess_download(remote_filename, 60)
        if body:
            with open(local_path.name, 'wb') as f:
                for chunk in body:
                    f.write(chunk)

    def _list(self):
        headers, objs = self.conn.get_container(self.container,
                                                full_listing=True,
                                                path=self.prefix)
        # removes prefix from return values. should check for the prefix ?
        return [o['name'][len(self.prefix):] for o in objs]

    def _delete(self, filename):
        self.conn.delete_object(self.container, self.prefix + filename)

    def _query(self, filename):
        sobject = self.conn.head_object(self.container, self.prefix + filename)
        return {'size': int(sobject['content-length'])}

    def preprocess_download(self, remote_filename, retry_period, wait=True):
        body = self.unseal(remote_filename)
        try:
            if wait:
                while not body:
                    time.sleep(retry_period)
                    self.conn = self.conn_cls(**self.conn_kwargs)
                    body = self.unseal(remote_filename)
                    self.conn.close()
        except Exception as e:
            log.FatalError(
                "Connection failed: %s %s" % (e.__class__.__name__, str(e)),
                log.ErrorCode.connection_failed)
        return body

    def unseal(self, remote_filename):
        try:
            _, body = self.conn.get_object(self.container,
                                           self.prefix + remote_filename,
                                           resp_chunk_size=1024)
            log.Info("File %s was successfully unsealed." % remote_filename)
            return body
        except self.resp_exc as e:
            # The object is sealed but being released.
            if e.http_status == 429:
                # The retry-after header contains the remaining duration before
                # the unsealing operation completes.
                duration = int(e.http_response_headers['Retry-After'])
                m, s = divmod(duration, 60)
                h, m = divmod(m, 60)
                eta = "%dh%02dm%02ds" % (h, m, s)
                log.Info("File %s is being unsealed, operation ETA is %s." %
                         (remote_filename, eta))
            else:
                raise
Esempio n. 5
0
class SwiftBackend(duplicity.backend.Backend):
    """
    Backend for Swift
    """
    def __init__(self, parsed_url):
        duplicity.backend.Backend.__init__(self, parsed_url)

        try:
            from swiftclient import Connection
            from swiftclient import ClientException
        except ImportError as e:
            raise BackendException("""\
Swift backend requires the python-swiftclient library.
Exception: %s""" % str(e))

        self.resp_exc = ClientException
        conn_kwargs = {}

        # if the user has already authenticated
        if 'SWIFT_PREAUTHURL' in os.environ and 'SWIFT_PREAUTHTOKEN' in os.environ:
            conn_kwargs['preauthurl'] = os.environ['SWIFT_PREAUTHURL']
            conn_kwargs['preauthtoken'] = os.environ['SWIFT_PREAUTHTOKEN']

        else:
            if 'SWIFT_USERNAME' not in os.environ:
                raise BackendException('SWIFT_USERNAME environment variable '
                                       'not set.')

            if 'SWIFT_PASSWORD' not in os.environ:
                raise BackendException('SWIFT_PASSWORD environment variable '
                                       'not set.')

            if 'SWIFT_AUTHURL' not in os.environ:
                raise BackendException('SWIFT_AUTHURL environment variable '
                                       'not set.')

            conn_kwargs['user'] = os.environ['SWIFT_USERNAME']
            conn_kwargs['key'] = os.environ['SWIFT_PASSWORD']
            conn_kwargs['authurl'] = os.environ['SWIFT_AUTHURL']

        os_options = {}

        if 'SWIFT_AUTHVERSION' in os.environ:
            conn_kwargs['auth_version'] = os.environ['SWIFT_AUTHVERSION']
            if os.environ['SWIFT_AUTHVERSION'] == '3':
                if 'SWIFT_USER_DOMAIN_NAME' in os.environ:
                    os_options.update({
                        'user_domain_name':
                        os.environ['SWIFT_USER_DOMAIN_NAME']
                    })
                if 'SWIFT_USER_DOMAIN_ID' in os.environ:
                    os_options.update(
                        {'user_domain_id': os.environ['SWIFT_USER_DOMAIN_ID']})
                if 'SWIFT_PROJECT_DOMAIN_NAME' in os.environ:
                    os_options.update({
                        'project_domain_name':
                        os.environ['SWIFT_PROJECT_DOMAIN_NAME']
                    })
                if 'SWIFT_PROJECT_DOMAIN_ID' in os.environ:
                    os_options.update({
                        'project_domain_id':
                        os.environ['SWIFT_PROJECT_DOMAIN_ID']
                    })
                if 'SWIFT_TENANTNAME' in os.environ:
                    os_options.update(
                        {'tenant_name': os.environ['SWIFT_TENANTNAME']})
                if 'SWIFT_ENDPOINT_TYPE' in os.environ:
                    os_options.update(
                        {'endpoint_type': os.environ['SWIFT_ENDPOINT_TYPE']})
                if 'SWIFT_USERID' in os.environ:
                    os_options.update({'user_id': os.environ['SWIFT_USERID']})
                if 'SWIFT_TENANTID' in os.environ:
                    os_options.update(
                        {'tenant_id': os.environ['SWIFT_TENANTID']})
                if 'SWIFT_REGIONNAME' in os.environ:
                    os_options.update(
                        {'region_name': os.environ['SWIFT_REGIONNAME']})

        else:
            conn_kwargs['auth_version'] = '1'
        if 'SWIFT_TENANTNAME' in os.environ:
            conn_kwargs['tenant_name'] = os.environ['SWIFT_TENANTNAME']
        if 'SWIFT_REGIONNAME' in os.environ:
            os_options.update({'region_name': os.environ['SWIFT_REGIONNAME']})

        conn_kwargs['os_options'] = os_options

        # This folds the null prefix and all null parts, which means that:
        #  //MyContainer/ and //MyContainer are equivalent.
        #  //MyContainer//My/Prefix/ and //MyContainer/My/Prefix are equivalent.
        url_parts = [x for x in parsed_url.path.split('/') if x != '']

        self.container = url_parts.pop(0)
        if url_parts:
            self.prefix = '%s/' % '/'.join(url_parts)
        else:
            self.prefix = ''

        policy = globals.swift_storage_policy
        policy_header = 'X-Storage-Policy'

        container_metadata = None
        try:
            self.conn = Connection(**conn_kwargs)
            container_metadata = self.conn.head_container(self.container)
        except ClientException:
            pass
        except Exception as e:
            log.FatalError(
                "Connection failed: %s %s" % (e.__class__.__name__, str(e)),
                log.ErrorCode.connection_failed)

        if container_metadata is None:
            log.Info("Creating container %s" % self.container)
            try:
                headers = dict([[policy_header, policy]]) if policy else None
                self.conn.put_container(self.container, headers=headers)
            except Exception as e:
                log.FatalError(
                    "Container creation failed: %s %s" %
                    (e.__class__.__name__, str(e)),
                    log.ErrorCode.connection_failed)
        elif policy and container_metadata[policy_header.lower()] != policy:
            log.FatalError(
                "Container '%s' exists but its storage policy is '%s' not '%s'."
                % (self.container, container_metadata[policy_header.lower()],
                   policy))

    def _error_code(self, operation, e):
        if isinstance(e, self.resp_exc):
            if e.http_status == 404:
                return log.ErrorCode.backend_not_found

    def _put(self, source_path, remote_filename):
        self.conn.put_object(self.container, self.prefix + remote_filename,
                             file(source_path.name))

    def _get(self, remote_filename, local_path):
        headers, body = self.conn.get_object(self.container,
                                             self.prefix + remote_filename,
                                             resp_chunk_size=1024)
        with open(local_path.name, 'wb') as f:
            for chunk in body:
                f.write(chunk)

    def _list(self):
        headers, objs = self.conn.get_container(self.container,
                                                full_listing=True,
                                                path=self.prefix)
        # removes prefix from return values. should check for the prefix ?
        return [o['name'][len(self.prefix):] for o in objs]

    def _delete(self, filename):
        self.conn.delete_object(self.container, self.prefix + filename)

    def _query(self, filename):
        sobject = self.conn.head_object(self.container, self.prefix + filename)
        return {'size': int(sobject['content-length'])}
Esempio n. 6
0
class SwiftBackend(duplicity.backend.Backend):
    """
    Backend for Swift
    """
    def __init__(self, parsed_url):
        try:
            from swiftclient import Connection
            from swiftclient import ClientException
        except ImportError:
            raise BackendException("This backend requires "
                                   "the python-swiftclient library.")

        self.resp_exc = ClientException
        conn_kwargs = {}

        # if the user has already authenticated
        if 'SWIFT_PREAUTHURL' in os.environ and 'SWIFT_PREAUTHTOKEN' in os.environ:
            conn_kwargs['preauthurl'] = os.environ['SWIFT_PREAUTHURL']
            conn_kwargs['preauthtoken'] = os.environ['SWIFT_PREAUTHTOKEN']

        else:
            if 'SWIFT_USERNAME' not in os.environ:
                raise BackendException('SWIFT_USERNAME environment variable '
                                       'not set.')

            if 'SWIFT_PASSWORD' not in os.environ:
                raise BackendException('SWIFT_PASSWORD environment variable '
                                       'not set.')

            if 'SWIFT_AUTHURL' not in os.environ:
                raise BackendException('SWIFT_AUTHURL environment variable '
                                       'not set.')

            conn_kwargs['user'] = os.environ['SWIFT_USERNAME']
            conn_kwargs['key'] = os.environ['SWIFT_PASSWORD']
            conn_kwargs['authurl'] = os.environ['SWIFT_AUTHURL']

        os_options = {}

        if 'SWIFT_AUTHVERSION' in os.environ:
            conn_kwargs['auth_version'] = os.environ['SWIFT_AUTHVERSION']
            if os.environ['SWIFT_AUTHVERSION'] == '3':
                if 'SWIFT_USER_DOMAIN_NAME' in os.environ:
                    os_options.update({'user_domain_name': os.environ['SWIFT_USER_DOMAIN_NAME']})
                if 'SWIFT_USER_DOMAIN_ID' in os.environ:
                    os_options.update({'user_domain_id': os.environ['SWIFT_USER_DOMAIN_ID']})
                if 'SWIFT_PROJECT_DOMAIN_NAME' in os.environ:
                    os_options.update({'project_domain_name': os.environ['SWIFT_PROJECT_DOMAIN_NAME']})
                if 'SWIFT_PROJECT_DOMAIN_ID' in os.environ:
                    os_options.update({'project_domain_id': os.environ['SWIFT_PROJECT_DOMAIN_ID']})
                if 'SWIFT_TENANTNAME' in os.environ:
                    os_options.update({'tenant_name': os.environ['SWIFT_TENANTNAME']})
                if 'SWIFT_ENDPOINT_TYPE' in os.environ:
                    os_options.update({'endpoint_type': os.environ['SWIFT_ENDPOINT_TYPE']})
                if 'SWIFT_USERID' in os.environ:
                    os_options.update({'user_id': os.environ['SWIFT_USERID']})
                if 'SWIFT_TENANTID' in os.environ:
                    os_options.update({'tenant_id': os.environ['SWIFT_TENANTID']})
                if 'SWIFT_REGIONNAME' in os.environ:
                    os_options.update({'region_name': os.environ['SWIFT_REGIONNAME']})

        else:
            conn_kwargs['auth_version'] = '1'
        if 'SWIFT_TENANTNAME' in os.environ:
            conn_kwargs['tenant_name'] = os.environ['SWIFT_TENANTNAME']
        if 'SWIFT_REGIONNAME' in os.environ:
            os_options.update({'region_name': os.environ['SWIFT_REGIONNAME']})

        conn_kwargs['os_options'] = os_options

        # This folds the null prefix and all null parts, which means that:
        #  //MyContainer/ and //MyContainer are equivalent.
        #  //MyContainer//My/Prefix/ and //MyContainer/My/Prefix are equivalent.
        url_parts = [x for x in parsed_url.path.split('/') if x != '']

        self.container = url_parts.pop(0)
        if url_parts:
            self.prefix = '%s/' % '/'.join(url_parts)
        else:
            self.prefix = ''

        container_metadata = None
        try:
            self.conn = Connection(**conn_kwargs)
            container_metadata = self.conn.head_container(self.container)
        except ClientException:
            pass
        except Exception as e:
            log.FatalError("Connection failed: %s %s"
                           % (e.__class__.__name__, str(e)),
                           log.ErrorCode.connection_failed)

        if container_metadata is None:
            log.Info("Creating container %s" % self.container)
            try:
                self.conn.put_container(self.container)
            except Exception as e:
                log.FatalError("Container creation failed: %s %s"
                               % (e.__class__.__name__, str(e)),
                               log.ErrorCode.connection_failed)

    def _error_code(self, operation, e):
        if isinstance(e, self.resp_exc):
            if e.http_status == 404:
                return log.ErrorCode.backend_not_found

    def _put(self, source_path, remote_filename):
        self.conn.put_object(self.container, self.prefix + remote_filename,
                             file(source_path.name))

    def _get(self, remote_filename, local_path):
        headers, body = self.conn.get_object(self.container, self.prefix + remote_filename)
        with open(local_path.name, 'wb') as f:
            for chunk in body:
                f.write(chunk)

    def _list(self):
        headers, objs = self.conn.get_container(self.container, full_listing=True, path=self.prefix)
        # removes prefix from return values. should check for the prefix ?
        return [o['name'][len(self.prefix):] for o in objs]

    def _delete(self, filename):
        self.conn.delete_object(self.container, self.prefix + filename)

    def _query(self, filename):
        sobject = self.conn.head_object(self.container, self.prefix + filename)
        return {'size': int(sobject['content-length'])}
Esempio n. 7
0
class SwiftBackend(duplicity.backend.Backend):
    """
    Backend for Swift
    """
    def __init__(self, parsed_url):
        try:
            from swiftclient import Connection
            from swiftclient import ClientException
        except ImportError:
            raise BackendException("This backend requires "
                                   "the python-swiftclient library.")

        self.resp_exc = ClientException
        conn_kwargs = {}

        # if the user has already authenticated
        if 'SWIFT_PREAUTHURL' in os.environ and 'SWIFT_PREAUTHTOKEN' in os.environ:
            conn_kwargs['preauthurl'] = os.environ['SWIFT_PREAUTHURL']
            conn_kwargs['preauthtoken'] = os.environ['SWIFT_PREAUTHTOKEN']

        else:
            if 'SWIFT_USERNAME' not in os.environ:
                raise BackendException('SWIFT_USERNAME environment variable '
                                       'not set.')

            if 'SWIFT_PASSWORD' not in os.environ:
                raise BackendException('SWIFT_PASSWORD environment variable '
                                       'not set.')

            if 'SWIFT_AUTHURL' not in os.environ:
                raise BackendException('SWIFT_AUTHURL environment variable '
                                       'not set.')

            conn_kwargs['user'] = os.environ['SWIFT_USERNAME']
            conn_kwargs['key'] = os.environ['SWIFT_PASSWORD']
            conn_kwargs['authurl'] = os.environ['SWIFT_AUTHURL']

        os_options = {}

        if 'SWIFT_AUTHVERSION' in os.environ:
            conn_kwargs['auth_version'] = os.environ['SWIFT_AUTHVERSION']
            if os.environ['SWIFT_AUTHVERSION'] == '3':
                if 'SWIFT_USER_DOMAIN_NAME' in os.environ:
                    os_options.update({
                        'user_domain_name':
                        os.environ['SWIFT_USER_DOMAIN_NAME']
                    })
                if 'SWIFT_USER_DOMAIN_ID' in os.environ:
                    os_options.update(
                        {'user_domain_id': os.environ['SWIFT_USER_DOMAIN_ID']})
                if 'SWIFT_PROJECT_DOMAIN_NAME' in os.environ:
                    os_options.update({
                        'project_domain_name':
                        os.environ['SWIFT_PROJECT_DOMAIN_NAME']
                    })
                if 'SWIFT_PROJECT_DOMAIN_ID' in os.environ:
                    os_options.update({
                        'project_domain_id':
                        os.environ['SWIFT_PROJECT_DOMAIN_ID']
                    })
                if 'SWIFT_TENANTNAME' in os.environ:
                    os_options.update(
                        {'tenant_name': os.environ['SWIFT_TENANTNAME']})
                if 'SWIFT_ENDPOINT_TYPE' in os.environ:
                    os_options.update(
                        {'endpoint_type': os.environ['SWIFT_ENDPOINT_TYPE']})

        else:
            conn_kwargs['auth_version'] = '1'
        if 'SWIFT_TENANTNAME' in os.environ:
            conn_kwargs['tenant_name'] = os.environ['SWIFT_TENANTNAME']
        if 'SWIFT_REGIONNAME' in os.environ:
            os_options.update({'region_name': os.environ['SWIFT_REGIONNAME']})

        conn_kwargs['os_options'] = os_options

        self.container = parsed_url.path.lstrip('/')

        container_metadata = None
        try:
            self.conn = Connection(**conn_kwargs)
            container_metadata = self.conn.head_container(self.container)
        except ClientException:
            pass
        except Exception as e:
            log.FatalError(
                "Connection failed: %s %s" % (e.__class__.__name__, str(e)),
                log.ErrorCode.connection_failed)

        if container_metadata is None:
            log.Info("Creating container %s" % self.container)
            try:
                self.conn.put_container(self.container)
            except Exception as e:
                log.FatalError(
                    "Container creation failed: %s %s" %
                    (e.__class__.__name__, str(e)),
                    log.ErrorCode.connection_failed)

    def _error_code(self, operation, e):
        if isinstance(e, self.resp_exc):
            if e.http_status == 404:
                return log.ErrorCode.backend_not_found

    def _put(self, source_path, remote_filename):
        self.conn.put_object(self.container, remote_filename,
                             file(source_path.name))

    def _get(self, remote_filename, local_path):
        headers, body = self.conn.get_object(self.container, remote_filename)
        with open(local_path.name, 'wb') as f:
            for chunk in body:
                f.write(chunk)

    def _list(self):
        headers, objs = self.conn.get_container(self.container,
                                                full_listing=True)
        return [o['name'] for o in objs]

    def _delete(self, filename):
        self.conn.delete_object(self.container, filename)

    def _query(self, filename):
        sobject = self.conn.head_object(self.container, filename)
        return {'size': int(sobject['content-length'])}
Esempio n. 8
0
class SwiftBackend(duplicity.backend.Backend):
    u"""
    Backend for Swift
    """
    def __init__(self, parsed_url):
        duplicity.backend.Backend.__init__(self, parsed_url)

        try:
            from swiftclient.service import SwiftService
            from swiftclient import Connection
            from swiftclient import ClientException
        except ImportError as e:
            raise BackendException(u"""\
Swift backend requires the python-swiftclient library.
Exception: %s""" % str(e))

        self.resp_exc = ClientException
        conn_kwargs = {}
        os_options = {}
        svc_options = {}

        # if the user has already authenticated
        if u'SWIFT_PREAUTHURL' in os.environ and u'SWIFT_PREAUTHTOKEN' in os.environ:
            conn_kwargs[u'preauthurl'] = os.environ[u'SWIFT_PREAUTHURL']
            conn_kwargs[u'preauthtoken'] = os.environ[u'SWIFT_PREAUTHTOKEN']

        else:
            if u'SWIFT_USERNAME' not in os.environ:
                raise BackendException(u'SWIFT_USERNAME environment variable '
                                       u'not set.')

            if u'SWIFT_PASSWORD' not in os.environ:
                raise BackendException(u'SWIFT_PASSWORD environment variable '
                                       u'not set.')

            if u'SWIFT_AUTHURL' not in os.environ:
                raise BackendException(u'SWIFT_AUTHURL environment variable '
                                       u'not set.')

            svc_options[u'os_username'] = conn_kwargs[u'user'] = os.environ[
                u'SWIFT_USERNAME']
            svc_options[u'os_password'] = conn_kwargs[u'key'] = os.environ[
                u'SWIFT_PASSWORD']
            svc_options[u'os_auth_url'] = conn_kwargs[u'authurl'] = os.environ[
                u'SWIFT_AUTHURL']

        if u'SWIFT_AUTHVERSION' in os.environ:
            svc_options[u'auth_version'] = conn_kwargs[
                u'auth_version'] = os.environ[u'SWIFT_AUTHVERSION']
            if os.environ[u'SWIFT_AUTHVERSION'] == u'3':
                if u'SWIFT_USER_DOMAIN_NAME' in os.environ:
                    os_options.update({
                        u'user_domain_name':
                        os.environ[u'SWIFT_USER_DOMAIN_NAME']
                    })
                if u'SWIFT_USER_DOMAIN_ID' in os.environ:
                    os_options.update({
                        u'user_domain_id':
                        os.environ[u'SWIFT_USER_DOMAIN_ID']
                    })
                if u'SWIFT_PROJECT_DOMAIN_NAME' in os.environ:
                    os_options.update({
                        u'project_domain_name':
                        os.environ[u'SWIFT_PROJECT_DOMAIN_NAME']
                    })
                if u'SWIFT_PROJECT_DOMAIN_ID' in os.environ:
                    os_options.update({
                        u'project_domain_id':
                        os.environ[u'SWIFT_PROJECT_DOMAIN_ID']
                    })
                if u'SWIFT_TENANTNAME' in os.environ:
                    os_options.update(
                        {u'tenant_name': os.environ[u'SWIFT_TENANTNAME']})
                if u'SWIFT_ENDPOINT_TYPE' in os.environ:
                    os_options.update(
                        {u'endpoint_type': os.environ[u'SWIFT_ENDPOINT_TYPE']})
                if u'SWIFT_USERID' in os.environ:
                    os_options.update(
                        {u'user_id': os.environ[u'SWIFT_USERID']})
                if u'SWIFT_TENANTID' in os.environ:
                    os_options.update(
                        {u'tenant_id': os.environ[u'SWIFT_TENANTID']})
                if u'SWIFT_REGIONNAME' in os.environ:
                    os_options.update(
                        {u'region_name': os.environ[u'SWIFT_REGIONNAME']})

        else:
            conn_kwargs[u'auth_version'] = u'1'
        if u'SWIFT_TENANTNAME' in os.environ:
            conn_kwargs[u'tenant_name'] = os.environ[u'SWIFT_TENANTNAME']
        if u'SWIFT_REGIONNAME' in os.environ:
            os_options.update(
                {u'region_name': os.environ[u'SWIFT_REGIONNAME']})

        # formatting options for swiftclient.SwiftService
        for key in os_options.keys():
            svc_options[u'os_' + key] = os_options[key]

        conn_kwargs[u'os_options'] = os_options

        # This folds the null prefix and all null parts, which means that:
        #  //MyContainer/ and //MyContainer are equivalent.
        #  //MyContainer//My/Prefix/ and //MyContainer/My/Prefix are equivalent.
        url_parts = [x for x in parsed_url.path.split(u'/') if x != u'']

        self.container = url_parts.pop(0)
        if url_parts:
            self.prefix = u'%s/' % u'/'.join(url_parts)
        else:
            self.prefix = u''

        policy = config.swift_storage_policy
        policy_header = u'X-Storage-Policy'

        container_metadata = None
        try:
            log.Debug(u"Starting connection with arguments:'%s'" % conn_kwargs)
            self.conn = Connection(**conn_kwargs)
            container_metadata = self.conn.head_container(self.container)
        except ClientException as e:
            log.Debug(u"Connection failed: %s %s" %
                      (e.__class__.__name__, str(e)))
            pass
        except Exception as e:
            log.FatalError(
                u"Connection failed: %s %s" % (e.__class__.__name__, str(e)),
                log.ErrorCode.connection_failed)

        if container_metadata is None:
            log.Info(u"Creating container %s" % self.container)
            try:
                headers = dict([[policy_header, policy]]) if policy else None
                self.conn.put_container(self.container, headers=headers)
            except Exception as e:
                log.FatalError(
                    u"Container creation failed: %s %s" %
                    (e.__class__.__name__, str(e)),
                    log.ErrorCode.connection_failed)
        elif policy and container_metadata[policy_header.lower()] != policy:
            log.FatalError(
                u"Container '%s' exists but its storage policy is '%s' not '%s'."
                % (self.container, container_metadata[policy_header.lower()],
                   policy))
        else:
            log.Debug(u"Container already created: %s" % container_metadata)

        # checking service connection
        try:
            log.Debug(u"Starting  Swiftservice: '%s'" % svc_options)
            self.svc = SwiftService(options=svc_options)
            container_stat = self.svc.stat(self.container)
        except ClientException as e:
            log.FatalError(
                u"Connection failed: %s %s" % (e.__class__.__name__, str(e)),
                log.ErrorCode.connection_failed)
        log.Debug(u"Container stats: %s" % container_stat)

    def _error_code(self, operation, e):  # pylint: disable=unused-argument
        if isinstance(e, self.resp_exc):
            if e.http_status == 404:
                return log.ErrorCode.backend_not_found

    def _put(self, source_path, remote_filename):
        lp = util.fsdecode(source_path.name)
        if config.mp_segment_size > 0:
            from swiftclient.service import SwiftUploadObject
            st = os.stat(lp)
            # only upload using Dynamic Large Object if mpvolsize is triggered
            if st.st_size >= config.mp_segment_size:
                log.Debug(u"Uploading Dynamic Large Object")
                mp = self.svc.upload(
                    self.container, [
                        SwiftUploadObject(lp,
                                          object_name=self.prefix +
                                          util.fsdecode(remote_filename))
                    ],
                    options={u'segment_size': config.mp_segment_size})
                uploads = [a for a in mp if u'container' not in a[u'action']]
                for upload in uploads:
                    if not upload[u'success']:
                        raise BackendException(upload[u'traceback'])
                return
        rp = self.prefix + util.fsdecode(remote_filename)
        log.Debug(u"Uploading '%s' to '%s' in remote container '%s'" %
                  (lp, rp, self.container))
        self.conn.put_object(container=self.container,
                             obj=self.prefix + util.fsdecode(remote_filename),
                             contents=open(lp, u'rb'))

    def _get(self, remote_filename, local_path):
        headers, body = self.conn.get_object(self.container,
                                             self.prefix +
                                             util.fsdecode(remote_filename),
                                             resp_chunk_size=1024)
        with open(local_path.name, u'wb') as f:
            for chunk in body:
                f.write(chunk)

    def _list(self):
        headers, objs = self.conn.get_container(self.container,
                                                full_listing=True,
                                                path=self.prefix)
        # removes prefix from return values. should check for the prefix ?
        return [o[u'name'][len(self.prefix):] for o in objs]

    def _delete(self, filename):
        # use swiftservice to correctly delete all segments in case of multipart uploads
        deleted = [
            a for a in self.svc.delete(self.container,
                                       [self.prefix + util.fsdecode(filename)])
        ]

    def _query(self, filename):
        # use swiftservice to correctly report filesize in case of multipart uploads
        sobject = [
            a for a in self.svc.stat(self.container,
                                     [self.prefix + util.fsdecode(filename)])
        ][0]
        sobj = {u'size': int(sobject[u'headers'][u'content-length'])}
        log.Debug(u"Objectquery: '%s' has size %s." %
                  (util.fsdecode(filename), sobj[u'size']))
        return sobj
Esempio n. 9
0
class SwiftCheck:

    """

    Functional test for Swift - basic workflow as follows:

    List containers
    List objects in those containers
    Update Metadata of the account
    Retrieve Metadata of the account
    Create a Container
    Select that container
    Update Metadata of the container
    Retrieve Metadata of the container
    Create an object in the container
    Retrieve the object in the container
    Update metadata of the object
    Retrieve metadata of the object
    Delete object
    Delete container.

    """

    def __init__(self, logger, exec_time, **kwargs):
        self.swift_client = Connection(
            user=kwargs['os_username'],
            key=kwargs['os_password'],
            tenant_name=kwargs['os_tenant_name'],
            authurl=kwargs['os_auth_url'],
            auth_version="2.0",
            os_options={'region_name': kwargs['os_region']}
        )

        self.service = 'Object Storage'
        self.logger = logger
        self.exec_time = exec_time
        self.zone = kwargs['os_zone']
        self.region = kwargs['os_region']
        self.failure = None
        self.overall_success = True
        self.tenant_name = kwargs['os_tenant_name']

    @monitoring.timeit
    def list_containers(self):
        """
        List all the existing containers
        """

        try:
            self.success = True
            self.containers = self.swift_client.get_account()[1]
            self.logger.warning("Listing Containers:")
            for self.container in self.containers:
                self.logger.warning(self.container['name'])

        except Exception as e:
            self.success, self.overall_success = False, False
            self.failure = e
            self.logger.error(
                "Listing Swift containers failed %s", sys.exc_info()[1])

    @monitoring.timeit
    def list_objects(self):
        """
        List all the objects contained in each container
        """

        try:
            self.success = True
            self.containers = self.swift_client.get_account()[1]
            self.logger.warning("Listing Objects:")
            for self.container in self.containers:
                try:
                    for swift_container in self.swift_client.get_container(
                            self.container['name'])[1]:
                        self.logger.warning(
                            self.container['name'] + " " +
                            swift_container['name'])
                except:
                    pass

        except Exception as e:
            self.success, self.overall_success = False, False
            self.failure = e
            self.logger.error(
                "Listing objects failed %s", sys.exc_info()[1])

    @monitoring.timeit
    def create_container(self):
        """
        Create a new container
        """

        try:
            self.success = True
            self.container = 'swiftcheck' + str(time.time())
            self.swift_client.put_container(self.container)

            self.logger.warning("Created Container %s", self.container)

        except Exception as e:
            self.success, self.overall_success = False, False
            self.failure = e
            self.logger.error(
                "Create Container failed %s", sys.exc_info()[1])

    @monitoring.timeit
    def create_object(self):
        """
        Create a new object in the newly created container
        """

        try:
            self.success = True
            self.object = 'swiftcheckob' + str(time.time())
            contents = 'yeah this is a pretty small object'
            self.swift_client.put_object(self.container, self.object, contents)
            self.logger.warning("Created Object %s", self.object)

        except Exception as e:
            self.success, self.overall_success = False, False
            self.failure = e
            self.logger.error(
                "Create Object Failed %s", sys.exc_info()[1])

    @monitoring.timeit
    def get_container(self):
        """
        Select the newly created container
        """

        try:
            self.success = True
            self.containers = self.swift_client.get_account()[1]
            self.flag = 0

            for container in self.containers:
                if container['name'] == self.container:
                    self.logger.warning(
                        "Getting Container Succeeded: %s", container['name'])
                    self.flag = self.flag + 1
            if self.flag == 0:
                raise cdn_exceptions.ClientException("Container not found")

        except cdn_exceptions.ClientException:
            self.success, self.overall_success = False, False
            self.failure = "Not found"
            self.logger.error("Container not found")

        except Exception as e:
            self.success, self.overall_success = False, False
            self.failure = e
            self.logger.error(
                "Selecting a container failed %s", sys.exc_info()[1])

    @monitoring.timeit
    def get_object(self):
        """
        Get the newly created object from the newly created container
        """

        try:
            self.success = True
            object = self.swift_client.get_object(self.container, self.object)
            self.logger.warning("Getting Object Succeded: %s", object)

        except Exception as e:
            self.success, self.overall_success = False, False
            self.failure = e
            self.logger.error(
                "Getting object failed %s", sys.exc_info()[1])

    @monitoring.timeit
    def retrieve_metadata_object(self):
        """
        Retrieve the metadata of the selected object
        """

        try:
            self.success = True
            object_info = self.swift_client.head_object(
                self.container, self.object)
            self.logger.warning(
                "Getting Object Head succeeded: %s", object_info)

        except Exception as e:
            self.success, self.overall_success = False, False
            self.failure = e
            self.logger.error(
                "Retrieving Object Head failed %s", sys.exc_info()[1])

    @monitoring.timeit
    def retrieve_metadata_account(self):
        """
        Retrieve the metadata of the account
        """

        try:
            self.success = True
            account_info = self.swift_client.head_account()
            self.logger.warning(
                "Getting Account Head succeeded: %s", account_info)

        except Exception as e:
            self.success, self.overall_success = False, False
            self.failure = e
            self.logger.error(
                "Retrieving Account Head failed %s", sys.exc_info()[1])

    @monitoring.timeit
    def retrieve_metadata_container(self):
        """
        Retrieve the metadata of the container
        """

        try:
            self.success = True
            container_info = self.swift_client.head_container(self.container)
            self.logger.warning(
                "Getting Container Head succeeded: %s", container_info)

        except Exception as e:
            self.success, self.overall_success = False, False
            self.failure = e
            self.logger.error(
                "Getting Container Head failed: %s", sys.exc_info()[1])

    @monitoring.timeit
    def update_metadata_account(self):
        """
        Update the metadata of the account
        """

        try:
            self.success = True
            headers = {"X-Account-Meta-Temp-User": "******"}
            self.swift_client.post_account(headers)
            self.logger.warning(
                "Posting Metadata to Account succeeded: %s", headers)

        except Exception as e:
            self.success, self.overall_success = False, False
            self.failure = e
            self.logger.error(
                "Posting Metadata to Account failed %s", sys.exc_info()[1])

    @monitoring.timeit
    def update_metadata_container(self):
        """
        Update the metadata of the container
        """

        try:
            self.success = True
            headers = {"X-Container-Meta-Temp-User": "******"}
            self.swift_client.post_container(self.container, headers)
            self.logger.warning(
                "Posting Metadata to Container succeeded: %s", headers)

        except Exception as e:
            self.success, self.overall_success = False, False
            self.failure = e
            self.logger.error(
                "Posting Metadata to Container failed %s", sys.exc_info()[1])

    @monitoring.timeit
    def update_metadata_object(self):
        """
        Update the metadata of the object
        """

        try:
            self.success = True
            headers = {"X-Object-Meta-Temp-User": "******"}
            self.swift_client.post_object(self.container, self.object, headers)
            self.logger.warning(
                "Posting Metadata to Object succeeded: %s", headers)

        except Exception as e:
            self.success, self.overall_success = False, False
            self.failure = e
            self.logger.error(
                "Posting Metadata to Object failed %s", sys.exc_info()[1])

    @monitoring.timeit
    def delete_object(self):
        """
        Delete the object created earlier in the process flow
        """

        try:
            self.swift_client.delete_object(self.container, self.object)
            self.logger.warning("Object Deleted")

        except Exception as e:
            self.success, self.overall_success = False, False
            self.failure = e
            self.logger.error(
                "Object Delete failed %s", sys.exc_info()[1])

    @monitoring.timeit
    def delete_container(self):
        """
        Delete the container that was created earlier in the process flow
        """

        try:
            self.swift_client.delete_container(self.container)
            self.logger.warning("Container %s Deleted", self.container)

        except Exception as e:
            self.success, self.overall_success = False, False
            self.failure = e
            self.logger.error(
                "Container Delete Failed %s", sys.exc_info()[1])

    def run(self):
        """
        The driver method that runs all the steps in the process flow
        """

        self.list_containers()
        self.list_objects()

        self.update_metadata_account()
        self.retrieve_metadata_account()

        self.create_container()
        self.get_container()
        self.update_metadata_container()
        self.retrieve_metadata_container()

        self.create_object()
        self.get_object()
        self.update_metadata_object()
        self.retrieve_metadata_object()

        if hasattr(self, 'object'):
            self.delete_object()

        if hasattr(self, 'container'):
            self.delete_container()

        if self.overall_success is True:
            exit(0)
        else:
            exit(1)
Esempio n. 10
0
class Swift:

    def __init__(self, username, password, tenant, auth_url, region):
        self.swift_client = Connection(
            user=username,
            key=password,
            tenant_name=tenant,
            authurl=auth_url,
            auth_version="2.0",
            os_options={'region_name': region}
        )

        # Setup logging
        self.logger = logging.getLogger()
        self.logger.setLevel(logging.WARNING)
    
        # Allow the logger to write all output to stdout too
        self.logger.addHandler(logging.StreamHandler())
  
    def list_containers(self):
        try:
            self.containers = self.swift_client.get_account()[1]
      
            self.logger.warning("Listing Containers")
            for container in self.containers:
                self.logger.warning(container['name'])
    
        except Exception:
            self.logger.error("Listing Containers Failed")
            self.exit(1)

    def list_objects(self):
        try:
            self.logger.warning("Listing Objects")
            for container in self.containers:
                for swift_container in self.swift_client.get_container(container['name'])[1]:
                    self.logger.warning(container['name'] + " " + swift_container['name'])
    
        except Exception as e:
            self.logger.error('Listing Objects Failed, Got exception: %s' % e)
            self.exit(1)

    def create_container(self, name):
        try:
            self.container = name
            self.swift_client.put_container(self.container)
            self.logger.warning("Created Container %s", self.container)

        except Exception:
            self.logger.error("Creating Container Failed")
            self.exit(1)

    def create_object(self, name, contents):
        try:
            self.object = name
            self.swift_client.put_object(self.container, self.object, contents)
            self.logger.warning("Created Object %s", self.object)

        except Exception:
            self.logger.error("Creating Object Failed")
            self.exit(1)

    def get_container(self):
        try:
            self.containers = self.swift_client.get_account()[1]

            for container in self.containers:
                if container['name'] == self.container:
                    self.logger.warning("Getting Container Succeeded")

                    return True

            self.logger.error("Getting Container Failed")
            self.exit(1)

        except Exception as e:
            self.logger.error('Getting Container Failed: Got exception: ' % e)
            self.exit(1)

    def get_object(self):
        try:
            object = self.swift_client.get_object(self.container, self.object)
            self.logger.warning("Object Get: %s", object)

        except Exception as e:
            self.logger.error('Object Get Failed, Got exception: %s' % e)
            self.exit(1)

    def delete_object(self):
        try:
            self.swift_client.delete_object(self.container, self.object)
            self.logger.warning("Object Deleted")

        except Exception as e:
            self.logger.error('Object Deletion Failed: Got exception: ' % e)
      
    def delete_container(self):
        try:
            self.swift_client.delete_container(self.container)
            self.logger.warning("Container Deleted")

        except Exception as e:
            self.logger.error('Container Deletion Failed: Got exception: ' % e)

    def exit(self, value):
        if hasattr(self, 'object'):
            self.delete_object()

        if hasattr(self, 'container'):
            self.delete_container()
    
        exit(value)

    def run(self):
        self.list_containers()
        self.list_objects()

        self.create_container()
        self.create_object()

        self.get_container()
        self.get_object()
    
        self.exit(0)
Esempio n. 11
0
class SwiftStore(MutableMapping):
    """Storage class using Openstack Swift Object Store.

    Parameters
    ----------
    container: string
        swift container to use. It is created if it does not already exists
    prefix: string
        sub-directory path with in the container to store data
    storage_options: dict
        authentication information to connect to the swift store.

    Examples
    --------

    >>> import os
    >>> from zarrswift import SwiftStore
    >>> getenv = os.environ.get
    >>> options = {'preauthurl': getenv('OS_STORAGE_URL'),
    ...            'preauthtoken': getenv('OS_AUTH_TOKEN')}
    >>> store = SwiftStore(container="demo", prefix="zarr_demo", storage_options=options)
    >>> root = zarr.group(store=store, overwrite=True)
    >>> z = root.zeros('foo/bar', shape=(10, 10), chunks=(5, 5), dtype='i4')
    >>> z[:] = 42
    """
    def __init__(self, container, prefix="", storage_options=None):
        self.container = container
        self.prefix = normalize_storage_path(prefix)
        self.storage_options = storage_options or {}
        self.conn = Connection(**self.storage_options)
        self._ensure_container()

    def __getstate__(self):
        state = self.__dict__.copy()
        del state["conn"]
        return state

    def __setstate__(self, state):
        self.__dict__.update(state)
        self.conn = Connection(**self.storage_options)

    def __getitem__(self, name):
        name = self._add_prefix(name)
        try:
            resp, content = self.conn.get_object(self.container, name)
        except ClientException:
            raise KeyError("Object {} not found".format(name))
        return content

    def __setitem__(self, name, value):
        name = self._add_prefix(name)
        value = ensure_bytes(value)
        self.conn.put_object(self.container, name, value)

    def __delitem__(self, name):
        name = self._add_prefix(name)
        try:
            self.conn.delete_object(self.container, name)
        except ClientException:
            raise KeyError("Object {} not found".format(name))

    def __eq__(self, other):
        return (isinstance(other, SwiftStore)
                and self.container == other.container
                and self.prefix == other.prefix)

    def __contains__(self, name):
        return name in self.keys()

    def __iter__(self):
        contents = self._list_container(strip_prefix=True)
        for entry in contents:
            yield entry["name"]

    def __len__(self):
        return len(self.keys())

    def _ensure_container(self):
        _, contents = self.conn.get_account()
        listings = [item["name"] for item in contents]
        if self.container not in listings:
            self.conn.put_container(self.container)

    def _add_prefix(self, path):
        path = normalize_storage_path(path)
        path = "/".join([self.prefix, path])
        return normalize_storage_path(path)

    def _list_container(self,
                        path=None,
                        delimiter=None,
                        strip_prefix=False,
                        treat_path_as_dir=True):
        path = self.prefix if path is None else self._add_prefix(path)
        if path and treat_path_as_dir:
            path += "/"
        _, contents = self.conn.get_container(self.container,
                                              prefix=path,
                                              delimiter=delimiter)
        if strip_prefix:
            prefix_size = len(path)
            for entry in contents:
                name = entry.get('name', entry.get('subdir', ''))
                entry["name"] = normalize_storage_path(name[prefix_size:])
        for entry in contents:
            entry["bytes"] = entry.get("bytes", 0)
        return contents

    def keys(self):
        return list(self.__iter__())

    def listdir(self, path=None):
        contents = self._list_container(path, delimiter="/", strip_prefix=True)
        listings = [entry["name"] for entry in contents]
        return sorted(listings)

    def getsize(self, path=None):
        contents = self._list_container(path,
                                        strip_prefix=True,
                                        treat_path_as_dir=False)
        contents = [entry for entry in contents if "/" not in entry["name"]]
        return sum([entry["bytes"] for entry in contents])

    def rmdir(self, path=None):
        contents = self._list_container(path)
        for entry in contents:
            self.conn.delete_object(self.container, entry["name"])

    def clear(self):
        self.rmdir()

    @property
    def url(self):
        _url = '/'.join([self.conn.url, self.container, self.prefix])
        if not self.prefix:
            _url = _url.rstrip('/')
        return _url
Esempio n. 12
0
class SwiftBackend(duplicity.backend.Backend):
    """
    Backend for Swift
    """
    def __init__(self, parsed_url):
        try:
            from swiftclient import Connection
            from swiftclient import ClientException
        except ImportError:
            raise BackendException("This backend requires "
                                   "the python-swiftclient library.")

        self.resp_exc = ClientException
        conn_kwargs = {}

        # if the user has already authenticated
        if 'SWIFT_PREAUTHURL' in os.environ and 'SWIFT_PREAUTHTOKEN' in os.environ:
            conn_kwargs['preauthurl'] = os.environ['SWIFT_PREAUTHURL']
            conn_kwargs['preauthtoken'] = os.environ['SWIFT_PREAUTHTOKEN']

        else:
            if 'SWIFT_USERNAME' not in os.environ:
                raise BackendException('SWIFT_USERNAME environment variable '
                                       'not set.')

            if 'SWIFT_PASSWORD' not in os.environ:
                raise BackendException('SWIFT_PASSWORD environment variable '
                                       'not set.')

            if 'SWIFT_AUTHURL' not in os.environ:
                raise BackendException('SWIFT_AUTHURL environment variable '
                                       'not set.')

            conn_kwargs['user'] = os.environ['SWIFT_USERNAME']
            conn_kwargs['key'] = os.environ['SWIFT_PASSWORD']
            conn_kwargs['authurl'] = os.environ['SWIFT_AUTHURL']

        if 'SWIFT_AUTHVERSION' in os.environ:
            conn_kwargs['auth_version'] = os.environ['SWIFT_AUTHVERSION']
        else:
            conn_kwargs['auth_version'] = '1'
        if 'SWIFT_TENANTNAME' in os.environ:
            conn_kwargs['tenant_name'] = os.environ['SWIFT_TENANTNAME']
        if 'SWIFT_REGIONNAME' in os.environ:
            conn_kwargs['os_options'] = {'region_name': os.environ['SWIFT_REGIONNAME']}

        self.container = parsed_url.path.lstrip('/')

        container_metadata = None
        try:
            self.conn = Connection(**conn_kwargs)
            container_metadata = self.conn.head_container(self.container)
        except ClientException:
            pass
        except Exception as e:
            log.FatalError("Connection failed: %s %s"
                           % (e.__class__.__name__, str(e)),
                           log.ErrorCode.connection_failed)

        if container_metadata is None:
            log.Info("Creating container %s" % self.container)
            try:
                self.conn.put_container(self.container)
            except Exception as e:
                log.FatalError("Container creation failed: %s %s"
                               % (e.__class__.__name__, str(e)),
                               log.ErrorCode.connection_failed)

    def _error_code(self, operation, e):
        if isinstance(e, self.resp_exc):
            if e.http_status == 404:
                return log.ErrorCode.backend_not_found

    def _put(self, source_path, remote_filename):
        self.conn.put_object(self.container, remote_filename,
                             file(source_path.name))

    def _get(self, remote_filename, local_path):
        headers, body = self.conn.get_object(self.container, remote_filename)
        with open(local_path.name, 'wb') as f:
            for chunk in body:
                f.write(chunk)

    def _list(self):
        headers, objs = self.conn.get_container(self.container, full_listing=True)
        return [o['name'] for o in objs]

    def _delete(self, filename):
        self.conn.delete_object(self.container, filename)

    def _query(self, filename):
        sobject = self.conn.head_object(self.container, filename)
        return {'size': int(sobject['content-length'])}
Esempio n. 13
0
class PCABackend(duplicity.backend.Backend):
    """
    Backend for OVH PCA
    """
    def __init__(self, parsed_url):
        duplicity.backend.Backend.__init__(self, parsed_url)

        try:
            from swiftclient import Connection
            from swiftclient import ClientException
        except ImportError as e:
            raise BackendException("""\
PCA backend requires the python-swiftclient library.
Exception: %s""" % str(e))

        self.resp_exc = ClientException
        self.conn_cls = Connection
        conn_kwargs = {}

        # if the user has already authenticated
        if 'PCA_PREAUTHURL' in os.environ and 'PCA_PREAUTHTOKEN' in os.environ:
            conn_kwargs['preauthurl'] = os.environ['PCA_PREAUTHURL']
            conn_kwargs['preauthtoken'] = os.environ['PCA_PREAUTHTOKEN']

        else:
            if 'PCA_USERNAME' not in os.environ:
                raise BackendException('PCA_USERNAME environment variable '
                                       'not set.')

            if 'PCA_PASSWORD' not in os.environ:
                raise BackendException('PCA_PASSWORD environment variable '
                                       'not set.')

            if 'PCA_AUTHURL' not in os.environ:
                raise BackendException('PCA_AUTHURL environment variable '
                                       'not set.')

            conn_kwargs['user'] = os.environ['PCA_USERNAME']
            conn_kwargs['key'] = os.environ['PCA_PASSWORD']
            conn_kwargs['authurl'] = os.environ['PCA_AUTHURL']

        os_options = {}

        if 'PCA_AUTHVERSION' in os.environ:
            conn_kwargs['auth_version'] = os.environ['PCA_AUTHVERSION']
            if os.environ['PCA_AUTHVERSION'] == '3':
                if 'PCA_USER_DOMAIN_NAME' in os.environ:
                    os_options.update({'user_domain_name': os.environ['PCA_USER_DOMAIN_NAME']})
                if 'PCA_USER_DOMAIN_ID' in os.environ:
                    os_options.update({'user_domain_id': os.environ['PCA_USER_DOMAIN_ID']})
                if 'PCA_PROJECT_DOMAIN_NAME' in os.environ:
                    os_options.update({'project_domain_name': os.environ['PCA_PROJECT_DOMAIN_NAME']})
                if 'PCA_PROJECT_DOMAIN_ID' in os.environ:
                    os_options.update({'project_domain_id': os.environ['PCA_PROJECT_DOMAIN_ID']})
                if 'PCA_TENANTNAME' in os.environ:
                    os_options.update({'tenant_name': os.environ['PCA_TENANTNAME']})
                if 'PCA_ENDPOINT_TYPE' in os.environ:
                    os_options.update({'endpoint_type': os.environ['PCA_ENDPOINT_TYPE']})
                if 'PCA_USERID' in os.environ:
                    os_options.update({'user_id': os.environ['PCA_USERID']})
                if 'PCA_TENANTID' in os.environ:
                    os_options.update({'tenant_id': os.environ['PCA_TENANTID']})
                if 'PCA_REGIONNAME' in os.environ:
                    os_options.update({'region_name': os.environ['PCA_REGIONNAME']})

        else:
            conn_kwargs['auth_version'] = '2'
        if 'PCA_TENANTNAME' in os.environ:
            conn_kwargs['tenant_name'] = os.environ['PCA_TENANTNAME']
        if 'PCA_REGIONNAME' in os.environ:
            os_options.update({'region_name': os.environ['PCA_REGIONNAME']})

        conn_kwargs['os_options'] = os_options
        conn_kwargs['retries'] = 0

        self.conn_kwargs = conn_kwargs

        # This folds the null prefix and all null parts, which means that:
        #  //MyContainer/ and //MyContainer are equivalent.
        #  //MyContainer//My/Prefix/ and //MyContainer/My/Prefix are equivalent.
        url_parts = [x for x in parsed_url.path.split('/') if x != '']

        self.container = url_parts.pop(0)
        if url_parts:
            self.prefix = '%s/' % '/'.join(url_parts)
        else:
            self.prefix = ''

        policy = 'PCA'
        policy_header = 'X-Storage-Policy'

        container_metadata = None
        try:
            self.conn = Connection(**self.conn_kwargs)
            container_metadata = self.conn.head_container(self.container)
        except ClientException:
            pass
        except Exception as e:
            log.FatalError("Connection failed: %s %s"
                           % (e.__class__.__name__, str(e)),
                           log.ErrorCode.connection_failed)

        if container_metadata is None:
            log.Info("Creating container %s" % self.container)
            try:
                headers = dict([[policy_header, policy]])
                self.conn.put_container(self.container, headers=headers)
            except Exception as e:
                log.FatalError("Container creation failed: %s %s"
                               % (e.__class__.__name__, str(e)),
                               log.ErrorCode.connection_failed)
        elif policy and container_metadata[policy_header.lower()] != policy:
            log.FatalError("Container '%s' exists but its storage policy is '%s' not '%s'."
                           % (self.container, container_metadata[policy_header.lower()], policy))

    def _error_code(self, operation, e):
        if isinstance(e, self.resp_exc):
            if e.http_status == 404:
                return log.ErrorCode.backend_not_found

    def _put(self, source_path, remote_filename):
        self.conn.put_object(self.container, self.prefix + remote_filename,
                             file(source_path.name))

    def _get(self, remote_filename, local_path):
        body = self.preprocess_download(remote_filename, 60)
        if body:
            with open(local_path.name, 'wb') as f:
                for chunk in body:
                    f.write(chunk)

    def _list(self):
        headers, objs = self.conn.get_container(self.container, full_listing=True, path=self.prefix)
        # removes prefix from return values. should check for the prefix ?
        return [o['name'][len(self.prefix):] for o in objs]

    def _delete(self, filename):
        self.conn.delete_object(self.container, self.prefix + filename)

    def _query(self, filename):
        sobject = self.conn.head_object(self.container, self.prefix + filename)
        return {'size': int(sobject['content-length'])}

    def preprocess_download(self, remote_filename, retry_period, wait=True):
        body = self.unseal(remote_filename)
        try:
            if wait:
                while not body:
                    time.sleep(retry_period)
                    self.conn = self.conn_cls(**self.conn_kwargs)
                    body = self.unseal(remote_filename)
                    self.conn.close()
        except Exception as e:
            log.FatalError("Connection failed: %s %s" % (e.__class__.__name__, str(e)),
                           log.ErrorCode.connection_failed)
        return body

    def unseal(self, remote_filename):
        try:
            _, body = self.conn.get_object(self.container, self.prefix + remote_filename,
                                           resp_chunk_size=1024)
            log.Info("File %s was successfully unsealed." % remote_filename)
            return body
        except self.resp_exc as e:
            # The object is sealed but being released.
            if e.http_status == 429:
                # The retry-after header contains the remaining duration before
                # the unsealing operation completes.
                duration = int(e.http_response_headers['Retry-After'])
                m, s = divmod(duration, 60)
                h, m = divmod(m, 60)
                eta = "%dh%02dm%02ds" % (h, m, s)
                log.Info("File %s is being unsealed, operation ETA is %s." %
                         (remote_filename, eta))
            else:
                raise
Esempio n. 14
0
raw_input("Any Containers listed this time?...")

# Upload object to container
print ' '
print 'creating object %s in %s...' % (object_name, container_name)
swift.put_object(container=container_name,
                 obj=object_name,
                 contents='Hello World!',
                 content_type='text/plain')
print 'done!'
print '------------------------\n'

# Read object back
print ' '
print 'reading object %s in %s...\n' % (object_name, container_name)
(headers, content) = swift.get_object(container=container_name,
                                      obj=object_name)
print 'content: %s\n' % content

# Uncomment to print out http header
#print 'headers: '
#pprint(headers)

print 'done!'
print '------------------------\n'
raw_input("Object created, now verify with Cyberduck...")

# Delete object...clean up
print ' '
print 'Deleting object %s in %s...\n' % (object_name, container_name)
swift.delete_object(container=container_name, obj=object_name)
print 'done!'
Esempio n. 15
0
class PCABackend(duplicity.backend.Backend):
    u"""
    Backend for OVH PCA
    """
    def __init__(self, parsed_url):
        duplicity.backend.Backend.__init__(self, parsed_url)

        try:
            from swiftclient import Connection
            from swiftclient import ClientException
        except ImportError as e:
            raise BackendException(u"""\
PCA backend requires the python-swiftclient library.
Exception: %s""" % str(e))

        self.resp_exc = ClientException
        self.conn_cls = Connection
        conn_kwargs = {}

        # if the user has already authenticated
        if u'PCA_PREAUTHURL' in os.environ and u'PCA_PREAUTHTOKEN' in os.environ:
            conn_kwargs[u'preauthurl'] = os.environ[u'PCA_PREAUTHURL']
            conn_kwargs[u'preauthtoken'] = os.environ[u'PCA_PREAUTHTOKEN']

        else:
            if u'PCA_USERNAME' not in os.environ:
                raise BackendException(u'PCA_USERNAME environment variable '
                                       u'not set.')

            if u'PCA_PASSWORD' not in os.environ:
                raise BackendException(u'PCA_PASSWORD environment variable '
                                       u'not set.')

            if u'PCA_AUTHURL' not in os.environ:
                raise BackendException(u'PCA_AUTHURL environment variable '
                                       u'not set.')

            conn_kwargs[u'user'] = os.environ[u'PCA_USERNAME']
            conn_kwargs[u'key'] = os.environ[u'PCA_PASSWORD']
            conn_kwargs[u'authurl'] = os.environ[u'PCA_AUTHURL']

        os_options = {}

        if u'PCA_AUTHVERSION' in os.environ:
            conn_kwargs[u'auth_version'] = os.environ[u'PCA_AUTHVERSION']
            if os.environ[u'PCA_AUTHVERSION'] == u'3':
                if u'PCA_USER_DOMAIN_NAME' in os.environ:
                    os_options.update({u'user_domain_name': os.environ[u'PCA_USER_DOMAIN_NAME']})
                if u'PCA_USER_DOMAIN_ID' in os.environ:
                    os_options.update({u'user_domain_id': os.environ[u'PCA_USER_DOMAIN_ID']})
                if u'PCA_PROJECT_DOMAIN_NAME' in os.environ:
                    os_options.update({u'project_domain_name': os.environ[u'PCA_PROJECT_DOMAIN_NAME']})
                if u'PCA_PROJECT_DOMAIN_ID' in os.environ:
                    os_options.update({u'project_domain_id': os.environ[u'PCA_PROJECT_DOMAIN_ID']})
                if u'PCA_TENANTNAME' in os.environ:
                    os_options.update({u'tenant_name': os.environ[u'PCA_TENANTNAME']})
                if u'PCA_ENDPOINT_TYPE' in os.environ:
                    os_options.update({u'endpoint_type': os.environ[u'PCA_ENDPOINT_TYPE']})
                if u'PCA_USERID' in os.environ:
                    os_options.update({u'user_id': os.environ[u'PCA_USERID']})
                if u'PCA_TENANTID' in os.environ:
                    os_options.update({u'tenant_id': os.environ[u'PCA_TENANTID']})
                if u'PCA_REGIONNAME' in os.environ:
                    os_options.update({u'region_name': os.environ[u'PCA_REGIONNAME']})

        else:
            conn_kwargs[u'auth_version'] = u'2'
        if u'PCA_TENANTNAME' in os.environ:
            conn_kwargs[u'tenant_name'] = os.environ[u'PCA_TENANTNAME']
        if u'PCA_REGIONNAME' in os.environ:
            os_options.update({u'region_name': os.environ[u'PCA_REGIONNAME']})

        conn_kwargs[u'os_options'] = os_options
        conn_kwargs[u'retries'] = 0

        self.conn_kwargs = conn_kwargs

        # This folds the null prefix and all null parts, which means that:
        #  //MyContainer/ and //MyContainer are equivalent.
        #  //MyContainer//My/Prefix/ and //MyContainer/My/Prefix are equivalent.
        url_parts = [x for x in parsed_url.path.split(u'/') if x != u'']

        self.container = url_parts.pop(0)
        if url_parts:
            self.prefix = u'%s/' % u'/'.join(url_parts)
        else:
            self.prefix = u''

        policy = u'PCA'
        policy_header = u'X-Storage-Policy'

        container_metadata = None
        try:
            self.conn = Connection(**self.conn_kwargs)
            container_metadata = self.conn.head_container(self.container)
        except ClientException:
            pass
        except Exception as e:
            log.FatalError(u"Connection failed: %s %s"
                           % (e.__class__.__name__, str(e)),
                           log.ErrorCode.connection_failed)

        if container_metadata is None:
            log.Info(u"Creating container %s" % self.container)
            try:
                headers = dict([[policy_header, policy]])
                self.conn.put_container(self.container, headers=headers)
            except Exception as e:
                log.FatalError(u"Container creation failed: %s %s"
                               % (e.__class__.__name__, str(e)),
                               log.ErrorCode.connection_failed)
        elif policy and container_metadata[policy_header.lower()] != policy:
            log.FatalError(u"Container '%s' exists but its storage policy is '%s' not '%s'."
                           % (self.container, container_metadata[policy_header.lower()], policy))

    def _error_code(self, operation, e):  # pylint: disable= unused-argument
        if isinstance(e, self.resp_exc):
            if e.http_status == 404:
                return log.ErrorCode.backend_not_found

    def _put(self, source_path, remote_filename):
        self.conn.put_object(self.container, self.prefix + util.fsdecode(remote_filename),
                             open(util.fsdecode(source_path.name), u'rb'))

    def _get(self, remote_filename, local_path):
        body = self.unseal(util.fsdecode(remote_filename))
        if body:
            with open(util.fsdecode(local_path.name), u'wb') as f:
                for chunk in body:
                    f.write(chunk)

    def __list_objs(self, ffilter=None):
        # full_listing should be set to True but a bug in python-swiftclient
        # doesn't forward query_string in this case...
        # bypass here using a patched copy (with query_string) from swiftclient code
        rv = self.conn.get_container(self.container, full_listing=False,
                                     path=self.prefix, query_string=u'policy_extra=true')
        listing = rv[1]
        while listing:
            marker = listing[-1][u'name']
            version_marker = listing[-1].get(u'version_id')
            listing = self.conn.get_container(self.container, marker=marker, version_marker=version_marker,
                                              full_listing=False, path=self.prefix,
                                              query_string=u'policy_extra=true')[1]
            if listing:
                rv[1].extend(listing)
        if ffilter is not None:
            return filter(ffilter, rv[1])
        return rv[1]

    def _list(self):
        return [util.fsencode(o[u'name'][len(self.prefix):]) for o in self.__list_objs()]

    def _delete(self, filename):
        self.conn.delete_object(self.container, self.prefix + util.fsdecode(filename))

    def _query(self, filename):
        sobject = self.conn.head_object(self.container, self.prefix + util.fsdecode(filename))
        return {u'size': int(sobject[u'content-length'])}

    def unseal(self, remote_filename):
        try:
            _, body = self.conn.get_object(self.container, self.prefix + remote_filename,
                                           resp_chunk_size=1024)
            log.Info(u"File %s was successfully unsealed." % remote_filename)
            return body
        except self.resp_exc as e:
            # The object is sealed but being released.
            if e.http_status == 429:
                # The retry-after header contains the remaining duration before
                # the unsealing operation completes.
                duration = int(e.http_response_headers[u'Retry-After'])
                m, s = divmod(duration, 60)
                h, m = divmod(m, 60)
                eta = u"%dh%02dm%02ds" % (h, m, s)
                log.Info(u"File %s is being unsealed, operation ETA is %s." %
                         (remote_filename, eta))
            else:
                log.FatalError(u"Connection failed: %s %s" % (e.__class__.__name__, str(e)),
                               log.ErrorCode.connection_failed)
        return None

    def pre_process_download_batch(self, remote_filenames):
        u"""
        This is called before downloading volumes from this backend
        by main engine. For PCA, volumes passed as argument need to be unsealed.
        This method is blocking, showing a status at regular interval
        """
        retry_interval = 60  # status will be shown every 60s
        # remote_filenames are bytes string
        u_remote_filenames = list(map(util.fsdecode, remote_filenames))
        objs = self.__list_objs(ffilter=lambda x: x[u'name'] in u_remote_filenames)
        # first step: retrieve pca seal status for all required volumes
        # and launch unseal for all sealed files
        one_object_not_unsealed = False
        for o in objs:
            filename = util.fsdecode(o[u'name'])
            # see ovh documentation for policy_retrieval_state definition
            policy_retrieval_state = o[u'policy_retrieval_state']
            log.Info(u"Volume %s. State : %s. " % (filename, policy_retrieval_state))
            if policy_retrieval_state == u'sealed':
                log.Notice(u"Launching unseal of volume %s." % (filename))
                self.unseal(o[u'name'])
                one_object_not_unsealed = True
            elif policy_retrieval_state == u"unsealing":
                one_object_not_unsealed = True
        # second step: display estimated time for last volume unseal
        # and loop until all volumes are unsealed
        while one_object_not_unsealed:
            one_object_not_unsealed = self.unseal_status(u_remote_filenames)
            time.sleep(retry_interval)
            # might be a good idea to show a progress bar here...
        else:
            log.Notice(u"All volumes to download are unsealed.")

    def unseal_status(self, u_remote_filenames):
        u"""
        Shows unsealing status for input volumes
        """
        one_object_not_unsealed = False
        objs = self.__list_objs(ffilter=lambda x: util.fsdecode(x[u'name']) in u_remote_filenames)
        max_duration = 0
        for o in objs:
            policy_retrieval_state = o[u'policy_retrieval_state']
            filename = util.fsdecode(o[u'name'])
            if policy_retrieval_state == u'sealed':
                log.Notice(u"Error: volume is still in sealed state : %s." % (filename))
                log.Notice(u"Launching unseal of volume %s." % (filename))
                self.unseal(o[u'name'])
                one_object_not_unsealed = True
            elif policy_retrieval_state == u"unsealing":
                duration = int(o[u'policy_retrieval_delay'])
                log.Info(u"%s available in %d seconds." % (filename, duration))
                if duration > max_duration:
                    max_duration = duration
                one_object_not_unsealed = True

        m, s = divmod(max_duration, 60)
        h, m = divmod(m, 60)
        max_duration_eta = u"%dh%02dm%02ds" % (h, m, s)
        log.Notice(u"Need to wait %s before all volumes are unsealed." % (max_duration_eta))
        return one_object_not_unsealed