def get_connection(self, config): timeout = (int(config['timeout']) if config.get('timeout') else None) if config['auth_version'] == '3': # Keystone v3 os_options = {} for option, value in config.items(): if option.startswith('os_options_') and value: short_option = option[len('os_options_'):] os_options[short_option] = config.get(option) connection = Connection(auth_version='3', authurl=config['authurl'], user=config['user'], key=config['key'], os_options=os_options, timeout=timeout) elif config['auth_version'] == '1': # Legacy auth connection = Connection(auth_version='1', authurl=config['authurl'], user=config['user'], key=config['key'], tenant_name=config['tenant_name'], timeout=timeout) else: raise NotImplementedError('auth_version? {!r}'.format(config)) return connection
def get_swift(self): if self.swift_authver == '3': os_options = { 'project_name': self.swift_project, 'project_domain_name': self.swift_pdomain, 'user_domain_name': self.swift_udomain, } connection = Connection(auth_version='3', authurl=self.swift_auth, user=self.swift_user, key=self.swift_key, os_options=os_options) elif self.swift_authver == '1': connection = Connection(auth_version='1', authurl=self.swift_auth, user=self.swift_user, key=self.swift_key, tenant_name='UNUSED') else: raise NotImplementedError('auth_version? {!r}'.format( self.swift_authver)) return connection
def connect_swift(auth_version=None, auth_url=None, access_key=None, user_domain_name=None, secret_key=None, tenant_name=None, project_domain_name=None, node_settings=None, timeout=None): """Helper to build an swiftclient.Connection object """ if node_settings is not None: if node_settings.external_account is not None: provider = SwiftProvider(node_settings.external_account) auth_url, tenant_name = provider.auth_url, provider.tenant_name auth_version = provider.auth_version user_domain_name = provider.user_domain_name project_domain_name = provider.project_domain_name access_key, secret_key = provider.username, provider.password if auth_version == '2': connection = Connection(auth_version='2', authurl=auth_url, user=access_key, key=secret_key, tenant_name=tenant_name, timeout=timeout) else: os_options = {'user_domain_name': user_domain_name, 'project_domain_name': project_domain_name, 'project_name': tenant_name} connection = Connection(auth_version='3', authurl=auth_url, user=access_key, key=secret_key, os_options=os_options, timeout=timeout) return connection
def __init__(self, *args, **kwargs): super(SwiftStorage, self).__init__() self.__dict__.update(kwargs) config_data = get_config_json() # Load config variables. self.auth_url = config_data.get('swift').get('auth_url') self.access_key = config_data.get('swift').get('access_key') self.secret_key = config_data.get('swift').get('secret_key') self.auth_version = config_data.get('swift').get('auth_version') self.tenant_name = config_data.get('swift').get('tenant_name') self.insecure = True self.container = config_data.get('swift').get('container') self.name_backup = kwargs.get('name_backup', None) self.conn = Connection(authurl=self.auth_url, user=self.access_key, key=self.secret_key, auth_version=self.auth_version, tenant_name=self.tenant_name, insecure=self.insecure) try: self.conn.head_container(self.container) except: self.conn.put_container(self.container) if not self.name_backup: raise SystemExit(_error_codes.get(103))
def __init__(self): # Read configuration from environment variables (openstack.rc) auth_url = environ["OPENSTACK_AUTH_URL"] user_domain_name = environ["OPENSTACK_USER_DOMAIN_NAME"] user_name = environ["OPENSTACK_USERNAME"] password = environ["OPENSTACK_PASSWORD"] project_domain_name = environ["OPENSTACK_PROJECT_DOMAIN_NAME"] project_name = environ["OPENSTACK_PROJECT_NAME"] container_name = environ["OPENSTACK_CONTAINER_NAME"] ca_certificate = environ["OPENSTACK_CA_CERTIFICATE"] self.container_name = container_name options = { 'user_domain_name': user_domain_name, 'project_domain_name': project_domain_name, 'project_name': project_name, } # Establish the connection with the object storage API cacert=ca_certificate, self.conn = Connection(user=user_name, key=password, authurl=auth_url, auth_version=3, os_options=options, cacert=ca_certificate, timeout=120) found = False for container in self.conn.get_account()[1]: cname = container['name'] if cname == container_name: found = True if found is not True: # Create a new container container_name = container_name self.conn.put_container(container_name)
def create_sw_conn(): global swift_auth, swift_auth_token, storage_url if swift_auth_token and storage_url: return Connection(preauthtoken=swift_auth_token, preauthurl=storage_url) if swift_auth: swift_user = os.environ.get("ST_USER") swift_key = os.environ.get("ST_KEY") if swift_user and swift_key: return Connection(authurl=swift_auth, user=swift_user, key=swift_key) print("Error: Swift environment not configured!") sys.exit()
def connect_to_swift(): try: # establish connection conn = Connection(auth_url, username, password, tenant_name=tenant_name, auth_version="2.0") except ClientException, err: LOG.critical("No Swift Connection: %s", str(err))
def get_swiftclient(**kwargs): # nmap has entries that need name changes from a 'get_service_conn_info' # to a swift Connection name. # pt has names that pass straight through nmap = {'endpoint': 'preauthurl', 'token': 'preauthtoken'} pt = ('insecure', 'cacert') connargs = {v: kwargs.get(k) for k, v in nmap.items() if k in kwargs} connargs.update({k: kwargs.get(k) for k in pt if k in kwargs}) if kwargs.get('session'): sess = kwargs.get('session') try: # If session is available try it return Connection(session=sess) except TypeError: # The edge case where session is availble but swiftclient is # < 3.3.0. Use the old style method for Connection. pass return Connection(**connargs)
def create_sw_conn(): global swift_auth swift_user = os.environ.get("ST_USER") swift_key = os.environ.get("ST_KEY") if swift_auth and swift_user and swift_key: return Connection(authurl=swift_auth, user=swift_user, key=swift_key) print("Error: Swift environment not configured!")
def create_sw_conn(): swift_auth = _default_global_options['auth'] swift_auth_token = _default_global_options['os_auth_token'] storage_url = _default_global_options['os_storage_url'] if swift_auth_token and storage_url: return Connection(preauthtoken=swift_auth_token, preauthurl=storage_url) if swift_auth: swift_user = os.environ.get("ST_USER") swift_key = os.environ.get("ST_KEY") if swift_user and swift_key: return Connection(authurl=swift_auth, user=swift_user, key=swift_key) print("Error: Swift environment not configured!") sys.exit()
def create_connection(authurl, username, password, tenant_name, region): cfconn = Connection(authurl, username, password, snet=False, tenant_name=tenant_name, auth_version='2.0', os_options={'region_name': region, 'endpoint_type': 'publicURL', 'service_type': 'object-store'}) return cfconn
def get_connection(self): """ Connect to swift storage and return the connection object. """ if self._conn is not None: return self._conn try: self._conn = Connection(**self.conn_params) except ClientException as e: logger.error(str(e)) raise return self._conn
def get_conn(options): """ Return a SWIFT connection from the options dict. """ return Connection(options['authurl'], options['user'], options['key'], options['retries'], auth_version=options['auth_version'], os_options=options['os_options'], snet=options['snet'], cacert=options['os_cacert'], insecure=options['insecure'], ssl_compression=options['ssl_compression'])
def __init__(self, parsed_url): try: from swiftclient import Connection from swiftclient import ClientException except ImportError: raise BackendException("This backend requires " "the python-swiftclient library.") self.resp_exc = ClientException conn_kwargs = {} # if the user has already authenticated if os.environ.has_key('SWIFT_PREAUTHURL') and os.environ.has_key( 'SWIFT_PREAUTHTOKEN'): conn_kwargs['preauthurl'] = os.environ['SWIFT_PREAUTHURL'] conn_kwargs['preauthtoken'] = os.environ['SWIFT_PREAUTHTOKEN'] else: if not os.environ.has_key('SWIFT_USERNAME'): raise BackendException('SWIFT_USERNAME environment variable ' 'not set.') if not os.environ.has_key('SWIFT_PASSWORD'): raise BackendException('SWIFT_PASSWORD environment variable ' 'not set.') if not os.environ.has_key('SWIFT_AUTHURL'): raise BackendException('SWIFT_AUTHURL environment variable ' 'not set.') conn_kwargs['user'] = os.environ['SWIFT_USERNAME'] conn_kwargs['key'] = os.environ['SWIFT_PASSWORD'] conn_kwargs['authurl'] = os.environ['SWIFT_AUTHURL'] if os.environ.has_key('SWIFT_AUTHVERSION'): conn_kwargs['auth_version'] = os.environ['SWIFT_AUTHVERSION'] else: conn_kwargs['auth_version'] = '1' if os.environ.has_key('SWIFT_TENANTNAME'): conn_kwargs['tenant_name'] = os.environ['SWIFT_TENANTNAME'] self.container = parsed_url.path.lstrip('/') try: self.conn = Connection(**conn_kwargs) self.conn.put_container(self.container) except Exception, e: log.FatalError( "Connection failed: %s %s" % (e.__class__.__name__, str(e)), log.ErrorCode.connection_failed)
def get_connection(self): """ Connect to swift storage and return the connection object. """ if self._conn is not None: return self._conn for i in range(5): # 5 retries at most try: self._conn = Connection(**self.conn_params) except ClientException as e: logger.error(str(e)) if i == 4: raise # give up time.sleep(0.4) else: return self._conn
def test_delete(self, mocked_get_container_path, mocked_get_connection): """ Test and assert that delete_object is called with the correct arguments """ mocked_get_container_path.return_value = 'container/path/' objstore = ObjectStore(config='this is the config') connection = Connection() connection.delete_object = Mock() mocked_get_connection.return_value = connection document = Document(filename='doc_file_name.pdf', type=Document.DocumentType.Ontwerp) objstore.delete(document) connection.delete_object.assert_called_with('container/path/', 'doc_file_name.pdf')
def __init__(self, conf={}, profile="default"): BakthatBackend.__init__(self, conf, profile) from swiftclient import Connection, ClientException self.con = Connection(self.conf["auth_url"], self.conf["access_key"], self.conf["secret_key"], auth_version=self.conf["auth_version"], insecure=True) region_name = self.conf["region_name"] if region_name == DEFAULT_LOCATION: region_name = "" try: self.con.head_container(self.conf["s3_bucket"]) except ClientException, e: self.con.put_container(self.conf["s3_bucket"])
def test_delete_non_existing_file(self, mocked_get_container_path, mocked_get_connection): """ Test and assert that delete_object is called with the correct arguments """ mocked_get_container_path.return_value = 'container/path/' objstore = ObjectStore(config='this is the config') connection = Connection() connection.delete_object = Mock(side_effect=ClientException('')) mocked_get_connection.return_value = connection document = Document(id=1, filename='doc_file_name.pdf', type=Document.DocumentType.Ontwerp) with self.assertLogs(level='INFO') as logs: objstore.delete(document) connection.delete_object.assert_called_with('container/path/', 'doc_file_name.pdf') self.assertIn( 'INFO:storage.objectstore:Failed to delete object for document id 1', logs.output)
def __init__(self, username, password, tenant_name, authurl): self.username = username self.password = password self.tenant_name = tenant_name self.authurl = authurl self.os_options = { 'tenant_id': None, 'tenant_name': tenant_name, 'service_type': None, 'endpoint_type': None, 'auth_token': None, 'object_storage_url': None, 'region_name': None, } self.conn = Connection( self.authurl, self.username, self.password, 1, auth_version=2.0, os_options=self.os_options)
def __init__(self, conf): ''' 根据配置文件初始化连接Swift云存储的客户端,包括认证系统和存储系统。 ''' self.conf = conf colon = conf['storuser'].find(":") if colon > 0: self.user = conf['storuser'][:colon] self.tenant = conf['storuser'][colon + 1:] else: self.user = self.tenant = conf['storuser'] print self.user, self.tenant self.pawd = conf['storpass'] self.authurl = "http://" + conf['storip'] + ":" + conf[ 'storport'] + "/v2.0/" # self.keystone = client.Client(username="******", password="******", # tenant_name="admin", auth_url=self.authurl) self.swift = Connection(authurl=self.authurl, user=self.user, key=self.pawd, auth_version="2", tenant_name=self.tenant, insecure=True)
def __init__(self, parsed_url): duplicity.backend.Backend.__init__(self, parsed_url) try: from swiftclient import Connection from swiftclient import ClientException except ImportError as e: raise BackendException("""\ PCA backend requires the python-swiftclient library. Exception: %s""" % str(e)) self.resp_exc = ClientException self.conn_cls = Connection conn_kwargs = {} # if the user has already authenticated if 'PCA_PREAUTHURL' in os.environ and 'PCA_PREAUTHTOKEN' in os.environ: conn_kwargs['preauthurl'] = os.environ['PCA_PREAUTHURL'] conn_kwargs['preauthtoken'] = os.environ['PCA_PREAUTHTOKEN'] else: if 'PCA_USERNAME' not in os.environ: raise BackendException('PCA_USERNAME environment variable ' 'not set.') if 'PCA_PASSWORD' not in os.environ: raise BackendException('PCA_PASSWORD environment variable ' 'not set.') if 'PCA_AUTHURL' not in os.environ: raise BackendException('PCA_AUTHURL environment variable ' 'not set.') conn_kwargs['user'] = os.environ['PCA_USERNAME'] conn_kwargs['key'] = os.environ['PCA_PASSWORD'] conn_kwargs['authurl'] = os.environ['PCA_AUTHURL'] os_options = {} if 'PCA_AUTHVERSION' in os.environ: conn_kwargs['auth_version'] = os.environ['PCA_AUTHVERSION'] if os.environ['PCA_AUTHVERSION'] == '3': if 'PCA_USER_DOMAIN_NAME' in os.environ: os_options.update({ 'user_domain_name': os.environ['PCA_USER_DOMAIN_NAME'] }) if 'PCA_USER_DOMAIN_ID' in os.environ: os_options.update( {'user_domain_id': os.environ['PCA_USER_DOMAIN_ID']}) if 'PCA_PROJECT_DOMAIN_NAME' in os.environ: os_options.update({ 'project_domain_name': os.environ['PCA_PROJECT_DOMAIN_NAME'] }) if 'PCA_PROJECT_DOMAIN_ID' in os.environ: os_options.update({ 'project_domain_id': os.environ['PCA_PROJECT_DOMAIN_ID'] }) if 'PCA_TENANTNAME' in os.environ: os_options.update( {'tenant_name': os.environ['PCA_TENANTNAME']}) if 'PCA_ENDPOINT_TYPE' in os.environ: os_options.update( {'endpoint_type': os.environ['PCA_ENDPOINT_TYPE']}) if 'PCA_USERID' in os.environ: os_options.update({'user_id': os.environ['PCA_USERID']}) if 'PCA_TENANTID' in os.environ: os_options.update( {'tenant_id': os.environ['PCA_TENANTID']}) if 'PCA_REGIONNAME' in os.environ: os_options.update( {'region_name': os.environ['PCA_REGIONNAME']}) else: conn_kwargs['auth_version'] = '2' if 'PCA_TENANTNAME' in os.environ: conn_kwargs['tenant_name'] = os.environ['PCA_TENANTNAME'] if 'PCA_REGIONNAME' in os.environ: os_options.update({'region_name': os.environ['PCA_REGIONNAME']}) conn_kwargs['os_options'] = os_options conn_kwargs['retries'] = 0 self.conn_kwargs = conn_kwargs # This folds the null prefix and all null parts, which means that: # //MyContainer/ and //MyContainer are equivalent. # //MyContainer//My/Prefix/ and //MyContainer/My/Prefix are equivalent. url_parts = [x for x in parsed_url.path.split('/') if x != ''] self.container = url_parts.pop(0) if url_parts: self.prefix = '%s/' % '/'.join(url_parts) else: self.prefix = '' policy = 'PCA' policy_header = 'X-Storage-Policy' container_metadata = None try: self.conn = Connection(**self.conn_kwargs) container_metadata = self.conn.head_container(self.container) except ClientException: pass except Exception as e: log.FatalError( "Connection failed: %s %s" % (e.__class__.__name__, str(e)), log.ErrorCode.connection_failed) if container_metadata is None: log.Info("Creating container %s" % self.container) try: headers = dict([[policy_header, policy]]) self.conn.put_container(self.container, headers=headers) except Exception as e: log.FatalError( "Container creation failed: %s %s" % (e.__class__.__name__, str(e)), log.ErrorCode.connection_failed) elif policy and container_metadata[policy_header.lower()] != policy: log.FatalError( "Container '%s' exists but its storage policy is '%s' not '%s'." % (self.container, container_metadata[policy_header.lower()], policy))
os_password = os.environ.get('OS_PASSWORD') os_project_name = os.environ.get('OS_TENANT_NAME') os_domain_name = os.environ.get('OS_DOMAIN_NAME', 'Default') src_data_path = os.environ.get('SRC_DATA_PATH', '/registry') destination_path = os.environ.get('DST_DATA_PATH', 'files') swift_container_name = os.environ.get('SWIFT_CONTAINER_NAME') # create a password auth plugin auth = v2.Password(auth_url=os_auth_url, username=os_username, password=os_password, tenant_name=os_project_name) # create a Swift client Connection keystone_session = session.Session(auth=auth) swift = Connection(session=keystone_session) # create a Swift container if the target one does not exist swift.put_container(swift_container_name) # copy all the data to object storage print "Start copying local data" for root, directories, filenames in os.walk(src_data_path): for filename in filenames: abs_filename = os.path.join(root, filename) print "copying: " + abs_filename with open(abs_filename, 'r') as file: swift.put_object(swift_container_name, abs_filename.replace(src_data_path,
def __init__(self, container, prefix="", storage_options=None): self.container = container self.prefix = normalize_storage_path(prefix) self.storage_options = storage_options or {} self.conn = Connection(**self.storage_options) self._ensure_container()
def __init__(self, parsed_url): try: from swiftclient import Connection from swiftclient import ClientException except ImportError: raise BackendException("This backend requires " "the python-swiftclient library.") self.resp_exc = ClientException conn_kwargs = {} # if the user has already authenticated if 'SWIFT_PREAUTHURL' in os.environ and 'SWIFT_PREAUTHTOKEN' in os.environ: conn_kwargs['preauthurl'] = os.environ['SWIFT_PREAUTHURL'] conn_kwargs['preauthtoken'] = os.environ['SWIFT_PREAUTHTOKEN'] else: if 'SWIFT_USERNAME' not in os.environ: raise BackendException('SWIFT_USERNAME environment variable ' 'not set.') if 'SWIFT_PASSWORD' not in os.environ: raise BackendException('SWIFT_PASSWORD environment variable ' 'not set.') if 'SWIFT_AUTHURL' not in os.environ: raise BackendException('SWIFT_AUTHURL environment variable ' 'not set.') conn_kwargs['user'] = os.environ['SWIFT_USERNAME'] conn_kwargs['key'] = os.environ['SWIFT_PASSWORD'] conn_kwargs['authurl'] = os.environ['SWIFT_AUTHURL'] os_options = {} if 'SWIFT_AUTHVERSION' in os.environ: conn_kwargs['auth_version'] = os.environ['SWIFT_AUTHVERSION'] if os.environ['SWIFT_AUTHVERSION'] == '3': if 'SWIFT_USER_DOMAIN_NAME' in os.environ: os_options.update({ 'user_domain_name': os.environ['SWIFT_USER_DOMAIN_NAME'] }) if 'SWIFT_USER_DOMAIN_ID' in os.environ: os_options.update( {'user_domain_id': os.environ['SWIFT_USER_DOMAIN_ID']}) if 'SWIFT_PROJECT_DOMAIN_NAME' in os.environ: os_options.update({ 'project_domain_name': os.environ['SWIFT_PROJECT_DOMAIN_NAME'] }) if 'SWIFT_PROJECT_DOMAIN_ID' in os.environ: os_options.update({ 'project_domain_id': os.environ['SWIFT_PROJECT_DOMAIN_ID'] }) if 'SWIFT_TENANTNAME' in os.environ: os_options.update( {'tenant_name': os.environ['SWIFT_TENANTNAME']}) if 'SWIFT_ENDPOINT_TYPE' in os.environ: os_options.update( {'endpoint_type': os.environ['SWIFT_ENDPOINT_TYPE']}) else: conn_kwargs['auth_version'] = '1' if 'SWIFT_TENANTNAME' in os.environ: conn_kwargs['tenant_name'] = os.environ['SWIFT_TENANTNAME'] if 'SWIFT_REGIONNAME' in os.environ: os_options.update({'region_name': os.environ['SWIFT_REGIONNAME']}) conn_kwargs['os_options'] = os_options self.container = parsed_url.path.lstrip('/') container_metadata = None try: self.conn = Connection(**conn_kwargs) container_metadata = self.conn.head_container(self.container) except ClientException: pass except Exception as e: log.FatalError( "Connection failed: %s %s" % (e.__class__.__name__, str(e)), log.ErrorCode.connection_failed) if container_metadata is None: log.Info("Creating container %s" % self.container) try: self.conn.put_container(self.container) except Exception as e: log.FatalError( "Container creation failed: %s %s" % (e.__class__.__name__, str(e)), log.ErrorCode.connection_failed)
def __setstate__(self, state): self.__dict__.update(state) self.conn = Connection(**self.storage_options)
def __init__(self, parsed_url): duplicity.backend.Backend.__init__(self, parsed_url) try: from swiftclient.service import SwiftService from swiftclient import Connection from swiftclient import ClientException except ImportError as e: raise BackendException(u"""\ Swift backend requires the python-swiftclient library. Exception: %s""" % str(e)) self.resp_exc = ClientException conn_kwargs = {} os_options = {} svc_options = {} # if the user has already authenticated if u'SWIFT_PREAUTHURL' in os.environ and u'SWIFT_PREAUTHTOKEN' in os.environ: conn_kwargs[u'preauthurl'] = os.environ[u'SWIFT_PREAUTHURL'] conn_kwargs[u'preauthtoken'] = os.environ[u'SWIFT_PREAUTHTOKEN'] else: if u'SWIFT_USERNAME' not in os.environ: raise BackendException(u'SWIFT_USERNAME environment variable ' u'not set.') if u'SWIFT_PASSWORD' not in os.environ: raise BackendException(u'SWIFT_PASSWORD environment variable ' u'not set.') if u'SWIFT_AUTHURL' not in os.environ: raise BackendException(u'SWIFT_AUTHURL environment variable ' u'not set.') svc_options[u'os_username'] = conn_kwargs[u'user'] = os.environ[ u'SWIFT_USERNAME'] svc_options[u'os_password'] = conn_kwargs[u'key'] = os.environ[ u'SWIFT_PASSWORD'] svc_options[u'os_auth_url'] = conn_kwargs[u'authurl'] = os.environ[ u'SWIFT_AUTHURL'] if u'SWIFT_AUTHVERSION' in os.environ: svc_options[u'auth_version'] = conn_kwargs[ u'auth_version'] = os.environ[u'SWIFT_AUTHVERSION'] if os.environ[u'SWIFT_AUTHVERSION'] == u'3': if u'SWIFT_USER_DOMAIN_NAME' in os.environ: os_options.update({ u'user_domain_name': os.environ[u'SWIFT_USER_DOMAIN_NAME'] }) if u'SWIFT_USER_DOMAIN_ID' in os.environ: os_options.update({ u'user_domain_id': os.environ[u'SWIFT_USER_DOMAIN_ID'] }) if u'SWIFT_PROJECT_DOMAIN_NAME' in os.environ: os_options.update({ u'project_domain_name': os.environ[u'SWIFT_PROJECT_DOMAIN_NAME'] }) if u'SWIFT_PROJECT_DOMAIN_ID' in os.environ: os_options.update({ u'project_domain_id': os.environ[u'SWIFT_PROJECT_DOMAIN_ID'] }) if u'SWIFT_TENANTNAME' in os.environ: os_options.update( {u'tenant_name': os.environ[u'SWIFT_TENANTNAME']}) if u'SWIFT_ENDPOINT_TYPE' in os.environ: os_options.update( {u'endpoint_type': os.environ[u'SWIFT_ENDPOINT_TYPE']}) if u'SWIFT_USERID' in os.environ: os_options.update( {u'user_id': os.environ[u'SWIFT_USERID']}) if u'SWIFT_TENANTID' in os.environ: os_options.update( {u'tenant_id': os.environ[u'SWIFT_TENANTID']}) if u'SWIFT_REGIONNAME' in os.environ: os_options.update( {u'region_name': os.environ[u'SWIFT_REGIONNAME']}) else: conn_kwargs[u'auth_version'] = u'1' if u'SWIFT_TENANTNAME' in os.environ: conn_kwargs[u'tenant_name'] = os.environ[u'SWIFT_TENANTNAME'] if u'SWIFT_REGIONNAME' in os.environ: os_options.update( {u'region_name': os.environ[u'SWIFT_REGIONNAME']}) # formatting options for swiftclient.SwiftService for key in os_options.keys(): svc_options[u'os_' + key] = os_options[key] conn_kwargs[u'os_options'] = os_options # This folds the null prefix and all null parts, which means that: # //MyContainer/ and //MyContainer are equivalent. # //MyContainer//My/Prefix/ and //MyContainer/My/Prefix are equivalent. url_parts = [x for x in parsed_url.path.split(u'/') if x != u''] self.container = url_parts.pop(0) if url_parts: self.prefix = u'%s/' % u'/'.join(url_parts) else: self.prefix = u'' policy = config.swift_storage_policy policy_header = u'X-Storage-Policy' container_metadata = None try: log.Debug(u"Starting connection with arguments:'%s'" % conn_kwargs) self.conn = Connection(**conn_kwargs) container_metadata = self.conn.head_container(self.container) except ClientException as e: log.Debug(u"Connection failed: %s %s" % (e.__class__.__name__, str(e))) pass except Exception as e: log.FatalError( u"Connection failed: %s %s" % (e.__class__.__name__, str(e)), log.ErrorCode.connection_failed) if container_metadata is None: log.Info(u"Creating container %s" % self.container) try: headers = dict([[policy_header, policy]]) if policy else None self.conn.put_container(self.container, headers=headers) except Exception as e: log.FatalError( u"Container creation failed: %s %s" % (e.__class__.__name__, str(e)), log.ErrorCode.connection_failed) elif policy and container_metadata[policy_header.lower()] != policy: log.FatalError( u"Container '%s' exists but its storage policy is '%s' not '%s'." % (self.container, container_metadata[policy_header.lower()], policy)) else: log.Debug(u"Container already created: %s" % container_metadata) # checking service connection try: log.Debug(u"Starting Swiftservice: '%s'" % svc_options) self.svc = SwiftService(options=svc_options) container_stat = self.svc.stat(self.container) except ClientException as e: log.FatalError( u"Connection failed: %s %s" % (e.__class__.__name__, str(e)), log.ErrorCode.connection_failed) log.Debug(u"Container stats: %s" % container_stat)
import os env = os.environ loader = loading.get_plugin_loader('password') auth = loader.load_from_options(auth_url=env['OS_AUTH_URL'], username=env['OS_USERNAME'], password=env['OS_PASSWORD'], project_name=env['OS_PROJECT_NAME'], project_domain_name=env['OS_USER_DOMAIN_NAME'], project_id=env['OS_PROJECT_ID'], user_domain_name=env['OS_USER_DOMAIN_NAME']) sess = session.Session(auth=auth) swift = Connection(session=sess) resp_headers, containers = swift.get_account() container_name = 'fabian_data' container = swift.get_container(container_name) files = os.listdir('data') for filename in files: file_obj = os.path.join('data/', filename) with open(file_obj, 'r') as local: #print(local) swift.put_object(container_name, file_obj, contents=local, content_type="plain/text")
container = 'db_backup' backup_command = '/usr/bin/innobackupex --no-timestamp' prepare_command = '/usr/bin/innobackupex --apply-log' backup_dir = '/var/lib/mysql-backup' backup_name = "%s-%s-backup" % (time.strftime("%Y-%m-%d-%H%M-%Z-%a"), gethostname()) backup_file = backup_name + ".tar.gz" backup_file_enc = backup_file + ".enc" backup_key = "/etc/mysql-backup/.backup.key" content_type = 'application/gzip' try: # establish connection conn = Connection(os.environ['OS_AUTH_URL'], os.environ['OS_USERNAME'], os.environ['OS_PASSWORD'], tenant_name=os.environ['OS_TENANT_NAME'], auth_version="2.0") except ClientException, err: LOG.critical("No Swift Connection: %s", str(err)) def md5_for_file(filename, block_size=2**20): with open(filename, 'rb') as fh: md5 = hashlib.md5() while True: data = fh.read(block_size) if not data: break md5.update(data)
self.container = container self.conn = None retry_times = 3 if self.internal: try: conf_path = join(abspath(dirname(__file__)), 'stub.conf') self.conn = InternalClient(conf_path, 'swift_ring_sync', retry_times) except IOError, msg: raise RingSyncError('InternalClient Init Error: [%s]' % msg) except UnexpectedResponse, (msg, resp): raise RingSyncError('InternalClient Init Error: [%s]' % msg) else: try: self.conn = (lambda: Connection(authurl=self.auth_url, user=self.account, key=self.password))() except ClientException, msg: raise RingSyncError('SwiftClient Init Error: [%s]' % msg) def get_rings(self, ring_name, ring_dir=''): """ """ if self.internal: return self._get_rings_internal(ring_name, ring_dir) return self._get_rings(ring_name, ring_dir) def put_rings(self, ring_file, ring_name=None): """ """ if self.internal: return self._put_rings_internal(ring_file, ring_name) return self._put_rings(ring_file, ring_name)
chunk = infile.read(chunksize) if len(chunk) == 0: break elif len(chunk) % 16 != 0: chunk += ' ' * (16 - len(chunk) % 16) outfile.write(encryptor.encrypt(chunk)) if __name__ == '__main__': LOG.setLevel(logging.INFO) try: # establish connection conn = Connection(auth_url, username, password, tenant_name=tenant_name, auth_version="2.0") except ClientException, err: LOG.critical("No Swift Connection: %s", str(err)) ensure_container_exists(conn, container) binlogs_on_disk = get_binlogs_on_disk() binlogs_in_swift = get_binlogs_in_swift(conn, container) # print "before:" # print "binlogs_on_disk" # print binlogs_on_disk # print "binlogs_in_swift" # print binlogs_in_swift