def get_connection(self, config):
timeout = (int(config['timeout']) if config.get('timeout') else None)
if config['auth_version'] == '3':
# Keystone v3
os_options = {}
for option, value in config.items():
if option.startswith('os_options_') and value:
short_option = option[len('os_options_'):]
os_options[short_option] = config.get(option)
connection = Connection(auth_version='3',
authurl=config['authurl'],
user=config['user'],
key=config['key'],
os_options=os_options,
timeout=timeout)
elif config['auth_version'] == '1':
# Legacy auth
connection = Connection(auth_version='1',
authurl=config['authurl'],
user=config['user'],
key=config['key'],
tenant_name=config['tenant_name'],
timeout=timeout)
else:
raise NotImplementedError('auth_version? {!r}'.format(config))
return connection
def get_swift(self):
if self.swift_authver == '3':
os_options = {
'project_name': self.swift_project,
'project_domain_name': self.swift_pdomain,
'user_domain_name': self.swift_udomain,
}
connection = Connection(auth_version='3',
authurl=self.swift_auth,
user=self.swift_user,
key=self.swift_key,
os_options=os_options)
elif self.swift_authver == '1':
connection = Connection(auth_version='1',
authurl=self.swift_auth,
user=self.swift_user,
key=self.swift_key,
tenant_name='UNUSED')
else:
raise NotImplementedError('auth_version? {!r}'.format(
self.swift_authver))
return connection
def connect_swift(auth_version=None, auth_url=None, access_key=None,
user_domain_name=None, secret_key=None,
tenant_name=None, project_domain_name=None,
node_settings=None, timeout=None):
"""Helper to build an swiftclient.Connection object
"""
if node_settings is not None:
if node_settings.external_account is not None:
provider = SwiftProvider(node_settings.external_account)
auth_url, tenant_name = provider.auth_url, provider.tenant_name
auth_version = provider.auth_version
user_domain_name = provider.user_domain_name
project_domain_name = provider.project_domain_name
access_key, secret_key = provider.username, provider.password
if auth_version == '2':
connection = Connection(auth_version='2',
authurl=auth_url,
user=access_key,
key=secret_key,
tenant_name=tenant_name,
timeout=timeout)
else:
os_options = {'user_domain_name': user_domain_name,
'project_domain_name': project_domain_name,
'project_name': tenant_name}
connection = Connection(auth_version='3',
authurl=auth_url,
user=access_key,
key=secret_key,
os_options=os_options,
timeout=timeout)
return connection
def __init__(self, *args, **kwargs):
super(SwiftStorage, self).__init__()
self.__dict__.update(kwargs)
config_data = get_config_json()
# Load config variables.
self.auth_url = config_data.get('swift').get('auth_url')
self.access_key = config_data.get('swift').get('access_key')
self.secret_key = config_data.get('swift').get('secret_key')
self.auth_version = config_data.get('swift').get('auth_version')
self.tenant_name = config_data.get('swift').get('tenant_name')
self.insecure = True
self.container = config_data.get('swift').get('container')
self.name_backup = kwargs.get('name_backup', None)
self.conn = Connection(authurl=self.auth_url,
user=self.access_key,
key=self.secret_key,
auth_version=self.auth_version,
tenant_name=self.tenant_name,
insecure=self.insecure)
try:
self.conn.head_container(self.container)
except:
self.conn.put_container(self.container)
if not self.name_backup:
raise SystemExit(_error_codes.get(103))
def __init__(self):
# Read configuration from environment variables (openstack.rc)
auth_url = environ["OPENSTACK_AUTH_URL"]
user_domain_name = environ["OPENSTACK_USER_DOMAIN_NAME"]
user_name = environ["OPENSTACK_USERNAME"]
password = environ["OPENSTACK_PASSWORD"]
project_domain_name = environ["OPENSTACK_PROJECT_DOMAIN_NAME"]
project_name = environ["OPENSTACK_PROJECT_NAME"]
container_name = environ["OPENSTACK_CONTAINER_NAME"]
ca_certificate = environ["OPENSTACK_CA_CERTIFICATE"]
self.container_name = container_name
options = {
'user_domain_name': user_domain_name,
'project_domain_name': project_domain_name,
'project_name': project_name,
}
# Establish the connection with the object storage API cacert=ca_certificate,
self.conn = Connection(user=user_name,
key=password,
authurl=auth_url,
auth_version=3,
os_options=options,
cacert=ca_certificate,
timeout=120)
found = False
for container in self.conn.get_account()[1]:
cname = container['name']
if cname == container_name:
found = True
if found is not True:
# Create a new container
container_name = container_name
self.conn.put_container(container_name)
def create_sw_conn():
global swift_auth, swift_auth_token, storage_url
if swift_auth_token and storage_url:
return Connection(preauthtoken=swift_auth_token,
preauthurl=storage_url)
if swift_auth:
swift_user = os.environ.get("ST_USER")
swift_key = os.environ.get("ST_KEY")
if swift_user and swift_key:
return Connection(authurl=swift_auth,
user=swift_user,
key=swift_key)
print("Error: Swift environment not configured!")
sys.exit()
def connect_to_swift():
try:
# establish connection
conn = Connection(auth_url,
username,
password,
tenant_name=tenant_name,
auth_version="2.0")
except ClientException, err:
LOG.critical("No Swift Connection: %s", str(err))
def get_swiftclient(**kwargs):
# nmap has entries that need name changes from a 'get_service_conn_info'
# to a swift Connection name.
# pt has names that pass straight through
nmap = {'endpoint': 'preauthurl', 'token': 'preauthtoken'}
pt = ('insecure', 'cacert')
connargs = {v: kwargs.get(k) for k, v in nmap.items() if k in kwargs}
connargs.update({k: kwargs.get(k) for k in pt if k in kwargs})
if kwargs.get('session'):
sess = kwargs.get('session')
try:
# If session is available try it
return Connection(session=sess)
except TypeError:
# The edge case where session is availble but swiftclient is
# < 3.3.0. Use the old style method for Connection.
pass
return Connection(**connargs)
def create_sw_conn():
global swift_auth
swift_user = os.environ.get("ST_USER")
swift_key = os.environ.get("ST_KEY")
if swift_auth and swift_user and swift_key:
return Connection(authurl=swift_auth, user=swift_user, key=swift_key)
print("Error: Swift environment not configured!")
def create_sw_conn():
swift_auth = _default_global_options['auth']
swift_auth_token = _default_global_options['os_auth_token']
storage_url = _default_global_options['os_storage_url']
if swift_auth_token and storage_url:
return Connection(preauthtoken=swift_auth_token,
preauthurl=storage_url)
if swift_auth:
swift_user = os.environ.get("ST_USER")
swift_key = os.environ.get("ST_KEY")
if swift_user and swift_key:
return Connection(authurl=swift_auth,
user=swift_user,
key=swift_key)
print("Error: Swift environment not configured!")
sys.exit()
def create_connection(authurl, username, password, tenant_name, region):
cfconn = Connection(authurl,
username,
password,
snet=False,
tenant_name=tenant_name,
auth_version='2.0',
os_options={'region_name': region,
'endpoint_type': 'publicURL',
'service_type': 'object-store'})
return cfconn
def get_connection(self):
"""
Connect to swift storage and return the connection object.
"""
if self._conn is not None:
return self._conn
try:
self._conn = Connection(**self.conn_params)
except ClientException as e:
logger.error(str(e))
raise
return self._conn
def get_conn(options):
"""
Return a SWIFT connection from the options dict.
"""
return Connection(options['authurl'],
options['user'],
options['key'],
options['retries'],
auth_version=options['auth_version'],
os_options=options['os_options'],
snet=options['snet'],
cacert=options['os_cacert'],
insecure=options['insecure'],
ssl_compression=options['ssl_compression'])
def __init__(self, parsed_url):
try:
from swiftclient import Connection
from swiftclient import ClientException
except ImportError:
raise BackendException("This backend requires "
"the python-swiftclient library.")
self.resp_exc = ClientException
conn_kwargs = {}
# if the user has already authenticated
if os.environ.has_key('SWIFT_PREAUTHURL') and os.environ.has_key(
'SWIFT_PREAUTHTOKEN'):
conn_kwargs['preauthurl'] = os.environ['SWIFT_PREAUTHURL']
conn_kwargs['preauthtoken'] = os.environ['SWIFT_PREAUTHTOKEN']
else:
if not os.environ.has_key('SWIFT_USERNAME'):
raise BackendException('SWIFT_USERNAME environment variable '
'not set.')
if not os.environ.has_key('SWIFT_PASSWORD'):
raise BackendException('SWIFT_PASSWORD environment variable '
'not set.')
if not os.environ.has_key('SWIFT_AUTHURL'):
raise BackendException('SWIFT_AUTHURL environment variable '
'not set.')
conn_kwargs['user'] = os.environ['SWIFT_USERNAME']
conn_kwargs['key'] = os.environ['SWIFT_PASSWORD']
conn_kwargs['authurl'] = os.environ['SWIFT_AUTHURL']
if os.environ.has_key('SWIFT_AUTHVERSION'):
conn_kwargs['auth_version'] = os.environ['SWIFT_AUTHVERSION']
else:
conn_kwargs['auth_version'] = '1'
if os.environ.has_key('SWIFT_TENANTNAME'):
conn_kwargs['tenant_name'] = os.environ['SWIFT_TENANTNAME']
self.container = parsed_url.path.lstrip('/')
try:
self.conn = Connection(**conn_kwargs)
self.conn.put_container(self.container)
except Exception, e:
log.FatalError(
"Connection failed: %s %s" % (e.__class__.__name__, str(e)),
log.ErrorCode.connection_failed)
def get_connection(self):
"""
Connect to swift storage and return the connection object.
"""
if self._conn is not None:
return self._conn
for i in range(5): # 5 retries at most
try:
self._conn = Connection(**self.conn_params)
except ClientException as e:
logger.error(str(e))
if i == 4:
raise # give up
time.sleep(0.4)
else:
return self._conn
def test_delete(self, mocked_get_container_path, mocked_get_connection):
"""
Test and assert that delete_object is called with the correct arguments
"""
mocked_get_container_path.return_value = 'container/path/'
objstore = ObjectStore(config='this is the config')
connection = Connection()
connection.delete_object = Mock()
mocked_get_connection.return_value = connection
document = Document(filename='doc_file_name.pdf',
type=Document.DocumentType.Ontwerp)
objstore.delete(document)
connection.delete_object.assert_called_with('container/path/',
'doc_file_name.pdf')
def __init__(self, conf={}, profile="default"):
BakthatBackend.__init__(self, conf, profile)
from swiftclient import Connection, ClientException
self.con = Connection(self.conf["auth_url"], self.conf["access_key"],
self.conf["secret_key"],
auth_version=self.conf["auth_version"],
insecure=True)
region_name = self.conf["region_name"]
if region_name == DEFAULT_LOCATION:
region_name = ""
try:
self.con.head_container(self.conf["s3_bucket"])
except ClientException, e:
self.con.put_container(self.conf["s3_bucket"])
def test_delete_non_existing_file(self, mocked_get_container_path,
mocked_get_connection):
"""
Test and assert that delete_object is called with the correct arguments
"""
mocked_get_container_path.return_value = 'container/path/'
objstore = ObjectStore(config='this is the config')
connection = Connection()
connection.delete_object = Mock(side_effect=ClientException(''))
mocked_get_connection.return_value = connection
document = Document(id=1,
filename='doc_file_name.pdf',
type=Document.DocumentType.Ontwerp)
with self.assertLogs(level='INFO') as logs:
objstore.delete(document)
connection.delete_object.assert_called_with('container/path/',
'doc_file_name.pdf')
self.assertIn(
'INFO:storage.objectstore:Failed to delete object for document id 1',
logs.output)
def __init__(self, username, password, tenant_name, authurl):
self.username = username
self.password = password
self.tenant_name = tenant_name
self.authurl = authurl
self.os_options = {
'tenant_id': None,
'tenant_name': tenant_name,
'service_type': None,
'endpoint_type': None,
'auth_token': None,
'object_storage_url': None,
'region_name': None,
}
self.conn = Connection(
self.authurl,
self.username,
self.password,
1,
auth_version=2.0,
os_options=self.os_options)
def __init__(self, conf):
'''
根据配置文件初始化连接Swift云存储的客户端，包括认证系统和存储系统。
'''
self.conf = conf
colon = conf['storuser'].find(":")
if colon > 0:
self.user = conf['storuser'][:colon]
self.tenant = conf['storuser'][colon + 1:]
else:
self.user = self.tenant = conf['storuser']
print self.user, self.tenant
self.pawd = conf['storpass']
self.authurl = "http://" + conf['storip'] + ":" + conf[
'storport'] + "/v2.0/"
# self.keystone = client.Client(username="******", password="******",
# tenant_name="admin", auth_url=self.authurl)
self.swift = Connection(authurl=self.authurl,
user=self.user,
key=self.pawd,
auth_version="2",
tenant_name=self.tenant,
insecure=True)
def __init__(self, parsed_url):
duplicity.backend.Backend.__init__(self, parsed_url)
try:
from swiftclient import Connection
from swiftclient import ClientException
except ImportError as e:
raise BackendException("""\
PCA backend requires the python-swiftclient library.
Exception: %s""" % str(e))
self.resp_exc = ClientException
self.conn_cls = Connection
conn_kwargs = {}
# if the user has already authenticated
if 'PCA_PREAUTHURL' in os.environ and 'PCA_PREAUTHTOKEN' in os.environ:
conn_kwargs['preauthurl'] = os.environ['PCA_PREAUTHURL']
conn_kwargs['preauthtoken'] = os.environ['PCA_PREAUTHTOKEN']
else:
if 'PCA_USERNAME' not in os.environ:
raise BackendException('PCA_USERNAME environment variable '
'not set.')
if 'PCA_PASSWORD' not in os.environ:
raise BackendException('PCA_PASSWORD environment variable '
'not set.')
if 'PCA_AUTHURL' not in os.environ:
raise BackendException('PCA_AUTHURL environment variable '
'not set.')
conn_kwargs['user'] = os.environ['PCA_USERNAME']
conn_kwargs['key'] = os.environ['PCA_PASSWORD']
conn_kwargs['authurl'] = os.environ['PCA_AUTHURL']
os_options = {}
if 'PCA_AUTHVERSION' in os.environ:
conn_kwargs['auth_version'] = os.environ['PCA_AUTHVERSION']
if os.environ['PCA_AUTHVERSION'] == '3':
if 'PCA_USER_DOMAIN_NAME' in os.environ:
os_options.update({
'user_domain_name':
os.environ['PCA_USER_DOMAIN_NAME']
})
if 'PCA_USER_DOMAIN_ID' in os.environ:
os_options.update(
{'user_domain_id': os.environ['PCA_USER_DOMAIN_ID']})
if 'PCA_PROJECT_DOMAIN_NAME' in os.environ:
os_options.update({
'project_domain_name':
os.environ['PCA_PROJECT_DOMAIN_NAME']
})
if 'PCA_PROJECT_DOMAIN_ID' in os.environ:
os_options.update({
'project_domain_id':
os.environ['PCA_PROJECT_DOMAIN_ID']
})
if 'PCA_TENANTNAME' in os.environ:
os_options.update(
{'tenant_name': os.environ['PCA_TENANTNAME']})
if 'PCA_ENDPOINT_TYPE' in os.environ:
os_options.update(
{'endpoint_type': os.environ['PCA_ENDPOINT_TYPE']})
if 'PCA_USERID' in os.environ:
os_options.update({'user_id': os.environ['PCA_USERID']})
if 'PCA_TENANTID' in os.environ:
os_options.update(
{'tenant_id': os.environ['PCA_TENANTID']})
if 'PCA_REGIONNAME' in os.environ:
os_options.update(
{'region_name': os.environ['PCA_REGIONNAME']})
else:
conn_kwargs['auth_version'] = '2'
if 'PCA_TENANTNAME' in os.environ:
conn_kwargs['tenant_name'] = os.environ['PCA_TENANTNAME']
if 'PCA_REGIONNAME' in os.environ:
os_options.update({'region_name': os.environ['PCA_REGIONNAME']})
conn_kwargs['os_options'] = os_options
conn_kwargs['retries'] = 0
self.conn_kwargs = conn_kwargs
# This folds the null prefix and all null parts, which means that:
# //MyContainer/ and //MyContainer are equivalent.
# //MyContainer//My/Prefix/ and //MyContainer/My/Prefix are equivalent.
url_parts = [x for x in parsed_url.path.split('/') if x != '']
self.container = url_parts.pop(0)
if url_parts:
self.prefix = '%s/' % '/'.join(url_parts)
else:
self.prefix = ''
policy = 'PCA'
policy_header = 'X-Storage-Policy'
container_metadata = None
try:
self.conn = Connection(**self.conn_kwargs)
container_metadata = self.conn.head_container(self.container)
except ClientException:
pass
except Exception as e:
log.FatalError(
"Connection failed: %s %s" % (e.__class__.__name__, str(e)),
log.ErrorCode.connection_failed)
if container_metadata is None:
log.Info("Creating container %s" % self.container)
try:
headers = dict([[policy_header, policy]])
self.conn.put_container(self.container, headers=headers)
except Exception as e:
log.FatalError(
"Container creation failed: %s %s" %
(e.__class__.__name__, str(e)),
log.ErrorCode.connection_failed)
elif policy and container_metadata[policy_header.lower()] != policy:
log.FatalError(
"Container '%s' exists but its storage policy is '%s' not '%s'."
% (self.container, container_metadata[policy_header.lower()],
policy))
os_password = os.environ.get('OS_PASSWORD')
os_project_name = os.environ.get('OS_TENANT_NAME')
os_domain_name = os.environ.get('OS_DOMAIN_NAME', 'Default')
src_data_path = os.environ.get('SRC_DATA_PATH', '/registry')
destination_path = os.environ.get('DST_DATA_PATH', 'files')
swift_container_name = os.environ.get('SWIFT_CONTAINER_NAME')
# create a password auth plugin
auth = v2.Password(auth_url=os_auth_url,
username=os_username,
password=os_password,
tenant_name=os_project_name)
# create a Swift client Connection
keystone_session = session.Session(auth=auth)
swift = Connection(session=keystone_session)
# create a Swift container if the target one does not exist
swift.put_container(swift_container_name)
# copy all the data to object storage
print "Start copying local data"
for root, directories, filenames in os.walk(src_data_path):
for filename in filenames:
abs_filename = os.path.join(root, filename)
print "copying: " + abs_filename
with open(abs_filename, 'r') as file:
swift.put_object(swift_container_name,
abs_filename.replace(src_data_path,
def __init__(self, container, prefix="", storage_options=None):
self.container = container
self.prefix = normalize_storage_path(prefix)
self.storage_options = storage_options or {}
self.conn = Connection(**self.storage_options)
self._ensure_container()
def __init__(self, parsed_url):
try:
from swiftclient import Connection
from swiftclient import ClientException
except ImportError:
raise BackendException("This backend requires "
"the python-swiftclient library.")
self.resp_exc = ClientException
conn_kwargs = {}
# if the user has already authenticated
if 'SWIFT_PREAUTHURL' in os.environ and 'SWIFT_PREAUTHTOKEN' in os.environ:
conn_kwargs['preauthurl'] = os.environ['SWIFT_PREAUTHURL']
conn_kwargs['preauthtoken'] = os.environ['SWIFT_PREAUTHTOKEN']
else:
if 'SWIFT_USERNAME' not in os.environ:
raise BackendException('SWIFT_USERNAME environment variable '
'not set.')
if 'SWIFT_PASSWORD' not in os.environ:
raise BackendException('SWIFT_PASSWORD environment variable '
'not set.')
if 'SWIFT_AUTHURL' not in os.environ:
raise BackendException('SWIFT_AUTHURL environment variable '
'not set.')
conn_kwargs['user'] = os.environ['SWIFT_USERNAME']
conn_kwargs['key'] = os.environ['SWIFT_PASSWORD']
conn_kwargs['authurl'] = os.environ['SWIFT_AUTHURL']
os_options = {}
if 'SWIFT_AUTHVERSION' in os.environ:
conn_kwargs['auth_version'] = os.environ['SWIFT_AUTHVERSION']
if os.environ['SWIFT_AUTHVERSION'] == '3':
if 'SWIFT_USER_DOMAIN_NAME' in os.environ:
os_options.update({
'user_domain_name':
os.environ['SWIFT_USER_DOMAIN_NAME']
})
if 'SWIFT_USER_DOMAIN_ID' in os.environ:
os_options.update(
{'user_domain_id': os.environ['SWIFT_USER_DOMAIN_ID']})
if 'SWIFT_PROJECT_DOMAIN_NAME' in os.environ:
os_options.update({
'project_domain_name':
os.environ['SWIFT_PROJECT_DOMAIN_NAME']
})
if 'SWIFT_PROJECT_DOMAIN_ID' in os.environ:
os_options.update({
'project_domain_id':
os.environ['SWIFT_PROJECT_DOMAIN_ID']
})
if 'SWIFT_TENANTNAME' in os.environ:
os_options.update(
{'tenant_name': os.environ['SWIFT_TENANTNAME']})
if 'SWIFT_ENDPOINT_TYPE' in os.environ:
os_options.update(
{'endpoint_type': os.environ['SWIFT_ENDPOINT_TYPE']})
else:
conn_kwargs['auth_version'] = '1'
if 'SWIFT_TENANTNAME' in os.environ:
conn_kwargs['tenant_name'] = os.environ['SWIFT_TENANTNAME']
if 'SWIFT_REGIONNAME' in os.environ:
os_options.update({'region_name': os.environ['SWIFT_REGIONNAME']})
conn_kwargs['os_options'] = os_options
self.container = parsed_url.path.lstrip('/')
container_metadata = None
try:
self.conn = Connection(**conn_kwargs)
container_metadata = self.conn.head_container(self.container)
except ClientException:
pass
except Exception as e:
log.FatalError(
"Connection failed: %s %s" % (e.__class__.__name__, str(e)),
log.ErrorCode.connection_failed)
if container_metadata is None:
log.Info("Creating container %s" % self.container)
try:
self.conn.put_container(self.container)
except Exception as e:
log.FatalError(
"Container creation failed: %s %s" %
(e.__class__.__name__, str(e)),
log.ErrorCode.connection_failed)
def __setstate__(self, state):
self.__dict__.update(state)
self.conn = Connection(**self.storage_options)
def __init__(self, parsed_url):
duplicity.backend.Backend.__init__(self, parsed_url)
try:
from swiftclient.service import SwiftService
from swiftclient import Connection
from swiftclient import ClientException
except ImportError as e:
raise BackendException(u"""\
Swift backend requires the python-swiftclient library.
Exception: %s""" % str(e))
self.resp_exc = ClientException
conn_kwargs = {}
os_options = {}
svc_options = {}
# if the user has already authenticated
if u'SWIFT_PREAUTHURL' in os.environ and u'SWIFT_PREAUTHTOKEN' in os.environ:
conn_kwargs[u'preauthurl'] = os.environ[u'SWIFT_PREAUTHURL']
conn_kwargs[u'preauthtoken'] = os.environ[u'SWIFT_PREAUTHTOKEN']
else:
if u'SWIFT_USERNAME' not in os.environ:
raise BackendException(u'SWIFT_USERNAME environment variable '
u'not set.')
if u'SWIFT_PASSWORD' not in os.environ:
raise BackendException(u'SWIFT_PASSWORD environment variable '
u'not set.')
if u'SWIFT_AUTHURL' not in os.environ:
raise BackendException(u'SWIFT_AUTHURL environment variable '
u'not set.')
svc_options[u'os_username'] = conn_kwargs[u'user'] = os.environ[
u'SWIFT_USERNAME']
svc_options[u'os_password'] = conn_kwargs[u'key'] = os.environ[
u'SWIFT_PASSWORD']
svc_options[u'os_auth_url'] = conn_kwargs[u'authurl'] = os.environ[
u'SWIFT_AUTHURL']
if u'SWIFT_AUTHVERSION' in os.environ:
svc_options[u'auth_version'] = conn_kwargs[
u'auth_version'] = os.environ[u'SWIFT_AUTHVERSION']
if os.environ[u'SWIFT_AUTHVERSION'] == u'3':
if u'SWIFT_USER_DOMAIN_NAME' in os.environ:
os_options.update({
u'user_domain_name':
os.environ[u'SWIFT_USER_DOMAIN_NAME']
})
if u'SWIFT_USER_DOMAIN_ID' in os.environ:
os_options.update({
u'user_domain_id':
os.environ[u'SWIFT_USER_DOMAIN_ID']
})
if u'SWIFT_PROJECT_DOMAIN_NAME' in os.environ:
os_options.update({
u'project_domain_name':
os.environ[u'SWIFT_PROJECT_DOMAIN_NAME']
})
if u'SWIFT_PROJECT_DOMAIN_ID' in os.environ:
os_options.update({
u'project_domain_id':
os.environ[u'SWIFT_PROJECT_DOMAIN_ID']
})
if u'SWIFT_TENANTNAME' in os.environ:
os_options.update(
{u'tenant_name': os.environ[u'SWIFT_TENANTNAME']})
if u'SWIFT_ENDPOINT_TYPE' in os.environ:
os_options.update(
{u'endpoint_type': os.environ[u'SWIFT_ENDPOINT_TYPE']})
if u'SWIFT_USERID' in os.environ:
os_options.update(
{u'user_id': os.environ[u'SWIFT_USERID']})
if u'SWIFT_TENANTID' in os.environ:
os_options.update(
{u'tenant_id': os.environ[u'SWIFT_TENANTID']})
if u'SWIFT_REGIONNAME' in os.environ:
os_options.update(
{u'region_name': os.environ[u'SWIFT_REGIONNAME']})
else:
conn_kwargs[u'auth_version'] = u'1'
if u'SWIFT_TENANTNAME' in os.environ:
conn_kwargs[u'tenant_name'] = os.environ[u'SWIFT_TENANTNAME']
if u'SWIFT_REGIONNAME' in os.environ:
os_options.update(
{u'region_name': os.environ[u'SWIFT_REGIONNAME']})
# formatting options for swiftclient.SwiftService
for key in os_options.keys():
svc_options[u'os_' + key] = os_options[key]
conn_kwargs[u'os_options'] = os_options
# This folds the null prefix and all null parts, which means that:
# //MyContainer/ and //MyContainer are equivalent.
# //MyContainer//My/Prefix/ and //MyContainer/My/Prefix are equivalent.
url_parts = [x for x in parsed_url.path.split(u'/') if x != u'']
self.container = url_parts.pop(0)
if url_parts:
self.prefix = u'%s/' % u'/'.join(url_parts)
else:
self.prefix = u''
policy = config.swift_storage_policy
policy_header = u'X-Storage-Policy'
container_metadata = None
try:
log.Debug(u"Starting connection with arguments:'%s'" % conn_kwargs)
self.conn = Connection(**conn_kwargs)
container_metadata = self.conn.head_container(self.container)
except ClientException as e:
log.Debug(u"Connection failed: %s %s" %
(e.__class__.__name__, str(e)))
pass
except Exception as e:
log.FatalError(
u"Connection failed: %s %s" % (e.__class__.__name__, str(e)),
log.ErrorCode.connection_failed)
if container_metadata is None:
log.Info(u"Creating container %s" % self.container)
try:
headers = dict([[policy_header, policy]]) if policy else None
self.conn.put_container(self.container, headers=headers)
except Exception as e:
log.FatalError(
u"Container creation failed: %s %s" %
(e.__class__.__name__, str(e)),
log.ErrorCode.connection_failed)
elif policy and container_metadata[policy_header.lower()] != policy:
log.FatalError(
u"Container '%s' exists but its storage policy is '%s' not '%s'."
% (self.container, container_metadata[policy_header.lower()],
policy))
else:
log.Debug(u"Container already created: %s" % container_metadata)
# checking service connection
try:
log.Debug(u"Starting Swiftservice: '%s'" % svc_options)
self.svc = SwiftService(options=svc_options)
container_stat = self.svc.stat(self.container)
except ClientException as e:
log.FatalError(
u"Connection failed: %s %s" % (e.__class__.__name__, str(e)),
log.ErrorCode.connection_failed)
log.Debug(u"Container stats: %s" % container_stat)
import os
env = os.environ
loader = loading.get_plugin_loader('password')
auth = loader.load_from_options(auth_url=env['OS_AUTH_URL'],
username=env['OS_USERNAME'],
password=env['OS_PASSWORD'],
project_name=env['OS_PROJECT_NAME'],
project_domain_name=env['OS_USER_DOMAIN_NAME'],
project_id=env['OS_PROJECT_ID'],
user_domain_name=env['OS_USER_DOMAIN_NAME'])
sess = session.Session(auth=auth)
swift = Connection(session=sess)
resp_headers, containers = swift.get_account()
container_name = 'fabian_data'
container = swift.get_container(container_name)
files = os.listdir('data')
for filename in files:
file_obj = os.path.join('data/', filename)
with open(file_obj, 'r') as local:
#print(local)
swift.put_object(container_name,
file_obj,
contents=local,
content_type="plain/text")
container = 'db_backup'
backup_command = '/usr/bin/innobackupex --no-timestamp'
prepare_command = '/usr/bin/innobackupex --apply-log'
backup_dir = '/var/lib/mysql-backup'
backup_name = "%s-%s-backup" % (time.strftime("%Y-%m-%d-%H%M-%Z-%a"),
gethostname())
backup_file = backup_name + ".tar.gz"
backup_file_enc = backup_file + ".enc"
backup_key = "/etc/mysql-backup/.backup.key"
content_type = 'application/gzip'
try:
# establish connection
conn = Connection(os.environ['OS_AUTH_URL'],
os.environ['OS_USERNAME'],
os.environ['OS_PASSWORD'],
tenant_name=os.environ['OS_TENANT_NAME'],
auth_version="2.0")
except ClientException, err:
LOG.critical("No Swift Connection: %s", str(err))
def md5_for_file(filename, block_size=2**20):
with open(filename, 'rb') as fh:
md5 = hashlib.md5()
while True:
data = fh.read(block_size)
if not data:
break
md5.update(data)
self.container = container
self.conn = None
retry_times = 3
if self.internal:
try:
conf_path = join(abspath(dirname(__file__)), 'stub.conf')
self.conn = InternalClient(conf_path, 'swift_ring_sync',
retry_times)
except IOError, msg:
raise RingSyncError('InternalClient Init Error: [%s]' % msg)
except UnexpectedResponse, (msg, resp):
raise RingSyncError('InternalClient Init Error: [%s]' % msg)
else:
try:
self.conn = (lambda: Connection(authurl=self.auth_url,
user=self.account,
key=self.password))()
except ClientException, msg:
raise RingSyncError('SwiftClient Init Error: [%s]' % msg)
def get_rings(self, ring_name, ring_dir=''):
""" """
if self.internal:
return self._get_rings_internal(ring_name, ring_dir)
return self._get_rings(ring_name, ring_dir)
def put_rings(self, ring_file, ring_name=None):
""" """
if self.internal:
return self._put_rings_internal(ring_file, ring_name)
return self._put_rings(ring_file, ring_name)
chunk = infile.read(chunksize)
if len(chunk) == 0:
break
elif len(chunk) % 16 != 0:
chunk += ' ' * (16 - len(chunk) % 16)
outfile.write(encryptor.encrypt(chunk))
if __name__ == '__main__':
LOG.setLevel(logging.INFO)
try:
# establish connection
conn = Connection(auth_url,
username,
password,
tenant_name=tenant_name,
auth_version="2.0")
except ClientException, err:
LOG.critical("No Swift Connection: %s", str(err))
ensure_container_exists(conn, container)
binlogs_on_disk = get_binlogs_on_disk()
binlogs_in_swift = get_binlogs_in_swift(conn, container)
# print "before:"
# print "binlogs_on_disk"
# print binlogs_on_disk
# print "binlogs_in_swift"
# print binlogs_in_swift
