def __init__(self, *args, **kwargs):
super(SwiftStorage, self).__init__()
self.__dict__.update(kwargs)
config_data = get_config_json()
# Load config variables.
self.auth_url = config_data.get('swift').get('auth_url')
self.access_key = config_data.get('swift').get('access_key')
self.secret_key = config_data.get('swift').get('secret_key')
self.auth_version = config_data.get('swift').get('auth_version')
self.tenant_name = config_data.get('swift').get('tenant_name')
self.insecure = True
self.container = config_data.get('swift').get('container')
self.name_backup = kwargs.get('name_backup', None)
self.conn = Connection(authurl=self.auth_url,
user=self.access_key,
key=self.secret_key,
auth_version=self.auth_version,
tenant_name=self.tenant_name,
insecure=self.insecure)
try:
self.conn.head_container(self.container)
except:
self.conn.put_container(self.container)
if not self.name_backup:
raise SystemExit(_error_codes.get(103))
def get_connection(self, config):
timeout = (int(config['timeout']) if config.get('timeout') else None)
if config['auth_version'] == '3':
# Keystone v3
os_options = {}
for option, value in config.items():
if option.startswith('os_options_') and value:
short_option = option[len('os_options_'):]
os_options[short_option] = config.get(option)
connection = Connection(auth_version='3',
authurl=config['authurl'],
user=config['user'],
key=config['key'],
os_options=os_options,
timeout=timeout)
elif config['auth_version'] == '1':
# Legacy auth
connection = Connection(auth_version='1',
authurl=config['authurl'],
user=config['user'],
key=config['key'],
tenant_name=config['tenant_name'],
timeout=timeout)
else:
raise NotImplementedError('auth_version? {!r}'.format(config))
return connection
def get_swift(self):
if self.swift_authver == '3':
os_options = {
'project_name': self.swift_project,
'project_domain_name': self.swift_pdomain,
'user_domain_name': self.swift_udomain,
}
connection = Connection(auth_version='3',
authurl=self.swift_auth,
user=self.swift_user,
key=self.swift_key,
os_options=os_options)
elif self.swift_authver == '1':
connection = Connection(auth_version='1',
authurl=self.swift_auth,
user=self.swift_user,
key=self.swift_key,
tenant_name='UNUSED')
else:
raise NotImplementedError('auth_version? {!r}'.format(
self.swift_authver))
return connection
class SwiftBackend(BakthatBackend):
"""Backend to handle OpenStack Swift upload/download."""
def __init__(self, conf={}, profile="default"):
BakthatBackend.__init__(self, conf, profile)
from swiftclient import Connection, ClientException
self.con = Connection(self.conf["auth_url"],
self.conf["access_key"],
self.conf["secret_key"],
auth_version=self.conf["auth_version"],
tenant_name=self.conf["tenant_name"],
insecure=self.conf["insecure"])
region_name = self.conf["region_name"]
if region_name == DEFAULT_LOCATION:
region_name = ""
try:
self.con.head_container(self.conf["s3_bucket"])
except ClientException, e:
self.con.put_container(self.conf["s3_bucket"])
self.container = self.conf["s3_bucket"]
self.container_key = "s3_bucket"
def __init__(self):
# Read configuration from environment variables (openstack.rc)
auth_url = environ["OPENSTACK_AUTH_URL"]
user_domain_name = environ["OPENSTACK_USER_DOMAIN_NAME"]
user_name = environ["OPENSTACK_USERNAME"]
password = environ["OPENSTACK_PASSWORD"]
project_domain_name = environ["OPENSTACK_PROJECT_DOMAIN_NAME"]
project_name = environ["OPENSTACK_PROJECT_NAME"]
container_name = environ["OPENSTACK_CONTAINER_NAME"]
ca_certificate = environ["OPENSTACK_CA_CERTIFICATE"]
self.container_name = container_name
options = {
'user_domain_name': user_domain_name,
'project_domain_name': project_domain_name,
'project_name': project_name,
}
# Establish the connection with the object storage API cacert=ca_certificate,
self.conn = Connection(user=user_name,
key=password,
authurl=auth_url,
auth_version=3,
os_options=options,
cacert=ca_certificate,
timeout=120)
found = False
for container in self.conn.get_account()[1]:
cname = container['name']
if cname == container_name:
found = True
if found is not True:
# Create a new container
container_name = container_name
self.conn.put_container(container_name)
def connect_swift(auth_version=None, auth_url=None, access_key=None,
user_domain_name=None, secret_key=None,
tenant_name=None, project_domain_name=None,
node_settings=None, timeout=None):
"""Helper to build an swiftclient.Connection object
"""
if node_settings is not None:
if node_settings.external_account is not None:
provider = SwiftProvider(node_settings.external_account)
auth_url, tenant_name = provider.auth_url, provider.tenant_name
auth_version = provider.auth_version
user_domain_name = provider.user_domain_name
project_domain_name = provider.project_domain_name
access_key, secret_key = provider.username, provider.password
if auth_version == '2':
connection = Connection(auth_version='2',
authurl=auth_url,
user=access_key,
key=secret_key,
tenant_name=tenant_name,
timeout=timeout)
else:
os_options = {'user_domain_name': user_domain_name,
'project_domain_name': project_domain_name,
'project_name': tenant_name}
connection = Connection(auth_version='3',
authurl=auth_url,
user=access_key,
key=secret_key,
os_options=os_options,
timeout=timeout)
return connection
class SwiftBackend(BakthatBackend):
"""Backend to handle OpenStack Swift upload/download."""
def __init__(self, conf={}, profile="default"):
BakthatBackend.__init__(self, conf, profile)
from swiftclient import Connection, ClientException
self.con = Connection(self.conf["auth_url"],
self.conf["access_key"],
self.conf["secret_key"],
auth_version=self.conf["auth_version"],
tenant_name=self.conf["tenant_name"],
insecure=self.conf["insecure"])
region_name = self.conf["region_name"]
if region_name == DEFAULT_LOCATION:
region_name = ""
try:
self.con.head_container(self.conf["s3_bucket"])
except ClientException, e:
self.con.put_container(self.conf["s3_bucket"])
self.container = self.conf["s3_bucket"]
self.container_key = "s3_bucket"
class swift_backend(object):
'''
Swift client to store object to and retrieve object from SWIFT
'''
def __init__(self, username, password, tenant_name, authurl):
self.username = username
self.password = password
self.tenant_name = tenant_name
self.authurl = authurl
self.os_options = {
'tenant_id': None,
'tenant_name': tenant_name,
'service_type': None,
'endpoint_type': None,
'auth_token': None,
'object_storage_url': None,
'region_name': None,
}
self.conn = Connection(
self.authurl,
self.username,
self.password,
1,
auth_version=2.0,
os_options=self.os_options)
def put(self, container_name, filename, path):
#put_headers = {'x-object-meta-mtime': "%f" % getmtime(path)}
'''resps = self.conn.get_container(container_name)
for resp in resps:
print "resp is " + str(resp)'''
self.conn.put_object(
container_name, filename, open(path, 'rb'))
return
def get(self, container_name, filename, output_name):
headers, body = \
self.conn.get_object(container_name, filename, resp_chunk_size=65536)
fp = open(output_name, 'wb')
for chunk in body:
fp.write(chunk)
fp.close()
return
def get_session():
SWIFT_USER = get_reg_value('HKLM', APP_STORAGE_REG_KEY, 'account')
SWIFT_PASS = get_reg_value('HKLM', APP_STORAGE_REG_KEY, 'password')
SWIFT_KEY = get_reg_value('HKLM', APP_STORAGE_REG_KEY, 'key')
SWIFT_AUTH_URL = get_reg_value('HKLM', APP_STORAGE_REG_KEY, 'auth_url')
cnx = Connection(SWIFT_AUTH_URL, '%s:%s' % (SWIFT_USER,SWIFT_PASS), SWIFT_KEY)
cnx.get_auth()
return cnx
class SwiftStorage(Storage):
def __init__(self, config):
self._config = config
self._root_path = self._config.storage_path
self._connection = Connection(self._config.storage_auth_url, self._config.storage_account + ':' + self._config.storage_username, self._config.storage_password)
self._container = self._config.storage_container
self._index_lastupdate = None
def _update_index(self):
logger.info("_update_index")
try:
headers = self._connection.head_object(self._container, 'docker-registry.index')
lastupdatestr = headers['last-modified']
if not lastupdatestr:
return
lastupdate = datetime.strptime(lastupdatestr, "%a, %d %b %Y %H:%M:%S %Z")
logger.info('lastupdate: ' + str(lastupdate))
refresh_index = False
if not self._index_lastupdate:
refresh_index = True
elif self._index_lastupdate < lastupdate:
refresh_index = True
if not refresh_index:
return
logger.info("refresh index...")
(headers, data) = self._connection.get_object(self._container, 'docker-registry.index')
(level, temppath) = mkstemp()
tempfile = open(temppath, 'w')
tempfile.write(data)
tempfile.close()
with open(temppath, 'r') as f:
for line in f:
line = line.strip()
if line.startswith("#"):
continue
tokens = line.split('\t')
if len(tokens) < 2:
continue
logger.info("Index: " + str(tokens))
realpath = os.path.join(self._root_path, tokens[0])
if not os.path.exists(realpath):
os.makedirs(realpath)
metafile = open(os.path.join(realpath, '.object'), 'w')
metafile.write(tokens[1])
metafile.close()
os.remove(temppath)
self._index_lastupdate = lastupdate
except ClientException, inst:
if inst.http_status != 404:
raise inst
class SwiftBackend(duplicity.backend.Backend):
"""
Backend for Swift
"""
def __init__(self, parsed_url):
try:
from swiftclient import Connection
from swiftclient import ClientException
except ImportError:
raise BackendException("This backend requires "
"the python-swiftclient library.")
self.resp_exc = ClientException
conn_kwargs = {}
# if the user has already authenticated
if os.environ.has_key('SWIFT_PREAUTHURL') and os.environ.has_key(
'SWIFT_PREAUTHTOKEN'):
conn_kwargs['preauthurl'] = os.environ['SWIFT_PREAUTHURL']
conn_kwargs['preauthtoken'] = os.environ['SWIFT_PREAUTHTOKEN']
else:
if not os.environ.has_key('SWIFT_USERNAME'):
raise BackendException('SWIFT_USERNAME environment variable '
'not set.')
if not os.environ.has_key('SWIFT_PASSWORD'):
raise BackendException('SWIFT_PASSWORD environment variable '
'not set.')
if not os.environ.has_key('SWIFT_AUTHURL'):
raise BackendException('SWIFT_AUTHURL environment variable '
'not set.')
conn_kwargs['user'] = os.environ['SWIFT_USERNAME']
conn_kwargs['key'] = os.environ['SWIFT_PASSWORD']
conn_kwargs['authurl'] = os.environ['SWIFT_AUTHURL']
if os.environ.has_key('SWIFT_AUTHVERSION'):
conn_kwargs['auth_version'] = os.environ['SWIFT_AUTHVERSION']
else:
conn_kwargs['auth_version'] = '1'
if os.environ.has_key('SWIFT_TENANTNAME'):
conn_kwargs['tenant_name'] = os.environ['SWIFT_TENANTNAME']
self.container = parsed_url.path.lstrip('/')
try:
self.conn = Connection(**conn_kwargs)
self.conn.put_container(self.container)
except Exception, e:
log.FatalError(
"Connection failed: %s %s" % (e.__class__.__name__, str(e)),
log.ErrorCode.connection_failed)
class SwiftBackend(duplicity.backend.Backend):
"""
Backend for Swift
"""
def __init__(self, parsed_url):
try:
from swiftclient import Connection
from swiftclient import ClientException
except ImportError:
raise BackendException("This backend requires "
"the python-swiftclient library.")
self.resp_exc = ClientException
conn_kwargs = {}
# if the user has already authenticated
if os.environ.has_key('SWIFT_PREAUTHURL') and os.environ.has_key('SWIFT_PREAUTHTOKEN'):
conn_kwargs['preauthurl'] = os.environ['SWIFT_PREAUTHURL']
conn_kwargs['preauthtoken'] = os.environ['SWIFT_PREAUTHTOKEN']
else:
if not os.environ.has_key('SWIFT_USERNAME'):
raise BackendException('SWIFT_USERNAME environment variable '
'not set.')
if not os.environ.has_key('SWIFT_PASSWORD'):
raise BackendException('SWIFT_PASSWORD environment variable '
'not set.')
if not os.environ.has_key('SWIFT_AUTHURL'):
raise BackendException('SWIFT_AUTHURL environment variable '
'not set.')
conn_kwargs['user'] = os.environ['SWIFT_USERNAME']
conn_kwargs['key'] = os.environ['SWIFT_PASSWORD']
conn_kwargs['authurl'] = os.environ['SWIFT_AUTHURL']
if os.environ.has_key('SWIFT_AUTHVERSION'):
conn_kwargs['auth_version'] = os.environ['SWIFT_AUTHVERSION']
else:
conn_kwargs['auth_version'] = '1'
if os.environ.has_key('SWIFT_TENANTNAME'):
conn_kwargs['tenant_name'] = os.environ['SWIFT_TENANTNAME']
self.container = parsed_url.path.lstrip('/')
try:
self.conn = Connection(**conn_kwargs)
self.conn.put_container(self.container)
except Exception, e:
log.FatalError("Connection failed: %s %s"
% (e.__class__.__name__, str(e)),
log.ErrorCode.connection_failed)
class SwiftBackend(duplicity.backend.Backend):
"""
Backend for Swift
"""
def __init__(self, parsed_url):
try:
from swiftclient import Connection
from swiftclient import ClientException
except ImportError:
raise BackendException("This backend requires " "the python-swiftclient library.")
self.resp_exc = ClientException
conn_kwargs = {}
# if the user has already authenticated
if os.environ.has_key("SWIFT_PREAUTHURL") and os.environ.has_key("SWIFT_PREAUTHTOKEN"):
conn_kwargs["preauthurl"] = os.environ["SWIFT_PREAUTHURL"]
conn_kwargs["preauthtoken"] = os.environ["SWIFT_PREAUTHTOKEN"]
else:
if not os.environ.has_key("SWIFT_USERNAME"):
raise BackendException("SWIFT_USERNAME environment variable " "not set.")
if not os.environ.has_key("SWIFT_PASSWORD"):
raise BackendException("SWIFT_PASSWORD environment variable " "not set.")
if not os.environ.has_key("SWIFT_AUTHURL"):
raise BackendException("SWIFT_AUTHURL environment variable " "not set.")
conn_kwargs["user"] = os.environ["SWIFT_USERNAME"]
conn_kwargs["key"] = os.environ["SWIFT_PASSWORD"]
conn_kwargs["authurl"] = os.environ["SWIFT_AUTHURL"]
if os.environ.has_key("SWIFT_AUTHVERSION"):
conn_kwargs["auth_version"] = os.environ["SWIFT_AUTHVERSION"]
else:
conn_kwargs["auth_version"] = "1"
if os.environ.has_key("SWIFT_TENANTNAME"):
conn_kwargs["tenant_name"] = os.environ["SWIFT_TENANTNAME"]
self.container = parsed_url.path.lstrip("/")
try:
self.conn = Connection(**conn_kwargs)
self.conn.put_container(self.container)
except Exception, e:
log.FatalError("Connection failed: %s %s" % (e.__class__.__name__, str(e)), log.ErrorCode.connection_failed)
def __init__(self, *args, **kwargs):
super(SwiftStorage, self).__init__()
self.__dict__.update(kwargs)
config_data = get_config_json()
# Load config variables.
self.auth_url = config_data.get('swift').get('auth_url')
self.access_key = config_data.get('swift').get('access_key')
self.secret_key = config_data.get('swift').get('secret_key')
self.auth_version = config_data.get('swift').get('auth_version')
self.tenant_name = config_data.get('swift').get('tenant_name')
self.insecure = True
self.container = config_data.get('swift').get('container')
self.name_backup = kwargs.get('name_backup', None)
self.conn = Connection(
authurl=self.auth_url,
user=self.access_key,
key=self.secret_key,
auth_version=self.auth_version,
tenant_name=self.tenant_name,
insecure=self.insecure)
try:
self.conn.head_container(self.container)
except:
self.conn.put_container(self.container)
if not self.name_backup:
raise SystemExit(_error_codes.get(103))
def test_delete(self, mocked_get_container_path, mocked_get_connection):
"""
Test and assert that delete_object is called with the correct arguments
"""
mocked_get_container_path.return_value = 'container/path/'
objstore = ObjectStore(config='this is the config')
connection = Connection()
connection.delete_object = Mock()
mocked_get_connection.return_value = connection
document = Document(filename='doc_file_name.pdf',
type=Document.DocumentType.Ontwerp)
objstore.delete(document)
connection.delete_object.assert_called_with('container/path/',
'doc_file_name.pdf')
def __init__(self, logger, exec_time, **kwargs):
self.keystone_client = keystone_client.Client(
username=kwargs['os_username'],
password=kwargs['os_password'],
tenant_name=kwargs['os_tenant_name'],
auth_url=kwargs['os_auth_url'])
self.token = self.keystone_client.auth_ref['token']['id']
self.tenant_id = self.keystone_client.auth_ref['token']['tenant']['id']
self.end_points = self.keystone_client.service_catalog.get_endpoints()
self.region1 = str(self.end_points['hpext:cdn'][0]['region'])
self.region2 = str(self.end_points['hpext:cdn'][1]['region'])
self.url1 = str(self.end_points['hpext:cdn'][0]['versionList'])
self.url1 = self.url1[:-1]
self.url2 = str(self.end_points['hpext:cdn'][1]['versionList'])
self.url2 = self.url2[:-1]
self.END_POINTS = {
self.region1: self.url1,
self.region2: self.url2
}
self.cdn_client = Connection(
user=kwargs['os_username'],
key=kwargs['os_password'],
tenant_name=kwargs['os_tenant_name'],
authurl=kwargs['os_auth_url'],
auth_version="2.0",
os_options={'region_name': kwargs['os_region'],
'service_type': 'hpext:cdn'})
self.swift_client = Connection(
user=kwargs['os_username'],
key=kwargs['os_password'],
tenant_name=kwargs['os_tenant_name'],
authurl=kwargs['os_auth_url'],
auth_version="2.0",
os_options={'region_name': kwargs['os_region']})
self.service = 'cdn'
self.authurl = kwargs['os_auth_url']
self.logger = logger
self.exec_time = exec_time
self.zone = kwargs['os_zone']
self.failure = None
self.overall_success = True
self.region = kwargs['os_region']
self.tenant_name = kwargs['os_tenant_name']
def _upload_swift(self):
tag = str(int(time.time()))
(tenant, user) = self.cfg["s3user"].split(':')
auth_url = "http://" + self.cfg["s3host"] + "/v2.0/"
conn = Connection(auth_url, user, self.cfg["s3pass"],
snet=False, tenant_name=tenant, auth_version="2")
# Not calling conn.put_container() for the same reason of permissions.
key_name = self.cfg["prefix"] + '/' + "i" + tag
mimetype = "image/jpeg"
headers = { "Content-Type": mimetype }
fp = open(self.cfg["file"], 'rb')
conn.put_object(self.cfg["bucket"], key_name, fp, headers=headers)
fp.close()
def __init__(self, conf={}, profile="default"):
BakthatBackend.__init__(self, conf, profile)
from swiftclient import Connection, ClientException
self.con = Connection(self.conf["auth_url"], self.conf["access_key"],
self.conf["secret_key"],
auth_version=self.conf["auth_version"],
insecure=True)
region_name = self.conf["region_name"]
if region_name == DEFAULT_LOCATION:
region_name = ""
try:
self.con.head_container(self.conf["s3_bucket"])
except ClientException, e:
self.con.put_container(self.conf["s3_bucket"])
def create_sw_conn():
global swift_auth, swift_auth_token, storage_url
if swift_auth_token and storage_url:
return Connection(preauthtoken=swift_auth_token,
preauthurl=storage_url)
if swift_auth:
swift_user = os.environ.get("ST_USER")
swift_key = os.environ.get("ST_KEY")
if swift_user and swift_key:
return Connection(authurl=swift_auth,
user=swift_user,
key=swift_key)
print("Error: Swift environment not configured!")
sys.exit()
def swift_upload(container, path, auth, user, key):
conn = SwiftConnection(auth, user, key, snet=False, insecure=True)
put_headers = { 'x-object-meta-mtime': "%f" % getmtime(path) }
retry = SWIFT_RETRY_TIMES
while retry > 0:
try:
with open(path, 'rb') as fd:
conn.put_object(container, path, fd,
content_length=getsize(path), headers=put_headers)
return True
except ClientException:
log_normal(logger, {
'action': 'upload-error',
'error': 'swift client exception'
}, LOG_ERROR)
conn.put_container(container, headers={})
retry -= 1
return False
def get_swiftclient(**kwargs):
# nmap has entries that need name changes from a 'get_service_conn_info'
# to a swift Connection name.
# pt has names that pass straight through
nmap = {'endpoint': 'preauthurl', 'token': 'preauthtoken'}
pt = ('insecure', 'cacert')
connargs = {v: kwargs.get(k) for k, v in nmap.items() if k in kwargs}
connargs.update({k: kwargs.get(k) for k in pt if k in kwargs})
if kwargs.get('session'):
sess = kwargs.get('session')
try:
# If session is available try it
return Connection(session=sess)
except TypeError:
# The edge case where session is availble but swiftclient is
# < 3.3.0. Use the old style method for Connection.
pass
return Connection(**connargs)
def create_sw_conn():
global swift_auth
swift_user = os.environ.get("ST_USER")
swift_key = os.environ.get("ST_KEY")
if swift_auth and swift_user and swift_key:
return Connection(authurl=swift_auth, user=swift_user, key=swift_key)
print("Error: Swift environment not configured!")
def connect_to_swift():
try:
# establish connection
conn = Connection(auth_url,
username,
password,
tenant_name=tenant_name,
auth_version="2.0")
except ClientException, err:
LOG.critical("No Swift Connection: %s", str(err))
def create_sw_conn():
swift_auth = _default_global_options['auth']
swift_auth_token = _default_global_options['os_auth_token']
storage_url = _default_global_options['os_storage_url']
if swift_auth_token and storage_url:
return Connection(preauthtoken=swift_auth_token,
preauthurl=storage_url)
if swift_auth:
swift_user = os.environ.get("ST_USER")
swift_key = os.environ.get("ST_KEY")
if swift_user and swift_key:
return Connection(authurl=swift_auth,
user=swift_user,
key=swift_key)
print("Error: Swift environment not configured!")
sys.exit()
def create_connection(authurl, username, password, tenant_name, region):
cfconn = Connection(authurl,
username,
password,
snet=False,
tenant_name=tenant_name,
auth_version='2.0',
os_options={'region_name': region,
'endpoint_type': 'publicURL',
'service_type': 'object-store'})
return cfconn
def get_connection(self):
"""
Connect to swift storage and return the connection object.
"""
if self._conn is not None:
return self._conn
try:
self._conn = Connection(**self.conn_params)
except ClientException as e:
logger.error(str(e))
raise
return self._conn
class SwiftStorage(object):
def __init__(self, *args, **kwargs):
super(SwiftStorage, self).__init__()
self.__dict__.update(kwargs)
config_data = get_config_json()
# Load config variables.
self.auth_url = config_data.get('swift').get('auth_url')
self.access_key = config_data.get('swift').get('access_key')
self.secret_key = config_data.get('swift').get('secret_key')
self.auth_version = config_data.get('swift').get('auth_version')
self.tenant_name = config_data.get('swift').get('tenant_name')
self.insecure = True
self.container = config_data.get('swift').get('container')
self.name_backup = kwargs.get('name_backup', None)
self.conn = Connection(
authurl=self.auth_url,
user=self.access_key,
key=self.secret_key,
auth_version=self.auth_version,
tenant_name=self.tenant_name,
insecure=self.insecure)
try:
self.conn.head_container(self.container)
except:
self.conn.put_container(self.container)
if not self.name_backup:
raise SystemExit(_error_codes.get(103))
def send_file(self, filename, **kwargs):
try:
backup_file = open(filename, 'rb')
response = self.conn.put_object(
self.container, filename, backup_file)
print('Uploading file {0} to container "{1}" on swift'.
format(filename,
self.container))
except Exception, e:
raise SystemExit(_error_codes.get(115).format(
filename, 'Swift', e))
def __init__(self, auth_url, user_name, key, logger, cache_file_path,
object_store_url=None, auth_version="2",
os_options=None):
socket.setdefaulttimeout(30.0) # timeout set at socket level
Connection.__init__(self, auth_url, user_name, key, retries=0,
os_options=os_options,
auth_version=auth_version)
# needed if keystone-get-token does not work
self.object_store_url = object_store_url
self.state = dict()
for sev in severities:
for component_name in self.component_names():
self.state[component_name + sev] = \
dict(current_state=component_states.unknown,
reason='',
metrics={})
self.latency = dict()
self.metric_data = []
self.latency_reset()
self.logger = logger
self.cache_file_path = cache_file_path
class SwiftStorage(object):
def __init__(self, *args, **kwargs):
super(SwiftStorage, self).__init__()
self.__dict__.update(kwargs)
config_data = get_config_json()
# Load config variables.
self.auth_url = config_data.get('swift').get('auth_url')
self.access_key = config_data.get('swift').get('access_key')
self.secret_key = config_data.get('swift').get('secret_key')
self.auth_version = config_data.get('swift').get('auth_version')
self.tenant_name = config_data.get('swift').get('tenant_name')
self.insecure = True
self.container = config_data.get('swift').get('container')
self.name_backup = kwargs.get('name_backup', None)
self.conn = Connection(authurl=self.auth_url,
user=self.access_key,
key=self.secret_key,
auth_version=self.auth_version,
tenant_name=self.tenant_name,
insecure=self.insecure)
try:
self.conn.head_container(self.container)
except:
self.conn.put_container(self.container)
if not self.name_backup:
raise SystemExit(_error_codes.get(103))
def send_file(self, filename, **kwargs):
try:
backup_file = open(filename, 'rb')
response = self.conn.put_object(self.container, filename,
backup_file)
print('Uploading file {0} to container "{1}" on swift'.format(
filename, self.container))
except Exception, e:
raise SystemExit(
_error_codes.get(115).format(filename, 'Swift', e))
def __init__(self, username, password, tenant_name, authurl):
self.username = username
self.password = password
self.tenant_name = tenant_name
self.authurl = authurl
self.os_options = {
'tenant_id': None,
'tenant_name': tenant_name,
'service_type': None,
'endpoint_type': None,
'auth_token': None,
'object_storage_url': None,
'region_name': None,
}
self.conn = Connection(
self.authurl,
self.username,
self.password,
1,
auth_version=2.0,
os_options=self.os_options)
def get_conn(options):
"""
Return a SWIFT connection from the options dict.
"""
return Connection(options['authurl'],
options['user'],
options['key'],
options['retries'],
auth_version=options['auth_version'],
os_options=options['os_options'],
snet=options['snet'],
cacert=options['os_cacert'],
insecure=options['insecure'],
ssl_compression=options['ssl_compression'])
def test_delete_non_existing_file(self, mocked_get_container_path,
mocked_get_connection):
"""
Test and assert that delete_object is called with the correct arguments
"""
mocked_get_container_path.return_value = 'container/path/'
objstore = ObjectStore(config='this is the config')
connection = Connection()
connection.delete_object = Mock(side_effect=ClientException(''))
mocked_get_connection.return_value = connection
document = Document(id=1,
filename='doc_file_name.pdf',
type=Document.DocumentType.Ontwerp)
with self.assertLogs(level='INFO') as logs:
objstore.delete(document)
connection.delete_object.assert_called_with('container/path/',
'doc_file_name.pdf')
self.assertIn(
'INFO:storage.objectstore:Failed to delete object for document id 1',
logs.output)
def __init__(self, conf):
'''
根据配置文件初始化连接Swift云存储的客户端，包括认证系统和存储系统。
'''
self.conf = conf
colon = conf['storuser'].find(":")
if colon > 0:
self.user = conf['storuser'][:colon]
self.tenant = conf['storuser'][colon + 1:]
else:
self.user = self.tenant = conf['storuser']
print self.user, self.tenant
self.pawd = conf['storpass']
self.authurl = "http://" + conf['storip'] + ":" + conf[
'storport'] + "/v2.0/"
# self.keystone = client.Client(username="******", password="******",
# tenant_name="admin", auth_url=self.authurl)
self.swift = Connection(authurl=self.authurl,
user=self.user,
key=self.pawd,
auth_version="2",
tenant_name=self.tenant,
insecure=True)
def __init__(self,
auth_url,
user_name,
key,
logger,
cache_file_path,
object_store_url=None,
auth_version="2",
os_options=None,
latency_log_interval=LATENCY_LOG_INTERVAL):
socket.setdefaulttimeout(30.0) # timeout set at socket level
Connection.__init__(self,
auth_url,
user_name,
key,
retries=0,
os_options=os_options,
auth_version=auth_version)
# needed if keystone-get-token does not work
self.object_store_url = object_store_url
self.state = dict()
for component_name in self.component_names():
self.state[component_name] = \
dict(current_state=component_states.unknown,
reason='',
metrics={})
self.uptime = OrderedDict()
self.latency = dict()
self.metric_data = []
self.latency_reset()
self.logger = logger
self.cache_file_path = cache_file_path
self.loop_end_time = time.time()
self.latency_log_interval = latency_log_interval
self.last_latency_logged = 0 # Will trigger immediate log
def get_connection(self):
"""
Connect to swift storage and return the connection object.
"""
if self._conn is not None:
return self._conn
for i in range(5): # 5 retries at most
try:
self._conn = Connection(**self.conn_params)
except ClientException as e:
logger.error(str(e))
if i == 4:
raise # give up
time.sleep(0.4)
else:
return self._conn
def __init__(self, username, password, tenant, auth_url, region):
self.swift_client = Connection(
user=username,
key=password,
tenant_name=tenant,
authurl=auth_url,
auth_version="2.0",
os_options={'region_name': region}
)
# Setup logging
self.logger = logging.getLogger()
self.logger.setLevel(logging.WARNING)
# Allow the logger to write all output to stdout too
self.logger.addHandler(logging.StreamHandler())
def __init__(self, logger, exec_time, **kwargs):
self.swift_client = Connection(
user=kwargs['os_username'],
key=kwargs['os_password'],
tenant_name=kwargs['os_tenant_name'],
authurl=kwargs['os_auth_url'],
auth_version="2.0",
os_options={'region_name': kwargs['os_region']}
)
self.service = 'Object Storage'
self.logger = logger
self.exec_time = exec_time
self.zone = kwargs['os_zone']
self.region = kwargs['os_region']
self.failure = None
self.overall_success = True
self.tenant_name = kwargs['os_tenant_name']
def __init__(self, conf):
'''
根据配置文件初始化连接Swift云存储的客户端，包括认证系统和存储系统。
'''
self.conf = conf
colon = conf['storuser'].find(":")
if colon > 0 :
self.user = conf['storuser'][:colon]
self.tenant = conf['storuser'][colon+1:]
else:
self.user = self.tenant = conf['storuser']
print self.user, self.tenant
self.pawd = conf['storpass']
self.authurl = "http://" + conf['storip'] + ":" + conf['storport'] + "/v2.0/"
# self.keystone = client.Client(username="******", password="******",
# tenant_name="admin", auth_url=self.authurl)
self.swift = Connection(authurl=self.authurl, user=self.user, key=self.pawd,
auth_version="2", tenant_name=self.tenant, insecure = True)
def __init__(self, parsed_url):
duplicity.backend.Backend.__init__(self, parsed_url)
try:
from swiftclient import Connection
from swiftclient import ClientException
except ImportError as e:
raise BackendException("""\
PCA backend requires the python-swiftclient library.
Exception: %s""" % str(e))
self.resp_exc = ClientException
self.conn_cls = Connection
conn_kwargs = {}
# if the user has already authenticated
if 'PCA_PREAUTHURL' in os.environ and 'PCA_PREAUTHTOKEN' in os.environ:
conn_kwargs['preauthurl'] = os.environ['PCA_PREAUTHURL']
conn_kwargs['preauthtoken'] = os.environ['PCA_PREAUTHTOKEN']
else:
if 'PCA_USERNAME' not in os.environ:
raise BackendException('PCA_USERNAME environment variable '
'not set.')
if 'PCA_PASSWORD' not in os.environ:
raise BackendException('PCA_PASSWORD environment variable '
'not set.')
if 'PCA_AUTHURL' not in os.environ:
raise BackendException('PCA_AUTHURL environment variable '
'not set.')
conn_kwargs['user'] = os.environ['PCA_USERNAME']
conn_kwargs['key'] = os.environ['PCA_PASSWORD']
conn_kwargs['authurl'] = os.environ['PCA_AUTHURL']
os_options = {}
if 'PCA_AUTHVERSION' in os.environ:
conn_kwargs['auth_version'] = os.environ['PCA_AUTHVERSION']
if os.environ['PCA_AUTHVERSION'] == '3':
if 'PCA_USER_DOMAIN_NAME' in os.environ:
os_options.update({'user_domain_name': os.environ['PCA_USER_DOMAIN_NAME']})
if 'PCA_USER_DOMAIN_ID' in os.environ:
os_options.update({'user_domain_id': os.environ['PCA_USER_DOMAIN_ID']})
if 'PCA_PROJECT_DOMAIN_NAME' in os.environ:
os_options.update({'project_domain_name': os.environ['PCA_PROJECT_DOMAIN_NAME']})
if 'PCA_PROJECT_DOMAIN_ID' in os.environ:
os_options.update({'project_domain_id': os.environ['PCA_PROJECT_DOMAIN_ID']})
if 'PCA_TENANTNAME' in os.environ:
os_options.update({'tenant_name': os.environ['PCA_TENANTNAME']})
if 'PCA_ENDPOINT_TYPE' in os.environ:
os_options.update({'endpoint_type': os.environ['PCA_ENDPOINT_TYPE']})
if 'PCA_USERID' in os.environ:
os_options.update({'user_id': os.environ['PCA_USERID']})
if 'PCA_TENANTID' in os.environ:
os_options.update({'tenant_id': os.environ['PCA_TENANTID']})
if 'PCA_REGIONNAME' in os.environ:
os_options.update({'region_name': os.environ['PCA_REGIONNAME']})
else:
conn_kwargs['auth_version'] = '2'
if 'PCA_TENANTNAME' in os.environ:
conn_kwargs['tenant_name'] = os.environ['PCA_TENANTNAME']
if 'PCA_REGIONNAME' in os.environ:
os_options.update({'region_name': os.environ['PCA_REGIONNAME']})
conn_kwargs['os_options'] = os_options
conn_kwargs['retries'] = 0
self.conn_kwargs = conn_kwargs
# This folds the null prefix and all null parts, which means that:
# //MyContainer/ and //MyContainer are equivalent.
# //MyContainer//My/Prefix/ and //MyContainer/My/Prefix are equivalent.
url_parts = [x for x in parsed_url.path.split('/') if x != '']
self.container = url_parts.pop(0)
if url_parts:
self.prefix = '%s/' % '/'.join(url_parts)
else:
self.prefix = ''
policy = 'PCA'
policy_header = 'X-Storage-Policy'
container_metadata = None
try:
self.conn = Connection(**self.conn_kwargs)
container_metadata = self.conn.head_container(self.container)
except ClientException:
pass
except Exception as e:
log.FatalError("Connection failed: %s %s"
% (e.__class__.__name__, str(e)),
log.ErrorCode.connection_failed)
if container_metadata is None:
log.Info("Creating container %s" % self.container)
try:
headers = dict([[policy_header, policy]])
self.conn.put_container(self.container, headers=headers)
except Exception as e:
log.FatalError("Container creation failed: %s %s"
% (e.__class__.__name__, str(e)),
log.ErrorCode.connection_failed)
elif policy and container_metadata[policy_header.lower()] != policy:
log.FatalError("Container '%s' exists but its storage policy is '%s' not '%s'."
% (self.container, container_metadata[policy_header.lower()], policy))
def __init__(self, logger, exec_time, **kwargs):
self.neutron_client = neutron_client.Client(
username=kwargs['os_username'],
password=kwargs['os_password'],
tenant_name=kwargs['os_tenant_name'],
auth_url=kwargs['os_auth_url'],
region_name=kwargs['os_region']
)
self.nova_client = nova_client.Client(
kwargs['os_username'],
kwargs['os_password'],
kwargs['os_tenant_name'],
kwargs['os_auth_url'],
region_name=kwargs['os_region']
)
self.cinder_client = cinder_client.Client(
kwargs['os_username'],
kwargs['os_password'],
kwargs['os_tenant_name'],
kwargs['os_auth_url'],
region_name=kwargs['os_region'],
service_type="volume"
)
self.trove_client = trove_client.Client(
kwargs['os_username'],
kwargs['os_password'],
kwargs['os_tenant_name'],
kwargs['os_auth_url'],
region_name=kwargs['os_region'],
)
self.keystone_client = keystone_client.Client(
username=kwargs['os_username'],
password=kwargs['os_password'],
tenant_name=kwargs['os_tenant_name'],
auth_url=kwargs['os_auth_url']
)
try:
self.designate_client = Designate(
auth_url=kwargs['os_auth_url'],
username=kwargs['os_username'],
password=kwargs['os_password'],
tenant_name=kwargs['os_tenant_name'],
region_name=kwargs['os_region'],
service_type='hpext:dns'
)
except EndpointNotFound:
self.designate_client = None
self.cdn_client = Connection(
user=kwargs['os_username'],
key=kwargs['os_password'],
tenant_name=kwargs['os_tenant_name'],
authurl=kwargs['os_auth_url'],
auth_version="2.0",
os_options={'region_name': kwargs['os_region'],
'service_type': 'hpext:cdn'}
)
self.swift_client = Connection(
user=kwargs['os_username'],
key=kwargs['os_password'],
tenant_name=kwargs['os_tenant_name'],
authurl=kwargs['os_auth_url'],
auth_version="2.0",
os_options={'region_name': kwargs['os_region']}
)
self.region = kwargs['os_region']
self.zone = kwargs['os_zone']
self.container = None
self.cdn_object = None
self.logger = logger
self.exec_time = exec_time
self.success = True
self.overall_success = True
self.service = 'Cleanup'
self.failure = None
self.tenant_name = kwargs['os_tenant_name']
class SwiftBackend(duplicity.backend.Backend):
"""
Backend for Swift
"""
def __init__(self, parsed_url):
duplicity.backend.Backend.__init__(self, parsed_url)
try:
from swiftclient import Connection
from swiftclient import ClientException
except ImportError as e:
raise BackendException("""\
Swift backend requires the python-swiftclient library.
Exception: %s""" % str(e))
self.resp_exc = ClientException
conn_kwargs = {}
# if the user has already authenticated
if 'SWIFT_PREAUTHURL' in os.environ and 'SWIFT_PREAUTHTOKEN' in os.environ:
conn_kwargs['preauthurl'] = os.environ['SWIFT_PREAUTHURL']
conn_kwargs['preauthtoken'] = os.environ['SWIFT_PREAUTHTOKEN']
else:
if 'SWIFT_USERNAME' not in os.environ:
raise BackendException('SWIFT_USERNAME environment variable '
'not set.')
if 'SWIFT_PASSWORD' not in os.environ:
raise BackendException('SWIFT_PASSWORD environment variable '
'not set.')
if 'SWIFT_AUTHURL' not in os.environ:
raise BackendException('SWIFT_AUTHURL environment variable '
'not set.')
conn_kwargs['user'] = os.environ['SWIFT_USERNAME']
conn_kwargs['key'] = os.environ['SWIFT_PASSWORD']
conn_kwargs['authurl'] = os.environ['SWIFT_AUTHURL']
os_options = {}
if 'SWIFT_AUTHVERSION' in os.environ:
conn_kwargs['auth_version'] = os.environ['SWIFT_AUTHVERSION']
if os.environ['SWIFT_AUTHVERSION'] == '3':
if 'SWIFT_USER_DOMAIN_NAME' in os.environ:
os_options.update({
'user_domain_name':
os.environ['SWIFT_USER_DOMAIN_NAME']
})
if 'SWIFT_USER_DOMAIN_ID' in os.environ:
os_options.update(
{'user_domain_id': os.environ['SWIFT_USER_DOMAIN_ID']})
if 'SWIFT_PROJECT_DOMAIN_NAME' in os.environ:
os_options.update({
'project_domain_name':
os.environ['SWIFT_PROJECT_DOMAIN_NAME']
})
if 'SWIFT_PROJECT_DOMAIN_ID' in os.environ:
os_options.update({
'project_domain_id':
os.environ['SWIFT_PROJECT_DOMAIN_ID']
})
if 'SWIFT_TENANTNAME' in os.environ:
os_options.update(
{'tenant_name': os.environ['SWIFT_TENANTNAME']})
if 'SWIFT_ENDPOINT_TYPE' in os.environ:
os_options.update(
{'endpoint_type': os.environ['SWIFT_ENDPOINT_TYPE']})
if 'SWIFT_USERID' in os.environ:
os_options.update({'user_id': os.environ['SWIFT_USERID']})
if 'SWIFT_TENANTID' in os.environ:
os_options.update(
{'tenant_id': os.environ['SWIFT_TENANTID']})
if 'SWIFT_REGIONNAME' in os.environ:
os_options.update(
{'region_name': os.environ['SWIFT_REGIONNAME']})
else:
conn_kwargs['auth_version'] = '1'
if 'SWIFT_TENANTNAME' in os.environ:
conn_kwargs['tenant_name'] = os.environ['SWIFT_TENANTNAME']
if 'SWIFT_REGIONNAME' in os.environ:
os_options.update({'region_name': os.environ['SWIFT_REGIONNAME']})
conn_kwargs['os_options'] = os_options
# This folds the null prefix and all null parts, which means that:
# //MyContainer/ and //MyContainer are equivalent.
# //MyContainer//My/Prefix/ and //MyContainer/My/Prefix are equivalent.
url_parts = [x for x in parsed_url.path.split('/') if x != '']
self.container = url_parts.pop(0)
if url_parts:
self.prefix = '%s/' % '/'.join(url_parts)
else:
self.prefix = ''
policy = globals.swift_storage_policy
policy_header = 'X-Storage-Policy'
container_metadata = None
try:
self.conn = Connection(**conn_kwargs)
container_metadata = self.conn.head_container(self.container)
except ClientException:
pass
except Exception as e:
log.FatalError(
"Connection failed: %s %s" % (e.__class__.__name__, str(e)),
log.ErrorCode.connection_failed)
if container_metadata is None:
log.Info("Creating container %s" % self.container)
try:
headers = dict([[policy_header, policy]]) if policy else None
self.conn.put_container(self.container, headers=headers)
except Exception as e:
log.FatalError(
"Container creation failed: %s %s" %
(e.__class__.__name__, str(e)),
log.ErrorCode.connection_failed)
elif policy and container_metadata[policy_header.lower()] != policy:
log.FatalError(
"Container '%s' exists but its storage policy is '%s' not '%s'."
% (self.container, container_metadata[policy_header.lower()],
policy))
def _error_code(self, operation, e):
if isinstance(e, self.resp_exc):
if e.http_status == 404:
return log.ErrorCode.backend_not_found
def _put(self, source_path, remote_filename):
self.conn.put_object(self.container, self.prefix + remote_filename,
file(source_path.name))
def _get(self, remote_filename, local_path):
headers, body = self.conn.get_object(self.container,
self.prefix + remote_filename,
resp_chunk_size=1024)
with open(local_path.name, 'wb') as f:
for chunk in body:
f.write(chunk)
def _list(self):
headers, objs = self.conn.get_container(self.container,
full_listing=True,
path=self.prefix)
# removes prefix from return values. should check for the prefix ?
return [o['name'][len(self.prefix):] for o in objs]
def _delete(self, filename):
self.conn.delete_object(self.container, self.prefix + filename)
def _query(self, filename):
sobject = self.conn.head_object(self.container, self.prefix + filename)
return {'size': int(sobject['content-length'])}
os_password = os.environ.get('OS_PASSWORD')
os_project_name = os.environ.get('OS_TENANT_NAME')
os_domain_name = os.environ.get('OS_DOMAIN_NAME', 'Default')
src_data_path = os.environ.get('SRC_DATA_PATH', '/registry')
destination_path = os.environ.get('DST_DATA_PATH', 'files')
swift_container_name = os.environ.get('SWIFT_CONTAINER_NAME')
# create a password auth plugin
auth = v2.Password(auth_url=os_auth_url,
username=os_username,
password=os_password,
tenant_name=os_project_name)
# create a Swift client Connection
keystone_session = session.Session(auth=auth)
swift = Connection(session=keystone_session)
# create a Swift container if the target one does not exist
swift.put_container(swift_container_name)
# copy all the data to object storage
print "Start copying local data"
for root, directories, filenames in os.walk(src_data_path):
for filename in filenames:
abs_filename = os.path.join(root, filename)
print "copying: " + abs_filename
with open(abs_filename, 'r') as file:
swift.put_object(swift_container_name,
abs_filename.replace(src_data_path,
class CdnCheck:
"""
Functional test for CDN - basic workflow as follows
Create a CDN enabled container
List all the CDN containers
Select a CDN enabled container
Upload Metadata to the account
Retrieve Metadata to the account
Create and upload an object
Access the object with the CDN http url
Access the object with the CDN https url
Delete the object created in the previous step
Delete the CDN container we created in our first step
"""
def __init__(self, logger, exec_time, **kwargs):
self.keystone_client = keystone_client.Client(
username=kwargs['os_username'],
password=kwargs['os_password'],
tenant_name=kwargs['os_tenant_name'],
auth_url=kwargs['os_auth_url'])
self.token = self.keystone_client.auth_ref['token']['id']
self.tenant_id = self.keystone_client.auth_ref['token']['tenant']['id']
self.end_points = self.keystone_client.service_catalog.get_endpoints()
self.region1 = str(self.end_points['hpext:cdn'][0]['region'])
self.region2 = str(self.end_points['hpext:cdn'][1]['region'])
self.url1 = str(self.end_points['hpext:cdn'][0]['versionList'])
self.url1 = self.url1[:-1]
self.url2 = str(self.end_points['hpext:cdn'][1]['versionList'])
self.url2 = self.url2[:-1]
self.END_POINTS = {
self.region1: self.url1,
self.region2: self.url2
}
self.cdn_client = Connection(
user=kwargs['os_username'],
key=kwargs['os_password'],
tenant_name=kwargs['os_tenant_name'],
authurl=kwargs['os_auth_url'],
auth_version="2.0",
os_options={'region_name': kwargs['os_region'],
'service_type': 'hpext:cdn'})
self.swift_client = Connection(
user=kwargs['os_username'],
key=kwargs['os_password'],
tenant_name=kwargs['os_tenant_name'],
authurl=kwargs['os_auth_url'],
auth_version="2.0",
os_options={'region_name': kwargs['os_region']})
self.service = 'cdn'
self.authurl = kwargs['os_auth_url']
self.logger = logger
self.exec_time = exec_time
self.zone = kwargs['os_zone']
self.failure = None
self.overall_success = True
self.region = kwargs['os_region']
self.tenant_name = kwargs['os_tenant_name']
@monitoring.timeit
def list_containers(self):
"""
List all the CDN containers
"""
try:
self.success = True
self.cdn_containers = self.cdn_client.get_account()[1]
self.logger.warning('Listing CDN Containers')
for container in self.cdn_containers:
self.logger.warning(
container['name'] + ' ' + container['x-cdn-uri'] +
' ' + container['x-cdn-ssl-uri'])
except Exception as e:
self.success, self.overall_success = False, False
self.failure = e
self.logger.error(
"Listing CDN containers failed %s", sys.exc_info()[1])
@monitoring.timeit
def select_container(self):
"""
Select a CDN enabled container. Usually this is the container we created.
"""
try:
self.success = True
self.enabled_container = None
self.container = None
self.cdn_containers = self.cdn_client.get_account()[1]
"""
Select the container that was created in the previous step.
"""
for x in range(len(self.cdn_containers)):
if self.cdn_containers[x]['name'] == self.container_name:
self.enabled_container = x
break
if self.enabled_container is not None:
self.container = self.cdn_containers[self.enabled_container]
else:
raise cdn_exceptions.ClientException("CDN container not found")
self.logger.warning(
"The following CDN-enabled container has been selected %s",
self.container)
except cdn_exceptions.ClientException:
self.success, self.overall_success = False, False
self.failure = "Not found"
self.logger.error("CDN container not found")
except Exception as e:
self.success, self.overall_success = False, False
self.failure = e
self.logger.error(
"Selecting a CDN container failed %s", sys.exc_info()[1])
@monitoring.timeit
def upload_object(self):
"""
Create and upload a small object to the container
"""
try:
self.success = True
self.object = 'cdncheckob' + str(time.time())
contents = 'yeah this is a pretty small object'
self.swift_client.put_object(
self.container['name'], self.object, contents)
self.logger.warning("Created Object %s", self.object)
except Exception as e:
self.success, self.overall_success = False, False
self.failure = e
self.logger.error(
"Uploading object to a CDN container failed %s",
sys.exc_info()[1])
@monitoring.timeit
def check_http_url(self):
"""
Access the object we created using the CDN http url
"""
try:
self.success = True
uri = self.container['x-cdn-uri'] + '/' + self.object
self.logger.warning("HTTP URL: %s", uri)
for x in range(1, 51):
sleep(10)
http_client = httplib2.Http(
timeout=9, disable_ssl_certificate_validation=True)
try:
response, content = http_client.request(uri, "GET")
except:
continue
if response['status'] == '200':
self.logger.warning("Checking HTTP CDN URL Succeeded")
return True
self.logger.error("Checking HTTP CDN URL Timed Out")
exit(1)
except Exception as e:
self.success, self.overall_success = False, False
self.failure = e
self.logger.error(
"Checking HTTP CDN URL Failed %s", sys.exc_info()[1])
@monitoring.timeit
def check_https_url(self):
"""
Access the object we created using the CDN https url
"""
try:
self.success = True
uri = self.container['x-cdn-ssl-uri'] + self.object
self.logger.warning("HTTPS URL: %s", uri)
for x in range(1, 51):
sleep(10)
http_client = httplib2.Http(
timeout=9, disable_ssl_certificate_validation=True)
response, content = http_client.request(uri, "GET")
if response['status'] == '200':
self.logger.warning("Checking HTTPS CDN URL Succeeded")
return True
self.logger.error("Checking HTTPS CDN URL Timed Out")
exit(1)
except Exception as e:
self.success, self.overall_success = False, False
self.failure = e
self.logger.error(
"Checking HTTPS CDN URL Failed %s", sys.exc_info()[1])
def delete_object(self):
"""
Delete the object we created
"""
try:
self.success = True
self.swift_client.delete_object(
self.container['name'], self.object)
self.logger.warning("Object %s Deleted", self.object)
except Exception as e:
self.success, self.overall_success = False, False
self.failure = e
self.logger.error(
"Object Deletion Failed %s", sys.exc_info()[1])
@monitoring.timeit
def create_cdn_enabled_container(self):
"""
Create a CDN enabled container. We use Swift client to do this except
we use a CDN endpoint
"""
try:
self.success = True
self.container_name = 'cdncheck' + str(time.time())
self.cdn_client.put_container(self.container_name)
self.logger.warning(
"Created New Container %s", self.container_name)
except Exception as e:
self.success, self.overall_success = False, False
self.failure = e
self.logger.error(
"Create CDN Container Failed %s", sys.exc_info()[1])
@monitoring.timeit
def delete_container(self):
"""
Delete the container we created
"""
try:
self.success = True
self.cdn_containers[self.enabled_container]['cdn_enabled'] = False
# This would take some time to be propagated
# We will keep checking until it's disabled.
self.timer = 0
while self.cdn_containers[
self.enabled_container]['cdn_enabled'] is True:
if self.timer > 90:
break
sleep(1)
self.timer = self.timer + 1
continue
self.logger.warning("Deleting Container: %s", self.container['name'])
self.swift_client.delete_container(self.container_name)
self.cdn_client.delete_container(self.container_name)
except Exception as e:
self.success, self.overall_success = False, False
self.failure = e
self.logger.error(
"Delete CDN Container Failed %s", sys.exc_info()[1])
@monitoring.timeit
def update_metadata(self):
"""
Update the metadata of the account
"""
try:
self.success = True
headers = {"X-Account-Meta-Temp-User": "******"}
self.swift_client.post_account(headers)
self.logger.warning(
"Posting Metadata to Account succeeded: %s", headers)
except Exception as e:
self.success, self.overall_success = False, False
self.failure = e
self.logger.error(
"Updating MetaData for CDN Container Failed %s",
sys.exc_info()[1])
@monitoring.timeit
def retrieve_metadata(self):
"""
Retrieve the metadata of the account
"""
try:
self.success = True
account_info = self.swift_client.head_account()
self.logger.warning(
"Getting Account Head succeeded: %s", account_info)
except Exception as e:
self.success, self.overall_success = False, False
self.failure = e
self.logger.error(
"Retrieve MetaData from CDN Container Failed %s",
sys.exc_info()[1])
def run(self):
"""
This is a driver function that runs all the other methods in the Class
"""
self.create_cdn_enabled_container()
self.list_containers()
self.select_container()
# Uploading metadata and retrieving it
self.update_metadata()
self.retrieve_metadata()
# First create a test object. Next, test the presence of it, using
# bot a http and hppts url. Then delete the object.
self.upload_object()
self.check_http_url()
self.check_https_url()
if hasattr(self, 'object'):
self.delete_object()
self.delete_container()
if self.overall_success is True:
exit(0)
else:
exit(1)
access_token = get_access_token(
client_id=client_id,
client_secret=client_secret,
refresh_token=refresh_token,
)
credentials = get_open_stack_credentials(access_token=access_token)
except Exception, e:
lastExc = e
if lastExc is None:
break
time.sleep(5)
if lastExc is not None: raise lastExc
conn_kwargs = {}
if ('endpoint' in credentials) and ('token' in credentials):
conn_kwargs['preauthurl'] = credentials['endpoint']
conn_kwargs['preauthtoken'] = credentials['token']
else:
print "Asplod"
conn_kwargs['auth_version'] = '1'
conn = Connection(**conn_kwargs)
return conn
if len(sys.argv) < 6:
print "Wrong args. Use: client_id client_secret refresh_token out_path file"
conn = get_credentials(sys.argv[1], sys.argv[2], sys.argv[3])
conn.put_object(None, sys.argv[4] + os.path.basename(sys.argv[5]), open(sys.argv[5], "rb"))
def __init__(self, parsed_url):
duplicity.backend.Backend.__init__(self, parsed_url)
try:
from swiftclient import Connection
from swiftclient import ClientException
except ImportError as e:
raise BackendException("""\
PCA backend requires the python-swiftclient library.
Exception: %s""" % str(e))
self.resp_exc = ClientException
self.conn_cls = Connection
conn_kwargs = {}
# if the user has already authenticated
if 'PCA_PREAUTHURL' in os.environ and 'PCA_PREAUTHTOKEN' in os.environ:
conn_kwargs['preauthurl'] = os.environ['PCA_PREAUTHURL']
conn_kwargs['preauthtoken'] = os.environ['PCA_PREAUTHTOKEN']
else:
if 'PCA_USERNAME' not in os.environ:
raise BackendException('PCA_USERNAME environment variable '
'not set.')
if 'PCA_PASSWORD' not in os.environ:
raise BackendException('PCA_PASSWORD environment variable '
'not set.')
if 'PCA_AUTHURL' not in os.environ:
raise BackendException('PCA_AUTHURL environment variable '
'not set.')
conn_kwargs['user'] = os.environ['PCA_USERNAME']
conn_kwargs['key'] = os.environ['PCA_PASSWORD']
conn_kwargs['authurl'] = os.environ['PCA_AUTHURL']
os_options = {}
if 'PCA_AUTHVERSION' in os.environ:
conn_kwargs['auth_version'] = os.environ['PCA_AUTHVERSION']
if os.environ['PCA_AUTHVERSION'] == '3':
if 'PCA_USER_DOMAIN_NAME' in os.environ:
os_options.update({
'user_domain_name':
os.environ['PCA_USER_DOMAIN_NAME']
})
if 'PCA_USER_DOMAIN_ID' in os.environ:
os_options.update(
{'user_domain_id': os.environ['PCA_USER_DOMAIN_ID']})
if 'PCA_PROJECT_DOMAIN_NAME' in os.environ:
os_options.update({
'project_domain_name':
os.environ['PCA_PROJECT_DOMAIN_NAME']
})
if 'PCA_PROJECT_DOMAIN_ID' in os.environ:
os_options.update({
'project_domain_id':
os.environ['PCA_PROJECT_DOMAIN_ID']
})
if 'PCA_TENANTNAME' in os.environ:
os_options.update(
{'tenant_name': os.environ['PCA_TENANTNAME']})
if 'PCA_ENDPOINT_TYPE' in os.environ:
os_options.update(
{'endpoint_type': os.environ['PCA_ENDPOINT_TYPE']})
if 'PCA_USERID' in os.environ:
os_options.update({'user_id': os.environ['PCA_USERID']})
if 'PCA_TENANTID' in os.environ:
os_options.update(
{'tenant_id': os.environ['PCA_TENANTID']})
if 'PCA_REGIONNAME' in os.environ:
os_options.update(
{'region_name': os.environ['PCA_REGIONNAME']})
else:
conn_kwargs['auth_version'] = '2'
if 'PCA_TENANTNAME' in os.environ:
conn_kwargs['tenant_name'] = os.environ['PCA_TENANTNAME']
if 'PCA_REGIONNAME' in os.environ:
os_options.update({'region_name': os.environ['PCA_REGIONNAME']})
conn_kwargs['os_options'] = os_options
conn_kwargs['retries'] = 0
self.conn_kwargs = conn_kwargs
# This folds the null prefix and all null parts, which means that:
# //MyContainer/ and //MyContainer are equivalent.
# //MyContainer//My/Prefix/ and //MyContainer/My/Prefix are equivalent.
url_parts = [x for x in parsed_url.path.split('/') if x != '']
self.container = url_parts.pop(0)
if url_parts:
self.prefix = '%s/' % '/'.join(url_parts)
else:
self.prefix = ''
policy = 'PCA'
policy_header = 'X-Storage-Policy'
container_metadata = None
try:
self.conn = Connection(**self.conn_kwargs)
container_metadata = self.conn.head_container(self.container)
except ClientException:
pass
except Exception as e:
log.FatalError(
"Connection failed: %s %s" % (e.__class__.__name__, str(e)),
log.ErrorCode.connection_failed)
if container_metadata is None:
log.Info("Creating container %s" % self.container)
try:
headers = dict([[policy_header, policy]])
self.conn.put_container(self.container, headers=headers)
except Exception as e:
log.FatalError(
"Container creation failed: %s %s" %
(e.__class__.__name__, str(e)),
log.ErrorCode.connection_failed)
elif policy and container_metadata[policy_header.lower()] != policy:
log.FatalError(
"Container '%s' exists but its storage policy is '%s' not '%s'."
% (self.container, container_metadata[policy_header.lower()],
policy))
def __init__(self, parsed_url):
try:
from swiftclient import Connection
from swiftclient import ClientException
except ImportError:
raise BackendException("This backend requires "
"the python-swiftclient library.")
self.resp_exc = ClientException
conn_kwargs = {}
# if the user has already authenticated
if 'SWIFT_PREAUTHURL' in os.environ and 'SWIFT_PREAUTHTOKEN' in os.environ:
conn_kwargs['preauthurl'] = os.environ['SWIFT_PREAUTHURL']
conn_kwargs['preauthtoken'] = os.environ['SWIFT_PREAUTHTOKEN']
else:
if 'SWIFT_USERNAME' not in os.environ:
raise BackendException('SWIFT_USERNAME environment variable '
'not set.')
if 'SWIFT_PASSWORD' not in os.environ:
raise BackendException('SWIFT_PASSWORD environment variable '
'not set.')
if 'SWIFT_AUTHURL' not in os.environ:
raise BackendException('SWIFT_AUTHURL environment variable '
'not set.')
conn_kwargs['user'] = os.environ['SWIFT_USERNAME']
conn_kwargs['key'] = os.environ['SWIFT_PASSWORD']
conn_kwargs['authurl'] = os.environ['SWIFT_AUTHURL']
os_options = {}
if 'SWIFT_AUTHVERSION' in os.environ:
conn_kwargs['auth_version'] = os.environ['SWIFT_AUTHVERSION']
if os.environ['SWIFT_AUTHVERSION'] == '3':
if 'SWIFT_USER_DOMAIN_NAME' in os.environ:
os_options.update({'user_domain_name': os.environ['SWIFT_USER_DOMAIN_NAME']})
if 'SWIFT_USER_DOMAIN_ID' in os.environ:
os_options.update({'user_domain_id': os.environ['SWIFT_USER_DOMAIN_ID']})
if 'SWIFT_PROJECT_DOMAIN_NAME' in os.environ:
os_options.update({'project_domain_name': os.environ['SWIFT_PROJECT_DOMAIN_NAME']})
if 'SWIFT_PROJECT_DOMAIN_ID' in os.environ:
os_options.update({'project_domain_id': os.environ['SWIFT_PROJECT_DOMAIN_ID']})
if 'SWIFT_TENANTNAME' in os.environ:
os_options.update({'tenant_name': os.environ['SWIFT_TENANTNAME']})
if 'SWIFT_ENDPOINT_TYPE' in os.environ:
os_options.update({'endpoint_type': os.environ['SWIFT_ENDPOINT_TYPE']})
if 'SWIFT_USERID' in os.environ:
os_options.update({'user_id': os.environ['SWIFT_USERID']})
if 'SWIFT_TENANTID' in os.environ:
os_options.update({'tenant_id': os.environ['SWIFT_TENANTID']})
if 'SWIFT_REGIONNAME' in os.environ:
os_options.update({'region_name': os.environ['SWIFT_REGIONNAME']})
else:
conn_kwargs['auth_version'] = '1'
if 'SWIFT_TENANTNAME' in os.environ:
conn_kwargs['tenant_name'] = os.environ['SWIFT_TENANTNAME']
if 'SWIFT_REGIONNAME' in os.environ:
os_options.update({'region_name': os.environ['SWIFT_REGIONNAME']})
conn_kwargs['os_options'] = os_options
# This folds the null prefix and all null parts, which means that:
# //MyContainer/ and //MyContainer are equivalent.
# //MyContainer//My/Prefix/ and //MyContainer/My/Prefix are equivalent.
url_parts = [x for x in parsed_url.path.split('/') if x != '']
self.container = url_parts.pop(0)
if url_parts:
self.prefix = '%s/' % '/'.join(url_parts)
else:
self.prefix = ''
container_metadata = None
try:
self.conn = Connection(**conn_kwargs)
container_metadata = self.conn.head_container(self.container)
except ClientException:
pass
except Exception as e:
log.FatalError("Connection failed: %s %s"
% (e.__class__.__name__, str(e)),
log.ErrorCode.connection_failed)
if container_metadata is None:
log.Info("Creating container %s" % self.container)
try:
self.conn.put_container(self.container)
except Exception as e:
log.FatalError("Container creation failed: %s %s"
% (e.__class__.__name__, str(e)),
log.ErrorCode.connection_failed)
class SwiftBackend(duplicity.backend.Backend):
"""
Backend for Swift
"""
def __init__(self, parsed_url):
try:
from swiftclient import Connection
from swiftclient import ClientException
except ImportError:
raise BackendException("This backend requires "
"the python-swiftclient library.")
self.resp_exc = ClientException
conn_kwargs = {}
# if the user has already authenticated
if 'SWIFT_PREAUTHURL' in os.environ and 'SWIFT_PREAUTHTOKEN' in os.environ:
conn_kwargs['preauthurl'] = os.environ['SWIFT_PREAUTHURL']
conn_kwargs['preauthtoken'] = os.environ['SWIFT_PREAUTHTOKEN']
else:
if 'SWIFT_USERNAME' not in os.environ:
raise BackendException('SWIFT_USERNAME environment variable '
'not set.')
if 'SWIFT_PASSWORD' not in os.environ:
raise BackendException('SWIFT_PASSWORD environment variable '
'not set.')
if 'SWIFT_AUTHURL' not in os.environ:
raise BackendException('SWIFT_AUTHURL environment variable '
'not set.')
conn_kwargs['user'] = os.environ['SWIFT_USERNAME']
conn_kwargs['key'] = os.environ['SWIFT_PASSWORD']
conn_kwargs['authurl'] = os.environ['SWIFT_AUTHURL']
os_options = {}
if 'SWIFT_AUTHVERSION' in os.environ:
conn_kwargs['auth_version'] = os.environ['SWIFT_AUTHVERSION']
if os.environ['SWIFT_AUTHVERSION'] == '3':
if 'SWIFT_USER_DOMAIN_NAME' in os.environ:
os_options.update({'user_domain_name': os.environ['SWIFT_USER_DOMAIN_NAME']})
if 'SWIFT_USER_DOMAIN_ID' in os.environ:
os_options.update({'user_domain_id': os.environ['SWIFT_USER_DOMAIN_ID']})
if 'SWIFT_PROJECT_DOMAIN_NAME' in os.environ:
os_options.update({'project_domain_name': os.environ['SWIFT_PROJECT_DOMAIN_NAME']})
if 'SWIFT_PROJECT_DOMAIN_ID' in os.environ:
os_options.update({'project_domain_id': os.environ['SWIFT_PROJECT_DOMAIN_ID']})
if 'SWIFT_TENANTNAME' in os.environ:
os_options.update({'tenant_name': os.environ['SWIFT_TENANTNAME']})
if 'SWIFT_ENDPOINT_TYPE' in os.environ:
os_options.update({'endpoint_type': os.environ['SWIFT_ENDPOINT_TYPE']})
if 'SWIFT_USERID' in os.environ:
os_options.update({'user_id': os.environ['SWIFT_USERID']})
if 'SWIFT_TENANTID' in os.environ:
os_options.update({'tenant_id': os.environ['SWIFT_TENANTID']})
if 'SWIFT_REGIONNAME' in os.environ:
os_options.update({'region_name': os.environ['SWIFT_REGIONNAME']})
else:
conn_kwargs['auth_version'] = '1'
if 'SWIFT_TENANTNAME' in os.environ:
conn_kwargs['tenant_name'] = os.environ['SWIFT_TENANTNAME']
if 'SWIFT_REGIONNAME' in os.environ:
os_options.update({'region_name': os.environ['SWIFT_REGIONNAME']})
conn_kwargs['os_options'] = os_options
# This folds the null prefix and all null parts, which means that:
# //MyContainer/ and //MyContainer are equivalent.
# //MyContainer//My/Prefix/ and //MyContainer/My/Prefix are equivalent.
url_parts = [x for x in parsed_url.path.split('/') if x != '']
self.container = url_parts.pop(0)
if url_parts:
self.prefix = '%s/' % '/'.join(url_parts)
else:
self.prefix = ''
container_metadata = None
try:
self.conn = Connection(**conn_kwargs)
container_metadata = self.conn.head_container(self.container)
except ClientException:
pass
except Exception as e:
log.FatalError("Connection failed: %s %s"
% (e.__class__.__name__, str(e)),
log.ErrorCode.connection_failed)
if container_metadata is None:
log.Info("Creating container %s" % self.container)
try:
self.conn.put_container(self.container)
except Exception as e:
log.FatalError("Container creation failed: %s %s"
% (e.__class__.__name__, str(e)),
log.ErrorCode.connection_failed)
def _error_code(self, operation, e):
if isinstance(e, self.resp_exc):
if e.http_status == 404:
return log.ErrorCode.backend_not_found
def _put(self, source_path, remote_filename):
self.conn.put_object(self.container, self.prefix + remote_filename,
file(source_path.name))
def _get(self, remote_filename, local_path):
headers, body = self.conn.get_object(self.container, self.prefix + remote_filename)
with open(local_path.name, 'wb') as f:
for chunk in body:
f.write(chunk)
def _list(self):
headers, objs = self.conn.get_container(self.container, full_listing=True, path=self.prefix)
# removes prefix from return values. should check for the prefix ?
return [o['name'][len(self.prefix):] for o in objs]
def _delete(self, filename):
self.conn.delete_object(self.container, self.prefix + filename)
def _query(self, filename):
sobject = self.conn.head_object(self.container, self.prefix + filename)
return {'size': int(sobject['content-length'])}
class Swift:
def __init__(self, username, password, tenant, auth_url, region):
self.swift_client = Connection(
user=username,
key=password,
tenant_name=tenant,
authurl=auth_url,
auth_version="2.0",
os_options={'region_name': region}
)
# Setup logging
self.logger = logging.getLogger()
self.logger.setLevel(logging.WARNING)
# Allow the logger to write all output to stdout too
self.logger.addHandler(logging.StreamHandler())
def list_containers(self):
try:
self.containers = self.swift_client.get_account()[1]
self.logger.warning("Listing Containers")
for container in self.containers:
self.logger.warning(container['name'])
except Exception:
self.logger.error("Listing Containers Failed")
self.exit(1)
def list_objects(self):
try:
self.logger.warning("Listing Objects")
for container in self.containers:
for swift_container in self.swift_client.get_container(container['name'])[1]:
self.logger.warning(container['name'] + " " + swift_container['name'])
except Exception as e:
self.logger.error('Listing Objects Failed, Got exception: %s' % e)
self.exit(1)
def create_container(self, name):
try:
self.container = name
self.swift_client.put_container(self.container)
self.logger.warning("Created Container %s", self.container)
except Exception:
self.logger.error("Creating Container Failed")
self.exit(1)
def create_object(self, name, contents):
try:
self.object = name
self.swift_client.put_object(self.container, self.object, contents)
self.logger.warning("Created Object %s", self.object)
except Exception:
self.logger.error("Creating Object Failed")
self.exit(1)
def get_container(self):
try:
self.containers = self.swift_client.get_account()[1]
for container in self.containers:
if container['name'] == self.container:
self.logger.warning("Getting Container Succeeded")
return True
self.logger.error("Getting Container Failed")
self.exit(1)
except Exception as e:
self.logger.error('Getting Container Failed: Got exception: ' % e)
self.exit(1)
def get_object(self):
try:
object = self.swift_client.get_object(self.container, self.object)
self.logger.warning("Object Get: %s", object)
except Exception as e:
self.logger.error('Object Get Failed, Got exception: %s' % e)
self.exit(1)
def delete_object(self):
try:
self.swift_client.delete_object(self.container, self.object)
self.logger.warning("Object Deleted")
except Exception as e:
self.logger.error('Object Deletion Failed: Got exception: ' % e)
def delete_container(self):
try:
self.swift_client.delete_container(self.container)
self.logger.warning("Container Deleted")
except Exception as e:
self.logger.error('Container Deletion Failed: Got exception: ' % e)
def exit(self, value):
if hasattr(self, 'object'):
self.delete_object()
if hasattr(self, 'container'):
self.delete_container()
exit(value)
def run(self):
self.list_containers()
self.list_objects()
self.create_container()
self.create_object()
self.get_container()
self.get_object()
self.exit(0)
def __init__(self, parsed_url):
try:
from swiftclient import Connection
from swiftclient import ClientException
except ImportError:
raise BackendException("This backend requires "
"the python-swiftclient library.")
self.resp_exc = ClientException
conn_kwargs = {}
# if the user has already authenticated
if 'SWIFT_PREAUTHURL' in os.environ and 'SWIFT_PREAUTHTOKEN' in os.environ:
conn_kwargs['preauthurl'] = os.environ['SWIFT_PREAUTHURL']
conn_kwargs['preauthtoken'] = os.environ['SWIFT_PREAUTHTOKEN']
else:
if 'SWIFT_USERNAME' not in os.environ:
raise BackendException('SWIFT_USERNAME environment variable '
'not set.')
if 'SWIFT_PASSWORD' not in os.environ:
raise BackendException('SWIFT_PASSWORD environment variable '
'not set.')
if 'SWIFT_AUTHURL' not in os.environ:
raise BackendException('SWIFT_AUTHURL environment variable '
'not set.')
conn_kwargs['user'] = os.environ['SWIFT_USERNAME']
conn_kwargs['key'] = os.environ['SWIFT_PASSWORD']
conn_kwargs['authurl'] = os.environ['SWIFT_AUTHURL']
os_options = {}
if 'SWIFT_AUTHVERSION' in os.environ:
conn_kwargs['auth_version'] = os.environ['SWIFT_AUTHVERSION']
if os.environ['SWIFT_AUTHVERSION'] == '3':
if 'SWIFT_USER_DOMAIN_NAME' in os.environ:
os_options.update({'user_domain_name': os.environ['SWIFT_USER_DOMAIN_NAME']})
if 'SWIFT_USER_DOMAIN_ID' in os.environ:
os_options.update({'user_domain_id': os.environ['SWIFT_USER_DOMAIN_ID']})
if 'SWIFT_PROJECT_DOMAIN_NAME' in os.environ:
os_options.update({'project_domain_name': os.environ['SWIFT_PROJECT_DOMAIN_NAME']})
if 'SWIFT_PROJECT_DOMAIN_ID' in os.environ:
os_options.update({'project_domain_id': os.environ['SWIFT_PROJECT_DOMAIN_ID']})
if 'SWIFT_TENANTNAME' in os.environ:
os_options.update({'tenant_name': os.environ['SWIFT_TENANTNAME']})
else:
conn_kwargs['auth_version'] = '1'
if 'SWIFT_TENANTNAME' in os.environ:
conn_kwargs['tenant_name'] = os.environ['SWIFT_TENANTNAME']
if 'SWIFT_REGIONNAME' in os.environ:
os_options.update({'region_name': os.environ['SWIFT_REGIONNAME']})
conn_kwargs['os_options'] = os_options
self.container = parsed_url.path.lstrip('/')
container_metadata = None
try:
self.conn = Connection(**conn_kwargs)
container_metadata = self.conn.head_container(self.container)
except ClientException:
pass
except Exception as e:
log.FatalError("Connection failed: %s %s"
% (e.__class__.__name__, str(e)),
log.ErrorCode.connection_failed)
if container_metadata is None:
log.Info("Creating container %s" % self.container)
try:
self.conn.put_container(self.container)
except Exception as e:
log.FatalError("Container creation failed: %s %s"
% (e.__class__.__name__, str(e)),
log.ErrorCode.connection_failed)
class SwiftCheck:
"""
Functional test for Swift - basic workflow as follows:
List containers
List objects in those containers
Update Metadata of the account
Retrieve Metadata of the account
Create a Container
Select that container
Update Metadata of the container
Retrieve Metadata of the container
Create an object in the container
Retrieve the object in the container
Update metadata of the object
Retrieve metadata of the object
Delete object
Delete container.
"""
def __init__(self, logger, exec_time, **kwargs):
self.swift_client = Connection(
user=kwargs['os_username'],
key=kwargs['os_password'],
tenant_name=kwargs['os_tenant_name'],
authurl=kwargs['os_auth_url'],
auth_version="2.0",
os_options={'region_name': kwargs['os_region']}
)
self.service = 'Object Storage'
self.logger = logger
self.exec_time = exec_time
self.zone = kwargs['os_zone']
self.region = kwargs['os_region']
self.failure = None
self.overall_success = True
self.tenant_name = kwargs['os_tenant_name']
@monitoring.timeit
def list_containers(self):
"""
List all the existing containers
"""
try:
self.success = True
self.containers = self.swift_client.get_account()[1]
self.logger.warning("Listing Containers:")
for self.container in self.containers:
self.logger.warning(self.container['name'])
except Exception as e:
self.success, self.overall_success = False, False
self.failure = e
self.logger.error(
"Listing Swift containers failed %s", sys.exc_info()[1])
@monitoring.timeit
def list_objects(self):
"""
List all the objects contained in each container
"""
try:
self.success = True
self.containers = self.swift_client.get_account()[1]
self.logger.warning("Listing Objects:")
for self.container in self.containers:
try:
for swift_container in self.swift_client.get_container(
self.container['name'])[1]:
self.logger.warning(
self.container['name'] + " " +
swift_container['name'])
except:
pass
except Exception as e:
self.success, self.overall_success = False, False
self.failure = e
self.logger.error(
"Listing objects failed %s", sys.exc_info()[1])
@monitoring.timeit
def create_container(self):
"""
Create a new container
"""
try:
self.success = True
self.container = 'swiftcheck' + str(time.time())
self.swift_client.put_container(self.container)
self.logger.warning("Created Container %s", self.container)
except Exception as e:
self.success, self.overall_success = False, False
self.failure = e
self.logger.error(
"Create Container failed %s", sys.exc_info()[1])
@monitoring.timeit
def create_object(self):
"""
Create a new object in the newly created container
"""
try:
self.success = True
self.object = 'swiftcheckob' + str(time.time())
contents = 'yeah this is a pretty small object'
self.swift_client.put_object(self.container, self.object, contents)
self.logger.warning("Created Object %s", self.object)
except Exception as e:
self.success, self.overall_success = False, False
self.failure = e
self.logger.error(
"Create Object Failed %s", sys.exc_info()[1])
@monitoring.timeit
def get_container(self):
"""
Select the newly created container
"""
try:
self.success = True
self.containers = self.swift_client.get_account()[1]
self.flag = 0
for container in self.containers:
if container['name'] == self.container:
self.logger.warning(
"Getting Container Succeeded: %s", container['name'])
self.flag = self.flag + 1
if self.flag == 0:
raise cdn_exceptions.ClientException("Container not found")
except cdn_exceptions.ClientException:
self.success, self.overall_success = False, False
self.failure = "Not found"
self.logger.error("Container not found")
except Exception as e:
self.success, self.overall_success = False, False
self.failure = e
self.logger.error(
"Selecting a container failed %s", sys.exc_info()[1])
@monitoring.timeit
def get_object(self):
"""
Get the newly created object from the newly created container
"""
try:
self.success = True
object = self.swift_client.get_object(self.container, self.object)
self.logger.warning("Getting Object Succeded: %s", object)
except Exception as e:
self.success, self.overall_success = False, False
self.failure = e
self.logger.error(
"Getting object failed %s", sys.exc_info()[1])
@monitoring.timeit
def retrieve_metadata_object(self):
"""
Retrieve the metadata of the selected object
"""
try:
self.success = True
object_info = self.swift_client.head_object(
self.container, self.object)
self.logger.warning(
"Getting Object Head succeeded: %s", object_info)
except Exception as e:
self.success, self.overall_success = False, False
self.failure = e
self.logger.error(
"Retrieving Object Head failed %s", sys.exc_info()[1])
@monitoring.timeit
def retrieve_metadata_account(self):
"""
Retrieve the metadata of the account
"""
try:
self.success = True
account_info = self.swift_client.head_account()
self.logger.warning(
"Getting Account Head succeeded: %s", account_info)
except Exception as e:
self.success, self.overall_success = False, False
self.failure = e
self.logger.error(
"Retrieving Account Head failed %s", sys.exc_info()[1])
@monitoring.timeit
def retrieve_metadata_container(self):
"""
Retrieve the metadata of the container
"""
try:
self.success = True
container_info = self.swift_client.head_container(self.container)
self.logger.warning(
"Getting Container Head succeeded: %s", container_info)
except Exception as e:
self.success, self.overall_success = False, False
self.failure = e
self.logger.error(
"Getting Container Head failed: %s", sys.exc_info()[1])
@monitoring.timeit
def update_metadata_account(self):
"""
Update the metadata of the account
"""
try:
self.success = True
headers = {"X-Account-Meta-Temp-User": "******"}
self.swift_client.post_account(headers)
self.logger.warning(
"Posting Metadata to Account succeeded: %s", headers)
except Exception as e:
self.success, self.overall_success = False, False
self.failure = e
self.logger.error(
"Posting Metadata to Account failed %s", sys.exc_info()[1])
@monitoring.timeit
def update_metadata_container(self):
"""
Update the metadata of the container
"""
try:
self.success = True
headers = {"X-Container-Meta-Temp-User": "******"}
self.swift_client.post_container(self.container, headers)
self.logger.warning(
"Posting Metadata to Container succeeded: %s", headers)
except Exception as e:
self.success, self.overall_success = False, False
self.failure = e
self.logger.error(
"Posting Metadata to Container failed %s", sys.exc_info()[1])
@monitoring.timeit
def update_metadata_object(self):
"""
Update the metadata of the object
"""
try:
self.success = True
headers = {"X-Object-Meta-Temp-User": "******"}
self.swift_client.post_object(self.container, self.object, headers)
self.logger.warning(
"Posting Metadata to Object succeeded: %s", headers)
except Exception as e:
self.success, self.overall_success = False, False
self.failure = e
self.logger.error(
"Posting Metadata to Object failed %s", sys.exc_info()[1])
@monitoring.timeit
def delete_object(self):
"""
Delete the object created earlier in the process flow
"""
try:
self.swift_client.delete_object(self.container, self.object)
self.logger.warning("Object Deleted")
except Exception as e:
self.success, self.overall_success = False, False
self.failure = e
self.logger.error(
"Object Delete failed %s", sys.exc_info()[1])
@monitoring.timeit
def delete_container(self):
"""
Delete the container that was created earlier in the process flow
"""
try:
self.swift_client.delete_container(self.container)
self.logger.warning("Container %s Deleted", self.container)
except Exception as e:
self.success, self.overall_success = False, False
self.failure = e
self.logger.error(
"Container Delete Failed %s", sys.exc_info()[1])
def run(self):
"""
The driver method that runs all the steps in the process flow
"""
self.list_containers()
self.list_objects()
self.update_metadata_account()
self.retrieve_metadata_account()
self.create_container()
self.get_container()
self.update_metadata_container()
self.retrieve_metadata_container()
self.create_object()
self.get_object()
self.update_metadata_object()
self.retrieve_metadata_object()
if hasattr(self, 'object'):
self.delete_object()
if hasattr(self, 'container'):
self.delete_container()
if self.overall_success is True:
exit(0)
else:
exit(1)
def __init__(self, config):
self._config = config
self._root_path = self._config.storage_path
self._connection = Connection(self._config.storage_auth_url, self._config.storage_account + ':' + self._config.storage_username, self._config.storage_password)
self._container = self._config.storage_container
self._index_lastupdate = None
class PCABackend(duplicity.backend.Backend):
"""
Backend for OVH PCA
"""
def __init__(self, parsed_url):
duplicity.backend.Backend.__init__(self, parsed_url)
try:
from swiftclient import Connection
from swiftclient import ClientException
except ImportError as e:
raise BackendException("""\
PCA backend requires the python-swiftclient library.
Exception: %s""" % str(e))
self.resp_exc = ClientException
self.conn_cls = Connection
conn_kwargs = {}
# if the user has already authenticated
if 'PCA_PREAUTHURL' in os.environ and 'PCA_PREAUTHTOKEN' in os.environ:
conn_kwargs['preauthurl'] = os.environ['PCA_PREAUTHURL']
conn_kwargs['preauthtoken'] = os.environ['PCA_PREAUTHTOKEN']
else:
if 'PCA_USERNAME' not in os.environ:
raise BackendException('PCA_USERNAME environment variable '
'not set.')
if 'PCA_PASSWORD' not in os.environ:
raise BackendException('PCA_PASSWORD environment variable '
'not set.')
if 'PCA_AUTHURL' not in os.environ:
raise BackendException('PCA_AUTHURL environment variable '
'not set.')
conn_kwargs['user'] = os.environ['PCA_USERNAME']
conn_kwargs['key'] = os.environ['PCA_PASSWORD']
conn_kwargs['authurl'] = os.environ['PCA_AUTHURL']
os_options = {}
if 'PCA_AUTHVERSION' in os.environ:
conn_kwargs['auth_version'] = os.environ['PCA_AUTHVERSION']
if os.environ['PCA_AUTHVERSION'] == '3':
if 'PCA_USER_DOMAIN_NAME' in os.environ:
os_options.update({
'user_domain_name':
os.environ['PCA_USER_DOMAIN_NAME']
})
if 'PCA_USER_DOMAIN_ID' in os.environ:
os_options.update(
{'user_domain_id': os.environ['PCA_USER_DOMAIN_ID']})
if 'PCA_PROJECT_DOMAIN_NAME' in os.environ:
os_options.update({
'project_domain_name':
os.environ['PCA_PROJECT_DOMAIN_NAME']
})
if 'PCA_PROJECT_DOMAIN_ID' in os.environ:
os_options.update({
'project_domain_id':
os.environ['PCA_PROJECT_DOMAIN_ID']
})
if 'PCA_TENANTNAME' in os.environ:
os_options.update(
{'tenant_name': os.environ['PCA_TENANTNAME']})
if 'PCA_ENDPOINT_TYPE' in os.environ:
os_options.update(
{'endpoint_type': os.environ['PCA_ENDPOINT_TYPE']})
if 'PCA_USERID' in os.environ:
os_options.update({'user_id': os.environ['PCA_USERID']})
if 'PCA_TENANTID' in os.environ:
os_options.update(
{'tenant_id': os.environ['PCA_TENANTID']})
if 'PCA_REGIONNAME' in os.environ:
os_options.update(
{'region_name': os.environ['PCA_REGIONNAME']})
else:
conn_kwargs['auth_version'] = '2'
if 'PCA_TENANTNAME' in os.environ:
conn_kwargs['tenant_name'] = os.environ['PCA_TENANTNAME']
if 'PCA_REGIONNAME' in os.environ:
os_options.update({'region_name': os.environ['PCA_REGIONNAME']})
conn_kwargs['os_options'] = os_options
conn_kwargs['retries'] = 0
self.conn_kwargs = conn_kwargs
# This folds the null prefix and all null parts, which means that:
# //MyContainer/ and //MyContainer are equivalent.
# //MyContainer//My/Prefix/ and //MyContainer/My/Prefix are equivalent.
url_parts = [x for x in parsed_url.path.split('/') if x != '']
self.container = url_parts.pop(0)
if url_parts:
self.prefix = '%s/' % '/'.join(url_parts)
else:
self.prefix = ''
policy = 'PCA'
policy_header = 'X-Storage-Policy'
container_metadata = None
try:
self.conn = Connection(**self.conn_kwargs)
container_metadata = self.conn.head_container(self.container)
except ClientException:
pass
except Exception as e:
log.FatalError(
"Connection failed: %s %s" % (e.__class__.__name__, str(e)),
log.ErrorCode.connection_failed)
if container_metadata is None:
log.Info("Creating container %s" % self.container)
try:
headers = dict([[policy_header, policy]])
self.conn.put_container(self.container, headers=headers)
except Exception as e:
log.FatalError(
"Container creation failed: %s %s" %
(e.__class__.__name__, str(e)),
log.ErrorCode.connection_failed)
elif policy and container_metadata[policy_header.lower()] != policy:
log.FatalError(
"Container '%s' exists but its storage policy is '%s' not '%s'."
% (self.container, container_metadata[policy_header.lower()],
policy))
def _error_code(self, operation, e):
if isinstance(e, self.resp_exc):
if e.http_status == 404:
return log.ErrorCode.backend_not_found
def _put(self, source_path, remote_filename):
self.conn.put_object(self.container, self.prefix + remote_filename,
file(source_path.name))
def _get(self, remote_filename, local_path):
body = self.preprocess_download(remote_filename, 60)
if body:
with open(local_path.name, 'wb') as f:
for chunk in body:
f.write(chunk)
def _list(self):
headers, objs = self.conn.get_container(self.container,
full_listing=True,
path=self.prefix)
# removes prefix from return values. should check for the prefix ?
return [o['name'][len(self.prefix):] for o in objs]
def _delete(self, filename):
self.conn.delete_object(self.container, self.prefix + filename)
def _query(self, filename):
sobject = self.conn.head_object(self.container, self.prefix + filename)
return {'size': int(sobject['content-length'])}
def preprocess_download(self, remote_filename, retry_period, wait=True):
body = self.unseal(remote_filename)
try:
if wait:
while not body:
time.sleep(retry_period)
self.conn = self.conn_cls(**self.conn_kwargs)
body = self.unseal(remote_filename)
self.conn.close()
except Exception as e:
log.FatalError(
"Connection failed: %s %s" % (e.__class__.__name__, str(e)),
log.ErrorCode.connection_failed)
return body
def unseal(self, remote_filename):
try:
_, body = self.conn.get_object(self.container,
self.prefix + remote_filename,
resp_chunk_size=1024)
log.Info("File %s was successfully unsealed." % remote_filename)
return body
except self.resp_exc as e:
# The object is sealed but being released.
if e.http_status == 429:
# The retry-after header contains the remaining duration before
# the unsealing operation completes.
duration = int(e.http_response_headers['Retry-After'])
m, s = divmod(duration, 60)
h, m = divmod(m, 60)
eta = "%dh%02dm%02ds" % (h, m, s)
log.Info("File %s is being unsealed, operation ETA is %s." %
(remote_filename, eta))
else:
raise
def __init__(self, logger, exec_time, **kwargs):
self.neutron_client = neutron_client.Client(
username=kwargs['os_username'],
password=kwargs['os_password'],
tenant_name=kwargs['os_tenant_name'],
auth_url=kwargs['os_auth_url'],
region_name=kwargs['os_region']
)
self.nova_client = nova_client.Client(
kwargs['os_username'],
kwargs['os_password'],
kwargs['os_tenant_name'],
kwargs['os_auth_url'],
region_name=kwargs['os_region']
)
self.cinder_client = cinder_client.Client(
kwargs['os_username'],
kwargs['os_password'],
kwargs['os_tenant_name'],
kwargs['os_auth_url'],
region_name=kwargs['os_region'],
service_type="volume"
)
self.client = client.Client(
kwargs['os_username'],
kwargs['os_password'],
kwargs['os_tenant_name'],
kwargs['os_auth_url'],
region_name=kwargs['os_region'],
)
self.keystone_client = keystone_client.Client(
username=kwargs['os_username'],
password=kwargs['os_password'],
tenant_name=kwargs['os_tenant_name'],
auth_url=kwargs['os_auth_url']
)
self.designate_client = Designate(
auth_url=kwargs['os_auth_url'],
username=kwargs['os_username'],
password=kwargs['os_password'],
tenant_name=kwargs['os_tenant_name'],
region_name=kwargs['os_region'],
service_type='hpext:dns'
)
self.swift_cdn_client = Connection(
user=kwargs['os_username'],
key=kwargs['os_password'],
tenant_name=kwargs['os_tenant_name'],
authurl=kwargs['os_auth_url'],
auth_version="2.0",
os_options={'region_name': kwargs['os_region'],
'service_type': 'hpext:cdn'}
)
self.swift_client = Connection(
user=kwargs['os_username'],
key=kwargs['os_password'],
tenant_name=kwargs['os_tenant_name'],
authurl=kwargs['os_auth_url'],
auth_version="2.0",
os_options={'region_name': kwargs['os_region']}
)
#self.cdn_client = cdn.Client(
# self.token, self.tenant_id, kwargs['os_region'], self.END_POINTS
#)
self.region = str(kwargs['os_region'])
self.container = None
self.cdn_object = None
self.logger = logger
self.exec_time = exec_time
self.success = True
self.overall_success = True
self.purge_service = kwargs['os_purge_service']
self.service = 'Purge Service'
self.failure = None
self.tenant_name = kwargs['os_tenant_name']
class PCABackend(duplicity.backend.Backend):
"""
Backend for OVH PCA
"""
def __init__(self, parsed_url):
duplicity.backend.Backend.__init__(self, parsed_url)
try:
from swiftclient import Connection
from swiftclient import ClientException
except ImportError as e:
raise BackendException("""\
PCA backend requires the python-swiftclient library.
Exception: %s""" % str(e))
self.resp_exc = ClientException
self.conn_cls = Connection
conn_kwargs = {}
# if the user has already authenticated
if 'PCA_PREAUTHURL' in os.environ and 'PCA_PREAUTHTOKEN' in os.environ:
conn_kwargs['preauthurl'] = os.environ['PCA_PREAUTHURL']
conn_kwargs['preauthtoken'] = os.environ['PCA_PREAUTHTOKEN']
else:
if 'PCA_USERNAME' not in os.environ:
raise BackendException('PCA_USERNAME environment variable '
'not set.')
if 'PCA_PASSWORD' not in os.environ:
raise BackendException('PCA_PASSWORD environment variable '
'not set.')
if 'PCA_AUTHURL' not in os.environ:
raise BackendException('PCA_AUTHURL environment variable '
'not set.')
conn_kwargs['user'] = os.environ['PCA_USERNAME']
conn_kwargs['key'] = os.environ['PCA_PASSWORD']
conn_kwargs['authurl'] = os.environ['PCA_AUTHURL']
os_options = {}
if 'PCA_AUTHVERSION' in os.environ:
conn_kwargs['auth_version'] = os.environ['PCA_AUTHVERSION']
if os.environ['PCA_AUTHVERSION'] == '3':
if 'PCA_USER_DOMAIN_NAME' in os.environ:
os_options.update({'user_domain_name': os.environ['PCA_USER_DOMAIN_NAME']})
if 'PCA_USER_DOMAIN_ID' in os.environ:
os_options.update({'user_domain_id': os.environ['PCA_USER_DOMAIN_ID']})
if 'PCA_PROJECT_DOMAIN_NAME' in os.environ:
os_options.update({'project_domain_name': os.environ['PCA_PROJECT_DOMAIN_NAME']})
if 'PCA_PROJECT_DOMAIN_ID' in os.environ:
os_options.update({'project_domain_id': os.environ['PCA_PROJECT_DOMAIN_ID']})
if 'PCA_TENANTNAME' in os.environ:
os_options.update({'tenant_name': os.environ['PCA_TENANTNAME']})
if 'PCA_ENDPOINT_TYPE' in os.environ:
os_options.update({'endpoint_type': os.environ['PCA_ENDPOINT_TYPE']})
if 'PCA_USERID' in os.environ:
os_options.update({'user_id': os.environ['PCA_USERID']})
if 'PCA_TENANTID' in os.environ:
os_options.update({'tenant_id': os.environ['PCA_TENANTID']})
if 'PCA_REGIONNAME' in os.environ:
os_options.update({'region_name': os.environ['PCA_REGIONNAME']})
else:
conn_kwargs['auth_version'] = '2'
if 'PCA_TENANTNAME' in os.environ:
conn_kwargs['tenant_name'] = os.environ['PCA_TENANTNAME']
if 'PCA_REGIONNAME' in os.environ:
os_options.update({'region_name': os.environ['PCA_REGIONNAME']})
conn_kwargs['os_options'] = os_options
conn_kwargs['retries'] = 0
self.conn_kwargs = conn_kwargs
# This folds the null prefix and all null parts, which means that:
# //MyContainer/ and //MyContainer are equivalent.
# //MyContainer//My/Prefix/ and //MyContainer/My/Prefix are equivalent.
url_parts = [x for x in parsed_url.path.split('/') if x != '']
self.container = url_parts.pop(0)
if url_parts:
self.prefix = '%s/' % '/'.join(url_parts)
else:
self.prefix = ''
policy = 'PCA'
policy_header = 'X-Storage-Policy'
container_metadata = None
try:
self.conn = Connection(**self.conn_kwargs)
container_metadata = self.conn.head_container(self.container)
except ClientException:
pass
except Exception as e:
log.FatalError("Connection failed: %s %s"
% (e.__class__.__name__, str(e)),
log.ErrorCode.connection_failed)
if container_metadata is None:
log.Info("Creating container %s" % self.container)
try:
headers = dict([[policy_header, policy]])
self.conn.put_container(self.container, headers=headers)
except Exception as e:
log.FatalError("Container creation failed: %s %s"
% (e.__class__.__name__, str(e)),
log.ErrorCode.connection_failed)
elif policy and container_metadata[policy_header.lower()] != policy:
log.FatalError("Container '%s' exists but its storage policy is '%s' not '%s'."
% (self.container, container_metadata[policy_header.lower()], policy))
def _error_code(self, operation, e):
if isinstance(e, self.resp_exc):
if e.http_status == 404:
return log.ErrorCode.backend_not_found
def _put(self, source_path, remote_filename):
self.conn.put_object(self.container, self.prefix + remote_filename,
file(source_path.name))
def _get(self, remote_filename, local_path):
body = self.preprocess_download(remote_filename, 60)
if body:
with open(local_path.name, 'wb') as f:
for chunk in body:
f.write(chunk)
def _list(self):
headers, objs = self.conn.get_container(self.container, full_listing=True, path=self.prefix)
# removes prefix from return values. should check for the prefix ?
return [o['name'][len(self.prefix):] for o in objs]
def _delete(self, filename):
self.conn.delete_object(self.container, self.prefix + filename)
def _query(self, filename):
sobject = self.conn.head_object(self.container, self.prefix + filename)
return {'size': int(sobject['content-length'])}
def preprocess_download(self, remote_filename, retry_period, wait=True):
body = self.unseal(remote_filename)
try:
if wait:
while not body:
time.sleep(retry_period)
self.conn = self.conn_cls(**self.conn_kwargs)
body = self.unseal(remote_filename)
self.conn.close()
except Exception as e:
log.FatalError("Connection failed: %s %s" % (e.__class__.__name__, str(e)),
log.ErrorCode.connection_failed)
return body
def unseal(self, remote_filename):
try:
_, body = self.conn.get_object(self.container, self.prefix + remote_filename,
resp_chunk_size=1024)
log.Info("File %s was successfully unsealed." % remote_filename)
return body
except self.resp_exc as e:
# The object is sealed but being released.
if e.http_status == 429:
# The retry-after header contains the remaining duration before
# the unsealing operation completes.
duration = int(e.http_response_headers['Retry-After'])
m, s = divmod(duration, 60)
h, m = divmod(m, 60)
eta = "%dh%02dm%02ds" % (h, m, s)
log.Info("File %s is being unsealed, operation ETA is %s." %
(remote_filename, eta))
else:
raise
class PurgeCheck:
"""
Description - Purge left over elements created by each service.
Service is selected and then specific deletes for
that service are executed.
Basic workflow
if cdn
delete_cdn_containers
if cinder
delete_snapshots
delete_volume
if domains
delete_domains
if keystone
pass
if libra
delete_lb
if neutron
delete_floating
delete_port
delete_router
delete_subnet
delete_network
if nova
delete_instance
delete_floating
delete_keypair
if swift
delete_swift_containers
if trove
delete_db
"""
def __init__(self, logger, exec_time, **kwargs):
self.neutron_client = neutron_client.Client(
username=kwargs['os_username'],
password=kwargs['os_password'],
tenant_name=kwargs['os_tenant_name'],
auth_url=kwargs['os_auth_url'],
region_name=kwargs['os_region']
)
self.nova_client = nova_client.Client(
kwargs['os_username'],
kwargs['os_password'],
kwargs['os_tenant_name'],
kwargs['os_auth_url'],
region_name=kwargs['os_region']
)
self.cinder_client = cinder_client.Client(
kwargs['os_username'],
kwargs['os_password'],
kwargs['os_tenant_name'],
kwargs['os_auth_url'],
region_name=kwargs['os_region'],
service_type="volume"
)
self.client = client.Client(
kwargs['os_username'],
kwargs['os_password'],
kwargs['os_tenant_name'],
kwargs['os_auth_url'],
region_name=kwargs['os_region'],
)
self.keystone_client = keystone_client.Client(
username=kwargs['os_username'],
password=kwargs['os_password'],
tenant_name=kwargs['os_tenant_name'],
auth_url=kwargs['os_auth_url']
)
self.designate_client = Designate(
auth_url=kwargs['os_auth_url'],
username=kwargs['os_username'],
password=kwargs['os_password'],
tenant_name=kwargs['os_tenant_name'],
region_name=kwargs['os_region'],
service_type='hpext:dns'
)
self.swift_cdn_client = Connection(
user=kwargs['os_username'],
key=kwargs['os_password'],
tenant_name=kwargs['os_tenant_name'],
authurl=kwargs['os_auth_url'],
auth_version="2.0",
os_options={'region_name': kwargs['os_region'],
'service_type': 'hpext:cdn'}
)
self.swift_client = Connection(
user=kwargs['os_username'],
key=kwargs['os_password'],
tenant_name=kwargs['os_tenant_name'],
authurl=kwargs['os_auth_url'],
auth_version="2.0",
os_options={'region_name': kwargs['os_region']}
)
#self.cdn_client = cdn.Client(
# self.token, self.tenant_id, kwargs['os_region'], self.END_POINTS
#)
self.region = str(kwargs['os_region'])
self.container = None
self.cdn_object = None
self.logger = logger
self.exec_time = exec_time
self.success = True
self.overall_success = True
self.purge_service = kwargs['os_purge_service']
self.service = 'Purge Service'
self.failure = None
self.tenant_name = kwargs['os_tenant_name']
@monitoring.timeit
def delete_keypair(self):
"""
Description - Delete all keypairs where OSfunctest is part of the name
"""
try:
self.logger.warning('Delete Keypairs:')
for kp in self.nova_client.keypairs.list():
if 'OSfuncTest' in kp.name:
self.keypair = self.nova_client.keypairs.delete(kp.name)
self.success = True
except nova_exceptions.NotFound as e:
self.success, self.overall_success = False, True
self.failure = e
self.logger.error("<*>404 keypairs not found delete_keypair %s", e)
except Exception as e:
self.success, self.overall_success = False, False
self.failure = e
self.logger.error("<*>delete_keypair Failed %s", e)
@monitoring.timeit
def delete_floating(self):
"""
Description - Delete all floating IPs where floatingip['port_id'] is
none and floatingip['fixed_ip_address'] is none
"""
try:
floatingips = self.neutron_client.list_floatingips()['floatingips']
self.success = True
self.logger.warning('Delete Floating IP:')
for floatingip in floatingips:
if floatingip['port_id'] is None:
if floatingip['fixed_ip_address'] is None:
floating_ips = self.neutron_client.delete_floatingip(
floatingip['id'])
self.logger.warning(
'Delete ' + floatingip['id'])
except Exception as e:
self.logger.error("<*>delete_floating Failed %s", e)
self.success, self.overall_success = False, False
self.failure = e
@monitoring.timeit
def delete_network(self):
"""
Description - Delete all networks where network['name'] is new_network
or contains 'neutroncheck'
"""
try:
networks = self.neutron_client.list_networks()['networks']
self.success = True
self.logger.warning('Deleting Networks:')
for network in networks:
try:
if network['name'] == 'new_network' or \
'neutroncheck' in network['name']:
self.neutron_client.delete_network(network['id'])
self.logger.warning(
'Delete ' + network['id'] + " " + network['name'])
else:
self.logger.warning("Did not delete %s", network['id'])
except Exception as e:
self.logger.warning("Did not delete %s", network['id'])
except Exception as e:
self.logger.error("<*>delete_network Failed %s", e)
self.success, self.overall_success = False, False
self.failure = e
@monitoring.timeit
def delete_port(self):
"""
Description - Delete all ports where port['name'] contains
'neutroncheck'
"""
try:
ports = self.neutron_client.list_ports()['ports']
self.logger.warning('Deleting Ports:')
self.success = True
for port in ports:
try:
if 'neutroncheck' in port['name']:
self.neutron_client.delete_port(port['id'])
self.logger.warning(
'Deleting ' + port['id'] + " " + port['name'])
else:
self.logger.warning("Did not delete " + port['id'])
except:
self.logger.warning("Did not delete %s", port['id'])
except Exception as e:
self.logger.error("<*>delete_port Failed %s", e)
self.success, self.overall_success = False, False
self.failure = e
@monitoring.timeit
def delete_router(self):
"""
Description - Delete all routers where router['name'] contains
'neutroncheck'
"""
try:
routers = self.neutron_client.list_routers()['routers']
self.success = True
self.logger.warning('Deleting Routers:')
for router in routers:
try:
if 'neutroncheck' in router['name']:
self.neutron_client.delete_router(router['id'])
self.logger.warning(
'Deleting ' + router['id'] + " " + router['name'])
else:
self.logger.warning("Did not delete %s", router['id'])
except:
self.logger.warning("Did not delete %s", router['id'])
except Exception as e:
self.logger.error("<*>delete_router Failed %s", e)
self.success, self.overall_success = False, False
self.failure = e
@monitoring.timeit
def delete_subnet(self):
"""
Description - Delete all subnets where subnet['name'] contains
'neutroncheck'
"""
try:
subnets = self.neutron_client.list_subnets()['subnets']
self.success = True
self.logger.warning('Deleting Subnets:')
for subnet in subnets:
try:
if 'neutroncheck' in subnet['name']:
self.neutron_client.delete_subnet(subnet['id'])
self.logger.warning(
'Delete ' + subnet['id'] + " " + subnet['name'])
else:
self.logger.warning("Did not delete %s", subnet['id'])
except:
self.logger.warning("Did not delete %s", subnet['id'])
except Exception as e:
self.logger.error("<*>list_subnet Failed %s", e)
self.success, self.overall_success = False, False
self.failure = e
@monitoring.timeit
def delete_volume(self):
"""
Description - Delete all volumes where volume.display_name contains
'cinder'
"""
volumes = self.cinder_client.volumes.list()
try:
self.logger.warning('Deleting Volumes:')
for volume in volumes:
self.success = True
if 'cinder' in volume.display_name:
if self.cinder_client.volumes.get(volume.id).status\
== 'available':
try:
volume.delete()
self.logger.warning('Deleting unused volume: %s' %
volume.id)
except Exception as e:
self.logger.error('Unable to delete volume: {}'
.format(e))
else:
self.logger.warning('Did not delete ' + volume.id)
except cinder_exceptions.NotFound:
self.logger.error("No Volumes found to delete")
self.success, self.overall_success = False, True
self.failure = "Not Found"
except Exception as e:
self.logger.warning(
'Caught exception trying to delete old volumes: %s' % e)
self.success, self.overall_success = False, False
self.failure = e
@monitoring.timeit
def delete_snapshots(self):
"""
Description - Delete all snapshots
"""
try:
snapshots = self.cinder_client.volume_snapshots.list()
self.success = True
self.logger.warning("deleting snapshots")
for snapshot in snapshots:
try:
snapshot.delete()
self.logger.warning("Deleting snapshot %s", snapshot.id)
except Exception as e:
self.logger.error("Delete snapshot failed %s", e)
except cinder_exceptions.NotFound:
self.logger.error("No Snapshots found")
self.success, self.overall_success = False, True
self.failure = "Not found"
except Exception as e:
self.logger.error("Deleting Snapshots Failed")
self.success, self.overall_success = False, False
self.failure = e
@monitoring.timeit
def delete_instance(self):
"""
Description - Delete all instances where instance.name contains
'novacheck'
"""
try:
instances = self.nova_client.servers.list()
self.success = True
self.logger.warning("Deleting instances")
for instance in instances:
if 'novacheck' in instance.name:
instance.delete()
self.logger.warning("delete %s", instance.id)
else:
self.logger.warning("did not delete " + instance.id)
except Exception as e:
self.success, self.overall_success = False, False
self.failure = e
self.logger.error("<*>delete_instance %s", e)
@monitoring.timeit
def delete_db(self):
"""
Description - Delete all databases where instance.name contains
'DBaas' and the instance.status is active
"""
try:
instances = self.client.instances.list()
self.logger.warning('Delete db instances')
self.success = True
for instance in instances:
if instance.status == "ACTIVE":
if 'DBaaS' in instance.name:
try:
instance.delete()
self.logger.warning(
'delete ' + instance.id + ' ' + str(instance.name))
except Exception as e:
self.logger.error('could not delete %s', instance.name)
else:
self.logger.warning('did not delete %s', instance.name)
else:
self.logger.warning('Not active %s', instance.name)
except Exception as e:
self.success, self.overall_success = False, False
self.failure = e
self.logger.warning('Error Deleting Instance %s', e)
@monitoring.timeit
def delete_lb(self):
"""
Description - Delete all loadbalancers where name equals TestLB
"""
name = "TestLB"
try:
self.logger.warning('Delete Load Balancers:')
self.success = True
proc = subprocess.Popen(['libra',
'--insecure',
'--os-region-name',
self.region,
'list'], stdout=PIPE)
proc.wait()
outs, err = proc.communicate()
sum_output = ''
for lists in outs:
sum_output = sum_output + lists
if lists == '\n':
sum_output = ''
if name in sum_output:
sum_split = sum_output.split()
proc = subprocess.Popen(['libra',
'--insecure',
'--os-region-name',
self.region,
'delete', sum_split[1]])
sum_output = ''
proc.wait()
if proc.returncode == 0:
self.logger.warning('deleting %s', sum_split[1])
else:
self.failure = "Return code " + str(proc.returncode)
self.logger.error("did not delete lb return code %s",
proc.returncode)
except Exception as e:
self.success, self.overall_success = False, True
self.failure = e
self.logger.error("<*>delete_lb Failed %s", e)
@monitoring.timeit
def delete_cdn_containers(self):
"""
Description - Delete all containers where name starts with
'cdncheck' and all objects in that container
"""
try:
self.success = True
self.cdn_containers = self.cdn_client.get_account()[1]
if len(self.cdn_containers) == 0:
self.logger.warning(
'No CDN containers to clean up')
return
self.cdn_container_names = []
for i in self.cdn_containers:
if str(i['name']).startswith('cdncheck'):
self.cdn_container_names.append(str(i['name']))
if len(self.cdn_container_names) == 0:
self.logger.warning(
'No CDN containers (cdncheck string) to delete')
return
for self.container_name in self.cdn_container_names:
try:
self.objects = self.swift_client.get_container(
self.container_name)[1]
except Exception:
self.logger.warning(
"Couldn't retrieve objects for %s", self.container_name)
self.logger.warning("Deleting the container directly")
self.cdn_client.delete_container(self.container_name)
continue
if len(self.objects) == 0:
self.logger.warning(
'No objects found for %s', self.container_name)
self.logger.warning(
'Deleting CDN container %s', self.container_name)
self.swift_client.delete_container(self.container_name)
self.cdn_client.delete_container(self.container_name)
continue
self.object_names = []
for i in self.swift_client.get_container(self.container_name)[1]:
self.object_names.append(str(i['name']))
self.logger.warning(
'Deleting objects for %s', self.container_name)
for i in self.object_names:
self.logger.warning(' Deleting object %s', i)
self.swift_client.delete_object(self.container_name, i)
self.logger.warning(
'Deleting CDN container %s', self.container_name)
self.swift_client.delete_container(self.container_name)
self.cdn_client.delete_container(self.container_name)
except Exception as e:
self.success, self.overall_success = False, False
self.failure = e
self.logger.error("<*>delete_cdn_containers Failed %s", e)
@monitoring.timeit
def delete_swift_containers(self):
"""
Description - Delete all containers where name starts with
'swiftcheck' and all objects in that container
"""
try:
self.success = True
self.swift_containers = self.swift_client.get_account()[1]
if len(self.swift_containers) == 0:
self.logger.warning('No Swift containers to delete')
return
self.swift_container_names = []
for i in self.swift_containers:
if i['name'].startswith('swiftcheck'):
self.swift_container_names.append(str(i['name']))
for self.container_name in self.swift_container_names:
self.logger.warning('Container: %s', self.container_name)
self.objects = self.swift_client.get_container(
self.container_name)[1]
if len(self.objects) == 0:
self.logger.warning(
'No objects found for %s', self.container_name)
self.logger.warning(
'Deleting container: %s', self.container_name)
self.swift_client.delete_container(self.container_name)
continue
self.object_names = []
for i in self.swift_client.get_container(self.container_name)[1]:
self.object_names.append(str(i['name']))
self.logger.warning(
'Deleting objects for %s', self.container_name)
for i in self.object_names:
self.logger.warning(' Deleting %s', i)
self.swift_client.delete_object(self.container_name, i)
self.logger.warning(
'Deleting container: %s', self.container_name)
self.swift_client.delete_container(self.container_name)
except Exception as e:
self.success, self.overall_success = False, False
self.failure = e
self.logger.error("<*>delete_swift_containers Failed %s", e)
def delete_domains(self):
"""
Description - Delete all domains where domain.name contains
'OSfunctional'
"""
try:
self.logger.warning("Delete Domains")
for domain in self.designate_client.domains.list():
if 'OSfunctional' in domain.name:
self.logger.warning('Deleting Domain {}'.format(domain.name))
self.designate_client.domains.delete(domain.id)
except Exception as e:
self.logger.error('<*>Domain Deletion Failed: {}'.format(e))
def run(self):
if self.purge_service == "cdn":
self.delete_cdn_containers()
elif self.purge_service == "cinder":
self.delete_snapshots()
self.delete_volume()
elif self.purge_service == "domains":
self.delete_domains()
elif self.purge_service == "keystone":
pass
elif self.purge_service == "libra":
self.delete_lb()
elif self.purge_service == "neutron":
self.delete_floating()
self.delete_port()
self.delete_router()
self.delete_subnet()
self.delete_network()
elif self.purge_service == "nova":
self.delete_instance()
self.delete_floating()
self.delete_keypair()
elif self.purge_service == "swift":
self.delete_swift_containers()
elif self.purge_service == "trove":
self.delete_db()
else:
self.logger.error("Invalid service entered")
if self.overall_success is True:
exit(0)
else:
exit(1)
class SwiftBackend(duplicity.backend.Backend):
"""
Backend for Swift
"""
def __init__(self, parsed_url):
try:
from swiftclient import Connection
from swiftclient import ClientException
except ImportError:
raise BackendException("This backend requires "
"the python-swiftclient library.")
self.resp_exc = ClientException
conn_kwargs = {}
# if the user has already authenticated
if 'SWIFT_PREAUTHURL' in os.environ and 'SWIFT_PREAUTHTOKEN' in os.environ:
conn_kwargs['preauthurl'] = os.environ['SWIFT_PREAUTHURL']
conn_kwargs['preauthtoken'] = os.environ['SWIFT_PREAUTHTOKEN']
else:
if 'SWIFT_USERNAME' not in os.environ:
raise BackendException('SWIFT_USERNAME environment variable '
'not set.')
if 'SWIFT_PASSWORD' not in os.environ:
raise BackendException('SWIFT_PASSWORD environment variable '
'not set.')
if 'SWIFT_AUTHURL' not in os.environ:
raise BackendException('SWIFT_AUTHURL environment variable '
'not set.')
conn_kwargs['user'] = os.environ['SWIFT_USERNAME']
conn_kwargs['key'] = os.environ['SWIFT_PASSWORD']
conn_kwargs['authurl'] = os.environ['SWIFT_AUTHURL']
if 'SWIFT_AUTHVERSION' in os.environ:
conn_kwargs['auth_version'] = os.environ['SWIFT_AUTHVERSION']
else:
conn_kwargs['auth_version'] = '1'
if 'SWIFT_TENANTNAME' in os.environ:
conn_kwargs['tenant_name'] = os.environ['SWIFT_TENANTNAME']
if 'SWIFT_REGIONNAME' in os.environ:
conn_kwargs['os_options'] = {'region_name': os.environ['SWIFT_REGIONNAME']}
self.container = parsed_url.path.lstrip('/')
container_metadata = None
try:
self.conn = Connection(**conn_kwargs)
container_metadata = self.conn.head_container(self.container)
except ClientException:
pass
except Exception as e:
log.FatalError("Connection failed: %s %s"
% (e.__class__.__name__, str(e)),
log.ErrorCode.connection_failed)
if container_metadata is None:
log.Info("Creating container %s" % self.container)
try:
self.conn.put_container(self.container)
except Exception as e:
log.FatalError("Container creation failed: %s %s"
% (e.__class__.__name__, str(e)),
log.ErrorCode.connection_failed)
def _error_code(self, operation, e):
if isinstance(e, self.resp_exc):
if e.http_status == 404:
return log.ErrorCode.backend_not_found
def _put(self, source_path, remote_filename):
self.conn.put_object(self.container, remote_filename,
file(source_path.name))
def _get(self, remote_filename, local_path):
headers, body = self.conn.get_object(self.container, remote_filename)
with open(local_path.name, 'wb') as f:
for chunk in body:
f.write(chunk)
def _list(self):
headers, objs = self.conn.get_container(self.container, full_listing=True)
return [o['name'] for o in objs]
def _delete(self, filename):
self.conn.delete_object(self.container, filename)
def _query(self, filename):
sobject = self.conn.head_object(self.container, filename)
return {'size': int(sobject['content-length'])}
def __init__(self, parsed_url):
try:
from swiftclient import Connection
from swiftclient import ClientException
except ImportError:
raise BackendException("This backend requires "
"the python-swiftclient library.")
try:
import requests
except ImportError:
raise BackendException("This backend requires "
"the python-requests library.")
self.session = requests.Session()
self.resp_exc = ClientException
conn_kwargs = {}
# if the user has already authenticated
if os.environ.has_key('HUBIC_ACCESS_TOKEN'):
access_token = os.environ['HUBIC_ACCESS_TOKEN']
else:
if not os.environ.has_key('HUBIC_CLIENT_ID'):
raise BackendException('HUBIC_CLIENT_ID environment variable '
'not set.')
if not os.environ.has_key('HUBIC_CLIENT_SECRET'):
raise BackendException('HUBIC_CLIENT_SECRET environment variable '
'not set.')
if not os.environ.has_key('HUBIC_REFRESH_TOKEN'):
raise BackendException('HUBIC_REFRESH_TOKEN environment variable '
'not set.')
# With these 3 keys, generate an access token
access_token = self.getAccessToken(
client_id = os.environ['HUBIC_CLIENT_ID'],
client_secret = os.environ['HUBIC_CLIENT_SECRET'],
refresh_token = os.environ['HUBIC_REFRESH_TOKEN'],
)
if access_token == None:
raise BackendException('access_token not defined')
# Now obtain hubic openstack credentials
credentials = self.getOpenStackCredentials(access_token=access_token)
if ('endpoint' in credentials) and ('token' in credentials):
conn_kwargs['preauthurl'] = credentials['endpoint']
conn_kwargs['preauthtoken'] = credentials['token']
else:
raise BackendException('Unable to getOpenStackCredentials from access_token')
conn_kwargs['auth_version'] = '1'
self.container = parsed_url.path.lstrip('/')
try:
self.conn = Connection(**conn_kwargs)
self.conn.put_container(self.container)
except Exception, e:
log.FatalError("Connection failed: %s %s"
% (e.__class__.__name__, str(e)),
log.ErrorCode.connection_failed)