def _parse_v2_region():
"""Pull region/auth url information from context."""
if ARGS.get('os_rax_auth'):
region = ARGS.get('os_rax_auth')
auth_url = 'identity.api.rackspacecloud.com/v2.0/tokens'
if region is 'LON':
return ARGS.get('os_auth_url', 'https://lon.%s' % auth_url)
elif region.lower() in info.__rax_regions__:
return ARGS.get('os_auth_url', 'https://%s' % auth_url)
else:
raise turbo.SystemProblem('No Known RAX Region Was Specified')
elif ARGS.get('os_hp_auth'):
region = ARGS.get('os_hp_auth')
auth_url = 'https://%s.identity.hpcloudsvc.com:35357/v2.0/tokens'
if region.lower() in info.__hpc_regions__:
return ARGS.get('os_auth_url', auth_url % region)
else:
raise turbo.SystemProblem('No Known HP Region Was Specified')
elif 'os_auth_url' in ARGS:
auth_url = ARGS.get('os_auth_url')
if not auth_url.endswith('/tokens'):
auth_url = '%s/tokens' % auth_url
return auth_url
else:
raise turbo.SystemProblem(
'You Are required to specify an Auth URL, Region or Plugin')
def resp_exception(self, resp, rty):
"""If we encounter an exception in our upload.
we will look at how we can attempt to resolve the exception.
:param resp:
:param rty:
"""
try:
if resp.status == 401:
report.reporter(
msg='MESSAGE: Forced Re-authentication is happening.',
lvl='error',
log=True)
basic.stupid_hack()
self.payload['headers']['X-Auth-Token'] = auth.get_new_token()
rty()
elif resp.status == 404:
report.reporter(
msg=('Not found STATUS: %s, REASON: %s, MESSAGE: %s' %
(resp.status, resp.reason, resp.msg)),
prt=False,
lvl='debug')
elif resp.status == 413:
_di = dict(resp.getheaders())
basic.stupid_hack(wait=_di.get('retry_after', 10))
raise turbo.SystemProblem(
'The System encountered an API limitation and will'
' continue in "%s" Seconds' % _di.get('retry_after'))
elif resp.status == 502:
raise turbo.SystemProblem('Failure making Connection')
elif resp.status == 503:
basic.stupid_hack(wait=10)
raise turbo.SystemProblem('SWIFT-API FAILURE')
elif resp.status == 504:
basic.stupid_hack(wait=10)
raise turbo.SystemProblem('Gateway Time-out')
elif resp.status >= 300:
raise turbo.SystemProblem('SWIFT-API FAILURE -> REQUEST')
except turbo.SystemProblem as exp:
report.reporter(
msg=('FAIL-MESSAGE %s FAILURE STATUS %s FAILURE REASON %s '
'TYPE %s MESSAGE %s' %
(exp, resp.status, resp.reason, resp._method, resp.msg)),
prt=True,
lvl='warn',
log=True)
rty()
def parse_region():
"""Pull region/auth url information from context."""
base_auth_url = 'identity.api.rackspacecloud.com/v2.0/tokens'
if ARGS.get('os_region'):
region = ARGS.get('os_region')
elif ARGS.get('os_rax_auth'):
region = ARGS.get('os_rax_auth')
else:
raise turbo.SystemProblem('You Are required to specify a REGION')
if region is 'LON':
return ARGS.get('os_auth_url', 'lon.%s' % base_auth_url), True
elif region.lower() in info.__rax_regions__:
return ARGS.get('os_auth_url', '%s' % base_auth_url), True
else:
if ARGS.get('os_auth_url'):
if 'racksapce' in ARGS.get('os_auth_url'):
return ARGS.get('os_auth_url', '%s' % base_auth_url), True
else:
return ARGS.get('os_auth_url'), False
else:
LOG.error('FAILURE: No Region Found. ARGS DUMP:\t %s', ARGS)
raise turbo.AuthenticationProblem('You Are required to specify a'
' REGION and an AUTHURL')
def resp_exception(self, resp):
"""If we encounter an exception in our upload.
we will look at how we can attempt to resolve the exception.
:param resp:
"""
# Check to make sure we have all the bits needed
if not hasattr(resp, 'status_code'):
raise turbo.SystemProblem('No Status to check.')
elif resp is None:
raise turbo.SystemProblem('No response information.')
elif resp.status_code == 401:
report.reporter(
msg=('Turbolift experienced an Authentication issue.'
' STATUS %s REASON %s REQUEST %s. Turbolift will retry' %
(resp.status_code, resp.reason, resp.request)),
lvl='warn',
log=True,
prt=False)
# This was done in this manor due to how manager dicts are proxied
# related : http://bugs.python.org/issue6766
headers = self.payload['headers']
headers['X-Auth-Token'] = get_new_token()
self.payload['headers'] = headers
raise turbo.AuthenticationProblem(
'Attempting to resolve the Authentication issue.')
elif resp.status_code == 404:
report.reporter(
msg=('Not found STATUS: %s, REASON: %s, MESSAGE: %s' %
(resp.status_code, resp.reason, resp.request)),
prt=False,
lvl='debug')
elif resp.status_code == 413:
_di = resp.headers
basic.stupid_hack(wait=_di.get('retry_after', 10))
raise turbo.SystemProblem(
'The System encountered an API limitation and will'
' continue in "%s" Seconds' % _di.get('retry_after'))
elif resp.status_code == 502:
raise turbo.SystemProblem('Failure making Connection')
elif resp.status_code == 503:
basic.stupid_hack(wait=10)
raise turbo.SystemProblem('SWIFT-API FAILURE')
elif resp.status_code == 504:
basic.stupid_hack(wait=10)
raise turbo.SystemProblem('Gateway Time-out')
elif resp.status_code >= 300:
raise turbo.SystemProblem(
'SWIFT-API FAILURE -> REASON %s REQUEST %s' %
(resp.reason, resp.request))
else:
report.reporter(
msg=('MESSAGE %s %s %s' %
(resp.status_code, resp.reason, resp.request)),
prt=False,
lvl='debug')
def get_local_files():
"""Find all files specified in the "source" path.
This creates a list for all of files using the full path.
"""
def not_list(item):
"""Exclude items.
:param item:
:return True|False:
"""
if all([not os.path.islink(item), not os.path.ismount(item)]):
if not os.path.getsize(item) > 4831838208:
return True
else:
return False
def indexer(location):
"""Return a list of indexed files.
:param location:
:return:
"""
_location = basic.real_full_path(location.encode('utf8'))
if os.path.isdir(_location):
r_walk = os.walk(_location)
indexes = [(root, fls) for root, sfs, fls in r_walk]
return [
basic.jpath(root=inx[0], inode=inode) for inx in indexes
for inode in inx[1]
]
elif os.path.isfile(_location):
return [_location]
else:
raise turbo.NoFileProvided('No Path was Found for %s' % _location)
try:
d_paths = ARGS.get('source')
if not isinstance(d_paths, list):
d_paths = [d_paths]
# Local Index Path
c_index = [indexer(location=d_path) for d_path in d_paths]
# make sure my files are only files, and compare it with the not_list
f_index = [
item for subl in c_index for item in subl if not_list(item=item)
]
except Exception as exp:
raise turbo.SystemProblem('Died for some reason. MESSAGE:\t%s' % exp)
else:
report.reporter(msg='FILE LIST:\t%s' % f_index, lvl='debug', prt=False)
return f_index
def authenticate():
"""Authentication For Openstack API.
Pulls the full Openstack Service Catalog Credentials are the Users API
Username and Key/Password "osauth" has a Built in Rackspace Method for
Authentication
Set a DC Endpoint and Authentication URL for the OpenStack environment
"""
# Setup the request variables
a_url = "https://zebra.zerovm.org/auth/v1.0"
#a_url = http.parse_url(url=url, auth=True)
auth_json = auth.parse_reqtype()
print auth_json
# remove the prefix for the Authentication URL if Found
# LOG.debug('POST == REQUEST DICT > JSON DUMP %s', auth_json)
# auth_json_req = json.dumps(auth_json)
headers = {
'Content-Type': 'application/json',
"X-Auth-User": auth_json['auth']['passwordCredentials']['username'],
"X-Auth-Key": auth_json['auth']['passwordCredentials']['password']
}
# Send Request
try:
auth_resp = requests.get(url=a_url, headers=headers)
if auth_resp.status_code >= 300:
raise SystemExit('Authentication Failure, %s %s' %
(auth_resp.status_code, auth_resp.reason))
except ValueError as exp:
LOG.error('Authentication Failure %s\n%s', exp, traceback.format_exc())
raise turbo.SystemProblem('JSON Decode Failure. ERROR: %s' % exp)
else:
LOG.debug('POST Authentication Response %s', auth_resp.json())
#auth_info = auth.parse_auth_response(auth_resp.json())
#token, tenant, user, inet, enet, cnet, acfep = auth_info
token = auth_resp.headers['x-auth-token']
tenant, user = auth_json['auth']['passwordCredentials'][
'username'].split(":")
inet = urlparse.urlparse(auth_resp.headers['x-storage-url'])
enet = inet
cnet = None
acfep = inet
report.reporter(msg=('API Access Granted. TenantID: %s Username: %s' %
(tenant, user)),
prt=False,
log=True)
return token, tenant, user, inet, enet, cnet, urlparse.urlparse(
a_url), acfep
def get_surl(region, cf_list, lookup):
"""Lookup a service URL.
:param region:
:param cf_list:
:param lookup:
:return net:
"""
for srv in cf_list:
if region in srv.get('region'):
net = http.parse_url(url=srv.get(lookup))
return net
else:
raise turbo.SystemProblem(
'Region "%s" was not found in your Service Catalog.' % region)
def authenticate():
"""Authentication For Openstack API.
Pulls the full Openstack Service Catalog Credentials are the Users API
Username and Key/Password "osauth" has a Built in Rackspace Method for
Authentication
Set a DC Endpoint and Authentication URL for the OpenStack environment
"""
# Setup the request variables
url = auth.parse_region()
LOG.debug('Raw Auth URL: [ %s ]', url)
a_url = http.parse_url(url=url, auth=True)
headers = {
'Content-Type': 'application/json',
'Accept': 'application/json'
}
headers.update(auth.get_headers() or {})
auth_json = auth.parse_reqtype() or {}
LOG.debug('Parsed Auth URL: [ %s ]', a_url)
# remove the prefix for the Authentication URL if Found
auth_json_req = json.dumps(auth_json)
LOG.debug('Request JSON: [ %s ]', auth_json_req)
LOG.debug('Request Headers: [ %s ]', headers)
# Send Request
try:
auth_resp = auth.auth_request(a_url,
headers=headers,
body=auth_json_req)
if auth_resp.status_code >= 300:
raise SystemExit('Authentication Failure, %s %s' %
(auth_resp.status_code, auth_resp.reason))
except ValueError as exp:
LOG.error('Authentication Failure %s\n%s', exp, traceback.format_exc())
raise turbo.SystemProblem('JSON Decode Failure. ERROR: %s' % exp)
else:
auth_info = auth.parse_auth_response(auth_resp)
token, tenant, user, inet, enet, cnet, acfep = auth_info
report.reporter(msg=('API Access Granted. TenantID: %s Username: %s' %
(tenant, user)),
prt=False,
log=True)
return token, tenant, user, inet, enet, cnet, a_url, acfep
def parse_auth_response(auth_response):
"""Parse the auth response and return the tenant, token, and username.
:param auth_response: the full object returned from an auth call
:returns: tuple (token, tenant, username, internalurl, externalurl, cdnurl)
"""
access = auth_response.get('access')
token = access.get('token').get('id')
if 'tenant' in access.get('token'):
tenant = access.get('token').get('tenant').get('name')
user = access.get('user').get('name')
elif 'user' in access:
tenant = None
user = access.get('user').get('name')
else:
LOG.error('No Token Found to Parse.\nHere is the DATA: %s\n%s',
auth_response, traceback.format_exc())
raise turbo.NoTenantIdFound('When attempting to grab the '
'tenant or user nothing was found.')
scat = access.pop('serviceCatalog')
for srv in scat:
if srv.get('name') in info.__srv_types__:
if srv.get('name') == 'cloudFilesCDN':
cdn = srv.get('endpoints')
if srv.get('name') == ARGS.get('service_type'):
cfl = srv.get('endpoints')
if ARGS.get('os_region') is not None:
region = ARGS.get('os_region')
elif ARGS.get('os_rax_auth') is not None:
region = ARGS.get('os_rax_auth')
else:
raise turbo.SystemProblem('No Region Set')
if cfl is not None:
inet = get_surl(region=region, cf_list=cfl, lookup='internalURL')
enet = get_surl(region=region, cf_list=cfl, lookup='publicURL')
if cdn is not None:
cnet = get_surl(region=region, cf_list=cdn, lookup='publicURL')
return token, tenant, user, inet, enet, cnet, cfl
def bcolors(msg, color):
"""return a colorizes string.
:param msg:
:param color:
:return str:
"""
# Available Colors
colors = {'debug': '\033[94m',
'info': '\033[92m',
'warn': '\033[93m',
'error': '\033[91m',
'critical': '\033[95m',
'ENDC': '\033[0m'}
if color in colors:
return '%s%s%s' % (colors[color], msg, colors['ENDC'])
else:
raise turbo.SystemProblem('"%s" was not a known color.' % color)
def get_surl(region, cf_list, lookup):
"""Lookup a service URL.
:param region:
:param cf_list:
:param lookup:
:return net:
"""
for srv in cf_list:
region_get = srv.get('region')
lookup_get = srv.get(lookup)
if any([region in region_get, region.lower() in region_get]):
if lookup_get is None:
return None
else:
return http.parse_url(url=lookup_get)
else:
raise turbo.SystemProblem(
'Region "%s" was not found in your Service Catalog.' % region)
def authenticate():
"""Authentication For Openstack API.
Pulls the full Openstack Service Catalog Credentials are the Users API
Username and Key/Password "osauth" has a Built in Rackspace Method for
Authentication
Set a DC Endpoint and Authentication URL for the OpenStack environment
:param auth_dict: required parameters are auth_url
"""
# Setup the request variables
url, rax = auth.parse_region()
a_url = http.parse_url(url=url, auth=True)
auth_json = auth.parse_reqtype()
# remove the prefix for the Authentication URL if Found
LOG.debug('POST == REQUEST DICT > JSON DUMP %s', auth_json)
auth_json_req = json.dumps(auth_json)
headers = {'Content-Type': 'application/json'}
# Send Request
request = ('POST', a_url.path, auth_json_req, headers)
resp_read = auth.request_process(aurl=a_url, req=request)
LOG.debug('POST Authentication Response %s', resp_read)
try:
auth_resp = json.loads(resp_read)
except ValueError as exp:
LOG.error('Authentication Failure %s\n%s', exp, traceback.format_exc())
raise turbo.SystemProblem('JSON Decode Failure. ERROR: %s - RESP %s' %
(exp, resp_read))
else:
auth_info = auth.parse_auth_response(auth_resp)
token, tenant, user, inet, enet, cnet, acfep = auth_info
report.reporter(msg=('API Access Granted. TenantID: %s Username: %s' %
(tenant, user)),
prt=False,
log=True)
return token, tenant, user, inet, enet, cnet, a_url, acfep
def _parse_v2_auth_response(auth_response):
"""Parse the auth response and return the tenant, token, and username.
:param auth_response: the full object returned from an auth call
:returns: tuple (token, tenant, username, internalurl, externalurl, cdnurl)
"""
try:
auth_response = auth_response.json()
except AttributeError:
pass
else:
LOG.debug('Authentication Response Body %s', auth_response)
def _service_ep(scat, types_list):
for srv in scat:
if srv.get('name') in types_list:
index_id = types_list.index(srv.get('name'))
index = types_list[index_id]
if srv.get('name') == index:
return srv.get('endpoints')
else:
return None
access = auth_response.get('access')
token = access.get('token').get('id')
if 'tenant' in access.get('token'):
tenant = access.get('token').get('tenant').get('name')
user = access.get('user').get('name')
elif 'user' in access:
tenant = None
user = access.get('user').get('name')
else:
LOG.error('No Token Found to Parse.\nHere is the DATA: %s\n%s',
auth_response, traceback.format_exc())
raise turbo.NoTenantIdFound('When attempting to grab the '
'tenant or user nothing was found.')
if ARGS.get('os_rax_auth') is not None:
region = ARGS.get('os_rax_auth')
elif ARGS.get('os_hp_auth') is not None:
if ARGS.get('os_tenant') is None:
raise turbo.NoTenantIdFound(
'You need to have a tenant set to use HP Cloud')
region = ARGS.get('os_hp_auth')
elif ARGS.get('os_region') is not None:
region = ARGS.get('os_region')
else:
raise turbo.SystemProblem('No Region Set')
scat = access.pop('serviceCatalog')
cfl = _service_ep(scat, info.__srv_types__)
cdn = _service_ep(scat, info.__cdn_types__)
if cfl is not None:
inet = get_surl(region=region, cf_list=cfl, lookup='internalURL')
enet = get_surl(region=region, cf_list=cfl, lookup='publicURL')
else:
need_tenant = ' Maybe you need to specify "os-tenant"?'
gen_message = ('No Service Endpoints were found for use '
'with Swift. If you have Swift available to you, '
'Check Your Credentials and/or Swift\'s '
'availability using Token Auth.')
if ARGS.get('os_tenant') is None:
gen_message += need_tenant
raise turbo.SystemProblem(gen_message)
if cdn is not None:
cnet = get_surl(region=region, cf_list=cdn, lookup='publicURL')
else:
cnet = None
return token, tenant, user, inet, enet, cnet, cfl
def get_local_files():
"""Find all files specified in the "source" path.
This creates a list for all of files using the full path.
"""
def not_list(item):
"""Exclude items.
:param item:
:return True|False:
"""
if os.path.islink(item):
dest = os.readlink(item)
if not os.path.exists(dest):
return True
elif not os.path.ismount(item):
if not os.path.getsize(item) > 4831838208:
return True
return False
def indexer(location):
"""Return a list of indexed files.
:param location:
:return:
"""
_location = basic.real_full_path(location.encode('utf8'))
if os.path.isdir(_location):
r_walk = os.walk(_location)
objects = list()
for root_inx, inx in [(root, fls) for root, sfs, fls in r_walk]:
for inode in inx:
object_path = basic.jpath(root=root_inx, inode=inode)
objects.append(unicode(object_path.decode('utf-8')))
else:
return objects
elif os.path.isfile(_location):
return [_location]
else:
raise turbo.NoFileProvided('No Path was Found for %s' % _location)
try:
d_paths = ARGS.get('source')
if not isinstance(d_paths, list):
d_paths = [d_paths]
# Local Index Path
c_index = [indexer(location=d_path) for d_path in d_paths]
# make sure my files are only files, and not in the the not_list
f_index = [
item for subl in c_index for item in subl if not_list(item=item)
]
if ARGS.get('exclude'):
for item in f_index:
for exclude in ARGS['exclude']:
if exclude in item:
try:
index = f_index.index(item)
except ValueError:
pass
else:
f_index.pop(index)
except Exception as exp:
raise turbo.SystemProblem('Died for some reason. MESSAGE:\t%s' % exp)
else:
report.reporter(msg='FILE LIST:\t%s' % f_index, lvl='debug', prt=False)
return f_index
def compress_files(file_list):
"""If the archive function is used, create a compressed archive.
:param file_list:
This function allows for multiple sources to be added to the
compressed archive.
"""
tmp_file = None
try:
# Set date and time
date_format = '%a%b%d.%H.%M.%S.%Y'
today = datetime.datetime.today()
_ts = today.strftime(date_format)
# Get Home Directory
home_dir = os.getenv('HOME')
# Set the name of the archive.
set_name = ARGS.get('tar_name', '%s_%s' % ('Archive', _ts))
file_name = '%s.tgz' % set_name
# Set the working File.
tmp_file = basic.jpath(root=home_dir, inode=file_name)
# Begin creating the Archive.
tar = tarfile.open(tmp_file, 'w:gz')
for name in file_list:
if basic.file_exists(name) is True:
tar.add(name)
tar.close()
report.reporter(msg='ARCHIVE CREATED: %s' % tmp_file, prt=False)
if ARGS.get('verify'):
tar_len = tarfile.open(tmp_file, 'r')
ver_array = []
for member_info in tar_len.getmembers():
ver_array.append(member_info.name)
count = len(ver_array)
orig_count = len(file_list)
if orig_count != count:
raise turbo.SystemProblem(
'ARCHIVE NOT VERIFIED: Archive and File List do not Match.'
' Original File Count = %s, Found Archive Contents = %s' %
(orig_count, count))
report.reporter(msg='ARCHIVE CONTENTS VERIFIED: %s files' %
count, )
except KeyboardInterrupt:
if basic.file_exists(tmp_file):
basic.remove_file(tmp_file)
turbo.emergency_exit('I have stopped at your command,'
' I removed Local Copy of the Archive')
except Exception as exp:
if basic.file_exists(tmp_file):
basic.remove_file(tmp_file)
turbo.emergency_exit(
'I am sorry i just don\'t know what you put into me, Removing'
' Local Copy of the Archive.')
turbo.emergency_exit(
'Exception while working on the archive. MESSAGE: %s' % exp)
else:
return tmp_file