def parse_region():
"""Pull region/auth url information from context."""
base_auth_url = 'identity.api.rackspacecloud.com/v2.0/tokens'
if ARGS.get('os_region'):
region = ARGS.get('os_region')
elif ARGS.get('os_rax_auth'):
region = ARGS.get('os_rax_auth')
else:
raise turbo.SystemProblem('You Are required to specify a REGION')
if region is 'LON':
return ARGS.get('os_auth_url', 'lon.%s' % base_auth_url), True
elif region.lower() in info.__rax_regions__:
return ARGS.get('os_auth_url', '%s' % base_auth_url), True
else:
if ARGS.get('os_auth_url'):
if 'racksapce' in ARGS.get('os_auth_url'):
return ARGS.get('os_auth_url', '%s' % base_auth_url), True
else:
return ARGS.get('os_auth_url'), False
else:
LOG.error('FAILURE: No Region Found. ARGS DUMP:\t %s', ARGS)
raise turbo.AuthenticationProblem('You Are required to specify a'
' REGION and an AUTHURL')
def post_request(url, headers, body=None, rpath=None):
"""Perform HTTP(s) POST request based on Provided Params.
:param url:
:param rpath:
:param headers:
:param body:
:return resp:
"""
try:
if rpath is not None:
_url = urlparse.urljoin(urlparse.urlunparse(url), rpath)
else:
_url = urlparse.urlunparse(url)
kwargs = {'timeout': ARGS.get('timeout', 60)}
resp = requests.post(_url, data=body, headers=headers, **kwargs)
except Exception as exp:
LOG.error('Not able to perform Request ERROR: %s', exp)
raise AttributeError(
"Failure to perform Authentication %s ERROR:\n%s" %
(exp, traceback.format_exc()))
else:
return resp
def parse_reqtype():
"""Setup our Authentication POST.
username and setup are only used in APIKEY/PASSWORD Authentication
"""
setup = {'username': ARGS.get('os_user')}
if ARGS.get('os_token') is not None:
auth_body = {'auth': {'token': {'id': ARGS.get('os_token')}}}
elif ARGS.get('os_password') is not None:
prefix = 'passwordCredentials'
setup['password'] = ARGS.get('os_password')
auth_body = {'auth': {prefix: setup}}
elif ARGS.get('os_apikey') is not None:
prefix = 'RAX-KSKEY:apiKeyCredentials'
setup['apiKey'] = ARGS.get('os_apikey')
auth_body = {'auth': {prefix: setup}}
else:
LOG.error(traceback.format_exc())
raise AttributeError('No Password, APIKey, or Token Specified')
if ARGS.get('os_tenant'):
auth_body['auth']['tenantName'] = ARGS.get('os_tenant')
LOG.debug('AUTH Request Type > %s', auth_body)
return auth_body
def parse_reqtype():
"""Setup our Authentication POST.
username and setup are only used in APIKEY/PASSWORD Authentication
"""
setup = {'username': ARGS.get('os_user')}
if ARGS.get('os_token') is not None:
auth_body = {'auth': {'token': {'id': ARGS.get('os_token')}}}
elif ARGS.get('os_password') is not None:
prefix = 'passwordCredentials'
setup['password'] = ARGS.get('os_password')
auth_body = {'auth': {prefix: setup}}
elif ARGS.get('os_apikey') is not None:
prefix = 'RAX-KSKEY:apiKeyCredentials'
setup['apiKey'] = ARGS.get('os_apikey')
auth_body = {'auth': {prefix: setup}}
else:
LOG.error(traceback.format_exc())
raise AttributeError('No Password, APIKey, or Token Specified')
if ARGS.get('os_tenant'):
auth_body['auth']['tenantName'] = ARGS.get('os_tenant')
LOG.debug('AUTH Request Type > %s', auth_body)
return auth_body
def parse_region():
"""Pull region/auth url information from context."""
base_auth_url = 'identity.api.rackspacecloud.com/v2.0/tokens'
if ARGS.get('os_region'):
region = ARGS.get('os_region')
elif ARGS.get('os_rax_auth'):
region = ARGS.get('os_rax_auth')
else:
raise turbo.SystemProblem('You Are required to specify a REGION')
if region is 'LON':
return ARGS.get('os_auth_url', 'lon.%s' % base_auth_url), True
elif region.lower() in info.__rax_regions__:
return ARGS.get('os_auth_url', '%s' % base_auth_url), True
else:
if ARGS.get('os_auth_url'):
if 'racksapce' in ARGS.get('os_auth_url'):
return ARGS.get('os_auth_url', '%s' % base_auth_url), True
else:
return ARGS.get('os_auth_url'), False
else:
LOG.error('FAILURE: No Region Found. ARGS DUMP:\t %s', ARGS)
raise turbo.AuthenticationProblem('You Are required to specify a'
' REGION and an AUTHURL')
def start(self):
"""This is the upload method.
Uses file_upload is to simply upload all files and folders to a
specified container.
"""
# Index Local Files for Upload
with multi.spinner():
f_indexed = methods.get_local_files()
if ARGS.get('pattern_match'):
f_indexed = basic.match_filter(
idx_list=f_indexed, pattern=ARGS['pattern_match']
)
num_files = len(f_indexed)
# Get The rate of concurrency
concurrency = multi.set_concurrency(args=ARGS, file_count=num_files)
# Package up the Payload
payload = multi.manager_dict(
http.prep_payload(
auth=self.auth,
container=ARGS.get('container', basic.rand_string()),
source=basic.get_local_source(),
args=ARGS
)
)
LOG.info('MESSAGE\t: "%s" Files have been found.', num_files)
LOG.debug('PAYLOAD\t: "%s"', payload)
# Set the actions class up
self.go = actions.CloudActions(payload=payload)
kwargs = {'url': payload['url'],
'container': payload['c_name']}
# get that the container exists if not create it.
self.go.container_create(**kwargs)
kwargs['source'] = payload['source']
kwargs['cf_job'] = getattr(self.go, 'object_putter')
multi.job_processer(
num_jobs=num_files,
objects=f_indexed,
job_action=multi.doerator,
concur=concurrency,
kwargs=kwargs
)
if ARGS.get('delete_remote') is True:
self.remote_delete(payload=payload,
f_indexed=f_indexed)
def start(self):
"""This is the upload method.
Uses file_upload is to simply upload all files and folders to a
specified container.
"""
# Index Local Files for Upload
f_indexed = methods.get_local_files()
if ARGS.get('pattern_match'):
f_indexed = basic.match_filter(
idx_list=f_indexed, pattern=ARGS['pattern_match']
)
num_files = len(f_indexed)
# Get The rate of concurrency
concurrency = multi.set_concurrency(args=ARGS, file_count=num_files)
# Package up the Payload
payload = multi.manager_dict(
http.prep_payload(
auth=self.auth,
container=ARGS.get('container', basic.rand_string()),
source=basic.get_local_source(),
args=ARGS
)
)
LOG.info('MESSAGE\t: "%s" Files have been found.', num_files)
LOG.debug('PAYLOAD\t: "%s"', payload)
# Set the actions class up
self.go = actions.CloudActions(payload=payload)
kwargs = {'url': payload['url'],
'container': payload['c_name']}
# get that the container exists if not create it.
self.go.container_create(**kwargs)
kwargs['source'] = payload['source']
kwargs['cf_job'] = getattr(self.go, 'object_putter')
multi.job_processer(
num_jobs=num_files,
objects=f_indexed,
job_action=multi.doerator,
concur=concurrency,
kwargs=kwargs
)
if ARGS.get('delete_remote') is True:
self.remote_delete(payload=payload,
f_indexed=f_indexed)
def remote_delete(self, payload, f_indexed):
"""If Remote Delete was True run.
NOTE: Remote delete will delete ALL Objects in a remote container
which differ from the objects in the SOURCED LOCAL FILESYSTEM.
IE: If this option is used, on one directory and then another directory
and the files were different any difference would be deleted and based
on the index information found in LOCAL FILE SYSTEM on the LAST
command run.
:return:
"""
report.reporter(msg='Getting file list for REMOTE DELETE')
objects = self.go.object_lister(
url=payload['url'], container=payload['c_name']
)
source = payload['source']
obj_names = [basic.jpath(root=source, inode=obj.get('name'))
for obj in objects[0]]
# From the remote system see if we have differences in the local system
objects = multi.return_diff().difference(target=f_indexed,
source=obj_names)
if objects:
# Set Basic Data for file delete.
num_files = len(objects)
LOG.info('MESSAGE\t: "%s" Files have been found to be removed from'
' the REMOTE CONTAINER.', num_files)
concurrency = multi.set_concurrency(
args=ARGS, file_count=num_files
)
# Delete the difference in Files.
report.reporter(msg='Performing Remote Delete')
objects = [basic.get_sfile(
ufile=obj, source=payload['source']
) for obj in objects]
kwargs = {'url': payload['url'],
'container': payload['c_name'],
'cf_job': getattr(self.go, 'object_deleter')}
multi.job_processer(
num_jobs=num_files,
objects=objects,
job_action=multi.doerator,
concur=concurrency,
kwargs=kwargs
)
else:
report.reporter(
msg='No Difference between REMOTE and LOCAL Directories.'
)
def authenticate():
"""Authentication For Openstack API.
Pulls the full Openstack Service Catalog Credentials are the Users API
Username and Key/Password "osauth" has a Built in Rackspace Method for
Authentication
Set a DC Endpoint and Authentication URL for the OpenStack environment
"""
# Setup the request variables
a_url = "https://zebra.zerovm.org/auth/v1.0"
#a_url = http.parse_url(url=url, auth=True)
auth_json = auth.parse_reqtype()
print auth_json
# remove the prefix for the Authentication URL if Found
# LOG.debug('POST == REQUEST DICT > JSON DUMP %s', auth_json)
# auth_json_req = json.dumps(auth_json)
headers = {
'Content-Type': 'application/json',
"X-Auth-User": auth_json['auth']['passwordCredentials']['username'],
"X-Auth-Key": auth_json['auth']['passwordCredentials']['password']}
# Send Request
try:
auth_resp = requests.get(
url=a_url, headers=headers
)
if auth_resp.status_code >= 300:
raise SystemExit(
'Authentication Failure, %s %s' % (auth_resp.status_code,
auth_resp.reason)
)
except ValueError as exp:
LOG.error('Authentication Failure %s\n%s', exp, traceback.format_exc())
raise turbo.SystemProblem('JSON Decode Failure. ERROR: %s' % exp)
else:
LOG.debug('POST Authentication Response %s', auth_resp.json())
#auth_info = auth.parse_auth_response(auth_resp.json())
#token, tenant, user, inet, enet, cnet, acfep = auth_info
token = auth_resp.headers['x-auth-token']
tenant, user = auth_json['auth']['passwordCredentials']['username'].split(":")
inet = urlparse.urlparse(auth_resp.headers['x-storage-url'])
enet = inet
cnet = None
acfep = inet
report.reporter(
msg=('API Access Granted. TenantID: %s Username: %s'
% (tenant, user)),
prt=False,
log=True
)
return token, tenant, user, inet, enet, cnet, urlparse.urlparse(a_url), acfep
def remote_delete(self, payload, f_indexed):
"""If Remote Delete was True run.
NOTE: Remote delete will delete ALL Objects in a remote container
which differ from the objects in the SOURCED LOCAL FILESYSTEM.
IE: If this option is used, on one directory and then another directory
and the files were different any difference would be deleted and based
on the index information found in LOCAL FILE SYSTEM on the LAST
command run.
:return:
"""
report.reporter(msg='Getting file list for REMOTE DELETE')
objects = self.go.object_lister(
url=payload['url'], container=payload['c_name']
)
source = payload['source']
obj_names = [basic.jpath(root=source, inode=obj.get('name'))
for obj in objects[0]]
# From the remote system see if we have differences in the local system
objects = multi.return_diff().difference(target=f_indexed,
source=obj_names)
if objects:
# Set Basic Data for file delete.
num_files = len(objects)
LOG.info('MESSAGE\t: "%s" Files have been found to be removed from'
' the REMOTE CONTAINER.', num_files)
concurrency = multi.set_concurrency(
args=ARGS, file_count=num_files
)
# Delete the difference in Files.
report.reporter(msg='Performing Remote Delete')
objects = [basic.get_sfile(
ufile=obj, source=payload['source']
) for obj in objects]
kwargs = {'url': payload['url'],
'container': payload['c_name'],
'cf_job': getattr(self.go, 'object_deleter')}
multi.job_processer(
num_jobs=num_files,
objects=objects,
job_action=multi.doerator,
concur=concurrency,
kwargs=kwargs
)
else:
report.reporter(
msg='No Difference between REMOTE and LOCAL Directories.'
)
def get_request(url, headers, rpath, stream=False):
try:
_url = urlparse.urljoin(urlparse.urlunparse(url), rpath)
kwargs = {'timeout': ARGS.get('timeout')}
resp = requests.get(_url, headers=headers, stream=stream, **kwargs)
report.reporter(msg='INFO: %s %s %s' %
(resp.status_code, resp.reason, resp.request),
prt=False)
except Exception as exp:
LOG.error('Not able to perform Request ERROR: %s', exp)
else:
return resp
def authenticate():
"""Authentication For Openstack API.
Pulls the full Openstack Service Catalog Credentials are the Users API
Username and Key/Password "osauth" has a Built in Rackspace Method for
Authentication
Set a DC Endpoint and Authentication URL for the OpenStack environment
"""
# Setup the request variables
a_url = "https://zebra.zerovm.org/auth/v1.0"
#a_url = http.parse_url(url=url, auth=True)
auth_json = auth.parse_reqtype()
print auth_json
# remove the prefix for the Authentication URL if Found
# LOG.debug('POST == REQUEST DICT > JSON DUMP %s', auth_json)
# auth_json_req = json.dumps(auth_json)
headers = {
'Content-Type': 'application/json',
"X-Auth-User": auth_json['auth']['passwordCredentials']['username'],
"X-Auth-Key": auth_json['auth']['passwordCredentials']['password']
}
# Send Request
try:
auth_resp = requests.get(url=a_url, headers=headers)
if auth_resp.status_code >= 300:
raise SystemExit('Authentication Failure, %s %s' %
(auth_resp.status_code, auth_resp.reason))
except ValueError as exp:
LOG.error('Authentication Failure %s\n%s', exp, traceback.format_exc())
raise turbo.SystemProblem('JSON Decode Failure. ERROR: %s' % exp)
else:
LOG.debug('POST Authentication Response %s', auth_resp.json())
#auth_info = auth.parse_auth_response(auth_resp.json())
#token, tenant, user, inet, enet, cnet, acfep = auth_info
token = auth_resp.headers['x-auth-token']
tenant, user = auth_json['auth']['passwordCredentials'][
'username'].split(":")
inet = urlparse.urlparse(auth_resp.headers['x-storage-url'])
enet = inet
cnet = None
acfep = inet
report.reporter(msg=('API Access Granted. TenantID: %s Username: %s' %
(tenant, user)),
prt=False,
log=True)
return token, tenant, user, inet, enet, cnet, urlparse.urlparse(
a_url), acfep
def get_request(url, headers, rpath, stream=False):
try:
_url = urlparse.urljoin(urlparse.urlunparse(url), rpath)
resp = requests.get(_url, headers=headers, stream=stream)
report.reporter(
msg='INFO: %s %s %s' % (resp.status_code,
resp.reason,
resp.request),
prt=False
)
except Exception as exp:
LOG.error('Not able to perform Request ERROR: %s', exp)
else:
return resp
def delete_request(url, headers, rpath):
try:
_url = urlparse.urljoin(urlparse.urlunparse(url), rpath)
kwargs = {'timeout': ARGS.get('timeout')}
resp = requests.delete(_url, headers=headers, **kwargs)
report.reporter(
msg='INFO: %s %s %s' % (resp.status_code,
resp.reason,
resp.request),
prt=False
)
except Exception as exp:
LOG.error('Not able to perform Request ERROR: %s', exp)
else:
return resp
def parse_auth_response(auth_response):
"""Parse the auth response and return the tenant, token, and username.
:param auth_response: the full object returned from an auth call
:returns: tuple (token, tenant, username, internalurl, externalurl, cdnurl)
"""
access = auth_response.get('access')
token = access.get('token').get('id')
if 'tenant' in access.get('token'):
tenant = access.get('token').get('tenant').get('name')
user = access.get('user').get('name')
elif 'user' in access:
tenant = None
user = access.get('user').get('name')
else:
LOG.error('No Token Found to Parse.\nHere is the DATA: %s\n%s',
auth_response, traceback.format_exc())
raise turbo.NoTenantIdFound('When attempting to grab the '
'tenant or user nothing was found.')
scat = access.pop('serviceCatalog')
for srv in scat:
if srv.get('name') in info.__srv_types__:
if srv.get('name') == 'cloudFilesCDN':
cdn = srv.get('endpoints')
if srv.get('name') == ARGS.get('service_type'):
cfl = srv.get('endpoints')
if ARGS.get('os_region') is not None:
region = ARGS.get('os_region')
elif ARGS.get('os_rax_auth') is not None:
region = ARGS.get('os_rax_auth')
else:
raise turbo.SystemProblem('No Region Set')
if cfl is not None:
inet = get_surl(region=region, cf_list=cfl, lookup='internalURL')
enet = get_surl(region=region, cf_list=cfl, lookup='publicURL')
if cdn is not None:
cnet = get_surl(region=region, cf_list=cdn, lookup='publicURL')
return token, tenant, user, inet, enet, cnet, cfl
def parse_auth_response(auth_response):
"""Parse the auth response and return the tenant, token, and username.
:param auth_response: the full object returned from an auth call
:returns: tuple (token, tenant, username, internalurl, externalurl, cdnurl)
"""
access = auth_response.get('access')
token = access.get('token').get('id')
if 'tenant' in access.get('token'):
tenant = access.get('token').get('tenant').get('name')
user = access.get('user').get('name')
elif 'user' in access:
tenant = None
user = access.get('user').get('name')
else:
LOG.error('No Token Found to Parse.\nHere is the DATA: %s\n%s',
auth_response, traceback.format_exc())
raise turbo.NoTenantIdFound('When attempting to grab the '
'tenant or user nothing was found.')
scat = access.pop('serviceCatalog')
for srv in scat:
if srv.get('name') in info.__srv_types__:
if srv.get('name') == 'cloudFilesCDN':
cdn = srv.get('endpoints')
if srv.get('name') == ARGS.get('service_type'):
cfl = srv.get('endpoints')
if ARGS.get('os_region') is not None:
region = ARGS.get('os_region')
elif ARGS.get('os_rax_auth') is not None:
region = ARGS.get('os_rax_auth')
else:
raise turbo.SystemProblem('No Region Set')
if cfl is not None:
inet = get_surl(region=region, cf_list=cfl, lookup='internalURL')
enet = get_surl(region=region, cf_list=cfl, lookup='publicURL')
if cdn is not None:
cnet = get_surl(region=region, cf_list=cdn, lookup='publicURL')
return token, tenant, user, inet, enet, cnet, cfl
def authenticate():
"""Authentication For Openstack API.
Pulls the full Openstack Service Catalog Credentials are the Users API
Username and Key/Password "osauth" has a Built in Rackspace Method for
Authentication
Set a DC Endpoint and Authentication URL for the OpenStack environment
"""
# Setup the request variables
url = auth.parse_region()
a_url = http.parse_url(url=url, auth=True)
auth_json = auth.parse_reqtype()
# remove the prefix for the Authentication URL if Found
LOG.debug("POST == REQUEST DICT > JSON DUMP %s", auth_json)
auth_json_req = json.dumps(auth_json)
headers = {"Content-Type": "application/json"}
# Send Request
request = ("POST", a_url.path, auth_json_req, headers)
resp_read = auth.request_process(aurl=a_url, req=request)
LOG.debug("POST Authentication Response %s", resp_read)
try:
auth_resp = json.loads(resp_read)
except ValueError as exp:
LOG.error("Authentication Failure %s\n%s", exp, traceback.format_exc())
raise turbo.SystemProblem("JSON Decode Failure. ERROR: %s - RESP %s" % (exp, resp_read))
else:
auth_info = auth.parse_auth_response(auth_resp)
token, tenant, user, inet, enet, cnet, acfep = auth_info
report.reporter(msg=("API Access Granted. TenantID: %s Username: %s" % (tenant, user)), prt=False, log=True)
return token, tenant, user, inet, enet, cnet, a_url, acfep
def request_process(aurl, req):
"""Perform HTTP(s) request based on Provided Params.
:param aurl:
:param req:
:param https:
:return read_resp:
"""
conn = http.open_connection(url=aurl)
# Make the request for authentication
try:
_method, _url, _body, _headers = req
conn.request(method=_method, url=_url, body=_body, headers=_headers)
resp = conn.getresponse()
except Exception as exc:
LOG.error('Not able to perform Request ERROR: %s', exc)
raise AttributeError("Failure to perform Authentication %s ERROR:\n%s"
% (exc, traceback.format_exc()))
else:
resp_read = resp.read()
status_code = resp.status
if status_code >= 300:
LOG.error('HTTP connection exception: '
'Response %s - Response Code %s\n%s',
resp_read, status_code, traceback.format_exc())
raise httplib.HTTPException('Failed to authenticate %s'
% status_code)
LOG.debug('Connection successful MSG: %s - STATUS: %s', resp.reason,
resp.status)
return resp_read
finally:
conn.close()
def post_request(url, headers, body=None, rpath=None):
"""Perform HTTP(s) POST request based on Provided Params.
:param url:
:param rpath:
:param headers:
:param body:
:return resp:
"""
try:
if rpath is not None:
_url = urlparse.urljoin(urlparse.urlunparse(url), rpath)
else:
_url = urlparse.urlunparse(url)
resp = requests.post(_url, data=body, headers=headers)
except Exception as exp:
LOG.error('Not able to perform Request ERROR: %s', exp)
raise AttributeError("Failure to perform Authentication %s ERROR:\n%s"
% (exp, traceback.format_exc()))
else:
return resp
def authenticate():
"""Authentication For Openstack API.
Pulls the full Openstack Service Catalog Credentials are the Users API
Username and Key/Password "osauth" has a Built in Rackspace Method for
Authentication
Set a DC Endpoint and Authentication URL for the OpenStack environment
"""
# Setup the request variables
url = auth.parse_region()
a_url = http.parse_url(url=url, auth=True)
auth_json = auth.parse_reqtype()
# remove the prefix for the Authentication URL if Found
LOG.debug('POST == REQUEST DICT > JSON DUMP %s', auth_json)
auth_json_req = json.dumps(auth_json)
headers = {'Content-Type': 'application/json'}
# Send Request
try:
auth_resp = http.post_request(
url=a_url, headers=headers, body=auth_json_req
)
if auth_resp.status_code >= 300:
raise SystemExit(
'Authentication Failure, %s %s' % (auth_resp.status_code,
auth_resp.reason)
)
except ValueError as exp:
LOG.error('Authentication Failure %s\n%s', exp, traceback.format_exc())
raise turbo.SystemProblem('JSON Decode Failure. ERROR: %s' % exp)
else:
LOG.debug('POST Authentication Response %s', auth_resp.json())
auth_info = auth.parse_auth_response(auth_resp.json())
token, tenant, user, inet, enet, cnet, acfep = auth_info
report.reporter(
msg=('API Access Granted. TenantID: %s Username: %s'
% (tenant, user)),
prt=False,
log=True
)
return token, tenant, user, inet, enet, cnet, a_url, acfep
def authenticate():
"""Authentication For Openstack API.
Pulls the full Openstack Service Catalog Credentials are the Users API
Username and Key/Password "osauth" has a Built in Rackspace Method for
Authentication
Set a DC Endpoint and Authentication URL for the OpenStack environment
:param auth_dict: required parameters are auth_url
"""
# Setup the request variables
url, rax = auth.parse_region()
a_url = http.parse_url(url=url, auth=True)
auth_json = auth.parse_reqtype()
# remove the prefix for the Authentication URL if Found
LOG.debug('POST == REQUEST DICT > JSON DUMP %s', auth_json)
auth_json_req = json.dumps(auth_json)
headers = {'Content-Type': 'application/json'}
# Send Request
request = ('POST', a_url.path, auth_json_req, headers)
resp_read = auth.request_process(aurl=a_url, req=request)
LOG.debug('POST Authentication Response %s', resp_read)
try:
auth_resp = json.loads(resp_read)
except ValueError as exp:
LOG.error('Authentication Failure %s\n%s', exp, traceback.format_exc())
raise turbo.SystemProblem('JSON Decode Failure. ERROR: %s - RESP %s' %
(exp, resp_read))
else:
auth_info = auth.parse_auth_response(auth_resp)
token, tenant, user, inet, enet, cnet, acfep = auth_info
report.reporter(msg=('API Access Granted. TenantID: %s Username: %s' %
(tenant, user)),
prt=False,
log=True)
return token, tenant, user, inet, enet, cnet, a_url, acfep
def authenticate():
"""Authentication For Openstack API.
Pulls the full Openstack Service Catalog Credentials are the Users API
Username and Key/Password "osauth" has a Built in Rackspace Method for
Authentication
Set a DC Endpoint and Authentication URL for the OpenStack environment
"""
# Setup the request variables
url = auth.parse_region()
a_url = http.parse_url(url=url, auth=True)
auth_json = auth.parse_reqtype()
# remove the prefix for the Authentication URL if Found
LOG.debug('POST == REQUEST DICT > JSON DUMP %s', auth_json)
auth_json_req = json.dumps(auth_json)
headers = {'Content-Type': 'application/json'}
# Send Request
try:
auth_resp = http.post_request(url=a_url,
headers=headers,
body=auth_json_req)
if auth_resp.status_code >= 300:
raise SystemExit('Authentication Failure, %s %s' %
(auth_resp.status_code, auth_resp.reason))
except ValueError as exp:
LOG.error('Authentication Failure %s\n%s', exp, traceback.format_exc())
raise turbo.SystemProblem('JSON Decode Failure. ERROR: %s' % exp)
else:
LOG.debug('POST Authentication Response %s', auth_resp.json())
auth_info = auth.parse_auth_response(auth_resp.json())
token, tenant, user, inet, enet, cnet, acfep = auth_info
report.reporter(msg=('API Access Granted. TenantID: %s Username: %s' %
(tenant, user)),
prt=False,
log=True)
return token, tenant, user, inet, enet, cnet, a_url, acfep
def request_process(aurl, req):
"""Perform HTTP(s) request based on Provided Params.
:param aurl:
:param req:
:param https:
:return read_resp:
"""
conn = http.open_connection(url=aurl)
# Make the request for authentication
try:
_method, _url, _body, _headers = req
conn.request(method=_method, url=_url, body=_body, headers=_headers)
resp = conn.getresponse()
except Exception as exc:
LOG.error('Not able to perform Request ERROR: %s', exc)
raise AttributeError(
"Failure to perform Authentication %s ERROR:\n%s" %
(exc, traceback.format_exc()))
else:
resp_read = resp.read()
status_code = resp.status
if status_code >= 300:
LOG.error(
'HTTP connection exception: '
'Response %s - Response Code %s\n%s', resp_read, status_code,
traceback.format_exc())
raise httplib.HTTPException('Failed to authenticate %s' %
status_code)
LOG.debug('Connection successful MSG: %s - STATUS: %s', resp.reason,
resp.status)
return resp_read
finally:
conn.close()
def start(self):
"""Retrieve a long list of all files in a container."""
# Package up the Payload
payload = http.prep_payload(
auth=self.auth,
container=ARGS.get('container'),
source=ARGS.get('source'),
args=ARGS
)
self.go = actions.CloudActions(payload=payload)
self.action = getattr(self.go, 'object_lister')
LOG.info('Attempting Download of Remote path %s', payload['c_name'])
if ARGS.get('verbose'):
LOG.info(
'Accessing API for a list of Objects in %s', payload['c_name']
)
report.reporter(
msg='PAYLOAD\t: "%s"' % payload,
log=True,
lvl='debug',
prt=False
)
report.reporter(msg='getting file list')
with multi.spinner():
# Get all objects in a Container
objects, list_count, last_obj = self.action(
url=payload['url'],
container=payload['c_name'],
last_obj=ARGS.get('index_from')
)
if ARGS.get('pattern_match'):
objects = basic.match_filter(
idx_list=objects,
pattern=ARGS['pattern_match'],
dict_type=True
)
# Count the number of objects returned.
if objects is False:
report.reporter(msg='No Container found.')
return
elif objects is not None:
num_files = len(objects)
if num_files < 1:
report.reporter(msg='No Objects found.')
return
else:
report.reporter(msg='No Objects found.')
return
# Get The rate of concurrency
concurrency = multi.set_concurrency(args=ARGS,
file_count=num_files)
# Load the queue
obj_list = [obj['name'] for obj in objects if obj.get('name')]
report.reporter(msg='Building Directory Structure.')
with multi.spinner():
if ARGS.get('object'):
obj_names = ARGS.get('object')
obj_list = [obj for obj in obj_list if obj in obj_names]
num_files = len(obj_list)
elif ARGS.get('dir'):
objpath = ARGS.get('dir')
obj_list = [obj for obj in obj_list if obj.startswith(objpath)]
num_files = len(obj_list)
# from objects found set a unique list of directories
unique_dirs = basic.set_unique_dirs(object_list=obj_list,
root_dir=payload['source'])
for udir in unique_dirs:
basic.mkdir_p(path=udir)
kwargs = {'url': payload['url'],
'container': payload['c_name'],
'source': payload['source'],
'cf_job': getattr(self.go, 'object_downloader')}
report.reporter(msg='Performing Object Download.')
multi.job_processer(
num_jobs=num_files,
objects=obj_list,
job_action=multi.doerator,
concur=concurrency,
kwargs=kwargs
)
if ARGS.get('max_jobs') is not None:
report.reporter(
msg=('This is the last object downloaded. [ %s ]'
% last_obj),
log=True
)
def parse_auth_response(auth_response):
"""Parse the auth response and return the tenant, token, and username.
:param auth_response: the full object returned from an auth call
:returns: tuple (token, tenant, username, internalurl, externalurl, cdnurl)
"""
def _service_ep(scat, types_list):
for srv in scat:
if srv.get('name') in types_list:
index_id = types_list.index(srv.get('name'))
index = types_list[index_id]
if srv.get('name') == index:
return srv.get('endpoints')
else:
return None
access = auth_response.get('access')
token = access.get('token').get('id')
if 'tenant' in access.get('token'):
tenant = access.get('token').get('tenant').get('name')
user = access.get('user').get('name')
elif 'user' in access:
tenant = None
user = access.get('user').get('name')
else:
LOG.error('No Token Found to Parse.\nHere is the DATA: %s\n%s',
auth_response, traceback.format_exc())
raise turbo.NoTenantIdFound('When attempting to grab the '
'tenant or user nothing was found.')
if ARGS.get('os_rax_auth') is not None:
region = ARGS.get('os_rax_auth')
elif ARGS.get('os_hp_auth') is not None:
if ARGS.get('os_tenant') is None:
raise turbo.NoTenantIdFound(
'You need to have a tenant set to use HP Cloud'
)
region = ARGS.get('os_hp_auth')
elif ARGS.get('os_region') is not None:
region = ARGS.get('os_region')
else:
raise turbo.SystemProblem('No Region Set')
scat = access.pop('serviceCatalog')
cfl = _service_ep(scat, info.__srv_types__)
cdn = _service_ep(scat, info.__cdn_types__)
if cfl is not None:
inet = get_surl(region=region, cf_list=cfl, lookup='internalURL')
enet = get_surl(region=region, cf_list=cfl, lookup='publicURL')
else:
need_tenant = ' Maybe you need to specify "os-tenant"?'
gen_message = ('No Service Endpoints were found for use with Swift.'
' If you have Swift available to you,'
' Check Your Credentials and/or Swift\'s availability'
' using Token Auth.')
if ARGS.get('os_tenant') is None:
gen_message += need_tenant
raise turbo.SystemProblem(gen_message)
if cdn is not None:
cnet = get_surl(region=region, cf_list=cdn, lookup='publicURL')
else:
cnet = None
return token, tenant, user, inet, enet, cnet, cfl
def start(self):
"""Retrieve a long list of all files in a container."""
# Package up the Payload
payload = http.prep_payload(auth=self.auth,
container=ARGS.get('container'),
source=ARGS.get('source'),
args=ARGS)
self.go = actions.CloudActions(payload=payload)
self.action = getattr(self.go, 'object_lister')
LOG.info('Attempting Download of Remote path %s', payload['c_name'])
if ARGS.get('verbose'):
LOG.info('Accessing API for a list of Objects in %s',
payload['c_name'])
report.reporter(msg='PAYLOAD\t: "%s"' % payload,
log=True,
lvl='debug',
prt=False)
report.reporter(msg='getting file list')
with multi.spinner():
# Get all objects in a Container
objects, list_count, last_obj = self.action(
url=payload['url'], container=payload['c_name'])
if ARGS.get('pattern_match'):
objects = basic.match_filter(idx_list=objects,
pattern=ARGS['pattern_match'],
dict_type=True)
# Count the number of objects returned.
if objects is False:
report.reporter(msg='No Container found.')
return
elif objects is not None:
num_files = len(objects)
if num_files < 1:
report.reporter(msg='No Objects found.')
return
else:
report.reporter(msg='No Objects found.')
return
# Get The rate of concurrency
concurrency = multi.set_concurrency(args=ARGS,
file_count=num_files)
# Load the queue
obj_list = [obj['name'] for obj in objects]
report.reporter(msg='Building Directory Structure.')
with multi.spinner():
if ARGS.get('object'):
obj_names = ARGS.get('object')
obj_list = [obj for obj in obj_list if obj in obj_names]
num_files = len(obj_list)
elif ARGS.get('dir'):
objpath = ARGS.get('dir')
obj_list = [obj for obj in obj_list if obj.startswith(objpath)]
num_files = len(obj_list)
# from objects found set a unique list of directories
unique_dirs = basic.set_unique_dirs(object_list=obj_list,
root_dir=payload['source'])
for udir in unique_dirs:
basic.mkdir_p(path=udir)
kwargs = {
'url': payload['url'],
'container': payload['c_name'],
'source': payload['source'],
'cf_job': getattr(self.go, 'object_downloader')
}
report.reporter(msg='Performing Object Download.')
multi.job_processer(num_jobs=num_files,
objects=obj_list,
job_action=multi.doerator,
concur=concurrency,
kwargs=kwargs)
def parse_auth_response(auth_response):
"""Parse the auth response and return the tenant, token, and username.
:param auth_response: the full object returned from an auth call
:returns: tuple (token, tenant, username, internalurl, externalurl, cdnurl)
"""
def _service_ep(scat, types_list):
for srv in scat:
if srv.get('name') in types_list:
index_id = types_list.index(srv.get('name'))
index = types_list[index_id]
if srv.get('name') == index:
return srv.get('endpoints')
else:
return None
access = auth_response.get('access')
token = access.get('token').get('id')
if 'tenant' in access.get('token'):
tenant = access.get('token').get('tenant').get('name')
user = access.get('user').get('name')
elif 'user' in access:
tenant = None
user = access.get('user').get('name')
else:
LOG.error('No Token Found to Parse.\nHere is the DATA: %s\n%s',
auth_response, traceback.format_exc())
raise turbo.NoTenantIdFound('When attempting to grab the '
'tenant or user nothing was found.')
if ARGS.get('os_rax_auth') is not None:
region = ARGS.get('os_rax_auth')
elif ARGS.get('os_hp_auth') is not None:
if ARGS.get('os_tenant') is None:
raise turbo.NoTenantIdFound(
'You need to have a tenant set to use HP Cloud')
region = ARGS.get('os_hp_auth')
elif ARGS.get('os_region') is not None:
region = ARGS.get('os_region')
else:
raise turbo.SystemProblem('No Region Set')
scat = access.pop('serviceCatalog')
cfl = _service_ep(scat, info.__srv_types__)
cdn = _service_ep(scat, info.__cdn_types__)
if cfl is not None:
inet = get_surl(region=region, cf_list=cfl, lookup='internalURL')
enet = get_surl(region=region, cf_list=cfl, lookup='publicURL')
else:
need_tenant = ' Maybe you need to specify "os-tenant"?'
gen_message = ('No Service Endpoints were found for use with Swift.'
' If you have Swift available to you,'
' Check Your Credentials and/or Swift\'s availability'
' using Token Auth.')
if ARGS.get('os_tenant') is None:
gen_message += need_tenant
raise turbo.SystemProblem(gen_message)
if cdn is not None:
cnet = get_surl(region=region, cf_list=cdn, lookup='publicURL')
else:
cnet = None
return token, tenant, user, inet, enet, cnet, cfl
