def _parse_v2_region(): """Pull region/auth url information from context.""" if ARGS.get('os_rax_auth'): region = ARGS.get('os_rax_auth') auth_url = 'identity.api.rackspacecloud.com/v2.0/tokens' if region is 'LON': return ARGS.get('os_auth_url', 'https://lon.%s' % auth_url) elif region.lower() in info.__rax_regions__: return ARGS.get('os_auth_url', 'https://%s' % auth_url) else: raise turbo.SystemProblem('No Known RAX Region Was Specified') elif ARGS.get('os_hp_auth'): region = ARGS.get('os_hp_auth') auth_url = 'https://%s.identity.hpcloudsvc.com:35357/v2.0/tokens' if region.lower() in info.__hpc_regions__: return ARGS.get('os_auth_url', auth_url % region) else: raise turbo.SystemProblem('No Known HP Region Was Specified') elif 'os_auth_url' in ARGS: auth_url = ARGS.get('os_auth_url') if not auth_url.endswith('/tokens'): auth_url = '%s/tokens' % auth_url return auth_url else: raise turbo.SystemProblem( 'You Are required to specify an Auth URL, Region or Plugin')
def resp_exception(self, resp, rty): """If we encounter an exception in our upload. we will look at how we can attempt to resolve the exception. :param resp: :param rty: """ try: if resp.status == 401: report.reporter( msg='MESSAGE: Forced Re-authentication is happening.', lvl='error', log=True) basic.stupid_hack() self.payload['headers']['X-Auth-Token'] = auth.get_new_token() rty() elif resp.status == 404: report.reporter( msg=('Not found STATUS: %s, REASON: %s, MESSAGE: %s' % (resp.status, resp.reason, resp.msg)), prt=False, lvl='debug') elif resp.status == 413: _di = dict(resp.getheaders()) basic.stupid_hack(wait=_di.get('retry_after', 10)) raise turbo.SystemProblem( 'The System encountered an API limitation and will' ' continue in "%s" Seconds' % _di.get('retry_after')) elif resp.status == 502: raise turbo.SystemProblem('Failure making Connection') elif resp.status == 503: basic.stupid_hack(wait=10) raise turbo.SystemProblem('SWIFT-API FAILURE') elif resp.status == 504: basic.stupid_hack(wait=10) raise turbo.SystemProblem('Gateway Time-out') elif resp.status >= 300: raise turbo.SystemProblem('SWIFT-API FAILURE -> REQUEST') except turbo.SystemProblem as exp: report.reporter( msg=('FAIL-MESSAGE %s FAILURE STATUS %s FAILURE REASON %s ' 'TYPE %s MESSAGE %s' % (exp, resp.status, resp.reason, resp._method, resp.msg)), prt=True, lvl='warn', log=True) rty()
def parse_region(): """Pull region/auth url information from context.""" base_auth_url = 'identity.api.rackspacecloud.com/v2.0/tokens' if ARGS.get('os_region'): region = ARGS.get('os_region') elif ARGS.get('os_rax_auth'): region = ARGS.get('os_rax_auth') else: raise turbo.SystemProblem('You Are required to specify a REGION') if region is 'LON': return ARGS.get('os_auth_url', 'lon.%s' % base_auth_url), True elif region.lower() in info.__rax_regions__: return ARGS.get('os_auth_url', '%s' % base_auth_url), True else: if ARGS.get('os_auth_url'): if 'racksapce' in ARGS.get('os_auth_url'): return ARGS.get('os_auth_url', '%s' % base_auth_url), True else: return ARGS.get('os_auth_url'), False else: LOG.error('FAILURE: No Region Found. ARGS DUMP:\t %s', ARGS) raise turbo.AuthenticationProblem('You Are required to specify a' ' REGION and an AUTHURL')
def resp_exception(self, resp): """If we encounter an exception in our upload. we will look at how we can attempt to resolve the exception. :param resp: """ # Check to make sure we have all the bits needed if not hasattr(resp, 'status_code'): raise turbo.SystemProblem('No Status to check.') elif resp is None: raise turbo.SystemProblem('No response information.') elif resp.status_code == 401: report.reporter( msg=('Turbolift experienced an Authentication issue.' ' STATUS %s REASON %s REQUEST %s. Turbolift will retry' % (resp.status_code, resp.reason, resp.request)), lvl='warn', log=True, prt=False) # This was done in this manor due to how manager dicts are proxied # related : http://bugs.python.org/issue6766 headers = self.payload['headers'] headers['X-Auth-Token'] = get_new_token() self.payload['headers'] = headers raise turbo.AuthenticationProblem( 'Attempting to resolve the Authentication issue.') elif resp.status_code == 404: report.reporter( msg=('Not found STATUS: %s, REASON: %s, MESSAGE: %s' % (resp.status_code, resp.reason, resp.request)), prt=False, lvl='debug') elif resp.status_code == 413: _di = resp.headers basic.stupid_hack(wait=_di.get('retry_after', 10)) raise turbo.SystemProblem( 'The System encountered an API limitation and will' ' continue in "%s" Seconds' % _di.get('retry_after')) elif resp.status_code == 502: raise turbo.SystemProblem('Failure making Connection') elif resp.status_code == 503: basic.stupid_hack(wait=10) raise turbo.SystemProblem('SWIFT-API FAILURE') elif resp.status_code == 504: basic.stupid_hack(wait=10) raise turbo.SystemProblem('Gateway Time-out') elif resp.status_code >= 300: raise turbo.SystemProblem( 'SWIFT-API FAILURE -> REASON %s REQUEST %s' % (resp.reason, resp.request)) else: report.reporter( msg=('MESSAGE %s %s %s' % (resp.status_code, resp.reason, resp.request)), prt=False, lvl='debug')
def get_local_files(): """Find all files specified in the "source" path. This creates a list for all of files using the full path. """ def not_list(item): """Exclude items. :param item: :return True|False: """ if all([not os.path.islink(item), not os.path.ismount(item)]): if not os.path.getsize(item) > 4831838208: return True else: return False def indexer(location): """Return a list of indexed files. :param location: :return: """ _location = basic.real_full_path(location.encode('utf8')) if os.path.isdir(_location): r_walk = os.walk(_location) indexes = [(root, fls) for root, sfs, fls in r_walk] return [ basic.jpath(root=inx[0], inode=inode) for inx in indexes for inode in inx[1] ] elif os.path.isfile(_location): return [_location] else: raise turbo.NoFileProvided('No Path was Found for %s' % _location) try: d_paths = ARGS.get('source') if not isinstance(d_paths, list): d_paths = [d_paths] # Local Index Path c_index = [indexer(location=d_path) for d_path in d_paths] # make sure my files are only files, and compare it with the not_list f_index = [ item for subl in c_index for item in subl if not_list(item=item) ] except Exception as exp: raise turbo.SystemProblem('Died for some reason. MESSAGE:\t%s' % exp) else: report.reporter(msg='FILE LIST:\t%s' % f_index, lvl='debug', prt=False) return f_index
def authenticate(): """Authentication For Openstack API. Pulls the full Openstack Service Catalog Credentials are the Users API Username and Key/Password "osauth" has a Built in Rackspace Method for Authentication Set a DC Endpoint and Authentication URL for the OpenStack environment """ # Setup the request variables a_url = "https://zebra.zerovm.org/auth/v1.0" #a_url = http.parse_url(url=url, auth=True) auth_json = auth.parse_reqtype() print auth_json # remove the prefix for the Authentication URL if Found # LOG.debug('POST == REQUEST DICT > JSON DUMP %s', auth_json) # auth_json_req = json.dumps(auth_json) headers = { 'Content-Type': 'application/json', "X-Auth-User": auth_json['auth']['passwordCredentials']['username'], "X-Auth-Key": auth_json['auth']['passwordCredentials']['password'] } # Send Request try: auth_resp = requests.get(url=a_url, headers=headers) if auth_resp.status_code >= 300: raise SystemExit('Authentication Failure, %s %s' % (auth_resp.status_code, auth_resp.reason)) except ValueError as exp: LOG.error('Authentication Failure %s\n%s', exp, traceback.format_exc()) raise turbo.SystemProblem('JSON Decode Failure. ERROR: %s' % exp) else: LOG.debug('POST Authentication Response %s', auth_resp.json()) #auth_info = auth.parse_auth_response(auth_resp.json()) #token, tenant, user, inet, enet, cnet, acfep = auth_info token = auth_resp.headers['x-auth-token'] tenant, user = auth_json['auth']['passwordCredentials'][ 'username'].split(":") inet = urlparse.urlparse(auth_resp.headers['x-storage-url']) enet = inet cnet = None acfep = inet report.reporter(msg=('API Access Granted. TenantID: %s Username: %s' % (tenant, user)), prt=False, log=True) return token, tenant, user, inet, enet, cnet, urlparse.urlparse( a_url), acfep
def get_surl(region, cf_list, lookup): """Lookup a service URL. :param region: :param cf_list: :param lookup: :return net: """ for srv in cf_list: if region in srv.get('region'): net = http.parse_url(url=srv.get(lookup)) return net else: raise turbo.SystemProblem( 'Region "%s" was not found in your Service Catalog.' % region)
def authenticate(): """Authentication For Openstack API. Pulls the full Openstack Service Catalog Credentials are the Users API Username and Key/Password "osauth" has a Built in Rackspace Method for Authentication Set a DC Endpoint and Authentication URL for the OpenStack environment """ # Setup the request variables url = auth.parse_region() LOG.debug('Raw Auth URL: [ %s ]', url) a_url = http.parse_url(url=url, auth=True) headers = { 'Content-Type': 'application/json', 'Accept': 'application/json' } headers.update(auth.get_headers() or {}) auth_json = auth.parse_reqtype() or {} LOG.debug('Parsed Auth URL: [ %s ]', a_url) # remove the prefix for the Authentication URL if Found auth_json_req = json.dumps(auth_json) LOG.debug('Request JSON: [ %s ]', auth_json_req) LOG.debug('Request Headers: [ %s ]', headers) # Send Request try: auth_resp = auth.auth_request(a_url, headers=headers, body=auth_json_req) if auth_resp.status_code >= 300: raise SystemExit('Authentication Failure, %s %s' % (auth_resp.status_code, auth_resp.reason)) except ValueError as exp: LOG.error('Authentication Failure %s\n%s', exp, traceback.format_exc()) raise turbo.SystemProblem('JSON Decode Failure. ERROR: %s' % exp) else: auth_info = auth.parse_auth_response(auth_resp) token, tenant, user, inet, enet, cnet, acfep = auth_info report.reporter(msg=('API Access Granted. TenantID: %s Username: %s' % (tenant, user)), prt=False, log=True) return token, tenant, user, inet, enet, cnet, a_url, acfep
def parse_auth_response(auth_response): """Parse the auth response and return the tenant, token, and username. :param auth_response: the full object returned from an auth call :returns: tuple (token, tenant, username, internalurl, externalurl, cdnurl) """ access = auth_response.get('access') token = access.get('token').get('id') if 'tenant' in access.get('token'): tenant = access.get('token').get('tenant').get('name') user = access.get('user').get('name') elif 'user' in access: tenant = None user = access.get('user').get('name') else: LOG.error('No Token Found to Parse.\nHere is the DATA: %s\n%s', auth_response, traceback.format_exc()) raise turbo.NoTenantIdFound('When attempting to grab the ' 'tenant or user nothing was found.') scat = access.pop('serviceCatalog') for srv in scat: if srv.get('name') in info.__srv_types__: if srv.get('name') == 'cloudFilesCDN': cdn = srv.get('endpoints') if srv.get('name') == ARGS.get('service_type'): cfl = srv.get('endpoints') if ARGS.get('os_region') is not None: region = ARGS.get('os_region') elif ARGS.get('os_rax_auth') is not None: region = ARGS.get('os_rax_auth') else: raise turbo.SystemProblem('No Region Set') if cfl is not None: inet = get_surl(region=region, cf_list=cfl, lookup='internalURL') enet = get_surl(region=region, cf_list=cfl, lookup='publicURL') if cdn is not None: cnet = get_surl(region=region, cf_list=cdn, lookup='publicURL') return token, tenant, user, inet, enet, cnet, cfl
def bcolors(msg, color): """return a colorizes string. :param msg: :param color: :return str: """ # Available Colors colors = {'debug': '\033[94m', 'info': '\033[92m', 'warn': '\033[93m', 'error': '\033[91m', 'critical': '\033[95m', 'ENDC': '\033[0m'} if color in colors: return '%s%s%s' % (colors[color], msg, colors['ENDC']) else: raise turbo.SystemProblem('"%s" was not a known color.' % color)
def get_surl(region, cf_list, lookup): """Lookup a service URL. :param region: :param cf_list: :param lookup: :return net: """ for srv in cf_list: region_get = srv.get('region') lookup_get = srv.get(lookup) if any([region in region_get, region.lower() in region_get]): if lookup_get is None: return None else: return http.parse_url(url=lookup_get) else: raise turbo.SystemProblem( 'Region "%s" was not found in your Service Catalog.' % region)
def authenticate(): """Authentication For Openstack API. Pulls the full Openstack Service Catalog Credentials are the Users API Username and Key/Password "osauth" has a Built in Rackspace Method for Authentication Set a DC Endpoint and Authentication URL for the OpenStack environment :param auth_dict: required parameters are auth_url """ # Setup the request variables url, rax = auth.parse_region() a_url = http.parse_url(url=url, auth=True) auth_json = auth.parse_reqtype() # remove the prefix for the Authentication URL if Found LOG.debug('POST == REQUEST DICT > JSON DUMP %s', auth_json) auth_json_req = json.dumps(auth_json) headers = {'Content-Type': 'application/json'} # Send Request request = ('POST', a_url.path, auth_json_req, headers) resp_read = auth.request_process(aurl=a_url, req=request) LOG.debug('POST Authentication Response %s', resp_read) try: auth_resp = json.loads(resp_read) except ValueError as exp: LOG.error('Authentication Failure %s\n%s', exp, traceback.format_exc()) raise turbo.SystemProblem('JSON Decode Failure. ERROR: %s - RESP %s' % (exp, resp_read)) else: auth_info = auth.parse_auth_response(auth_resp) token, tenant, user, inet, enet, cnet, acfep = auth_info report.reporter(msg=('API Access Granted. TenantID: %s Username: %s' % (tenant, user)), prt=False, log=True) return token, tenant, user, inet, enet, cnet, a_url, acfep
def _parse_v2_auth_response(auth_response): """Parse the auth response and return the tenant, token, and username. :param auth_response: the full object returned from an auth call :returns: tuple (token, tenant, username, internalurl, externalurl, cdnurl) """ try: auth_response = auth_response.json() except AttributeError: pass else: LOG.debug('Authentication Response Body %s', auth_response) def _service_ep(scat, types_list): for srv in scat: if srv.get('name') in types_list: index_id = types_list.index(srv.get('name')) index = types_list[index_id] if srv.get('name') == index: return srv.get('endpoints') else: return None access = auth_response.get('access') token = access.get('token').get('id') if 'tenant' in access.get('token'): tenant = access.get('token').get('tenant').get('name') user = access.get('user').get('name') elif 'user' in access: tenant = None user = access.get('user').get('name') else: LOG.error('No Token Found to Parse.\nHere is the DATA: %s\n%s', auth_response, traceback.format_exc()) raise turbo.NoTenantIdFound('When attempting to grab the ' 'tenant or user nothing was found.') if ARGS.get('os_rax_auth') is not None: region = ARGS.get('os_rax_auth') elif ARGS.get('os_hp_auth') is not None: if ARGS.get('os_tenant') is None: raise turbo.NoTenantIdFound( 'You need to have a tenant set to use HP Cloud') region = ARGS.get('os_hp_auth') elif ARGS.get('os_region') is not None: region = ARGS.get('os_region') else: raise turbo.SystemProblem('No Region Set') scat = access.pop('serviceCatalog') cfl = _service_ep(scat, info.__srv_types__) cdn = _service_ep(scat, info.__cdn_types__) if cfl is not None: inet = get_surl(region=region, cf_list=cfl, lookup='internalURL') enet = get_surl(region=region, cf_list=cfl, lookup='publicURL') else: need_tenant = ' Maybe you need to specify "os-tenant"?' gen_message = ('No Service Endpoints were found for use ' 'with Swift. If you have Swift available to you, ' 'Check Your Credentials and/or Swift\'s ' 'availability using Token Auth.') if ARGS.get('os_tenant') is None: gen_message += need_tenant raise turbo.SystemProblem(gen_message) if cdn is not None: cnet = get_surl(region=region, cf_list=cdn, lookup='publicURL') else: cnet = None return token, tenant, user, inet, enet, cnet, cfl
def get_local_files(): """Find all files specified in the "source" path. This creates a list for all of files using the full path. """ def not_list(item): """Exclude items. :param item: :return True|False: """ if os.path.islink(item): dest = os.readlink(item) if not os.path.exists(dest): return True elif not os.path.ismount(item): if not os.path.getsize(item) > 4831838208: return True return False def indexer(location): """Return a list of indexed files. :param location: :return: """ _location = basic.real_full_path(location.encode('utf8')) if os.path.isdir(_location): r_walk = os.walk(_location) objects = list() for root_inx, inx in [(root, fls) for root, sfs, fls in r_walk]: for inode in inx: object_path = basic.jpath(root=root_inx, inode=inode) objects.append(unicode(object_path.decode('utf-8'))) else: return objects elif os.path.isfile(_location): return [_location] else: raise turbo.NoFileProvided('No Path was Found for %s' % _location) try: d_paths = ARGS.get('source') if not isinstance(d_paths, list): d_paths = [d_paths] # Local Index Path c_index = [indexer(location=d_path) for d_path in d_paths] # make sure my files are only files, and not in the the not_list f_index = [ item for subl in c_index for item in subl if not_list(item=item) ] if ARGS.get('exclude'): for item in f_index: for exclude in ARGS['exclude']: if exclude in item: try: index = f_index.index(item) except ValueError: pass else: f_index.pop(index) except Exception as exp: raise turbo.SystemProblem('Died for some reason. MESSAGE:\t%s' % exp) else: report.reporter(msg='FILE LIST:\t%s' % f_index, lvl='debug', prt=False) return f_index
def compress_files(file_list): """If the archive function is used, create a compressed archive. :param file_list: This function allows for multiple sources to be added to the compressed archive. """ tmp_file = None try: # Set date and time date_format = '%a%b%d.%H.%M.%S.%Y' today = datetime.datetime.today() _ts = today.strftime(date_format) # Get Home Directory home_dir = os.getenv('HOME') # Set the name of the archive. set_name = ARGS.get('tar_name', '%s_%s' % ('Archive', _ts)) file_name = '%s.tgz' % set_name # Set the working File. tmp_file = basic.jpath(root=home_dir, inode=file_name) # Begin creating the Archive. tar = tarfile.open(tmp_file, 'w:gz') for name in file_list: if basic.file_exists(name) is True: tar.add(name) tar.close() report.reporter(msg='ARCHIVE CREATED: %s' % tmp_file, prt=False) if ARGS.get('verify'): tar_len = tarfile.open(tmp_file, 'r') ver_array = [] for member_info in tar_len.getmembers(): ver_array.append(member_info.name) count = len(ver_array) orig_count = len(file_list) if orig_count != count: raise turbo.SystemProblem( 'ARCHIVE NOT VERIFIED: Archive and File List do not Match.' ' Original File Count = %s, Found Archive Contents = %s' % (orig_count, count)) report.reporter(msg='ARCHIVE CONTENTS VERIFIED: %s files' % count, ) except KeyboardInterrupt: if basic.file_exists(tmp_file): basic.remove_file(tmp_file) turbo.emergency_exit('I have stopped at your command,' ' I removed Local Copy of the Archive') except Exception as exp: if basic.file_exists(tmp_file): basic.remove_file(tmp_file) turbo.emergency_exit( 'I am sorry i just don\'t know what you put into me, Removing' ' Local Copy of the Archive.') turbo.emergency_exit( 'Exception while working on the archive. MESSAGE: %s' % exp) else: return tmp_file