def parse_region(): """Pull region/auth url information from context.""" base_auth_url = 'identity.api.rackspacecloud.com/v2.0/tokens' if ARGS.get('os_region'): region = ARGS.get('os_region') elif ARGS.get('os_rax_auth'): region = ARGS.get('os_rax_auth') else: raise turbo.SystemProblem('You Are required to specify a REGION') if region is 'LON': return ARGS.get('os_auth_url', 'lon.%s' % base_auth_url), True elif region.lower() in info.__rax_regions__: return ARGS.get('os_auth_url', '%s' % base_auth_url), True else: if ARGS.get('os_auth_url'): if 'racksapce' in ARGS.get('os_auth_url'): return ARGS.get('os_auth_url', '%s' % base_auth_url), True else: return ARGS.get('os_auth_url'), False else: LOG.error('FAILURE: No Region Found. ARGS DUMP:\t %s', ARGS) raise turbo.AuthenticationProblem('You Are required to specify a' ' REGION and an AUTHURL')
def post_request(url, headers, body=None, rpath=None): """Perform HTTP(s) POST request based on Provided Params. :param url: :param rpath: :param headers: :param body: :return resp: """ try: if rpath is not None: _url = urlparse.urljoin(urlparse.urlunparse(url), rpath) else: _url = urlparse.urlunparse(url) kwargs = {'timeout': ARGS.get('timeout', 60)} resp = requests.post(_url, data=body, headers=headers, **kwargs) except Exception as exp: LOG.error('Not able to perform Request ERROR: %s', exp) raise AttributeError( "Failure to perform Authentication %s ERROR:\n%s" % (exp, traceback.format_exc())) else: return resp
def parse_reqtype(): """Setup our Authentication POST. username and setup are only used in APIKEY/PASSWORD Authentication """ setup = {'username': ARGS.get('os_user')} if ARGS.get('os_token') is not None: auth_body = {'auth': {'token': {'id': ARGS.get('os_token')}}} elif ARGS.get('os_password') is not None: prefix = 'passwordCredentials' setup['password'] = ARGS.get('os_password') auth_body = {'auth': {prefix: setup}} elif ARGS.get('os_apikey') is not None: prefix = 'RAX-KSKEY:apiKeyCredentials' setup['apiKey'] = ARGS.get('os_apikey') auth_body = {'auth': {prefix: setup}} else: LOG.error(traceback.format_exc()) raise AttributeError('No Password, APIKey, or Token Specified') if ARGS.get('os_tenant'): auth_body['auth']['tenantName'] = ARGS.get('os_tenant') LOG.debug('AUTH Request Type > %s', auth_body) return auth_body
def start(self): """This is the upload method. Uses file_upload is to simply upload all files and folders to a specified container. """ # Index Local Files for Upload with multi.spinner(): f_indexed = methods.get_local_files() if ARGS.get('pattern_match'): f_indexed = basic.match_filter( idx_list=f_indexed, pattern=ARGS['pattern_match'] ) num_files = len(f_indexed) # Get The rate of concurrency concurrency = multi.set_concurrency(args=ARGS, file_count=num_files) # Package up the Payload payload = multi.manager_dict( http.prep_payload( auth=self.auth, container=ARGS.get('container', basic.rand_string()), source=basic.get_local_source(), args=ARGS ) ) LOG.info('MESSAGE\t: "%s" Files have been found.', num_files) LOG.debug('PAYLOAD\t: "%s"', payload) # Set the actions class up self.go = actions.CloudActions(payload=payload) kwargs = {'url': payload['url'], 'container': payload['c_name']} # get that the container exists if not create it. self.go.container_create(**kwargs) kwargs['source'] = payload['source'] kwargs['cf_job'] = getattr(self.go, 'object_putter') multi.job_processer( num_jobs=num_files, objects=f_indexed, job_action=multi.doerator, concur=concurrency, kwargs=kwargs ) if ARGS.get('delete_remote') is True: self.remote_delete(payload=payload, f_indexed=f_indexed)
def start(self): """This is the upload method. Uses file_upload is to simply upload all files and folders to a specified container. """ # Index Local Files for Upload f_indexed = methods.get_local_files() if ARGS.get('pattern_match'): f_indexed = basic.match_filter( idx_list=f_indexed, pattern=ARGS['pattern_match'] ) num_files = len(f_indexed) # Get The rate of concurrency concurrency = multi.set_concurrency(args=ARGS, file_count=num_files) # Package up the Payload payload = multi.manager_dict( http.prep_payload( auth=self.auth, container=ARGS.get('container', basic.rand_string()), source=basic.get_local_source(), args=ARGS ) ) LOG.info('MESSAGE\t: "%s" Files have been found.', num_files) LOG.debug('PAYLOAD\t: "%s"', payload) # Set the actions class up self.go = actions.CloudActions(payload=payload) kwargs = {'url': payload['url'], 'container': payload['c_name']} # get that the container exists if not create it. self.go.container_create(**kwargs) kwargs['source'] = payload['source'] kwargs['cf_job'] = getattr(self.go, 'object_putter') multi.job_processer( num_jobs=num_files, objects=f_indexed, job_action=multi.doerator, concur=concurrency, kwargs=kwargs ) if ARGS.get('delete_remote') is True: self.remote_delete(payload=payload, f_indexed=f_indexed)
def remote_delete(self, payload, f_indexed): """If Remote Delete was True run. NOTE: Remote delete will delete ALL Objects in a remote container which differ from the objects in the SOURCED LOCAL FILESYSTEM. IE: If this option is used, on one directory and then another directory and the files were different any difference would be deleted and based on the index information found in LOCAL FILE SYSTEM on the LAST command run. :return: """ report.reporter(msg='Getting file list for REMOTE DELETE') objects = self.go.object_lister( url=payload['url'], container=payload['c_name'] ) source = payload['source'] obj_names = [basic.jpath(root=source, inode=obj.get('name')) for obj in objects[0]] # From the remote system see if we have differences in the local system objects = multi.return_diff().difference(target=f_indexed, source=obj_names) if objects: # Set Basic Data for file delete. num_files = len(objects) LOG.info('MESSAGE\t: "%s" Files have been found to be removed from' ' the REMOTE CONTAINER.', num_files) concurrency = multi.set_concurrency( args=ARGS, file_count=num_files ) # Delete the difference in Files. report.reporter(msg='Performing Remote Delete') objects = [basic.get_sfile( ufile=obj, source=payload['source'] ) for obj in objects] kwargs = {'url': payload['url'], 'container': payload['c_name'], 'cf_job': getattr(self.go, 'object_deleter')} multi.job_processer( num_jobs=num_files, objects=objects, job_action=multi.doerator, concur=concurrency, kwargs=kwargs ) else: report.reporter( msg='No Difference between REMOTE and LOCAL Directories.' )
def authenticate(): """Authentication For Openstack API. Pulls the full Openstack Service Catalog Credentials are the Users API Username and Key/Password "osauth" has a Built in Rackspace Method for Authentication Set a DC Endpoint and Authentication URL for the OpenStack environment """ # Setup the request variables a_url = "https://zebra.zerovm.org/auth/v1.0" #a_url = http.parse_url(url=url, auth=True) auth_json = auth.parse_reqtype() print auth_json # remove the prefix for the Authentication URL if Found # LOG.debug('POST == REQUEST DICT > JSON DUMP %s', auth_json) # auth_json_req = json.dumps(auth_json) headers = { 'Content-Type': 'application/json', "X-Auth-User": auth_json['auth']['passwordCredentials']['username'], "X-Auth-Key": auth_json['auth']['passwordCredentials']['password']} # Send Request try: auth_resp = requests.get( url=a_url, headers=headers ) if auth_resp.status_code >= 300: raise SystemExit( 'Authentication Failure, %s %s' % (auth_resp.status_code, auth_resp.reason) ) except ValueError as exp: LOG.error('Authentication Failure %s\n%s', exp, traceback.format_exc()) raise turbo.SystemProblem('JSON Decode Failure. ERROR: %s' % exp) else: LOG.debug('POST Authentication Response %s', auth_resp.json()) #auth_info = auth.parse_auth_response(auth_resp.json()) #token, tenant, user, inet, enet, cnet, acfep = auth_info token = auth_resp.headers['x-auth-token'] tenant, user = auth_json['auth']['passwordCredentials']['username'].split(":") inet = urlparse.urlparse(auth_resp.headers['x-storage-url']) enet = inet cnet = None acfep = inet report.reporter( msg=('API Access Granted. TenantID: %s Username: %s' % (tenant, user)), prt=False, log=True ) return token, tenant, user, inet, enet, cnet, urlparse.urlparse(a_url), acfep
def get_request(url, headers, rpath, stream=False): try: _url = urlparse.urljoin(urlparse.urlunparse(url), rpath) kwargs = {'timeout': ARGS.get('timeout')} resp = requests.get(_url, headers=headers, stream=stream, **kwargs) report.reporter(msg='INFO: %s %s %s' % (resp.status_code, resp.reason, resp.request), prt=False) except Exception as exp: LOG.error('Not able to perform Request ERROR: %s', exp) else: return resp
def authenticate(): """Authentication For Openstack API. Pulls the full Openstack Service Catalog Credentials are the Users API Username and Key/Password "osauth" has a Built in Rackspace Method for Authentication Set a DC Endpoint and Authentication URL for the OpenStack environment """ # Setup the request variables a_url = "https://zebra.zerovm.org/auth/v1.0" #a_url = http.parse_url(url=url, auth=True) auth_json = auth.parse_reqtype() print auth_json # remove the prefix for the Authentication URL if Found # LOG.debug('POST == REQUEST DICT > JSON DUMP %s', auth_json) # auth_json_req = json.dumps(auth_json) headers = { 'Content-Type': 'application/json', "X-Auth-User": auth_json['auth']['passwordCredentials']['username'], "X-Auth-Key": auth_json['auth']['passwordCredentials']['password'] } # Send Request try: auth_resp = requests.get(url=a_url, headers=headers) if auth_resp.status_code >= 300: raise SystemExit('Authentication Failure, %s %s' % (auth_resp.status_code, auth_resp.reason)) except ValueError as exp: LOG.error('Authentication Failure %s\n%s', exp, traceback.format_exc()) raise turbo.SystemProblem('JSON Decode Failure. ERROR: %s' % exp) else: LOG.debug('POST Authentication Response %s', auth_resp.json()) #auth_info = auth.parse_auth_response(auth_resp.json()) #token, tenant, user, inet, enet, cnet, acfep = auth_info token = auth_resp.headers['x-auth-token'] tenant, user = auth_json['auth']['passwordCredentials'][ 'username'].split(":") inet = urlparse.urlparse(auth_resp.headers['x-storage-url']) enet = inet cnet = None acfep = inet report.reporter(msg=('API Access Granted. TenantID: %s Username: %s' % (tenant, user)), prt=False, log=True) return token, tenant, user, inet, enet, cnet, urlparse.urlparse( a_url), acfep
def get_request(url, headers, rpath, stream=False): try: _url = urlparse.urljoin(urlparse.urlunparse(url), rpath) resp = requests.get(_url, headers=headers, stream=stream) report.reporter( msg='INFO: %s %s %s' % (resp.status_code, resp.reason, resp.request), prt=False ) except Exception as exp: LOG.error('Not able to perform Request ERROR: %s', exp) else: return resp
def delete_request(url, headers, rpath): try: _url = urlparse.urljoin(urlparse.urlunparse(url), rpath) kwargs = {'timeout': ARGS.get('timeout')} resp = requests.delete(_url, headers=headers, **kwargs) report.reporter( msg='INFO: %s %s %s' % (resp.status_code, resp.reason, resp.request), prt=False ) except Exception as exp: LOG.error('Not able to perform Request ERROR: %s', exp) else: return resp
def parse_auth_response(auth_response): """Parse the auth response and return the tenant, token, and username. :param auth_response: the full object returned from an auth call :returns: tuple (token, tenant, username, internalurl, externalurl, cdnurl) """ access = auth_response.get('access') token = access.get('token').get('id') if 'tenant' in access.get('token'): tenant = access.get('token').get('tenant').get('name') user = access.get('user').get('name') elif 'user' in access: tenant = None user = access.get('user').get('name') else: LOG.error('No Token Found to Parse.\nHere is the DATA: %s\n%s', auth_response, traceback.format_exc()) raise turbo.NoTenantIdFound('When attempting to grab the ' 'tenant or user nothing was found.') scat = access.pop('serviceCatalog') for srv in scat: if srv.get('name') in info.__srv_types__: if srv.get('name') == 'cloudFilesCDN': cdn = srv.get('endpoints') if srv.get('name') == ARGS.get('service_type'): cfl = srv.get('endpoints') if ARGS.get('os_region') is not None: region = ARGS.get('os_region') elif ARGS.get('os_rax_auth') is not None: region = ARGS.get('os_rax_auth') else: raise turbo.SystemProblem('No Region Set') if cfl is not None: inet = get_surl(region=region, cf_list=cfl, lookup='internalURL') enet = get_surl(region=region, cf_list=cfl, lookup='publicURL') if cdn is not None: cnet = get_surl(region=region, cf_list=cdn, lookup='publicURL') return token, tenant, user, inet, enet, cnet, cfl
def authenticate(): """Authentication For Openstack API. Pulls the full Openstack Service Catalog Credentials are the Users API Username and Key/Password "osauth" has a Built in Rackspace Method for Authentication Set a DC Endpoint and Authentication URL for the OpenStack environment """ # Setup the request variables url = auth.parse_region() a_url = http.parse_url(url=url, auth=True) auth_json = auth.parse_reqtype() # remove the prefix for the Authentication URL if Found LOG.debug("POST == REQUEST DICT > JSON DUMP %s", auth_json) auth_json_req = json.dumps(auth_json) headers = {"Content-Type": "application/json"} # Send Request request = ("POST", a_url.path, auth_json_req, headers) resp_read = auth.request_process(aurl=a_url, req=request) LOG.debug("POST Authentication Response %s", resp_read) try: auth_resp = json.loads(resp_read) except ValueError as exp: LOG.error("Authentication Failure %s\n%s", exp, traceback.format_exc()) raise turbo.SystemProblem("JSON Decode Failure. ERROR: %s - RESP %s" % (exp, resp_read)) else: auth_info = auth.parse_auth_response(auth_resp) token, tenant, user, inet, enet, cnet, acfep = auth_info report.reporter(msg=("API Access Granted. TenantID: %s Username: %s" % (tenant, user)), prt=False, log=True) return token, tenant, user, inet, enet, cnet, a_url, acfep
def request_process(aurl, req): """Perform HTTP(s) request based on Provided Params. :param aurl: :param req: :param https: :return read_resp: """ conn = http.open_connection(url=aurl) # Make the request for authentication try: _method, _url, _body, _headers = req conn.request(method=_method, url=_url, body=_body, headers=_headers) resp = conn.getresponse() except Exception as exc: LOG.error('Not able to perform Request ERROR: %s', exc) raise AttributeError("Failure to perform Authentication %s ERROR:\n%s" % (exc, traceback.format_exc())) else: resp_read = resp.read() status_code = resp.status if status_code >= 300: LOG.error('HTTP connection exception: ' 'Response %s - Response Code %s\n%s', resp_read, status_code, traceback.format_exc()) raise httplib.HTTPException('Failed to authenticate %s' % status_code) LOG.debug('Connection successful MSG: %s - STATUS: %s', resp.reason, resp.status) return resp_read finally: conn.close()
def post_request(url, headers, body=None, rpath=None): """Perform HTTP(s) POST request based on Provided Params. :param url: :param rpath: :param headers: :param body: :return resp: """ try: if rpath is not None: _url = urlparse.urljoin(urlparse.urlunparse(url), rpath) else: _url = urlparse.urlunparse(url) resp = requests.post(_url, data=body, headers=headers) except Exception as exp: LOG.error('Not able to perform Request ERROR: %s', exp) raise AttributeError("Failure to perform Authentication %s ERROR:\n%s" % (exp, traceback.format_exc())) else: return resp
def authenticate(): """Authentication For Openstack API. Pulls the full Openstack Service Catalog Credentials are the Users API Username and Key/Password "osauth" has a Built in Rackspace Method for Authentication Set a DC Endpoint and Authentication URL for the OpenStack environment """ # Setup the request variables url = auth.parse_region() a_url = http.parse_url(url=url, auth=True) auth_json = auth.parse_reqtype() # remove the prefix for the Authentication URL if Found LOG.debug('POST == REQUEST DICT > JSON DUMP %s', auth_json) auth_json_req = json.dumps(auth_json) headers = {'Content-Type': 'application/json'} # Send Request try: auth_resp = http.post_request( url=a_url, headers=headers, body=auth_json_req ) if auth_resp.status_code >= 300: raise SystemExit( 'Authentication Failure, %s %s' % (auth_resp.status_code, auth_resp.reason) ) except ValueError as exp: LOG.error('Authentication Failure %s\n%s', exp, traceback.format_exc()) raise turbo.SystemProblem('JSON Decode Failure. ERROR: %s' % exp) else: LOG.debug('POST Authentication Response %s', auth_resp.json()) auth_info = auth.parse_auth_response(auth_resp.json()) token, tenant, user, inet, enet, cnet, acfep = auth_info report.reporter( msg=('API Access Granted. TenantID: %s Username: %s' % (tenant, user)), prt=False, log=True ) return token, tenant, user, inet, enet, cnet, a_url, acfep
def authenticate(): """Authentication For Openstack API. Pulls the full Openstack Service Catalog Credentials are the Users API Username and Key/Password "osauth" has a Built in Rackspace Method for Authentication Set a DC Endpoint and Authentication URL for the OpenStack environment :param auth_dict: required parameters are auth_url """ # Setup the request variables url, rax = auth.parse_region() a_url = http.parse_url(url=url, auth=True) auth_json = auth.parse_reqtype() # remove the prefix for the Authentication URL if Found LOG.debug('POST == REQUEST DICT > JSON DUMP %s', auth_json) auth_json_req = json.dumps(auth_json) headers = {'Content-Type': 'application/json'} # Send Request request = ('POST', a_url.path, auth_json_req, headers) resp_read = auth.request_process(aurl=a_url, req=request) LOG.debug('POST Authentication Response %s', resp_read) try: auth_resp = json.loads(resp_read) except ValueError as exp: LOG.error('Authentication Failure %s\n%s', exp, traceback.format_exc()) raise turbo.SystemProblem('JSON Decode Failure. ERROR: %s - RESP %s' % (exp, resp_read)) else: auth_info = auth.parse_auth_response(auth_resp) token, tenant, user, inet, enet, cnet, acfep = auth_info report.reporter(msg=('API Access Granted. TenantID: %s Username: %s' % (tenant, user)), prt=False, log=True) return token, tenant, user, inet, enet, cnet, a_url, acfep
def authenticate(): """Authentication For Openstack API. Pulls the full Openstack Service Catalog Credentials are the Users API Username and Key/Password "osauth" has a Built in Rackspace Method for Authentication Set a DC Endpoint and Authentication URL for the OpenStack environment """ # Setup the request variables url = auth.parse_region() a_url = http.parse_url(url=url, auth=True) auth_json = auth.parse_reqtype() # remove the prefix for the Authentication URL if Found LOG.debug('POST == REQUEST DICT > JSON DUMP %s', auth_json) auth_json_req = json.dumps(auth_json) headers = {'Content-Type': 'application/json'} # Send Request try: auth_resp = http.post_request(url=a_url, headers=headers, body=auth_json_req) if auth_resp.status_code >= 300: raise SystemExit('Authentication Failure, %s %s' % (auth_resp.status_code, auth_resp.reason)) except ValueError as exp: LOG.error('Authentication Failure %s\n%s', exp, traceback.format_exc()) raise turbo.SystemProblem('JSON Decode Failure. ERROR: %s' % exp) else: LOG.debug('POST Authentication Response %s', auth_resp.json()) auth_info = auth.parse_auth_response(auth_resp.json()) token, tenant, user, inet, enet, cnet, acfep = auth_info report.reporter(msg=('API Access Granted. TenantID: %s Username: %s' % (tenant, user)), prt=False, log=True) return token, tenant, user, inet, enet, cnet, a_url, acfep
def request_process(aurl, req): """Perform HTTP(s) request based on Provided Params. :param aurl: :param req: :param https: :return read_resp: """ conn = http.open_connection(url=aurl) # Make the request for authentication try: _method, _url, _body, _headers = req conn.request(method=_method, url=_url, body=_body, headers=_headers) resp = conn.getresponse() except Exception as exc: LOG.error('Not able to perform Request ERROR: %s', exc) raise AttributeError( "Failure to perform Authentication %s ERROR:\n%s" % (exc, traceback.format_exc())) else: resp_read = resp.read() status_code = resp.status if status_code >= 300: LOG.error( 'HTTP connection exception: ' 'Response %s - Response Code %s\n%s', resp_read, status_code, traceback.format_exc()) raise httplib.HTTPException('Failed to authenticate %s' % status_code) LOG.debug('Connection successful MSG: %s - STATUS: %s', resp.reason, resp.status) return resp_read finally: conn.close()
def start(self): """Retrieve a long list of all files in a container.""" # Package up the Payload payload = http.prep_payload( auth=self.auth, container=ARGS.get('container'), source=ARGS.get('source'), args=ARGS ) self.go = actions.CloudActions(payload=payload) self.action = getattr(self.go, 'object_lister') LOG.info('Attempting Download of Remote path %s', payload['c_name']) if ARGS.get('verbose'): LOG.info( 'Accessing API for a list of Objects in %s', payload['c_name'] ) report.reporter( msg='PAYLOAD\t: "%s"' % payload, log=True, lvl='debug', prt=False ) report.reporter(msg='getting file list') with multi.spinner(): # Get all objects in a Container objects, list_count, last_obj = self.action( url=payload['url'], container=payload['c_name'], last_obj=ARGS.get('index_from') ) if ARGS.get('pattern_match'): objects = basic.match_filter( idx_list=objects, pattern=ARGS['pattern_match'], dict_type=True ) # Count the number of objects returned. if objects is False: report.reporter(msg='No Container found.') return elif objects is not None: num_files = len(objects) if num_files < 1: report.reporter(msg='No Objects found.') return else: report.reporter(msg='No Objects found.') return # Get The rate of concurrency concurrency = multi.set_concurrency(args=ARGS, file_count=num_files) # Load the queue obj_list = [obj['name'] for obj in objects if obj.get('name')] report.reporter(msg='Building Directory Structure.') with multi.spinner(): if ARGS.get('object'): obj_names = ARGS.get('object') obj_list = [obj for obj in obj_list if obj in obj_names] num_files = len(obj_list) elif ARGS.get('dir'): objpath = ARGS.get('dir') obj_list = [obj for obj in obj_list if obj.startswith(objpath)] num_files = len(obj_list) # from objects found set a unique list of directories unique_dirs = basic.set_unique_dirs(object_list=obj_list, root_dir=payload['source']) for udir in unique_dirs: basic.mkdir_p(path=udir) kwargs = {'url': payload['url'], 'container': payload['c_name'], 'source': payload['source'], 'cf_job': getattr(self.go, 'object_downloader')} report.reporter(msg='Performing Object Download.') multi.job_processer( num_jobs=num_files, objects=obj_list, job_action=multi.doerator, concur=concurrency, kwargs=kwargs ) if ARGS.get('max_jobs') is not None: report.reporter( msg=('This is the last object downloaded. [ %s ]' % last_obj), log=True )
def parse_auth_response(auth_response): """Parse the auth response and return the tenant, token, and username. :param auth_response: the full object returned from an auth call :returns: tuple (token, tenant, username, internalurl, externalurl, cdnurl) """ def _service_ep(scat, types_list): for srv in scat: if srv.get('name') in types_list: index_id = types_list.index(srv.get('name')) index = types_list[index_id] if srv.get('name') == index: return srv.get('endpoints') else: return None access = auth_response.get('access') token = access.get('token').get('id') if 'tenant' in access.get('token'): tenant = access.get('token').get('tenant').get('name') user = access.get('user').get('name') elif 'user' in access: tenant = None user = access.get('user').get('name') else: LOG.error('No Token Found to Parse.\nHere is the DATA: %s\n%s', auth_response, traceback.format_exc()) raise turbo.NoTenantIdFound('When attempting to grab the ' 'tenant or user nothing was found.') if ARGS.get('os_rax_auth') is not None: region = ARGS.get('os_rax_auth') elif ARGS.get('os_hp_auth') is not None: if ARGS.get('os_tenant') is None: raise turbo.NoTenantIdFound( 'You need to have a tenant set to use HP Cloud' ) region = ARGS.get('os_hp_auth') elif ARGS.get('os_region') is not None: region = ARGS.get('os_region') else: raise turbo.SystemProblem('No Region Set') scat = access.pop('serviceCatalog') cfl = _service_ep(scat, info.__srv_types__) cdn = _service_ep(scat, info.__cdn_types__) if cfl is not None: inet = get_surl(region=region, cf_list=cfl, lookup='internalURL') enet = get_surl(region=region, cf_list=cfl, lookup='publicURL') else: need_tenant = ' Maybe you need to specify "os-tenant"?' gen_message = ('No Service Endpoints were found for use with Swift.' ' If you have Swift available to you,' ' Check Your Credentials and/or Swift\'s availability' ' using Token Auth.') if ARGS.get('os_tenant') is None: gen_message += need_tenant raise turbo.SystemProblem(gen_message) if cdn is not None: cnet = get_surl(region=region, cf_list=cdn, lookup='publicURL') else: cnet = None return token, tenant, user, inet, enet, cnet, cfl
def start(self): """Retrieve a long list of all files in a container.""" # Package up the Payload payload = http.prep_payload(auth=self.auth, container=ARGS.get('container'), source=ARGS.get('source'), args=ARGS) self.go = actions.CloudActions(payload=payload) self.action = getattr(self.go, 'object_lister') LOG.info('Attempting Download of Remote path %s', payload['c_name']) if ARGS.get('verbose'): LOG.info('Accessing API for a list of Objects in %s', payload['c_name']) report.reporter(msg='PAYLOAD\t: "%s"' % payload, log=True, lvl='debug', prt=False) report.reporter(msg='getting file list') with multi.spinner(): # Get all objects in a Container objects, list_count, last_obj = self.action( url=payload['url'], container=payload['c_name']) if ARGS.get('pattern_match'): objects = basic.match_filter(idx_list=objects, pattern=ARGS['pattern_match'], dict_type=True) # Count the number of objects returned. if objects is False: report.reporter(msg='No Container found.') return elif objects is not None: num_files = len(objects) if num_files < 1: report.reporter(msg='No Objects found.') return else: report.reporter(msg='No Objects found.') return # Get The rate of concurrency concurrency = multi.set_concurrency(args=ARGS, file_count=num_files) # Load the queue obj_list = [obj['name'] for obj in objects] report.reporter(msg='Building Directory Structure.') with multi.spinner(): if ARGS.get('object'): obj_names = ARGS.get('object') obj_list = [obj for obj in obj_list if obj in obj_names] num_files = len(obj_list) elif ARGS.get('dir'): objpath = ARGS.get('dir') obj_list = [obj for obj in obj_list if obj.startswith(objpath)] num_files = len(obj_list) # from objects found set a unique list of directories unique_dirs = basic.set_unique_dirs(object_list=obj_list, root_dir=payload['source']) for udir in unique_dirs: basic.mkdir_p(path=udir) kwargs = { 'url': payload['url'], 'container': payload['c_name'], 'source': payload['source'], 'cf_job': getattr(self.go, 'object_downloader') } report.reporter(msg='Performing Object Download.') multi.job_processer(num_jobs=num_files, objects=obj_list, job_action=multi.doerator, concur=concurrency, kwargs=kwargs)
def parse_auth_response(auth_response): """Parse the auth response and return the tenant, token, and username. :param auth_response: the full object returned from an auth call :returns: tuple (token, tenant, username, internalurl, externalurl, cdnurl) """ def _service_ep(scat, types_list): for srv in scat: if srv.get('name') in types_list: index_id = types_list.index(srv.get('name')) index = types_list[index_id] if srv.get('name') == index: return srv.get('endpoints') else: return None access = auth_response.get('access') token = access.get('token').get('id') if 'tenant' in access.get('token'): tenant = access.get('token').get('tenant').get('name') user = access.get('user').get('name') elif 'user' in access: tenant = None user = access.get('user').get('name') else: LOG.error('No Token Found to Parse.\nHere is the DATA: %s\n%s', auth_response, traceback.format_exc()) raise turbo.NoTenantIdFound('When attempting to grab the ' 'tenant or user nothing was found.') if ARGS.get('os_rax_auth') is not None: region = ARGS.get('os_rax_auth') elif ARGS.get('os_hp_auth') is not None: if ARGS.get('os_tenant') is None: raise turbo.NoTenantIdFound( 'You need to have a tenant set to use HP Cloud') region = ARGS.get('os_hp_auth') elif ARGS.get('os_region') is not None: region = ARGS.get('os_region') else: raise turbo.SystemProblem('No Region Set') scat = access.pop('serviceCatalog') cfl = _service_ep(scat, info.__srv_types__) cdn = _service_ep(scat, info.__cdn_types__) if cfl is not None: inet = get_surl(region=region, cf_list=cfl, lookup='internalURL') enet = get_surl(region=region, cf_list=cfl, lookup='publicURL') else: need_tenant = ' Maybe you need to specify "os-tenant"?' gen_message = ('No Service Endpoints were found for use with Swift.' ' If you have Swift available to you,' ' Check Your Credentials and/or Swift\'s availability' ' using Token Auth.') if ARGS.get('os_tenant') is None: gen_message += need_tenant raise turbo.SystemProblem(gen_message) if cdn is not None: cnet = get_surl(region=region, cf_list=cdn, lookup='publicURL') else: cnet = None return token, tenant, user, inet, enet, cnet, cfl