def container_check(self, container_name): """ container_name = 'The name of the Container' check a container to see if it exists """ self.connection_prep() r_loc = '%s/%s' % (self.c_path, container_name) path = quote(r_loc) for retry in generators.retryloop(attempts=self.tur_arg['error_retry'], delay=5): c_headers = self.headers # Check to see if the container exists self.conn.request('HEAD', path, headers=c_headers) resp_info = self.response_type() if resp_info[1] is not True: retry() resp = resp_info[0] resp.read() # Check that the status was a good one if resp.status == 404: print('Container Not Found') if resp.status >= 300 or resp.status == None: self.result_exception(resp=resp, headers=c_headers, authurl=self.url, jsonreq=path) retry() return resp
def object_deleter(self, file_path, container): """check a container to see if it exists :param file_path: :param container: """ for retry in gen.retryloop(attempts=self.retry_atmp, delay=5): try: conn = self.connection_prep() r_loc = '%s/%s/%s' % (self.c_path, container, file_path) remote_path = urllib.quote(r_loc) f_headers = self.set_headers() conn.request('DELETE', remote_path, headers=f_headers) resp = self.response_get(conn=conn, rty=retry) if self.result_exception(resp=resp, authurl=self.url, jsonreq=remote_path, dfc=True): raise exceptions.SystemProblem(resp) except exceptions.SystemProblem: retry() else: # Give us more data if we requested it if any([self.tur_arg['os_verbose'], self.tur_arg['debug']]): print('INFO\t: %s %s %s' % (resp.status, resp.reason, file_path)) if self.tur_arg['debug']: print('MESSAGE\t: Delete path = %s ==> %s' % (file_path, container)) finally: conn.close()
def container_deleter(self, container): """ container_name = 'The name of the Container' check a container to see if it exists """ self.connection_prep() r_loc = '%s/%s' % (self.c_path, container) path = quote(r_loc) for retry in generators.retryloop(attempts=self.tur_arg['error_retry'], delay=5): c_headers = self.headers # Check to see if the container exists self.conn.request('DELETE', path, headers=c_headers) resp_info = self.response_type() if resp_info[1] is not True: retry() resp = resp_info[0] resp.read() if resp.status >= 300 or resp.status == None: self.result_exception(resp=resp, headers=c_headers, authurl=self.url, jsonreq=path) retry() # Give us more data if we requested it if self.tur_arg['os_verbose'] or self.tur_arg['debug']: print 'INFO\t: %s %s %s' % (resp.status, resp.reason, container) if self.tur_arg['debug']: print 'MESSAGE\t: Delete path = %s' % (container) self.connection_prep(conn_close=True)
def container_check(self, container_name): """check a container to see if it exists. :param container_name: The name of the Container """ try: conn = self.connection_prep() r_loc = '%s/%s' % (self.c_path, container_name) path = urllib.quote(r_loc) for retry in gen.retryloop(attempts=self.retry_atmp, delay=5): c_headers = self.set_headers() # Check to see if the container exists conn.request('HEAD', path, headers=c_headers) resp = self.response_get(conn=conn, rty=retry) # Check that the status was a good one if resp.status == 404: print('Container Not Found') elif self.result_exception(resp=resp, authurl=self.url, jsonreq=path): raise exceptions.SystemProblem(resp) except exceptions.SystemProblem: retry() else: return resp finally: conn.close()
def enable_cdn(self, container_name): """Enable the CDN on a provided Container. If custom meta data is specified, and the container exists the method will put the metadata onto the object. :param container_name: The name of the Container """ cdnurl_data = self.tur_arg['simple_cdn_endpoint'].split('/') cdnurl = cdnurl_data[0] cdn_path = urllib.quote('/%s' % ('/'.join(cdnurl_data[1:]))) r_loc = '%s/%s' % (cdn_path, container_name) path = urllib.quote(r_loc) c_headers = self.set_headers(cdn=True) for retry in gen.retryloop(attempts=self.retry_atmp, delay=5): try: conn = self.connection(url=cdnurl) conn.request('PUT', path, headers=c_headers) resp = self.response_get(conn=conn, rty=retry) if self.tur_arg['os_verbose']: print('ENABLING CDN ON CONTAINER: %s %s %s' % (resp.status, resp.reason, container_name)) if self.result_exception(resp=resp, authurl=cdnurl, jsonreq=path): raise exceptions.SystemProblem(resp) except exceptions.SystemProblem: retry() except Exception as exp: print('ERROR\t: Shits broke son, here comes the' ' stack trace:\t %s' % traceback.format_exc()) print(exp) finally: conn.close()
def get_downloader(self, file_path, file_name, container): """ This is the simple download Method. There is no file level checking at the target. The files are simply downloaded. """ try: for retry in generators.retryloop( attempts=self.tur_arg['error_retry'], delay=5, backoff=2): try: # Set the headers if some custom ones were specified f_headers = self.headers if self.tur_arg['object_headers']: f_headers.update(self.tur_arg['object_headers']) # Get a file list ready for action r_loc = '%s/%s/%s' % (self.c_path, container, file_path) filepath = quote(r_loc) self.conn.request('GET', filepath, headers=f_headers) resp_info = self.response_type() if resp_info[1] is not True: retry() resp = resp_info[0] _rr = resp.read() # Check that the status was a good one if resp.status >= 300 or resp.status == None: self.result_exception(resp=resp, headers=f_headers, authurl=self.url, jsonreq=filepath, file_path=file_path) retry() # Open our source file with open(file_name, 'wb') as f_name: f_name.write(_rr) f_name.close() # Give us more data if we requested it if self.tur_arg['os_verbose'] or self.tur_arg['debug']: print 'INFO\t: %s %s %s' % (resp.status, resp.reason, file_name) if self.tur_arg['debug']: print('MESSAGE\t: Upload path = %s ==> %s' % (file_path, filepath)) except Exception, exp: print(exp) print('Exception from within an Download Action, placing' ' the failed Download back in Queue') self.work_q.put(file_path) except IOError: print('ERROR\t: path "%s" does not exist or is a broken symlink' % file_name) except ValueError: print('ERROR\t: The data for "%s" got all jacked up, so it' ' got skipped' % file_name) except KeyboardInterrupt: pass
def enable_cdn(self, container_name): """ container_name = 'The name of the Container' Enable the CDN on a provided Container. If custom meta data is specified, and the container exists the method will put the metadata onto the object. """ cdnurl_data = self.tur_arg['simple_cdn_endpoint'].split('/') cdnurl = cdnurl_data[0] cdn_path = quote('/%s' % ('/'.join(cdnurl_data[1:]))) r_loc = '%s/%s' % (cdn_path, container_name) path = quote(r_loc) c_headers = self.headers c_headers.update({'X-CDN-Enabled': True, 'X-TTL': self.tur_arg['cdn_ttl'], 'X-Log-Retention': self.tur_arg['cdn_logs']}) for retry in generators.retryloop(attempts=self.tur_arg['error_retry'], delay=5): try: conn = httplib.HTTPSConnection(cdnurl) for retry in generators.retryloop( attempts=self.tur_arg['error_retry'], delay=5): conn.request('PUT', path, headers=c_headers) resp_info = self.response_type() if resp_info[1] is not True: retry() resp = resp_info[0] resp.read() status_codes = (resp.status, resp.reason, container_name) if self.tur_arg['os_verbose']: print('ENABLING CDN ON CONTAINER: %s %s %s' % status_codes) if resp.status >= 300 or resp.status == None: print('Failure happened, will retry %s %s %s' % status_codes) retry() continue except Exception, exp: print('ERROR\t: Shits broke son, here comes the' ' stack trace:\t %s' % (sys.exc_info()[1])) print exp finally:
def sync_uploader(self, file_path, file_name, container): """This is the Sync method which uploads files to the swift repository if they are not already found. If a file "name" is found locally and in the swift repository an MD5 comparison is done between the two files. If the MD5 is miss-matched the local file is uploaded to the repository. If custom meta data is specified, and the object exists the method will put the metadata onto the object. :param file_path: :param file_name: :param container: """ for retry in gen.retryloop(attempts=self.retry_atmp, delay=5): try: conn = self.connection_prep() f_headers = self.set_headers() # Get the path ready for action r_loc = '%s/%s/%s' % (self.c_path, container, file_name) remote_path = urllib.quote(r_loc) conn.request('HEAD', remote_path, headers=f_headers) resp = self.response_get(conn=conn, rty=retry) if any([resp.status == 404, self.md5_checker(resp=resp, local_file=file_path) is True]): # If different or not found, perform Upload. self.object_putter(fpath=file_path, rpath=remote_path, fname=file_name, fheaders=f_headers, retry=retry) except IOError: print('ERROR\t: path "%s" does not exist or is a broken' ' symlink' % file_path) except ValueError: print('ERROR\t: The data for "%s" got all jacked up,' ' so it got skipped' % file_path) except KeyboardInterrupt: pass except Exception as exp: print('\nFile Failed to be uploaded %s. Error ==> %s\n\n%s' % (file_path, exp, traceback.format_exc())) else: # Put headers on the object if custom headers if self.tur_arg['object_headers']: conn.request('POST', remote_path, headers=f_headers) resp = self.response_get(conn=conn, rty=retry) if self.result_exception(resp=resp, authurl=self.url, jsonreq=remote_path): retry() finally: conn.close()
def container_create(self, container_name): """Create a container if the container specified on the command. If custom meta data is specified, and the container exists the method will put the metadata onto the object. :param container_name: The name of the Container """ for retry in gen.retryloop(attempts=self.retry_atmp, delay=5): try: conn = self.connection_prep() r_loc = '%s/%s' % (self.c_path, container_name) path = urllib.quote(r_loc) c_headers = self.set_headers(ctr=True) resp = self.container_check(container_name) status_codes = (resp.status, resp.reason, container_name) # Check that the status was a good one if resp.status == 404: print('Creating Container ==> %s' % container_name) conn.request('PUT', path, headers=c_headers) resp = self.response_get(conn=conn, rty=retry) if resp.status == 404: print('Container Not Found %s' % resp.status) elif self.result_exception(resp=resp, authurl=self.url, jsonreq=path): raise exceptions.SystemProblem(resp) status_codes = (resp.status, resp.reason, container_name) if self.tur_arg['os_verbose']: print('CREATING CONTAINER: %s %s %s' % status_codes) elif self.result_exception(resp=resp, authurl=self.url, jsonreq=path): raise exceptions.SystemProblem(resp) except exceptions.SystemProblem: retry() except Exception as exp: print('ERROR\t: Shits broke son, here comes the' ' stack trace:\t %s -> Exception\t: %s' % (traceback.format_exc(), exp)) else: if self.tur_arg['os_verbose']: print('Container Found %s %s %s' % status_codes) # Put headers on the object if custom headers used if self.tur_arg['object_headers']: conn.request('POST', path, headers=c_headers) resp = self.response_get(conn=conn, rty=retry) if self.result_exception(resp=resp, authurl=self.url, jsonreq=path): retry() finally: conn.close()
def make_request(self, jsonreq, url): """Make an API request. :param jsonreq: :param url: """ try: conn = self.connection(url=url) for retry in gen.retryloop(attempts=self.retry_atmp, timeout=960, delay=5): if self.tur_arg['os_verbose']: print('JSON REQUEST: %s' % jsonreq) conn.set_debuglevel(1) headers = {'Content-Type': 'application/json'} tokenurl = '/%s/tokens' % self.tur_arg.get('os_version') conn.request('POST', tokenurl, jsonreq, headers) resp, resp_read = self.response_get(conn=conn, rty=retry, ret_read=True, mcr=True) jrp = json.loads(resp_read) # Check that the status was a good one if resp.status >= 500: print('500 Error => Attempting HTTP connection') conn = self.connection(url=url, http=True) raise exceptions.SystemProblem(resp) elif self.result_exception(resp=resp, authurl=url, jsonreq=jsonreq): raise exceptions.SystemProblem(resp) else: if self.tur_arg['os_verbose']: print('JSON decoded and pretty') print(json.dumps(jrp, indent=2)) except exceptions.SystemProblem: retry() else: # Send Response to Parser return self.parse_request(json_response=jrp) finally: conn.close()
def response_type(self): for retry in generators.retryloop(attempts=self.tur_arg['error_retry'], timeout=960, delay=5): try: # Compatibility with Python 2.6 if sys.version_info < (2, 7, 0): resp = self.conn.getresponse() else: resp = self.conn.getresponse(buffering=True) return resp, True except httplib.BadStatusLine: self.conn.close() self.connection_prep() retry() continue else: return None, False
def response_type(self, conn, mcr=False): """Understand the response type and provide for the connection. :param mcr: :param conn: """ for retry in gen.retryloop(attempts=self.retry_atmp, timeout=960, delay=5): try: resp = conn.getresponse() except httplib.BadStatusLine as exp: if mcr is False: retry() else: raise exceptions.SystemProblem('Failed to perform Action' ' %s' % exp) else: return resp, True
def put_uploader(self, file_path, file_name, container): """This is the simple upload Method. There is no file level checking at the target. The files are simply uploaded. :param file_path: :param file_name: :param container: """ for retry in gen.retryloop(attempts=self.retry_atmp, delay=5, backoff=2): try: f_headers = self.set_headers() # Get the path ready for action r_loc = '%s/%s/%s' % (self.c_path, container, file_name) remote_path = urllib.quote(r_loc) self.object_putter(fpath=file_path, rpath=remote_path, fname=file_name, fheaders=f_headers, retry=retry) except IOError: print('ERROR\t: path "%s" does not exist or is a broken' ' symlink' % file_path) except ValueError: print('ERROR\t: The data for "%s" got all jacked up,' ' so it got skipped' % file_path) except KeyboardInterrupt: pass except Exception as exp: print('\nFile Failed to be uploaded %s. Error ==> %s\n\n%s' % (file_path, exp, traceback.format_exc())) else: if self.tur_arg['debug']: print('MESSAGE\t: Upload path = %s ==> %s' % (file_path, remote_path))
def sync_uploader(self, file_path, file_name, container): """ This is the Sync method which uploads files to the swift repository' if they are not already found. If a file "name" is found locally and in the swift repository an MD5 comparison is done between the two files. If the MD5 is miss-matched the local file is uploaded to the repository. If custom meta data is specified, and the object exists the method will put the metadata onto the object. """ #noinspection PyBroadException try: for retry in generators.retryloop( attempts=self.tur_arg['error_retry'], delay=5): try: # Set the headers if some custom ones were specified f_headers = self.headers if self.tur_arg['object_headers']: f_headers.update(self.tur_arg['object_headers']) # Get the path ready for action r_loc = '%s/%s/%s' % (self.c_path, container, file_name) filepath = quote(r_loc) self.conn.request('HEAD', filepath, headers=f_headers) resp_info = self.response_type() if resp_info[1] is not True: retry() resp = resp_info[0] resp.read() if resp.status == 404: with open(file_path, 'rb') as f_path: self.conn.request('PUT', filepath, body=f_path, headers=f_headers) f_path.close() try: # Compatibility with Python 2.6 if sys.version_info < (2, 7, 0): resp = self.conn.getresponse() else: resp = self.conn.getresponse(buffering=True) except httplib.BadStatusLine: self.conn.close() self.connection_prep() retry() resp.read() if resp.status >= 300 or resp.status == None: self.result_exception(resp=resp, headers=f_headers, authurl=filepath, jsonreq=r_loc, file_path=file_path) retry() if self.tur_arg['verbose']: print 'INFO\t: %s %s %s' % (resp.status, resp.reason, file_name) elif resp.status >= 300 or resp.status == None: self.result_exception(resp=resp, headers=f_headers, authurl=filepath, jsonreq=r_loc, file_path=file_path) retry() else: remotemd5sum = resp.getheader('etag') md5 = hashlib.md5() with open(file_path, 'rb') as f_hash: for chunk in iter(lambda: f_hash.read( 128 * md5.block_size), ''): md5.update(chunk) f_hash.close() localmd5sum = md5.hexdigest() if remotemd5sum != localmd5sum: with open(file_path, 'rb') as f_path: self.conn.request('PUT', filepath, body=f_path, headers=f_headers) f_path.close() resp_info = self.response_type() if resp_info[1] is not True: retry() resp = resp_info[0] resp.read() if resp.status >= 300: self.result_exception(resp=resp, headers=f_headers, authurl=filepath, jsonreq=r_loc, file_path=file_path) retry() if self.tur_arg['verbose']: proc_dict = {'lmd5': localmd5sum, 'rmd5': remotemd5sum, 'rs': resp.status, 'rr': resp.reason, 'sjf': file_name} print('MESSAGE\t: CheckSumm Mis-Match %(lmd5)s' ' != %(rmd5)s\n\t File Upload : %(rs)s' ' %(rr)s %(sjf)s' % proc_dict) else: if self.tur_arg['verbose']: print 'MESSAGE\t: CheckSum Match', localmd5sum # Put headers on the object if custom headers if self.tur_arg['object_headers']: self.conn.request('POST', filepath, headers=f_headers) resp_info = self.response_type() if resp_info[1] is not True: retry() resp = resp_info[0] resp.read() if resp.status >= 300: self.result_exception(resp=resp, headers=self.headers, authurl=filepath, jsonreq=r_loc, file_path=file_path) retry() except Exception: print(traceback.format_exc()) print('Exception from within an upload Action, placing' ' the failed upload back in Queue') self.work_q.put(file_path) except IOError: print('ERROR\t: path "%s" does not exist or is a broken symlink' % file_path) except ValueError: print('ERROR\t: The data for "%s" got all jacked up,' ' so it got skipped' % file_path) except KeyboardInterrupt: pass
def get_object_list(self, container_name, lastobj=None): """Builds a long list of files found in a container :param container_name: :param lastobj: """ for retry in gen.retryloop(attempts=self.retry_atmp, delay=5, backoff=2): try: conn = self.connection_prep() f_headers = self.set_headers() # Determine how many files are in the container r_loc = '%s/%s' % (self.c_path, container_name) filepath = urllib.quote(r_loc) conn.request('HEAD', filepath, headers=f_headers) resp = self.response_get(conn=conn, rty=retry) if self.result_exception(resp=resp, authurl=self.url, jsonreq=filepath): raise exceptions.SystemProblem(resp) count = int(resp.getheader('X-Container-Object-Count')) # Build the List file_list = [] # Set the number of loops that we are going to do jobs = count / 10000 + 1 filepath = '%s/?limit=10000&format=json' % filepath filepath_m = filepath f_headers.update({'Content-type': 'application/json'}) for _ in xrange(jobs): for nest_rty in gen.retryloop(attempts=self.retry_atmp, delay=5): conn.request('GET', filepath, headers=f_headers) resp, resp_read = self.response_get(conn=conn, rty=retry, ret_read=True) if self.result_exception(resp=resp, authurl=self.url, jsonreq=filepath): raise exceptions.NoSource(resp) _rr = json.loads(resp_read) for obj in _rr: file_list.append(obj['name']) if count - 10000 > 0: count = count - 10000 lastobj = file_list[-1] filepath = ( '%s&marker=%s' % (filepath_m, urllib.quote(lastobj)) ) except exceptions.NoSource: nest_rty() except exceptions.SystemProblem: retry() except KeyboardInterrupt: pass except Exception as exp: print('Exception from within an Download Action\n%s\n%s' % (traceback.format_exc(), exp)) else: # Give us more data if we requested it if any([self.tur_arg['os_verbose'], self.tur_arg['debug']]): print('INFO\t: %s %s %s' % (resp.status, resp.reason, file_list)) if self.tur_arg['debug']: print('MESSAGE\t: Path => %s\nMESSAGE\t: %s' % (filepath, _rr)) return file_list finally: conn.close()
def get_object_list(self, container_name): """ Builds a long list of files found in a container """ lastobj = None try: for retry in generators.retryloop( attempts=self.tur_arg['error_retry'], delay=5, backoff=2): try: self.connection_prep() # Set the headers if some custom ones were specified c_headers = self.headers if self.tur_arg['object_headers']: c_headers.update(self.tur_arg['object_headers']) # Determine how many files are in the container r_loc = '%s/%s' % (self.c_path, container_name) filepath = quote(r_loc) self.conn.request('HEAD', filepath, headers=c_headers) resp_info = self.response_type() if resp_info[1] is not True: retry() resp = resp_info[0] resp.read() if resp.status >= 300: self.result_exception(resp=resp, headers=c_headers, authurl=self.url, jsonreq=filepath) count = int(resp.getheader('X-Container-Object-Count')) # Build the List file_list = [] # Set the number of loops that we are going to do jobs = count / 10000 + 1 filepath = '%s/?limit=10000&format=json' % filepath filepath_m = filepath f_headers = self.headers f_headers.update({'Content-type': 'application/json'}) for _ in xrange(jobs): self.conn.request('GET', filepath, headers=f_headers) resp_info = self.response_type() if resp_info[1] is not True: retry() resp = resp_info[0] if resp.status >= 300: self.result_exception(resp=resp, headers=c_headers, authurl=self.url, jsonreq=filepath) _rr = json.loads(resp.read()) for obj in _rr: file_list.append(obj['name']) if count - 10000 > 0: count = count - 10000 lastobj = file_list[-1] filepath = '%s&marker=%s' % (filepath_m, quote(lastobj)) # Give us more data if we requested it if self.tur_arg['os_verbose'] or self.tur_arg['debug']: print 'INFO\t: %s %s %s' % (resp.status, resp.reason, file_list) if self.tur_arg['debug']: print('MESSAGE\t: Path => %s' % (filepath)) print(_rr) self.connection_prep(conn_close=True) return file_list except Exception, exp: print exp print traceback.format_exc() print('Exception from within an Download Action') except KeyboardInterrupt: pass
def container_create(self, container_name): """ container_name = 'The name of the Container' Create a container if the container specified on the command line did not already exist. If custom meta data is specified, and the container exists the method will put the metadata onto the object. """ r_loc = '%s/%s' % (self.c_path, container_name) path = quote(r_loc) try: for retry in generators.retryloop( attempts=self.tur_arg['error_retry'], delay=5): c_headers = self.headers if self.tur_arg['container_headers']: c_headers.update(self.tur_arg['container_headers']) resp = self.container_check(container_name) status_codes = (resp.status, resp.reason, container_name) # Check that the status was a good one if resp.status == 404: self.conn.request('PUT', path, headers=c_headers) resp_info = self.response_type() if resp_info[1] is not True: retry() resp = resp_info[0] resp.read() if resp.status >= 300 or resp.status == None: self.result_exception(resp=resp, headers=c_headers, authurl=self.url, jsonreq=path) retry() status_codes = (resp.status, resp.reason, container_name) if self.tur_arg['os_verbose']: print('CREATING CONTAINER: %s %s %s' % status_codes) elif resp.status >= 300 or resp.status == None: self.result_exception(resp=resp, headers=c_headers, authurl=self.url, jsonreq=path) retry() else: if self.tur_arg['os_verbose']: print('Container Found %s %s %s' % status_codes) # Put headers on the object if custom headers were specified if self.tur_arg['object_headers']: self.conn.request('POST', path, headers=c_headers) resp_info = self.response_type() if resp_info[1] is not True: retry() resp = resp_info[0] resp.read() if resp.status >= 300: self.result_exception(resp=resp, headers=c_headers, authurl=self.url, jsonreq=path) retry() except Exception, exp: print('ERROR\t: Shits broke son, here comes the' ' stack trace:\t %s -> Exception\t: %s' % (sys.exc_info()[1], exp))
def get_downloader(self, file_path, file_name, container): """This is the simple download Method. There is no file level checking at the target. The files are simply downloaded. :param file_path: :param file_name: :param container: """ def get_action(conn, filepath, local_file, headers, url, retry): """Get a target file and save it locally. :param conn: :param filepath: :param file_name: :param headers: :param url: :param retry: """ conn.request('GET', filepath, headers=headers) resp, resp_read = self.response_get(conn=conn, rty=retry, ret_read=True) # Check that the status was a good one if self.result_exception(resp=resp, authurl=url, jsonreq=filepath): raise exceptions.SystemProblem(resp) # Open our source file and write it with open(local_file, 'wb') as f_name: f_name.write(resp_read) # Give us more data if we requested it if self.tur_arg.get('verbose'): print('INFO\t: %s %s %s' % (resp.status, resp.reason, local_file)) if self.tur_arg.get('debug'): print('MESSAGE\t: Download path = %s ==> %s' % (file_name, filepath)) for retry in gen.retryloop(attempts=self.retry_atmp, delay=5, backoff=1): try: conn = self.connection_prep() f_headers = self.set_headers() # Get a file list ready for action remote_path = '%s/%s/%s' % (self.c_path, container, file_path) filepath = urllib.quote(remote_path) if self.tur_arg.get('dl_sync') is True: conn.request('HEAD', remote_path, headers=f_headers) resp = self.response_get(conn=conn, rty=retry) if self.md5_checker(resp=resp, local_file=file_name) is True: get_action(conn=conn, filepath=filepath, local_file=file_name, headers=f_headers, url=self.url, retry=retry) else: get_action(conn=conn, filepath=filepath, local_file=file_name, headers=f_headers, url=self.url, retry=retry) except exceptions.SystemProblem: retry() except IOError: print('ERROR\t: path "%s" does not exist or is a broken' ' symlink' % file_name) except ValueError: print('ERROR\t: The data for "%s" got all jacked up, so it' ' got skipped' % file_name) except KeyboardInterrupt: pass except Exception as exp: print traceback.format_exc() print('ERROR\t: Exception from within an Download Action ' 'Message == %s' % exp) finally: conn.close()
def osauth(self): """ Authentication For Openstack API, Pulls the full Openstack Service Catalog Credentials are the Users API Username and Key/Password "osauth" has a Built in Rackspace Method for Authentication Set a DC Endpoint and Authentication URL for the Open Stack environment """ rax_us_dc = (self.tur_arg['os_rax_auth'] == 'DFW', self.tur_arg['os_rax_auth'] == 'ORD') rax_eu_dc = (self.tur_arg['os_rax_auth'] == 'LON', self.tur_arg['os_rax_auth'] == 'NEWDC') if any(rax_eu_dc): self.tur_arg['os_region'] = self.tur_arg['os_rax_auth'] if self.tur_arg['os_auth_url']: authurl = self.tur_arg['os_auth_url'] else: authurl = 'lon.identity.api.rackspacecloud.com' elif any(rax_us_dc): self.tur_arg['os_region'] = self.tur_arg['os_rax_auth'] if self.tur_arg['os_auth_url']: authurl = self.tur_arg['os_auth_url'] else: authurl = 'identity.api.rackspacecloud.com' else: if not self.tur_arg['os_region']: sys.exit('FAIL\t: You have to specify ' 'a Region along with an Auth URL') if self.tur_arg['os_auth_url']: authurl = self.tur_arg['os_auth_url'] else: sys.exit('FAIL\t: You have to specify an Auth URL' ' along with the Region') if self.tur_arg['os_apikey'] or self.tur_arg['os_rax_auth']: jsonreq = json.dumps({'auth': {'RAX-KSKEY:apiKeyCredentials': {'username': self.tur_arg['os_user'], 'apiKey': self.tur_arg['os_apikey']}}}) else: jsonreq = json.dumps({'auth': {'passwordCredentials': {'username': self.tur_arg['os_user'], 'password': self.tur_arg['os_password']}}}) authurl = self.tur_arg['os_auth_url'] # remove the prefix for the Authentication URL authurl = authurl.strip('http?s://') url_data = authurl.split('/') url = url_data[0] for retry in generators.retryloop(attempts=self.tur_arg['error_retry'], timeout=960, delay=5): conn = httplib.HTTPSConnection(url) if self.tur_arg['os_verbose']: print('JSON REQUEST: %s' % jsonreq) conn.set_debuglevel(1) headers = {'Content-Type': 'application/json'} tokenurl = '/%s/tokens' % self.tur_arg['os_version'] conn.request('POST', tokenurl, jsonreq, headers) try: # Compatibility with Python 2.6 if sys.version_info < (2, 7, 0): resp = conn.getresponse() else: resp = conn.getresponse(buffering=True) except httplib.BadStatusLine: conn.close() retry() readresp = resp.read() json_response = json.loads(readresp) conn.close() # Check that the status was a good one if resp.status >= 500: print('Attempting HTTP connection') conn = httplib.HTTPConnection(url) retry() elif resp.status >= 300 or resp.status == None: self.result_exception(resp=resp, headers=self.headers, authurl=self.url, jsonreq=self.c_path) try: retry() except Exception, exp: print('Authentication has FAILED "%s %s %s %s"' % (resp.status, resp.reason, readresp, exp)) continue else: if self.tur_arg['os_verbose']: print('JSON decoded and pretty') print json.dumps(json_response, indent=2) try: for service in json_response['access']['serviceCatalog']: if service['name'] == 'cloudFiles': for _ep in service['endpoints']: if _ep['region'] == self.tur_arg['os_region']: if self.tur_arg['internal']: endpt = _ep['internalURL'] self.tur_arg['endpoint'] = endpt else: self.tur_arg['endpoint'] = _ep['publicURL'] elif service['name'] == 'swift': for _ep in service['endpoints']: if _ep['region'] == self.tur_arg['os_region']: if self.tur_arg['internal']: endpt = _ep['internalURL'] self.tur_arg['endpoint'] = endpt else: self.tur_arg['endpoint'] = _ep['publicURL'] if service['name'] == 'cloudFilesCDN': for _ep in service['endpoints']: if _ep['region'] == self.tur_arg['os_region']: self.tur_arg['CDNendpoint'] = _ep['publicURL'] tenant_id = json_response['access']['token']['tenant']['id'] self.tur_arg['tenantid'] = tenant_id token = json_response['access']['token']['id'] headers = self.tur_arg['base_headers'] headers.update({'X-Auth-Token': token}) self.tur_arg['base_headers'] = headers cdn_encode = self.tur_arg['CDNendpoint'].encode('utf8') cdn_split = cdn_encode.split('//')[1] self.tur_arg['simple_cdn_endpoint'] = cdn_split url_encode = self.tur_arg['endpoint'].encode('utf8') url_split = url_encode.split('//')[1] self.tur_arg['simple_endpoint'] = url_split if self.tur_arg['os_verbose']: print('SimpleURL\t: %s' '\nPublicURL\t: %s' '\nTenant\t\t: %s' '\nCDN_manage\t: %s' % (url_split, self.tur_arg['endpoint'], tenant_id, cdn_encode)) return self.tur_arg except (KeyError, IndexError): print('Error while getting answers from auth server.' ' Check the endpoint and auth credentials.')