def object_deleter(self, url, container, u_file): """Deletes an objects in a container. :param url: :param container: :param u_file: """ rty_count = ARGS.get('error_retry') for retry in basic.retryloop(attempts=rty_count, delay=2, obj=u_file): # Open Connection conn = http.open_connection(url=url) # Open connection and perform operation with meth.operation(retry, conn): rpath = http.quoter(url=url.path, cont=container, ufile=u_file) # Make a connection resp = self._header_getter(conn=conn, rpath=rpath, fheaders=self.payload['headers'], retry=retry) if not resp.status == 404: # Perform delete. self._deleter(conn=conn, rpath=rpath, fheaders=self.payload['headers'], retry=retry)
def object_downloader(self, url, container, source, u_file): """Download an Object from a Container. :param url: :param container: :param u_file: """ rty_count = ARGS.get('error_retry') for retry in basic.retryloop(attempts=rty_count, delay=2, obj=u_file): # Open Connection conn = http.open_connection(url=url) # Perform operation with meth.operation(retry, conn): fheaders = self.payload['headers'] rpath = http.quoter(url=url.path, cont=container, ufile=u_file) # Perform Download. self._downloader(conn=conn, rpath=rpath, fheaders=fheaders, lfile=u_file, source=source, retry=retry)
def container_cdn_command(self, url, container, sfile=None): """Command your CDN enabled Container. :param url: :param container: """ rty_count = ARGS.get('error_retry') for retry in basic.retryloop(attempts=rty_count, delay=2, obj=sfile): # Open Connection conn = http.open_connection(url=url) with meth.operation(retry, conn): cheaders = self.payload['headers'] if sfile is not None: rpath = http.quoter(url=url.path, cont=container, ufile=sfile) # perform CDN Object DELETE conn.request('DELETE', rpath, headers=cheaders) resp, read = http.response_get(conn=conn, retry=retry) self.resp_exception(resp=resp, rty=retry) else: rpath = http.quoter(url=url.path, cont=container) http.cdn_toggle(headers=cheaders) # perform CDN Enable POST conn.request('PUT', rpath, headers=cheaders) resp, read = http.response_get(conn=conn, retry=retry) self.resp_exception(resp=resp, rty=retry) report.reporter( msg=('OBJECT %s MESSAGE %s %s %s' % (rpath, resp.status, resp.reason, resp.msg)), prt=False, lvl='debug' )
def container_cdn_command(self, url, container, sfile=None): """Command your CDN enabled Container. :param url: :param container: """ rty_count = ARGS.get('error_retry') for retry in basic.retryloop(attempts=rty_count, delay=2, obj=sfile): # Open Connection conn = http.open_connection(url=url) with meth.operation(retry, conn): cheaders = self.payload['headers'] if sfile is not None: rpath = http.quoter(url=url.path, cont=container, ufile=sfile) # perform CDN Object DELETE conn.request('DELETE', rpath, headers=cheaders) resp, read = http.response_get(conn=conn, retry=retry) self.resp_exception(resp=resp, rty=retry) else: rpath = http.quoter(url=url.path, cont=container) http.cdn_toggle(headers=cheaders) # perform CDN Enable POST conn.request('PUT', rpath, headers=cheaders) resp, read = http.response_get(conn=conn, retry=retry) self.resp_exception(resp=resp, rty=retry) report.reporter( msg=('OBJECT %s MESSAGE %s %s %s' % (rpath, resp.status, resp.reason, resp.msg)), prt=False, lvl='debug')
def request_process(aurl, req): """Perform HTTP(s) request based on Provided Params. :param aurl: :param req: :param https: :return read_resp: """ conn = http.open_connection(url=aurl) # Make the request for authentication try: _method, _url, _body, _headers = req conn.request(method=_method, url=_url, body=_body, headers=_headers) resp = conn.getresponse() except Exception as exc: LOG.error('Not able to perform Request ERROR: %s', exc) raise AttributeError("Failure to perform Authentication %s ERROR:\n%s" % (exc, traceback.format_exc())) else: resp_read = resp.read() status_code = resp.status if status_code >= 300: LOG.error('HTTP connection exception: ' 'Response %s - Response Code %s\n%s', resp_read, status_code, traceback.format_exc()) raise httplib.HTTPException('Failed to authenticate %s' % status_code) LOG.debug('Connection successful MSG: %s - STATUS: %s', resp.reason, resp.status) return resp_read finally: conn.close()
def object_putter(self, url, container, source, u_file): """This is the Sync method which uploads files to the swift repository if they are not already found. If a file "name" is found locally and in the swift repository an MD5 comparison is done between the two files. If the MD5 is miss-matched the local file is uploaded to the repository. If custom meta data is specified, and the object exists the method will put the metadata onto the object. :param url: :param container: :param source: :param u_file: """ for retry in basic.retryloop(attempts=ARGS.get('error_retry'), delay=2, obj=u_file): # Open Connection conn = http.open_connection(url=url) # Open connection and perform operation with meth.operation(retry, conn, obj=u_file): # Get the path ready for action sfile = basic.get_sfile(ufile=u_file, source=source) rpath = http.quoter(url=url.path, cont=container, ufile=sfile) fheaders = self.payload['headers'] # Perform Upload. self._putter(conn=conn, fpath=u_file, rpath=rpath, fheaders=fheaders, retry=retry) # Put headers on the object if custom headers, or save perms. if any([ARGS.get('object_headers') is not None, ARGS.get('save_perms') is not None]): if ARGS.get('object_headers') is not None: fheaders.update(ARGS.get('object_headers')) if ARGS.get('save_perms') is not None: fheaders.update(basic.stat_file(local_file=u_file)) self._header_poster(conn=conn, rpath=rpath, fheaders=fheaders, retry=retry)
def object_putter(self, url, container, source, u_file): """This is the Sync method which uploads files to the swift repository if they are not already found. If a file "name" is found locally and in the swift repository an MD5 comparison is done between the two files. If the MD5 is miss-matched the local file is uploaded to the repository. If custom meta data is specified, and the object exists the method will put the metadata onto the object. :param url: :param container: :param source: :param u_file: """ for retry in basic.retryloop(attempts=ARGS.get('error_retry'), delay=2, obj=u_file): # Open Connection conn = http.open_connection(url=url) # Open connection and perform operation with meth.operation(retry, conn, obj=u_file): # Get the path ready for action sfile = basic.get_sfile(ufile=u_file, source=source) rpath = http.quoter(url=url.path, cont=container, ufile=sfile) fheaders = self.payload['headers'] # Perform Upload. self._putter(conn=conn, fpath=u_file, rpath=rpath, fheaders=fheaders, retry=retry) # Put headers on the object if custom headers, or save perms. if any([ ARGS.get('object_headers') is not None, ARGS.get('save_perms') is not None ]): if ARGS.get('object_headers') is not None: fheaders.update(ARGS.get('object_headers')) if ARGS.get('save_perms') is not None: fheaders.update(basic.stat_file(local_file=u_file)) self._header_poster(conn=conn, rpath=rpath, fheaders=fheaders, retry=retry)
def object_lister(self, url, container, object_count=None, last_obj=None): """Builds a long list of objects found in a container. NOTE: This could be millions of Objects. :param url: :param container: :param object_count: :param last_obj: :return None | list: """ for retry in basic.retryloop(attempts=ARGS.get('error_retry'), obj='Object List'): # Open Connection conn = http.open_connection(url=url) # Open connection and perform operation with meth.operation(retry, conn): # Determine how many files are in the container fpath = http.quoter(url=url.path, cont=container) # Make a connection resp = self._header_getter(conn=conn, rpath=fpath, fheaders=self.payload['headers'], retry=retry) if resp.status == 404: report.reporter( msg='Not found. %s | %s' % (resp.status, resp.msg) ) return None, None, None else: if object_count is None: head_check = dict(resp.getheaders()) object_count = head_check.get( 'x-container-object-count' ) if object_count: object_count = int(object_count) if not object_count > 0: return None, None, None else: return None, None, None # Set the number of loops that we are going to do return self._list_getter(conn=conn, count=object_count, filepath=fpath, fheaders=self.payload['headers'], last_obj=last_obj)
def object_lister(self, url, container, object_count=None, last_obj=None): """Builds a long list of objects found in a container. NOTE: This could be millions of Objects. :param url: :param container: :param object_count: :param last_obj: :return None | list: """ for retry in basic.retryloop(attempts=ARGS.get('error_retry'), obj='Object List'): # Open Connection conn = http.open_connection(url=url) # Open connection and perform operation with meth.operation(retry, conn): # Determine how many files are in the container fpath = http.quoter(url=url.path, cont=container) # Make a connection resp = self._header_getter(conn=conn, rpath=fpath, fheaders=self.payload['headers'], retry=retry) if resp.status == 404: report.reporter(msg='Not found. %s | %s' % (resp.status, resp.msg)) return None, None, None else: if object_count is None: head_check = dict(resp.getheaders()) object_count = head_check.get( 'x-container-object-count') if object_count: object_count = int(object_count) if not object_count > 0: return None, None, None else: return None, None, None # Set the number of loops that we are going to do return self._list_getter(conn=conn, count=object_count, filepath=fpath, fheaders=self.payload['headers'], last_obj=last_obj)
def container_deleter(self, url, container): """Delete all objects in a container. :param url: :param container: """ for retry in basic.retryloop(attempts=ARGS.get('error_retry'), delay=2, obj=container): # Open Connection conn = http.open_connection(url=url) # Open connection and perform operation with meth.operation(retry, conn): rpath = http.quoter(url=url.path, cont=container) # Perform delete. self._deleter(conn=conn, rpath=rpath, fheaders=self.payload['headers'], retry=retry)
def container_lister(self, url, last_obj=None): """Builds a long list of objects found in a container. NOTE: This could be millions of Objects. :param url: :return None | list: """ for retry in basic.retryloop(attempts=ARGS.get('error_retry'), obj='Container List'): # Open Connection conn = http.open_connection(url=url) # Open connection and perform operation with meth.operation(retry, conn): # Determine how many files are in the container fpath = http.quoter(url=url.path) # Make a connection resp = self._header_getter(conn=conn, rpath=fpath, fheaders=self.payload['headers'], retry=retry) head_check = dict(resp.getheaders()) container_count = head_check.get('x-account-container-count') if container_count: container_count = int(container_count) if not container_count > 0: return None else: return None # Set the number of loops that we are going to do return self._list_getter(conn=conn, count=container_count, filepath=fpath, fheaders=self.payload['headers'], last_obj=last_obj)
def container_create(self, url, container): """Create a container if it is not Found. :param url: :param container: """ rty_count = ARGS.get('error_retry') for retry in basic.retryloop(attempts=rty_count, delay=5, obj=container): conn = http.open_connection(url=url) rpath = http.quoter(url=url.path, cont=container) # Open connection and perform operation with meth.operation(retry, conn): resp = self._header_getter(conn=conn, rpath=rpath, fheaders=self.payload['headers'], retry=retry) # Check that the status was a good one if resp.status == 404: report.reporter( msg='Creating Container ==> %s' % container ) conn.request('PUT', rpath, headers=self.payload['headers']) resp, read = http.response_get(conn=conn, retry=retry) self.resp_exception(resp=resp, rty=retry) report.reporter(msg='Container "%s" Created' % container) return True else: report.reporter(msg='Container "%s" Found' % container) return False
def detail_show(self, url): """Return Details on an object or container.""" rty_count = ARGS.get('error_retry') for retry in basic.retryloop(attempts=rty_count, delay=5, obj=ARGS.get('container')): conn = http.open_connection(url=url) if ARGS.get('object') is not None: rpath = http.quoter(url=url.path, cont=ARGS.get('container'), ufile=ARGS.get('object')) else: rpath = http.quoter(url=url.path, cont=ARGS.get('container')) resp = self._header_getter(conn=conn, rpath=rpath, fheaders=self.payload['headers'], retry=retry) if resp.status == 404: return 'Nothing found.' else: return resp.getheaders()
def container_create(self, url, container): """Create a container if it is not Found. :param url: :param container: """ rty_count = ARGS.get('error_retry') for retry in basic.retryloop(attempts=rty_count, delay=5, obj=container): conn = http.open_connection(url=url) rpath = http.quoter(url=url.path, cont=container) # Open connection and perform operation with meth.operation(retry, conn): resp = self._header_getter(conn=conn, rpath=rpath, fheaders=self.payload['headers'], retry=retry) # Check that the status was a good one if resp.status == 404: report.reporter(msg='Creating Container ==> %s' % container) conn.request('PUT', rpath, headers=self.payload['headers']) resp, read = http.response_get(conn=conn, retry=retry) self.resp_exception(resp=resp, rty=retry) report.reporter(msg='Container "%s" Created' % container) return True else: report.reporter(msg='Container "%s" Found' % container) return False
def request_process(aurl, req): """Perform HTTP(s) request based on Provided Params. :param aurl: :param req: :param https: :return read_resp: """ conn = http.open_connection(url=aurl) # Make the request for authentication try: _method, _url, _body, _headers = req conn.request(method=_method, url=_url, body=_body, headers=_headers) resp = conn.getresponse() except Exception as exc: LOG.error('Not able to perform Request ERROR: %s', exc) raise AttributeError( "Failure to perform Authentication %s ERROR:\n%s" % (exc, traceback.format_exc())) else: resp_read = resp.read() status_code = resp.status if status_code >= 300: LOG.error( 'HTTP connection exception: ' 'Response %s - Response Code %s\n%s', resp_read, status_code, traceback.format_exc()) raise httplib.HTTPException('Failed to authenticate %s' % status_code) LOG.debug('Connection successful MSG: %s - STATUS: %s', resp.reason, resp.status) return resp_read finally: conn.close()
def object_syncer(self, surl, turl, scontainer, tcontainer, obj): """Download an Object from one Container and the upload it to a target. :param surl: :param turl: :param scontainer: :param tcontainer: :param obj: """ def _cleanup(): """Ensure that our temp file is removed.""" if locals().get('tfile') is not None: basic.remove_file(tfile) def _time_difference(resp, obj): if ARGS.get('save_newer') is True: # Get the source object last modified time. compare_time = resp.getheader('last_modified') if compare_time is None: return True elif cloud.time_delta(compare_time=compare_time, lmobj=obj['last_modified']) is True: return False else: return True else: return True def _compare(resp, obj): if resp.status == 404: report.reporter(msg='Target Object %s not found' % obj['name'], prt=False) return True elif resp.getheader('etag') != obj['hash']: report.reporter(msg='Checksum Mismatch on Target Object %s' % obj['name'], prt=False, lvl='debug') return _time_difference(resp, obj) else: return False fheaders = self.payload['headers'] for retry in basic.retryloop(attempts=ARGS.get('error_retry'), delay=5, obj=obj['name']): # Open connection and perform operation fmt, date, date_delta, now = basic.time_stamp() spath = http.quoter(url=surl.path, cont=scontainer, ufile=obj['name']) tpath = http.quoter(url=turl.path, cont=tcontainer, ufile=obj['name']) conn = http.open_connection(url=turl) with meth.operation(retry, conn=conn, obj=obj): resp = self._header_getter(conn=conn, rpath=tpath, fheaders=fheaders, retry=retry) # If object comparison is True GET then PUT object if _compare(resp=resp, obj=obj) is not True: return None try: # Open Connection for source Download conn = http.open_connection(url=surl) with meth.operation(retry, conn=conn, obj=obj): # make a temp file. tfile = basic.create_tmp() # Make a connection resp = self._header_getter(conn=conn, rpath=spath, fheaders=fheaders, retry=retry) sheaders = dict(resp.getheaders()) # TODO(kevin) add the ability to short upload if timestamp # TODO(kevin) ... is newer on the target. # GET remote Object self._downloader(conn=conn, rpath=spath, fheaders=fheaders, lfile=tfile, source=None, retry=retry, skip=True) for nretry in basic.retryloop(attempts=ARGS.get('error_retry'), delay=5, obj=obj): # open connection for target upload. conn = http.open_connection(url=turl) with meth.operation(retry, conn=conn, obj=obj, cleanup=_cleanup): resp = self._header_getter(conn=conn, rpath=tpath, fheaders=fheaders, retry=nretry) self.resp_exception(resp=resp, rty=nretry) # PUT remote object self._putter(conn=conn, fpath=tfile, rpath=tpath, fheaders=fheaders, retry=nretry, skip=True) # let the system rest for 3 seconds. basic.stupid_hack(wait=3) # With the source headers POST new headers on target if ARGS.get('clone_headers') is True: resp = self._header_getter(conn=conn, rpath=tpath, fheaders=fheaders, retry=nretry) theaders = dict(resp.getheaders()) for key in sheaders.keys(): if key not in theaders: fheaders.update({key: sheaders[key]}) # Force the SOURCE content Type on the Target. fheaders.update( {'content-type': sheaders.get('content-type')}) self._header_poster(conn=conn, rpath=tpath, fheaders=fheaders, retry=nretry) finally: _cleanup()
def object_syncer(self, surl, turl, scontainer, tcontainer, obj): """Download an Object from one Container and the upload it to a target. :param surl: :param turl: :param scontainer: :param tcontainer: :param obj: """ def _cleanup(): """Ensure that our temp file is removed.""" if locals().get('tfile') is not None: basic.remove_file(tfile) def _time_difference(resp, obj): if ARGS.get('save_newer') is True: # Get the source object last modified time. compare_time = resp.getheader('last_modified') if compare_time is None: return True elif cloud.time_delta(compare_time=compare_time, lmobj=obj['last_modified']) is True: return False else: return True else: return True def _compare(resp, obj): if resp.status == 404: report.reporter( msg='Target Object %s not found' % obj['name'], prt=False ) return True elif resp.getheader('etag') != obj['hash']: report.reporter( msg='Checksum Mismatch on Target Object %s' % obj['name'], prt=False, lvl='debug' ) return _time_difference(resp, obj) else: return False fheaders = self.payload['headers'] for retry in basic.retryloop(attempts=ARGS.get('error_retry'), delay=5, obj=obj['name']): # Open connection and perform operation fmt, date, date_delta, now = basic.time_stamp() spath = http.quoter(url=surl.path, cont=scontainer, ufile=obj['name']) tpath = http.quoter(url=turl.path, cont=tcontainer, ufile=obj['name']) conn = http.open_connection(url=turl) with meth.operation(retry, conn=conn, obj=obj): resp = self._header_getter(conn=conn, rpath=tpath, fheaders=fheaders, retry=retry) # If object comparison is True GET then PUT object if _compare(resp=resp, obj=obj) is not True: return None try: # Open Connection for source Download conn = http.open_connection(url=surl) with meth.operation(retry, conn=conn, obj=obj): # make a temp file. tfile = basic.create_tmp() # Make a connection resp = self._header_getter(conn=conn, rpath=spath, fheaders=fheaders, retry=retry) sheaders = dict(resp.getheaders()) # TODO(kevin) add the ability to short upload if timestamp # TODO(kevin) ... is newer on the target. # GET remote Object self._downloader( conn=conn, rpath=spath, fheaders=fheaders, lfile=tfile, source=None, retry=retry, skip=True ) for nretry in basic.retryloop(attempts=ARGS.get('error_retry'), delay=5, obj=obj): # open connection for target upload. conn = http.open_connection(url=turl) with meth.operation(retry, conn=conn, obj=obj, cleanup=_cleanup): resp = self._header_getter(conn=conn, rpath=tpath, fheaders=fheaders, retry=nretry) self.resp_exception(resp=resp, rty=nretry) # PUT remote object self._putter(conn=conn, fpath=tfile, rpath=tpath, fheaders=fheaders, retry=nretry, skip=True) # let the system rest for 3 seconds. basic.stupid_hack(wait=3) # With the source headers POST new headers on target if ARGS.get('clone_headers') is True: resp = self._header_getter(conn=conn, rpath=tpath, fheaders=fheaders, retry=nretry) theaders = dict(resp.getheaders()) for key in sheaders.keys(): if key not in theaders: fheaders.update({key: sheaders[key]}) # Force the SOURCE content Type on the Target. fheaders.update( {'content-type': sheaders.get('content-type')} ) self._header_poster( conn=conn, rpath=tpath, fheaders=fheaders, retry=nretry ) finally: _cleanup()