def test_file_exists(self): return_file = tempfile.mkstemp()[1] self.assertEqual(basic_utils.file_exists(filename=return_file), True) try: os.remove(return_file) except OSError: pass else: self.assertEqual(basic_utils.file_exists(filename=return_file), False)
def test_file_exists(self): return_file = tempfile.mkstemp()[1] self.assertEqual( basic_utils.file_exists(filename=return_file), True ) try: os.remove(return_file) except OSError: pass else: self.assertEqual( basic_utils.file_exists(filename=return_file), False )
def _putter(self, conn, fpath, rpath, fheaders, retry, skip=False): """Place object into the container. :param conn: :param fpath: :param rpath: :param fheaders: :param retry: """ if self._checker(conn, rpath, fpath, fheaders, retry, skip) is True: report.reporter(msg='OBJECT ORIGIN %s RPATH %s' % (fpath, rpath), prt=False, lvl='debug') if basic.file_exists(fpath) is False: return None else: with open(fpath, 'rb') as f_open: conn.request('PUT', rpath, body=f_open, headers=fheaders) resp, read = http.response_get(conn=conn, retry=retry) self.resp_exception(resp=resp, rty=retry) report.reporter(msg=('MESSAGE %s %s %s' % (resp.status, resp.reason, resp.msg)), prt=False, lvl='debug')
def _putter(self, conn, fpath, rpath, fheaders, retry, skip=False): """Place object into the container. :param conn: :param fpath: :param rpath: :param fheaders: :param retry: """ if self._checker(conn, rpath, fpath, fheaders, retry, skip) is True: report.reporter( msg='OBJECT ORIGIN %s RPATH %s' % (fpath, rpath), prt=False, lvl='debug' ) if basic.file_exists(fpath) is False: return None else: with open(fpath, 'rb') as f_open: conn.request('PUT', rpath, body=f_open, headers=fheaders) resp, read = http.response_get(conn=conn, retry=retry) self.resp_exception(resp=resp, rty=retry) report.reporter( msg=('MESSAGE %s %s %s' % (resp.status, resp.reason, resp.msg)), prt=False, lvl='debug' )
def _putter(self, url, fpath, rpath, fheaders, skip=False): """Place object into the container. :param url: :param fpath: :param rpath: :param fheaders: """ if self._checker(url, rpath, fpath, fheaders, skip) is True: report.reporter( msg='OBJECT ORIGIN %s RPATH %s' % (fpath, rpath), prt=False, lvl='debug' ) if basic.file_exists(fpath) is False: return None else: with open(fpath, 'rb') as f_open: resp = http.put_request( url=url, rpath=rpath, body=f_open, headers=fheaders ) self.resp_exception(resp=resp) report.reporter( msg=('MESSAGE %s %s %s' % (resp.status_code, resp.reason, resp.request)), prt=False, lvl='debug' )
def _putter(self, url, fpath, rpath, fheaders, skip=False): """Place object into the container. :param url: :param fpath: :param rpath: :param fheaders: """ if self._checker(url, rpath, fpath, fheaders, skip) is True: report.reporter(msg="OBJECT ORIGIN %s RPATH %s" % (fpath, rpath), prt=False, lvl="debug") if basic.file_exists(fpath) is False: return None else: if os.path.islink(fpath): link = os.readlink(fpath) lpath = os.path.abspath(os.path.join(os.path.dirname(fpath), link)) rpath_reversed = rpath.split("/")[::-1] fpath_reversed = fpath.split("/")[::-1] a, b = sorted((rpath_reversed, fpath_reversed), key=len) for i, j in enumerate(a): if j != b[i]: index = i break container = rpath_reversed[index] container_dir = fpath.replace("/".join(rpath_reversed[:index][::-1]), "") if container_dir in lpath: manafest = container + "/" + lpath.replace(container_dir, "") fheaders["X-Object-Manifest"] = manafest resp = http.put_request(url=url, rpath=rpath, body=None, headers=fheaders) self.resp_exception(resp=resp) else: report.reporter( msg="symlink %s points to location %s which is" " outside uploading directory" % (fpath, lpath), lvl="warning", ) else: with open(fpath, "rb") as f_open: resp = http.put_request(url=url, rpath=rpath, body=f_open, headers=fheaders) self.resp_exception(resp=resp)
def _putter(self, url, fpath, rpath, fheaders, skip=False): """Place object into the container. :param url: :param fpath: :param rpath: :param fheaders: """ if self._checker(url, rpath, fpath, fheaders, skip) is True: report.reporter(msg="OBJECT ORIGIN %s RPATH %s" % (fpath, rpath), prt=False, lvl="debug") if basic.file_exists(fpath) is False: return None else: with open(fpath, "rb") as f_open: resp = http.put_request(url=url, rpath=rpath, body=f_open, headers=fheaders) self.resp_exception(resp=resp)
def _putter(self, url, fpath, rpath, fheaders, skip=False): """Place object into the container. :param url: :param fpath: :param rpath: :param fheaders: """ if self._checker(url, rpath, fpath, fheaders, skip) is True: report.reporter(msg='OBJECT ORIGIN %s RPATH %s' % (fpath, rpath), prt=False, lvl='debug') if basic.file_exists(fpath) is False: return None else: with open(fpath, 'rb') as f_open: resp = http.put_request(url=url, rpath=rpath, body=f_open, headers=fheaders) self.resp_exception(resp=resp)
def compress_files(file_list): """If the archive function is used, create a compressed archive. :param file_list: This function allows for multiple sources to be added to the compressed archive. """ tmp_file = None try: # Set date and time date_format = '%a%b%d.%H.%M.%S.%Y' today = datetime.datetime.today() _ts = today.strftime(date_format) # Get Home Directory home_dir = os.getenv('HOME') # Set the name of the archive. set_name = ARGS.get('tar_name', '%s_%s' % ('Archive', _ts)) file_name = '%s.tgz' % set_name # Set the working File. tmp_file = basic.jpath(root=home_dir, inode=file_name) # Begin creating the Archive. tar = tarfile.open(tmp_file, 'w:gz') for name in file_list: if basic.file_exists(name) is True: tar.add(name) tar.close() report.reporter(msg='ARCHIVE CREATED: %s' % tmp_file, prt=False) if ARGS.get('verify'): tar_len = tarfile.open(tmp_file, 'r') ver_array = [] for member_info in tar_len.getmembers(): ver_array.append(member_info.name) count = len(ver_array) orig_count = len(file_list) if orig_count != count: raise turbo.SystemProblem( 'ARCHIVE NOT VERIFIED: Archive and File List do not Match.' ' Original File Count = %s, Found Archive Contents = %s' % (orig_count, count) ) report.reporter( msg='ARCHIVE CONTENTS VERIFIED: %s files' % count, ) except KeyboardInterrupt: if basic.file_exists(tmp_file): basic.remove_file(tmp_file) turbo.emergency_exit('I have stopped at your command,' ' I removed Local Copy of the Archive') except Exception as exp: if basic.file_exists(tmp_file): basic.remove_file(tmp_file) turbo.emergency_exit( 'I am sorry i just don\'t know what you put into me, Removing' ' Local Copy of the Archive.' ) turbo.emergency_exit( 'Exception while working on the archive. MESSAGE: %s' % exp ) else: return tmp_file
def _putter(self, url, fpath, rpath, fheaders, skip=False): """Place object into the container. :param url: :param fpath: :param rpath: :param fheaders: """ if self._checker(url, rpath, fpath, fheaders, skip) is True: report.reporter(msg='OBJECT ORIGIN %s RPATH %s' % (fpath, rpath), prt=False, lvl='debug') if basic.file_exists(fpath) is False: return None else: fheaders['content-type'] = "application/x-bzip2" opener = self._dispatch_open(fpath) if opener is None: return None with opener(fpath, 'rb') as f_open: compressor = bz2.BZ2Compressor() buf = "" buf_len = 0 chunk_size = 2**20 * 100 line_number = 1 last_line_number = 1 resp = self._header_getter(url=url, rpath="%s-%s.bz2" % (rpath, last_line_number), fheaders=fheaders) skip = resp.status_code >= 200 \ and resp.status_code < 300 if skip: report.reporter( msg="Skipping upload of part %s in %s" % \ (last_line_number, rpath), lvl="info", prt=False, log=True) while True: line = f_open.readline() line_number += 1 if line == "": if buf_len != 0: data = compressor.flush() if data: buf += data if not skip: resp = http.put_request( url=url, rpath="%s-%s.bz2" % (rpath, last_line_number), body=buf, headers=fheaders) self.resp_exception(resp=resp) break if buf_len + len(line) > chunk_size: if line_number == last_line_number + 1: report.reporter( msg= "Skipping upload due to %d size lines: %s" % (chunk_size, fpath), lvl="info", prt=False, log=True) break data = compressor.flush() if data is not None: buf += data if not skip: resp = http.put_request( url=url, rpath="%s-%s.bz2" % (rpath, last_line_number), body=buf, headers=fheaders) self.resp_exception(resp=resp) last_line_number = line_number - 1 buf = "" buf_len = 0 resp = self._header_getter( url=url, rpath="%s-%s.bz2" % (rpath, last_line_number), fheaders=fheaders) skip = resp.status_code >= 200 \ and resp.status_code < 300 if skip: report.reporter( msg="Skipping upload of part %s in %s" % \ (last_line_number, rpath), lvl="info", prt=False, log=True) compressor = bz2.BZ2Compressor() buf_len += len(line) data = compressor.compress(line if not skip else "") if data is not None: buf += data
def compress_files(file_list): """If the archive function is used, create a compressed archive. :param file_list: This function allows for multiple sources to be added to the compressed archive. """ tmp_file = None try: # Set date and time date_format = '%a%b%d.%H.%M.%S.%Y' today = datetime.datetime.today() _ts = today.strftime(date_format) # Get Home Directory home_dir = os.getenv('HOME') # Set the name of the archive. set_name = ARGS.get('tar_name', '%s_%s' % ('Archive', _ts)) file_name = '%s.tgz' % set_name # Set the working File. tmp_file = basic.jpath(root=home_dir, inode=file_name) # Begin creating the Archive. tar = tarfile.open(tmp_file, 'w:gz') for name in file_list: if basic.file_exists(name) is True: tar.add(name) tar.close() report.reporter(msg='ARCHIVE CREATED: %s' % tmp_file, prt=False) if ARGS.get('verify'): tar_len = tarfile.open(tmp_file, 'r') ver_array = [] for member_info in tar_len.getmembers(): ver_array.append(member_info.name) count = len(ver_array) orig_count = len(file_list) if orig_count != count: raise turbo.SystemProblem( 'ARCHIVE NOT VERIFIED: Archive and File List do not Match.' ' Original File Count = %s, Found Archive Contents = %s' % (orig_count, count)) report.reporter(msg='ARCHIVE CONTENTS VERIFIED: %s files' % count, ) except KeyboardInterrupt: if basic.file_exists(tmp_file): basic.remove_file(tmp_file) turbo.emergency_exit('I have stopped at your command,' ' I removed Local Copy of the Archive') except Exception as exp: if basic.file_exists(tmp_file): basic.remove_file(tmp_file) turbo.emergency_exit( 'I am sorry i just don\'t know what you put into me, Removing' ' Local Copy of the Archive.') turbo.emergency_exit( 'Exception while working on the archive. MESSAGE: %s' % exp) else: return tmp_file