def _upload(self, package): data = package.metadata_dictionary() data.update({ # action ":action": "file_upload", "protcol_version": "1", }) data_to_send = self._convert_data_to_list_of_tuples(data) print("Uploading {0}".format(package.basefilename)) with open(package.filename, "rb") as fp: data_to_send.append(( "content", (package.basefilename, fp, "application/octet-stream"), )) encoder = MultipartEncoder(data_to_send) bar = ProgressBar(expected_size=encoder.len, filled_char='=') monitor = MultipartEncoderMonitor( encoder, lambda monitor: bar.show(monitor.bytes_read)) resp = self.session.post( self.url, data=monitor, allow_redirects=False, headers={'Content-Type': monitor.content_type}, ) bar.done() return resp
def download_artifacts(session): response = session.get("{0}/lastBuild/api/json/".format(JENKINS_URL), headers={"Accept": "application/json"}) response.raise_for_status() json_response = response.json() assert not json_response["building"] assert json_response["result"] == "SUCCESS" paths = [] for artifact in json_response["artifacts"]: response = session.get("{0}artifact/{1}".format( json_response["url"], artifact["relativePath"]), stream=True) assert response.headers["content-length"] print("Downloading {0}".format(artifact["fileName"])) bar = ProgressBar(expected_size=int( response.headers["content-length"]), filled_char="=") content = io.BytesIO() for data in response.iter_content(chunk_size=8192): content.write(data) bar.show(content.tell()) assert bar.expected_size == content.tell() bar.done() out_path = os.path.join( os.path.dirname(__file__), "dist", artifact["fileName"], ) with open(out_path, "wb") as f: f.write(content.getvalue()) paths.append(out_path) return paths
def _upload(self, package): data = package.metadata_dictionary() data.update({ # action ":action": "file_upload", "protcol_version": "1", }) data_to_send = self._convert_data_to_list_of_tuples(data) print("Uploading {0}".format(package.basefilename)) with open(package.filename, "rb") as fp: data_to_send.append(( "content", (package.basefilename, fp, "application/octet-stream"), )) encoder = MultipartEncoder(data_to_send) bar = ProgressBar(expected_size=encoder.len, filled_char='=') monitor = MultipartEncoderMonitor( encoder, lambda monitor: bar.show(monitor.bytes_read) ) resp = self.session.post( self.url, data=monitor, allow_redirects=False, headers={'Content-Type': monitor.content_type}, ) bar.done() return resp
def resume_upload(self, file_path, file_endpoint, chunk_size=None, headers=None, auth=None, offset=None): chunk_size = chunk_size or self.chunk_size try: offset = self._get_offset(file_endpoint, headers=headers, auth=auth) except FloydException as e: floyd_logger.error( "Failed to fetch offset data from upload server! %s", e.message) return False except requests.exceptions.ConnectionError as e: floyd_logger.error( "Cannot connect to the Floyd data upload server for offset. " "Check your internet connection.") return False total_sent = 0 file_size = os.path.getsize(file_path) with open(file_path, 'rb') as f: pb = ProgressBar(filled_char="=", expected_size=file_size) while offset < file_size: pb.show(offset) f.seek(offset) data = f.read(chunk_size) try: offset = self._upload_chunk(data, offset, file_endpoint, headers=headers, auth=auth) total_sent += len(data) floyd_logger.debug("%s bytes sent", total_sent) except FloydException as e: floyd_logger.error( "Failed to fetch offset data from upload server! %s", e.message) return False except requests.exceptions.ConnectionError as e: floyd_logger.error( "Cannot connect to the Floyd data upload server. " "Check your internet connection.") return False # Complete the progress bar with one more call to show() pb.show(offset) pb.done() return True
def download_artifacts(session): response = session.get( "{0}/lastBuild/api/json/".format(JENKINS_URL), headers={ "Accept": "application/json" } ) response.raise_for_status() assert not response.json()["building"] assert response.json()["result"] == "SUCCESS" paths = [] last_build_number = response.json()["number"] for run in response.json()["runs"]: if run["number"] != last_build_number: print( "Skipping {0} as it is not from the latest build ({1})".format( run["url"], last_build_number ) ) continue response = session.get( run["url"] + "api/json/", headers={ "Accept": "application/json", } ) response.raise_for_status() for artifact in response.json()["artifacts"]: response = session.get( "{0}artifact/{1}".format(run["url"], artifact["relativePath"]), stream=True ) assert response.headers["content-length"] print("Downloading {0}".format(artifact["fileName"])) bar = ProgressBar( expected_size=int(response.headers["content-length"]), filled_char="=" ) content = io.BytesIO() for data in response.iter_content(chunk_size=8192): content.write(data) bar.show(content.tell()) assert bar.expected_size == content.tell() bar.done() out_path = os.path.join( os.path.dirname(__file__), "dist", artifact["fileName"], ) with open(out_path, "wb") as f: f.write(content.getvalue()) paths.append(out_path) return paths
class FileChunks(object): def __init__(self, filename, chunk_size=2 * 1024 * 1024, progress_hook=None): self.chunk_size = chunk_size self.amount_seen = 0 self.filename = filename self.file_obj = open(filename, 'rb') self.file_size = os.fstat(self.file_obj.fileno()).st_size self.progress_hook = progress_hook if self.progress_hook is None and ProgressBar is not None: self.progress_bar = ProgressBar(label=filename, expected_size=self.file_size, filled_char='=') #self.progress_bar.show(self.amount_seen) else: self.progress_bar = None def get(self): try: data = self.file_obj.read(self.chunk_size) while len(data) > 0: self.amount_seen += len(data) if self.progress_hook is not None: self.progress_hook(self.filename, self.amount_seen, self.file_size) if self.progress_bar is not None: self.progress_bar.show(self.amount_seen) if self.progress_bar is None and self.progress_hook is None: print('Uploading %s: %s / %s\r' % (self.filename, self.amount_seen, self.file_size)) yield data data = self.file_obj.read(self.chunk_size) finally: if self.progress_bar is not None: self.progress_bar.done() if self.progress_bar is None and self.progress_hook is None: print('Done Uploading %s' % (self.filename, )) self.file_obj.close() def close(self): if not self.file_obj.closed: self.file_obj.close()
def wget(url,target=None,printhook=None,proxies=None,connect_timeout=10,download_timeout=None,verify_cert=False,referer=None,user_agent=None,cert=None,resume=False,md5=None,sha1=None,sha256=None,cache_dir=None): r"""Copy the contents of a file from a given URL to a local file. Args: url (str): URL to document target (str) : full file path of downloaded file. If None, put in a temporary dir with supplied url filename (final part of url) proxies (dict) : proxies to use. eg {'http':'http://wpad:3128','https':'http://wpad:3128'} timeout (int) : seconds to wait for answer before giving up auth (list) : (user,password) to authenticate wirh basic auth verify_cert (bool or str) : either False, True (verify with embedded CA list), or path to a directory or PEM encoded CA bundle file to check https certificate signature against. cert (list) : pair of (x509certfilename,pemkeyfilename) for authenticating the client referer (str): user_agent: resume (bool): md5 (str) : sha1 (str) : sha256 (str) : cache_dir (str) : if file exists here, and md5 matches, copy from here instead of downloading. If not, put a copy of the file here after downloading. Returns: str : path to downloaded file >>> respath = wget('http://wapt.tranquil.it/wapt/tis-firefox_28.0.0-1_all.wapt','c:\\tmp\\test.wapt',proxies={'http':'http://proxy:3128'}) ??? >>> os.stat(respath).st_size>10000 True >>> respath = wget('http://localhost:8088/runstatus','c:\\tmp\\test.json') ??? """ start_time = time.time() last_time_display = 0.0 last_downloaded = 0 def reporthook(received,total): total = float(total) if total>1 and received>1: # print only every second or at end if (time.time()-start_time>1) and ((time.time()-last_time_display>=1) or (received>=total)): speed = received /(1024.0 * (time.time()-start_time)) if printhook: printhook(received,total,speed,url) elif sys.stdout is not None: try: if received == 0: print(u"Downloading %s (%.1f Mb)" % (url,int(total)/1024/1024)) elif received>=total: print(u" -> download finished (%.0f Kb/s)" % (total /(1024.0*(time.time()+.001-start_time)))) else: print(u'%i / %i (%.0f%%) (%.0f KB/s)\r' % (received,total,100.0*received/total,speed)) except: return False return True else: return False if target is None: target = tempfile.gettempdir() if os.path.isdir(target): target = os.path.join(target,'') (dir,filename) = os.path.split(target) if not filename: url_parts = urlparse.urlparse(url) filename = url_parts.path.split('/')[-1] if not dir: dir = os.getcwd() if not os.path.isdir(dir): os.makedirs(dir) if verify_cert == False: requests.packages.urllib3.disable_warnings() # pylint: disable=no-member header=default_http_headers() if referer != None: header.update({'referer': '%s' % referer}) if user_agent != None: header.update({'user-agent': '%s' % user_agent}) target_fn = os.path.join(dir,filename) # return cached file if md5 matches. if (md5 is not None or sha1 is not None or sha256 is not None) and cache_dir is not None and os.path.isdir(cache_dir): cached_filename = os.path.join(cache_dir,filename) if os.path.isfile(cached_filename): if _check_hash_for_file(cached_filename,md5=md5,sha1=sha1,sha256=sha256): resume = False if cached_filename != target_fn: shutil.copy2(cached_filename,target_fn) return target_fn else: cached_filename = None if os.path.isfile(target_fn) and resume: try: actual_size = os.stat(target_fn).st_size size_req = requests.head(url, proxies=proxies, timeout=connect_timeout, verify=verify_cert, headers=header, cert = cert, allow_redirects=True) target_size = int(size_req.headers['content-length']) file_date = size_req.headers.get('last-modified',None) if target_size > actual_size: header.update({'Range':'bytes=%s-' % (actual_size,)}) write_mode = 'ab' elif target_size < actual_size: target_size = None write_mode = 'wb' except Exception as e: target_size = None write_mode = 'wb' else: file_date = None actual_size = 0 target_size = None write_mode = 'wb' # check hashes if size equal if resume and (md5 is not None or sha1 is not None or sha256 is not None) and target_size is not None and (target_size == actual_size): if not _check_hash_for_file(target_fn,md5=md5,sha1=sha1,sha256=sha256): # restart download... target_size = None write_mode = 'wb' if not resume or target_size is None or (target_size - actual_size) > 0: httpreq = requests.get(url,stream=True, proxies=proxies, timeout=connect_timeout, verify=verify_cert, headers=header, cert = cert, allow_redirects=True) httpreq.raise_for_status() total_bytes = int(httpreq.headers['content-length']) target_free_bytes = get_disk_free_space(os.path.dirname(os.path.abspath(target))) if total_bytes > target_free_bytes: raise Exception('wget : not enough free space on target drive to get %s MB. Total size: %s MB. Free space: %s MB' % (url,total_bytes // (1024*1024),target_free_bytes // (1024*1024))) # 1Mb max, 1kb min chunk_size = min([1024*1024,max([total_bytes/100,2048])]) cnt = 0 if printhook is None and ProgressBar is not None: progress_bar = ProgressBar(label=filename,expected_size=target_size or total_bytes, filled_char='=') progress_bar.show(actual_size) with open(target_fn,write_mode) as output_file: last_time_display = time.time() last_downloaded = 0 if httpreq.ok: for chunk in httpreq.iter_content(chunk_size=chunk_size): output_file.write(chunk) output_file.flush() if download_timeout is not None and (time.time()-start_time>download_timeout): raise requests.Timeout(r'Download of %s takes more than the requested %ss'%(url,download_timeout)) if printhook is None and ProgressBar is not None: if (time.time()-start_time>0.2) and (time.time()-last_time_display>=0.2): progress_bar.show(actual_size + cnt*len(chunk)) last_time_display = time.time() else: if reporthook(cnt*len(chunk),total_bytes): last_time_display = time.time() last_downloaded += len(chunk) cnt +=1 if printhook is None and ProgressBar is not None: progress_bar.show(total_bytes) progress_bar.done() last_time_display = time.time() elif reporthook(last_downloaded,total_bytes): last_time_display = time.time() # check hashes if sha256 is not None: file_hash = _hash_file(target_fn,hash_func=hashlib.sha256) if file_hash != sha256: raise Exception(u'Downloaded file %s sha256 %s does not match expected %s' % (url,file_hash,sha256)) elif sha1 is not None: file_hash = _hash_file(target_fn,hash_func=hashlib.sha1) if file_hash != sha1: raise Exception(u'Downloaded file %s sha1 %s does not match expected %s' % (url,file_hash,sha1)) elif md5 is not None: file_hash = _hash_file(target_fn,hash_func=hashlib.md5) if file_hash != md5: raise Exception(u'Downloaded file %s md5 %s does not match expected %s' % (url,file_hash,md5)) file_date = httpreq.headers.get('last-modified',None) if file_date: file_datetime = httpdatetime2time(file_date) os.utime(target_fn,(file_datetime,file_datetime)) # cache result if cache_dir: if not os.path.isdir(cache_dir): os.makedirs(cache_dir) cached_filename = os.path.join(cache_dir,filename) if target_fn != cached_filename: shutil.copy2(target_fn,cached_filename) return target_fn
class DataCompressor(object): """ Local Data Compression with progress bar. """ def __init__(self, source_dir, filename): # Data directory to compress self.source_dir = source_dir # Archive (Tar file) name # e.g. "/tmp/contents.tar.gz" self.filename = filename # Prgress Bar for tracking data compression self.__compression_bar = None # Number of files to compress self.__files_to_compress = 0 self.__get_nfiles_to_compress() # Number of files already compressed self.__files_compressed = 0 def __get_nfiles_to_compress(self): """ Return the number of files to compress Note: it should take about 0.1s for counting 100k files on a dual core machine """ floyd_logger.info( "Get number of files to compress... (this could take a few seconds)" ) paths = [self.source_dir] try: # Traverse each subdirs of source_dir and count files/dirs while paths: path = paths.pop() for item in scandir(path): if item.is_dir(): paths.append(item.path) self.__files_to_compress += 1 elif item.is_file(): self.__files_to_compress += 1 except OSError as e: # OSError: [Errno 13] Permission denied if e.errno == errno.EACCES: self.source_dir = os.getcwd( ) if self.source_dir == '.' else self.source_dir # Expand cwd sys.exit( ("Permission denied. Make sure to have read permission " "for all the files and directories in the path: %s") % (self.source_dir)) floyd_logger.info("Compressing %d files", self.__files_to_compress) def create_tarfile(self): """ Create a tar file with the contents of the current directory """ floyd_logger.info("Compressing data...") # Show progress bar (file_compressed/file_to_compress) self.__compression_bar = ProgressBar( expected_size=self.__files_to_compress, filled_char='=') # Auxiliary functions def dfilter_file_counter(tarinfo): """ Dummy filter function used to track the progression at file levels. """ self.__compression_bar.show(self.__files_compressed) self.__files_compressed += 1 return tarinfo def warn_purge_exit(info_msg, filename, progress_bar, exit_msg): """ Warn the user that's something went wrong, remove the tarball and provide an exit message. """ progress_bar.done() floyd_logger.info(info_msg) rmtree(os.path.dirname(filename)) sys.exit(exit_msg) try: # Define the default signal handler for catching: Ctrl-C signal.signal(signal.SIGINT, signal.default_int_handler) with tarfile.open(self.filename, "w:gz") as tar: tar.add(self.source_dir, arcname=os.path.basename(self.source_dir), filter=dfilter_file_counter) self.__compression_bar.done() except (OSError, IOError) as e: # OSError: [Errno 13] Permission denied if e.errno == errno.EACCES: self.source_dir = os.getcwd( ) if self.source_dir == '.' else self.source_dir # Expand cwd warn_purge_exit( info_msg="Permission denied. Removing compressed data...", filename=self.filename, progress_bar=self.__compression_bar, exit_msg=( "Permission denied. Make sure to have read permission " "for all the files and directories in the path: %s") % (self.source_dir)) # OSError: [Errno 28] No Space Left on Device (IOError on python2.7) elif e.errno == errno.ENOSPC: dir_path = os.path.dirname(self.filename) warn_purge_exit( info_msg="No space left. Removing compressed data...", filename=self.filename, progress_bar=self.__compression_bar, exit_msg= ("No space left when compressing your data in: %s.\n" "Make sure to have enough space before uploading your data." ) % (os.path.abspath(dir_path))) except KeyboardInterrupt: # Purge tarball on Ctrl-C warn_purge_exit( info_msg="Ctrl-C signal detected: Removing compressed data...", filename=self.filename, progress_bar=self.__compression_bar, exit_msg="Stopped the data upload gracefully.")