def _upload(self, package): data = package.metadata_dictionary() data.update({ # action ":action": "file_upload", "protcol_version": "1", }) data_to_send = self._convert_data_to_list_of_tuples(data) print("Uploading {0}".format(package.basefilename)) with open(package.filename, "rb") as fp: data_to_send.append(( "content", (package.basefilename, fp, "application/octet-stream"), )) encoder = MultipartEncoder(data_to_send) bar = ProgressBar(expected_size=encoder.len, filled_char='=') monitor = MultipartEncoderMonitor( encoder, lambda monitor: bar.show(monitor.bytes_read)) resp = self.session.post( self.url, data=monitor, allow_redirects=False, headers={'Content-Type': monitor.content_type}, ) bar.done() return resp
def _upload(self, package): data = package.metadata_dictionary() data.update({ # action ":action": "file_upload", "protcol_version": "1", }) data_to_send = self._convert_data_to_list_of_tuples(data) print("Uploading {0}".format(package.basefilename)) with open(package.filename, "rb") as fp: data_to_send.append(( "content", (package.basefilename, fp, "application/octet-stream"), )) encoder = MultipartEncoder(data_to_send) bar = ProgressBar(expected_size=encoder.len, filled_char='=') monitor = MultipartEncoderMonitor( encoder, lambda monitor: bar.show(monitor.bytes_read) ) resp = self.session.post( self.url, data=monitor, allow_redirects=False, headers={'Content-Type': monitor.content_type}, ) bar.done() return resp
def download_artifacts(session): response = session.get("{0}/lastBuild/api/json/".format(JENKINS_URL), headers={"Accept": "application/json"}) response.raise_for_status() json_response = response.json() assert not json_response["building"] assert json_response["result"] == "SUCCESS" paths = [] for artifact in json_response["artifacts"]: response = session.get("{0}artifact/{1}".format( json_response["url"], artifact["relativePath"]), stream=True) assert response.headers["content-length"] print("Downloading {0}".format(artifact["fileName"])) bar = ProgressBar(expected_size=int( response.headers["content-length"]), filled_char="=") content = io.BytesIO() for data in response.iter_content(chunk_size=8192): content.write(data) bar.show(content.tell()) assert bar.expected_size == content.tell() bar.done() out_path = os.path.join( os.path.dirname(__file__), "dist", artifact["fileName"], ) with open(out_path, "wb") as f: f.write(content.getvalue()) paths.append(out_path) return paths
def download(url, dest): m = re.match(r'^https://send.firefox.com/download/(\w+)/#([\w_-]+)$', url) if not m: raise ValueError("URL format appears to be incorrect") fid = m.group(1) key = base64.urlsafe_b64decode(m.group(2) + '==') print("Downloading %s..." % url) url = "https://send.firefox.com/api/download/" + fid resp = requests.get(url, stream=True) resp.raise_for_status() flen = int(resp.headers.get('Content-Length')) metadata = json.loads(resp.headers.get('X-File-Metadata')) filename = metadata['filename'] if os.path.isdir(dest): filename = os.path.join(dest, filename) else: filename = dest iv = binascii.unhexlify(metadata['id']) cipher = AES.new(key, AES.MODE_GCM, iv) ho = sha256() print("Downloading to %s..." % filename) try: with open(filename + '.tmp', 'wb') as outf: bar = ProgressBar(expected_size=flen, filled_char='=') dl = 0 tag = b'' taglen = 16 for data in resp.iter_content(chunk_size=8192): dl += len(data) bar.show(dl) if dl > flen - taglen: dend = max(len(data) - (dl - (flen - taglen)), 0) tag += data[dend:] data = data[:dend] chunk = cipher.decrypt(data) ho.update(chunk) outf.write(chunk) if len(tag) == taglen: break print() cipher.verify(tag) except Exception as e: print("File download failed:", e) os.unlink(filename + '.tmp') else: os.rename(filename + '.tmp', filename) print("Done, file verified!")
def download(fid, secret, dest, password=None, url=None): metadata, nonce = get_metadata(fid, secret, password, url) encryptKey = deriveFileKey(secret) authKey = deriveAuthKey(secret, password, url) sig = hmac.new(authKey, nonce, sha256).digest() url = "https://send.firefox.com/api/download/" + fid resp = requests.get(url, headers={'Authorization': 'send-v1 ' + b64encode(sig)}, stream=True) resp.raise_for_status() flen = int(resp.headers.get('Content-Length')) filename = metadata['metadata']['name'] if os.path.isdir(dest): filename = os.path.join(dest, filename) else: filename = dest iv = b64decode(metadata['metadata']['iv']) cipher = AES.new(encryptKey, AES.MODE_GCM, iv, mac_len=16) ho = sha256() print("Downloading to %s..." % filename) try: with open(filename + '.tmp', 'wb') as outf: bar = ProgressBar(expected_size=flen, filled_char='=') dl = 0 tag = b'' taglen = 16 for data in resp.iter_content(chunk_size=8192): dl += len(data) bar.show(dl) if dl > flen - taglen: dend = max(len(data) - (dl - (flen - taglen)), 0) tag += data[dend:] data = data[:dend] chunk = cipher.decrypt(data) ho.update(chunk) outf.write(chunk) if len(tag) == taglen: break print() cipher.verify(tag) except Exception as e: print("File download failed:", e) os.unlink(filename + '.tmp') else: os.rename(filename + '.tmp', filename) print("Done, file verified!")
class CallBack: def __init__(self): self.bar = None def __call__(self, size_rd, total): if self.bar is None: self.bar = ProgressBar(expected_size=total, filled_char='=') self.bar.show(size_rd)
def download_artifacts(session): response = session.get( "{0}/lastBuild/api/json/".format(JENKINS_URL), headers={ "Accept": "application/json" } ) response.raise_for_status() assert not response.json()["building"] assert response.json()["result"] == "SUCCESS" paths = [] last_build_number = response.json()["number"] for run in response.json()["runs"]: if run["number"] != last_build_number: print( "Skipping {0} as it is not from the latest build ({1})".format( run["url"], last_build_number ) ) continue response = session.get( run["url"] + "api/json/", headers={ "Accept": "application/json", } ) response.raise_for_status() for artifact in response.json()["artifacts"]: response = session.get( "{0}artifact/{1}".format(run["url"], artifact["relativePath"]), stream=True ) assert response.headers["content-length"] print("Downloading {0}".format(artifact["fileName"])) bar = ProgressBar( expected_size=int(response.headers["content-length"]), filled_char="=" ) content = io.BytesIO() for data in response.iter_content(chunk_size=8192): content.write(data) bar.show(content.tell()) assert bar.expected_size == content.tell() bar.done() out_path = os.path.join( os.path.dirname(__file__), "dist", artifact["fileName"], ) with open(out_path, "wb") as f: f.write(content.getvalue()) paths.append(out_path) return paths
class ProgressBar(): def __init__(self, length): self.bar = Bar(expected_size=length, filled_char='=') self.status = 0 def increment(self, value): self.status += value def progress(self, value): self.bar.show(self.status + value)
def mkprogress(encoder): """ Returns a function that can be used as a callback for :class:`requests_toolbelt.MultipartEncoderMonitor`. :param encoder: instance of :class:`requests_toolbelt.MultipartEncoder` """ expected_size = encoder.len bar = Bar(expected_size=expected_size, filled_char='=') waiting = [False] def callback(monitor): """ prints progess bar :param monitor: instance of :class:`requests_toolbelt.MultipartEncoderMonitor` """ if monitor.bytes_read < expected_size: bar.show(monitor.bytes_read) elif not waiting[0]: waiting[0] = True bar.done() print('\nwaiting for server-side processing ...') return callback
def create_progress_callback(encoder): encoder_len = encoder.len progress_bar = Bar(expected_size=encoder_len, filled_char='=') def callback(monitor): progress_bar.show(monitor.bytes_read) return callback, progress_bar
def do_artifacts_artifact_add(opts): logging.debug('add artifact %r', opts) url = artifacts_url(opts.service) image = { 'name': opts.name, 'description': opts.description, } # build contents of multipart/form-data, image meta must come first, hence # we use an OrderedDict to preserve the order files = OrderedDict() for k, v in image.items(): files[k] = (None, io.StringIO(v)) # followed by firmware data # but first, try to find out the size of firmware data files['size'] = str(os.stat(opts.infile).st_size) files['artifact'] = (opts.infile, open(opts.infile, 'rb'), "application/octet-stream", {}) encoder = MultipartEncoder(files) if sys.stderr.isatty(): try: from requests_toolbelt import MultipartEncoderMonitor from clint.textui.progress import Bar as ProgressBar pb = ProgressBar(expected_size=encoder.len, filled_char='=', every=1024 * 1024) monitor = MultipartEncoderMonitor( encoder, lambda mon: pb.show(mon.bytes_read)) encoder = monitor except ImportError: pass with api_from_opts(opts) as api: rsp = api.post(url, data=encoder, headers={'Content-Type': encoder.content_type}) if rsp.status_code == 201: # created location = rsp.headers.get('Location', '') print("created with URL: {}".format(location)) print('artifact ID: ', location.rsplit('/')[-1]) else: errorprinter(rsp)
class ProgressDownloadStream(object): """ Progress bar buffer class that can monitor a file download by displaying progress to the user while also writing to the desired file. Basic usage: download_stream = ProgressDownloadStream(stream=api_response_obj, expected_size=total_file_bytes) download_stream.stream_to_file(desired_file_path) """ def __init__(self, stream=None, expected_size=None, chunk_size=512): """ :param stream: Streaming API response object returned by the 'requests' library :type: requests.Response :param int expected_size: Total size of file being downloaded (in bytes) :param int chunk_size: Chunk size to use when iterating over streamed response content """ self.stream = stream self.prog_bar = ProgressBar(expected_size=expected_size, filled_char='=') self.progress = 0 self.chunk_size = chunk_size def stream_to_file(self, path): """ Download file to given path and display progress to user :param str path: Desired file path """ # Using default state of files being overwritten for now if os.path.exists(path): os.remove(path) # Stream downloaded contents to file and show progress with open(path, 'wb') as f: for chunk in self.stream.iter_content(chunk_size=self.chunk_size): f.write(chunk) self.progress += int(len(chunk)) self.prog_bar.show(self.progress)
def _download_with_progressbar(response, content_length): """ :type response: requests.Response :type content_length: int """ with Bar(expected_size=content_length) as bar: for i, block in enumerate(response.iter_content(1024), 1): yield block bar.show(min(i * 1024, content_length))
class FileChunks(object): def __init__(self, filename, chunk_size=2 * 1024 * 1024, progress_hook=None): self.chunk_size = chunk_size self.amount_seen = 0 self.filename = filename self.file_obj = open(filename, 'rb') self.file_size = os.fstat(self.file_obj.fileno()).st_size self.progress_hook = progress_hook if self.progress_hook is None and ProgressBar is not None: self.progress_bar = ProgressBar(label=filename, expected_size=self.file_size, filled_char='=') #self.progress_bar.show(self.amount_seen) else: self.progress_bar = None def get(self): try: data = self.file_obj.read(self.chunk_size) while len(data) > 0: self.amount_seen += len(data) if self.progress_hook is not None: self.progress_hook(self.filename, self.amount_seen, self.file_size) if self.progress_bar is not None: self.progress_bar.show(self.amount_seen) if self.progress_bar is None and self.progress_hook is None: print('Uploading %s: %s / %s\r' % (self.filename, self.amount_seen, self.file_size)) yield data data = self.file_obj.read(self.chunk_size) finally: if self.progress_bar is not None: self.progress_bar.done() if self.progress_bar is None and self.progress_hook is None: print('Done Uploading %s' % (self.filename, )) self.file_obj.close() def close(self): if not self.file_obj.closed: self.file_obj.close()
def resume_upload(self, file_path, file_endpoint, chunk_size=None, headers=None, auth=None, offset=None): chunk_size = chunk_size or self.chunk_size try: offset = self._get_offset(file_endpoint, headers=headers, auth=auth) except FloydException as e: floyd_logger.error( "Failed to fetch offset data from upload server! %s", e.message) return False except requests.exceptions.ConnectionError as e: floyd_logger.error( "Cannot connect to the Floyd data upload server for offset. " "Check your internet connection.") return False total_sent = 0 file_size = os.path.getsize(file_path) with open(file_path, 'rb') as f: pb = ProgressBar(filled_char="=", expected_size=file_size) while offset < file_size: pb.show(offset) f.seek(offset) data = f.read(chunk_size) try: offset = self._upload_chunk(data, offset, file_endpoint, headers=headers, auth=auth) total_sent += len(data) floyd_logger.debug("%s bytes sent", total_sent) except FloydException as e: floyd_logger.error( "Failed to fetch offset data from upload server! %s", e.message) return False except requests.exceptions.ConnectionError as e: floyd_logger.error( "Cannot connect to the Floyd data upload server. " "Check your internet connection.") return False # Complete the progress bar with one more call to show() pb.show(offset) pb.done() return True
def __init__(self, filename, chunk_size=2*1024*1024,progress_hook=None): self.chunk_size = chunk_size self.amount_seen = 0 self.filename = filename self.file_obj = open(filename,'rb') self.file_size = os.fstat(self.file_obj.fileno()).st_size self.progress_hook = progress_hook if self.progress_hook is None and ProgressBar is not None: self.progress_bar = ProgressBar(label=filename,expected_size=self.file_size, filled_char='=') #self.progress_bar.show(self.amount_seen) else: self.progress_bar = None
def course_exports(course_list): '''Do course exports for each course in the list''' #pprint.pprint(course_list) bar = Bar(label='Exporting Courses ',expected_size=len(course_list)) export_list = [] for idx,course in enumerate(course_list): export_list.append(course_export(course)) bar.show(idx) blen = len(export_list) bar.label = "Checking Export Statuses: " bar.expected_length = blen bar.show(1) while export_list: if len(export_list) < 10: time.sleep(2) bar.show(blen-len(export_list)) #print export_list for idx,cm in enumerate(export_list): course,res = cm url = res.get('progress_url',None) if not url: del(export_list[idx]) print 'not able to generate export for course',course else: download_progress = requests.get(url,headers=config['headers']).json() bar.label = "Checking Export Status: {} {}% ".format(course[course_id_field],download_progress['completion']) if download_progress['workflow_state'] not in ['queued','running']: if download_progress['workflow_state'] == 'completed': url = "https://{domain}/api/v1/courses/{}/content_exports/{}".format(course[course_id_field],res['id'],**config) export_info = requests.get(url,headers=config['headers']).json() #print 'export_info',export_info export_url = export_info['attachment']['url'] course['export_url'] = export_url yield course else: course['export_url'] = None del(export_list[idx])
def create_console_callback(size): """ Create progress callback suitable for displaying the progress of a requests_toolbelt.MultipartEncoderMonitor to a text console. @param size int representing the number of bytes to be read be the MultipartEncoderMonitor """ bar = Bar(expected_size=size, filled_char='\u25A0', every=1024) def callback(monitor): bar.show(monitor.bytes_read) return callback
def __init__(self, stream=None, expected_size=None, chunk_size=512): """ :param stream: Streaming API response object returned by the 'requests' library :type: requests.Response :param int expected_size: Total size of file being downloaded (in bytes) :param int chunk_size: Chunk size to use when iterating over streamed response content """ self.stream = stream self.prog_bar = ProgressBar(expected_size=expected_size, filled_char='=') self.progress = 0 self.chunk_size = chunk_size
def runMigrations(copies): pool = Pool(processes=num_processes) #copies.reverse() bar = Bar() #res = pool.map(massDoCopies,((bar,x) for x in copies)) #for x in copies: # massDoCopies((bar,x)) res = pool.map_async(massDoCopies, ((bar, x) for x in copies)) stats = [] try: stats.append(res.get(0xFFFF)) except KeyboardInterrupt: print 'kill processes' #pool.terminate() exit() except TypeError, err: print 'err', err pass
def wget(url,target=None,printhook=None,proxies=None,connect_timeout=10,download_timeout=None,verify_cert=False,referer=None,user_agent=None,cert=None,resume=False,md5=None,sha1=None,sha256=None,cache_dir=None): r"""Copy the contents of a file from a given URL to a local file. Args: url (str): URL to document target (str) : full file path of downloaded file. If None, put in a temporary dir with supplied url filename (final part of url) proxies (dict) : proxies to use. eg {'http':'http://wpad:3128','https':'http://wpad:3128'} timeout (int) : seconds to wait for answer before giving up auth (list) : (user,password) to authenticate wirh basic auth verify_cert (bool or str) : either False, True (verify with embedded CA list), or path to a directory or PEM encoded CA bundle file to check https certificate signature against. cert (list) : pair of (x509certfilename,pemkeyfilename) for authenticating the client referer (str): user_agent: resume (bool): md5 (str) : sha1 (str) : sha256 (str) : cache_dir (str) : if file exists here, and md5 matches, copy from here instead of downloading. If not, put a copy of the file here after downloading. Returns: str : path to downloaded file >>> respath = wget('http://wapt.tranquil.it/wapt/tis-firefox_28.0.0-1_all.wapt','c:\\tmp\\test.wapt',proxies={'http':'http://proxy:3128'}) ??? >>> os.stat(respath).st_size>10000 True >>> respath = wget('http://localhost:8088/runstatus','c:\\tmp\\test.json') ??? """ start_time = time.time() last_time_display = 0.0 last_downloaded = 0 def reporthook(received,total): total = float(total) if total>1 and received>1: # print only every second or at end if (time.time()-start_time>1) and ((time.time()-last_time_display>=1) or (received>=total)): speed = received /(1024.0 * (time.time()-start_time)) if printhook: printhook(received,total,speed,url) elif sys.stdout is not None: try: if received == 0: print(u"Downloading %s (%.1f Mb)" % (url,int(total)/1024/1024)) elif received>=total: print(u" -> download finished (%.0f Kb/s)" % (total /(1024.0*(time.time()+.001-start_time)))) else: print(u'%i / %i (%.0f%%) (%.0f KB/s)\r' % (received,total,100.0*received/total,speed)) except: return False return True else: return False if target is None: target = tempfile.gettempdir() if os.path.isdir(target): target = os.path.join(target,'') (dir,filename) = os.path.split(target) if not filename: url_parts = urlparse.urlparse(url) filename = url_parts.path.split('/')[-1] if not dir: dir = os.getcwd() if not os.path.isdir(dir): os.makedirs(dir) if verify_cert == False: requests.packages.urllib3.disable_warnings() # pylint: disable=no-member header=default_http_headers() if referer != None: header.update({'referer': '%s' % referer}) if user_agent != None: header.update({'user-agent': '%s' % user_agent}) target_fn = os.path.join(dir,filename) # return cached file if md5 matches. if (md5 is not None or sha1 is not None or sha256 is not None) and cache_dir is not None and os.path.isdir(cache_dir): cached_filename = os.path.join(cache_dir,filename) if os.path.isfile(cached_filename): if _check_hash_for_file(cached_filename,md5=md5,sha1=sha1,sha256=sha256): resume = False if cached_filename != target_fn: shutil.copy2(cached_filename,target_fn) return target_fn else: cached_filename = None if os.path.isfile(target_fn) and resume: try: actual_size = os.stat(target_fn).st_size size_req = requests.head(url, proxies=proxies, timeout=connect_timeout, verify=verify_cert, headers=header, cert = cert, allow_redirects=True) target_size = int(size_req.headers['content-length']) file_date = size_req.headers.get('last-modified',None) if target_size > actual_size: header.update({'Range':'bytes=%s-' % (actual_size,)}) write_mode = 'ab' elif target_size < actual_size: target_size = None write_mode = 'wb' except Exception as e: target_size = None write_mode = 'wb' else: file_date = None actual_size = 0 target_size = None write_mode = 'wb' # check hashes if size equal if resume and (md5 is not None or sha1 is not None or sha256 is not None) and target_size is not None and (target_size == actual_size): if not _check_hash_for_file(target_fn,md5=md5,sha1=sha1,sha256=sha256): # restart download... target_size = None write_mode = 'wb' if not resume or target_size is None or (target_size - actual_size) > 0: httpreq = requests.get(url,stream=True, proxies=proxies, timeout=connect_timeout, verify=verify_cert, headers=header, cert = cert, allow_redirects=True) httpreq.raise_for_status() total_bytes = int(httpreq.headers['content-length']) target_free_bytes = get_disk_free_space(os.path.dirname(os.path.abspath(target))) if total_bytes > target_free_bytes: raise Exception('wget : not enough free space on target drive to get %s MB. Total size: %s MB. Free space: %s MB' % (url,total_bytes // (1024*1024),target_free_bytes // (1024*1024))) # 1Mb max, 1kb min chunk_size = min([1024*1024,max([total_bytes/100,2048])]) cnt = 0 if printhook is None and ProgressBar is not None: progress_bar = ProgressBar(label=filename,expected_size=target_size or total_bytes, filled_char='=') progress_bar.show(actual_size) with open(target_fn,write_mode) as output_file: last_time_display = time.time() last_downloaded = 0 if httpreq.ok: for chunk in httpreq.iter_content(chunk_size=chunk_size): output_file.write(chunk) output_file.flush() if download_timeout is not None and (time.time()-start_time>download_timeout): raise requests.Timeout(r'Download of %s takes more than the requested %ss'%(url,download_timeout)) if printhook is None and ProgressBar is not None: if (time.time()-start_time>0.2) and (time.time()-last_time_display>=0.2): progress_bar.show(actual_size + cnt*len(chunk)) last_time_display = time.time() else: if reporthook(cnt*len(chunk),total_bytes): last_time_display = time.time() last_downloaded += len(chunk) cnt +=1 if printhook is None and ProgressBar is not None: progress_bar.show(total_bytes) progress_bar.done() last_time_display = time.time() elif reporthook(last_downloaded,total_bytes): last_time_display = time.time() # check hashes if sha256 is not None: file_hash = _hash_file(target_fn,hash_func=hashlib.sha256) if file_hash != sha256: raise Exception(u'Downloaded file %s sha256 %s does not match expected %s' % (url,file_hash,sha256)) elif sha1 is not None: file_hash = _hash_file(target_fn,hash_func=hashlib.sha1) if file_hash != sha1: raise Exception(u'Downloaded file %s sha1 %s does not match expected %s' % (url,file_hash,sha1)) elif md5 is not None: file_hash = _hash_file(target_fn,hash_func=hashlib.md5) if file_hash != md5: raise Exception(u'Downloaded file %s md5 %s does not match expected %s' % (url,file_hash,md5)) file_date = httpreq.headers.get('last-modified',None) if file_date: file_datetime = httpdatetime2time(file_date) os.utime(target_fn,(file_datetime,file_datetime)) # cache result if cache_dir: if not os.path.isdir(cache_dir): os.makedirs(cache_dir) cached_filename = os.path.join(cache_dir,filename) if target_fn != cached_filename: shutil.copy2(target_fn,cached_filename) return target_fn
progr.show(monitor.bytes_read) fileobj = jfs.up(p, testfile, upload_callback=UP) _end = time.time() puts(colored.magenta("Network upload speed %s/sec \n" % ( humanizeFileSize( (filesize / (_end-_start)) ) ))) # DOWNLOAD TEST 1 puts(colored.green('Test 2 | requests | Read | File size: %s' % humanizeFileSize(filesize))) _start = time.time() x = fileobj.read() _end = time.time() puts(colored.magenta("Network download speed %s/sec \n" % ( humanizeFileSize( (filesize / (_end-_start)) ) ))) del x # DOWNLOAD TEST 2 puts(colored.green('Test 2 | requests | Stream | File size: %s' % humanizeFileSize(filesize))) progr2 = ProgressBar(expected_size=filesize) _start = time.time() _bytesread = 0 for chunk in fileobj.stream(): _bytesread = _bytesread + len(chunk) progr2.show(_bytesread) _end = time.time() puts(colored.magenta("Network download speed %s/sec \n" % ( humanizeFileSize( (filesize / (_end-_start)) ) ))) # TODO: PRINT STATS IN A TABLE FOR COMPARISON / BOOKKEEPING # Versions: jottalib, urllib3, requests, jottaAPI # Server version from jottacloud.com # TEST WITHOUR REQUESTS, ONLY urllib3 # UPLOAD TEST #
def runMigrations(copies): pool = Pool(processes=num_processes) #copies.reverse() bar = Bar() res = pool.map(massDoCopies,((bar,x) for x in copies))
puts( colored.green('Test 2 | requests | Read | File size: %s' % humanizeFileSize(filesize))) _start = time.time() x = fileobj.read() _end = time.time() puts( colored.magenta("Network download speed %s/sec \n" % (humanizeFileSize( (filesize / (_end - _start)))))) del x # DOWNLOAD TEST 2 puts( colored.green('Test 2 | requests | Stream | File size: %s' % humanizeFileSize(filesize))) progr2 = ProgressBar(expected_size=filesize) _start = time.time() _bytesread = 0 for chunk in fileobj.stream(): _bytesread = _bytesread + len(chunk) progr2.show(_bytesread) _end = time.time() puts( colored.magenta("Network download speed %s/sec \n" % (humanizeFileSize( (filesize / (_end - _start)))))) # TODO: PRINT STATS IN A TABLE FOR COMPARISON / BOOKKEEPING # Versions: jottalib, urllib3, requests, jottaAPI # Server version from jottacloud.com # TEST WITHOUR REQUESTS, ONLY urllib3
def create_tarfile(self): """ Create a tar file with the contents of the current directory """ floyd_logger.info("Compressing data...") # Show progress bar (file_compressed/file_to_compress) self.__compression_bar = ProgressBar( expected_size=self.__files_to_compress, filled_char='=') # Auxiliary functions def dfilter_file_counter(tarinfo): """ Dummy filter function used to track the progression at file levels. """ self.__compression_bar.show(self.__files_compressed) self.__files_compressed += 1 return tarinfo def warn_purge_exit(info_msg, filename, progress_bar, exit_msg): """ Warn the user that's something went wrong, remove the tarball and provide an exit message. """ progress_bar.done() floyd_logger.info(info_msg) rmtree(os.path.dirname(filename)) sys.exit(exit_msg) try: # Define the default signal handler for catching: Ctrl-C signal.signal(signal.SIGINT, signal.default_int_handler) with tarfile.open(self.filename, "w:gz") as tar: tar.add(self.source_dir, arcname=os.path.basename(self.source_dir), filter=dfilter_file_counter) self.__compression_bar.done() except (OSError, IOError) as e: # OSError: [Errno 13] Permission denied if e.errno == errno.EACCES: self.source_dir = os.getcwd( ) if self.source_dir == '.' else self.source_dir # Expand cwd warn_purge_exit( info_msg="Permission denied. Removing compressed data...", filename=self.filename, progress_bar=self.__compression_bar, exit_msg=( "Permission denied. Make sure to have read permission " "for all the files and directories in the path: %s") % (self.source_dir)) # OSError: [Errno 28] No Space Left on Device (IOError on python2.7) elif e.errno == errno.ENOSPC: dir_path = os.path.dirname(self.filename) warn_purge_exit( info_msg="No space left. Removing compressed data...", filename=self.filename, progress_bar=self.__compression_bar, exit_msg= ("No space left when compressing your data in: %s.\n" "Make sure to have enough space before uploading your data." ) % (os.path.abspath(dir_path))) except KeyboardInterrupt: # Purge tarball on Ctrl-C warn_purge_exit( info_msg="Ctrl-C signal detected: Removing compressed data...", filename=self.filename, progress_bar=self.__compression_bar, exit_msg="Stopped the data upload gracefully.")
class DataCompressor(object): """ Local Data Compression with progress bar. """ def __init__(self, source_dir, filename): # Data directory to compress self.source_dir = source_dir # Archive (Tar file) name # e.g. "/tmp/contents.tar.gz" self.filename = filename # Prgress Bar for tracking data compression self.__compression_bar = None # Number of files to compress self.__files_to_compress = 0 self.__get_nfiles_to_compress() # Number of files already compressed self.__files_compressed = 0 def __get_nfiles_to_compress(self): """ Return the number of files to compress Note: it should take about 0.1s for counting 100k files on a dual core machine """ floyd_logger.info( "Get number of files to compress... (this could take a few seconds)" ) paths = [self.source_dir] try: # Traverse each subdirs of source_dir and count files/dirs while paths: path = paths.pop() for item in scandir(path): if item.is_dir(): paths.append(item.path) self.__files_to_compress += 1 elif item.is_file(): self.__files_to_compress += 1 except OSError as e: # OSError: [Errno 13] Permission denied if e.errno == errno.EACCES: self.source_dir = os.getcwd( ) if self.source_dir == '.' else self.source_dir # Expand cwd sys.exit( ("Permission denied. Make sure to have read permission " "for all the files and directories in the path: %s") % (self.source_dir)) floyd_logger.info("Compressing %d files", self.__files_to_compress) def create_tarfile(self): """ Create a tar file with the contents of the current directory """ floyd_logger.info("Compressing data...") # Show progress bar (file_compressed/file_to_compress) self.__compression_bar = ProgressBar( expected_size=self.__files_to_compress, filled_char='=') # Auxiliary functions def dfilter_file_counter(tarinfo): """ Dummy filter function used to track the progression at file levels. """ self.__compression_bar.show(self.__files_compressed) self.__files_compressed += 1 return tarinfo def warn_purge_exit(info_msg, filename, progress_bar, exit_msg): """ Warn the user that's something went wrong, remove the tarball and provide an exit message. """ progress_bar.done() floyd_logger.info(info_msg) rmtree(os.path.dirname(filename)) sys.exit(exit_msg) try: # Define the default signal handler for catching: Ctrl-C signal.signal(signal.SIGINT, signal.default_int_handler) with tarfile.open(self.filename, "w:gz") as tar: tar.add(self.source_dir, arcname=os.path.basename(self.source_dir), filter=dfilter_file_counter) self.__compression_bar.done() except (OSError, IOError) as e: # OSError: [Errno 13] Permission denied if e.errno == errno.EACCES: self.source_dir = os.getcwd( ) if self.source_dir == '.' else self.source_dir # Expand cwd warn_purge_exit( info_msg="Permission denied. Removing compressed data...", filename=self.filename, progress_bar=self.__compression_bar, exit_msg=( "Permission denied. Make sure to have read permission " "for all the files and directories in the path: %s") % (self.source_dir)) # OSError: [Errno 28] No Space Left on Device (IOError on python2.7) elif e.errno == errno.ENOSPC: dir_path = os.path.dirname(self.filename) warn_purge_exit( info_msg="No space left. Removing compressed data...", filename=self.filename, progress_bar=self.__compression_bar, exit_msg= ("No space left when compressing your data in: %s.\n" "Make sure to have enough space before uploading your data." ) % (os.path.abspath(dir_path))) except KeyboardInterrupt: # Purge tarball on Ctrl-C warn_purge_exit( info_msg="Ctrl-C signal detected: Removing compressed data...", filename=self.filename, progress_bar=self.__compression_bar, exit_msg="Stopped the data upload gracefully.")
def __call__(self, size_rd, total): if self.bar is None: self.bar = ProgressBar(expected_size=total, filled_char='=') self.bar.show(size_rd)
def course_exports(course_list): '''Do course exports for each course in the list''' #pprint.pprint(course_list) bar = Bar(label='Exporting Courses ', expected_size=len(course_list)) export_list = [] for idx, course in enumerate(course_list): export_list.append(course_export(course)) bar.show(idx) blen = len(export_list) bar.label = "Checking Export Statuses: " bar.expected_length = blen bar.show(1) while export_list: if len(export_list) < 10: time.sleep(2) bar.show(blen - len(export_list)) #print export_list for idx, cm in enumerate(export_list): course, res = cm url = res.get('progress_url', None) if not url: del (export_list[idx]) print 'not able to generate export for course', course else: download_progress = requests.get( url, headers=config['headers']).json() bar.label = "Checking Export Status: {} {}% ".format( course[course_id_field], download_progress['completion']) if download_progress['workflow_state'] not in [ 'queued', 'running' ]: if download_progress['workflow_state'] == 'completed': url = "https://{domain}/api/v1/courses/{}/content_exports/{}".format( course[course_id_field], res['id'], **config) export_info = requests.get( url, headers=config['headers']).json() #print 'export_info',export_info export_url = export_info['attachment']['url'] course['export_url'] = export_url yield course else: course['export_url'] = None del (export_list[idx])
def __init__(self, length): self.bar = Bar(expected_size=length, filled_char='=') self.status = 0