def stream_to_file(fi, path, info={}): """ Read data from the filehandle and write a the file. Optionally check the MD5. """ size = info['size'] md5 = info.get('md5') print "Fetching: %s (%s)" % (basename(path), human_bytes(size)) n = 0 h = hashlib.new('md5') with open(path + '.part', 'wb') as fo: while True: chunk = fi.read(16384) if not chunk: break fo.write(chunk) if md5: h.update(chunk) n += len(chunk) fi.close() if md5 and h.hexdigest() != md5: raise Exception("Error: received data MD5 sums mismatch") os.rename(path + '.part', path)
def remove(self): if not isdir(self.meta_dir): print "Error: Can't find meta data for:", self.cname return self.read_meta() cur = n = 0 nof = len(self.files) # number of files sys.stdout.write('%9s [' % human_bytes(self.installed_size)) self.run('pre_egguninst.py') for p in self.files: n += 1 rat = float(n) / nof if rat * 64 >= cur: sys.stdout.write('.') sys.stdout.flush() cur += 1 rm_rf(p) if p.endswith('.py') and isfile(p + 'c'): # remove the corresponding .pyc rm_rf(p + 'c') self.rmdirs() rm_rf(self.meta_dir) sys.stdout.write('.' * (65-cur) + ']\n') sys.stdout.flush()
def fetch(self, key, execution_aborted=None): """ Fetch the given key. execution_aborted: a threading.Event object which signals when the execution needs to be aborted, or None, if we don't want to abort the fetching at all. """ path = self.path(key) fi, info = self.remote.get(key) size = info['size'] md5 = info.get('md5') if self.evt_mgr: from encore.events.api import ProgressManager else: from egginst.console import ProgressManager progress = ProgressManager( self.evt_mgr, source=self, operation_id=uuid4(), message="fetching", steps=size, # --- progress_type="fetching", filename=basename(path), disp_amount=human_bytes(size), super_id=getattr(self, 'super_id', None)) n = 0 h = hashlib.new('md5') if size < 256: buffsize = 1 else: buffsize = 2 ** int(math.log(size / 256.0) / math.log(2.0) + 1) pp = path + '.part' if sys.platform == 'win32': rm_rf(pp) with progress: with open(pp, 'wb') as fo: while True: if execution_aborted is not None and execution_aborted.is_set(): close_file_or_response(fi) return chunk = fi.read(buffsize) if not chunk: break fo.write(chunk) if md5: h.update(chunk) n += len(chunk) progress(step=n) close_file_or_response(fi) if md5 and h.hexdigest() != md5: raise ValueError("received data MD5 sums mismatch") if sys.platform == 'win32': rm_rf(path) os.rename(pp, path)
def fetch(self, key): path = self.path(key) fi, info = self.remote.get(key) size = info['size'] md5 = info.get('md5') if self.evt_mgr: from encore.events.api import ProgressManager else: from egginst.console import ProgressManager progress = ProgressManager( self.evt_mgr, source=self, operation_id=uuid4(), message="fetching", steps=size, # --- progress_type="fetching", filename=basename(path), disp_amount=human_bytes(size), super_id=getattr(self, 'super_id', None)) n = 0 h = hashlib.new('md5') if size < 256: buffsize = 1 else: buffsize = 2 ** int(math.log(size / 256.0) / math.log(2.0) + 1) pp = path + '.part' if sys.platform == 'win32': rm_rf(pp) with progress: with open(pp, 'wb') as fo: while True: chunk = fi.read(buffsize) if not chunk: break fo.write(chunk) if md5: h.update(chunk) n += len(chunk) progress(step=n) fi.close() if md5 and h.hexdigest() != md5: raise ValueError("received data MD5 sums mismatch") if sys.platform == 'win32': rm_rf(path) os.rename(pp, path)
def console_progress_manager_factory(message, filename, size, steps=None, show_speed=False): if steps is None: steps = size first_line = _compute_optimal_first_line(message, filename) sys.stdout.write(first_line + "\n") left_align = 10 # 2 for '[' and ']' width = len(first_line) - left_align - 2 - 1 bar_template = "{0:>10}".format(human_bytes(size)) bar_template += "%(label)s [%(bar)s] %(info)s" return ProgressBar(length=steps, bar_template=bar_template, width=width, fill_char=".", show_speed=show_speed)
def extract(self): cur = n = 0 size = sum(self.z.getinfo(name).file_size for name in self.arcnames) sys.stdout.write('%9s [' % human_bytes(size)) for name in self.arcnames: n += self.z.getinfo(name).file_size if size == 0: rat = 1 else: rat = float(n) / size if rat * 64 >= cur: sys.stdout.write('.') sys.stdout.flush() cur += 1 self.write_arcname(name) self.installed_size = size sys.stdout.write('.' * (65-cur) + ']\n') sys.stdout.flush()
def write_data_from_url(fo, url, md5=None, size=None): """ Read data from the url and write to the file handle fo, which must be open for writing. Optionally check the MD5. When the size in bytes is provided, a progress bar is displayed using the download/copy. """ if size: sys.stdout.write('%9s [' % human_bytes(size)) sys.stdout.flush() n = cur = 0 if url.startswith('file://'): path = url[7:] fi = open(path, 'rb') elif url.startswith('http://'): try: fi = open_url(url) except urllib2.URLError, e: raise urllib2.URLError("\n%s\nCannot open URL:\n %s" % (e, url))
def write_data_from_url(fo, url, md5=None, size=None): """ Read data from the url and write to the file handle fo, which must be open for writing. Optionally check the MD5. When the size in bytes is provided, a progress bar is displayed using the download/copy. """ if size: sys.stdout.write("%9s [" % human_bytes(size)) sys.stdout.flush() n = cur = 0 if url.startswith("file://"): path = url[7:] fi = open(path, "rb") elif url.startswith("http://"): try: fi = open_url(url) except urllib2.URLError, e: raise urllib2.URLError("\n%s\nCannot open URL:\n %s" % (e, url))
def stream_to_file(fi, path, info={}): """ Read data from the filehandle and write a the file. Optionally check the MD5. """ size = info['size'] md5 = info.get('md5') getLogger('progress.start').info(dict( amount = size, disp_amount = human_bytes(size), filename = basename(path), action = 'fetching')) n = 0 h = hashlib.new('md5') if size and size < 16384: buffsize = 1 else: buffsize = 256 with open(path + '.part', 'wb') as fo: while True: chunk = fi.read(buffsize) if not chunk: break fo.write(chunk) if md5: h.update(chunk) n += len(chunk) getLogger('progress.update').info(n) fi.close() getLogger('progress.stop').info(None) if md5 and h.hexdigest() != md5: raise MD5Mismatch("Error: received data MD5 sums mismatch") os.rename(path + '.part', path)