def stream_to_file(fi, path, info={}):
"""
Read data from the filehandle and write a the file.
Optionally check the MD5.
"""
size = info['size']
md5 = info.get('md5')
print "Fetching: %s (%s)" % (basename(path), human_bytes(size))
n = 0
h = hashlib.new('md5')
with open(path + '.part', 'wb') as fo:
while True:
chunk = fi.read(16384)
if not chunk:
break
fo.write(chunk)
if md5:
h.update(chunk)
n += len(chunk)
fi.close()
if md5 and h.hexdigest() != md5:
raise Exception("Error: received data MD5 sums mismatch")
os.rename(path + '.part', path)
def remove(self):
if not isdir(self.meta_dir):
print "Error: Can't find meta data for:", self.cname
return
self.read_meta()
cur = n = 0
nof = len(self.files) # number of files
sys.stdout.write('%9s [' % human_bytes(self.installed_size))
self.run('pre_egguninst.py')
for p in self.files:
n += 1
rat = float(n) / nof
if rat * 64 >= cur:
sys.stdout.write('.')
sys.stdout.flush()
cur += 1
rm_rf(p)
if p.endswith('.py') and isfile(p + 'c'):
# remove the corresponding .pyc
rm_rf(p + 'c')
self.rmdirs()
rm_rf(self.meta_dir)
sys.stdout.write('.' * (65-cur) + ']\n')
sys.stdout.flush()
def fetch(self, key, execution_aborted=None):
""" Fetch the given key.
execution_aborted: a threading.Event object which signals when the execution
needs to be aborted, or None, if we don't want to abort the fetching at all.
"""
path = self.path(key)
fi, info = self.remote.get(key)
size = info['size']
md5 = info.get('md5')
if self.evt_mgr:
from encore.events.api import ProgressManager
else:
from egginst.console import ProgressManager
progress = ProgressManager(
self.evt_mgr, source=self,
operation_id=uuid4(),
message="fetching",
steps=size,
# ---
progress_type="fetching", filename=basename(path),
disp_amount=human_bytes(size),
super_id=getattr(self, 'super_id', None))
n = 0
h = hashlib.new('md5')
if size < 256:
buffsize = 1
else:
buffsize = 2 ** int(math.log(size / 256.0) / math.log(2.0) + 1)
pp = path + '.part'
if sys.platform == 'win32':
rm_rf(pp)
with progress:
with open(pp, 'wb') as fo:
while True:
if execution_aborted is not None and execution_aborted.is_set():
close_file_or_response(fi)
return
chunk = fi.read(buffsize)
if not chunk:
break
fo.write(chunk)
if md5:
h.update(chunk)
n += len(chunk)
progress(step=n)
close_file_or_response(fi)
if md5 and h.hexdigest() != md5:
raise ValueError("received data MD5 sums mismatch")
if sys.platform == 'win32':
rm_rf(path)
os.rename(pp, path)
def stream_to_file(fi, path, info={}):
"""
Read data from the filehandle and write a the file.
Optionally check the MD5.
"""
size = info['size']
md5 = info.get('md5')
print "Fetching: %s (%s)" % (basename(path), human_bytes(size))
n = 0
h = hashlib.new('md5')
with open(path + '.part', 'wb') as fo:
while True:
chunk = fi.read(16384)
if not chunk:
break
fo.write(chunk)
if md5:
h.update(chunk)
n += len(chunk)
fi.close()
if md5 and h.hexdigest() != md5:
raise Exception("Error: received data MD5 sums mismatch")
os.rename(path + '.part', path)
def fetch(self, key):
path = self.path(key)
fi, info = self.remote.get(key)
size = info['size']
md5 = info.get('md5')
if self.evt_mgr:
from encore.events.api import ProgressManager
else:
from egginst.console import ProgressManager
progress = ProgressManager(
self.evt_mgr, source=self,
operation_id=uuid4(),
message="fetching",
steps=size,
# ---
progress_type="fetching", filename=basename(path),
disp_amount=human_bytes(size),
super_id=getattr(self, 'super_id', None))
n = 0
h = hashlib.new('md5')
if size < 256:
buffsize = 1
else:
buffsize = 2 ** int(math.log(size / 256.0) / math.log(2.0) + 1)
pp = path + '.part'
if sys.platform == 'win32':
rm_rf(pp)
with progress:
with open(pp, 'wb') as fo:
while True:
chunk = fi.read(buffsize)
if not chunk:
break
fo.write(chunk)
if md5:
h.update(chunk)
n += len(chunk)
progress(step=n)
fi.close()
if md5 and h.hexdigest() != md5:
raise ValueError("received data MD5 sums mismatch")
if sys.platform == 'win32':
rm_rf(path)
os.rename(pp, path)
def console_progress_manager_factory(message, filename, size, steps=None,
show_speed=False):
if steps is None:
steps = size
first_line = _compute_optimal_first_line(message, filename)
sys.stdout.write(first_line + "\n")
left_align = 10
# 2 for '[' and ']'
width = len(first_line) - left_align - 2 - 1
bar_template = "{0:>10}".format(human_bytes(size))
bar_template += "%(label)s [%(bar)s] %(info)s"
return ProgressBar(length=steps, bar_template=bar_template, width=width,
fill_char=".", show_speed=show_speed)
def extract(self):
cur = n = 0
size = sum(self.z.getinfo(name).file_size for name in self.arcnames)
sys.stdout.write('%9s [' % human_bytes(size))
for name in self.arcnames:
n += self.z.getinfo(name).file_size
if size == 0:
rat = 1
else:
rat = float(n) / size
if rat * 64 >= cur:
sys.stdout.write('.')
sys.stdout.flush()
cur += 1
self.write_arcname(name)
self.installed_size = size
sys.stdout.write('.' * (65-cur) + ']\n')
sys.stdout.flush()
def write_data_from_url(fo, url, md5=None, size=None):
"""
Read data from the url and write to the file handle fo, which must be
open for writing. Optionally check the MD5. When the size in bytes
is provided, a progress bar is displayed using the download/copy.
"""
if size:
sys.stdout.write('%9s [' % human_bytes(size))
sys.stdout.flush()
n = cur = 0
if url.startswith('file://'):
path = url[7:]
fi = open(path, 'rb')
elif url.startswith('http://'):
try:
fi = open_url(url)
except urllib2.URLError, e:
raise urllib2.URLError("\n%s\nCannot open URL:\n %s" % (e, url))
def write_data_from_url(fo, url, md5=None, size=None):
"""
Read data from the url and write to the file handle fo, which must be
open for writing. Optionally check the MD5. When the size in bytes
is provided, a progress bar is displayed using the download/copy.
"""
if size:
sys.stdout.write("%9s [" % human_bytes(size))
sys.stdout.flush()
n = cur = 0
if url.startswith("file://"):
path = url[7:]
fi = open(path, "rb")
elif url.startswith("http://"):
try:
fi = open_url(url)
except urllib2.URLError, e:
raise urllib2.URLError("\n%s\nCannot open URL:\n %s" % (e, url))
def console_progress_manager_factory(message,
filename,
size,
steps=None,
show_speed=False):
if steps is None:
steps = size
first_line = _compute_optimal_first_line(message, filename)
sys.stdout.write(first_line + "\n")
left_align = 10
# 2 for '[' and ']'
width = len(first_line) - left_align - 2 - 1
bar_template = "{0:>10}".format(human_bytes(size))
bar_template += "%(label)s [%(bar)s] %(info)s"
return ProgressBar(length=steps,
bar_template=bar_template,
width=width,
fill_char=".",
show_speed=show_speed)
def stream_to_file(fi, path, info={}):
"""
Read data from the filehandle and write a the file.
Optionally check the MD5.
"""
size = info['size']
md5 = info.get('md5')
getLogger('progress.start').info(dict(
amount = size,
disp_amount = human_bytes(size),
filename = basename(path),
action = 'fetching'))
n = 0
h = hashlib.new('md5')
if size and size < 16384:
buffsize = 1
else:
buffsize = 256
with open(path + '.part', 'wb') as fo:
while True:
chunk = fi.read(buffsize)
if not chunk:
break
fo.write(chunk)
if md5:
h.update(chunk)
n += len(chunk)
getLogger('progress.update').info(n)
fi.close()
getLogger('progress.stop').info(None)
if md5 and h.hexdigest() != md5:
raise MD5Mismatch("Error: received data MD5 sums mismatch")
os.rename(path + '.part', path)
