def _get_md5_from_file(target_file, link): download_hash = md5() fp = open(target_file, 'rb') while 1: chunk = fp.read(4096) if not chunk: break download_hash.update(chunk) fp.close() return download_hash
def _download_url(resp, link, temp_location): fp = open(temp_location, 'wb') download_hash = None if link.md5_hash: download_hash = md5() try: total_length = int(resp.info()['content-length']) except (ValueError, KeyError): total_length = 0 downloaded = 0 show_progress = total_length > 40 * 1000 or not total_length show_url = link.show_url try: if show_progress: ## FIXME: the URL can get really long in this message: if total_length: logger.start_progress('Downloading %s (%s): ' % (show_url, format_size(total_length))) else: logger.start_progress('Downloading %s (unknown size): ' % show_url) else: logger.notify('Downloading %s' % show_url) logger.debug('Downloading from URL %s' % link) while 1: chunk = resp.read(4096) if not chunk: break downloaded += len(chunk) if show_progress: if not total_length: logger.show_progress('%s' % format_size(downloaded)) else: logger.show_progress('%3i%% %s' % (100 * downloaded / total_length, format_size(downloaded))) if link.md5_hash: download_hash.update(chunk) fp.write(chunk) fp.close() finally: if show_progress: logger.end_progress('%s downloaded' % format_size(downloaded)) return download_hash
def _download_url(resp, link, temp_location): fp = open(temp_location, 'wb') download_hash = None if link.md5_hash: download_hash = md5() try: total_length = int(resp.info()['content-length']) except (ValueError, KeyError): total_length = 0 downloaded = 0 show_progress = total_length > 40*1000 or not total_length show_url = link.show_url try: if show_progress: ## FIXME: the URL can get really long in this message: if total_length: logger.start_progress('Downloading %s (%s): ' % (show_url, format_size(total_length))) else: logger.start_progress('Downloading %s (unknown size): ' % show_url) else: logger.notify('Downloading %s' % show_url) logger.debug('Downloading from URL %s' % link) while 1: chunk = resp.read(4096) if not chunk: break downloaded += len(chunk) if show_progress: if not total_length: logger.show_progress('%s' % format_size(downloaded)) else: logger.show_progress('%3i%% %s' % (100*downloaded/total_length, format_size(downloaded))) if link.md5_hash: download_hash.update(chunk) fp.write(chunk) fp.close() finally: if show_progress: logger.end_progress('%s downloaded' % format_size(downloaded)) return download_hash