def test_bandwidth_meter(self):
meter = BandwidthMeter()
self.assertEqual(0, meter.speed())
time.sleep(0.2)
meter.feed(1000)
self.assertTrue(meter.speed())
def __init__(self):
self.start_time = None
self.stop_time = None
self.files = 0
self.size = 0
self.errors = Counter()
self.quota = None
self._temp_dir = None
self._required_urls_db = None
self.bandwidth_meter = BandwidthMeter()
def __init__(self, update_interval=0.5, bar_width=25, **kwargs):
super().__init__(**kwargs)
self._last_flush_time = 0
self._update_interval = update_interval
self._bytes_continued = 0
self._total_size = None
self._bar_width = bar_width
self._throbber_index = 0
self._throbber_iter = itertools.cycle(
itertools.chain(range(bar_width), reversed(range(1,
bar_width - 1))))
self._bandwidth_meter = BandwidthMeter()
self._start_time = time.time()
def __init__(self):
self.start_time = None
self.stop_time = None
self.files = 0
self.size = 0
self.errors = Counter()
self.quota = None
self.required_url_infos = set()
self.bandwidth_meter = BandwidthMeter()
def test_bandwidth_meter(self):
meter = BandwidthMeter()
self.assertEqual(0, meter.speed())
meter.feed(1000, feed_time=time.time() + 0.2)
self.assertTrue(meter.speed())
def __init__(self, update_interval=0.5, bar_width=25, **kwargs):
super().__init__(**kwargs)
self._last_flush_time = 0
self._update_interval = update_interval
self._bytes_continued = 0
self._total_size = None
self._bar_width = bar_width
self._throbber_index = 0
self._throbber_iter = itertools.cycle(
itertools.chain(
range(bar_width), reversed(range(1, bar_width - 1))
))
self._bandwidth_meter = BandwidthMeter()
self._start_time = time.time()
class Statistics(object):
'''Statistics.
Attributes:
start_time (float): Timestamp when the engine started.
stop_time (float): Timestamp when the engine stopped.
files (int): Number of files downloaded.
size (int): Size of files in bytes.
errors: a Counter mapping error types to integer.
quota (int): Threshold of number of bytes when the download quota is
exceeded.
required_urls_db (dict): A mapping of :class:`.url.URLInfo` to bool
that must be completed before the quota can be exceeded. The
mapping uses a disk store so it is created on demand.
bandwidth_meter (:class:`.network.BandwidthMeter`): The bandwidth
meter.
'''
def __init__(self):
self.start_time = None
self.stop_time = None
self.files = 0
self.size = 0
self.errors = Counter()
self.quota = None
self._temp_dir = None
self._required_urls_db = None
self.bandwidth_meter = BandwidthMeter()
@property
def required_urls_db(self):
if not self._required_urls_db:
self._required_urls_db = self._new_required_urls_db()
return self._required_urls_db
def _new_required_urls_db(self):
self._temp_dir = tempfile.TemporaryDirectory(
prefix='tmp-wpull-quota', dir=os.getcwd())
return shelve.open(os.path.join(self._temp_dir.name, 'quota.db'))
def start(self):
'''Record the start time.'''
self.start_time = time.time()
self.bandwidth_meter.feed(1)
def stop(self):
'''Record the stop time.'''
self.stop_time = time.time()
if self._temp_dir:
self._temp_dir.cleanup()
self._temp_dir = None
@property
def duration(self):
'''Return the time in seconds the interval.'''
return self.stop_time - self.start_time
def increment(self, size):
'''Increment the number of files downloaded.
Args:
size: The size of the file
'''
self.files += 1
self.size += size
self.bandwidth_meter.feed(size)
@property
def is_quota_exceeded(self):
'''Return whether the quota is exceeded.'''
if self._required_urls_db is None:
return False
try:
# bool(db) is not guaranteed to be supported, esp on PyPy
if next(iter(self._required_urls_db.keys())):
return False
except StopIteration:
pass
if self.quota:
return self.size >= self.quota
def mark_done(self, url_info):
'''Set the URLInfo as completed.'''
if self._required_urls_db and url_info.url in self._required_urls_db:
del self.required_urls_db[url_info.url]
def increment_error(self, error):
'''Increment the error counter preferring base exceptions.'''
_logger.debug('Increment error %s', error)
for error_class in ERROR_PRIORITIES:
if isinstance(error, error_class):
self.errors[error_class] += 1
return
self.errors[type(error)] += 1
class BarProgressRecorderSession(BaseProgressRecorderSession):
'''Bar Progress Recorder Session.
This session is responsible for displaying the ASCII bar
and stats.
'''
def __init__(self, update_interval=0.5, bar_width=25, **kwargs):
super().__init__(**kwargs)
self._last_flush_time = 0
self._update_interval = update_interval
self._bytes_continued = 0
self._total_size = None
self._bar_width = bar_width
self._throbber_index = 0
self._throbber_iter = itertools.cycle(
itertools.chain(
range(bar_width), reversed(range(1, bar_width - 1))
))
self._bandwidth_meter = BandwidthMeter()
self._start_time = time.time()
def pre_response(self, response):
super().pre_response(response)
if response.status_code == http.client.PARTIAL_CONTENT:
match = re.search(
r'bytes +([0-9]+)-([0-9]+)/([0-9]+)',
response.fields.get('Content-Range', '')
)
if match:
self._bytes_continued = int(match.group(1))
self._total_size = int(match.group(3))
else:
self._total_size = self._content_length
def response_data(self, data):
super().response_data(data)
if not self._response:
return
self._bandwidth_meter.feed(len(data))
time_now = time.time()
if time_now - self._last_flush_time > self._update_interval:
self._print_status()
self._stream.flush()
self._last_flush_time = time_now
def response(self, response):
self._print_status()
self._stream.flush()
super().response(response)
def _print_status(self):
self._clear_line()
if self._total_size:
self._print_percent()
self._print(' ')
self._print_bar()
else:
self._print_throbber()
self._print(' ')
self._print_size_downloaded()
self._print(' ')
self._print_duration()
self._print(' ')
self._print_speed()
self._flush()
def _clear_line(self):
self._print('\x1b[1G')
self._print('\x1b[2K')
def _print_throbber(self):
self._print('[')
for position in range(self._bar_width):
self._print('O' if position == self._throbber_index else ' ')
self._print(']')
self._throbber_index = next(self._throbber_iter)
def _print_bar(self):
self._print('[')
for position in range(self._bar_width):
position_fraction = position / (self._bar_width - 1)
position_bytes = position_fraction * self._total_size
if position_bytes < self._bytes_continued:
self._print('+')
elif (position_bytes <=
self._bytes_continued + self._bytes_received):
self._print('=')
else:
self._print(' ')
self._print(']')
def _print_size_downloaded(self):
self._print(wpull.string.format_size(self._bytes_received))
def _print_duration(self):
duration = int(time.time() - self._start_time)
self._print(datetime.timedelta(seconds=duration))
def _print_speed(self):
if self._bandwidth_meter.num_samples:
speed = self._bandwidth_meter.speed()
speed_str = _('{preformatted_file_size}/s').format(
preformatted_file_size=wpull.string.format_size(speed)
)
else:
speed_str = _('-- B/s')
self._print(speed_str)
def _print_percent(self):
fraction_done = ((self._bytes_continued + self._bytes_received) /
self._total_size)
self._print('{fraction_done:.1%}'.format(fraction_done=fraction_done))
class Statistics(object):
'''Statistics.
Attributes:
start_time (float): Timestamp when the engine started.
stop_time (float): Timestamp when the engine stopped.
files (int): Number of files downloaded.
size (int): Size of files in bytes.
errors: a Counter mapping error types to integer.
quota (int): Threshold of number of bytes when the download quota is
exceeded.
required_url_infos (set): A set of :class:`.url.URLInfo` that must
be completed before the quota can be exceeded.
bandwidth_meter (:class:`.network.BandwidthMeter`): The bandwidth
meter.
'''
def __init__(self):
self.start_time = None
self.stop_time = None
self.files = 0
self.size = 0
self.errors = Counter()
self.quota = None
self.required_url_infos = set()
self.bandwidth_meter = BandwidthMeter()
def start(self):
'''Record the start time.'''
self.start_time = time.time()
self.bandwidth_meter.feed(1)
def stop(self):
'''Record the stop time.'''
self.stop_time = time.time()
@property
def duration(self):
'''Return the time in seconds the interval.'''
return self.stop_time - self.start_time
def increment(self, size):
'''Increment the number of files downloaded.
Args:
size: The size of the file
'''
self.files += 1
self.size += size
self.bandwidth_meter.feed(size)
@property
def is_quota_exceeded(self):
'''Return whether the quota is exceeded.'''
if self.required_url_infos:
return False
if self.quota:
return self.size >= self.quota
def mark_done(self, url_info):
'''Set the URLInfo as completed.'''
if url_info in self.required_url_infos:
self.required_url_infos.remove(url_info)
class BarProgressRecorderSession(BaseProgressRecorderSession):
'''Bar Progress Recorder Session.
This session is responsible for displaying the ASCII bar
and stats.
'''
def __init__(self, update_interval=0.5, bar_width=25, **kwargs):
super().__init__(**kwargs)
self._last_flush_time = 0
self._update_interval = update_interval
self._bytes_continued = 0
self._total_size = None
self._bar_width = bar_width
self._throbber_index = 0
self._throbber_iter = itertools.cycle(
itertools.chain(range(bar_width), reversed(range(1,
bar_width - 1))))
self._bandwidth_meter = BandwidthMeter()
self._start_time = time.time()
def pre_response(self, response):
super().pre_response(response)
if response.protocol == 'http' and \
response.status_code == http.client.PARTIAL_CONTENT:
match = re.search(r'bytes +([0-9]+)-([0-9]+)/([0-9]+)',
response.fields.get('Content-Range', ''))
if match:
self._bytes_continued = int(match.group(1))
self._total_size = int(match.group(3))
elif response.protocol == 'ftp' and response.restart_value:
self._bytes_continued = response.restart_value
self._total_size = self._content_length
else:
self._total_size = self._content_length
def response_data(self, data):
super().response_data(data)
if not self._response:
return
self._bandwidth_meter.feed(len(data))
time_now = time.time()
if time_now - self._last_flush_time > self._update_interval:
self._print_status()
self._stream.flush()
self._last_flush_time = time_now
def response(self, response):
self._print_status()
self._stream.flush()
super().response(response)
def _print_status(self):
'''Print an entire status line including bar and stats.'''
self._clear_line()
self._print(' ')
if self._total_size:
self._print_percent()
self._print(' ')
self._print_bar()
else:
self._print_throbber()
self._print(' ')
self._print_size_downloaded()
self._print(' ')
self._print_duration()
self._print(' ')
self._print_speed()
self._flush()
def _clear_line(self):
'''Print ANSI code to clear the current line.'''
self._print('\x1b[1G')
self._print('\x1b[2K')
def _print_throbber(self):
'''Print an indefinite progress bar.'''
self._print('[')
for position in range(self._bar_width):
self._print('O' if position == self._throbber_index else ' ')
self._print(']')
self._throbber_index = next(self._throbber_iter)
def _print_bar(self):
'''Print a progress bar.'''
self._print('[')
for position in range(self._bar_width):
position_fraction = position / (self._bar_width - 1)
position_bytes = position_fraction * self._total_size
if position_bytes < self._bytes_continued:
self._print('+')
elif (position_bytes <=
self._bytes_continued + self._bytes_received):
self._print('=')
else:
self._print(' ')
self._print(']')
def _print_size_downloaded(self):
'''Print the bytes downloaded.'''
self._print(wpull.string.format_size(self._bytes_received))
def _print_duration(self):
'''Print the elapsed download time.'''
duration = int(time.time() - self._start_time)
self._print(datetime.timedelta(seconds=duration))
def _print_speed(self):
'''Print the current speed.'''
if self._bandwidth_meter.num_samples:
speed = self._bandwidth_meter.speed()
if self._human_format:
file_size_str = wpull.string.format_size(speed)
else:
file_size_str = '{:.1f} b'.format(speed * 8)
speed_str = _('{preformatted_file_size}/s').format(
preformatted_file_size=file_size_str)
else:
speed_str = _('-- B/s')
self._print(speed_str)
def _print_percent(self):
'''Print how much is done in percentage.'''
fraction_done = ((self._bytes_continued + self._bytes_received) /
self._total_size)
self._print('{fraction_done:.1%}'.format(fraction_done=fraction_done))