def __init__(self, url_table: Optional[BaseURLTable] = None):
self.start_time = None
self.stop_time = None
self.files = 0
self.size = 0
self.errors = Counter()
self.quota = None
self.bandwidth_meter = BandwidthMeter()
self._url_table = url_table
def __init__(self, *args, draw_interval: float=0.5, bar_width: int=25,
human_format: bool=True, **kwargs):
super().__init__(*args, **kwargs)
self._draw_interval = draw_interval
self._bar_width = bar_width
self._human_format = human_format
self._throbber_index = 0
self._throbber_iter = itertools.cycle(
itertools.chain(
range(bar_width), reversed(range(1, bar_width - 1))
))
self._bandwidth_meter = BandwidthMeter()
self._previous_value = 0
self._last_draw_time = 0
self._start_time = time.time()
def __init__(self, url_table: Optional[BaseURLTable]=None):
self.start_time = None
self.stop_time = None
self.files = 0
self.size = 0
self.errors = Counter()
self.quota = None
self.bandwidth_meter = BandwidthMeter()
self._url_table = url_table
def test_bandwidth_meter(self):
meter = BandwidthMeter()
self.assertEqual(0, meter.speed())
meter.feed(1000, feed_time=time.time() + 0.2)
self.assertTrue(meter.speed())
class BarProgress(ProgressPrinter):
def __init__(self, *args, draw_interval: float=0.5, bar_width: int=25,
human_format: bool=True, **kwargs):
super().__init__(*args, **kwargs)
self._draw_interval = draw_interval
self._bar_width = bar_width
self._human_format = human_format
self._throbber_index = 0
self._throbber_iter = itertools.cycle(
itertools.chain(
range(bar_width), reversed(range(1, bar_width - 1))
))
self._bandwidth_meter = BandwidthMeter()
self._previous_value = 0
self._last_draw_time = 0
self._start_time = time.time()
def update(self):
super().update()
if self._state != self.State.receiving_body:
return
difference = self.current_value - self._previous_value
self._previous_value = self.current_value
self._bandwidth_meter.feed(difference)
time_now = time.time()
if time_now - self._last_draw_time > self._draw_interval or self.current_value == self.max_value:
self._print_status()
self._flush()
self._last_draw_time = time_now
def _print_status(self):
'''Print an entire status line including bar and stats.'''
self._clear_line()
self._print(' ')
if self.max_value:
self._print_percent()
self._print(' ')
self._print_bar()
else:
self._print_throbber()
self._print(' ')
if self.measurement == Measurement.bytes:
self._print_size_downloaded()
else:
self._print(self.current_value)
self._print(' ')
self._print_duration()
self._print(' ')
if self.measurement == Measurement.bytes:
self._print_speed()
self._flush()
def _clear_line(self):
'''Print ANSI code to clear the current line.'''
self._print('\x1b[1G')
self._print('\x1b[2K')
def _print_throbber(self):
'''Print an indefinite progress bar.'''
self._print('[')
for position in range(self._bar_width):
self._print('O' if position == self._throbber_index else ' ')
self._print(']')
self._throbber_index = next(self._throbber_iter)
def _print_bar(self):
'''Print a progress bar.'''
self._print('[')
for position in range(self._bar_width):
position_fraction = position / (self._bar_width - 1)
position_bytes = position_fraction * self.max_value
if position_bytes < (self.continue_value or 0):
self._print('+')
elif position_bytes <= (self.continue_value or 0) + self.current_value:
self._print('=')
else:
self._print(' ')
self._print(']')
def _print_size_downloaded(self):
'''Print the bytes downloaded.'''
self._print(wpull.string.format_size(self.current_value))
def _print_duration(self):
'''Print the elapsed download time.'''
duration = int(time.time() - self._start_time)
self._print(datetime.timedelta(seconds=duration))
def _print_speed(self):
'''Print the current speed.'''
if self._bandwidth_meter.num_samples:
speed = self._bandwidth_meter.speed()
if self._human_format:
file_size_str = wpull.string.format_size(speed)
else:
file_size_str = '{:.1f} b'.format(speed * 8)
speed_str = _('{preformatted_file_size}/s').format(
preformatted_file_size=file_size_str
)
else:
speed_str = _('-- B/s')
self._print(speed_str)
def _print_percent(self):
'''Print how much is done in percentage.'''
fraction_done = ((self.continue_value or 0 + self.current_value) /
self.max_value)
self._print('{fraction_done:.1%}'.format(fraction_done=fraction_done))
class Statistics(object):
'''Statistics.
Attributes:
start_time (float): Timestamp when the engine started.
stop_time (float): Timestamp when the engine stopped.
files (int): Number of files downloaded.
size (int): Size of files in bytes.
errors: a Counter mapping error types to integer.
quota (int): Threshold of number of bytes when the download quota is
exceeded.
bandwidth_meter (:class:`.network.BandwidthMeter`): The bandwidth
meter.
'''
def __init__(self, url_table: Optional[BaseURLTable] = None):
self.start_time = None
self.stop_time = None
self.files = 0
self.size = 0
self.errors = Counter()
self.quota = None
self.bandwidth_meter = BandwidthMeter()
self._url_table = url_table
def start(self):
'''Record the start time.'''
self.start_time = time.time()
self.bandwidth_meter.feed(1)
def stop(self):
'''Record the stop time.'''
self.stop_time = time.time()
@property
def duration(self) -> float:
'''Return the time in seconds the interval.'''
return self.stop_time - self.start_time
def increment(self, size: int):
'''Increment the number of files downloaded.
Args:
size: The size of the file
'''
assert size >= 0, size
self.files += 1
self.size += size
self.bandwidth_meter.feed(size)
@property
def is_quota_exceeded(self) -> bool:
'''Return whether the quota is exceeded.'''
if self.quota and self._url_table is not None:
return self.size >= self.quota and \
self._url_table.get_root_url_todo_count() == 0
def increment_error(self, error: Exception):
'''Increment the error counter preferring base exceptions.'''
_logger.debug('Increment error %s', error)
for error_class in ERROR_PRIORITIES:
if isinstance(error, error_class):
self.errors[error_class] += 1
return
self.errors[type(error)] += 1
class Statistics(object):
'''Statistics.
Attributes:
start_time (float): Timestamp when the engine started.
stop_time (float): Timestamp when the engine stopped.
files (int): Number of files downloaded.
size (int): Size of files in bytes.
errors: a Counter mapping error types to integer.
quota (int): Threshold of number of bytes when the download quota is
exceeded.
bandwidth_meter (:class:`.network.BandwidthMeter`): The bandwidth
meter.
'''
def __init__(self, url_table: Optional[BaseURLTable]=None):
self.start_time = None
self.stop_time = None
self.files = 0
self.size = 0
self.errors = Counter()
self.quota = None
self.bandwidth_meter = BandwidthMeter()
self._url_table = url_table
def start(self):
'''Record the start time.'''
self.start_time = time.time()
self.bandwidth_meter.feed(1)
def stop(self):
'''Record the stop time.'''
self.stop_time = time.time()
@property
def duration(self) -> float:
'''Return the time in seconds the interval.'''
return self.stop_time - self.start_time
def increment(self, size: int):
'''Increment the number of files downloaded.
Args:
size: The size of the file
'''
assert size >= 0, size
self.files += 1
self.size += size
self.bandwidth_meter.feed(size)
@property
def is_quota_exceeded(self) -> bool:
'''Return whether the quota is exceeded.'''
if self.quota and self._url_table is not None:
return self.size >= self.quota and \
self._url_table.get_root_url_todo_count() == 0
def increment_error(self, error: Exception):
'''Increment the error counter preferring base exceptions.'''
_logger.debug('Increment error %s', error)
for error_class in ERROR_PRIORITIES:
if isinstance(error, error_class):
self.errors[error_class] += 1
return
self.errors[type(error)] += 1