예제 #1
0
    def test_bandwidth_meter(self):
        meter = BandwidthMeter()

        self.assertEqual(0, meter.speed())

        time.sleep(0.2)
        meter.feed(1000)

        self.assertTrue(meter.speed())
예제 #2
0
파일: stats.py 프로젝트: flatron18116/wpull
 def __init__(self):
     self.start_time = None
     self.stop_time = None
     self.files = 0
     self.size = 0
     self.errors = Counter()
     self.quota = None
     self._temp_dir = None
     self._required_urls_db = None
     self.bandwidth_meter = BandwidthMeter()
예제 #3
0
 def __init__(self, update_interval=0.5, bar_width=25, **kwargs):
     super().__init__(**kwargs)
     self._last_flush_time = 0
     self._update_interval = update_interval
     self._bytes_continued = 0
     self._total_size = None
     self._bar_width = bar_width
     self._throbber_index = 0
     self._throbber_iter = itertools.cycle(
         itertools.chain(range(bar_width), reversed(range(1,
                                                          bar_width - 1))))
     self._bandwidth_meter = BandwidthMeter()
     self._start_time = time.time()
예제 #4
0
파일: stats.py 프로젝트: mback2k/wpull
 def __init__(self):
     self.start_time = None
     self.stop_time = None
     self.files = 0
     self.size = 0
     self.errors = Counter()
     self.quota = None
     self.required_url_infos = set()
     self.bandwidth_meter = BandwidthMeter()
예제 #5
0
    def test_bandwidth_meter(self):
        meter = BandwidthMeter()

        self.assertEqual(0, meter.speed())

        meter.feed(1000, feed_time=time.time() + 0.2)

        self.assertTrue(meter.speed())
예제 #6
0
파일: recorder.py 프로젝트: mback2k/wpull
 def __init__(self, update_interval=0.5, bar_width=25, **kwargs):
     super().__init__(**kwargs)
     self._last_flush_time = 0
     self._update_interval = update_interval
     self._bytes_continued = 0
     self._total_size = None
     self._bar_width = bar_width
     self._throbber_index = 0
     self._throbber_iter = itertools.cycle(
         itertools.chain(
             range(bar_width), reversed(range(1, bar_width - 1))
         ))
     self._bandwidth_meter = BandwidthMeter()
     self._start_time = time.time()
예제 #7
0
파일: stats.py 프로젝트: flatron18116/wpull
class Statistics(object):
    '''Statistics.

    Attributes:
        start_time (float): Timestamp when the engine started.
        stop_time (float): Timestamp when the engine stopped.
        files (int): Number of files downloaded.
        size (int): Size of files in bytes.
        errors: a Counter mapping error types to integer.
        quota (int): Threshold of number of bytes when the download quota is
            exceeded.
        required_urls_db (dict): A mapping of :class:`.url.URLInfo` to bool
            that must be completed before the quota can be exceeded. The
            mapping uses a disk store so it is created on demand.
        bandwidth_meter (:class:`.network.BandwidthMeter`): The bandwidth
            meter.
    '''
    def __init__(self):
        self.start_time = None
        self.stop_time = None
        self.files = 0
        self.size = 0
        self.errors = Counter()
        self.quota = None
        self._temp_dir = None
        self._required_urls_db = None
        self.bandwidth_meter = BandwidthMeter()

    @property
    def required_urls_db(self):
        if not self._required_urls_db:
            self._required_urls_db = self._new_required_urls_db()

        return self._required_urls_db

    def _new_required_urls_db(self):
        self._temp_dir = tempfile.TemporaryDirectory(
            prefix='tmp-wpull-quota', dir=os.getcwd())

        return shelve.open(os.path.join(self._temp_dir.name, 'quota.db'))

    def start(self):
        '''Record the start time.'''
        self.start_time = time.time()
        self.bandwidth_meter.feed(1)

    def stop(self):
        '''Record the stop time.'''
        self.stop_time = time.time()

        if self._temp_dir:
            self._temp_dir.cleanup()
            self._temp_dir = None

    @property
    def duration(self):
        '''Return the time in seconds the interval.'''
        return self.stop_time - self.start_time

    def increment(self, size):
        '''Increment the number of files downloaded.

        Args:
            size: The size of the file
        '''
        self.files += 1
        self.size += size
        self.bandwidth_meter.feed(size)

    @property
    def is_quota_exceeded(self):
        '''Return whether the quota is exceeded.'''
        if self._required_urls_db is None:
            return False

        try:
            # bool(db) is not guaranteed to be supported, esp on PyPy
            if next(iter(self._required_urls_db.keys())):
                return False
        except StopIteration:
            pass

        if self.quota:
            return self.size >= self.quota

    def mark_done(self, url_info):
        '''Set the URLInfo as completed.'''
        if self._required_urls_db and url_info.url in self._required_urls_db:
            del self.required_urls_db[url_info.url]

    def increment_error(self, error):
        '''Increment the error counter preferring base exceptions.'''
        _logger.debug('Increment error %s', error)

        for error_class in ERROR_PRIORITIES:
            if isinstance(error, error_class):
                self.errors[error_class] += 1
                return

        self.errors[type(error)] += 1
예제 #8
0
파일: recorder.py 프로젝트: mback2k/wpull
class BarProgressRecorderSession(BaseProgressRecorderSession):
    '''Bar Progress Recorder Session.

    This session is responsible for displaying the ASCII bar
    and stats.
    '''
    def __init__(self, update_interval=0.5, bar_width=25, **kwargs):
        super().__init__(**kwargs)
        self._last_flush_time = 0
        self._update_interval = update_interval
        self._bytes_continued = 0
        self._total_size = None
        self._bar_width = bar_width
        self._throbber_index = 0
        self._throbber_iter = itertools.cycle(
            itertools.chain(
                range(bar_width), reversed(range(1, bar_width - 1))
            ))
        self._bandwidth_meter = BandwidthMeter()
        self._start_time = time.time()

    def pre_response(self, response):
        super().pre_response(response)

        if response.status_code == http.client.PARTIAL_CONTENT:
            match = re.search(
                r'bytes +([0-9]+)-([0-9]+)/([0-9]+)',
                response.fields.get('Content-Range', '')
            )

            if match:
                self._bytes_continued = int(match.group(1))
                self._total_size = int(match.group(3))
        else:
            self._total_size = self._content_length

    def response_data(self, data):
        super().response_data(data)

        if not self._response:
            return

        self._bandwidth_meter.feed(len(data))

        time_now = time.time()

        if time_now - self._last_flush_time > self._update_interval:
            self._print_status()
            self._stream.flush()

            self._last_flush_time = time_now

    def response(self, response):
        self._print_status()
        self._stream.flush()
        super().response(response)

    def _print_status(self):
        self._clear_line()

        if self._total_size:
            self._print_percent()
            self._print(' ')
            self._print_bar()
        else:
            self._print_throbber()

        self._print(' ')
        self._print_size_downloaded()
        self._print(' ')
        self._print_duration()
        self._print(' ')
        self._print_speed()
        self._flush()

    def _clear_line(self):
        self._print('\x1b[1G')
        self._print('\x1b[2K')

    def _print_throbber(self):
        self._print('[')

        for position in range(self._bar_width):
            self._print('O' if position == self._throbber_index else ' ')

        self._print(']')

        self._throbber_index = next(self._throbber_iter)

    def _print_bar(self):
        self._print('[')

        for position in range(self._bar_width):
            position_fraction = position / (self._bar_width - 1)
            position_bytes = position_fraction * self._total_size

            if position_bytes < self._bytes_continued:
                self._print('+')
            elif (position_bytes <=
                  self._bytes_continued + self._bytes_received):
                self._print('=')
            else:
                self._print(' ')

        self._print(']')

    def _print_size_downloaded(self):
        self._print(wpull.string.format_size(self._bytes_received))

    def _print_duration(self):
        duration = int(time.time() - self._start_time)
        self._print(datetime.timedelta(seconds=duration))

    def _print_speed(self):
        if self._bandwidth_meter.num_samples:
            speed = self._bandwidth_meter.speed()
            speed_str = _('{preformatted_file_size}/s').format(
                preformatted_file_size=wpull.string.format_size(speed)
            )
        else:
            speed_str = _('-- B/s')

        self._print(speed_str)

    def _print_percent(self):
        fraction_done = ((self._bytes_continued + self._bytes_received) /
                         self._total_size)

        self._print('{fraction_done:.1%}'.format(fraction_done=fraction_done))
예제 #9
0
파일: stats.py 프로젝트: mback2k/wpull
class Statistics(object):
    '''Statistics.

    Attributes:
        start_time (float): Timestamp when the engine started.
        stop_time (float): Timestamp when the engine stopped.
        files (int): Number of files downloaded.
        size (int): Size of files in bytes.
        errors: a Counter mapping error types to integer.
        quota (int): Threshold of number of bytes when the download quota is
            exceeded.
        required_url_infos (set): A set of :class:`.url.URLInfo` that must
            be completed before the quota can be exceeded.
        bandwidth_meter (:class:`.network.BandwidthMeter`): The bandwidth
            meter.
    '''
    def __init__(self):
        self.start_time = None
        self.stop_time = None
        self.files = 0
        self.size = 0
        self.errors = Counter()
        self.quota = None
        self.required_url_infos = set()
        self.bandwidth_meter = BandwidthMeter()

    def start(self):
        '''Record the start time.'''
        self.start_time = time.time()
        self.bandwidth_meter.feed(1)

    def stop(self):
        '''Record the stop time.'''
        self.stop_time = time.time()

    @property
    def duration(self):
        '''Return the time in seconds the interval.'''
        return self.stop_time - self.start_time

    def increment(self, size):
        '''Increment the number of files downloaded.

        Args:
            size: The size of the file
        '''
        self.files += 1
        self.size += size
        self.bandwidth_meter.feed(size)

    @property
    def is_quota_exceeded(self):
        '''Return whether the quota is exceeded.'''
        if self.required_url_infos:
            return False

        if self.quota:
            return self.size >= self.quota

    def mark_done(self, url_info):
        '''Set the URLInfo as completed.'''
        if url_info in self.required_url_infos:
            self.required_url_infos.remove(url_info)
예제 #10
0
class BarProgressRecorderSession(BaseProgressRecorderSession):
    '''Bar Progress Recorder Session.

    This session is responsible for displaying the ASCII bar
    and stats.
    '''
    def __init__(self, update_interval=0.5, bar_width=25, **kwargs):
        super().__init__(**kwargs)
        self._last_flush_time = 0
        self._update_interval = update_interval
        self._bytes_continued = 0
        self._total_size = None
        self._bar_width = bar_width
        self._throbber_index = 0
        self._throbber_iter = itertools.cycle(
            itertools.chain(range(bar_width), reversed(range(1,
                                                             bar_width - 1))))
        self._bandwidth_meter = BandwidthMeter()
        self._start_time = time.time()

    def pre_response(self, response):
        super().pre_response(response)

        if response.protocol == 'http' and \
                response.status_code == http.client.PARTIAL_CONTENT:
            match = re.search(r'bytes +([0-9]+)-([0-9]+)/([0-9]+)',
                              response.fields.get('Content-Range', ''))

            if match:
                self._bytes_continued = int(match.group(1))
                self._total_size = int(match.group(3))

        elif response.protocol == 'ftp' and response.restart_value:
            self._bytes_continued = response.restart_value
            self._total_size = self._content_length
        else:
            self._total_size = self._content_length

    def response_data(self, data):
        super().response_data(data)

        if not self._response:
            return

        self._bandwidth_meter.feed(len(data))

        time_now = time.time()

        if time_now - self._last_flush_time > self._update_interval:
            self._print_status()
            self._stream.flush()

            self._last_flush_time = time_now

    def response(self, response):
        self._print_status()
        self._stream.flush()
        super().response(response)

    def _print_status(self):
        '''Print an entire status line including bar and stats.'''
        self._clear_line()

        self._print('  ')

        if self._total_size:
            self._print_percent()
            self._print(' ')
            self._print_bar()
        else:
            self._print_throbber()

        self._print(' ')
        self._print_size_downloaded()
        self._print(' ')
        self._print_duration()
        self._print(' ')
        self._print_speed()
        self._flush()

    def _clear_line(self):
        '''Print ANSI code to clear the current line.'''
        self._print('\x1b[1G')
        self._print('\x1b[2K')

    def _print_throbber(self):
        '''Print an indefinite progress bar.'''
        self._print('[')

        for position in range(self._bar_width):
            self._print('O' if position == self._throbber_index else ' ')

        self._print(']')

        self._throbber_index = next(self._throbber_iter)

    def _print_bar(self):
        '''Print a progress bar.'''
        self._print('[')

        for position in range(self._bar_width):
            position_fraction = position / (self._bar_width - 1)
            position_bytes = position_fraction * self._total_size

            if position_bytes < self._bytes_continued:
                self._print('+')
            elif (position_bytes <=
                  self._bytes_continued + self._bytes_received):
                self._print('=')
            else:
                self._print(' ')

        self._print(']')

    def _print_size_downloaded(self):
        '''Print the bytes downloaded.'''
        self._print(wpull.string.format_size(self._bytes_received))

    def _print_duration(self):
        '''Print the elapsed download time.'''
        duration = int(time.time() - self._start_time)
        self._print(datetime.timedelta(seconds=duration))

    def _print_speed(self):
        '''Print the current speed.'''
        if self._bandwidth_meter.num_samples:
            speed = self._bandwidth_meter.speed()

            if self._human_format:
                file_size_str = wpull.string.format_size(speed)
            else:
                file_size_str = '{:.1f} b'.format(speed * 8)

            speed_str = _('{preformatted_file_size}/s').format(
                preformatted_file_size=file_size_str)
        else:
            speed_str = _('-- B/s')

        self._print(speed_str)

    def _print_percent(self):
        '''Print how much is done in percentage.'''
        fraction_done = ((self._bytes_continued + self._bytes_received) /
                         self._total_size)

        self._print('{fraction_done:.1%}'.format(fraction_done=fraction_done))