Esempio n. 1
0
    def test_ratelimit_sleep(self):
        running_time = 0
        start = time.time()
        for i in range(100):
            running_time = utils.ratelimit_sleep(running_time, 0)
        self.assertTrue(abs((time.time() - start) * 100) < 1)

        running_time = 0
        start = time.time()
        for i in range(50):
            running_time = utils.ratelimit_sleep(running_time, 200)
        # make sure its accurate to 10th of a second
        self.assertTrue(abs(25 - (time.time() - start) * 100) < 10)
Esempio n. 2
0
    def test_ratelimit_sleep(self):
        running_time = 0
        start = time.time()
        for i in range(100):
            running_time = utils.ratelimit_sleep(running_time, 0)
        self.assertTrue(abs((time.time() - start) * 100) < 1)

        running_time = 0
        start = time.time()
        for i in range(50):
            running_time = utils.ratelimit_sleep(running_time, 200)
        # make sure its accurate to 10th of a second
        self.assertTrue(abs(25 - (time.time() - start) * 100) < 10)
Esempio n. 3
0
 def audit_all_objects(self, mode='once'):
     self.logger.info(_('Begin object audit "%s" mode (%s)' %
                        (mode, self.auditor_type)))
     begin = reported = time.time()
     self.total_bytes_processed = 0
     self.total_files_processed = 0
     total_quarantines = 0
     total_errors = 0
     files_running_time = 0
     time_auditing = 0
     all_locs = audit_location_generator(self.devices,
                                         object_server.DATADIR,
                                         mount_check=self.mount_check,
                                         logger=self.logger)
     for path, device, partition in all_locs:
         loop_time = time.time()
         self.object_audit(path, device, partition)
         self.files_running_time = ratelimit_sleep(
             self.files_running_time, self.max_files_per_second)
         self.total_files_processed += 1
         now = time.time()
         if now - reported >= self.log_time:
             self.logger.info(_(
                 'Object audit (%(type)s). '
                 'Since %(start_time)s: Locally: %(passes)d passed, '
                 '%(quars)d quarantined, %(errors)d errors '
                 'files/sec: %(frate).2f , bytes/sec: %(brate).2f, '
                 'Total time: %(total).2f, Auditing time: %(audit).2f, '
                 'Rate: %(audit_rate).2f') % {
                         'type': self.auditor_type,
                         'start_time': time.ctime(reported),
                         'passes': self.passes, 'quars': self.quarantines,
                         'errors': self.errors,
                         'frate': self.passes / (now - reported),
                         'brate': self.bytes_processed / (now - reported),
                         'total': (now - begin), 'audit': time_auditing,
                         'audit_rate': time_auditing / (now - begin)})
             reported = now
             total_quarantines += self.quarantines
             total_errors += self.errors
             self.passes = 0
             self.quarantines = 0
             self.errors = 0
             self.bytes_processed = 0
         time_auditing += (now - loop_time)
     elapsed = time.time() - begin
     self.logger.info(_(
         'Object audit (%(type)s) "%(mode)s" mode '
         'completed: %(elapsed).02fs. Total quarantined: %(quars)d, '
         'Total errors: %(errors)d, Total files/sec: %(frate).2f , '
         'Total bytes/sec: %(brate).2f, Auditing time: %(audit).2f, '
         'Rate: %(audit_rate).2f') % {
             'type': self.auditor_type, 'mode': mode, 'elapsed': elapsed,
             'quars': total_quarantines, 'errors': total_errors,
             'frate': self.total_files_processed / elapsed,
             'brate': self.total_bytes_processed / elapsed,
             'audit': time_auditing, 'audit_rate': time_auditing / elapsed})
Esempio n. 4
0
 def test_ratelimit_sleep_with_incr(self):
     running_time = 0
     start = time.time()
     vals = [5, 17, 0, 3, 11, 30, 40, 4, 13, 2, -1
             ] * 2  # adds up to 250 (with no -1)
     total = 0
     for i in vals:
         running_time = utils.ratelimit_sleep(running_time, 500, incr_by=i)
         total += i
     self.assertTrue(abs(50 - (time.time() - start) * 100) < 10)
Esempio n. 5
0
 def test_ratelimit_sleep_with_sleep(self):
     running_time = 0
     start = time.time()
     sleeps = [0] * 7 + [.2] * 3 + [0] * 30
     for i in sleeps:
         running_time = utils.ratelimit_sleep(running_time, 40,
                                              rate_buffer=1)
         time.sleep(i)
     # make sure its accurate to 10th of a second
     self.assertTrue(abs(100 - (time.time() - start) * 100) < 10)
Esempio n. 6
0
 def test_ratelimit_sleep_with_sleep(self):
     running_time = 0
     start = time.time()
     sleeps = [0] * 7 + [.2] * 3 + [0] * 30
     for i in sleeps:
         running_time = utils.ratelimit_sleep(running_time,
                                              40,
                                              rate_buffer=1)
         time.sleep(i)
     # make sure its accurate to 10th of a second
     self.assertTrue(abs(100 - (time.time() - start) * 100) < 10)
Esempio n. 7
0
 def test_ratelimit_sleep_with_incr(self):
     running_time = 0
     start = time.time()
     vals = [5, 17, 0, 3, 11, 30,
             40, 4, 13, 2, -1] * 2  # adds up to 250 (with no -1)
     total = 0
     for i in vals:
         running_time = utils.ratelimit_sleep(running_time,
                                              500, incr_by=i)
         total += i
     self.assertTrue(abs(50 - (time.time() - start) * 100) < 10)
Esempio n. 8
0
    def object_audit(self, path, device, partition):
        """
        Audits the given object path.

        :param path: a path to an object
        :param device: the device the path is on
        :param partition: the partition the path is on
        """
        try:
            if not path.endswith('.data'):
                return
            try:
                name = object_server.read_metadata(path)['name']
            except (Exception, Timeout), exc:
                raise AuditException('Error when reading metadata: %s' % exc)
            _junk, account, container, obj = name.split('/', 3)
            df = object_server.DiskFile(self.devices,
                                        device,
                                        partition,
                                        account,
                                        container,
                                        obj,
                                        self.logger,
                                        keep_data_fp=True)
            try:
                if df.data_file is None:
                    # file is deleted, we found the tombstone
                    return
                try:
                    obj_size = df.get_data_file_size()
                except DiskFileError, e:
                    raise AuditException(str(e))
                except DiskFileNotExist:
                    return
                if self.zero_byte_only_at_fps and obj_size:
                    self.passes += 1
                    return
                for chunk in df:
                    self.bytes_running_time = ratelimit_sleep(
                        self.bytes_running_time,
                        self.max_bytes_per_second,
                        incr_by=len(chunk))
                    self.bytes_processed += len(chunk)
                    self.total_bytes_processed += len(chunk)
                df.close()
                if df.quarantined_dir:
                    self.quarantines += 1
                    self.logger.error(
                        _("ERROR Object %(path)s failed audit and will be "
                          "quarantined: ETag and file's md5 do not match"),
                        {'path': path})
Esempio n. 9
0
    def object_audit(self, path, device, partition):
        """
        Audits the given object path.

        :param path: a path to an object
        :param device: the device the path is on
        :param partition: the partition the path is on
        """
        try:
            if not path.endswith('.data'):
                return
            try:
                name = object_server.read_metadata(path)['name']
            except (Exception, Timeout), exc:
                raise AuditException('Error when reading metadata: %s' % exc)
            _junk, account, container, obj = name.split('/', 3)
            df = object_server.DiskFile(self.devices, device, partition,
                                        account, container, obj, self.logger,
                                        keep_data_fp=True)
            try:
                if df.data_file is None:
                    # file is deleted, we found the tombstone
                    return
                try:
                    obj_size = df.get_data_file_size()
                except DiskFileError, e:
                    raise AuditException(str(e))
                except DiskFileNotExist:
                    return
                if self.zero_byte_only_at_fps and obj_size:
                    self.passes += 1
                    return
                for chunk in df:
                    self.bytes_running_time = ratelimit_sleep(
                        self.bytes_running_time, self.max_bytes_per_second,
                        incr_by=len(chunk))
                    self.bytes_processed += len(chunk)
                    self.total_bytes_processed += len(chunk)
                df.close()
                if df.quarantined_dir:
                    self.quarantines += 1
                    self.logger.error(
                        _("ERROR Object %(path)s failed audit and will be "
                          "quarantined: ETag and file's md5 do not match"),
                        {'path': path})
Esempio n. 10
0
 def audit_all_objects(self, mode='once'):
     self.logger.info(
         _('Begin object audit "%s" mode (%s)' % (mode, self.auditor_type)))
     begin = reported = time.time()
     self.total_bytes_processed = 0
     self.total_files_processed = 0
     total_quarantines = 0
     total_errors = 0
     files_running_time = 0
     time_auditing = 0
     all_locs = audit_location_generator(self.devices,
                                         object_server.DATADIR,
                                         mount_check=self.mount_check,
                                         logger=self.logger)
     for path, device, partition in all_locs:
         loop_time = time.time()
         self.object_audit(path, device, partition)
         self.files_running_time = ratelimit_sleep(
             self.files_running_time, self.max_files_per_second)
         self.total_files_processed += 1
         now = time.time()
         if now - reported >= self.log_time:
             self.logger.info(
                 _('Object audit (%(type)s). '
                   'Since %(start_time)s: Locally: %(passes)d passed, '
                   '%(quars)d quarantined, %(errors)d errors '
                   'files/sec: %(frate).2f , bytes/sec: %(brate).2f, '
                   'Total time: %(total).2f, Auditing time: %(audit).2f, '
                   'Rate: %(audit_rate).2f') % {
                       'type': self.auditor_type,
                       'start_time': time.ctime(reported),
                       'passes': self.passes,
                       'quars': self.quarantines,
                       'errors': self.errors,
                       'frate': self.passes / (now - reported),
                       'brate': self.bytes_processed / (now - reported),
                       'total': (now - begin),
                       'audit': time_auditing,
                       'audit_rate': time_auditing / (now - begin)
                   })
             reported = now
             total_quarantines += self.quarantines
             total_errors += self.errors
             self.passes = 0
             self.quarantines = 0
             self.errors = 0
             self.bytes_processed = 0
         time_auditing += (now - loop_time)
     elapsed = time.time() - begin
     self.logger.info(
         _('Object audit (%(type)s) "%(mode)s" mode '
           'completed: %(elapsed).02fs. Total quarantined: %(quars)d, '
           'Total errors: %(errors)d, Total files/sec: %(frate).2f , '
           'Total bytes/sec: %(brate).2f, Auditing time: %(audit).2f, '
           'Rate: %(audit_rate).2f') % {
               'type': self.auditor_type,
               'mode': mode,
               'elapsed': elapsed,
               'quars': total_quarantines,
               'errors': total_errors,
               'frate': self.total_files_processed / elapsed,
               'brate': self.total_bytes_processed / elapsed,
               'audit': time_auditing,
               'audit_rate': time_auditing / elapsed
           })