Пример #1
0
    def object_audit(self, path, device, partition):
        """
        Audits the given object path.

        :param path: a path to an object
        :param device: the device the path is on
        :param partition: the partition the path is on
        """
        try:
            try:
                name = diskfile.read_metadata(path)['name']
            except (Exception, Timeout) as exc:
                raise AuditException('Error when reading metadata: %s' % exc)
            _junk, account, container, obj = name.split('/', 3)
            df = diskfile.DiskFile(self.devices, device, partition,
                                   account, container, obj, self.logger,
                                   keep_data_fp=True)
            try:
                try:
                    obj_size = df.get_data_file_size()
                except DiskFileNotExist:
                    return
                except DiskFileError as e:
                    raise AuditException(str(e))
                if self.stats_sizes:
                    self.record_stats(obj_size)
                if self.zero_byte_only_at_fps and obj_size:
                    self.passes += 1
                    return
                for chunk in df:
                    self.bytes_running_time = ratelimit_sleep(
                        self.bytes_running_time, self.max_bytes_per_second,
                        incr_by=len(chunk))
                    self.bytes_processed += len(chunk)
                    self.total_bytes_processed += len(chunk)
                df.close()
                if df.quarantined_dir:
                    self.quarantines += 1
                    self.logger.error(
                        _("ERROR Object %(path)s failed audit and will be "
                          "quarantined: ETag and file's md5 do not match"),
                        {'path': path})
            finally:
                df.close(verify_file=False)
        except AuditException as err:
            self.logger.increment('quarantines')
            self.quarantines += 1
            self.logger.error(_('ERROR Object %(obj)s failed audit and will '
                                'be quarantined: %(err)s'),
                              {'obj': path, 'err': err})
            diskfile.quarantine_renamer(
                os.path.join(self.devices, device), path)
            return
        except (Exception, Timeout):
            self.logger.increment('errors')
            self.errors += 1
            self.logger.exception(_('ERROR Trying to audit %s'), path)
            return
        self.passes += 1
Пример #2
0
    def object_audit(self, path, device, partition):
        """
        Audits the given object path.

        :param path: a path to an object
        :param device: the device the path is on
        :param partition: the partition the path is on
        """
        try:
            try:
                name = diskfile.read_metadata(path)['name']
            except (Exception, Timeout) as exc:
                raise AuditException('Error when reading metadata: %s' % exc)
            _junk, account, container, obj = name.split('/', 3)
            df = self.diskfile_mgr.get_diskfile(
                device, partition, account, container, obj)
            try:
                with df.open():
                    metadata = df.get_metadata()
                    obj_size = int(metadata['Content-Length'])
                    if self.stats_sizes:
                        self.record_stats(obj_size)
                    if self.zero_byte_only_at_fps and obj_size:
                        self.passes += 1
                        return
                    reader = df.reader()
                with closing(reader):
                    for chunk in reader:
                        chunk_len = len(chunk)
                        self.bytes_running_time = ratelimit_sleep(
                            self.bytes_running_time,
                            self.max_bytes_per_second,
                            incr_by=chunk_len)
                        self.bytes_processed += chunk_len
                        self.total_bytes_processed += chunk_len
                if reader.was_quarantined:
                    self.quarantines += 1
                    self.logger.error(_('ERROR Object %(obj)s failed audit and'
                                        ' was quarantined: %(err)s'),
                                      {'obj': path,
                                       'err': reader.was_quarantined})
                    return
            except DiskFileNotExist:
                return
        except DiskFileQuarantined as err:
            self.quarantines += 1
            self.logger.error(_('ERROR Object %(obj)s failed audit and was'
                                ' quarantined: %(err)s'),
                              {'obj': path, 'err': err})
        except AuditException as err:
            self.logger.increment('quarantines')
            self.quarantines += 1
            self.logger.error(_('ERROR Object %(obj)s failed audit and will'
                                ' be quarantined: %(err)s'),
                              {'obj': path, 'err': err})
            diskfile.quarantine_renamer(
                os.path.join(self.devices, device), path)
            return
        self.passes += 1
Пример #3
0
    def object_audit(self, path, device, partition):
        """
        Audits the given object path.

        :param path: a path to an object
        :param device: the device the path is on
        :param partition: the partition the path is on
        """
        try:
            try:
                name = diskfile.read_metadata(path)['name']
            except (Exception, Timeout) as exc:
                raise AuditException('Error when reading metadata: %s' % exc)
            _junk, account, container, obj = name.split('/', 3)
            df = diskfile.DiskFile(self.devices, device, partition, account,
                                   container, obj, self.logger)
            df.open()
            try:
                try:
                    obj_size = df.get_data_file_size()
                except DiskFileNotExist:
                    return
                except DiskFileError as e:
                    raise AuditException(str(e))
                if self.stats_sizes:
                    self.record_stats(obj_size)
                if self.zero_byte_only_at_fps and obj_size:
                    self.passes += 1
                    return
                for chunk in df:
                    self.bytes_running_time = ratelimit_sleep(
                        self.bytes_running_time,
                        self.max_bytes_per_second,
                        incr_by=len(chunk))
                    self.bytes_processed += len(chunk)
                    self.total_bytes_processed += len(chunk)
                df.close()
                if df.quarantined_dir:
                    self.quarantines += 1
                    self.logger.error(
                        _("ERROR Object %(path)s failed audit and will be "
                          "quarantined: ETag and file's md5 do not match"),
                        {'path': path})
            finally:
                df.close(verify_file=False)
        except AuditException as err:
            self.logger.increment('quarantines')
            self.quarantines += 1
            self.logger.error(
                _('ERROR Object %(obj)s failed audit and will '
                  'be quarantined: %(err)s'), {
                      'obj': path,
                      'err': err
                  })
            diskfile.quarantine_renamer(os.path.join(self.devices, device),
                                        path)
            return
        self.passes += 1
Пример #4
0
    def object_audit(self, path, device, partition):
        """
        Audits the given object path.

        :param path: a path to an object
        :param device: the device the path is on
        :param partition: the partition the path is on
        """
        try:
            try:
                name = diskfile.read_metadata(path)['name']
            except (Exception, Timeout) as exc:
                raise AuditException('Error when reading metadata: %s' % exc)
            _junk, account, container, obj = name.split('/', 3)
            df = self.diskfile_mgr.get_diskfile(device, partition, account,
                                                container, obj)
            try:
                with df.open():
                    metadata = df.get_metadata()
                    obj_size = int(metadata['Content-Length'])
                    if self.stats_sizes:
                        self.record_stats(obj_size)
                    if self.zero_byte_only_at_fps and obj_size:
                        self.passes += 1
                        return
                    reader = df.reader()
                with closing(reader):
                    for chunk in reader:
                        chunk_len = len(chunk)
                        self.bytes_running_time = ratelimit_sleep(
                            self.bytes_running_time,
                            self.max_bytes_per_second,
                            incr_by=chunk_len)
                        self.bytes_processed += chunk_len
                        self.total_bytes_processed += chunk_len
                if reader.was_quarantined:
                    self.quarantines += 1
                    self.logger.error(
                        _('ERROR Object %(obj)s failed audit and'
                          ' was quarantined: %(err)s'), {
                              'obj': path,
                              'err': reader.was_quarantined
                          })
                    return
            except DiskFileNotExist:
                return
        except DiskFileQuarantined as err:
            self.quarantines += 1
            self.logger.error(
                _('ERROR Object %(obj)s failed audit and was'
                  ' quarantined: %(err)s'), {
                      'obj': path,
                      'err': err
                  })
        except AuditException as err:
            self.logger.increment('quarantines')
            self.quarantines += 1
            self.logger.error(
                _('ERROR Object %(obj)s failed audit and will'
                  ' be quarantined: %(err)s'), {
                      'obj': path,
                      'err': err
                  })
            diskfile.quarantine_renamer(os.path.join(self.devices, device),
                                        path)
            return
        self.passes += 1