Ejemplo n.º 1
17
 def test_non_dir_contents(self):
     with temptree([]) as tmpdir:
         data = os.path.join(tmpdir, "drive", "data")
         os.makedirs(data)
         with open(os.path.join(data, "partition1"), "w"):
             pass
         partition = os.path.join(data, "partition2")
         os.makedirs(partition)
         with open(os.path.join(partition, "suffix1"), "w"):
             pass
         suffix = os.path.join(partition, "suffix2")
         os.makedirs(suffix)
         with open(os.path.join(suffix, "hash1"), "w"):
             pass
         locations = ondisk.audit_location_generator(
             tmpdir, "data", mount_check=False
         )
         self.assertEqual(list(locations), [])
Ejemplo n.º 2
0
 def _one_audit_pass(self, reported):
     all_locs = audit_location_generator(self.devices,
                                         container_server.DATADIR, '.db',
                                         mount_check=self.mount_check,
                                         logger=self.logger)
     for path, device, partition in all_locs:
         self.container_audit(path)
         if time.time() - reported >= 3600:  # once an hour
             self.logger.info(
                 _('Since %(time)s: Container audits: %(pass)s passed '
                   'audit, %(fail)s failed audit'),
                 {'time': time.ctime(reported),
                  'pass': self.container_passes,
                  'fail': self.container_failures})
             dump_recon_cache(
                 {'container_audits_since': reported,
                  'container_audits_passed': self.container_passes,
                  'container_audits_failed': self.container_failures},
                 self.rcache, self.logger)
             reported = time.time()
             self.container_passes = 0
             self.container_failures = 0
         self.containers_running_time = ratelimit_sleep(
             self.containers_running_time, self.max_containers_per_second)
     return reported
Ejemplo n.º 3
0
 def _one_audit_pass(self, reported):
     all_locs = audit_location_generator(self.devices,
                                         container_server.DATADIR,
                                         '.db',
                                         mount_check=self.mount_check,
                                         logger=self.logger)
     for path, device, partition in all_locs:
         self.container_audit(path)
         if time.time() - reported >= 3600:  # once an hour
             self.logger.info(
                 _('Since %(time)s: Container audits: %(pass)s passed '
                   'audit, %(fail)s failed audit'), {
                       'time': time.ctime(reported),
                       'pass': self.container_passes,
                       'fail': self.container_failures
                   })
             dump_recon_cache(
                 {
                     'container_audits_since': reported,
                     'container_audits_passed': self.container_passes,
                     'container_audits_failed': self.container_failures
                 }, self.rcache, self.logger)
             reported = time.time()
             self.container_passes = 0
             self.container_failures = 0
         self.containers_running_time = ratelimit_sleep(
             self.containers_running_time, self.max_containers_per_second)
     return reported
Ejemplo n.º 4
0
 def run_once(self):
     """
     Runs a single container sync scan.
     """
     self.logger.info(_('Begin container sync "once" mode'))
     begin = time()
     all_locs = audit_location_generator(self.devices,
                                         container_server.DATADIR, '.db',
                                         mount_check=self.mount_check,
                                         logger=self.logger)
     for path, device, partition in all_locs:
         self.container_sync(path)
         if time() - self.reported >= 3600:  # once an hour
             self.report()
     self.report()
     elapsed = time() - begin
     self.logger.info(
         _('Container sync "once" mode completed: %.02fs'), elapsed)
Ejemplo n.º 5
0
 def run_forever(self):
     """
     Runs container sync scans until stopped.
     """
     sleep(random() * self.interval)
     while True:
         begin = time()
         all_locs = audit_location_generator(self.devices,
                                             container_server.DATADIR,
                                             '.db',
                                             mount_check=self.mount_check,
                                             logger=self.logger)
         for path, device, partition in all_locs:
             self.container_sync(path)
             if time() - self.reported >= 3600:  # once an hour
                 self.report()
         elapsed = time() - begin
         if elapsed < self.interval:
             sleep(self.interval - elapsed)
Ejemplo n.º 6
0
 def run_once(self):
     """
     Runs a single container sync scan.
     """
     self.logger.info(_('Begin container sync "once" mode'))
     begin = time()
     all_locs = audit_location_generator(self.devices,
                                         container_server.DATADIR,
                                         '.db',
                                         mount_check=self.mount_check,
                                         logger=self.logger)
     for path, device, partition in all_locs:
         self.container_sync(path)
         if time() - self.reported >= 3600:  # once an hour
             self.report()
     self.report()
     elapsed = time() - begin
     self.logger.info(_('Container sync "once" mode completed: %.02fs'),
                      elapsed)
Ejemplo n.º 7
0
 def run_forever(self):
     """
     Runs container sync scans until stopped.
     """
     sleep(random() * self.interval)
     while True:
         begin = time()
         all_locs = audit_location_generator(self.devices,
                                             container_server.DATADIR,
                                             '.db',
                                             mount_check=self.mount_check,
                                             logger=self.logger)
         for path, device, partition in all_locs:
             self.container_sync(path)
             if time() - self.reported >= 3600:  # once an hour
                 self.report()
         elapsed = time() - begin
         if elapsed < self.interval:
             sleep(self.interval - elapsed)
Ejemplo n.º 8
0
 def audit_all_objects(self, mode='once'):
     self.logger.info(_('Begin object audit "%s" mode (%s)') %
                      (mode, self.auditor_type))
     begin = reported = time.time()
     self.total_bytes_processed = 0
     self.total_files_processed = 0
     total_quarantines = 0
     total_errors = 0
     time_auditing = 0
     all_locs = audit_location_generator(self.devices,
                                         object_server.DATADIR, '.data',
                                         mount_check=self.mount_check,
                                         logger=self.logger)
     for path, device, partition in all_locs:
         loop_time = time.time()
         self.failsafe_object_audit(path, device, partition)
         self.logger.timing_since('timing', loop_time)
         self.files_running_time = ratelimit_sleep(
             self.files_running_time, self.max_files_per_second)
         self.total_files_processed += 1
         now = time.time()
         if now - reported >= self.log_time:
             self.logger.info(_(
                 'Object audit (%(type)s). '
                 'Since %(start_time)s: Locally: %(passes)d passed, '
                 '%(quars)d quarantined, %(errors)d errors '
                 'files/sec: %(frate).2f , bytes/sec: %(brate).2f, '
                 'Total time: %(total).2f, Auditing time: %(audit).2f, '
                 'Rate: %(audit_rate).2f') % {
                     'type': self.auditor_type,
                     'start_time': time.ctime(reported),
                     'passes': self.passes, 'quars': self.quarantines,
                     'errors': self.errors,
                     'frate': self.passes / (now - reported),
                     'brate': self.bytes_processed / (now - reported),
                     'total': (now - begin), 'audit': time_auditing,
                     'audit_rate': time_auditing / (now - begin)})
             dump_recon_cache({'object_auditor_stats_%s' %
                               self.auditor_type: {
                                   'errors': self.errors,
                                   'passes': self.passes,
                                   'quarantined': self.quarantines,
                                   'bytes_processed': self.bytes_processed,
                                   'start_time': reported,
                                   'audit_time': time_auditing}},
                              self.rcache, self.logger)
             reported = now
             total_quarantines += self.quarantines
             total_errors += self.errors
             self.passes = 0
             self.quarantines = 0
             self.errors = 0
             self.bytes_processed = 0
         time_auditing += (now - loop_time)
     # Avoid divide by zero during very short runs
     elapsed = (time.time() - begin) or 0.000001
     self.logger.info(_(
         'Object audit (%(type)s) "%(mode)s" mode '
         'completed: %(elapsed).02fs. Total quarantined: %(quars)d, '
         'Total errors: %(errors)d, Total files/sec: %(frate).2f , '
         'Total bytes/sec: %(brate).2f, Auditing time: %(audit).2f, '
         'Rate: %(audit_rate).2f') % {
             'type': self.auditor_type, 'mode': mode, 'elapsed': elapsed,
             'quars': total_quarantines, 'errors': total_errors,
             'frate': self.total_files_processed / elapsed,
             'brate': self.total_bytes_processed / elapsed,
             'audit': time_auditing, 'audit_rate': time_auditing / elapsed})
     if self.stats_sizes:
         self.logger.info(
             _('Object audit stats: %s') % json.dumps(self.stats_buckets))
Ejemplo n.º 9
0
 def audit_all_objects(self, mode='once'):
     self.logger.info(
         _('Begin object audit "%s" mode (%s)') % (mode, self.auditor_type))
     begin = reported = time.time()
     self.total_bytes_processed = 0
     self.total_files_processed = 0
     total_quarantines = 0
     total_errors = 0
     time_auditing = 0
     all_locs = audit_location_generator(self.devices,
                                         object_server.DATADIR,
                                         '.data',
                                         mount_check=self.mount_check,
                                         logger=self.logger)
     for path, device, partition in all_locs:
         loop_time = time.time()
         self.failsafe_object_audit(path, device, partition)
         self.logger.timing_since('timing', loop_time)
         self.files_running_time = ratelimit_sleep(
             self.files_running_time, self.max_files_per_second)
         self.total_files_processed += 1
         now = time.time()
         if now - reported >= self.log_time:
             self.logger.info(
                 _('Object audit (%(type)s). '
                   'Since %(start_time)s: Locally: %(passes)d passed, '
                   '%(quars)d quarantined, %(errors)d errors '
                   'files/sec: %(frate).2f , bytes/sec: %(brate).2f, '
                   'Total time: %(total).2f, Auditing time: %(audit).2f, '
                   'Rate: %(audit_rate).2f') % {
                       'type': self.auditor_type,
                       'start_time': time.ctime(reported),
                       'passes': self.passes,
                       'quars': self.quarantines,
                       'errors': self.errors,
                       'frate': self.passes / (now - reported),
                       'brate': self.bytes_processed / (now - reported),
                       'total': (now - begin),
                       'audit': time_auditing,
                       'audit_rate': time_auditing / (now - begin)
                   })
             dump_recon_cache(
                 {
                     'object_auditor_stats_%s' % self.auditor_type: {
                         'errors': self.errors,
                         'passes': self.passes,
                         'quarantined': self.quarantines,
                         'bytes_processed': self.bytes_processed,
                         'start_time': reported,
                         'audit_time': time_auditing
                     }
                 }, self.rcache, self.logger)
             reported = now
             total_quarantines += self.quarantines
             total_errors += self.errors
             self.passes = 0
             self.quarantines = 0
             self.errors = 0
             self.bytes_processed = 0
         time_auditing += (now - loop_time)
     # Avoid divide by zero during very short runs
     elapsed = (time.time() - begin) or 0.000001
     self.logger.info(
         _('Object audit (%(type)s) "%(mode)s" mode '
           'completed: %(elapsed).02fs. Total quarantined: %(quars)d, '
           'Total errors: %(errors)d, Total files/sec: %(frate).2f , '
           'Total bytes/sec: %(brate).2f, Auditing time: %(audit).2f, '
           'Rate: %(audit_rate).2f') % {
               'type': self.auditor_type,
               'mode': mode,
               'elapsed': elapsed,
               'quars': total_quarantines,
               'errors': total_errors,
               'frate': self.total_files_processed / elapsed,
               'brate': self.total_bytes_processed / elapsed,
               'audit': time_auditing,
               'audit_rate': time_auditing / elapsed
           })
     if self.stats_sizes:
         self.logger.info(
             _('Object audit stats: %s') % json.dumps(self.stats_buckets))
Ejemplo n.º 10
0
 def audit_all_objects(self, mode="once"):
     self.logger.info(_('Begin object audit "%s" mode (%s)') % (mode, self.auditor_type))
     begin = reported = time.time()
     self.total_bytes_processed = 0
     self.total_files_processed = 0
     total_quarantines = 0
     total_errors = 0
     time_auditing = 0
     all_locs = audit_location_generator(
         self.devices, object_server.DATADIR, ".data", mount_check=self.mount_check, logger=self.logger
     )
     for path, device, partition in all_locs:
         loop_time = time.time()
         self.failsafe_object_audit(path, device, partition)
         self.logger.timing_since("timing", loop_time)
         self.files_running_time = ratelimit_sleep(self.files_running_time, self.max_files_per_second)
         self.total_files_processed += 1
         now = time.time()
         if now - reported >= self.log_time:
             self.logger.info(
                 _(
                     "Object audit (%(type)s). "
                     "Since %(start_time)s: Locally: %(passes)d passed, "
                     "%(quars)d quarantined, %(errors)d errors "
                     "files/sec: %(frate).2f , bytes/sec: %(brate).2f, "
                     "Total time: %(total).2f, Auditing time: %(audit).2f, "
                     "Rate: %(audit_rate).2f"
                 )
                 % {
                     "type": self.auditor_type,
                     "start_time": time.ctime(reported),
                     "passes": self.passes,
                     "quars": self.quarantines,
                     "errors": self.errors,
                     "frate": self.passes / (now - reported),
                     "brate": self.bytes_processed / (now - reported),
                     "total": (now - begin),
                     "audit": time_auditing,
                     "audit_rate": time_auditing / (now - begin),
                 }
             )
             dump_recon_cache(
                 {
                     "object_auditor_stats_%s"
                     % self.auditor_type: {
                         "errors": self.errors,
                         "passes": self.passes,
                         "quarantined": self.quarantines,
                         "bytes_processed": self.bytes_processed,
                         "start_time": reported,
                         "audit_time": time_auditing,
                     }
                 },
                 self.rcache,
                 self.logger,
             )
             reported = now
             total_quarantines += self.quarantines
             total_errors += self.errors
             self.passes = 0
             self.quarantines = 0
             self.errors = 0
             self.bytes_processed = 0
         time_auditing += now - loop_time
     # Avoid divide by zero during very short runs
     elapsed = (time.time() - begin) or 0.000001
     self.logger.info(
         _(
             'Object audit (%(type)s) "%(mode)s" mode '
             "completed: %(elapsed).02fs. Total quarantined: %(quars)d, "
             "Total errors: %(errors)d, Total files/sec: %(frate).2f , "
             "Total bytes/sec: %(brate).2f, Auditing time: %(audit).2f, "
             "Rate: %(audit_rate).2f"
         )
         % {
             "type": self.auditor_type,
             "mode": mode,
             "elapsed": elapsed,
             "quars": total_quarantines,
             "errors": total_errors,
             "frate": self.total_files_processed / elapsed,
             "brate": self.total_bytes_processed / elapsed,
             "audit": time_auditing,
             "audit_rate": time_auditing / elapsed,
         }
     )
     if self.stats_sizes:
         self.logger.info(_("Object audit stats: %s") % json.dumps(self.stats_buckets))