def index_pass(self): start_time = report_time = time.time() total_errors = 0 paths = paths_gen(self.volume) for path in paths: self.safe_update_index(path) self.chunks_run_time = ratelimit(self.chunks_run_time, self.max_chunks_per_second) self.total_chunks_processed += 1 now = time.time() if now - self.last_reported >= self.report_interval: self.logger.info( 'started=%(start_time)s ' 'passes=%(passes)d ' 'errors=%(errors)d ' 'chunks=%(nb_chunks)d %(c_rate).2f/s ' 'total=%(total).2f ' % { 'start_time': datetime.fromtimestamp(int(report_time)).isoformat(), 'passes': self.passes, 'errors': self.errors, 'nb_chunks': self.total_chunks_processed, 'c_rate': self.passes / (now - report_time), 'total': (now - start_time) }) report_time = now total_errors += self.errors self.passes = 0 self.errors = 0 self.last_reported = now end_time = time.time() elapsed = (end_time - start_time) or 0.000001 self.logger.info( 'started=%(start_time)s ' 'ended=%(end_time)s ' 'elapsed=%(elapsed).02f ' 'errors=%(errors)d ' 'chunks=%(nb_chunks)d %(c_rate).2f/s ' % { 'start_time': datetime.fromtimestamp( int(start_time)).isoformat(), 'end_time': datetime.fromtimestamp(int(end_time)).isoformat(), 'elapsed': elapsed, 'errors': total_errors + self.errors, 'nb_chunks': self.total_chunks_processed, 'c_rate': self.total_chunks_processed / elapsed }) if elapsed < self.interval: time.sleep(self.interval - elapsed)
def pass_without_lock(self): last_report = now() count, success, fail = 0, 0, 0 if self.namespace != self.volume_ns: self.logger.warn("Forcing the NS to [%s] (previously [%s])", self.namespace, self.volume_ns) self.logger.info("START %s", self.volume) paths = paths_gen(self.volume) for path in paths: chunk_id = path.rsplit('/', 1)[-1] if len(chunk_id) != STRLEN_CHUNKID: self.logger.warn('WARN Not a chunk %s' % path) return for c in chunk_id: if c not in hexdigits: self.logger.warn('WARN Not a chunk %s' % path) return # Action try: with open(path) as f: meta = read_chunk_metadata(f, chunk_id) self.action(self, path, f, meta) success = success + 1 except NotFound as e: fail = fail + 1 self.logger.info("ORPHAN %s/%s in %s/%s %s", meta['content_id'], meta['chunk_id'], meta['container_id'], meta['content_path'], str(e)) except Conflict as e: fail = fail + 1 self.logger.info("ALREADY %s/%s in %s/%s %s", meta['content_id'], meta['chunk_id'], meta['container_id'], meta['content_path'], str(e)) except Exception as e: fail = fail + 1 self.logger.warn("ERROR %s/%s in %s/%s %s", meta['content_id'], meta['chunk_id'], meta['container_id'], meta['content_path'], str(e)) count = count + 1 # TODO(jfs): do the throttling # periodical reporting t = now() if t - last_report > self.report_interval: self.logger.info("STEP %d ok %d ko %d", count, success, fail) self.logger.info("FINAL %s %d ok %d ko %d", self.volume, count, success, fail)
def index_pass(self): def safe_update_index(path): chunk_id = path.rsplit('/', 1)[-1] if len(chunk_id) != STRLEN_CHUNKID: return for c in chunk_id: if c not in hexdigits: return try: self.update_index(path) self.successes += 1 self.logger.debug('Updated %s', path) except OioNetworkException as exc: self.errors += 1 self.logger.warn('ERROR while updating %s: %s', path, exc) except Exception: self.errors += 1 self.logger.exception('ERROR while updating %s', path) def report(tag): total = self.errors + self.successes now = time.time() elapsed = (now - start_time) or 0.000001 self.logger.info( '%(tag)s=%(current_time)s ' 'elapsed=%(elapsed).02f ' 'pass=%(pass)d ' 'errors=%(errors)d ' 'chunks=%(nb_chunks)d %(c_rate).2f/s' % { 'tag': tag, 'current_time': datetime.fromtimestamp( int(now)).isoformat(), 'pass': self.passes, 'errors': self.errors, 'nb_chunks': total, 'c_rate': total / (now - self.last_reported), 'elapsed': elapsed }) self.last_reported = now start_time = time.time() self.last_reported = start_time self.errors = 0 self.successes = 0 paths = paths_gen(self.volume) report('started') for path in paths: safe_update_index(path) self.chunks_run_time = ratelimit(self.chunks_run_time, self.max_chunks_per_second) now = time.time() if now - self.last_reported >= self.report_interval: report('running') report('ended')
def index_pass(self): start_time = report_time = time.time() total_errors = 0 paths = paths_gen(self.volume) for path in paths: self.safe_update_index(path) self.chunks_run_time = ratelimit( self.chunks_run_time, self.max_chunks_per_second ) self.total_chunks_processed += 1 now = time.time() if now - self.last_reported >= self.report_interval: self.logger.info( '%(start_time)s ' '%(passes)d ' '%(errors)d ' '%(c_rate).2f ' '%(total).2f ' % { 'start_time': time.ctime(report_time), 'passes': self.passes, 'errors': self.errors, 'c_rate': self.passes / (now - report_time), 'total': (now - start_time) } ) report_time = now total_errors += self.errors self.passes = 0 self.errors = 0 self.last_reported = now elapsed = (time.time() - start_time) or 0.000001 self.logger.info( '%(elapsed).02f ' '%(errors)d ' '%(chunk_rate).2f ' % { 'elapsed': elapsed, 'errors': total_errors + self.errors, 'chunk_rate': self.total_chunks_processed / elapsed } ) if elapsed < self.interval: time.sleep(self.interval - elapsed)
def index_pass(self): start_time = report_time = time.time() total_errors = 0 paths = paths_gen(self.volume) for path in paths: self.safe_update_index(path) self.chunks_run_time = ratelimit( self.chunks_run_time, self.max_chunks_per_second ) self.total_chunks_processed += 1 now = time.time() if now - self.last_reported >= self.report_interval: self.logger.info( '%(start_time)s ' '%(passes)d ' '%(errors)d ' '%(c_rate).2f ' '%(total).2f ' % { 'start_time': time.ctime(report_time), 'passes': self.passes, 'errors': self.errors, 'c_rate': self.passes / (now - report_time), 'total': (now - start_time) } ) report_time = now total_errors += self.errors self.passes = 0 self.errors = 0 self.last_reported = now elapsed = (time.time() - start_time) or 0.000001 self.logger.info( '%(elapsed).02f ' '%(errors)d ' '%(chunk_rate).2f ' % { 'elapsed': elapsed, 'errors': total_errors + self.errors, 'chunk_rate': self.total_chunks_processed / elapsed } )
def index_pass(self): start_time = time.time() self.last_reported = start_time self.errors = 0 self.successes = 0 paths = paths_gen(self.volume) self.report('started', start_time) for path in paths: self.safe_update_index(path) self.chunks_run_time = ratelimit( self.chunks_run_time, self.max_chunks_per_second ) now = time.time() if now - self.last_reported >= self.report_interval: self.report('running', start_time) self.report('ended', start_time)
def setUp(self): super(TestBlobIndexer, self).setUp() self.rdir_client = RdirClient(self.conf) self.blob_client = BlobClient(self.conf) _, self.rawx_path, rawx_addr, _ = \ self.get_service_url('rawx') services = self.conscience.all_services('rawx') self.rawx_id = None for rawx in services: if rawx_addr == rawx['addr']: self.rawx_id = rawx['tags'].get('tag.service_id', None) if self.rawx_id is None: self.rawx_id = rawx_addr conf = self.conf.copy() conf['volume'] = self.rawx_path self.blob_indexer = BlobIndexer(conf) # clear rawx/rdir chunk_files = paths_gen(self.rawx_path) for chunk_file in chunk_files: os.remove(chunk_file) self.rdir_client.admin_clear(self.rawx_id, clear_all=True)
def pass_volume(self): self.start_time = self.last_report = time.time() self.log_report('START', force=True) paths = paths_gen(self.volume) for path in paths: try: self.pass_chunk_file(path) self.chunks_processed += 1 except Exception as exc: self.logger.error( 'Failed to pass chunk file (chunk_file=%s): %s', path, exc) self.chunk_errors += 1 self.log_report('RUN') self.chunks_run_time = ratelimit(self.chunks_run_time, self.max_chunks_per_second) self.log_report('DONE', force=True) return self.chunk_errors == 0 \ and all(errors == 0 for errors in self.bean_errors.values())
def crawl_volume(self): """ Crawl the volume assigned to this worker, and index every database. """ paths = paths_gen(self.volume) self.full_scan_nb += 1 self.success_nb = 0 self.failed_nb = 0 now = time.time() self.last_report_time = now self.report("starting") for db_path in paths: # Graceful exit, hopefully if self._stop: break db_id = db_path.rsplit("/")[-1].rsplit(".") if len(db_id) != 3: self.warn("Malformed db file name !", db_path) continue db_id = ".".join(db_id[:2]) self.index_meta2_database(db_id) self.last_index_time = ratelimit( self.last_index_time, self.max_indexed_per_second ) now = time.time() if now - self.last_report_time >= self.report_interval: self.report("running") self.report("ended")
def mover_pass(self): self.namespace, self.address = check_volume(self.volume) start_time = report_time = time.time() total_errors = 0 mover_time = 0 paths = paths_gen(self.volume) for path in paths: loop_time = time.time() now = time.time() if now - self.last_usage_check >= self.usage_check_interval: used, total = statfs(self.volume) usage = (float(used) / total) * 100 if usage <= self.usage_target: self.logger.info("current usage %.2f%%: target reached (%.2f%%)", usage, self.usage_target) self.last_usage_check = now break self.safe_chunk_move(path) self.chunks_run_time = ratelimit(self.chunks_run_time, self.max_chunks_per_second) self.total_chunks_processed += 1 now = time.time() if now - self.last_reported >= self.report_interval: self.logger.info( "%(start_time)s " "%(passes)d " "%(errors)d " "%(c_rate).2f " "%(b_rate).2f " "%(total).2f " "%(mover_time).2f" "%(mover_rate).2f" % { "start_time": time.ctime(report_time), "passes": self.passes, "errors": self.errors, "c_rate": self.passes / (now - report_time), "b_rate": self.bytes_processed / (now - report_time), "total": (now - start_time), "mover_time": mover_time, "mover_rate": mover_time / (now - start_time), } ) report_time = now total_errors += self.errors self.passes = 0 self.bytes_processed = 0 self.last_reported = now mover_time += now - loop_time elapsed = (time.time() - start_time) or 0.000001 self.logger.info( "%(elapsed).02f " "%(errors)d " "%(chunk_rate).2f " "%(bytes_rate).2f " "%(mover_time).2f " "%(mover_rate).2f" % { "elapsed": elapsed, "errors": total_errors + self.errors, "chunk_rate": self.total_chunks_processed / elapsed, "bytes_rate": self.total_bytes_processed / elapsed, "mover_time": mover_time, "mover_rate": mover_time / elapsed, } )
def mover_pass(self, **kwargs): start_time = report_time = time.time() total_errors = 0 mover_time = 0 pool = GreenPool(self.concurrency) paths = paths_gen(self.volume) for path in paths: loop_time = time.time() now = time.time() if now - self.last_usage_check >= self.usage_check_interval: free_ratio = statfs(self.volume) usage = (1 - float(free_ratio)) * 100 if usage <= self.usage_target: self.logger.info( 'current usage %.2f%%: target reached (%.2f%%)', usage, self.usage_target) break self.last_usage_check = now # Spawn a chunk move task. # The call will block if no green thread is available. pool.spawn_n(self.safe_chunk_move, path) self.chunks_run_time = ratelimit(self.chunks_run_time, self.max_chunks_per_second) self.total_chunks_processed += 1 now = time.time() if now - self.last_reported >= self.report_interval: self.logger.info( '%(start_time)s ' '%(passes)d ' '%(errors)d ' '%(c_rate).2f ' '%(b_rate).2f ' '%(total).2f ' '%(mover_time).2f' '%(mover_rate).2f' % { 'start_time': time.ctime(report_time), 'passes': self.passes, 'errors': self.errors, 'c_rate': self.passes / (now - report_time), 'b_rate': self.bytes_processed / (now - report_time), 'total': (now - start_time), 'mover_time': mover_time, 'mover_rate': mover_time / (now - start_time) }) report_time = now total_errors += self.errors self.passes = 0 self.bytes_processed = 0 self.last_reported = now mover_time += (now - loop_time) if self.limit != 0 and self.total_chunks_processed >= self.limit: break pool.waitall() elapsed = (time.time() - start_time) or 0.000001 self.logger.info( '%(elapsed).02f ' '%(errors)d ' '%(chunk_rate).2f ' '%(bytes_rate).2f ' '%(mover_time).2f ' '%(mover_rate).2f' % { 'elapsed': elapsed, 'errors': total_errors + self.errors, 'chunk_rate': self.total_chunks_processed / elapsed, 'bytes_rate': self.total_bytes_processed / elapsed, 'mover_time': mover_time, 'mover_rate': mover_time / elapsed })
def audit_pass(self): self.namespace, self.address = check_volume(self.volume) start_time = report_time = time.time() total_errors = 0 total_corrupted = 0 total_orphans = 0 total_faulty = 0 audit_time = 0 paths = paths_gen(self.volume) for path in paths: loop_time = time.time() self.safe_chunk_audit(path) self.chunks_run_time = ratelimit( self.chunks_run_time, self.max_chunks_per_second ) self.total_chunks_processed += 1 now = time.time() if now - self.last_reported >= self.report_interval: self.logger.info( '%(start_time)s ' '%(passes)d ' '%(corrupted)d ' '%(faulty)d ' '%(orphans)d ' '%(errors)d ' '%(c_rate).2f ' '%(b_rate).2f ' '%(total).2f ' '%(audit_time).2f' '%(audit_rate).2f' % { 'start_time': time.ctime(report_time), 'passes': self.passes, 'corrupted': self.corrupted_chunks, 'faulty': self.faulty_chunks, 'orphans': self.orphan_chunks, 'errors': self.errors, 'c_rate': self.passes / (now - report_time), 'b_rate': self.bytes_processed / (now - report_time), 'total': (now - start_time), 'audit_time': audit_time, 'audit_rate': audit_time / (now - start_time) } ) report_time = now total_corrupted += self.corrupted_chunks total_orphans += self.orphan_chunks total_faulty += self.faulty_chunks total_errors += self.errors self.passes = 0 self.corrupted_chunks = 0 self.orphan_chunks = 0 self.faulty_chunks = 0 self.errors = 0 self.bytes_processed = 0 self.last_reported = now audit_time += (now - loop_time) elapsed = (time.time() - start_time) or 0.000001 self.logger.info( '%(elapsed).02f ' '%(corrupted)d ' '%(faulty)d ' '%(orphans)d ' '%(errors)d ' '%(chunk_rate).2f ' '%(bytes_rate).2f ' '%(audit_time).2f ' '%(audit_rate).2f' % { 'elapsed': elapsed, 'corrupted': total_corrupted + self.corrupted_chunks, 'faulty': total_faulty + self.faulty_chunks, 'orphans': total_orphans + self.orphan_chunks, 'errors': total_errors + self.errors, 'chunk_rate': self.total_chunks_processed / elapsed, 'bytes_rate': self.total_bytes_processed / elapsed, 'audit_time': audit_time, 'audit_rate': audit_time / elapsed } )
def paths_gen(self, input_file=None): if input_file: return self._fetch_chunks_from_file(input_file) else: return paths_gen(self.volume)
def audit_pass(self): self.namespace, self.address = check_volume(self.volume) start_time = report_time = time.time() total_errors = 0 total_corrupted = 0 total_orphans = 0 total_faulty = 0 audit_time = 0 paths = paths_gen(self.volume) for path in paths: loop_time = time.time() self.safe_chunk_audit(path) self.chunks_run_time = ratelimit(self.chunks_run_time, self.max_chunks_per_second) self.total_chunks_processed += 1 now = time.time() if now - self.last_reported >= self.report_interval: self.logger.info( "%(start_time)s " "%(passes)d " "%(corrupted)d " "%(faulty)d " "%(orphans)d " "%(errors)d " "%(c_rate).2f " "%(b_rate).2f " "%(total).2f " "%(audit_time).2f" "%(audit_rate).2f" % { "start_time": time.ctime(report_time), "passes": self.passes, "corrupted": self.corrupted_chunks, "faulty": self.faulty_chunks, "orphans": self.orphan_chunks, "errors": self.errors, "c_rate": self.passes / (now - report_time), "b_rate": self.bytes_processed / (now - report_time), "total": (now - start_time), "audit_time": audit_time, "audit_rate": audit_time / (now - start_time), } ) report_time = now total_corrupted += self.corrupted_chunks total_orphans += self.orphan_chunks total_faulty += self.faulty_chunks total_errors += self.errors self.passes = 0 self.corrupted_chunks = 0 self.orphan_chunks = 0 self.faulty_chunks = 0 self.errors = 0 self.bytes_processed = 0 self.last_reported = now audit_time += now - loop_time elapsed = (time.time() - start_time) or 0.000001 self.logger.info( "%(elapsed).02f " "%(corrupted)d " "%(faulty)d " "%(orphans)d " "%(errors)d " "%(chunk_rate).2f " "%(bytes_rate).2f " "%(audit_time).2f " "%(audit_rate).2f" % { "elapsed": elapsed, "corrupted": total_corrupted + self.corrupted_chunks, "faulty": total_faulty + self.faulty_chunks, "orphans": total_orphans + self.orphan_chunks, "errors": total_errors + self.errors, "chunk_rate": self.total_chunks_processed / elapsed, "bytes_rate": self.total_bytes_processed / elapsed, "audit_time": audit_time, "audit_rate": audit_time / elapsed, } )
def audit_pass(self): self.namespace, self.address = check_volume(self.volume) start_time = report_time = time.time() total_errors = 0 total_corrupted = 0 total_orphans = 0 total_faulty = 0 audit_time = 0 paths = paths_gen(self.volume) for path in paths: loop_time = time.time() self.safe_chunk_audit(path) self.chunks_run_time = ratelimit(self.chunks_run_time, self.max_chunks_per_second) self.total_chunks_processed += 1 now = time.time() if now - self.last_reported >= self.report_interval: self.logger.info( '%(start_time)s ' '%(passes)d ' '%(corrupted)d ' '%(faulty)d ' '%(orphans)d ' '%(errors)d ' '%(c_rate).2f ' '%(b_rate).2f ' '%(total).2f ' '%(audit_time).2f' '%(audit_rate).2f' % { 'start_time': time.ctime(report_time), 'passes': self.passes, 'corrupted': self.corrupted_chunks, 'faulty': self.faulty_chunks, 'orphans': self.orphan_chunks, 'errors': self.errors, 'c_rate': self.passes / (now - report_time), 'b_rate': self.bytes_processed / (now - report_time), 'total': (now - start_time), 'audit_time': audit_time, 'audit_rate': audit_time / (now - start_time) }) report_time = now total_corrupted += self.corrupted_chunks total_orphans += self.orphan_chunks total_faulty += self.faulty_chunks total_errors += self.errors self.passes = 0 self.corrupted_chunks = 0 self.orphan_chunks = 0 self.faulty_chunks = 0 self.errors = 0 self.bytes_processed = 0 self.last_reported = now audit_time += (now - loop_time) elapsed = (time.time() - start_time) or 0.000001 self.logger.info( '%(elapsed).02f ' '%(corrupted)d ' '%(faulty)d ' '%(orphans)d ' '%(errors)d ' '%(chunk_rate).2f ' '%(bytes_rate).2f ' '%(audit_time).2f ' '%(audit_rate).2f' % { 'elapsed': elapsed, 'corrupted': total_corrupted + self.corrupted_chunks, 'faulty': total_faulty + self.faulty_chunks, 'orphans': total_orphans + self.orphan_chunks, 'errors': total_errors + self.errors, 'chunk_rate': self.total_chunks_processed / elapsed, 'bytes_rate': self.total_bytes_processed / elapsed, 'audit_time': audit_time, 'audit_rate': audit_time / elapsed })
def mover_pass(self): self.namespace, self.address = check_volume(self.volume) start_time = report_time = time.time() total_errors = 0 mover_time = 0 paths = paths_gen(self.volume) for path in paths: loop_time = time.time() now = time.time() if now - self.last_usage_check >= self.usage_check_interval: used, total = statfs(self.volume) usage = (float(used) / total) * 100 if usage <= self.usage_target: self.logger.info( 'current usage %.2f%%: target reached (%.2f%%)', usage, self.usage_target) self.last_usage_check = now break self.safe_chunk_move(path) self.chunks_run_time = ratelimit(self.chunks_run_time, self.max_chunks_per_second) self.total_chunks_processed += 1 now = time.time() if now - self.last_reported >= self.report_interval: self.logger.info( '%(start_time)s ' '%(passes)d ' '%(errors)d ' '%(c_rate).2f ' '%(b_rate).2f ' '%(total).2f ' '%(mover_time).2f' '%(mover_rate).2f' % { 'start_time': time.ctime(report_time), 'passes': self.passes, 'errors': self.errors, 'c_rate': self.passes / (now - report_time), 'b_rate': self.bytes_processed / (now - report_time), 'total': (now - start_time), 'mover_time': mover_time, 'mover_rate': mover_time / (now - start_time) }) report_time = now total_errors += self.errors self.passes = 0 self.bytes_processed = 0 self.last_reported = now mover_time += (now - loop_time) elapsed = (time.time() - start_time) or 0.000001 self.logger.info( '%(elapsed).02f ' '%(errors)d ' '%(chunk_rate).2f ' '%(bytes_rate).2f ' '%(mover_time).2f ' '%(mover_rate).2f' % { 'elapsed': elapsed, 'errors': total_errors + self.errors, 'chunk_rate': self.total_chunks_processed / elapsed, 'bytes_rate': self.total_bytes_processed / elapsed, 'mover_time': mover_time, 'mover_rate': mover_time / elapsed })
def mover_pass(self): self.namespace, self.address = check_volume(self.volume) start_time = report_time = time.time() total_errors = 0 mover_time = 0 paths = paths_gen(self.volume) for path in paths: loop_time = time.time() now = time.time() if now - self.last_usage_check >= self.usage_check_interval: used, total = statfs(self.volume) usage = (float(used) / total) * 100 if usage <= self.usage_target: self.logger.info( 'current usage %.2f%%: target reached (%.2f%%)', usage, self.usage_target) self.last_usage_check = now break self.safe_chunk_move(path) self.chunks_run_time = ratelimit( self.chunks_run_time, self.max_chunks_per_second ) self.total_chunks_processed += 1 now = time.time() if now - self.last_reported >= self.report_interval: self.logger.info( '%(start_time)s ' '%(passes)d ' '%(errors)d ' '%(c_rate).2f ' '%(b_rate).2f ' '%(total).2f ' '%(mover_time).2f' '%(mover_rate).2f' % { 'start_time': time.ctime(report_time), 'passes': self.passes, 'errors': self.errors, 'c_rate': self.passes / (now - report_time), 'b_rate': self.bytes_processed / (now - report_time), 'total': (now - start_time), 'mover_time': mover_time, 'mover_rate': mover_time / (now - start_time) } ) report_time = now total_errors += self.errors self.passes = 0 self.bytes_processed = 0 self.last_reported = now mover_time += (now - loop_time) elapsed = (time.time() - start_time) or 0.000001 self.logger.info( '%(elapsed).02f ' '%(errors)d ' '%(chunk_rate).2f ' '%(bytes_rate).2f ' '%(mover_time).2f ' '%(mover_rate).2f' % { 'elapsed': elapsed, 'errors': total_errors + self.errors, 'chunk_rate': self.total_chunks_processed / elapsed, 'bytes_rate': self.total_bytes_processed / elapsed, 'mover_time': mover_time, 'mover_rate': mover_time / elapsed } )
def index_pass(self): def safe_update_index(path): chunk_id = path.rsplit('/', 1)[-1] if len(chunk_id) != STRLEN_CHUNKID: self.logger.warn('WARN Not a chunk %s' % path) return for c in chunk_id: if c not in hexdigits: self.logger.warn('WARN Not a chunk %s' % path) return try: self.update_index(path, chunk_id) self.successes += 1 self.logger.debug('Updated %s', path) except OioNetworkException as exc: self.errors += 1 self.logger.warn('ERROR while updating %s: %s', path, exc) except VolumeException as exc: self.errors += 1 self.logger.error('Cannot index %s: %s', path, exc) # All chunks of this volume are indexed in the same service, # no need to try another chunk, it will generate the same # error. Let the upper level retry later. raise except Exception: self.errors += 1 self.logger.exception('ERROR while updating %s', path) self.total_since_last_reported += 1 def report(tag): total = self.errors + self.successes now = time.time() elapsed = (now - start_time) or 0.000001 self.logger.info( '%(tag)s=%(current_time)s ' 'elapsed=%(elapsed).02f ' 'pass=%(pass)d ' 'errors=%(errors)d ' 'chunks=%(nb_chunks)d %(c_rate).2f/s' % { 'tag': tag, 'current_time': datetime.fromtimestamp(int(now)).isoformat(), 'pass': self.passes, 'errors': self.errors, 'nb_chunks': total, 'c_rate': self.total_since_last_reported / (now - self.last_reported), 'elapsed': elapsed }) self.last_reported = now self.total_since_last_reported = 0 start_time = time.time() self.last_reported = start_time self.errors = 0 self.successes = 0 paths = paths_gen(self.volume) report('started') for path in paths: safe_update_index(path) self.chunks_run_time = ratelimit(self.chunks_run_time, self.max_chunks_per_second) now = time.time() if now - self.last_reported >= self.report_interval: report('running') report('ended')