def run(self, accessor, opts, on_progress=None): """Run some repairs. See command.BaseCommand """ accessor.connect() # TODO: use multiprocessing + progressbar here. Probably remove some # of the current arguments and generate them instead based on a number # of processes to do a full scan. if opts.storage_dir: settings = {"path": opts.storage_dir} with metadata_cache.DiskCache(accessor, settings) as cache: cache.repair( shard=opts.shard, nshards=opts.nshards, start_key=opts.start_key, end_key=opts.end_key, ) else: logging.warning("Skipping disk cache repair because storage_dir is empty") out_fd = sys.stderr if opts.quiet: out_fd = _DEV_NULL if self.pbar is None: self.pbar = progressbar.ProgressBar(fd=out_fd, redirect_stderr=False) self.pbar.start() if on_progress is None: def _on_progress(done, total): self.pbar.max_value = max(total, done) self.pbar.update(done) on_progress = _on_progress accessor.repair( shard=opts.shard, nshards=opts.nshards, start_key=opts.start_key, end_key=opts.end_key, callback_on_progress=on_progress, ) self.pbar.finish()
def run(self, accessor, opts, on_progress=None): """Run some cleanups. See command.BaseCommand """ out_fd = sys.stderr if opts.quiet: out_fd = _DEV_NULL if self.pbar is None: self.pbar = progressbar.ProgressBar(fd=out_fd, redirect_stderr=False) self.pbar.start() if on_progress is None: def _on_progress(done, total): self.pbar.max_value = total self.pbar.update(done) on_progress = _on_progress with accessor as bg_acc: if opts.clean_cache: if opts.storage_dir: settings = {"path": opts.storage_dir, "ttl": opts.max_age} logging.info("Cleaning cache from %s", settings) with metadata_cache.DiskCache(bg_acc, settings) as cache: cache.clean() else: logging.error('Cannot clean disk cache because storage_dir' ' is empty') if opts.clean_backend: logging.info("Cleaning backend, removing things before %d", opts.max_age) bg_acc.clean(opts.max_age, shard=opts.shard, nshards=opts.nshards, start_key=opts.start_key, end_key=opts.end_key, callback_on_progress=on_progress) self.pbar.finish()
def run(self, accessor, opts, on_progress=None): """Run some repairs. See command.BaseCommand """ accessor.connect() # TODO: use multiprocessing + progressbar here. Probably remove some # of the current arguments and generate them instead based on a number # of processes to do a full scan. self.metrics_file_path = opts.metrics_file_path if opts.storage_dir: settings = {"path": opts.storage_dir} with metadata_cache.DiskCache(accessor, settings) as cache: cache.repair( shard=opts.shard, nshards=opts.nshards, start_key=opts.start_key, end_key=opts.end_key, ) else: logging.warning( "Skipping disk cache repair because storage_dir is empty") out_fd = sys.stderr if opts.quiet: out_fd = _DEV_NULL if self.pbar is None: start_key = -1 * 2**63 end_key = 2**63 - 1 if opts.start_key is not None: start_key = int(opts.start_key) if opts.end_key is not None: end_key = int(opts.end_key) widgets = [ progressbar.Variable('token', format='(current: {formatted_value})'), ' ', progressbar.Percentage(), ' ', progressbar.SimpleProgress( format='(%s)' % progressbar.SimpleProgress.DEFAULT_FORMAT), ' ', progressbar.Bar(), ' ', progressbar.Timer(), ' ', progressbar.AdaptiveETA(), ] # max_value = end_key - start_key self.pbar = progressbar.ProgressBar(widgets=widgets, fd=out_fd, redirect_stderr=False, min_value=0, max_value=end_key - start_key) self.pbar.start() if on_progress is None: def _on_progress(total, done, token): self.pbar.update(total, token=token) if self.metrics_file_path != "": write_to_textfile(self.metrics_file_path, REGISTRY) on_progress = _on_progress accessor.repair( shard=opts.shard, nshards=opts.nshards, start_key=opts.start_key, end_key=opts.end_key, callback_on_progress=on_progress, ) self.pbar.finish() # Final metric dump if self.metrics_file_path != "": write_to_textfile(self.metrics_file_path, REGISTRY)
def run(self, accessor, opts, on_progress=None): """Run some cleanups. See command.BaseCommand """ out_fd = sys.stderr if opts.quiet: out_fd = _DEV_NULL if self.pbar is None: self.pbar = progressbar.ProgressBar(fd=out_fd, redirect_stderr=False) self.pbar.start() if on_progress is None: def _on_progress(done, total): self.pbar.max_value = max(total, done) self.pbar.update(done) on_progress = _on_progress accessor.connect() if opts.clean_cache: if opts.storage_dir: settings = {"path": opts.storage_dir, "ttl": opts.max_age} logging.info("Cleaning cache from %s", settings) with metadata_cache.DiskCache(accessor, settings) as cache: cache.clean() else: logging.error("Cannot clean disk cache because storage_dir" " is empty") if opts.clean_backend: logging.info("Cleaning backend, removing things before %d", opts.max_age) accessor.clean( max_age=opts.max_age, shard=opts.shard, nshards=opts.nshards, start_key=opts.start_key, end_key=opts.end_key, callback_on_progress=on_progress, ) if opts.clean_corrupted: # Remove corrupt metrics. now = time.time() def callback(metric, done, total): # TODO: Probably worth removing old metrics here # instead of in the driver... The index doesn't work # well anyway. if metric.updated_on: delta = now - time.mktime(metric.updated_on.timetuple()) else: delta = now if delta > opts.max_age: logging.info("Removing %s (%s)" % (metric.name, delta)) accessor.delete_metric(metric.name) on_progress(done, total) def errback(metric): logging.info("Removing %s" % metric) accessor.delete_metric(metric) logging.info("Cleaning corrupted metrics") accessor.map( callback, shard=opts.shard, nshards=opts.nshards, start_key=opts.start_key, end_key=opts.end_key, errback=errback, ) self.pbar.finish()
def run(self, accessor, opts, on_progress=None): """Run some cleanups. See command.BaseCommand """ out_fd = sys.stderr if opts.quiet: out_fd = _DEV_NULL self.metrics_file_path = opts.metrics_file_path if self.pbar is None: start_key = -1 * 2**63 end_key = 2**63 - 1 if opts.start_key is not None: start_key = int(opts.start_key) if opts.end_key is not None: end_key = int(opts.end_key) widgets = [ progressbar.Variable('token', format='(current: {formatted_value})'), ' ', progressbar.Percentage(), ' ', progressbar.SimpleProgress( format='(%s)' % progressbar.SimpleProgress.DEFAULT_FORMAT), ' ', progressbar.Bar(), ' ', progressbar.Timer(), ' ', progressbar.AdaptiveETA(), ] # max_value = end_key - start_key self.pbar = progressbar.ProgressBar(widgets=widgets, fd=out_fd, redirect_stderr=False, min_value=0, max_value=end_key - start_key) self.pbar.start() if on_progress is None: def _on_progress(total, done, token): self.pbar.update(total, token=token) if self.metrics_file_path != "": write_to_textfile(self.metrics_file_path, REGISTRY) on_progress = _on_progress accessor.connect() if opts.clean_cache: if opts.storage_dir: settings = {"path": opts.storage_dir, "ttl": opts.max_age} logging.info("Cleaning cache from %s", settings) with metadata_cache.DiskCache(accessor, settings) as cache: cache.clean() else: logging.error("Cannot clean disk cache because storage_dir" " is empty") if opts.clean_backend: logging.info("Cleaning backend, removing things before %d", opts.max_age) accessor.clean( max_age=opts.max_age, shard=opts.shard, nshards=opts.nshards, start_key=opts.start_key, end_key=opts.end_key, callback_on_progress=on_progress, disable_clean_directories=opts.disable_clean_directories, disable_clean_metrics=opts.disable_clean_metrics, ) if opts.clean_corrupted: # Remove corrupt metrics. now = time.time() def callback(metric, done, total): # TODO: Probably worth removing old metrics here # instead of in the driver... The index doesn't work # well anyway. if metric.updated_on: delta = now - time.mktime(metric.updated_on.timetuple()) else: delta = now if delta > opts.max_age: logging.info("Removing %s (%s)" % (metric.name, delta)) accessor.delete_metric(metric.name) on_progress(done, total) def errback(metric): logging.info("Removing %s" % metric) accessor.delete_metric(metric) logging.info("Cleaning corrupted metrics") accessor.map( callback, shard=opts.shard, nshards=opts.nshards, start_key=opts.start_key, end_key=opts.end_key, errback=errback, ) self.pbar.finish() # Final metric dump if self.metrics_file_path != "": write_to_textfile(self.metrics_file_path, REGISTRY)