def run(self, accessor, opts, on_progress=None):
"""Run some repairs.
See command.BaseCommand
"""
accessor.connect()
# TODO: use multiprocessing + progressbar here. Probably remove some
# of the current arguments and generate them instead based on a number
# of processes to do a full scan.
if opts.storage_dir:
settings = {"path": opts.storage_dir}
with metadata_cache.DiskCache(accessor, settings) as cache:
cache.repair(
shard=opts.shard,
nshards=opts.nshards,
start_key=opts.start_key,
end_key=opts.end_key,
)
else:
logging.warning("Skipping disk cache repair because storage_dir is empty")
out_fd = sys.stderr
if opts.quiet:
out_fd = _DEV_NULL
if self.pbar is None:
self.pbar = progressbar.ProgressBar(fd=out_fd, redirect_stderr=False)
self.pbar.start()
if on_progress is None:
def _on_progress(done, total):
self.pbar.max_value = max(total, done)
self.pbar.update(done)
on_progress = _on_progress
accessor.repair(
shard=opts.shard,
nshards=opts.nshards,
start_key=opts.start_key,
end_key=opts.end_key,
callback_on_progress=on_progress,
)
self.pbar.finish()
def run(self, accessor, opts, on_progress=None):
"""Run some cleanups.
See command.BaseCommand
"""
out_fd = sys.stderr
if opts.quiet:
out_fd = _DEV_NULL
if self.pbar is None:
self.pbar = progressbar.ProgressBar(fd=out_fd,
redirect_stderr=False)
self.pbar.start()
if on_progress is None:
def _on_progress(done, total):
self.pbar.max_value = total
self.pbar.update(done)
on_progress = _on_progress
with accessor as bg_acc:
if opts.clean_cache:
if opts.storage_dir:
settings = {"path": opts.storage_dir, "ttl": opts.max_age}
logging.info("Cleaning cache from %s", settings)
with metadata_cache.DiskCache(bg_acc, settings) as cache:
cache.clean()
else:
logging.error('Cannot clean disk cache because storage_dir'
' is empty')
if opts.clean_backend:
logging.info("Cleaning backend, removing things before %d",
opts.max_age)
bg_acc.clean(opts.max_age,
shard=opts.shard,
nshards=opts.nshards,
start_key=opts.start_key,
end_key=opts.end_key,
callback_on_progress=on_progress)
self.pbar.finish()
def run(self, accessor, opts, on_progress=None):
"""Run some repairs.
See command.BaseCommand
"""
accessor.connect()
# TODO: use multiprocessing + progressbar here. Probably remove some
# of the current arguments and generate them instead based on a number
# of processes to do a full scan.
self.metrics_file_path = opts.metrics_file_path
if opts.storage_dir:
settings = {"path": opts.storage_dir}
with metadata_cache.DiskCache(accessor, settings) as cache:
cache.repair(
shard=opts.shard,
nshards=opts.nshards,
start_key=opts.start_key,
end_key=opts.end_key,
)
else:
logging.warning(
"Skipping disk cache repair because storage_dir is empty")
out_fd = sys.stderr
if opts.quiet:
out_fd = _DEV_NULL
if self.pbar is None:
start_key = -1 * 2**63
end_key = 2**63 - 1
if opts.start_key is not None:
start_key = int(opts.start_key)
if opts.end_key is not None:
end_key = int(opts.end_key)
widgets = [
progressbar.Variable('token',
format='(current: {formatted_value})'),
' ',
progressbar.Percentage(),
' ',
progressbar.SimpleProgress(
format='(%s)' % progressbar.SimpleProgress.DEFAULT_FORMAT),
' ',
progressbar.Bar(),
' ',
progressbar.Timer(),
' ',
progressbar.AdaptiveETA(),
]
# max_value = end_key - start_key
self.pbar = progressbar.ProgressBar(widgets=widgets,
fd=out_fd,
redirect_stderr=False,
min_value=0,
max_value=end_key - start_key)
self.pbar.start()
if on_progress is None:
def _on_progress(total, done, token):
self.pbar.update(total, token=token)
if self.metrics_file_path != "":
write_to_textfile(self.metrics_file_path, REGISTRY)
on_progress = _on_progress
accessor.repair(
shard=opts.shard,
nshards=opts.nshards,
start_key=opts.start_key,
end_key=opts.end_key,
callback_on_progress=on_progress,
)
self.pbar.finish()
# Final metric dump
if self.metrics_file_path != "":
write_to_textfile(self.metrics_file_path, REGISTRY)
def run(self, accessor, opts, on_progress=None):
"""Run some cleanups.
See command.BaseCommand
"""
out_fd = sys.stderr
if opts.quiet:
out_fd = _DEV_NULL
if self.pbar is None:
self.pbar = progressbar.ProgressBar(fd=out_fd, redirect_stderr=False)
self.pbar.start()
if on_progress is None:
def _on_progress(done, total):
self.pbar.max_value = max(total, done)
self.pbar.update(done)
on_progress = _on_progress
accessor.connect()
if opts.clean_cache:
if opts.storage_dir:
settings = {"path": opts.storage_dir, "ttl": opts.max_age}
logging.info("Cleaning cache from %s", settings)
with metadata_cache.DiskCache(accessor, settings) as cache:
cache.clean()
else:
logging.error("Cannot clean disk cache because storage_dir" " is empty")
if opts.clean_backend:
logging.info("Cleaning backend, removing things before %d", opts.max_age)
accessor.clean(
max_age=opts.max_age,
shard=opts.shard,
nshards=opts.nshards,
start_key=opts.start_key,
end_key=opts.end_key,
callback_on_progress=on_progress,
)
if opts.clean_corrupted:
# Remove corrupt metrics.
now = time.time()
def callback(metric, done, total):
# TODO: Probably worth removing old metrics here
# instead of in the driver... The index doesn't work
# well anyway.
if metric.updated_on:
delta = now - time.mktime(metric.updated_on.timetuple())
else:
delta = now
if delta > opts.max_age:
logging.info("Removing %s (%s)" % (metric.name, delta))
accessor.delete_metric(metric.name)
on_progress(done, total)
def errback(metric):
logging.info("Removing %s" % metric)
accessor.delete_metric(metric)
logging.info("Cleaning corrupted metrics")
accessor.map(
callback,
shard=opts.shard,
nshards=opts.nshards,
start_key=opts.start_key,
end_key=opts.end_key,
errback=errback,
)
self.pbar.finish()
def run(self, accessor, opts, on_progress=None):
"""Run some cleanups.
See command.BaseCommand
"""
out_fd = sys.stderr
if opts.quiet:
out_fd = _DEV_NULL
self.metrics_file_path = opts.metrics_file_path
if self.pbar is None:
start_key = -1 * 2**63
end_key = 2**63 - 1
if opts.start_key is not None:
start_key = int(opts.start_key)
if opts.end_key is not None:
end_key = int(opts.end_key)
widgets = [
progressbar.Variable('token',
format='(current: {formatted_value})'),
' ',
progressbar.Percentage(),
' ',
progressbar.SimpleProgress(
format='(%s)' % progressbar.SimpleProgress.DEFAULT_FORMAT),
' ',
progressbar.Bar(),
' ',
progressbar.Timer(),
' ',
progressbar.AdaptiveETA(),
]
# max_value = end_key - start_key
self.pbar = progressbar.ProgressBar(widgets=widgets,
fd=out_fd,
redirect_stderr=False,
min_value=0,
max_value=end_key - start_key)
self.pbar.start()
if on_progress is None:
def _on_progress(total, done, token):
self.pbar.update(total, token=token)
if self.metrics_file_path != "":
write_to_textfile(self.metrics_file_path, REGISTRY)
on_progress = _on_progress
accessor.connect()
if opts.clean_cache:
if opts.storage_dir:
settings = {"path": opts.storage_dir, "ttl": opts.max_age}
logging.info("Cleaning cache from %s", settings)
with metadata_cache.DiskCache(accessor, settings) as cache:
cache.clean()
else:
logging.error("Cannot clean disk cache because storage_dir"
" is empty")
if opts.clean_backend:
logging.info("Cleaning backend, removing things before %d",
opts.max_age)
accessor.clean(
max_age=opts.max_age,
shard=opts.shard,
nshards=opts.nshards,
start_key=opts.start_key,
end_key=opts.end_key,
callback_on_progress=on_progress,
disable_clean_directories=opts.disable_clean_directories,
disable_clean_metrics=opts.disable_clean_metrics,
)
if opts.clean_corrupted:
# Remove corrupt metrics.
now = time.time()
def callback(metric, done, total):
# TODO: Probably worth removing old metrics here
# instead of in the driver... The index doesn't work
# well anyway.
if metric.updated_on:
delta = now - time.mktime(metric.updated_on.timetuple())
else:
delta = now
if delta > opts.max_age:
logging.info("Removing %s (%s)" % (metric.name, delta))
accessor.delete_metric(metric.name)
on_progress(done, total)
def errback(metric):
logging.info("Removing %s" % metric)
accessor.delete_metric(metric)
logging.info("Cleaning corrupted metrics")
accessor.map(
callback,
shard=opts.shard,
nshards=opts.nshards,
start_key=opts.start_key,
end_key=opts.end_key,
errback=errback,
)
self.pbar.finish()
# Final metric dump
if self.metrics_file_path != "":
write_to_textfile(self.metrics_file_path, REGISTRY)