def recover(ctx):
ret = True
stats = ctx.monitor.stats['recover']
stats.timer('recover', 'started')
it = iterate_key(ctx.merged_filename, ctx.groups)
node = elliptics_create_node(address=ctx.address,
elog=ctx.elog,
wait_timeout=ctx.wait_timeout,
net_thread_num=4,
io_thread_num=1,
remotes=ctx.remotes)
for batch_id, batch in groupby(enumerate(it),
key=lambda x: x[0] / ctx.batch_size):
recovers = []
rs = RecoverStat()
for _, val in batch:
rec = KeyRecover(ctx, *val, node=node)
recovers.append(rec)
for r in recovers:
r.wait()
ret &= r.succeeded()
rs += r.stats
rs.apply(stats)
stats.timer('recover', 'finished')
return ret
def recover(ctx):
ret = True
stats = ctx.monitor.stats['recover']
filtered = filter(ctx.merged_filename, ctx.groups)
node = elliptics_create_node(address=ctx.address,
elog=ctx.elog,
wait_timeout=ctx.wait_timeout,
net_thread_num=4,
io_thread_num=1,
remotes=ctx.remotes)
for batch_id, batch in groupby(enumerate(filtered),
key=lambda x: x[0] / ctx.batch_size):
recovers = []
rs = RecoverStat()
for _, val in batch:
rec = KeyRecover(*val, node=node)
recovers.append(rec)
rec.run()
for r in recovers:
r.wait()
ret &= r.succeeded()
rs += r.stats
rs.apply(stats)
return ret
def recover(ctx):
stats = ctx.stats['recover']
stats.timer('recover', 'started')
node = elliptics_create_node(address=ctx.address,
elog=elliptics.Logger(ctx.log_file, int(ctx.log_level)),
wait_timeout=ctx.wait_timeout,
net_thread_num=4,
io_thread_num=24,
remotes=ctx.remotes)
result = WindowedDC(ctx, node).run()
stats.timer('recover', 'finished')
return result
def recover(ctx):
stats = ctx.stats['recover']
stats.timer('recover', 'started')
node = elliptics_create_node(address=ctx.address,
elog=elliptics.Logger(ctx.log_file,
int(ctx.log_level)),
wait_timeout=ctx.wait_timeout,
net_thread_num=4,
io_thread_num=24,
remotes=ctx.remotes)
result = WindowedDC(ctx, node).run()
stats.timer('recover', 'finished')
return result
def recover(ctx):
from itertools import islice
import time
ret = True
stats = ctx.stats['recover']
stats.timer('recover', 'started')
it = iterate_key(ctx.merged_filename, ctx.groups)
elog = elliptics.Logger(ctx.log_file, int(ctx.log_level))
node = elliptics_create_node(address=ctx.address,
elog=elog,
wait_timeout=ctx.wait_timeout,
net_thread_num=4,
io_thread_num=1,
remotes=ctx.remotes)
processed_keys = 0
start = time.time()
while 1:
batch = tuple(islice(it, ctx.batch_size))
if not batch:
break
recovers = []
rs = RecoverStat()
for val in batch:
rec = KeyRecover(ctx, *val, node=node)
recovers.append(rec)
successes, failures = 0, 0
for r in recovers:
r.wait()
ret &= r.succeeded()
rs += r.stats
if r.succeeded():
successes += 1
else:
failures += 1
processed_keys += successes + failures
rs.apply(stats)
stats.counter('recovered_keys', successes)
ctx.stats.counter('recovered_keys', successes)
stats.counter('recovered_keys', -failures)
ctx.stats.counter('recovered_keys', -failures)
stats.set_counter('recovery_speed',
processed_keys / (time.time() - start))
stats.timer('recover', 'finished')
return ret
def recover(ctx):
stats = ctx.stats['recover']
stats.timer('recover', 'started')
node = elliptics_create_node(address=ctx.address,
elog=elliptics.Logger(ctx.log_file, int(ctx.log_level)),
wait_timeout=ctx.wait_timeout,
flags=elliptics.config_flags.no_route_list,
net_thread_num=4,
io_thread_num=24,
remotes=ctx.remotes)
result = ServerSendRecovery(ctx, node).recover()
result &= WindowedDC(ctx, node).run()
cleanup(ctx)
stats.timer('recover', 'finished')
return result
def get_routes(ctx):
log.debug('Requesting routes')
log.debug("Creating logger")
elog = elliptics.Logger(ctx.log_file, int(ctx.log_level))
log.debug("Creating node")
node = elliptics_create_node(address=ctx.address,
elog=elog,
wait_timeout=ctx.wait_timeout,
remotes=ctx.remotes)
log.debug("Creating session for: {0}".format(ctx.address))
session = elliptics_create_session(node=node, group=0, trace_id=ctx.trace_id)
log.debug("Parsing routing table")
return RouteList.from_session(session)
def recover(ctx):
stats = ctx.stats['recover']
stats.timer('recover', 'started')
node = elliptics_create_node(address=ctx.address,
elog=elliptics.Logger(ctx.log_file, int(ctx.log_level), True),
wait_timeout=ctx.wait_timeout,
flags=elliptics.config_flags.no_route_list,
net_thread_num=4,
io_thread_num=24,
remotes=ctx.remotes)
result = ServerSendRecovery(ctx, node).recover()
result &= WindowedDC(ctx, node).run()
cleanup(ctx)
stats.timer('recover', 'finished')
return result
def recover(ctx):
from itertools import islice
import time
ret = True
stats = ctx.stats['recover']
stats.timer('recover', 'started')
it = iterate_key(ctx.merged_filename, ctx.groups)
elog = elliptics.Logger(ctx.log_file, int(ctx.log_level))
node = elliptics_create_node(address=ctx.address,
elog=elog,
wait_timeout=ctx.wait_timeout,
net_thread_num=4,
io_thread_num=1,
remotes=ctx.remotes)
processed_keys = 0
start = time.time()
while 1:
batch = tuple(islice(it, ctx.batch_size))
if not batch:
break
recovers = []
rs = RecoverStat()
for val in batch:
rec = KeyRecover(ctx, *val, node=node)
recovers.append(rec)
successes, failures = 0, 0
for r in recovers:
r.wait()
ret &= r.succeeded()
rs += r.stats
if r.succeeded():
successes += 1
else:
failures += 1
processed_keys += successes + failures
rs.apply(stats)
stats.counter('recovered_keys', successes)
ctx.stats.counter('recovered_keys', successes)
stats.counter('recovered_keys', -failures)
ctx.stats.counter('recovered_keys', -failures)
stats.set_counter('recovery_speed', processed_keys / (time.time() - start))
stats.timer('recover', 'finished')
return ret
def get_routes(ctx):
log.debug('Requesting routes')
log.debug("Creating logger")
elog = elliptics.Logger(ctx.log_file, int(ctx.log_level), True)
log.debug("Creating node")
node = elliptics_create_node(address=ctx.address,
elog=elog,
wait_timeout=ctx.wait_timeout,
flags=elliptics.config_flags.no_route_list,
remotes=ctx.remotes)
log.debug("Creating session for: {0}".format(ctx.address))
session = elliptics_create_session(node=node, group=0, trace_id=ctx.trace_id)
log.debug("Parsing routing table")
return session.routes.filter_by_groups(ctx.groups)
def get_routes(ctx):
log.debug('Requesting routes')
log.debug("Creating logger")
elog = elliptics.Logger(ctx.log_file, int(ctx.log_level))
log.debug("Creating node")
node = elliptics_create_node(address=ctx.address,
elog=elog,
wait_timeout=ctx.wait_timeout,
remotes=ctx.remotes)
log.debug("Creating session for: {0}".format(ctx.address))
session = elliptics_create_session(node=node,
group=0,
trace_id=ctx.trace_id)
log.debug("Parsing routing table")
return RouteList.from_session(session)
repr(e), traceback.format_exc()))
log.info("Initializing monitor")
monitor = Monitor(ctx, ctx.monitor_port)
ctx.stats = monitor.stats
log.debug("Using following context:\n{0}".format(ctx))
log.debug("Setting up elliptics client")
log.debug("Creating logger")
elog = elliptics.Logger(ctx.log_file, int(ctx.log_level))
log.debug("Creating node")
node = elliptics_create_node(address=ctx.address,
elog=elog,
wait_timeout=ctx.wait_timeout,
remotes=ctx.remotes)
log.debug("Creating session for: {0}".format(ctx.address))
session = elliptics_create_session(node=node, group=0)
log.debug("Parsing routing table")
ctx.routes = RouteList.from_session(session)
log.debug("Parsed routing table:\n{0}".format(ctx.routes))
if not ctx.routes:
raise RuntimeError("No routes was parsed from session")
log.debug("Total routes: {0}".format(len(ctx.routes)))
if len(ctx.groups) == 0:
ctx.groups = ctx.routes.groups()
log.info("No groups specified: using all available groups: {0}".format(
.format(repr(e), traceback.format_exc()))
log.info("Initializing monitor")
monitor = Monitor(ctx, ctx.monitor_port)
ctx.stats = monitor.stats
log.debug("Using following context:\n{0}".format(ctx))
log.debug("Setting up elliptics client")
log.debug("Creating logger")
elog = elliptics.Logger(ctx.log_file, int(ctx.log_level))
log.debug("Creating node")
node = elliptics_create_node(address=ctx.address,
elog=elog,
wait_timeout=ctx.wait_timeout,
remotes=ctx.remotes)
log.debug("Creating session for: {0}".format(ctx.address))
session = elliptics_create_session(node=node, group=0)
log.debug("Parsing routing table")
ctx.routes = RouteList.from_session(session)
log.debug("Parsed routing table:\n{0}".format(ctx.routes))
if not ctx.routes:
raise RuntimeError("No routes was parsed from session")
log.debug("Total routes: {0}".format(len(ctx.routes)))
if len(ctx.groups) == 0:
ctx.groups = ctx.routes.groups()
log.info("No groups specified: using all available groups: {0}".format(ctx.groups))
