def recover(ctx): ret = True stats = ctx.monitor.stats['recover'] stats.timer('recover', 'started') it = iterate_key(ctx.merged_filename, ctx.groups) node = elliptics_create_node(address=ctx.address, elog=ctx.elog, wait_timeout=ctx.wait_timeout, net_thread_num=4, io_thread_num=1, remotes=ctx.remotes) for batch_id, batch in groupby(enumerate(it), key=lambda x: x[0] / ctx.batch_size): recovers = [] rs = RecoverStat() for _, val in batch: rec = KeyRecover(ctx, *val, node=node) recovers.append(rec) for r in recovers: r.wait() ret &= r.succeeded() rs += r.stats rs.apply(stats) stats.timer('recover', 'finished') return ret
def recover(ctx): ret = True stats = ctx.monitor.stats['recover'] filtered = filter(ctx.merged_filename, ctx.groups) node = elliptics_create_node(address=ctx.address, elog=ctx.elog, wait_timeout=ctx.wait_timeout, net_thread_num=4, io_thread_num=1, remotes=ctx.remotes) for batch_id, batch in groupby(enumerate(filtered), key=lambda x: x[0] / ctx.batch_size): recovers = [] rs = RecoverStat() for _, val in batch: rec = KeyRecover(*val, node=node) recovers.append(rec) rec.run() for r in recovers: r.wait() ret &= r.succeeded() rs += r.stats rs.apply(stats) return ret
def recover(ctx): stats = ctx.stats['recover'] stats.timer('recover', 'started') node = elliptics_create_node(address=ctx.address, elog=elliptics.Logger(ctx.log_file, int(ctx.log_level)), wait_timeout=ctx.wait_timeout, net_thread_num=4, io_thread_num=24, remotes=ctx.remotes) result = WindowedDC(ctx, node).run() stats.timer('recover', 'finished') return result
def recover(ctx): stats = ctx.stats['recover'] stats.timer('recover', 'started') node = elliptics_create_node(address=ctx.address, elog=elliptics.Logger(ctx.log_file, int(ctx.log_level)), wait_timeout=ctx.wait_timeout, net_thread_num=4, io_thread_num=24, remotes=ctx.remotes) result = WindowedDC(ctx, node).run() stats.timer('recover', 'finished') return result
def recover(ctx): from itertools import islice import time ret = True stats = ctx.stats['recover'] stats.timer('recover', 'started') it = iterate_key(ctx.merged_filename, ctx.groups) elog = elliptics.Logger(ctx.log_file, int(ctx.log_level)) node = elliptics_create_node(address=ctx.address, elog=elog, wait_timeout=ctx.wait_timeout, net_thread_num=4, io_thread_num=1, remotes=ctx.remotes) processed_keys = 0 start = time.time() while 1: batch = tuple(islice(it, ctx.batch_size)) if not batch: break recovers = [] rs = RecoverStat() for val in batch: rec = KeyRecover(ctx, *val, node=node) recovers.append(rec) successes, failures = 0, 0 for r in recovers: r.wait() ret &= r.succeeded() rs += r.stats if r.succeeded(): successes += 1 else: failures += 1 processed_keys += successes + failures rs.apply(stats) stats.counter('recovered_keys', successes) ctx.stats.counter('recovered_keys', successes) stats.counter('recovered_keys', -failures) ctx.stats.counter('recovered_keys', -failures) stats.set_counter('recovery_speed', processed_keys / (time.time() - start)) stats.timer('recover', 'finished') return ret
def recover(ctx): stats = ctx.stats['recover'] stats.timer('recover', 'started') node = elliptics_create_node(address=ctx.address, elog=elliptics.Logger(ctx.log_file, int(ctx.log_level)), wait_timeout=ctx.wait_timeout, flags=elliptics.config_flags.no_route_list, net_thread_num=4, io_thread_num=24, remotes=ctx.remotes) result = ServerSendRecovery(ctx, node).recover() result &= WindowedDC(ctx, node).run() cleanup(ctx) stats.timer('recover', 'finished') return result
def get_routes(ctx): log.debug('Requesting routes') log.debug("Creating logger") elog = elliptics.Logger(ctx.log_file, int(ctx.log_level)) log.debug("Creating node") node = elliptics_create_node(address=ctx.address, elog=elog, wait_timeout=ctx.wait_timeout, remotes=ctx.remotes) log.debug("Creating session for: {0}".format(ctx.address)) session = elliptics_create_session(node=node, group=0, trace_id=ctx.trace_id) log.debug("Parsing routing table") return RouteList.from_session(session)
def recover(ctx): stats = ctx.stats['recover'] stats.timer('recover', 'started') node = elliptics_create_node(address=ctx.address, elog=elliptics.Logger(ctx.log_file, int(ctx.log_level), True), wait_timeout=ctx.wait_timeout, flags=elliptics.config_flags.no_route_list, net_thread_num=4, io_thread_num=24, remotes=ctx.remotes) result = ServerSendRecovery(ctx, node).recover() result &= WindowedDC(ctx, node).run() cleanup(ctx) stats.timer('recover', 'finished') return result
def recover(ctx): from itertools import islice import time ret = True stats = ctx.stats['recover'] stats.timer('recover', 'started') it = iterate_key(ctx.merged_filename, ctx.groups) elog = elliptics.Logger(ctx.log_file, int(ctx.log_level)) node = elliptics_create_node(address=ctx.address, elog=elog, wait_timeout=ctx.wait_timeout, net_thread_num=4, io_thread_num=1, remotes=ctx.remotes) processed_keys = 0 start = time.time() while 1: batch = tuple(islice(it, ctx.batch_size)) if not batch: break recovers = [] rs = RecoverStat() for val in batch: rec = KeyRecover(ctx, *val, node=node) recovers.append(rec) successes, failures = 0, 0 for r in recovers: r.wait() ret &= r.succeeded() rs += r.stats if r.succeeded(): successes += 1 else: failures += 1 processed_keys += successes + failures rs.apply(stats) stats.counter('recovered_keys', successes) ctx.stats.counter('recovered_keys', successes) stats.counter('recovered_keys', -failures) ctx.stats.counter('recovered_keys', -failures) stats.set_counter('recovery_speed', processed_keys / (time.time() - start)) stats.timer('recover', 'finished') return ret
def get_routes(ctx): log.debug('Requesting routes') log.debug("Creating logger") elog = elliptics.Logger(ctx.log_file, int(ctx.log_level), True) log.debug("Creating node") node = elliptics_create_node(address=ctx.address, elog=elog, wait_timeout=ctx.wait_timeout, flags=elliptics.config_flags.no_route_list, remotes=ctx.remotes) log.debug("Creating session for: {0}".format(ctx.address)) session = elliptics_create_session(node=node, group=0, trace_id=ctx.trace_id) log.debug("Parsing routing table") return session.routes.filter_by_groups(ctx.groups)
def get_routes(ctx): log.debug('Requesting routes') log.debug("Creating logger") elog = elliptics.Logger(ctx.log_file, int(ctx.log_level)) log.debug("Creating node") node = elliptics_create_node(address=ctx.address, elog=elog, wait_timeout=ctx.wait_timeout, remotes=ctx.remotes) log.debug("Creating session for: {0}".format(ctx.address)) session = elliptics_create_session(node=node, group=0, trace_id=ctx.trace_id) log.debug("Parsing routing table") return RouteList.from_session(session)
repr(e), traceback.format_exc())) log.info("Initializing monitor") monitor = Monitor(ctx, ctx.monitor_port) ctx.stats = monitor.stats log.debug("Using following context:\n{0}".format(ctx)) log.debug("Setting up elliptics client") log.debug("Creating logger") elog = elliptics.Logger(ctx.log_file, int(ctx.log_level)) log.debug("Creating node") node = elliptics_create_node(address=ctx.address, elog=elog, wait_timeout=ctx.wait_timeout, remotes=ctx.remotes) log.debug("Creating session for: {0}".format(ctx.address)) session = elliptics_create_session(node=node, group=0) log.debug("Parsing routing table") ctx.routes = RouteList.from_session(session) log.debug("Parsed routing table:\n{0}".format(ctx.routes)) if not ctx.routes: raise RuntimeError("No routes was parsed from session") log.debug("Total routes: {0}".format(len(ctx.routes))) if len(ctx.groups) == 0: ctx.groups = ctx.routes.groups() log.info("No groups specified: using all available groups: {0}".format(
.format(repr(e), traceback.format_exc())) log.info("Initializing monitor") monitor = Monitor(ctx, ctx.monitor_port) ctx.stats = monitor.stats log.debug("Using following context:\n{0}".format(ctx)) log.debug("Setting up elliptics client") log.debug("Creating logger") elog = elliptics.Logger(ctx.log_file, int(ctx.log_level)) log.debug("Creating node") node = elliptics_create_node(address=ctx.address, elog=elog, wait_timeout=ctx.wait_timeout, remotes=ctx.remotes) log.debug("Creating session for: {0}".format(ctx.address)) session = elliptics_create_session(node=node, group=0) log.debug("Parsing routing table") ctx.routes = RouteList.from_session(session) log.debug("Parsed routing table:\n{0}".format(ctx.routes)) if not ctx.routes: raise RuntimeError("No routes was parsed from session") log.debug("Total routes: {0}".format(len(ctx.routes))) if len(ctx.groups) == 0: ctx.groups = ctx.routes.groups() log.info("No groups specified: using all available groups: {0}".format(ctx.groups))