def test_2_backends_with_equal_ids_and_group(self, servers):
'''
These test case check correct handling situation when some backend from one nodes has the same group and ids
like another backend from another node.
For this test all nodes mustn't know about each other, their `remote` are empty.
In this test creates 2 client nodes and both connects to 2 different nodes.
At each node selects one backend from one group, equal to both backend.
Updates ids for both backends to make it be equal.
To the second client node adds remote to the first node. It raises route-list update error: -EEXIST and raises exception.
After that makes read some noexistent key from all groups - it will raise an exception.
With old bug thes test case caused `Segmentation fault` on read_data_from_groups.
At the end reverts ids of both backends.
'''
address1, address2 = servers.remotes
address1 = elliptics.Address.from_host_port_family(address1)
session1 = elliptics.Session(
elliptics.Node(
elliptics.Logger("client.log", elliptics.log_level.debug)))
session1._node.add_remotes(address1)
routes = session1.routes.filter_by_address(address1)
group = routes.groups()[-1]
groups = routes.groups()
routes = session1.routes.filter_by_group(group)
ids = [session1.transform('somekey')]
backend_id1 = routes.get_address_backends(address1)[0]
old_ids1 = [
r.id for r in routes.filter_by_address(address1).filter_by_backend(
backend_id1)
]
session1.set_backend_ids(address1, backend_id1, ids).get()
address2 = elliptics.Address.from_host_port_family(address2)
session2 = elliptics.Session(
elliptics.Node(
elliptics.Logger("client.log", elliptics.log_level.debug)))
session2._node.add_remotes(address2)
routes = session2.routes.filter_by_group(group)
backend_id2 = routes.get_address_backends(address2)[0]
old_ids2 = [
r.id for r in routes.filter_by_address(address2).filter_by_backend(
backend_id2)
]
session2.set_backend_ids(address2, backend_id2, ids).get()
with pytest.raises(elliptics.core.Error):
session2._node.add_remotes(address1)
assert session2.routes.addresses() == (address2, )
for g in groups:
with pytest.raises(elliptics.core.Error):
session2.read_data_from_groups(
'unique key for test_2_backends_with_equal_ids_and_group',
[g]).get()
session1.set_backend_ids(address1, backend_id1, old_ids1).get()
session2.set_backend_ids(address2, backend_id2, old_ids2).get()
def dump_process_group((ctx, group)):
log.debug("Processing group: {0}".format(group))
stats = ctx.stats['group_{0}'.format(group)]
stats.timer('process', 'started')
if group not in ctx.routes.groups():
log.error("Group: {0} is not presented in route list".format(group))
return False
elog = elliptics.Logger(ctx.log_file, int(ctx.log_level))
node = elliptics_create_node(address=ctx.address,
elog=elog,
wait_timeout=ctx.wait_timeout,
net_thread_num=1,
io_thread_num=1,
remotes=ctx.remotes)
ret = True
with open(ctx.dump_file, 'r') as dump:
#splits ids from dump file in batchs and recovers it
for batch_id, batch in groupby(enumerate(dump), key=lambda x: x[0] / ctx.batch_size):
recovers = []
rs = RecoverStat()
for _, val in batch:
rec = DumpRecover(routes=ctx.routes, node=node, id=elliptics.Id(val), group=group, ctx=ctx)
recovers.append(rec)
rec.run()
for r in recovers:
r.wait()
ret &= r.succeeded()
rs += r.stats
rs.apply(stats)
stats.timer('process', 'finished')
return ret
def make_meta_session():
log = elliptics.Logger('/tmp/ell-namespace-convert.log',
config["dnet_log_mask"])
node_config = elliptics.Config()
meta_node = elliptics.Node(log, node_config)
nodes = config['metadata']['nodes']
addresses = [
elliptics.Address(host=host, port=port, family=family)
for (host, port, family) in nodes
]
logger.info('Connecting to meta nodes: {0}'.format(nodes))
try:
meta_node.add_remotes(addresses)
except Exception:
raise ValueError(
'Failed to connect to any elliptics storage META node')
meta_session = elliptics.Session(meta_node)
meta_wait_timeout = config['metadata'].get('wait_timeout', 5)
meta_session.set_timeout(meta_wait_timeout)
meta_session.add_groups(list(config["metadata"]["groups"]))
time.sleep(meta_wait_timeout)
return meta_session
def __init__(self, config):
cfg = elliptics.Config()
# The parameter which sets the time to wait for the operation complete
cfg.config.wait_timeout = config.get("wait-timeout", 60)
# The parameter which sets the timeout for pinging node
cfg.config.check_timeout = config.get("check_timeout", 60)
# Number of IO threads in processing pool
cfg.config.io_thread_num = config.get("io-thread-num", 2)
# Number of threads in network processing pool
cfg.config.net_thread_num = config.get("net-thread-num", 2)
# Number of IO threads in processing pool dedicated to nonblocking ops
nonblock_io_threads = config.get("nonblocking_io_thread_num", 2)
cfg.config.nonblocking_io_thread_num = nonblock_io_threads
groups = config.get('groups', [])
if len(groups) == 0:
raise ValueError("Specify groups")
# loglevel of elliptics logger
elliptics_log_level = config.get('verbosity', 0)
# path to logfile
elliptics_log_file = config.get('logfile', '/dev/stderr')
log = elliptics.Logger(elliptics_log_file, elliptics_log_level)
self._elliptics_node = elliptics.Node(log, cfg)
for host, port in config.get('nodes').iteritems():
self._elliptics_node.add_remote(host, port)
self._session = elliptics.Session(self._elliptics_node)
self._session.groups = groups
self._session.set_namespace(NAMESPACE)
def dump_process_group((ctx, group)):
try:
log.debug("Processing group: {0}".format(group))
stats = ctx.stats['group_{0}'.format(group)]
stats.timer('process', 'started')
if group not in ctx.routes.groups():
log.error(
"Group: {0} is not presented in route list".format(group))
return False
elog = elliptics.Logger(ctx.log_file, int(ctx.log_level), True)
node = elliptics_create_node(address=ctx.address,
elog=elog,
wait_timeout=ctx.wait_timeout,
net_thread_num=1,
io_thread_num=1,
remotes=ctx.remotes)
ret = True
with open(ctx.dump_file, 'r') as dump:
backend_id = ctx.backend_id if ctx.one_node else None
ss_rec = ServerSendRecovery(ctx, node, group, stats, ctx.address,
backend_id)
# splits ids from dump file in batchs and recovers it
for batch_id, batch in groupby(
enumerate(dump), key=lambda x: x[0] / ctx.batch_size):
keys = [elliptics.Id(val) for _, val in batch]
ret &= ss_rec.recover(keys)
stats.timer('process', 'finished')
return ret
except Exception as e:
log.error(
"Processing group failed for: {0}, group: {1}: {2}, traceback: {3}"
.format(ctx.address, group, repr(e), traceback.format_exc()))
return False
def make_meta_session():
log = elliptics.Logger('/tmp/ell-namespace-convert.log',
config["dnet_log_mask"])
node_config = elliptics.Config()
meta_node = elliptics.Node(log, node_config)
addresses = [
elliptics.Address(host=str(node[0]), port=node[1], family=node[2])
for node in config["metadata"]["nodes"]
]
logger.info('Connecting to meta nodes: {0}'.format(
config["metadata"]["nodes"]))
meta_wait_timeout = config['metadata'].get('wait_timeout', 5)
try:
meta_node.add_remotes(addresses)
except Exception as e:
logger.error(
'Failed to connect to any elliptics meta storage node: {0}'.format(
e))
raise ValueError(
'Failed to connect to any elliptics storage META node')
meta_session = elliptics.Session(meta_node)
meta_session.set_timeout(meta_wait_timeout)
meta_session.add_groups(list(config["metadata"]["groups"]))
time.sleep(5)
return meta_session
def recover((address, )):
"""
Recovers difference between remote and local data.
"""
ctx = g_ctx
result = True
stats_name = 'recover_{0}'.format(address)
stats = ctx.monitor.stats[stats_name]
log.info("Recovering ranges for: {0}".format(address))
stats.timer('recover', 'started')
filename = os.path.join(ctx.tmp_dir, mk_container_name(address, "merge_"))
diff = IteratorResult.load_filename(filename,
address=address,
is_sorted=True,
tmp_dir=ctx.tmp_dir,
leave_file=False)
ctx.elog = elliptics.Logger(ctx.log_file, int(ctx.log_level))
local_node = elliptics_create_node(address=ctx.address,
elog=ctx.elog,
io_thread_num=10,
net_thread_num=10,
nonblocking_io_thread_num=10,
wait_timeout=ctx.wait_timeout)
log.debug("Creating direct session: {0}".format(ctx.address))
local_session = elliptics_create_session(
node=local_node,
group=ctx.group_id,
)
local_session.set_direct_id(*ctx.address)
remote_node = elliptics_create_node(address=diff.address,
elog=ctx.elog,
io_thread_num=10,
net_thread_num=10,
nonblocking_io_thread_num=10,
wait_timeout=ctx.wait_timeout)
log.debug("Creating direct session: {0}".format(diff.address))
remote_session = elliptics_create_session(
node=remote_node,
group=diff.address.group_id,
)
remote_session.set_direct_id(*diff.address)
for batch_id, batch in groupby(enumerate(diff),
key=lambda x: x[0] / ctx.batch_size):
result &= recover_keys(ctx=ctx,
address=diff.address,
group_id=diff.address.group_id,
keys=[r.key for _, r in batch],
local_session=local_session,
remote_session=remote_session,
stats=stats)
stats.timer('recover', 'finished')
return result
def iterate_node(arg):
ctx, address, backend_id, ranges = arg
elog = elliptics.Logger(ctx.log_file, int(ctx.log_level))
stats = ctx.stats["iterate"][str(address)][str(backend_id)]
stats.timer('process', 'started')
log.info("Running iterator on node: {0}/{1}".format(address, backend_id))
log.debug("Ranges:")
for range in ranges:
log.debug(repr(range))
stats.timer('process', 'iterate')
node_id = ctx.routes.get_address_backend_route_id(address, backend_id)
node = elliptics_create_node(address=address,
elog=elog,
wait_timeout=ctx.wait_timeout,
net_thread_num=4,
io_thread_num=1)
try:
flags = elliptics.iterator_flags.key_range
timestamp_range = ctx.timestamp.to_etime(), Time.time_max().to_etime()
if ctx.no_meta:
flags |= elliptics.iterator_flags.no_meta
else:
flags |= elliptics.iterator_flags.ts_range
log.debug("Running iterator on node: {0}/{1}".format(address, backend_id))
iterator = Iterator(node, node_id.group_id, separately=True, trace_id=ctx.trace_id)
results, results_len = iterator.iterate_with_stats(
eid=node_id,
timestamp_range=timestamp_range,
key_ranges=ranges,
tmp_dir=ctx.tmp_dir,
address=address,
backend_id=backend_id,
group_id=node_id.group_id,
batch_size=ctx.batch_size,
stats=stats,
flags=flags,
leave_file=True)
if results is None or results_len == 0:
return None
except Exception as e:
log.error("Iteration failed for node {0}/{1}: {2}, traceback: {3}"
.format(address, backend_id, repr(e), traceback.format_exc()))
return None
log.debug("Iterator for node {0}/{1} obtained: {2} record(s)"
.format(address, backend_id, results_len))
stats.timer('process', 'sort')
for range_id in results:
results[range_id].sort()
stats.timer('process', 'finished')
return [(range_id, container.filename, container.address, container.backend_id, container.group_id)
for range_id, container in results.items()]
def simple_node(request):
simple_node = elliptics.Node(elliptics.Logger("client.log", 4))
simple_node.add_remote(request.config.option.remotes)
def fin():
print "Finilizing simple node"
request.addfinalizer(fin)
return simple_node
def test_add_remotes_by_one(self, server):
elog = elliptics.Logger('client.log', 4)
node = elliptics.Node(elog)
remotes = server.remotes
node.add_remote(remotes[0])
node.add_remote(elliptics.Address.from_host_port_family(remotes[1]))
host, port, family = remotes[2].split(':')
node.add_remote(host, int(port), int(family))
def simple_node(request):
if len(request.config.option.remotes) == 0:
pytest.fail(
'`simple_node` fixture should go after `server` fixture, check your test'
)
simple_node = elliptics.Node(
elliptics.Logger("client.log", elliptics.log_level.debug))
simple_node.add_remotes(request.config.option.remotes)
return simple_node
def iterate_node(arg):
address, backend_id, ranges = arg
ctx = g_ctx
ctx.elog = elliptics.Logger(ctx.log_file, int(ctx.log_level))
stats_name = 'iterate_{0}/{1}'.format(address, backend_id)
stats = ctx.monitor.stats[stats_name]
stats.timer('process', 'started')
log.info("Running iterator")
log.debug("Ranges:")
for range in ranges:
log.debug(repr(range))
stats.timer('process', 'iterate')
node_id = ctx.routes.get_address_backend_route_id(address, backend_id)
node = elliptics_create_node(address=address,
elog=ctx.elog,
wait_timeout=ctx.wait_timeout,
net_thread_num=4,
io_thread_num=1)
try:
timestamp_range = ctx.timestamp.to_etime(), Time.time_max().to_etime()
log.debug("Running iterator on node: {0}/{1}".format(address, backend_id))
results, results_len = Iterator.iterate_with_stats(
node=node,
eid=node_id,
timestamp_range=timestamp_range,
key_ranges=ranges,
tmp_dir=ctx.tmp_dir,
address=address,
backend_id=backend_id,
group_id = node_id.group_id,
batch_size=ctx.batch_size,
stats=stats,
leave_file=True,
separately=True)
except Exception as e:
log.error("Iteration failed for node {0}/{1}: {2}, traceback: {3}"
.format(address, backend_id, repr(e), traceback.format_exc()))
stats.counter('iterations', -1)
return None
log.debug("Iterator for node {0}/{1} obtained: {2} record(s)"
.format(address, backend_id, results_len))
stats.counter('iterations', 1)
stats.timer('process', 'sort')
for range_id in results:
results[range_id].sort()
stats.timer('process', 'finished')
return [(range_id, container.filename, container.address, container.backend_id, container.group_id)
for range_id, container in results.items()]
def test_add_several_remotes(self, server):
elog = elliptics.Logger('client.log', 4)
node = elliptics.Node(elog)
remotes = []
for num, remote in enumerate(server.remotes):
if num % 2 == 0:
remotes.append(remote)
else:
remotes.append(elliptics.Address.from_host_port_family(remote))
node.add_remote(remotes)
def lookup_keys(ctx):
log.info("Start looking up keys")
stats = ctx.stats["lookup"]
stats.timer('process', 'started')
elog = elliptics.Logger(ctx.log_file, int(ctx.log_level))
node = elliptics_create_node(address=ctx.address,
elog=elog,
wait_timeout=ctx.wait_timeout,
net_thread_num=1,
io_thread_num=1,
remotes=ctx.remotes)
session = elliptics.Session(node)
session.trace_id = ctx.trace_id
filename = os.path.join(ctx.tmp_dir, 'merged_result')
rest_keys_filename = os.path.join(ctx.tmp_dir, 'rest_keys')
ctx.rest_file = open(rest_keys_filename, 'wb')
ctx.bucket_files = dict()
group_freq = dict()
with open(ctx.dump_file, 'r') as dump_f:
for str_id in dump_f:
id = elliptics.Id(str_id)
lookups = []
for g in ctx.groups:
session.groups = [g]
lookups.append(session.read_data(id, size=1))
key_infos = []
for i, l in enumerate(lookups):
try:
result = l.get()[0]
address = result.address
key_infos.append(
KeyInfo(address, ctx.groups[i], result.timestamp,
result.total_size, result.user_flags,
result.record_flags))
except Exception, e:
log.debug(
"Failed to lookup key: {0} in group: {1}: {2}, traceback: {3}"
.format(id, ctx.groups[i], repr(e),
traceback.format_exc()))
stats.counter("lookups", -1)
if len(key_infos) > 0:
key_data = (id, key_infos)
if not skip_key_data(ctx, key_data):
key_infos.sort(key=lambda x: (x.timestamp, x.size),
reverse=True)
newest_key_group = key_infos[0].group_id
dump_key(ctx, id, key_infos, newest_key_group)
group_freq[newest_key_group] = group_freq.get(
newest_key_group, 0) + 1
stats.counter("lookups", len(key_infos))
else:
log.error(
"Key: {0} is missing in all specified groups: {1}. It won't be recovered."
.format(id, ctx.groups))
def make_elliptics_node():
log = elliptics.Logger('/tmp/ell-namespace-convert.log', config["dnet_log_mask"])
node_config = elliptics.Config()
meta_node = elliptics.Node(log, node_config)
addresses = [elliptics.Address(host=str(node[0]), port=node[1], family=node[2])
for node in config["metadata"]["nodes"]]
logger.info('Connecting to meta nodes: {0}'.format(config["metadata"]["nodes"]))
meta_wait_timeout = config['metadata'].get('wait_timeout', 5)
try:
meta_node.add_remotes(addresses)
except Exception as e:
logger.error('Failed to connect to any elliptics meta storage node: {0}'.format(
e))
raise ValueError('Failed to connect to any elliptics storage META node')
meta_session = elliptics.Session(meta_node)
meta_session.set_timeout(meta_wait_timeout)
meta_session.add_groups(list(config["metadata"]["groups"]))
nodes = config.get('elliptics', {}).get('nodes', []) or config["elliptics_nodes"]
logger.debug("config elliptics nodes: %s" % str(nodes))
node_config = elliptics.Config()
node_config.io_thread_num = config.get('io_thread_num', 1)
node_config.nonblocking_io_thread_num = config.get('nonblocking_io_thread_num', 1)
node_config.net_thread_num = config.get('net_thread_num', 1)
logger.info('Node config: io_thread_num {0}, nonblocking_io_thread_num {1}, '
'net_thread_num {2}'.format(node_config.io_thread_num, node_config.nonblocking_io_thread_num,
node_config.net_thread_num))
n = elliptics.Node(log, node_config)
addresses = [elliptics.Address(host=str(node[0]), port=node[1], family=node[2])
for node in nodes]
try:
n.add_remotes(addresses)
except Exception as e:
logger.error('Failed to connect to any elliptics storage node: {0}'.format(
e))
raise ValueError('Failed to connect to any elliptics storage node')
n.meta_session = meta_session
wait_timeout = config.get('elliptics', {}).get('wait_timeout', 5)
s = elliptics.Session(n)
s.set_timeout(wait_timeout)
print 'sleeping for wait timeout: {0} seconds'.format(wait_timeout)
time.sleep(wait_timeout)
return n
def process_node_backend(ctx, address, backend_id, group, ranges):
try:
log.debug(
"Processing node: {0}/{1} from group: {2} for ranges: {3}".format(
address, backend_id, group, ranges))
stats = ctx.stats['node_{0}/{1}'.format(address, backend_id)]
stats.timer('process', 'started')
elog = elliptics.Logger(ctx.log_file, int(ctx.log_level), True)
node = elliptics_create_node(
address=ctx.address,
elog=elog,
wait_timeout=ctx.wait_timeout,
flags=elliptics.config_flags.no_route_list,
remotes=ctx.remotes,
io_thread_num=4)
stats.timer('process', 'iterate')
results = iterate_node(ctx=ctx,
node=node,
address=address,
backend_id=backend_id,
ranges=ranges,
eid=ctx.routes.get_address_backend_route_id(
address, backend_id),
stats=stats)
if results is None or len(results) == 0:
log.warning('Iterator result is empty, skipping')
return True
stats.timer('process', 'dump_keys')
dump_path = os.path.join(ctx.tmp_dir,
'dump_{0}.{1}'.format(address, backend_id))
log.debug("Dump iterated keys to file: {0}".format(dump_path))
with open(dump_path, 'w') as dump_f:
for r in results:
dump_f.write('{0}\n'.format(r.key))
stats.timer('process', 'recover')
ss_rec = ServerSendRecovery(ctx, node, group, stats, address,
backend_id)
ret = True
for batch_id, batch in groupby(enumerate(results),
key=lambda x: x[0] / ctx.batch_size):
keys = [val.key for _, val in batch]
ret &= ss_rec.recover(keys)
stats.timer('process', 'finished')
return ret
except Exception as e:
log.error(
"Processing node failed for: {0}/{1}: {2}, traceback: {3}".format(
address, backend_id, repr(e), traceback.format_exc()))
return False
def __init__(self, path, level):
logging.Handler.__init__(self)
if level == elliptics.log_level.error:
logging.Handler.setLevel(self, logging.ERROR)
elif level == elliptics.log_level.info:
logging.Handler.setLevel(self, logging.INFO)
elif level == elliptics.log_level.notice:
logging.Handler.setLevel(self, logging.INFO)
elif level == elliptics.log_level.debug:
logging.Handler.setLevel(self, logging.DEBUG)
self.logger = elliptics.Logger(path, level)
def recover(ctx):
stats = ctx.stats['recover']
stats.timer('recover', 'started')
node = elliptics_create_node(address=ctx.address,
elog=elliptics.Logger(ctx.log_file,
int(ctx.log_level)),
wait_timeout=ctx.wait_timeout,
net_thread_num=4,
io_thread_num=24,
remotes=ctx.remotes)
result = WindowedDC(ctx, node).run()
stats.timer('recover', 'finished')
return result
def recover(ctx):
from itertools import islice
import time
ret = True
stats = ctx.stats['recover']
stats.timer('recover', 'started')
it = iterate_key(ctx.merged_filename, ctx.groups)
elog = elliptics.Logger(ctx.log_file, int(ctx.log_level))
node = elliptics_create_node(address=ctx.address,
elog=elog,
wait_timeout=ctx.wait_timeout,
net_thread_num=4,
io_thread_num=1,
remotes=ctx.remotes)
processed_keys = 0
start = time.time()
while 1:
batch = tuple(islice(it, ctx.batch_size))
if not batch:
break
recovers = []
rs = RecoverStat()
for val in batch:
rec = KeyRecover(ctx, *val, node=node)
recovers.append(rec)
successes, failures = 0, 0
for r in recovers:
r.wait()
ret &= r.succeeded()
rs += r.stats
if r.succeeded():
successes += 1
else:
failures += 1
processed_keys += successes + failures
rs.apply(stats)
stats.counter('recovered_keys', successes)
ctx.stats.counter('recovered_keys', successes)
stats.counter('recovered_keys', -failures)
ctx.stats.counter('recovered_keys', -failures)
stats.set_counter('recovery_speed',
processed_keys / (time.time() - start))
stats.timer('recover', 'finished')
return ret
def test_dc_isolated_groups(self, servers):
'''
Write one key into every group,
run dc server-send recovery,
check that keys were recovered into all groups.
'''
scope.node = elliptics.Node(
elliptics.Logger("client.log", elliptics.log_level.debug))
scope.node.add_remotes(servers.remotes)
session = make_session(
node=scope.node,
test_name='TestIsolatedRecovery.test_dc_isolated_groups',
test_namespace=self.namespace)
groups = session.routes.groups()
scope.test_group = groups[0]
scope.test_group2 = groups[1]
routes = session.routes.filter_by_group(scope.test_group)
scope.test_address = routes[0].address
keys = []
data = 'isolated_data'
groups = (scope.test_group, scope.test_group2)
session.timestamp = elliptics.Time.now()
for group_id in groups:
key = 'isolated_key_{}'.format(group_id)
session.groups = [group_id]
write_data(scope, session, [key], [data])
check_data(scope, session, [key], [data], session.timestamp)
keys.append(key)
recovery(one_node=False,
remotes=map(elliptics.Address.from_host_port_family,
servers.remotes),
backend_id=None,
address=scope.test_address,
groups=groups,
rtype=RECOVERY.DC,
log_file='dc_isolated_groups.log',
tmp_dir='dc_isolated_groups')
for group_id in groups:
session.groups = [group_id]
check_data(scope, session, keys, [data] * len(keys),
session.timestamp)
def recover(ctx):
stats = ctx.stats['recover']
stats.timer('recover', 'started')
node = elliptics_create_node(address=ctx.address,
elog=elliptics.Logger(ctx.log_file, int(ctx.log_level), True),
wait_timeout=ctx.wait_timeout,
flags=elliptics.config_flags.no_route_list,
net_thread_num=4,
io_thread_num=24,
remotes=ctx.remotes)
result = ServerSendRecovery(ctx, node).recover()
result &= WindowedDC(ctx, node).run()
cleanup(ctx)
stats.timer('recover', 'finished')
return result
def lookup_keys(ctx):
log.info("Start looking up keys")
stats = ctx.monitor.stats["lookup"]
stats.timer('process', 'started')
ctx.elog = elliptics.Logger(ctx.log_file, int(ctx.log_level))
node = elliptics_create_node(address=ctx.address,
elog=ctx.elog,
wait_timeout=ctx.wait_timeout,
net_thread_num=1,
io_thread_num=1,
remotes=ctx.remotes)
session = elliptics.Session(node)
filename = os.path.join(ctx.tmp_dir, 'merged_result')
merged_f = open(filename, 'w')
pickler = pickle.Pickler(merged_f)
with open(ctx.dump_file, 'r') as dump:
for str_id in dump:
id = elliptics.Id(str_id)
lookups = []
for g in ctx.groups:
session.groups = [g]
lookups.append(session.read_data(id, size=1))
key_infos = []
for i, l in enumerate(lookups):
try:
result = l.get()[0]
address = result.address
address.group_id = ctx.groups[i]
key_infos.append(
KeyInfo(address, result.timestamp, result.size,
result.user_flags))
except Exception, e:
log.error(
"Failed to lookup key: {0} in group: {1}: {2}, traceback: {3}"
.format(id, ctx.groups[i], repr(e),
traceback.format_exc()))
stats.counter("lookups", -1)
if len(key_infos) > 0:
key_data = (id, key_infos)
pickler.dump(key_data)
stats.counter("lookups", len(key_infos))
else:
log.error(
"Key: {0} is missing in all specified groups: {1}. It won't be recovered."
.format(id, ctx.groups))
def get_routes(ctx):
log.debug('Requesting routes')
log.debug("Creating logger")
elog = elliptics.Logger(ctx.log_file, int(ctx.log_level), True)
log.debug("Creating node")
node = elliptics_create_node(address=ctx.address,
elog=elog,
wait_timeout=ctx.wait_timeout,
flags=elliptics.config_flags.no_route_list,
remotes=ctx.remotes)
log.debug("Creating session for: {0}".format(ctx.address))
session = elliptics_create_session(node=node, group=0, trace_id=ctx.trace_id)
log.debug("Parsing routing table")
return session.routes.filter_by_groups(ctx.groups)
def __init__(self, config):
cfg = elliptics.Config()
log = elliptics.Logger("/dev/stderr", config.get('verbosity', 0))
cfg.config.wait_timeout = config.get("wait-timeout", 60)
cfg.config.io_thread_num = config.get("io-thread-num", 1)
cfg.config.net_thread_num = config.get("net-thread-num", 1)
cfg.config.groups = config.get('groups', [])
self._elliptics_node = elliptics.Node(log, cfg)
for host, port in config.get('nodes').iteritems():
self._elliptics_node.add_remote(host, port)
self._session = elliptics.Session(self._elliptics_node)
self._session.groups = config.get('groups', [])
if len(self._session.groups) == 0:
raise ValueError("Specify groups")
self._session.set_namespace(NAMESPACE)
def get_routes(ctx):
log.debug('Requesting routes')
log.debug("Creating logger")
elog = elliptics.Logger(ctx.log_file, int(ctx.log_level))
log.debug("Creating node")
node = elliptics_create_node(address=ctx.address,
elog=elog,
wait_timeout=ctx.wait_timeout,
remotes=ctx.remotes)
log.debug("Creating session for: {0}".format(ctx.address))
session = elliptics_create_session(node=node,
group=0,
trace_id=ctx.trace_id)
log.debug("Parsing routing table")
return RouteList.from_session(session)
def combine_logs(remotes, groups, min_write, keys, new_key):
elog = elliptics.Logger("/dev/stderr", 0)
cfg = elliptics.Config()
cfg.config.wait_timeout = 60
cfg.config.check_timeout = 60
cfg.config.io_thread_num = 16
cfg.config.nonblocking_io_thread_num = 16
cfg.config.net_thread_num = 16
node = elliptics.Node(elog, cfg)
for r in remotes:
try:
node.add_remote(addr=r[0], port=r[1], family=r[2])
except Exception as e:
print "Coudn't connect to elliptics node: {0}: {1}".format(r, e)
log_s = elliptics.Session(node)
log_s.set_groups(groups)
log_s.set_ioflags(elliptics.io_flags.append)
index_s = elliptics.Session(node)
index_s.set_groups(groups)
index_s.set_ioflags(elliptics.io_flags.cache)
users = Set()
print "Keys: {0}".format(keys)
for key in keys:
try:
users.update(process_key(key, log_s, index_s, new_key))
except Exception as e:
print "Process key failed: {0}".format(e)
print "Users: {0}".format(users)
for u in users:
try:
index_s.update_indexes(elliptics.Id(u), [new_key + ".0"], [u])
except Exception as e:
print "Update_indexes failed: {0}".format(e)
def process_node_backend(ctx, address, backend_id, group, ranges):
log.debug(
"Processing node: {0}/{1} from group: {2} for ranges: {3}".format(
address, backend_id, group, ranges))
stats = ctx.stats['node_{0}/{1}'.format(address, backend_id)]
stats.timer('process', 'started')
elog = elliptics.Logger(ctx.log_file, int(ctx.log_level))
node = elliptics_create_node(address=ctx.address,
elog=elog,
wait_timeout=ctx.wait_timeout,
remotes=ctx.remotes,
io_thread_num=4)
stats.timer('process', 'iterate')
results = iterate_node(ctx=ctx,
node=node,
address=address,
backend_id=backend_id,
ranges=ranges,
eid=ctx.routes.get_address_backend_route_id(
address, backend_id),
stats=stats)
if results is None or len(results) == 0:
log.warning('Iterator result is empty, skipping')
return True
stats.timer('process', 'dump_keys')
dump_path = os.path.join(ctx.tmp_dir,
'dump_{0}.{1}'.format(address, backend_id))
log.debug("Dump iterated keys to file: {0}".format(dump_path))
with open(dump_path, 'w') as dump_f:
for r in results:
dump_f.write('{0}\n'.format(r.key))
stats.timer('process', 'recover')
ret = recover(ctx, address, backend_id, group, node, results, stats)
stats.timer('process', 'finished')
return ret
def __init__(self, config):
cfg = elliptics.Config()
# The parameter which sets the time to wait for the operation complete
cfg.config.wait_timeout = config.get("elliptics_wait_timeout", 60)
# The parameter which sets the timeout for pinging node
cfg.config.check_timeout = config.get("elliptics_check_timeout", 60)
# Number of IO threads in processing pool
cfg.config.io_thread_num = config.get("elliptics_io_thread_num", 2)
# Number of threads in network processing pool
cfg.config.net_thread_num = config.get("elliptics_net_thread_num", 2)
# Number of IO threads in processing pool dedicated to nonblocking ops
nblock_iothreads = config.get("elliptics_nonblocking_io_thread_num", 2)
cfg.config.nonblocking_io_thread_num = nblock_iothreads
self.groups = config.get('elliptics_groups', [])
if len(self.groups) == 0:
raise ValueError("Specify groups")
# loglevel of elliptics logger
elliptics_log_level = config.get('elliptics_verbosity', 0)
# path to logfile
elliptics_log_file = config.get('elliptics_logfile', '/dev/stderr')
log = elliptics.Logger(elliptics_log_file, elliptics_log_level)
self._elliptics_node = elliptics.Node(log, cfg)
self.namespace = config.get('elliptics_namespace', DEFAULT_NAMESPACE)
logger.info("Using namespace %s", self.namespace)
at_least_one = False
for host, port in config.get('elliptics_nodes').iteritems():
try:
self._elliptics_node.add_remote(
host, port, config.get('elliptics_addr_family', 2))
at_least_one = True
except Exception as err:
logger.error("Failed to add remote %s:%d %s", host, port, err)
if not at_least_one:
raise Exception("Unable to connect to Elliptics")
def iterate_node(address_ranges):
"""Iterates node range, sorts it and returns"""
ctx = g_ctx
ctx.elog = elliptics.Logger(ctx.log_file, int(ctx.log_level))
stats_name = 'iterate_{0}'.format(address_ranges.address)
if address_ranges.address == ctx.address:
stats_name = 'iterate_local'
stats = ctx.monitor.stats[stats_name]
stats.timer('process', 'started')
log.info("Running iterator")
stats.timer('process', 'iterate')
result = run_iterator(ctx=ctx,
address=address_ranges.address,
eid=address_ranges.eid,
ranges=address_ranges.id_ranges,
stats=stats)
if result is None or len(result) == 0:
log.warning("Iterator result is empty, skipping")
stats.timer('process', 'finished')
return None
stats.timer('process', 'sort')
sorted_result = sort(ctx=ctx, result=result, stats=stats)
assert len(result) >= len(sorted_result)
log.info("Sorted successfully: {0} result(s)".format(len(sorted_result)))
if sorted_result is None or len(sorted_result) == 0:
log.warning("Sorted results are empty, skipping")
stats.timer('process', 'finished')
return None
stats.timer('process', 'finished')
return (sorted_result.address, sorted_result.filename)
log.info("Starting cleanup...")
cleanup(ctx.tmp_dir)
except Exception as e:
log.error("Cleanup failed: {0}, traceback: {1}".format(
repr(e), traceback.format_exc()))
log.info("Initializing monitor")
monitor = Monitor(ctx, ctx.monitor_port)
ctx.stats = monitor.stats
log.debug("Using following context:\n{0}".format(ctx))
log.debug("Setting up elliptics client")
log.debug("Creating logger")
elog = elliptics.Logger(ctx.log_file, int(ctx.log_level))
log.debug("Creating node")
node = elliptics_create_node(address=ctx.address,
elog=elog,
wait_timeout=ctx.wait_timeout,
remotes=ctx.remotes)
log.debug("Creating session for: {0}".format(ctx.address))
session = elliptics_create_session(node=node, group=0)
log.debug("Parsing routing table")
ctx.routes = RouteList.from_session(session)
log.debug("Parsed routing table:\n{0}".format(ctx.routes))
if not ctx.routes:
raise RuntimeError("No routes was parsed from session")
