def test_2_backends_with_equal_ids_and_group(self, servers):
'''
These test case check correct handling situation when some backend from one nodes has the same group and ids
like another backend from another node.
For this test all nodes mustn't know about each other, their `remote` are empty.
In this test creates 2 client nodes and both connects to 2 different nodes.
At each node selects one backend from one group, equal to both backend.
Updates ids for both backends to make it be equal.
To the second client node adds remote to the first node. It raises route-list update error: -EEXIST and raises exception.
After that makes read some noexistent key from all groups - it will raise an exception.
With old bug thes test case caused `Segmentation fault` on read_data_from_groups.
At the end reverts ids of both backends.
'''
address1, address2 = servers.remotes
address1 = elliptics.Address.from_host_port_family(address1)
session1 = elliptics.Session(
elliptics.Node(
elliptics.Logger("client.log", elliptics.log_level.debug)))
session1._node.add_remotes(address1)
routes = session1.routes.filter_by_address(address1)
group = routes.groups()[-1]
groups = routes.groups()
routes = session1.routes.filter_by_group(group)
ids = [session1.transform('somekey')]
backend_id1 = routes.get_address_backends(address1)[0]
old_ids1 = [
r.id for r in routes.filter_by_address(address1).filter_by_backend(
backend_id1)
]
session1.set_backend_ids(address1, backend_id1, ids).get()
address2 = elliptics.Address.from_host_port_family(address2)
session2 = elliptics.Session(
elliptics.Node(
elliptics.Logger("client.log", elliptics.log_level.debug)))
session2._node.add_remotes(address2)
routes = session2.routes.filter_by_group(group)
backend_id2 = routes.get_address_backends(address2)[0]
old_ids2 = [
r.id for r in routes.filter_by_address(address2).filter_by_backend(
backend_id2)
]
session2.set_backend_ids(address2, backend_id2, ids).get()
with pytest.raises(elliptics.core.Error):
session2._node.add_remotes(address1)
assert session2.routes.addresses() == (address2, )
for g in groups:
with pytest.raises(elliptics.core.Error):
session2.read_data_from_groups(
'unique key for test_2_backends_with_equal_ids_and_group',
[g]).get()
session1.set_backend_ids(address1, backend_id1, old_ids1).get()
session2.set_backend_ids(address2, backend_id2, old_ids2).get()
def make_elliptics_node():
log = elliptics.Logger('/tmp/ell-namespace-convert.log', config["dnet_log_mask"])
node_config = elliptics.Config()
meta_node = elliptics.Node(log, node_config)
addresses = [elliptics.Address(host=str(node[0]), port=node[1], family=node[2])
for node in config["metadata"]["nodes"]]
logger.info('Connecting to meta nodes: {0}'.format(config["metadata"]["nodes"]))
meta_wait_timeout = config['metadata'].get('wait_timeout', 5)
try:
meta_node.add_remotes(addresses)
except Exception as e:
logger.error('Failed to connect to any elliptics meta storage node: {0}'.format(
e))
raise ValueError('Failed to connect to any elliptics storage META node')
meta_session = elliptics.Session(meta_node)
meta_session.set_timeout(meta_wait_timeout)
meta_session.add_groups(list(config["metadata"]["groups"]))
nodes = config.get('elliptics', {}).get('nodes', []) or config["elliptics_nodes"]
logger.debug("config elliptics nodes: %s" % str(nodes))
node_config = elliptics.Config()
node_config.io_thread_num = config.get('io_thread_num', 1)
node_config.nonblocking_io_thread_num = config.get('nonblocking_io_thread_num', 1)
node_config.net_thread_num = config.get('net_thread_num', 1)
logger.info('Node config: io_thread_num {0}, nonblocking_io_thread_num {1}, '
'net_thread_num {2}'.format(node_config.io_thread_num, node_config.nonblocking_io_thread_num,
node_config.net_thread_num))
n = elliptics.Node(log, node_config)
addresses = [elliptics.Address(host=str(node[0]), port=node[1], family=node[2])
for node in nodes]
try:
n.add_remotes(addresses)
except Exception as e:
logger.error('Failed to connect to any elliptics storage node: {0}'.format(
e))
raise ValueError('Failed to connect to any elliptics storage node')
n.meta_session = meta_session
wait_timeout = config.get('elliptics', {}).get('wait_timeout', 5)
s = elliptics.Session(n)
s.set_timeout(wait_timeout)
print 'sleeping for wait timeout: {0} seconds'.format(wait_timeout)
time.sleep(wait_timeout)
return n
def make_meta_session():
log = elliptics.Logger('/tmp/ell-namespace-convert.log',
config["dnet_log_mask"])
node_config = elliptics.Config()
meta_node = elliptics.Node(log, node_config)
addresses = [
elliptics.Address(host=str(node[0]), port=node[1], family=node[2])
for node in config["metadata"]["nodes"]
]
logger.info('Connecting to meta nodes: {0}'.format(
config["metadata"]["nodes"]))
meta_wait_timeout = config['metadata'].get('wait_timeout', 5)
try:
meta_node.add_remotes(addresses)
except Exception as e:
logger.error(
'Failed to connect to any elliptics meta storage node: {0}'.format(
e))
raise ValueError(
'Failed to connect to any elliptics storage META node')
meta_session = elliptics.Session(meta_node)
meta_session.set_timeout(meta_wait_timeout)
meta_session.add_groups(list(config["metadata"]["groups"]))
time.sleep(5)
return meta_session
def __init__(self, config):
cfg = elliptics.Config()
# The parameter which sets the time to wait for the operation complete
cfg.config.wait_timeout = config.get("wait-timeout", 60)
# The parameter which sets the timeout for pinging node
cfg.config.check_timeout = config.get("check_timeout", 60)
# Number of IO threads in processing pool
cfg.config.io_thread_num = config.get("io-thread-num", 2)
# Number of threads in network processing pool
cfg.config.net_thread_num = config.get("net-thread-num", 2)
# Number of IO threads in processing pool dedicated to nonblocking ops
nonblock_io_threads = config.get("nonblocking_io_thread_num", 2)
cfg.config.nonblocking_io_thread_num = nonblock_io_threads
groups = config.get('groups', [])
if len(groups) == 0:
raise ValueError("Specify groups")
# loglevel of elliptics logger
elliptics_log_level = config.get('verbosity', 0)
# path to logfile
elliptics_log_file = config.get('logfile', '/dev/stderr')
log = elliptics.Logger(elliptics_log_file, elliptics_log_level)
self._elliptics_node = elliptics.Node(log, cfg)
for host, port in config.get('nodes').iteritems():
self._elliptics_node.add_remote(host, port)
self._session = elliptics.Session(self._elliptics_node)
self._session.groups = groups
self._session.set_namespace(NAMESPACE)
def elliptics_create_node(address=None,
elog=None,
wait_timeout=3600,
check_timeout=60,
flags=0,
io_thread_num=1,
net_thread_num=1,
nonblocking_io_thread_num=1):
"""
Connects to elliptics cloud
"""
log.info("Creating node using: {0}".format(address))
cfg = elliptics.Config()
cfg.config.wait_timeout = wait_timeout
cfg.config.check_timeout = check_timeout
cfg.config.flags = flags
cfg.config.io_thread_num = io_thread_num
cfg.config.nonblocking_io_thread_num = nonblocking_io_thread_num
cfg.config.net_thread_num = net_thread_num
node = elliptics.Node(elog, cfg)
node.add_remote(addr=address.host,
port=address.port,
family=address.family)
log.info("Created node: {0}".format(node))
return node
def make_meta_session():
log = elliptics.Logger('/tmp/ell-namespace-convert.log',
config["dnet_log_mask"])
node_config = elliptics.Config()
meta_node = elliptics.Node(log, node_config)
nodes = config['metadata']['nodes']
addresses = [
elliptics.Address(host=host, port=port, family=family)
for (host, port, family) in nodes
]
logger.info('Connecting to meta nodes: {0}'.format(nodes))
try:
meta_node.add_remotes(addresses)
except Exception:
raise ValueError(
'Failed to connect to any elliptics storage META node')
meta_session = elliptics.Session(meta_node)
meta_wait_timeout = config['metadata'].get('wait_timeout', 5)
meta_session.set_timeout(meta_wait_timeout)
meta_session.add_groups(list(config["metadata"]["groups"]))
time.sleep(meta_wait_timeout)
return meta_session
def elliptics_create_node(address=None,
elog=None,
wait_timeout=3600,
check_timeout=60,
flags=0,
io_thread_num=1,
net_thread_num=1,
nonblocking_io_thread_num=1,
remotes=[]):
"""
Connects to elliptics cloud
"""
log.debug(
"Creating node using: {0}, wait_timeout: {1}, remotes: {2}".format(
address, wait_timeout, remotes))
cfg = elliptics.Config()
cfg.config.wait_timeout = wait_timeout
cfg.config.check_timeout = check_timeout
cfg.config.flags = flags
cfg.config.io_thread_num = io_thread_num
cfg.config.nonblocking_io_thread_num = nonblocking_io_thread_num
cfg.config.net_thread_num = net_thread_num
node = elliptics.Node(elog, cfg)
node.add_remotes([address] + remotes)
log.debug("Created node: {0}".format(node))
return node
def simple_node(request):
simple_node = elliptics.Node(elliptics.Logger("client.log", 4))
simple_node.add_remote(request.config.option.remotes)
def fin():
print "Finilizing simple node"
request.addfinalizer(fin)
return simple_node
def test_add_remotes_by_one(self, server):
elog = elliptics.Logger('client.log', 4)
node = elliptics.Node(elog)
remotes = server.remotes
node.add_remote(remotes[0])
node.add_remote(elliptics.Address.from_host_port_family(remotes[1]))
host, port, family = remotes[2].split(':')
node.add_remote(host, int(port), int(family))
def simple_node(request):
if len(request.config.option.remotes) == 0:
pytest.fail(
'`simple_node` fixture should go after `server` fixture, check your test'
)
simple_node = elliptics.Node(
elliptics.Logger("client.log", elliptics.log_level.debug))
simple_node.add_remotes(request.config.option.remotes)
return simple_node
def test_add_several_remotes(self, server):
elog = elliptics.Logger('client.log', 4)
node = elliptics.Node(elog)
remotes = []
for num, remote in enumerate(server.remotes):
if num % 2 == 0:
remotes.append(remote)
else:
remotes.append(elliptics.Address.from_host_port_family(remote))
node.add_remote(remotes)
def test_dc_isolated_groups(self, servers):
'''
Write one key into every group,
run dc server-send recovery,
check that keys were recovered into all groups.
'''
scope.node = elliptics.Node(
elliptics.Logger("client.log", elliptics.log_level.debug))
scope.node.add_remotes(servers.remotes)
session = make_session(
node=scope.node,
test_name='TestIsolatedRecovery.test_dc_isolated_groups',
test_namespace=self.namespace)
groups = session.routes.groups()
scope.test_group = groups[0]
scope.test_group2 = groups[1]
routes = session.routes.filter_by_group(scope.test_group)
scope.test_address = routes[0].address
keys = []
data = 'isolated_data'
groups = (scope.test_group, scope.test_group2)
session.timestamp = elliptics.Time.now()
for group_id in groups:
key = 'isolated_key_{}'.format(group_id)
session.groups = [group_id]
write_data(scope, session, [key], [data])
check_data(scope, session, [key], [data], session.timestamp)
keys.append(key)
recovery(one_node=False,
remotes=map(elliptics.Address.from_host_port_family,
servers.remotes),
backend_id=None,
address=scope.test_address,
groups=groups,
rtype=RECOVERY.DC,
log_file='dc_isolated_groups.log',
tmp_dir='dc_isolated_groups')
for group_id in groups:
session.groups = [group_id]
check_data(scope, session, keys, [data] * len(keys),
session.timestamp)
def __init__(self, config):
cfg = elliptics.Config()
log = elliptics.Logger("/dev/stderr", config.get('verbosity', 0))
cfg.config.wait_timeout = config.get("wait-timeout", 60)
cfg.config.io_thread_num = config.get("io-thread-num", 1)
cfg.config.net_thread_num = config.get("net-thread-num", 1)
cfg.config.groups = config.get('groups', [])
self._elliptics_node = elliptics.Node(log, cfg)
for host, port in config.get('nodes').iteritems():
self._elliptics_node.add_remote(host, port)
self._session = elliptics.Session(self._elliptics_node)
self._session.groups = config.get('groups', [])
if len(self._session.groups) == 0:
raise ValueError("Specify groups")
self._session.set_namespace(NAMESPACE)
def combine_logs(remotes, groups, min_write, keys, new_key):
elog = elliptics.Logger("/dev/stderr", 0)
cfg = elliptics.Config()
cfg.config.wait_timeout = 60
cfg.config.check_timeout = 60
cfg.config.io_thread_num = 16
cfg.config.nonblocking_io_thread_num = 16
cfg.config.net_thread_num = 16
node = elliptics.Node(elog, cfg)
for r in remotes:
try:
node.add_remote(addr=r[0], port=r[1], family=r[2])
except Exception as e:
print "Coudn't connect to elliptics node: {0}: {1}".format(r, e)
log_s = elliptics.Session(node)
log_s.set_groups(groups)
log_s.set_ioflags(elliptics.io_flags.append)
index_s = elliptics.Session(node)
index_s.set_groups(groups)
index_s.set_ioflags(elliptics.io_flags.cache)
users = Set()
print "Keys: {0}".format(keys)
for key in keys:
try:
users.update(process_key(key, log_s, index_s, new_key))
except Exception as e:
print "Process key failed: {0}".format(e)
print "Users: {0}".format(users)
for u in users:
try:
index_s.update_indexes(elliptics.Id(u), [new_key + ".0"], [u])
except Exception as e:
print "Update_indexes failed: {0}".format(e)
def __init__(self, config):
cfg = elliptics.Config()
# The parameter which sets the time to wait for the operation complete
cfg.config.wait_timeout = config.get("elliptics_wait_timeout", 60)
# The parameter which sets the timeout for pinging node
cfg.config.check_timeout = config.get("elliptics_check_timeout", 60)
# Number of IO threads in processing pool
cfg.config.io_thread_num = config.get("elliptics_io_thread_num", 2)
# Number of threads in network processing pool
cfg.config.net_thread_num = config.get("elliptics_net_thread_num", 2)
# Number of IO threads in processing pool dedicated to nonblocking ops
nblock_iothreads = config.get("elliptics_nonblocking_io_thread_num", 2)
cfg.config.nonblocking_io_thread_num = nblock_iothreads
self.groups = config.get('elliptics_groups', [])
if len(self.groups) == 0:
raise ValueError("Specify groups")
# loglevel of elliptics logger
elliptics_log_level = config.get('elliptics_verbosity', 0)
# path to logfile
elliptics_log_file = config.get('elliptics_logfile', '/dev/stderr')
log = elliptics.Logger(elliptics_log_file, elliptics_log_level)
self._elliptics_node = elliptics.Node(log, cfg)
self.namespace = config.get('elliptics_namespace', DEFAULT_NAMESPACE)
logger.info("Using namespace %s", self.namespace)
at_least_one = False
for host, port in config.get('elliptics_nodes').iteritems():
try:
self._elliptics_node.add_remote(
host, port, config.get('elliptics_addr_family', 2))
at_least_one = True
except Exception as err:
logger.error("Failed to add remote %s:%d %s", host, port, err)
if not at_least_one:
raise Exception("Unable to connect to Elliptics")
if options.route_file and options.route_addr:
ranges = parse_route_ranges(options.route_file, options.route_addr)
ctx.ranges += ranges
except Exception as e:
raise ValueError(
"Can't parse route_file '{0}' and route_addr '{1}' options: {2}".
format(options.route_file, options.route_addr, repr(e)))
return ctx
if __name__ == '__main__':
ctx = parse_args()
ctx.elog = elliptics.Logger(ctx.log_file, ctx.log_level)
ctx.node = elliptics.Node(ctx.elog)
ctx.node.add_remotes(ctx.remotes)
ctx.session = elliptics.Session(ctx.node)
ctx.session.set_timeout(60)
if ctx.groups:
ctx.session.groups = ctx.groups
else:
ctx.session.groups = ctx.session.routes.groups()
#print ctx.session.routes
if ctx.iterate_mode == MODE_NODES:
for r in ctx.remotes:
eid = elliptics.Id([0] * 64, 0)
if len(ctx.ranges) != 0:
def init_elliptics_node():
nodes = config.get('elliptics', {}).get('nodes', []) or config["elliptics_nodes"]
logger.debug("config: %s" % str(nodes))
log = elliptics.Logger(str(config["dnet_log"]), config["dnet_log_mask"])
node_config = elliptics.Config()
node_config.io_thread_num = config.get('io_thread_num', 1)
node_config.nonblocking_io_thread_num = config.get('nonblocking_io_thread_num', 1)
node_config.net_thread_num = config.get('net_thread_num', 1)
logger.info('Node config: io_thread_num {0}, nonblocking_io_thread_num {1}, '
'net_thread_num {2}'.format(
node_config.io_thread_num, node_config.nonblocking_io_thread_num,
node_config.net_thread_num))
n = elliptics.Node(log, node_config)
addresses = []
for node in nodes:
try:
addresses.append(elliptics.Address(
host=str(node[0]), port=node[1], family=node[2]))
except Exception as e:
logger.error('Failed to connect to storage node: {0}:{1}:{2}'.format(
node[0], node[1], node[2]))
pass
try:
n.add_remotes(addresses)
except Exception as e:
logger.error('Failed to connect to any elliptics storage node: {0}'.format(
e))
raise ValueError('Failed to connect to any elliptics storage node')
meta_node = elliptics.Node(log, node_config)
addresses = []
for node in config["metadata"]["nodes"]:
try:
addresses.append(elliptics.Address(
host=str(node[0]), port=node[1], family=node[2]))
except Exception as e:
logger.error('Failed to connect to meta node: {0}:{1}:{2}'.format(
node[0], node[1], node[2]))
pass
logger.info('Connecting to meta nodes: {0}'.format(config["metadata"]["nodes"]))
try:
meta_node.add_remotes(addresses)
except Exception as e:
logger.error('Failed to connect to any elliptics meta storage node: {0}'.format(
e))
raise ValueError('Failed to connect to any elliptics storage META node')
meta_wait_timeout = config['metadata'].get('wait_timeout', 5)
meta_session = elliptics.Session(meta_node)
meta_session.set_timeout(meta_wait_timeout)
meta_session.add_groups(list(config["metadata"]["groups"]))
n.meta_session = meta_session
wait_timeout = config.get('elliptics', {}).get('wait_timeout', 5)
time.sleep(wait_timeout)
return n
import sys
import random
rnd = random.randint(0, 999999999)
test_id = "test_id_" + str(rnd)
test_ind = "test_ind_" + str(rnd)
test_data = "test_data_" + str(rnd)
sys.path.insert(0, "bindings/python/")
import elliptics
elog = elliptics.Logger("/dev/stderr", 0)
cfg = elliptics.Config()
node = elliptics.Node(elog, cfg)
node.add_remote("localhost", 1025)
s = elliptics.Session(node)
s.set_groups([1])
r = s.set_indexes(elliptics.Id(test_id), [test_ind], [test_data])
r.wait()
assert r.successful()
r = s.find_any_indexes([test_ind])
r.wait()
assert r.successful()
assert len(r.get()) >= 1
assert r.get()[0].indexes[0].data == test_data
import cache
import minions
import node_info_updater
import statistics
from config import config
logger = logging.getLogger('mm.init')
i = iter(xrange(100))
logger.info("trace %d" % (i.next()))
logger.debug("config: %s" % str(config["elliptics_nodes"]))
logger.info("trace %d" % (i.next()))
log = elliptics.Logger(str(config["dnet_log"]), config["dnet_log_mask"])
n = elliptics.Node(log)
connected = False
logger.info("trace %d" % (i.next()))
for host in config["elliptics_nodes"]:
logger.debug("Adding node %s" % str(host))
try:
logger.info("host: " + str(host))
n.add_remote(str(host[0]), host[1])
connected = True
except Exception as e:
logger.error("Error: " + str(e) + "\n" + traceback.format_exc())
if not connected:
logger.error('Failed to connect to any elliptics storage node')
ch = logging.StreamHandler(sys.stderr)
ch.setFormatter(formatter)
ch.setLevel(logging.INFO)
log.addHandler(ch)
hand_re = re.compile(
"POST \/(.*) HTTP/1.1.*"
"user=([0-9]*).*"
"data=(.*)&"
"key=([a-zA-Z_0-9]*).*", re.DOTALL)
cfg = elliptics.Config()
cfg.config.check_timeout = 1000
cfg.config.wait_timeout = 1000
n = elliptics.Node(elliptics.Logger("/dev/stderr", 0), cfg)
n.add_remote("s16h.xxx.yandex.net", 1025)
activities = dict()
offsets = dict()
counter = 0
start = None
def process_packet(packet, s, perc):
global activities, offsets, counter, start
ret = True
log.debug("packet: '{0}'".format(packet))
hand, user, data, key = hand_re.search(packet).groups()
log.debug("hand: '{0}', user = '******', data = '{2}', key = '{3}'".format(
'--percentage',
dest='percentage',
action='store_true',
help=
'if present, dump parts of DHT ring each node occupies (in percents)')
parser.add_argument('--log', default='/dev/stdout', help='log file')
parser.add_argument('--log-level',
type=int,
default=elliptics.log_level.error,
help='log level: %d-%d' %
(elliptics.log_level.error, elliptics.log_level.debug))
args = parser.parse_args()
if len(args.remotes) == 0:
args.remotes = "localhost:1025:2"
log = elliptics.Logger(args.log, args.log_level)
n = elliptics.Node(log)
s = elliptics.Session(n)
try:
n.add_remotes(args.remotes)
except Exception as e:
print e
pass
if args.percentage:
percentage(s.routes)
else:
print routes
logger.info("trace %d" % (i.next()))
log = elliptics.Logger(str(config["dnet_log"]), config["dnet_log_mask"])
node_config = elliptics.Config()
node_config.io_thread_num = config.get('io_thread_num', 1)
node_config.nonblocking_io_thread_num = config.get('nonblocking_io_thread_num',
1)
node_config.net_thread_num = config.get('net_thread_num', 1)
logger.info('Node config: io_thread_num {0}, nonblocking_io_thread_num {1}, '
'net_thread_num {2}'.format(node_config.io_thread_num,
node_config.nonblocking_io_thread_num,
node_config.net_thread_num))
n = elliptics.Node(log, node_config)
logger.info("trace %d" % (i.next()))
addresses = []
for node in nodes:
try:
addresses.append(
elliptics.Address(host=str(node[0]), port=node[1], family=node[2]))
except Exception as e:
logger.error('Failed to connect to storage node: {0}:{1}:{2}'.format(
node[0], node[1], node[2]))
pass
try:
n.add_remotes(addresses)
def init_node(self, log_file, log_level, remotes):
if not self.node:
self.elog = elliptics.Logger(log_file, log_level)
self.node = elliptics.Node(self.elog)
self.node.add_remotes(remotes)
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# =============================================================================
import sys
import elliptics
import binascii
from pprint import pprint
try:
log = elliptics.Logger("/dev/stderr", 31)
cfg = elliptics.Config()
cfg.cookie = "0123456789012345678901234567890123456789"
cfg.config.wait_timeout = 60
n = elliptics.Node(log, cfg)
n.add_remotes("localhost:1025:2")
s = elliptics.Session(n)
s.add_groups([1, 2, 3])
group = 1
try:
obj = "qwerty.xml"
addr = s.lookup_address(obj, group)
print "object", obj, "should live at", addr, "in group", group
except Exception as e:
print "Failed to lookup in group", group, ":", e
def simple_node(request):
simple_node = elliptics.Node(
elliptics.Logger("client.log", elliptics.log_level.debug))
simple_node.add_remotes(request.config.option.remotes)
return simple_node
