def test_2_backends_with_equal_ids_and_group(self, servers): ''' These test case check correct handling situation when some backend from one nodes has the same group and ids like another backend from another node. For this test all nodes mustn't know about each other, their `remote` are empty. In this test creates 2 client nodes and both connects to 2 different nodes. At each node selects one backend from one group, equal to both backend. Updates ids for both backends to make it be equal. To the second client node adds remote to the first node. It raises route-list update error: -EEXIST and raises exception. After that makes read some noexistent key from all groups - it will raise an exception. With old bug thes test case caused `Segmentation fault` on read_data_from_groups. At the end reverts ids of both backends. ''' address1, address2 = servers.remotes address1 = elliptics.Address.from_host_port_family(address1) session1 = elliptics.Session( elliptics.Node( elliptics.Logger("client.log", elliptics.log_level.debug))) session1._node.add_remotes(address1) routes = session1.routes.filter_by_address(address1) group = routes.groups()[-1] groups = routes.groups() routes = session1.routes.filter_by_group(group) ids = [session1.transform('somekey')] backend_id1 = routes.get_address_backends(address1)[0] old_ids1 = [ r.id for r in routes.filter_by_address(address1).filter_by_backend( backend_id1) ] session1.set_backend_ids(address1, backend_id1, ids).get() address2 = elliptics.Address.from_host_port_family(address2) session2 = elliptics.Session( elliptics.Node( elliptics.Logger("client.log", elliptics.log_level.debug))) session2._node.add_remotes(address2) routes = session2.routes.filter_by_group(group) backend_id2 = routes.get_address_backends(address2)[0] old_ids2 = [ r.id for r in routes.filter_by_address(address2).filter_by_backend( backend_id2) ] session2.set_backend_ids(address2, backend_id2, ids).get() with pytest.raises(elliptics.core.Error): session2._node.add_remotes(address1) assert session2.routes.addresses() == (address2, ) for g in groups: with pytest.raises(elliptics.core.Error): session2.read_data_from_groups( 'unique key for test_2_backends_with_equal_ids_and_group', [g]).get() session1.set_backend_ids(address1, backend_id1, old_ids1).get() session2.set_backend_ids(address2, backend_id2, old_ids2).get()
def make_elliptics_node(): log = elliptics.Logger('/tmp/ell-namespace-convert.log', config["dnet_log_mask"]) node_config = elliptics.Config() meta_node = elliptics.Node(log, node_config) addresses = [elliptics.Address(host=str(node[0]), port=node[1], family=node[2]) for node in config["metadata"]["nodes"]] logger.info('Connecting to meta nodes: {0}'.format(config["metadata"]["nodes"])) meta_wait_timeout = config['metadata'].get('wait_timeout', 5) try: meta_node.add_remotes(addresses) except Exception as e: logger.error('Failed to connect to any elliptics meta storage node: {0}'.format( e)) raise ValueError('Failed to connect to any elliptics storage META node') meta_session = elliptics.Session(meta_node) meta_session.set_timeout(meta_wait_timeout) meta_session.add_groups(list(config["metadata"]["groups"])) nodes = config.get('elliptics', {}).get('nodes', []) or config["elliptics_nodes"] logger.debug("config elliptics nodes: %s" % str(nodes)) node_config = elliptics.Config() node_config.io_thread_num = config.get('io_thread_num', 1) node_config.nonblocking_io_thread_num = config.get('nonblocking_io_thread_num', 1) node_config.net_thread_num = config.get('net_thread_num', 1) logger.info('Node config: io_thread_num {0}, nonblocking_io_thread_num {1}, ' 'net_thread_num {2}'.format(node_config.io_thread_num, node_config.nonblocking_io_thread_num, node_config.net_thread_num)) n = elliptics.Node(log, node_config) addresses = [elliptics.Address(host=str(node[0]), port=node[1], family=node[2]) for node in nodes] try: n.add_remotes(addresses) except Exception as e: logger.error('Failed to connect to any elliptics storage node: {0}'.format( e)) raise ValueError('Failed to connect to any elliptics storage node') n.meta_session = meta_session wait_timeout = config.get('elliptics', {}).get('wait_timeout', 5) s = elliptics.Session(n) s.set_timeout(wait_timeout) print 'sleeping for wait timeout: {0} seconds'.format(wait_timeout) time.sleep(wait_timeout) return n
def make_meta_session(): log = elliptics.Logger('/tmp/ell-namespace-convert.log', config["dnet_log_mask"]) node_config = elliptics.Config() meta_node = elliptics.Node(log, node_config) addresses = [ elliptics.Address(host=str(node[0]), port=node[1], family=node[2]) for node in config["metadata"]["nodes"] ] logger.info('Connecting to meta nodes: {0}'.format( config["metadata"]["nodes"])) meta_wait_timeout = config['metadata'].get('wait_timeout', 5) try: meta_node.add_remotes(addresses) except Exception as e: logger.error( 'Failed to connect to any elliptics meta storage node: {0}'.format( e)) raise ValueError( 'Failed to connect to any elliptics storage META node') meta_session = elliptics.Session(meta_node) meta_session.set_timeout(meta_wait_timeout) meta_session.add_groups(list(config["metadata"]["groups"])) time.sleep(5) return meta_session
def __init__(self, config): cfg = elliptics.Config() # The parameter which sets the time to wait for the operation complete cfg.config.wait_timeout = config.get("wait-timeout", 60) # The parameter which sets the timeout for pinging node cfg.config.check_timeout = config.get("check_timeout", 60) # Number of IO threads in processing pool cfg.config.io_thread_num = config.get("io-thread-num", 2) # Number of threads in network processing pool cfg.config.net_thread_num = config.get("net-thread-num", 2) # Number of IO threads in processing pool dedicated to nonblocking ops nonblock_io_threads = config.get("nonblocking_io_thread_num", 2) cfg.config.nonblocking_io_thread_num = nonblock_io_threads groups = config.get('groups', []) if len(groups) == 0: raise ValueError("Specify groups") # loglevel of elliptics logger elliptics_log_level = config.get('verbosity', 0) # path to logfile elliptics_log_file = config.get('logfile', '/dev/stderr') log = elliptics.Logger(elliptics_log_file, elliptics_log_level) self._elliptics_node = elliptics.Node(log, cfg) for host, port in config.get('nodes').iteritems(): self._elliptics_node.add_remote(host, port) self._session = elliptics.Session(self._elliptics_node) self._session.groups = groups self._session.set_namespace(NAMESPACE)
def elliptics_create_node(address=None, elog=None, wait_timeout=3600, check_timeout=60, flags=0, io_thread_num=1, net_thread_num=1, nonblocking_io_thread_num=1): """ Connects to elliptics cloud """ log.info("Creating node using: {0}".format(address)) cfg = elliptics.Config() cfg.config.wait_timeout = wait_timeout cfg.config.check_timeout = check_timeout cfg.config.flags = flags cfg.config.io_thread_num = io_thread_num cfg.config.nonblocking_io_thread_num = nonblocking_io_thread_num cfg.config.net_thread_num = net_thread_num node = elliptics.Node(elog, cfg) node.add_remote(addr=address.host, port=address.port, family=address.family) log.info("Created node: {0}".format(node)) return node
def make_meta_session(): log = elliptics.Logger('/tmp/ell-namespace-convert.log', config["dnet_log_mask"]) node_config = elliptics.Config() meta_node = elliptics.Node(log, node_config) nodes = config['metadata']['nodes'] addresses = [ elliptics.Address(host=host, port=port, family=family) for (host, port, family) in nodes ] logger.info('Connecting to meta nodes: {0}'.format(nodes)) try: meta_node.add_remotes(addresses) except Exception: raise ValueError( 'Failed to connect to any elliptics storage META node') meta_session = elliptics.Session(meta_node) meta_wait_timeout = config['metadata'].get('wait_timeout', 5) meta_session.set_timeout(meta_wait_timeout) meta_session.add_groups(list(config["metadata"]["groups"])) time.sleep(meta_wait_timeout) return meta_session
def elliptics_create_node(address=None, elog=None, wait_timeout=3600, check_timeout=60, flags=0, io_thread_num=1, net_thread_num=1, nonblocking_io_thread_num=1, remotes=[]): """ Connects to elliptics cloud """ log.debug( "Creating node using: {0}, wait_timeout: {1}, remotes: {2}".format( address, wait_timeout, remotes)) cfg = elliptics.Config() cfg.config.wait_timeout = wait_timeout cfg.config.check_timeout = check_timeout cfg.config.flags = flags cfg.config.io_thread_num = io_thread_num cfg.config.nonblocking_io_thread_num = nonblocking_io_thread_num cfg.config.net_thread_num = net_thread_num node = elliptics.Node(elog, cfg) node.add_remotes([address] + remotes) log.debug("Created node: {0}".format(node)) return node
def simple_node(request): simple_node = elliptics.Node(elliptics.Logger("client.log", 4)) simple_node.add_remote(request.config.option.remotes) def fin(): print "Finilizing simple node" request.addfinalizer(fin) return simple_node
def test_add_remotes_by_one(self, server): elog = elliptics.Logger('client.log', 4) node = elliptics.Node(elog) remotes = server.remotes node.add_remote(remotes[0]) node.add_remote(elliptics.Address.from_host_port_family(remotes[1])) host, port, family = remotes[2].split(':') node.add_remote(host, int(port), int(family))
def simple_node(request): if len(request.config.option.remotes) == 0: pytest.fail( '`simple_node` fixture should go after `server` fixture, check your test' ) simple_node = elliptics.Node( elliptics.Logger("client.log", elliptics.log_level.debug)) simple_node.add_remotes(request.config.option.remotes) return simple_node
def test_add_several_remotes(self, server): elog = elliptics.Logger('client.log', 4) node = elliptics.Node(elog) remotes = [] for num, remote in enumerate(server.remotes): if num % 2 == 0: remotes.append(remote) else: remotes.append(elliptics.Address.from_host_port_family(remote)) node.add_remote(remotes)
def test_dc_isolated_groups(self, servers): ''' Write one key into every group, run dc server-send recovery, check that keys were recovered into all groups. ''' scope.node = elliptics.Node( elliptics.Logger("client.log", elliptics.log_level.debug)) scope.node.add_remotes(servers.remotes) session = make_session( node=scope.node, test_name='TestIsolatedRecovery.test_dc_isolated_groups', test_namespace=self.namespace) groups = session.routes.groups() scope.test_group = groups[0] scope.test_group2 = groups[1] routes = session.routes.filter_by_group(scope.test_group) scope.test_address = routes[0].address keys = [] data = 'isolated_data' groups = (scope.test_group, scope.test_group2) session.timestamp = elliptics.Time.now() for group_id in groups: key = 'isolated_key_{}'.format(group_id) session.groups = [group_id] write_data(scope, session, [key], [data]) check_data(scope, session, [key], [data], session.timestamp) keys.append(key) recovery(one_node=False, remotes=map(elliptics.Address.from_host_port_family, servers.remotes), backend_id=None, address=scope.test_address, groups=groups, rtype=RECOVERY.DC, log_file='dc_isolated_groups.log', tmp_dir='dc_isolated_groups') for group_id in groups: session.groups = [group_id] check_data(scope, session, keys, [data] * len(keys), session.timestamp)
def __init__(self, config): cfg = elliptics.Config() log = elliptics.Logger("/dev/stderr", config.get('verbosity', 0)) cfg.config.wait_timeout = config.get("wait-timeout", 60) cfg.config.io_thread_num = config.get("io-thread-num", 1) cfg.config.net_thread_num = config.get("net-thread-num", 1) cfg.config.groups = config.get('groups', []) self._elliptics_node = elliptics.Node(log, cfg) for host, port in config.get('nodes').iteritems(): self._elliptics_node.add_remote(host, port) self._session = elliptics.Session(self._elliptics_node) self._session.groups = config.get('groups', []) if len(self._session.groups) == 0: raise ValueError("Specify groups") self._session.set_namespace(NAMESPACE)
def combine_logs(remotes, groups, min_write, keys, new_key): elog = elliptics.Logger("/dev/stderr", 0) cfg = elliptics.Config() cfg.config.wait_timeout = 60 cfg.config.check_timeout = 60 cfg.config.io_thread_num = 16 cfg.config.nonblocking_io_thread_num = 16 cfg.config.net_thread_num = 16 node = elliptics.Node(elog, cfg) for r in remotes: try: node.add_remote(addr=r[0], port=r[1], family=r[2]) except Exception as e: print "Coudn't connect to elliptics node: {0}: {1}".format(r, e) log_s = elliptics.Session(node) log_s.set_groups(groups) log_s.set_ioflags(elliptics.io_flags.append) index_s = elliptics.Session(node) index_s.set_groups(groups) index_s.set_ioflags(elliptics.io_flags.cache) users = Set() print "Keys: {0}".format(keys) for key in keys: try: users.update(process_key(key, log_s, index_s, new_key)) except Exception as e: print "Process key failed: {0}".format(e) print "Users: {0}".format(users) for u in users: try: index_s.update_indexes(elliptics.Id(u), [new_key + ".0"], [u]) except Exception as e: print "Update_indexes failed: {0}".format(e)
def __init__(self, config): cfg = elliptics.Config() # The parameter which sets the time to wait for the operation complete cfg.config.wait_timeout = config.get("elliptics_wait_timeout", 60) # The parameter which sets the timeout for pinging node cfg.config.check_timeout = config.get("elliptics_check_timeout", 60) # Number of IO threads in processing pool cfg.config.io_thread_num = config.get("elliptics_io_thread_num", 2) # Number of threads in network processing pool cfg.config.net_thread_num = config.get("elliptics_net_thread_num", 2) # Number of IO threads in processing pool dedicated to nonblocking ops nblock_iothreads = config.get("elliptics_nonblocking_io_thread_num", 2) cfg.config.nonblocking_io_thread_num = nblock_iothreads self.groups = config.get('elliptics_groups', []) if len(self.groups) == 0: raise ValueError("Specify groups") # loglevel of elliptics logger elliptics_log_level = config.get('elliptics_verbosity', 0) # path to logfile elliptics_log_file = config.get('elliptics_logfile', '/dev/stderr') log = elliptics.Logger(elliptics_log_file, elliptics_log_level) self._elliptics_node = elliptics.Node(log, cfg) self.namespace = config.get('elliptics_namespace', DEFAULT_NAMESPACE) logger.info("Using namespace %s", self.namespace) at_least_one = False for host, port in config.get('elliptics_nodes').iteritems(): try: self._elliptics_node.add_remote( host, port, config.get('elliptics_addr_family', 2)) at_least_one = True except Exception as err: logger.error("Failed to add remote %s:%d %s", host, port, err) if not at_least_one: raise Exception("Unable to connect to Elliptics")
if options.route_file and options.route_addr: ranges = parse_route_ranges(options.route_file, options.route_addr) ctx.ranges += ranges except Exception as e: raise ValueError( "Can't parse route_file '{0}' and route_addr '{1}' options: {2}". format(options.route_file, options.route_addr, repr(e))) return ctx if __name__ == '__main__': ctx = parse_args() ctx.elog = elliptics.Logger(ctx.log_file, ctx.log_level) ctx.node = elliptics.Node(ctx.elog) ctx.node.add_remotes(ctx.remotes) ctx.session = elliptics.Session(ctx.node) ctx.session.set_timeout(60) if ctx.groups: ctx.session.groups = ctx.groups else: ctx.session.groups = ctx.session.routes.groups() #print ctx.session.routes if ctx.iterate_mode == MODE_NODES: for r in ctx.remotes: eid = elliptics.Id([0] * 64, 0) if len(ctx.ranges) != 0:
def init_elliptics_node(): nodes = config.get('elliptics', {}).get('nodes', []) or config["elliptics_nodes"] logger.debug("config: %s" % str(nodes)) log = elliptics.Logger(str(config["dnet_log"]), config["dnet_log_mask"]) node_config = elliptics.Config() node_config.io_thread_num = config.get('io_thread_num', 1) node_config.nonblocking_io_thread_num = config.get('nonblocking_io_thread_num', 1) node_config.net_thread_num = config.get('net_thread_num', 1) logger.info('Node config: io_thread_num {0}, nonblocking_io_thread_num {1}, ' 'net_thread_num {2}'.format( node_config.io_thread_num, node_config.nonblocking_io_thread_num, node_config.net_thread_num)) n = elliptics.Node(log, node_config) addresses = [] for node in nodes: try: addresses.append(elliptics.Address( host=str(node[0]), port=node[1], family=node[2])) except Exception as e: logger.error('Failed to connect to storage node: {0}:{1}:{2}'.format( node[0], node[1], node[2])) pass try: n.add_remotes(addresses) except Exception as e: logger.error('Failed to connect to any elliptics storage node: {0}'.format( e)) raise ValueError('Failed to connect to any elliptics storage node') meta_node = elliptics.Node(log, node_config) addresses = [] for node in config["metadata"]["nodes"]: try: addresses.append(elliptics.Address( host=str(node[0]), port=node[1], family=node[2])) except Exception as e: logger.error('Failed to connect to meta node: {0}:{1}:{2}'.format( node[0], node[1], node[2])) pass logger.info('Connecting to meta nodes: {0}'.format(config["metadata"]["nodes"])) try: meta_node.add_remotes(addresses) except Exception as e: logger.error('Failed to connect to any elliptics meta storage node: {0}'.format( e)) raise ValueError('Failed to connect to any elliptics storage META node') meta_wait_timeout = config['metadata'].get('wait_timeout', 5) meta_session = elliptics.Session(meta_node) meta_session.set_timeout(meta_wait_timeout) meta_session.add_groups(list(config["metadata"]["groups"])) n.meta_session = meta_session wait_timeout = config.get('elliptics', {}).get('wait_timeout', 5) time.sleep(wait_timeout) return n
import sys import random rnd = random.randint(0, 999999999) test_id = "test_id_" + str(rnd) test_ind = "test_ind_" + str(rnd) test_data = "test_data_" + str(rnd) sys.path.insert(0, "bindings/python/") import elliptics elog = elliptics.Logger("/dev/stderr", 0) cfg = elliptics.Config() node = elliptics.Node(elog, cfg) node.add_remote("localhost", 1025) s = elliptics.Session(node) s.set_groups([1]) r = s.set_indexes(elliptics.Id(test_id), [test_ind], [test_data]) r.wait() assert r.successful() r = s.find_any_indexes([test_ind]) r.wait() assert r.successful() assert len(r.get()) >= 1 assert r.get()[0].indexes[0].data == test_data
import cache import minions import node_info_updater import statistics from config import config logger = logging.getLogger('mm.init') i = iter(xrange(100)) logger.info("trace %d" % (i.next())) logger.debug("config: %s" % str(config["elliptics_nodes"])) logger.info("trace %d" % (i.next())) log = elliptics.Logger(str(config["dnet_log"]), config["dnet_log_mask"]) n = elliptics.Node(log) connected = False logger.info("trace %d" % (i.next())) for host in config["elliptics_nodes"]: logger.debug("Adding node %s" % str(host)) try: logger.info("host: " + str(host)) n.add_remote(str(host[0]), host[1]) connected = True except Exception as e: logger.error("Error: " + str(e) + "\n" + traceback.format_exc()) if not connected: logger.error('Failed to connect to any elliptics storage node')
ch = logging.StreamHandler(sys.stderr) ch.setFormatter(formatter) ch.setLevel(logging.INFO) log.addHandler(ch) hand_re = re.compile( "POST \/(.*) HTTP/1.1.*" "user=([0-9]*).*" "data=(.*)&" "key=([a-zA-Z_0-9]*).*", re.DOTALL) cfg = elliptics.Config() cfg.config.check_timeout = 1000 cfg.config.wait_timeout = 1000 n = elliptics.Node(elliptics.Logger("/dev/stderr", 0), cfg) n.add_remote("s16h.xxx.yandex.net", 1025) activities = dict() offsets = dict() counter = 0 start = None def process_packet(packet, s, perc): global activities, offsets, counter, start ret = True log.debug("packet: '{0}'".format(packet)) hand, user, data, key = hand_re.search(packet).groups() log.debug("hand: '{0}', user = '******', data = '{2}', key = '{3}'".format(
'--percentage', dest='percentage', action='store_true', help= 'if present, dump parts of DHT ring each node occupies (in percents)') parser.add_argument('--log', default='/dev/stdout', help='log file') parser.add_argument('--log-level', type=int, default=elliptics.log_level.error, help='log level: %d-%d' % (elliptics.log_level.error, elliptics.log_level.debug)) args = parser.parse_args() if len(args.remotes) == 0: args.remotes = "localhost:1025:2" log = elliptics.Logger(args.log, args.log_level) n = elliptics.Node(log) s = elliptics.Session(n) try: n.add_remotes(args.remotes) except Exception as e: print e pass if args.percentage: percentage(s.routes) else: print routes
logger.info("trace %d" % (i.next())) log = elliptics.Logger(str(config["dnet_log"]), config["dnet_log_mask"]) node_config = elliptics.Config() node_config.io_thread_num = config.get('io_thread_num', 1) node_config.nonblocking_io_thread_num = config.get('nonblocking_io_thread_num', 1) node_config.net_thread_num = config.get('net_thread_num', 1) logger.info('Node config: io_thread_num {0}, nonblocking_io_thread_num {1}, ' 'net_thread_num {2}'.format(node_config.io_thread_num, node_config.nonblocking_io_thread_num, node_config.net_thread_num)) n = elliptics.Node(log, node_config) logger.info("trace %d" % (i.next())) addresses = [] for node in nodes: try: addresses.append( elliptics.Address(host=str(node[0]), port=node[1], family=node[2])) except Exception as e: logger.error('Failed to connect to storage node: {0}:{1}:{2}'.format( node[0], node[1], node[2])) pass try: n.add_remotes(addresses)
def init_node(self, log_file, log_level, remotes): if not self.node: self.elog = elliptics.Logger(log_file, log_level) self.node = elliptics.Node(self.elog) self.node.add_remotes(remotes)
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # ============================================================================= import sys import elliptics import binascii from pprint import pprint try: log = elliptics.Logger("/dev/stderr", 31) cfg = elliptics.Config() cfg.cookie = "0123456789012345678901234567890123456789" cfg.config.wait_timeout = 60 n = elliptics.Node(log, cfg) n.add_remotes("localhost:1025:2") s = elliptics.Session(n) s.add_groups([1, 2, 3]) group = 1 try: obj = "qwerty.xml" addr = s.lookup_address(obj, group) print "object", obj, "should live at", addr, "in group", group except Exception as e: print "Failed to lookup in group", group, ":", e
def simple_node(request): simple_node = elliptics.Node( elliptics.Logger("client.log", elliptics.log_level.debug)) simple_node.add_remotes(request.config.option.remotes) return simple_node