def make_elliptics_node(): log = elliptics.Logger('/tmp/ell-namespace-convert.log', config["dnet_log_mask"]) node_config = elliptics.Config() meta_node = elliptics.Node(log, node_config) addresses = [elliptics.Address(host=str(node[0]), port=node[1], family=node[2]) for node in config["metadata"]["nodes"]] logger.info('Connecting to meta nodes: {0}'.format(config["metadata"]["nodes"])) meta_wait_timeout = config['metadata'].get('wait_timeout', 5) try: meta_node.add_remotes(addresses) except Exception as e: logger.error('Failed to connect to any elliptics meta storage node: {0}'.format( e)) raise ValueError('Failed to connect to any elliptics storage META node') meta_session = elliptics.Session(meta_node) meta_session.set_timeout(meta_wait_timeout) meta_session.add_groups(list(config["metadata"]["groups"])) nodes = config.get('elliptics', {}).get('nodes', []) or config["elliptics_nodes"] logger.debug("config elliptics nodes: %s" % str(nodes)) node_config = elliptics.Config() node_config.io_thread_num = config.get('io_thread_num', 1) node_config.nonblocking_io_thread_num = config.get('nonblocking_io_thread_num', 1) node_config.net_thread_num = config.get('net_thread_num', 1) logger.info('Node config: io_thread_num {0}, nonblocking_io_thread_num {1}, ' 'net_thread_num {2}'.format(node_config.io_thread_num, node_config.nonblocking_io_thread_num, node_config.net_thread_num)) n = elliptics.Node(log, node_config) addresses = [elliptics.Address(host=str(node[0]), port=node[1], family=node[2]) for node in nodes] try: n.add_remotes(addresses) except Exception as e: logger.error('Failed to connect to any elliptics storage node: {0}'.format( e)) raise ValueError('Failed to connect to any elliptics storage node') n.meta_session = meta_session wait_timeout = config.get('elliptics', {}).get('wait_timeout', 5) s = elliptics.Session(n) s.set_timeout(wait_timeout) print 'sleeping for wait timeout: {0} seconds'.format(wait_timeout) time.sleep(wait_timeout) return n
def make_meta_session(): log = elliptics.Logger('/tmp/ell-namespace-convert.log', config["dnet_log_mask"]) node_config = elliptics.Config() meta_node = elliptics.Node(log, node_config) addresses = [ elliptics.Address(host=str(node[0]), port=node[1], family=node[2]) for node in config["metadata"]["nodes"] ] logger.info('Connecting to meta nodes: {0}'.format( config["metadata"]["nodes"])) meta_wait_timeout = config['metadata'].get('wait_timeout', 5) try: meta_node.add_remotes(addresses) except Exception as e: logger.error( 'Failed to connect to any elliptics meta storage node: {0}'.format( e)) raise ValueError( 'Failed to connect to any elliptics storage META node') meta_session = elliptics.Session(meta_node) meta_session.set_timeout(meta_wait_timeout) meta_session.add_groups(list(config["metadata"]["groups"])) time.sleep(5) return meta_session
def make_meta_session(): log = elliptics.Logger('/tmp/ell-namespace-convert.log', config["dnet_log_mask"]) node_config = elliptics.Config() meta_node = elliptics.Node(log, node_config) nodes = config['metadata']['nodes'] addresses = [ elliptics.Address(host=host, port=port, family=family) for (host, port, family) in nodes ] logger.info('Connecting to meta nodes: {0}'.format(nodes)) try: meta_node.add_remotes(addresses) except Exception: raise ValueError( 'Failed to connect to any elliptics storage META node') meta_session = elliptics.Session(meta_node) meta_wait_timeout = config['metadata'].get('wait_timeout', 5) meta_session.set_timeout(meta_wait_timeout) meta_session.add_groups(list(config["metadata"]["groups"])) time.sleep(meta_wait_timeout) return meta_session
def load(cls, data): return cls(elliptics.Address(data[0][0], data[0][1], data[0][2]), data[1], elliptics.Time(data[2][0], data[2][1]), data[3], data[4], data[5])
def monitor_stats(self, groups=None): if groups: hosts = set((nb.node.host.addr, nb.node.port, nb.node.family) for g in groups for nb in g.node_backends) host_addrs = [elliptics.Address(*host) for host in hosts] else: logger.info('Before calculating routes') host_addrs = self.__session.routes.addresses() logger.info('Unique routes calculated') for ha in host_addrs: node_addr = '{host}:{port}'.format( host=ha.host, port=ha.port, ) if ha.host not in storage.hosts: logger.debug('Adding host {}'.format(ha.host)) host = storage.hosts.add(ha.host) else: host = storage.hosts[ha.host] if node_addr not in storage.nodes: logger.debug('Adding node {}'.format(node_addr)) storage.nodes.add(host, ha.port, ha.family) responses_collected = 0 for node, result in self._do_get_monitor_stats(host_addrs): responses_collected += 1 self.update_statistics(node, result['content'], elapsed_time=result['request_time']) logger.info( 'Number of hosts in route table: {}, responses collected {}'. format( len(host_addrs), responses_collected, )) # TODO: can we use iterkeys? nbs = (groups and [nb for g in groups for nb in g.node_backends] or storage.node_backends.keys()) for nb in nbs: nb.update_statistics_status() nb.update_status() # TODO: can we use iterkeys? fss = (groups and set(nb.fs for nb in nbs) or storage.fs.keys()) for fs in fss: fs.update_status() for group in groups or storage.groups.keys(): logger.info('Updating status for group {0}'.format(group.group_id)) group.update_status() if groups is None: storage.dc_host_view.update() load_manager.update(storage)
node_config.net_thread_num = config.get('net_thread_num', 1) logger.info('Node config: io_thread_num {0}, nonblocking_io_thread_num {1}, ' 'net_thread_num {2}'.format(node_config.io_thread_num, node_config.nonblocking_io_thread_num, node_config.net_thread_num)) n = elliptics.Node(log, node_config) logger.info("trace %d" % (i.next())) addresses = [] for node in nodes: try: addresses.append( elliptics.Address(host=str(node[0]), port=node[1], family=node[2])) except Exception as e: logger.error('Failed to connect to storage node: {0}:{1}:{2}'.format( node[0], node[1], node[2])) pass try: n.add_remotes(addresses) except Exception as e: logger.error( 'Failed to connect to any elliptics storage node: {0}'.format(e)) raise ValueError('Failed to connect to any elliptics storage node') logger.info("trace %d" % (i.next())) meta_node = elliptics.Node(log, node_config)
'Node config: io_thread_num {0}, nonblocking_io_thread_num {1}, ' 'net_thread_num {2}'.format( node_config.io_thread_num, node_config.nonblocking_io_thread_num, node_config.net_thread_num ) ) n = elliptics.Node(log, node_config) logger.info("trace %d" % (i.next())) addresses = [] for node in nodes: try: addresses.append(elliptics.Address( host=str(node[0]), port=node[1], family=node[2])) except Exception as e: logger.error('Failed to connect to storage node: {0}:{1}:{2}'.format( node[0], node[1], node[2])) pass try: n.add_remotes(addresses) except Exception as e: logger.error('Failed to connect to any elliptics storage node: {0}'.format( e)) raise ValueError('Failed to connect to any elliptics storage node') logger.info("trace %d" % (i.next())) meta_node = elliptics.Node(log, node_config)
def init_elliptics_node(): nodes = config.get('elliptics', {}).get('nodes', []) or config["elliptics_nodes"] logger.debug("config: %s" % str(nodes)) log = elliptics.Logger(str(config["dnet_log"]), config["dnet_log_mask"]) node_config = elliptics.Config() node_config.io_thread_num = config.get('io_thread_num', 1) node_config.nonblocking_io_thread_num = config.get('nonblocking_io_thread_num', 1) node_config.net_thread_num = config.get('net_thread_num', 1) logger.info('Node config: io_thread_num {0}, nonblocking_io_thread_num {1}, ' 'net_thread_num {2}'.format( node_config.io_thread_num, node_config.nonblocking_io_thread_num, node_config.net_thread_num)) n = elliptics.Node(log, node_config) addresses = [] for node in nodes: try: addresses.append(elliptics.Address( host=str(node[0]), port=node[1], family=node[2])) except Exception as e: logger.error('Failed to connect to storage node: {0}:{1}:{2}'.format( node[0], node[1], node[2])) pass try: n.add_remotes(addresses) except Exception as e: logger.error('Failed to connect to any elliptics storage node: {0}'.format( e)) raise ValueError('Failed to connect to any elliptics storage node') meta_node = elliptics.Node(log, node_config) addresses = [] for node in config["metadata"]["nodes"]: try: addresses.append(elliptics.Address( host=str(node[0]), port=node[1], family=node[2])) except Exception as e: logger.error('Failed to connect to meta node: {0}:{1}:{2}'.format( node[0], node[1], node[2])) pass logger.info('Connecting to meta nodes: {0}'.format(config["metadata"]["nodes"])) try: meta_node.add_remotes(addresses) except Exception as e: logger.error('Failed to connect to any elliptics meta storage node: {0}'.format( e)) raise ValueError('Failed to connect to any elliptics storage META node') meta_wait_timeout = config['metadata'].get('wait_timeout', 5) meta_session = elliptics.Session(meta_node) meta_session.set_timeout(meta_wait_timeout) meta_session.add_groups(list(config["metadata"]["groups"])) n.meta_session = meta_session wait_timeout = config.get('elliptics', {}).get('wait_timeout', 5) time.sleep(wait_timeout) return n