def make_elliptics_node(): log = elliptics.Logger('/tmp/ell-namespace-convert.log', config["dnet_log_mask"]) node_config = elliptics.Config() meta_node = elliptics.Node(log, node_config) addresses = [elliptics.Address(host=str(node[0]), port=node[1], family=node[2]) for node in config["metadata"]["nodes"]] logger.info('Connecting to meta nodes: {0}'.format(config["metadata"]["nodes"])) meta_wait_timeout = config['metadata'].get('wait_timeout', 5) try: meta_node.add_remotes(addresses) except Exception as e: logger.error('Failed to connect to any elliptics meta storage node: {0}'.format( e)) raise ValueError('Failed to connect to any elliptics storage META node') meta_session = elliptics.Session(meta_node) meta_session.set_timeout(meta_wait_timeout) meta_session.add_groups(list(config["metadata"]["groups"])) nodes = config.get('elliptics', {}).get('nodes', []) or config["elliptics_nodes"] logger.debug("config elliptics nodes: %s" % str(nodes)) node_config = elliptics.Config() node_config.io_thread_num = config.get('io_thread_num', 1) node_config.nonblocking_io_thread_num = config.get('nonblocking_io_thread_num', 1) node_config.net_thread_num = config.get('net_thread_num', 1) logger.info('Node config: io_thread_num {0}, nonblocking_io_thread_num {1}, ' 'net_thread_num {2}'.format(node_config.io_thread_num, node_config.nonblocking_io_thread_num, node_config.net_thread_num)) n = elliptics.Node(log, node_config) addresses = [elliptics.Address(host=str(node[0]), port=node[1], family=node[2]) for node in nodes] try: n.add_remotes(addresses) except Exception as e: logger.error('Failed to connect to any elliptics storage node: {0}'.format( e)) raise ValueError('Failed to connect to any elliptics storage node') n.meta_session = meta_session wait_timeout = config.get('elliptics', {}).get('wait_timeout', 5) s = elliptics.Session(n) s.set_timeout(wait_timeout) print 'sleeping for wait timeout: {0} seconds'.format(wait_timeout) time.sleep(wait_timeout) return n
def make_meta_session(): log = elliptics.Logger('/tmp/ell-namespace-convert.log', config["dnet_log_mask"]) node_config = elliptics.Config() meta_node = elliptics.Node(log, node_config) addresses = [ elliptics.Address(host=str(node[0]), port=node[1], family=node[2]) for node in config["metadata"]["nodes"] ] logger.info('Connecting to meta nodes: {0}'.format( config["metadata"]["nodes"])) meta_wait_timeout = config['metadata'].get('wait_timeout', 5) try: meta_node.add_remotes(addresses) except Exception as e: logger.error( 'Failed to connect to any elliptics meta storage node: {0}'.format( e)) raise ValueError( 'Failed to connect to any elliptics storage META node') meta_session = elliptics.Session(meta_node) meta_session.set_timeout(meta_wait_timeout) meta_session.add_groups(list(config["metadata"]["groups"])) time.sleep(5) return meta_session
def make_meta_session(): log = elliptics.Logger('/tmp/ell-namespace-convert.log', config["dnet_log_mask"]) node_config = elliptics.Config() meta_node = elliptics.Node(log, node_config) nodes = config['metadata']['nodes'] addresses = [ elliptics.Address(host=host, port=port, family=family) for (host, port, family) in nodes ] logger.info('Connecting to meta nodes: {0}'.format(nodes)) try: meta_node.add_remotes(addresses) except Exception: raise ValueError( 'Failed to connect to any elliptics storage META node') meta_session = elliptics.Session(meta_node) meta_wait_timeout = config['metadata'].get('wait_timeout', 5) meta_session.set_timeout(meta_wait_timeout) meta_session.add_groups(list(config["metadata"]["groups"])) time.sleep(meta_wait_timeout) return meta_session
def elliptics_create_node(address=None, elog=None, wait_timeout=3600, check_timeout=60, flags=0, io_thread_num=1, net_thread_num=1, nonblocking_io_thread_num=1): """ Connects to elliptics cloud """ log.info("Creating node using: {0}".format(address)) cfg = elliptics.Config() cfg.config.wait_timeout = wait_timeout cfg.config.check_timeout = check_timeout cfg.config.flags = flags cfg.config.io_thread_num = io_thread_num cfg.config.nonblocking_io_thread_num = nonblocking_io_thread_num cfg.config.net_thread_num = net_thread_num node = elliptics.Node(elog, cfg) node.add_remote(addr=address.host, port=address.port, family=address.family) log.info("Created node: {0}".format(node)) return node
def elliptics_create_node(address=None, elog=None, wait_timeout=3600, check_timeout=60, flags=0, io_thread_num=1, net_thread_num=1, nonblocking_io_thread_num=1, remotes=[]): """ Connects to elliptics cloud """ log.debug( "Creating node using: {0}, wait_timeout: {1}, remotes: {2}".format( address, wait_timeout, remotes)) cfg = elliptics.Config() cfg.config.wait_timeout = wait_timeout cfg.config.check_timeout = check_timeout cfg.config.flags = flags cfg.config.io_thread_num = io_thread_num cfg.config.nonblocking_io_thread_num = nonblocking_io_thread_num cfg.config.net_thread_num = net_thread_num node = elliptics.Node(elog, cfg) node.add_remotes([address] + remotes) log.debug("Created node: {0}".format(node)) return node
def __init__(self, config): cfg = elliptics.Config() # The parameter which sets the time to wait for the operation complete cfg.config.wait_timeout = config.get("wait-timeout", 60) # The parameter which sets the timeout for pinging node cfg.config.check_timeout = config.get("check_timeout", 60) # Number of IO threads in processing pool cfg.config.io_thread_num = config.get("io-thread-num", 2) # Number of threads in network processing pool cfg.config.net_thread_num = config.get("net-thread-num", 2) # Number of IO threads in processing pool dedicated to nonblocking ops nonblock_io_threads = config.get("nonblocking_io_thread_num", 2) cfg.config.nonblocking_io_thread_num = nonblock_io_threads groups = config.get('groups', []) if len(groups) == 0: raise ValueError("Specify groups") # loglevel of elliptics logger elliptics_log_level = config.get('verbosity', 0) # path to logfile elliptics_log_file = config.get('logfile', '/dev/stderr') log = elliptics.Logger(elliptics_log_file, elliptics_log_level) self._elliptics_node = elliptics.Node(log, cfg) for host, port in config.get('nodes').iteritems(): self._elliptics_node.add_remote(host, port) self._session = elliptics.Session(self._elliptics_node) self._session.groups = groups self._session.set_namespace(NAMESPACE)
def __init__(self, config): cfg = elliptics.Config() log = elliptics.Logger("/dev/stderr", config.get('verbosity', 0)) cfg.config.wait_timeout = config.get("wait-timeout", 60) cfg.config.io_thread_num = config.get("io-thread-num", 1) cfg.config.net_thread_num = config.get("net-thread-num", 1) cfg.config.groups = config.get('groups', []) self._elliptics_node = elliptics.Node(log, cfg) for host, port in config.get('nodes').iteritems(): self._elliptics_node.add_remote(host, port) self._session = elliptics.Session(self._elliptics_node) self._session.groups = config.get('groups', []) if len(self._session.groups) == 0: raise ValueError("Specify groups") self._session.set_namespace(NAMESPACE)
def combine_logs(remotes, groups, min_write, keys, new_key): elog = elliptics.Logger("/dev/stderr", 0) cfg = elliptics.Config() cfg.config.wait_timeout = 60 cfg.config.check_timeout = 60 cfg.config.io_thread_num = 16 cfg.config.nonblocking_io_thread_num = 16 cfg.config.net_thread_num = 16 node = elliptics.Node(elog, cfg) for r in remotes: try: node.add_remote(addr=r[0], port=r[1], family=r[2]) except Exception as e: print "Coudn't connect to elliptics node: {0}: {1}".format(r, e) log_s = elliptics.Session(node) log_s.set_groups(groups) log_s.set_ioflags(elliptics.io_flags.append) index_s = elliptics.Session(node) index_s.set_groups(groups) index_s.set_ioflags(elliptics.io_flags.cache) users = Set() print "Keys: {0}".format(keys) for key in keys: try: users.update(process_key(key, log_s, index_s, new_key)) except Exception as e: print "Process key failed: {0}".format(e) print "Users: {0}".format(users) for u in users: try: index_s.update_indexes(elliptics.Id(u), [new_key + ".0"], [u]) except Exception as e: print "Update_indexes failed: {0}".format(e)
def __init__(self, config): cfg = elliptics.Config() # The parameter which sets the time to wait for the operation complete cfg.config.wait_timeout = config.get("elliptics_wait_timeout", 60) # The parameter which sets the timeout for pinging node cfg.config.check_timeout = config.get("elliptics_check_timeout", 60) # Number of IO threads in processing pool cfg.config.io_thread_num = config.get("elliptics_io_thread_num", 2) # Number of threads in network processing pool cfg.config.net_thread_num = config.get("elliptics_net_thread_num", 2) # Number of IO threads in processing pool dedicated to nonblocking ops nblock_iothreads = config.get("elliptics_nonblocking_io_thread_num", 2) cfg.config.nonblocking_io_thread_num = nblock_iothreads self.groups = config.get('elliptics_groups', []) if len(self.groups) == 0: raise ValueError("Specify groups") # loglevel of elliptics logger elliptics_log_level = config.get('elliptics_verbosity', 0) # path to logfile elliptics_log_file = config.get('elliptics_logfile', '/dev/stderr') log = elliptics.Logger(elliptics_log_file, elliptics_log_level) self._elliptics_node = elliptics.Node(log, cfg) self.namespace = config.get('elliptics_namespace', DEFAULT_NAMESPACE) logger.info("Using namespace %s", self.namespace) at_least_one = False for host, port in config.get('elliptics_nodes').iteritems(): try: self._elliptics_node.add_remote( host, port, config.get('elliptics_addr_family', 2)) at_least_one = True except Exception as err: logger.error("Failed to add remote %s:%d %s", host, port, err) if not at_least_one: raise Exception("Unable to connect to Elliptics")
# (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # ============================================================================= import sys import elliptics import binascii from pprint import pprint try: log = elliptics.Logger("/dev/stderr", 31) cfg = elliptics.Config() cfg.cookie = "0123456789012345678901234567890123456789" cfg.config.wait_timeout = 60 n = elliptics.Node(log, cfg) n.add_remotes("localhost:1025:2") s = elliptics.Session(n) s.add_groups([1, 2, 3]) group = 1 try: obj = "qwerty.xml" addr = s.lookup_address(obj, group)
def term_handler(signo, frame): # required to guarantee execution of cleanup functions registered # with atexit.register sys.exit(0) signal.signal(signal.SIGTERM, term_handler) nodes = config.get('elliptics', {}).get('nodes', []) or config["elliptics_nodes"] logger.debug("config: %s" % str(nodes)) logger.info("trace %d" % (i.next())) log = elliptics.Logger(str(config["dnet_log"]), config["dnet_log_mask"]) node_config = elliptics.Config() node_config.io_thread_num = config.get('io_thread_num', 1) node_config.nonblocking_io_thread_num = config.get('nonblocking_io_thread_num', 1) node_config.net_thread_num = config.get('net_thread_num', 1) logger.info('Node config: io_thread_num {0}, nonblocking_io_thread_num {1}, ' 'net_thread_num {2}'.format(node_config.io_thread_num, node_config.nonblocking_io_thread_num, node_config.net_thread_num)) n = elliptics.Node(log, node_config) logger.info("trace %d" % (i.next())) addresses = []
def cfg(): import elliptics return elliptics.Config()
def init_elliptics_node(): nodes = config.get('elliptics', {}).get('nodes', []) or config["elliptics_nodes"] logger.debug("config: %s" % str(nodes)) log = elliptics.Logger(str(config["dnet_log"]), config["dnet_log_mask"]) node_config = elliptics.Config() node_config.io_thread_num = config.get('io_thread_num', 1) node_config.nonblocking_io_thread_num = config.get('nonblocking_io_thread_num', 1) node_config.net_thread_num = config.get('net_thread_num', 1) logger.info('Node config: io_thread_num {0}, nonblocking_io_thread_num {1}, ' 'net_thread_num {2}'.format( node_config.io_thread_num, node_config.nonblocking_io_thread_num, node_config.net_thread_num)) n = elliptics.Node(log, node_config) addresses = [] for node in nodes: try: addresses.append(elliptics.Address( host=str(node[0]), port=node[1], family=node[2])) except Exception as e: logger.error('Failed to connect to storage node: {0}:{1}:{2}'.format( node[0], node[1], node[2])) pass try: n.add_remotes(addresses) except Exception as e: logger.error('Failed to connect to any elliptics storage node: {0}'.format( e)) raise ValueError('Failed to connect to any elliptics storage node') meta_node = elliptics.Node(log, node_config) addresses = [] for node in config["metadata"]["nodes"]: try: addresses.append(elliptics.Address( host=str(node[0]), port=node[1], family=node[2])) except Exception as e: logger.error('Failed to connect to meta node: {0}:{1}:{2}'.format( node[0], node[1], node[2])) pass logger.info('Connecting to meta nodes: {0}'.format(config["metadata"]["nodes"])) try: meta_node.add_remotes(addresses) except Exception as e: logger.error('Failed to connect to any elliptics meta storage node: {0}'.format( e)) raise ValueError('Failed to connect to any elliptics storage META node') meta_wait_timeout = config['metadata'].get('wait_timeout', 5) meta_session = elliptics.Session(meta_node) meta_session.set_timeout(meta_wait_timeout) meta_session.add_groups(list(config["metadata"]["groups"])) n.meta_session = meta_session wait_timeout = config.get('elliptics', {}).get('wait_timeout', 5) time.sleep(wait_timeout) return n