def make_elliptics_node():
log = elliptics.Logger('/tmp/ell-namespace-convert.log', config["dnet_log_mask"])
node_config = elliptics.Config()
meta_node = elliptics.Node(log, node_config)
addresses = [elliptics.Address(host=str(node[0]), port=node[1], family=node[2])
for node in config["metadata"]["nodes"]]
logger.info('Connecting to meta nodes: {0}'.format(config["metadata"]["nodes"]))
meta_wait_timeout = config['metadata'].get('wait_timeout', 5)
try:
meta_node.add_remotes(addresses)
except Exception as e:
logger.error('Failed to connect to any elliptics meta storage node: {0}'.format(
e))
raise ValueError('Failed to connect to any elliptics storage META node')
meta_session = elliptics.Session(meta_node)
meta_session.set_timeout(meta_wait_timeout)
meta_session.add_groups(list(config["metadata"]["groups"]))
nodes = config.get('elliptics', {}).get('nodes', []) or config["elliptics_nodes"]
logger.debug("config elliptics nodes: %s" % str(nodes))
node_config = elliptics.Config()
node_config.io_thread_num = config.get('io_thread_num', 1)
node_config.nonblocking_io_thread_num = config.get('nonblocking_io_thread_num', 1)
node_config.net_thread_num = config.get('net_thread_num', 1)
logger.info('Node config: io_thread_num {0}, nonblocking_io_thread_num {1}, '
'net_thread_num {2}'.format(node_config.io_thread_num, node_config.nonblocking_io_thread_num,
node_config.net_thread_num))
n = elliptics.Node(log, node_config)
addresses = [elliptics.Address(host=str(node[0]), port=node[1], family=node[2])
for node in nodes]
try:
n.add_remotes(addresses)
except Exception as e:
logger.error('Failed to connect to any elliptics storage node: {0}'.format(
e))
raise ValueError('Failed to connect to any elliptics storage node')
n.meta_session = meta_session
wait_timeout = config.get('elliptics', {}).get('wait_timeout', 5)
s = elliptics.Session(n)
s.set_timeout(wait_timeout)
print 'sleeping for wait timeout: {0} seconds'.format(wait_timeout)
time.sleep(wait_timeout)
return n
def make_meta_session():
log = elliptics.Logger('/tmp/ell-namespace-convert.log',
config["dnet_log_mask"])
node_config = elliptics.Config()
meta_node = elliptics.Node(log, node_config)
addresses = [
elliptics.Address(host=str(node[0]), port=node[1], family=node[2])
for node in config["metadata"]["nodes"]
]
logger.info('Connecting to meta nodes: {0}'.format(
config["metadata"]["nodes"]))
meta_wait_timeout = config['metadata'].get('wait_timeout', 5)
try:
meta_node.add_remotes(addresses)
except Exception as e:
logger.error(
'Failed to connect to any elliptics meta storage node: {0}'.format(
e))
raise ValueError(
'Failed to connect to any elliptics storage META node')
meta_session = elliptics.Session(meta_node)
meta_session.set_timeout(meta_wait_timeout)
meta_session.add_groups(list(config["metadata"]["groups"]))
time.sleep(5)
return meta_session
def make_meta_session():
log = elliptics.Logger('/tmp/ell-namespace-convert.log',
config["dnet_log_mask"])
node_config = elliptics.Config()
meta_node = elliptics.Node(log, node_config)
nodes = config['metadata']['nodes']
addresses = [
elliptics.Address(host=host, port=port, family=family)
for (host, port, family) in nodes
]
logger.info('Connecting to meta nodes: {0}'.format(nodes))
try:
meta_node.add_remotes(addresses)
except Exception:
raise ValueError(
'Failed to connect to any elliptics storage META node')
meta_session = elliptics.Session(meta_node)
meta_wait_timeout = config['metadata'].get('wait_timeout', 5)
meta_session.set_timeout(meta_wait_timeout)
meta_session.add_groups(list(config["metadata"]["groups"]))
time.sleep(meta_wait_timeout)
return meta_session
def load(cls, data):
return cls(elliptics.Address(data[0][0], data[0][1], data[0][2]),
data[1],
elliptics.Time(data[2][0], data[2][1]),
data[3],
data[4],
data[5])
def monitor_stats(self, groups=None):
if groups:
hosts = set((nb.node.host.addr, nb.node.port, nb.node.family)
for g in groups for nb in g.node_backends)
host_addrs = [elliptics.Address(*host) for host in hosts]
else:
logger.info('Before calculating routes')
host_addrs = self.__session.routes.addresses()
logger.info('Unique routes calculated')
for ha in host_addrs:
node_addr = '{host}:{port}'.format(
host=ha.host,
port=ha.port,
)
if ha.host not in storage.hosts:
logger.debug('Adding host {}'.format(ha.host))
host = storage.hosts.add(ha.host)
else:
host = storage.hosts[ha.host]
if node_addr not in storage.nodes:
logger.debug('Adding node {}'.format(node_addr))
storage.nodes.add(host, ha.port, ha.family)
responses_collected = 0
for node, result in self._do_get_monitor_stats(host_addrs):
responses_collected += 1
self.update_statistics(node,
result['content'],
elapsed_time=result['request_time'])
logger.info(
'Number of hosts in route table: {}, responses collected {}'.
format(
len(host_addrs),
responses_collected,
))
# TODO: can we use iterkeys?
nbs = (groups and [nb for g in groups for nb in g.node_backends]
or storage.node_backends.keys())
for nb in nbs:
nb.update_statistics_status()
nb.update_status()
# TODO: can we use iterkeys?
fss = (groups and set(nb.fs for nb in nbs) or storage.fs.keys())
for fs in fss:
fs.update_status()
for group in groups or storage.groups.keys():
logger.info('Updating status for group {0}'.format(group.group_id))
group.update_status()
if groups is None:
storage.dc_host_view.update()
load_manager.update(storage)
node_config.net_thread_num = config.get('net_thread_num', 1)
logger.info('Node config: io_thread_num {0}, nonblocking_io_thread_num {1}, '
'net_thread_num {2}'.format(node_config.io_thread_num,
node_config.nonblocking_io_thread_num,
node_config.net_thread_num))
n = elliptics.Node(log, node_config)
logger.info("trace %d" % (i.next()))
addresses = []
for node in nodes:
try:
addresses.append(
elliptics.Address(host=str(node[0]), port=node[1], family=node[2]))
except Exception as e:
logger.error('Failed to connect to storage node: {0}:{1}:{2}'.format(
node[0], node[1], node[2]))
pass
try:
n.add_remotes(addresses)
except Exception as e:
logger.error(
'Failed to connect to any elliptics storage node: {0}'.format(e))
raise ValueError('Failed to connect to any elliptics storage node')
logger.info("trace %d" % (i.next()))
meta_node = elliptics.Node(log, node_config)
'Node config: io_thread_num {0}, nonblocking_io_thread_num {1}, '
'net_thread_num {2}'.format(
node_config.io_thread_num,
node_config.nonblocking_io_thread_num,
node_config.net_thread_num
)
)
n = elliptics.Node(log, node_config)
logger.info("trace %d" % (i.next()))
addresses = []
for node in nodes:
try:
addresses.append(elliptics.Address(
host=str(node[0]), port=node[1], family=node[2]))
except Exception as e:
logger.error('Failed to connect to storage node: {0}:{1}:{2}'.format(
node[0], node[1], node[2]))
pass
try:
n.add_remotes(addresses)
except Exception as e:
logger.error('Failed to connect to any elliptics storage node: {0}'.format(
e))
raise ValueError('Failed to connect to any elliptics storage node')
logger.info("trace %d" % (i.next()))
meta_node = elliptics.Node(log, node_config)
def init_elliptics_node():
nodes = config.get('elliptics', {}).get('nodes', []) or config["elliptics_nodes"]
logger.debug("config: %s" % str(nodes))
log = elliptics.Logger(str(config["dnet_log"]), config["dnet_log_mask"])
node_config = elliptics.Config()
node_config.io_thread_num = config.get('io_thread_num', 1)
node_config.nonblocking_io_thread_num = config.get('nonblocking_io_thread_num', 1)
node_config.net_thread_num = config.get('net_thread_num', 1)
logger.info('Node config: io_thread_num {0}, nonblocking_io_thread_num {1}, '
'net_thread_num {2}'.format(
node_config.io_thread_num, node_config.nonblocking_io_thread_num,
node_config.net_thread_num))
n = elliptics.Node(log, node_config)
addresses = []
for node in nodes:
try:
addresses.append(elliptics.Address(
host=str(node[0]), port=node[1], family=node[2]))
except Exception as e:
logger.error('Failed to connect to storage node: {0}:{1}:{2}'.format(
node[0], node[1], node[2]))
pass
try:
n.add_remotes(addresses)
except Exception as e:
logger.error('Failed to connect to any elliptics storage node: {0}'.format(
e))
raise ValueError('Failed to connect to any elliptics storage node')
meta_node = elliptics.Node(log, node_config)
addresses = []
for node in config["metadata"]["nodes"]:
try:
addresses.append(elliptics.Address(
host=str(node[0]), port=node[1], family=node[2]))
except Exception as e:
logger.error('Failed to connect to meta node: {0}:{1}:{2}'.format(
node[0], node[1], node[2]))
pass
logger.info('Connecting to meta nodes: {0}'.format(config["metadata"]["nodes"]))
try:
meta_node.add_remotes(addresses)
except Exception as e:
logger.error('Failed to connect to any elliptics meta storage node: {0}'.format(
e))
raise ValueError('Failed to connect to any elliptics storage META node')
meta_wait_timeout = config['metadata'].get('wait_timeout', 5)
meta_session = elliptics.Session(meta_node)
meta_session.set_timeout(meta_wait_timeout)
meta_session.add_groups(list(config["metadata"]["groups"]))
n.meta_session = meta_session
wait_timeout = config.get('elliptics', {}).get('wait_timeout', 5)
time.sleep(wait_timeout)
return n