def make_elliptics_node():
log = elliptics.Logger('/tmp/ell-namespace-convert.log', config["dnet_log_mask"])
node_config = elliptics.Config()
meta_node = elliptics.Node(log, node_config)
addresses = [elliptics.Address(host=str(node[0]), port=node[1], family=node[2])
for node in config["metadata"]["nodes"]]
logger.info('Connecting to meta nodes: {0}'.format(config["metadata"]["nodes"]))
meta_wait_timeout = config['metadata'].get('wait_timeout', 5)
try:
meta_node.add_remotes(addresses)
except Exception as e:
logger.error('Failed to connect to any elliptics meta storage node: {0}'.format(
e))
raise ValueError('Failed to connect to any elliptics storage META node')
meta_session = elliptics.Session(meta_node)
meta_session.set_timeout(meta_wait_timeout)
meta_session.add_groups(list(config["metadata"]["groups"]))
nodes = config.get('elliptics', {}).get('nodes', []) or config["elliptics_nodes"]
logger.debug("config elliptics nodes: %s" % str(nodes))
node_config = elliptics.Config()
node_config.io_thread_num = config.get('io_thread_num', 1)
node_config.nonblocking_io_thread_num = config.get('nonblocking_io_thread_num', 1)
node_config.net_thread_num = config.get('net_thread_num', 1)
logger.info('Node config: io_thread_num {0}, nonblocking_io_thread_num {1}, '
'net_thread_num {2}'.format(node_config.io_thread_num, node_config.nonblocking_io_thread_num,
node_config.net_thread_num))
n = elliptics.Node(log, node_config)
addresses = [elliptics.Address(host=str(node[0]), port=node[1], family=node[2])
for node in nodes]
try:
n.add_remotes(addresses)
except Exception as e:
logger.error('Failed to connect to any elliptics storage node: {0}'.format(
e))
raise ValueError('Failed to connect to any elliptics storage node')
n.meta_session = meta_session
wait_timeout = config.get('elliptics', {}).get('wait_timeout', 5)
s = elliptics.Session(n)
s.set_timeout(wait_timeout)
print 'sleeping for wait timeout: {0} seconds'.format(wait_timeout)
time.sleep(wait_timeout)
return n
def make_meta_session():
log = elliptics.Logger('/tmp/ell-namespace-convert.log',
config["dnet_log_mask"])
node_config = elliptics.Config()
meta_node = elliptics.Node(log, node_config)
addresses = [
elliptics.Address(host=str(node[0]), port=node[1], family=node[2])
for node in config["metadata"]["nodes"]
]
logger.info('Connecting to meta nodes: {0}'.format(
config["metadata"]["nodes"]))
meta_wait_timeout = config['metadata'].get('wait_timeout', 5)
try:
meta_node.add_remotes(addresses)
except Exception as e:
logger.error(
'Failed to connect to any elliptics meta storage node: {0}'.format(
e))
raise ValueError(
'Failed to connect to any elliptics storage META node')
meta_session = elliptics.Session(meta_node)
meta_session.set_timeout(meta_wait_timeout)
meta_session.add_groups(list(config["metadata"]["groups"]))
time.sleep(5)
return meta_session
def make_meta_session():
log = elliptics.Logger('/tmp/ell-namespace-convert.log',
config["dnet_log_mask"])
node_config = elliptics.Config()
meta_node = elliptics.Node(log, node_config)
nodes = config['metadata']['nodes']
addresses = [
elliptics.Address(host=host, port=port, family=family)
for (host, port, family) in nodes
]
logger.info('Connecting to meta nodes: {0}'.format(nodes))
try:
meta_node.add_remotes(addresses)
except Exception:
raise ValueError(
'Failed to connect to any elliptics storage META node')
meta_session = elliptics.Session(meta_node)
meta_wait_timeout = config['metadata'].get('wait_timeout', 5)
meta_session.set_timeout(meta_wait_timeout)
meta_session.add_groups(list(config["metadata"]["groups"]))
time.sleep(meta_wait_timeout)
return meta_session
def elliptics_create_node(address=None,
elog=None,
wait_timeout=3600,
check_timeout=60,
flags=0,
io_thread_num=1,
net_thread_num=1,
nonblocking_io_thread_num=1):
"""
Connects to elliptics cloud
"""
log.info("Creating node using: {0}".format(address))
cfg = elliptics.Config()
cfg.config.wait_timeout = wait_timeout
cfg.config.check_timeout = check_timeout
cfg.config.flags = flags
cfg.config.io_thread_num = io_thread_num
cfg.config.nonblocking_io_thread_num = nonblocking_io_thread_num
cfg.config.net_thread_num = net_thread_num
node = elliptics.Node(elog, cfg)
node.add_remote(addr=address.host,
port=address.port,
family=address.family)
log.info("Created node: {0}".format(node))
return node
def elliptics_create_node(address=None,
elog=None,
wait_timeout=3600,
check_timeout=60,
flags=0,
io_thread_num=1,
net_thread_num=1,
nonblocking_io_thread_num=1,
remotes=[]):
"""
Connects to elliptics cloud
"""
log.debug(
"Creating node using: {0}, wait_timeout: {1}, remotes: {2}".format(
address, wait_timeout, remotes))
cfg = elliptics.Config()
cfg.config.wait_timeout = wait_timeout
cfg.config.check_timeout = check_timeout
cfg.config.flags = flags
cfg.config.io_thread_num = io_thread_num
cfg.config.nonblocking_io_thread_num = nonblocking_io_thread_num
cfg.config.net_thread_num = net_thread_num
node = elliptics.Node(elog, cfg)
node.add_remotes([address] + remotes)
log.debug("Created node: {0}".format(node))
return node
def __init__(self, config):
cfg = elliptics.Config()
# The parameter which sets the time to wait for the operation complete
cfg.config.wait_timeout = config.get("wait-timeout", 60)
# The parameter which sets the timeout for pinging node
cfg.config.check_timeout = config.get("check_timeout", 60)
# Number of IO threads in processing pool
cfg.config.io_thread_num = config.get("io-thread-num", 2)
# Number of threads in network processing pool
cfg.config.net_thread_num = config.get("net-thread-num", 2)
# Number of IO threads in processing pool dedicated to nonblocking ops
nonblock_io_threads = config.get("nonblocking_io_thread_num", 2)
cfg.config.nonblocking_io_thread_num = nonblock_io_threads
groups = config.get('groups', [])
if len(groups) == 0:
raise ValueError("Specify groups")
# loglevel of elliptics logger
elliptics_log_level = config.get('verbosity', 0)
# path to logfile
elliptics_log_file = config.get('logfile', '/dev/stderr')
log = elliptics.Logger(elliptics_log_file, elliptics_log_level)
self._elliptics_node = elliptics.Node(log, cfg)
for host, port in config.get('nodes').iteritems():
self._elliptics_node.add_remote(host, port)
self._session = elliptics.Session(self._elliptics_node)
self._session.groups = groups
self._session.set_namespace(NAMESPACE)
def __init__(self, config):
cfg = elliptics.Config()
log = elliptics.Logger("/dev/stderr", config.get('verbosity', 0))
cfg.config.wait_timeout = config.get("wait-timeout", 60)
cfg.config.io_thread_num = config.get("io-thread-num", 1)
cfg.config.net_thread_num = config.get("net-thread-num", 1)
cfg.config.groups = config.get('groups', [])
self._elliptics_node = elliptics.Node(log, cfg)
for host, port in config.get('nodes').iteritems():
self._elliptics_node.add_remote(host, port)
self._session = elliptics.Session(self._elliptics_node)
self._session.groups = config.get('groups', [])
if len(self._session.groups) == 0:
raise ValueError("Specify groups")
self._session.set_namespace(NAMESPACE)
def combine_logs(remotes, groups, min_write, keys, new_key):
elog = elliptics.Logger("/dev/stderr", 0)
cfg = elliptics.Config()
cfg.config.wait_timeout = 60
cfg.config.check_timeout = 60
cfg.config.io_thread_num = 16
cfg.config.nonblocking_io_thread_num = 16
cfg.config.net_thread_num = 16
node = elliptics.Node(elog, cfg)
for r in remotes:
try:
node.add_remote(addr=r[0], port=r[1], family=r[2])
except Exception as e:
print "Coudn't connect to elliptics node: {0}: {1}".format(r, e)
log_s = elliptics.Session(node)
log_s.set_groups(groups)
log_s.set_ioflags(elliptics.io_flags.append)
index_s = elliptics.Session(node)
index_s.set_groups(groups)
index_s.set_ioflags(elliptics.io_flags.cache)
users = Set()
print "Keys: {0}".format(keys)
for key in keys:
try:
users.update(process_key(key, log_s, index_s, new_key))
except Exception as e:
print "Process key failed: {0}".format(e)
print "Users: {0}".format(users)
for u in users:
try:
index_s.update_indexes(elliptics.Id(u), [new_key + ".0"], [u])
except Exception as e:
print "Update_indexes failed: {0}".format(e)
def __init__(self, config):
cfg = elliptics.Config()
# The parameter which sets the time to wait for the operation complete
cfg.config.wait_timeout = config.get("elliptics_wait_timeout", 60)
# The parameter which sets the timeout for pinging node
cfg.config.check_timeout = config.get("elliptics_check_timeout", 60)
# Number of IO threads in processing pool
cfg.config.io_thread_num = config.get("elliptics_io_thread_num", 2)
# Number of threads in network processing pool
cfg.config.net_thread_num = config.get("elliptics_net_thread_num", 2)
# Number of IO threads in processing pool dedicated to nonblocking ops
nblock_iothreads = config.get("elliptics_nonblocking_io_thread_num", 2)
cfg.config.nonblocking_io_thread_num = nblock_iothreads
self.groups = config.get('elliptics_groups', [])
if len(self.groups) == 0:
raise ValueError("Specify groups")
# loglevel of elliptics logger
elliptics_log_level = config.get('elliptics_verbosity', 0)
# path to logfile
elliptics_log_file = config.get('elliptics_logfile', '/dev/stderr')
log = elliptics.Logger(elliptics_log_file, elliptics_log_level)
self._elliptics_node = elliptics.Node(log, cfg)
self.namespace = config.get('elliptics_namespace', DEFAULT_NAMESPACE)
logger.info("Using namespace %s", self.namespace)
at_least_one = False
for host, port in config.get('elliptics_nodes').iteritems():
try:
self._elliptics_node.add_remote(
host, port, config.get('elliptics_addr_family', 2))
at_least_one = True
except Exception as err:
logger.error("Failed to add remote %s:%d %s", host, port, err)
if not at_least_one:
raise Exception("Unable to connect to Elliptics")
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# =============================================================================
import sys
import elliptics
import binascii
from pprint import pprint
try:
log = elliptics.Logger("/dev/stderr", 31)
cfg = elliptics.Config()
cfg.cookie = "0123456789012345678901234567890123456789"
cfg.config.wait_timeout = 60
n = elliptics.Node(log, cfg)
n.add_remotes("localhost:1025:2")
s = elliptics.Session(n)
s.add_groups([1, 2, 3])
group = 1
try:
obj = "qwerty.xml"
addr = s.lookup_address(obj, group)
def term_handler(signo, frame):
# required to guarantee execution of cleanup functions registered
# with atexit.register
sys.exit(0)
signal.signal(signal.SIGTERM, term_handler)
nodes = config.get('elliptics', {}).get('nodes',
[]) or config["elliptics_nodes"]
logger.debug("config: %s" % str(nodes))
logger.info("trace %d" % (i.next()))
log = elliptics.Logger(str(config["dnet_log"]), config["dnet_log_mask"])
node_config = elliptics.Config()
node_config.io_thread_num = config.get('io_thread_num', 1)
node_config.nonblocking_io_thread_num = config.get('nonblocking_io_thread_num',
1)
node_config.net_thread_num = config.get('net_thread_num', 1)
logger.info('Node config: io_thread_num {0}, nonblocking_io_thread_num {1}, '
'net_thread_num {2}'.format(node_config.io_thread_num,
node_config.nonblocking_io_thread_num,
node_config.net_thread_num))
n = elliptics.Node(log, node_config)
logger.info("trace %d" % (i.next()))
addresses = []
def cfg():
import elliptics
return elliptics.Config()
def init_elliptics_node():
nodes = config.get('elliptics', {}).get('nodes', []) or config["elliptics_nodes"]
logger.debug("config: %s" % str(nodes))
log = elliptics.Logger(str(config["dnet_log"]), config["dnet_log_mask"])
node_config = elliptics.Config()
node_config.io_thread_num = config.get('io_thread_num', 1)
node_config.nonblocking_io_thread_num = config.get('nonblocking_io_thread_num', 1)
node_config.net_thread_num = config.get('net_thread_num', 1)
logger.info('Node config: io_thread_num {0}, nonblocking_io_thread_num {1}, '
'net_thread_num {2}'.format(
node_config.io_thread_num, node_config.nonblocking_io_thread_num,
node_config.net_thread_num))
n = elliptics.Node(log, node_config)
addresses = []
for node in nodes:
try:
addresses.append(elliptics.Address(
host=str(node[0]), port=node[1], family=node[2]))
except Exception as e:
logger.error('Failed to connect to storage node: {0}:{1}:{2}'.format(
node[0], node[1], node[2]))
pass
try:
n.add_remotes(addresses)
except Exception as e:
logger.error('Failed to connect to any elliptics storage node: {0}'.format(
e))
raise ValueError('Failed to connect to any elliptics storage node')
meta_node = elliptics.Node(log, node_config)
addresses = []
for node in config["metadata"]["nodes"]:
try:
addresses.append(elliptics.Address(
host=str(node[0]), port=node[1], family=node[2]))
except Exception as e:
logger.error('Failed to connect to meta node: {0}:{1}:{2}'.format(
node[0], node[1], node[2]))
pass
logger.info('Connecting to meta nodes: {0}'.format(config["metadata"]["nodes"]))
try:
meta_node.add_remotes(addresses)
except Exception as e:
logger.error('Failed to connect to any elliptics meta storage node: {0}'.format(
e))
raise ValueError('Failed to connect to any elliptics storage META node')
meta_wait_timeout = config['metadata'].get('wait_timeout', 5)
meta_session = elliptics.Session(meta_node)
meta_session.set_timeout(meta_wait_timeout)
meta_session.add_groups(list(config["metadata"]["groups"]))
n.meta_session = meta_session
wait_timeout = config.get('elliptics', {}).get('wait_timeout', 5)
time.sleep(wait_timeout)
return n
