def add_server(self, server_enum, timeout=30):
# add new server (collrelay or snmp_relay)
identity_str = process_tools.zmq_identity_str(self.__msg_prefix,
short=True)
zmq_context = self.__process.zmq_context
cur_timeout = timeout
client_send = zmq_context.socket(zmq.PUSH)
client_recv = zmq_context.socket(zmq.SUB)
client_send.setsockopt(zmq.LINGER, cur_timeout * 2)
client_recv.setsockopt_string(zmq.SUBSCRIBE, identity_str)
send_conn_str = "{}".format(
process_tools.get_zmq_ipc_name(
"receiver",
s_name=server_enum.value,
connect_to_root_instance=True,
))
recv_conn_str = "{}".format(
process_tools.get_zmq_ipc_name(
"sender",
s_name=server_enum.value,
connect_to_root_instance=True,
))
self.log("adding server ({} -> {} -> {})".format(
send_conn_str,
server_enum.value,
recv_conn_str,
))
self.__sock_list.append(client_send)
self.__sock_list.append(client_recv)
self.__sock_lut[server_enum] = (client_send, client_recv)
client_send.connect(send_conn_str)
client_recv.connect(recv_conn_str)
self.register_poller(client_recv, zmq.POLLIN, self._handle_message)
def _init_network(self):
conn_str = process_tools.get_zmq_ipc_name(
"vector", s_name="collserver", connect_to_root_instance=True)
vector_socket = self.zmq_context.socket(zmq.PUSH)
vector_socket.setsockopt(zmq.LINGER, 0)
vector_socket.connect(conn_str)
self.vector_socket = vector_socket
def init_connection(self):
if self.args.identity_string:
self.identity_str = self.args.identity_string
else:
self.identity_str = process_tools.zmq_identity_str(
self.args.identity_substring)
s_type = "DEALER" if not self.args.split else "PUSH"
client = self.zmq_context.socket(getattr(zmq, s_type))
client.setsockopt_string(zmq.IDENTITY, self.identity_str)
client.setsockopt(zmq.LINGER, self.args.timeout)
if self.args.protocoll == "ipc":
if self.args.root:
process_tools.ALLOW_MULTIPLE_INSTANCES = False
conn_str = "{}".format(
process_tools.get_zmq_ipc_name(
self.args.host,
s_name=self.args.server_name,
connect_to_root_instance=self.args.root))
else:
conn_str = "{}://{}:{:d}".format(self.args.protocoll,
self.args.host, self.args.port)
if self.args.split:
recv_conn_str = "{}".format(
process_tools.get_zmq_ipc_name(
self.args.split,
s_name=self.args.server_name,
connect_to_root_instance=self.args.root))
recv_sock = self.zmq_context.socket(zmq.ROUTER)
recv_sock.setsockopt_string(zmq.IDENTITY, self.identity_str)
recv_sock.setsockopt(zmq.LINGER, self.args.timeout)
else:
recv_conn_str = None
recv_sock = None
self.send_sock = client
self.recv_sock = recv_sock
self.conn_str = conn_str
self.recv_conn_str = recv_conn_str
self.verbose(
"socket_type is {}\nIdentity_string is '{}'\nconnection_string is '{}'"
.format(
s_type,
self.identity_str,
conn_str,
))
if self.args.split:
self.verbose(
"receive connection string is '{}'".format(recv_conn_str))
def _init_network(self):
_v_conn_str = process_tools.get_zmq_ipc_name("vector", s_name="collserver", connect_to_root_instance=True)
vector_socket = self.zmq_context.socket(zmq.PUSH) # @UndefinedVariable
vector_socket.setsockopt(zmq.LINGER, 0) # @UndefinedVariable
vector_socket.connect(_v_conn_str)
self.vector_socket = vector_socket
_c_conn_str = "tcp://localhost:8002"
collectd_socket = self.zmq_context.socket(zmq.PUSH) # @UndefinedVariable
collectd_socket.setsockopt(zmq.LINGER, 0) # @UndefinedVariable
collectd_socket.setsockopt(zmq.IMMEDIATE, 1), # @UndefinedVariable
collectd_socket.connect(_c_conn_str)
self.collectd_socket = collectd_socket
def _init_ipc_sockets(self):
self.__num_messages = 0
sock_list = [
("receiver", zmq.PULL, 2), # @UndefinedVariable
("sender", zmq.PUB, 1024), # @UndefinedVariable
]
[setattr(self, "{}_socket".format(short_sock_name), None) for short_sock_name, _a0, _b0 in sock_list]
for short_sock_name, sock_type, hwm_size in sock_list:
sock_name = process_tools.get_zmq_ipc_name(short_sock_name, s_name="snmp-relay")
file_name = sock_name[5:]
self.log(
"init {} ipc_socket '{}' (HWM: {:d})".format(
short_sock_name,
sock_name,
hwm_size
)
)
if os.path.exists(file_name):
self.log("removing previous file")
try:
os.unlink(file_name)
except:
self.log("... %s" % (process_tools.get_except_info()), logging_tools.LOG_LEVEL_ERROR)
wait_iter = 0
while os.path.exists(file_name) and wait_iter < 100:
self.log("socket %s still exists, waiting" % (sock_name))
time.sleep(0.1)
wait_iter += 1
cur_socket = self.zmq_context.socket(sock_type)
try:
process_tools.bind_zmq_socket(cur_socket, sock_name)
# client.bind("tcp://*:8888")
except zmq.ZMQError:
self.log(
"error binding %s: %s" % (
short_sock_name,
process_tools.get_except_info()
),
logging_tools.LOG_LEVEL_CRITICAL)
raise
else:
setattr(self, "%s_socket" % (short_sock_name), cur_socket)
os.chmod(file_name, 0777)
self.receiver_socket.setsockopt(zmq.LINGER, 0) # @UndefinedVariable
self.receiver_socket.setsockopt(zmq.RCVHWM, hwm_size) # @UndefinedVariable
self.receiver_socket.setsockopt(zmq.SNDHWM, hwm_size) # @UndefinedVariable
if sock_type == zmq.PULL: # @UndefinedVariable
self.register_poller(cur_socket, zmq.POLLIN, self._recv_command) # @UndefinedVariable
def _init_network(self):
_v_conn_str = process_tools.get_zmq_ipc_name(
"vector", s_name="collserver", connect_to_root_instance=True)
vector_socket = self.zmq_context.socket(zmq.PUSH)
vector_socket.setsockopt(zmq.LINGER, 0)
vector_socket.connect(_v_conn_str)
self.vector_socket = vector_socket
c_port = InstanceXML(quiet=True).get_port_dict(
icswServiceEnum.collectd_server,
ptype="receive",
)
_c_conn_str = "tcp://127.0.0.1:{:d}".format(c_port)
collectd_socket = self.zmq_context.socket(zmq.PUSH)
collectd_socket.setsockopt(zmq.LINGER, 0)
collectd_socket.setsockopt(zmq.IMMEDIATE, 1)
collectd_socket.connect(_c_conn_str)
self.collectd_socket = collectd_socket
def _init_network(self):
# connection to local collserver socket
conn_str = process_tools.get_zmq_ipc_name(
"vector", s_name="collserver", connect_to_root_instance=True)
vector_socket = self.zmq_context.socket(zmq.PUSH) # @UndefinedVariable
vector_socket.setsockopt(zmq.LINGER, 0) # @UndefinedVariable
vector_socket.connect(conn_str)
self.vector_socket = vector_socket
self.log("connected vector_socket to {}".format(conn_str))
# connection to local collectd server
_cc_str = "tcp://localhost:{:d}".format(
# get receive port for collectd-server drop
self._instance.get_port_dict(icswServiceEnum.collectd_server,
ptype="receive"))
collectd_socket = self.zmq_context.socket(zmq.PUSH)
collectd_socket.setsockopt(zmq.LINGER, 0)
collectd_socket.connect(_cc_str)
self.log("connected collectd_socket to {}".format(_cc_str))
self.collectd_socket = collectd_socket
def _init_network_sockets(self):
self.network_bind(
bind_to_localhost=True,
pollin=self._recv_command,
service_type_enum=icswServiceEnum.meta_server,
client_type=icswServiceEnum.meta_server,
)
conn_str = process_tools.get_zmq_ipc_name(
"vector", s_name="collserver", connect_to_root_instance=True)
if hm_classes and global_config["TRACK_CSW_MEMORY"]:
self.log(
"CSW memory tracking enabled, target is {}".format(conn_str))
vector_socket = self.zmq_context.socket(zmq.PUSH)
vector_socket.setsockopt(zmq.LINGER, 0)
vector_socket.connect(conn_str)
else:
vector_socket = None
self.vector_socket = vector_socket
# memory info send dict
self.mis_dict = {}
# -*- coding: utf-8 -*-
#
# Copyright (C) 2013,2015 Andreas Lang-Nevyjel
#
# Send feedback to: <*****@*****.**>
#
# This file is part of icsw-client
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License Version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
""" SNMP-relay, config """
from initat.tools import configfile, process_tools
global_config = configfile.get_global_config(process_tools.get_programm_name())
IPC_SOCK_SNMP = process_tools.get_zmq_ipc_name("snmp", connect_to_root_instance=True, s_name="snmp-relay")
def _init_network_sockets(self):
self.network_bind(
need_all_binds=False,
bind_to_localhost=True,
service_type_enum=icswServiceEnum.monitor_slave,
client_type=icswServiceEnum.monitor_slave,
simple_server_bind=True,
pollin=self.remote_call,
)
self.__slaves = {}
conn_str = process_tools.get_zmq_ipc_name(
"vector", s_name="collserver", connect_to_root_instance=True)
vector_socket = self.zmq_context.socket(zmq.PUSH)
vector_socket.setsockopt(zmq.LINGER, 0)
vector_socket.connect(conn_str)
self.vector_socket = vector_socket
# copy from relay, Refactor
self.__nhm_connections = set()
sock_list = [
("ipc", "receiver", zmq.PULL, 2),
]
[
setattr(self, "{}_socket".format(short_sock_name), None)
for _sock_proto, short_sock_name, _a0, _b0 in sock_list
]
for _sock_proto, short_sock_name, sock_type, hwm_size in sock_list:
sock_name = process_tools.get_zmq_ipc_name(
short_sock_name,
s_name="md-sync-server",
connect_to_root_instance=True)
file_name = sock_name[5:]
self.log("init {} ipc_socket '{}' (HWM: {:d})".format(
short_sock_name, sock_name, hwm_size))
if os.path.exists(file_name):
self.log("removing previous file")
try:
os.unlink(file_name)
except:
self.log("... {}".format(process_tools.get_except_info()),
logging_tools.LOG_LEVEL_ERROR)
wait_iter = 0
while os.path.exists(file_name) and wait_iter < 100:
self.log("socket {} still exists, waiting".format(sock_name))
time.sleep(0.1)
wait_iter += 1
cur_socket = self.zmq_context.socket(sock_type)
try:
process_tools.bind_zmq_socket(cur_socket, sock_name)
# client.bind("tcp://*:8888")
except zmq.ZMQError:
self.log(
"error binding {}: {}".format(
short_sock_name, process_tools.get_except_info()),
logging_tools.LOG_LEVEL_CRITICAL)
raise
else:
setattr(self, "{}_socket".format(short_sock_name), cur_socket)
os.chmod(file_name, 0o777)
cur_socket.setsockopt(zmq.LINGER, 0)
cur_socket.setsockopt(zmq.SNDHWM, hwm_size)
cur_socket.setsockopt(zmq.RCVHWM, hwm_size)
if sock_type == zmq.PULL:
self.register_poller(cur_socket, zmq.POLLIN,
self._recv_command_ipc)
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
""" base constants for collectd """
from initat.tools import process_tools, configfile
from enum import Enum
IPC_SOCK_SNMP = process_tools.get_zmq_ipc_name("snmp",
connect_to_root_instance=True,
s_name="collectd-init")
# constant, change to limit RRDs to be converted at once
MAX_FOUND = 0
class _KeyEmitter(object):
def __init__(self, pfix):
self.pfix = pfix
def __call__(self, device):
# device is a device instance or the uuid
if isinstance(device, str):
return "{}_{}".format(
self.pfix,
# Copyright (C) 2014-2015 Andreas Lang-Nevyjel, init.at
#
# Send feedback to: <*****@*****.**>
#
# this file is part of discovery-server
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License Version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
""" discovery-server, configuration and constants """
from initat.tools import configfile, process_tools
from initat.server_version import VERSION_STRING
global_config = configfile.get_global_config(process_tools.get_programm_name())
IPC_SOCK_SNMP = process_tools.get_zmq_ipc_name("snmp",
connect_to_root_instance=True,
s_name="discovery-server")
def _init_network_sockets(self):
zmq_id_name = "/etc/sysconfig/host-monitoring.d/0mq_id"
my_0mq_id = uuid_tools.get_uuid().get_urn()
if not config_store.ConfigStore.exists(ZMQ_ID_MAP_STORE):
create_0mq_cs = True
if os.path.exists(zmq_id_name):
try:
zmq_id_dict = {
cur_el.attrib["bind_address"]:
(cur_el.text,
True if "virtual" in cur_el.attrib else False)
for cur_el in
etree.fromstring(file(zmq_id_name, "r").read()).xpath(
".//zmq_id[@bind_address]", smart_strings=False)
}
except:
self.log(
"error reading from {}: {}".format(
zmq_id_name, process_tools.get_except_info()),
logging_tools.LOG_LEVEL_ERROR)
zmq_id_dict = {}
else:
zmq_id_dict = {}
if "*" not in zmq_id_dict:
zmq_id_dict["*"] = (my_0mq_id, False)
_cs = config_store.ConfigStore(
ZMQ_ID_MAP_STORE,
log_com=self.log,
read=False,
prefix="bind",
access_mode=config_store.AccessModeEnum.LOCAL,
fix_access_mode=True,
)
for _idx, _key in enumerate(["*"] + sorted(
[_key for _key in zmq_id_dict.keys() if _key not in ["*"]])):
_cs["{:d}".format(_idx)] = {
"address": _key,
"uuid": zmq_id_dict[_key][0],
"virtual": zmq_id_dict[_key][1]
}
else:
# read from cs
_cs = config_store.ConfigStore(
ZMQ_ID_MAP_STORE,
log_com=self.log,
prefix="bind",
access_mode=config_store.AccessModeEnum.LOCAL,
fix_access_mode=True,
)
create_0mq_cs = False
if "0" not in _cs:
_cs["0"] = {
"address": "*",
"virtual": False,
"uuid": my_0mq_id,
}
if _cs["0"]["uuid"] != my_0mq_id:
self.log(
"0MQ id from cluster ({}) differs from host-monitoring 0MQ id ({})"
.format(
my_0mq_id,
_cs["bind_0_uuid"],
))
# code snippet to update value
_cur = _cs["0"]
_cur["uuid"] = my_0mq_id
_cs["0"] = _cur
create_0mq_cs = True
if create_0mq_cs:
_cs.write()
# get all ipv4 interfaces with their ip addresses, dict: interfacename -> IPv4
zmq_id_dict = {}
for _idx in _cs.keys():
_bind = _cs[_idx]
zmq_id_dict[_bind["address"]] = (
_bind["uuid"],
_bind["virtual"],
)
ipv4_dict = {
cur_if_name: [ip_tuple["addr"] for ip_tuple in value[2]][0]
for cur_if_name, value in [(if_name,
netifaces.ifaddresses(if_name))
for if_name in netifaces.interfaces()]
if 2 in value
}
# ipv4_lut = dict([(value, key) for key, value in ipv4_dict.iteritems()])
ipv4_addresses = ipv4_dict.values()
if zmq_id_dict.keys() == ["*"]:
# wildcard bind
pass
else:
if "*" in zmq_id_dict:
wc_urn, wc_virtual = zmq_id_dict.pop("*")
for target_ip in ipv4_addresses:
if target_ip not in zmq_id_dict:
zmq_id_dict[target_ip] = (wc_urn, wc_virtual)
ref_id = "*" if "*" in zmq_id_dict else "127.0.0.1"
self.zeromq_id = zmq_id_dict[ref_id][0].split(":")[-1]
self.log("0MQ bind info (global 0MQ id is {})".format(self.zeromq_id))
for key in sorted(zmq_id_dict.iterkeys()):
self.log("bind address {:<15s}: {}{}".format(
key, zmq_id_dict[key][0],
" is virtual" if zmq_id_dict[key][1] else ""))
self.zmq_id_dict = zmq_id_dict
self._bind_external()
sock_list = [
("ipc", "vector", zmq.PULL, 512, None, ""), # @UndefinedVariable
("ipc", "command", zmq.PULL, 512, self._recv_ext_command,
""), # @UndefinedVariable
("ipc", "result", zmq.ROUTER, 512, None,
process_tools.zmq_identity_str("host_monitor")
) # @UndefinedVariable
]
for _sock_proto, short_sock_name, sock_type, hwm_size, dst_func, zmq_id in sock_list:
sock_name = process_tools.get_zmq_ipc_name(short_sock_name,
s_name="collserver")
file_name = sock_name[5:]
self.log("init {} ipc_socket '{}' (HWM: {:d})".format(
short_sock_name, sock_name, hwm_size))
if os.path.exists(file_name):
self.log("removing previous file")
try:
os.unlink(file_name)
except:
self.log("... {}".format(process_tools.get_except_info()),
logging_tools.LOG_LEVEL_ERROR)
wait_iter = 0
while os.path.exists(file_name) and wait_iter < 100:
self.log("socket {} still exists, waiting".format(sock_name))
time.sleep(0.1)
wait_iter += 1
cur_socket = self.zmq_context.socket(sock_type)
if zmq_id:
cur_socket.setsockopt_string(zmq.IDENTITY,
zmq_id) # @UndefinedVariable
try:
process_tools.bind_zmq_socket(cur_socket, sock_name)
# client.bind("tcp://*:8888")
except zmq.ZMQError:
self.log(
"error binding {}: {}".format(
short_sock_name, process_tools.get_except_info()),
logging_tools.LOG_LEVEL_CRITICAL)
raise
else:
setattr(self, "{}_socket".format(short_sock_name), cur_socket)
_backlog_size = self.CC.CS["hm.socket.backlog.size"]
os.chmod(file_name, 0777)
cur_socket.setsockopt(zmq.LINGER, 0) # @UndefinedVariable
cur_socket.setsockopt(zmq.SNDHWM,
hwm_size) # @UndefinedVariable
cur_socket.setsockopt(zmq.RCVHWM,
hwm_size) # @UndefinedVariable
if dst_func:
self.register_poller(cur_socket, zmq.POLLIN,
dst_func) # @UndefinedVariable
def _init_ipc_sockets(self):
# init IP lookup table
# if self.__force_resolve:
# self.__ip_lut = {}
# self.__forward_lut = {}
self.__num_messages = 0
# nhm (not host monitoring) dictionary for timeout
self.__nhm_dict = {}
# raw_nhm (not host monitoring) dictionary for timeout, raw connections (no XML)
self.__raw_nhm_dict = {}
self.__nhm_connections = set()
# also used in md-sync-server/server, ToDo: Refactor
sock_list = [
{
"proto": "ipc",
"name": "receiver",
"type": zmq.PULL,
"hwm_size": 2,
},
{
"proto": "ipc",
"name": "sender",
"type": zmq.PUB,
"hwm_size": 1024,
},
]
[
setattr(self, "{}_socket".format(_sock["name"]), None) for _sock in sock_list
]
for _sock in sock_list:
sock_name = process_tools.get_zmq_ipc_name(_sock["name"], s_name="collrelay")
file_name = sock_name[5:]
self.log(
"init {} ipc_socket '{}' (HWM: {:d})".format(
_sock["name"],
sock_name,
_sock["hwm_size"],
)
)
if os.path.exists(file_name):
self.log("removing previous file")
try:
os.unlink(file_name)
except:
self.log(
"... {}".format(process_tools.get_except_info()),
logging_tools.LOG_LEVEL_ERROR
)
wait_iter = 0
while os.path.exists(file_name) and wait_iter < 100:
self.log("socket {} still exists, waiting".format(sock_name))
time.sleep(0.1)
wait_iter += 1
cur_socket = self.zmq_context.socket(_sock["type"])
try:
process_tools.bind_zmq_socket(cur_socket, sock_name)
# client.bind("tcp://*:8888")
except zmq.ZMQError:
self.log(
"error binding {}: {}" .format(
_sock["name"],
process_tools.get_except_info()
),
logging_tools.LOG_LEVEL_CRITICAL
)
raise
else:
setattr(self, "{}_socket".format(_sock["name"]), cur_socket)
os.chmod(file_name, 0o777)
cur_socket.setsockopt(zmq.LINGER, 0)
cur_socket.setsockopt(zmq.SNDHWM, _sock["hwm_size"])
cur_socket.setsockopt(zmq.RCVHWM, _sock["hwm_size"])
if _sock["type"] == zmq.PULL:
self.register_poller(cur_socket, zmq.POLLIN, self._recv_command_ipc)
self.client_socket = process_tools.get_socket(
self.zmq_context,
"ROUTER",
identity="ccollclient:{}".format(process_tools.get_machine_name()),
linger=0,
sndhwm=2,
rcvhwm=2,
immediate=True,
)
self.register_poller(self.client_socket, zmq.POLLIN, self._recv_nhm_result)
def send_and_receive_zmq(target_host, command, *args, **kwargs):
identity_str = process_tools.zmq_identity_str(
kwargs.pop("identity_string", "srv_com"))
zmq_context = kwargs.pop("zmq_context")
cur_timeout = kwargs.pop("timeout", 20)
client_send = zmq_context.socket(zmq.PUSH)
client_recv = zmq_context.socket(zmq.SUB)
client_send.setsockopt(zmq.LINGER, cur_timeout * 2)
client_recv.setsockopt(zmq.SUBSCRIBE, identity_str)
# kwargs["server"] : collrelay or snmprelay
server_name = kwargs.pop("server")
send_conn_str = "{}".format(
process_tools.get_zmq_ipc_name(
kwargs.pop("process", "receiver"),
s_name=server_name,
connect_to_root_instance=True,
))
recv_conn_str = "{}".format(
process_tools.get_zmq_ipc_name(
kwargs.pop("process", "sender"),
s_name=server_name,
connect_to_root_instance=True,
))
client_send.connect(send_conn_str)
client_recv.connect(recv_conn_str)
srv_com = server_command.srv_command(command=command,
identity=identity_str)
srv_com["host"] = target_host
srv_com["raw"] = "True"
srv_com["arg_list"] = " ".join(args)
# add additional keys
for key, value in kwargs.items():
srv_com[key] = "{:d}".format(value) if isinstance(value,
int) else value
s_time = time.time()
client_send.send_unicode(str(srv_com))
client_send.close()
if client_recv.poll(cur_timeout * 1000):
id_str = client_recv.recv()
else:
id_str = None
if id_str and client_recv.getsockopt(zmq.RCVMORE):
e_time = time.time()
if client_recv.poll((cur_timeout - (e_time - s_time)) * 1000):
recv_str = client_recv.recv()
else:
recv_str = None
else:
recv_str = None
client_recv.close()
if recv_str and id_str:
try:
srv_reply = server_command.srv_command(source=recv_str)
except:
# srv_reply = None
raise
else:
# srv_reply = None
raise SystemError("timeout ({:d} seconds) exceeded".format(
int(cur_timeout)))
return srv_reply
