def do_chpasswd(cur_opts, log_com):
srv_com = server_command.srv_command(command="modify_password")
srv_com["server_key:user_name"] = cur_opts.username
print("changing password for user '{}'".format(cur_opts.username))
srv_com["server_key:old_password"] = base64.b64encode(
bz2.compress(
get_pass("please enter current password:"******"server_key:new_password_1"] = base64.b64encode(
bz2.compress(
get_pass("please enter the new password:"******"server_key:new_password_2"] = base64.b64encode(
bz2.compress(
get_pass("please reenter the new password:"******"pwd_change_request",
timeout=cur_opts.timeout,
)
_conn.add_connection("tcp://localhost:8004", srv_com, immediate=True)
_result = _conn.loop()[0]
_res_str, _res_state = _result.get_log_tuple()
# _res_str, _res_state = ("ok", logging_tools.LOG_LEVEL_OK)
print(
"change gave [{}]: {}".format(
logging_tools.get_log_level_str(_res_state)
, _res_str
)
)
if _res_state == logging_tools.LOG_LEVEL_OK:
_conn = net_tools.ZMQConnection(
"ldap_update_request",
timeout=cur_opts.timeout,
)
upd_com = server_command.srv_command(command="sync_ldap_config")
_conn.add_connection("tcp://localhost:8004", upd_com, immediate=True)
_res_str, _res_state = _conn.loop()[0].get_log_tuple()
print(
"syncing the LDAP tree returned ({}) {}".format(
logging_tools.get_log_level_str(_res_state),
_res_str,
)
)
# print(_result.pretty_print())
return 0
def signal_localhost():
# signal clusterserver running on localhost
_sender = net_tools.ZMQConnection("wf_server_notify")
# only send no receive
_sender.add_connection("tcp://localhost:8004", notify_command(), multi=True)
# close connection / terminate context
_sender.close()
def _call(self, cur_inst):
cur_inst.srv_com["command"] = "state{}".format(cur_inst.srv_com["*control"])
_result = net_tools.ZMQConnection(
"icsw_cssc_{:d}".format(os.getpid())
).add_connection(
"tcp://localhost:8012",
cur_inst.srv_com,
)
cur_inst.srv_com.set_result(*_result.get_log_tuple(map_to_log_level=False))
def query_local_meta_server(inst_xml, command, services=None):
srv_com = server_command.srv_command(command="state{}".format(command), )
if services:
srv_com["services"] = ",".join(services)
return net_tools.ZMQConnection("icsw_state_{:d}".format(
os.getpid())).add_connection(
"tcp://localhost:{:d}".format(
inst_xml.get_port_dict("meta-server", command=True)),
srv_com,
)
def _signal_restart(self):
if self.name in ["meta-server"]:
from initat.tools import net_tools, server_command
self.log("sending signal-restart")
_result = net_tools.ZMQConnection("icsw_restart_{:d}".format(
os.getpid())).add_connection(
"tcp://localhost:8012",
server_command.srv_command(
command="next-stop-is-restart", ),
)
if _result is not None:
self.log(*_result.get_log_tuple())
def send_msg(self, *args, **kwargs):
_from_name, _from_pid, run_idx, conn_str, srv_com = args
srv_com = server_command.srv_command(source=srv_com)
srv_com["discovery_run_idx"] = "{:d}".format(run_idx)
_new_con = net_tools.ZMQConnection(
"ext_con_discovery_{:d}".format(run_idx),
context=self.zmq_context,
poller_base=self,
callback=self._ext_receive,
)
self.__ext_con_dict[run_idx] = _new_con
_new_con.add_connection(conn_str, srv_com)
def send_host_monitor_command(self, *args, **kwargs):
_from_name, _from_pid, run_index, conn_str, srv_com = args
srv_com = server_command.srv_command(source=srv_com)
srv_com["run_index"] = "{:d}".format(run_index)
_new_con = net_tools.ZMQConnection(
"host_monitor_command_{:d}".format(run_index),
context=self.zmq_context,
poller_base=self,
callback=self.__send_host_monitor_command_callback,
)
self.__pending_host_monitoring_commands[run_index] = _new_con
_new_con.add_connection(conn_str, srv_com)
def query_local_server(inst, cmd):
_port = inst.get_port_dict(icswServiceEnum.cluster_server, ptype="command")
_result = net_tools.ZMQConnection("icsw_image_{:d}".format(
os.getpid())).add_connection(
"tcp://localhost:{:d}".format(_port),
server_command.srv_command(command=cmd),
)
if _result is None:
print("Unable to send '{}' to local cluster-server".format(cmd))
sys.exit(1)
_ret, _state = _result.get_log_tuple()
if _state != logging_tools.LOG_LEVEL_OK:
print("a problem occured ({}): {}".format(
logging_tools.get_log_level_str(_state), _ret))
sys.exit(_state)
return _result
def query_local_meta_server(inst_xml,
command,
services=None,
days_to_consider=1,
db_limit=100):
srv_com = server_command.srv_command(command="state{}".format(command), )
if services:
srv_com["services"] = ",".join(services)
srv_com["days_to_consider"] = "{:d}".format(days_to_consider)
srv_com["db_limit"] = "{:d}".format(db_limit)
return net_tools.ZMQConnection("icsw_state_{:d}".format(
os.getpid())).add_connection(
"tcp://localhost:{:d}".format(
inst_xml.get_port_dict(icswServiceEnum.meta_server,
command=True)),
srv_com,
)
def handle(self, **options):
if options["parse_file"]:
f = open("disk_output_data")
lines = f.read()
lines = lines.split("\n")
f = open(ASSET_MANAGEMENT_TEST_LOCATION, "rb")
data = pickle.load(f)
f.close()
new_device_for_next_line = False
current_device = None
parse_partition = False
parse_partition_cnt = 0
parse_disk = False
parse_disk_cnt = 0
parse_disk_device_name = ""
parse_disk_size = ""
parse_partition_mountpount = ""
parse_partition_size = ""
parse_logical = False
parse_logical_cnt = 0
parse_logical_device_name = ""
parse_logical_size = ""
parse_logical_free = ""
parse_logical_filesystem = ""
for line in lines:
if line == "------":
new_device_for_next_line = True
continue
if new_device_for_next_line:
new_device_for_next_line = False
for device in data:
if device.identifier == line:
current_device = device
current_device.expected_hdds = []
current_device.expected_partitions = []
current_device.expected_logical_volumes = []
if line == "--DISK--":
parse_partition = False
parse_partition_cnt = 0
parse_disk = True
parse_disk_cnt = 0
parse_logical = False
parse_logical_cnt = 0
continue
if parse_disk:
if parse_disk_cnt == 0:
parse_disk_device_name = line
parse_disk_cnt += 1
elif parse_disk_cnt == 1:
parse_disk_size = int(line)
parse_disk_cnt += 1
elif parse_disk_cnt == 2:
parse_disk_serial = line
parse_disk = False
parse_disk_cnt = 0
new_hdd = ExpectedHdd(parse_disk_device_name,
parse_disk_serial,
parse_disk_size)
current_device.expected_hdds.append(new_hdd)
if line == "--PARTITION--":
parse_partition = True
parse_partition_cnt = 0
parse_disk = False
parse_disk_cnt = 0
parse_logical = False
parse_logical_cnt = 0
continue
if parse_partition:
if parse_partition_cnt == 0:
parse_partition_mountpount = line
parse_partition_cnt += 1
elif parse_partition_cnt == 1:
parse_partition_size = int(line)
parse_partition_cnt += 1
elif parse_partition_cnt == 2:
parse_partition = False
parse_partition_cnt = 0
_partition = ExpectedPartition(
parse_disk_device_name, parse_partition_mountpount,
parse_partition_size, line)
current_device.expected_partitions.append(_partition)
if line == "--LOGICAL--":
parse_partition = False
parse_partition_cnt = 0
parse_disk = False
parse_disk_cnt = 0
parse_logical = True
parse_logical_cnt = 0
continue
if parse_logical:
if parse_logical_cnt == 0:
parse_logical_device_name = line
parse_logical_cnt += 1
elif parse_logical_cnt == 1:
parse_logical_size = None
if line != "None":
parse_logical_size = int(line)
parse_logical_cnt += 1
elif parse_logical_cnt == 2:
parse_logical_free = None
if line != "None":
parse_logical_free = int(line)
parse_logical_cnt += 1
elif parse_logical_cnt == 3:
parse_logical_filesystem = line
parse_logical_cnt += 1
elif parse_logical_cnt == 4:
parse_logical = False
parse_logical_cnt = 0
elv = ExpectedLogicalVolume(parse_logical_device_name,
parse_logical_size,
parse_logical_free,
parse_logical_filesystem,
line)
current_device.expected_logical_volumes.append(elv)
f = open(ASSET_MANAGEMENT_TEST_LOCATION, "wb")
pickle.dump(data, f)
f.close()
return
if options['delete_index'] is not None:
self.handle_delete(options['delete_index'])
return
if options['list']:
self.handle_list()
return
if options['expected_hdd'] is not None:
self.handle_add_expected_hdd(options['expected_hdd'])
return
if options['expected_partition'] is not None:
self.handle_add_expected_partition(options['expected_partition'])
return
if options['expected_logical_volume'] is not None:
self.handle_add_expected_logical_volume(
options['expected_logical_volume'])
return
for _property in options['ignore_tests']:
if _property not in self.ignorable_properties:
print("Invalid property: {}".format(_property))
return
if options['scan_type'] is None:
print("Scan Type missing")
return
if options['scan_type'] not in ['HM', 'NRPE']:
print("Invalid Scan Type: {}".format(options['scan_type']))
if options['ip'] is None:
print("IP/Hostname missing")
return
if options['identifier'] is None:
print("Identifier for this entry missing")
return
result_dict = {}
if options['scan_type'] == "HM":
scan_type = ScanType.HM
hm_port = InstanceXML(quiet=True).get_port_dict("host-monitoring",
command=True)
conn_str = "tcp://{}:{:d}".format(options['ip'], hm_port)
for asset_type, hm_command in list(
ASSETTYPE_HM_COMMAND_MAP.items()):
result_dict[asset_type] = None
print("Running command [{}] on {}".format(
hm_command, conn_str))
srv_com = server_command.srv_command(command=hm_command)
new_con = net_tools.ZMQConnection(hm_command, timeout=30)
new_con.add_connection(conn_str, srv_com)
result = new_con.loop()
if result:
result = result[0]
if result:
(status_string,
server_result_code) = result.get_result()
if server_result_code == server_command.SRV_REPLY_STATE_OK:
result_dict[asset_type] = etree.tostring(
result.tree)
valid = all([
result_dict[asset_type] is not None
for asset_type in ASSETTYPE_HM_COMMAND_MAP
])
else:
scan_type = ScanType.NRPE
for asset_type, nrpe_command in list(
ASSETTYPE_NRPE_COMMAND_MAP.items()):
result_dict[asset_type] = None
_com = "/opt/cluster/sbin/check_nrpe -H{} -2 -P1048576 -p{} -n -c{} -t{}".format(
options['ip'],
DEFAULT_NRPE_PORT,
nrpe_command,
1000,
)
output = subprocess.check_output(_com.split(" "))
if output and len(output) > 0:
result_dict[asset_type] = output
valid = all([
result_dict[asset_type] is not None
for asset_type in ASSETTYPE_NRPE_COMMAND_MAP
])
if valid:
try:
f = open(ASSET_MANAGEMENT_TEST_LOCATION, "rb")
data = pickle.load(f)
f.close()
except IOError:
data = []
data.append(
ResultObject(options['identifier'], options['ignore_tests'],
result_dict, scan_type))
f = open(ASSET_MANAGEMENT_TEST_LOCATION, "wb")
pickle.dump(data, f)
f.close()
print("New entry added")
else:
print("Failed to generate new entry")
missing_types = []
for asset_type, result in list(result_dict.items()):
if result is None:
missing_types.append(asset_type)
print("No result for: {}".format(missing_types))
def contact_server(request, srv_type_enum, send_com, **kwargs):
# log lines
_log_lines = []
# xml request
_xml_req = kwargs.get("xml_request", hasattr(request, "xml_response"))
# simple mapping
cur_router = routing.SrvTypeRouting()
if srv_type_enum.name not in cur_router:
# try again harder (rebuild routing table)
cur_router = routing.SrvTypeRouting(force=True)
if srv_type_enum.name in cur_router:
# print send_com.pretty_print()
if request.user:
send_com["user_id"] = request.user.pk
_conn = net_tools.ZMQConnection(kwargs.get("connection_id",
"webfrontend"),
timeout=kwargs.get("timeout", 10))
# split to node-local servers ?
if kwargs.get("split_send", True):
send_list = cur_router.check_for_split_send(
srv_type_enum.name, send_com)
if cur_router.no_bootserver_devices:
# for _miss_pk, _miss_name in cur_router.no_bootserver_devices:
cur_router._log(
request, _log_lines, "no bootserver for {}: {}".format(
logging_tools.get_plural(
"device", len(cur_router.no_bootserver_devices)),
", ".join(
sorted([
_v[1]
for _v in cur_router.no_bootserver_devices
])),
), logging_tools.LOG_LEVEL_WARN)
else:
send_list = [(None, send_com)]
if send_list:
_conn_strs = []
for _send_id, _send_com in send_list:
if _send_id is None:
# get send_id from target_server_id
_send_id = kwargs.get("target_server_id", None)
# no split send, decide based on target_server_id
# _conn_str = cur_router.get_connection_string(srv_type, server_id=)
# else:
# print "*", _send_id
_connect_port_enum = kwargs.get("connect_port_enum", None)
_conn_str = cur_router.get_connection_string(
srv_type_enum,
server_id=_send_id,
connect_port_enum=_connect_port_enum)
_conn_strs.append(_conn_str)
_conn.add_connection(_conn_str,
_send_com,
multi=True,
immediate=True)
log_result = kwargs.get("log_result", True)
log_error = kwargs.get("log_error", True)
cur_router.start_result_feed()
# merge results
[
cur_router.feed_srv_result(
send_com,
_res,
request if _xml_req else None,
_conn_str,
_log_lines,
log_result,
log_error,
srv_type_enum,
) for _res, _conn_str in zip(_conn.loop(), _conn_strs)
]
result = cur_router.result
else:
result = None
else:
result = None
_err_str = u"ServiceType '{}' not defined in routing".format(
srv_type_enum.name)
cur_router._log(request, _log_lines, _err_str,
logging_tools.LOG_LEVEL_ERROR)
if _xml_req:
return result
else:
return result, _log_lines
def device_syslog(opt_ns, cur_dev, j_logs):
print(
"Information about device '{}' (full name {}, devicegroup {})".format(
str(cur_dev), str(cur_dev.full_name), str(cur_dev.device_group)))
print("UUID is '{}', database-ID is {:d}".format(cur_dev.uuid, cur_dev.pk))
_cr = routing.SrvTypeRouting(force=True, ignore_errors=True)
_ST = "logcheck-server"
if _ST in _cr.service_types:
_inst_xml = InstanceXML(quiet=True)
# get logcheck-server IP
_ls_ip = _cr[_ST][0][1]
# get logcheck-server Port
_ls_port = _inst_xml.get_port_dict(_ST, ptype="command")
_sc = server_command.srv_command(command="get_syslog", )
_sc["devices"] = _sc.builder(
"devices", *[
_sc.builder(
"device",
pk="{:d}".format(cur_dev.pk),
lines="{:d}".format(opt_ns.loglines),
minutes="{:d}".format(opt_ns.minutes),
)
])
_conn_str = "tcp://{}:{:d}".format(_ls_ip, _ls_port)
_result = net_tools.ZMQConnection("icsw_state_{:d}".format(
os.getpid())).add_connection(
_conn_str,
_sc,
)
if _result is not None:
_dev = _result.xpath(".//ns:devices/ns:device[@pk]")[0]
_lines = _result.xpath("ns:lines", start_el=_dev)[0]
_rates = _result.xpath("ns:rates", start_el=_dev)
if _rates:
_rates = {
int(_el.get("timeframe")): float(_el.get("rate"))
for _el in _rates[0]
}
print("rate info: {}".format(", ".join([
"{:.2f} lines/sec in {}".format(
_rates[_seconds],
logging_tools.get_diff_time_str(_seconds))
for _seconds in sorted(_rates)
])))
else:
print("no rate info found")
print(_rates)
_out_lines = logging_tools.NewFormList()
for _entry in server_command.decompress(_lines.text, json=True):
_out_lines.append([
logging_tools.form_entry(_entry["line_id"], header="idx"),
logging_tools.form_entry(
"{:04d}-{:02d}-{:02d} {:02d}:{:02d}:{:02d}".format(
*_entry["line_datetime_parsed"]),
header="Timestamp",
),
] + [
logging_tools.form_entry(_entry[_key], header=_key)
for _key in ["hostname", "priority", "facility", "tag"]
] + [
logging_tools.form_entry(_entry["text"], header="text"),
])
print(str(_out_lines))
else:
print("got no result from {} ({})".format(_conn_str, _ST))
else:
print("No logcheck-server found, skipping syslog display")
def ClientCode(global_config):
from initat.host_monitoring import modules
if global_config["VERBOSE"] > 1:
print("{:d} import errors:".format(len(modules.IMPORT_ERRORS)))
for mod, com, _str in modules.IMPORT_ERRORS:
print("{:<30s} {:<20s} {}".format(com, mod.split(".")[-1], _str))
conn_str = "tcp://{}:{:d}".format(global_config["HOST"],
global_config["COMMAND_PORT"])
arg_stuff = global_config.get_argument_stuff()
arg_list = arg_stuff["arg_list"]
com_name = arg_list.pop(0)
if com_name in modules.command_dict:
srv_com = server_command.srv_command(command=com_name)
for src_key, dst_key in [("HOST", "host"), ("COMMAND_PORT", "port")]:
srv_com[dst_key] = global_config[src_key]
com_struct = modules.command_dict[com_name]
try:
cur_ns, rest = com_struct.handle_commandline(arg_list)
except ValueError, what:
ret = ExtReturn(limits.mon_STATE_CRITICAL,
"error parsing: {}".format(what[1]))
else:
# see also struct.py in collrelay
if hasattr(cur_ns, "arguments"):
for arg_index, arg in enumerate(cur_ns.arguments):
srv_com["arguments:arg{:d}".format(arg_index)] = arg
srv_com["arguments:rest"] = " ".join(rest)
for key, value in vars(cur_ns).iteritems():
srv_com["namespace:{}".format(key)] = value
result = net_tools.ZMQConnection(
"{}:{:d}".format(global_config["IDENTITY_STRING"],
os.getpid()),
timeout=global_config["TIMEOUT"],
).add_connection(
conn_str,
srv_com,
immediate=True,
)
if result:
if global_config["COMMAND_PORT"] == InstanceXML(
quiet=True).get_port_dict("host-monitoring",
command=True):
error_result = result.xpath(".//ns:result[@state != '0']",
smart_strings=False)
if error_result:
error_result = error_result[0]
ret = ExtReturn(int(error_result.attrib["state"]),
error_result.attrib["reply"])
else:
if hasattr(com_struct, "interpret"):
ret = ExtReturn.get_ext_return(
com_struct.interpret(result, cur_ns))
else:
_result = result.xpath(".//ns:result",
smart_strings=False)[0]
ret = ExtReturn(
server_command.srv_reply_to_nag_state(
int(_result.attrib["state"])),
result.attrib["reply"])
else:
ret_str, ret_state = result.get_log_tuple()
ret = ExtReturn(
server_command.srv_reply_to_nag_state(ret_state),
ret_str)
else:
ret = ExtReturn(limits.mon_STATE_CRITICAL, "timeout")
def contact_server(request, srv_type_enum, send_com, **kwargs):
# log lines
_log_lines = []
# xml request
_xml_req = kwargs.get("xml_request", hasattr(request, "xml_response"))
# simple mapping
cur_router = routing.SrvTypeRouting()
if srv_type_enum.name not in cur_router:
# try again harder (rebuild routing table)
cur_router = routing.SrvTypeRouting(force=True)
if srv_type_enum.name in cur_router:
# com = send_com["*command"]
# connection id
_conn_id = kwargs.get("connection_id", "webfrontend")
# memcache key to catch multi-calls to server with this is
# mc_key = "$$MCTS_KD_{}".format(_conn_id)
# memcache key for connection dict
mc_key = "{}_WebConDict".format(settings.ICSW_CACHE_KEY)
mc_client = _C_MCC.client
# print(dir(mc_client))
# print("c=", _conn_id)
# try to set default value (will most likely fail but never mind)
_default_c = {"open": {}}
# the open dict has the format
# conn_id -> (last_time_used, request_pending)
mc_client.add(mc_key, _default_c)
_cur_time = time.time()
_default_time = _cur_time - 3600
_run_idx = 0
while True:
_c = mc_client.gets(mc_key)
# print("gets={}".format(str(_c)))
if _c is None:
# should never happen, set default value again
mc_client.add(mc_key, _default_c)
continue
if isinstance(_c["open"], list):
_c["open"] = {}
while True:
_run_idx += 1
cur_conn_id = "{}_{:d}".format(_conn_id, _run_idx)
_open_entry = _c["open"].get(cur_conn_id,
(_default_time, False))
# connection has to be closed for at least 10 seconds
if not _open_entry[1] and _open_entry[0] < _cur_time - 10:
# print("reuse")
break
_c["open"][cur_conn_id] = (_cur_time, True)
# import pprint
# pprint.pprint(_c["open"])
_ret = mc_client.cas(mc_key, _c)
# print("_ret={}".format(_ret))
if _ret:
break
else:
# print("ERRLoop")
pass
# print("CurConnId={}".format(cur_conn_id))
# print("list='{}' ({})".format(_c["open"], com))
# print send_com.pretty_print()
if request.user:
send_com["user_id"] = request.user.pk
_conn = net_tools.ZMQConnection(cur_conn_id,
timeout=kwargs.get("timeout", 10))
# split to node-local servers ?
if kwargs.get("split_send", True):
send_list = cur_router.check_for_split_send(
srv_type_enum.name, send_com)
if cur_router.no_bootserver_devices:
# for _miss_pk, _miss_name in cur_router.no_bootserver_devices:
cur_router._log(
request, _log_lines, "no bootserver for {}: {}".format(
logging_tools.get_plural(
"device", len(cur_router.no_bootserver_devices)),
", ".join(
sorted([
_v[1]
for _v in cur_router.no_bootserver_devices
])),
), logging_tools.LOG_LEVEL_WARN)
else:
send_list = [(None, send_com)]
if send_list:
_conn_strs = []
for _send_id, _send_com in send_list:
if _send_id is None:
# get send_id from target_server_id
_send_id = kwargs.get("target_server_id", None)
# no split send, decide based on target_server_id
# _conn_str = cur_router.get_connection_string(srv_type, server_id=)
# else:
# print "*", _send_id
_connect_port_enum = kwargs.get("connect_port_enum", None)
_conn_str = cur_router.get_connection_string(
srv_type_enum,
server_id=_send_id,
connect_port_enum=_connect_port_enum)
_conn_strs.append(_conn_str)
_conn.add_connection(_conn_str,
_send_com,
multi=True,
immediate=True)
log_result = kwargs.get("log_result", True)
log_error = kwargs.get("log_error", True)
cur_router.start_result_feed()
# merge results
[
cur_router.feed_srv_result(
send_com,
_res,
request if _xml_req else None,
_conn_str,
_log_lines,
log_result,
log_error,
srv_type_enum,
) for _res, _conn_str in zip(_conn.loop(), _conn_strs)
]
result = cur_router.result
else:
result = None
# remove cur_conn_id from cache
while True:
_c = mc_client.gets(mc_key)
if _c is None:
# should never happen, set default value again
mc_client.add(mc_key, _default_c)
continue
# if cur_conn_id in _c["open"]:
# _c["open"].remove(cur_conn_id)
_c["open"][cur_conn_id] = (time.time(), False)
_ret = mc_client.cas(mc_key, _c)
# print("_ret={}".format(_ret))
if _ret:
break
else:
# try again
continue
# print("done", cur_conn_id)
else:
result = None
_err_str = "ServiceType '{}' not defined in routing".format(
srv_type_enum.name)
cur_router._log(request, _log_lines, _err_str,
logging_tools.LOG_LEVEL_ERROR)
if _xml_req:
return result
else:
return result, _log_lines
def ClientCode(global_config):
from initat.host_monitoring.modules import local_mc
from initat.debug import ICSW_DEBUG_MODE
if ICSW_DEBUG_MODE:
# dummy logger of module loader
def dummy_log(what, log_level=logging_tools.LOG_LEVEL_OK):
print("{} {}".format(logging_tools.get_log_level_str(log_level),
what))
local_mc.set_log_command(dummy_log)
local_mc.build_structure(None, None)
conn_str = "tcp://{}:{:d}".format(global_config["HOST"],
global_config["COMMAND_PORT"])
arg_list = global_config["ARGUMENTS"]
com_name = arg_list.pop(0)
if com_name in local_mc:
srv_com = server_command.srv_command(command=com_name)
for src_key, dst_key in [("HOST", "host"), ("COMMAND_PORT", "port")]:
srv_com[dst_key] = global_config[src_key]
com_struct = local_mc[com_name]
try:
cur_ns, rest = com_struct.handle_commandline(arg_list)
except ValueError as what:
ret = ExtReturn(limits.mon_STATE_CRITICAL,
"error parsing: {}".format(what.args[1]))
else:
# see also struct.py in collrelay
if hasattr(cur_ns, "arguments"):
for arg_index, arg in enumerate(cur_ns.arguments):
srv_com["arguments:arg{:d}".format(arg_index)] = arg
srv_com["arguments:rest"] = " ".join(rest)
for key, value in vars(cur_ns).items():
srv_com["namespace:{}".format(key)] = value
result = net_tools.ZMQConnection(
"{}:{:d}".format(global_config["IDENTITY_STRING"],
os.getpid()),
timeout=global_config["TIMEOUT"],
).add_connection(
conn_str,
srv_com,
immediate=True,
)
if result:
if global_config["COMMAND_PORT"] == InstanceXML(
quiet=True).get_port_dict(
icswServiceEnum.host_monitoring, command=True):
error_result = result.xpath(".//ns:result[@state != '0']",
smart_strings=False)
if error_result:
error_result = error_result[0]
ret = ExtReturn(int(error_result.attrib["state"]),
error_result.attrib["reply"])
else:
if hasattr(com_struct, "interpret"):
ret = ExtReturn.get_ext_return(
com_struct.interpret(result, cur_ns))
else:
_result = result.xpath(".//ns:result",
smart_strings=False)[0]
ret = ExtReturn(
server_command.srv_reply_to_nag_state(
int(_result.attrib["state"])),
result.attrib["reply"])
else:
ret_str, ret_state = result.get_log_tuple()
ret = ExtReturn(
server_command.srv_reply_to_nag_state(ret_state),
ret_str)
else:
ret = ExtReturn(limits.mon_STATE_CRITICAL, "timeout")
else:
import difflib
c_matches = difflib.get_close_matches(com_name, list(local_mc.keys()))
if c_matches:
cm_str = "close matches: {}".format(", ".join(c_matches))
else:
cm_str = "no matches found"
ret = ExtReturn(limits.mon_STATE_CRITICAL,
"unknown command {}, {}".format(com_name, cm_str))
if ret.ascii_chunk:
print("Ignoring ascii_chunk with {:d} bytes".format(
len(ret.ascii_chunk)))
# output
print(ret.ret_str)
return ret.ret_state
def scan_network_info(self, dev_com, scan_dev):
hm_port = InstanceXML(quiet=True).get_port_dict(
icswServiceEnum.host_monitoring, command=True)
res_node = ResultNode()
strict_mode = True if int(dev_com.get("strict_mode")) else False
modify_peering = True if int(dev_com.get("modify_peering")) else False
scan_address = dev_com.get("scan_address")
self.log(
"scanning network for device '{}' ({:d}), scan_address is '{}', strict_mode is {}"
.format(
str(scan_dev),
scan_dev.pk,
scan_address,
"on" if strict_mode else "off",
))
zmq_con = net_tools.ZMQConnection("server:{}".format(
process_tools.get_machine_name()),
context=self.zmq_context)
conn_str = "tcp://{}:{:d}".format(
scan_address,
hm_port,
)
self.log("connection_str for {} is {}".format(str(scan_dev), conn_str))
zmq_con.add_connection(
conn_str,
server_command.srv_command(command="network_info"),
multi=True)
res_list = zmq_con.loop()
self.log("length of result list: {:d}".format(len(res_list)))
num_taken, num_ignored = (0, 0)
nds_list = netdevice_speed.objects.filter(
Q(speed_bps__in=[1000000000, 100000000])).order_by(
"-speed_bps", "-full_duplex", "-check_via_ethtool")
default_nds = nds_list[0]
self.log("default nds is {}".format(str(default_nds)))
for _idx, (result, target_dev) in enumerate(zip(res_list, [scan_dev])):
self.log("device {} ...".format(str(target_dev)))
res_state = -1 if result is None else int(
result["result"].attrib["state"])
if res_state:
# num_errors += 1
if res_state == -1:
res_node.error("{}: no result".format(str(target_dev)))
else:
res_node.error("{}: error {:d}: {}".format(
str(target_dev), int(result["result"].attrib["state"]),
result["result"].attrib["reply"]))
else:
try:
bridges = result["bridges"]
networks = result["networks"]
except:
res_node.error(
"{}: error missing keys in dict".format(target_dev))
else:
# clear current network
with transaction.atomic():
sid = transaction.savepoint()
# store current peers
_peers = [
_obj.store_before_delete(target_dev)
for _obj in peer_information.objects.filter(
Q(s_netdevice__in=target_dev.netdevice_set.all(
)) | Q(d_netdevice__in=target_dev.
netdevice_set.all()))
]
_old_peer_dict = {}
for _old_peer in _peers:
_old_peer_dict.setdefault(_old_peer["my_name"],
[]).append(_old_peer)
self.log("removing current network devices")
target_dev.netdevice_set.all().delete()
all_ok = True
exc_dict = {}
_all_devs = set(networks)
_br_devs = set(bridges)
# build bond dict
bond_dict = {}
for dev_name in _all_devs:
_struct = networks[dev_name]
if "MASTER" in _struct["flags"]:
bond_dict[dev_name] = {"slaves": []}
for dev_name in _all_devs:
_struct = networks[dev_name]
if "SLAVE" in _struct["flags"]:
master_name = _struct["features"]["master"]
bond_dict[master_name]["slaves"].append(
dev_name)
NDStruct.setup(self, target_dev, default_nds,
bond_dict)
for dev_name in sorted(
list(_all_devs & _br_devs)) + sorted(
list(_all_devs - _br_devs)):
if any([
dev_name.startswith(_ignore_pf)
for _ignore_pf in IGNORE_LIST
]):
self.log("ignoring device {}".format(dev_name))
num_ignored += 1
continue
_struct = networks[dev_name]
cur_nd = NDStruct(dev_name, _struct,
bridges.get(dev_name, None))
try:
cur_nd.create()
except (NoMatchingNetworkDeviceTypeFoundError,
NoMatchingNetworkFoundError) as exc:
_name = exc.__class__.__name__
self.log(
"caught {} for {}".format(_name, dev_name),
logging_tools.LOG_LEVEL_ERROR)
exc_dict.setdefault(_name, []).append(dev_name)
all_ok = False
except:
err_str = "error creating netdevice {}: {}".format(
dev_name, process_tools.get_except_info())
if strict_mode:
res_node.error(err_str)
for _log in process_tools.icswExceptionInfo(
).log_lines:
self.log(" {}".format(_log),
logging_tools.LOG_LEVEL_CRITICAL)
all_ok = False
else:
num_taken += 1
if cur_nd.nd is not None and cur_nd.nd.devname in _old_peer_dict:
# relink peers
for _peer in _old_peer_dict[cur_nd.nd.devname]:
_new_peer = peer_information.create_from_store(
_peer, cur_nd.nd)
del _old_peer_dict[cur_nd.nd.devname]
if all_ok:
NDStruct.handle_bonds()
if exc_dict:
for key in sorted(exc_dict.keys()):
res_node.error("{} for {}: {}".format(
key,
logging_tools.get_plural(
"netdevice", len(exc_dict[key])),
", ".join(sorted(exc_dict[key]))))
if list(_old_peer_dict.keys()):
_err_str = "not all peers migrated: {}".format(
", ".join(list(_old_peer_dict.keys())))
if strict_mode:
res_node.error(_err_str)
all_ok = False
else:
res_node.warn(_err_str)
[
NDStruct.dict[_bridge_name].link_bridge_slaves()
for _bridge_name in _br_devs
& set(NDStruct.dict.keys())
]
if not all_ok and strict_mode:
self.log(
"rolling back to savepoint because strict_mode is enabled",
logging_tools.LOG_LEVEL_WARN)
num_taken -= target_dev.netdevice_set.all().count()
transaction.savepoint_rollback(sid)
else:
transaction.savepoint_commit(sid)
if num_taken:
res_node.ok("{} taken".format(
logging_tools.get_plural("netdevice", num_taken)))
if num_ignored:
res_node.ok("{} ignored".format(
logging_tools.get_plural("netdevice", num_ignored)))
return res_node
def scan_system_info(self, dev_com, scan_dev):
hm_port = InstanceXML(quiet=True).get_port_dict(
icswServiceEnum.host_monitoring, command=True)
res_node = ResultNode()
s_time = time.time()
# global config should be a kwarg
self.get_route_to_devices(self.global_config, [scan_dev])
self.log(
"scanning system for device '{}' ({:d}), scan_address is '{}'".
format(
str(scan_dev),
scan_dev.pk,
scan_dev.target_ip,
))
zmq_con = net_tools.ZMQConnection("server:{}".format(
process_tools.get_machine_name()),
context=self.zmq_context)
conn_str = "tcp://{}:{:d}".format(
scan_dev.target_ip,
hm_port,
)
self.log("connection_str for {} is {}".format(str(scan_dev), conn_str))
zmq_con.add_connection(
conn_str,
server_command.srv_command(command="sysinfo"),
multi=True,
)
res_list = zmq_con.loop()
e_time = time.time()
for _idx, (_dev, _res) in enumerate(zip([scan_dev], res_list)):
if _res:
_to_save = []
for _dt, _kwargs, _cbf in [
("lstopo_dump", {}, self._interpret_lstopo),
("dmi_dump", {}, self._interpret_dmiinfo),
("pci_dump", {
"marshal": True
}, self._interpret_pciinfo),
]:
if _dt in _res:
_data = _cbf(
server_command.decompress(_res["*{}".format(_dt)],
**_kwargs))
if _data is not None:
_to_save.append((_dt.split("_")[0], _data))
else:
self.log("missing {} in result".format(_dt),
logging_tools.LOG_LEVEL_WARN)
if _to_save:
_highest_run = _dev.deviceinventory_set.all().order_by(
"-idx")
if _highest_run.count():
_highest_run = _highest_run[0].run_idx
else:
_highest_run = 0
_highest_run += 1
self.log("{} trees to save ({}), run_idx is {:d}".format(
logging_tools.get_plural("inventory tree",
len(_to_save)),
", ".join(sorted([_tuple[0] for _tuple in _to_save])),
_highest_run,
))
for _name, _tree in _to_save:
inv = DeviceInventory(device=_dev,
inventory_type=_name,
run_idx=_highest_run,
value=etree.tostring(_tree))
inv.save()
else:
self.log("no trees to save", logging_tools.LOG_LEVEL_WARN)
DeviceLogEntry.new(
device=_dev,
source=global_config["LOG_SOURCE_IDX"],
level=logging_tools.LOG_LEVEL_OK,
text="system scan took {}, {:d} trees".format(
logging_tools.get_diff_time_str(e_time - s_time, ),
len(_to_save),
),
)
else:
DeviceLogEntry.new(
device=_dev,
source=global_config["LOG_SOURCE_IDX"],
level=logging_tools.LOG_LEVEL_ERROR,
text="error scanning system (took {})".format(
logging_tools.get_diff_time_str(e_time - s_time, )),
)
res_node.ok("system scanned")
return res_node