def get_route_to_devices(self, dev_list):
src_dev = device.objects.get(Q(pk=global_config["SERVER_IDX"]))
src_nds = src_dev.netdevice_set.all().values_list("pk", flat=True)
self.log(u"device list: {}".format(u", ".join(
[unicode(cur_dev) for cur_dev in dev_list])))
router_obj = config_tools.RouterObject(self.log)
for cur_dev in dev_list:
routes = router_obj.get_ndl_ndl_pathes(
src_nds,
cur_dev.netdevice_set.all().values_list("pk", flat=True),
only_endpoints=True,
add_penalty=True)
cur_dev.target_ip = None
if routes:
for route in sorted(routes):
found_ips = net_ip.objects.filter(Q(netdevice=route[2]))
if found_ips:
cur_dev.target_ip = found_ips[0].ip
break
if cur_dev.target_ip:
self.log("contact device {} via {}".format(
unicode(cur_dev), cur_dev.target_ip))
else:
self.log(
u"no route to device {} found".format(unicode(cur_dev)),
logging_tools.LOG_LEVEL_ERROR)
del router_obj
def post(self, request):
_post = request.POST
to_dev_pk = int(_post["device"])
to_dev = device.objects.prefetch_related(
"netdevice_set__net_ip_set__network__network_type"
).get(
Q(pk=to_dev_pk)
)
# from-device is where virtual desktop client config is set
server_by_type = config_tools.server_check(server_type="virtual_desktop_client")
from_dev = server_by_type.effective_device
if from_dev is None:
# fall back to local device
cur_routing = routing.SrvTypeRouting(force=True)
from_dev = cur_routing.local_device
from_server_check = config_tools.server_check(device=from_dev, config=None, server_type="node")
to_server_check = config_tools.server_check(device=to_dev, config=None, server_type="node")
# calc route to it and use target ip
_router = config_tools.RouterObject(logger)
route = from_server_check.get_route_to_other_device(_router, to_server_check, allow_route_to_other_networks=True, prefer_production_net=True)
if route:
ip = route[0][3][1][0]
else:
ip = "127.0.0.1" # try fallback (it might not work, but it will not make things more broken)
return HttpResponse(json.dumps({"ip": ip}), content_type="application/json")
def init(srv_process):
# cluster device group
ConfigControl.cdg = device.objects.get(Q(device_group__cluster_device_group=True))
ConfigControl.srv_process = srv_process
ConfigControl.cc_log("init config_control")
ConfigControl.__cc_dict = {}
ConfigControl.__lut_dict = {}
ConfigControl.__queue_dict = {}
ConfigControl.__queue_num = 0
ConfigControl.pending_config_requests = {}
ConfigControl.done_config_requests = {}
ConfigControl.router_last_update = time.time() - 3600
ConfigControl.router_obj = config_tools.RouterObject(ConfigControl.cc_log)
def process_init(self):
global_config.enable_pm(self)
self.__log_template = logging_tools.get_logger(
global_config["LOG_NAME"],
global_config["LOG_DESTINATION"],
context=self.zmq_context,
)
# close database connection
db_tools.close_connection()
self.router_obj = config_tools.RouterObject(self.log)
self.config_src = LogSource.objects.get(
Q(pk=global_config["LOG_SOURCE_IDX"]))
self.register_func("generate_config", self._generate_config)
# for requests from config_control
self.register_func("complex_request", self._complex_request)
build_client.init(self)
def process_init(self):
global_config.enable_pm(self)
self.__log_template = logging_tools.get_logger(
global_config["LOG_NAME"],
global_config["LOG_DESTINATION"],
context=self.zmq_context,
)
# close database connection
db_tools.close_connection()
MotherSimpleCommand.setup(self)
self.router_obj = config_tools.RouterObject(self.log)
self.snmp_sink = SNMPSink(self.log)
self.sc = config_tools.icswServerCheck(
service_type_enum=icswServiceEnum.mother_server)
self.register_func("delay_command", self._delay_command)
self.register_func("hard_control", self._hard_control)
self.register_func("snmp_finished", self._snmp_finished)
self.register_timer(self._check_commands, 10)
HardControlCommand.setup(self)
self.send_pool_message("register_return",
"command",
target="snmp_process")
def process_init(self):
global_config.enable_pm(self)
self.__log_template = logging_tools.get_logger(
global_config["LOG_NAME"],
global_config["LOG_DESTINATION"],
context=self.zmq_context,
)
db_tools.close_connection()
self.router_obj = config_tools.RouterObject(self.log)
self.register_func("check_for_slaves", self._check_for_slaves)
self.register_func("check_for_redistribute",
self._check_for_redistribute)
self.register_func("build_info", self._build_info)
self.register_func("slave_info", self._slave_info)
self.register_func("get_sys_info", self._get_sys_info)
self.register_func("mon_process_handling", self._mon_process_handling)
# handling external commands (acks and so on)
self.register_func("ext_command", self._ext_command)
self.__build_in_progress, self.__build_version = (False, 0)
self.__master_config = None
# this used to be just set in _check_for_slaves, but apparently check_for_redistribute can be called before that
self.__slave_configs, self.__slave_lut = ({}, {})
def reconnect_to_clients(self):
router_obj = config_tools.RouterObject(self.log)
self.log("reconnecting to {}".format(
logging_tools.get_plural("client", len(Client.name_set))))
all_servers = config_tools.device_with_config("package_server")
if "package_server" not in all_servers:
self.log("no package_server defined, strange...",
logging_tools.LOG_LEVEL_ERROR)
else:
_pserver = all_servers["package_server"][0]
if _pserver.effective_device.pk != global_config["SERVER_IDX"]:
self.log(
"effective_device pk differs from SERVER_IDX: {:d} != {:d}"
.format(_pserver.effective_device.pk,
global_config["SERVER_IDX"]),
logging_tools.LOG_LEVEL_ERROR)
else:
for target_name in Client.name_set:
cur_c = Client.get(target_name)
dev_sc = config_tools.server_check(device=cur_c.device,
config="",
server_type="node",
fetch_network_info=True)
act_routing_info = _pserver.get_route_to_other_device(
router_obj,
dev_sc,
allow_route_to_other_networks=True,
prefer_production_net=True,
)
if act_routing_info:
_ip = act_routing_info[0][3][1][0]
self.log("found routing_info for {}, IP is {}".format(
unicode(cur_c.device), _ip))
self.connect_client(cur_c.device, _ip)
# self.send_reply(cur_c.uid, server_command.srv_command(command="hello"))
else:
self.log("no routing_info found for {}".format(
unicode(cur_c.device)))
def _get_router_obj(self):
return config_tools.RouterObject(self.log)
def post(self, request):
r_obj = config_tools.RouterObject(self.log)
return HttpResponse(json.dumps(r_obj.get_clusters()),
content_type="application/json")
def _call(self, cur_inst):
file_list = []
server_idxs = [self.server_idx]
# get additional idx if host is virtual server
sc_result = config_tools.icswServerCheck(service_type_enum=icswServiceEnum.cluster_server).get_result()
if sc_result.effective_device is not None and sc_result.effective_device.idx != self.server_idx:
server_idxs.append(sc_result.effective_device.idx)
# recognize for which devices i am responsible
dev_r = cluster_location.DeviceRecognition()
server_idxs = list(set(server_idxs) | set(dev_r.device_dict.keys()))
# get all peers to local machine and local netdevices
my_idxs = netdevice.objects.exclude(
Q(enabled=False)
).filter(
Q(device__in=server_idxs) &
Q(device__enabled=True) &
Q(device__device_group__enabled=True)
).values_list("pk", flat=True)
# ref_table
route_obj = config_tools.RouterObject(cur_inst.log)
all_paths = []
for s_ndev in my_idxs:
all_paths.extend(list(networkx.shortest_path(route_obj.nx, s_ndev, weight="weight").values()))
# pprint.pprint(all_paths)
nd_lut = {
cur_nd.pk: cur_nd for cur_nd in netdevice.objects.all().select_related(
"device"
).prefetch_related(
"net_ip_set", "net_ip_set__network", "net_ip_set__domain_tree_node"
)
}
# fetch key-information
ssh_vars = device_variable.objects.filter(Q(name="ssh_host_rsa_key_pub")).select_related("device")
rsa_key_dict = {}
for _db_rec in ssh_vars:
pass
# read pre/post lines from /etc/hosts
pre_host_lines, post_host_lines = ([], [])
# parse pre/post host_lines
try:
host_lines = [line.strip() for line in codecs.open(ETC_HOSTS_FILENAME, "r", "utf-8").read().split("\n")]
except:
self.log(
"error reading / parsing {}: {}".format(
ETC_HOSTS_FILENAME,
process_tools.get_except_info()),
logging_tools.LOG_LEVEL_ERROR)
else:
mode, any_modes_found = (0, False)
for line in host_lines:
if line.lower().startswith("### aeh-start-pre"):
mode, any_modes_found = (1, True)
elif line.lower().startswith("### aeh-start-post"):
mode, any_modes_found = (2, True)
elif line.lower().startswith("### aeh-end"):
mode, any_modes_found = (0, True)
else:
if mode == 1:
pre_host_lines.append(line)
elif mode == 2:
post_host_lines.append(line)
if not any_modes_found:
self.log(
"no ### aeh-.* stuff found in {}, copying to {}.orig".format(
ETC_HOSTS_FILENAME, ETC_HOSTS_FILENAME
)
)
try:
pass
except:
self.log(
"error writing {}.orig: {}".format(
ETC_HOSTS_FILENAME,
process_tools.get_except_info()
)
)
# mapping from device_name to all names for ssh_host_keys
name_dict = {}
# ip dictionary
ip_dict = {}
# min_target_dict
min_target_dict = {}
for cur_path in all_paths:
min_value = route_obj.get_penalty(cur_path)
target_nd = nd_lut[cur_path[-1]]
min_target_dict[target_nd] = min(min_target_dict.get(target_nd, 999999999), min_value)
tl_dtn = domain_tree_node.objects.get(Q(depth=0))
for cur_path in all_paths:
target_nd = nd_lut[cur_path[-1]]
min_value = min_target_dict[target_nd]
for cur_ip in nd_lut[cur_path[-1]].net_ip_set.all():
# get names
host_names = []
cur_dtn = cur_ip.domain_tree_node or tl_dtn
if not (cur_ip.alias.strip() and cur_ip.alias_excl):
host_names.append("{}{}".format(target_nd.device.name, cur_dtn.node_postfix))
host_names.extend(["{}".format(cur_entry) for cur_entry in cur_ip.alias.strip().split()])
if "localhost" in [x.split(".")[0] for x in host_names]:
host_names = [host_name for host_name in host_names if host_name.split(".")[0] == "localhost"]
if cur_dtn.full_name:
if cur_dtn.create_short_names:
# also create short_names
out_names = (
" ".join(
[
"{}.{} {}".format(host_name, cur_dtn.full_name, host_name) for host_name in host_names if not host_name.count(".")
]
)
).split()
else:
# only print the long names
out_names = ["{}.{}".format(host_name, cur_dtn.full_name) for host_name in host_names if not host_name.count(".")]
else:
if cur_dtn.create_short_names:
# also create short_names
out_names = (" ".join(["{}".format(host_name) for host_name in host_names if not host_name.count(".")])).split()
else:
# only print the long names
out_names = ["{}".format(host_name) for host_name in host_names if not host_name.count(".")]
# add names with dot
out_names.extend([host_name for host_name in host_names if host_name.count(".")])
# name_dict without localhost
name_dict.setdefault(
target_nd.device.name, []
).extend(
[
out_name for out_name in out_names if out_name not in name_dict[target_nd.device.name] and not out_name.startswith("localhost")
]
)
ip_dict.setdefault(cur_ip.ip, [])
if out_names not in [entry[1] for entry in ip_dict[cur_ip.ip]]:
if cur_ip.ip != "0.0.0.0":
ip_dict[cur_ip.ip].append((min_value, out_names))
# out_list
loc_dict = {}
for ip, h_list in ip_dict.items():
all_values = sorted([entry[0] for entry in h_list])
if all_values:
min_value = all_values[0]
out_names = []
for val in all_values:
for _act_val, act_list in [(x_value, x_list) for x_value, x_list in h_list if x_value == val]:
out_names.extend([value for value in act_list if value not in out_names])
# print min_value, ip, out_names
loc_dict.setdefault(min_value, []).append([ipvx_tools.IPv4(ip)] + out_names)
pen_list = sorted(loc_dict.keys())
out_file = []
for pen_value in pen_list:
act_out_list = logging_tools.NewFormList()
for entry in sorted(loc_dict[pen_value]):
act_out_list.append(
[
logging_tools.form_entry(entry[0])
] + [
logging_tools.form_entry(cur_e) for cur_e in entry[1:]
]
)
host_lines = str(act_out_list).split("\n")
out_file.extend(
[
"# penalty {:d}, {}".format(
pen_value,
logging_tools.get_plural("host entry", len(host_lines))
),
""
] + host_lines + [""]
)
if not os.path.isdir(GROUP_DIR):
try:
os.makedirs(GROUP_DIR)
except:
pass
if os.path.isdir(GROUP_DIR):
# remove old files
for file_name in os.listdir(GROUP_DIR):
try:
os.unlink(os.path.join(GROUP_DIR, file_name))
except:
pass
# get all devices with netips
all_devs = device.objects.filter(
Q(enabled=True) &
Q(device_group__enabled=True) &
Q(netdevice__net_ip__ip__contains=".")
).values_list(
"name",
"device_group__name"
).order_by(
"device_group__name",
"name"
)
dg_dict = {}
for dev_name, dg_name in all_devs:
dg_dict.setdefault(dg_name, []).append(dev_name)
for file_name, content in dg_dict.items():
codecs.open(
os.path.join(GROUP_DIR, file_name),
"w",
"utf-8"
).write("\n".join(sorted(set(content)) + [""]))
file_list.append(ETC_HOSTS_FILENAME)
codecs.open(ETC_HOSTS_FILENAME, "w+", "utf-8").write(
"\n".join(
[
"### AEH-START-PRE insert pre-host lines below"
] + pre_host_lines +
[
"### AEH-END-PRE insert pre-host lines above",
""
] + out_file +
[
"",
"### AEH-START-POST insert post-host lines below"
] + post_host_lines +
[
"### AEH-END-POST insert post-host lines above",
""
]
)
)
# write known_hosts_file
if os.path.isdir(os.path.dirname(SSH_KNOWN_HOSTS_FILENAME)):
skh_f = open(SSH_KNOWN_HOSTS_FILENAME, "w")
for ssh_key_node in sorted(rsa_key_dict.keys()):
skh_f.write(
"{} {}\n".format(
",".join(name_dict.get(ssh_key_node, [ssh_key_node])), rsa_key_dict[ssh_key_node]
)
)
skh_f.close()
file_list.append(SSH_KNOWN_HOSTS_FILENAME)
cur_inst.srv_com.set_result(
"wrote {}".format(", ".join(sorted(file_list)))
)
