def init(r_process, backlog_size, timeout, verbose): HostConnection.relayer_process = r_process # 2 queues for 0MQ and tcp, 0MQ is (True, conn_str), TCP is (False, conn_str) HostConnection.hc_dict = {} # lut to map message_ids to host_connections HostConnection.message_lut = {} HostConnection.backlog_size = backlog_size HostConnection.timeout = timeout HostConnection.verbose = verbose HostConnection.g_log( "backlog size is {:d}, timeout is {:d}, verbose is {}".format( HostConnection.backlog_size, HostConnection.timeout, str(HostConnection.verbose), ) ) # router socket id_str = "relayer_rtr_{}".format(process_tools.get_machine_name()) new_sock = process_tools.get_socket( HostConnection.relayer_process.zmq_context, "ROUTER", identity=id_str, linger=0, sndhwm=HostConnection.backlog_size, rcvhwm=HostConnection.backlog_size, backlog=HostConnection.backlog_size, immediate=True, ) HostConnection.zmq_socket = new_sock HostConnection.relayer_process.register_poller(new_sock, zmq.POLLIN, HostConnection.get_result) # @UndefinedVariable
def process_init(self): self.__log_template = logging_functions.get_logger( config_store.ConfigStore("client", quiet=True), "{}/{}".format( process_tools.get_machine_name(), self.global_config["LOG_NAME"], ), process_name=self.name, ) self.register_func("connection", self._connection) # clear flag for extra twisted thread self.__extra_twisted_threads = 0 # print self.start_kwargs if self.start_kwargs.get("icmp", True): self.icmp_protocol = HMIcmpProtocol( self, self.__log_template, debug=self.global_config["DEBUG"] ) # reactor.listenWith(icmp_twisted.icmp_port, self.icmp_protocol) # reactor.listen_ICMP(self.icmp_protocol) self.register_func("ping", self._ping) else: self.icmp_protocol = None self.register_func("resolved", self._resolved) self.register_timer(self._check_timeout, 5) self.__ip_re = re.compile("^\d+\.\d+\.\d+\.\d+$") self.__pending_id, self.__pending_dict = (0, {})
def process_init(self): self.__log_template = logging_functions.get_logger( config_store.ConfigStore("client", quiet=True), "{}/{}".format( process_tools.get_machine_name(), self.global_config["LOG_NAME"], ), process_name=self.name, ) self.__watcher = inotify_tools.InotifyWatcher() # was INOTIFY_IDLE_TIMEOUT in self.global_config, now static self.__idle_timeout = 5 # self.__watcher.add_watcher("internal", "/etc/sysconfig/host-monitoring.d", inotify_tools.IN_CREATE | inotify_tools.IN_MODIFY, self._trigger) self.__file_watcher_dict = {} self.__target_dict = {} # self.register_func("connection", self._connection) self.send_pool_message("register_callback", "register_file_watch", "fw_handle") self.send_pool_message("register_callback", "unregister_file_watch", "fw_handle") self.register_exception("term_error", self._sigint) self.allow_signal(15) self.register_func("fw_handle", self._fw_handle) # register watcher fd with 0MQ poller self.register_poller(self.__watcher._fd, zmq.POLLIN, self._inotify_check) self.log("idle_timeout is {:d}".format(self.__idle_timeout)) self.register_timer(self._fw_timeout, 1000)
def init(cls, r_process, backlog_size, timeout, zmq_discovery): cls.relayer_process = r_process # 2 queues for 0MQ and tcp, 0MQ is (True, conn_str), TCP is (False, conn_str) cls.hc_dict = {} # lut to map message_ids to host_connections cls.message_lut = {} cls.backlog_size = backlog_size cls.timeout = timeout cls.g_log( "backlog size is {:d}, timeout is {:d}".format( cls.backlog_size, cls.timeout, ) ) # router socket id_str = "relayer_rtr_{}".format( process_tools.get_machine_name() ) new_sock = process_tools.get_socket( cls.relayer_process.zmq_context, "ROUTER", identity=id_str, linger=0, sndhwm=cls.backlog_size, rcvhwm=cls.backlog_size, backlog=cls.backlog_size, immediate=True, ) cls.zmq_socket = new_sock cls.relayer_process.register_poller(new_sock, zmq.POLLIN, cls.get_result) # ZMQDiscovery instance cls.zmq_discovery = zmq_discovery
def _insert_bg_job(cmd, cause, obj): if getattr(obj, "_no_bg_job", False): # used in boot_views return # create entry to be handled by the cluster-server # get local device, key is defined in routing.py # see routing.py, SrvTypeRouting definition _routing_key = "_WF_ROUTING_{}".format(settings.ICSW_CACHE_KEY) _resolv_dict = cache.get(_routing_key) if _resolv_dict: _r_dict = json.loads(_resolv_dict) if "_local_device" in _r_dict: _local_pk = _r_dict["_local_device"][0] else: _local_pk = 0 else: try: _local_pk = device.objects.get( Q(name=process_tools.get_machine_name())).pk except device.DoesNotExist: _local_pk = 0 # we need local_pk and a valid user (so we have to be called via webfrontend) if _local_pk and thread_local_middleware().user and isinstance( thread_local_middleware().user, user): create_bg_job(_local_pk, thread_local_middleware().user, cmd, cause, obj) # init if not already done if not hasattr(thread_local_obj, "num_bg_jobs"): thread_local_obj.num_bg_jobs = 1 else: thread_local_obj.num_bg_jobs += 1
def _resolve_address(self, target): # to avoid loops in the 0MQ connection scheme (will result to nasty asserts) if target in self.__forward_lut: ip_addr = self.__forward_lut[target] else: orig_target = target if target.lower() in ["localhost", "127.0.0.1", "localhost.localdomain"]: target = process_tools.get_machine_name() # step 1: resolve to ip ip_addr = socket.gethostbyname(target) try: # step 2: try to get full name full_name, _aliases, _ip_addrs = socket.gethostbyaddr(ip_addr) except: # forget it pass else: # resolve full name try: ip_addr = socket.gethostbyname(full_name) except: self.log("error looking up {}: {}".format(full_name, process_tools.get_except_info()), logging_tools.LOG_LEVEL_ERROR) if ip_addr not in self.__ip_lut: self.log("resolved %s to %s" % (target, ip_addr)) self.__ip_lut[ip_addr] = target self.__forward_lut[target] = ip_addr self.log("ip resolving: %s -> %s" % (target, ip_addr)) if orig_target != target: self.__forward_lut[orig_target] = ip_addr self.log("ip resolving: %s -> %s" % (orig_target, ip_addr)) return ip_addr
def _check_error_dict(self, force=False): c_name = process_tools.get_cluster_name() mails_sent = 0 s_time = time.time() ep_dels = [] for ep, es in self.__eg_dict.items(): t_diff = s_time - es["last_update"] if force or (t_diff < 0 or t_diff > 60): subject = "Python error for pid {:d} on {}@{} ({}, {})".format( ep, process_tools.get_fqdn()[0], c_name, process_tools.get_machine_name(), clusterid.get_cluster_id() or "N/A", ) err_lines = "".join(es["error_str"]).split("\n") msg_body = "\n".join( ["Processinfo {}".format(self._get_process_info(es))] + [ "{:3d} {}".format(line_num + 1, line) for line_num, line in enumerate(err_lines) ]) if self.CC.CS["log.send.errormails"]: self._send_mail(subject, msg_body) mails_sent += 1 ep_dels.append(ep) for epd in ep_dels: del self.__eg_dict[epd] e_time = time.time() if mails_sent: self.log("Sent {} in {}".format( logging_tools.get_plural("mail", mails_sent), logging_tools.get_diff_time_str(e_time - s_time)))
def _add_info_parser(self, sub_parser, server_mode): _mach_name = process_tools.get_machine_name(short=True) parser = sub_parser.add_parser("info", help="show command help") parser.set_defaults(subcom="info", execute=self._execute) _choices = ["host-monitoring"] if server_mode: _choices.extend(["cluster-server"]) parser.add_argument( "--subsys", type=str, default=_choices[0], choices=_choices, help="show command info for given subsystem [%(default)s]") parser.add_argument("--detail", default=False, action="store_true", help="show detailed help [%(default)s]") parser.add_argument("--no-overview", default=True, dest="overview", action="store_false", help="show overview [%(default)s]") parser.add_argument( "--update-json", default=False, action="store_true", help="update json file for all defined commands [%(default)s]") parser.add_argument("args", nargs="*") return parser
def __init__(self, **kwargs): self.short_host_name = kwargs.get("short_host_name", process_tools.get_machine_name()) try: self.device = device.all_enabled.get(Q(name=self.short_host_name)) except device.DoesNotExist: self.device = None # get IP-adresses (from IP) self.local_ips = list( net_ip.objects.filter( Q(netdevice__device__name=self.short_host_name) & Q(netdevice__device__enabled=True) & Q(netdevice__device__device_group__enabled=True)). values_list("ip", flat=True)) # get configured IP-Adresses ipv4_dict = { cur_if_name: [ip_tuple["addr"] for ip_tuple in value[2]] for cur_if_name, value in [(if_name, netifaces.ifaddresses(if_name)) for if_name in netifaces.interfaces()] if 2 in value } # remove loopback addresses self_ips = [ _ip for _ip in sum(list(ipv4_dict.values()), []) if not _ip.startswith("127.") ] self.ip_lut = {} self.ip_r_lut = {} if self_ips: _do = device.all_enabled # get IPs self.device_dict = { cur_dev.pk: cur_dev for cur_dev in _do.filter(Q( netdevice__net_ip__ip__in=self_ips)).prefetch_related( "netdevice_set__net_ip_set__network__network_type") } for _ip in self.local_ips: self.ip_lut[_ip] = self.device self.ip_r_lut[self.device] = self.local_ips # build lut for _dev in self.device_dict.values(): # gather all ips _dev_ips = sum([ list(_ndev.net_ip_set.all()) for _ndev in _dev.netdevice_set.all() ], []) # filter for valid ips (no loopback addresses) _dev_ips = [ _ip.ip for _ip in _dev_ips if _ip.network.network_type.identifier != "l" and not _ip.ip.startswith("127.") ] for _dev_ip in _dev_ips: self.ip_lut[_dev_ip] = _dev self.ip_r_lut[_dev] = _dev_ips else: self.device_dict = {}
def _add_lw_parser(self, sub_parser): client_cs = config_store.ConfigStore("client", quiet=True) _mach_name = process_tools.get_machine_name(short=True) parser = sub_parser.add_parser("logwatch", help="watch icsw logs") parser.set_defaults(subcom="status", execute=self._execute) # get logroot from config_store parser.add_argument("--root", type=str, default=client_cs["log.logdir"], help="root directory [%(default)s]") parser.add_argument("--machine", type=str, default=_mach_name, help="machine to use [%(default)s]") parser.add_argument("-n", type=int, default=400, help="show latest [%(default)d] lines") parser.add_argument("--format", type=str, default="%a %b %d %H:%M:%S %Y", help="argument for parsing loglines [%(default)s]") parser.add_argument( "-f", dest="follow", default=True, action="store_true", help="enable follow mode, always enabled [%(default)s]") parser.add_argument("-F", dest="follow", default=True, action="store_false", help="disable follow mode") parser.add_argument("--system-filter", type=str, default=".*", help="regexp filter for system [%(default)s]") parser.add_argument("--with-nodes", default=False, action="store_true", help="add node logs [%(default)s]") parser.add_argument("--node-filter", type=str, default=".*", help="regexp filter for nodes [%(default)s]") parser.add_argument("--verbose", default=False, action="store_true", help="enable verbose mode [%(default)s]") parser.add_argument("--show-unparseable", default=False, action="store_true", help="show unparseable lines [%(default)s]") parser.add_argument( "filter", nargs="*", type=str, help="list of regexp filter for system [%(default)s]") return parser
def open_socket(zmq_context): send_sock = zmq_context.socket(zmq.DEALER) send_sock.setsockopt_string( zmq.IDENTITY, "{}:syslog_scan".format(process_tools.get_machine_name())) send_sock.setsockopt(zmq.LINGER, 0) send_sock.connect("tcp://localhost:8000") return send_sock
def _resolve_address_resolve(self, target): # to avoid loops in the 0MQ connection scheme (will result to nasty asserts) if target in self.__forward_lut: ip_addr = self.__forward_lut[target] else: orig_target = target if target.lower().startswith( "localhost") or target.lower().startswith("127.0.0."): # map localhost to something 0MQ can handle target = process_tools.get_machine_name() # step 1: resolve to ip try: ip_addr = socket.gethostbyname(target) except: self.log( "cannot resolve target '{}': {}".format( target, process_tools.get_except_info()), logging_tools.LOG_LEVEL_CRITICAL) raise try: # step 2: try to get full name full_name, aliases, ip_addrs = socket.gethostbyaddr(ip_addr) except: # forget it pass else: # resolve full name self.log("ip_addr {} resolved to '{}' ({}), {}".format( ip_addr, full_name, ", ".join(aliases) or "N/A", ", ".join(ip_addrs) or "N/A")) try: new_ip_addr = socket.gethostbyname(full_name) except: self.log( "cannot resolve full_name '{}': {}".format( full_name, process_tools.get_except_info()), logging_tools.LOG_LEVEL_CRITICAL) raise else: self.log( "full_name {} resolves back to {} (was: {})".format( full_name, new_ip_addr, ip_addr), logging_tools.LOG_LEVEL_OK if new_ip_addr == ip_addr else logging_tools.LOG_LEVEL_ERROR) # should we use the new ip_addr ? dangerous, FIXME # ip_addr = new_ip_addr if ip_addr not in self.__ip_lut: self.log("resolved {} to {}".format(target, ip_addr)) self.__ip_lut[ip_addr] = target self.__forward_lut[target] = ip_addr self.log("ip resolving: {} -> {}".format(target, ip_addr)) if orig_target != target: self.__forward_lut[orig_target] = ip_addr self.log("ip resolving: {} -> {}".format(orig_target, ip_addr)) return ip_addr
def __init__(self): threading_tools.process_pool.__init__(self, "main", zmq=True) self.register_exception("int_error", self._int_error) self.register_exception("term_error", self._int_error) self.CC.init(icswServiceEnum.discovery_server, global_config) self.CC.check_config() # close connection (daemonize) db_tools.close_connection() self.CC.read_config_from_db([ ("SNMP_PROCESSES", configfile.int_c_var( 4, help_string="number of SNMP processes [%(default)d]", short_options="n")), ("MAX_CALLS", configfile.int_c_var( 100, help_string="number of calls per helper process [%(default)d]" )), ]) self.CC.re_insert_config() self.CC.log_config() self.add_process(DiscoveryProcess("discovery"), start=True) self.add_process(EventLogPollerProcess( EventLogPollerProcess.PROCESS_NAME), start=True) self.add_process(GenerateAssetsProcess("generate_assets"), start=True) self._init_network_sockets() self.register_func("snmp_run", self._snmp_run) self.register_func("generate_assets", self._generate_assets) self.register_func( "process_assets_finished", self._process_assets_finished, ) self.register_func( "process_batch_assets_finished", self._process_batch_assets_finished, ) self.register_func("send_msg", self.send_msg) db_tools.close_connection() self.__max_calls = global_config[ "MAX_CALLS"] if not global_config["DEBUG"] else 5 self.__snmp_running = True self._init_processes() # not really necessary self.install_remote_call_handlers() # clear pending scans self.clear_pending_scans() self.__run_idx = 0 self.__pending_commands = {} if process_tools.get_machine_name( ) == "eddiex" and global_config["DEBUG"]: self._test()
def __init__(self, srv_com, src_id, xml_input): if "conn_str" in srv_com: _conn_str = srv_com["*conn_str"] _parts = _conn_str.split(":") if len(_parts) == 3: _parts.pop(0) self.host = _parts.pop(0).split("/")[-1] self.port = int(_parts.pop(0)) else: self.port = int(srv_com["port"].text) self.host = srv_com["host"].text self.raw_connect = True if int(srv_com.get("raw_connect", "0")) else False self.conn_str = "tcp://{}:{:d}".format(self.host, self.port) self.init_time = time.time() self.srv_com = srv_com self.src_id = src_id self.xml_input = xml_input cur_time = time.time() if self.conn_str in ZMQDiscovery.last_try and abs( ZMQDiscovery.last_try[self.conn_str] - cur_time) < 60: # need 60 seconds between tries self.socket = None self.send_return("last 0MQ discovery less than 60 seconds ago") else: ZMQDiscovery._pending[self.conn_str] = self new_sock = ZMQDiscovery.relayer_process.zmq_context.socket( zmq.DEALER) id_str = "relayer_dlr_{}_{}".format( process_tools.get_machine_name(), self.src_id) new_sock.setsockopt_string(zmq.IDENTITY, id_str) new_sock.setsockopt(zmq.LINGER, 0) new_sock.setsockopt(zmq.SNDHWM, ZMQDiscovery.backlog_size) new_sock.setsockopt(zmq.RCVHWM, ZMQDiscovery.backlog_size) new_sock.setsockopt(zmq.BACKLOG, ZMQDiscovery.backlog_size) new_sock.setsockopt(zmq.TCP_KEEPALIVE, 1) new_sock.setsockopt(zmq.TCP_KEEPALIVE_IDLE, 300) self.socket = new_sock ZMQDiscovery.relayer_process.register_poller( new_sock, zmq.POLLIN, self.get_result) # ZMQDiscovery.relayer_process.register_poller(new_sock, zmq.POLLIN, self.error) self.socket.connect(self.conn_str) if self.raw_connect: self.log("send raw discovery message") self.socket.send_unicode("get_0mq_id") else: self.log("send discovery message") dealer_message = server_command.srv_command( command="get_0mq_id") dealer_message["target_ip"] = self.host self.socket.send_unicode(str(dealer_message))
def process_init(self): self.__log_template = logging_functions.get_logger( config_store.ConfigStore("client", quiet=True), "{}/{}".format( process_tools.get_machine_name(), self.global_config["LOG_NAME"], ), process_name=self.name, ) # log.startLoggingWithObserver(my_observer, setStdout=False) self.__debug = self.global_config["DEBUG"] self.register_func("resolve", self._resolve, greedy=True) # clear flag for extra twisted thread self.__cache = {}
def loop_post(self): if self.__build_lock: self.log("removing buildlock") cur_img = self._get_image() cur_img.build_lock = False cur_img.release += 1 if not cur_img.builds: cur_img.builds = 1 else: cur_img.builds += 1 cur_img.build_machine = process_tools.get_machine_name(short=False) cur_img.save() e_time = time.time() self.log("build took {}".format(logging_tools.get_diff_time_str(e_time - self.__start_time))) self.__log_template.close()
def _add_info_parser(self, sub_parser, server_mode): _mach_name = process_tools.get_machine_name(short=True) parser = sub_parser.add_parser("call", help="call subsystems") parser.set_defaults(subcom="call", execute=self._execute) _choices = ["host-monitoring"] if server_mode: _choices.extend(["cluster-server"]) parser.add_argument( "--subsys", type=str, default=_choices[0], choices=_choices, help="show command info for given subsystem [%(default)s]") parser.add_argument("args", nargs="+") return parser
def process_init(self): self.__log_template = logging_functions.get_logger( config_store.ConfigStore("client", quiet=True), "{}/{}".format( process_tools.get_machine_name(), self.global_config["LOG_NAME"], ), process_name=self.name, ) self.CS = config_store.ConfigStore("client", self.log) self.commands = [] self.register_func("command_batch", self._command_batch) # commands pending becaus of missing package list self.pending_commands = [] # list of pending package commands self.package_commands = [] self.register_timer(self._check_commands, 10)
def _add_job_parser(self, sub_parser, server_mode): _mach_name = process_tools.get_machine_name(short=True) parser = sub_parser.add_parser("job", help="job helper commands") if server_mode: _choices = ["info", "setvar", "listvars"] else: _choices = ["info", "setvar"] parser.set_defaults(subcom="job", execute=self._execute) parser.add_argument("--job-id", default="", type=str, help="job ID (gets evaluated automatically via environ) [%(default)s]") parser.add_argument("--task-id", default=0, type=int, help="task ID (gets evaluated automatically via environ) [%(default)d]") parser.add_argument("--server-address", default="", type=str, help="RMS server address [%(default)s]") parser.add_argument("--server-port", default=8009, type=int, help="RMS server address [%(default)d]") parser.add_argument("--mode", default="info", type=str, choices=_choices, help="job subcommand [%(default)s]") parser.add_argument("--name", default="", type=str, help="variable name [%(default)s]") parser.add_argument("--value", default="", type=str, help="variable value [%(default)s]") parser.add_argument("--unit", default="", type=str, help="variable unit [%(default)s]") return parser
def _enable_syslog_config(self): syslog_srvcs = self.srv_helper.find_services(".*syslog", active=True) self.__syslog_type = None if syslog_srvcs: self.__syslog_type = syslog_srvcs[0] self.log("syslog type found: {}".format(self.__syslog_type)) # hack for old sles11sp3 (liebherr) and Centos6 (Ac2T) if self.__syslog_type.count("rsys") or (self.__syslog_type in [ "syslog" ] and process_tools.get_machine_name() in ["lwnsu62020", "admin"]): self._enable_rsyslog() else: self.log( "syslog-type {} not supported".format(self.__syslog_type), logging_tools.LOG_LEVEL_ERROR) else: self.log("found no valid syslog service", logging_tools.LOG_LEVEL_ERROR)
def _send_to_rms_server(self, srv_com): if not self.__server_socket: self.__server_socket = process_tools.get_socket( self.module_info.main_proc.zmq_context, "DEALER", linger=10, identity="afm_{}_{:d}".format(process_tools.get_machine_name(), os.getpid()), immediate=False, ) _srv_address = "tcp://{}:{:d}".format( self._config_dict["SGE_SERVER"], self._config_dict["SGE_SERVER_PORT"], ) self.__server_socket.connect(_srv_address) self.log("connected to {}".format(_srv_address)) try: self.__server_socket.send_unicode(str(srv_com)) except: self.log( "error sending affinity info: {}".format( process_tools.get_except_info()), logging_tools.LOG_LEVEL_ERROR)
# database config _cs = config_store.ConfigStore(GEN_CS_NAME, quiet=True, access_mode=config_store.AccessModeEnum.GLOBAL) # version config # TODO: check for local config when running in debug (development) mode _vers = config_store.ConfigStore(VERSION_CS_NAME, quiet=True) _DEF_NAMES = ["database", "software", "models"] ICSW_VERSION_DICT = {_name: _vers.get(_name, "???") for _name in _DEF_NAMES} ICSW_DATABASE_VERSION = ICSW_VERSION_DICT["database"] ICSW_SOFTWARE_VERSION = ICSW_VERSION_DICT["software"] ICSW_MODELS_VERSION = ICSW_VERSION_DICT["models"] ICSW_DEBUG = process_tools.get_machine_name() in ["eddie", "lemmy"] # validate settings if _cs["password.hash.function"] not in ["SHA1", "CRYPT"]: raise ImproperlyConfigured("password hash function '{}' not known".format( _cs["password.hash.function"])) ICSW_ALLOWED_OVERALL_STYLES = {"normal", "condensed"} ICSW_ALLOWED_MENU_LAYOUTS = {"newstyle"} if "overall.style" in _cs: ICSW_OVERALL_STYLE = _cs["overall.style"] else: ICSW_OVERALL_STYLE = "normal" if _cs.get("missing.timezone.is.critical", True):
# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License Version 3 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # """ tools for icsw commands """ from __future__ import unicode_literals, print_function import os from initat.tools import process_tools __all__ = [ "ICSW_DEBUG_MODE", ] ICSW_DEBUG_MODE = process_tools.get_machine_name() in [ "eddie", "lemmy" ] and (os.environ.get("DEBUG_ICSW_SOFTWARE") or os.environ.get("ICSW_DEBUG_SOFTWARE"))
def __call__(self, srv_com, cur_ns): srv_com["heartbeat_info"] = { "host": process_tools.get_machine_name(), "output": self.module._exec_command("/usr/sbin/crm_mon -1") }
""" classes for building rpm-packages """ import argparse import subprocess import os import pwd import stat import sys import codecs from initat.tools import logging_tools, process_tools SCRIPT_TYPES = ["post", "pre", "postun", "preun"] default_ns = argparse.Namespace(packager="{}@{}".format( pwd.getpwuid(os.getuid())[0], process_tools.get_machine_name()), user="******", group="root", provides="", arch={"i686": "i586"}.get(os.uname()[4], os.uname()[4])) class package_parser(argparse.ArgumentParser): def __init__(self): argparse.ArgumentParser.__init__(self) self.add_argument("-s", "--summary", dest="summary", help="Summary string [%(default)s]",
def get_python_handle(self, record): if isinstance(record, icswLogHandleTypes): # special type for direct handles (log, log_py, err_py) sub_dirs = [] record_host = "localhost" record_name, record_process, record_parent_process = (record.value, os.getpid(), os.getppid()) else: if not hasattr(record, "host"): # no host set: use local machine name record.host = process_tools.get_machine_name() sub_dirs = [record.host] record_host = record.host record_name, record_process, record_parent_process = ( record.name, record.process, getattr(record, "ppid", 0)) # init.at logger, create subdirectories # generate list of dirs and file_name # strip init.at. prefix (no longer in use) if record_name.startswith("init.at."): record_name = record_name[8:] scr1_name = record_name.replace("\.", "#").replace(".", "/").replace("#", ".") for path_part in os.path.dirname(scr1_name).split(os.path.sep): if path_part: path_part = "{}.d".format(path_part) if sub_dirs: sub_dirs.append(os.path.join(sub_dirs[-1], path_part)) else: sub_dirs.append(path_part) if sub_dirs: h_name = os.path.join(sub_dirs[-1], os.path.basename(scr1_name)) else: h_name = os.path.basename(scr1_name) # create logger_name logger_name = "{}.{}".format(record_host, record_name) if h_name in self.__handles: if not ({record_process, record_parent_process} & { self.__handles[h_name].process_id, self.__handles[h_name].parent_process_id }) and not self.__handles[h_name].ignore_process_id: self.log("access mismatch detected for {}".format(h_name, ), logging_tools.LOG_LEVEL_ERROR) self.remove_handle(h_name) if h_name not in self.__handles: # TODO, FIXME, unify with native_logging code in server_mixins.py self.log("logger '{}' requested".format(logger_name, )) full_name = os.path.join(self.CC.CS["log.logdir"], h_name) base_dir, base_name = (os.path.dirname(full_name), os.path.basename(full_name)) self.log("attempting to create log_file '{}' in dir '{}'".format( base_name, base_dir)) # add new sub_dirs sub_dirs = [] for new_sub_dir in os.path.dirname(h_name).split("/"): if not sub_dirs: sub_dirs.append(new_sub_dir) else: sub_dirs.append(os.path.join(sub_dirs[-1], new_sub_dir)) # create sub_dirs for sub_dir in sub_dirs: act_dir = os.path.join(self.CC.CS["log.logdir"], sub_dir) if not os.path.isdir(act_dir): try: os.makedirs(act_dir) except OSError: self.log( "cannot create directory {}: {}".format( act_dir, process_tools.get_except_info(), ), logging_tools.LOG_LEVEL_ERROR) else: self.log("created directory {}".format(act_dir)) # get logger logger = logging_functions.get_logger(self.CC.CS, h_name, logger_name) self.__num_open += 1 # save process_id to handle open / close logger.process_id = record_process logger.parent_process_id = record_parent_process # set ignore_process_id flag, usefull for apache process / threadpools logger.ignore_process_id = False logger.handle_name = h_name self.__handles[h_name] = logger self.__handle_usage[h_name] = set() self.__handle_usecount[h_name] = 0 self.log( "added handle {} (file {} in dir {}), total open: {}".format( h_name, base_name, base_dir, logging_tools.get_plural("handle", len(list( self.__handles.keys()))))) return self.__handles[h_name]
def show_command(options): for f_name in options.files: _obj_name = f_name if not options.short_path else os.path.basename( f_name) for _rc in ["/", ".", "-"]: _obj_name = _obj_name.replace(_rc, "_") while _obj_name.startswith("_"): _obj_name = _obj_name[1:] obj_name = "{}_object".format(_obj_name) try: f_stat = os.stat(f_name) content = open(f_name).read() except: print("error reading file '{}': {}".format( f_name, process_tools.get_except_info())) else: if not options.binary: f_lines = content.split("\n") _f_info = logging_tools.get_plural("line", f_lines) else: _f_info = "binary" out_lines = [ "", "# from {} ({}, host {}, size was {}, {})".format( f_name, datetime.datetime.now(), process_tools.get_machine_name(short=False), logging_tools.get_size_str(f_stat[stat.ST_SIZE]), _f_info, ), "", "{} = config.add_file_object('{}')".format(obj_name, f_name), ] if options.binary: out_lines.extend([ "import bz2", "import base64", "", "{} += bz2.decompress(base.b64decode('{}'))".format( obj_name, base64.b64encode(bz2.compress(content)), ) ]) else: if options.full_strip: f_lines = [ _line.strip() for _line in f_lines if _line.strip() ] if options.remove_hashes: f_lines = [ _line for _line in f_lines if (not _line.startswith("#") or _line.startswith("#!") ) ] p_line = " " * 4 try: out_lines.append("{} += [\n{}]\n".format( obj_name, "".join([ "{}'{}',\n".format( p_line, _line.replace("'", '"').replace("\\", "\\\\")) for _line in f_lines ]))) except UnicodeDecodeError: print() print( "'{}' seems to be a binary file, please use -b switch". format(f_name)) print() sys.exit(3) out_lines.append("{}.mode = 0o{:o}".format( obj_name, stat.S_IMODE(f_stat[stat.ST_MODE]))) print("\n".join(out_lines))
def create_noctua_fixtures(): print("Creating Noctua fixtures...") # first config catalog first_cc = config_catalog.objects.all()[0] # category tree ct = category_tree() cat_serv = ct.add_category("/mon/services") cat_web = ct.add_category("/mon/services/web") cat_mail = ct.add_category("/mon/services/mail") # config print("Creeating configurations.") ping_config = factories.Config( name="check_ping", config_catalog=first_cc, ) snmp_config = factories.Config( name="check_snmp_info", config_catalog=first_cc, ) ssh_config = factories.Config( name="check_ssh", config_catalog=first_cc, ) http_config = factories.Config( name="check_http", config_catalog=first_cc, ) https_config = factories.Config( name="check_https", config_catalog=first_cc, ) ldap_config = factories.Config( name="check_ldap", config_catalog=first_cc, ) imap_config = factories.Config( name="check_imap", config_catalog=first_cc, ) imaps_config = factories.Config( name="check_imaps", config_catalog=first_cc, ) pop3s_config = factories.Config( name="check_pop3s", config_catalog=first_cc, ) smtps_config = factories.Config( name="check_smtps", config_catalog=first_cc, ) print("Creating monitoring checks.") factories.MonCheckCommand( name="snmp_info", command_line="$USER3$ -m $HOSTADDRESS$ -C $ARG1$ -V $ARG2$ snmp_info", config=snmp_config).categories.add(cat_serv) factories.MonCheckCommand( name="check_ping", command_line="$USER2$ -m localhost ping $HOSTADDRESS$ 5 5.0", config=ping_config).categories.add(cat_serv) factories.MonCheckCommand(name="check_ssh", command_line="$USER1$/check_ssh $HOSTADDRESS$", config=ssh_config).categories.add(cat_serv) factories.MonCheckCommand( name="check_http", command_line="$USER1$/check_http -H $HOSTADDRESS$", config=http_config).categories.add(cat_web) factories.MonCheckCommand( name="check_imaps", command_line="$USER1$/check_imap -H $HOSTADDRESS$ -p 993 -S", config=imaps_config).categories.add(cat_mail) factories.MonCheckCommand( name="check_ldap", command_line="$USER1$/check_ldap -H $HOSTADDRESS$ -b dc=init,dc=at -3", config=ldap_config).categories.add(cat_serv) factories.MonCheckCommand( name="check_https", command_line="$USER1$/check_http -S -H $HOSTADDRESS$ -C 30", config=https_config).categories.add(cat_web) factories.MonCheckCommand( name="check_imap", command_line="$USER1$/check_imap -H $HOSTADDRESS$ -p 143", config=imap_config).categories.add(cat_mail) factories.MonCheckCommand( name="check_pop3s", command_line="$USER1$/check_pop3 -H $HOSTADDRESS$ -p 995 -S", config=pop3s_config).categories.add(cat_mail) factories.MonCheckCommand( name="check_smtps", command_line="$USER1$/check_smtps -H $HOSTADDRESS$ -p 465 -S", config=smtps_config).categories.add(cat_mail) # domain name tree dnt = domain_name_tree() _top_level_dtn = dnt.get_domain_tree_node("") # device_group print("Creating device and device group.") first_devg = factories.DeviceGroup(name="server_group") first_dev = factories.Device( name=process_tools.get_machine_name(), device_group=first_devg, domain_tree_node=_top_level_dtn, ) print("Creating device configurations.") factories.DeviceConfig( device=first_dev, config=factories.Config(name="monitor_server", config_catalog=first_cc), ) factories.DeviceConfig( device=first_dev, config=factories.Config(name="rrd_server", config_catalog=first_cc), ) factories.DeviceConfig( device=first_dev, config=factories.Config(name="server", config_catalog=first_cc), ) factories.DeviceConfig( device=first_dev, config=factories.Config(name="rrd_collector", config_catalog=first_cc), ) factories.DeviceConfig( device=first_dev, config=factories.Config(name="discovery_server", config_catalog=first_cc), ) factories.DeviceConfig( device=first_dev, config=ssh_config, ) print("Creating monitoring periods.") initial_mon_period = factories.MonPeriod(name="always", sun_range="00:00-24:00", mon_range="00:00-24:00", tue_range="00:00-24:00", wed_range="00:00-24:00", thu_range="00:00-24:00", fri_range="00:00-24:00", sat_range="00:00-24:00") first_st = factories.MonServiceTempl( name="dummy_service_template", nsc_period=initial_mon_period, nsn_period=initial_mon_period, ) _first_dt = factories.MonDeviceTempl( name="dummy_device_template", mon_service_templ=first_st, mon_period=initial_mon_period, not_period=initial_mon_period, host_check_command=host_check_command.objects.get( Q(name="check-host-alive")), ) is_ucs = os.path.isfile("/usr/sbin/ucr") # the create_cluster script adds an admin user # if there are no users, or in case of an ucs system, if only this one new admin exists, # then we want an admin and a user user users = user.objects.all() empty_install = users.count() == 0 new_install = (users.count() == 1 and users[0].login == 'admin' and users[0].login_count == 0) if empty_install or (is_ucs and new_install): print('Creating user and groups.') user.objects.all().delete() group.objects.all().delete() # group / users _group = factories.Group( groupname="group", homestart="/", gid=100, ) _group.allowed_device_groups.add(first_devg) _user = factories.User( login="******", uid=400, group=_group, password="******", ) _user.allowed_device_groups.add(first_devg) _first_mc = factories.MonContact( user=_user, snperiod=initial_mon_period, hnperiod=initial_mon_period, ) _admin = user.objects.create_superuser( "admin", "*****@*****.**", "admin", ) # we need contacts for all initial users so that they can access icinga factories.MonContact( user=_admin, snperiod=initial_mon_period, hnperiod=initial_mon_period, ) _admin.allowed_device_groups.add(first_devg) # network if is_ucs: if_address = get_local_ip_address("62.99.204.238") # print if_address if_name = get_interface_by_ip(if_address) # print if_name p = subprocess.Popen( ['ucr', 'get', 'interfaces/%s/address' % (if_name)], stdout=subprocess.PIPE) if_address = p.stdout.read().strip().split("\n")[0] p = subprocess.Popen( ['ucr', 'get', 'interfaces/%s/network' % (if_name)], stdout=subprocess.PIPE) if_network = p.stdout.read().strip().split("\n")[0] p = subprocess.Popen( ['ucr', 'get', 'interfaces/%s/broadcast' % (if_name)], stdout=subprocess.PIPE) if_broadcast = p.stdout.read().strip().split("\n")[0] p = subprocess.Popen( ['ucr', 'get', 'interfaces/%s/netmask' % (if_name)], stdout=subprocess.PIPE) if_netmask = p.stdout.read().strip().split("\n")[0] p = subprocess.Popen(['ucr', 'get', 'gateway'], stdout=subprocess.PIPE) out = p.stdout.read().strip().split("\n")[0] if_gateway = out else: print( "Not installed on UCS, /usr/sbin/ucr not found. Using python-netifaces." ) if_address = get_local_ip_address("62.99.204.238") if_name = get_interface_by_ip(if_address) if_netmask = get_netmask_by_interface(if_name) if_broadcast = get_broadcast_by_interface(if_name) if_network = str( ipvx_tools.ipv4(if_netmask) & ipvx_tools.ipv4(if_broadcast)) if_gateway = get_default_gateway_linux() print('Creating network objects.') _network = factories.Network( identifier="lan", network_type=network_type.objects.get(Q(identifier="o")), name="lan", network=if_network, broadcast=if_broadcast, netmask=if_netmask, gateway=if_gateway, ) _netdevice = factories.NetDevice( device=first_dev, devname=if_name, routing=True, netdevice_speed=netdevice_speed.objects.get( Q(speed_bps=1000000000) & Q(full_duplex=True) & Q(check_via_ethtool=True)), network_device_type=network_device_type.objects.get( Q(identifier="eth")), ) _net_ip = factories.NetIp( ip=if_address, network=_network, netdevice=_netdevice, domain_tree_node=_top_level_dtn, )
def add_fixtures(**kwargs): _fact_dict = {} for _name, _prefix, _descr, _pri, _fixed, _sys_scope in [ ("normal", "", "default Scope", 100, False, True), ("inventory", "__$$ICSW_INV$$__", "Scope for device inventory", 50, True, False), ("comm", "__$$ICSW_COM$$__", "Scope for device communication", 20, True, True), ]: _fact_dict[_name] = factories.device_variable_scope_factory( name=_name, prefix=_prefix, description=_descr, priority=_pri, fixed=_fixed, system_scope=_sys_scope, ) # set default scope _fact_dict["normal"].default_scope = True _fact_dict["normal"].save() comm_list = [] _used_vars = set() for dci in DeviceConnectionEnum: # name is group for cur_var in dci.value.var_list + dci.value.opt_list: if cur_var.name not in _used_vars: _used_vars.add(cur_var.name) comm_list.append(( "*{}".format(cur_var.name), cur_var.info, dci.name, cur_var.var_type, cur_var.is_password, )) _defaults = { "inventory": [ ("serial", "Serial number", "admin", DeviceVarTypeEnum.string, False), ("id", "Numeric ID", "admin", DeviceVarTypeEnum.integer, False), ("time_of_purchase", "Date of purchase", "admin", DeviceVarTypeEnum.date, False), ], "comm": comm_list, } for _scope_name, _var_list in _defaults.items(): for _name, _descr, _group, _forced_type, _passwd_field in _var_list: editable = not _name.startswith("*") if not editable: _name = _name[1:] factories.DVSAllowedNameFactory( name=_name, description=_descr, device_variable_scope=_fact_dict[_scope_name], forced_type=_forced_type.value, group=_group, editable=editable, password_field=_passwd_field, ) if process_tools.get_machine_name() in ["eddie"]: # debug output for _e in device_variable_scope.objects.all(): print(str(_e)) for _e in dvs_allowed_name.objects.all(): print(str(_e))
def _send_vector(self, *args, **kwargs): send_id = args[0] _struct = self.cs[send_id] _p_until = _struct.get("pause_until", 0) cur_time = int(time.time()) # print "_", _p_until, cur_time if _p_until: if _p_until > cur_time: return else: self.log("clearing pause_until") del _struct["pause_until"] cur_id = _struct["sent"] full = cur_id % _struct.get("full_info_every", 10) == 0 cur_id += 1 _struct["sent"] = cur_id try: fqdn, _short_name = process_tools.get_fqdn() except: fqdn = process_tools.get_machine_name() send_format = _struct.get("format", "xml") if send_format == "xml": send_vector = self.build_xml(E, simple=not full) send_vector.attrib["name"] = _struct.get("send_name", fqdn) or fqdn send_vector.attrib["interval"] = "{:d}".format( _struct.get("send_every")) send_vector.attrib["uuid"] = self.module.main_proc.zeromq_id else: send_vector = self.build_json(simple=not full) send_vector[1]["name"] = _struct.get("send_name", fqdn) or fqdn send_vector[1]["interval"] = _struct.get("send_every") send_vector[1]["uuid"] = self.module.main_proc.zeromq_id # send to server t_host, t_port = ( _struct.get("target", "127.0.0.1"), _struct.get("port", 8002), ) try: if send_format == "xml": self.__socket_dict[send_id].send_unicode( unicode(etree.tostring(send_vector))) # @UndefinedVariable else: # print json.dumps(send_vector) self.__socket_dict[send_id].send_unicode( json.dumps(send_vector)) except: exc_info = process_tools.get_except_info() # ignore errors self.log( "error sending to ({}, {:d}): {}".format( t_host, t_port, exc_info), logging_tools.LOG_LEVEL_ERROR) if exc_info.count("int_error"): raise else: # problem sending, wait 2 minutes _diff_t = 120 _w_time = cur_time + _diff_t self.log( "setting pause_until to {:d} (+{:d} seconds)".format( _w_time, _diff_t), logging_tools.LOG_LEVEL_WARN) _struct["pause_until"] = _w_time self.cs[send_id] = _struct