def main(): parser = argparse.ArgumentParser("log a message with standard metwork" "logging system") parser.add_argument('--application-name', '-a', action="store", default="default", help="application name") parser.add_argument( 'LEVEL', action='store', help="Niveau de log", choices=['ERROR', 'CRITICAL', 'WARNING', 'INFO', 'DEBUG']) parser.add_argument('MESSAGE', action='store', help="message to log") options = parser.parse_args() logger = getLogger(options.application_name) if options.LEVEL == 'DEBUG': logger.debug(options.MESSAGE) elif options.LEVEL == 'INFO': logger.info(options.MESSAGE) elif options.LEVEL == 'WARNING': logger.warning(options.MESSAGE) elif options.LEVEL == 'CRITICAL': logger.critical(options.MESSAGE) elif options.LEVEL == 'ERROR': logger.error(options.MESSAGE) else: raise Exception("Bad message level: %s", options.LEVEL)
def main(): parser = argparse.ArgumentParser("log a message with standard metwork" "logging system") parser.add_argument('--application-name', '-a', action="store", default="default", help="application name") parser.add_argument('LEVEL', action='store', help="Niveau de log", choices=['ERROR', 'CRITICAL', 'WARNING', 'INFO', 'DEBUG']) parser.add_argument('MESSAGE', action='store', help="message to log") options = parser.parse_args() logger = getLogger(options.application_name) if options.LEVEL == 'DEBUG': logger.debug(options.MESSAGE) elif options.LEVEL == 'INFO': logger.info(options.MESSAGE) elif options.LEVEL == 'WARNING': logger.warning(options.MESSAGE) elif options.LEVEL == 'CRITICAL': logger.critical(options.MESSAGE) elif options.LEVEL == 'ERROR': logger.error(options.MESSAGE) else: raise Exception("Bad message level: %s", options.LEVEL)
def _get_logger(self): """Get a logger.""" if not self.__logger: logger_name = "mfdata.%s.%s" % ( self.plugin_name, self._get_step_or_daemon_name(), ) self.__logger = mflog.getLogger(logger_name) return self.__logger
def _get_logger(self): """Get a logger.""" if not self.__logger: logger_name = "mfdata.%s.%s" % ( self.plugin_name, self.step_name if self.step_name is not None else "notset", ) self.__logger = mflog.getLogger(logger_name) return self.__logger
# State timeout redis. redisTimeoutState = False # Flag request "clean" stop. stopRequest = False # Date of last event issued independently from the Notifier instance dtLastEvent = 0 # Value of application security timeout # Maximum duration of non-receipt of event before stopping the application inactivityTimeout = DEFAULT_INACTIVITY_TIMEOUT # Definition logger logger = getLogger("directory_observer") class Monitor(pyinotify.ProcessEvent): redis = None server = None port = None wm = None ## # Initialization of the instance # # Classes derived from ProcessEvent must not overload __init __ () # But implement my_init () which will be automatically called # by __init __ () of the base class. # Attention my_init only supports named parameters.
#!/usr/bin/env python3 import os import psutil import sys import argparse import time import json from mflog import getLogger LOG = getLogger('list_metwork_processes') USER = os.environ.get('MODULE_RUNTIME_USER', None) MODULE = os.environ.get('MODULE', None) MODULE_RUNTIME_HOME = os.environ.get('MODULE_RUNTIME_HOME', None) MODULE_RUNTIME_HOME_TMP = MODULE_RUNTIME_HOME + "/tmp" if MODULE_RUNTIME_HOME \ is not None else None if USER is None: LOG.critical("can't read MODULE_RUNTIME_USER env var") sys.exit(1) if MODULE is None: LOG.critical("can't read MODULE env var") sys.exit(1) CURRENT_PROCESS = psutil.Process() CURRENT_PLUGIN_ENV_VAR = "%s_CURRENT_PLUGIN_NAME" % MODULE def is_same_family(child, proc): if child.pid == proc.pid: return True try:
from circus.stream import FileStream import psutil from mflog import getLogger log = getLogger("circus_streammultifiles") class MultiFilesStream(FileStream): _filename = None _time_format = None _numprocesses = None # dict pid => (slot number, opened FileStream) _slots = None def clean_old_pids(self): pids = list(self._slots.keys()) for pid in pids: if not psutil.pid_exists(pid): slot_number, file_stream = self._slots[pid] file_stream.close() del(self._slots[pid]) def pid_to_slot_number_stream(self, pid): if pid in self._slots: return self._slots[pid] self.clean_old_pids() used_slot_numbers = [x[0] for x in self._slots.values()] for i in range(0, self._numprocesses): if i not in used_slot_numbers:
def get_logger(*args, **kwargs): os.environ['MFCOM_LOG_STDERR'] = '/dev/null' set_logging_config() return getLogger(*args, **kwargs)
#!/usr/bin/env python import os import time import psutil from datetime import datetime from telegraf_unixsocket_client import TelegrafUnixSocketClient from mflog import getLogger MFMODULE_RUNTIME_HOME = os.environ["MFMODULE_RUNTIME_HOME"] SOCKET_PATH = os.path.join(MFMODULE_RUNTIME_HOME, "var", "telegraf.socket") LOGGER = getLogger("telegraf_collector_custom_diskio") old_read_bytes = None old_write_bytes = None old_dt = None first = True while True: LOGGER.debug("waiting 10s...") time.sleep(10) client = TelegrafUnixSocketClient(SOCKET_PATH) try: client.connect() except Exception: LOGGER.warning("can't connect to %s, wait 10s and try again...", SOCKET_PATH) continue stats = psutil.disk_io_counters(perdisk=False) new_dt = datetime.now() new_read_bytes = stats.read_bytes new_write_bytes = stats.write_bytes
def get_logger(self): if not self.__logger: logger_name = "mfdata.%s.%s" % (self.plugin_name, self.daemon_name) self.__logger = mflog.getLogger(logger_name) return self.__logger
import os from mflog import getLogger import json from aiohttp import web, ClientSession, BasicAuth, ClientTimeout import aiohttp_github_helpers as h GITHUB_USER = os.environ['GITHUB_USER'] GITHUB_PASS = os.environ['GITHUB_PASS'] GITHUB_SECRET = os.environ['GITHUB_SECRET'].encode('utf8') LOGGER = getLogger("github_webhook_pr_labelling") TIMEOUT = ClientTimeout(total=20) AUTH = BasicAuth(GITHUB_USER, GITHUB_PASS) IGNORE_CONTEXT_GLOBS = ["mergify*"] async def handle(request): event = request['github_event'] if event != 'status': LOGGER.info("ignoring %s event" % event) return web.Response(text="ignoring %s event" % event) body = await request.read() decoded_body = json.loads(body.decode('utf8')) repo = decoded_body['repository']['name'] owner = decoded_body['repository']['owner']['login'] sha = decoded_body['sha'] async with ClientSession(auth=AUTH, timeout=TIMEOUT) as session: topics = await h.github_get_repo_topics(session, owner, repo) if ('integration-level-2' not in topics) and \ ('integration-level-3' not in topics) and \ ('integration-level-4' not in topics) and \
#!/usr/bin/env python3 import sys import os import argparse from xattrfile import XattrFile from mflog import getLogger from mfutil import get_unique_hexa_identifier from acquisition.utils import get_plugin_step_directory_path logger = getLogger("mfdata.reinject_plugin_step") def main(): parser = argparse.ArgumentParser("Inject a file into a plugin/step") parser.add_argument("filepath", type=str, help="filepath to inject") parser.add_argument("--plugin", type=str, help="plugin name (default :guess_file_type)", default="guess_file_type") parser.add_argument("--step", type=str, help="step name (default: main)", default="main") parser.add_argument("--move", action="store_true", help="move the file instead of copying it " "(default: copy)") parser.add_argument("--random-basename", action="store_true", help="use a random basename for copying/moving "
from circus.plugins import CircusPlugin from zmq.eventloop import ioloop from mfutil import add_inotify_watch from mflog import getLogger from inotify_simple import INotify, flags log = getLogger("circus_autorestart") class CircusAutorestart(CircusPlugin): """Circus plugin to automatically restart watchers. Watchers with a working_dir containing the string "/plugins/" are checked every second. If a change is detected in the configured working dir, the watcher is restarted. Args: name : the name of the plugin as a string. periodic: a PeriodicCallback object to call the ping() method every second. periodic10: a PeriodicCallback object to call the fill_watchers() method every 10 seconds. watchers: a set with watcher names (with "/plugins/" in configured working_dir). watchers_working_dir: a dict with watcher name as a key and corresponding inotify watcher as a value. """ name = 'autorestart' periodic = None watchers = None
from mflog import getLogger from mfutil import BashWrapper LOGGER = getLogger("circus_hooks") def _call(cmd): LOGGER.info("Calling %s..." % cmd) r = BashWrapper(cmd) if r.code != 0: msg = "Bad return code: %i from cmd: %s with output: %s" % \ (r.code, cmd, str(r).replace("\n", " ")) LOGGER.warning(msg) return False return True def _conditional_call(prefix, watcher_name, params=None): if watcher_name is not None: cmd = "%s_%s" % (prefix, watcher_name) else: cmd = prefix r = BashWrapper("which %s" % cmd) if r.code == 0: if params is not None: cmd = "%s %s" % (cmd, " ".join(params)) return _call(cmd) else: return True
from mflog import getLogger from mfutil import BashWrapper LOGGER = getLogger("circus_hooks") def _call(cmd): LOGGER.info("Calling %s..." % cmd) r = BashWrapper(cmd) if r.code != 0: msg = "Bad return code: %i from cmd: %s with output: %s" % \ (r.code, cmd, str(r).replace("\n", " ")) LOGGER.warning(msg) return False return True def _conditional_call(prefix, watcher_name): cmd = "%s_%s" % (prefix, watcher_name) r = BashWrapper("which %s" % cmd) if r.code == 0: return _call(cmd) else: return True def before_start_shell(watcher, arbiter, hook_name, **kwargs): return _conditional_call("before_start", watcher.name) def after_stop_shell(watcher, arbiter, hook_name, **kwargs):
import os from mflog import getLogger import json from aiohttp import web, ClientSession, BasicAuth, ClientTimeout import aiohttp_github_helpers as h GITHUB_USER = os.environ['GITHUB_USER'] GITHUB_PASS = os.environ['GITHUB_PASS'] GITHUB_SECRET = os.environ['GITHUB_SECRET'].encode('utf8') LOGGER = getLogger("github_webhook_no_pullrequest_on_master") TIMEOUT = ClientTimeout(total=20) AUTH = BasicAuth(GITHUB_USER, GITHUB_PASS) GUIDE_URL = "https://help.github.com/articles/" \ "changing-the-base-branch-of-a-pull-request/" COMMENT1 = """ Hi ! I'm the MetworkBot. Thank you for contributing to this project. But we don't accept pull requests on `master` branch as said in our contributing guide. => You have to change the base branch of your pull request to `integration` branch. It's very easy to do that by following this [github guide](%s). Many thanks !
#!/usr/bin/env python # inspired from # https://github.com/monitoring-tools/telegraf-plugins/tree/master/netstat import os import time import json import fnmatch from telegraf_unixsocket_client import TelegrafUnixSocketClient from mflog import getLogger from mfutil import BashWrapper MFMODULE_RUNTIME_HOME = os.environ["MFMODULE_RUNTIME_HOME"] SOCKET_PATH = os.path.join(MFMODULE_RUNTIME_HOME, "var", "telegraf.socket") LOGGER = getLogger("telegraf_collector_metwork_module") MFMODULE = os.environ['MFMODULE'] CMD = "list_metwork_processes.py --output-format=json --include-current-family" MONITORING_CMDLINE_PATTERNS = [ '*telegraf*', '*list_metwork_processes*', '*jsonlog2elasticsearch*' ] IS_MONITORING_MODULE = (MFMODULE in ['MFSYSMON', 'MFADMIN']) def is_cmdline_monitoring(cmdline): if IS_MONITORING_MODULE: return True for pattern in MONITORING_CMDLINE_PATTERNS: if fnmatch.fnmatch(cmdline, pattern): return True return False
#!/usr/bin/env python # inspired from # https://github.com/monitoring-tools/telegraf-plugins/tree/master/netstat import os import time import psutil from datetime import datetime from telegraf_unixsocket_client import TelegrafUnixSocketClient from mflog import getLogger from mfutil import BashWrapper MODULE_RUNTIME_HOME = os.environ["MODULE_RUNTIME_HOME"] SOCKET_PATH = os.path.join(MODULE_RUNTIME_HOME, "var", "telegraf.socket") LOGGER = getLogger("telegraf_collector_custom_netstat") CMD = "ss -t -a -n" STATES = { "ESTAB": "tcp_established", "SYN-SENT": "tcp_syn_sent", "SYN-RECV": "tcp_syn_recv", "FIN-WAIT-1": "tcp_fin_wait1", "FIN-WAIT-2": "tcp_fin_wait2", "TIME-WAIT": "tcp_time_wait", "UNCONN": "tcp_close", "CLOSE-WAIT": "tcp_close_wait", "LAST-ACK": "tcp_last_ack", "LISTEN": "tcp_listen", "CLOSING": "tcp_closing", "UNKNOWN": "tcp_none", "__TOTAL": "tcp_total"
#!/usr/bin/env python3 import psutil import sys import argparse import json from mfutil import kill_process_and_children, BashWrapper from mfutil.cli import MFProgress from mflog import getLogger LOG = getLogger('kill_remaining_processes') def get_processes_to_kill(): x = BashWrapper("list_metwork_processes.py --pids-only " "--output-format=json") if not x: LOG.warning("can't execute: %s" % x) sys.exit(1) try: pids = json.loads(x.stdout) except Exception: LOG.warning("bad output: %s" % x) sys.exit(1) return pids argparser = argparse.ArgumentParser(description="kill remaining non-terminal " "processes after a module stop") debug_doc = "debug mode: show processes killed"
#!/usr/bin/env python3 import os import time from mflog import getLogger from mfutil import BashWrapper, BashWrapperOrRaise from mfext.conf_monitor import ConfMonitorRunner, md5sumfile LOGGER = getLogger("conf_monitor") MFMODULE_RUNTIME_HOME = os.environ['MFMODULE_RUNTIME_HOME'] NGINX_FLAG = (int(os.environ['MFSERV_NGINX_FLAG']) == 1) def make_new_nginx_conf(): new_nginx_conf = "%s/tmp/tmp_nginx_conf2" % MFMODULE_RUNTIME_HOME cmd = "_make_nginx_conf >%s" % new_nginx_conf BashWrapperOrRaise(cmd) return (new_nginx_conf, md5sumfile(new_nginx_conf)) def get_old_nginx_conf(): old_nginx_conf = "%s/tmp/config_auto/nginx.conf" % MFMODULE_RUNTIME_HOME return (old_nginx_conf, md5sumfile(old_nginx_conf)) def restart_nginx(old_conf, new_conf): os.unlink(old_conf) os.rename(new_conf, old_conf) x = BashWrapper("_nginx.reload") if not x: LOGGER.warning(x)
#!/usr/bin/env python3 import os import psutil import sys import argparse from mflog import getLogger LOG = getLogger('kill_zombies_nginx_workers') USER = os.environ.get('MODULE_RUNTIME_USER', None) if USER is None: LOG.critical("can't read MODULE_RUNTIME_USER env var") sys.exit(1) MODULE_RUNTIME_HOME = os.environ.get('MODULE_RUNTIME_HOME', None) if MODULE_RUNTIME_HOME is None: LOG.critical("can't read MODULE_RUNTIME_HOME env var") sys.exit(1) def get_pids(): pids = [] for proc in psutil.process_iter(): try: if proc.username() != USER: continue if proc.ppid() != 1: continue cmdline = " ".join(proc.cmdline()) if "nginx: worker process" not in cmdline: continue
#!/usr/bin/env python3 import psutil import sys import argparse import json from mfutil import kill_process_and_children, BashWrapper from mflog import getLogger LOG = getLogger('kill_remaining_processes') def get_processes_to_kill(): x = BashWrapper("list_metwork_processes.py --pids-only " "--output-format=json") if not x: LOG.warning("can't execute: %s" % x) sys.exit(1) try: pids = json.loads(x.stdout) except Exception: LOG.warning("bad output: %s" % x) sys.exit(1) return pids argparser = argparse.ArgumentParser(description="kill remaining non-terminal " "processes after a module stop") silent_doc = "silent mode: return only the number of processes killed and " \ "the number of remaining processes"
#!/usr/bin/env python3 import os import time from telegraf_unixsocket_client import TelegrafUnixSocketClient from mfutil import BashWrapper from mflog import getLogger MODULE_RUNTIME_HOME = os.environ["MODULE_RUNTIME_HOME"] VAR_IN_PATH = os.path.join(MODULE_RUNTIME_HOME, "var", "in") SOCKET_PATH = os.path.join(MODULE_RUNTIME_HOME, "var", "telegraf.socket") LOGGER = getLogger("telegraf_collector_var_in_files_count") def get_var_in_directories(): result = [] var_in_content = os.listdir(VAR_IN_PATH) for name in var_in_content: full_path = os.path.join(VAR_IN_PATH, name) if os.path.isdir(full_path): result.append(full_path) return result def get_file_count(directory): cmd = "find %s -type f 2>/dev/null |wc -l" % directory x = BashWrapper(cmd) if x: return int(x.stdout)