def _setup_logging(self): config_ctrl = self.config["controlserver"] logfile = os.path.join( config_ctrl["log_path"], config_ctrl["log_name"].format(bl=self.beamline)) utils.check_writable(logfile) # Get queue self.log_queue = Queue(-1) handler = utils.get_log_handlers(logfile, config_ctrl["log_size"], config_ctrl["verbose"], config_ctrl["onscreen"]) # Start queue listener using the stream handler above self.log_queue_listener = utils.CustomQueueListener( self.log_queue, *handler) self.log_queue_listener.start() # Create log and set handler to queue handle self.log = utils.get_logger("ControlServer", self.log_queue) self.log.info("Init")
def __init__(self, log_queue, log_level): self.confirmations = {} self.jobs = {} self.log = utils.get_logger(self.__class__.__name__, queue=log_queue, log_level=log_level)
def __init__(self, log_queue, lock, config): threading.Thread.__init__(self) self.log = utils.get_logger("Synchronizing", log_queue) self.all_data = {} self.lock = lock self.server = config["kafka_server"] self.topic = config["kafka_topic"] self.operation = config["operation"] self.detids = config["detids"] self.n_detectors = config["n_detectors"] self.monitored_dir = pathlib.Path(config["monitored_dir"]) self.fix_subdirs = config["fix_subdirs"] self.paths = None self.consumer = None self.sync_buffer = collections.deque(maxlen=config["buffer_size"]) self.keep_running = True self._setup()
def __init__(self, handler_id, config, lock, log_queue): self.handler_id = handler_id self.lock = lock # Suppress logging messages of watchdog observer logging.getLogger("watchdog.observers.inotify_buffer").setLevel( logging.WARNING) self.log = utils.get_logger( "WatchdogEventHandler-{}".format(self.handler_id), log_queue ) self.log.debug("init") self.paths = [os.path.normpath(config["monitored_dir"])] # learn what events to detect self.detect_all = False self.detect_create = False self.detect_modify = False self.detect_delete = False self.detect_move_from = False self.detect_move_to = False self.detect_close = False regexes = [] for event, regex in iteritems(config["monitored_events"]): self.log.debug("event: %s, pattern: %s", event, regex) regex = convert_suffix_list_to_regex(regex, compile_regex=True, log=self.log) regexes.append(regex) if "all" in event.lower(): self.log.info("Activate all event types") self.detect_all = regex elif "create" in event.lower(): self.log.info("Activate on create event types") self.detect_create = regex elif "modify" in event.lower(): self.log.info("Activate on modify event types") self.detect_modify = regex elif "delete" in event.lower(): self.log.info("Activate on delete event types") self.detect_delete = regex elif "move_from" in event.lower(): self.log.info("Activate on move from event types") self.detect_move_from = regex elif "move_to" in event.lower(): self.log.info("Activate on move to event types") self.detect_move_to = regex elif "close" in event.lower(): self.log.info("Activate on close event types") self.detect_close = regex WatchdogEventHandler.regexes = regexes self.log.debug("init: super") super().__init__()
def _setup_logging(self): config_gen = self.config["general"] # Get queue self.log_queue = multiprocessing.Queue(-1) handler = utils.get_log_handlers(config_gen["log_file"], config_gen["log_size"], config_gen["verbose"], config_gen["onscreen"]) # Start queue listener using the stream handler above. self.log_queue_listener = utils.CustomQueueListener( self.log_queue, *handler) self.log_queue_listener.start() # the least sever log level to forward to the queuelistener file_log_level = "debug" if config_gen["verbose"] else "error" if config_gen["onscreen"]: self.log_level = utils.get_least_sever_log_level( log_levels=(config_gen["onscreen"], file_log_level)) else: self.log_level = file_log_level # Create log and set handler to queue handle self.log = utils.get_logger(self.__class__.__name__, queue=self.log_queue, log_level=self.log_level) self.log.info("Setting process log level to '%s'", self.log_level)
def __init__(self, beamline, backup_file, log_queue): self.log = utils.get_logger(self.__class__.__name__, log_queue) self.beamline = beamline self.backup_file = backup_file.format(bl=self.beamline) self.reply_codes = REPLYCODES self.instances = None self._set_instances()
def _setup(self): """Initializes parameters and creates sockets. """ self.log = utils.get_logger(self.__class__.__name__, queue=self.log_queue, log_level=self.log_level) self.log.info("%s started (PID %s).", self.__class__.__name__, os.getpid()) signal.signal(signal.SIGTERM, self.signal_term_handler) try: self.timeout = self.config["general"]["taskprovider_timeout"] except KeyError: self.timeout = 1000 self.log.debug("Set timeout to %s ms", self.timeout) # remember if the context was created outside this class or not self.log.info("Registering ZMQ context") self.context = zmq.Context() try: self.ignore_accumulated_events = ( self.config["eventdetector"]["ignore_accumulated_events"]) except KeyError: self.ignore_accumulated_events = False ed_type = self.config["eventdetector"]["type"] self.log.info("Loading event detector: %s", ed_type) try: eventdetector_m = import_module(ed_type) except Exception: self.log.error("Could not load event detector %s", ed_type, exc_info=True) raise self.eventdetector = eventdetector_m.EventDetector({ "config": self.config, "log_queue": self.log_queue, "check_dep": True, "context": self.context }) try: self.create_sockets() except Exception: self.log.error("Cannot create sockets", exc_info=True) self.stop()
def _setup(self): self.log = utils.get_logger(self.__class__.__name__, queue=self.log_queue, log_level=self.log_level) # -------------------------------------------------------------------- # zmq setup # -------------------------------------------------------------------- self.context = zmq.Context() endpoints = self.config["network"]["endpoints"] # to enable communication between the StatServer and # hidra-control running as different users # this has to be called even if the ipc dir was already created by # another hidra instance old_umask = os.umask(self.ipc_dir_umask) self.log.info("Changed umask on ipc_dir to %s", self.ipc_dir_umask) self.stats_collect_socket = self.start_socket( name="stats_collect_socket", sock_type=zmq.PULL, sock_con="bind", endpoint=endpoints.stats_collect_bind) # socket to get control signals from self.control_socket = self.start_socket( name="control_socket", sock_type=zmq.SUB, sock_con="connect", endpoint=endpoints.control_sub_con) self.control_socket.setsockopt_string(zmq.SUBSCRIBE, u"control") self.stats_expose_socket = self.start_socket( name="stats_expose_socket", sock_type=zmq.REP, sock_con="bind", endpoint=endpoints.stats_expose_bind) # revert change of umask after ipc dir was created to not interfere # with anything else (e.g. data files stored) os.umask(old_umask) self.log.info("Changed back to old umask %s", old_umask) self.poller = zmq.Poller() self.poller.register(self.control_socket, zmq.POLLIN) self.poller.register(self.stats_collect_socket, zmq.POLLIN) self.poller.register(self.stats_expose_socket, zmq.POLLIN)
def __init__(self, config, log_queue): threading.Thread.__init__(self) self.config = config self.continue_run = True # Send all logs to the main process self.log = utils.get_logger("RequestResponder", log_queue) self.context = zmq.Context() endpoint = self.config["endpoints"].request_fw_con self.request_fw_socket = self.context.socket(zmq.REP) self.request_fw_socket.bind(endpoint) self.log.info("request_fw_socket started (bind) for '%s'", endpoint)
def __init__(self, number_of_threads, time_till_closed, mon_dir, action_time, lock, stop_request, log_queue): threading.Thread.__init__(self) self.log = utils.get_logger("CheckModTime", log_queue, log_level="info") self.log.debug("init") # Make the Pool of workers self.pool = ThreadPool(number_of_threads) self.mon_dir = mon_dir self.time_till_closed = time_till_closed # s self.action_time = action_time self.lock = lock self.stopper = stop_request self.pool_running = True
def __init__(self, lock, config, data_queue, shutdown_event, log_queue): threading.Thread.__init__(self) self.log = utils.get_logger("Synchronizing", log_queue) self.lock = lock self.data_queue = data_queue self.shutdown_event = shutdown_event self.n_detectors = len(config["device_names"]) self.timeout = 1 self.sync_buffer = collections.deque(maxlen=config["buffer_size"])
def __init__(self, config, context, log_queue): super().__init__() self.config = config["datafetcher"] self.context = context self.log = utils.get_logger(self.__class__.__name__, log_queue) self.test_socket = None self.action_name = None self.address = None self.socket_conf = None self.test_signal = None self.is_req = None self.reestablish_time = 600 # in sec self.zmq_again_occurred = 0 self.show_check_warning = True
def __init__(self, eventdetector_base_config, name): """Initial setup Args: eventdetector_base_config: A dictionary containing all needed parameters encapsulated into a dictionary to prevent the event detector modules being affected by adding and removing of parameters. eventdetector_base_args should contain the following keys: config (dict): A dictionary containing the configuration parameters. check_dep (boolean): context: A zmq context log_queue: The multiprocessing queue which is used for logging. name (str): The name to be used for the logger. """ super().__init__() self.config_all = eventdetector_base_config["config"] check_dep = eventdetector_base_config["check_dep"] self.context = eventdetector_base_config["context"] self.log_queue = eventdetector_base_config["log_queue"] self.log = utils.get_logger(name, self.log_queue) # base_parameters self.required_params_base = {"eventdetector": ["type"]} self.required_params_dep = {} self.config_reduced = {} self._base_check(module_class="eventdetector", check_dep=check_dep) self.config_ed = self.config_all["eventdetector"] self.ed_type = self.config_ed["type"] if self.required_params_dep: self.config = self.config_ed[self.ed_type] else: self.config = {} self.required_params = []
def __init__(self, det_id, device_name, wait_time, shutdown_event, data_queue, log_queue): self.log = utils.get_logger("Connecting-{}".format(det_id), log_queue) self.n_attempts = 20 self.det_id = det_id self.device_name = device_name self.wait_time = wait_time self.shutdown_event = shutdown_event self.data_queue = data_queue self.device = None self._setup()
def __init__(self, config, log_queue, log_level, endpoints, stop_request, context=None): super().__init__() self.log = utils.get_logger(self.__class__.__name__, queue=log_queue, log_level=log_level) self.log.info("%s started (PID %s).", self.__class__.__name__, os.getpid()) self.config = config self.endpoints = endpoints self.stop_request = stop_request self.stopped = None self.job_socket = None self.confirmation_socket = None self.control_socket = None self.confirm_topic = None self.poller = None self.tracker = ConfirmationTracking(log_queue=log_queue, log_level=log_level) if context: self.context = context self.ext_context = True else: self.context = zmq.Context() self.ext_context = False self.create_sockets()
def __init__(self, context, beamline, det_id, config, instances, log_queue): logger_name = "{}_{}".format(self.__class__.__name__, det_id) self.log = utils.get_logger(logger_name, log_queue) super().__init__(beamline, self.log) self.context = context self.beamline = beamline self.det_id = det_id self.config = config self.log_queue = log_queue self.reply_codes = REPLYCODES self.confighandling = None self.instances = instances self.supported_keys = [] self.supported_remote_keys = [] self._setup()
def _setup(self): """Initializes parameters and creates sockets. """ log_name = "DataHandler-{}".format(self.dispatcher_id) self.log = utils.get_logger(log_name, queue=self.log_queue, log_level=self.log_level) # dict with information of all open sockets to which a data stream is # opened (host, port,...) self.open_connections = dict() self.log.info("Loading data fetcher: %s", self.config_df["type"]) try: datafetcher_m = import_module(self.config_df["type"]) except Exception: self.log.error("Could not load data fetcher %s", self.config_df["type"], exc_info=True) raise datafetcher_base_config = { "config": self.config_all, "log_queue": self.log_queue, "fetcher_id": self.dispatcher_id, "context": self.context, "lock": self.lock, "stop_request": self.stop_request, "check_dep": True } self.datafetcher = datafetcher_m.DataFetcher(datafetcher_base_config) try: self.create_sockets() except Exception: self.log.error("Cannot create sockets", ext_info=True) self.stop()
def _setup(self): """Initializes parameters and creates sockets. """ log_name = "DataDispatcher-{}".format(self.dispatcher_id) self.log = utils.get_logger(log_name, queue=self.log_queue, log_level=self.log_level) signal.signal(signal.SIGTERM, self.signal_term_handler) signal.signal(signal.SIGINT, self.signal_term_handler) self.log.info("DataDispatcher-%s started (PID %s).", self.dispatcher_id, os.getpid()) super().print_config(self.config) self.context = zmq.Context() if ("context" in self.config and not self.config["context"]): self.config["context"] = self.context try: self.create_sockets() except Exception: self.log.error("Cannot create sockets", ext_info=True) self.stop() self.datahandler = threading.Thread( target=run_datahandler, kwargs=dict(dispatcher_id=self.dispatcher_id, endpoints=self.endpoints, fixed_stream_addr=self.fixed_stream_addr, config=self.config, log_queue=self.log_queue, log_level=self.log_level, context=self.context, stop_request=self.stop_request)) self.datahandler.start()
def __init__(self, context, beamline, det_id, config, log_queue): super().__init__() self.context = context self.beamline = beamline self.det_id = det_id logger_name = "{}_{}".format(self.__class__.__name__, self.det_id) self.log = utils.get_logger(logger_name, log_queue) # the general config of the control server self.config = config self.config_static = None self.config_variable = None # the configuration hidra is running with (the one it reported back) self.remote_config = None # the original config read from the config file for this instance self.active_config = {} # hidra configuration depending on the host which set it: # the parameters set via the control client. When a config is marked # as active it means, that this changed in comparison to the one in # the config file (i.e. active_config) self.client_dep_configs = {} self.ctemplate = {} self.required_params = {} self.reply_codes = REPLYCODES self.use_statserver = None self.stats_expose_socket = None self.stats_expose_endpt_tmpl = None self.timeout = 1000 self._setup()
def setup(self): """Initializes parameters and creates sockets. """ whitelist = self.init_args["whitelist"] ldapuri = self.init_args["ldapuri"] context = self.init_args["context"] log_queue = self.init_args["log_queue"] log_level = self.init_args["log_level"] # Send all logs to the main process self.log = utils.get_logger("SignalHandler", queue=log_queue, log_level=log_level) self.log.info("SignalHandler started (PID %s).", os.getpid()) self.whitelist = utils.extend_whitelist(whitelist, ldapuri, self.log) # remember if the context was created outside this class or not if context: self.context = context self.ext_context = True else: self.log.info("Registering ZMQ context") self.context = zmq.Context() self.ext_context = False if self.config["general"]["use_statserver"]: self.setup_stats_collection() try: self.create_sockets() except Exception: self.log.error("Cannot create sockets", exc_info=True) self.stop() raise
def main(): # see https://docs.python.org/2/library/multiprocessing.html#windows freeze_support() config = { "log_file": os.path.join(BASE_DIR, "logs", "test_logging.log"), "log_size": 10485760, "verbose": True, "onscreen": "debug" } # Get queue log_queue = Queue(-1) handler = utils.get_log_handlers( config["log_file"], config["log_size"], config["verbose"], config["onscreen"] ) # Start queue listener using the stream handler above. log_queue_listener = utils.CustomQueueListener( log_queue, *handler ) log_queue_listener.start() # Create log and set handler to queue handle log = utils.get_logger("TestLogging", log_queue) log.debug("START") while True: log.debug("run") time.sleep(1)
def __init__(self, datafetcher_base_config, name): """Initial setup Checks if the required parameters are set in the configuration for the base setup. Args: datafetcher_base_config: A dictionary containing all needed parameters encapsulated into a dictionary to prevent the data fetcher modules to be affected by adding and removing of parameters. datafetcher_base_args should contain the following keys: config (dict): A dictionary containing the configuration parameters. log_queue: The multiprocessing queue which is used for logging. fetcher_id (int): The ID of this datafetcher instance. context: The ZMQ context to be used. lock: A threading lock object to handle control signal access. stop_request: A threading event to notify the data fetcher to stop. name (str): The name of the derived data fetcher module. This is used for logging. """ super().__init__() self.log_queue = datafetcher_base_config["log_queue"] self.config_all = datafetcher_base_config["config"] self.fetcher_id = datafetcher_base_config["fetcher_id"] self.context = datafetcher_base_config["context"] self.lock = datafetcher_base_config["lock"] self.stop_request = datafetcher_base_config["stop_request"] check_dep = datafetcher_base_config["check_dep"] logger_name = "{}-{}".format(name, self.fetcher_id) self.log = utils.get_logger(logger_name, self.log_queue) # base_parameters self.required_params_base = { "network": ["endpoints", "main_pid"], "datafetcher": [ "type", "chunksize", "local_target", "use_cleaner", [ "remove_data", [True, False, "stop_on_error", "with_confirmation"] ] ] } self.required_params_dep = {} self.config_reduced = {} self._base_check(module_class="datafetcher", check_dep=check_dep) self.config_df = self.config_all["datafetcher"] self.df_type = self.config_df["type"] if self.required_params_dep: self.config = self.config_df[self.df_type] else: self.config = {} self.cleaner_job_socket = None self.confirmation_topic = None self.source_file = None self.target_file = None self.control_signal = None self.required_params = [] self.base_setup()