def _setup_logging(self):
config_ctrl = self.config["controlserver"]
logfile = os.path.join(
config_ctrl["log_path"],
config_ctrl["log_name"].format(bl=self.beamline))
utils.check_writable(logfile)
# Get queue
self.log_queue = Queue(-1)
handler = utils.get_log_handlers(logfile, config_ctrl["log_size"],
config_ctrl["verbose"],
config_ctrl["onscreen"])
# Start queue listener using the stream handler above
self.log_queue_listener = utils.CustomQueueListener(
self.log_queue, *handler)
self.log_queue_listener.start()
# Create log and set handler to queue handle
self.log = utils.get_logger("ControlServer", self.log_queue)
self.log.info("Init")
def __init__(self, log_queue, log_level):
self.confirmations = {}
self.jobs = {}
self.log = utils.get_logger(self.__class__.__name__,
queue=log_queue,
log_level=log_level)
def __init__(self, log_queue, lock, config):
threading.Thread.__init__(self)
self.log = utils.get_logger("Synchronizing", log_queue)
self.all_data = {}
self.lock = lock
self.server = config["kafka_server"]
self.topic = config["kafka_topic"]
self.operation = config["operation"]
self.detids = config["detids"]
self.n_detectors = config["n_detectors"]
self.monitored_dir = pathlib.Path(config["monitored_dir"])
self.fix_subdirs = config["fix_subdirs"]
self.paths = None
self.consumer = None
self.sync_buffer = collections.deque(maxlen=config["buffer_size"])
self.keep_running = True
self._setup()
def __init__(self, handler_id, config, lock, log_queue):
self.handler_id = handler_id
self.lock = lock
# Suppress logging messages of watchdog observer
logging.getLogger("watchdog.observers.inotify_buffer").setLevel(
logging.WARNING)
self.log = utils.get_logger(
"WatchdogEventHandler-{}".format(self.handler_id),
log_queue
)
self.log.debug("init")
self.paths = [os.path.normpath(config["monitored_dir"])]
# learn what events to detect
self.detect_all = False
self.detect_create = False
self.detect_modify = False
self.detect_delete = False
self.detect_move_from = False
self.detect_move_to = False
self.detect_close = False
regexes = []
for event, regex in iteritems(config["monitored_events"]):
self.log.debug("event: %s, pattern: %s", event, regex)
regex = convert_suffix_list_to_regex(regex,
compile_regex=True,
log=self.log)
regexes.append(regex)
if "all" in event.lower():
self.log.info("Activate all event types")
self.detect_all = regex
elif "create" in event.lower():
self.log.info("Activate on create event types")
self.detect_create = regex
elif "modify" in event.lower():
self.log.info("Activate on modify event types")
self.detect_modify = regex
elif "delete" in event.lower():
self.log.info("Activate on delete event types")
self.detect_delete = regex
elif "move_from" in event.lower():
self.log.info("Activate on move from event types")
self.detect_move_from = regex
elif "move_to" in event.lower():
self.log.info("Activate on move to event types")
self.detect_move_to = regex
elif "close" in event.lower():
self.log.info("Activate on close event types")
self.detect_close = regex
WatchdogEventHandler.regexes = regexes
self.log.debug("init: super")
super().__init__()
def _setup_logging(self):
config_gen = self.config["general"]
# Get queue
self.log_queue = multiprocessing.Queue(-1)
handler = utils.get_log_handlers(config_gen["log_file"],
config_gen["log_size"],
config_gen["verbose"],
config_gen["onscreen"])
# Start queue listener using the stream handler above.
self.log_queue_listener = utils.CustomQueueListener(
self.log_queue, *handler)
self.log_queue_listener.start()
# the least sever log level to forward to the queuelistener
file_log_level = "debug" if config_gen["verbose"] else "error"
if config_gen["onscreen"]:
self.log_level = utils.get_least_sever_log_level(
log_levels=(config_gen["onscreen"], file_log_level))
else:
self.log_level = file_log_level
# Create log and set handler to queue handle
self.log = utils.get_logger(self.__class__.__name__,
queue=self.log_queue,
log_level=self.log_level)
self.log.info("Setting process log level to '%s'", self.log_level)
def __init__(self, beamline, backup_file, log_queue):
self.log = utils.get_logger(self.__class__.__name__, log_queue)
self.beamline = beamline
self.backup_file = backup_file.format(bl=self.beamline)
self.reply_codes = REPLYCODES
self.instances = None
self._set_instances()
def _setup(self):
"""Initializes parameters and creates sockets.
"""
self.log = utils.get_logger(self.__class__.__name__,
queue=self.log_queue,
log_level=self.log_level)
self.log.info("%s started (PID %s).", self.__class__.__name__,
os.getpid())
signal.signal(signal.SIGTERM, self.signal_term_handler)
try:
self.timeout = self.config["general"]["taskprovider_timeout"]
except KeyError:
self.timeout = 1000
self.log.debug("Set timeout to %s ms", self.timeout)
# remember if the context was created outside this class or not
self.log.info("Registering ZMQ context")
self.context = zmq.Context()
try:
self.ignore_accumulated_events = (
self.config["eventdetector"]["ignore_accumulated_events"])
except KeyError:
self.ignore_accumulated_events = False
ed_type = self.config["eventdetector"]["type"]
self.log.info("Loading event detector: %s", ed_type)
try:
eventdetector_m = import_module(ed_type)
except Exception:
self.log.error("Could not load event detector %s",
ed_type,
exc_info=True)
raise
self.eventdetector = eventdetector_m.EventDetector({
"config":
self.config,
"log_queue":
self.log_queue,
"check_dep":
True,
"context":
self.context
})
try:
self.create_sockets()
except Exception:
self.log.error("Cannot create sockets", exc_info=True)
self.stop()
def _setup(self):
self.log = utils.get_logger(self.__class__.__name__,
queue=self.log_queue,
log_level=self.log_level)
# --------------------------------------------------------------------
# zmq setup
# --------------------------------------------------------------------
self.context = zmq.Context()
endpoints = self.config["network"]["endpoints"]
# to enable communication between the StatServer and
# hidra-control running as different users
# this has to be called even if the ipc dir was already created by
# another hidra instance
old_umask = os.umask(self.ipc_dir_umask)
self.log.info("Changed umask on ipc_dir to %s", self.ipc_dir_umask)
self.stats_collect_socket = self.start_socket(
name="stats_collect_socket",
sock_type=zmq.PULL,
sock_con="bind",
endpoint=endpoints.stats_collect_bind)
# socket to get control signals from
self.control_socket = self.start_socket(
name="control_socket",
sock_type=zmq.SUB,
sock_con="connect",
endpoint=endpoints.control_sub_con)
self.control_socket.setsockopt_string(zmq.SUBSCRIBE, u"control")
self.stats_expose_socket = self.start_socket(
name="stats_expose_socket",
sock_type=zmq.REP,
sock_con="bind",
endpoint=endpoints.stats_expose_bind)
# revert change of umask after ipc dir was created to not interfere
# with anything else (e.g. data files stored)
os.umask(old_umask)
self.log.info("Changed back to old umask %s", old_umask)
self.poller = zmq.Poller()
self.poller.register(self.control_socket, zmq.POLLIN)
self.poller.register(self.stats_collect_socket, zmq.POLLIN)
self.poller.register(self.stats_expose_socket, zmq.POLLIN)
def __init__(self, config, log_queue):
threading.Thread.__init__(self)
self.config = config
self.continue_run = True
# Send all logs to the main process
self.log = utils.get_logger("RequestResponder", log_queue)
self.context = zmq.Context()
endpoint = self.config["endpoints"].request_fw_con
self.request_fw_socket = self.context.socket(zmq.REP)
self.request_fw_socket.bind(endpoint)
self.log.info("request_fw_socket started (bind) for '%s'", endpoint)
def __init__(self, number_of_threads, time_till_closed, mon_dir,
action_time, lock, stop_request, log_queue):
threading.Thread.__init__(self)
self.log = utils.get_logger("CheckModTime",
log_queue,
log_level="info")
self.log.debug("init")
# Make the Pool of workers
self.pool = ThreadPool(number_of_threads)
self.mon_dir = mon_dir
self.time_till_closed = time_till_closed # s
self.action_time = action_time
self.lock = lock
self.stopper = stop_request
self.pool_running = True
def __init__(self,
lock,
config,
data_queue,
shutdown_event,
log_queue):
threading.Thread.__init__(self)
self.log = utils.get_logger("Synchronizing", log_queue)
self.lock = lock
self.data_queue = data_queue
self.shutdown_event = shutdown_event
self.n_detectors = len(config["device_names"])
self.timeout = 1
self.sync_buffer = collections.deque(maxlen=config["buffer_size"])
def __init__(self, config, context, log_queue):
super().__init__()
self.config = config["datafetcher"]
self.context = context
self.log = utils.get_logger(self.__class__.__name__, log_queue)
self.test_socket = None
self.action_name = None
self.address = None
self.socket_conf = None
self.test_signal = None
self.is_req = None
self.reestablish_time = 600 # in sec
self.zmq_again_occurred = 0
self.show_check_warning = True
def __init__(self, eventdetector_base_config, name):
"""Initial setup
Args:
eventdetector_base_config: A dictionary containing all needed
parameters encapsulated into a dictionary to prevent the event
detector modules being affected by adding and removing of
parameters.
eventdetector_base_args should contain the following keys:
config (dict): A dictionary containing the configuration
parameters.
check_dep (boolean):
context: A zmq context
log_queue: The multiprocessing queue which is used for
logging.
name (str): The name to be used for the logger.
"""
super().__init__()
self.config_all = eventdetector_base_config["config"]
check_dep = eventdetector_base_config["check_dep"]
self.context = eventdetector_base_config["context"]
self.log_queue = eventdetector_base_config["log_queue"]
self.log = utils.get_logger(name, self.log_queue)
# base_parameters
self.required_params_base = {"eventdetector": ["type"]}
self.required_params_dep = {}
self.config_reduced = {}
self._base_check(module_class="eventdetector", check_dep=check_dep)
self.config_ed = self.config_all["eventdetector"]
self.ed_type = self.config_ed["type"]
if self.required_params_dep:
self.config = self.config_ed[self.ed_type]
else:
self.config = {}
self.required_params = []
def __init__(self,
det_id,
device_name,
wait_time,
shutdown_event,
data_queue,
log_queue):
self.log = utils.get_logger("Connecting-{}".format(det_id), log_queue)
self.n_attempts = 20
self.det_id = det_id
self.device_name = device_name
self.wait_time = wait_time
self.shutdown_event = shutdown_event
self.data_queue = data_queue
self.device = None
self._setup()
def __init__(self,
config,
log_queue,
log_level,
endpoints,
stop_request,
context=None):
super().__init__()
self.log = utils.get_logger(self.__class__.__name__,
queue=log_queue,
log_level=log_level)
self.log.info("%s started (PID %s).", self.__class__.__name__,
os.getpid())
self.config = config
self.endpoints = endpoints
self.stop_request = stop_request
self.stopped = None
self.job_socket = None
self.confirmation_socket = None
self.control_socket = None
self.confirm_topic = None
self.poller = None
self.tracker = ConfirmationTracking(log_queue=log_queue,
log_level=log_level)
if context:
self.context = context
self.ext_context = True
else:
self.context = zmq.Context()
self.ext_context = False
self.create_sockets()
def __init__(self, context, beamline, det_id, config, instances,
log_queue):
logger_name = "{}_{}".format(self.__class__.__name__, det_id)
self.log = utils.get_logger(logger_name, log_queue)
super().__init__(beamline, self.log)
self.context = context
self.beamline = beamline
self.det_id = det_id
self.config = config
self.log_queue = log_queue
self.reply_codes = REPLYCODES
self.confighandling = None
self.instances = instances
self.supported_keys = []
self.supported_remote_keys = []
self._setup()
def _setup(self):
"""Initializes parameters and creates sockets.
"""
log_name = "DataHandler-{}".format(self.dispatcher_id)
self.log = utils.get_logger(log_name,
queue=self.log_queue,
log_level=self.log_level)
# dict with information of all open sockets to which a data stream is
# opened (host, port,...)
self.open_connections = dict()
self.log.info("Loading data fetcher: %s", self.config_df["type"])
try:
datafetcher_m = import_module(self.config_df["type"])
except Exception:
self.log.error("Could not load data fetcher %s",
self.config_df["type"],
exc_info=True)
raise
datafetcher_base_config = {
"config": self.config_all,
"log_queue": self.log_queue,
"fetcher_id": self.dispatcher_id,
"context": self.context,
"lock": self.lock,
"stop_request": self.stop_request,
"check_dep": True
}
self.datafetcher = datafetcher_m.DataFetcher(datafetcher_base_config)
try:
self.create_sockets()
except Exception:
self.log.error("Cannot create sockets", ext_info=True)
self.stop()
def _setup(self):
"""Initializes parameters and creates sockets.
"""
log_name = "DataDispatcher-{}".format(self.dispatcher_id)
self.log = utils.get_logger(log_name,
queue=self.log_queue,
log_level=self.log_level)
signal.signal(signal.SIGTERM, self.signal_term_handler)
signal.signal(signal.SIGINT, self.signal_term_handler)
self.log.info("DataDispatcher-%s started (PID %s).",
self.dispatcher_id, os.getpid())
super().print_config(self.config)
self.context = zmq.Context()
if ("context" in self.config and not self.config["context"]):
self.config["context"] = self.context
try:
self.create_sockets()
except Exception:
self.log.error("Cannot create sockets", ext_info=True)
self.stop()
self.datahandler = threading.Thread(
target=run_datahandler,
kwargs=dict(dispatcher_id=self.dispatcher_id,
endpoints=self.endpoints,
fixed_stream_addr=self.fixed_stream_addr,
config=self.config,
log_queue=self.log_queue,
log_level=self.log_level,
context=self.context,
stop_request=self.stop_request))
self.datahandler.start()
def __init__(self, context, beamline, det_id, config, log_queue):
super().__init__()
self.context = context
self.beamline = beamline
self.det_id = det_id
logger_name = "{}_{}".format(self.__class__.__name__, self.det_id)
self.log = utils.get_logger(logger_name, log_queue)
# the general config of the control server
self.config = config
self.config_static = None
self.config_variable = None
# the configuration hidra is running with (the one it reported back)
self.remote_config = None
# the original config read from the config file for this instance
self.active_config = {}
# hidra configuration depending on the host which set it:
# the parameters set via the control client. When a config is marked
# as active it means, that this changed in comparison to the one in
# the config file (i.e. active_config)
self.client_dep_configs = {}
self.ctemplate = {}
self.required_params = {}
self.reply_codes = REPLYCODES
self.use_statserver = None
self.stats_expose_socket = None
self.stats_expose_endpt_tmpl = None
self.timeout = 1000
self._setup()
def setup(self):
"""Initializes parameters and creates sockets.
"""
whitelist = self.init_args["whitelist"]
ldapuri = self.init_args["ldapuri"]
context = self.init_args["context"]
log_queue = self.init_args["log_queue"]
log_level = self.init_args["log_level"]
# Send all logs to the main process
self.log = utils.get_logger("SignalHandler",
queue=log_queue,
log_level=log_level)
self.log.info("SignalHandler started (PID %s).", os.getpid())
self.whitelist = utils.extend_whitelist(whitelist, ldapuri, self.log)
# remember if the context was created outside this class or not
if context:
self.context = context
self.ext_context = True
else:
self.log.info("Registering ZMQ context")
self.context = zmq.Context()
self.ext_context = False
if self.config["general"]["use_statserver"]:
self.setup_stats_collection()
try:
self.create_sockets()
except Exception:
self.log.error("Cannot create sockets", exc_info=True)
self.stop()
raise
def main():
# see https://docs.python.org/2/library/multiprocessing.html#windows
freeze_support()
config = {
"log_file": os.path.join(BASE_DIR, "logs", "test_logging.log"),
"log_size": 10485760,
"verbose": True,
"onscreen": "debug"
}
# Get queue
log_queue = Queue(-1)
handler = utils.get_log_handlers(
config["log_file"],
config["log_size"],
config["verbose"],
config["onscreen"]
)
# Start queue listener using the stream handler above.
log_queue_listener = utils.CustomQueueListener(
log_queue, *handler
)
log_queue_listener.start()
# Create log and set handler to queue handle
log = utils.get_logger("TestLogging", log_queue)
log.debug("START")
while True:
log.debug("run")
time.sleep(1)
def __init__(self, datafetcher_base_config, name):
"""Initial setup
Checks if the required parameters are set in the configuration for
the base setup.
Args:
datafetcher_base_config: A dictionary containing all needed
parameters encapsulated into a dictionary to prevent the data
fetcher modules to be affected by adding and removing of
parameters.
datafetcher_base_args should contain the following keys:
config (dict): A dictionary containing the configuration
parameters.
log_queue: The multiprocessing queue which is used for
logging.
fetcher_id (int): The ID of this datafetcher instance.
context: The ZMQ context to be used.
lock: A threading lock object to handle control signal
access.
stop_request: A threading event to notify the data fetcher
to stop.
name (str): The name of the derived data fetcher module. This is
used for logging.
"""
super().__init__()
self.log_queue = datafetcher_base_config["log_queue"]
self.config_all = datafetcher_base_config["config"]
self.fetcher_id = datafetcher_base_config["fetcher_id"]
self.context = datafetcher_base_config["context"]
self.lock = datafetcher_base_config["lock"]
self.stop_request = datafetcher_base_config["stop_request"]
check_dep = datafetcher_base_config["check_dep"]
logger_name = "{}-{}".format(name, self.fetcher_id)
self.log = utils.get_logger(logger_name, self.log_queue)
# base_parameters
self.required_params_base = {
"network": ["endpoints", "main_pid"],
"datafetcher": [
"type", "chunksize", "local_target", "use_cleaner",
[
"remove_data",
[True, False, "stop_on_error", "with_confirmation"]
]
]
}
self.required_params_dep = {}
self.config_reduced = {}
self._base_check(module_class="datafetcher", check_dep=check_dep)
self.config_df = self.config_all["datafetcher"]
self.df_type = self.config_df["type"]
if self.required_params_dep:
self.config = self.config_df[self.df_type]
else:
self.config = {}
self.cleaner_job_socket = None
self.confirmation_topic = None
self.source_file = None
self.target_file = None
self.control_signal = None
self.required_params = []
self.base_setup()
