def __init__(self, handler_id, config, lock, log_queue):
self.handler_id = handler_id
self.lock = lock
# Suppress logging messages of watchdog observer
logging.getLogger("watchdog.observers.inotify_buffer").setLevel(
logging.WARNING)
self.log = utils.get_logger(
"WatchdogEventHandler-{}".format(self.handler_id),
log_queue
)
self.log.debug("init")
self.paths = [os.path.normpath(config["monitored_dir"])]
# learn what events to detect
self.detect_all = False
self.detect_create = False
self.detect_modify = False
self.detect_delete = False
self.detect_move_from = False
self.detect_move_to = False
self.detect_close = False
regexes = []
for event, regex in iteritems(config["monitored_events"]):
self.log.debug("event: %s, pattern: %s", event, regex)
regex = convert_suffix_list_to_regex(regex,
compile_regex=True,
log=self.log)
regexes.append(regex)
if "all" in event.lower():
self.log.info("Activate all event types")
self.detect_all = regex
elif "create" in event.lower():
self.log.info("Activate on create event types")
self.detect_create = regex
elif "modify" in event.lower():
self.log.info("Activate on modify event types")
self.detect_modify = regex
elif "delete" in event.lower():
self.log.info("Activate on delete event types")
self.detect_delete = regex
elif "move_from" in event.lower():
self.log.info("Activate on move from event types")
self.detect_move_from = regex
elif "move_to" in event.lower():
self.log.info("Activate on move to event types")
self.detect_move_to = regex
elif "close" in event.lower():
self.log.info("Activate on close event types")
self.detect_close = regex
WatchdogEventHandler.regexes = regexes
self.log.debug("init: super")
super().__init__()
def _setup(self):
"""Initiate class variables and environment.
Sets static configuration parameters creates ring buffer and starts
cleanup thread.
"""
try:
self.timeout = self.config["event_timeout"]
except KeyError:
# when using old config file type
self.timeout = 1
self.file_descriptor = inotifyx.init()
# TODO why is this necessary
self.paths = [self.config["monitored_dir"]]
self.mon_subdirs = self.config["fix_subdirs"]
self.mon_regex_per_event = self.config["monitored_events"]
self.log.debug("monitored_events=%s", self.config["monitored_events"])
regexes = []
for key, value in iteritems(self.config["monitored_events"]):
self.mon_regex_per_event[key] = (convert_suffix_list_to_regex(
value, compile_regex=False, log=self.log))
regexes.append(self.mon_regex_per_event[key])
# cannot be compiled before because regexes needs to be a list
# of string
try:
self.mon_regex_per_event[key] = (re.compile(
self.mon_regex_per_event[key]))
except Exception:
self.log.error("Could not compile regex '%s'",
self.mon_regex_per_event[key],
exc_info=True)
raise
self.log.debug("regexes=%s", regexes)
self.mon_regex = convert_suffix_list_to_regex(regexes,
suffix=False,
compile_regex=True,
log=self.log)
self.history = collections.deque(maxlen=self.config["history_size"])
self.lock = threading.Lock()
self._add_watch()
if self.config["use_cleanup"]:
self.cleanup_time = self.config["time_till_closed"]
self.action_time = self.config["action_time"]
self.get_remaining_events = self._get_events_from_cleanup
self.cleanup_thread = CleanUp(paths=self.paths,
mon_subdirs=self.mon_subdirs,
mon_regex=self.mon_regex,
cleanup_time=self.cleanup_time,
action_time=self.action_time,
lock=self.lock,
log_queue=self.log_queue)
self.cleanup_thread.start()
else:
self.get_remaining_events = self._get_no_events
def _start_signal(self, signal, send_type, appid, socket_ids,
registered_ids, vari_requests, perm_requests):
"""Register socket ids and updated related lists accordingly.
Updated registered_ids, vari_requests and perm_requests in place and
send confirmation back.
Args:
signal: Signal to send after finishing
send_type: The type of data the socket ids should get.
appid: The application ID to identify where the signal came from.
socket_ids: Socket ids to be registered.
registered_ids: Already registered socket ids.
vari_requests: List of open requests (query mode).
perm_requests: List of next node number to serve (stream mode).
"""
socket_ids = utils.convert_socket_to_fqdn(socket_ids, self.log)
# Convert suffixes to regex
# for compatibility with API versions 3.1.2 or older
# socket_ids is of the format [[<host>, <prio>, <suffix>], ...]
for socket_conf in socket_ids:
self.log.debug("suffix=%s", socket_conf[2])
socket_conf[2] = convert_suffix_list_to_regex(socket_conf[2],
suffix=True,
compile_regex=False,
log=self.log)
targets = copy.deepcopy(sorted([i + [send_type] for i in socket_ids]))
# compile regex
# This cannot be done before because deepcopy does not support it
# for python versions < 3.7, see http://bugs.python.org/issue10076
for socket_conf in targets:
try:
socket_conf[2] = re.compile(socket_conf[2])
except Exception:
self.log.error("Error message was:", exc_info=True)
raise FormatError("Could not compile regex '{}'".format(
socket_conf[2]))
current_time = datetime.datetime.now().isoformat()
targetset = TargetProperties(targets=targets,
appid=appid,
time_registered=current_time)
overwrite_index = None
for i, target_properties in enumerate(registered_ids):
if target_properties.appid != appid:
continue
# the registered disjoint socket ids for each node set
# set(<host>:<port>, <host>:<port>, ...)
targets_flatlist = set([j[0] for j in target_properties.targets])
# the disjoint socket_ids to be register
# "set" is used to eliminated duplications
# set(<host>:<port>, <host>:<port>, ...) created from socket_ids
socket_ids_flatlist = set(
[socket_conf[0] for socket_conf in socket_ids])
# If the socket_ids of the node set to be register are either a
# subset or a superset of an already registered node set
# overwrite the old one with it
# new registration | registered | what to done
# (h:p, h:p2) | (h:p) | overwrite: (h:p, h:p2)
# (h:p) | (h:p, h:p2) | overwrite: (h:p)
# (h:p, h:p2) | (h:p, h:p3) | ?
# Check if socket_ids is sublist of one entry of registered_ids
# -> overwrite existing entry
if socket_ids_flatlist.issubset(targets_flatlist):
self.log.debug("socket_ids already contained, override")
overwrite_index = i
# Check if one entry of registered_ids is sublist in socket_ids
# -> overwrite existing entry
elif targets_flatlist.issubset(socket_ids_flatlist):
self.log.debug("socket_ids is superset of already "
"contained set, override")
overwrite_index = i
# TODO Mixture ?
elif not socket_ids_flatlist.isdisjoint(targets_flatlist):
self.log.error("socket_ids is neither a subset nor "
"superset of already contained set")
self.log.debug("Currently: no idea what to do with this.")
self.log.debug("socket_ids=%s", socket_ids_flatlist)
self.log.debug("registered_socketids=%s", targets_flatlist)
if overwrite_index is None:
registered_ids.append(targetset)
if perm_requests is not None:
perm_requests.append(0)
if vari_requests is not None:
vari_requests.append([])
else:
# overriding is necessary because the new request may contain
# different parameters like monitored file suffix, priority or
# connection type also this means the old socket_id set should be
# replaced in total and not only partially
self.log.debug("overwrite_index=%s", overwrite_index)
registered_ids[overwrite_index] = targetset
if perm_requests is not None:
perm_requests[overwrite_index] = 0
if vari_requests is not None:
vari_requests[overwrite_index] = []
self.log.debug("after start handling: registered_ids=%s",
registered_ids)
# send signal back to receiver
self.send_response([signal])