def addTail(self, newlocation):
self.__tailLock.acquire()
for location in self.__monitorLocations:
if location == newlocation:
self.__tails.append(TailFollowBookmark(location, 1, self.__bookmark_dir, self._plugin.get("config", "encoding")))
self.__monitorLocations.remove(location)
break
self.__tailLock.release()
def process(self):
"""Starts to process events!
"""
number_of_lines = 0
# Check if we have to create the location file
self.check_file_path(self.__location)
while not self.__shutdown_event.is_set() and not os.path.isfile(
self.__location):
time.sleep(1)
tail = TailFollowBookmark(filename=self.__location,
track=True,
encoding=self.__plugin_configuration.get(
'config', 'encoding'))
logger.info("Plugin[%s] Reading from %s " %
(self.__plugin_id, self.__location))
while not self.__shutdown_event.is_set():
try:
# stop processing tails if requested
if self.__shutdown_event.is_set():
break
for line in tail:
try:
json_event = json.loads(line)
event = Event()
if 'plugin_sid' not in json_event:
event[
'plugin_sid'] = self.__plugin_configuration.get(
"DEFAULT", 'plugin_sid'
) if self.__plugin_configuration.has_option(
"DEFAULT", 'plugin_sid') else 1
event['plugin_id'] = self.__plugin_configuration.get(
"DEFAULT", 'plugin_id')
for key, value in json_event.iteritems():
event_key = key
if self.__plugin_configuration.has_section(
"mapping"):
if self.__plugin_configuration.has_option(
"mapping", key):
event_key = self.__plugin_configuration.get(
"mapping", key)
if event_key == "date":
value = normalize_date(value)
event[event_key] = value
event['log'] = json.dumps(json_event)
self.send_message(event)
number_of_lines += 1
except Exception as exp:
print "CRG %s" % exp
logger.warning("Invalid Json event: %s" % str(line))
# Added small sleep to avoid the excessive cpu usage
time.sleep(0.01)
except Exception, e:
logger.error("Plugin[{0}]: {1}".format(self.__plugin_id,
str(e)))
def process(self):
self.__notifier = pyinotify.ThreadedNotifier(
self.__watchdog, FileEventHandler(self.addTail))
try:
self.__bookmark_dir = self._plugin.get("config", "bookmark_dir")
except:
self.__bookmark_dir = ""
mask = EventsCodes.IN_CREATE #pyinotify.IN_CREATE
#check if the plugin has rlocation
locations = []
rlocationvalue = self._plugin.get("config", "rlocation")
if rlocationvalue != "":
files = glob.glob(rlocationvalue)
for f in files:
logger.debug("Adding location :%s" % f)
locations.append(f)
else:
locations = self._plugin.get("config", "location")
locations = locations.split(',')
self.__notifier.start()
# first check if file exists
for location in locations:
if self.check_file_path(location):
self.__locations.append(location)
else:
self.__monitorLocations.append(location)
dir = os.path.dirname(location)
# if not self.__watchdog.watches.has_key(dir): #--version python-pyinotify 0.8.9
self.__watchdog.add_watch(dir, mask, rec=True)
# compile the list of regexp
unsorted_rules = self._plugin.rules()
keys = unsorted_rules.keys()
keys.sort()
for key in keys:
item = unsorted_rules[key]
self.rules.append(SnortSyslogBinRuleMatch(key, item, self._plugin))
# Move to the end of file
# fd.seek(0, 2)
for location in self.__locations:
self.__tails.append(
TailFollowBookmark(location, 1, self.__bookmark_dir,
self._plugin.get("config", "encoding")))
while not self.stop_processing:
# is plugin enabled?
if not self._plugin.getboolean("config", "enable"):
# wait until plugin is enabled
while not self._plugin.getboolean("config", "enable"):
time.sleep(1)
# plugin is now enabled, skip events generated on
# 'disable' state, so move to the end of file
self._thresholding()
for tail in self.__tails:
# stop processing tails if requested
if self.stop_processing:
break
for line in tail:
matches = 0
rules = 0
# stop processing lines if requested
if self.stop_processing:
break
rule_matched = False
for rule in self.rules:
rules += 1
rule.feed(line)
if rule.match() and not rule_matched:
matches += 1
event = rule.generate_event()
self.resetAllrules()
# send the event as appropriate
if event is not None:
self.send_message(event)
# one rule matched, no need to check more
rule_matched = True
break
#Added small sleep to avoid the excessive cpu usage
time.sleep(0.01)
for tail in self.__tails:
tail.close()
logger.debug("Processing completed.")
def process(self):
try:
bookmark_dir = self._plugin.get("config", "bookmark_dir")
except:
bookmark_dir = ""
locations = self._plugin.get("config", "location")
locations = locations.split(',')
# first check if file exists
for location in locations:
if self.check_file_path(location):
self.__locations.append(location)
# compile the list of regexp
unsorted_rules = self._plugin.rules()
keys = unsorted_rules.keys()
keys.sort()
for key in keys:
item = unsorted_rules[key]
self.rules.append(RuleMatch(key, item, self._plugin))
# Move to the end of file
# fd.seek(0, 2)
tails = []
for location in self.__locations:
tails.append(
TailFollowBookmark(
location, 1, bookmark_dir,
self._plugin.getboolean("config", "unicode_support")))
while not self.stop_processing:
# is plugin enabled?
if not self._plugin.getboolean("config", "enable"):
# wait until plugin is enabled
while not self._plugin.getboolean("config", "enable"):
time.sleep(1)
# plugin is now enabled, skip events generated on
# 'disable' state, so move to the end of file
self._thresholding()
for tail in tails:
# stop processing tails if requested
if self.stop_processing:
break
for line in tail:
matches = 0
rules = 0
# stop processing lines if requested
if self.stop_processing:
break
rule_matched = False
for rule in self.rules:
rules += 1
rule.feed(line)
if rule.match() and not rule_matched:
matches += 1
logger.debug('Match rule: [%s] -> %s' %
(rule.name, line))
event = rule.generate_event()
# send the event as appropriate
if event is not None:
self.send_message(event)
# one rule matched, no need to check more
rule_matched = True
break
time.sleep(0.1)
for tail in tails:
tail.close()
logger.debug("Processing completed.")