def runStartQuery(self, plugin_source_type, rules):
cVal = "NA"
if self.__myDataBaseCursor is None:
return cVal
try:
if plugin_source_type != "db2":
sql = rules['start_query']['query']
self.logdebug(Lazyformat("Running Start query: {}", sql))
self.__myDataBaseCursor.execute(sql)
rows = self.__myDataBaseCursor.fetchone()
if not rows:
self.logwarn("Initial query empty, please double-check")
return cVal
cVal = str((rows[0]))
sql = rules['query']['query']
elif plugin_source_type == "db2":
sql = rules['start_query']['query']
self.logdebug(Lazyformat("Start query: {}", sql))
result = ibm_db.exec_immediate(self.__objDBConn, sql)
dictionary = ibm_db.fetch_both(result)
if not dictionary:
self.logwarn("Initial query empty, please double-check")
return cVal
cVal = str((dictionary[0]))
self.loginfo("Connection closed")
if cVal == None or cVal == "None" or len(cVal) <= 0:
cVal = "NA"
except Exception, e:
cVal = "NA"
self.logerror(Lazyformat("Error running the start query: {}", e))
def stop(self):
self.logdebug(Lazyformat("Scheduling plugin stop"))
self.stop_processing = True
try:
self.join(1)
except RuntimeError:
self.logwarn(Lazyformat("Stopping thread that likely hasn't started"))
def getLastRecordTimeString(self):
"""
Gets the last record time
# To get las record by time: "Select TimeWritten from Win32_NTLogEvent Where Logfile = 'Application' and TimeWritten >\"20110803103502.000000-000\""
"""
last_record_file = ParserWMI.LAST_RECORD_FILE_TMP % (self.__hostname,
self.__section)
self.__last_record_time = ""
if os.path.exists(last_record_file):
file = open(last_record_file, 'r')
data = file.readline()
self.logdebug(Lazyformat("Last record time: {}", data.rstrip()))
self.__last_record_time = data.rstrip()
file.close()
if self.__last_record_time != "":
return
#cmd_run = "wmic -U %s%%%s //%s \"Select TimeWritten from Win32_NTLogEvent Where Logfile = '%s'\" | sort | head -n 1 | tr \"|\" \" \" |awk '{print$3;}'" % (self.__username, self.__password, self.__hostname, self.__section)
cmd_run = "wmic -U %s%%%s //%s \"Select TimeWritten from Win32_NTLogEvent Where Logfile = '%s'\" | grep %s | tr \"\|\" \" \" |awk '{print$3;}' | sort -r | head -n 1" % (
self.__username, self.__password, self.__hostname, self.__section,
self.__section)
status, output = commands.getstatusoutput(cmd_run)
if status != 0 or output == "":
self.logwarn(
Lazyformat(
"[GET_LAST_RECORD] An error occurred while trying to get logs from: {}, section: {}",
self.__hostname, self.__section))
self.__last_record_time = ""
else:
self.logdebug(
Lazyformat("[GET_LAST_RECORD] Last record time: {}", output))
self.__last_record_time = output
self.updateLastRecordTimeString()
def __init__(self, conf, plugin, conn, hostname, username, password):
self.__conf = conf
self.__plugin = plugin
self.__rules = [] # list of RuleMatch objects
self.__conn = conn
self.__hostname = hostname
self.__username = username
self.__password = password.strip()
self.__section = self.__plugin.get("config", "section")
self.__last_record_time = ""
if self.__section == "":
#search into the command to find the section
rules = self.__plugin.rules()
cmd_str = rules['cmd']['cmd']
for sec in ParserWMI.VALID_SECTIONS:
if cmd_str.find(sec) >= 0:
self.__section = sec
self.logwarn(
Lazyformat(
"The section was not found in [config]. Section deduced: {}",
self.__section))
break
if self.__section == "":
self.__section = "Security"
self.logwarn(
Lazyformat(
"The section was not found in [config]. It can't be deduced: applying default value: {}",
self.__section))
self.__pluginID = self.__plugin.get("DEFAULT", "plugin_id")
self.__stop_processing = False
self.__sectionExists = False
Detector.__init__(self, conf, plugin, conn)
def process(self):
self.loginfo(Lazyformat("Starting process ParserElastic"))
try:
es = ElasticDetector(self.elastic_url, plugin_name=self.name, store_index=self.store_index,
verify_certs=self.verify_certs,credentials=self.elastic_credentials)
except Exception as ex:
exc_type, exc_obj, exc_tb = sys.exc_info()
self.logerror(Lazyformat("process[{}]:{} {}".format(exc_tb.tb_lineno, exc_type, ex)))
return
while not self.stop_processing:
for rule in self.rules:
try:
es.rule_name = rule.name
es.plugin_sid = rule.plugin_sid
timestamp = es.get_last_timestamp()
self.loginfo(Lazyformat("Getting last documents since {}".format(timestamp)))
documents = es.get_matches_since(data_index=rule.data_index, timestamp=timestamp, query=rule.query)
for doc in documents:
wdoc = DotAccessibleDict(doc['_source'])
self.loginfo(wdoc)
group = self._fetch(wdoc, rule.fields)
self.generate(group, rule)
es.insert_timestamp()
except Exception, ex:
self.logerror(Lazyformat("Elasticsearch operation to {} failed: {}", self.elastic_url, ex))
sleep(float(self.sleep_time))
def process(self):
self.loadRules()
ftp_host = self._plugin.get("config", "ftp_server")
ftp_port = self._plugin.get("config", "ftp_port")
ftp_user = self._plugin.get("config", "ftp_user")
ftp_pswd = self._plugin.get("config", "ftp_user_password")
sleep_time = self._plugin.get("config", "tSleep")
remote_file = self._plugin.get("config", "remote_filename")
while not self.stop_processing:
#connecto to ftp server and retreive the file
try:
ftp_conn = FTP(ftp_host)
filename = "/tmp/file_%s"% time()
file_tmp = open(filename,'wb')
ftp_conn.login(ftp_user,ftp_pswd)
cmd = "RETR " + remote_file
self.loginfo(Lazyformat("FTP cmd: {}", cmd))
ftp_conn.retrbinary(cmd,file_tmp.write)
file_tmp.close()
#read the file
file_tmp = open(filename)
for line in file_tmp:
self.tryMatchRule(line)
sleep(0.001)
file_tmp.close()
os.remove(filename)
except Exception,e:
self.logerror(Lazyformat("FTP connection to {} failed: {}", ftp_host, e))
sleep(float(sleep_time))
def check_file_path(self, location):
can_read = True
if self._plugin.has_option("config", "create_file"):
create_file = self._plugin.getboolean("config", "create_file")
else:
create_file = False
if not os.path.exists(location) and create_file:
if not os.path.exists(os.path.dirname(location)):
self.logwarn(
Lazyformat("Creating the {} directory...",
os.path.dirname(location)))
os.makedirs(os.path.dirname(location), 0755)
self.logwarn(
Lazyformat("The {} file is missing. Creating a new one...",
location))
fd = open(location, 'w')
fd.close()
# open file
fd = None
try:
#check if file exist.
if os.path.isfile(location):
fd = open(location, 'r')
else:
self.logwarn(Lazyformat("File: {} does not exist!", location))
can_read = False
except IOError, e:
self.logerror(
Lazyformat("Failed to read the file {}: {}", location, e))
can_read = False
def process(self):
"""Starts to process events!
"""
number_of_lines = 0
# Check if we have to create the location file
self.check_file_path(self.__location)
while not self.__shutdown_event.is_set() and not os.path.isfile(
self.__location):
time.sleep(1)
tail = TailFollowBookmark(filename=self.__location,
track=True,
encoding=self.__plugin_configuration.get(
'config', 'encoding'))
self.loginfo(Lazyformat("Reading from {}", self.__location))
while not self.__shutdown_event.is_set():
try:
# stop processing tails if requested
if self.__shutdown_event.is_set():
break
for line in tail:
try:
json_event = json.loads(line)
event = Event()
if 'plugin_sid' not in json_event:
event[
'plugin_sid'] = self.__plugin_configuration.get(
"DEFAULT", 'plugin_sid'
) if self.__plugin_configuration.has_option(
"DEFAULT", 'plugin_sid') else 1
event['plugin_id'] = self.__plugin_configuration.get(
"DEFAULT", 'plugin_id')
for key, value in json_event.iteritems():
event_key = key
if self.__plugin_configuration.has_section(
"mapping"):
if self.__plugin_configuration.has_option(
"mapping", key):
event_key = self.__plugin_configuration.get(
"mapping", key)
if event_key == "date":
value = normalize_date(value)
event[event_key] = value
event['log'] = json.dumps(json_event)
self.send_message(event)
number_of_lines += 1
except Exception as exp:
print "CRG %s" % exp
self.logwarn(Lazyformat("Invalid Json event: {}",
line))
# Added small sleep to avoid the excessive cpu usage
time.sleep(0.01)
except Exception, e:
self.logerror(Lazyformat("Processing failed: {}", e))
def existsLogForSeciton(self):
"""
Checks whether the specified section exists.
"""
returnValue = False
"""
Example query output:
CLASS: Win32_NTEventlogFile
LogfileName|Name
Application|C:\WINDOWS\system32\config\AppEvent.Evt
Security|C:\WINDOWS\System32\config\SecEvent.Evt
System|C:\WINDOWS\system32\config\SysEvent.Evt
"""
self.__plugin.get("config", "section")
query = ParserWMI.CMD_CHECK_SECTION % (
self.__username, self.__password, self.__hostname, self.__section)
status, output = commands.getstatusoutput(query)
if status != 0 or output == "":
self.logwarn(
Lazyformat(
"An error occurred while trying to get logs from: {} - status:{} --output: {}",
self.__hostname, status, output))
else:
returnValue = True
return returnValue
def __init__(self, conf, plugin, conn, idm=False):
Detector.__init__(self, conf, plugin, conn)
self._conf = conf
self._plugin = plugin
self.rules = [] # list of RuleMatch objects
self.conn = conn
self.__myDataBaseCursor = None
self.__objDBConn = None
self.__tries = 0
self.stop_processing = False
self._databasetype = self._plugin.get("config", "source_type")
self._canrun = True
self.__idm = True if self._plugin.get("config",
"idm") == "true" else False
self.loginfo(
Lazyformat("IDM is {}", "enabled" if self.__idm else "disabled"))
if self._databasetype == "db2" and db2notloaded:
self.loginfo("You need python ibm_db module installed")
self._canrun = False
elif self._databasetype == "mysql" and mysqlnotloaded:
self.loginfo("You need python mysqldb module installed")
self._canrun = False
self.stop()
elif self._databasetype == "oracle" and oraclenotloaded:
self.loginfo("You need python cx_Oracle module installed")
self._canrun = False
elif self._databasetype == "mssql" and mssqlnotloaded:
self.loginfo("You need python pymssql module installed")
self._canrun = False
def checkTimeZone(self):
if self._timezone in all_timezones:
used_tzone = self._timezone
self.logdebug(
Lazyformat("Using custom plugin tzone data: {}", used_tzone))
elif self._agenttimezone in all_timezones:
used_tzone = self._agenttimezone
self.loginfo(
Lazyformat(
"Failed to find the {} timezone in database. Falling back to agent timezone: {}",
self._timezone, used_tzone))
else:
self.set_system_tzone()
used_tzone = self.systemtzone
self._EventTimeZone = used_tzone
def parse(self, data):
doc = xml.dom.minidom.parseString(data)
alertlist = doc.getElementsByTagName('sd:evIdsAlert')
alert_obj_list = []
for alert in alertlist:
sig = alert.getElementsByTagName('sd:signature')[0]
self.logdebug("SDEE Parsing Alert")
#Plugin sid
sid = sig.attributes['id'].nodeValue
desc = sig.attributes['description'].nodeValue
participants = alert.getElementsByTagName('sd:participants')[0]
if not participants.hasChildNodes():
self.logdebug("Ignoring SDEE alert. Possible TCP/UDP/ARP DoS")
continue
attacker = participants.getElementsByTagName('sd:attacker')[0]
#Src addr
attAddr = attacker.getElementsByTagName('sd:addr')[0].firstChild.data
#Src port
try:
attPort = attacker.getElementsByTagName('sd:port')[0].firstChild.data
except:
attPort = 0
for dst in alert.getElementsByTagName('sd:target'):
data1 = self.sanitize(alert.toxml())
self.logdebug(Lazyformat("SDEE: {}", data1))
#Dst Address
dstAddr = dst.getElementsByTagName('sd:addr')[0].firstChild.data
#Dst Port
try:
dstPort = dst.getElementsByTagName('sd:port')[0].firstChild.data
except:
dstPort = 0
self.logdebug(Lazyformat("{}:{}, {}:{}, {}:{}", sid, desc, attAddr, attPort, dstAddr, dstPort))
self.generate(sid, attAddr, attPort, dstAddr, dstPort, data1)
def updateLastRecordTimeString(self):
last_record_file = ParserWMI.LAST_RECORD_FILE_TMP % (self.__hostname,
self.__section)
thefile = open(last_record_file, 'w')
thefile.write(self.__last_record_time + "\n")
self.logdebug(
Lazyformat("Updating last_record_file: {}",
self.__last_record_time))
thefile.close()
def generate(self, groups, rule):
self.logwarn(Lazyformat(groups))
event = Event()
for key, value in rule.original_rule.iteritems():
if key not in self.SKIP_RULE_FIELD:
data = self._plugin.get_replace_array_value(value.encode('utf-8'), groups)
if data is not None:
event[key] = data
if event is not None:
self.send_message(event)
def _parse_rules(self):
try:
rules = self._plugin.rules()
for rule_name in rules:
rule = rules[rule_name]
el_rule = ElasticRules(rule_name, rule)
self.rules.append(el_rule)
except Exception as ex:
exc_type, exc_obj, exc_tb = sys.exc_info()
self.logerror(Lazyformat("_parse_rules[{}]:{} {}".format(exc_tb.tb_lineno, exc_type, ex.message)))
def _exclude_event(self, event):
if self._plugin.has_option("config", "exclude_sids"):
exclude_sids = self._plugin.get("config", "exclude_sids")
if event["plugin_sid"] in Config.split_sids(exclude_sids):
self.logdebug(
Lazyformat(
"Excluding event with plugin_id={} and plugin_sid={}",
event["plugin_id"], event["plugin_sid"]))
return True
return False
def tryConnectDB(self):
connected = False
while not connected and self.__tries < MAX_TRIES_DB_CONNECT:
time.sleep(10)
connected = self.openDataBaseCursor(
self._plugin.get("config", "source_type"))
if not connected:
self.loginfo(
Lazyformat(
"Failed to establish the DB connection, retrying in 10 seconds...try: {}",
self.__tries))
self.__tries += 1
else:
if connected:
self.loginfo(
Lazyformat("DB connection established after {} tries",
self.__tries))
if self.__tries >= MAX_TRIES_DB_CONNECT:
self.loginfo("Max connection attempts reached")
self.__tries = 0
return connected
def process(self):
self._dir = self._plugin.get("config", "directory")
self._linklayer = self._plugin.get("config", "linklayer")
self._unified_version = int(
self._plugin.get("config", "unified_version"))
if self._linklayer in ['ethernet', 'cookedlinux']:
if os.path.isdir(self._dir):
self._prefix = self._plugin.get("config", "prefix")
if self._prefix != "":
snort = ParserSnort(linklayer=self._linklayer,
unified_version=self._unified_version)
snort.init_log_dir(self._dir, self._prefix)
while 1:
# get next snort event (blocking)
ev = snort.get_snort_event()
# create the Snort event
event = Snort()
event["event_type"] = Snort.EVENT_TYPE
if event['interface'] is None:
event["interface"] = self._plugin.get(
"config", "interface")
(event["unziplen"], event["gzipdata"]) = ev.strgzip()
if event['plugin_id'] is None:
event['plugin_id'] = self._plugin.get(
"config", "plugin_id")
if event['type'] is None:
event['type'] = self._plugin.get("config", "type")
if ev.isIPV6():
event['src_ip'] = ev.sip
event['dst_ip'] = ev.dip
event['ipv6'] = "1"
self.send_message(event)
else:
self.send_message(event)
else:
self.logerror(
Lazyformat("Bad config parameter: directory ({})",
dir))
sys.exit(-1)
else:
self.logerror("Unknown link layer")
sys.exit(-1)
def __init__(self, conf, plugin, conn):
self._conf = conf # config.cfg info
self._plugin = plugin # plugins/X.cfg info
self.rules = [] # list of ElasticRules objects
self.conn = conn
self.stop_processing = False
self.sleep_time = 10
Detector.__init__(self, conf, plugin, conn)
# Initialize values with config
self._plugin_config()
self._parse_rules()
self.loginfo(Lazyformat("Init ParserElastic"))
def connectMysql(self):
host = self._plugin.get("config", "source_ip")
user = self._plugin.get("config", "user")
passwd = self._plugin.get("config", "password")
db = self._plugin.get("config", "db")
try:
self.__objDBConn = MySQLdb.connect(host=host,
user=user,
passwd=passwd,
db=db)
except Exception, e:
self.logerror(Lazyformat("DB connection failed: {}", e))
return None
def tryMatchRule(self,line):
rule_matched = False
for rule in self.rules:
rule.feed(line)
if rule.match() and not rule_matched:
self.logdebug(Lazyformat("Matching rule: [{}] -> {}", rule.name, line))
event = rule.generate_event()
self.resetAllrules()
# send the event as appropriate
if event is not None:
self.send_message(event)
# one rule matched, no need to check more
rule_matched = True
break
def getLastRecord(self):
"""
Gets the last record.
"""
last_record = 0
query = ParserWMI.CMD_GET_LAST_RECORD % (
self.__username, self.__password, self.__hostname, self.__section)
status, output = commands.getstatusoutput(query)
if status != 0:
self.logwarn(
Lazyformat(
"[GET_LAST_RECORD] An error occurred while trying to get logs from: {}, section: {}",
self.__hostname, self.__section))
elif output == "":
last_record = 0
else:
last_record = output
return last_record
def _getLocalIP(self):
if self.override_sensor:
self.loginfo("override_sensor detected")
if self._conf.has_section("plugin-defaults"):
mylocalip = self._conf.get("plugin-defaults", "sensor")
return mylocalip
hostname, aliaslist, ipaddrlist = socket.gethostbyname_ex(
socket.gethostname())
for ip in ipaddrlist:
if not ip.startswith("127"):
return ip
# In this case we try to parse the output of ip a
lines = commands.getoutput(
"ip a | grep inet | grep -v inet6 | awk '{print $2}'| grep -v \"127.0.0.1\" | awk -F '/' '{print $1}'"
).split("\n")
if len(lines) > 0:
self.loginfo(Lazyformat("Using sensor ip: {}", lines[0]))
return lines[0]
def __init__(self, conf, plugin, conn):
self._conf = conf
self._plugin = plugin
self.os_hash = {}
self.conn = conn
self.consolidation = EventConsolidation(self._conf)
self.systemtzone = None
self.__plugin_id = self._plugin.get("DEFAULT", "plugin_id")
self.__plugin_name = self._plugin.get("config", "name")
if "tzone" in self._plugin.hitems("DEFAULT"):
self._timezone = self._plugin.get("DEFAULT", "tzone")
else:
self._timezone = self._conf.get("plugin-defaults", "tzone")
self._sensorID = None
if "sensor_id" in self._conf.hitems("plugin-defaults"):
self._sensorID = self._conf.get("plugin-defaults", "sensor_id")
self.loginfo(
Lazyformat("Starting plugin with the following tzone: {}",
self._timezone))
threading.Thread.__init__(self)
self._agenttimezone = self._conf.get("plugin-defaults", "tzone")
self._EventTimeZone = None
if self._conf.has_option("plugin-defaults", "override_sensor"):
self.override_sensor = self._conf.getboolean(
"plugin-defaults", "override_sensor")
else:
self.override_sensor = False
# 2011-02-01 17:00:16
self.patterndate = re.compile('(\d{10})')
self.patternISO_date = re.compile(
'(?P<year>\d+)[\s-](?P<month>\d+)[\s-](?P<day>\d+)\s+(?P<hour>\d+):(?P<minute>\d+):(?P<second>\d+)'
)
self.set_system_tzone()
self.checkTimeZone()
def process(self):
self.__notifier = pyinotify.ThreadedNotifier(
self.__watchdog, FileEventHandler(self.addTail))
try:
self.__bookmark_dir = self._plugin.get("config", "bookmark_dir")
except:
self.__bookmark_dir = ""
mask = pyinotify.IN_CREATE
#check if the plugin has rlocation
locations = []
rlocationvalue = self._plugin.get("config", "rlocation")
if rlocationvalue != "":
files = glob.glob(rlocationvalue)
for f in files:
self.logdebug(Lazyformat("Adding location: {}", f))
locations.append(f)
else:
locations = self._plugin.get("config", "location")
locations = locations.split(',')
self.__notifier.start()
# first check if file exists
for location in locations:
if self.check_file_path(location):
self.__locations.append(location)
else:
self.__monitorLocations.append(location)
dir = os.path.dirname(location)
# if not self.__watchdog.watches.has_key(dir): #--version python-pyinotify 0.8.9
self.__watchdog.add_watch(dir, mask, rec=True)
# compile the list of regexp
unsorted_rules = self._plugin.rules()
keys = unsorted_rules.keys()
keys.sort()
for key in keys:
item = unsorted_rules[key]
self.rules.append(RuleMatch(key, item, self._plugin))
# Move to the end of file
# fd.seek(0, 2)
for location in self.__locations:
self.__tails.append(
TailFollowBookmark(location, 1, self.__bookmark_dir,
self._plugin.get('config', 'encoding')))
while not self.stop_processing:
# is plugin enabled?
if not self._plugin.getboolean("config", "enable"):
# wait until plugin is enabled
while not self._plugin.getboolean("config", "enable"):
time.sleep(1)
# plugin is now enabled, skip events generated on
# 'disable' state, so move to the end of file
self._thresholding()
for tail in self.__tails:
# stop processing tails if requested
if self.stop_processing:
break
for line in tail:
matches = 0
rules = 0
# stop processing lines if requested
if self.stop_processing:
break
rule_matched = False
for rule in self.rules:
rules += 1
rule.feed(line)
if rule.match() and not rule_matched:
matches += 1
self.logdebug(
Lazyformat("Match rule: [{}] -> {}", rule.name,
line))
event = rule.generate_event()
self.resetAllrules()
# send the event as appropriate
if event is not None:
self.send_message(event)
# one rule matched, no need to check more
rule_matched = True
break
#Added small sleep to avoid the excessive cpu usage
time.sleep(0.01)
for tail in self.__tails:
tail.close()
self.logdebug("Processing completed.")
def process(self):
self.loginfo("Started SDEE Collector")
if self.__hostname:
self.host = self.__hostname
else:
self.host = self._plugin.get("config", "source_ip")
if self.__username:
self.username = self.__username
else:
self.username = self._plugin.get("config", "user")
if self.__password:
self.password = self.__password
else:
self.password = self._plugin.get("config", "password")
self.sleepField = self._plugin.get("config", "sleep")
self.plugin_id = self._plugin.get("DEFAULT", "plugin_id")
sdee = SDEE(user=self.username,password=self.password,host=self.host,method='https', force='yes')
try:
sdee.open()
self.loginfo(Lazyformat("SDEE subscriberId: {}", sdee._subscriptionid))
f = open(self.sIdFile, 'w')
f.write("%s\n" % sdee._subscriptionid)
f.close()
except:
self.logerror(Lazyformat("Failed to open SDEE connection to device {}", self.host))
self.loginfo("SDEE: Trying to close last session")
try:
f = open(self.sIdFile, 'r')
except IOError:
self.logerror("SDEE: Cannot read subscriber ID")
return
subs = f.readline()
try:
sdee = SDEE(user=self.username,password=self.password,host=self.host,method='https', force='yes')
sdee._subscriptionid = subs
sdee.close()
except:
self.logerror("SDEE: losing last session Failed")
return
try:
sdee = SDEE(user=self.username,password=self.password,host=self.host,method='https', force='yes')
sdee.open()
self.loginfo(Lazyformat("SDEE subscriberId: {}", sdee._subscriptionid))
f = open(self.sIdFile, 'w')
f.write("%s\n" % sdee._subscriptionid)
f.close()
except:
self.logerror("SDEE Failed")
return
while 1:
sdee.get()
self.loginfo("Requesting SDEE Data...")
data = sdee.data()
self.logdebug(data)
self.parse(data)
time.sleep(int(self.sleepField))
class ParserElastic(Detector):
SKIP_RULE_FIELD = {'query', 'fields', 'data_index'}
def __init__(self, conf, plugin, conn):
self._conf = conf # config.cfg info
self._plugin = plugin # plugins/X.cfg info
self.rules = [] # list of ElasticRules objects
self.conn = conn
self.stop_processing = False
self.sleep_time = 10
Detector.__init__(self, conf, plugin, conn)
# Initialize values with config
self._plugin_config()
self._parse_rules()
self.loginfo(Lazyformat("Init ParserElastic"))
def _fetch(self, document, fields):
group = []
for field in fields:
try:
group.append(document[field])
except KeyError:
group.append("")
self.logwarn("{} doesn't exists in document {}".format(field, document))
return group
def _plugin_config(self):
self.plugin_id = self._plugin.get("DEFAULT", "plugin_id")
self.name = self._plugin.get("config", "name")
self.elastic_url = self._plugin.get("config", "elastic_url")
self.store_index = self._plugin.get("config", "store_index")
self.verify_certs = self._plugin.getboolean("config", "verify_certs")
user = self._plugin.get("config", "elastic_user")
password = self._plugin.get("config", "elastic_password")
self.elastic_credentials = None
if user and password:
self.elastic_credentials = (user, password)
def _parse_rules(self):
try:
rules = self._plugin.rules()
for rule_name in rules:
rule = rules[rule_name]
el_rule = ElasticRules(rule_name, rule)
self.rules.append(el_rule)
except Exception as ex:
exc_type, exc_obj, exc_tb = sys.exc_info()
self.logerror(Lazyformat("_parse_rules[{}]:{} {}".format(exc_tb.tb_lineno, exc_type, ex.message)))
def process(self):
self.loginfo(Lazyformat("Starting process ParserElastic"))
try:
es = ElasticDetector(self.elastic_url, plugin_name=self.name, store_index=self.store_index,
verify_certs=self.verify_certs,credentials=self.elastic_credentials)
except Exception as ex:
exc_type, exc_obj, exc_tb = sys.exc_info()
self.logerror(Lazyformat("process[{}]:{} {}".format(exc_tb.tb_lineno, exc_type, ex)))
return
while not self.stop_processing:
for rule in self.rules:
try:
es.rule_name = rule.name
es.plugin_sid = rule.plugin_sid
timestamp = es.get_last_timestamp()
self.loginfo(Lazyformat("Getting last documents since {}".format(timestamp)))
documents = es.get_matches_since(data_index=rule.data_index, timestamp=timestamp, query=rule.query)
for doc in documents:
wdoc = DotAccessibleDict(doc['_source'])
self.loginfo(wdoc)
group = self._fetch(wdoc, rule.fields)
self.generate(group, rule)
es.insert_timestamp()
except Exception, ex:
self.logerror(Lazyformat("Elasticsearch operation to {} failed: {}", self.elastic_url, ex))
sleep(float(self.sleep_time))
self.loginfo(Lazyformat("Exiting process()"))
self.logerror(
"Couldn't connect to database, maximum retries exceeded"
)
return
else:
sql = rules['query']['query']
sql = sql.replace("$1", str(cVal))
self.logdebug(sql)
result = ibm_db.exec_immediate(self.__objDBConn, sql)
row = ibm_db.fetch_tuple(result)
ret = []
while row:
self.loginfo(str(row))
ret.append(row)
row = ibm_db.fetch_tuple(result)
self.loginfo(Lazyformat("len ret {} y ref {}", len(ret), ref))
if len(ret) > 0:
cVal = ret[len(ret) - 1][ref]
for e in ret:
self.loginfo(Lazyformat("-.-->{}", e))
self.generate(e)
time.sleep(tSleep)
def connectMysql(self):
host = self._plugin.get("config", "source_ip")
user = self._plugin.get("config", "user")
passwd = self._plugin.get("config", "password")
db = self._plugin.get("config", "db")
try:
self.__objDBConn = MySQLdb.connect(host=host,
def process(self):
'''
Process the snort file.
'''
self.__setKeepWorkingValue(True)
keepWorking = self.__getKeepWorkingValue()
if self.__pluginConfig.get("config", "linklayer") != "ethernet":
self.logerror(
"This kind of snort parser only works for 'ethernet' linklayer. Please update the configuration"
)
return
if int(self.__pluginConfig.get("config", "unified_version")) != 2:
self.logerror(
"This kind of snort parser only works for 'UNIFIED 2' version. Please update the configuration"
)
return
self.__filePrefix = self.__pluginConfig.get("config", "prefix")
if self.__filePrefix == "":
self.logerror("Invalid prefix used")
return
self.__logDirectory = self.__pluginConfig.get("config", "directory")
self.__lookForFiles()
#return
SnortUnpack.startPurgeEventsThread()
last_valid_position = 0
last_valid_packet_size = 0
#testconter = 0
while keepWorking:
sleep(0.02) #to avoid excessive cpu usage when no snort events.
if self.__currentOpenedLogFile_fd == None:
if len(self.__logfiles) == 0:
#There's no files ....waiting for it
self.__lookForFiles(True)
#and wait for some time...
sleep(10)
continue
else:
#read the file!
self.__currentOpenedLogFile_name = self.__logfiles[0]
del self.__logfiles[0]
self.__timestamp = self.__currentOpenedLogFile_name[
self.__currentOpenedLogFile_name.rindex('.') + 1:]
try:
self.__currentOpenedLogFile_fd = open(
self.__currentOpenedLogFile_name, 'r')
except IOError:
self.logerror(
Lazyformat(
"Error reading file {}: it no longer exists",
self.__currentOpenedLogFile_name))
# For unified (version 2) files there is no dedicated file header. The endianess
# is always in NETWORK byte order.
else:
#there's an opened file..
self.logdebug(
Lazyformat("Processing file: {}",
self.__currentOpenedLogFile_name))
filestat = os.fstat(self.__currentOpenedLogFile_fd.fileno())
self.__currentOpenedLogFile_size = filestat[stat.ST_SIZE]
position = self.__currentOpenedLogFile_fd.tell()
if self.__skipOldEvents:
self.logdebug("Event skipping is enabled")
self.__do_skipOldEvents()
self.__skipOldEvents = False
position = self.__currentOpenedLogFile_fd.tell()
if (position + SNORT_FILE_HEADER_SIZE
) <= self.__currentOpenedLogFile_size:
data = self.__currentOpenedLogFile_fd.read(
SNORT_FILE_HEADER_SIZE)
event_type, size = struct.unpack("!II", data)
else:
self.__tryRotate()
continue
position = self.__currentOpenedLogFile_fd.tell()
#wait until the packet bytes are written by snort
max_tries = 10 # Max tries until the data should be there
while ((position + size) >
self.__currentOpenedLogFile_size) and max_tries > 0:
self.loginfo("waiting until Snort writes the packet data")
filestat = os.fstat(
self.__currentOpenedLogFile_fd.fileno())
self.__currentOpenedLogFile_size = filestat[stat.ST_SIZE]
max_tries = max_tries - 1
sleep(0.1)
if (position + size) <= self.__currentOpenedLogFile_size:
data = self.__currentOpenedLogFile_fd.read(size)
position = self.__currentOpenedLogFile_fd.tell()
if self.snort_events_by_type.has_key(event_type):
last_valid_position = position - size - SNORT_FILE_HEADER_SIZE
last_valid_packet_size = size
if event_type == UNIFIED2_EVENT: #1
SnortUnpack.get_UNIFIED2_EVENT(
data) # --Not information
elif event_type == UNIFIED2_PACKET: #2
ev = SnortUnpack.get_Serial_Unified2Packet(
data, event_type, size) # -- done
if ev:
self.send_message(ev)
elif event_type == EVENT_TYPE_EXTRA_DATA:
SnortUnpack.get_EVENT_TYPE_EXTRA_DATA(
data
) #4 -not information
elif event_type == UNIFIED2_IDS_EVENT: #7 -- done
SnortUnpack.get_Serial_Unified2IDSEvent_legacy(
data, event_type, size)
elif event_type == UNIFIED2_IDS_EVENT_IPV6: #72 -- done
SnortUnpack.get_Serial_Unified2IDSEventIPv6_legacy(
data)
elif event_type == UNIFIED2_IDS_EVENT_MPLS: #99 -not information
SnortUnpack.get_UNIFIED2_IDS_EVENT_MPLS(data)
elif event_type == UNIFIED2_IDS_EVENT_IPV6_MPLS: #100 -not information
SnortUnpack.get_UNIFIED2_IDS_EVENT_IPV6_MPLS(data)
elif event_type == UNIFIED2_IDS_EVENT_VLAN: #104 -- done
SnortUnpack.get_Unified2IDSEvent(data)
elif event_type == UNIFIED2_IDS_EVENT_IPV6_VLAN: #105 --done
SnortUnpack.get_Unified2IDSEventIPv6(data)
elif event_type == UNIFIED2_EXTRA_DATA: #110
SnortUnpack.get_SerialUnified2ExtraData(data)
elif event_type == UNIFIED2_IDS_EVENT_NG: #207 NOT YET INFORMATION
SnortUnpack.get_Unified2IDSEventNG(data)
elif event_type == UNIFIED2_IDS_EVENT_IPV6_NG: #208 - NOT YET INFORMATION
SnortUnpack.get_Unified2IDSEventIPv6_NG(data)
else:
self.logerror(
Lazyformat(
"Unknown record type: {}, last valid cursor: {}, last valid packet size: {}, current_cursor: {}, theoric packet size: {}",
event_type, last_valid_position,
last_valid_packet_size, position, size))
self.__currentOpenedLogFile_fd.seek(
position, os.SEEK_CUR)
else:
self.loginfo(
"Snort Log file size is less than packet size... we have been waiting for a second, try rotate.."
)
#Set the current position of file descriptor fd to position pos, modified by how: SEEK_SET or 0 to set the position relative to the beginning of the file; SEEK_CUR or 1 to set it relative to the current position; os.SEEK_END or 2 to set it relative to the end of the file.
self.__currentOpenedLogFile_fd.seek(position, os.SEEK_SET)
self.__tryRotate()
#testconter+=1
keepWorking = self.__getKeepWorkingValue()
SnortUnpack.stopPurgeEventsThread()
def process(self):
tSleep = DEFAULT_SLEEP
try:
tSleep = int(self._plugin.get("config", "sleep"))
except ValueError:
self.logerror(
Lazyformat(
"sleep should be an integer number...using default value: {}",
DEFAULT_SLEEP))
if not self._canrun:
self.loginfo("We can't start the process, missing modules")
return
self.loginfo("Starting Database plugin")
rules = self._plugin.rules()
run_process = False
if not self.tryConnectDB():
self.stop()
return
cVal = "NA"
plugin_source_type = self._plugin.get("config", "source_type")
while cVal == "NA" and not self.stop_processing:
cVal = self.runStartQuery(plugin_source_type, rules)
if cVal == "NA":
self.loginfo(
"No data retrieved in the start quere, retrying in 10 seconds"
)
time.sleep(10)
else:
run_process = True
ref = int(rules['query']['ref'])
while run_process and not self.stop_processing:
if self._plugin.get("config", "source_type") != "db2":
try:
if self.__myDataBaseCursor:
self.__myDataBaseCursor.close()
if self.__objDBConn:
self.__objDBConn.close()
except Exception, e:
self.loginfo(
Lazyformat("Failed to close the cursor: {}", e))
if self.tryConnectDB():
sql = rules['query']['query']
sql = sql.replace("$1", str(cVal))
self.logdebug(sql)
try:
self.__myDataBaseCursor.execute(sql)
ret = self.__myDataBaseCursor.fetchall()
except Exception, e:
self.logerror(
Lazyformat("DB query failed: {} -> {}", sql, e))
time.sleep(1)
continue
try:
if len(ret) > 0:
#We have to think about event order when processing
cVal = ret[len(ret) - 1][ref]
for e in ret:
self.generate(e)
except Exception, e:
self.logerror(
Lazyformat("Error building the event: {}", e))
time.sleep(tSleep)
