def test_SnareParser():
sp = GetParser('SnareParser')
sp.write(
'<23> Jan 12 23:59:59 hello world\n<24> Jan 13 23:59:59 hello foobar\n'
)
for i, event in enumerate(sp):
if i == 0:
expected = dict(
msg='<23> Jan 12 23:59:59 hello world',
_type_str='msg',
_type_num='log_ts severity facility',
facility=2,
severity=7,
)
expected['log_ts'] = calendar.timegm(
time.strptime("%d Jan 12 23:59:59" % this_year,
"%Y %b %d %H:%M:%S"))
eq_(event, expected)
if i == 1:
expected = dict(
msg='<24> Jan 13 23:59:59 hello foobar',
_type_str='msg',
_type_num='log_ts severity facility',
facility=3,
severity=0,
log_ts=1294963199,
)
expected['log_ts'] = calendar.timegm(
time.strptime("%d Jan 13 23:59:59" % this_year,
"%Y %b %d %H:%M:%S"))
eq_(event, expected)
eq_(i, 1)
def test_unspecified_charset():
log = u"あいうえお नेपाल"
chunk = (log + '\n').encode('utf-8')
lp = GetParser('LineParser')
lp.write(chunk)
eq_({'msg': log.encode('utf-8'), '_type_str': 'msg'}, lp.next())
def _check_charset(charset):
log = u"あいうえお नेपाल"
chunk = (log + '\n').encode(charset)
lp = GetParser('LineParser', charset=charset)
lp.write(chunk)
eq_({'msg': log.encode('utf-8'), '_type_str': 'msg'}, lp.next(), charset)
def test_no_pri_no_date():
sp = GetParser('SyslogParser')
sp.write('hello world\n')
for event in sp:
eq_(event, dict(
msg='hello world',
_type_str='msg',
))
def test_log_time_with_year():
sp = GetParser('SyslogParser')
sp.write('Aug 11 2011 12:58:28\n')
for event in sp:
eq_(
event,
dict(msg='Aug 11 2011 12:58:28',
_type_str='msg',
_type_num='log_ts',
_normalized_fields=dict(log_ts=1313067508)))
def test_no_pri_no_date():
sp = GetParser('SnareParser')
sp.write('hello world\n')
for event in sp:
eq_(
event,
dict(
msg='hello world',
_type_str='msg',
_type_num='log_ts',
log_ts=int(time.mktime(time.gmtime())),
))
def test_log_time_with_year():
sp = GetParser('SnareParser')
sp.write('Aug 11 2011 12:58:28\n')
for event in sp:
eq_(
event,
dict(
msg='Aug 11 2011 12:58:28',
_type_str='msg',
_type_num='log_ts',
log_ts=1313067508,
))
def __handle_tcp_client(self, sock, addr, protocol="TCP"):
"""
"""
logging.debug("tcp collector; %s connected;" % str(addr))
ip = inet.get_ip(addr)
try:
# config_ip can be changed if any device whose cidr belong to this ip is added
old_config_ips = None
old_parser_name = None
old_charset = None
parser = None
while True:
config_ips = self.__config["client_map"].keys()
if config_ips != old_config_ips:
old_config_ips = config_ips
config_ip = config_reader.get_config_ip(ip, self.__config)
if not config_ip:
return
sid = "%s|%s" % (self.__config["col_type"], config_ip)
profile = self.__config["client_map"][config_ip]
parser_name = profile.get("parser") or None
charset = profile["charset"]
if parser_name and (parser_name != old_parser_name or charset != old_charset):
if old_parser_name and old_charset:
logging.warn("settings changed for ip %s, old_parser=%s, new_parser=%s, "
"old_charset=%s, new_charset=%s", ip, old_parser_name, parser_name,
old_charset, charset)
old_parser_name = parser_name
old_charset = charset
new_parser = GetParser(parser_name, sid, charset,
profile.get('regex_pattern'), profile.get('regexparser_name'))
if parser and parser.buffer:
new_parser.write(parser.buffer)
parser = new_parser
data = sock.recv(4096)
if not data:
break
if protocol == "SSL":
self.handle_tcp_ssl_data(data, ip=ip, sid=sid, parser=parser, device_name=profile["device_name"],
normalizer=profile["normalizer"], repo=profile["repo"])
else:
self.handle_tcp_data(data, ip=ip, sid=sid, parser=parser, device_name=profile["device_name"],
normalizer=profile["normalizer"], repo=profile["repo"])
except Exception, e:
logging.warn('Exception receiving message: %s' % str(e))
def test_no_pri():
sp = GetParser('SyslogParser')
sp.write('Jan 12 23:59:59 hello world\n')
for event in sp:
expected = dict(msg='Jan 12 23:59:59 hello world',
_type_str='msg',
_type_num='log_ts',
_normalized_fields=dict(log_ts=1294876799))
expected['_normalized_fields']['log_ts'] = calendar.timegm(
time.strptime("%d Jan 12 23:59:59" % this_year,
"%Y %b %d %H:%M:%S"))
eq_(event, expected)
def _handle_tcp_client(sock, addr, config, snare_out):
log.debug("tcp collector; %s connected;" % str(addr))
ip = inet.get_ip(addr)
try:
# config_ip can be changed if any device whose cidr belong to this ip is added
old_config_ips = None
old_parser_name = None
old_charset = None
parser = None
while True:
config_ips = config["client_map"].keys()
if config_ips != old_config_ips:
old_config_ips = config_ips
config_ip = config_reader.get_config_ip(ip, config)
if not config_ip:
return
sid = '%s|%s' % (config['col_type'], config_ip)
profile = config["client_map"][config_ip]
parser_name = profile["parser"]
charset = profile["charset"]
if parser_name != old_parser_name or charset != old_charset:
if old_parser_name and old_charset:
log.warn(
'settings changed for ip %s, old_parser=%s, new_parser=%s, '
'old_charset=%s, new_charset=%s', ip, old_parser_name,
parser_name, old_charset, charset)
old_parser_name = parser_name
old_charset = charset
new_parser = GetParser(parser_name, sid, charset,
profile.get('regex_pattern'),
profile.get('regexparser_name'))
if parser and parser.buffer:
new_parser.write(parser.buffer)
parser = new_parser
data = sock.recv(4096)
log.debug("tcp collector; ip=%s, got data=%s", config_ip, data)
if not data:
break
device_name = profile["device_name"]
collected_at = config["loginspect_name"]
_handle_data(data, sid, parser, snare_out, device_name, ip,
collected_at)
finally:
sock.close()
def on_file_received(self, localfile):
logging.debug("file transfer; completed")
profile = self.config["client_map"][self.config_ip][self.username]
sid = profile["sid"]
profile["device_ip"] = self.ip
vc = shelves.VersionChecker(self.db_file, sid, localfile)
cursor = vc.get_old_cursor(localfile)
if cursor < 0:
return
profile["time_received"] = time.time()
profile["cursor"] = cursor
profile["wiring_conf_path"] = self.config.get("wiring_conf_path") or None
profile["col_type"] = self.config["col_type"]
if not self.parser_name_only:
parser_name = profile.get("parser")
charset = profile.get("charset")
parser = None
if parser_name:
try:
parser = GetParser(parser_name, sid, charset,
profile.get("regex_pattern"), profile.get("regexparser_name"))
except InvalidParserException, err:
logging.warn(err)
profile["parser"] = parser
def test_no_pri():
sp = GetParser('SnareParser')
sp.write('Jan 12 23:59:59 hello world\n')
for event in sp:
expected = dict(
msg='Jan 12 23:59:59 hello world',
_type_str='msg',
_type_num='log_ts',
log_ts=1326392099,
)
expected['log_ts'] = calendar.timegm(
time.strptime("%d Jan 12 23:59:59" % this_year,
"%Y %b %d %H:%M:%S"))
eq_(event, expected)
def file_processor(path, parser_name, sid, charset, cursor, regex_pattern,
regexparser_name):
try:
parser = GetParser(parser_name, sid, charset, regex_pattern,
regexparser_name)
except InvalidParserException, err:
log.warn(err)
return
def fetch_job(host, config, wmi_out):
global LAST_COL_TS
global LOG_COUNTER
while True:
try:
prop = config['client_map'][host]
except KeyError:
log.debug('source for %s has been deleted' % host)
return
username = prop['username']
password = prop['password']
parser_name = prop['parser']
charset = prop['charset']
facility = prop['facility']
severity = prop['severity']
device_name = prop['device_name']
normalizer = prop['normalizer']
repo = prop['repo']
try:
assert parser_name == 'WmiParser'
parser = GetParser(parser_name, charset=charset)
except InvalidParserException, err:
log.warn(err)
return
parser.set_facility_severity(facility, severity)
interval = prop['fetch_interval_seconds']
log.debug('Starting WMI fethcer for host %s with user %s' %
(host, username))
#timeGenerated, interval_ago = get_timegenerated_intervalago(host, username, password, parser, interval)
timeGenerated = get_current_host_time(host, username, password, parser)
if not timeGenerated:
log.debug(
'Unable to connect to host %s, reconnecting in 5 seconds' %
host)
gevent.sleep(interval)
continue
break
def _set_parser(self):
parser = None
if self.client_map.get("parser"):
try:
parser = GetParser(self.client_map["parser"], self.sid,
self.charset,
self.client_map.get("regex_pattern"),
self.client_map.get("regexparser_name"))
except InvalidParserException, err:
logging.warn(err)
def test_RegexParser():
rp = GetParser('RegexParser', regex_pattern=regex_pattern)
rp.write('boxabbbbbbbbbaccccccccccadddddddddddarrrrr')
rp.write('nut\ngrapes')
count = 0
event = []
for events in rp:
#if events != None:
event.append(events)
count = count + 1
len_event = len(event)
assert count == len_event
for i, e in enumerate(event):
if i == 0:
expected = dict(
_normalized_fields=dict(msg_type = 'unmatched_data'),
msg='box',
_type_str='msg msg_type'
)
eq_(e, expected)
if i == 1:
expected = dict(
_normalized_fields=dict(same = 'b', axxx = 'abbbb'),
msg='abbbbbbb',
_type_str='msg same axxx',
)
def __get_sid_parser(self, config_ip, profile):
sid = "%s|%s" % (self.__config["col_type"], config_ip)
parser_name = profile.get("parser")
charset = profile.get("charset")
parser = None
if parser_name:
try:
parser = GetParser(parser_name, sid, charset,
profile.get("regex_pattern"), profile.get("regexparser_name"))
except InvalidParserException, err:
logging.warn(err)
def _get_sid_parser(config, config_ip, profile):
sid = '%s|%s' % (config['col_type'], config_ip)
parser_name = profile.get('parser')
if parser_name is None:
log.warn("parser not found for sid=%s", sid)
return sid, None
charset = profile.get('charset')
try:
parser = GetParser(parser_name, sid, charset,
profile.get('regex_pattern'),
profile.get('regexparser_name'))
except InvalidParserException, err:
log.warn(err)
return sid, None
def _get_sid_parser(ip, config, config_ip):
sid = '%s|%s' % (config['col_type'], config_ip)
profile = config['client_map'].get(config_ip)
if not profile:
log.warn(
"logpoint agent collector; Connection attempt from unregistered IP %s",
ip)
return sid, None
parser_name = profile.get('parser')
if parser_name is None:
log.warn("logpoint agent collector; parser not found for sid=%s", sid)
return sid, None
charset = profile.get('charset')
try:
parser = GetParser(parser_name, sid, charset,
profile.get('regex_pattern'),
profile.get('regexparser_name'))
except InvalidParserException, err:
log.warn(err)
return sid, None
def test_LineParser():
lp = GetParser('LineParser')
chunk1 = (
"5 1 2010-11-22T05:25:37+00:00 Nov 22 11:08:28 192.168.2.170 pc MSWinEventLog 4 Security 1395 Mon Nov 22 11:08:26 2010 593 Security User User Success Audit PC Detailed Tracking A process has exited: Process ID: 3104 Image File Name: C:\Python25\python.exe User Name: User Domain: PC Logon ID: (0x0,0x143B1) 1346\n \n"
"5 1 2010-11-22T05:26:34+00:00 Nov 22 11:09:25 192.168.2.170 pc MSWinEventLog 4 Security 1396 Mon Nov 22 11:09:25 2010 592 Security User User Success Audit PC Detailed Tracking A new process has been created: New Process ID: 3148 Image File Name: C:\Python25\python.exe Creator Process ID: 2808 User Name: User Domain: PC Logon ID: (0x0,0x143B1) 1347\n"
"5 1 2010-11-22T05:26:42+00:00 Nov 22 11:09:33 192.168.2.170 pc MSWinEventLog 4 Security 1397 Mon Nov 22 11:09:29 2010 593 Security User User Success Audit PC Detailed Tracking A process"
)
chunk2 = (
"has exited: Process ID: 3148 Image File Name: C:\Python25\python.exe User Name: User Domain: PC Logon ID: (0x0,0x143B1) 1348\n"
"5 1 2010-11-22T05:27:12+00:00 Nov 22 11:10:04 192.168.2.170 pc MSWinEventLog 4 Security 1398 Mon Nov 22 11:10:04 2010 861 Security NETWORK SERVICE Well Known Group Failure Audit PC Detailed Tracking The Windows Firewall has detected an application listening for incoming traffic. Name: - Path: C:\WINDOWS\system32\svchost.exe Process identifier: 748 User account: NETWORK SERVICE User domain: NT AUTHORITY Service: Yes RPC server: No IP version: IPv4 IP protocol: UDP Port number: 55561 Allowed: No User notified: No 1349\n"
)
lp.write(chunk1)
count1 = sum(1 for msg in lp)
assert count1 == 3
lp.write(chunk2)
count2 = sum(1 for msg in lp)
assert count2 == 2
def test_invalid_parser():
GetParser('InvalidParser')