def _handle_data(event, ip, col_type, device_name, collected_at, normalizer,
repo, ipssdee_out):
global LAST_COL_TS
global LOG_COUNTER
ipssdee_out.start_benchmarker_processing()
col_ts = int(time.time())
if col_ts > LAST_COL_TS:
LAST_COL_TS = col_ts
LOG_COUNTER = 0
mid_prefix = '%s|%s|%s|%d|' % (collected_at, col_type, ip, col_ts)
LOG_COUNTER += 1
event['mid'] = mid_prefix + "%d" % LOG_COUNTER
event['col_ts'] = col_ts
event['device_name'] = device_name
event["collected_at"] = collected_at
event["col_type"] = col_type
event["device_ip"] = ip
##update msg_types here
event = remove_NULL_fields(event)
msg_types_dict = fill_msg_types(event)
event.update(msg_types_dict)
event['_counter'] = LOG_COUNTER
event['device_ip'] = ip
msgfilling.add_types(event, '_type_str', 'device_ip')
msgfilling.add_types(event, '_type_ip', 'device_ip')
event['normalizer'] = normalizer
event['repo'] = repo
ipssdee_out.send_with_norm_policy_and_repo(event)
def _handle_data(event, sid, netflow_out, device_name, col_type, ip,
collected_at):
netflow_out.start_benchmarker_processing()
global LAST_COL_TS
global LOG_COUNTER
col_ts = int(time.time())
if col_ts > LAST_COL_TS:
LAST_COL_TS = col_ts
LOG_COUNTER = 0
mid_prefix = '%s|%s|%s|' % (collected_at, sid, col_ts)
LOG_COUNTER += 1
event['mid'] = mid_prefix + "%d" % LOG_COUNTER
event['device_name'] = device_name
event['col_type'] = col_type
event['msg'] = ''
event['device_ip'] = ip
event['collected_at'] = collected_at
msgfilling.add_types(event, '_type_str',
'msg col_type device_name collected_at')
msgfilling.add_types(event, '_type_ip', 'device_ip')
#msgfilling.add_types(event, '_type_str', '_raw_msg_b')
netflow_out.send_with_mid(event)
def _handle_data(data, sid, parser, snare_out, device_name, ip, collected_at):
snare_out.start_benchmarker_processing()
global LAST_COL_TS
global LOG_COUNTER
col_ts = int(time.time())
if col_ts > LAST_COL_TS:
LAST_COL_TS = col_ts
LOG_COUNTER = 0
mid_prefix = '%s|%s|%d|' % (collected_at, sid, col_ts)
parser.write(data)
#log.info("checking data: %s", data)
for event in parser:
LOG_COUNTER += 1
event['mid'] = mid_prefix + "%d" % LOG_COUNTER
event['device_name'] = device_name
event['device_ip'] = ip
event['collected_at'] = collected_at
msgfilling.add_types(event, '_type_str', 'device_name collected_at')
msgfilling.add_types(event, '_type_ip', 'device_ip')
snare_out.send_with_mid(event)
def transform(grouped_files, col_out, estimator, config, updater):
last_ts = 0
last_path = ""
recorded_col_ts = updater.get_col_ts()
recorded_col_ts_human = time.strftime("%Y%m%d %H:%M:%S",
time.localtime(recorded_col_ts))
start = time.time()
throttler = timing.speed_throttler(MAX_SPEED)
for day, filepaths in grouped_files:
if time.mktime(time.strptime(str(day),
"%Y%m%d")) + 86400 <= recorded_col_ts:
logging.warning(
"skipping logs for day %d, last tracked col_ts is %d (%s)",
day, recorded_col_ts, recorded_col_ts_human)
continue
for col_ts, event, path in reader.event_generator_by_time(
filepaths, config):
throttler.next()
if col_ts < recorded_col_ts:
continue
if col_ts < last_ts:
assert path == last_path, "path=%s; last_path=%s" % (path,
last_path)
logging.warning(
"outdated msg with col_ts=%s, last_ts=%s; path=%s", col_ts,
last_ts, path)
elif col_ts > last_ts:
last_ts = col_ts
counter = 0
# save col_ts every col_ts minute
if col_ts % 60 == 0:
updater.save_col_ts(col_ts)
gevent.sleep()
counter += 1
last_path = path
event["mid"] = event["mid"] % counter
event["_counter"] = counter
event["collected_at"] = config["loginspect_name"]
msgfilling.add_types(event, '_type_str', 'collected_at')
event['normalizer'] = config['normalizer']
event['repo'] = config['repo']
col_out.send_with_norm_policy_and_repo(event)
estimator.modify(filepaths, add=False)
duration = time.time() - start
total = reader.COUNTER
logging.warning(
"%d messages processed in %d sec, @ %d msg/sec, @ %d bytes/sec", total,
duration, total / duration, reader.BYTES_READ / duration)
def prepare_event(self, event, field, value, _type="_type_str", _normalized=True):
"""
Update event with field/value and msgfilling done
"""
if _normalized:
if event.get("_normalized_fields"):
event["_normalized_fields"][field] = value
else:
event["_normalized_fields"] = dict(field=value)
else:
event[field] = value
msgfilling.add_types(event, _type, field)
def next(self):
event = super(SyslogParser, self).next()
found = self.regex.match(event['msg'])
if not found:
# this never happens because everything is optional in regex
log.warn(
'syslog parser; received message with incorrect syslog format;'
' sid=%s; msg=%r', self.sid, event['msg'])
else:
pri = found.group('pri')
log_time = found.group('log_time')
year = found.group('year')
if not pri and not log_time:
log.info(
'syslog parser; received message with incorrect syslog format;'
' sid=%s; msg=%r', self.sid, event['msg'])
else:
event["_normalized_fields"] = {}
if log_time:
if year:
log_ts = timestamp.fromsyslogtime_with_year(log_time)
else:
try:
log_ts = timestamp.fromsyslogtime(log_time)
except ValueError:
log.warn(
"error while parsing log_ts; possibly because client timezone is not UTC; sid=%s; log_time=%s",
self.sid, log_time)
log_ts = None
if log_ts:
event["_normalized_fields"]["log_ts"] = log_ts
#event['log_ts'] = log_ts
msgfilling.add_types(event, '_type_num', 'log_ts')
if pri:
pri = int(pri)
sev = pri % 8
fac = pri // 8
event["_normalized_fields"]["severity"] = sev
event["_normalized_fields"]["facility"] = fac
#event['severity'] = sev
#event['facility'] = fac
msgfilling.add_types(event, '_type_num',
'severity facility')
return event
def _handle_unparsed_raw_data(unparsed_data_info, sid, netflow_out,
device_name, col_type, ip, collected_at):
for template_id, data_list in unparsed_data_info.iteritems():
unparsed_msg = {}
unparsed_msg["template_id"] = template_id
unparsed_msg["msg_status"] = "unparsed"
unparsed_msg["version"] = 9
msgfilling.add_types(unparsed_msg, '_type_num', 'template_id version')
msgfilling.add_types(unparsed_msg, '_type_str', 'msg_status')
for length, datum in data_list:
unparsed_msg["_p__raw_msg_b"] = binascii.b2a_base64(str(datum))
_handle_data(unparsed_msg, sid, netflow_out, device_name, col_type,
ip, collected_at)
def _handle_template(self, sid, netflow_out, device_name, col_type,
device_ip, collected_at):
for (k, v) in self.template_data_dict.iteritems():
_raw_msg_b = binascii.b2a_base64(self.template_data_dict[k])
event = dict(
_p___raw_msg_b=_raw_msg_b,
packet_type="template",
version=self.version,
template_id=k,
)
msgfilling.add_types(event, '_type_str', 'packet_type')
msgfilling.add_types(event, '_type_num', 'template_id version')
_handle_data(event, sid, netflow_out, device_name, col_type,
device_ip, collected_at)
def next(self):
event = super(SnareParser, self).next()
found = self.regex.match(event['msg'])
if not found:
# this never happens because everything is optional in regex
log.warn('snare parser; received message with incorrect snare format;'
' sid=%s; msg=%r', self.sid, event['msg'])
else:
pri = found.group('pri')
log_time = found.group('log_time')
year = found.group('year')
flag_syslog = True
if not log_time:
log.debug("Set current time as log_time")
log_time = time.strftime("%b %d %Y %H:%M:%S ", time.gmtime())
#year = time.gmtime().tm_year
log_ts = int(time.mktime(time.gmtime()))
flag_syslog = False
if not pri and not log_time:
log.info('snare parser; received message with incorrect snare format;'
' sid=%s; msg=%r', self.sid, event['msg'])
else:
if log_time and flag_syslog:
if year:
log_ts = timestamp.fromsyslogtime_with_year(log_time)
else:
try:
log_ts = timestamp.fromsyslogtime(log_time)
except ValueError, e:
log.warn("error while parsing log_ts; possibly because client timezone is not UTC; sid=%s; log_time=%s", self.sid, log_time)
log_ts = None
if log_ts:
event['log_ts'] = log_ts
msgfilling.add_types(event, '_type_num', 'log_ts')
if pri:
pri = int(pri)
sev = pri % 8
fac = pri // 8
event['severity'] = sev
event['facility'] = fac
msgfilling.add_types(event, '_type_num', 'severity facility')
def _add_types(event, oid, oid_val):
msgfilling.add_types(event, '_type_str', 'msg')
oid_type = oid, str(oid_val.__class__).split('.')[-1:][0].upper()
if oid_type in [
'INTEGER', 'INTEGER32', 'TIMETICKS', 'COUNTER32', 'GAUGE32',
"COUNTER64"
]:
msgfilling.add_types(event, '_type_num', oid)
elif oid_type in ['OCTETSTRING', 'OID', 'HEXSTRING', 'PHYSADDRESS']:
msgfilling.add_types(event, '_type_str', oid)
elif oid_type in ['IPADDRESS']:
msgfilling.add_types(event, '_type_ip', oid)
else:
msgfilling.add_types(event, '_type_str', oid)
def fill_msg_types(d):
msg_types = {}
for (k, v) in d.iteritems():
#print k, ": ", v
if v and type(v) is not type(list()):
if is_valid_ipv4(v) or is_valid_ipv6(v):
msgfilling.add_types(msg_types, '_type_ip', k)
msgfilling.add_types(msg_types, '_type_str', k)
elif is_valid_num(v):
msgfilling.add_types(msg_types, '_type_num', k)
elif isinstance(v, list):
pass
else:
msgfilling.add_types(msg_types, '_type_str', k)
else:
msgfilling.add_types(msg_types, '_type_str', k)
log.debug("msg_types: %s", msg_types)
return msg_types
def parse_wmi_log_query(self, wmi_data):
msgs = wmi_data[2:]
keys = wmi_data[1].split('|')
events = []
appended_msgs = []
prev_msg = ''
for each_msg in msgs:
start_msg = each_msg.split('|', 1)
if start_msg[0].isdigit():
if prev_msg != '':
appended_msgs.append(prev_msg)
prev_msg = each_msg
else:
prev_msg = prev_msg + each_msg
appended_msgs.append(prev_msg)
priority = self.facility * self.severity + 8
for index, each_msg in enumerate(appended_msgs):
event = {"_normalized_fields": {}}
splitted_msg = []
split1 = each_msg.split('|', 9)
splitted_msg.extend(split1[0:9])
split2 = split1[9].rsplit('|', 6)
splitted_msg.extend(split2)
for index, each_key in enumerate(keys):
each_key = each_key.strip()
if each_key in ('Message', 'TimeGenerated','RecordNumber'):
if each_key == 'TimeGenerated':
wmi_time = splitted_msg[index].strip()
if not self.last_time or (wmi_time[-3:]!=self.last_time[-3:] or self.last_time < wmi_time):
self.last_time = wmi_time
event["_normalized_fields"]["log_ts"] = parse_wmi_time(wmi_time)
#event['log_ts'] = parse_wmi_time(wmi_time)
msgfilling.add_types(event, '_type_num', 'log_ts')
elif each_key == 'Message':
event['msg'] = parse_wmi_message(splitted_msg, index, priority)
msgfilling.add_types(event, '_type_str', 'msg')
elif each_key == 'RecordNumber':
event["_normalized_fields"]["record_number"] = int(splitted_msg[index].strip())
#event['record_number'] = int(splitted_msg[index].strip())
msgfilling.add_types(event, '_type_num', 'record_number')
#event['severity'] = self.severity
#event['facility'] = self.facility
event["_normalized_fields"]["severity"] = self.severity
event["_normalized_fields"]["facility"] = self.facility
msgfilling.add_types(event, '_type_num', 'severity facility')
events.append(event)
return events
def _handle_data(event, col_type, ip, device_name, loginspect_name, opsecfetcher_out, normalizer, repo):
#sends data to the normalizer
global LAST_COL_TS
global LOG_COUNTER
opsecfetcher_out.start_benchmarker_processing()
col_ts = int(time.time())
if col_ts > LAST_COL_TS:
LAST_COL_TS = col_ts
LOG_COUNTER = 0
mid_prefix = '%s|%s|%s|%d|' % (loginspect_name, col_type, ip, col_ts)
LOG_COUNTER += 1
event['mid'] = mid_prefix + "%d" % LOG_COUNTER
event['device_name'] = device_name
event['collected_at'] = loginspect_name
event['col_ts'] = col_ts
event['col_type'] = col_type
for key, value in event.items():
if key == 'mid':
continue
if key in ['orig']:
msgfilling.add_types(event, '_type_ip', key)
elif key in ['has_accounting', 'loc']:
msgfilling.add_types(event, '_type_num', key)
else:
msgfilling.add_types(event, '_type_str', key)
event['_counter'] = LOG_COUNTER
event['device_ip'] = ip
msgfilling.add_types(event, '_type_str', 'device_ip')
msgfilling.add_types(event, '_type_ip', 'device_ip')
event['normalizer'] = normalizer
event['repo'] = repo
opsecfetcher_out.send_with_norm_policy_and_repo(event)
def parse_record(record):
global PROTO_DIC
d = {}
d['interface_index'] = int(record.input_iface)
d['source_address'] = socket.inet_ntoa(struct.pack(
'!L', record.src_addr)) #record[0:4]
d['destination_address'] = socket.inet_ntoa(
struct.pack('!L', record.dst_addr)) #record[4:8]
d['packet_count'] = int(record.pkts_sent) #record[16:20]
d['bytes_count'] = int(record.bytes_sent) #record[20:24])[0])
d['start_uptime_ms'] = int(record.start_time) #record[24:28])[0])
d['end_uptime_ms'] = int(record.end_time) #record[28:32])[0])
d['source_port'] = int(record.src_port) #record[32:34])[0])
d['destination_port'] = int(record.dst_port) #record[34:36])[0])
protocol = int(record.ip_proto) #record[38])
d['types_of_service'] = int(record.tos)
d['protocol_name'] = PROTO_DIC[protocol]
msgfilling.add_types(d, '_type_str',
"protocol_name source_address destination_address")
msgfilling.add_types(d, '_type_ip', "source_address destination_address")
msgfilling.add_types(
d, '_type_num',
"interface_index start_uptime_ms end_uptime_ms source_port destination_port packet_count bytes_count types_of_service"
)
#log.debug("this is parsed result %s", d)
return d
def _forward_event(self, message):
"""
"""
global LAST_COL_TS
global LOG_COUNTER
self.opsecfetcher_out.start_benchmarker_processing()
col_ts = int(time.time())
if col_ts > LAST_COL_TS:
LAST_COL_TS = col_ts
LOG_COUNTER = 0
mid_prefix = '%s|%s|%s|%d|' % (self.collected_at, self.col_type, self.device_ip, col_ts)
LOG_COUNTER += 1
event = dict(
msg = self.encode(message),
mid = mid_prefix + "%d" % LOG_COUNTER,
device_ip = self.device_ip,
device_name = self.device_name,
collected_at = self.collected_at,
col_ts = col_ts,
col_type = self.col_type,
_counter = LOG_COUNTER,
normalizer = self.normalizer,
repo = self.repo
)
msgfilling.add_types(event, "_type_num", "col_ts")
msgfilling.add_types(event, "_type_str", "msg device_ip device_name collected_at col_type")
msgfilling.add_types(event, "_type_ip", "device_ip")
self.opsecfetcher_out.send_with_norm_policy_and_repo(event)
def _handle_template(self, sid, netflow_out, device_name, col_type,
device_ip, collected_at, msg_dict, option_template):
for (k, v) in self.template_data_dict.iteritems():
_raw_msg_b = binascii.b2a_base64(self.template_data_dict[k])
if option_template:
packet_t = "option_template"
else:
packet_t = "template"
event = dict(
_p__raw_msg_b=_raw_msg_b,
packet_type=packet_t,
version=self.version,
template_id=k,
)
event.update(msg_dict)
msgfilling.add_types(event, '_type_str', 'packet_type')
if self.version == 9:
msgfilling.add_types(
event, '_type_num',
'template_id version sys_uptime_ms package_sequence source_id unix_secs flowset_count'
)
elif self.version == 10:
msgfilling.add_types(
event, '_type_num',
'template_id version sys_uptime_ms package_sequence source_id flowset_length'
)
_handle_data(event, sid, netflow_out, device_name, col_type,
device_ip, collected_at)
def _fill_msg_types(d):
event = {}
for (k, v) in d.iteritems():
log.debug("%s : %s ", k, v)
if v and k is not "_p__raw_msg_b":
if _is_valid_ipv4(v) or _is_valid_ipv6(v):
msgfilling.add_types(event, '_type_ip', k)
msgfilling.add_types(event, '_type_str', k)
elif _is_valid_num(v):
msgfilling.add_types(event, '_type_num', k)
else:
msgfilling.add_types(event, '_type_str', k)
event[k] = v
log.debug("event: %s", event)
return event
def next(self):
event = super(LIv4Parser, self).next()
found = self.v4_logstorage_re.match(event['msg'])
if not found:
logging.warning(
'LIv4Parser; received message with incorrect format;'
' sid=%s; msg=%r', self.sid, event['msg'])
return event
severity = found.group(1)
facility = found.group(2)
#syslog_time = found.group(3)
col_year = found.group(4)
col_month = found.group(5)
col_day = found.group(6)
col_hour = found.group(7)
col_minute = found.group(8)
col_second = found.group(9)
client_time = found.group(10)
src_ip = found.group(11)
msg = found.group(12)
priority = int(facility) * 8 + int(severity)
col_ts = _get_col_ts(col_year, col_month, col_day, col_hour,
col_minute, col_second)
log_ts = timestamp.fromsyslogtime(client_time, int(col_year))
# for sorting only, col_ts will be reparsed from mid by indexsearcher
event["col_ts"] = col_ts
event["mid"] = "%s|%s|%s|%d|%%d" % (self.collected_at, self.col_type,
src_ip, col_ts)
event["msg"] = "<%s> %s %s" % (priority, client_time, msg)
event["_normalized_fields"] = dict(severity=severity,
facility=facility,
log_ts=log_ts,
log_type="syslog")
# event["severity"] = severity
# event["facility"] = facility
# event["log_ts"] = log_ts
# event["log_type"] = "syslog"
event["col_type"] = self.col_type
event["device_ip"] = src_ip
msgfilling.add_types(event, "_type_num",
"severity facility log_ts col_ts")
msgfilling.add_types(event, "_type_str", "log_type col_type device_ip")
msgfilling.add_types(event, "_type_ip", "device_ip")
return event
def parse_record(record):
global PROTO_DIC
d = {}
d['interface_index'] = int(record.input_iface)
d['source_address'] = socket.inet_ntoa(struct.pack(
'!L', record.src_addr)) #record[0:4]
d['destination_address'] = socket.inet_ntoa(
struct.pack('!L', record.dst_addr)) #record[4:8]
d['packet_count'] = int(record.pkts_sent) #record[16:20]
d['bytes_count'] = int(record.bytes_sent) #record[20:24])[0])
d['start_uptime_ms'] = int(record.start_time) #record[24:28])[0])
d['end_uptime_ms'] = int(record.end_time) #record[28:32])[0])
d['source_port'] = int(record.src_port) #record[32:34])[0])
d['destination_port'] = int(record.dst_port) #record[34:36])[0])
protocol = int(record.ip_proto) #record[38])
d['types_of_service'] = int(record.tos)
d['protocol_name'] = PROTO_DIC[protocol]
msgfilling.add_types(d, '_type_str',
"protocol_name source_address destination_address")
msgfilling.add_types(d, '_type_ip', "source_address destination_address")
msgfilling.add_types(
d, '_type_num',
"interface_index start_uptime_ms end_uptime_ms source_port destination_port packet_count bytes_count types_of_service"
)
ts = int(time.time())
timestamp = datetime.datetime.fromtimestamp(ts).strftime(
'%Y-%m-%d %H:%M:%S')
#d['timestamp'] = timestamp
# format: timestamp (interface_index) srcIP(srcPort)-(PROTO)->dstIP(dstPort) from start_uptime to end_uptime (tos) nPacket nBytes
result = "%s (%d) %s(%d) -(%s)-> %s(%d) from %s to %s, (%s) pcount:%s, bcount:%s" % (
timestamp, d['interface_index'], d['source_address'], d['source_port'], PROTO_DIC[protocol], \
d['destination_address'], d['destination_port'], \
d['start_uptime_ms'], d['end_uptime_ms'], d['types_of_service'], d['packet_count'], d['bytes_count'])
log.debug("Record %s", result)
log.debug("this is parsed result %s", d)
return d
def next(self):
event = super(LIv4SNMPParser, self).next()
found = self.v4_logstorage_re.match(event['msg'])
if not found:
logging.warning('LIv4SNMPParser; received message with incorrect format;'
' sid=%s; msg=%r', self.sid, event['msg'])
return event
#server_time = found.group(1)
col_year = found.group(2)
col_month = found.group(3)
col_day = found.group(4)
col_hour = found.group(5)
col_minute = found.group(6)
col_second = found.group(7)
src_ip = found.group(8)
col_ts = _get_col_ts(col_year, col_month, col_day,
col_hour, col_minute, col_second)
# for sorting only, col_ts will be reparsed from mid by indexsearcher
event["col_ts"] = col_ts
event["mid"] = "%s|%s|%s|%d|%%d" % (self.collected_at, self.col_type, src_ip, col_ts)
event["_normalized_fields"] = dict(log_type="snmp")
#event["log_type"] = "snmp"
event["col_type"] = self.col_type
event["device_ip"] = src_ip
msgfilling.add_types(event, "_type_num", "col_ts")
msgfilling.add_types(event, "_type_str", "log_type col_type device_ip")
msgfilling.add_types(event, "_type_ip", "device_ip")
return event
def _handle_message_request(sock, addr, config, fi_out):
global LAST_COL_TS
global LOG_COUNTER
log.debug("tcp collector; %s connected;" % str(addr))
try:
client_ip = inet.get_ip(addr)
config_ip = config_reader.get_config_ip(client_ip, config)
sid, parser = _get_sid_parser(client_ip, config, config_ip)
if not parser:
return
device_name = config['client_map'][config_ip]["device_name"]
normalizer = config['client_map'][config_ip]["normalizer"]
repo = config['client_map'][config_ip]["repo"]
while True:
data = sock.recv(4096)
if not data:
break
try:
message = cPickle.loads(data)
except:
#in case if complete data is not received
try:
data += sock.recv(4096)
message = cPickle.loads(data)
except:
log.warn("Dropping the log; log is more than 4 KB")
sock.send(cPickle.dumps({'received': False}))
continue
client_id = message['id']
if message.get('message') and message.get('app_name'):
app_name = message['app_name']
extra_info = message.get('extra_info') or {}
fi_out.start_benchmarker_processing()
if app_name == "windows_eventlog_reader":
event = {
"msg": textual.utf8(message["message"]),
"_type_str": "msg"
}
if extra_info.get("_is_event_xml"):
extra_info.pop("_is_event_xml")
try:
more_info = _get_extra_key_values_from_xml(
message["message"])
except:
more_info = {}
log.warn(
"Couldnot parse windows xml event log sent from LogPoint Agent"
)
if more_info:
extra_info.update(more_info)
parser_data = [event]
else:
parser.write(textual.utf8(message['message']),
old_parser=True)
parser_data = []
if parser:
for event in parser:
if event:
parser_data.append(event)
for event in parser_data:
col_ts = int(time.time())
if col_ts > LAST_COL_TS:
LAST_COL_TS = col_ts
LOG_COUNTER = 0
col_type = "lpagent"
mid_prefix = '%s|%s|%s|%s|' % (config['loginspect_name'],
col_type, config_ip, col_ts)
LOG_COUNTER += 1
event['mid'] = mid_prefix + "%d" % LOG_COUNTER
event['col_ts'] = col_ts
event['_counter'] = LOG_COUNTER
event['col_type'] = col_type
msgfilling.add_types(event, '_type_num', 'col_ts')
msgfilling.add_types(event, '_type_str', 'col_type')
event['app_name'] = message['app_name']
event['fi_client_id'] = client_id
event['device_name'] = device_name
event['device_ip'] = client_ip
event['collected_at'] = config['loginspect_name']
if extra_info:
event.update(extra_info)
for key, value in extra_info.iteritems():
if type(value) is int:
msgfilling.add_types(event, '_type_num', key)
else:
msgfilling.add_types(event, '_type_str', key)
msgfilling.add_types(event, '_type_str', 'app_name')
msgfilling.add_types(event, '_type_str', 'device_name')
msgfilling.add_types(event, '_type_str', 'fi_client_id')
msgfilling.add_types(event, '_type_ip', 'device_ip')
msgfilling.add_types(event, '_type_str', 'device_ip')
msgfilling.add_types(event, '_type_str', 'collected_at')
log.debug('sending message to normalizer: %s' % event)
event['normalizer'] = normalizer
event['repo'] = repo
fi_out.send_with_norm_policy_and_repo(event)
sock.send(cPickle.dumps({'received': True}))
else:
sock.send(cPickle.dumps({'received': False}))
except Exception, e:
log.warn('fileinspect collector exception: %s' % str(e))
def prepare_msgfilling(self, event, field, _type="_type_str"):
"""
Update the event with msgfilling the given field with the given type
"""
msgfilling.add_types(event, _type, field)
def forward(self, data):
self.parser.write(data, old_parser=True)
col_ts = int(time.time())
if col_ts > self.last_timestamp:
self.last_timestamp = col_ts
self.counter = 0
if self.counter >= 100:
time.sleep(1)
for event in self.parser:
self.counter += 1
loginspect_name = self.config["loginspect_name"]
event['mid'] = '%s|%s|%s|%d' % (loginspect_name, self.sid, col_ts,
self.counter)
event['col_ts'] = col_ts
event['_counter'] = self.counter
event['col_type'] = self.col_type
msgfilling.add_types(event, '_type_num', 'col_ts')
msgfilling.add_types(event, '_type_str', 'col_type')
event["collected_at"] = loginspect_name
event["device_name"] = self.device_name
msgfilling.add_types(event, '_type_str',
'device_name collected_at')
event['device_ip'] = '127.0.0.1'
msgfilling.add_types(event, '_type_str', 'device_ip')
msgfilling.add_types(event, '_type_ip', 'device_ip')
event['normalizer'] = self.normalizer
event['repo'] = self.repo
if event.get('_normalized_fields'):
event['_normalized_fields']['source_name'] = self.path
else:
event['_normalized_fields'] = {'source_name': self.path}
msgfilling.add_types(event, '_type_str', 'source_name')
self.collector_out.send_with_norm_policy_and_repo(event)
def main():
config = _parse_args()
log_level = config['core']['log_level']
port = config['port']
expire_time = config['expire_time']
col_type = config['col_type']
collected_at = config["loginspect_name"]
zmq_context = zmq.Context()
netflow_out = wiring.Wire('collector_out',
zmq_context=zmq_context,
conf_path=config.get('wiring_conf_path') or None)
sock = start_udp_server(port)
while True:
data, addr = sock.recvfrom(9216)
if not data:
continue
log.debug('udp collector; from ip=%s, got msg=%s;', addr, data)
ip = inet.get_ip(addr)
config_ip = config_reader.get_config_ip(ip, config)
if not config_ip:
continue
try:
version = get_netflow_packet_version(data[0:2])
count = socket.ntohs(struct.unpack('H', data[2:4])[0])
current_unix_sec = (struct.unpack('I', data[8:12])[0])
log.debug("Version: %s", version)
log.debug("Count of no. of records: %s", count)
log.debug("Count of no. of seconds since 0000 UTC 1970: %s",
current_unix_sec)
netflow1 = netflow.Netflow1()
netflow5 = netflow.Netflow5()
netflow6 = netflow.Netflow6()
netflow7 = netflow.Netflow7()
global VERSION
global netflowdata
if ((version == 1) or (version == 5) or (version == 6)
or (version == 7)):
if version == 1:
log.info("version 1 unpacking...")
VERSION = 1
netflow1.unpack(data)
netflowdata = netflow1.data
elif version == 5:
log.info("version 5 unpacking...")
VERSION = 5
netflow5.unpack(data)
netflowdata = netflow5.data
elif version == 6:
log.info("version 6 unpacking...")
VERSION = 6
netflow6.unpack(data)
netflowdata = netflow6.data
elif version == 7:
log.info("version 7 unpacking...")
VERSION = 7
netflow7.unpack(data)
netflowdata = netflow7.data
i = 1
if not netflowdata:
continue
for netflow_record in netflowdata:
try:
i = i + 1
try:
parsed_msg_dict = parse_record(netflow_record)
except Exception, e:
log.error("Could not parse the given record. %s",
repr(e))
parsed_msg_dict['_p__raw_msg_b'] = binascii.b2a_base64(
str(netflow_record))
parsed_msg_dict['version'] = VERSION
parsed_msg_dict['current_unix_sec'] = current_unix_sec
msgfilling.add_types(parsed_msg_dict, '_type_num',
'version current_unix_sec')
sid = _get_sid(config_ip, config)
device_name = config['client_map'][config_ip][
"device_name"]
log.debug("device: %s", device_name)
log.debug("descrete ip: %s", ip)
try:
_handle_data(parsed_msg_dict, sid, netflow_out,
device_name, col_type, ip,
collected_at)
except Exception, e:
log.error("Device name not found. %s", repr(e))
except Exception, e:
log.error(
"Error in constructing message, Necessary field not supplied in Netflow"
)
log.error(repr(e))
def msgfill_parsed_record_v9(dd):
log.info("parsing v9 records")
global FIELDS
global PROTO_DIC
d = {}
e = {}
for (k, v) in dd.iteritems():
#log.debug("********** %s ******* %s", k, v)
if k in _filter_ipv4:
#log.debug("#pack the ipv4")
msgfilling.add_types(e, '_type_ip', FIELDS[k])
msgfilling.add_types(e, '_type_str', FIELDS[k])
d[k] = socket.inet_ntoa(struct.pack('!I', dd[k]))
elif k in _filter_ipv6:
#log.debug("#pace the ipv6")
msgfilling.add_types(e, '_type_ip', FIELDS[k])
msgfilling.add_types(e, '_type_str', FIELDS[k])
d[k] = dd[k]
else:
#log.debug("leave as it is")
d[k] = dd[k]
if (k == 4):
try:
e['protocol_name'] = PROTO_DIC[d[k]]
except:
log.info("Protocol Name not found in PROTO_DIC")
e['protocol_name'] = "Unknown"
msgfilling.add_types(e, '_type_str', "protocol_name")
try:
msgfilling.add_types(e, '_type_num', FIELDS[k])
except:
log.info(
"Fields not found in netflow_v9 field_type; Not filling msg_type"
)
#msgfilling.add_types(e, '_type_num', "Unknown_field_"+str(k))
try:
e[FIELDS[k]] = d[k]
except:
log.info(
"Fields not found in netflow_v9 field_type; Not assigining values."
)
#e["Unknown_field_"+str(k)] = d[k]
log.debug("this is v9 parsed result %s", e)
return e
data_dict = data_dict_tuple[0]
_p__raw_msg_b = data_dict_tuple[1]
if not data_dict:
parsed_msg_dict = {}
parsed_msg_dict[
'_p__raw_msg_b'] = binascii.b2a_base64(
str(_p__raw_msg_b))
parsed_msg_dict['version'] = VERSION
parsed_msg_dict[
'sys_uptime_ms'] = sys_uptime_ms
parsed_msg_dict['unix_secs'] = unix_secs
parsed_msg_dict[
'package_sequence'] = package_sequence
parsed_msg_dict['source_id'] = source_id
msgfilling.add_types(
parsed_msg_dict, '_type_num',
'version sys_uptime_ms unix_secs package_sequence source_id'
)
#sid = _get_sid(config_ip, config)
#device_name = config['client_map'][config_ip]["device_name"]
try:
_handle_data(parsed_msg_dict, sid,
netflow_out, device_name,
col_type, ip,
collected_at)
except Exception, e:
log.error("Device name not found; %s",
repr(e))
else: ###parse record
for (k, v) in data_dict.iteritems():
try:
def cbFun(snmpEngine, stateReference, contextEngineId, contextName, varBinds,
cbCtx):
snmptrapd_out = cbCtx.get("out_wire")
snmptrapd_out.start_benchmarker_processing()
event = {"_normalized_fields": {}}
MIBView = cbCtx.get("MIBView")
last_col_ts = cbCtx.get("last_col_ts")
log_counter = cbCtx.get("log_counter")
app_config = cbCtx.get("app_config")
transportDomain, transportAddress = snmpEngine.msgAndPduDsp.getTransportInfo(
stateReference)
log_msg = 'Notification from %s :\n' % str(transportAddress)
log.debug("SNMP Trap; Notification received; client_ip=%s",
str(transportAddress))
#check if from valid client address
client_list = app_config["client_list"]
loginspect_name = app_config["loginspect_name"]
if not transportAddress[0] in client_list:
log.warn("SNMP Trap; warning; being sent from unregistered IP (%s)",
transportAddress[0])
return
col_ts = int(time.time())
if col_ts > last_col_ts:
last_col_ts = col_ts
log_counter = 0
type_str = "msg "
type_num = ""
for name, val in varBinds:
oid, label, suffix = MIBView.getNodeName(name)
oidLabel = ".".join(map(str, label))
oidSuffix = ".".join(map(str, suffix))
oid_string = oid.prettyPrint()
value = val.prettyPrint()
log_msg += "%s (%s %s) = %s\n" % (oid_string, oidLabel, oidSuffix,
value)
if (name[:10] == SNMP_TRAP_OID):
oid, label, suffix = MIBView.getNodeName(val)
oidLabel = ".".join(map(str, label))
oidSuffix = ".".join(map(str, suffix))
log_msg += " The snmp trap %s (%s %s) received\n" % (
value, oidLabel, oidSuffix)
oid_string = "snmp_trap_oid"
if (name[:8] == SNMP_SYS_UPTIME):
oid_string = "sys_uptime"
if value.isdigit():
#print "Integer Value found ", val, " =", value
event["_normalized_fields"][oid_string] = int(value)
#event[oid_string] = int(value)
type_num += oid_string + " "
else:
event["_normalized_fields"][oid_string] = value
#event[oid_string] = value
type_str += oid_string + " "
#send to the upper layer
log_counter += 1
device_ip = transportAddress[0]
device_name = app_config["name_ip_mapping"][device_ip]
sid = 'snmp|%s' % device_ip
event['mid'] = '%s|%s|%d|%d' % (loginspect_name, sid, col_ts, log_counter)
event['col_type'] = 'snmp'
event['col_ts'] = col_ts
event['_counter'] = log_counter
type_num += 'col_ts '
type_str += 'col_type '
event['device_ip'] = device_ip
type_str += 'device_ip '
event['msg'] = log_msg
event['_type_str'] = type_str.strip()
event['_type_num'] = type_num.strip()
event['_type_ip'] = 'device_ip'
event['device_name'] = device_name
event['collected_at'] = loginspect_name
event['device_ip'] = device_ip
normalizer = app_config["repo_norm_mapping"][device_ip]["normalizer"]
repo = app_config["repo_norm_mapping"][device_ip]["repo"]
msgfilling.add_types(event, '_type_str',
'device_ip device_name collected_at')
event['normalizer'] = normalizer
event['repo'] = repo
snmptrapd_out.send_with_norm_policy_and_repo(event)
cbCtx["last_col_ts"] = last_col_ts
cbCtx["log_counter"] = log_counter
def _handle_data(event, ip, col_type, device_name, loginspect_name,
snmpfetcher_out, normalizer, repo):
global LAST_COL_TS
global LOG_COUNTER
snmpfetcher_out.start_benchmarker_processing()
col_ts = int(time.time())
if col_ts > LAST_COL_TS:
LAST_COL_TS = col_ts
LOG_COUNTER = 0
mid_prefix = '%s|%s|%s|%d|' % (loginspect_name, col_type, ip, col_ts)
LOG_COUNTER += 1
event['mid'] = mid_prefix + "%d" % LOG_COUNTER
event['col_ts'] = col_ts
event['_counter'] = LOG_COUNTER
event['col_type'] = col_type
msgfilling.add_types(event, '_type_num', 'col_ts')
msgfilling.add_types(event, '_type_str', 'col_type')
event['device_ip'] = ip
msgfilling.add_types(event, '_type_str', 'device_ip')
msgfilling.add_types(event, '_type_ip', 'device_ip')
event['device_name'] = device_name
event['collected_at'] = loginspect_name
msgfilling.add_types(event, '_type_str', 'device_name')
msgfilling.add_types(event, '_type_str', 'collected_at')
event['normalizer'] = normalizer
event['repo'] = repo
snmpfetcher_out.send_with_norm_policy_and_repo(event)
def prepare_msgfilling(self, event, _type, msg_str):
"""
msgfilling multiple fields at once
"""
msgfilling.add_types(event, _type, msg_str)
def get_raw_template_dict_raw_data_dict(self, sid, netflow_out,
device_name, col_type, device_ip,
collected_at):
log.debug(
".. inside separate the data and template flowsets and store with template_id in the ;;tempalate_data_dict ---> raw template flowset ;;data_data_dict --------> raw data flowset ..."
)
for header_num in range(0, len(self.template_header_list)):
if self.template_header_list[header_num][0] == 0:
'''
do the task of templates here
'''
#log.debug("Inside ; this is template set")
if self.template_header_list[header_num][1] > 4:
#log.debug("Template Diagnonsis; template length: %s", self.template_header_list[header_num][1])
do = True
next_count = 0
while do:
template_id = socket.ntohs(
struct.unpack(
'H', self.flow_data_list[header_num]
[4 + next_count:6 + next_count])[0])
field_count = socket.ntohs(
struct.unpack(
'H', self.flow_data_list[header_num]
[6 + next_count:8 + next_count])[0])
for i in range(len(self.template_template_list)):
if template_id == self.template_template_list[i][
0]: #replace the template if template_id is repeated
self.template_template_list[i] = (template_id,
field_count)
self.template_template_list.append(
(template_id, field_count))
self.template_data_dict[
template_id] = self.flow_data_list[header_num][
4:] #removing flowset_id and length
#self.template_template_list.append((2344, 4567))
#self.template_data_dict[2344] = "\x00\x12\00"
if (self.template_template_list[-1][1] * 4 + 4 + 4
) != (self.template_header_list[header_num][1]):
log.debug(
"Here is true error; Invalid field count vs template data length"
)
del self.template_template_list[-1]
#del self.template_data_dict[2344]
break
'''
log.debug("inside template_id: %s", template_id)
log.debug("inside field_count: %s", field_count)
log.debug("inside template_template_list: %s", self.template_template_list)
log.debug("inside template_data_dict: %s", self.template_data_dict)
'''
next_count += (field_count * 2 * 2) + 4
if (next_count +
4) >= self.template_header_list[header_num][1]:
do = False
else:
log.info(
"Length Problem;;;Deleteing template header with incorrect lengths: %s",
self.template_header_list[header_num])
#del self.template_header_list[header_num]
else:
'''
do the task of data here
'''
#log.debug("Inside;; this is data set")
template_id = self.template_header_list[header_num][0]
data_length = socket.ntohs(
struct.unpack('H',
self.flow_data_list[header_num][2:4])[0])
replaced = False
self.data_header_list.append(
(template_id, data_length,
self.flow_data_list[header_num][4:]))
'''
log.debug("inside data template_id: %s", template_id)
log.debug("inside data dataflow length: %s", data_length)
log.debug("inside self.data_header_list: %s", self.data_header_list)
log.debug("inside self.data_data_dict: %s", self.data_data_dict)
'''
#handle template data
self.update_template_data_list_dict()
for (k, v) in self.template_data_dict.iteritems():
_raw_msg_b = binascii.b2a_base64(self.template_data_dict[k])
event = dict(
_p___raw_msg_b=_raw_msg_b,
packet_type="template",
template_id=k,
)
msgfilling.add_types(event, '_type_str', 'packet_type')
msgfilling.add_types(event, '_type_num', 'template_id')
_handle_data(event, sid, netflow_out, device_name, col_type,
device_ip, collected_at)
