def _handle_data(event, ip, col_type, device_name, collected_at, normalizer, repo, ipssdee_out): global LAST_COL_TS global LOG_COUNTER ipssdee_out.start_benchmarker_processing() col_ts = int(time.time()) if col_ts > LAST_COL_TS: LAST_COL_TS = col_ts LOG_COUNTER = 0 mid_prefix = '%s|%s|%s|%d|' % (collected_at, col_type, ip, col_ts) LOG_COUNTER += 1 event['mid'] = mid_prefix + "%d" % LOG_COUNTER event['col_ts'] = col_ts event['device_name'] = device_name event["collected_at"] = collected_at event["col_type"] = col_type event["device_ip"] = ip ##update msg_types here event = remove_NULL_fields(event) msg_types_dict = fill_msg_types(event) event.update(msg_types_dict) event['_counter'] = LOG_COUNTER event['device_ip'] = ip msgfilling.add_types(event, '_type_str', 'device_ip') msgfilling.add_types(event, '_type_ip', 'device_ip') event['normalizer'] = normalizer event['repo'] = repo ipssdee_out.send_with_norm_policy_and_repo(event)
def _handle_data(event, sid, netflow_out, device_name, col_type, ip, collected_at): netflow_out.start_benchmarker_processing() global LAST_COL_TS global LOG_COUNTER col_ts = int(time.time()) if col_ts > LAST_COL_TS: LAST_COL_TS = col_ts LOG_COUNTER = 0 mid_prefix = '%s|%s|%s|' % (collected_at, sid, col_ts) LOG_COUNTER += 1 event['mid'] = mid_prefix + "%d" % LOG_COUNTER event['device_name'] = device_name event['col_type'] = col_type event['msg'] = '' event['device_ip'] = ip event['collected_at'] = collected_at msgfilling.add_types(event, '_type_str', 'msg col_type device_name collected_at') msgfilling.add_types(event, '_type_ip', 'device_ip') #msgfilling.add_types(event, '_type_str', '_raw_msg_b') netflow_out.send_with_mid(event)
def _handle_data(data, sid, parser, snare_out, device_name, ip, collected_at): snare_out.start_benchmarker_processing() global LAST_COL_TS global LOG_COUNTER col_ts = int(time.time()) if col_ts > LAST_COL_TS: LAST_COL_TS = col_ts LOG_COUNTER = 0 mid_prefix = '%s|%s|%d|' % (collected_at, sid, col_ts) parser.write(data) #log.info("checking data: %s", data) for event in parser: LOG_COUNTER += 1 event['mid'] = mid_prefix + "%d" % LOG_COUNTER event['device_name'] = device_name event['device_ip'] = ip event['collected_at'] = collected_at msgfilling.add_types(event, '_type_str', 'device_name collected_at') msgfilling.add_types(event, '_type_ip', 'device_ip') snare_out.send_with_mid(event)
def transform(grouped_files, col_out, estimator, config, updater): last_ts = 0 last_path = "" recorded_col_ts = updater.get_col_ts() recorded_col_ts_human = time.strftime("%Y%m%d %H:%M:%S", time.localtime(recorded_col_ts)) start = time.time() throttler = timing.speed_throttler(MAX_SPEED) for day, filepaths in grouped_files: if time.mktime(time.strptime(str(day), "%Y%m%d")) + 86400 <= recorded_col_ts: logging.warning( "skipping logs for day %d, last tracked col_ts is %d (%s)", day, recorded_col_ts, recorded_col_ts_human) continue for col_ts, event, path in reader.event_generator_by_time( filepaths, config): throttler.next() if col_ts < recorded_col_ts: continue if col_ts < last_ts: assert path == last_path, "path=%s; last_path=%s" % (path, last_path) logging.warning( "outdated msg with col_ts=%s, last_ts=%s; path=%s", col_ts, last_ts, path) elif col_ts > last_ts: last_ts = col_ts counter = 0 # save col_ts every col_ts minute if col_ts % 60 == 0: updater.save_col_ts(col_ts) gevent.sleep() counter += 1 last_path = path event["mid"] = event["mid"] % counter event["_counter"] = counter event["collected_at"] = config["loginspect_name"] msgfilling.add_types(event, '_type_str', 'collected_at') event['normalizer'] = config['normalizer'] event['repo'] = config['repo'] col_out.send_with_norm_policy_and_repo(event) estimator.modify(filepaths, add=False) duration = time.time() - start total = reader.COUNTER logging.warning( "%d messages processed in %d sec, @ %d msg/sec, @ %d bytes/sec", total, duration, total / duration, reader.BYTES_READ / duration)
def prepare_event(self, event, field, value, _type="_type_str", _normalized=True): """ Update event with field/value and msgfilling done """ if _normalized: if event.get("_normalized_fields"): event["_normalized_fields"][field] = value else: event["_normalized_fields"] = dict(field=value) else: event[field] = value msgfilling.add_types(event, _type, field)
def next(self): event = super(SyslogParser, self).next() found = self.regex.match(event['msg']) if not found: # this never happens because everything is optional in regex log.warn( 'syslog parser; received message with incorrect syslog format;' ' sid=%s; msg=%r', self.sid, event['msg']) else: pri = found.group('pri') log_time = found.group('log_time') year = found.group('year') if not pri and not log_time: log.info( 'syslog parser; received message with incorrect syslog format;' ' sid=%s; msg=%r', self.sid, event['msg']) else: event["_normalized_fields"] = {} if log_time: if year: log_ts = timestamp.fromsyslogtime_with_year(log_time) else: try: log_ts = timestamp.fromsyslogtime(log_time) except ValueError: log.warn( "error while parsing log_ts; possibly because client timezone is not UTC; sid=%s; log_time=%s", self.sid, log_time) log_ts = None if log_ts: event["_normalized_fields"]["log_ts"] = log_ts #event['log_ts'] = log_ts msgfilling.add_types(event, '_type_num', 'log_ts') if pri: pri = int(pri) sev = pri % 8 fac = pri // 8 event["_normalized_fields"]["severity"] = sev event["_normalized_fields"]["facility"] = fac #event['severity'] = sev #event['facility'] = fac msgfilling.add_types(event, '_type_num', 'severity facility') return event
def _handle_unparsed_raw_data(unparsed_data_info, sid, netflow_out, device_name, col_type, ip, collected_at): for template_id, data_list in unparsed_data_info.iteritems(): unparsed_msg = {} unparsed_msg["template_id"] = template_id unparsed_msg["msg_status"] = "unparsed" unparsed_msg["version"] = 9 msgfilling.add_types(unparsed_msg, '_type_num', 'template_id version') msgfilling.add_types(unparsed_msg, '_type_str', 'msg_status') for length, datum in data_list: unparsed_msg["_p__raw_msg_b"] = binascii.b2a_base64(str(datum)) _handle_data(unparsed_msg, sid, netflow_out, device_name, col_type, ip, collected_at)
def _handle_template(self, sid, netflow_out, device_name, col_type, device_ip, collected_at): for (k, v) in self.template_data_dict.iteritems(): _raw_msg_b = binascii.b2a_base64(self.template_data_dict[k]) event = dict( _p___raw_msg_b=_raw_msg_b, packet_type="template", version=self.version, template_id=k, ) msgfilling.add_types(event, '_type_str', 'packet_type') msgfilling.add_types(event, '_type_num', 'template_id version') _handle_data(event, sid, netflow_out, device_name, col_type, device_ip, collected_at)
def next(self): event = super(SnareParser, self).next() found = self.regex.match(event['msg']) if not found: # this never happens because everything is optional in regex log.warn('snare parser; received message with incorrect snare format;' ' sid=%s; msg=%r', self.sid, event['msg']) else: pri = found.group('pri') log_time = found.group('log_time') year = found.group('year') flag_syslog = True if not log_time: log.debug("Set current time as log_time") log_time = time.strftime("%b %d %Y %H:%M:%S ", time.gmtime()) #year = time.gmtime().tm_year log_ts = int(time.mktime(time.gmtime())) flag_syslog = False if not pri and not log_time: log.info('snare parser; received message with incorrect snare format;' ' sid=%s; msg=%r', self.sid, event['msg']) else: if log_time and flag_syslog: if year: log_ts = timestamp.fromsyslogtime_with_year(log_time) else: try: log_ts = timestamp.fromsyslogtime(log_time) except ValueError, e: log.warn("error while parsing log_ts; possibly because client timezone is not UTC; sid=%s; log_time=%s", self.sid, log_time) log_ts = None if log_ts: event['log_ts'] = log_ts msgfilling.add_types(event, '_type_num', 'log_ts') if pri: pri = int(pri) sev = pri % 8 fac = pri // 8 event['severity'] = sev event['facility'] = fac msgfilling.add_types(event, '_type_num', 'severity facility')
def _add_types(event, oid, oid_val): msgfilling.add_types(event, '_type_str', 'msg') oid_type = oid, str(oid_val.__class__).split('.')[-1:][0].upper() if oid_type in [ 'INTEGER', 'INTEGER32', 'TIMETICKS', 'COUNTER32', 'GAUGE32', "COUNTER64" ]: msgfilling.add_types(event, '_type_num', oid) elif oid_type in ['OCTETSTRING', 'OID', 'HEXSTRING', 'PHYSADDRESS']: msgfilling.add_types(event, '_type_str', oid) elif oid_type in ['IPADDRESS']: msgfilling.add_types(event, '_type_ip', oid) else: msgfilling.add_types(event, '_type_str', oid)
def fill_msg_types(d): msg_types = {} for (k, v) in d.iteritems(): #print k, ": ", v if v and type(v) is not type(list()): if is_valid_ipv4(v) or is_valid_ipv6(v): msgfilling.add_types(msg_types, '_type_ip', k) msgfilling.add_types(msg_types, '_type_str', k) elif is_valid_num(v): msgfilling.add_types(msg_types, '_type_num', k) elif isinstance(v, list): pass else: msgfilling.add_types(msg_types, '_type_str', k) else: msgfilling.add_types(msg_types, '_type_str', k) log.debug("msg_types: %s", msg_types) return msg_types
def parse_wmi_log_query(self, wmi_data): msgs = wmi_data[2:] keys = wmi_data[1].split('|') events = [] appended_msgs = [] prev_msg = '' for each_msg in msgs: start_msg = each_msg.split('|', 1) if start_msg[0].isdigit(): if prev_msg != '': appended_msgs.append(prev_msg) prev_msg = each_msg else: prev_msg = prev_msg + each_msg appended_msgs.append(prev_msg) priority = self.facility * self.severity + 8 for index, each_msg in enumerate(appended_msgs): event = {"_normalized_fields": {}} splitted_msg = [] split1 = each_msg.split('|', 9) splitted_msg.extend(split1[0:9]) split2 = split1[9].rsplit('|', 6) splitted_msg.extend(split2) for index, each_key in enumerate(keys): each_key = each_key.strip() if each_key in ('Message', 'TimeGenerated','RecordNumber'): if each_key == 'TimeGenerated': wmi_time = splitted_msg[index].strip() if not self.last_time or (wmi_time[-3:]!=self.last_time[-3:] or self.last_time < wmi_time): self.last_time = wmi_time event["_normalized_fields"]["log_ts"] = parse_wmi_time(wmi_time) #event['log_ts'] = parse_wmi_time(wmi_time) msgfilling.add_types(event, '_type_num', 'log_ts') elif each_key == 'Message': event['msg'] = parse_wmi_message(splitted_msg, index, priority) msgfilling.add_types(event, '_type_str', 'msg') elif each_key == 'RecordNumber': event["_normalized_fields"]["record_number"] = int(splitted_msg[index].strip()) #event['record_number'] = int(splitted_msg[index].strip()) msgfilling.add_types(event, '_type_num', 'record_number') #event['severity'] = self.severity #event['facility'] = self.facility event["_normalized_fields"]["severity"] = self.severity event["_normalized_fields"]["facility"] = self.facility msgfilling.add_types(event, '_type_num', 'severity facility') events.append(event) return events
def _handle_data(event, col_type, ip, device_name, loginspect_name, opsecfetcher_out, normalizer, repo): #sends data to the normalizer global LAST_COL_TS global LOG_COUNTER opsecfetcher_out.start_benchmarker_processing() col_ts = int(time.time()) if col_ts > LAST_COL_TS: LAST_COL_TS = col_ts LOG_COUNTER = 0 mid_prefix = '%s|%s|%s|%d|' % (loginspect_name, col_type, ip, col_ts) LOG_COUNTER += 1 event['mid'] = mid_prefix + "%d" % LOG_COUNTER event['device_name'] = device_name event['collected_at'] = loginspect_name event['col_ts'] = col_ts event['col_type'] = col_type for key, value in event.items(): if key == 'mid': continue if key in ['orig']: msgfilling.add_types(event, '_type_ip', key) elif key in ['has_accounting', 'loc']: msgfilling.add_types(event, '_type_num', key) else: msgfilling.add_types(event, '_type_str', key) event['_counter'] = LOG_COUNTER event['device_ip'] = ip msgfilling.add_types(event, '_type_str', 'device_ip') msgfilling.add_types(event, '_type_ip', 'device_ip') event['normalizer'] = normalizer event['repo'] = repo opsecfetcher_out.send_with_norm_policy_and_repo(event)
def parse_record(record): global PROTO_DIC d = {} d['interface_index'] = int(record.input_iface) d['source_address'] = socket.inet_ntoa(struct.pack( '!L', record.src_addr)) #record[0:4] d['destination_address'] = socket.inet_ntoa( struct.pack('!L', record.dst_addr)) #record[4:8] d['packet_count'] = int(record.pkts_sent) #record[16:20] d['bytes_count'] = int(record.bytes_sent) #record[20:24])[0]) d['start_uptime_ms'] = int(record.start_time) #record[24:28])[0]) d['end_uptime_ms'] = int(record.end_time) #record[28:32])[0]) d['source_port'] = int(record.src_port) #record[32:34])[0]) d['destination_port'] = int(record.dst_port) #record[34:36])[0]) protocol = int(record.ip_proto) #record[38]) d['types_of_service'] = int(record.tos) d['protocol_name'] = PROTO_DIC[protocol] msgfilling.add_types(d, '_type_str', "protocol_name source_address destination_address") msgfilling.add_types(d, '_type_ip', "source_address destination_address") msgfilling.add_types( d, '_type_num', "interface_index start_uptime_ms end_uptime_ms source_port destination_port packet_count bytes_count types_of_service" ) #log.debug("this is parsed result %s", d) return d
def _forward_event(self, message): """ """ global LAST_COL_TS global LOG_COUNTER self.opsecfetcher_out.start_benchmarker_processing() col_ts = int(time.time()) if col_ts > LAST_COL_TS: LAST_COL_TS = col_ts LOG_COUNTER = 0 mid_prefix = '%s|%s|%s|%d|' % (self.collected_at, self.col_type, self.device_ip, col_ts) LOG_COUNTER += 1 event = dict( msg = self.encode(message), mid = mid_prefix + "%d" % LOG_COUNTER, device_ip = self.device_ip, device_name = self.device_name, collected_at = self.collected_at, col_ts = col_ts, col_type = self.col_type, _counter = LOG_COUNTER, normalizer = self.normalizer, repo = self.repo ) msgfilling.add_types(event, "_type_num", "col_ts") msgfilling.add_types(event, "_type_str", "msg device_ip device_name collected_at col_type") msgfilling.add_types(event, "_type_ip", "device_ip") self.opsecfetcher_out.send_with_norm_policy_and_repo(event)
def _handle_template(self, sid, netflow_out, device_name, col_type, device_ip, collected_at, msg_dict, option_template): for (k, v) in self.template_data_dict.iteritems(): _raw_msg_b = binascii.b2a_base64(self.template_data_dict[k]) if option_template: packet_t = "option_template" else: packet_t = "template" event = dict( _p__raw_msg_b=_raw_msg_b, packet_type=packet_t, version=self.version, template_id=k, ) event.update(msg_dict) msgfilling.add_types(event, '_type_str', 'packet_type') if self.version == 9: msgfilling.add_types( event, '_type_num', 'template_id version sys_uptime_ms package_sequence source_id unix_secs flowset_count' ) elif self.version == 10: msgfilling.add_types( event, '_type_num', 'template_id version sys_uptime_ms package_sequence source_id flowset_length' ) _handle_data(event, sid, netflow_out, device_name, col_type, device_ip, collected_at)
def _fill_msg_types(d): event = {} for (k, v) in d.iteritems(): log.debug("%s : %s ", k, v) if v and k is not "_p__raw_msg_b": if _is_valid_ipv4(v) or _is_valid_ipv6(v): msgfilling.add_types(event, '_type_ip', k) msgfilling.add_types(event, '_type_str', k) elif _is_valid_num(v): msgfilling.add_types(event, '_type_num', k) else: msgfilling.add_types(event, '_type_str', k) event[k] = v log.debug("event: %s", event) return event
def next(self): event = super(LIv4Parser, self).next() found = self.v4_logstorage_re.match(event['msg']) if not found: logging.warning( 'LIv4Parser; received message with incorrect format;' ' sid=%s; msg=%r', self.sid, event['msg']) return event severity = found.group(1) facility = found.group(2) #syslog_time = found.group(3) col_year = found.group(4) col_month = found.group(5) col_day = found.group(6) col_hour = found.group(7) col_minute = found.group(8) col_second = found.group(9) client_time = found.group(10) src_ip = found.group(11) msg = found.group(12) priority = int(facility) * 8 + int(severity) col_ts = _get_col_ts(col_year, col_month, col_day, col_hour, col_minute, col_second) log_ts = timestamp.fromsyslogtime(client_time, int(col_year)) # for sorting only, col_ts will be reparsed from mid by indexsearcher event["col_ts"] = col_ts event["mid"] = "%s|%s|%s|%d|%%d" % (self.collected_at, self.col_type, src_ip, col_ts) event["msg"] = "<%s> %s %s" % (priority, client_time, msg) event["_normalized_fields"] = dict(severity=severity, facility=facility, log_ts=log_ts, log_type="syslog") # event["severity"] = severity # event["facility"] = facility # event["log_ts"] = log_ts # event["log_type"] = "syslog" event["col_type"] = self.col_type event["device_ip"] = src_ip msgfilling.add_types(event, "_type_num", "severity facility log_ts col_ts") msgfilling.add_types(event, "_type_str", "log_type col_type device_ip") msgfilling.add_types(event, "_type_ip", "device_ip") return event
def parse_record(record): global PROTO_DIC d = {} d['interface_index'] = int(record.input_iface) d['source_address'] = socket.inet_ntoa(struct.pack( '!L', record.src_addr)) #record[0:4] d['destination_address'] = socket.inet_ntoa( struct.pack('!L', record.dst_addr)) #record[4:8] d['packet_count'] = int(record.pkts_sent) #record[16:20] d['bytes_count'] = int(record.bytes_sent) #record[20:24])[0]) d['start_uptime_ms'] = int(record.start_time) #record[24:28])[0]) d['end_uptime_ms'] = int(record.end_time) #record[28:32])[0]) d['source_port'] = int(record.src_port) #record[32:34])[0]) d['destination_port'] = int(record.dst_port) #record[34:36])[0]) protocol = int(record.ip_proto) #record[38]) d['types_of_service'] = int(record.tos) d['protocol_name'] = PROTO_DIC[protocol] msgfilling.add_types(d, '_type_str', "protocol_name source_address destination_address") msgfilling.add_types(d, '_type_ip', "source_address destination_address") msgfilling.add_types( d, '_type_num', "interface_index start_uptime_ms end_uptime_ms source_port destination_port packet_count bytes_count types_of_service" ) ts = int(time.time()) timestamp = datetime.datetime.fromtimestamp(ts).strftime( '%Y-%m-%d %H:%M:%S') #d['timestamp'] = timestamp # format: timestamp (interface_index) srcIP(srcPort)-(PROTO)->dstIP(dstPort) from start_uptime to end_uptime (tos) nPacket nBytes result = "%s (%d) %s(%d) -(%s)-> %s(%d) from %s to %s, (%s) pcount:%s, bcount:%s" % ( timestamp, d['interface_index'], d['source_address'], d['source_port'], PROTO_DIC[protocol], \ d['destination_address'], d['destination_port'], \ d['start_uptime_ms'], d['end_uptime_ms'], d['types_of_service'], d['packet_count'], d['bytes_count']) log.debug("Record %s", result) log.debug("this is parsed result %s", d) return d
def next(self): event = super(LIv4SNMPParser, self).next() found = self.v4_logstorage_re.match(event['msg']) if not found: logging.warning('LIv4SNMPParser; received message with incorrect format;' ' sid=%s; msg=%r', self.sid, event['msg']) return event #server_time = found.group(1) col_year = found.group(2) col_month = found.group(3) col_day = found.group(4) col_hour = found.group(5) col_minute = found.group(6) col_second = found.group(7) src_ip = found.group(8) col_ts = _get_col_ts(col_year, col_month, col_day, col_hour, col_minute, col_second) # for sorting only, col_ts will be reparsed from mid by indexsearcher event["col_ts"] = col_ts event["mid"] = "%s|%s|%s|%d|%%d" % (self.collected_at, self.col_type, src_ip, col_ts) event["_normalized_fields"] = dict(log_type="snmp") #event["log_type"] = "snmp" event["col_type"] = self.col_type event["device_ip"] = src_ip msgfilling.add_types(event, "_type_num", "col_ts") msgfilling.add_types(event, "_type_str", "log_type col_type device_ip") msgfilling.add_types(event, "_type_ip", "device_ip") return event
def _handle_message_request(sock, addr, config, fi_out): global LAST_COL_TS global LOG_COUNTER log.debug("tcp collector; %s connected;" % str(addr)) try: client_ip = inet.get_ip(addr) config_ip = config_reader.get_config_ip(client_ip, config) sid, parser = _get_sid_parser(client_ip, config, config_ip) if not parser: return device_name = config['client_map'][config_ip]["device_name"] normalizer = config['client_map'][config_ip]["normalizer"] repo = config['client_map'][config_ip]["repo"] while True: data = sock.recv(4096) if not data: break try: message = cPickle.loads(data) except: #in case if complete data is not received try: data += sock.recv(4096) message = cPickle.loads(data) except: log.warn("Dropping the log; log is more than 4 KB") sock.send(cPickle.dumps({'received': False})) continue client_id = message['id'] if message.get('message') and message.get('app_name'): app_name = message['app_name'] extra_info = message.get('extra_info') or {} fi_out.start_benchmarker_processing() if app_name == "windows_eventlog_reader": event = { "msg": textual.utf8(message["message"]), "_type_str": "msg" } if extra_info.get("_is_event_xml"): extra_info.pop("_is_event_xml") try: more_info = _get_extra_key_values_from_xml( message["message"]) except: more_info = {} log.warn( "Couldnot parse windows xml event log sent from LogPoint Agent" ) if more_info: extra_info.update(more_info) parser_data = [event] else: parser.write(textual.utf8(message['message']), old_parser=True) parser_data = [] if parser: for event in parser: if event: parser_data.append(event) for event in parser_data: col_ts = int(time.time()) if col_ts > LAST_COL_TS: LAST_COL_TS = col_ts LOG_COUNTER = 0 col_type = "lpagent" mid_prefix = '%s|%s|%s|%s|' % (config['loginspect_name'], col_type, config_ip, col_ts) LOG_COUNTER += 1 event['mid'] = mid_prefix + "%d" % LOG_COUNTER event['col_ts'] = col_ts event['_counter'] = LOG_COUNTER event['col_type'] = col_type msgfilling.add_types(event, '_type_num', 'col_ts') msgfilling.add_types(event, '_type_str', 'col_type') event['app_name'] = message['app_name'] event['fi_client_id'] = client_id event['device_name'] = device_name event['device_ip'] = client_ip event['collected_at'] = config['loginspect_name'] if extra_info: event.update(extra_info) for key, value in extra_info.iteritems(): if type(value) is int: msgfilling.add_types(event, '_type_num', key) else: msgfilling.add_types(event, '_type_str', key) msgfilling.add_types(event, '_type_str', 'app_name') msgfilling.add_types(event, '_type_str', 'device_name') msgfilling.add_types(event, '_type_str', 'fi_client_id') msgfilling.add_types(event, '_type_ip', 'device_ip') msgfilling.add_types(event, '_type_str', 'device_ip') msgfilling.add_types(event, '_type_str', 'collected_at') log.debug('sending message to normalizer: %s' % event) event['normalizer'] = normalizer event['repo'] = repo fi_out.send_with_norm_policy_and_repo(event) sock.send(cPickle.dumps({'received': True})) else: sock.send(cPickle.dumps({'received': False})) except Exception, e: log.warn('fileinspect collector exception: %s' % str(e))
def prepare_msgfilling(self, event, field, _type="_type_str"): """ Update the event with msgfilling the given field with the given type """ msgfilling.add_types(event, _type, field)
def forward(self, data): self.parser.write(data, old_parser=True) col_ts = int(time.time()) if col_ts > self.last_timestamp: self.last_timestamp = col_ts self.counter = 0 if self.counter >= 100: time.sleep(1) for event in self.parser: self.counter += 1 loginspect_name = self.config["loginspect_name"] event['mid'] = '%s|%s|%s|%d' % (loginspect_name, self.sid, col_ts, self.counter) event['col_ts'] = col_ts event['_counter'] = self.counter event['col_type'] = self.col_type msgfilling.add_types(event, '_type_num', 'col_ts') msgfilling.add_types(event, '_type_str', 'col_type') event["collected_at"] = loginspect_name event["device_name"] = self.device_name msgfilling.add_types(event, '_type_str', 'device_name collected_at') event['device_ip'] = '127.0.0.1' msgfilling.add_types(event, '_type_str', 'device_ip') msgfilling.add_types(event, '_type_ip', 'device_ip') event['normalizer'] = self.normalizer event['repo'] = self.repo if event.get('_normalized_fields'): event['_normalized_fields']['source_name'] = self.path else: event['_normalized_fields'] = {'source_name': self.path} msgfilling.add_types(event, '_type_str', 'source_name') self.collector_out.send_with_norm_policy_and_repo(event)
def main(): config = _parse_args() log_level = config['core']['log_level'] port = config['port'] expire_time = config['expire_time'] col_type = config['col_type'] collected_at = config["loginspect_name"] zmq_context = zmq.Context() netflow_out = wiring.Wire('collector_out', zmq_context=zmq_context, conf_path=config.get('wiring_conf_path') or None) sock = start_udp_server(port) while True: data, addr = sock.recvfrom(9216) if not data: continue log.debug('udp collector; from ip=%s, got msg=%s;', addr, data) ip = inet.get_ip(addr) config_ip = config_reader.get_config_ip(ip, config) if not config_ip: continue try: version = get_netflow_packet_version(data[0:2]) count = socket.ntohs(struct.unpack('H', data[2:4])[0]) current_unix_sec = (struct.unpack('I', data[8:12])[0]) log.debug("Version: %s", version) log.debug("Count of no. of records: %s", count) log.debug("Count of no. of seconds since 0000 UTC 1970: %s", current_unix_sec) netflow1 = netflow.Netflow1() netflow5 = netflow.Netflow5() netflow6 = netflow.Netflow6() netflow7 = netflow.Netflow7() global VERSION global netflowdata if ((version == 1) or (version == 5) or (version == 6) or (version == 7)): if version == 1: log.info("version 1 unpacking...") VERSION = 1 netflow1.unpack(data) netflowdata = netflow1.data elif version == 5: log.info("version 5 unpacking...") VERSION = 5 netflow5.unpack(data) netflowdata = netflow5.data elif version == 6: log.info("version 6 unpacking...") VERSION = 6 netflow6.unpack(data) netflowdata = netflow6.data elif version == 7: log.info("version 7 unpacking...") VERSION = 7 netflow7.unpack(data) netflowdata = netflow7.data i = 1 if not netflowdata: continue for netflow_record in netflowdata: try: i = i + 1 try: parsed_msg_dict = parse_record(netflow_record) except Exception, e: log.error("Could not parse the given record. %s", repr(e)) parsed_msg_dict['_p__raw_msg_b'] = binascii.b2a_base64( str(netflow_record)) parsed_msg_dict['version'] = VERSION parsed_msg_dict['current_unix_sec'] = current_unix_sec msgfilling.add_types(parsed_msg_dict, '_type_num', 'version current_unix_sec') sid = _get_sid(config_ip, config) device_name = config['client_map'][config_ip][ "device_name"] log.debug("device: %s", device_name) log.debug("descrete ip: %s", ip) try: _handle_data(parsed_msg_dict, sid, netflow_out, device_name, col_type, ip, collected_at) except Exception, e: log.error("Device name not found. %s", repr(e)) except Exception, e: log.error( "Error in constructing message, Necessary field not supplied in Netflow" ) log.error(repr(e))
def msgfill_parsed_record_v9(dd): log.info("parsing v9 records") global FIELDS global PROTO_DIC d = {} e = {} for (k, v) in dd.iteritems(): #log.debug("********** %s ******* %s", k, v) if k in _filter_ipv4: #log.debug("#pack the ipv4") msgfilling.add_types(e, '_type_ip', FIELDS[k]) msgfilling.add_types(e, '_type_str', FIELDS[k]) d[k] = socket.inet_ntoa(struct.pack('!I', dd[k])) elif k in _filter_ipv6: #log.debug("#pace the ipv6") msgfilling.add_types(e, '_type_ip', FIELDS[k]) msgfilling.add_types(e, '_type_str', FIELDS[k]) d[k] = dd[k] else: #log.debug("leave as it is") d[k] = dd[k] if (k == 4): try: e['protocol_name'] = PROTO_DIC[d[k]] except: log.info("Protocol Name not found in PROTO_DIC") e['protocol_name'] = "Unknown" msgfilling.add_types(e, '_type_str', "protocol_name") try: msgfilling.add_types(e, '_type_num', FIELDS[k]) except: log.info( "Fields not found in netflow_v9 field_type; Not filling msg_type" ) #msgfilling.add_types(e, '_type_num', "Unknown_field_"+str(k)) try: e[FIELDS[k]] = d[k] except: log.info( "Fields not found in netflow_v9 field_type; Not assigining values." ) #e["Unknown_field_"+str(k)] = d[k] log.debug("this is v9 parsed result %s", e) return e
data_dict = data_dict_tuple[0] _p__raw_msg_b = data_dict_tuple[1] if not data_dict: parsed_msg_dict = {} parsed_msg_dict[ '_p__raw_msg_b'] = binascii.b2a_base64( str(_p__raw_msg_b)) parsed_msg_dict['version'] = VERSION parsed_msg_dict[ 'sys_uptime_ms'] = sys_uptime_ms parsed_msg_dict['unix_secs'] = unix_secs parsed_msg_dict[ 'package_sequence'] = package_sequence parsed_msg_dict['source_id'] = source_id msgfilling.add_types( parsed_msg_dict, '_type_num', 'version sys_uptime_ms unix_secs package_sequence source_id' ) #sid = _get_sid(config_ip, config) #device_name = config['client_map'][config_ip]["device_name"] try: _handle_data(parsed_msg_dict, sid, netflow_out, device_name, col_type, ip, collected_at) except Exception, e: log.error("Device name not found; %s", repr(e)) else: ###parse record for (k, v) in data_dict.iteritems(): try:
def cbFun(snmpEngine, stateReference, contextEngineId, contextName, varBinds, cbCtx): snmptrapd_out = cbCtx.get("out_wire") snmptrapd_out.start_benchmarker_processing() event = {"_normalized_fields": {}} MIBView = cbCtx.get("MIBView") last_col_ts = cbCtx.get("last_col_ts") log_counter = cbCtx.get("log_counter") app_config = cbCtx.get("app_config") transportDomain, transportAddress = snmpEngine.msgAndPduDsp.getTransportInfo( stateReference) log_msg = 'Notification from %s :\n' % str(transportAddress) log.debug("SNMP Trap; Notification received; client_ip=%s", str(transportAddress)) #check if from valid client address client_list = app_config["client_list"] loginspect_name = app_config["loginspect_name"] if not transportAddress[0] in client_list: log.warn("SNMP Trap; warning; being sent from unregistered IP (%s)", transportAddress[0]) return col_ts = int(time.time()) if col_ts > last_col_ts: last_col_ts = col_ts log_counter = 0 type_str = "msg " type_num = "" for name, val in varBinds: oid, label, suffix = MIBView.getNodeName(name) oidLabel = ".".join(map(str, label)) oidSuffix = ".".join(map(str, suffix)) oid_string = oid.prettyPrint() value = val.prettyPrint() log_msg += "%s (%s %s) = %s\n" % (oid_string, oidLabel, oidSuffix, value) if (name[:10] == SNMP_TRAP_OID): oid, label, suffix = MIBView.getNodeName(val) oidLabel = ".".join(map(str, label)) oidSuffix = ".".join(map(str, suffix)) log_msg += " The snmp trap %s (%s %s) received\n" % ( value, oidLabel, oidSuffix) oid_string = "snmp_trap_oid" if (name[:8] == SNMP_SYS_UPTIME): oid_string = "sys_uptime" if value.isdigit(): #print "Integer Value found ", val, " =", value event["_normalized_fields"][oid_string] = int(value) #event[oid_string] = int(value) type_num += oid_string + " " else: event["_normalized_fields"][oid_string] = value #event[oid_string] = value type_str += oid_string + " " #send to the upper layer log_counter += 1 device_ip = transportAddress[0] device_name = app_config["name_ip_mapping"][device_ip] sid = 'snmp|%s' % device_ip event['mid'] = '%s|%s|%d|%d' % (loginspect_name, sid, col_ts, log_counter) event['col_type'] = 'snmp' event['col_ts'] = col_ts event['_counter'] = log_counter type_num += 'col_ts ' type_str += 'col_type ' event['device_ip'] = device_ip type_str += 'device_ip ' event['msg'] = log_msg event['_type_str'] = type_str.strip() event['_type_num'] = type_num.strip() event['_type_ip'] = 'device_ip' event['device_name'] = device_name event['collected_at'] = loginspect_name event['device_ip'] = device_ip normalizer = app_config["repo_norm_mapping"][device_ip]["normalizer"] repo = app_config["repo_norm_mapping"][device_ip]["repo"] msgfilling.add_types(event, '_type_str', 'device_ip device_name collected_at') event['normalizer'] = normalizer event['repo'] = repo snmptrapd_out.send_with_norm_policy_and_repo(event) cbCtx["last_col_ts"] = last_col_ts cbCtx["log_counter"] = log_counter
def _handle_data(event, ip, col_type, device_name, loginspect_name, snmpfetcher_out, normalizer, repo): global LAST_COL_TS global LOG_COUNTER snmpfetcher_out.start_benchmarker_processing() col_ts = int(time.time()) if col_ts > LAST_COL_TS: LAST_COL_TS = col_ts LOG_COUNTER = 0 mid_prefix = '%s|%s|%s|%d|' % (loginspect_name, col_type, ip, col_ts) LOG_COUNTER += 1 event['mid'] = mid_prefix + "%d" % LOG_COUNTER event['col_ts'] = col_ts event['_counter'] = LOG_COUNTER event['col_type'] = col_type msgfilling.add_types(event, '_type_num', 'col_ts') msgfilling.add_types(event, '_type_str', 'col_type') event['device_ip'] = ip msgfilling.add_types(event, '_type_str', 'device_ip') msgfilling.add_types(event, '_type_ip', 'device_ip') event['device_name'] = device_name event['collected_at'] = loginspect_name msgfilling.add_types(event, '_type_str', 'device_name') msgfilling.add_types(event, '_type_str', 'collected_at') event['normalizer'] = normalizer event['repo'] = repo snmpfetcher_out.send_with_norm_policy_and_repo(event)
def prepare_msgfilling(self, event, _type, msg_str): """ msgfilling multiple fields at once """ msgfilling.add_types(event, _type, msg_str)
def get_raw_template_dict_raw_data_dict(self, sid, netflow_out, device_name, col_type, device_ip, collected_at): log.debug( ".. inside separate the data and template flowsets and store with template_id in the ;;tempalate_data_dict ---> raw template flowset ;;data_data_dict --------> raw data flowset ..." ) for header_num in range(0, len(self.template_header_list)): if self.template_header_list[header_num][0] == 0: ''' do the task of templates here ''' #log.debug("Inside ; this is template set") if self.template_header_list[header_num][1] > 4: #log.debug("Template Diagnonsis; template length: %s", self.template_header_list[header_num][1]) do = True next_count = 0 while do: template_id = socket.ntohs( struct.unpack( 'H', self.flow_data_list[header_num] [4 + next_count:6 + next_count])[0]) field_count = socket.ntohs( struct.unpack( 'H', self.flow_data_list[header_num] [6 + next_count:8 + next_count])[0]) for i in range(len(self.template_template_list)): if template_id == self.template_template_list[i][ 0]: #replace the template if template_id is repeated self.template_template_list[i] = (template_id, field_count) self.template_template_list.append( (template_id, field_count)) self.template_data_dict[ template_id] = self.flow_data_list[header_num][ 4:] #removing flowset_id and length #self.template_template_list.append((2344, 4567)) #self.template_data_dict[2344] = "\x00\x12\00" if (self.template_template_list[-1][1] * 4 + 4 + 4 ) != (self.template_header_list[header_num][1]): log.debug( "Here is true error; Invalid field count vs template data length" ) del self.template_template_list[-1] #del self.template_data_dict[2344] break ''' log.debug("inside template_id: %s", template_id) log.debug("inside field_count: %s", field_count) log.debug("inside template_template_list: %s", self.template_template_list) log.debug("inside template_data_dict: %s", self.template_data_dict) ''' next_count += (field_count * 2 * 2) + 4 if (next_count + 4) >= self.template_header_list[header_num][1]: do = False else: log.info( "Length Problem;;;Deleteing template header with incorrect lengths: %s", self.template_header_list[header_num]) #del self.template_header_list[header_num] else: ''' do the task of data here ''' #log.debug("Inside;; this is data set") template_id = self.template_header_list[header_num][0] data_length = socket.ntohs( struct.unpack('H', self.flow_data_list[header_num][2:4])[0]) replaced = False self.data_header_list.append( (template_id, data_length, self.flow_data_list[header_num][4:])) ''' log.debug("inside data template_id: %s", template_id) log.debug("inside data dataflow length: %s", data_length) log.debug("inside self.data_header_list: %s", self.data_header_list) log.debug("inside self.data_data_dict: %s", self.data_data_dict) ''' #handle template data self.update_template_data_list_dict() for (k, v) in self.template_data_dict.iteritems(): _raw_msg_b = binascii.b2a_base64(self.template_data_dict[k]) event = dict( _p___raw_msg_b=_raw_msg_b, packet_type="template", template_id=k, ) msgfilling.add_types(event, '_type_str', 'packet_type') msgfilling.add_types(event, '_type_num', 'template_id') _handle_data(event, sid, netflow_out, device_name, col_type, device_ip, collected_at)