def start(config, basedir, db, is_compressed_file_present, updater): logging.warn("transforming logs") repo = config["repo"] zmq_context = zmq.Context() col_out = wiring.Wire('collector_out', zmq_context=zmq_context) grouped_files, ips, estimator = get_grouped_files(basedir) device_router.add(db, ips, config) updater.update_stat('running') gevent.spawn_link_exception(progress.responder, estimator, zmq_context, repo) gevent.sleep( 2 ) # allow config-regeneration to complete successfully called by device_router.add transform(grouped_files, col_out, estimator, config, updater) # delete uploaded compressed file and extracted dir if is_compressed_file_present: dir_path = os.path.dirname(basedir) if os.path.exists(dir_path): shutil.rmtree(dir_path) # stopping immediately will remove sid in norm_front and store_handler # so sleep 1 hour, because events may be in queue of norm_front and store_handler time.sleep(3600) updater.update_stat('finished')
def main(): app_config = _parse_args() port = app_config["port"] address4 = ('0.0.0.0', port) address6 = ('::1', port) zmq_context = zmq.Context() snmptrapd_out = wiring.Wire('collector_out', zmq_context=zmq_context, conf_path=app_config.get('wiring_conf_path') or None) snmpEngine = getSNMPEngine(address4, address6, app_config["snmpv12_agents"], app_config["SNMPv3_users"]) cbCtx = { "out_wire": snmptrapd_out, "MIBView": getMIBViewer(), "app_config": app_config, "last_col_ts": 0, "log_counter": 0 } ntfrcv.NotificationReceiver(snmpEngine, cbFun, cbCtx) snmpEngine.transportDispatcher.jobStarted(1) # this job would never finish snmpEngine.transportDispatcher.runDispatcher()
def main(): """main function""" context = zmq.Context() socket = context.socket(zmq.DEALER) identity = u'client-3' socket.identity = identity.encode('ascii') socket.connect('tcp://localhost:5570') print 'Client %s started' % (identity) poll = zmq.Poller() poll.register(socket, zmq.POLLIN) reqs = 0 count = 0 begin = time.time() while True: reqs = reqs + 1 #print 'Req #%d sent..' % (reqs) socket.send_string(u'request #%d' % (reqs)) sockets = dict(poll.poll(10)) if socket in sockets: msg = socket.recv() count = count + 1 print 'Client %s received: %s count %d time %d' % ( identity, msg, count, time.time() - begin) socket.close() context.term()
def test(): config = _parse_args() port = config['port'] expire_time = config['expire_time'] col_type = config['col_type'] collected_at = config["loginspect_name"] zmq_context = zmq.Context() netflow_out = wiring.Wire('collector_out', zmq_context=zmq_context, conf_path=config.get('wiring_conf_path') or None) q = Queue.Queue() for i in range(5000): q.put(DATA) #sock = start_udp_server(port) netflow1 = netflow.Netflow1() netflow5 = netflow.Netflow5() netflow6 = netflow.Netflow6() netflow7 = netflow.Netflow7() num_of_process = multiprocessing.cpu_count() start = time.time() benchmark_file = open("multiprocess.benchmark", "w") netflow_proc = [multiprocessing.Process(target=work, args=(i, q, config, netflow_out, col_type, collected_at, expire_time, start, \ netflow1, netflow5, netflow6, netflow7, benchmark_file)) for i in xrange(num_of_process)] for nf in netflow_proc: nf.start()
class test_syslog_collector(unittest.TestCase): zmq_context = zmq.Context() syslog_out = wiring.Wire('collector_out', zmq_context=zmq_context) def setUp(self): config_path = disk.get_sibling(__file__, "test-config.json") self.config = conf.load(config_path) self.port = self.config["port"] self.ssl_port = self.config["ssl_port"] self.collected_at = self.config["loginspect_name"] def tearDown(self): pass def test__create_listener(self): result = _create_listener(self.port) sock, sockaddr = inet.create_external_address(self.port) expected = sock eq_(type(result), type(expected)) def test__get_profile_info(self): addr = '127.0.0.1', self.port ip = addr[0] sid = 'syslog|127.0.0.1' device_name = 'localhost' result = _get_profile_info(addr, self.config) result = list(result) parser = result.pop(2) expected = [ip, sid, device_name, self.collected_at] eq_(result, expected) def test__get_sid_parser(self): profile = self.config["client_map"]['127.0.0.1'] sid, parser = _get_sid_parser(self.config, '127.0.0.1', profile) eq_(sid, 'syslog|127.0.0.1')
def main(): """main function""" context = zmq.Context() frontend = context.socket(zmq.ROUTER) frontend.bind('tcp://*:5570') backend = context.socket(zmq.DEALER) backend.bind('tcp://*:5571') poll = zmq.Poller() poll.register(frontend, zmq.POLLIN) poll.register(backend, zmq.POLLIN) while True: sockets = dict(poll.poll()) if frontend in sockets: ident, msg = frontend.recv_multipart() print 'Server received %s id %s' % (msg, ident) backend.send_multipart([ident, msg]) if backend in sockets: ident, msg = backend.recv_multipart() print 'Sending to frontend %s id %s' % (msg, ident) frontend.send_multipart([ident, msg]) frontend.close() backend.close() context.term()
def __create_context(self, wire_name): """ Creates wiring context for sending events to normalizer """ zmq_context = zmq.Context() collector_out = wiring.Wire(wire_name, zmq_context=zmq_context, conf_path=self.__config.get('wiring_conf_path', None)) return collector_out
def _create_context(config): """ Creates wiring context for sending events to normalizer """ zmq_context = zmq.Context() return wiring.Wire("collector_out", zmq_context=zmq_context, conf_path=config.get("wiring_conf_path"))
class TestFTPCollector(unittest.TestCase): zmq_context = zmq.Context() def setUp(self): self.ftp_server = subprocess.Popen( ['python', 'ftp_collector.py', 'tests/data/test-config.json']) self.batch_processor_in = wiring.Wire('batch_processor_in', use_gevent=True) # dont know why it fails when zmq_context reused #self.batch_processor_in = wiring.Wire('batch_processor_in', # zmq_context=self.zmq_context) # Allow to prepare for serving time.sleep(0.5) def tearDown(self): self.ftp_server.kill() self.batch_processor_in.close() time.sleep(0.5) def login(self): ftp = FTP() ftp.connect('0.0.0.0', 2021) ftp.login('alpha', 'alpha') return ftp def test_login(self): self.login() def test_bad_authentication(self): ftp = FTP() ftp.connect('0.0.0.0', 2021) self.assertRaises(Exception, ftp.login, 'sujan', 'asdf') def test_normal_logfile(self): ftp = self.login() logfile = 'tests/data/1308216000' f = open(logfile) ftp.storbinary('STOR test_logfile_%s' % time.time(), f) event = gevent.with_timeout(5, self.batch_processor_in.recv, timeout_value=None) eq_(event['parser'], 'SyslogParser') eq_(event['sid'], 'ftpc|127.0.0.1-ubuntu') def test_gz_logfile(self): ftp = self.login() logfile = 'tests/data/1308216000.gz' f = open(logfile) ftp.storbinary('STOR test_logfile_%s.gz' % time.time(), f) event = gevent.with_timeout(5, self.batch_processor_in.recv, timeout_value=None) eq_(event['parser'], 'SyslogParser') eq_(event['sid'], 'ftpc|127.0.0.1-ubuntu')
def main(): config = _parse_args() zmq_context = zmq.Context() db_file = homing.home_join('storage/col/filesystem_collector', 'checksums.shelve') disk.prepare_path(db_file) cursor_shelve = shelve.open(db_file, protocol=2) watcher.monitor(config, cursor_shelve, zmq_context)
def main(): zmq_context = zmq.Context() config = _parse_args() opsecfetcher_out = wiring.Wire("collector_out", zmq_context=zmq_context, conf_path=config.get('wiring_conf_path') or None) log.info('opsec fetcher starting..') fetcherloop.start(config, opsecfetcher_out)
def main(): zmq_context = zmq.Context() config = _parse_args() opsecfetcher_out = wiring.Wire("collector_out", zmq_context=zmq_context, conf_path=config.get("wiring_conf_path") or None) running_opsecf_jobs = {} while not STOPIT: if config["_onreload"](timeout = 1): update_jobs(config, running_opsecf_jobs, opsecfetcher_out)
def main(): config = _parse_args() port = config["port"] zmq_context = zmq.Context() snare_out = wiring.Wire('collector_out', zmq_context=zmq_context, conf_path=config.get('wiring_conf_path') or None) start_tcp_server(port, config, snare_out) start_udp_server(port, config, snare_out)
def main(): zmq_context = zmq.Context() config = _parse_args() #config = textual.utf8(config) #_prepare_application_directory(config) fi_out = wiring.Wire('collector_out', zmq_context=zmq_context, conf_path=config.get('wiring_conf_path') or None) log.info('LogPoint_agent_collector starting...') fi_collector.main(config, fi_out)
def main(): """main function""" context = zmq.Context() worker = context.socket(zmq.ROUTER) worker.bind('tcp://*:5570') print 'Worker started' receiver = context.socket(zmq.PULL) receiver.bind('tcp://*:5503') poll = zmq.Poller() poll.register(worker, zmq.POLLIN) poll.register(receiver, zmq.POLLIN) identifier = "" queue = Queue(10000) count = 0 received_count = 0 begin = time.time() while True: sockets = dict(poll.poll(10)) if receiver in sockets: msg = receiver.recv() queue.put(msg) received_count = received_count + 1 print 'message received %d time %d' % (received_count, time.time() - begin) if worker in sockets: ident, req_msg = worker.recv_multipart() #print 'Worker received %s from %s' % (req_msg, ident) # replies = randint(0,4) # for i in range(replies): # time.sleep(1. / (randint(1,10))) if not queue.empty(): msg = queue.get() queue.task_done() worker.send_multipart([ident, msg]) count = count + 1 worker.close()
def main(): """ Main program loop where the storage listens for the incomming connection and from the collector layer, finds the appropriate Repo for the given log and forwards them to the RepoHandler. """ zmq_context = zmq.Context() client_address = "PUSH:connect:tcp://127.0.0.1:5503" client_1 = wiring.create_wire(zmq_context, dict(format="json", socket=client_address)) count = 1 msg = "{'msg':'Hello World - 2'}" while True: client_1.send_raw(json.dumps(msg)) if count == 100000: break count = count + 1 #print json.loads(raw_data) print count
class test_sflow_collector(unittest.TestCase): zmq_context = zmq.Context() def setUp(self): # starting sflow collector config_path = disk.get_sibling(__file__, "test-config.json") config = conf.load(config_path) self.port = config["port"] self.sflow_out = wiring.Wire('norm_front_in', zmq_context=self.zmq_context) self.sflow_collector = Popen( ['python', 'sflow_collector.py', config_path]) time.sleep(0.5) def tearDown(self): self.sflow_collector.kill() self.sflow_out.close() time.sleep(0.5) def send_message(self, address=None, message=None): address = address or ('127.0.0.1', self.port) host, port = address data_file_path = disk.get_sibling(__file__, "sflow-data-v5.txt") message = message or open(data_file_path, "rb").read() client, sockaddr = inet.create_address(host, port, socket.SOCK_DGRAM) client.sendto(message, sockaddr) event = gevent.with_timeout(5, self.sflow_out.recv, timeout_value=None) mid = event.pop('mid') assert re.match(r'^LogInspect500\|sflow\|(127.0.0.1|::1)\|\d+\|1$', mid) device_name = event.pop('device_name') eq_(device_name, 'localhost') expected = dict( col_type='sflow', switch_uptime=240000L, _type_num= 'switch_uptime samples_count sample_source_id_index sub_agent_id sample_sequence_number sample_source_id_type version datagram_sequence_number version', samples_count=1L, sub_agent_id=100000L, device_ip=address[0], sample_type='COUNTER_SAMPLE', _p__raw_msg_b= 'AAAABQAAAAHAqAIoAAGGoAAAAAwAA6mAAAAAAQAAAAIAAAGMAAAADAIAAAEAAAAGAAAH0QAAACQAAAACAAAAAQAAAAEAAAAAAAAAAAAAAAIAAAABCAAnPETIAAAAAAfVAAAANAAAAAHdFswAAAAAAY+B4AAAAAZmAAAiyAAAAAAKiYQAABA0sAAACKYAAAAAAOuQAAAAE9wAAAfUAAAASAAAAAAt99AAAAAAACl0AAAAAAAAAAAAAAAAAAAA/mAAAAAAAAHBIAAAAAAAGD/gAAAAAAAYP+AAAAEMRAAAE6YAAAAAAAAAAAAAB9MAAABEAAAAADwj1woAAAAAAAAAAAAAAEwAAAABAAAJZgAAAU0AAAWgAAAAAAAAEoQABN6aAAAeAAAAADwAAAAeAAAyogAAaOoAAAfWAAAAKAAAAAAAAk54AAAECgAAAAAAAAAAAAAAAAAAlKAAAACMAAAAAAAAAAAAAAfQAAAARAAAAAlyaXR1YnVudHUAAAAfQisf1SlOi4jO6A5NVlugAAAAAgAAAAIAAAAVMi42LjMyLTI4LWdlbmVyaWMtcGFlAAAA\n', sample_sequence_number=12L, sample_source_id_index=1L, sample_source_id_type=2L, version=5L, address_type='IP_V4', datagram_sequence_number=12L, collected_at='LogInspect500', msg='', ip_address='192.168.2.40', _type_ip='ip_address device_ip', _type_str= 'sample_type address_type ip_address msg col_type device_name collected_at', ) eq_(event, expected) def test_udp_flow(self): self.send_message(('127.0.0.1', self.port)) def test_udp6_flow(self): self.send_message(('::1', self.port))
def main(): log.info('Batch Processor for Collector apps starting...') config = _parse_args() zmq_context = zmq.Context() processor_in = wiring.Wire('batch_processor_in', zmq_context=zmq_context, conf_path=config.get('wiring_conf_path') or None) collector_out = wiring.Wire('collector_out', zmq_context=zmq_context, conf_path=config.get('wiring_conf_path') or None) last_timestamp = 0 throttler = timing.speed_throttler(MAX_SPEED) while True: file_info = processor_in.recv() sid = file_info['sid'] col_type = file_info['col_type'] col_ts = file_info['col_ts'] parser = file_info['parser'] file = file_info['file'] cursor = file_info['cursor'] charset = file_info['charset'] device_name = file_info['device_name'] device_ip = file_info['device_ip'] regex_pattern = file_info['regex_pattern'] regexparser_name = file_info['regexparser_name'] normalizer = file_info['normalizer'] repo = file_info['repo'] source_name = file_info['source_name'] current_timestamp = int(time.time()) if current_timestamp > last_timestamp: last_timestamp = current_timestamp counter = 0 for event in file_processor(file, parser, sid, charset, cursor, regex_pattern, regexparser_name): throttler.next() counter += 1 loginspect_name = config["loginspect_name"] event['mid'] = '%s|%s|%s|%d' % (loginspect_name, sid, col_ts, counter) event['collected_at'] = loginspect_name event['device_name'] = device_name msgfilling.add_types(event, '_type_str', 'device_name collected_at col_type') event['col_ts'] = col_ts event['_counter'] = counter event['col_type'] = col_type msgfilling.add_types(event, '_type_num', 'col_ts') if device_ip is not None: event['device_ip'] = device_ip msgfilling.add_types(event, '_type_str', 'device_ip') msgfilling.add_types(event, '_type_ip', 'device_ip') event['normalizer'] = normalizer event['repo'] = repo if source_name: if event.get('_normalized_fields'): event['_normalized_fields']['source_name'] = source_name else: event['_normalized_fields'] = {'source_name': source_name} msgfilling.add_types(event, '_type_str', 'source_name') collector_out.send_with_norm_policy_and_repo(event) try: os.unlink(file) except: pass
class test_netflow_collector(unittest.TestCase): zmq_context = zmq.Context() def setUp(self): # netflow collector forwards the received msg to normalizer_in # starting netflow collector config_path = disk.get_sibling(__file__, "test-config-netflow.json") config = conf.load(config_path) self.port = config['port'] self.normalizer = wiring.Wire('norm_front_in', zmq_context=self.zmq_context) self.netflow_collector = Popen( ['python', 'netflow_collector.py', config_path]) # Allow to prepare for serving time.sleep(0.5) def tearDown(self): self.netflow_collector.kill() self.normalizer.close() time.sleep(0.5) def send_message(self, address=None, message=None, version=5): address = address or ('127.0.0.1', self.port) host, port = address client, sockaddr = inet.create_address(host, port, socket.SOCK_DGRAM) if version == 5: file_path = disk.get_sibling(__file__, "v5-data.txt") msg = open(file_path, "rb").read() message = message or msg client.sendto(message, sockaddr) event = gevent.with_timeout(5, self.normalizer.recv, timeout_value=None) mid = event.pop('mid') assert re.match( r'^LogInspect500\|netflow\|(127.0.0.1|::1)\|\d+\|1$', mid) #'mid': u'LogInspect500|netflow|192.168.2.0/24|1353399814|1', eq_( event, dict( #msg=message.rstrip('\n'), destination_address='10.0.0.3', protocol_name='UDP', _p__raw_msg_b= 'CgAAAgoAAAMAAAAAAAMABQAAAAEAAABAAbw5vQG9JB0QkgBQAAARAQACAAMgHwAA\n', version=5, msg='', source_address='10.0.0.2', current_unix_sec=1026403152, bytes_count=64, end_uptime_ms=29172765, types_of_service=1, destination_port=80, interface_index=3, start_uptime_ms=29112765, device_name='localhost', packet_count=1, col_type='netflow', source_port=4242, device_ip=address[0], collected_at='LogInspect500', _type_num= 'interface_index start_uptime_ms end_uptime_ms source_port destination_port packet_count bytes_count types_of_service version current_unix_sec', _type_str= 'protocol_name source_address destination_address msg col_type device_name collected_at', _type_ip='source_address destination_address device_ip', )) elif version == 9: file_path = disk.get_sibling(__file__, "v9-data.txt") msg = open(file_path, "rb").read() message = message or msg client.sendto(message, sockaddr) event = gevent.with_timeout(5, self.normalizer.recv, timeout_value=None) mid = event.pop('mid') assert re.match( r'^LogInspect500\|netflow\|(127.0.0.1|::1)\|\d+\|1$', mid) #'mid': u'LogInspect500|netflow|192.168.2.0/24|1353399814|1', expected = { '_type_str': 'packet_type msg col_type device_name collected_at', '_type_num': 'template_id', 'template_id': 300, 'device_ip': '127.0.0.1', 'device_name': u'localhost', 'packet_type': 'template', 'col_type': u'netflow', 'collected_at': u'LogInspect500', 'msg': '', '_p___raw_msg_b': 'ASwAEgAIAAQADAAEAA8ABAAKAAQADgAEAAIABAABAAQABwACAAsAAgAGAAEABAABAAUAAQARAAIAEAACAAkAAQANAAEAFQAEABYABA==\n', '_type_ip': 'device_ip' } eq_(event, expected) #eq_(event, dict( # protocol=17, # first_switched=29074919, # unix_secs=0, # sys_uptime_ms=29134919, # package_sequence=111, # destination_address='10.0.0.3', # protocol_name='UDP', # _p__raw_msg_b='CgAAAgoAAAMAAAAAAAAAAwAAAAUAAAABAAAAQBCSAFAAEQEAAwACIB8BvJBHAbul5wAAAA==\n', # version=9, # msg='', # source_address='10.0.0.2', # bytes_count=64, # destination_mask=31, # source_mask=32, # next_hop='0.0.0.0', # source_as=2, # output_interface_index=5, # source_id=0, # last_switched=29134919, # tcp_flag=0, # destination_as=3, # types_of_service=1, # destination_port=80, # input_interface_index=3, # device_name='localhost', # packet_count=1, # col_type='netflow', # source_port=4242, # device_ip=address[0], # collected_at='LogInspect500', # _type_num='bytes_count packet_count protocol types_of_service tcp_flag source_port source_mask input_interface_index destination_port destination_mask output_interface_index source_as destination_as last_switched first_switched version sys_uptime_ms unix_secs package_sequence source_id', # _type_str='protocol_name source_address destination_address next_hop msg col_type device_name collected_at', # _type_ip='source_address destination_address next_hop device_ip', # )) else: raise ValueError("Unknown netflow version type: %r" % version) def test_udp_version5(self): self.send_message(version=5) def test_udp_version9(self): self.send_message(version=9) def test_udp6_version5(self): self.send_message(address=('::1', self.port), version=5)
def main(): config = _parse_args() log_level = config['core']['log_level'] port = config['port'] expire_time = config['expire_time'] col_type = config['col_type'] collected_at = config["loginspect_name"] zmq_context = zmq.Context() netflow_out = wiring.Wire('collector_out', zmq_context=zmq_context, conf_path=config.get('wiring_conf_path') or None) sock = start_udp_server(port) while True: data, addr = sock.recvfrom(9216) if not data: continue log.debug('udp collector; from ip=%s, got msg=%s;', addr, data) ip = inet.get_ip(addr) config_ip = config_reader.get_config_ip(ip, config) if not config_ip: continue try: version = get_netflow_packet_version(data[0:2]) count = socket.ntohs(struct.unpack('H', data[2:4])[0]) current_unix_sec = (struct.unpack('I', data[8:12])[0]) log.debug("Version: %s", version) log.debug("Count of no. of records: %s", count) log.debug("Count of no. of seconds since 0000 UTC 1970: %s", current_unix_sec) netflow1 = netflow.Netflow1() netflow5 = netflow.Netflow5() netflow6 = netflow.Netflow6() netflow7 = netflow.Netflow7() global VERSION global netflowdata if ((version == 1) or (version == 5) or (version == 6) or (version == 7)): if version == 1: log.info("version 1 unpacking...") VERSION = 1 netflow1.unpack(data) netflowdata = netflow1.data elif version == 5: log.info("version 5 unpacking...") VERSION = 5 netflow5.unpack(data) netflowdata = netflow5.data elif version == 6: log.info("version 6 unpacking...") VERSION = 6 netflow6.unpack(data) netflowdata = netflow6.data elif version == 7: log.info("version 7 unpacking...") VERSION = 7 netflow7.unpack(data) netflowdata = netflow7.data i = 1 if not netflowdata: continue for netflow_record in netflowdata: try: i = i + 1 try: parsed_msg_dict = parse_record(netflow_record) except Exception, e: log.error("Could not parse the given record. %s", repr(e)) parsed_msg_dict['_p__raw_msg_b'] = binascii.b2a_base64( str(netflow_record)) parsed_msg_dict['version'] = VERSION parsed_msg_dict['current_unix_sec'] = current_unix_sec msgfilling.add_types(parsed_msg_dict, '_type_num', 'version current_unix_sec') sid = _get_sid(config_ip, config) device_name = config['client_map'][config_ip][ "device_name"] log.debug("device: %s", device_name) log.debug("descrete ip: %s", ip) try: _handle_data(parsed_msg_dict, sid, netflow_out, device_name, col_type, ip, collected_at) except Exception, e: log.error("Device name not found. %s", repr(e)) except Exception, e: log.error( "Error in constructing message, Necessary field not supplied in Netflow" ) log.error(repr(e))
class test_snare_collector(unittest.TestCase): os.environ["TZ"] = "UTC" zmq_context = zmq.Context() def setUp(self): # snare collector forwards the received msg to normalizer_in # starting snare collector config_path = disk.get_sibling(__file__, 'test-config.json') config = conf.load(config_path) self.port = config['port'] self.normalizer = wiring.Wire('norm_front_in', zmq_context=self.zmq_context) self.snare_collector = Popen(['python', 'snare_collector.py', config_path]) # Allow to prepare for serving time.sleep(0.5) def tearDown(self): self.snare_collector.kill() self.normalizer.close() time.sleep(0.5) def send_message(self, address=None, message=None, flow='udp'): address = address or ('127.0.0.1', self.port) message = message or "<124> May 06 2012 15:02:24 [emerg] (17)File exists: Couldn't create accept lock (/private/var/log/apache2/accept.lock.19) (5)\n" host, port = address if flow == 'tcp': client, sockaddr = inet.create_address(host, port) client.connect(sockaddr) client.send(message) elif flow == 'ssl': client, sockaddr = inet.create_address(host, port) client = ssl.wrap_socket(client) client.connect(sockaddr) client.send(message) elif flow == 'udp': client, sockaddr = inet.create_address(host, port, socket.SOCK_DGRAM) client.sendto(message, sockaddr) else: raise ValueError('Unknown flow type: %r' % flow) event = gevent.with_timeout(5, self.normalizer.recv, timeout_value=None) mid = event.pop('mid') assert re.match(r'^LogInspect500\|snare\|(127.0.0.1|::1)\|\d+\|1$', mid) eq_(event, dict( msg=message.rstrip('\n'), severity=4, facility=15, log_ts=1336316544, device_ip=address[0], device_name='localhost', collected_at='LogInspect500', _type_num='log_ts severity facility', _type_str='msg device_name collected_at', _type_ip='device_ip', )) def test_tcp_basic_flow(self): self.send_message(flow='tcp') def test_udp_basic_flow(self): self.send_message(flow='udp') def test_tcp6_flow(self): self.send_message(('::1', self.port), flow='tcp') def test_udp6_flow(self): self.send_message(('::1', self.port), flow='udp')
from netflow_collector import parse_record from netflow_collector import _handle_data from netflow_collector import _netflow9 from netflow_collector import _New_Netflow_v9 from netflow_collector import msgfill_parsed_record_v9 import unittest sample_v5 = '\x00\x05\x00\x01\x01\xbd$\x1dP\xab-=\x00\x00\x00\x00\x00\x00\x00\x07\x00\x00\x00\x00\n\x00\x00\x02\n\x00\x00\x03\x00\x00\x00\x00\x00\x03\x00\x05\x00\x00\x00\x01\x00\x00\x00@\x01\xbc9\xbd\x01\xbd$\x1d\x10\x92\x00P\x00\x00\x11\x01\x00\x02\x00\x03 \x1f\x00\x00' sample_v9 = '\x00\t\x00\x02\x01\xbc\x90G\x00\x00\x00\x00\x00\x00\x00o\x00\x00\x00\x00\x00\x00\x00P\x01,\x00\x12\x00\x08\x00\x04\x00\x0c\x00\x04\x00\x0f\x00\x04\x00\n\x00\x04\x00\x0e\x00\x04\x00\x02\x00\x04\x00\x01\x00\x04\x00\x07\x00\x02\x00\x0b\x00\x02\x00\x06\x00\x01\x00\x04\x00\x01\x00\x05\x00\x01\x00\x11\x00\x02\x00\x10\x00\x02\x00\t\x00\x01\x00\r\x00\x01\x00\x15\x00\x04\x00\x16\x00\x04\x01,\x008\n\x00\x00\x02\n\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x05\x00\x00\x00\x01\x00\x00\x00@\x10\x92\x00P\x00\x11\x01\x00\x03\x00\x02 \x1f\x01\xbc\x90G\x01\xbb\xa5\xe7\x00\x00\x00' config = {"col_type": "netflow"} collected_at = "Loginspect500" col_type = "netflowc" zmq_context = zmq.Context() netflow_out = wiring.Wire('collector_out', zmq_context=zmq_context, conf_path=config.get('wiring_conf_path') or None) ip = '127.0.0.1' sid = "%s|%s" % (col_type, ip) device_name = "device_name" def test_test(): eq_('test', 'test', msg="Checking test") def test_start_udp_server(): port = 9001 sock = start_udp_server(port)
def main(): #print "This is my sflow_collector 1 ." log.debug("Started.") config = _parse_args() log_level = config['core']['log_level'] port = config['port'] log.debug("This is log level set to %s.", log_level) col_type = config['col_type'] log.debug("Col_type : %s", col_type) zmq_context = zmq.Context() sflow_out = wiring.Wire('collector_out', zmq_context=zmq_context, conf_path=config.get('wiring_conf_path') or None) sock = start_udp_server(port) while True: global data data, addr = sock.recvfrom(9216) log.info("data: %s, addr: %s", data, addr) if not data: log.debug("no data") continue ip = inet.get_ip(addr) config_ip = config_reader.get_config_ip(ip, config) if not config_ip: continue try: _p__raw_msg_b = data # Datagram version = get_data32(data) address_type = get_data32(data) if address_type == Address_type['IPV4']: log.debug("IPV4 agent found.") address_type = 'IP_V4' ip_address = get_data32_addr(data) elif address_type == Address_type['IPV6']: address_type = 'IP_V6' ip_address = get_data128_addr(data) else: address_type = None sub_agent_id = get_data32(data) datagram_sequence_number = get_data32(data) switch_uptime = get_data32(data) #in ms samples_count = get_data32(data) datagram_dict = dict( _p__raw_msg_b=binascii.b2a_base64(str(_p__raw_msg_b)), version=version, address_type=address_type, ip_address=ip_address, sub_agent_id=sub_agent_id, datagram_sequence_number=datagram_sequence_number, switch_uptime=switch_uptime, samples_count=samples_count) EVENT.clear() EACH_EVENT.clear() EVENT.update(datagram_dict) EACH_EVENT.update(datagram_dict) log.info("Version: %s", version) # samples if version == Versions['VERSION5'] or address_type is not None: log.info("Version %s unpacking...", version) try: for i in range(samples_count): log.debug("datagram samples : %s", i) try: parse_sample(data) #Parse the obtained datagram except Exception, e: log.error("Unable to parse the data: %s", repr(e)) complete_event_dict = _fill_msg_types(EVENT) sid = _get_sid(config_ip, config) device_name = config['client_map'][config_ip][ "device_name"] collected_at = config["loginspect_name"] _handle_data(complete_event_dict, sid, sflow_out, device_name, col_type, ip, collected_at) EVENT.clear() EVENT.update(EACH_EVENT) except Exception, e: log.error( "Error in constructing sflow message, Necessary field not supplied in Sflow" ) log.error(repr(e)) else: #we donot accept this agent log.error( "Datagram from Unknown agent: %s. Or incorrect version type.", address_type)