def setUp(self): # input source self.batch_collector_out = wiring.Wire('batch_collector_out') # 'starting batch_processor' self.batch_processor = subprocess.Popen('python batch_processor.py', shell=True) # output sink self.normalizer = wiring.Wire('normalizer_in', use_gevent=True) # Allow syslog_collector to prepare for serving time.sleep(0.5)
def start(config, basedir, db, is_compressed_file_present, updater): logging.warn("transforming logs") repo = config["repo"] zmq_context = zmq.Context() col_out = wiring.Wire('collector_out', zmq_context=zmq_context) grouped_files, ips, estimator = get_grouped_files(basedir) device_router.add(db, ips, config) updater.update_stat('running') gevent.spawn_link_exception(progress.responder, estimator, zmq_context, repo) gevent.sleep( 2 ) # allow config-regeneration to complete successfully called by device_router.add transform(grouped_files, col_out, estimator, config, updater) # delete uploaded compressed file and extracted dir if is_compressed_file_present: dir_path = os.path.dirname(basedir) if os.path.exists(dir_path): shutil.rmtree(dir_path) # stopping immediately will remove sid in norm_front and store_handler # so sleep 1 hour, because events may be in queue of norm_front and store_handler time.sleep(3600) updater.update_stat('finished')
def test(): config = _parse_args() port = config['port'] expire_time = config['expire_time'] col_type = config['col_type'] collected_at = config["loginspect_name"] zmq_context = zmq.Context() netflow_out = wiring.Wire('collector_out', zmq_context=zmq_context, conf_path=config.get('wiring_conf_path') or None) q = Queue.Queue() for i in range(5000): q.put(DATA) #sock = start_udp_server(port) netflow1 = netflow.Netflow1() netflow5 = netflow.Netflow5() netflow6 = netflow.Netflow6() netflow7 = netflow.Netflow7() num_of_process = multiprocessing.cpu_count() start = time.time() benchmark_file = open("multiprocess.benchmark", "w") netflow_proc = [multiprocessing.Process(target=work, args=(i, q, config, netflow_out, col_type, collected_at, expire_time, start, \ netflow1, netflow5, netflow6, netflow7, benchmark_file)) for i in xrange(num_of_process)] for nf in netflow_proc: nf.start()
class test_syslog_collector(unittest.TestCase): zmq_context = zmq.Context() syslog_out = wiring.Wire('collector_out', zmq_context=zmq_context) def setUp(self): config_path = disk.get_sibling(__file__, "test-config.json") self.config = conf.load(config_path) self.port = self.config["port"] self.ssl_port = self.config["ssl_port"] self.collected_at = self.config["loginspect_name"] def tearDown(self): pass def test__create_listener(self): result = _create_listener(self.port) sock, sockaddr = inet.create_external_address(self.port) expected = sock eq_(type(result), type(expected)) def test__get_profile_info(self): addr = '127.0.0.1', self.port ip = addr[0] sid = 'syslog|127.0.0.1' device_name = 'localhost' result = _get_profile_info(addr, self.config) result = list(result) parser = result.pop(2) expected = [ip, sid, device_name, self.collected_at] eq_(result, expected) def test__get_sid_parser(self): profile = self.config["client_map"]['127.0.0.1'] sid, parser = _get_sid_parser(self.config, '127.0.0.1', profile) eq_(sid, 'syslog|127.0.0.1')
def main(): app_config = _parse_args() port = app_config["port"] address4 = ('0.0.0.0', port) address6 = ('::1', port) zmq_context = zmq.Context() snmptrapd_out = wiring.Wire('collector_out', zmq_context=zmq_context, conf_path=app_config.get('wiring_conf_path') or None) snmpEngine = getSNMPEngine(address4, address6, app_config["snmpv12_agents"], app_config["SNMPv3_users"]) cbCtx = { "out_wire": snmptrapd_out, "MIBView": getMIBViewer(), "app_config": app_config, "last_col_ts": 0, "log_counter": 0 } ntfrcv.NotificationReceiver(snmpEngine, cbFun, cbCtx) snmpEngine.transportDispatcher.jobStarted(1) # this job would never finish snmpEngine.transportDispatcher.runDispatcher()
def _create_context(config): """ Creates wiring context for sending events to normalizer """ zmq_context = zmq.Context() return wiring.Wire("collector_out", zmq_context=zmq_context, conf_path=config.get("wiring_conf_path"))
def __create_context(self, wire_name): """ Creates wiring context for sending events to normalizer """ zmq_context = zmq.Context() collector_out = wiring.Wire(wire_name, zmq_context=zmq_context, conf_path=self.__config.get('wiring_conf_path', None)) return collector_out
def main(): zmq_context = zmq.Context() config = _parse_args() opsecfetcher_out = wiring.Wire("collector_out", zmq_context=zmq_context, conf_path=config.get('wiring_conf_path') or None) log.info('opsec fetcher starting..') fetcherloop.start(config, opsecfetcher_out)
def setUp(self): # starting sflow collector config_path = disk.get_sibling(__file__, "test-config.json") config = conf.load(config_path) self.port = config["port"] self.sflow_out = wiring.Wire('norm_front_in', zmq_context=self.zmq_context) self.sflow_collector = Popen( ['python', 'sflow_collector.py', config_path]) time.sleep(0.5)
def setUp(self): self.ftp_server = subprocess.Popen( ['python', 'ftp_collector.py', 'tests/data/test-config.json']) self.batch_processor_in = wiring.Wire('batch_processor_in', use_gevent=True) # dont know why it fails when zmq_context reused #self.batch_processor_in = wiring.Wire('batch_processor_in', # zmq_context=self.zmq_context) # Allow to prepare for serving time.sleep(0.5)
def main(): zmq_context = zmq.Context() config = _parse_args() opsecfetcher_out = wiring.Wire("collector_out", zmq_context=zmq_context, conf_path=config.get("wiring_conf_path") or None) running_opsecf_jobs = {} while not STOPIT: if config["_onreload"](timeout = 1): update_jobs(config, running_opsecf_jobs, opsecfetcher_out)
def main(): config = _parse_args() port = config["port"] zmq_context = zmq.Context() snare_out = wiring.Wire('collector_out', zmq_context=zmq_context, conf_path=config.get('wiring_conf_path') or None) start_tcp_server(port, config, snare_out) start_udp_server(port, config, snare_out)
def setUp(self): # netflow collector forwards the received msg to normalizer_in # starting netflow collector config_path = disk.get_sibling(__file__, "test-config-netflow.json") config = conf.load(config_path) self.port = config['port'] self.normalizer = wiring.Wire('norm_front_in', zmq_context=self.zmq_context) self.netflow_collector = Popen( ['python', 'netflow_collector.py', config_path]) # Allow to prepare for serving time.sleep(0.5)
def main(): zmq_context = zmq.Context() config = _parse_args() #config = textual.utf8(config) #_prepare_application_directory(config) fi_out = wiring.Wire('collector_out', zmq_context=zmq_context, conf_path=config.get('wiring_conf_path') or None) log.info('LogPoint_agent_collector starting...') fi_collector.main(config, fi_out)
def monitor(config, cursor_shelve, zmq_context): collector_out = wiring.Wire('collector_out', zmq_context=zmq_context, conf_path=config.get('wiring_conf_path') or None) new_modified_config = {"client_map": {}} while True: if config['_onreload'](): old_modified_config = new_modified_config new_modified_config = _get_modified_config(config) if new_modified_config != old_modified_config: update_norm_repo_db(new_modified_config) _del_unused_paths(new_modified_config, cursor_shelve) _read_file_content_changes(new_modified_config, cursor_shelve, collector_out) gevent.sleep(10)
def responder(estimator, zmq_context, repo_name): responder = wiring.Wire('liv4_collector_progress_reply', zmq_context=zmq_context, repo_name=repo_name) start_time = time.time() while True: request = responder.recv() while True: est_byte_size = float(estimator.get_est_total_bytes()) completed_bytes = reader.BYTES_READ completed_msgs = reader.COUNTER completed_percent = completed_bytes / est_byte_size * 100 if completed_percent < 0.1: logging.warn( "completed_bytes=%s, completed_msgs=%s, est_byte_size=%s, completed_percent=%s", completed_bytes, completed_msgs, est_byte_size, completed_percent) gevent.sleep(2) elif completed_percent > 100: completed_percent = 100 else: break duration = time.time() - start_time rate_bytes = completed_bytes / duration rate_msgs = completed_msgs / duration rate_unit = _get_human_size(rate_bytes) time_left = _calculate_time_left(est_byte_size, completed_bytes, rate_bytes) if int(time_left) == 0 and completed_percent != 100: time_left = 1 elif time_left < 0: time_left = 0 human_time = _get_human_time(time_left) response = dict( percent_completed="%.1f%%" % (completed_percent), rate="%.1f %s/sec (%d msg/sec)" % (rate_unit[0], rate_unit[1], rate_msgs), time_left="%.1f %s" % tuple(human_time), ) responder.send(response)
def main(): try: repo = sys.argv[1] except: sys.exit("Usage python requester.py <repo>") db = mongo.get_makalu() requester = wiring.Wire('liv4_collector_progress_request', repo_name=repo) while True: status = db.datatransport.find_one({'repo': repo}).get("status") if status == "run": requester.send(()) print requester.recv() time.sleep(5) elif status == "finished": print "finished" return else: print 'invalid stat %r' % status return
def test_file_handler(): col_type = 'ftp' sid = 'ftp|192.168.1.2' col_ts = '1234567890' parser = 'SyslogParser' file = '/dummy/path' charset = 'utf-8' device_name = "makalu" normalizer = "norm1" repo = "repo1" device_ip = "192.168.1.2" file_handler.main(sid, col_type, col_ts, parser, file, charset, device_name, normalizer, repo, device_ip=device_ip) batch_processor_in = wiring.Wire('batch_processor_in') event = batch_processor_in.recv() eq_( event, dict(sid=sid, col_ts=col_ts, parser=parser, file=file, charset=charset, device_name=device_name, cursor=0, normalizer=normalizer, repo=repo, device_ip=device_ip, regexparser_name=None, regex_pattern=None, col_type=col_type))
def main(): log.info('Batch Processor for Collector apps starting...') config = _parse_args() zmq_context = zmq.Context() processor_in = wiring.Wire('batch_processor_in', zmq_context=zmq_context, conf_path=config.get('wiring_conf_path') or None) collector_out = wiring.Wire('collector_out', zmq_context=zmq_context, conf_path=config.get('wiring_conf_path') or None) last_timestamp = 0 throttler = timing.speed_throttler(MAX_SPEED) while True: file_info = processor_in.recv() sid = file_info['sid'] col_type = file_info['col_type'] col_ts = file_info['col_ts'] parser = file_info['parser'] file = file_info['file'] cursor = file_info['cursor'] charset = file_info['charset'] device_name = file_info['device_name'] device_ip = file_info['device_ip'] regex_pattern = file_info['regex_pattern'] regexparser_name = file_info['regexparser_name'] normalizer = file_info['normalizer'] repo = file_info['repo'] source_name = file_info['source_name'] current_timestamp = int(time.time()) if current_timestamp > last_timestamp: last_timestamp = current_timestamp counter = 0 for event in file_processor(file, parser, sid, charset, cursor, regex_pattern, regexparser_name): throttler.next() counter += 1 loginspect_name = config["loginspect_name"] event['mid'] = '%s|%s|%s|%d' % (loginspect_name, sid, col_ts, counter) event['collected_at'] = loginspect_name event['device_name'] = device_name msgfilling.add_types(event, '_type_str', 'device_name collected_at col_type') event['col_ts'] = col_ts event['_counter'] = counter event['col_type'] = col_type msgfilling.add_types(event, '_type_num', 'col_ts') if device_ip is not None: event['device_ip'] = device_ip msgfilling.add_types(event, '_type_str', 'device_ip') msgfilling.add_types(event, '_type_ip', 'device_ip') event['normalizer'] = normalizer event['repo'] = repo if source_name: if event.get('_normalized_fields'): event['_normalized_fields']['source_name'] = source_name else: event['_normalized_fields'] = {'source_name': source_name} msgfilling.add_types(event, '_type_str', 'source_name') collector_out.send_with_norm_policy_and_repo(event) try: os.unlink(file) except: pass
def main(): #print "This is my sflow_collector 1 ." log.debug("Started.") config = _parse_args() log_level = config['core']['log_level'] port = config['port'] log.debug("This is log level set to %s.", log_level) col_type = config['col_type'] log.debug("Col_type : %s", col_type) zmq_context = zmq.Context() sflow_out = wiring.Wire('collector_out', zmq_context=zmq_context, conf_path=config.get('wiring_conf_path') or None) sock = start_udp_server(port) while True: global data data, addr = sock.recvfrom(9216) log.info("data: %s, addr: %s", data, addr) if not data: log.debug("no data") continue ip = inet.get_ip(addr) config_ip = config_reader.get_config_ip(ip, config) if not config_ip: continue try: _p__raw_msg_b = data # Datagram version = get_data32(data) address_type = get_data32(data) if address_type == Address_type['IPV4']: log.debug("IPV4 agent found.") address_type = 'IP_V4' ip_address = get_data32_addr(data) elif address_type == Address_type['IPV6']: address_type = 'IP_V6' ip_address = get_data128_addr(data) else: address_type = None sub_agent_id = get_data32(data) datagram_sequence_number = get_data32(data) switch_uptime = get_data32(data) #in ms samples_count = get_data32(data) datagram_dict = dict( _p__raw_msg_b=binascii.b2a_base64(str(_p__raw_msg_b)), version=version, address_type=address_type, ip_address=ip_address, sub_agent_id=sub_agent_id, datagram_sequence_number=datagram_sequence_number, switch_uptime=switch_uptime, samples_count=samples_count) EVENT.clear() EACH_EVENT.clear() EVENT.update(datagram_dict) EACH_EVENT.update(datagram_dict) log.info("Version: %s", version) # samples if version == Versions['VERSION5'] or address_type is not None: log.info("Version %s unpacking...", version) try: for i in range(samples_count): log.debug("datagram samples : %s", i) try: parse_sample(data) #Parse the obtained datagram except Exception, e: log.error("Unable to parse the data: %s", repr(e)) complete_event_dict = _fill_msg_types(EVENT) sid = _get_sid(config_ip, config) device_name = config['client_map'][config_ip][ "device_name"] collected_at = config["loginspect_name"] _handle_data(complete_event_dict, sid, sflow_out, device_name, col_type, ip, collected_at) EVENT.clear() EVENT.update(EACH_EVENT) except Exception, e: log.error( "Error in constructing sflow message, Necessary field not supplied in Sflow" ) log.error(repr(e)) else: #we donot accept this agent log.error( "Datagram from Unknown agent: %s. Or incorrect version type.", address_type)
from netflow_collector import _netflow9 from netflow_collector import _New_Netflow_v9 from netflow_collector import msgfill_parsed_record_v9 import unittest sample_v5 = '\x00\x05\x00\x01\x01\xbd$\x1dP\xab-=\x00\x00\x00\x00\x00\x00\x00\x07\x00\x00\x00\x00\n\x00\x00\x02\n\x00\x00\x03\x00\x00\x00\x00\x00\x03\x00\x05\x00\x00\x00\x01\x00\x00\x00@\x01\xbc9\xbd\x01\xbd$\x1d\x10\x92\x00P\x00\x00\x11\x01\x00\x02\x00\x03 \x1f\x00\x00' sample_v9 = '\x00\t\x00\x02\x01\xbc\x90G\x00\x00\x00\x00\x00\x00\x00o\x00\x00\x00\x00\x00\x00\x00P\x01,\x00\x12\x00\x08\x00\x04\x00\x0c\x00\x04\x00\x0f\x00\x04\x00\n\x00\x04\x00\x0e\x00\x04\x00\x02\x00\x04\x00\x01\x00\x04\x00\x07\x00\x02\x00\x0b\x00\x02\x00\x06\x00\x01\x00\x04\x00\x01\x00\x05\x00\x01\x00\x11\x00\x02\x00\x10\x00\x02\x00\t\x00\x01\x00\r\x00\x01\x00\x15\x00\x04\x00\x16\x00\x04\x01,\x008\n\x00\x00\x02\n\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x05\x00\x00\x00\x01\x00\x00\x00@\x10\x92\x00P\x00\x11\x01\x00\x03\x00\x02 \x1f\x01\xbc\x90G\x01\xbb\xa5\xe7\x00\x00\x00' config = {"col_type": "netflow"} collected_at = "Loginspect500" col_type = "netflowc" zmq_context = zmq.Context() netflow_out = wiring.Wire('collector_out', zmq_context=zmq_context, conf_path=config.get('wiring_conf_path') or None) ip = '127.0.0.1' sid = "%s|%s" % (col_type, ip) device_name = "device_name" def test_test(): eq_('test', 'test', msg="Checking test") def test_start_udp_server(): port = 9001 sock = start_udp_server(port) sock_obj, sockaddr = inet.create_external_address(port, socket.SOCK_DGRAM,
def main(): config = _parse_args() log_level = config['core']['log_level'] port = config['port'] expire_time = config['expire_time'] col_type = config['col_type'] collected_at = config["loginspect_name"] zmq_context = zmq.Context() netflow_out = wiring.Wire('collector_out', zmq_context=zmq_context, conf_path=config.get('wiring_conf_path') or None) sock = start_udp_server(port) while True: data, addr = sock.recvfrom(9216) if not data: continue log.debug('udp collector; from ip=%s, got msg=%s;', addr, data) ip = inet.get_ip(addr) config_ip = config_reader.get_config_ip(ip, config) if not config_ip: continue try: version = get_netflow_packet_version(data[0:2]) count = socket.ntohs(struct.unpack('H', data[2:4])[0]) current_unix_sec = (struct.unpack('I', data[8:12])[0]) log.debug("Version: %s", version) log.debug("Count of no. of records: %s", count) log.debug("Count of no. of seconds since 0000 UTC 1970: %s", current_unix_sec) netflow1 = netflow.Netflow1() netflow5 = netflow.Netflow5() netflow6 = netflow.Netflow6() netflow7 = netflow.Netflow7() global VERSION global netflowdata if ((version == 1) or (version == 5) or (version == 6) or (version == 7)): if version == 1: log.info("version 1 unpacking...") VERSION = 1 netflow1.unpack(data) netflowdata = netflow1.data elif version == 5: log.info("version 5 unpacking...") VERSION = 5 netflow5.unpack(data) netflowdata = netflow5.data elif version == 6: log.info("version 6 unpacking...") VERSION = 6 netflow6.unpack(data) netflowdata = netflow6.data elif version == 7: log.info("version 7 unpacking...") VERSION = 7 netflow7.unpack(data) netflowdata = netflow7.data i = 1 if not netflowdata: continue for netflow_record in netflowdata: try: i = i + 1 try: parsed_msg_dict = parse_record(netflow_record) except Exception, e: log.error("Could not parse the given record. %s", repr(e)) parsed_msg_dict['_p__raw_msg_b'] = binascii.b2a_base64( str(netflow_record)) parsed_msg_dict['version'] = VERSION parsed_msg_dict['current_unix_sec'] = current_unix_sec msgfilling.add_types(parsed_msg_dict, '_type_num', 'version current_unix_sec') sid = _get_sid(config_ip, config) device_name = config['client_map'][config_ip][ "device_name"] log.debug("device: %s", device_name) log.debug("descrete ip: %s", ip) try: _handle_data(parsed_msg_dict, sid, netflow_out, device_name, col_type, ip, collected_at) except Exception, e: log.error("Device name not found. %s", repr(e)) except Exception, e: log.error( "Error in constructing message, Necessary field not supplied in Netflow" ) log.error(repr(e))