def setUp(self):
# input source
self.batch_collector_out = wiring.Wire('batch_collector_out')
# 'starting batch_processor'
self.batch_processor = subprocess.Popen('python batch_processor.py',
shell=True)
# output sink
self.normalizer = wiring.Wire('normalizer_in', use_gevent=True)
# Allow syslog_collector to prepare for serving
time.sleep(0.5)
def start(config, basedir, db, is_compressed_file_present, updater):
logging.warn("transforming logs")
repo = config["repo"]
zmq_context = zmq.Context()
col_out = wiring.Wire('collector_out', zmq_context=zmq_context)
grouped_files, ips, estimator = get_grouped_files(basedir)
device_router.add(db, ips, config)
updater.update_stat('running')
gevent.spawn_link_exception(progress.responder, estimator, zmq_context,
repo)
gevent.sleep(
2
) # allow config-regeneration to complete successfully called by device_router.add
transform(grouped_files, col_out, estimator, config, updater)
# delete uploaded compressed file and extracted dir
if is_compressed_file_present:
dir_path = os.path.dirname(basedir)
if os.path.exists(dir_path):
shutil.rmtree(dir_path)
# stopping immediately will remove sid in norm_front and store_handler
# so sleep 1 hour, because events may be in queue of norm_front and store_handler
time.sleep(3600)
updater.update_stat('finished')
def test():
config = _parse_args()
port = config['port']
expire_time = config['expire_time']
col_type = config['col_type']
collected_at = config["loginspect_name"]
zmq_context = zmq.Context()
netflow_out = wiring.Wire('collector_out',
zmq_context=zmq_context,
conf_path=config.get('wiring_conf_path') or None)
q = Queue.Queue()
for i in range(5000):
q.put(DATA)
#sock = start_udp_server(port)
netflow1 = netflow.Netflow1()
netflow5 = netflow.Netflow5()
netflow6 = netflow.Netflow6()
netflow7 = netflow.Netflow7()
num_of_process = multiprocessing.cpu_count()
start = time.time()
benchmark_file = open("multiprocess.benchmark", "w")
netflow_proc = [multiprocessing.Process(target=work,
args=(i, q, config, netflow_out, col_type, collected_at, expire_time, start, \
netflow1, netflow5, netflow6, netflow7, benchmark_file)) for i in xrange(num_of_process)]
for nf in netflow_proc:
nf.start()
class test_syslog_collector(unittest.TestCase):
zmq_context = zmq.Context()
syslog_out = wiring.Wire('collector_out', zmq_context=zmq_context)
def setUp(self):
config_path = disk.get_sibling(__file__, "test-config.json")
self.config = conf.load(config_path)
self.port = self.config["port"]
self.ssl_port = self.config["ssl_port"]
self.collected_at = self.config["loginspect_name"]
def tearDown(self):
pass
def test__create_listener(self):
result = _create_listener(self.port)
sock, sockaddr = inet.create_external_address(self.port)
expected = sock
eq_(type(result), type(expected))
def test__get_profile_info(self):
addr = '127.0.0.1', self.port
ip = addr[0]
sid = 'syslog|127.0.0.1'
device_name = 'localhost'
result = _get_profile_info(addr, self.config)
result = list(result)
parser = result.pop(2)
expected = [ip, sid, device_name, self.collected_at]
eq_(result, expected)
def test__get_sid_parser(self):
profile = self.config["client_map"]['127.0.0.1']
sid, parser = _get_sid_parser(self.config, '127.0.0.1', profile)
eq_(sid, 'syslog|127.0.0.1')
def main():
app_config = _parse_args()
port = app_config["port"]
address4 = ('0.0.0.0', port)
address6 = ('::1', port)
zmq_context = zmq.Context()
snmptrapd_out = wiring.Wire('collector_out',
zmq_context=zmq_context,
conf_path=app_config.get('wiring_conf_path')
or None)
snmpEngine = getSNMPEngine(address4, address6,
app_config["snmpv12_agents"],
app_config["SNMPv3_users"])
cbCtx = {
"out_wire": snmptrapd_out,
"MIBView": getMIBViewer(),
"app_config": app_config,
"last_col_ts": 0,
"log_counter": 0
}
ntfrcv.NotificationReceiver(snmpEngine, cbFun, cbCtx)
snmpEngine.transportDispatcher.jobStarted(1) # this job would never finish
snmpEngine.transportDispatcher.runDispatcher()
def _create_context(config):
"""
Creates wiring context for sending events to normalizer
"""
zmq_context = zmq.Context()
return wiring.Wire("collector_out",
zmq_context=zmq_context,
conf_path=config.get("wiring_conf_path"))
def __create_context(self, wire_name):
"""
Creates wiring context for sending events to normalizer
"""
zmq_context = zmq.Context()
collector_out = wiring.Wire(wire_name, zmq_context=zmq_context,
conf_path=self.__config.get('wiring_conf_path', None))
return collector_out
def main():
zmq_context = zmq.Context()
config = _parse_args()
opsecfetcher_out = wiring.Wire("collector_out", zmq_context=zmq_context,
conf_path=config.get('wiring_conf_path') or None)
log.info('opsec fetcher starting..')
fetcherloop.start(config, opsecfetcher_out)
def setUp(self):
# starting sflow collector
config_path = disk.get_sibling(__file__, "test-config.json")
config = conf.load(config_path)
self.port = config["port"]
self.sflow_out = wiring.Wire('norm_front_in',
zmq_context=self.zmq_context)
self.sflow_collector = Popen(
['python', 'sflow_collector.py', config_path])
time.sleep(0.5)
def setUp(self):
self.ftp_server = subprocess.Popen(
['python', 'ftp_collector.py', 'tests/data/test-config.json'])
self.batch_processor_in = wiring.Wire('batch_processor_in',
use_gevent=True)
# dont know why it fails when zmq_context reused
#self.batch_processor_in = wiring.Wire('batch_processor_in',
# zmq_context=self.zmq_context)
# Allow to prepare for serving
time.sleep(0.5)
def main():
zmq_context = zmq.Context()
config = _parse_args()
opsecfetcher_out = wiring.Wire("collector_out", zmq_context=zmq_context,
conf_path=config.get("wiring_conf_path") or None)
running_opsecf_jobs = {}
while not STOPIT:
if config["_onreload"](timeout = 1):
update_jobs(config, running_opsecf_jobs, opsecfetcher_out)
def main():
config = _parse_args()
port = config["port"]
zmq_context = zmq.Context()
snare_out = wiring.Wire('collector_out',
zmq_context=zmq_context,
conf_path=config.get('wiring_conf_path') or None)
start_tcp_server(port, config, snare_out)
start_udp_server(port, config, snare_out)
def setUp(self):
# netflow collector forwards the received msg to normalizer_in
# starting netflow collector
config_path = disk.get_sibling(__file__, "test-config-netflow.json")
config = conf.load(config_path)
self.port = config['port']
self.normalizer = wiring.Wire('norm_front_in',
zmq_context=self.zmq_context)
self.netflow_collector = Popen(
['python', 'netflow_collector.py', config_path])
# Allow to prepare for serving
time.sleep(0.5)
def main():
zmq_context = zmq.Context()
config = _parse_args()
#config = textual.utf8(config)
#_prepare_application_directory(config)
fi_out = wiring.Wire('collector_out',
zmq_context=zmq_context,
conf_path=config.get('wiring_conf_path') or None)
log.info('LogPoint_agent_collector starting...')
fi_collector.main(config, fi_out)
def monitor(config, cursor_shelve, zmq_context):
collector_out = wiring.Wire('collector_out',
zmq_context=zmq_context,
conf_path=config.get('wiring_conf_path')
or None)
new_modified_config = {"client_map": {}}
while True:
if config['_onreload']():
old_modified_config = new_modified_config
new_modified_config = _get_modified_config(config)
if new_modified_config != old_modified_config:
update_norm_repo_db(new_modified_config)
_del_unused_paths(new_modified_config, cursor_shelve)
_read_file_content_changes(new_modified_config, cursor_shelve,
collector_out)
gevent.sleep(10)
def responder(estimator, zmq_context, repo_name):
responder = wiring.Wire('liv4_collector_progress_reply',
zmq_context=zmq_context,
repo_name=repo_name)
start_time = time.time()
while True:
request = responder.recv()
while True:
est_byte_size = float(estimator.get_est_total_bytes())
completed_bytes = reader.BYTES_READ
completed_msgs = reader.COUNTER
completed_percent = completed_bytes / est_byte_size * 100
if completed_percent < 0.1:
logging.warn(
"completed_bytes=%s, completed_msgs=%s, est_byte_size=%s, completed_percent=%s",
completed_bytes, completed_msgs, est_byte_size,
completed_percent)
gevent.sleep(2)
elif completed_percent > 100:
completed_percent = 100
else:
break
duration = time.time() - start_time
rate_bytes = completed_bytes / duration
rate_msgs = completed_msgs / duration
rate_unit = _get_human_size(rate_bytes)
time_left = _calculate_time_left(est_byte_size, completed_bytes,
rate_bytes)
if int(time_left) == 0 and completed_percent != 100:
time_left = 1
elif time_left < 0:
time_left = 0
human_time = _get_human_time(time_left)
response = dict(
percent_completed="%.1f%%" % (completed_percent),
rate="%.1f %s/sec (%d msg/sec)" %
(rate_unit[0], rate_unit[1], rate_msgs),
time_left="%.1f %s" % tuple(human_time),
)
responder.send(response)
def main():
try:
repo = sys.argv[1]
except:
sys.exit("Usage python requester.py <repo>")
db = mongo.get_makalu()
requester = wiring.Wire('liv4_collector_progress_request', repo_name=repo)
while True:
status = db.datatransport.find_one({'repo': repo}).get("status")
if status == "run":
requester.send(())
print requester.recv()
time.sleep(5)
elif status == "finished":
print "finished"
return
else:
print 'invalid stat %r' % status
return
def test_file_handler():
col_type = 'ftp'
sid = 'ftp|192.168.1.2'
col_ts = '1234567890'
parser = 'SyslogParser'
file = '/dummy/path'
charset = 'utf-8'
device_name = "makalu"
normalizer = "norm1"
repo = "repo1"
device_ip = "192.168.1.2"
file_handler.main(sid,
col_type,
col_ts,
parser,
file,
charset,
device_name,
normalizer,
repo,
device_ip=device_ip)
batch_processor_in = wiring.Wire('batch_processor_in')
event = batch_processor_in.recv()
eq_(
event,
dict(sid=sid,
col_ts=col_ts,
parser=parser,
file=file,
charset=charset,
device_name=device_name,
cursor=0,
normalizer=normalizer,
repo=repo,
device_ip=device_ip,
regexparser_name=None,
regex_pattern=None,
col_type=col_type))
def main():
log.info('Batch Processor for Collector apps starting...')
config = _parse_args()
zmq_context = zmq.Context()
processor_in = wiring.Wire('batch_processor_in',
zmq_context=zmq_context,
conf_path=config.get('wiring_conf_path')
or None)
collector_out = wiring.Wire('collector_out',
zmq_context=zmq_context,
conf_path=config.get('wiring_conf_path')
or None)
last_timestamp = 0
throttler = timing.speed_throttler(MAX_SPEED)
while True:
file_info = processor_in.recv()
sid = file_info['sid']
col_type = file_info['col_type']
col_ts = file_info['col_ts']
parser = file_info['parser']
file = file_info['file']
cursor = file_info['cursor']
charset = file_info['charset']
device_name = file_info['device_name']
device_ip = file_info['device_ip']
regex_pattern = file_info['regex_pattern']
regexparser_name = file_info['regexparser_name']
normalizer = file_info['normalizer']
repo = file_info['repo']
source_name = file_info['source_name']
current_timestamp = int(time.time())
if current_timestamp > last_timestamp:
last_timestamp = current_timestamp
counter = 0
for event in file_processor(file, parser, sid, charset, cursor,
regex_pattern, regexparser_name):
throttler.next()
counter += 1
loginspect_name = config["loginspect_name"]
event['mid'] = '%s|%s|%s|%d' % (loginspect_name, sid, col_ts,
counter)
event['collected_at'] = loginspect_name
event['device_name'] = device_name
msgfilling.add_types(event, '_type_str',
'device_name collected_at col_type')
event['col_ts'] = col_ts
event['_counter'] = counter
event['col_type'] = col_type
msgfilling.add_types(event, '_type_num', 'col_ts')
if device_ip is not None:
event['device_ip'] = device_ip
msgfilling.add_types(event, '_type_str', 'device_ip')
msgfilling.add_types(event, '_type_ip', 'device_ip')
event['normalizer'] = normalizer
event['repo'] = repo
if source_name:
if event.get('_normalized_fields'):
event['_normalized_fields']['source_name'] = source_name
else:
event['_normalized_fields'] = {'source_name': source_name}
msgfilling.add_types(event, '_type_str', 'source_name')
collector_out.send_with_norm_policy_and_repo(event)
try:
os.unlink(file)
except:
pass
def main():
#print "This is my sflow_collector 1 ."
log.debug("Started.")
config = _parse_args()
log_level = config['core']['log_level']
port = config['port']
log.debug("This is log level set to %s.", log_level)
col_type = config['col_type']
log.debug("Col_type : %s", col_type)
zmq_context = zmq.Context()
sflow_out = wiring.Wire('collector_out',
zmq_context=zmq_context,
conf_path=config.get('wiring_conf_path') or None)
sock = start_udp_server(port)
while True:
global data
data, addr = sock.recvfrom(9216)
log.info("data: %s, addr: %s", data, addr)
if not data:
log.debug("no data")
continue
ip = inet.get_ip(addr)
config_ip = config_reader.get_config_ip(ip, config)
if not config_ip:
continue
try:
_p__raw_msg_b = data
# Datagram
version = get_data32(data)
address_type = get_data32(data)
if address_type == Address_type['IPV4']:
log.debug("IPV4 agent found.")
address_type = 'IP_V4'
ip_address = get_data32_addr(data)
elif address_type == Address_type['IPV6']:
address_type = 'IP_V6'
ip_address = get_data128_addr(data)
else:
address_type = None
sub_agent_id = get_data32(data)
datagram_sequence_number = get_data32(data)
switch_uptime = get_data32(data) #in ms
samples_count = get_data32(data)
datagram_dict = dict(
_p__raw_msg_b=binascii.b2a_base64(str(_p__raw_msg_b)),
version=version,
address_type=address_type,
ip_address=ip_address,
sub_agent_id=sub_agent_id,
datagram_sequence_number=datagram_sequence_number,
switch_uptime=switch_uptime,
samples_count=samples_count)
EVENT.clear()
EACH_EVENT.clear()
EVENT.update(datagram_dict)
EACH_EVENT.update(datagram_dict)
log.info("Version: %s", version)
# samples
if version == Versions['VERSION5'] or address_type is not None:
log.info("Version %s unpacking...", version)
try:
for i in range(samples_count):
log.debug("datagram samples : %s", i)
try:
parse_sample(data) #Parse the obtained datagram
except Exception, e:
log.error("Unable to parse the data: %s", repr(e))
complete_event_dict = _fill_msg_types(EVENT)
sid = _get_sid(config_ip, config)
device_name = config['client_map'][config_ip][
"device_name"]
collected_at = config["loginspect_name"]
_handle_data(complete_event_dict, sid, sflow_out,
device_name, col_type, ip, collected_at)
EVENT.clear()
EVENT.update(EACH_EVENT)
except Exception, e:
log.error(
"Error in constructing sflow message, Necessary field not supplied in Sflow"
)
log.error(repr(e))
else:
#we donot accept this agent
log.error(
"Datagram from Unknown agent: %s. Or incorrect version type.",
address_type)
from netflow_collector import _netflow9
from netflow_collector import _New_Netflow_v9
from netflow_collector import msgfill_parsed_record_v9
import unittest
sample_v5 = '\x00\x05\x00\x01\x01\xbd$\x1dP\xab-=\x00\x00\x00\x00\x00\x00\x00\x07\x00\x00\x00\x00\n\x00\x00\x02\n\x00\x00\x03\x00\x00\x00\x00\x00\x03\x00\x05\x00\x00\x00\x01\x00\x00\x00@\x01\xbc9\xbd\x01\xbd$\x1d\x10\x92\x00P\x00\x00\x11\x01\x00\x02\x00\x03 \x1f\x00\x00'
sample_v9 = '\x00\t\x00\x02\x01\xbc\x90G\x00\x00\x00\x00\x00\x00\x00o\x00\x00\x00\x00\x00\x00\x00P\x01,\x00\x12\x00\x08\x00\x04\x00\x0c\x00\x04\x00\x0f\x00\x04\x00\n\x00\x04\x00\x0e\x00\x04\x00\x02\x00\x04\x00\x01\x00\x04\x00\x07\x00\x02\x00\x0b\x00\x02\x00\x06\x00\x01\x00\x04\x00\x01\x00\x05\x00\x01\x00\x11\x00\x02\x00\x10\x00\x02\x00\t\x00\x01\x00\r\x00\x01\x00\x15\x00\x04\x00\x16\x00\x04\x01,\x008\n\x00\x00\x02\n\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x05\x00\x00\x00\x01\x00\x00\x00@\x10\x92\x00P\x00\x11\x01\x00\x03\x00\x02 \x1f\x01\xbc\x90G\x01\xbb\xa5\xe7\x00\x00\x00'
config = {"col_type": "netflow"}
collected_at = "Loginspect500"
col_type = "netflowc"
zmq_context = zmq.Context()
netflow_out = wiring.Wire('collector_out',
zmq_context=zmq_context,
conf_path=config.get('wiring_conf_path') or None)
ip = '127.0.0.1'
sid = "%s|%s" % (col_type, ip)
device_name = "device_name"
def test_test():
eq_('test', 'test', msg="Checking test")
def test_start_udp_server():
port = 9001
sock = start_udp_server(port)
sock_obj, sockaddr = inet.create_external_address(port,
socket.SOCK_DGRAM,
def main():
config = _parse_args()
log_level = config['core']['log_level']
port = config['port']
expire_time = config['expire_time']
col_type = config['col_type']
collected_at = config["loginspect_name"]
zmq_context = zmq.Context()
netflow_out = wiring.Wire('collector_out',
zmq_context=zmq_context,
conf_path=config.get('wiring_conf_path') or None)
sock = start_udp_server(port)
while True:
data, addr = sock.recvfrom(9216)
if not data:
continue
log.debug('udp collector; from ip=%s, got msg=%s;', addr, data)
ip = inet.get_ip(addr)
config_ip = config_reader.get_config_ip(ip, config)
if not config_ip:
continue
try:
version = get_netflow_packet_version(data[0:2])
count = socket.ntohs(struct.unpack('H', data[2:4])[0])
current_unix_sec = (struct.unpack('I', data[8:12])[0])
log.debug("Version: %s", version)
log.debug("Count of no. of records: %s", count)
log.debug("Count of no. of seconds since 0000 UTC 1970: %s",
current_unix_sec)
netflow1 = netflow.Netflow1()
netflow5 = netflow.Netflow5()
netflow6 = netflow.Netflow6()
netflow7 = netflow.Netflow7()
global VERSION
global netflowdata
if ((version == 1) or (version == 5) or (version == 6)
or (version == 7)):
if version == 1:
log.info("version 1 unpacking...")
VERSION = 1
netflow1.unpack(data)
netflowdata = netflow1.data
elif version == 5:
log.info("version 5 unpacking...")
VERSION = 5
netflow5.unpack(data)
netflowdata = netflow5.data
elif version == 6:
log.info("version 6 unpacking...")
VERSION = 6
netflow6.unpack(data)
netflowdata = netflow6.data
elif version == 7:
log.info("version 7 unpacking...")
VERSION = 7
netflow7.unpack(data)
netflowdata = netflow7.data
i = 1
if not netflowdata:
continue
for netflow_record in netflowdata:
try:
i = i + 1
try:
parsed_msg_dict = parse_record(netflow_record)
except Exception, e:
log.error("Could not parse the given record. %s",
repr(e))
parsed_msg_dict['_p__raw_msg_b'] = binascii.b2a_base64(
str(netflow_record))
parsed_msg_dict['version'] = VERSION
parsed_msg_dict['current_unix_sec'] = current_unix_sec
msgfilling.add_types(parsed_msg_dict, '_type_num',
'version current_unix_sec')
sid = _get_sid(config_ip, config)
device_name = config['client_map'][config_ip][
"device_name"]
log.debug("device: %s", device_name)
log.debug("descrete ip: %s", ip)
try:
_handle_data(parsed_msg_dict, sid, netflow_out,
device_name, col_type, ip,
collected_at)
except Exception, e:
log.error("Device name not found. %s", repr(e))
except Exception, e:
log.error(
"Error in constructing message, Necessary field not supplied in Netflow"
)
log.error(repr(e))