def setUp(self):
# starting sflow collector
config_path = disk.get_sibling(__file__, "test-config.json")
config = conf.load(config_path)
self.port = config["port"]
self.sflow_out = wiring.Wire('norm_front_in',
zmq_context=self.zmq_context)
self.sflow_collector = Popen(
['python', 'sflow_collector.py', config_path])
time.sleep(0.5)
def setUp(self):
# netflow collector forwards the received msg to normalizer_in
# starting netflow collector
config_path = disk.get_sibling(__file__, "test-config-netflow.json")
config = conf.load(config_path)
self.port = config['port']
self.normalizer = wiring.Wire('norm_front_in',
zmq_context=self.zmq_context)
self.netflow_collector = Popen(
['python', 'netflow_collector.py', config_path])
# Allow to prepare for serving
time.sleep(0.5)
def send_message(self, address=None, message=None):
address = address or ('127.0.0.1', self.port)
host, port = address
data_file_path = disk.get_sibling(__file__, "sflow-data-v5.txt")
message = message or open(data_file_path, "rb").read()
client, sockaddr = inet.create_address(host, port, socket.SOCK_DGRAM)
client.sendto(message, sockaddr)
event = gevent.with_timeout(5, self.sflow_out.recv, timeout_value=None)
mid = event.pop('mid')
assert re.match(r'^LogInspect500\|sflow\|(127.0.0.1|::1)\|\d+\|1$',
mid)
device_name = event.pop('device_name')
eq_(device_name, 'localhost')
expected = dict(
col_type='sflow',
switch_uptime=240000L,
_type_num=
'switch_uptime samples_count sample_source_id_index sub_agent_id sample_sequence_number sample_source_id_type version datagram_sequence_number version',
samples_count=1L,
sub_agent_id=100000L,
device_ip=address[0],
sample_type='COUNTER_SAMPLE',
_p__raw_msg_b=
'AAAABQAAAAHAqAIoAAGGoAAAAAwAA6mAAAAAAQAAAAIAAAGMAAAADAIAAAEAAAAGAAAH0QAAACQAAAACAAAAAQAAAAEAAAAAAAAAAAAAAAIAAAABCAAnPETIAAAAAAfVAAAANAAAAAHdFswAAAAAAY+B4AAAAAZmAAAiyAAAAAAKiYQAABA0sAAACKYAAAAAAOuQAAAAE9wAAAfUAAAASAAAAAAt99AAAAAAACl0AAAAAAAAAAAAAAAAAAAA/mAAAAAAAAHBIAAAAAAAGD/gAAAAAAAYP+AAAAEMRAAAE6YAAAAAAAAAAAAAB9MAAABEAAAAADwj1woAAAAAAAAAAAAAAEwAAAABAAAJZgAAAU0AAAWgAAAAAAAAEoQABN6aAAAeAAAAADwAAAAeAAAyogAAaOoAAAfWAAAAKAAAAAAAAk54AAAECgAAAAAAAAAAAAAAAAAAlKAAAACMAAAAAAAAAAAAAAfQAAAARAAAAAlyaXR1YnVudHUAAAAfQisf1SlOi4jO6A5NVlugAAAAAgAAAAIAAAAVMi42LjMyLTI4LWdlbmVyaWMtcGFlAAAA\n',
sample_sequence_number=12L,
sample_source_id_index=1L,
sample_source_id_type=2L,
version=5L,
address_type='IP_V4',
datagram_sequence_number=12L,
collected_at='LogInspect500',
msg='',
ip_address='192.168.2.40',
_type_ip='ip_address device_ip',
_type_str=
'sample_type address_type ip_address msg col_type device_name collected_at',
)
eq_(event, expected)
def _parse_args():
cli_parser = optparse.OptionParser()
cli_parser.add_option("-c", "--conf", help="Configuration file.")
cli_parser.set_defaults(conf=disk.get_sibling(__file__, 'sources.conf'))
logger.configure(cli_parser, syslog=False)
options, args = cli_parser.parse_args()
if len(args) > 0:
cli_parser.error("No extra arg is expected.")
try:
mapper = json.load(open(options.conf))
except IOError, err:
log.debug(err)
cli_parser.error("Configuration file '%s' does not exist." %
options.conf)
def send_message(self, address=None, message=None, version=5):
address = address or ('127.0.0.1', self.port)
host, port = address
client, sockaddr = inet.create_address(host, port, socket.SOCK_DGRAM)
if version == 5:
file_path = disk.get_sibling(__file__, "v5-data.txt")
msg = open(file_path, "rb").read()
message = message or msg
client.sendto(message, sockaddr)
event = gevent.with_timeout(5,
self.normalizer.recv,
timeout_value=None)
mid = event.pop('mid')
assert re.match(
r'^LogInspect500\|netflow\|(127.0.0.1|::1)\|\d+\|1$', mid)
#'mid': u'LogInspect500|netflow|192.168.2.0/24|1353399814|1',
eq_(
event,
dict(
#msg=message.rstrip('\n'),
destination_address='10.0.0.3',
protocol_name='UDP',
_p__raw_msg_b=
'CgAAAgoAAAMAAAAAAAMABQAAAAEAAABAAbw5vQG9JB0QkgBQAAARAQACAAMgHwAA\n',
version=5,
msg='',
source_address='10.0.0.2',
current_unix_sec=1026403152,
bytes_count=64,
end_uptime_ms=29172765,
types_of_service=1,
destination_port=80,
interface_index=3,
start_uptime_ms=29112765,
device_name='localhost',
packet_count=1,
col_type='netflow',
source_port=4242,
device_ip=address[0],
collected_at='LogInspect500',
_type_num=
'interface_index start_uptime_ms end_uptime_ms source_port destination_port packet_count bytes_count types_of_service version current_unix_sec',
_type_str=
'protocol_name source_address destination_address msg col_type device_name collected_at',
_type_ip='source_address destination_address device_ip',
))
elif version == 9:
file_path = disk.get_sibling(__file__, "v9-data.txt")
msg = open(file_path, "rb").read()
message = message or msg
client.sendto(message, sockaddr)
event = gevent.with_timeout(5,
self.normalizer.recv,
timeout_value=None)
mid = event.pop('mid')
assert re.match(
r'^LogInspect500\|netflow\|(127.0.0.1|::1)\|\d+\|1$', mid)
#'mid': u'LogInspect500|netflow|192.168.2.0/24|1353399814|1',
expected = {
'_type_str':
'packet_type msg col_type device_name collected_at',
'_type_num': 'template_id',
'template_id': 300,
'device_ip': '127.0.0.1',
'device_name': u'localhost',
'packet_type': 'template',
'col_type': u'netflow',
'collected_at': u'LogInspect500',
'msg': '',
'_p___raw_msg_b':
'ASwAEgAIAAQADAAEAA8ABAAKAAQADgAEAAIABAABAAQABwACAAsAAgAGAAEABAABAAUAAQARAAIAEAACAAkAAQANAAEAFQAEABYABA==\n',
'_type_ip': 'device_ip'
}
eq_(event, expected)
#eq_(event, dict(
# protocol=17,
# first_switched=29074919,
# unix_secs=0,
# sys_uptime_ms=29134919,
# package_sequence=111,
# destination_address='10.0.0.3',
# protocol_name='UDP',
# _p__raw_msg_b='CgAAAgoAAAMAAAAAAAAAAwAAAAUAAAABAAAAQBCSAFAAEQEAAwACIB8BvJBHAbul5wAAAA==\n',
# version=9,
# msg='',
# source_address='10.0.0.2',
# bytes_count=64,
# destination_mask=31,
# source_mask=32,
# next_hop='0.0.0.0',
# source_as=2,
# output_interface_index=5,
# source_id=0,
# last_switched=29134919,
# tcp_flag=0,
# destination_as=3,
# types_of_service=1,
# destination_port=80,
# input_interface_index=3,
# device_name='localhost',
# packet_count=1,
# col_type='netflow',
# source_port=4242,
# device_ip=address[0],
# collected_at='LogInspect500',
# _type_num='bytes_count packet_count protocol types_of_service tcp_flag source_port source_mask input_interface_index destination_port destination_mask output_interface_index source_as destination_as last_switched first_switched version sys_uptime_ms unix_secs package_sequence source_id',
# _type_str='protocol_name source_address destination_address next_hop msg col_type device_name collected_at',
# _type_ip='source_address destination_address next_hop device_ip',
# ))
else:
raise ValueError("Unknown netflow version type: %r" % version)
from pysnmp.entity.rfc3413.oneliner import cmdgen
from pysnmp.proto.rfc1155 import ObjectName
from pysnmp.proto.rfc1157 import univ
from pylib import disk
import re
import sys
sys.path.append(disk.get_sibling(__file__, '../../../snmp_fetcher/'))
from lib.fetcherloop import _get_mib_viewer
from lib.fetcherloop import process_snmp_fetcher_data
from lib.fetcherloop import _handle_data
import unittest
class MockColOut:
def send_with_norm_policy_and_repo(self, event):
pass
def start_benchmarker_processing(self):
pass
class TestSnmpFetcher(unittest.TestCase):
def setUp(self):
self.cmdGen = cmdgen.CommandGenerator()
self.mibView = _get_mib_viewer(self.cmdGen)
self.snmp_fetcher_out = MockColOut()
self.snmp_data = {
'cmdGen': None,
import sys
from pylib import disk
sys.path.append(disk.get_sibling(__file__, '../../../apps/scp_fetcher/lib'))
import scp
def test(ip, port, username, password, remotepath):
scp.setup(ip, port, username, password)
if remotepath.startswith('~'):
remotepath = '.' + remotepath[1:]
for filename, mtime in scp.fetch_file_mtime(remotepath):
break
if __name__ == '__main__':
test("192.168.2.205", 22, "sujan", "mypassword", ".profile")
import logging
import logging.handlers
import os
import time
from pylib import disk
LOG_DIR = disk.get_sibling(__file__, 'logs')
if not os.path.exists(LOG_DIR):
os.mkdir(LOG_DIR)
LOG_FILENAME = os.path.join(LOG_DIR, 'rotatingfile.out')
log = logging.getLogger(__name__)
log.setLevel(logging.DEBUG)
handler = logging.handlers.RotatingFileHandler(LOG_FILENAME,
maxBytes=100,
backupCount=5)
log.addHandler(handler)
for i in xrange(int(1e9)):
time.sleep(1)
log.debug('i = %d' % i)
#break
def setUp(self):
config_path = disk.get_sibling(__file__, "test-config.json")
self.config = conf.load(config_path)
self.port = self.config["port"]
self.ssl_port = self.config["ssl_port"]
self.collected_at = self.config["loginspect_name"]