def test_parse_valid_textfsm(self): test_dir = here+'/input/31_textfsm_parser' pm = parser_manager.ParserManager( parser_dirs=[test_dir + "/parsers"], default_parser_dir=False ) ## Read XML content xml_data = open( test_dir + "/rpc-reply/show_system_processes_extensive/command_short.xml").read() ## Return a list dict data = list(pm.parse( input="show-system-processes-extensive.parser.yaml", data=xml_data.encode())) # pp.pprint(data) expected_dict_1 = { 'fields': {'cpu': '0.59', 'memory': '112000000'}, 'measurement': 'jnpr_system_process', 'tags': {'process': 'authd'} } expected_dict_2 = { 'fields': {'cpu': '0.00', 'memory': '55532000'}, 'measurement': 'jnpr_system_process', 'tags': {'process': 'pfed'} } expected_dict_3 = { 'fields': {'cpu': '0.00', 'memory': '98872000'}, 'measurement': 'jnpr_system_process', 'tags': {'process': 'jdhcpd'} } self.assertDictEqual( expected_dict_1, data[0] ) self.assertDictEqual( expected_dict_2, data[1] ) self.assertDictEqual( expected_dict_3, data[2] ) self.assertTrue( len(data) == 4 )
def test_find_parser(self): test_dir = here + '/input/05_find_parsers/parsers' pm = parser_manager.ParserManager(parser_dir=test_dir) by_name = pm.get_parser_name_for( input=test_dir + '/type-regex-regex-command.parser.yaml') # by_name_2 = pm.get_parser_name_for(input='type-regex-regex-command.parser') regex_by_command = pm.get_parser_name_for( input='show system processes extensive') xml_by_regex = pm.get_parser_name_for(input='show ospf summary') xml_by_regex_2 = pm.get_parser_name_for( input='show ospf summary | display xml') xml_by_command = pm.get_parser_name_for(input='show bgp summary') xml_by_command_2 = pm.get_parser_name_for( input='show bgp summary | display xml') assert (pm.get_nbr_parsers() == 3) assert (by_name == test_dir + '/type-regex-regex-command.parser.yaml') # assert( by_name_2 == test_dir + '/type-regex-regex-command.parser.yaml' ) assert (regex_by_command == test_dir + '/type-regex-regex-command.parser.yaml') assert (xml_by_regex == test_dir + '/type-xml-regex-command.parser.yaml') assert (xml_by_regex_2 == test_dir + '/type-xml-regex-command.parser.yaml') assert (xml_by_command == test_dir + '/type-xml-command.parser.yaml') assert (xml_by_command_2 == test_dir + '/type-xml-command.parser.yaml')
def test_parser_show_interfaces_extensive(self): test_dir = here + '/input/41_validate_parsers' pm = parser_manager.ParserManager(parser_dir=parsers_dir) ## Read XML content xml_data = open(test_dir + "/rpc-reply/show_interfaces_extensive_01.xml").read() ## Return a list dict ## NOTE: input arg needs to include full path to the parser as line 109 of ## lib/metric_collector/parser_manager.py doesnt do a substring match data = pm.parse( input= "/source/parsers/juniper/show-interfaces-extensive.parser.yaml", data=xml_data.encode()) # pprint.pprint(data) ## Read XML content xml_data = open(test_dir + "/rpc-reply/show_interfaces_extensive_02.xml").read() ## Return a list dict data = pm.parse( input= "/source/parsers/juniper/show-interfaces-extensive.parser.yaml", data=xml_data.encode()) # pprint.pprint(data) self.assertTrue(True)
def test_parse_valid_xml_enum(self): test_dir = here+'/input/21_xml_enum_parser' pm = parser_manager.ParserManager( parser_dirs=[test_dir + "/parsers"], default_parser_dir=False ) ## Read XML content xml_data = open( test_dir + "/rpc-reply/show_bgp_neighbor/command.xml").read() xml_etree = etree.fromstring(xml_data) expected_dict_0 = { 'fields': {'flap-count': '13', 'status': 0}, 'measurement': None, 'tags': {'peer-id': '10.20.250.251', 'peer-as': '65003', 'peer-state': 'Established'}, } expected_dict_1 = { 'fields': {'flap-count': '4', 'status': 2}, 'measurement': None, 'tags': {'peer-id': '10.20.250.252', 'peer-as': '65001', 'peer-state': 'Idle'} } data = list(pm.parse( input="show-bgp-neighbor.parser.yaml", data=xml_etree)) self.assertDictEqual( expected_dict_0, data[0] ) self.assertDictEqual( expected_dict_1, data[1] ) self.assertTrue( len(data) == 2 )
def __init__(self, creds_conf, cmds_conf, parsers_dir, output_type, output_addr, max_worker_threads=1, use_threads=True, num_threads_per_worker=10): self.workers = {} self.working = set() self.host_mgr = host_manager.HostManager(credentials=creds_conf, commands=cmds_conf) self.parser_mgr = parser_manager.ParserManager(parser_dirs=parsers_dir) self.collector = collector.Collector(self.host_mgr, self.parser_mgr, output_type, output_addr) self.max_worker_threads = max_worker_threads self.output_type = output_type self.output_addr = output_addr self.use_threads = use_threads self.num_threads_per_worker = num_threads_per_worker # default worker that is started if there are no hosts to schedule self.default_worker = Worker(120, self.collector, self.output_type, self.output_addr, self.use_threads, self.num_threads_per_worker) self.default_worker.set_name('Default-120sec')
def test_parse_valid_xml(self): test_dir = here+'/input/20_xml_parser' pm = parser_manager.ParserManager( parser_dirs=[test_dir + "/parsers"], default_parser_dir=False ) ## Read XML content xml_data = open( test_dir + "/rpc-reply/show_route_summary/command.xml").read() xml_etree = etree.fromstring(xml_data) expected_dict_0 = { 'fields': { 'active-route-count': '16', 'destination-count': '16', 'hidden-route-count': '0', 'holddown-route-count': '0', 'total-route-count': '21'}, 'measurement': None, 'tags': { 'key': 'inet.0'} } expected_dict_1 = { 'fields': { 'active-route-count': '2', 'destination-count': '2', 'hidden-route-count': '0', 'holddown-route-count': '0', 'total-route-count': '2'}, 'measurement': None, 'tags': { 'key': 'inet6.0'} } data = list(pm.parse( input="show-route-summary.parser.yaml", data=xml_etree)) self.assertDictEqual( expected_dict_0, data[0] ) self.assertDictEqual( expected_dict_1, data[1] ) self.assertTrue( len(data) == 2 )
def test_parse_valid_json(self): test_dir = here+'/input/51_json_parser' pm = parser_manager.ParserManager(parser_dirs=[test_dir + "/parsers"], default_parser_dir=False) # Read JSON Content with open(test_dir + "/json-reply/f5-pools.json") as f: data = f.read() json_data = json.loads(data) data = pm.parse(input="f5-pools.yaml", data=json_data) expected_dict_1 = {'fields': {'active_member_count': 1, 'bits_in': 0, 'bits_out': 0, 'current_conns': 0, 'current_sessions': 0, 'max_conns': 0, 'min_active_members': 0, 'packets_in': 0, 'packets_out': 0, 'total_conns': 0, 'total_requests': 0}, 'measurement': None, 'tags': {'partition_poolname': '/Common/RA-WEB241-443'}} expected_dict_2 = {'fields': {'active_member_count': 1, 'bits_in': 0, 'bits_out': 0, 'current_conns': 0, 'current_sessions': 0, 'max_conns': 0, 'min_active_members': 0, 'packets_in': 0, 'packets_out': 0, 'total_conns': 0, 'total_requests': 0}, 'measurement': None, 'tags': {'partition_poolname': '/Common/matte_xff_test_pool'}} expected_dict_3 = {'fields': {'active_member_count': 1, 'bits_in': 1085496, 'bits_out': 172672, 'current_conns': 0, 'current_sessions': 0, 'max_conns': 2, 'min_active_members': 0, 'packets_in': 582, 'packets_out': 527, 'total_conns': 47, 'total_requests': 442}, 'measurement': None, 'tags': {'partition_poolname': '/Common/rkeller_syslog_pool'}} self.assertDictEqual( expected_dict_1, data[0] ) self.assertDictEqual( expected_dict_2, data[1] ) self.assertDictEqual( expected_dict_3, data[2] ) self.assertTrue( len(data) == 6 )
def test_load_valid_regex_parser(self): test_dir = here+'/input/04_valid_regex_parser/parsers' pm = parser_manager.ParserManager( parser_dirs=[test_dir], default_parser_dir=False ) name = pm.get_parser_name_for(input='show-system-processes-extensive.parser.yaml') self.assertTrue( pm.get_nbr_parsers() == 1 ) self.assertTrue( pm.nbr_regex_parsers == 1 ) self.assertTrue( name == "show-system-processes-extensive.parser.yaml" )
def test_load_valid_xml_parser(self): test_dir = here+'/input/03_valid_xml_parser/parsers' pm = parser_manager.ParserManager( parser_dirs=[test_dir], default_parser_dir=False ) name = pm.get_parser_name_for(input='show-bgp-summary.parser.yaml') self.assertTrue( pm.get_nbr_parsers() == 1 ) self.assertTrue( pm.nbr_xml_parsers == 1 ) self.assertTrue( name == "show-bgp-summary.parser.yaml" )
def test_parser_f5_pools_json(self): test_dir = here + '/input/41_validate_parsers' pm = parser_manager.ParserManager() # Read JSON Content with open(test_dir + "/json-reply/f5-pools.json") as f: data = f.read() json_data = json.loads(data) data = pm.parse(input="f5-pools.yaml", data=json_data) self.assertTrue(True)
def test_parser_show_interfaces_extensive(self): test_dir = here + '/input/41_validate_parsers' pm = parser_manager.ParserManager() ## Read XML content xml_data = open(test_dir + "/rpc-reply/show_interfaces_extensive_01.xml").read() ## Return a list dict data = pm.parse(input="show-interfaces-extensive.parser.yaml", data=xml_data.encode()) # pprint.pprint(data) self.assertTrue(True)
def test_no_parser_key(self): test_dir = here + '/input/02_no_parser_key/parsers' pm = parser_manager.ParserManager(parser_dir=test_dir) self.assertTrue(pm.get_nbr_parsers() == 0)
def test_invalid_yaml(self): test_dir = here + '/input/01_wrong_yaml/parsers' pm = parser_manager.ParserManager(parser_dir=test_dir) self.assertTrue(pm.get_nbr_parsers() == 0)
def main(): time_start = time.time() ### ------------------------------------------------------------------------------ ### Create and Parse Arguments ### ----------------------------------------------------------------------------- # if getattr(sys, 'frozen', False): # # frozen # BASE_DIR = os.path.dirname(sys.executable) # else: # # unfrozen # BASE_DIR = os.path.dirname(os.path.realpath(__file__)) BASE_DIR = os.getcwd() full_parser = argparse.ArgumentParser() full_parser.add_argument( "--tag", nargs='+', help="Collect data from hosts that matches the tag") full_parser.add_argument( "--cmd-tag", nargs='+', help="Collect data from command that matches the tag") full_parser.add_argument("-s", "--start", action='store_true', help="Start collecting (default 'no')") full_parser.add_argument("--loglvl", default=20, help="Logs verbosity, 10-debug, 50 Critical") full_parser.add_argument("--logdir", default="", help="Directory where to store logs") full_parser.add_argument( "--sharding", help= "Define if the script is part of a shard need to include the place in the shard and the size of the shard [0/3]" ) full_parser.add_argument( "--sharding-offset", default=True, help="Define an offset needs to be applied to the shard_id") full_parser.add_argument("--parserdir", default="parsers", help="Directory where to find parsers") full_parser.add_argument( "--collector-timeout", default=15, help="Timeout for collector device rpc/rest calls") full_parser.add_argument("--retry", default=5, help="Max retry") full_parser.add_argument("--host", default=None, help="Host DNS or IP") full_parser.add_argument("--hosts", default="hosts.yaml", help="Hosts file in yaml") full_parser.add_argument("--commands", default="commands.yaml", help="Commands file in Yaml") full_parser.add_argument("--credentials", default="credentials.yaml", help="Credentials file in Yaml") full_parser.add_argument( "--no-facts", action='store_false', help= "Disable facts collection on device (remove version and product name in results)" ) full_parser.add_argument("--output-format", default="influxdb", help="Format of the output") full_parser.add_argument("--output-type", default="stdout", choices=['stdout', 'http'], help="Type of output") full_parser.add_argument("--output-addr", default="http://localhost:8186/write", help="Addr information for output action") full_parser.add_argument( "--no-collector-threads", action='store_true', help= "Dont Spawn multiple threads to collect the information on the devices" ) full_parser.add_argument( "--nbr-collector-threads", type=int, default=10, help="Maximum number of collector thread to spawn (default 10)") full_parser.add_argument( "--max-worker-threads", type=int, default=1, help="Maximum number of worker threads per interval for scheduler") full_parser.add_argument("--use-scheduler", action='store_true', help="Use scheduler") full_parser.add_argument( "--hosts-refresh-interval", type=int, default=3 * 60 * 60, help="Interval to periodically refresh dynamic host inventory") full_parser.add_argument("--allow-zero-hosts", action='store_true', help="Allow scheduler to run even with 0 hosts") dynamic_args = vars(full_parser.parse_args()) # Print help if no parameters are provided if len(sys.argv) == 1: full_parser.print_help() sys.exit(1) ### ------------------------------------------------------------------------------ # Loading YAML Default Variables ### ------------------------------------------------------------------------------ max_connection_retries = dynamic_args['retry'] logging_level = int(dynamic_args['loglvl']) ### ------------------------------------------------------------------------------ ### Validate Arguments ### ------------------------------------------------------------------------------ pp = pprint.PrettyPrinter(indent=4) tag_list = [] ### Known and fixed arguments if dynamic_args['tag']: tag_list = dynamic_args['tag'] else: tag_list = [".*"] if not (dynamic_args['start']): print('Missing <start> option, so nothing to do') sys.exit(0) ### ------------------------------------------------------------------------------ ### Logging ### ------------------------------------------------------------------------------ formatter = logging.Formatter( '%(asctime)s %(name)s: %(levelname)s: %(message)s') sh = logging.StreamHandler() sh.setFormatter(formatter) handlers = [sh] if dynamic_args['logdir']: log_dir = BASE_DIR + "/" + dynamic_args['logdir'] ## Check that logs directory exist, create it if needed if not os.path.exists(log_dir): os.makedirs(log_dir) filename = log_dir + "/" + 'metric_collector.log', fh = logging.handlers.RotatingFileHandler(filename, maxSize=10 * 1024 * 1024, backupCount=5) fh.setFormatter(formatter) handlers.append(fh) logging.basicConfig(level=logging_level, handlers=handlers) ### ------------------------------------------------------------------------------ ### LOAD all credentials in a dict ### ------------------------------------------------------------------------------ credentials = {} credentials_yaml_file = '' if os.path.isfile(dynamic_args['credentials']): credentials_yaml_file = dynamic_args['credentials'] else: credentials_yaml_file = BASE_DIR + "/" + dynamic_args['credentials'] logger.info('Importing credentials file: %s ', credentials_yaml_file) try: with open(credentials_yaml_file) as f: credentials = yaml.full_load(f) except Exception as e: logger.error('Error importing credentials file: %s: %s', credentials_yaml_file, str(e)) sys.exit(0) ### ------------------------------------------------------------------------------ ### LOAD all commands with their tags in a dict ### ------------------------------------------------------------------------------ commands_yaml_file = '' commands = [] if os.path.isfile(dynamic_args['commands']): commands_yaml_file = dynamic_args['commands'] else: commands_yaml_file = BASE_DIR + "/" + dynamic_args['commands'] logger.info('Importing commands file: %s ', commands_yaml_file) with open(commands_yaml_file) as f: try: for document in yaml.load_all(f, yaml.FullLoader): commands.append(document) except Exception as e: logger.error('Error importing commands file: %s, %s', commands_yaml_file, str(e)) sys.exit(0) general_commands = commands[0] use_threads = not (dynamic_args['no_collector_threads']) if dynamic_args['cmd_tag']: command_tags = dynamic_args['cmd_tag'] else: command_tags = ['.*'] sharding = dynamic_args.get('sharding') sharding_offset = dynamic_args.get('sharding_offset') max_worker_threads = dynamic_args.get('max_worker_threads', 1) max_collector_threads = dynamic_args.get('nbr_collector_threads') if dynamic_args.get('use_scheduler', False): device_scheduler = scheduler.Scheduler( credentials, general_commands, dynamic_args['parserdir'], dynamic_args['output_type'], dynamic_args['output_addr'], max_worker_threads=max_worker_threads, use_threads=use_threads, num_threads_per_worker=max_collector_threads, collector_timeout=dynamic_args['collector_timeout']) hri = dynamic_args.get('hosts_refresh_interval', 6 * 60 * 60) select_hosts( dynamic_args['hosts'], tag_list, sharding, sharding_offset, scheduler=device_scheduler, refresh_interval=float(hri), allow_zero_hosts=dynamic_args.get('allow_zero_hosts', False), ) device_scheduler.start() # blocking call return ### ------------------------------------------------------------------------------ ### LOAD all parsers ### ------------------------------------------------------------------------------ parsers_manager = parser_manager.ParserManager( parser_dirs=dynamic_args['parserdir']) hosts_conf = select_hosts(dynamic_args['hosts'], tag_list, sharding, sharding_offset) hosts_manager = host_manager.HostManager(credentials=credentials, commands=general_commands) hosts_manager.update_hosts(hosts_conf) coll = collector.Collector(hosts_manager=hosts_manager, parser_manager=parsers_manager, output_type=dynamic_args['output_type'], output_addr=dynamic_args['output_addr'], collect_facts=dynamic_args.get( 'no_facts', True), timeout=dynamic_args['collector_timeout']) target_hosts = hosts_manager.get_target_hosts(tags=tag_list) if use_threads: target_hosts_lists = [ target_hosts[x:x + int(len(target_hosts) / max_collector_threads + 1)] for x in range(0, len(target_hosts), int(len(target_hosts) / max_collector_threads + 1)) ] jobs = [] for (i, target_hosts_list) in enumerate(target_hosts_lists, 1): logger.info( 'Collector Thread-%s scheduled with following hosts: %s', i, target_hosts_list) thread = threading.Thread(target=coll.collect, args=('global', ), kwargs={ "hosts": target_hosts_list, "cmd_tags": command_tags }) jobs.append(thread) i = i + 1 # Start the threads for j in jobs: j.start() # Ensure all of the threads have finished for j in jobs: j.join() else: # Execute everythings in the main thread coll.collect('global', hosts=target_hosts, cmd_tags=command_tags) ### ----------------------------------------------------- ### Collect Global Statistics ### ----------------------------------------------------- time_end = time.time() time_execution = time_end - time_start global_datapoint = [{ 'measurement': global_measurement_prefix + '_stats_agent', 'tags': {}, 'fields': { 'execution_time_sec': "%.4f" % time_execution, 'nbr_devices': len(target_hosts) }, 'timestamp': time.time_ns(), }] if 'sharding' in dynamic_args and dynamic_args['sharding'] != None: global_datapoint[0]['tags']['sharding'] = dynamic_args['sharding'] if use_threads: global_datapoint[0]['fields']['nbr_threads'] = dynamic_args[ 'nbr_collector_threads'] ### Send results to the right output try: if dynamic_args['output_type'] == 'stdout': utils.print_format_influxdb(global_datapoint) elif dynamic_args['output_type'] == 'http': utils.post_format_influxdb( global_datapoint, dynamic_args['output_addr'], ) else: logger.warn('Output format unknown: %s', dynamic_args['output_type']) except Exception as ex: logger.warn("Hit error trying to post to influx: ", str(ex))
def test_cleanup_tag(self): pm = parser_manager.ParserManager() self.assertEqual(pm.cleanup_tag('my tag'), 'my_tag') self.assertEqual(pm.cleanup_tag('my tag=true'), 'my_tag_true')
def test_str_2_int(self): pm = parser_manager.ParserManager() self.assertEqual(pm.str_2_int('100Gbps'), 100000000000) self.assertEqual(pm.str_2_int('100gbps'), 100000000000) self.assertEqual(pm.str_2_int('100G'), 100000000000)
def test_str_2_int_wrong_input(self): pm = parser_manager.ParserManager() self.assertEqual(pm.str_2_int('notanint'), None) self.assertEqual(pm.str_2_int('Undefined'), None)