def test_hypervisor_no_detect(self): switch = L3Switch() switch._add_switch_port(24, 'ethernet/1/24') switch._add_server_port('ethernet/1/24', '0025904EB5A4') switch._add_server_port('ethernet/1/24', '0025904EB5A5') switch._add_server_port('ethernet/1/24', 'CEA9ACD2080C') switch._add_server_port('ethernet/1/24', 'CEA9ACD2081C') switch._add_server_port('ethernet/1/24', 'CEA9ACD2082C') switch._add_server_port('ethernet/1/24', 'CEA9ACD2083C') switch._add_server_port('ethernet/1/24', 'CEA9ACD2084C') cmdb_importer = GenericCmdbImporter() sw = Switch.objects.create(label="switch") cmdb_importer.import_switch(sw.id, switch) self.assertEqual(1, len(SwitchPort.active.filter())) self.assertEqual(1, len(SwitchPort.active.filter(status=Resource.STATUS_INUSE))) self.assertEqual(2, len(Server.active.filter())) self.assertEqual(5, len(VirtualServer.active.filter())) self.assertEqual(5, len(VirtualServer.active.filter(parent=None))) self.assertEqual(2, len(ServerPort.active.filter())) self.assertEqual(2, len(ServerPort.active.filter(status=Resource.STATUS_INUSE))) self.assertEqual(0, len(ServerPort.active.filter(parent=None))) self.assertEqual(5, len(VirtualServerPort.active.filter())) self.assertEqual(5, len(VirtualServerPort.active.filter(status=Resource.STATUS_INUSE))) self.assertEqual(0, len(VirtualServerPort.active.filter(parent=None))) self.assertEqual(7, len(PortConnection.active.filter()))
def test_process_servers(self): new_server1 = Server.objects.create(label="test server") new_server2 = Server.objects.create(label="test server") cmdb_importer = GenericCmdbImporter() link_unresolved_to_container, created = RegionResource.objects.get_or_create(name='Unresolved servers') self.assertTrue(created) link_unresolved_to_container1, created = RegionResource.objects.get_or_create(name='Unresolved servers') self.assertFalse(created) cmdb_importer.process_servers(link_unresolved_to=link_unresolved_to_container) new_server1.refresh_from_db() new_server2.refresh_from_db() self.assertEqual(link_unresolved_to_container, new_server1.parent) self.assertEqual(link_unresolved_to_container, new_server2.parent)
def test_relink_ip(self): new_server1 = Server.objects.create(label="test server") new_server2 = Server.objects.create(label="test server") pool = IPAddressPoolFactory.from_name('test') cmdb_importer = GenericCmdbImporter() ip1 = IPAddressGeneric.objects.create(address='46.17.40.2', status=Resource.STATUS_LOCKED, pool=pool, parent=new_server1) self.assertEqual(Resource.STATUS_LOCKED, ip1.status) self.assertEqual(new_server1.id, ip1.parent.id) # used IP relink between parents cmdb_importer._add_ip(ip1.address, parent=new_server2) ip1.refresh_from_db() self.assertEqual(Resource.STATUS_INUSE, ip1.status) self.assertEqual(new_server2.id, ip1.parent.id)
def test_hypervisor_detection(self): switch = L3Switch() switch._add_switch_port(24, 'ethernet/1/24') switch._add_server_port('ethernet/1/24', '0025904EB5A4') switch._add_server_port('ethernet/1/24', '0025904EB5A5') switch._add_server_port('ethernet/1/24', 'CEA9ACD2080C') switch._add_server_port('ethernet/1/24', 'CEA9ACD2081C') switch._add_server_port('ethernet/1/24', 'CEA9ACD2082C') switch._add_server_port('ethernet/1/24', 'CEA9ACD2083C') switch._add_server_port('ethernet/1/24', 'CEA9ACD2084C') # manually identify hypervisor hv_server = Server.objects.create(label="hvisor", role='hypervisor') ServerPort.objects.create(mac='0025904EB5A4', parent=hv_server) cmdb_importer = GenericCmdbImporter() sw = Switch.objects.create(label="switch") cmdb_importer.import_switch(sw.id, switch) for sw_port in sw: cmdb_importer.process_hypervisors(sw_port) self.assertEqual(2, len(Server.active.filter())) self.assertEqual(5, len(VirtualServer.active.filter())) self.assertEqual(5, len(VirtualServer.active.filter(parent=hv_server))) self.assertEqual(0, len(VirtualServer.active.filter(parent=None))) self.assertEqual(2, len(ServerPort.active.filter())) self.assertEqual(0, len(ServerPort.active.filter(parent=None))) self.assertEqual(5, len(VirtualServerPort.active.filter())) self.assertEqual(0, len(VirtualServerPort.active.filter(parent=None))) self.assertEqual(7, len(PortConnection.active.filter()))
class Command(BaseCommand): cmdb_importer = GenericCmdbImporter() registered_handlers = {} registered_providers = { 'generic': L3Switch, '3com.2952': HP1910Switch, '3com.2250': Switch3Com2250, 'dlink.dsg3200': DSG3200Switch, 'hp.1910': HP1910Switch, 'qtech': QtechL3Switch, 'qtech.3400': Qtech3400Switch } def add_arguments(self, parser): subparsers = parser.add_subparsers(title="CMDB data importer", help="Commands help", dest='manager_name', parser_class=ArgumentParser) # IP address commands file_cmd_parser = subparsers.add_parser('fromfile', help='Import data from dump files') file_cmd_parser.add_argument('device-id', help="Resource ID of the device used to take the dump.") file_cmd_parser.add_argument('provider', choices=self.registered_providers.keys(), help="Type of the device (or dump file format).") file_types_group = file_cmd_parser.add_mutually_exclusive_group() file_types_group.add_argument('--arpdump', help="Path to the ARP dump.") file_types_group.add_argument('--macdump', help="Path to the MAC dump.") self._register_handler('fromfile', self._handle_file_dumps) snmp_cmd_parser = subparsers.add_parser('snmp', help='Import data from SNMP') snmp_cmd_parser.add_argument('device-id', help="Resource ID of the device used to take the dump.") snmp_cmd_parser.add_argument('provider', choices=self.registered_providers.keys(), help="Type of the device (or dump file format).") snmp_cmd_parser.add_argument('hostname', help="Hostname or IP address.") snmp_cmd_parser.add_argument('community', help="SNMP community string.") self._register_handler('snmp', self._handle_snmp) auto_cmd_parser = subparsers.add_parser('auto', help='Import and update CMDB data based on resources.') auto_cmd_parser.add_argument('--switch-id', help="ID of the switch to get SNMP data from.") auto_cmd_parser.add_argument('--skip-arp', action="store_true", help="Skip ARP analysis.") self._register_handler('auto', self._handle_auto) household_cmd_parser = subparsers.add_parser('household', help='Cleanup unused resources.') self._register_handler('household', self._handle_household) def _handle_household(self, *args, **options): last_seen_31days = timezone.now() - datetime.timedelta(days=31) last_seen_15days = timezone.now() - datetime.timedelta(days=15) # Clean IP with parent=ip pool (free) with last_seen older that 31 days. It means that IP is not # used and can be released. logger.info("Clean missing IP addresses: %s" % last_seen_31days) for free_ip_pool in Resource.active.filter(status=Resource.STATUS_FREE, type__in=IPAddressPool.ip_pool_types): logger.info(" pool %s" % free_ip_pool) for ip in IPAddress.active.filter( status=Resource.STATUS_INUSE, last_seen__lt=last_seen_31days, ipman_pool_id=free_ip_pool.id, version=4): logger.warning(" used ip %s from the FREE IP pool is not seen for 31 days. Free it." % ip) ip.free(cascade=True) for ip in IPAddress.active.filter( status=Resource.STATUS_LOCKED, last_seen__lt=last_seen_15days, ipman_pool_id=free_ip_pool.id, version=4): logger.warning(" locked ip %s from the FREE IP pool is not seen for 15 days. Free it." % ip) ip.free(cascade=True) logger.info("Clean missing virtual servers: %s" % last_seen_31days) for vm in VirtualServer.active.filter(last_seen__lt=last_seen_31days): logger.warning(" server %s not seen for 31 days. Removing..." % vm) for vm_child in vm: logger.info(" remove %s" % vm_child) vm_child.delete() vm.delete() logger.info("Clean unresolved PortConnections...") removed = 0 for connection in PortConnection.active.all(): if not connection.linked_port: connection.delete() removed += 1 logger.info(" removed: %s" % removed) Resource.objects.rebuild() def _handle_auto(self, *args, **options): # update via snmp query = dict(type__in=[GatewaySwitch.__name__, Switch.__name__]) if options['switch_id']: query['pk'] = options['switch_id'] if not options['skip_arp']: for switch in Resource.active.filter(**query): logger.info("* Found switch: %s" % switch) if switch.has_option('snmp_provider_key'): snmp_provider_key = switch.get_option_value('snmp_provider_key') if snmp_provider_key in self.registered_providers: hostname = switch.get_option_value('snmp_host') community = switch.get_option_value('snmp_community') logger.info("host: %s" % hostname) provider = self.registered_providers[snmp_provider_key]() provider.from_snmp(hostname, community) self.cmdb_importer.import_switch(switch.id, provider) else: logger.warning("Unknown SNMP data provider: %s" % snmp_provider_key) Resource.objects.rebuild() logger.info("Process hypervisors.") for switch in Switch.active.all(): for switch_port in SwitchPort.active.filter(parent=switch): self.cmdb_importer.process_hypervisors(switch_port) for switch in GatewaySwitch.active.all(): for switch_port in SwitchPort.active.filter(parent=switch): self.cmdb_importer.process_hypervisors(switch_port) logger.info("Process server mounts") link_unresolved_to_container, created = RegionResource.objects.get_or_create(name='Unresolved servers') self.cmdb_importer.process_servers(link_unresolved_to=link_unresolved_to_container) logger.info("Process virtual server mounts") link_unresolved_to_container, created = RegionResource.objects.get_or_create(name='Unresolved VPS') self.cmdb_importer.process_virtual_servers(link_unresolved_to=link_unresolved_to_container) Resource.objects.rebuild() def _handle_snmp(self, *args, **options): device_id = options['device-id'] provider_key = options['provider'] hostname = options['hostname'] community = options['community'] source_switch = Resource.active.get(pk=device_id) provider = self.registered_providers[provider_key]() provider.from_snmp(hostname, community) self.cmdb_importer.import_switch(device_id, provider) source_switch.set_option('snmp_host', hostname) source_switch.set_option('snmp_community', community) source_switch.set_option('snmp_provider_key', provider_key) def _handle_file_dumps(self, *args, **options): device_id = options['device-id'] provider_key = options['provider'] provider = self.registered_providers[provider_key]() if options['arpdump']: provider.from_arp_dump(options['arpdump']) elif options['macdump']: provider.from_mac_dump(options['macdump']) else: raise Exception("Specify one of the dump files.") self.cmdb_importer.import_switch(device_id, provider) def handle(self, *args, **options): if 'subcommand_name' in options: subcommand = "%s.%s" % (options['manager_name'], options['subcommand_name']) else: subcommand = options['manager_name'] # call handler self.registered_handlers[subcommand](*args, **options) def _register_handler(self, command_name, handler): assert command_name, "command_name must be defined." assert handler, "handler must be defined." self.registered_handlers[command_name] = handler
def test_import_data(self): arp_file_path = os.path.join(self.DATA_DIR, 'arp-table.txt') mac_file_path = os.path.join(self.DATA_DIR, 'mac-table.txt') cmdb_importer = GenericCmdbImporter() # create IP pools and basic structure dc_anders = RegionResource.objects.create(name="Anders") dc_rtcom = RegionResource.objects.create(name="Rostelecom") # create source switch anders_gw = GatewaySwitch.objects.create(name="baxet-gw-q", parent=dc_anders) # arp table provider qtech_switch = QtechL3Switch() qtech_switch.from_arp_dump(arp_file_path) switch_ports = list(qtech_switch.ports) self.assertEqual(12, len(switch_ports)) # Add our IP pools pool_list = [IPNetworkPool.objects.create(network='46.17.40.0/23', parent=dc_anders), IPNetworkPool.objects.create(network='46.17.44.0/23', parent=dc_anders), IPNetworkPool.objects.create(network='176.32.34.0/23', parent=dc_anders), IPNetworkPool.objects.create(network='176.32.36.0/24', parent=dc_anders), IPNetworkPool.objects.create(network='176.32.37.0/24', parent=dc_anders), IPNetworkPool.objects.create(network='176.32.38.0/24', parent=dc_anders), IPNetworkPool.objects.create(network='176.32.39.0/24', parent=dc_anders), IPNetworkPool.objects.create(network='2a00:b700::/48', parent=dc_anders), IPNetworkPool.objects.create(network='46.17.46.0/23', parent=dc_rtcom), IPNetworkPool.objects.create(network='46.29.160.0/23', parent=dc_rtcom), IPNetworkPool.objects.create(network='176.32.32.0/23', parent=dc_rtcom), IPNetworkPool.objects.create(network='46.29.162.0/23', parent=dc_rtcom), IPNetworkPool.objects.create(network='46.29.164.0/22', parent=dc_rtcom), IPNetworkPool.objects.create(network='185.22.152.0/22', parent=dc_rtcom), IPNetworkPool.objects.create(network='2a00:b700:1::/48', parent=dc_rtcom)] # Double the proccess, to test data update process cmdb_importer.import_switch(anders_gw.id, qtech_switch) cmdb_importer.import_switch(anders_gw.id, qtech_switch) # -1 IP: 87.251.133.9, /30 peering network address self.assertEqual(1328, len(IPAddress.active.filter())) for pool in pool_list: print "%s - %d" % (pool, pool.usage) # count servers self.assertEqual(71, len(Server.active.filter())) self.assertEqual(39, len(VirtualServer.active.filter())) self.assertEqual(71, len(ServerPort.active.filter())) self.assertEqual(0, len(ServerPort.active.filter(parent=None))) self.assertEqual(39, len(VirtualServerPort.active.filter())) self.assertEqual(10, len(PortConnection.active.filter())) # Servers and ports equality check self.assertEqual((len(ServerPort.active.filter()) + len(VirtualServerPort.active.filter())), (len(Server.active.filter()) + len(VirtualServer.active.filter()))) # import MAC data from mac table anders_sw1 = Switch.objects.create(name="baxet-sw-1", parent=dc_anders) qtech3400_switch = Qtech3400Switch() qtech3400_switch.from_mac_dump(mac_file_path) # double call, to check update cmdb_importer.import_switch(anders_sw1.id, qtech3400_switch) cmdb_importer.import_switch(anders_sw1.id, qtech3400_switch) # count servers self.assertEqual(76, len(Server.active.filter())) self.assertEqual(41, len(VirtualServer.active.filter())) self.assertEqual(76, len(ServerPort.active.filter())) self.assertEqual(0, len(ServerPort.active.filter(parent=None))) self.assertEqual(41, len(VirtualServerPort.active.filter())) self.assertEqual(54, len(PortConnection.active.filter())) self.assertEqual(1328, len(IPAddress.active.filter())) # update VPS links to hypervisors for switch in Switch.active.all(): for switch_port in SwitchPort.active.filter(parent=switch): cmdb_importer.process_hypervisors(switch_port) for switch in GatewaySwitch.active.all(): for switch_port in SwitchPort.active.filter(parent=switch): cmdb_importer.process_hypervisors(switch_port) self.assertEqual(3, len(Server.active.filter(role='hypervisor'))) for server in Server.active.filter(role='hypervisor'): print server.id # There are linked VPS, hypervisor detection logic test. self.assertEqual(4, len(VirtualServer.active.filter(parent=812))) self.assertEqual(2, len(VirtualServer.active.filter(parent=820))) self.assertEqual(3, len(VirtualServer.active.filter(parent=764))) events = HistoryEvent.objects.filter(type=HistoryEvent.CREATE) self.assertEqual(1677, len(events))
def test_import_data(self): arp_file_path = os.path.join(self.DATA_DIR, 'arp-table.txt') mac_file_path = os.path.join(self.DATA_DIR, 'mac-table.txt') cmdb_importer = GenericCmdbImporter() # create IP pools and basic structure dc_anders = RegionResource.objects.create(name="Anders") dc_rtcom = RegionResource.objects.create(name="Rostelecom") # create source switch anders_gw = GatewaySwitch.objects.create(name="baxet-gw-q", parent=dc_anders) # arp table provider qtech_switch = QtechL3Switch() qtech_switch.from_arp_dump(arp_file_path) switch_ports = list(qtech_switch.ports) self.assertEqual(12, len(switch_ports)) # Add our IP pools pool_list = [IPNetworkPool.objects.create(network='46.17.40.0/23', parent=dc_anders), IPNetworkPool.objects.create(network='46.17.44.0/23', parent=dc_anders), IPNetworkPool.objects.create(network='176.32.34.0/23', parent=dc_anders), IPNetworkPool.objects.create(network='176.32.36.0/24', parent=dc_anders), IPNetworkPool.objects.create(network='176.32.37.0/24', parent=dc_anders), IPNetworkPool.objects.create(network='176.32.38.0/24', parent=dc_anders), IPNetworkPool.objects.create(network='176.32.39.0/24', parent=dc_anders), IPNetworkPool.objects.create(network='2a00:b700::/48', parent=dc_anders), IPNetworkPool.objects.create(network='46.17.46.0/23', parent=dc_rtcom), IPNetworkPool.objects.create(network='46.29.160.0/23', parent=dc_rtcom), IPNetworkPool.objects.create(network='176.32.32.0/23', parent=dc_rtcom), IPNetworkPool.objects.create(network='46.29.162.0/23', parent=dc_rtcom), IPNetworkPool.objects.create(network='46.29.164.0/22', parent=dc_rtcom), IPNetworkPool.objects.create(network='185.22.152.0/22', parent=dc_rtcom), IPNetworkPool.objects.create(network='2a00:b700:1::/48', parent=dc_rtcom)] # Double the proccess, to test data update process cmdb_importer.import_switch(anders_gw.id, qtech_switch) cmdb_importer.import_switch(anders_gw.id, qtech_switch) # -1 IP: 87.251.133.9, /30 peering network address self.assertEqual(1328, len(IPAddress.active.filter())) for pool in pool_list: print "%s - %d" % (pool, pool.usage) # count servers self.assertEqual(71, len(Server.active.filter())) self.assertEqual(39, len(VirtualServer.active.filter())) self.assertEqual(71, len(ServerPort.active.filter())) self.assertEqual(0, len(ServerPort.active.filter(parent=None))) self.assertEqual(39, len(VirtualServerPort.active.filter())) self.assertEqual(10, len(PortConnection.active.filter())) # Servers and ports equality check self.assertEqual((len(ServerPort.active.filter()) + len(VirtualServerPort.active.filter())), (len(Server.active.filter()) + len(VirtualServer.active.filter()))) # import MAC data from mac table anders_sw1 = Switch.objects.create(name="baxet-sw-1", parent=dc_anders) qtech3400_switch = Qtech3400Switch() qtech3400_switch.from_mac_dump(mac_file_path) # double call, to check update cmdb_importer.import_switch(anders_sw1.id, qtech3400_switch) cmdb_importer.import_switch(anders_sw1.id, qtech3400_switch) # count servers self.assertEqual(76, len(Server.active.filter())) self.assertEqual(41, len(VirtualServer.active.filter())) self.assertEqual(76, len(ServerPort.active.filter())) self.assertEqual(0, len(ServerPort.active.filter(parent=None))) self.assertEqual(41, len(VirtualServerPort.active.filter())) self.assertEqual(54, len(PortConnection.active.filter())) self.assertEqual(1328, len(IPAddress.active.filter())) # update VPS links to hypervisors for switch in Switch.active.all(): for switch_port in SwitchPort.active.filter(parent=switch): cmdb_importer.process_hypervisors(switch_port) for switch in GatewaySwitch.active.all(): for switch_port in SwitchPort.active.filter(parent=switch): cmdb_importer.process_hypervisors(switch_port) self.assertEqual(3, len(Server.active.filter(role='hypervisor'))) for server in Server.active.filter(role='hypervisor'): print server.id # There are linked VPS, hypervisor detection logic test. self.assertEqual(4, len(VirtualServer.active.filter(parent=660))) self.assertEqual(2, len(VirtualServer.active.filter(parent=668))) self.assertEqual(3, len(VirtualServer.active.filter(parent=612))) events = HistoryEvent.objects.filter(type=HistoryEvent.CREATE) self.assertEqual(1677, len(events))