def test_hypervisor_no_detect(self):
switch = L3Switch()
switch._add_switch_port(24, 'ethernet/1/24')
switch._add_server_port('ethernet/1/24', '0025904EB5A4')
switch._add_server_port('ethernet/1/24', '0025904EB5A5')
switch._add_server_port('ethernet/1/24', 'CEA9ACD2080C')
switch._add_server_port('ethernet/1/24', 'CEA9ACD2081C')
switch._add_server_port('ethernet/1/24', 'CEA9ACD2082C')
switch._add_server_port('ethernet/1/24', 'CEA9ACD2083C')
switch._add_server_port('ethernet/1/24', 'CEA9ACD2084C')
cmdb_importer = GenericCmdbImporter()
sw = Switch.objects.create(label="switch")
cmdb_importer.import_switch(sw.id, switch)
self.assertEqual(1, len(SwitchPort.active.filter()))
self.assertEqual(1, len(SwitchPort.active.filter(status=Resource.STATUS_INUSE)))
self.assertEqual(2, len(Server.active.filter()))
self.assertEqual(5, len(VirtualServer.active.filter()))
self.assertEqual(5, len(VirtualServer.active.filter(parent=None)))
self.assertEqual(2, len(ServerPort.active.filter()))
self.assertEqual(2, len(ServerPort.active.filter(status=Resource.STATUS_INUSE)))
self.assertEqual(0, len(ServerPort.active.filter(parent=None)))
self.assertEqual(5, len(VirtualServerPort.active.filter()))
self.assertEqual(5, len(VirtualServerPort.active.filter(status=Resource.STATUS_INUSE)))
self.assertEqual(0, len(VirtualServerPort.active.filter(parent=None)))
self.assertEqual(7, len(PortConnection.active.filter()))
def test_process_servers(self):
new_server1 = Server.objects.create(label="test server")
new_server2 = Server.objects.create(label="test server")
cmdb_importer = GenericCmdbImporter()
link_unresolved_to_container, created = RegionResource.objects.get_or_create(name='Unresolved servers')
self.assertTrue(created)
link_unresolved_to_container1, created = RegionResource.objects.get_or_create(name='Unresolved servers')
self.assertFalse(created)
cmdb_importer.process_servers(link_unresolved_to=link_unresolved_to_container)
new_server1.refresh_from_db()
new_server2.refresh_from_db()
self.assertEqual(link_unresolved_to_container, new_server1.parent)
self.assertEqual(link_unresolved_to_container, new_server2.parent)
def test_relink_ip(self):
new_server1 = Server.objects.create(label="test server")
new_server2 = Server.objects.create(label="test server")
pool = IPAddressPoolFactory.from_name('test')
cmdb_importer = GenericCmdbImporter()
ip1 = IPAddressGeneric.objects.create(address='46.17.40.2',
status=Resource.STATUS_LOCKED,
pool=pool,
parent=new_server1)
self.assertEqual(Resource.STATUS_LOCKED, ip1.status)
self.assertEqual(new_server1.id, ip1.parent.id)
# used IP relink between parents
cmdb_importer._add_ip(ip1.address, parent=new_server2)
ip1.refresh_from_db()
self.assertEqual(Resource.STATUS_INUSE, ip1.status)
self.assertEqual(new_server2.id, ip1.parent.id)
def test_hypervisor_detection(self):
switch = L3Switch()
switch._add_switch_port(24, 'ethernet/1/24')
switch._add_server_port('ethernet/1/24', '0025904EB5A4')
switch._add_server_port('ethernet/1/24', '0025904EB5A5')
switch._add_server_port('ethernet/1/24', 'CEA9ACD2080C')
switch._add_server_port('ethernet/1/24', 'CEA9ACD2081C')
switch._add_server_port('ethernet/1/24', 'CEA9ACD2082C')
switch._add_server_port('ethernet/1/24', 'CEA9ACD2083C')
switch._add_server_port('ethernet/1/24', 'CEA9ACD2084C')
# manually identify hypervisor
hv_server = Server.objects.create(label="hvisor", role='hypervisor')
ServerPort.objects.create(mac='0025904EB5A4', parent=hv_server)
cmdb_importer = GenericCmdbImporter()
sw = Switch.objects.create(label="switch")
cmdb_importer.import_switch(sw.id, switch)
for sw_port in sw:
cmdb_importer.process_hypervisors(sw_port)
self.assertEqual(2, len(Server.active.filter()))
self.assertEqual(5, len(VirtualServer.active.filter()))
self.assertEqual(5, len(VirtualServer.active.filter(parent=hv_server)))
self.assertEqual(0, len(VirtualServer.active.filter(parent=None)))
self.assertEqual(2, len(ServerPort.active.filter()))
self.assertEqual(0, len(ServerPort.active.filter(parent=None)))
self.assertEqual(5, len(VirtualServerPort.active.filter()))
self.assertEqual(0, len(VirtualServerPort.active.filter(parent=None)))
self.assertEqual(7, len(PortConnection.active.filter()))
class Command(BaseCommand):
cmdb_importer = GenericCmdbImporter()
registered_handlers = {}
registered_providers = {
'generic': L3Switch,
'3com.2952': HP1910Switch,
'3com.2250': Switch3Com2250,
'dlink.dsg3200': DSG3200Switch,
'hp.1910': HP1910Switch,
'qtech': QtechL3Switch,
'qtech.3400': Qtech3400Switch
}
def add_arguments(self, parser):
subparsers = parser.add_subparsers(title="CMDB data importer",
help="Commands help",
dest='manager_name',
parser_class=ArgumentParser)
# IP address commands
file_cmd_parser = subparsers.add_parser('fromfile', help='Import data from dump files')
file_cmd_parser.add_argument('device-id', help="Resource ID of the device used to take the dump.")
file_cmd_parser.add_argument('provider', choices=self.registered_providers.keys(),
help="Type of the device (or dump file format).")
file_types_group = file_cmd_parser.add_mutually_exclusive_group()
file_types_group.add_argument('--arpdump', help="Path to the ARP dump.")
file_types_group.add_argument('--macdump', help="Path to the MAC dump.")
self._register_handler('fromfile', self._handle_file_dumps)
snmp_cmd_parser = subparsers.add_parser('snmp', help='Import data from SNMP')
snmp_cmd_parser.add_argument('device-id', help="Resource ID of the device used to take the dump.")
snmp_cmd_parser.add_argument('provider', choices=self.registered_providers.keys(),
help="Type of the device (or dump file format).")
snmp_cmd_parser.add_argument('hostname', help="Hostname or IP address.")
snmp_cmd_parser.add_argument('community', help="SNMP community string.")
self._register_handler('snmp', self._handle_snmp)
auto_cmd_parser = subparsers.add_parser('auto', help='Import and update CMDB data based on resources.')
auto_cmd_parser.add_argument('--switch-id', help="ID of the switch to get SNMP data from.")
auto_cmd_parser.add_argument('--skip-arp', action="store_true", help="Skip ARP analysis.")
self._register_handler('auto', self._handle_auto)
household_cmd_parser = subparsers.add_parser('household', help='Cleanup unused resources.')
self._register_handler('household', self._handle_household)
def _handle_household(self, *args, **options):
last_seen_31days = timezone.now() - datetime.timedelta(days=31)
last_seen_15days = timezone.now() - datetime.timedelta(days=15)
# Clean IP with parent=ip pool (free) with last_seen older that 31 days. It means that IP is not
# used and can be released.
logger.info("Clean missing IP addresses: %s" % last_seen_31days)
for free_ip_pool in Resource.active.filter(status=Resource.STATUS_FREE, type__in=IPAddressPool.ip_pool_types):
logger.info(" pool %s" % free_ip_pool)
for ip in IPAddress.active.filter(
status=Resource.STATUS_INUSE,
last_seen__lt=last_seen_31days,
ipman_pool_id=free_ip_pool.id,
version=4):
logger.warning(" used ip %s from the FREE IP pool is not seen for 31 days. Free it." % ip)
ip.free(cascade=True)
for ip in IPAddress.active.filter(
status=Resource.STATUS_LOCKED,
last_seen__lt=last_seen_15days,
ipman_pool_id=free_ip_pool.id,
version=4):
logger.warning(" locked ip %s from the FREE IP pool is not seen for 15 days. Free it." % ip)
ip.free(cascade=True)
logger.info("Clean missing virtual servers: %s" % last_seen_31days)
for vm in VirtualServer.active.filter(last_seen__lt=last_seen_31days):
logger.warning(" server %s not seen for 31 days. Removing..." % vm)
for vm_child in vm:
logger.info(" remove %s" % vm_child)
vm_child.delete()
vm.delete()
logger.info("Clean unresolved PortConnections...")
removed = 0
for connection in PortConnection.active.all():
if not connection.linked_port:
connection.delete()
removed += 1
logger.info(" removed: %s" % removed)
Resource.objects.rebuild()
def _handle_auto(self, *args, **options):
# update via snmp
query = dict(type__in=[GatewaySwitch.__name__, Switch.__name__])
if options['switch_id']:
query['pk'] = options['switch_id']
if not options['skip_arp']:
for switch in Resource.active.filter(**query):
logger.info("* Found switch: %s" % switch)
if switch.has_option('snmp_provider_key'):
snmp_provider_key = switch.get_option_value('snmp_provider_key')
if snmp_provider_key in self.registered_providers:
hostname = switch.get_option_value('snmp_host')
community = switch.get_option_value('snmp_community')
logger.info("host: %s" % hostname)
provider = self.registered_providers[snmp_provider_key]()
provider.from_snmp(hostname, community)
self.cmdb_importer.import_switch(switch.id, provider)
else:
logger.warning("Unknown SNMP data provider: %s" % snmp_provider_key)
Resource.objects.rebuild()
logger.info("Process hypervisors.")
for switch in Switch.active.all():
for switch_port in SwitchPort.active.filter(parent=switch):
self.cmdb_importer.process_hypervisors(switch_port)
for switch in GatewaySwitch.active.all():
for switch_port in SwitchPort.active.filter(parent=switch):
self.cmdb_importer.process_hypervisors(switch_port)
logger.info("Process server mounts")
link_unresolved_to_container, created = RegionResource.objects.get_or_create(name='Unresolved servers')
self.cmdb_importer.process_servers(link_unresolved_to=link_unresolved_to_container)
logger.info("Process virtual server mounts")
link_unresolved_to_container, created = RegionResource.objects.get_or_create(name='Unresolved VPS')
self.cmdb_importer.process_virtual_servers(link_unresolved_to=link_unresolved_to_container)
Resource.objects.rebuild()
def _handle_snmp(self, *args, **options):
device_id = options['device-id']
provider_key = options['provider']
hostname = options['hostname']
community = options['community']
source_switch = Resource.active.get(pk=device_id)
provider = self.registered_providers[provider_key]()
provider.from_snmp(hostname, community)
self.cmdb_importer.import_switch(device_id, provider)
source_switch.set_option('snmp_host', hostname)
source_switch.set_option('snmp_community', community)
source_switch.set_option('snmp_provider_key', provider_key)
def _handle_file_dumps(self, *args, **options):
device_id = options['device-id']
provider_key = options['provider']
provider = self.registered_providers[provider_key]()
if options['arpdump']:
provider.from_arp_dump(options['arpdump'])
elif options['macdump']:
provider.from_mac_dump(options['macdump'])
else:
raise Exception("Specify one of the dump files.")
self.cmdb_importer.import_switch(device_id, provider)
def handle(self, *args, **options):
if 'subcommand_name' in options:
subcommand = "%s.%s" % (options['manager_name'], options['subcommand_name'])
else:
subcommand = options['manager_name']
# call handler
self.registered_handlers[subcommand](*args, **options)
def _register_handler(self, command_name, handler):
assert command_name, "command_name must be defined."
assert handler, "handler must be defined."
self.registered_handlers[command_name] = handler
def test_import_data(self):
arp_file_path = os.path.join(self.DATA_DIR, 'arp-table.txt')
mac_file_path = os.path.join(self.DATA_DIR, 'mac-table.txt')
cmdb_importer = GenericCmdbImporter()
# create IP pools and basic structure
dc_anders = RegionResource.objects.create(name="Anders")
dc_rtcom = RegionResource.objects.create(name="Rostelecom")
# create source switch
anders_gw = GatewaySwitch.objects.create(name="baxet-gw-q", parent=dc_anders)
# arp table provider
qtech_switch = QtechL3Switch()
qtech_switch.from_arp_dump(arp_file_path)
switch_ports = list(qtech_switch.ports)
self.assertEqual(12, len(switch_ports))
# Add our IP pools
pool_list = [IPNetworkPool.objects.create(network='46.17.40.0/23', parent=dc_anders),
IPNetworkPool.objects.create(network='46.17.44.0/23', parent=dc_anders),
IPNetworkPool.objects.create(network='176.32.34.0/23', parent=dc_anders),
IPNetworkPool.objects.create(network='176.32.36.0/24', parent=dc_anders),
IPNetworkPool.objects.create(network='176.32.37.0/24', parent=dc_anders),
IPNetworkPool.objects.create(network='176.32.38.0/24', parent=dc_anders),
IPNetworkPool.objects.create(network='176.32.39.0/24', parent=dc_anders),
IPNetworkPool.objects.create(network='2a00:b700::/48', parent=dc_anders),
IPNetworkPool.objects.create(network='46.17.46.0/23', parent=dc_rtcom),
IPNetworkPool.objects.create(network='46.29.160.0/23', parent=dc_rtcom),
IPNetworkPool.objects.create(network='176.32.32.0/23', parent=dc_rtcom),
IPNetworkPool.objects.create(network='46.29.162.0/23', parent=dc_rtcom),
IPNetworkPool.objects.create(network='46.29.164.0/22', parent=dc_rtcom),
IPNetworkPool.objects.create(network='185.22.152.0/22', parent=dc_rtcom),
IPNetworkPool.objects.create(network='2a00:b700:1::/48', parent=dc_rtcom)]
# Double the proccess, to test data update process
cmdb_importer.import_switch(anders_gw.id, qtech_switch)
cmdb_importer.import_switch(anders_gw.id, qtech_switch)
# -1 IP: 87.251.133.9, /30 peering network address
self.assertEqual(1328, len(IPAddress.active.filter()))
for pool in pool_list:
print "%s - %d" % (pool, pool.usage)
# count servers
self.assertEqual(71, len(Server.active.filter()))
self.assertEqual(39, len(VirtualServer.active.filter()))
self.assertEqual(71, len(ServerPort.active.filter()))
self.assertEqual(0, len(ServerPort.active.filter(parent=None)))
self.assertEqual(39, len(VirtualServerPort.active.filter()))
self.assertEqual(10, len(PortConnection.active.filter()))
# Servers and ports equality check
self.assertEqual((len(ServerPort.active.filter()) + len(VirtualServerPort.active.filter())),
(len(Server.active.filter()) + len(VirtualServer.active.filter())))
# import MAC data from mac table
anders_sw1 = Switch.objects.create(name="baxet-sw-1", parent=dc_anders)
qtech3400_switch = Qtech3400Switch()
qtech3400_switch.from_mac_dump(mac_file_path)
# double call, to check update
cmdb_importer.import_switch(anders_sw1.id, qtech3400_switch)
cmdb_importer.import_switch(anders_sw1.id, qtech3400_switch)
# count servers
self.assertEqual(76, len(Server.active.filter()))
self.assertEqual(41, len(VirtualServer.active.filter()))
self.assertEqual(76, len(ServerPort.active.filter()))
self.assertEqual(0, len(ServerPort.active.filter(parent=None)))
self.assertEqual(41, len(VirtualServerPort.active.filter()))
self.assertEqual(54, len(PortConnection.active.filter()))
self.assertEqual(1328, len(IPAddress.active.filter()))
# update VPS links to hypervisors
for switch in Switch.active.all():
for switch_port in SwitchPort.active.filter(parent=switch):
cmdb_importer.process_hypervisors(switch_port)
for switch in GatewaySwitch.active.all():
for switch_port in SwitchPort.active.filter(parent=switch):
cmdb_importer.process_hypervisors(switch_port)
self.assertEqual(3, len(Server.active.filter(role='hypervisor')))
for server in Server.active.filter(role='hypervisor'):
print server.id
# There are linked VPS, hypervisor detection logic test.
self.assertEqual(4, len(VirtualServer.active.filter(parent=812)))
self.assertEqual(2, len(VirtualServer.active.filter(parent=820)))
self.assertEqual(3, len(VirtualServer.active.filter(parent=764)))
events = HistoryEvent.objects.filter(type=HistoryEvent.CREATE)
self.assertEqual(1677, len(events))
def test_import_data(self):
arp_file_path = os.path.join(self.DATA_DIR, 'arp-table.txt')
mac_file_path = os.path.join(self.DATA_DIR, 'mac-table.txt')
cmdb_importer = GenericCmdbImporter()
# create IP pools and basic structure
dc_anders = RegionResource.objects.create(name="Anders")
dc_rtcom = RegionResource.objects.create(name="Rostelecom")
# create source switch
anders_gw = GatewaySwitch.objects.create(name="baxet-gw-q", parent=dc_anders)
# arp table provider
qtech_switch = QtechL3Switch()
qtech_switch.from_arp_dump(arp_file_path)
switch_ports = list(qtech_switch.ports)
self.assertEqual(12, len(switch_ports))
# Add our IP pools
pool_list = [IPNetworkPool.objects.create(network='46.17.40.0/23', parent=dc_anders),
IPNetworkPool.objects.create(network='46.17.44.0/23', parent=dc_anders),
IPNetworkPool.objects.create(network='176.32.34.0/23', parent=dc_anders),
IPNetworkPool.objects.create(network='176.32.36.0/24', parent=dc_anders),
IPNetworkPool.objects.create(network='176.32.37.0/24', parent=dc_anders),
IPNetworkPool.objects.create(network='176.32.38.0/24', parent=dc_anders),
IPNetworkPool.objects.create(network='176.32.39.0/24', parent=dc_anders),
IPNetworkPool.objects.create(network='2a00:b700::/48', parent=dc_anders),
IPNetworkPool.objects.create(network='46.17.46.0/23', parent=dc_rtcom),
IPNetworkPool.objects.create(network='46.29.160.0/23', parent=dc_rtcom),
IPNetworkPool.objects.create(network='176.32.32.0/23', parent=dc_rtcom),
IPNetworkPool.objects.create(network='46.29.162.0/23', parent=dc_rtcom),
IPNetworkPool.objects.create(network='46.29.164.0/22', parent=dc_rtcom),
IPNetworkPool.objects.create(network='185.22.152.0/22', parent=dc_rtcom),
IPNetworkPool.objects.create(network='2a00:b700:1::/48', parent=dc_rtcom)]
# Double the proccess, to test data update process
cmdb_importer.import_switch(anders_gw.id, qtech_switch)
cmdb_importer.import_switch(anders_gw.id, qtech_switch)
# -1 IP: 87.251.133.9, /30 peering network address
self.assertEqual(1328, len(IPAddress.active.filter()))
for pool in pool_list:
print "%s - %d" % (pool, pool.usage)
# count servers
self.assertEqual(71, len(Server.active.filter()))
self.assertEqual(39, len(VirtualServer.active.filter()))
self.assertEqual(71, len(ServerPort.active.filter()))
self.assertEqual(0, len(ServerPort.active.filter(parent=None)))
self.assertEqual(39, len(VirtualServerPort.active.filter()))
self.assertEqual(10, len(PortConnection.active.filter()))
# Servers and ports equality check
self.assertEqual((len(ServerPort.active.filter()) + len(VirtualServerPort.active.filter())),
(len(Server.active.filter()) + len(VirtualServer.active.filter())))
# import MAC data from mac table
anders_sw1 = Switch.objects.create(name="baxet-sw-1", parent=dc_anders)
qtech3400_switch = Qtech3400Switch()
qtech3400_switch.from_mac_dump(mac_file_path)
# double call, to check update
cmdb_importer.import_switch(anders_sw1.id, qtech3400_switch)
cmdb_importer.import_switch(anders_sw1.id, qtech3400_switch)
# count servers
self.assertEqual(76, len(Server.active.filter()))
self.assertEqual(41, len(VirtualServer.active.filter()))
self.assertEqual(76, len(ServerPort.active.filter()))
self.assertEqual(0, len(ServerPort.active.filter(parent=None)))
self.assertEqual(41, len(VirtualServerPort.active.filter()))
self.assertEqual(54, len(PortConnection.active.filter()))
self.assertEqual(1328, len(IPAddress.active.filter()))
# update VPS links to hypervisors
for switch in Switch.active.all():
for switch_port in SwitchPort.active.filter(parent=switch):
cmdb_importer.process_hypervisors(switch_port)
for switch in GatewaySwitch.active.all():
for switch_port in SwitchPort.active.filter(parent=switch):
cmdb_importer.process_hypervisors(switch_port)
self.assertEqual(3, len(Server.active.filter(role='hypervisor')))
for server in Server.active.filter(role='hypervisor'):
print server.id
# There are linked VPS, hypervisor detection logic test.
self.assertEqual(4, len(VirtualServer.active.filter(parent=660)))
self.assertEqual(2, len(VirtualServer.active.filter(parent=668)))
self.assertEqual(3, len(VirtualServer.active.filter(parent=612)))
events = HistoryEvent.objects.filter(type=HistoryEvent.CREATE)
self.assertEqual(1677, len(events))
