def _handle_address_move(self, *args, **options):
target_pool = Resource.active.get(pk=options['target-pool-id'])
moved_ips = []
if options['file']:
with open(options['file']) as ip_list_file:
for ip_line in ip_list_file:
ip_line = ip_line.decode('utf-8').strip()
if not ip_line:
continue
ip_obj = GlobalIPManager.get_ip(ip_line)
if ip_obj.move_to_pool(target_pool):
moved_ips.append(ip_obj)
elif options['ip_start']:
start_ip = options['ip_start'].decode('utf-8')
end_ip = options['ip_end'].decode('utf-8')
count = options['count']
moved_ips = GlobalIPManager.move_ips(target_pool=target_pool,
start_ip=start_ip,
end_ip=end_ip,
count=count)
logger.info("IPs are moved to %s" % target_pool)
self._print_addresses(moved_ips)
def test_datacenter_pool_manager(self):
russia = RegionResource.objects.create(name='Russia')
moscow = RegionResource.objects.create(name='Moscow', parent=russia)
dc1 = Datacenter.objects.create(name='Test DC 1', parent=moscow)
dc2 = Datacenter.objects.create(name='Test DC 2', parent=moscow)
rack1 = Rack.objects.create(name='Test Rack 1', parent=dc1)
srv1 = Server.objects.create(name='Test server 1', parent=rack1)
pool1 = IPAddressPoolFactory.from_network('192.168.0.0/23')
dc1 += pool1
pool2 = IPAddressPoolFactory.from_network('192.169.0.0/23')
pool2.use()
dc2 += pool2
# testing manager
pools1 = GlobalIPManager.find_pools()
self.assertEqual(2, len(pools1))
pools2 = GlobalIPManager.find_pools(datacenter=dc1)
self.assertEqual(1, len(pools2))
ip_address = GlobalIPManager.get_ip('192.169.0.100')
self.assertEqual('192.169.0.100', ip_address.address)
self.assertEqual(pool2.id, ip_address.pool.id)
def _handle_household(self, *args, **options):
last_seen_31days = timezone.now() - datetime.timedelta(days=31)
last_seen_15days = timezone.now() - datetime.timedelta(days=15)
# Clean IP with parent=ip pool (free) with last_seen older that 31 days. It means that IP is not
# used and can be released.
logger.info("Clean missing IP addresses: %s" % last_seen_31days)
for free_ip_pool in GlobalIPManager.find_pools(
status=Resource.STATUS_FREE, version=4):
logger.info(" pool %s" % free_ip_pool)
for ip in GlobalIPManager.find_ips(status=Resource.STATUS_INUSE,
last_seen__lt=last_seen_31days,
pool=free_ip_pool):
logger.warning(
" used ip %s from the FREE IP pool is not seen for 31 days. Free it."
% ip)
ip.free()
for ip in GlobalIPManager.find_ips(status=Resource.STATUS_LOCKED,
last_seen__lt=last_seen_15days,
pool=free_ip_pool):
logger.warning(
" locked ip %s from the FREE IP pool is not seen for 15 days. Free it."
% ip)
ip.free()
logger.info("Clean missing virtual servers: %s" % last_seen_31days)
for vm in VirtualServer.active.filter(last_seen__lt=last_seen_31days):
logger.warning(" server %s not seen for 31 days. Removing..." %
vm)
for vm_child in vm:
logger.info(" remove %s" % vm_child)
vm_child.delete()
vm.delete()
logger.info("Clean unresolved PortConnections...")
removed = 0
for connection in PortConnection.active.all():
if not connection.linked_port:
connection.delete()
removed += 1
logger.info(" removed: %s" % removed)
logger.info("Clean missing PortConnections (not seen for 15 days)...")
removed = 0
for connection in PortConnection.active.filter(
last_seen__lt=last_seen_31days):
connection.delete()
removed += 1
logger.info(" removed: %s" % removed)
Resource.objects.rebuild()
def setUp(self):
super(ResourcesAPITests, self).setUp()
user_name = 'admin'
user, created = User.objects.get_or_create(username=user_name,
password=user_name,
email='*****@*****.**',
is_staff=True)
token, created = Token.objects.get_or_create(user=user)
self.client.credentials(HTTP_AUTHORIZATION='Token ' + token.key)
Resource.objects.all().delete()
self.moscow = RegionResource.objects.create(name='Moscow')
self.dc1 = Datacenter.objects.create(name='Test DC 1',
parent=self.moscow)
self.rack1 = Rack.objects.create(name='Test Rack 1', parent=self.dc1)
self.srv1 = Server.objects.create(name='Test hypervisor 1',
role='hypervisor',
parent=self.rack1)
self.pools_group1 = RegionResource.objects.create(
name='Test DC 1 IP Pools', parent=self.dc1)
self.pool1 = IPAddressPoolFactory.from_network(
'192.168.0.0/23',
parent=self.pools_group1,
dns1='46.17.46.200',
dns2='46.17.40.200')
GlobalIPManager.get_ip('192.168.0.1').lock()
self.pool11 = IPAddressPoolFactory.from_network(
'192.169.0.0/23',
parent=self.pools_group1,
dns1='46.17.46.200',
dns2='46.17.40.200')
self.pool11.use()
GlobalIPManager.get_ip('192.169.0.1').lock()
self.srv1.set_option('agentd_taskqueue', 'test_task_queue')
MockVpsControlTask.REMOTE_WORKER.sent_tasks = []
self.cloud = CmdbCloudConfig()
self.backend = ProxMoxJBONServiceBackend(self.cloud)
ProxMoxJBONServiceBackend.TASK_CREATE = MockVpsControlTask
ProxMoxJBONServiceBackend.TASK_START = MockVpsControlTask
ProxMoxJBONServiceBackend.TASK_STOP = MockVpsControlTask
def test_move_ips(self):
pool1 = IPAddressPoolFactory.from_network('192.168.1.0/24')
pool2 = IPAddressPoolFactory.from_network('192.168.2.0/23')
pool3 = IPAddressPoolFactory.from_name('Target pool')
self.assertEqual(254, pool1.get_ips().count())
self.assertEqual(510, pool2.get_ips().count())
self.assertEqual(0, pool3.get_ips().count())
GlobalIPManager.move_ips(pool3, '192.168.1.100', 50)
GlobalIPManager.move_ips(pool3, '192.168.2.100', 1)
self.assertEqual(204, pool1.get_ips().count())
self.assertEqual(509, pool2.get_ips().count())
self.assertEqual(51, pool3.get_ips().count())
def _list_pools(self):
resource_writer = ConsoleResourceWriter(GlobalIPManager.find_pools())
resource_writer.print_table(fields=[
'id', 'parent', 'self', 'status', 'used_addresses',
'total_addresses', 'usage'
],
sort_by='parent')
def _handle_address_delete(self, *args, **options):
ip_address = options['ip-address'].decode('utf-8')
ip = GlobalIPManager.get_ip(ip_address)
ip.delete()
logger.info("%s deleted" % ip_address)
def _dump_server(self, server):
assert server
logger.info("%s %s" % (server.type, server))
if server.is_mounted:
logger.info(
"mounted %s, position %s, rails %s" %
(server.typed_parent, server.position, server.on_rails))
else:
logger.info("not mounted")
for server_port in ServerPort.active.filter(parent=server):
conn = server_port.connection
if conn:
logger.info(" %s (seen %s)" % (conn, conn.last_seen))
else:
logger.info(" %s no connections" % server_port)
for ip_address in GlobalIPManager.find_ips(parent=server_port):
logger.info(" %s (seen %s)" %
(ip_address, ip_address.last_seen))
logger.info("Options:")
for option in server.get_options():
logger.info(" %s" % option)
def find_ip_info(self, ip_address):
"""
Search and return info about IP address: gateway and netmask.
Check only in IPNetworkPools that is free.
:param ip_address:
:return: tuple (ip, gw, netmask, dns1, dns2)
"""
assert ip_address
found_ip = GlobalIPManager.get_ip(ip_address)
target_net_pool = found_ip.pool
netmask = target_net_pool.get_option_value('netmask', default=None)
if not netmask:
target_net_pool = None
# checking IP pool
netmask = target_net_pool.get_option_value('netmask', default=None)
gateway = target_net_pool.get_option_value('gateway', default=None)
dns1 = target_net_pool.get_option_value('dns1', default=None)
dns2 = target_net_pool.get_option_value('dns2', default=None)
if not netmask or not gateway:
raise Exception("IP pool %s have no network settings." % target_net_pool)
return ip_address, gateway, netmask, dns1, dns2
def _handle_daimport(self, *args, **options):
cid = options['cid']
da_pool = IPAddressPoolFactory.from_name(name="Imported DirectAdmin")
with open(options['tsv-file']) as tsv_file:
for line in tsv_file:
line = line.strip()
if line == '':
continue
(lid_id, ipaddr,
lic_status) = line.decode('utf-8').split(None, 3)
logger.info("> Processing: %s %s %s" %
(lid_id, ipaddr, lic_status))
try:
ip_obj = GlobalIPManager.get_ip(ipaddr)
except:
# TODO: refactor IP creation
ip_obj, created = IPAddressGeneric.objects.update_or_create(
address=ipaddr, pool=da_pool)
if lic_status == Resource.STATUS_FREE:
if ip_obj.status == Resource.STATUS_FREE:
license = DirectAdminLicense.register_license(
pool=da_pool,
cid=cid,
lid=int(lid_id),
ip_address=ip_obj)
license.free()
logger.info("LIC %s (%s). Added as FREE" %
(lid_id, ipaddr))
else:
logger.warning(
"(!!) LIC %s (%s). You must change IP." %
(lid_id, ipaddr))
else:
if ip_obj.status == Resource.STATUS_FREE:
license = DirectAdminLicense.register_license(
pool=da_pool,
cid=cid,
lid=int(lid_id),
ip_address=ip_obj)
license.free()
logger.warning(
"(!) LIC %s (%s). Added as FREE (changed License status to FREE)"
% (lid_id, ipaddr))
else:
license = DirectAdminLicense.register_license(
pool=da_pool,
cid=cid,
lid=int(lid_id),
ip_address=ip_obj)
license.use()
logger.info("LIC %s (%s). Added as USED." %
(lid_id, ipaddr))
def register_license(cid, lid, ip_address, pool):
assert cid > 0
assert lid > 0
assert ip_address
assert pool
assert isinstance(pool, IPAddressPoolGeneric)
if not isinstance(ip_address, IPAddressGeneric):
ip_address = GlobalIPManager.get_ip(ip_address)
da_license, created = DirectAdminLicense.objects.update_or_create(
cid=cid,
lid=lid,
defaults=dict(status=ip_address.status, assigned_ip=ip_address))
GlobalIPManager.move_ips(pool, ip_address, count=1)
return da_license
def setUp(self):
Resource.objects.all().delete()
self.moscow = RegionResource.objects.create(name='Moscow')
self.dc1 = Datacenter.objects.create(name='Test DC 1',
parent=self.moscow)
self.rack1 = Rack.objects.create(name='Test Rack 1', parent=self.dc1)
self.srv1 = Server.objects.create(name='Test hypervisor 1',
role='hypervisor',
parent=self.rack1)
self.pools_group1 = RegionResource.objects.create(
name='Test DC 1 IP Pools', parent=self.dc1)
self.pool1 = IPAddressPoolFactory.from_network(
'192.168.0.0/23',
parent=self.pools_group1,
dns1='46.17.46.200',
dns2='46.17.40.200')
GlobalIPManager.get_ip('192.168.0.1').lock()
self.pool11 = IPAddressPoolFactory.from_network(
'192.169.0.0/23',
parent=self.pools_group1,
dns1='46.17.46.200',
dns2='46.17.40.200')
self.pool11.use()
GlobalIPManager.get_ip('192.169.0.1').lock()
self.srv1.set_option('agentd_taskqueue', 'test_task_queue')
MockVpsControlTask.REMOTE_WORKER.sent_tasks = []
self.cloud = CmdbCloudConfig()
self.backend = ProxMoxJBONServiceBackend(self.cloud)
ProxMoxJBONServiceBackend.TASK_CREATE = MockVpsControlTask
ProxMoxJBONServiceBackend.TASK_START = MockVpsControlTask
ProxMoxJBONServiceBackend.TASK_STOP = MockVpsControlTask
def _handle_pool_rent(self, *args, **options):
ip_count = options['count']
if options['dc']:
renter = IPAddressRenter.from_datacenter(
Resource.objects.get(pk=options['dc']),
ip_version=options['version'])
else:
renter = IPAddressRenter.from_pools(
GlobalIPManager.find_pools(pk__in=options['pools']))
rent_ips = renter.rent(ip_count)
self._print_addresses(rent_ips)
def test_mptt_tree_navigation_single_tree(self):
russia = RegionResource.objects.create(name='Russia')
moscow = RegionResource.objects.create(name='Moscow', parent=russia)
dc1 = Datacenter.objects.create(name='Test DC 1', parent=moscow)
rack1 = Rack.objects.create(name='Test Rack 1', parent=dc1)
srv1 = Server.objects.create(name='Test server 1', parent=rack1)
pools_group1 = RegionResource.objects.create(name='Test DC 1 IP Pools',
parent=dc1)
pool1 = IPAddressPoolFactory.from_network(network='192.168.0.0/23',
parent=pools_group1,
status=Resource.STATUS_FREE)
pool11 = IPAddressPoolFactory.from_network(
network='192.169.0.0/23',
parent=pools_group1,
status=Resource.STATUS_INUSE)
kazan = RegionResource.objects.create(name='Kazan', parent=russia)
dc2 = Datacenter.objects.create(name='Test DC 2', parent=kazan)
rack2 = Rack.objects.create(name='Test Rack 2', parent=dc2)
srv2 = Server.objects.create(name='Test server 1', parent=rack2)
pools_group2 = RegionResource.objects.create(name='Test DC 2 IP Pools',
parent=dc2)
pool2 = IPAddressPoolFactory.from_network(network='172.168.0.0/23',
parent=pools_group2)
# find Pool of the server
my_dcs_raw = Datacenter.active.filter(lft__lt=srv1.lft,
rght__gt=srv1.rght,
tree_id=srv1.tree_id)
self.assertEqual(1, len(my_dcs_raw))
self.assertEqual(dc1.id, my_dcs_raw[0].id)
# Find Datacenter
my_dcs = srv1.filter_parents(Datacenter)
self.assertEqual(1, len(my_dcs))
self.assertEqual(dc1.id, my_dcs[0].id)
# Find free IPNetworkPool (we have free and used IP pools in this DC)
my_ippools = GlobalIPManager.find_pools(datacenter=my_dcs[0],
status=Resource.STATUS_FREE)
self.assertEqual(1, len(my_ippools))
self.assertEqual('192.168.0.0/23', "%s" % my_ippools[0])
def _add_ip(self, ip_address, parent=None):
assert ip_address, "ip_address must be defined."
logger.info("Processing IP %s" % ip_address)
try:
added_ip = GlobalIPManager.get_ip(ip_address)
added_ip.use()
added_ip.touch()
if parent:
if added_ip.parent and added_ip.parent.id != parent.id:
logger.info("IP %s moved from %s to %s" %
(ip_address, added_ip.parent, parent))
added_ip.parent = parent
added_ip.save()
except Exception as ex:
logger.exception(ex.message)
def _get_server_by_ip_or_id(self, server_ip_id):
"""
Find server by IP or ID. If nothing found, return None
:param server_ip_id:
:return:
"""
assert server_ip_id
server = None
if server_ip_id.find('.') > -1:
ips = GlobalIPManager.get_ip(server_ip_id)
if len(ips) > 0 and isinstance(ips[0].parent.typed_parent,
AssetResource):
print(ips[0].parent.parent.type)
server = ips[0].parent.typed_parent
else:
logger.warning("IP %s is not assigned to any server." %
server_ip_id)
else:
server = Resource.objects.get(pk=server_ip_id)
return server
def __init__(self):
self.available_ip_pools = GlobalIPManager.find_pools(
status=Resource.STATUS_FREE)
def test_import_data(self):
arp_file_path = os.path.join(self.DATA_DIR, 'arp-table.txt')
mac_file_path = os.path.join(self.DATA_DIR, 'mac-table.txt')
cmdb_importer = GenericCmdbImporter()
# create IP pools and basic structure
dc_anders = RegionResource.objects.create(name="Anders")
dc_rtcom = RegionResource.objects.create(name="Rostelecom")
# create source switch
anders_gw = GatewaySwitch.objects.create(name="baxet-gw-q",
parent=dc_anders)
# arp table provider
qtech_switch = QtechL3Switch()
qtech_switch.from_arp_dump(arp_file_path)
switch_ports = list(qtech_switch.ports)
self.assertEqual(12, len(switch_ports))
# Add our IP pools
pool_list = [
IPAddressPoolFactory.from_network(network='46.17.40.0/23',
parent=dc_anders),
IPAddressPoolFactory.from_network(network='46.17.44.0/23',
parent=dc_anders),
IPAddressPoolFactory.from_network(network='176.32.34.0/23',
parent=dc_anders),
IPAddressPoolFactory.from_network(network='176.32.36.0/24',
parent=dc_anders),
IPAddressPoolFactory.from_network(network='176.32.37.0/24',
parent=dc_anders),
IPAddressPoolFactory.from_network(network='176.32.38.0/24',
parent=dc_anders),
IPAddressPoolFactory.from_network(network='176.32.39.0/24',
parent=dc_anders),
IPAddressPoolFactory.from_network(network='2a00:b700::/120',
parent=dc_anders),
IPAddressPoolFactory.from_network(network='46.17.46.0/23',
parent=dc_rtcom),
IPAddressPoolFactory.from_network(network='46.29.160.0/23',
parent=dc_rtcom),
IPAddressPoolFactory.from_network(network='176.32.32.0/23',
parent=dc_rtcom),
IPAddressPoolFactory.from_network(network='46.29.162.0/23',
parent=dc_rtcom),
IPAddressPoolFactory.from_network(network='46.29.164.0/22',
parent=dc_rtcom),
IPAddressPoolFactory.from_network(network='185.22.152.0/22',
parent=dc_rtcom),
IPAddressPoolFactory.from_network(network='2a00:b700:1::/120',
parent=dc_rtcom),
IPAddressPoolFactory.from_network(network='87.251.133.9/31',
parent=dc_rtcom)
]
# Double the proccess, to test data update process
cmdb_importer.import_switch(anders_gw.id, qtech_switch)
cmdb_importer.import_switch(anders_gw.id, qtech_switch)
# +1 IP: 87.251.133.9, /31 peering network address
self.assertEqual(
1329, len(GlobalIPManager.find_ips(status=Resource.STATUS_INUSE)))
for pool in pool_list:
print("%s - %d" % (pool, pool.usage))
# count servers
self.assertEqual(71, len(Server.active.filter()))
self.assertEqual(39, len(VirtualServer.active.filter()))
self.assertEqual(71, len(ServerPort.active.filter()))
self.assertEqual(0, len(ServerPort.active.filter(parent=None)))
self.assertEqual(39, len(VirtualServerPort.active.filter()))
self.assertEqual(10, len(PortConnection.active.filter()))
# Servers and ports equality check
self.assertEqual(
(len(ServerPort.active.filter()) +
len(VirtualServerPort.active.filter())),
(len(Server.active.filter()) + len(VirtualServer.active.filter())))
# import MAC data from mac table
anders_sw1 = Switch.objects.create(name="baxet-sw-1", parent=dc_anders)
qtech3400_switch = Qtech3400Switch()
qtech3400_switch.from_mac_dump(mac_file_path)
# double call, to check update
cmdb_importer.import_switch(anders_sw1.id, qtech3400_switch)
cmdb_importer.import_switch(anders_sw1.id, qtech3400_switch)
# count servers
self.assertEqual(76, len(Server.active.filter()))
self.assertEqual(41, len(VirtualServer.active.filter()))
self.assertEqual(76, len(ServerPort.active.filter()))
self.assertEqual(0, len(ServerPort.active.filter(parent=None)))
self.assertEqual(41, len(VirtualServerPort.active.filter()))
self.assertEqual(54, len(PortConnection.active.filter()))
self.assertEqual(
1329, len(GlobalIPManager.find_ips(status=Resource.STATUS_INUSE)))
# update VPS links to hypervisors
for switch in Switch.active.all():
for switch_port in SwitchPort.active.filter(parent=switch):
cmdb_importer.process_hypervisors(switch_port)
for switch in GatewaySwitch.active.all():
for switch_port in SwitchPort.active.filter(parent=switch):
cmdb_importer.process_hypervisors(switch_port)
hypervisors = Server.active.filter(role='hypervisor')
self.assertEqual(3, len(hypervisors))
for server in hypervisors:
print(server.id)
# There are linked VPS, hypervisor detection logic test.
self.assertEqual(
4, len(VirtualServer.active.filter(parent=hypervisors[0].id)))
self.assertEqual(
2, len(VirtualServer.active.filter(parent=hypervisors[1].id)))
self.assertEqual(
3, len(VirtualServer.active.filter(parent=hypervisors[2].id)))
events = HistoryEvent.objects.filter(event_type=HistoryEvent.CREATE)
self.assertEqual(7492, len(events)) # all IPs are beign created