def clean(self):
"""
Additional validation handler.
"""
pool = self.cleaned_data.get('pool')
if self.cleaned_data.get('network'):
try:
network, cidr = self.cleaned_data.get('network').split('/')
cidr = int(cidr)
if not IPV4_ADDR_RE.match(network) or network.startswith('127.'):
raise ValueError
except ValueError:
raise forms.ValidationError(_("Enter subnet in CIDR notation!"))
# Check if the given subnet already exists
if Subnet.is_allocated(network, cidr, exclude_node = self.__node):
raise forms.ValidationError(_("Specified subnet is already in use!"))
# Check if the given subnet is part of any allocation pools (unless it is allocatable)
if Pool.contains_network(network, cidr):
if pool.reserve_subnet(network, cidr, check_only = True):
self.cleaned_data['reserve'] = True
else:
raise forms.ValidationError(_("Specified subnet is part of another allocation pool and cannot be manually allocated!"))
self.cleaned_data['network'] = network
self.cleaned_data['cidr'] = cidr
return self.cleaned_data
def check_network_status():
"""
Performs the network status check.
"""
# Initialize the state of nodes and subnets, remove out of date ap clients and graph items
Node.objects.all().update(visible = False)
Subnet.objects.all().update(visible = False)
Link.objects.all().update(visible = False)
APClient.objects.filter(last_update__lt = datetime.now() - timedelta(minutes = 11)).delete()
# Reset some states
NodeWarning.objects.all().update(source = EventSource.Monitor, dirty = False)
Node.objects.all().update(warnings = False, conflicting_subnets = False)
# Fetch routing tables from OLSR
try:
nodes, hna = wifi_utils.get_tables(settings.MONITOR_OLSR_HOST)
except TypeError:
logging.error("Unable to fetch routing tables from '%s'!" % settings.MONITOR_OLSR_HOST)
return
# Ping nodes present in the database and visible in OLSR
dbNodes = {}
nodesToPing = []
for nodeIp in nodes.keys():
try:
# Try to get the node from the database
n = Node.get_exclusive(ip = nodeIp)
n.visible = True
n.peers = len(nodes[nodeIp].links)
# If we have succeeded, add to list (if not invalid)
if not n.is_invalid():
if n.awaiting_renumber:
# Reset any status from awaiting renumber to invalid
for notice in n.renumber_notices.all():
try:
rn = Node.objects.get(ip = notice.original_ip)
if rn.status == NodeStatus.AwaitingRenumber:
rn.status = NodeStatus.Invalid
rn.node_type = NodeType.Unknown
rn.awaiting_renumber = False
rn.save()
except Node.DoesNotExist:
pass
notice.delete()
n.awaiting_renumber = False
n.save()
nodesToPing.append(nodeIp)
else:
n.last_seen = datetime.now()
n.peers = len(nodes[nodeIp].links)
# Create a warning since node is not registered
NodeWarning.create(n, WarningCode.UnregisteredNode, EventSource.Monitor)
n.save()
dbNodes[nodeIp] = n
except Node.DoesNotExist:
# Node does not exist, create an invalid entry for it
n = Node(ip = nodeIp, status = NodeStatus.Invalid, last_seen = datetime.now())
n.visible = True
n.node_type = NodeType.Unknown
n.peers = len(nodes[nodeIp].links)
# Check if there are any renumber notices for this IP address
try:
notice = RenumberNotice.objects.get(original_ip = nodeIp)
n.status = NodeStatus.AwaitingRenumber
n.node_type = notice.node.node_type
n.awaiting_renumber = True
except RenumberNotice.DoesNotExist:
pass
n.save(force_insert = True)
dbNodes[nodeIp] = n
# Create an event and append a warning since an unknown node has appeared
NodeWarning.create(n, WarningCode.UnregisteredNode, EventSource.Monitor)
Event.create_event(n, EventCode.UnknownNodeAppeared, '', EventSource.Monitor)
# Add a warning to all nodes that have been stuck in renumbering state for over a week
for node in Node.objects.filter(renumber_notices__renumbered_at__lt = datetime.now() - timedelta(days = 7)):
NodeWarning.create(node, WarningCode.LongRenumber, EventSource.Monitor)
node.save()
# Mark invisible nodes as down
for node in Node.objects.exclude(status__in = (NodeStatus.Invalid, NodeStatus.AwaitingRenumber)):
oldStatus = node.status
if node.ip not in dbNodes:
if node.status == NodeStatus.New:
node.status = NodeStatus.Pending
elif node.status != NodeStatus.Pending:
node.status = NodeStatus.Down
node.save()
if oldStatus in (NodeStatus.Up, NodeStatus.Visible, NodeStatus.Duped) and node.status == NodeStatus.Down:
Event.create_event(node, EventCode.NodeDown, '', EventSource.Monitor)
# Invalidate uptime credit for this node
node.uptime_last = None
node.save()
# Generate timestamp and snapshot identifier
timestamp = datetime.now()
snapshot_id = int(time.time())
# Setup all node peerings
for nodeIp, node in nodes.iteritems():
n = dbNodes[nodeIp]
n.redundancy_link = False
links = []
# Find old VPN server peers
old_vpn_peers = set([p.dst for p in n.get_peers().filter(dst__vpn_server = True)])
for peerIp, lq, ilq, etx, vtime in node.links:
try:
l = Link.objects.get(src = n, dst = dbNodes[peerIp])
except Link.DoesNotExist:
l = Link(src = n, dst = dbNodes[peerIp])
l.lq = float(lq)
l.ilq = float(ilq)
l.etx = float(etx)
l.vtime = vtime
l.visible = True
l.save()
links.append(l)
# Check if any of the peers has never peered with us before
if n.is_adjacency_important() and l.dst.is_adjacency_important() and not n.peer_history.filter(pk = l.dst.pk).count():
n.peer_history.add(l.dst)
Event.create_event(n, EventCode.AdjacencyEstablished, '', EventSource.Monitor,
data = 'Peer node: %s' % l.dst, aggregate = False)
Event.create_event(l.dst, EventCode.AdjacencyEstablished, '', EventSource.Monitor,
data = 'Peer node: %s' % n, aggregate = False)
# Check if we have a peering with any VPN servers
if l.dst.vpn_server:
n.redundancy_link = True
if not n.is_invalid():
# Determine new VPN server peers
new_vpn_peers = set([p.dst for p in n.get_peers().filter(visible = True, dst__vpn_server = True)])
if old_vpn_peers != new_vpn_peers:
for p in old_vpn_peers:
if p not in new_vpn_peers:
# Redundancy loss has ocurred
Event.create_event(n, EventCode.RedundancyLoss, '', EventSource.Monitor,
data = 'VPN server: %s' % p)
for p in new_vpn_peers:
if p not in old_vpn_peers:
# Redundancy restoration has ocurred
Event.create_event(n, EventCode.RedundancyRestored, '', EventSource.Monitor,
data = 'VPN server: %s' % p)
# Issue a warning when node requires peering but has none
if n.redundancy_req and not n.redundancy_link:
NodeWarning.create(n, WarningCode.NoRedundancy, EventSource.Monitor)
n.save()
# Archive topology information
data_archive.record_topology_entry(snapshot_id, timestamp, n, links)
# Update valid subnet status in the database
for nodeIp, subnets in hna.iteritems():
if nodeIp not in dbNodes:
continue
for subnet in subnets:
subnet, cidr = subnet.split("/")
try:
s = Subnet.objects.get(node__ip = nodeIp, subnet = subnet, cidr = int(cidr))
s.last_seen = datetime.now()
s.visible = True
except Subnet.DoesNotExist:
s = Subnet(node = dbNodes[nodeIp], subnet = subnet, cidr = int(cidr), last_seen = datetime.now())
s.visible = True
s.allocated = False
# Save previous subnet status for later use
old_status = s.status
# Set status accoording to allocation flag
if s.allocated:
s.status = SubnetStatus.AnnouncedOk
else:
s.status = SubnetStatus.NotAllocated
# Check if this is a more specific prefix announce for an allocated prefix
if s.is_more_specific() and not s.allocated:
s.status = SubnetStatus.Subset
# Check if this is a hijack
try:
origin = Subnet.objects.ip_filter(
# Subnet overlaps with another one
ip_subnet__contains = '%s/%s' % (subnet, cidr)
).exclude(
# Of another node (= filter all subnets belonging to current node)
node = s.node
).get(
# That is allocated and visible
allocated = True,
visible = True
)
s.status = SubnetStatus.Hijacked
except Subnet.DoesNotExist:
pass
# Generate an event if status has changed
if old_status != s.status and s.status == SubnetStatus.Hijacked:
Event.create_event(n, EventCode.SubnetHijacked, '', EventSource.Monitor,
data = 'Subnet: %s/%s\n Allocated to: %s' % (s.subnet, s.cidr, origin.node))
# Flag node entry with warnings flag for unregistered announces
if not s.is_properly_announced():
if s.node.border_router and not s.is_from_known_pool():
# TODO when we have peering announce registration this should first check if
# the subnet is registered as a peering
s.status = SubnetStatus.Peering
if not s.node.border_router or s.status == SubnetStatus.Hijacked or s.is_from_known_pool():
# Add a warning message for unregistered announced subnets
NodeWarning.create(s.node, WarningCode.UnregisteredAnnounce, EventSource.Monitor)
s.node.save()
s.save()
# Detect subnets that cause conflicts and raise warning flags for all involved
# nodes
if s.is_conflicting():
NodeWarning.create(s.node, WarningCode.AnnounceConflict, EventSource.Monitor)
s.node.conflicting_subnets = True
s.node.save()
for cs in s.get_conflicting_subnets():
NodeWarning.create(cs.node, WarningCode.AnnounceConflict, EventSource.Monitor)
cs.node.conflicting_subnets = True
cs.node.save()
# Remove subnets that were hijacked but are not visible anymore
for s in Subnet.objects.filter(status = SubnetStatus.Hijacked, visible = False):
Event.create_event(s.node, EventCode.SubnetRestored, '', EventSource.Monitor, data = 'Subnet: %s/%s' % (s.subnet, s.cidr))
s.delete()
# Remove (or change their status) subnets that are not visible
Subnet.objects.filter(allocated = False, visible = False).delete()
Subnet.objects.filter(allocated = True, visible = False).update(status = SubnetStatus.NotAnnounced)
for subnet in Subnet.objects.filter(status = SubnetStatus.NotAnnounced, node__visible = True):
NodeWarning.create(subnet.node, WarningCode.OwnNotAnnounced, EventSource.Monitor)
subnet.node.save()
# Remove invisible unknown nodes
for node in Node.objects.filter(status = NodeStatus.Invalid, visible = False).all():
# Create an event since an unknown node has disappeared
Event.create_event(node, EventCode.UnknownNodeDisappeared, '', EventSource.Monitor)
Node.objects.filter(status__in = (NodeStatus.Invalid, NodeStatus.AwaitingRenumber), visible = False).delete()
# Remove invisible links
Link.objects.filter(visible = False).delete()
# Add nodes to topology map and generate output
if not getattr(settings, 'MONITOR_DISABLE_GRAPHS', None):
# Only generate topology when graphing is not disabled
topology = DotTopologyPlotter()
for node in dbNodes.values():
topology.addNode(node)
topology.save(os.path.join(settings.GRAPH_DIR, 'network_topology.png'), os.path.join(settings.GRAPH_DIR, 'network_topology.dot'))
# Ping the nodes to prepare information for later node processing
varsize_results = {}
results, dupes = wifi_utils.ping_hosts(10, nodesToPing)
for packet_size in (100, 500, 1000, 1480):
r, d = wifi_utils.ping_hosts(10, nodesToPing, packet_size - 8)
for node_ip in nodesToPing:
varsize_results.setdefault(node_ip, []).append(r[node_ip][3] if node_ip in r else None)
if getattr(settings, 'MONITOR_DISABLE_MULTIPROCESSING', None):
# Multiprocessing is disabled (the MONITOR_DISABLE_MULTIPROCESSING option is usually
# used for debug purpuses where a single process is prefered)
for node_ip in nodesToPing:
process_node(node_ip, results.get(node_ip), node_ip in dupes, nodes[node_ip].links, varsize_results.get(node_ip))
# Commit the transaction here since we do everything in the same session
transaction.commit()
else:
# We MUST commit the current transaction here, because we will be processing
# some transactions in parallel and must ensure that this transaction that has
# modified the nodes is commited. Otherwise this will deadlock!
transaction.commit()
worker_results = []
for node_ip in nodesToPing:
worker_results.append(
WORKER_POOL.apply_async(process_node, (node_ip, results.get(node_ip), node_ip in dupes, nodes[node_ip].links, varsize_results.get(node_ip)))
)
# Wait for all workers to finish processing
objects = {}
for result in worker_results:
try:
k, v = result.get()
objects[k] = v
except Exception, e:
logging.warning(format_exc())
# When GC debugging is enabled make some additional computations
if getattr(settings, 'MONITOR_ENABLE_GC_DEBUG', None):
global _MAX_GC_OBJCOUNT
objcount = sum(objects.values())
if '_MAX_GC_OBJCOUNT' not in globals():
_MAX_GC_OBJCOUNT = objcount
logging.debug("GC object count: %d %s" % (objcount, "!M" if objcount > _MAX_GC_OBJCOUNT else ""))
_MAX_GC_OBJCOUNT = max(_MAX_GC_OBJCOUNT, objcount)
def save(self, node):
"""
Completes subnet allocation.
"""
network = self.cleaned_data.get('network')
cidr = self.cleaned_data.get('cidr')
pool = self.cleaned_data.get('pool')
if network:
if self.cleaned_data['reserve']:
# We should reserve this manually allocated subnet in the project pool
if not pool.reserve_subnet(network, cidr):
return
subnet = Subnet(node = node)
subnet.subnet = network
subnet.cidr = cidr
subnet.allocated = True
subnet.allocated_at = datetime.now()
subnet.status = SubnetStatus.NotAnnounced
subnet.description = self.cleaned_data.get('description')
subnet.gen_iface_type = IfaceType.LAN
subnet.gen_dhcp = self.cleaned_data.get('dhcp')
subnet.save()
else:
allocation = pool.allocate_subnet(int(self.cleaned_data.get('prefix_len')) or 27)
subnet = Subnet(node = node, subnet = allocation.network, cidr = allocation.cidr)
subnet.allocated = True
subnet.allocated_at = datetime.now()
subnet.status = SubnetStatus.NotAnnounced
subnet.description = self.cleaned_data.get('description')
subnet.gen_iface_type = IfaceType.LAN
subnet.gen_dhcp = self.cleaned_data.get('dhcp')
subnet.save()
# Remove any already announced subnets that are the same subnet
Subnet.objects.filter(node = node, subnet = subnet.subnet, cidr = subnet.cidr, allocated = False).delete()
return node
def save(self, user):
"""
Completes node registration.
"""
ip = self.cleaned_data.get('ip')
project = self.cleaned_data.get('project')
pool = self.cleaned_data.get('pool')
prefix_len = self.cleaned_data.get('prefix_len')
if prefix_len == 0:
prefix_len = None
subnet = None
if not ip:
# Assign a new IP address from the selected pool (if no IP address selected)
node = Node()
fresh_subnet = pool.allocate_subnet(prefix_len)
net = ipcalc.Network(fresh_subnet.network, fresh_subnet.cidr)
node.ip = str(net.host_first())
# Create a new subnet for this node or use an existing one if available
subnet = Subnet(node = node, subnet = fresh_subnet.network, cidr = fresh_subnet.cidr)
subnet.allocated = True
subnet.allocated_at = datetime.now()
subnet.status = SubnetStatus.NotAnnounced
else:
# When prefix is not available we should use /32
if prefix_len is None:
prefix_len = 32
net = ipcalc.Network(ip, prefix_len)
sub_ip = str(net.network())
# Check if this node already exists
try:
node = Node.objects.get(ip = str(net.host_first()))
except Node.DoesNotExist:
node = Node(ip = str(net.host_first()))
# Reserve existing IP in the pool
pool.reserve_subnet(sub_ip, prefix_len)
try:
subnet = Subnet.objects.get(node = node, subnet = sub_ip, cidr = prefix_len)
subnet.status = SubnetStatus.AnnouncedOk
except Subnet.DoesNotExist:
subnet = Subnet(node = node, subnet = sub_ip, cidr = prefix_len)
subnet.status = SubnetStatus.NotAnnounced
subnet.allocated = True
subnet.allocated_at = datetime.now()
# Update node metadata
node.name = self.cleaned_data.get('name').lower()
node.project = project
node.owner = user
node.location = self.cleaned_data.get('location')
node.geo_lat = self.cleaned_data.get('geo_lat')
node.geo_long = self.cleaned_data.get('geo_long')
node.ant_external = self.cleaned_data.get('ant_external')
node.ant_polarization = self.cleaned_data.get('ant_polarization')
node.ant_type = self.cleaned_data.get('ant_type')
node.node_type = self.cleaned_data.get('node_type')
node.notes = self.cleaned_data.get('notes')
node.url = self.cleaned_data.get('url')
node.redundancy_req = self.cleaned_data.get('redundancy_req')
node.warnings = False
for i in xrange(10):
try:
mac = gen_mac_address()
Node.objects.get(vpn_mac_conf = mac)
except Node.DoesNotExist:
node.vpn_mac_conf = mac
break
else:
raise Exception, "unable to generate unique MAC"
if user.is_staff:
node.system_node = self.cleaned_data.get('system_node')
node.vpn_server = self.cleaned_data.get('vpn_server')
if user.is_staff or getattr(settings, 'NONSTAFF_BORDER_ROUTERS', False):
node.border_router = self.cleaned_data.get('border_router')
node.status = NodeStatus.New
node.save()
# Create node traffic control policy
tc_ingress = self.cleaned_data.get('tc_ingress')
if tc_ingress:
Policy.set_policy(node, node.vpn_mac_conf, PolicyAction.Shape, tc_ingress, PolicyFamily.Ethernet)
# Create node profile for image generator
if self.cleaned_data.get('template'):
profile = Profile(node = node, template = self.cleaned_data.get('template'))
if self.cleaned_data.get('channel') in ('', "0", None):
profile.channel = node.project.channel
else:
profile.channel = self.cleaned_data.get('channel')
profile.root_pass = self.cleaned_data.get('root_pass')
profile.use_vpn = self.cleaned_data.get('use_vpn')
profile.antenna = self.cleaned_data.get('ant_conn') or 0
profile.lan_bridge = self.cleaned_data.get('lan_bridge') or False
profile.wan_dhcp = self.cleaned_data.get('wan_dhcp')
profile.wan_ip = self.cleaned_data.get('wan_ip')
profile.wan_cidr = self.cleaned_data.get('wan_cidr')
profile.wan_gw = self.cleaned_data.get('wan_gw')
if self.cleaned_data.get('tc_egress'):
profile.vpn_egress_limit = self.cleaned_data.get('tc_egress').bandwidth
profile.save()
profile.optional_packages = self.cleaned_data.get('optional_packages')
profile.save()
if subnet:
subnet.node = node
subnet.save()
# Update DNS entries
Record.update_for_node(node)
# Registers node name
NodeNames(name = node.name, node = node).save()
# Generate node added event
Event.create_event(node, EventCode.NodeAdded, '', EventSource.NodeDatabase,
data = 'Maintainer: %s' % node.owner.username)
self.node = node
return node
def save(self):
"""
Performs the actual renumbering.
"""
# We must ensure exclusive access during node updates as otherwise this might happen
# in the middle of a monitor update and this would cause unwanted consequences
self.__node.ensure_exclusive_access()
# Determine what subnet primary IP belonged to
primary = self.__node.get_primary_subnet()
renumber_primary = False
old_router_id = self.__node.ip
# Renumber subnets first
for subnet in queryset_by_ip(self.__node.subnet_set.filter(allocated = True), 'ip_subnet')[:]:
action = int(self.cleaned_data.get('subnet_%s' % subnet.pk))
prefix_len = int(self.cleaned_data.get('prefix_%s' % subnet.pk) or 27)
manual_ip = self.cleaned_data.get('manual_%s' % subnet.pk)
if action == RenumberAction.Keep:
pass
elif action == RenumberAction.Remove:
subnet.delete()
else:
# This means we should renumber to some other pool
pool = Pool.objects.get(pk = action)
if manual_ip:
new_subnet = pool.reserve_subnet(manual_ip, prefix_len)
else:
new_subnet = pool.allocate_subnet(prefix_len)
# If the old subnet has been the source of node's primary IP remember that
save_primary = (not renumber_primary and primary and primary[0] == subnet)
# Remove old subnet and create a new one; it is deleted here so the old allocation
# is returned to the pool and all status info is reset
subnet.delete()
s = Subnet(node = self.__node, subnet = new_subnet.network, cidr = new_subnet.cidr)
s.allocated = True
s.allocated_at = datetime.now()
s.status = SubnetStatus.NotAnnounced
s.description = subnet.description
s.gen_iface_type = subnet.gen_iface_type
s.gen_dhcp = subnet.gen_dhcp
s.save()
if save_primary:
primary = s
renumber_primary = True
# The subnet have now been renumbered, check if we need to renumber the primary IP
router_id_changed = False
if renumber_primary:
net = ipcalc.Network(primary.subnet, primary.cidr)
self.__node.ip = str(net.host_first())
router_id_changed = True
# Remove conflicting invalid nodes (another node with the IP we just renumbered to)
existing_nodes = Node.objects.filter(ip = self.__node.ip, status = NodeStatus.Invalid)
if existing_nodes.count() > 0:
self.warning_or_continue(_("There is an existing but unregistered node with the same primary IP address you are currently renumbering to! If you continue with this operation, this invalid node will be replaced."))
existing_nodes.delete()
# Node has been renumbered, reset monitoring status as this node is obviously not
# visible right after renumbering.
if router_id_changed:
# Update node's DNS record
Record.update_for_node(self.__node)
if not self.__node.is_pending():
self.__node.status = NodeStatus.Down
self.__node.peers = 0
Link.objects.filter(src = self.__node).delete()
Link.objects.filter(dst = self.__node).delete()
self.__node.subnet_set.filter(allocated = False).delete()
self.__node.subnet_set.all().update(status = SubnetStatus.NotAnnounced)
# Setup a node renumbered notice (if one doesn't exist yet)
try:
notice = RenumberNotice.objects.get(node = self.__node)
if notice.original_ip == self.__node.ip:
notice.delete()
self.__node.awaiting_renumber = False
else:
self.__node.awaiting_renumber = True
except RenumberNotice.DoesNotExist:
notice = RenumberNotice(node = self.__node)
notice.original_ip = old_router_id
notice.renumbered_at = datetime.now()
notice.save()
self.__node.awaiting_renumber = True
self.__node.save()
# Generate node renumbered event
Event.create_event(self.__node, EventCode.NodeRenumbered, '', EventSource.NodeDatabase,
data = 'Old address: %s' % old_router_id)
