def sync_prefixes(self, prefixes):
"""
Apply prefixes to database
:param prefixes:
:return:
"""
# vpn_id -> [prefix, ]
vrf_prefixes = defaultdict(list)
for vpn_id, p in prefixes:
vrf_prefixes[vpn_id] += [p]
# build vpn_id -> VRF mapping
self.logger.debug("Building VRF map")
vrfs = {}
for vpn_id in vrf_prefixes:
vrf = VRF.get_by_vpn_id(vpn_id)
if vrf:
vrfs[vpn_id] = vrf
missed_vpn_id = set(vrf_prefixes) - set(vrfs)
if missed_vpn_id:
self.logger.info("RD missed in VRF database and to be ignored: %s", ", ".join(missed_vpn_id))
#
self.logger.debug("Getting prefixes to synchronize")
for vpn_id in vrfs:
vrf = vrfs[vpn_id]
seen = set()
for p in Prefix.objects.filter(vrf=vrf, prefix__in=vrf_prefixes[vpn_id]):
norm_prefix = IP.expand(p.prefix)
# Confirmed prefix, apply changes and touch
prefix = prefixes[vpn_id, norm_prefix]
self.apply_prefix_changes(p, prefix)
seen.add(norm_prefix)
for p in set(vrf_prefixes[vpn_id]) - seen:
# New prefix, create
self.create_prefix(prefixes[vpn_id, p])
def create_address(self, address):
"""
Create new address
:param address: DiscoveredAddress instance
:return:
"""
if self.is_ignored_address(address):
return
vrf = VRF.get_by_vpn_id(address.vpn_id)
self.ensure_afi(vrf, address)
if not self.has_address_permission(vrf, address):
self.logger.debug(
"Do not creating vpn_id=%s address=%s: Disabled by policy",
address.vpn_id, address.address)
metrics["address_creation_denied"] += 1
return
a = Address(vrf=vrf,
address=address.address,
name=self.get_address_name(address),
fqdn=address.fqdn,
profile=address.profile,
description=address.description,
source=address.source,
mac=address.mac)
if address.source in LOCAL_SRC:
a.managed_object = self.object
a.subinterface = address.subinterface
self.logger.info(
"Creating address %s (%s): name=%s fqdn=%s mac=%s profile=%s source=%s",
a.address, a.vrf.name, a.name, a.fqdn, a.mac, a.profile.name,
a.source)
a.save()
self.fire_seen(a)
metrics["address_created"] += 1
def sync_ipam(self):
"""
Synchronize FQDN and address with IPAM
"""
from noc.ip.models.address import Address
from noc.ip.models.vrf import VRF
# Generate FQDN from template
fqdn = self.object_profile.get_fqdn(self)
# Get existing IPAM record
vrf = self.vrf if self.vrf else VRF.get_global()
try:
a = Address.objects.get(vrf=vrf, address=self.address)
except Address.DoesNotExist:
# Create new address
Address(
vrf=vrf,
address=self.address,
fqdn=fqdn,
managed_object=self
).save()
return
# Update existing address
if (a.managed_object != self or
a.address != self.address or a.fqdn != fqdn):
a.managed_object = self
a.address = self.address
a.fqdn = fqdn
a.save()
def create_vpn(self, vpn):
"""
Create new vpn
:param vpn: DiscoveredVPN instance
:return:
"""
if not self.has_vpn_permission(vpn):
self.logger.debug(
"Do not creating rd=%s vpn_id=%s name=%s Disabled by policy",
vpn.rd, vpn.vpn_id, vpn.name)
metrics["vpn_creation_denied"] += 1
return
name = self.get_vpn_name(vpn)
# Check for naming clash
if VRF.objects.filter(name=name).exists():
# Naming clash
old_name = name
name = self.get_unique_vpn_name(vpn)
self.logger.info(
"Name '%s' is already exists with other vpn_id. Rename to '%s'",
old_name, name)
metrics["vpn_name_clash"] += 1
#
p = VRF(name=name,
rd=vpn.rd,
vpn_id=vpn.vpn_id,
profile=vpn.profile,
source=vpn.source)
self.logger.info("Creating vpn %s: name=%s rd=%s profile=%s source=%s",
p.vpn_id, p.name, p.rd, p.profile.name, p.source)
p.save()
p.fire_event("seen")
metrics["vpn_created"] += 1
def api_bulk_import(self, request, items):
n = 0
for i in items:
if not VRF.objects.filter(name=i["name"], rd=i["rd"]).exists():
# Add only new
VRF(name=i["name"],
vrf_group=i["vrf_group"],
rd=i["rd"],
afi_ipv4=i["afi_ipv4"],
afi_ipv6=i["afi_ipv6"],
description=i.get("description")).save()
n += 1
return {"status": True, "imported": n}
def unregister_dynamic_usage(self,
vrf=None,
name="default",
technology="IPoE"):
"""
Decrease dynamic pool usage
"""
## Avoid circular references
from noc.ip.models.vrf import VRF
from noc.ip.models.dynamicippoolusage import DynamicIPPoolUsage
self._check_technology(technology)
if not vrf:
vrf = VRF.get_global()
DynamicIPPoolUsage.unregister_usage(self, vrf, name, technology)
def get_data(self, **kwargs):
vrf_id = VRF.get_global().id
return self.from_query(title=self.title,
columns=[_("Prefix")],
query="""
SELECT prefix
FROM ip_prefix p
WHERE
vrf_id=%s
AND afi='4'
AND masklen(prefix)=30
AND NOT EXISTS (SELECT * FROM ip_address WHERE vrf_id=%s AND afi='4' AND prefix=p.prefix)
ORDER BY prefix
""",
params=[vrf_id, vrf_id],
enumerate=True)
def sync_addresses(self, addresses):
"""
Apply addresses to database
:param addresses:
:return:
"""
# vpn_id -> [address, ]
vrf_addresses = defaultdict(list)
for vpn_id, a in addresses:
vrf_addresses[vpn_id] += [a]
# build vpn_id -> VRF mapping
self.logger.debug("Building VRF map")
vrfs = {}
for vpn_id in vrf_addresses:
vrf = VRF.get_by_vpn_id(vpn_id)
if vrf:
vrfs[vpn_id] = vrf
missed_vpn_id = set(vrf_addresses) - set(vrfs)
if missed_vpn_id:
self.logger.info(
"VPN ID are missed in VRF database and to be ignored: %s",
", ".join(missed_vpn_id))
#
self.logger.debug("Getting addresses to synchronize")
for vpn_id in vrfs:
vrf = vrfs[vpn_id]
seen = set()
for a in Address.objects.filter(vrf=vrf,
address__in=vrf_addresses[vpn_id]):
norm_address = IP.expand(a.address)
# Confirmed address, apply changes and touch
address = addresses[vpn_id, norm_address]
self.apply_address_changes(a, address)
seen.add(norm_address)
for a in set(vrf_addresses[vpn_id]) - seen:
# New address, create
self.create_address(addresses[vpn_id, a])
# Detaching hanging addresses
self.logger.debug("Checking for hanging addresses")
for a in Address.objects.filter(managed_object=self.object):
norm_address = IP.expand(a.address)
address = addresses.get((a.vrf.vpn_id, norm_address))
if not address or address.source not in LOCAL_SRC:
self.logger.info("Detaching %s:%s", a.vrf.name, a.address)
a.managed_object = None
a.save()
def get_or_create(self, object, name, rd):
"""
:param object:
:param name:
:param rd:
:return:
"""
def set_cache(vrf):
self.cache_vrf_by_rd[vrf.rd] = vrf
self.cache_vrf_by_name[vrf.name] = vrf
return vrf
if name == "default":
if object.vrf:
# Use object's VRF is set
return object.vrf
# Get default VRF
try:
return self.cache_vrf_by_name["default"]
except KeyError:
return set_cache(VRF.get_global())
# Non-default VRF
if not rd:
rd = VRF.generate_rd(name)
# Lookup RD cache
try:
return self.cache_vrf_by_rd[rd]
except KeyError:
pass
# Lookup database
try:
return set_cache(VRF.objects.get(rd=rd))
except VRF.DoesNotExist:
pass
# VRF Not found, create
# Generate unique VRF in case of names clash
vrf_name = name
if VRF.objects.filter(name=vrf_name).exists():
# Name clash, generate new name by appending RD
vrf_name += "_%s" % rd
self.info(
object,
"Conflicting names for VRF %s. Using fallback name %s" %
(name, vrf_name))
# Create VRF
vrf = VRF(name=vrf_name, rd=rd, vrf_group=VRF.get_global().vrf_group)
vrf.save()
return set_cache(vrf)
def get_data(self, request, **kwargs):
vrf_id = VRF.get_global().id
return self.from_query(title=self.title,
columns=["FQDN", "AFI", "N", "Addresses"],
query="""
SELECT fqdn, afi, COUNT(*),
array_to_string(ARRAY(SELECT address
FROM ip_address
WHERE fqdn=a.fqdn
AND vrf_id=%s
AND afi=a.afi
ORDER BY address),', ')
FROM ip_address a
WHERE vrf_id=%s
GROUP BY 1, 2
HAVING COUNT(*)>1
ORDER BY 3 DESC""",
params=[vrf_id, vrf_id])
def get_usage(cls,
termination_group,
vrf=None,
pool_name="default",
technology="IPoE"):
if not vrf:
vrf = VRF.get_global()
r = cls._get_collection().find_one(
{
"termination_group": termination_group.id,
"vrf": vrf.id,
"pool_name": pool_name,
"technology": technology
}, {"usage": 1})
if r:
return r["usage"]
else:
return 0
def unregister_usage(cls,
termination_group,
vrf=None,
pool_name="default",
technology="IPoE"):
"""
Decrease usage counter
"""
if not vrf:
vrf = VRF.get_global()
cls._get_collection().update(
{
"termination_group": termination_group.id,
"vrf": vrf.id,
"pool_name": pool_name,
"technology": technology
}, {"$inc": {
"usage": -1
}},
upsert=True)
def create_prefix(self, prefix):
"""
Create new prefix
:param prefix: DiscoveredPrefix instance
:return:
"""
if self.is_ignored_prefix(prefix):
return
vrf = VRF.get_by_vpn_id(prefix.vpn_id)
self.ensure_afi(vrf, prefix)
if not self.has_prefix_permission(vrf, prefix):
self.logger.debug(
"Do not creating vpn_id=%s asn=%s prefix=%s: Disabled by policy",
prefix.vpn_id,
prefix.asn.asn if prefix.asn else None,
prefix.prefix,
)
metrics["prefix_creation_denied"] += 1
return
p = Prefix(
vrf=vrf,
prefix=prefix.prefix,
name=self.get_prefix_name(prefix),
profile=prefix.profile,
asn=prefix.asn,
description=prefix.description,
source=prefix.source,
)
self.logger.info(
"Creating prefix %s (%s): name=%s profile=%s source=%s",
p.prefix,
p.vrf.name,
p.name,
p.profile.name,
p.source,
)
p.save()
self.fire_seen(p)
metrics["prefix_created"] += 1
def get_data(self, **kwargs):
def reverse_format(p):
n, m = p.split("/")
n = n.split(".")[:-1]
n.reverse()
return "%s.%s.%s.in-addr.arpa" % (n[0], n[1], n[2])
vrf_id = VRF.get_global().id
return self.from_query(
title=self.title,
columns=["Prefix",
TableColumn("Zone", format=reverse_format)],
query="""
SELECT prefix,prefix
FROM ip_prefix
WHERE vrf_id=%s
AND masklen(prefix)=24
AND regexp_replace(host(prefix),E'([0-9]+)\\\\.([0-9]+)\\\\.([0-9]+)\\\\.([0-9]+)',E'\\\\3.\\\\2.\\\\1.in-addr.arpa')
NOT IN (SELECT name FROM dns_dnszone WHERE name LIKE '%%.in-addr.arpa')
ORDER BY 1
""",
params=[vrf_id],
enumerate=True)
def import_zone(
self,
path=None,
axfr=False,
zone_profile=None,
address_profile=None,
transfer_zone=None,
nameserver=None,
source_address=None,
dry_run=False,
force=False,
clean=False,
):
self.print("Loading zone file '%s'" % path)
self.print("Parsing zone file using BIND parser")
if path:
with open(path) as f:
rrs = self.iter_bind_zone_rr(f)
try:
soa = next(rrs)
except StopIteration:
raise CommandError("Unable to parse zone file from %s" %
path)
zone = self.from_idna(soa.zone)
z = self.dns_zone(zone, zone_profile, dry_run, clean)
# Populate zone
vrf = VRF.get_global()
zz = zone + "."
lz = len(zz)
if z.is_forward:
zp = None
elif z.is_reverse_ipv4:
# Calculate prefix for reverse zone
zp = ".".join(reversed(zone[:-13].split("."))) + "."
elif z.is_reverse_ipv6:
raise CommandError(
"IPv6 reverse import is not implemented")
else:
raise CommandError("Unknown zone type")
for rr in rrs:
name = rr.name
if name.endswith(zz):
name = name[:-lz]
if name.endswith("."):
name = name[:-1]
# rr = None
# Skip zone NS
if rr.type == "NS" and not name:
continue
if rr.type in ("A", "AAAA"):
self.create_address(
zone,
vrf,
rr.rdata,
"%s.%s" % (name, zone) if name else zone,
address_profile,
dry_run=dry_run,
force=force,
)
elif rr.type == "PTR":
if "." in name:
address = zp + ".".join(reversed(name.split(".")))
else:
address = zp + name
self.create_address(
zone,
vrf,
address,
rr.rdata,
address_profile,
dry_run=dry_run,
force=force,
)
else:
zrr = DNSZoneRecord(
zone=z,
name=name,
type=rr.type,
ttl=rr.ttl,
priority=rr.priority,
content=rr.rdata,
)
self.print("Creating %s %s" % (rr.type, rr.name))
if not dry_run:
zrr.save()
if axfr:
data = self.load_axfr(nameserver, transfer_zone, source_address)
zone = self.from_idna(transfer_zone)
z = self.dns_zone(zone, zone_profile, dry_run, clean)
# Populate zone
vrf = VRF.get_global()
zz = zone + "."
lz = len(zz)
if z.is_forward:
zp = None
elif z.is_reverse_ipv4:
# Calculate prefix for reverse zone
zp = ".".join(reversed(zone[:-13].split("."))) + "."
elif z.is_reverse_ipv6:
raise CommandError("IPv6 reverse import is not implemented")
else:
raise CommandError("Unknown zone type")
for row in data:
row = row.strip()
if row == "" or row.startswith(";"):
continue
row = row.split()
if len(row) != 5 or row[2] != "IN" or row[3] not in ("A",
"AAAA",
"PTR"):
continue
if row[3] in ("A", "AAAA"):
name = row[0]
if name.endswith(zz):
name = name[:-lz]
if name.endswith("."):
name = name[:-1]
self.create_address(
zone,
vrf,
row[4],
"%s.%s" % (name, zone) if name else zone,
address_profile,
dry_run=dry_run,
force=force,
)
if row[3] == "PTR":
name = row[4]
if name.endswith(zz):
name = name[:-lz]
if name.endswith("."):
name = name[:-1]
# @todo: IPv6
if "." in row[0]:
address = ".".join(reversed(row[0].split(".")[:-3]))
else:
address = zp + name
fqdn = row[4]
if fqdn.endswith("."):
fqdn = fqdn[:-1]
self.create_address(zone,
vrf,
address,
fqdn,
address_profile,
dry_run=dry_run,
force=force)
def import_zone(self,
path,
zone_profile,
address_profile,
dry_run=False,
force=False,
clean=False):
self.print("Loading zone file '%s'" % path)
self.print("Parsing zone file using BIND parser")
with open(path) as f:
rrs = self.iter_bind_zone_rr(f)
try:
soa = next(rrs)
except StopIteration:
raise CommandError("Unable to parse zone file from %s" % path)
zone = self.from_idna(soa.zone)
z = DNSZone.get_by_name(zone)
if z:
self.print("Using existing zone '%s'" % zone)
else:
self.print("Creating zone '%s'" % zone)
z = DNSZone(name=zone, profile=zone_profile)
clean = False # Nothing to clean
if z.profile.id != zone_profile.id:
self.print("Setting profile to '%s'" % zone_profile.name)
z.profile = zone_profile
# Apply changes
if dry_run:
z.clean() # Set type
else:
z.save()
# Clean zone when necessary
if clean:
self.print("Cleaning zone")
for rr in DNSZoneRecord.objects.filter(zone=z):
self.print("Removing %s %s" % (rr.type, rr.name))
if not dry_run:
rr.delete()
# Populate zone
vrf = VRF.get_global()
zz = zone + "."
lz = len(zz)
if z.is_forward:
zp = None
elif z.is_reverse_ipv4:
# Calculate prefix for reverse zone
zp = ".".join(reversed(zone[:-13].split("."))) + "."
elif z.is_reverse_ipv6:
raise CommandError("IPv6 reverse import is not implemented")
else:
raise CommandError("Unknown zone type")
for rr in rrs:
name = rr.name
if name.endswith(zz):
name = name[:-lz]
if name.endswith("."):
name = name[:-1]
# rr = None
# Skip zone NS
if rr.type == "NS" and not name:
continue
if rr.type in ("A", "AAAA"):
self.create_address(
zone,
vrf,
rr.rdata,
"%s.%s" % (name, zone) if name else zone,
address_profile,
dry_run=dry_run,
force=force,
)
elif rr.type == "PTR":
if "." in name:
address = zp + ".".join(reversed(name.split(".")))
else:
address = zp + name
self.create_address(zone,
vrf,
address,
rr.rdata,
address_profile,
dry_run=dry_run,
force=force)
else:
zrr = DNSZoneRecord(
zone=z,
name=name,
type=rr.type,
ttl=rr.ttl,
priority=rr.priority,
content=rr.rdata,
)
self.print("Creating %s %s" % (rr.type, rr.name))
if not dry_run:
zrr.save()
def import_zone(self, path, options):
is_test = bool(options["test"])
self.info("Loading zone file '%s'" % path)
with open(path) as f:
data = f.read()
self.info("Parsing zone file using BIND parser")
zone, rrs = self.parse_bind_zone(data)
if not zone or not rrs:
raise CommandError("Unable to parse zone file")
# Find profile
profile = self.get_profile(options)
# Get or create zone
to_clean = bool(options["clean"])
try:
z = DNSZone.objects.get(name=zone)
self.info("Using existing zone '%s'" % z)
except DNSZone.DoesNotExist:
self.info("Creating zone '%s'" % zone)
z = DNSZone(name=zone, profile=profile)
to_clean = False # Nothing to clean yet
if z.profile != profile:
self.info("Setting profile to '%s'" % profile)
z.profile = profile
if not is_test:
z.save()
# Clean zone when necessary
if to_clean and z.id:
self.info("Cleaning zone")
for rr in z.dnszonerecord_set.all():
self.info("Removing %s %s" % (rr.type, rr.name))
if not is_test:
rr.delete()
# Populate zone
vrf = VRF.get_global()
zz = zone + "."
lz = len(zz)
if zone.endswith(".in-addr.arpa"):
# Calculate prefix for reverse zone
zp = zone[:-13].split(".")
zp.reverse()
zp = ".".join(zp) + "."
else:
# @todo: IPv6 reverse
zp = None
for name, t, value, ttl, priority in rrs:
# print name, t, value
if name.endswith(zz):
name = name[:-lz]
if name.endswith("."):
name = name[:-1]
rr = None
if (t == "SOA") or (t == "NS" and not name):
continue
if t in ("A", "AAAA"):
afi = "4" if t == "A" else "6"
self.create_address(
zone, vrf, afi, value,
"%s.%s" % (name, zone) if name else zone, is_test)
elif t == "PTR":
if not zp:
raise CommandError("IPv6 reverse zone import is still not supported")
address = zp + name
afi = "6" if ":" in address else "4"
self.create_address(zone, vrf, afi, address, value, is_test)
else:
rr = DNSZoneRecord(
zone=z, name=name, type=t,
ttl=ttl, priority=priority,
content=value
)
if rr:
self.info("Creating %s %s" % (rr.type, rr.name))
if not is_test:
rr.save()