class RoutingNeighbors(Model):
class Meta:
db_table = "mac"
engine = MergeTree("date", ("managed_object", "ts"))
date = DateField(description=_("Date"))
ts = DateTimeField(description=_("Created"))
managed_object = ReferenceField(
ManagedObject,
description=_("Object Name")
)
interface = StringField(
description=_("Physical Interface")
)
subinterface = StringField(
description=_("Logical Interface")
)
neighbor_address = StringField(
description=_("Neighbor Address")
)
neighbor_object = ReferenceField(
ManagedObject,
description=_("Neighbor Object")
)
protocol = StringField(
description=_("Protocol")
)
bgp_local_as = UInt64Field(
description=_("BGP Local AS")
)
bgp_remote_as = UInt64Field(
description=_("BGP Remogte AS")
)
class NetworkSegment(Dictionary):
class Meta(object):
name = "networksegment"
layout = "hashed"
name = StringField()
parent = ReferenceField("self")
class Syslog(Model):
class Meta(object):
db_table = "syslog"
engine = MergeTree("date", ("managed_object", "ts"))
date = DateField(description=_("Date"))
ts = DateTimeField(description=_("Created"))
managed_object = ReferenceField(ManagedObject, description=_("Object Name"))
facility = UInt8Field(description=_("Facility"))
severity = UInt8Field(description=_("Severity"))
message = StringField(description=_("Syslog Message"))
@classmethod
def transform_query(cls, query, user):
if not user or user.is_superuser:
return query # No restrictions
# Get user domains
domains = UserAccess.get_domains(user)
# Resolve domains against dict
domain_ids = [x.bi_id for x in AdministrativeDomainM.objects.filter(id__in=domains)]
filter = query.get("filter", {})
dl = len(domain_ids)
if not dl:
return None
elif dl == 1:
q = {"$eq": [{"$field": "administrative_domain"}, domain_ids[0]]}
else:
q = {"$in": [{"$field": "administrative_domain"}, domain_ids]}
if filter:
query["filter"] = {"$and": [query["filter"], q]}
else:
query["filter"] = q
return query
class AdministrativeDomain(Dictionary):
class Meta(object):
name = "administrativedomain"
layout = "flat"
name = StringField()
parent = ReferenceField("self")
class Container(Dictionary):
class Meta(object):
name = "container"
layout = "hashed"
id = StringField()
name = StringField()
parent = ReferenceField("self")
address_text = StringField()
class Reboots(Model):
class Meta:
db_table = "reboots"
engine = MergeTree("date", ("ts", "managed_object"))
date = DateField(description=_("Date"))
ts = DateTimeField(description=_("Created"))
managed_object = ReferenceField(ManagedObject, description=_("Object Name"))
pool = ReferenceField(Pool, description=_("Pool Name"))
ip = IPv4Field(description=_("IP Address"))
profile = ReferenceField(Profile, description=_("Profile"))
object_profile = ReferenceField(ObjectProfile, description=_("Object Profile"))
vendor = ReferenceField(Vendor, description=_("Vendor Name"))
platform = ReferenceField(Platform, description=_("Platform"))
version = ReferenceField(Version, description=_("Version"))
administrative_domain = ReferenceField(AdministrativeDomain, description=_("Admin. Domain"))
segment = ReferenceField(NetworkSegment, description=_("Network Segment"))
container = ReferenceField(Container, description=_("Container"))
# Coordinates
x = Float64Field(description=_("Longitude"))
y = Float64Field(description=_("Latitude"))
@classmethod
def transform_query(cls, query, user):
if not user or user.is_superuser:
return query # No restrictions
# Get user domains
domains = UserAccess.get_domains(user)
# Resolve domains against dict
domain_ids = [x.bi_id for x in AdministrativeDomainM.objects.filter(id__in=domains)]
filter = query.get("filter", {})
dl = len(domain_ids)
if not dl:
return None
elif dl == 1:
q = {"$eq": [{"$field": "administrative_domain"}, domain_ids[0]]}
else:
q = {"$in": [{"$field": "administrative_domain"}, domain_ids]}
if filter:
query["filter"] = {"$and": [query["filter"], q]}
else:
query["filter"] = q
return query
class MAC(Model):
"""
MAC address table snapshot
Common queries:
Last seen MAC location:
SELECT timestamp, object, interface
FROM mac
WHERE
date >= ?
AND mac = ?
AND uni = 1
ORDER BY timestamp DESC LIMIT 1;
All MAC locations for date interval:
SELECT timestamp, object, interface
FROM mac
WHERE
date >= ?
AND mac = ?
AND uni = 1;
"""
class Meta(object):
db_table = "mac"
engine = MergeTree("date", ("ts", "managed_object"))
date = DateField(description=_("Date"))
ts = DateTimeField(description=_("Created"))
managed_object = ReferenceField(ManagedObject,
description=_("Object Name"))
mac = UInt64Field(description=_("MAC"))
interface = StringField(description=_("Interface"))
interface_profile = ReferenceField(InterfaceProfile,
description=_("Interface Profile"))
segment = ReferenceField(NetworkSegment, description=_("Network Segment"))
vlan = UInt16Field(description=_("VLAN"))
is_uni = UInt8Field(description=_("Is UNI"))
def mac_filter(self, query, offset=0, limit=400, convert_mac=False):
"""
Filter interface to MACDB
:param query: Query to MACDB
:param query: dict
:param offset: Offset output data
:type offset: int
:param limit: Offset output data
:type limit: limit data count
:param convert_mac: Conver MAC from int to str represent
:return: list query result from MACDB
select max(ts), managed_object, interface, vlan from mac
where like(MACNumToString(mac), 'A0:AB:1B%') group by managed_object, interface, vlan;
"""
query_field = ["mac", "managed_object"]
t0 = datetime.datetime.now() - datetime.timedelta(
seconds=config.web.macdb_window)
t0 = t0.replace(microsecond=0)
if not query:
return
f_filter = {
"$and": [
{
"$gte": [{
"$field": "date"
}, t0.date().isoformat()]
},
{
"$gte": [{
"$field": "ts"
}, t0.isoformat(sep=" ")]
},
]
}
for k in query:
field = k
q = "eq"
if "__" in field:
field, q = k.split("__")
if field not in query_field:
# Not implemented
continue
if q == "like" and not convert_mac:
field = "MACNumToString(mac)"
# @todo convert mac all
if isinstance(query[k], list) and len(query[k]) == 1:
arg = query[k][0].strip()
elif isinstance(query[k], six.string_types):
arg = query[k].strip()
else:
arg = query[k]
f_filter["$and"] += [{"$%s" % q: [{"$field": field}, arg]}]
if not f_filter:
return
fields = [
{
"expr": "argMax(ts, ts)",
"alias": "timestamp",
"order": 0
},
{
"expr": "mac",
"alias": "mac",
"group": 1
},
{
"expr": "vlan",
"alias": "vlan",
"group": 2
},
{
"expr": "managed_object",
"alias": "managed_object",
"group": 3
},
{
"expr": "argMax(interface, ts)",
"alias": "interface"
},
]
if convert_mac:
fields[1]["expr"] = "MACNumToString(mac)"
# @todo paging (offset and limit)
# @todo check in list
ch_query = {"fields": fields, "filter": f_filter, "limit": limit}
if offset:
ch_query["offset"] = offset
res = self.query(ch_query)
# print res
for r in res["result"]:
yield dict(zip(res["fields"], r))
def get_neighbors_by_mac(self, macs, mos=None):
"""
Return list BI ID MO by interfaces. Filter mo by macs
:param macs: list(int)
:param mos: list(int)
:return: Dict {mo_a: {iface1: [mo1, mo2], iface2: [mo3, mo4], ...}, mo_b: ...}
"""
if not macs and not mos:
return
neighbors = defaultdict(dict)
if mos:
query = {"managed_object__in": mos}
else:
query = {"mac__in": macs}
for r in self.mac_filter(query):
agg = int(r["managed_object"])
if r["interface"] in neighbors[agg]:
neighbors[agg][r["interface"]] += [int(r["mac"])]
else:
neighbors[agg][r["interface"]] = [int(r["mac"])]
return neighbors
def get_mac_history(self, mac):
return []
class ManagedObject(Model):
class Meta(object):
db_table = "managedobjects"
engine = MergeTree("date", ("managed_object", "ts"))
date = DateField(description=_("Date"))
ts = DateTimeField(description=_("Created"))
managed_object = ReferenceField(ManagedObjectDict,
description=_("Object Name"))
# Location
administrative_domain = ReferenceField(AdministrativeDomain,
description=_("Admin. Domain"))
segment = ReferenceField(NetworkSegment, description=_("Network Segment"))
container = ReferenceField(Container, description=_("Container"))
location = StringField(description="Geo location")
level = UInt16Field(description=_("Level"))
# Coordinates
x = Float64Field(description=_("Longitude"))
y = Float64Field(description=_("Latitude"))
# Management
pool = ReferenceField(Pool, description=_("Pool Name"))
name = StringField(description=_("Name"))
hostname = StringField(description=_("Hostaname"))
ip = IPv4Field(description=_("IP Address"))
is_managed = BooleanField(description=_("Is Managed"))
# Platform
profile = ReferenceField(Profile, description=_("Profile"))
vendor = ReferenceField(Vendor, description=_("Vendor"))
platform = ReferenceField(Platform, description=_("Platform"))
hw_version = StringField(description=_("HW. Version"))
version = ReferenceField(Version, description=_("Version"))
bootprom_version = StringField(description=_("BootPROM. Version"))
n_interfaces = Int32Field(description=_("Interface count"))
n_subscribers = Int32Field(description=_("Subscribers count"))
n_services = Int32Field(description=_("Services count"))
# Topology
n_neighbors = Int32Field(description=_("Neighbors"))
n_links = Int32Field(description=_("Links count"))
nri_links = Int32Field(description=_("Links (NRI)"))
mac_links = Int32Field(description=_("Links (MAC)"))
stp_links = Int32Field(description=_("Links (STP)"))
lldp_links = Int32Field(description=_("Links (LLDP)"))
cdp_links = Int32Field(description=_("Links (CDP)"))
# Capabilities
has_stp = BooleanField(description=_("Has STP"))
has_lldp = BooleanField(description=_("Has LLDP"))
has_cdp = BooleanField(description=_("Has CDP"))
has_snmp = BooleanField(description=_("Has SNMP"))
has_snmp_v1 = BooleanField(description=_("Has SNMP v1"))
has_snmp_v2c = BooleanField(description=_("Has SNMP v2c"))
# Counter
uptime = Float64Field(description=_("Uptime"))
# SerialNumber
serials = ArrayField(StringField(), description=_("Serial Numbers"))
# Tags
tags = ArrayField(StringField(), description=_("Tags"))
@classmethod
def transform_query(cls, query, user):
if not user or user.is_superuser:
return query # No restrictions
# Get user domains
domains = UserAccess.get_domains(user)
# Resolve domains against dict
domain_ids = [
x.bi_id
for x in AdministrativeDomainM.objects.filter(id__in=domains)
]
filter = query.get("filter", {})
dl = len(domain_ids)
if not dl:
return None
elif dl == 1:
q = {"$eq": [{"$field": "administrative_domain"}, domain_ids[0]]}
else:
q = {"$in": [{"$field": "administrative_domain"}, domain_ids]}
if filter:
query["filter"] = {"$and": [query["filter"], q]}
else:
query["filter"] = q
return query
class Flows(Model):
"""
Netflow collector model
"""
class Meta:
db_table = "flows"
engine = MergeTree("date", ("ts", "managed_object"))
date = DateField(description=_("Date"))
ts = DateTimeField(description=_("Created"))
managed_object = ReferenceField(ManagedObject,
description=_("Object Name"))
# Coordinates
x = Float64Field(description=_("Longitude"))
y = Float64Field(description=_("Latitude"))
#
version = UInt8Field(description=_("Flow version"))
in_bytes = UInt64Field(description=_("Incoming octets"))
in_pkts = UInt64Field(description=_("Incoming packets"))
flows = UInt16Field(description=_("Aggregated flows"))
protocol = UInt8Field(description=_("IP protocol type"))
src_tos = UInt8Field(description=_("Type of Service byte"))
tcp_flags = UInt8Field(description=_("TCP flags cumulative byte"))
l4_src_port = UInt8Field(description=_("TCP/UDP src port number"))
ipv4_src_addr = IPv4Field(description=_("IPv4 source address"))
src_mask = UInt8Field(description=_("Number of mask bits in src adr"))
input_snmp = UInt16Field(description=_("Input interface index"))
ipv4_dst_port = UInt8Field(description=_("TCP/UDP dst port number"))
ipv4_dst_addr = IPv4Field(description=_("IPv4 destination address"))
dst_mask = UInt8Field(description=_("Number of mask bits in dst adr"))
output_snmp = UInt16Field(description=_("Output interface index"))
ipv4_next_hop = IPv4Field(description=_("IPv4 adr of nexthop router"))
src_as = UInt32Field(description=_("Src BGP AS number"))
dst_as = UInt32Field(description=_("Dst BGP AS number"))
bgp_ipv4_next_hop = UInt32Field(
description=_("Nexthop router's IP in BGP domain"))
mul_dst_pkts = UInt64Field(
description=_("IP multicast outgoing packet counter"))
mul_dst_bytes = UInt64Field(
description=_("IP multicast outgoing packet counter"))
last_switched = UInt32Field(
description=_("System uptime when last pkt of flow was switched"))
first_switched = UInt32Field(
description=_("System uptime when last pkt of flow was switched"))
out_bytes = UInt64Field(description=_("Outgoing octets"))
out_pkts = UInt64Field(description=_("Outgoing packets"))
min_pkt_lngth = UInt16Field(
description=_("Min IP pkt length on inc packets of flow"))
max_pkt_lngth = UInt16Field(
description=_("Max IP pkt length on inc packets of flow"))
# ipv6_src_addr = IPv6Field(description=_("IPv6 source address"))
# ipv6_dst_addr = IPv6Field(description=_("IPv6 destination address"))
# ipv6_src_mask = UInt8Field(description=_("Length of IPv6 source mask in contiguous bits"))
# ipv6_dst_mask = UInt8Field(description=_("Length of IPv6 destination mask in contiguous bits"))
# ipv6_flow_label = UInt32Field(description=_("IPv6 flow label as in RFC2460"))
icmp_type = UInt16Field(description=_("ICMP packet type"))
mul_igmp_type = UInt8Field(description=_("IGMP packet type"))
sampling_interval = UInt32Field(
description=_("Rate of sampling of packets"))
sampling_algorithm = UInt8Field(description=_(
"Type of algorithm for sampled netflow: 01 deterministic or 02 random sampling"
))
flow_active_timeout = UInt16Field(description=_(
"Timeout value in seconds for active entries in netflow cache"))
flow_inactive_timeout = UInt16Field(description=_(
"Timeout value in seconds for inactive entries in netflow cache"))
engine_type = UInt8Field(
description=_("Type of flow switching engine: RP=0, VIP/Linecard=1"))
engine_id = UInt8Field(description=_("ID number of flow switching engine"))
total_bytes_exp = UInt32Field(description=_("Counter for bytes exported"))
total_pkts_exp = UInt32Field(description=_("Counter for pkts exported"))
total_flows_exp = UInt32Field(description=_("Counter for flows exported"))
ipv4_src_prefix = UInt32Field(
description=_("IPv4 source address prefix (Catalyst architecture)"))
ipv4_dst_prefix = UInt32Field(description=_(
"IPv4 destination address prefix (Catalyst architecture)"))
mpls_top_label_type = UInt8Field(description=_("MPLS top label type"))
mpls_top_label_ip_addr = UInt32Field(description=_(
"Forwarding equivalent class corresponding to MPLS top label"))
flow_sampler_id = UInt8Field(description=_("ID from 'show flow-sampler'"))
flow_sampler_mode = UInt8Field(
description=_("Type of algorithm used for sampling data"))
flow_sampler_random_interval = UInt32Field(
description=_("Packet interval at which to sample"))
min_ttl = UInt8Field(description=_("Minimum TTL on incoming packets"))
max_ttl = UInt8Field(description=_("Maximum TTL on incoming packets"))
ipv4_ident = UInt16Field(description=_("IPv4 identification field"))
dst_tos = UInt8Field(description=_(
"Type of Service byte setting when exiting outgoing interface"))
in_src_mac = UInt64Field(description=_("Incoming source MAC address"))
out_dst_mac = UInt64Field(
description=_("Outcoming destination MAC address"))
src_vlan = UInt16Field(
description=_("VLAN id associated with ingress interface"))
dst_vlan = UInt16Field(
description=_("VLAN id associated with egress interface"))
ip_protocol_version = UInt8Field(description=_(
"IP protocol version for IPv4 is 4 and 6 for IPv6. If none, assumed 4")
)
direction = UInt8Field(
description=_("Flow direction: 0 -- ingress, 1 -- egress flow"))
# ipv6_next_hop = IPv6Field(description=_("IPv6 address of nexthop router"))
# bgp_ipv6_next_hop = IPv6Field(description=_("IPv6 address of nexthop router in BGP domain"))
# ipv6_option_headers = IPv6Field(
# description=_("Bit-encoded field identifying IPv6 option headers found in the flow")
# )
mpls_label1 = UInt32Field(description=_(
"MPLS label at position 1 in the stack. This comprises 20 bits of MPLS label, 3 EXP (experimental) bits and 1 S (end-of-stack) bit"
))
mpls_label2 = UInt32Field(description=_(
"MPLS label at position 2 in the stack. This comprises 20 bits of MPLS label, 3 EXP (experimental) bits and 1 S (end-of-stack) bit"
))
mpls_label3 = UInt32Field(description=_(
"MPLS label at position 3 in the stack. This comprises 20 bits of MPLS label, 3 EXP (experimental) bits and 1 S (end-of-stack) bit"
))
mpls_label4 = UInt32Field(description=_(
"MPLS label at position 4 in the stack. This comprises 20 bits of MPLS label, 3 EXP (experimental) bits and 1 S (end-of-stack) bit"
))
mpls_label5 = UInt32Field(description=_(
"MPLS label at position 5 in the stack. This comprises 20 bits of MPLS label, 3 EXP (experimental) bits and 1 S (end-of-stack) bit"
))
mpls_label6 = UInt32Field(description=_(
"MPLS label at position 6 in the stack. This comprises 20 bits of MPLS label, 3 EXP (experimental) bits and 1 S (end-of-stack) bit"
))
mpls_label7 = UInt32Field(description=_(
"MPLS label at position 7 in the stack. This comprises 20 bits of MPLS label, 3 EXP (experimental) bits and 1 S (end-of-stack) bit"
))
mpls_label8 = UInt32Field(description=_(
"MPLS label at position 8 in the stack. This comprises 20 bits of MPLS label, 3 EXP (experimental) bits and 1 S (end-of-stack) bit"
))
mpls_label9 = UInt32Field(description=_(
"MPLS label at position 9 in the stack. This comprises 20 bits of MPLS label, 3 EXP (experimental) bits and 1 S (end-of-stack) bit"
))
mpls_label10 = UInt32Field(description=_(
"MPLS label at position 10 in the stack. This comprises 20 bits of MPLS label, 3 EXP (experimental) bits and 1 S (end-of-stack) bit"
))
in_dst_mac = UInt64Field(description=_("Incoming destination MAC address"))
out_src_mac = UInt64Field(description=_("Outcoming source MAC address"))
if_name = StringField(
description=_("Shortened interface name i.e.: 'FE1/0'"))
if_desc = StringField(
description=_("Full interface name i.e.: 'FastEthernet 1/0'"))
sampler_name = StringField(description=_("Name of the flow sampler"))
in_permanent_bytes = UInt32Field(
description=_("Running byte counter for a permanent flow"))
in_permanent_pkts = UInt32Field(
description=_("Running packet counter for a permanent flow"))
fragment_offset = UInt16Field(
description=_("The fragment-offset value from fragmented IP packets"))
forwarding_status = UInt8Field(description=_(
"Forwarding status is encoded on 1 byte with the 2 left bits giving the status and the 6 remaining bits giving the reason code."
))
mpls_pal_rd = UInt64Field(description=_("MPLS PAL Route Distinguisher"))
mpls_prefix_len = UInt8Field(
description=_("Number of consecutive bits in the MPLS prefix length"))
src_traffic_index = UInt32Field(
description=_("BGP Policy Accounting Source Traffic Index"))
dst_traffic_index = UInt32Field(
description=_("BGP Policy Accounting Destination Traffic Index"))
application_description = StringField(
description=_("Application description"))
application_tag = UInt16Field(description=_(
"8 bits of engine ID, followed by n bits of classification"))
application_name = StringField(
description=_("Name associated with a classification"))
postipDiffServCodePoint = UInt8Field(description=_(
"The value of a Differentiated Services Code Point (DSCP) encoded in the Differentiated Services Field, after modification"
))
replication_factor = UInt32Field(
description=_("Multicast replication factor"))
class Alarms(Model):
class Meta(object):
db_table = "alarms"
engine = MergeTree("date", ("ts", "managed_object"))
date = DateField(description=_("Date"))
ts = DateTimeField(description=_("Created"))
close_ts = DateTimeField(description=_("Close Time"))
duration = Int32Field(description=_("Duration"))
alarm_id = StringField(description=_("Id"))
root = StringField(description=_("Alarm Root"))
alarm_class = ReferenceField(AlarmClass, description=_("Alarm Class"))
severity = Int32Field(description=_("Severity"))
reopens = Int32Field(description=_("Reopens"))
direct_services = Int64Field(description=_("Direct Services"))
direct_subscribers = Int64Field(description=_("Direct Subscribers"))
total_objects = Int64Field(description=_("Total Objects"))
total_services = Int64Field(description=_("Total Services"))
total_subscribers = Int64Field(description=_("Total Subscribers"))
#
escalation_ts = DateTimeField(description=_("Escalation Time"))
escalation_tt = StringField(description=_("Number of Escalation"))
# Amount of reboots during alarm
reboots = Int16Field(description=_("Qty of Reboots"))
#
managed_object = ReferenceField(ManagedObject,
description=_("Object Name"))
pool = ReferenceField(Pool, description=_("Pool Name"))
object_profile = ReferenceField(ObjectProfile,
description=_("Object Profile"))
ip = IPv4Field(description=_("IP Address"))
profile = ReferenceField(Profile, description=_("Profile"))
vendor = ReferenceField(Vendor, description=_("Vendor Name"))
platform = ReferenceField(Platform, description=_("Platform"))
version = ReferenceField(Version, description=_("Version"))
administrative_domain = ReferenceField(AdministrativeDomain,
description=_("Admin. Domain"))
segment = ReferenceField(NetworkSegment, description=_("Network Segment"))
container = ReferenceField(Container, description=_("Container"))
# Coordinates
x = Float64Field(description=_("Longitude"))
y = Float64Field(description=_("Latitude"))
services = NestedField(Services, description=_("Affected Services"))
subscribers = NestedField(Subscribers,
description=_("Affected Subscribers"))
# location = StringField(description="Location")
# Ack info
ack_user = StringField(description=_("Manual acknowledgement user name"))
ack_ts = DateTimeField(description=_("Manual acknowledgement timestamp"))
@classmethod
def transform_query(cls, query, user):
if not user or user.is_superuser:
return query # No restrictions
# Get user domains
domains = UserAccess.get_domains(user)
# Resolve domains against dict
domain_ids = [
x.bi_id
for x in AdministrativeDomainM.objects.filter(id__in=domains)
]
filter = query.get("filter", {})
dl = len(domain_ids)
if not dl:
return None
elif dl == 1:
q = {"$eq": [{"$field": "administrative_domain"}, domain_ids[0]]}
else:
q = {"$in": [{"$field": "administrative_domain"}, domain_ids]}
if filter:
query["filter"] = {"$and": [query["filter"], q]}
else:
query["filter"] = q
return query
@classmethod
def transform_field(cls, field):
if field == "services":
return ",".join([
"arrayStringConcat(arrayMap(x -> concat(dictGetString('serviceprofile'",
" 'name', toUInt64(services.profile[indexOf(services.summary, x)]))",
" ':', toString(x)), services.summary),',')",
])
elif field == "subscribers":
return ",".join([
"arrayStringConcat(arrayMap(x -> concat(dictGetString('subscriberprofile'",
" 'name', toUInt64(subscribers.profile[indexOf(subscribers.summary, x)]))",
" ':', toString(x)), subscribers.summary),',')",
])
return field