class Task(Document): meta = {'collection': 'task'} _id = StringField(primary_key=True) detail = StringField(unique=True, required=True) duration = LongField() consumed_time = LongField() created_at = DateTimeField() duration = LongField() finished = BooleanField()
class Supplier(Document): meta = { "collection": "noc.suppliers", "indexes": ["name"], "strict": False, "auto_create_index": False } name = StringField() description = StringField() is_affilated = BooleanField(default=False) profile = PlainReferenceField(SupplierProfile) state = PlainReferenceField(State) # Integration with external NRI and TT systems # Reference to remote system object has been imported from remote_system = PlainReferenceField(RemoteSystem) # Object id in remote system remote_id = StringField() # Object id in BI bi_id = LongField(unique=True) tags = ListField(StringField()) _id_cache = cachetools.TTLCache(maxsize=100, ttl=60) def __unicode__(self): return self.name @classmethod @cachetools.cachedmethod(operator.attrgetter("_id_cache"), lock=lambda _: id_lock) def get_by_id(cls, id): return Supplier.objects.filter(id=id).first()
class ServiceProfile(Document): meta = { "collection": "noc.serviceprofiles", "strict": False, "auto_create_index": False } name = StringField(unique=True) description = StringField() # Jinja2 service label template card_title_template = StringField() # Short service code for reporting code = StringField() # FontAwesome glyph glyph = StringField() # Glyph order in summary display_order = IntField(default=100) # Show in total summary show_in_summary = BooleanField(default=True) workflow = PlainReferenceField(Workflow) # Auto-assign interface profile when service binds to interface interface_profile = ReferenceField(InterfaceProfile) # Alarm weight weight = IntField(default=0) # Capabilities caps = ListField(EmbeddedDocumentField(CapsItem)) # Integration with external NRI and TT systems # Reference to remote system object has been imported from remote_system = ReferenceField(RemoteSystem) # Object id in remote system remote_id = StringField() # Object id in BI bi_id = LongField(unique=True) # Labels labels = ListField(StringField()) effective_labels = ListField(StringField()) _id_cache = cachetools.TTLCache(maxsize=100, ttl=60) def __str__(self): return self.name @classmethod @cachetools.cachedmethod(operator.attrgetter("_id_cache"), lock=lambda _: id_lock) def get_by_id(cls, id): return ServiceProfile.objects.filter(id=id).first() def on_save(self): if not hasattr(self, "_changed_fields" ) or "interface_profile" in self._changed_fields: call_later( "noc.sa.models.serviceprofile.refresh_interface_profiles", sp_id=self.id, ip_id=self.interface_profile.id if self.interface_profile else None, ) @classmethod def can_set_label(cls, label): return Label.get_effective_setting(label, "enable_serviceprofile")
class AllocationGroup(Document): meta = { "collection": "allocationgroups", "strict": False, "auto_create_index": False } name = StringField(unique=True) description = StringField() tags = ListField(StringField()) # Integration with external NRI and TT systems # Reference to remote system object has been imported from remote_system = PlainReferenceField(RemoteSystem) # Object id in remote system remote_id = StringField() # Object id in BI bi_id = LongField(unique=True) _id_cache = cachetools.TTLCache(maxsize=100, ttl=60) _bi_id_cache = cachetools.TTLCache(maxsize=100, ttl=60) def __unicode__(self): return self.name @classmethod @cachetools.cachedmethod(operator.attrgetter("_id_cache"), lock=lambda _: id_lock) def get_by_id(cls, id): return AllocationGroup.objects.filter(id=id).first() @classmethod @cachetools.cachedmethod(operator.attrgetter("_bi_id_cache"), lock=lambda _: id_lock) def get_by_bi_id(cls, id): return AllocationGroup.objects.filter(bi_id=id).first()
class SupplierProfile(Document): meta = { "collection": "noc.supplierprofiles", "strict": False, "auto_create_index": False } name = StringField(unique=True) description = StringField() workflow = PlainReferenceField(Workflow) style = ForeignKeyField(Style, required=False) tags = ListField(StringField()) # Integration with external NRI and TT systems # Reference to remote system object has been imported from remote_system = PlainReferenceField(RemoteSystem) # Object id in remote system remote_id = StringField() # Object id in BI bi_id = LongField(unique=True) _id_cache = cachetools.TTLCache(maxsize=100, ttl=60) def __str__(self): return self.name @classmethod @cachetools.cachedmethod(operator.attrgetter("_id_cache"), lock=lambda _: id_lock) def get_by_id(cls, id): return SupplierProfile.objects.filter(id=id).first()
class Workflow(Document): meta = { "collection": "workflows", "strict": False, "auto_create_index": False } name = StringField(unique=True) is_active = BooleanField() description = StringField() # Integration with external NRI and TT systems # Reference to remote system object has been imported from remote_system = ReferenceField(RemoteSystem) # Object id in remote system remote_id = StringField() # Object id in BI bi_id = LongField(unique=True) _id_cache = cachetools.TTLCache(maxsize=1000, ttl=60) _bi_id_cache = cachetools.TTLCache(maxsize=1000, ttl=60) def __str__(self): return self.name @classmethod @cachetools.cachedmethod(operator.attrgetter("_id_cache"), lock=lambda _: id_lock) def get_by_id(cls, id): return Workflow.objects.filter(id=id).first() @classmethod @cachetools.cachedmethod(operator.attrgetter("_bi_id_cache"), lock=lambda _: id_lock) def get_by_bi_id(cls, id): return Workflow.objects.filter(bi_id=id).first() @cachetools.cached(_default_state_cache, key=lambda x: str(x.id), lock=id_lock) def get_default_state(self): from .state import State return State.objects.filter(workflow=self.id, is_default=True).first() def set_default_state(self, state): from .state import State logger.info("[%s] Set default state to: %s", self.name, state.name) for s in State.objects.filter(workflow=self.id): if s.is_default and s.id != state.id: logger.info("[%s] Removing default status from: %s", self.name, s.name) s.is_default = False s.save() # Invalidate caches key = str(self.id) if key in _default_state_cache: try: del _default_state_cache[key] except KeyError: pass
class SensorProfile(Document): meta = { "collection": "sensorprofiles", "strict": False, "auto_create_index": False } name = StringField(unique=True) description = StringField() workflow = PlainReferenceField(Workflow) style = ForeignKeyField(Style) enable_collect = BooleanField(default=False) # Labels labels = ListField(StringField()) effective_labels = ListField(StringField()) # BI ID bi_id = LongField(unique=True) _id_cache = cachetools.TTLCache(maxsize=100, ttl=60) _bi_id_cache = cachetools.TTLCache(maxsize=100, ttl=60) _default_cache = cachetools.TTLCache(maxsize=100, ttl=60) DEFAULT_PROFILE_NAME = "default" DEFAULT_WORKFLOW_NAME = "Sensor Default" def __str__(self): return self.name @classmethod @cachetools.cachedmethod(operator.attrgetter("_id_cache"), lock=lambda _: id_lock) def get_by_id(cls, id) -> "SensorProfile": return SensorProfile.objects.filter(id=id).first() @classmethod @cachetools.cachedmethod(operator.attrgetter("_bi_id_cache"), lock=lambda _: id_lock) def get_by_bi_id(cls, id) -> "SensorProfile": return SensorProfile.objects.filter(bi_id=id).first() @classmethod @cachetools.cachedmethod(operator.attrgetter("_default_cache"), lock=lambda _: id_lock) def get_default_profile(cls) -> "SensorProfile": sp = SensorProfile.objects.filter( name=cls.DEFAULT_PROFILE_NAME).first() if not sp: sp = SensorProfile( name=cls.DEFAULT_PROFILE_NAME, workflow=Workflow.objects.filter( name=cls.DEFAULT_WORKFLOW_NAME).first(), ) sp.save() return sp @classmethod def can_set_label(cls, label): if label.enable_sensorprofile: return True return False
class VPN(Document): meta = { "collection": "vpns", "strict": False, "auto_create_index": False } name = StringField(unique=True) profile = PlainReferenceField(VPNProfile) description = StringField() state = PlainReferenceField(State) # Link to parent overlay parent = PlainReferenceField("self") project = ForeignKeyField(Project) route_target = ListField(EmbeddedDocumentField(RouteTargetItem)) tags = ListField(StringField()) # Integration with external NRI and TT systems # Reference to remote system object has been imported from remote_system = PlainReferenceField(RemoteSystem) # Object id in remote system remote_id = StringField() # Object id in BI bi_id = LongField(unique=True) # @todo: last_seen # @todo: expired _id_cache = cachetools.TTLCache(maxsize=100, ttl=60) _bi_id_cache = cachetools.TTLCache(maxsize=100, ttl=60) def __unicode__(self): return self.name @classmethod @cachetools.cachedmethod(operator.attrgetter("_id_cache"), lock=lambda _: id_lock) def get_by_id(cls, id): return VPN.objects.filter(id=id).first() @classmethod @cachetools.cachedmethod(operator.attrgetter("_bi_id_cache"), lock=lambda _: id_lock) def get_by_bi_id(cls, id): return VPN.objects.filter(bi_id=id).first() def clean(self): super(VPN, self).clean() if self.id and "parent" in self._changed_fields and self.has_loop: raise ValidationError("Creating VPN loop") @property def has_loop(self): """ Check if object creates loop """ if not self.id: return False p = self.parent while p: if p.id == self.id: return True p = p.parent return False
class AddressProfile(Document): meta = { "collection": "addressprofiles", "strict": False, "auto_create_index": False } name = StringField(unique=True) description = StringField() # Address workflow workflow = PlainReferenceField(Workflow) style = ForeignKeyField(Style) # Template.subject to render Address.name name_template = ForeignKeyField(Template) # Template.subject to render Address.fqdn fqdn_template = ForeignKeyField(Template) # Send seen event to prefix seen_propagation_policy = StringField(choices=[("E", "Enable"), ("D", "Disable")], default="D") # Labels labels = ListField(StringField()) effective_labels = ListField(StringField()) # Integration with external NRI and TT systems # Reference to remote system object has been imported from remote_system = PlainReferenceField(RemoteSystem) # Object id in remote system remote_id = StringField() # Object id in BI bi_id = LongField(unique=True) _id_cache = cachetools.TTLCache(maxsize=100, ttl=60) _name_cache = cachetools.TTLCache(maxsize=100, ttl=60) _bi_id_cache = cachetools.TTLCache(maxsize=100, ttl=60) def __str__(self): return self.name @classmethod @cachetools.cachedmethod(operator.attrgetter("_id_cache"), lock=lambda _: id_lock) def get_by_id(cls, id): return AddressProfile.objects.filter(id=id).first() @classmethod @cachetools.cachedmethod(operator.attrgetter("_name_cache"), lock=lambda _: id_lock) def get_by_name(cls, name): return AddressProfile.objects.filter(name=name).first() @classmethod @cachetools.cachedmethod(operator.attrgetter("_bi_id_cache"), lock=lambda _: id_lock) def get_by_bi_id(cls, id): return AddressProfile.objects.filter(bi_id=id).first() @classmethod def can_set_label(cls, label): return Label.get_effective_setting(label, setting="enable_addressprofile")
class PrefixProfile(Document): meta = {"collection": "prefixprofiles", "strict": False, "auto_create_index": False} name = StringField(unique=True) description = StringField() # Enable nested Address discovery # via ARP cache enable_ip_discovery = BooleanField(default=False) # Enable nested Addresses discovery # via active PING probes enable_ip_ping_discovery = BooleanField(default=False) # Enable nested prefix prefix discovery enable_prefix_discovery = BooleanField(default=False) # Prefix workflow workflow = PlainReferenceField(Workflow) style = ForeignKeyField(Style) # Template.subject to render Prefix.name name_template = ForeignKeyField(Template) # Discovery policies prefix_discovery_policy = StringField(choices=[("E", "Enable"), ("D", "Disable")], default="D") address_discovery_policy = StringField(choices=[("E", "Enable"), ("D", "Disable")], default="D") # Send seen event to parent seen_propagation_policy = StringField( choices=[("P", "Propagate"), ("E", "Enable"), ("D", "Disable")], default="P" ) # Include/Exclude broadcast & network addresses from prefix prefix_special_address_policy = StringField( choices=[("I", "Include"), ("X", "Exclude")], default="X" ) # Labels labels = ListField(StringField()) effective_labels = ListField(StringField()) # Integration with external NRI and TT systems # Reference to remote system object has been imported from remote_system = PlainReferenceField(RemoteSystem) # Object id in remote system remote_id = StringField() # Object id in BI bi_id = LongField(unique=True) _id_cache = cachetools.TTLCache(maxsize=100, ttl=60) _bi_id_cache = cachetools.TTLCache(maxsize=100, ttl=60) def __str__(self): return self.name @classmethod @cachetools.cachedmethod(operator.attrgetter("_id_cache"), lock=lambda _: id_lock) def get_by_id(cls, id): return PrefixProfile.objects.filter(id=id).first() @classmethod @cachetools.cachedmethod(operator.attrgetter("_bi_id_cache"), lock=lambda _: id_lock) def get_by_bi_id(cls, id): return PrefixProfile.objects.filter(bi_id=id).first() @classmethod def can_set_label(cls, label): return Label.get_effective_setting(label, setting="enable_prefixprofile")
class StaticProfileResultDO(EmbeddedDocument): """ Static profiling result plain object """ # Number of parameters of this model parameters = IntField(required=True) # Floating point operations flops = LongField(required=True) # Memory consumption in Byte in order to load this model into GPU or CPU memory = LongField(required=True) # Memory read in Byte mread = LongField(required=True) # Memory write in Byte mwrite = LongField(required=True) # Memory readwrite in Byte mrw = LongField(required=True)
class Pool(Document): meta = { "collection": "noc.pools", "strict": False, "auto_create_index": False } name = StringField(unique=True, min_length=1, max_length=16, regex="^[0-9a-zA-Z]{1,16}$") description = StringField() discovery_reschedule_limit = IntField(default=50) # Object id in BI bi_id = LongField(unique=True) _id_cache = cachetools.TTLCache(1000, ttl=60) _bi_id_cache = cachetools.TTLCache(1000, ttl=60) _name_cache = cachetools.TTLCache(1000, ttl=60) reschedule_lock = threading.Lock() reschedule_ts = {} # pool id -> timestamp def __unicode__(self): return self.name @classmethod @cachetools.cachedmethod(operator.attrgetter("_id_cache"), lock=lambda _: id_lock) def get_by_id(cls, id): return Pool.objects.filter(id=id).first() @classmethod @cachetools.cachedmethod(operator.attrgetter("_bi_id_cache"), lock=lambda _: id_lock) def get_by_bi_id(cls, id): return Pool.objects.filter(bi_id=id).first() @classmethod @cachetools.cachedmethod(operator.attrgetter("_name_cache"), lock=lambda _: id_lock) def get_by_name(cls, name): return Pool.objects.filter(name=name).first() def get_delta(self): """ Get delta for next discovery, Limit runs to discovery_reschedule_limit tasks """ t = time.time() dt = 1.0 / float(self.discovery_reschedule_limit) with self.reschedule_lock: lt = self.reschedule_ts.get(self.id) if lt and lt > t: delta = lt - t else: delta = 0 self.reschedule_ts[self.id] = t + dt return delta
class ResourceGroup(Document): """ Technology Abstraction to restrict ResourceGroup links """ meta = { "collection": "resourcegroups", "strict": False, "auto_create_index": False } # Group | Name name = StringField() technology = PlainReferenceField(Technology) parent = PlainReferenceField("inv.ResourceGroup") description = StringField() # @todo: FM integration # Integration with external NRI and TT systems # Reference to remote system object has been imported from remote_system = PlainReferenceField(RemoteSystem) # Object id in remote system remote_id = StringField() # Object id in BI bi_id = LongField(unique=True) # Tags tags = ListField(StringField()) _id_cache = cachetools.TTLCache(maxsize=100, ttl=60) _bi_id_cache = cachetools.TTLCache(maxsize=100, ttl=60) def __str__(self): return "%s (%s)" % (self.name, self.technology.name) @classmethod @cachetools.cachedmethod(operator.attrgetter("_id_cache"), lock=lambda _: id_lock) def get_by_id(cls, id): return ResourceGroup.objects.filter(id=id).first() @classmethod @cachetools.cachedmethod(operator.attrgetter("_bi_id_cache"), lock=lambda _: id_lock) def get_by_bi_id(cls, id): return ResourceGroup.objects.filter(bi_id=id).first() def iter_changed_datastream(self, changed_fields=None): if config.datastream.enable_resourcegroup: yield "resourcegroup", self.id @property def has_children(self): return bool( ResourceGroup.objects.filter(parent=self.id).only("id").first())
class VPNProfile(Document): meta = { "collection": "vpnprofiles", "strict": False, "auto_create_index": False } name = StringField(unique=True) description = StringField() type = StringField(choices=[("vrf", "VRF"), ("vxlan", "VxLAN"), ("vpls", "VPLS"), ("vll", "VLL"), ("evpn", "EVPN"), ("ipsec", "IPSec"), ("gre", "GRE"), ("ipip", "IP-IP")], default="vrf") workflow = PlainReferenceField(Workflow) # Template.subject to render VPN/VRF.name name_template = ForeignKeyField(Template) # style = ForeignKeyField(Style) # For vrf type -- default prefix profile # Used to create AFI root prefixes default_prefix_profile = PlainReferenceField("ip.PrefixProfile") # tags = ListField(StringField()) # Integration with external NRI and TT systems # Reference to remote system object has been imported from remote_system = PlainReferenceField(RemoteSystem) # Object id in remote system remote_id = StringField() # Object id in BI bi_id = LongField(unique=True) _id_cache = cachetools.TTLCache(maxsize=100, ttl=60) _bi_id_cache = cachetools.TTLCache(maxsize=100, ttl=60) def __unicode__(self): return self.name @classmethod @cachetools.cachedmethod(operator.attrgetter("_id_cache"), lock=lambda _: id_lock) def get_by_id(cls, id): return VPNProfile.objects.filter(id=id).first() @classmethod @cachetools.cachedmethod(operator.attrgetter("_bi_id_cache"), lock=lambda _: id_lock) def get_by_bi_id(cls, id): return VPNProfile.objects.filter(bi_id=id).first() def clean(self): if self.type == "vrf" and not self.default_prefix_profile: raise ValidationError( "default_prefix_profile must be set for vrf type")
class VLANProfile(Document): meta = { "collection": "vlanprofiles", "strict": False, "auto_create_index": False } name = StringField(unique=True) description = StringField() # VLAN is management VLAN enable_management = BooleanField(default=False) # VLAN is multicast VLAN enable_multicast = BooleanField(default=False) # VLAN should be automatically provisioned enable_provisioning = BooleanField(default=False) # VLAN workflow workflow = PlainReferenceField(Workflow) style = ForeignKeyField(Style) # Labels labels = ListField(StringField()) effective_labels = ListField(StringField()) # Integration with external NRI and TT systems # Reference to remote system object has been imported from remote_system = PlainReferenceField(RemoteSystem) # Object id in remote system remote_id = StringField() # Object id in BI bi_id = LongField(unique=True) _id_cache = cachetools.TTLCache(maxsize=100, ttl=60) _bi_id_cache = cachetools.TTLCache(maxsize=100, ttl=60) def __str__(self): return self.name @classmethod @cachetools.cachedmethod(operator.attrgetter("_id_cache"), lock=lambda _: id_lock) def get_by_id(cls, id): return VLANProfile.objects.filter(id=id).first() @classmethod @cachetools.cachedmethod(operator.attrgetter("_bi_id_cache"), lock=lambda _: id_lock) def get_by_bi_id(cls, id): return VLANProfile.objects.filter(bi_id=id).first() @classmethod def can_set_label(cls, label): return Label.get_effective_setting(label, "enable_vlanprofile")
class SubscriberProfile(Document): meta = { "collection": "noc.subscriberprofiles", "strict": False, "auto_create_index": False } name = StringField(unique=True) description = StringField() style = ForeignKeyField(Style, required=False) workflow = PlainReferenceField(Workflow) # FontAwesome glyph glyph = StringField() # Glyph order in summary display_order = IntField(default=100) # Show in total summary show_in_summary = BooleanField(default=True) # Labels labels = ListField(StringField()) effective_labels = ListField(StringField()) # Alarm weight weight = IntField(default=0) # Integration with external NRI and TT systems # Reference to remote system object has been imported from remote_system = PlainReferenceField(RemoteSystem) # Object id in remote system remote_id = StringField() # Object id in BI bi_id = LongField(unique=True) _id_cache = cachetools.TTLCache(maxsize=100, ttl=60) def __str__(self): return self.name @classmethod @cachetools.cachedmethod(operator.attrgetter("_id_cache"), lock=lambda _: id_lock) def get_by_id(cls, id): return SubscriberProfile.objects.filter(id=id).first() @classmethod def can_set_label(cls, label): return Label.get_effective_setting(label, setting="enable_subscriberprofile")
class Subscriber(Document): meta = { "collection": "noc.subscribers", "indexes": ["name"], "strict": False, "auto_create_index": False, } name = StringField() description = StringField() profile = PlainReferenceField(SubscriberProfile) state = PlainReferenceField(State) # Main address address = StringField() # Technical contacts tech_contact_person = StringField() tech_contact_phone = StringField() project = ForeignKeyField(Project) # Labels labels = ListField(StringField()) effective_labels = ListField(StringField()) # Integration with external NRI and TT systems # Reference to remote system object has been imported from remote_system = PlainReferenceField(RemoteSystem) # Object id in remote system remote_id = StringField() # Object id in BI bi_id = LongField(unique=True) _id_cache = cachetools.TTLCache(maxsize=100, ttl=60) def __str__(self): return self.name @classmethod @cachetools.cachedmethod(operator.attrgetter("_id_cache"), lock=lambda _: id_lock) def get_by_id(cls, id): return Subscriber.objects.filter(id=id).first() @classmethod def can_set_label(cls, label): if label.enable_subscriber: return True return False
class PasswordRecovery(Document): meta = dict( collection="PwdRecovery", allow_inheritance=False, indexes=[ 'user', # Automatically delete records as they expire. dict(fields=['expires'], expireAfterSeconds=0) ]) user = ReferenceField(User, required=True) recovery_key = LongField(db_field='k', required=True) expires = DateTimeField( db_field='e', default=lambda: datetime.utcnow() + timedelta(minutes=15)) @property def created(self): return self.id.generation_time def __repr__(self): return 'PasswordRecovery({0}, {1}, {2})'.format( self.id.generation_time.isoformat(), self.user.username, self.recovery_key)
class Profile(Document): meta = { "collection": "noc.profiles", "strict": False, "auto_create_index": False, "json_collection": "sa.profiles", "json_unique_fields": ["name"] } name = StringField(unique=True) description = StringField(required=False) # Global ID uuid = UUIDField(binary=True) # Object id in BI bi_id = LongField(unique=True) _id_cache = cachetools.TTLCache(1000, ttl=60) _bi_id_cache = cachetools.TTLCache(1000, ttl=60) _name_cache = cachetools.TTLCache(1000, ttl=60) def __unicode__(self): return self.name @classmethod @cachetools.cachedmethod(operator.attrgetter("_id_cache"), lock=lambda _: id_lock) def get_by_id(cls, id): return Profile.objects.filter(id=id).first() @classmethod @cachetools.cachedmethod(operator.attrgetter("_bi_id_cache"), lock=lambda _: id_lock) def get_by_bi_id(cls, id): return Profile.objects.filter(bi_id=id).first() @classmethod @cachetools.cachedmethod(operator.attrgetter("_name_cache"), lock=lambda _: id_lock) def get_by_name(cls, name): return Profile.objects.filter(name=name).first() def to_json(self): return to_json( { "$collection": self._meta["json_collection"], "name": self.name, "uuid": self.uuid, "description": self.description }, order=["name", "uuid", "description"]) def get_json_path(self): vendor, soft = self.name.split(".") return os.path.join(vendor, "%s.json" % soft) def get_profile(self): return loader.get_profile(self.name)() @property def is_generic(self): return self.name == GENERIC_PROFILE @classmethod def get_generic_profile_id(cls): if not hasattr(cls, "_generic_profile_id"): cls._generic_profile_id = Profile.objects.filter( name=GENERIC_PROFILE).first().id return cls._generic_profile_id
class MetricType(Document): meta = { "collection": "noc.metrictypes", "strict": False, "auto_create_index": False, "json_collection": "pm.metrictypes", "json_depends_on": [ "pm.metricscopes" ], "json_unique_fields": ["name"] } # Metric type name, i.e. Interface | Load | In name = StringField(unique=True) # Global ID uuid = UUIDField(binary=True) # Metric scope reference scope = PlainReferenceField(MetricScope) # Database field name field_name = StringField() # Database field type field_type = StringField( choices=[ ("UInt8", "UInt8"), ("Int8", "Int8"), ("UInt16", "UInt16"), ("Int16", "Int16"), ("UInt32", "UInt32"), ("Int32", "Int32"), ("UInt64", "UInt64"), ("Int64", "Int64"), ("Float32", "Float32"), ("Float64", "Float64"), ("String", "String") ] ) # Text description description = StringField(required=False) # Measure name, like 'kbit/s' # Compatible to Grafana measure = StringField() # Optional required capability required_capability = PlainReferenceField(Capability) # Object id in BI, used for counter context hashing bi_id = LongField(unique=True) # category = ObjectIdField() _id_cache = cachetools.TTLCache(maxsize=100, ttl=60) _name_cache = cachetools.TTLCache(maxsize=100, ttl=60) _bi_id_cache = cachetools.TTLCache(maxsize=100, ttl=60) def __unicode__(self): return self.name @property def json_data(self): r = { "name": self.name, "$collection": self._meta["json_collection"], "uuid": self.uuid, "scope__name": self.scope.name, "field_name": self.field_name, "field_type": self.field_type, "description": self.description, "measure": self.measure } if self.required_capability: r["required_capability__name"] = self.required_capability.name return r def to_json(self): return to_json( self.json_data, order=[ "name", "$collection", "uuid", "scope__name", "field_name", "field_type", "description", "measure", "vector_tag"]) def get_json_path(self): p = [quote_safe_path(n.strip()) for n in self.name.split("|")] return os.path.join(*p) + ".json" @classmethod @cachetools.cachedmethod(operator.attrgetter("_id_cache"), lock=lambda _: id_lock) def get_by_id(cls, id): return MetricType.objects.filter(id=id).first() @classmethod @cachetools.cachedmethod(operator.attrgetter("_name_cache"), lock=lambda _: id_lock) def get_by_name(cls, name): return MetricType.objects.filter(name=name).first() @classmethod @cachetools.cachedmethod(operator.attrgetter("_bi_id_cache"), lock=lambda _: id_lock) def get_by_bi_id(cls, id): return MetricType.objects.filter(bi_id=id).first() def on_save(self): call_later( "noc.core.clickhouse.ensure.ensure_all_pm_scopes", scheduler="scheduler", delay=30 ) def clean_value(self, value): return getattr(self, "clean_%s" % self.field_type)(value) @staticmethod def clean_UInt8(value): try: v = int(value) except ValueError: raise if v < 0 or v > 255: raise ValueError("Value out of range") return v @staticmethod def clean_Int8(value): try: v = int(value) except ValueError: raise if v < -127 or v > 127: raise ValueError("Value out of range") return v @staticmethod def clean_UInt16(value): try: v = int(value) except ValueError: raise if v < 0 or v > 65535: raise ValueError("Value out of range") return v @staticmethod def clean_Int16(value): try: v = int(value) except ValueError: raise if v < -32767 or v > 32767: raise ValueError("Value out of range") return v @staticmethod def clean_UInt32(value): try: v = int(value) except ValueError: raise if v < 0 or v > 4294967295: raise ValueError("Value out of range") return v @staticmethod def clean_Int32(value): try: v = int(value) except ValueError: raise if v < -2147483647 or v > 2147483647: raise ValueError("Value out of range") return v @staticmethod def clean_UInt64(value): try: v = int(value) except ValueError: raise if v < 0 or v > 18446744073709551615: raise ValueError("Value out of range") return v @staticmethod def clean_Int64(value): try: v = int(value) except ValueError: raise if v < -9223372036854775807 or v > 9223372036854775807: raise ValueError("Value out of range") return v @staticmethod def clean_Float32(value): return float(value) @staticmethod def clean_Float64(value): return float(value) @staticmethod def clean_String(value): return str(value)
class Transition(Document): meta = { "collection": "transitions", "indexes": ["from_state", "to_state"], "strict": False, "auto_create_index": False, } workflow = PlainReferenceField(Workflow) from_state = PlainReferenceField(State) to_state = PlainReferenceField(State) is_active = BooleanField(default=True) # Event name # Some predefined names exists: # seen -- discovery confirms resource usage # expired - TTL expired event = StringField() # Text label label = StringField() # Arbbitrary description description = StringField() # Enable manual transition enable_manual = BooleanField(default=True) # Handler to be called on starting transitions # Any exception aborts transtion handlers = ListField(StringField()) # Visual vertices vertices = ListField(EmbeddedDocumentField(TransitionVertex)) # Integration with external NRI and TT systems # Reference to remote system object has been imported from remote_system = ReferenceField(RemoteSystem) # Object id in remote system remote_id = StringField() # Object id in BI bi_id = LongField(unique=True) _id_cache = cachetools.TTLCache(maxsize=100, ttl=60) _bi_id_cache = cachetools.TTLCache(maxsize=100, ttl=60) def __str__(self): return "%s: %s -> %s [%s]" % ( self.workflow.name, self.from_state.name, self.to_state.name, self.label, ) @classmethod @cachetools.cachedmethod(operator.attrgetter("_id_cache"), lock=lambda _: id_lock) def get_by_id(cls, id): return Transition.objects.filter(id=id).first() @classmethod @cachetools.cachedmethod(operator.attrgetter("_bi_id_cache"), lock=lambda _: id_lock) def get_by_bi_id(cls, id): return Transition.objects.filter(bi_id=id).first() def clean(self): if not self.from_state or not self.to_state: raise ValueError("Missed state") if self.from_state.workflow != self.to_state.workflow: raise ValueError("Workflow mismatch") self.workflow = self.from_state.workflow def on_transition(self, obj): """ Called during transition :param obj: :return: """ if self.handlers: logger.debug("[%s|%s|%s] Running transition handlers", obj, obj.state.name, self.label) for hn in self.handlers: try: h = get_handler(str(hn)) except ImportError as e: logger.error("Error import handler: %s" % e) h = None if h: logger.debug("[%s|%s|%s] Running %s", obj, obj.state.name, self.label, hn) h(obj) # @todo: Catch exceptions else: logger.debug( "[%s|%s|%s] Invalid handler %s, skipping", obj, obj.state.name, self.label, hn, )
class NetworkSegment(Document): meta = { "collection": "noc.networksegments", "strict": False, "auto_create_index": False, "indexes": ["parent", "sibling", "adm_domains"], } name = StringField(unique=True) parent = ReferenceField("self", required=False) profile = ReferenceField(NetworkSegmentProfile, required=True) description = StringField(required=False) # Management VLAN processing order # * d - disable management vlan # * e - enable management vlan and get from management_vlan field # * p - use profile settings management_vlan_policy = StringField( choices=[("d", "Disable"), ("p", "Profile"), ("e", "Enable")], default="p" ) management_vlan = IntField(required=False, min_value=1, max_value=4095) # MVR VLAN processing order # * d - disable multicast vlan # * e - enable multicast vlan and get from multicast_vlan field # * p - use profile settings multicast_vlan_policy = StringField( choices=[("d", "Disable"), ("p", "Profile"), ("e", "Enable")], default="p" ) multicast_vlan = IntField(required=False, min_value=1, max_value=4095) settings = DictField(default=lambda: {}.copy()) labels = ListField(StringField()) # Selectors for fake segments # Transition only, should not be used selector = ForeignKeyField(ManagedObjectSelector) # Sibling segment, if part of larger structure with # horizontal links sibling = ReferenceField("self") # True if segment has alternative paths is_redundant = BooleanField(default=False) # True if segment is redundant and redundancy # currently broken lost_redundancy = BooleanField(default=False) # VLAN namespace demarcation # * False - share namespace with parent VLAN # * True - split own namespace vlan_border = BooleanField(default=True) # VLAN translation policy when marking border # (vlan_border=True) # Dynamically recalculated and placed to VLAN.translation_rule # and VLAN.parent vlan_translation = ListField(EmbeddedDocumentField(VLANTranslation)) # Share allocation resources with another segments allocation_group = PlainReferenceField(AllocationGroup) # Provided L2 MTU l2_mtu = IntField(default=1504) # Administrative domains which have access to segment # Sum of all administrative domains adm_domains = ListField(IntField()) # Collapse object's downlinks on network map # when count is above the threshold max_shown_downlinks = IntField(default=1000) # Limit objects on network map for reach "Too many objects" error max_objects = IntField(default=300) # Horizontal transit policy horizontal_transit_policy = StringField( choices=[("E", "Always Enable"), ("C", "Calculate"), ("D", "Disable"), ("P", "Profile")], default="P", ) # Horizontal transit settings # i.e. Allow traffic flow not only from parent-to-childrens and # children-to-children, but parent-to-parent and parent-to-neighbors # Calculated automatically during topology research enable_horizontal_transit = BooleanField(default=False) # Objects, services and subscribers belonging to segment directly direct_objects = ListField(EmbeddedDocumentField(ObjectSummaryItem)) direct_services = ListField(EmbeddedDocumentField(SummaryItem)) direct_subscribers = ListField(EmbeddedDocumentField(SummaryItem)) # Objects, services and subscribers belonging to all nested segments total_objects = ListField(EmbeddedDocumentField(ObjectSummaryItem)) total_services = ListField(EmbeddedDocumentField(SummaryItem)) total_subscribers = ListField(EmbeddedDocumentField(SummaryItem)) # Integration with external NRI and TT systems # Reference to remote system object has been imported from remote_system = ReferenceField(RemoteSystem) # Object id in remote system remote_id = StringField() # Object id in BI bi_id = LongField(unique=True) _id_cache = cachetools.TTLCache(maxsize=100, ttl=60) _bi_id_cache = cachetools.TTLCache(maxsize=100, ttl=60) _border_cache = cachetools.TTLCache(maxsize=100, ttl=60) _vlan_domains_cache = cachetools.TTLCache(maxsize=100, ttl=60) _vlan_domains_mo_cache = cachetools.TTLCache(maxsize=100, ttl=60) DISCOVERY_JOB = "noc.services.discovery.jobs.segment.job.SegmentDiscoveryJob" def __str__(self): return self.name @classmethod @cachetools.cachedmethod(operator.attrgetter("_id_cache"), lock=lambda _: id_lock) def get_by_id(cls, id): return NetworkSegment.objects.filter(id=id).first() @classmethod @cachetools.cachedmethod(operator.attrgetter("_bi_id_cache"), lock=lambda _: id_lock) def get_by_bi_id(cls, id): return NetworkSegment.objects.filter(bi_id=id).first() @classmethod def _reset_caches(cls, id): try: del cls._id_cache[ str(id), # Tuple ] except KeyError: pass @cachetools.cached(_path_cache, key=lambda x: str(x.id), lock=id_lock) def get_path(self): """ Returns list of parent segment ids :return: """ if self.parent: return self.parent.get_path() + [self.id] return [self.id] def clean(self): if self.id and "parent" in self._changed_fields and self.has_loop: raise ValidationError("Creating segments loop") if self.horizontal_transit_policy == "E": self.enable_horizontal_transit = True elif self.horizontal_transit_policy == "D": self.enable_horizontal_transit = False elif self.profile and self.horizontal_transit_policy == "P": if self.profile.horizontal_transit_policy == "E": self.enable_horizontal_transit = True elif self.profile.horizontal_transit_policy == "D": self.enable_horizontal_transit = False super().clean() @property def effective_settings(self): """ Returns dict with effective settings """ if hasattr(self, "_es"): return self._es # Build full parent stack sstack = [self.settings or {}] p = self.parent while p: sstack = [p.settings or {}] + sstack p = p.parent # Get effective settings es = {} for s in sstack: for k in s: v = s[k] if v: # Override parent settings es[k] = v elif k in es: # Ignore parent settings del es[k] self._es = es return es @property def has_loop(self): """ Check if object creates loop """ if not self.id: return False p = self.parent while p: if p.id == self.id: return True p = p.parent return False @property def managed_objects(self): from noc.sa.models.managedobject import ManagedObject if self.selector: return self.selector.managed_objects else: siblings = self.get_siblings() if len(siblings) == 1: q = {"segment": str(siblings.pop().id)} else: q = {"segment__in": [str(s.id) for s in siblings]} return ManagedObject.objects.filter(**q) def get_siblings(self, seen=None): seen = seen or set() ss = {self} seen |= ss if self.sibling and self.sibling not in seen: ss |= self.sibling.get_siblings(seen) seen |= ss for s in NetworkSegment.objects.filter(sibling=self): ss |= s.get_siblings(seen) return ss def run_discovery(self): """ Run discovery on whole segment """ for o in self.managed_objects: if o.is_managed: o.run_discovery() @property def has_children(self): return bool(NetworkSegment.objects.filter(parent=self.id).only("id").first()) def set_redundancy(self, status): """ Change interface redundancy status :param status: :return: """ siblings = list(self.get_siblings()) filter = {"status": {"$ne": status}} if len(siblings) == 1: filter["_id"] = self.id else: filter["_id"] = {"$in": [s.id for s in siblings]} set_op = {"is_redundant": status} if not status: set_op["lost_redundancy"] = False NetworkSegment._get_collection().update_many(filter, {"$set": set_op}) def set_lost_redundancy(self, status): NetworkSegment._get_collection().update( {"_id": self.id}, {"$set": {"lost_redundancy": bool(status)}} ) def get_direct_summary(self): objects = { d["object_profile"]: d["count"] for d in self.managed_objects.values("object_profile") .annotate(count=Count("id")) .order_by("count") } # Direct services mo_ids = self.managed_objects.values_list("id", flat=True) services, subscribers = ServiceSummary.get_direct_summary(mo_ids) return services, subscribers, objects def get_summary(self): def to_list(v): return [{"profile": k, "summary": v[k]} for k in sorted(v)] def update_dict(d1, d2): for kk in d2: if kk in d1: d1[kk] += d2[kk] else: d1[kk] = d2[kk] services, subscribers, objects = self.get_direct_summary() r = { "direct_services": to_list(services), "direct_subscribers": to_list(subscribers), "direct_objects": to_list(objects), } # map(lambda x: update_dict(*x), zip([services, subscribers, objects], self.get_total_summary())) [ update_dict(k, v) for k, v in zip([services, subscribers, objects], self.get_total_summary()) ] r["total_services"] = to_list(services) r["total_subscribers"] = to_list(subscribers) r["total_objects"] = to_list(objects) return r @classmethod def update_summary(cls, network_segment): """ Update summaries :return: """ if not hasattr(network_segment, "id"): network_segment = NetworkSegment.get_by_id(network_segment) path = network_segment.get_path() # Update upwards path.reverse() for ns in sorted( NetworkSegment.objects.filter(id__in=path), key=lambda x: path.index(x.id) ): r = ns.get_summary() NetworkSegment._get_collection().update_one({"_id": ns.id}, {"$set": r}, upsert=True) def update_access(self): from noc.sa.models.administrativedomain import AdministrativeDomain # Get all own administrative domains adm_domains = set( d["administrative_domain"] for d in self.managed_objects.values("administrative_domain") .annotate(count=Count("id")) .order_by("count") ) p = set() for a in adm_domains: a = AdministrativeDomain.get_by_id(a) p |= set(a.get_path()) adm_domains |= p # Merge with children's administrative domains for s in NetworkSegment.objects.filter(parent=self.id).only("adm_domains"): adm_domains |= set(s.adm_domains or []) # Check for changes if set(self.adm_domains) != adm_domains: self.adm_domains = sorted(adm_domains) self.save() # Propagate to parents if self.parent: self.parent.update_access() def update_uplinks(self): # if self.profile.is_persistent: call_later("noc.core.topology.segment.update_uplinks", 60, segment_id=self.id) def get_horizontal_transit_policy(self): if self.horizontal_transit_policy in ("E", "C"): return self.horizontal_transit_policy elif self.horizontal_transit_policy == "P" and self.profile: return self.profile.horizontal_transit_policy else: return "D" def get_management_vlan(self): """ Returns Management VLAN for segment :return: vlan (integer) or None """ if self.management_vlan_policy == "e": return self.management_vlan or None elif self.management_vlan_policy == "p": return self.profile.management_vlan or None else: return None def get_multicast_vlan(self): """ Returns Multicast VLAN for segment :return: vlan (integer) or None """ if self.multicast_vlan_policy == "e": return self.multicast_vlan or None elif self.multicast_vlan_policy == "p": return self.profile.multicast_vlan or None else: return None def get_nested_ids(self): """ Return id of this and all nested segments :return: """ # $graphLookup hits 100Mb memory limit. Do not use it seen = {self.id} wave = {self.id} max_level = 10 coll = NetworkSegment._get_collection() for _ in range(max_level): # Get next wave wave = ( set(d["_id"] for d in coll.find({"parent": {"$in": list(wave)}}, {"_id": 1})) - seen ) if not wave: break seen |= wave return list(seen) def ensure_discovery_jobs(self): if self.profile and self.profile.discovery_interval > 0: Job.submit("scheduler", self.DISCOVERY_JOB, key=self.id, keep_ts=True) else: Job.remove("scheduler", self.DISCOVERY_JOB, key=self.id) def on_save(self): if hasattr(self, "_changed_fields") and "profile" in self._changed_fields: self.ensure_discovery_jobs() if ( hasattr(self, "_changed_fields") and self.vlan_border and "vlan_translation" in self._changed_fields ): from noc.vc.models.vlan import VLAN for vlan in VLAN.objects.filter(segment=self.id): vlan.refresh_translation() if hasattr(self, "_changed_fields") and "parent" in self._changed_fields: self.update_access() self.update_links() if self.parent: self.parent.update_links() @classmethod @cachetools.cachedmethod(operator.attrgetter("_border_cache"), lock=lambda _: id_lock) def get_border_segment(cls, segment): """ Proceed up until vlan border :return: """ current = segment while current: if current.vlan_border or not current.parent: return current current = current.parent @classmethod def iter_vlan_domain_segments(cls, segment): """ Get all segments related to same VLAN domains :param segment: :return: """ def iter_segments(ps): # Return segment yield ps # Iterate and recurse over all non vlan-border children for s in NetworkSegment.objects.filter(parent=ps.id): if s.vlan_border: continue yield from iter_segments(s) # Get domain root root = cls.get_border_segment(segment) # Yield all children segments yield from iter_segments(root) @classmethod @cachetools.cachedmethod(operator.attrgetter("_vlan_domains_cache"), lock=lambda _: id_lock) def get_vlan_domain_segments(cls, segment): """ Get list of all segments related to same VLAN domains :param segment: :return: """ return list(cls.iter_vlan_domain_segments(segment)) @classmethod @cachetools.cachedmethod(operator.attrgetter("_vlan_domains_mo_cache"), lock=lambda _: id_lock) def get_vlan_domain_object_ids(cls, segment): """ Get list of all managed object ids belonging to same VLAN domain :param segment: :return: """ from noc.sa.models.managedobject import ManagedObject return ManagedObject.objects.filter( segment__in=[s.id for s in cls.get_vlan_domain_segments(segment)] ).values_list("id", flat=True) def iter_links(self): yield from Link.objects.filter(linked_segments__in=[self.id]) def update_links(self): # @todo intersect link only for link in self.iter_links(): link.save() def get_total_summary(self, ids=None, parent_id=None): """ :param ids: Network segment ID list :param parent_id: Parent ID filter value :return: """ services = {} subscribers = {} objects = {} pipeline = [] # Exclude segment sibling set (sibling segments as one) match = {"sibling": None} if ids: # Filter by network segment match["_id"] = {"$in": ids} else: match["parent"] = parent_id or self.id if match: pipeline += [{"$match": match}] # Mark service and profile with type field pipeline += [ { "$project": { "_id": 0, "service": { "$map": { "input": "$total_services", "as": "svc", "in": { "type": "svc", "profile": "$$svc.profile", "summary": "$$svc.summary", }, } }, "subscriber": { "$map": { "input": "$total_subscribers", "as": "sub", "in": { "type": "sub", "profile": "$$sub.profile", "summary": "$$sub.summary", }, } }, "object": { "$map": { "input": "$total_objects", "as": "obj", "in": { "type": "obj", "profile": "$$obj.profile", "summary": "$$obj.summary", }, } }, } }, # Concatenate services and profiles {"$project": {"summary": {"$concatArrays": ["$service", "$subscriber", "$object"]}}}, # Unwind *summary* array to independed records {"$unwind": "$summary"}, # Group by (type, profile) { "$group": { "_id": {"type": "$summary.type", "profile": "$summary.profile"}, "summary": {"$sum": "$summary.summary"}, } }, ] # noqa try: for doc in NetworkSegment._get_collection().aggregate(pipeline): profile = doc["_id"]["profile"] if doc["_id"]["type"] == "svc": services[profile] = services.get(profile, 0) + doc["summary"] elif doc["_id"]["type"] == "sub": subscribers[profile] = subscribers.get(profile, 0) + doc["summary"] elif doc["_id"]["type"] == "obj": objects[profile] = objects.get(profile, 0) + doc["summary"] except OperationFailure: # for Mongo less 3.4 pass return services, subscribers, objects
class Firmware(Document): meta = { "collection": "noc.firmwares", "strict": False, "auto_create_index": False, "json_collection": "inv.firmwares", "json_depends_on": ["sa.profile"], "json_unique_fields": ["profile", "vendor", "version"], "indexes": [{ "fields": ["profile", "vendor", "version"], "unique": True }], } # Global ID uuid = UUIDField(binary=True) # profile = PlainReferenceField(Profile) vendor = PlainReferenceField(Vendor) version = StringField() description = StringField() download_url = StringField() # Full name, combined from profile and version full_name = StringField() # Object id in BI bi_id = LongField(unique=True) _id_cache = cachetools.TTLCache(1000, ttl=60) _bi_id_cache = cachetools.TTLCache(1000, ttl=60) _ensure_cache = cachetools.TTLCache(1000, ttl=60) def __str__(self): return self.full_name if self.full_name else self.version def clean(self): self.full_name = "%s %s" % (self.profile.name, self.version) super().clean() @classmethod @cachetools.cachedmethod(operator.attrgetter("_id_cache"), lock=lambda _: id_lock) def get_by_id(cls, id): return Firmware.objects.filter(id=id).first() @classmethod @cachetools.cachedmethod(operator.attrgetter("_bi_id_cache"), lock=lambda _: id_lock) def get_by_bi_id(cls, id): return Firmware.objects.filter(bi_id=id).first() def to_json(self): return to_json( { "$collection": self._meta["json_collection"], "profile__name": self.profile.name, "vendor__code": self.vendor.code[0], "version": self.version, "uuid": self.uuid, }, order=["profile__name", "vendor__code", "version", "uuid"], ) def get_json_path(self): return os.path.join(self.vendor.code[0], self.profile.name, "%s.json" % self.version.replace(os.sep, "_")) @classmethod @cachetools.cachedmethod( operator.attrgetter("_ensure_cache"), key=lambda p, v, vv: "%s-%s-%s" % (p.id, v.id, vv), lock=lambda _: id_lock, ) def ensure_firmware(cls, profile, vendor, version): """ Get or create firmware by profile, vendor and version :param profile: :param vendor: :param version: :return: """ while True: firmware = Firmware.objects.filter(profile=profile.id, vendor=vendor.id, version=version).first() if firmware: return firmware try: firmware = Firmware(profile=profile, vendor=vendor, version=version, uuid=uuid.uuid4()) firmware.save() return firmware except NotUniqueError: pass # Already created by concurrent process, reread
class DiscoveryID(Document): """ Managed Object's discovery identity """ meta = { "collection": "noc.inv.discovery_id", "strict": False, "auto_create_index": False, "indexes": ["object", "hostname", "udld_id", "macs"] } object = ForeignKeyField(ManagedObject) chassis_mac = ListField(EmbeddedDocumentField(MACRange)) hostname = StringField() router_id = StringField() udld_id = StringField() # UDLD local identifier # macs = ListField(LongField()) _mac_cache = cachetools.TTLCache(maxsize=10000, ttl=60) _udld_cache = cachetools.TTLCache(maxsize=1000, ttl=60) def __unicode__(self): return self.object.name @staticmethod def _macs_as_ints(ranges=None, additional=None): """ Get all MAC addresses within ranges as integers :param ranges: list of dicts {first_chassis_mac: ..., last_chassis_mac: ...} :param additional: Optional list of additional macs :return: List of integers """ ranges = ranges or [] additional = additional or [] # Apply ranges macs = set() for r in ranges: if not r: continue first = MAC(r["first_chassis_mac"]) last = MAC(r["last_chassis_mac"]) macs.update(m for m in range(int(first), int(last) + 1)) # Append additional macs macs.update(int(MAC(m)) for m in additional) return sorted(macs) @staticmethod def _macs_to_ranges(macs): """ Convert list of macs (as integers) to MACRange :param macs: List of integer :return: List of MACRange """ ranges = [] for mi in macs: if ranges and mi - ranges[-1][1] == 1: # Extend last range ranges[-1][1] = mi else: # New range ranges += [[mi, mi]] return [ MACRange(first_mac=str(MAC(r[0])), last_mac=str(MAC(r[1]))) for r in ranges ] @classmethod def submit(cls, object, chassis_mac=None, hostname=None, router_id=None, additional_macs=None): # Process ranges macs = cls._macs_as_ints(chassis_mac, additional_macs) ranges = cls._macs_to_ranges(macs) # Update database o = cls.objects.filter(object=object.id).first() if o: old_macs = set(m.first_mac for m in o.chassis_mac) o.chassis_mac = ranges o.hostname = hostname o.router_id = router_id old_macs -= set(m.first_mac for m in o.chassis_mac) if old_macs: cache.delete_many(["discoveryid-mac-%s" % m for m in old_macs]) # MAC index o.macs = macs o.save() else: cls(object=object, chassis_mac=ranges, hostname=hostname, router_id=router_id, macs=macs).save() @classmethod @cachedmethod(operator.attrgetter("_mac_cache"), key="discoveryid-mac-%s", lock=lambda _: mac_lock) def get_by_mac(cls, mac): return cls._get_collection().find_one({"macs": int(MAC(mac))}, { "_id": 0, "object": 1 }) @classmethod @cachetools.cachedmethod(operator.attrgetter("_udld_cache"), lock=lambda _: mac_lock) def get_by_udld_id(cls, device_id): return cls._get_collection().find_one({"udld_id": device_id}, { "_id": 0, "object": 1 }) @classmethod def find_object(cls, mac=None, ipv4_address=None): """ Find managed object :param mac: :param ipv4_address: :param cls: :return: Managed object instance or None """ def has_ip(ip, addresses): x = ip + "/" for a in addresses: if a.startswith(x): return True return False # Find by mac if mac: metrics["discoveryid_mac_requests"] += 1 r = cls.get_by_mac(mac) if r: return ManagedObject.get_by_id(r["object"]) if ipv4_address: metrics["discoveryid_ip_requests"] += 1 # Try router_id d = DiscoveryID.objects.filter(router_id=ipv4_address).first() if d: metrics["discoveryid_ip_routerid"] += 1 return d.object # Fallback to interface addresses o = set( d["managed_object"] for d in SubInterface._get_collection().with_options( read_preference=ReadPreference.SECONDARY_PREFERRED).find( { "ipv4_addresses": { "$gt": ipv4_address + "/", "$lt": ipv4_address + "/99" } }, { "_id": 0, "managed_object": 1, "ipv4_addresses": 1 }) if has_ip(ipv4_address, d["ipv4_addresses"])) if len(o) == 1: metrics["discoveryid_ip_interface"] += 1 return ManagedObject.get_by_id(list(o)[0]) metrics["discoveryid_ip_failed"] += 1 return None @classmethod def macs_for_object(cls, object): """ Get MAC addresses for object :param cls: :param object: :return: list of (fist_mac, last_mac) """ # Get discovered chassis id range o = DiscoveryID.objects.filter(object=object.id).first() if o and o.chassis_mac: c_macs = [(r.first_mac, r.last_mac) for r in o.chassis_mac] else: c_macs = [] # Get interface macs i_macs = set(i.mac for i in Interface.objects.filter( managed_object=object.id, mac__exists=True).only("mac") if i.mac) # Enrich discovered macs with additional interface's ones c_macs += [(m, m) for m in i_macs if not any(1 for f, t in c_macs if f <= m <= t)] return c_macs @classmethod def macs_for_objects(cls, objects_ids): """ Get MAC addresses for object :param cls: :param objects_ids: Lis IDs of Managed Object Instance :type: list :return: Dictionary mac: objects :rtype: dict """ if not objects_ids: return None if isinstance(objects_ids, list): objects = objects_ids else: objects = list(objects_ids) os = cls.objects.filter(object__in=objects) if not os: return None # Discovered chassis id range c_macs = { int(did[0][0]): did[1] for did in os.scalar("macs", "object") if did[0] } # c_macs = [r.macs for r in os] # Other interface macs i_macs = { int(MAC(i[0])): i[1] for i in Interface.objects.filter(managed_object__in=objects, mac__exists=True).scalar( "mac", "managed_object") if i[0] } # Other subinterface macs (actual for DSLAM) si_macs = { int(MAC(i[0])): i[1] for i in SubInterface.objects.filter(managed_object__in=objects, mac__exists=True).scalar( "mac", "managed_object") if i[0] } c_macs.update(i_macs) c_macs.update(si_macs) return c_macs def on_delete(self): # Reset cache macs = set(m.first_mac for m in self.chassis_mac) if macs: cache.delete_many(["discoveryid-mac-%s" % m for m in macs]) @classmethod def clean_for_object(cls, mo): if hasattr(mo, "id"): mo = mo.id for d in DiscoveryID.objects.filter(object=mo): d.delete() @classmethod def find_objects(cls, macs): """ Find objects for list of macs :param macs: List of MAC addresses :return: dict of MAC -> ManagedObject for resolved MACs """ r = {} if not macs: return r # Build list of macs to search mlist = sorted(int(MAC(m)) for m in macs) # Search for macs obj_ranges = {} # (first, last) -> mo for d in DiscoveryID._get_collection().find({"macs": { "$in": mlist }}, { "_id": 0, "object": 1, "chassis_mac": 1 }): mo = ManagedObject.get_by_id(d["object"]) if mo: for dd in d.get("chassis_mac", []): obj_ranges[int(MAC(dd["first_mac"])), int(MAC(dd["last_mac"]))] = mo n = 1 for s, e in obj_ranges: n += 1 # Resolve ranges start = 0 ll = len(mlist) for s, e in sorted(obj_ranges): mo = obj_ranges[s, e] start = bisect.bisect_left(mlist, s, start, ll) while start < ll and s <= mlist[start] <= e: r[MAC(mlist[start])] = mo start += 1 return r @classmethod def update_udld_id(cls, object, local_id): """ Update UDLD id if necessary :param object: Object for set :param local_id: Local UDLD id :return: """ DiscoveryID._get_collection().update_one( {"object": object.id}, {"$set": { "udld_id": local_id }}, upsert=True)
class NetworkSegmentProfile(Document): meta = { "collection": "noc.networksegmentprofiles", "strict": False, "auto_create_index": False } name = StringField(unique=True) description = StringField(required=False) # segment discovery interval discovery_interval = IntField(default=86400) # Segment style style = ForeignKeyField(Style, required=False) # Restrict MAC discovery to management vlan mac_restrict_to_management_vlan = BooleanField(default=False) # Management vlan, to restrict MAC search for MAC topology discovery management_vlan = IntField(required=False, min_value=1, max_value=4095) # MVR VLAN multicast_vlan = IntField(required=False, min_value=1, max_value=4095) # Detect lost redundancy condition enable_lost_redundancy = BooleanField(default=False) # Horizontal transit policy horizontal_transit_policy = StringField( choices=[ ("E", "Always Enable"), ("C", "Calculate"), ("D", "Disable") ], default="D" ) # Default profile for autocreated children segments # (i.e. during autosegmentation) # Copy this segment profile otherwise autocreated_profile = PlainReferenceField("self") # List of enabled topology method # in order of preference (most preferable first) topology_methods = ListField(EmbeddedDocumentField(SegmentTopologySettings)) # Enable VLAN discovery for appropriative management objects enable_vlan = BooleanField(default=False) # Default VLAN profile for discovered VLANs default_vlan_profile = PlainReferenceField("vc.VLANProfile") # Integration with external NRI and TT systems # Reference to remote system object has been imported from remote_system = PlainReferenceField(RemoteSystem) # Object id in remote system remote_id = StringField() # Object id in BI bi_id = LongField(unique=True) _id_cache = cachetools.TTLCache(maxsize=100, ttl=60) _bi_id_cache = cachetools.TTLCache(maxsize=100, ttl=60) def __unicode__(self): return self.name @classmethod @cachetools.cachedmethod(operator.attrgetter("_id_cache"), lock=lambda _: id_lock) def get_by_id(cls, id): return NetworkSegmentProfile.objects.filter(id=id).first() @classmethod @cachetools.cachedmethod(operator.attrgetter("_bi_id_cache"), lock=lambda _: id_lock) def get_by_bi_id(cls, id): return NetworkSegmentProfile.objects.filter(bi_id=id).first() def on_save(self): if hasattr(self, "_changed_fields") and "discovery_interval" in self._changed_fields: from .networksegment import NetworkSegment for ns in NetworkSegment.objects.filter(profile=self.id): ns.ensure_discovery_jobs() def get_topology_methods(self): ml = getattr(self, "_topology_methods", None) if not ml: ml = [m.method for m in self.topology_methods if m.is_active and m not in ("custom", "handler")] self._topology_methods = ml return ml def is_preferable_method(self, m1, m2): """ Returns True if m1 topology discovery method is preferable over m2 """ if m1 == m2: # Method can refine itself return True try: methods = self.get_topology_methods() i1 = methods.index(m1) i2 = methods.index(m2) except ValueError: return False return i1 <= i2
class Sensor(Document): meta = { "collection": "sensors", "strict": False, "auto_create_index": False } profile = PlainReferenceField(SensorProfile, default=SensorProfile.get_default_profile) object = PlainReferenceField(Object) managed_object = ForeignKeyField(ManagedObject) local_id = StringField() state = PlainReferenceField(State) units = PlainReferenceField(MeasurementUnits) label = StringField() dashboard_label = StringField() # Sources that find sensor sources = ListField(StringField(choices=list(SOURCES))) # Timestamp of last seen last_seen = DateTimeField() # Timestamp expired expired = DateTimeField() # Timestamp of first discovery first_discovered = DateTimeField(default=datetime.datetime.now) protocol = StringField( choices=["modbus_rtu", "modbus_ascii", "modbus_tcp", "snmp", "ipmi"]) modbus_register = IntField() snmp_oid = StringField() ipmi_id = StringField() bi_id = LongField(unique=True) # Labels labels = ListField(StringField()) effective_labels = ListField(StringField()) _id_cache = cachetools.TTLCache(maxsize=100, ttl=60) _bi_id_cache = cachetools.TTLCache(maxsize=100, ttl=60) def __str__(self): if self.object: return f"{self.object}: {self.local_id}" elif self.managed_object: return f"{self.managed_object}: {self.local_id}" return f"{self.units}: {self.local_id}" @classmethod @cachetools.cachedmethod(operator.attrgetter("_id_cache"), lock=lambda _: id_lock) def get_by_id(cls, id): return Sensor.objects.filter(id=id).first() @classmethod @cachetools.cachedmethod(operator.attrgetter("_bi_id_cache"), lock=lambda _: id_lock) def get_by_bi_id(cls, id): return Sensor.objects.filter(bi_id=id).first() def seen(self, source: Optional[str] = None): """ Seen sensor """ if source and source in SOURCES: self.sources = list(set(self.sources or []).union(set([source]))) self._get_collection().update_one( {"_id": self.id}, {"$addToSet": { "sources": source }}) self.fire_event("seen") self.touch() # Worflow expired def unseen(self, source: Optional[str] = None): """ Unseen sensor """ logger.info( "[%s|%s] Sensor is missed '%s'", self.object.name if self.object else "-", "-", self.local_id, ) if source and source in SOURCES: self.sources = list(set(self.sources or []) - set([source])) self._get_collection().update_one({"_id": self.id}, {"$pull": { "sources": source }}) elif not source: # For empty source, clean sources self.sources = [] self._get_collection().update_one({"_id": self.id}, {"$set": { "sources": [] }}) if not self.sources: # source - None, set sensor to missed self.fire_event("missed") self.touch() @classmethod def sync_object(cls, obj: Object) -> None: """ Synchronize sensors with object model :param obj: :return: """ # Get existing sensors obj_sensors: Dict[str, Sensor] = { s.local_id: s for s in Sensor.objects.filter(object=obj.id) } m_proto = [ d.value for d in obj.get_effective_data() if d.interface == "modbus" and d.attr == "type" ] or ["rtu"] # Create new sensors for sensor in obj.model.sensors: if sensor.name in obj_sensors: obj_sensors[sensor.name].seen("objectmodel") del obj_sensors[sensor.name] continue # logger.info("[%s|%s] Creating new sensor '%s'", obj.name if obj else "-", "-", sensor.name) s = Sensor( profile=SensorProfile.get_default_profile(), object=obj, local_id=sensor.name, units=sensor.units, label=sensor.description, ) # Get sensor protocol if sensor.modbus_register: if not m_proto: continue s.protocol = "modbus_%s" % m_proto[0].lower() s.modbus_register = sensor.modbus_register elif sensor.snmp_oid: s.protocol = "snmp" s.snmp_oid = sensor.snmp_oid else: logger.info( "[%s|%s] Unknown sensor protocol '%s'", obj.name if obj else "-", "-", sensor.name, ) s.save() s.seen("objectmodel") # Notify missed sensors for s in sorted(obj_sensors): sensor = obj_sensors[s] sensor.unseen(source="objectmodel") @classmethod def can_set_label(cls, label): return Label.get_effective_setting(label, setting="enable_sensor") @classmethod def iter_effective_labels(cls, intance: "Sensor") -> Iterable[List[str]]: yield intance.labels or [] + intance.profile.labels or []
class VLAN(Document): meta = { "collection": "vlans", "strict": False, "auto_create_index": False, "indexes": [{ "fields": ["segment", "vlan"], "unique": True }, "expired"], } name = StringField() profile = PlainReferenceField(VLANProfile) vlan = IntField(min_value=1, max_value=4095) segment = PlainReferenceField(NetworkSegment) description = StringField() state = PlainReferenceField(State) project = ForeignKeyField(Project) # Link to gathering VPN vpn = PlainReferenceField(VPN) # VxLAN VNI vni = IntField() # Translation rules when passing border translation_rule = StringField(choices=[ # Rewrite tag to parent vlan's ("map", "map"), # Append parent tag as S-VLAN ("push", "push"), ]) # parent = PlainReferenceField("self") # Automatically apply segment translation rule apply_translation = BooleanField(default=True) # Labels labels = ListField(StringField()) effective_labels = ListField(StringField()) # Integration with external NRI and TT systems # Reference to remote system object has been imported from remote_system = PlainReferenceField(RemoteSystem) # Object id in remote system remote_id = StringField() # Object id in BI bi_id = LongField(unique=True) # Discovery integration # Timestamp when object first discovered first_discovered = DateTimeField() # Timestamp when object last seen by discovery last_seen = DateTimeField() # Timestamp when send "expired" event expired = DateTimeField() _id_cache = cachetools.TTLCache(maxsize=100, ttl=60) _bi_id_cache = cachetools.TTLCache(maxsize=100, ttl=60) def __str__(self): return self.name @classmethod @cachetools.cachedmethod(operator.attrgetter("_id_cache"), lock=lambda _: id_lock) def get_by_id(cls, id): return VLAN.objects.filter(id=id).first() @classmethod @cachetools.cachedmethod(operator.attrgetter("_bi_id_cache"), lock=lambda _: id_lock) def get_by_bi_id(cls, id): return VLAN.objects.filter(bi_id=id).first() def clean(self): super().clean() self.segment = NetworkSegment.get_border_segment(self.segment) if self.translation_rule and not self.parent: self.translation_rule = None def refresh_translation(self): """ Set VLAN translation according to segment settings :return: """ if not self.apply_translation: return # Find matching rule for vt in self.segment.vlan_translation: if vt.filter.check(self.vlan): logger.debug( "[%s|%s|%s] Matching translation rule <%s|%s|%s>", self.segment.name, self.name, self.vlan, vt.filter.expression, vt.rule, vt.parent_vlan.vlan, ) if self.parent != vt.parent_vlan or self.translation_rule != vt.translation_rule: self.modify(parent=vt.parent_vlan, translation_rule=vt.rule) return # No matching rule if self.parent or self.translation_rule: logger.debug("[%s|%s|%s] No matching translation rule, resetting") if self.parent or self.translation_rule: self.modify(parent=None, translation_rule=None) def on_save(self): self.refresh_translation() @classmethod def can_set_label(cls, label): if label.enable_vlan: return True return False
class InterfaceProfile(Document): """ Interface SLA profile and settings """ meta = { "collection": "noc.interface_profiles", "strict": False, "auto_create_index": False } name = StringField(unique=True) description = StringField() style = ForeignKeyField(Style, required=False) # Interface-level events processing link_events = StringField( required=True, choices=[ ("I", "Ignore Events"), ("L", "Log events, do not raise alarms"), ("A", "Raise alarms"), ], default="A", ) # Discovery settings discovery_policy = StringField( choices=[("I", "Ignore"), ("O", "Create new"), ("R", "Replace"), ("C", "Add to cloud")], default="R", ) # Collect mac addresses on interface mac_discovery_policy = StringField( choices=[ ("d", "Disabled"), ("m", "Management VLAN"), ("e", "Transit"), ("i", "Direct Downlink"), ("c", "Chained Downlink"), ("u", "Direct Uplink"), ("C", "Cloud Downlink"), ], default="d", ) # Collect and keep interface status status_discovery = BooleanField(default=False) # allow_lag_mismatch = BooleanField(default=False) # Send up/down notifications status_change_notification = ForeignKeyField(NotificationGroup, required=False) # Interface profile metrics metrics = ListField(EmbeddedDocumentField(InterfaceProfileMetrics)) # Alarm weight weight = IntField(default=0) # User network interface # MAC discovery can be restricted to UNI is_uni = BooleanField(default=False) # Allow automatic segmentation allow_autosegmentation = BooleanField(default=False) # Allow collecting metrics from subinterfaces allow_subinterface_metrics = BooleanField(default=False) # allow_vacuum_bulling = BooleanField(default=False) # Validation policy interface_validation_policy = PlainReferenceField( InterfaceValidationPolicy) # ifdesc_patterns = PlainReferenceField(IfDescPatterns) ifdesc_handler = PlainReferenceField(Handler) # Enable abduct detection on interface enable_abduct_detection = BooleanField(default=False) # Integration with external NRI and TT systems # Reference to remote system object has been imported from remote_system = ReferenceField(RemoteSystem) # Object id in remote system remote_id = StringField() # Object id in BI bi_id = LongField(unique=True) _id_cache = cachetools.TTLCache(maxsize=100, ttl=60) _name_cache = cachetools.TTLCache(maxsize=100, ttl=60) _bi_id_cache = cachetools.TTLCache(maxsize=100, ttl=60) _default_cache = cachetools.TTLCache(maxsize=100, ttl=60) _status_discovery_cache = cachetools.TTLCache(maxsize=100, ttl=60) DEFAULT_PROFILE_NAME = "default" def __str__(self): return self.name @classmethod @cachetools.cachedmethod(operator.attrgetter("_id_cache"), lock=lambda _: id_lock) def get_by_id(cls, id): return InterfaceProfile.objects.filter(id=id).first() @classmethod @cachetools.cachedmethod(operator.attrgetter("_bi_id_cache"), lock=lambda _: id_lock) def get_by_bi_id(cls, id): return InterfaceProfile.objects.filter(bi_id=id).first() @classmethod @cachetools.cachedmethod(operator.attrgetter("_name_cache"), lock=lambda _: id_lock) def get_by_name(cls, name): return InterfaceProfile.objects.filter(name=name).first() @classmethod @cachetools.cachedmethod(operator.attrgetter("_default_cache"), lock=lambda _: id_lock) def get_default_profile(cls): return InterfaceProfile.objects.filter( name=cls.DEFAULT_PROFILE_NAME).first() @classmethod @cachetools.cachedmethod(operator.attrgetter("_status_discovery_cache"), lock=lambda _: id_lock) def get_with_status_discovery(cls): """ Get list of interface profile ids with status_discovery = True :return: """ return list(x["_id"] for x in InterfaceProfile._get_collection().find( {"status_discovery": True}, {"_id": 1}))
class Vendor(Document): """ Equipment vendor """ meta = { "collection": "noc.vendors", "strict": False, "auto_create_index": False, "json_collection": "inv.vendors", "json_unique_fields": ["name", "code"], } # Short vendor name, included as first part of platform name = StringField(unique=True) # Full vendor name full_name = StringField() # Unique id uuid = UUIDField(binary=True) # List of vendor codes to be searched via .get_by_code() code = ListField(StringField(), unique=True) # Vendor's site site = URLField(required=False) # Object id in BI bi_id = LongField(unique=True) _id_cache = cachetools.TTLCache(1000, ttl=60) _bi_id_cache = cachetools.TTLCache(1000, ttl=60) _code_cache = cachetools.TTLCache(1000, ttl=60) _ensure_cache = cachetools.TTLCache(1000, ttl=60) def __str__(self): return self.name @classmethod @cachetools.cachedmethod(operator.attrgetter("_id_cache"), lock=lambda _: id_lock) def get_by_id(cls, id): return Vendor.objects.filter(id=id).first() @classmethod @cachetools.cachedmethod(operator.attrgetter("_bi_id_cache"), lock=lambda _: id_lock) def get_by_bi_id(cls, id): return Vendor.objects.filter(bi_id=id).first() @classmethod def _get_by_code(cls, code): """ Uncached version of get_by_code :param code: :return: """ code = code.upper() return Vendor.objects.filter(code=code).first() @classmethod @cachetools.cachedmethod(operator.attrgetter("_code_cache"), lock=lambda _: id_lock) def get_by_code(cls, code): return cls._get_by_code(code) def clean(self): # Convert code to list if isinstance(self.code, str): self.code = [self.code] # Uppercase code self.code = [c.upper() for c in self.code] # Fill full name if not set if not self.full_name: self.full_name = self.name # super().clean() def on_save(self): if not hasattr(self, "_changed_fields") or "name" in self._changed_fields: from .platform import Platform for p in Platform.objects.filter(vendor=self.id): p.save() # Rebuild full name def to_json(self): return to_json( { "name": self.name, "$collection": self._meta["json_collection"], "full_name": self.full_name, "code": self.code, "site": self.site, "uuid": self.uuid, }, order=["name", "uuid", "full_name", "code", "site"], ) def get_json_path(self): return "%s.json" % self.name @classmethod @cachetools.cachedmethod(operator.attrgetter("_ensure_cache"), lock=lambda _: id_lock) def ensure_vendor(cls, code): """ Get or create vendor by code :param code: :return: """ while True: vendor = Vendor._get_by_code(code) if vendor: return vendor try: vendor = Vendor(name=code, full_name=code, code=[code], uuid=uuid.uuid4()) vendor.save() return vendor except NotUniqueError: pass # Already created by concurrent process, reread @classmethod def iter_lazy_labels(cls, vendor: "Vendor"): yield f"noc::vendor::{vendor.name}::="
class Object(Document): """ Inventory object """ meta = { "collection": "noc.objects", "strict": False, "auto_create_index": False, "indexes": [ "data", "container", ("name", "container"), ("data.interface", "data.attr", "data.value"), ], } name = StringField() model = PlainReferenceField(ObjectModel) data = ListField(EmbeddedDocumentField(ObjectAttr)) container = PlainReferenceField("self", required=False) comment = GridVCSField("object_comment") # Map layer = PlainReferenceField(Layer) point = PointField(auto_index=True) # Additional connection data connections = ListField(EmbeddedDocumentField(ObjectConnectionData)) # Labels labels = ListField(StringField()) effective_labels = ListField(StringField()) # Integration with external NRI and TT systems # Reference to remote system object has been imported from remote_system = ReferenceField(RemoteSystem) # Object id in remote system remote_id = StringField() # Object id in BI bi_id = LongField(unique=True) _id_cache = cachetools.TTLCache(maxsize=1000, ttl=60) _bi_id_cache = cachetools.TTLCache(maxsize=1000, ttl=60) REBUILD_CONNECTIONS = ["links", "conduits"] def __str__(self): return smart_text(self.name or self.id) @classmethod @cachetools.cachedmethod(operator.attrgetter("_id_cache"), lock=lambda _: id_lock) def get_by_id(cls, id) -> Optional["Object"]: return Object.objects.filter(id=id).first() @classmethod @cachetools.cachedmethod(operator.attrgetter("_bi_id_cache"), lock=lambda _: id_lock) def get_by_bi_id(cls, id) -> Optional["Object"]: return Object.objects.filter(bi_id=id).first() def iter_changed_datastream(self, changed_fields=None): if config.datastream.enable_managedobject: if self.data and self.get_data("management", "managed_object"): yield "managedobject", self.get_data("management", "managed_object") else: for _, o, _ in self.iter_outer_connections(): if o.data and o.get_data("management", "managed_object"): yield "managedobject", o.get_data( "management", "managed_object") def clean(self): self.set_point() def set_point(self): from noc.gis.map import map # Reset previous data self.layer = None self.point = None # Get points x, y, srid = self.get_data_tuple("geopoint", ("x", "y", "srid")) if x is None or y is None: return # No point data # Get layer layer_code = self.model.get_data("geopoint", "layer") if not layer_code: return layer = Layer.get_by_code(layer_code) if not layer: return # Update actual data self.layer = layer self.point = map.get_db_point(x, y, srid=srid) def on_save(self): def get_coordless_objects(o): r = {str(o.id)} for co in Object.objects.filter(container=o.id): cx, cy = co.get_data_tuple("geopoint", ("x", "y")) if cx is None and cy is None: r |= get_coordless_objects(co) return r x, y = self.get_data_tuple("geopoint", ("x", "y")) if x is not None and y is not None: # Rebuild connection layers for ct in self.REBUILD_CONNECTIONS: for c, _, _ in self.get_genderless_connections(ct): c.save() # Update nested objects from noc.sa.models.managedobject import ManagedObject mos = get_coordless_objects(self) if mos: ManagedObject.objects.filter(container__in=mos).update( x=x, y=y, default_zoom=self.layer.default_zoom if self.layer else DEFAULT_ZOOM) if self._created: if self.container: pop = self.get_pop() if pop: pop.update_pop_links() # Changed container elif hasattr( self, "_changed_fields") and "container" in self._changed_fields: # Old pop old_container_id = getattr(self, "_old_container", None) old_pop = None if old_container_id: c = Object.get_by_id(old_container_id) while c: if c.get_data("pop", "level"): old_pop = c break c = c.container # New pop new_pop = self.get_pop() # Check if pop moved if old_pop != new_pop: if old_pop: old_pop.update_pop_links() if new_pop: new_pop.update_pop_links() if self.model.sensors: self._sync_sensors() @cachetools.cached(_path_cache, key=lambda x: str(x.id), lock=id_lock) def get_path(self) -> List[str]: """ Returns list of parent segment ids :return: """ if self.container: return self.container.get_path() + [self.id] return [self.id] def get_nested_ids(self): """ Return id of this and all nested object :return: """ # $graphLookup hits 100Mb memory limit. Do not use it seen = {self.id} wave = {self.id} max_level = 4 coll = Object._get_collection() for _ in range(max_level): # Get next wave wave = (set(d["_id"] for d in coll.find({"container": { "$in": list(wave) }}, {"_id": 1})) - seen) if not wave: break seen |= wave return list(seen) def get_data(self, interface: str, key: str, scope: Optional[str] = None) -> Any: attr = ModelInterface.get_interface_attr(interface, key) if attr.is_const: # Lookup model return self.model.get_data(interface, key) for item in self.data: if item.interface == interface and item.attr == key: if not scope or item.scope == scope: return item.value return None def get_data_dict(self, interface: str, keys: Iterable, scope: Optional[str] = None) -> Dict[str, Any]: """ Get multiple keys from single interface. Returns dict with values for every given key. If key is missed, return None value :param interface: :param keys: Iterable contains key names :param scope: :return: """ kset = set(keys) r = {k: None for k in kset} for item in self.data: if item.interface == interface and item.attr in kset: if not scope or item.scope == scope: r[item.attr] = item.value return r def get_data_tuple(self, interface: str, keys: Union[List, Tuple], scope: Optional[str] = None) -> Tuple[Any, ...]: """ Get multiple keys from single interface. Returns tuple with values for every given key. If key is missed, return None value :param interface: :param keys: List or tuple with key names :param scope: :return: """ r = self.get_data_dict(interface, keys, scope) return tuple(r.get(k) for k in keys) def get_effective_data(self) -> List[ObjectAttr]: """ Return effective object data, including the model's defaults :return: """ seen: Set[Tuple[str, str, str]] = set() # (interface, attr, scope r: List[ObjectAttr] = [] # Object attributes for item in self.data: k = (item.interface, item.attr, item.scope or "") if k in seen: continue r += [item] seen.add(k) # Model attributes for i in self.model.data: for a in self.model.data[i]: k = (i, a, "") if k in seen: continue r += [ ObjectAttr(interface=i, attr=a, scope="", value=self.model.data[i][a]) ] seen.add(k) # Sort according to interface sorting_keys: Dict[str, str] = {} for ni, i in enumerate(sorted(set(x[0] for x in seen))): mi = ModelInterface.get_by_name(i) if not mi: continue for na, a in enumerate(mi.attrs): sorting_keys["%s.%s" % (i, a.name)] = "%06d.%06d" % (ni, na) # Return sorted result return list( sorted( r, key=lambda oa: "%s.%s" % (sorting_keys.get( "%s.%s" % (oa.interface, oa.attr), "999999.999999"), oa.scope), )) def set_data(self, interface: str, key: str, value: Any, scope: Optional[str] = None) -> None: attr = ModelInterface.get_interface_attr(interface, key) if attr.is_const: raise ModelDataError("Cannot set read-only value") value = attr._clean(value) for item in self.data: if item.interface == interface and item.attr == key: if not scope or item.scope == scope: item.value = value break else: # Insert new item self.data += [ ObjectAttr(interface=interface, attr=attr.name, value=value, scope=scope or "") ] def reset_data(self, interface: str, key: Union[str, Iterable], scope: Optional[str] = None) -> None: if isinstance(key, str): kset = {key} else: kset = set(key) v = [ ModelInterface.get_interface_attr(interface, k).is_const for k in kset ] if any(v): raise ModelDataError("Cannot reset read-only value") self.data = [ item for item in self.data if item.interface != interface or ( scope and item.scope != scope) or item.attr not in kset ] def has_connection(self, name): return self.model.has_connection(name) def get_p2p_connection( self, name: str ) -> Tuple[Optional["ObjectConnection"], Optional["Object"], Optional[str]]: """ Get neighbor for p2p connection (s and mf types) Returns connection, remote object, remote connection or None, None, None """ c = ObjectConnection.objects.filter(__raw__={ "connection": { "$elemMatch": { "object": self.id, "name": name } } }).first() if c: for x in c.connection: if x.object.id != self.id: return c, x.object, x.name # Strange things happen return None, None, None def get_genderless_connections( self, name: str) -> List[Tuple["ObjectConnection", "Object", str]]: r = [] for c in ObjectConnection.objects.filter(__raw__={ "connection": { "$elemMatch": { "object": self.id, "name": name } } }): for x in c.connection: if x.object.id != self.id: r += [[c, x.object, x.name]] return r def disconnect_p2p(self, name: str): """ Remove connection *name* """ c = self.get_p2p_connection(name)[0] if c: self.log("'%s' disconnected" % name, system="CORE", op="DISCONNECT") c.delete() def connect_p2p( self, name: str, remote_object: "Object", remote_name: str, data: Dict[str, Any], reconnect: bool = False, ) -> Optional["ObjectConnection"]: lc = self.model.get_model_connection(name) if lc is None: raise ConnectionError("Local connection not found: %s" % name) name = lc.name rc = remote_object.model.get_model_connection(remote_name) if rc is None: raise ConnectionError("Remote connection not found: %s" % remote_name) remote_name = rc.name valid, cause = self.model.check_connection(lc, rc) if not valid: raise ConnectionError(cause) # Check existing connecitons if lc.type.genders in ("s", "m", "f", "mf"): ec, r_object, r_name = self.get_p2p_connection(name) if ec is not None: # Connection exists if reconnect: if r_object.id == remote_object.id and r_name == remote_name: # Same connection exists n_data = deep_merge(ec.data, data) # Merge ObjectConnection if n_data != ec.data: # Update data ec.data = n_data ec.save() return self.disconnect_p2p(name) else: raise ConnectionError("Already connected") # Create connection c = ObjectConnection( connection=[ ObjectConnectionItem(object=self, name=name), ObjectConnectionItem(object=remote_object, name=remote_name), ], data=data, ).save() self.log("%s:%s -> %s:%s" % (self, name, remote_object, remote_name), system="CORE", op="CONNECT") # Disconnect from container on o-connection if lc.direction == "o" and self.container: self.log("Remove from %s" % self.container, system="CORE", op="REMOVE") self.container = None self.save() return c def connect_genderless( self, name: str, remote_object: "Object", remote_name: str, data: Dict[str, Any] = None, type: Optional[str] = None, layer: Optional[Layer] = None, ): """ Connect two genderless connections """ lc = self.model.get_model_connection(name) if lc is None: raise ConnectionError("Local connection not found: %s" % name) name = lc.name rc = remote_object.model.get_model_connection(remote_name) if rc is None: raise ConnectionError("Remote connection not found: %s" % remote_name) remote_name = rc.name if lc.gender != "s": raise ConnectionError("Local connection '%s' must be genderless" % name) if rc.gender != "s": raise ConnectionError("Remote connection '%s' must be genderless" % remote_name) # Check for connection for c, ro, rname in self.get_genderless_connections(name): if ro.id == remote_object.id and rname == remote_name: c.data = data or {} c.save() return # Normalize layer if layer and isinstance(layer, str): layer = Layer.get_by_code(layer) # Create connection ObjectConnection( connection=[ ObjectConnectionItem(object=self, name=name), ObjectConnectionItem(object=remote_object, name=remote_name), ], data=data or {}, type=type or None, layer=layer, ).save() self.log("%s:%s -> %s:%s" % (self, name, remote_object, remote_name), system="CORE", op="CONNECT") def put_into(self, container: "Object"): """ Put object into container """ if container and not container.get_data("container", "container"): raise ValueError("Must be put into container") # Disconnect all o-connections for c in self.model.connections: if c.direction == "o": c, _, _ = self.get_p2p_connection(c.name) if c: self.disconnect_p2p(c.name) # Connect to parent self.container = container.id if container else None # Reset previous rack position self.reset_data("rackmount", ("position", "side", "shift")) # self.save() self.log("Insert into %s" % (container or "Root"), system="CORE", op="INSERT") def get_content(self) -> "Object": """ Returns all items directly put into container """ return Object.objects.filter(container=self.id) def get_local_name_path(self): for _, ro, rn in self.get_outer_connections(): return ro.get_local_name_path() + [rn] return [] def get_name_path(self) -> List[str]: """ Return list of container names """ current = self.container if current is None: for _, ro, rn in self.get_outer_connections(): return ro.get_name_path() + [rn] return [smart_text(self)] np = [smart_text(self)] while current: np.insert(0, smart_text(current)) current = current.container return np def log(self, message, user=None, system=None, managed_object=None, op=None): if not user: user = get_user() if hasattr(user, "username"): user = user.username if not user: user = "******" if not isinstance(managed_object, str): managed_object = smart_text(managed_object) ObjectLog( object=self.id, user=user, ts=datetime.datetime.now(), message=message, system=system, managed_object=managed_object, op=op, ).save() def get_log(self): return ObjectLog.objects.filter(object=self.id).order_by("ts") def get_lost_and_found(self) -> Optional["Object"]: m = ObjectModel.get_by_name("Lost&Found") c = self.container while c: # Check siblings lf = Object.objects.filter(container=c, model=m).first() if lf: return lf # Up one level c = c.container return None @classmethod def detach_children(cls, sender, document, target=None): if not document.get_data("container", "container"): return if not target: target = document.get_lost_and_found() for o in Object.objects.filter(container=document.id): if o.get_data("container", "container"): cls.detach_children(sender, o, target) o.delete() else: o.put_into(target) def iter_connections( self, direction: Optional[str]) -> Iterable[Tuple[str, "Object", str]]: """ Yields connections of specified direction as tuples of (name, remote_object, remote_name) """ ic = set(c.name for c in self.model.connections if c.direction == direction) for c in ObjectConnection.objects.filter(connection__object=self.id): sn = None oc = None for cc in c.connection: if cc.object.id == self.id: if cc.name in ic: sn = cc.name else: oc = cc if sn and oc: yield sn, oc.object, oc.name def iter_inner_connections(self): """ Yields inner connections as tuples of (name, remote_object, remote_name) """ yield from self.iter_connections("i") def iter_outer_connections(self): """ Yields outer connections as tuples of (name, remote_object, remote_name) """ yield from self.iter_connections("o") def has_inner_connections(self): """ Returns True if object has any inner connections """ return any(self.iter_inner_connections()) def get_inner_connections(self): """ Returns a list of inner connections as (name, remote_object, remote_name) """ return list(self.iter_inner_connections()) def get_outer_connections(self): """ Returns a list of outer connections as (name, remote_object, remote_name) """ return list(self.iter_outer_connections()) @classmethod def delete_disconnect(cls, sender, document, target=None): for c in ObjectConnection.objects.filter( connection__object=document.id): left = [cc for cc in c.connection if cc.object.id != document.id] if len(left) < 2: c.delete() # Remove connection else: # Wipe object c.connection = left c.save() def get_pop(self) -> Optional["Object"]: """ Find enclosing PoP :returns: PoP instance or None """ c = self.container while c: if c.get_data("pop", "level"): return c c = c.container return None def get_coordinates_zoom( self) -> Tuple[Optional[float], Optional[float], Optional[int]]: """ Get managed object's coordinates # @todo: Speedup? :returns: x (lon), y (lat), zoom level """ c = self while c: if c.point and c.layer: x, y = c.get_data_tuple("geopoint", ("x", "y")) zoom = c.layer.default_zoom or DEFAULT_ZOOM return x, y, zoom if c.container: c = Object.get_by_id(c.container.id) if c: continue break return None, None, None @classmethod def get_managed(cls, mo): """ Get Object managed by managed object :param mo: Managed Object instance or id :returns: Objects managed by managed object, or empty list """ if hasattr(mo, "id"): mo = mo.id return cls.objects.filter(data__match={ "interface": "management", "attr": "managed_object", "value": mo }) def iter_managed_object_id(self) -> Iterator[int]: for d in Object._get_collection().aggregate([ { "$match": { "_id": self.id } }, # Get all nested objects and put them into the _path field { "$graphLookup": { "from": "noc.objects", "connectFromField": "_id", "connectToField": "container", "startWith": "$_id", "as": "_path", "maxDepth": 50, } }, # Leave only _path field { "$project": { "_id": 0, "_path": 1 } }, # Unwind _path array to separate documents { "$unwind": { "path": "$_path" } }, # Move data one level up { "$project": { "data": "$_path.data" } }, # Unwind data { "$unwind": { "path": "$data" } }, # Convert nested data to flat document { "$project": { "interface": "$data.interface", "attr": "$data.attr", "value": "$data.value", } }, # Leave only management objects { "$match": { "interface": "management", "attr": "managed_object" } }, # Leave only value { "$project": { "value": 1 } }, ]): mo = d.get("value") if mo: yield mo @classmethod def get_by_path(cls, path: List[str], hints=None) -> Optional["Object"]: """ Get object by given path. :param path: List of names following to path :param hints: {name: object_id} dictionary for getting object in path :returns: Object instance. None if not found """ current = None for p in path: current = Object.objects.filter(name=p, container=current).first() if not current: return None if hints: h = hints.get(p) if h: return Object.get_by_id(h) return current def update_pop_links(self, delay: int = 20): call_later("noc.inv.util.pop_links.update_pop_links", delay, pop_id=self.id) @classmethod def _pre_init(cls, sender, document, values, **kwargs): """ Object pre-initialization """ # Store original container id if "container" in values and values["container"]: document._cache_container = values["container"] def get_address_text(self) -> Optional[str]: """ Return first found address.text value upwards the path :return: Address text or None """ current = self while current: addr = current.get_data("address", "text") if addr: return addr if current.container: current = Object.get_by_id(current.container.id) else: break return None def get_object_serials(self, chassis_only: bool = True) -> List[str]: """ Gettint object serialNumber :param chassis_only: With serial numbers inner objects :return: """ serials = [self.get_data("asset", "serial")] if not chassis_only: for sn, oo, name in self.iter_inner_connections(): serials += oo.get_object_serials(chassis_only=False) return serials def iter_scope(self, scope: str) -> Iterable[Tuple[PathItem, ...]]: """ Yields Full physical path for all connections with given scopes behind the object :param scope: Scope name :return: """ connections = { name: ro for name, ro, _ in self.iter_inner_connections() } for c in self.model.connections: if c.type.is_matched_scope(scope, c.protocols): # Yield connection yield PathItem(object=self, connection=c), elif c.name in connections: ro = connections[c.name] for part_path in ro.iter_scope(scope): yield (PathItem(object=self, connection=c), ) + part_path def set_connection_interface(self, name, if_name): for cdata in self.connections: if cdata.name == name: cdata.interface_name = if_name return # New item self.connections += [ ObjectConnectionData(name=name, interface_name=if_name) ] def reset_connection_interface(self, name): self.connections = [c for c in self.connections if c.name != name] def _sync_sensors(self): """ Synchronize sensors :return: """ from .sensor import Sensor Sensor.sync_object(self) @classmethod def iter_by_address_id(cls, address: Union[str, List[str]], scope: str = None) -> Iterable["Object"]: """ Get objects :param address: :param scope: :return: """ q = { "interface": "address", "scope": scope or "", "attr": "id", } if isinstance(address, list): if len(address) == 1: q["value"] = address[0] else: q["value__in"] = address else: q["value"] = address yield from cls.objects.filter(data__match=q) @classmethod def can_set_label(cls, label): return Label.get_effective_setting(label, setting="enable_object")