class Poll(Document): date = DateTimeField(default=datetime.datetime.utcnow) author = StringField() labels = DictField() questions = ListField() possible_answers = ListField() related_answers = ListField() @classmethod def create(cls, poll_template: dict) -> dict: try: author = poll_template.get('author', 'Unknown') labels = poll_template.get('labels', ['default']) questions = poll_template.get('questions', []) possible_answers = poll_template.get('possible_answers', []) if questions != [] and len(questions) == len( possible_answers) and cls._check_answers_len( possible_answers): poll_instance = cls(author=author, labels=labels, questions=questions, possible_answers=possible_answers, related_answers=[]) poll_instance.save() status = True result = 'Everything is OK!' else: status = False result = "The questions list is either empty or there is a problem with answers length.." except Exception as ex: status = False result = f"An error ocurred while saving a Poll:{ex}" finally: return {"status": status, "result": result} @classmethod def get_all(cls): try: output = [] status = True result = cls.objects().as_pymongo() for index, item in enumerate(result): item['id'] = str(item['_id']) del (item['_id']) output.append(item) result = output except Exception as ex: status = False result = f"An error ocurred while querying all Polls:{ex}" finally: return {"status": status, "result": result} @classmethod def get_by_id(cls, id): try: status = True result = cls.objects(id=id) except Exception as ex: status = False result = f"An error ocurred while querying one Poll:{ex}" finally: return {"status": status, "result": result} @classmethod def get_by_user(cls, user_id): try: output = [] status = True result = cls.objects(author=user_id).as_pymongo() for index, item in enumerate(result): item['id'] = str(item['_id']) del (item['_id']) output.append(item) result = output except Exception as ex: status = False result = f"An error ocurred while querying user's Poll:{ex}" finally: return {"status": status, "result": result} @classmethod def get_by_labels(cls, labels: dict) -> list: try: output = [] result = cls.objects(labels=labels).as_pymongo() for index, item in enumerate(result): item['id'] = str(item['_id']) del (item['_id']) output.append(item) status = True result = output except Exception as ex: status = False result = f"An error ocurred while querying one Poll:{ex}" finally: return {"status": status, "result": result} @staticmethod def _check_answers_len(answers_list: list) -> bool: return all( [True if len(answers) <= 4 else False for answers in answers_list])
class ObjectCapabilities(Document): meta = { "collection": "noc.sa.objectcapabilities", "strict": False, "auto_create_index": False } object = ForeignKeyField(ManagedObject, primary_key=True) caps = ListField(EmbeddedDocumentField(CapsItem)) def __unicode__(self): return "%s caps" % self.object.name def on_save(self): cache.delete("cred-%s" % self.object.id) def iter_changed_datastream(self): yield "managedobject", self.object.id @classmethod def get_capabilities(cls, object): """ Resolve object capabilities :param object: ManagedObject instance or id :return: dict of capability name -> current value """ if hasattr(object, "id"): object = object.id caps = {} oc = ObjectCapabilities._get_collection().find_one({"_id": object}) if oc: for c in oc["caps"]: cc = Capability.get_by_id(c["capability"]) if cc: caps[cc.name] = c.get("value") return caps @classmethod def update_capabilities(cls, object, caps, source): """ Update stored capabilities :param object: :param caps: :param source: :return: """ o_label = object if hasattr(object, "id"): o_label = object.name object = object.id o_label += "|%s" % source oc = ObjectCapabilities._get_collection().find_one({"_id": object }) or {} # Update existing capabilities new_caps = [] seen = set() changed = False for ci in oc.get("caps", []): c = Capability.get_by_id(ci["capability"]) cs = ci.get("source") cv = ci.get("value") if not c: logger.info("[%s] Removing unknown capability id %s", o_label, ci["capability"]) continue cn = c.name seen.add(cn) if cs == source: if cn in caps: if caps[cn] != cv: logger.info("[%s] Changing capability %s: %s -> %s", o_label, cn, cv, caps[cn]) ci["value"] = caps[cn] changed = True else: logger.info("[%s] Removing capability %s", o_label, cn) changed = True continue elif cn in caps: logger.info( "[%s] Not changing capability %s: " "Already set with source '%s'", o_label, cn, cs) new_caps += [ci] # Add new capabilities for cn in set(caps) - seen: c = Capability.get_by_name(cn) if not c: logger.info("[%s] Unknown capability %s, ignoring", o_label, cn) continue logger.info("[%s] Adding capability %s = %s", o_label, cn, caps[cn]) new_caps += [{ "capability": c.id, "value": caps[cn], "source": source }] changed = True if changed: logger.info("[%s] Saving changes", o_label) ObjectCapabilities._get_collection().update( {"_id": object}, {"$set": { "caps": new_caps }}, upsert=True) cache.delete("cred-%s" % object) caps = {} for ci in new_caps: cn = Capability.get_by_id(ci["capability"]) if cn: caps[cn.name] = ci.get("value") return caps
class Session(Document): """ This class represents a running instance of the agent on a target system. It is responsible for running actions created by the user. This class also has an associated class SessionHistory, which stores less frequently accessed data, that tends to grow rapidly over time. """ meta = { "collection": COLLECTION_SESSIONS, "indexes": [ { "fields": ["session_id"], "unique": True }, { "fields": ["target_name", "archived"] }, ], } session_id = StringField(required=True, null=False, max_length=MAX_STR_LEN) target_name = StringField(required=True, null=False, max_length=MAX_STR_LEN) timestamp = FloatField(required=True, null=False) servers = ListField(StringField(required=True, null=False, max_length=MAX_STR_LEN), required=True, null=False) interval = FloatField(required=True, null=False) interval_delta = FloatField(required=True, null=False) config_dict = DictField(null=False) agent_version = StringField(null=True, max_length=MAX_STR_LEN) archived = BooleanField(required=False, null=False, default=False) @staticmethod def get_by_id(session_id): """ This method queries for the session object matching the name provided. """ return Session.objects.get(session_id=session_id) # pylint: disable=no-member @staticmethod def list_sessions(): """ This method queries for all session objects. """ return Session.objects(archived=False) # pylint: disable=no-member @property def config(self): """ This property returns the session configuration, overriding all reserved keys with their proper values. config_dict should never be used directly. """ self.config_dict["interval"] = self.interval # pylint: disable=unsupported-assignment-operation self.config_dict["interval_delta"] = self.interval_delta # pylint: disable=unsupported-assignment-operation self.config_dict["servers"] = self.servers # pylint: disable=unsupported-assignment-operation return self.config_dict @property def status(self): """ This property returns the session status, which is based on the current interval setting and the last seen timestamp. """ max_time = self.interval + self.interval_delta + SESSION_CHECK_THRESHOLD if time.time() > self.timestamp + (max_time * SESSION_CHECK_MODIFIER): return SESSION_STATUSES.get("inactive", "inactive") elif time.time() > self.timestamp + max_time: return SESSION_STATUSES.get("missing", "missing") return SESSION_STATUSES.get("active", "active") @property def history(self): """ Performs a query to retrieve history information about this session. """ return SessionHistory.objects.get(session_id=self.session_id) # pylint: disable=no-member @property def document(self): """ This property returns a filtered JSON document representation of the session. """ return { "session_id": self.session_id, "target_name": self.target_name, "status": self.status, "timestamp": self.timestamp, "config": self.config, "agent_version": self.agent_version, } def update_config(self, interval=None, interval_delta=None, servers=None, config=None): """ This function will update a sessions config according to the provided dict. It will also validate types of reserved keywords. """ if interval is not None: self.interval = interval if interval_delta is not None: self.interval_delta = interval_delta if servers is not None: self.servers = servers if config is not None and isinstance(config, dict): for key, value in config.items(): self.config_dict[key] = value # pylint: disable=unsupported-assignment-operation self.save() def update_timestamp(self, new_timestamp): """ This function will update a session's timestamp, and it's history document. It will also ensure that the session is unarchived. """ self.timestamp = new_timestamp self.history.add_checkin(new_timestamp) self.archived = False self.save() def archive(self): """ Archive a target. This will prevent it from being discovered by lists and from being in target documents. """ self.archived = True self.save()
class AnalyticConfigurationList(Document): analytics = ListField(EmbeddedDocumentField(AnalyticConfiguration)) name = StringField(unique=True, required=True)
class Candidate(TimestampMixin, DynamicDocument): """ State is included because in nearly all cases, a candidate is unique to a state (presidential races involve state-level candidacies). This helps with lookups and prevents duplicates. """ ### Meta fields ### source = StringField( required=True, help_text= "Name of data source (preferably from datasource.py). NOTE: this could be a single file among many for a given state, if results are split into different files by reporting level" ) election_id = StringField( required=True, help_text="election id, e.g. md-2012-11-06-general") state = StringField(required=True, choices=STATE_POSTALS) person = ReferenceField( Person, help_text= "Reference to unique Person record to link candidacies over time and/or across states for presidential cands." ) ### Contest fields #### contest = ReferenceField(Contest, reverse_delete_rule=CASCADE, required=True) contest_slug = StringField( required=True, help_text="Denormalized contest slug for easier querying and obj repr") ### Candidate fields ### #TODO: Add validation to require full_name or famly_name, assuming we allow full_name (see question above) full_name = StringField(max_length=200) family_name = StringField(max_length=75) given_name = StringField(max_length=50) suffix = StringField(max_length=20) additional_name = StringField( max_length=75, help_text="Middle name, nickname, etc., if provided in raw results.") #TODO: Add validation to require full_name or family_name #TODO: Add example to help_text for slugified name slug = StringField( max_length=300, required=True, help_text="Slugified name for easier querying and obj repr") identifiers = DictField( help_text= "Unique identifiers for candidate in other data sets, such as FEC Cand number. " "This should store IDs relevant to just this candidacy, such as FEC Cand number(s) for a particular election " "cycle. The Person model will store the full history of all FEC Cand Numbers" ) flags = ListField( StringField(choices=CANDIDATE_FLAG_CHOICES, help_text="Flags to unambiguously identify candidate " "records that represent special non-person candidates.")) meta = { 'indexes': [ 'election_id', ], } def __unicode__(self): name = u'%s - %s' % (self.contest_slug, self.name) return name @property def name(self): if self.full_name: name = self.full_name else: name = self.family_name if self.given_name: name += " %s" % self.given_name if self.additional_name: name += " %s" % self.additional_name if self.suffix: name += " %s" % self.suffix name = "%s" % self.family_name return name @property def key(self): return (self.election_id, self.contest_slug, self.slug) @classmethod def post_init(cls, sender, document, **kwargs): if not document.contest_slug: document.contest_slug = document.contest.slug if not document.slug: document.slug = cls.make_slug(full_name=document.full_name) @classmethod def make_slug(cls, **kwargs): return slugify(kwargs.get('full_name'), '-')
class Contributions(DynamicDocument): project = LazyReferenceField("Projects", required=True, passthrough=True, reverse_delete_rule=CASCADE) identifier = StringField(required=True, help_text="material/composition identifier") formula = StringField( help_text="formula (set dynamically if not provided)") is_public = BooleanField(required=True, default=False, help_text="public/private contribution") last_modified = DateTimeField(required=True, default=datetime.utcnow, help_text="time of last modification") data = DictField(default={}, validation=valid_dict, help_text="simple free-form data") structures = ListField(ReferenceField("Structures"), default=list, max_length=10) tables = ListField(ReferenceField("Tables"), default=list, max_length=10) notebook = ReferenceField("Notebooks") meta = { "collection": "contributions", "indexes": [ "project", "identifier", "formula", "is_public", "last_modified", { "fields": [(r"data.$**", 1)] }, ], } @classmethod def post_init(cls, sender, document, **kwargs): # replace existing structures/tables with according ObjectIds for component in ["structures", "tables"]: lst = getattr(document, component) if lst and lst[0].id is None: # id is None for incoming POST dmodule = import_module(f"mpcontribs.api.{component}.document") klass = component.capitalize() Docs = getattr(dmodule, klass) vmodule = import_module(f"mpcontribs.api.{component}.views") Resource = getattr(vmodule, f"{klass}Resource") resource = Resource() for i, o in enumerate(lst): d = resource.serialize( o, fields=["lattice", "sites", "charge"]) s = json.dumps(d, sort_keys=True).encode("utf-8") digest = md5(s).hexdigest() obj = Docs.objects(md5=digest).only("id").first() if obj: obj.reload() lst[i] = obj @classmethod def pre_save_post_validation(cls, sender, document, **kwargs): if kwargs.get("skip"): return # set formula field if hasattr(document, "formula") and not document.formula: formulae = current_app.config["FORMULAE"] document.formula = formulae.get(document.identifier, document.identifier) # project is LazyReferenceField project = document.project.fetch() # run data through Pint Quantities and save as dicts def make_quantities(path, key, value): if key in quantity_keys or not isinstance(value, (str, int, float)): return key, value str_value = str(value) if str_value.count(" ") > 1: return key, value q = get_quantity(str_value) if not q: return key, value # silently ignore "nan" if isnan(q.nominal_value): return False # try compact representation qq = q.value.to_compact() q = new_error_units(q, qq) # reduce dimensionality if possible if not q.check(0): qq = q.value.to_reduced_units() q = new_error_units(q, qq) # ensure that the same units are used across contributions field = delimiter.join(["data"] + list(path) + [key]) try: column = project.columns.get(path=field) if column.unit != str(q.value.units): qq = q.value.to(column.unit) q = new_error_units(q, qq) except DoesNotExist: pass # column doesn't exist yet (generated in post_save) except DimensionalityError: raise ValueError( f"Can't convert [{q.units}] to [{column.unit}]!") v = Decimal(str(q.nominal_value)) vt = v.as_tuple() if vt.exponent < 0: dgts = len(vt.digits) dgts = max_dgts if dgts > max_dgts else dgts s = f"{v:.{dgts}g}" if not isnan(q.std_dev): s += f"+/-{q.std_dev:.{dgts}g}" s += f" {q.units}" q = get_quantity(s) # return new value dict display = str(q.value) if isnan(q.std_dev) else str(q) value = { "display": display, "value": q.nominal_value, "error": q.std_dev, "unit": str(q.units), } return key, value document.data = remap(document.data, visit=make_quantities, enter=enter) @classmethod def post_save(cls, sender, document, **kwargs): if kwargs.get("skip"): return # project is LazyReferenceField project = document.project.fetch() # set columns field for project def update_columns(path, key, value): path = delimiter.join(["data"] + list(path) + [key]) is_quantity = isinstance(value, dict) and quantity_keys.issubset( value.keys()) is_text = bool(not is_quantity and isinstance(value, str) and key not in quantity_keys) if is_quantity or is_text: project.reload("columns") try: column = project.columns.get(path=path) if is_quantity: v = value["value"] if isnan(column.max) or v > column.max: column.max = v if isnan(column.min) or v < column.min: column.min = v except DoesNotExist: column = {"path": path} if is_quantity: column["unit"] = value["unit"] column["min"] = column["max"] = value["value"] project.columns.create(**column) project.save().reload("columns") ncolumns = len(project.columns) if ncolumns > 50: raise ValueError("Reached maximum number of columns (50)!") return True # run update_columns over document data remap(document.data, visit=update_columns, enter=enter) # add/remove columns for other components for path in ["structures", "tables"]: try: project.columns.get(path=path) except DoesNotExist: if getattr(document, path): project.columns.create(path=path) project.save().reload("columns") # generate notebook for this contribution if document.notebook is not None: document.notebook.delete() cells = [ nbf.new_code_cell("client = Client(\n" '\theaders={"X-Consumer-Groups": "admin"},\n' f'\thost="{MPCONTRIBS_API_HOST}"\n' ")"), nbf.new_code_cell( f'client.get_contribution("{document.id}").pretty()'), ] if document.tables: cells.append(nbf.new_markdown_cell("## Tables")) for table in document.tables: cells.append( nbf.new_code_cell( f'client.get_table("{table.id}").plot()')) if document.structures: cells.append(nbf.new_markdown_cell("## Structures")) for structure in document.structures: cells.append( nbf.new_code_cell( f'client.get_structure("{structure.id}")')) loop = asyncio.new_event_loop() task = loop.create_task( execute_cells(str(document.id), cells, loop=loop)) outputs = loop.run_until_complete(task) for task in asyncio.all_tasks(loop=loop): print(f"Cancelling {task}") task.cancel() outputs = loop.run_until_complete(task) loop.close() for idx, output in outputs.items(): cells[idx]["outputs"] = output cells[0] = nbf.new_code_cell("client = Client()") doc = deepcopy(seed_nb) doc["cells"] += cells # avoid circular imports from mpcontribs.api.notebooks.document import Notebooks document.notebook = Notebooks(**doc).save() document.last_modified = datetime.utcnow() document.save(signal_kwargs={"skip": True}) @classmethod def pre_delete(cls, sender, document, **kwargs): document.reload("notebook", "structures", "tables") # remove reference documents if document.notebook is not None: document.notebook.delete() for component in ["structures", "tables"]: # check if other contributions exist before deletion! for obj in getattr(document, component): q = {component: obj.id} if sender.objects(**q).count() < 2: obj.delete() @classmethod def post_delete(cls, sender, document, **kwargs): if kwargs.get("skip"): return # reset columns field for project project = document.project.fetch() for column in list(project.columns): if not isnan(column.min) and not isnan(column.max): column.min, column.max = get_min_max(sender, column.path) if isnan(column.min) and isnan(column.max): # just deleted last contribution with this column project.update(pull__columns__path=column.path) else: # use wildcard index if available -> single field query field = column.path.replace(delimiter, "__") + "__type" qs = sender.objects(**{field: "string"}).only(column.path) if qs.count() < 1 or qs.filter( project__name=project.name).count() < 1: project.update(pull__columns__path=column.path)
class LabeledSentences(EmbeddedDocument): id = ObjectIdField(required=True, default=lambda: ObjectId()) data = ListField(required=True)
class PhoneNumber(Document): meta = { "collection": "noc.phonenumbers", "strict": False, "auto_create_index": False, "indexes": [ "static_service_groups", "effective_service_groups", "static_client_groups", "effective_client_groups" ] } number = StringField() profile = PlainReferenceField(PhoneNumberProfile) state = PlainReferenceField(State) dialplan = PlainReferenceField(DialPlan) phone_range = PlainReferenceField(PhoneRange) category = PlainReferenceField(NumberCategory) description = StringField() service = PlainReferenceField(Service) project = ForeignKeyField(Project) protocol = StringField(default="SIP", choices=[("SIP", "SIP"), ("H323", "H.323"), ("SS7", "SS7"), ("MGCP", "MGCP"), ("H247", "H.247"), ("ISDN", "ISDN"), ("Skinny", "Skinny")]) # Auto-change status to F after *allocated_till* allocated_till = DateTimeField() # Last state change changed = DateTimeField() # administrative_domain = ForeignKeyField(AdministrativeDomain) # Resource groups static_service_groups = ListField(ObjectIdField()) effective_service_groups = ListField(ObjectIdField()) static_client_groups = ListField(ObjectIdField()) effective_client_groups = ListField(ObjectIdField()) _id_cache = cachetools.TTLCache(100, ttl=60) def __unicode__(self): return self.number @classmethod @cachetools.cachedmethod(operator.attrgetter("_id_cache"), lock=lambda _: id_lock) def get_by_id(cls, id): return PhoneNumber.objects.filter(id=id).first() def clean(self): super(PhoneNumber, self).clean() # Check number is valid integer self.number = clean_number(self.number or "") if not self.number: raise ValidationError("Empty phone number") # Change parent self.phone_range = PhoneRange.get_closest_range( dialplan=self.dialplan, from_number=self.number) # Set profile when necessary if not self.profile: if not self.phone_range: raise ValidationError("Either range or profile must be set") self.profile = self.phone_range.profile.default_number_profile @property def enum(self): return ".".join(reversed(self.number)) + ".e164.arpa"
class ModelInterface(Document): """ Equipment vendor """ meta = { "collection": "noc.modelinterfaces", "strict": False, "auto_create_index": False, "json_collection": "inv.modelinterfaces", "json_unique_fields": ["uuid", "name"], } name = StringField(unique=True) description = StringField() attrs = ListField(EmbeddedDocumentField(ModelInterfaceAttr)) uuid = UUIDField(binary=True) _id_cache = cachetools.TTLCache(1000, 10) _name_cache = cachetools.TTLCache(1000, 10) def __str__(self): return self.name @classmethod @cachetools.cachedmethod(operator.attrgetter("_id_cache"), lock=lambda _: id_lock) def get_by_id(cls, id) -> Optional["ModelInterface"]: return ModelInterface.objects.filter(id=id).first() @classmethod @cachetools.cachedmethod(operator.attrgetter("_name_cache"), lock=lambda _: id_lock) def get_by_name(cls, name: str) -> Optional["ModelInterface"]: return ModelInterface.objects.filter(name=name).first() def get_attr(self, name): for a in self.attrs: if a.name == name: return a return None def to_json(self): ar = [] for a in self.attrs: r = [" {"] r += [' "name": "%s",' % q(a.name)] r += [' "type": "%s",' % q(a.type)] r += [' "description": "%s",' % q(a.description)] r += [' "required": %s,' % q(a.required)] r += [' "is_const": %s' % q(a.is_const)] r += [" }"] ar += ["\n".join(r)] r = [ "{", ' "name": "%s",' % q(self.name), ' "$collection": "%s",' % self._meta["json_collection"], ' "uuid": "%s",' % str(self.uuid), ' "description": "%s",' % q(self.description), ' "attrs": [', ",\n".join(ar), " ]", "}", ] return "\n".join(r) + "\n" def get_json_path(self): p = [n.strip() for n in self.name.split("|")] return os.path.join(*p) + ".json" @classmethod def clean_data(cls, data): """ Convert types accoding to interface """ d = deep_copy(data) for i_name in d: mi = ModelInterface.objects.filter(name=i_name).first() if not mi: raise ModelDataError("Unknown interface '%s'" % i_name) v = d[i_name] for a in mi.attrs: if a.name in v: vv = v[a.name] if a.type == "strlist": if isinstance(vv, str): vv = [vv] r = set() for x in vv: r.update(x.split(",")) vv = [x.strip() for x in sorted(r) if x.strip()] v[a.name] = T_MAP[a.type].clean(vv) return d @classmethod @cachetools.cachedmethod(operator.attrgetter("_id_cache"), lock=lambda _: id_lock) def get_interface_attr(cls, interface, key): mi = ModelInterface.objects.filter(name=interface).first() if not mi: raise ModelDataError("Invalid interface '%s'" % interface) attr = mi.get_attr(key) if not attr: raise ModelDataError("Invalid attribute '%s.%s'" % (interface, key)) return attr
class Subscription(Document, DocumentHelperMixin): # BIG DB migration 20141028 bigmig_migrated = BooleanField(default=False) bigmig_reassigned = BooleanField(default=False) # END BIG DB migration feed = ReferenceField('Feed', reverse_delete_rule=CASCADE) user = ReferenceField('User', unique_with='feed', reverse_delete_rule=CASCADE) # allow the user to rename the field in its own subscription name = StringField(verbose_name=_(u'Name')) # TODO: convert to UserTag to use ReferenceField and reverse_delete_rule. tags = ListField(GenericReferenceField(), default=list, verbose_name=_(u'Tags'), help_text=_(u'Tags that will be applied to new reads in ' u'this subscription.')) folders = ListField(ReferenceField(Folder, reverse_delete_rule=PULL), verbose_name=_(u'Folders'), default=list, help_text=_(u'Folder(s) in which this subscription ' u'appears.')) all_articles_count = IntRedisDescriptor( attr_name='s.aa_c', default=subscription_all_articles_count_default, set_default=True, min_value=0) unread_articles_count = IntRedisDescriptor( attr_name='s.ua_c', default=subscription_unread_articles_count_default, set_default=True, min_value=0) starred_articles_count = IntRedisDescriptor( attr_name='s.sa_c', default=subscription_starred_articles_count_default, set_default=True, min_value=0) archived_articles_count = IntRedisDescriptor( attr_name='s.ra_c', default=subscription_archived_articles_count_default, set_default=True, min_value=0) bookmarked_articles_count = IntRedisDescriptor( attr_name='s.ba_c', default=subscription_bookmarked_articles_count_default, set_default=True, min_value=0) meta = { 'indexes': [ 'user', 'feed', 'folders', ] } def __unicode__(self): return _(u'{0}+{1} (#{2})').format( self.user.username, self.feed.name, self.id) @classmethod def signal_post_save_handler(cls, sender, document, created=False, **kwargs): subscription = document if created: if subscription._db_name != settings.MONGODB_NAME_ARCHIVE: # HEADS UP: this task name will be registered later # by the register_task_method() call. subscription_post_create_task.delay(subscription.id) # NOQA # else: # if subscription.feed.is_mailfeed: # mailfeed = MailFeed.get_from_stream_url(subscription.feed.url) # # if subscription.user.django == mailfeed.user: # # HEADS UP: we use save() to forward the # # name change to the Feed instance without # # duplicating the code here. # mailfeed.name = subscription.name # mailfeed.save() def post_create_task(self): """ Method meant to be run from a celery task. """ # The content of this method is done in subscribe_user_to_feed() # to avoid more-than-needed write operations on the database. pass @classmethod def signal_post_delete_handler(cls, sender, document, **kwargs): subscription = document if subscription._db_name != settings.MONGODB_NAME_ARCHIVE: # HEADS UP: we don't pass an ID, else the .get() fails # in the task for a good reason: the subscription doesn't # exist anymore. subscription_post_delete_task.delay(subscription) @classmethod def subscribe_user_to_feed(cls, user, feed, name=None, force=False, background=False): try: subscription = cls(user=user, feed=feed).save() except (NotUniqueError, DuplicateKeyError): if not force: LOGGER.info(u'User %s already subscribed to feed %s.', user, feed) return cls.objects.get(user=user, feed=feed) else: subscription.name = name or feed.name subscription.tags = feed.tags[:] subscription.save() if background: # HEADS UP: this task name will be registered later # by the register_task_method() call. # 'True' is for the 'force' argument. subscription_check_reads_task.delay(subscription.id, True) else: subscription.check_reads(force=True) LOGGER.info(u'Subscribed %s to %s via %s.', user, feed, subscription) return subscription @property def has_unread(self): # We need a boolean value for accurate template caching. return self.unread_articles_count != 0 @property def is_closed(self): return self.feed.closed def mark_all_read(self, latest_displayed_read=None): if self.unread_articles_count == 0: return # count = self.unread_articles_count # self.unread_articles_count = 0 # for folder in self.folders: # folder.unread_articles_count -= count # self.user.unread_articles_count -= count # Marking all read is not a database-friendly operation, # thus it's run via a task to be able to return now immediately, # with cache numbers updated. # # HEADS UP: this task name will be registered later # by the register_task_method() call. subscription_mark_all_read_in_database_task.delay( self.id, now() if latest_displayed_read is None # # TRICK: we use self.user.reads for 2 reasons: # - avoid importing `Read`, which would create a loop. # - in case of a folder/global initiated mark_all_read(), # the ID can be one of a read in another subscription # and in this case, self.reads.get() will fail. # else latest_displayed_read.date_created) def mark_all_read_in_database(self, prior_datetime): """ To avoid marking read the reads that could have been created between the task call and the moment it is effectively run, we define what to exactly mark as read with the datetime when the operation was done by the user. Also available as a task for background execution. .. note:: the archived reads stay archived, whatever their read status is. No need to test this attribute. """ # We touch only unread. This avoid altering the auto_read attribute # on reads that have been manually marked read by the user. params = {'is_read__ne': True, 'date_created__lte': prior_datetime} if self.user.preferences.read.bookmarked_marks_unread: # Let bookmarked reads stay unread. params['is_bookmarked__ne'] = True impacted_unread = self.reads.filter(**params) impacted_count = impacted_unread.count() impacted_unread.update(set__is_read=True, set__is_auto_read=True, set__date_read=prior_datetime, set__date_auto_read=prior_datetime) # If our caches are correctly computed, doing # one more full query just for this is too much. # # self.compute_cached_descriptors(unread=True) self.unread_articles_count -= impacted_count for folder in self.folders: folder.unread_articles_count -= impacted_count self.user.unread_articles_count -= impacted_count def check_reads(self, articles=None, force=False, extended_check=False): """ Also available as a task for background execution. """ if not force: LOGGER.info(u'Subscription.check_reads() is very costy and should ' u'not be needed in normal conditions. Call it with ' u'`force=True` if you are sure you want to run it.') return yesterday = combine(today() - timedelta(days=1), time(0, 0, 0)) is_older = False my_now = now() reads = 0 unreads = 0 failed = 0 missing = 0 rechecked = 0 # See generic_check_subscriptions_method() for comment about this. if articles is None: articles = self.feed.good_articles.order_by('-id') for article in articles: # # NOTE: Checking `article.is_good()` is done at a lower # level in the individual `self.create_read()`. # It has nothing to do with the dates-only checks # that we do here. # params = {} if is_older or article.date_published is None: params = { 'is_read': True, 'is_auto_read': True, 'date_read': my_now, 'date_auto_read': my_now, } else: # As they are ordered by date, switching is_older to True will # avoid more date comparisons. MongoDB already did the job. if article.date_published < yesterday: is_older = True params = { 'is_read': True, 'is_auto_read': True, 'date_read': my_now, 'date_auto_read': my_now, } # implicit: else: pass # No params == all by default == is_read is False # The `create_read()` methods is defined # in `nonrel/read.py` to avoid an import loop. _, created = self.create_read(article, False, **params) if created: missing += 1 if params.get('is_read', False): reads += 1 else: unreads += 1 elif created is False: rechecked += 1 if extended_check: try: article.activate_reads() except: LOGGER.exception(u'Problem while activating reads ' u'of Article #%s in Subscription ' u'#%s.check_reads(), continuing ' u'check.', article.id, self.id) else: failed += 1 if missing or rechecked: # # TODO: don't recompute everything, just # add or subscribe the changed counts. # self.compute_cached_descriptors(all=True, unread=True) for folder in self.folders: folder.compute_cached_descriptors(all=True, unread=True) LOGGER.info(u'Checked subscription #%s. ' u'%s/%s non-existing/re-checked, ' u'%s/%s read/unread and %s not created.', self.id, missing, rechecked, reads, unreads, failed) return missing, rechecked, reads, unreads, failed
class Operation(EmbeddedQueryTerm): """ An Operation performs some type of operation (so far only boolean: AND, OR, NOT) on multiple query terms to build a compound query. There is no implementation yet for more sophisticated mappings, such as SQL join, Splunk transaction, map, etc. :param QueryComparators operator: The abstract operation to be performed on the input expression(s) :param List[QueryTerm] terms: A list of all of the terms joined by the operator """ string_operator = StringField(db_field='operator') terms = ListField(EmbeddedDocumentField(EmbeddedQueryTerm)) def __init__(self, terms, operator=None, **kwargs): """ :type terms: List[QueryTerm] :type operator: QueryComparators """ kwargs["terms"] = list(term.prep_value() for term in terms) if operator is not None: kwargs['string_operator'] = operator.name self.operator = operator super(Operation, self).__init__(**kwargs) if operator is None: self.operator = QueryComparators[self.string_operator] def compare(self, event): if self.operator == QueryComparators.And: return all(term.compare(event) for term in self.terms) elif self.operator == QueryComparators.Or: return any(term.compare(event) for term in self.terms) elif self.operator == QueryComparators.Not: return not (self.terms[0].compare(event)) def get_fields(self): return {f for term in self.terms for f in term.get_fields()} def operation(self, other=None, operator=None): # if the operation is with something null ignore it if other is None or other is EmptyQuery: if operator is self.operator: return self # but not if the operation is inversion else: return Operation(terms=[self], operator=operator) # if the operators match, then create a new operation and merge them together if possible elif operator is self.operator: terms = list(self.terms) if isinstance(other, Operation) and other.operator is operator: terms.extend(other.terms) else: terms.append(other) return Operation(terms=terms, operator=operator) else: return Operation(terms=[self, other], operator=operator) def __repr__(self): return '{}(operator={}, terms={})'.format( type(self).__name__, self.string_operator, self.terms) __str__ = __repr__
class Stem(Document): meta = {'collection': 'stems'} stem = StringField() srclangs = ListField(StringField()) targetlangs = ListField(StringField()) dicts = ListField(StringField())
class MetricConfig(Document): meta = {"collection": "noc.pm.metricconfigs"} name = StringField(unique=True) is_active = BooleanField(default=True) handler = StringField() interval = IntField(default=60) description = StringField(required=False) probe = ReferenceField(Probe, reverse_delete_rule=DENY, required=False) metrics = ListField(EmbeddedDocumentField(MetricItem)) config = DictField(default={}) def __unicode__(self): return self.name def get_effective_settings(self, trace=False): """ Returns a list of MetricItems, containing all effective metrics and thresholds for group """ def q(s): return s.replace(" | ", ".").replace(" ", "_").replace("/", "-").lower() def apply_settings(name, mi): """ Apply settings to node and all children """ dst = mt_tree[name][0] dst.is_active = mi.is_active dst.low_warn = mi.low_warn dst.high_warn = mi.high_warn dst.low_error = mi.low_error dst.high_error = mi.high_error for c in mt_tree[name][1]: apply_settings(c, mi) # Build metric type tree mt_tree = {} # Metric type name -> (metric item, [children]) for mi in self.metrics: mt = mi.metric_type if mt.name in mt_tree: continue # Find all children nmt = [mt] + sorted( MetricType.objects.filter(name__startswith=mt.name + " | "), key=lambda x: len(x.name)) for m in nmt: if m.name in mt_tree: continue mt_tree[m.name] = [ MetricItem(metric_type=m, is_active=True), [] ] parent = " | ".join(p for p in m.name.split(" | ")[:-1]) if parent in mt_tree: mt_tree[parent][1] += [m.name] # Apply settings for mi in self.metrics: apply_settings(mi.metric_type.name, mi) # Fetch leaf nodes r = [] mt = [mi[0] for mi in mt_tree.itervalues() if not mi[1]] for mi in mt: if not mi.is_active: continue if mi.metric: metric = mi.metric else: # Auto-generate metric metric = "metric.%s.%s" % (q(self.name), q( mi.metric_type.name)) es = EffectiveSettings(object=self, model_id="pm.MetricConfig", metric=metric, metric_type=mi.metric_type, is_active=True, probe=self.probe, interval=self.interval, thresholds=[ mi.low_error, mi.low_warn, mi.high_warn, mi.high_error ]) for h in probe_registry.iter_class_handlers( self.handler, mi.metric_type.name): if trace: es.trace("Checking %s" % h.handler_name) config = {} failed = False if h.req: for name in h.req: if name in self.config and self.config[name] not in ( "", None): config[name] = self.config[name] else: failed = True if trace: es.trace("Cannot get required variable '%s'" % name) break if failed: if trace: es.trace("Giving up") continue # Get optional parameters for name in h.opt: if name in self.config and self.config[name] not in ("", None): config[name] = self.config[name] # Handler found if h.match(config): es.handler = h.handler_name es.config = config es.convert = h.convert es.scale = h.scale if trace: es.trace("Matched handler %s(%s)" % (h.handler_name, config)) break elif trace: es.trace("Handler mismatch") # es.is_active = bool(es.handler) if trace and not es.handler: es.error("No handler found") if es.is_active or trace: r += [es] # Collapse around handlers rr = {} for es in r: probe_id = es.probe.id if es.probe else None if es.handler: key = (probe_id, es.handler, es.interval) else: key = (probe_id, es.metric, es.metric_type, es.interval) if key in rr: e = rr[key] e.metrics += [ EffectiveSettingsMetric(metric=es.metric, metric_type=es.metric_type, thresholds=es.thresholds, convert=es.convert, scale=es.scale) ] else: es.metrics = [ EffectiveSettingsMetric(metric=es.metric, metric_type=es.metric_type, thresholds=es.thresholds, convert=es.convert, scale=es.scale) ] es.metric = None es.metric_type = None es.thresholds = None es.convert = None es.scale = None rr[key] = es return rr.values()
class Group(Document): """ This class represents a collection of target systems. Some are added to the group manually, while some are dynamically added based on group membership rules. """ meta = { 'collection': COLLECTION_GROUPS, 'indexes': [{ 'fields': ['name'], 'unique': True }] } name = StringField(required=True, null=False, unique=True) whitelist_members = ListField(StringField(required=True, null=False)) blacklist_members = ListField(StringField(required=True, null=False)) membership_rules = EmbeddedDocumentListField(GroupAutomemberRule, null=False) @staticmethod def get_target_groups(target_name): """ WARNING: Expensive Method This method returns a list of groups that a target is in. """ groups = [] for group in Group.objects(): #pylint: disable=no-member if target_name in group.member_names: groups.append(group) return list(set(groups)) @staticmethod def get_by_name(name): """ This method queries for the group with the given name. """ return Group.objects.get(name=name) #pylint: disable=no-member @staticmethod def list_groups(): """ This method queries for all group objects. """ return Group.objects() #pylint: disable=no-member @property def members(self): """ This property returns member objects of all group members. """ # TODO: Implement membership rules and blacklist return Target.objects(name__in=self.whitelist_members) #pylint: disable=no-member @property def member_names(self): """ This property returns member object names for all group members. """ # TODO: Implement membership rules and blacklist return self.whitelist_members @property def document(self): """ This property filters and returns the JSON information for a queried group. """ return { 'name': self.name, 'whitelist_members': self.whitelist_members, 'blacklist_members': self.blacklist_members } def whitelist_member(self, target_name): """ This function attempts to add a target to the member whitelist. The target will not be added if the target is in the blacklist. """ if target_name in self.blacklist_members: #pylint: disable=unsupported-membership-test raise MembershipError( 'Cannot whitelist a member that is on the blacklist.') self.whitelist_members.append(target_name) #pylint: disable=no-member self.save() def remove_member(self, target_name): """ This function removes a target from the member whitelist. """ if not target_name in self.whitelist_members: #pylint: disable=unsupported-membership-test raise MembershipError( 'Cannot remove member, member is not whitelisted.') self.whitelist_members.remove(target_name) #pylint: disable=no-member self.save() def blacklist_member(self, target_name): """ This function removes a target from the member whitelist (if they exist), and add them to the blacklist (if they are not yet on there). """ try: self.remove_member(target_name) except ValueError: pass if target_name in self.blacklist_members: #pylint: disable=unsupported-membership-test raise MembershipError('Member is already blacklisted.') self.blacklist_members.append(target_name) #pylint: disable=no-member self.save() def remove(self): """ Remove this document from the database, and perform any related cleanup. """ self.delete()
class MetricScope(Document): meta = { "collection": "noc.metricscopes", "strict": False, "auto_create_index": False, "json_collection": "pm.metricscopes", "json_unique_fields": ["name"], } name = StringField(unique=True) uuid = UUIDField(binary=True) # Database table name table_name = StringField() description = StringField(required=False) key_fields = ListField(EmbeddedDocumentField(KeyField)) path = ListField(EmbeddedDocumentField(PathItem)) _id_cache = cachetools.TTLCache(maxsize=100, ttl=60) def __str__(self): return self.name @classmethod @cachetools.cachedmethod(operator.attrgetter("_id_cache"), lock=lambda _: id_lock) def get_by_id(cls, id): return MetricScope.objects.filter(id=id).first() @property def json_data(self): r = { "name": self.name, "$collection": self._meta["json_collection"], "uuid": self.uuid, "table_name": self.table_name, "description": self.description, "key_fields": [kf.to_json() for kf in self.key_fields], "path": [p.to_json() for p in self.path], } return r def to_json(self): return to_json( self.json_data, order=[ "name", "$collection", "uuid", "table_name", "description", "key_fields", "path", ], ) def get_json_path(self): return "%s.json" % self.name def iter_fields(self): """ Yield (field_name, field_type) tuples :return: """ from .metrictype import MetricType yield ("date", "Date") yield ("ts", "DateTime") for f in self.key_fields: yield (f.field_name, f.field_type) if self.path: yield ("path", "Array(String)") for t in MetricType.objects.filter(scope=self.id).order_by("id"): yield (t.field_name, t.field_type) def get_create_sql(self): """ Get CREATE TABLE SQL statement :return: """ pk = [f.field_name for f in self.key_fields] if self.path: pk += ["path"] pk += ["ts"] r = [ "CREATE TABLE IF NOT EXISTS %s (" % self._get_raw_db_table(), ",\n".join(" %s %s" % (n, t) for n, t in self.iter_fields()), ") ENGINE = MergeTree(date, (%s), 8192)" % ", ".join(pk), ] return "\n".join(r) def get_create_distributed_sql(self): """ Get CREATE TABLE for Distributed engine :return: """ return ( "CREATE TABLE IF NOT EXISTS %s " "AS %s " "ENGINE = Distributed(%s, %s, %s)" % ( self.table_name, self._get_raw_db_table(), config.clickhouse.cluster, config.clickhouse.db, self._get_raw_db_table(), ) ) def _get_raw_db_table(self): if config.clickhouse.cluster: return "raw_%s" % self.table_name else: return self.table_name def ensure_table(self, connect=None): """ Ensure table is exists :return: True, if table has been changed """ from noc.core.clickhouse.connect import connection def ensure_columns(table_name): c = False # Alter when necessary existing = {} for name, type in ch.execute( """ SELECT name, type FROM system.columns WHERE database=%s AND table=%s """, [config.clickhouse.db, table_name], ): existing[name] = type after = None for f, t in self.iter_fields(): if f not in existing: ch.execute( post="ALTER TABLE %s ADD COLUMN %s %s AFTER %s" % (table_name, f, t, after) ) c = True after = f if f in existing and existing[f] != t: print("Warning! Type mismatch for column %s: %s <> %s" % (f, existing[f], t)) print( "Set command manually: ALTER TABLE %s MODIFY COLUMN %s %s" % (table_name, f, t) ) return c changed = False ch = connect or connection(read_only=False) if not ch.has_table(self._get_raw_db_table()): # Create new table ch.execute(post=self.get_create_sql()) changed = True else: changed |= ensure_columns(self._get_raw_db_table()) # Check for distributed table if config.clickhouse.cluster: if not ch.has_table(self.table_name): ch.execute(post=self.get_create_distributed_sql()) changed = True else: changed |= ensure_columns(self.table_name) return changed
class PostModel(DynamicDocument): url = StringField() abs_rshares = IntField() active = DateTimeField() allow_curation_rewards = BooleanField() allow_replies = BooleanField() allow_votes = BooleanField() active_votes = DictField() author = StringField() author_rewards = IntField() body = StringField() cashout_time = DateTimeField() category = StringField() children = IntField() # TODO сделать вывод количества всех комментариев children_abs_rshares = IntField() children_rshares2 = IntField() created = DateTimeField() curator_payout_symbol = StringField() curator_payout_value = DictField() depth = IntField() json_metadata = DictField() last_payout = DateTimeField() last_update = DateTimeField() max_accepted_payout_symbol = StringField() max_accepted_payout_value = FloatField() max_cashout_time = DateTimeField() mode = StringField() net_rshares = IntField() net_votes = IntField() parent_author = StringField() parent_permlink = StringField() percent_steem_dollars = IntField() permlink = StringField() removed = BooleanField() reward_weight = IntField() root_comment = ObjectId() title = StringField() total_payout_symbol = StringField() total_payout_value = FloatField() total_vote_weight = IntField() vote_rshares = IntField() author_reputation = IntField() beneficiaries = ListField(DictField()) body_length = IntField() community = StringField() identifier = StringField() max_accepted_payout = DictField() patched = BooleanField() pending_payout_value = DictField() promoted = DictField() reblogged_by = ListField(StringField()) replies = ListField(StringField()) # TODO StringField? root_author = StringField() root_identifier = StringField() root_permlink = StringField() root_title = StringField() tags = ListField(StringField()) total_payout_value = DictField() total_pending_payout_value = DictField() total_vote_weight = IntField() vote_rshares = IntField() meta = { 'ordering': ['-created'], 'indexes': [ 'author', 'permlink', 'created', 'category', 'json_metadata.location', 'depth', 'root_comment', 'parent_permlink', 'parent_author', 'mode', ], 'auto_create_index': True, 'index_background': True }
class ActiveAlarm(Document): meta = { "collection": "noc.alarms.active", "strict": False, "auto_create_index": False, "indexes": [ "timestamp", "root", "-severity", ("alarm_class", "managed_object"), ("discriminator", "managed_object"), ("timestamp", "managed_object"), "escalation_tt", "escalation_ts", "adm_path", "segment_path", "container_path", "uplinks", ("alarm_class", "rca_neighbors"), ], } status = "A" timestamp = DateTimeField(required=True) last_update = DateTimeField(required=True) managed_object = ForeignKeyField(ManagedObject) alarm_class = PlainReferenceField(AlarmClass) severity = IntField(required=True) vars = DictField() # Calculated alarm discriminator # Has meaning only for alarms with is_unique flag set # Calculated as sha1("value1\x00....\x00valueN").hexdigest() discriminator = StringField(required=False) log = ListField(EmbeddedDocumentField(AlarmLog)) # Manual acknowledgement timestamp ack_ts = DateTimeField(required=False) # Manual acknowledgement user name ack_user = StringField(required=False) # opening_event = ObjectIdField(required=False) closing_event = ObjectIdField(required=False) # List of subscribers subscribers = ListField(ForeignKeyField(User)) # custom_subject = StringField(required=False) custom_style = ForeignKeyField(Style, required=False) # reopens = IntField(required=False) # RCA # Reference to root cause (Active Alarm or Archived Alarm instance) root = ObjectIdField(required=False) # Escalated TT ID in form # <external system name>:<external tt id> escalation_ts = DateTimeField(required=False) escalation_tt = StringField(required=False) escalation_error = StringField(required=False) # span context escalation_ctx = LongField(required=False) # Close tt when alarm cleared close_tt = BooleanField(default=False) # Do not clear alarm until *wait_tt* is closed wait_tt = StringField() wait_ts = DateTimeField() # Directly affected services summary, grouped by profiles # (connected to the same managed object) direct_services = ListField(EmbeddedDocumentField(SummaryItem)) direct_subscribers = ListField(EmbeddedDocumentField(SummaryItem)) # Indirectly affected services summary, groupped by profiles # (covered by this and all inferred alarms) total_objects = ListField(EmbeddedDocumentField(ObjectSummaryItem)) total_services = ListField(EmbeddedDocumentField(SummaryItem)) total_subscribers = ListField(EmbeddedDocumentField(SummaryItem)) # Template and notification group to send close notification clear_template = ForeignKeyField(Template, required=False) clear_notification_group = ForeignKeyField(NotificationGroup, required=False) # Paths adm_path = ListField(IntField()) segment_path = ListField(ObjectIdField()) container_path = ListField(ObjectIdField()) # Uplinks, for topology_rca only uplinks = ListField(IntField()) # RCA neighbor cache, for topology_rca only rca_neighbors = ListField(IntField()) dlm_windows = ListField(IntField()) # RCA_* enums rca_type = IntField(default=RCA_NONE) # labels labels = ListField(StringField()) effective_labels = ListField(StringField()) def __str__(self): return "%s" % self.id def iter_changed_datastream(self, changed_fields=None): if config.datastream.enable_alarm: yield "alarm", self.id def clean(self): super().clean() if not self.last_update: self.last_update = self.timestamp data = self.managed_object.data self.adm_path = data.adm_path self.segment_path = data.segment_path self.container_path = data.container_path self.uplinks = data.uplinks self.rca_neighbors = data.rca_neighbors self.dlm_windows = data.dlm_windows if not self.id: self.effective_labels = [ label for label in self.iter_effective_labels() if self.can_set_label(label) ] def safe_save(self, **kwargs): """ Create new alarm or update existing if still exists :param kwargs: :return: """ if self.id: # Update existing only if exists if "save_condition" not in kwargs: kwargs["save_condition"] = {"id": self.id} try: self.save(**kwargs) except SaveConditionError: pass # Race condition, closed during update else: self.save() def change_severity(self, user="", delta=None, severity=None, to_save=True): """ Change alarm severity """ if isinstance(user, User): user = user.username if delta: self.severity = max(0, self.severity + delta) if delta > 0: self.log_message("%s has increased alarm severity by %s" % (user, delta)) else: self.log_message("%s has decreased alarm severity by %s" % (user, delta)) elif severity: if isinstance(severity, int) or isinstance(severity, float): self.severity = int(severity) self.log_message("%s has changed severity to %s" % (user, severity)) else: self.severity = severity.severity self.log_message("%s has changed severity to %s" % (user, severity.name)) if to_save: self.safe_save() def log_message(self, message, to_save=True, bulk=None, source=None): if bulk: bulk += [ UpdateOne( {"_id": self.id}, { "$push": { "log": { "timestamp": datetime.datetime.now(), "from_status": self.status, "to_status": self.status, "message": message, "source": source, } } }, ) ] self.log += [ AlarmLog( timestamp=datetime.datetime.now(), from_status=self.status, to_status=self.status, message=message, source=source, ) ] if to_save and not bulk: self.safe_save() def clear_alarm(self, message, ts=None, force=False, source=None): """ Clear alarm :param message: Log clearing message :param ts: Clearing timestamp :param force: Clear ever if wait_tt seg :param source: Source clear alarm """ ts = ts or datetime.datetime.now() if self.wait_tt and not force: # Wait for escalated tt to close if not self.wait_ts: self.wait_ts = ts self.log_message("Waiting for TT to close") call_later( "noc.services.escalator.wait_tt.wait_tt", scheduler="escalator", pool=self.managed_object.escalator_shard, alarm_id=self.id, ) return if self.alarm_class.clear_handlers: # Process clear handlers for h in self.alarm_class.get_clear_handlers(): try: h(self) except Exception: error_report() log = self.log + [ AlarmLog(timestamp=ts, from_status="A", to_status="C", message=message, source=source) ] a = ArchivedAlarm( id=self.id, timestamp=self.timestamp, clear_timestamp=ts, managed_object=self.managed_object, alarm_class=self.alarm_class, severity=self.severity, vars=self.vars, log=log, ack_ts=self.ack_ts, ack_user=self.ack_user, root=self.root, escalation_ts=self.escalation_ts, escalation_tt=self.escalation_tt, escalation_error=self.escalation_error, escalation_ctx=self.escalation_ctx, opening_event=self.opening_event, closing_event=self.closing_event, discriminator=self.discriminator, reopens=self.reopens, direct_services=self.direct_services, direct_subscribers=self.direct_subscribers, total_objects=self.total_objects, total_services=self.total_services, total_subscribers=self.total_subscribers, adm_path=self.adm_path, segment_path=self.segment_path, container_path=self.container_path, uplinks=self.uplinks, rca_neighbors=self.rca_neighbors, rca_type=self.rca_type, labels=self.labels, effective_labels=self.effective_labels, ) ct = self.alarm_class.get_control_time(self.reopens) if ct: a.control_time = datetime.datetime.now() + datetime.timedelta(seconds=ct) a.save() # Send notifications if not a.root and not self.reopens: a.managed_object.event( a.managed_object.EV_ALARM_CLEARED, { "alarm": a, "subject": a.subject, "body": a.body, "symptoms": a.alarm_class.symptoms, "recommended_actions": a.alarm_class.recommended_actions, "probable_causes": a.alarm_class.probable_causes, }, ) elif ct: pass # Set checks on all consequences for d in self._get_collection().find( {"root": self.id}, {"_id": 1, "alarm_class": 1, "managed_object": 1} ): ac = AlarmClass.get_by_id(d["alarm_class"]) if not ac: continue t = ac.recover_time if not t: continue call_later( "noc.services.correlator.check.check_close_consequence", scheduler="correlator", pool=self.managed_object.get_effective_fm_pool().name, delay=t, shard=d.get("managed_object"), alarm_id=d["_id"], ) # Clear alarm self.delete() # Close TT # MUST be after .delete() to prevent race conditions if a.escalation_tt or self.clear_template: if self.clear_template: ctx = {"alarm": a} subject = self.clear_template.render_subject(**ctx) body = self.clear_template.render_body(**ctx) else: subject = "Alarm cleared" body = "Alarm has been cleared" call_later( "noc.services.escalator.escalation.notify_close", scheduler="escalator", pool=self.managed_object.escalator_shard, max_runs=config.fm.alarm_close_retries, alarm_id=self.id, tt_id=self.escalation_tt, subject=subject, body=body, notification_group_id=self.clear_notification_group.id if self.clear_notification_group else None, close_tt=self.close_tt, login="******", queue=a.managed_object.tt_queue, ) # Gather diagnostics AlarmDiagnosticConfig.on_clear(a) # Return archived return a def get_template_vars(self): """ Prepare template variables """ vars = self.vars.copy() vars.update({"alarm": self}) return vars @property def subject(self): if self.custom_subject: s = self.custom_subject else: ctx = Context(self.get_template_vars()) s = DjangoTemplate(self.alarm_class.subject_template).render(ctx) if len(s) >= 255: s = s[:125] + " ... " + s[-125:] return s @property def body(self): ctx = Context(self.get_template_vars()) s = DjangoTemplate(self.alarm_class.body_template).render(ctx) return s def subscribe(self, user): """ Change alarm's subscribers """ if user.id not in self.subscribers: self.subscribers += [user.id] self.log_message( "%s(%s): has been subscribed" % ((" ".join([user.first_name, user.last_name]), user.username)), to_save=False, source=user.username, ) self.save() def unsubscribe(self, user): if self.is_subscribed(user): self.subscribers = [u.id for u in self.subscribers if u != user.id] self.log_message( "%s(%s) has been unsubscribed" % ((" ".join([user.first_name, user.last_name]), user.username)), to_save=False, source=user.username, ) self.save() def is_subscribed(self, user): return user.id in self.subscribers def acknowledge(self, user, msg=""): self.ack_ts = datetime.datetime.now() self.ack_user = user.username self.log = self.log + [ AlarmLog( timestamp=self.ack_ts, from_status="A", to_status="A", message="Acknowledged by %s(%s): %s" % (user.get_full_name(), user.username, msg), source=user.username, ) ] self.save() def unacknowledge(self, user, msg=""): self.ack_ts = None self.ack_user = None self.log = self.log + [ AlarmLog( timestamp=datetime.datetime.now(), from_status="A", to_status="A", message="Unacknowledged by %s(%s): %s" % (user.get_full_name(), user.username, msg), source=user.username, ) ] self.save() @property def duration(self): dt = datetime.datetime.now() - self.timestamp return dt.days * 86400 + dt.seconds @property def display_duration(self): duration = datetime.datetime.now() - self.timestamp secs = duration.seconds % 60 mins = (duration.seconds / 60) % 60 hours = (duration.seconds / 3600) % 24 days = duration.days r = "%02d:%02d:%02d" % (hours, mins, secs) if days: r = "%dd %s" % (days, r) return r @property def effective_style(self): if self.custom_style: return self.custom_style else: return AlarmSeverity.get_severity(self.severity).style def get_root(self): """ Get top-level root alarm """ root = self while root.root: root = get_alarm(root.root) return root def update_summary(self): """ Recalculate all summaries for given alarm. Performs recursive descent :return: """ def update_dict(d1, d2): for k in d2: if k in d1: d1[k] += d2[k] else: d1[k] = d2[k] services = SummaryItem.items_to_dict(self.direct_services) subscribers = SummaryItem.items_to_dict(self.direct_subscribers) objects = {self.managed_object.object_profile.id: 1} for a in ActiveAlarm.objects.filter(root=self.id): a.update_summary() update_dict(objects, SummaryItem.items_to_dict(a.total_objects)) update_dict(services, SummaryItem.items_to_dict(a.total_services)) update_dict(subscribers, SummaryItem.items_to_dict(a.total_subscribers)) obj_list = ObjectSummaryItem.dict_to_items(objects) svc_list = SummaryItem.dict_to_items(services) sub_list = SummaryItem.dict_to_items(subscribers) if ( svc_list != self.total_services or sub_list != self.total_subscribers or obj_list != self.total_objects ): ns = ServiceSummary.get_severity( {"service": services, "subscriber": subscribers, "objects": objects} ) self.total_objects = obj_list self.total_services = svc_list self.total_subscribers = sub_list if ns != self.severity: self.change_severity(severity=ns, to_save=False) self.safe_save() def _get_path_summary_bulk(self): def list_to_dict(summary): if not summary: return {} return {d["profile"]: d["summary"] for d in summary} def e_list_to_dict(summary): if not summary: return {} return {d.profile: d.summary for d in summary} def dict_to_list(d): return [{"profile": k, "summary": d[k]} for k in sorted(d)] def get_summary(docs, key, direct=None): r = direct.copy() if direct else {} for doc in docs: dv = doc.get(key) if not dv: continue for k in dv: nv = dv[k] if nv: r[k] = r.get(k, 0) + nv return r def get_root_path(alarm_id, path=None): path = path or [] if alarm_id in path: raise ValueError("Loop detected: %s" % (str(x) for x in path)) path = path + [alarm_id] root = alarms[alarm_id].get("root") if not root: return path return get_root_path(root, path) alarms = {} # id -> alarm doc children = defaultdict(list) # id -> [alarm doc, ..] # Inject current alarm alarms[self.id] = { "_id": self.id, "root": self.root, "severity": self.severity, "total_objects": e_list_to_dict(self.total_objects), "total_services": e_list_to_dict(self.total_services), "total_subscribers": e_list_to_dict(self.total_subscribers), } # Collect relevant neighbors for doc in ActiveAlarm._get_collection().aggregate( [ # Starting from given alarm {"$match": {"_id": self.root}}, # Add to 'path' field all alarm upwards { "$graphLookup": { "from": "noc.alarms.active", "connectFromField": "root", "connectToField": "_id", "startWith": "$root", "as": "path", "maxDepth": 50, } }, # Append the necessary fields of given alarm to 'path' field # and wipe out all other fields { "$project": { "_id": 0, "path": { "$concatArrays": [ "$path", [ { "_id": "$_id", "root": "$root", "severity": "$severity", "direct_services": "$direct_services", "direct_subscribers": "$direct_subscribers", "total_objects": "$total_objects", "total_services": "$total_services", "total_subscribers": "$total_subscribers", } ], ] }, } }, # Convert path field to the list of documents {"$unwind": "$path"}, # Normalize resulting documents { "$project": { "_id": "$path._id", "root": "$path.root", "severity": "$path.severity", "direct_services": "$path.direct_services", "direct_subscribers": "$path.direct_subscribers", "total_objects": "$path.total_objects", "total_services": "$path.total_services", "total_subscribers": "$path.total_subscribers", } }, # Add all children alarms to 'children' field { "$lookup": { "from": "noc.alarms.active", "localField": "_id", "foreignField": "root", "as": "children", } }, # Append the neccessary fields of path alarms to `children` field # and wipe out all other fields { "$project": { "_id": 0, "children": { "$concatArrays": [ "$children", [ { "_id": "$_id", "root": "$root", "severity": "$severity", "direct_services": "$direct_services", "direct_subscribers": "$direct_subscribers", "total_objects": "$total_objects", "total_services": "$total_services", "total_subscribers": "$total_subscribers", } ], ] }, } }, # Convert path field to the list of documents {"$unwind": "$children"}, # Normalize resulting documents { "$project": { "_id": "$children._id", "root": "$children.root", "severity": "$children.severity", "direct_services": "$children.direct_services", "direct_subscribers": "$children.direct_subscribers", "total_objects": "$children.total_objects", "total_services": "$children.total_services", "total_subscribers": "$children.total_subscribers", } }, ] ): # May contains duplicates, perform deduplication doc["direct_services"] = list_to_dict(doc.get("direct_services")) doc["direct_subscribers"] = list_to_dict(doc.get("direct_subscribers")) doc["total_objects"] = list_to_dict(doc.get("total_objects")) doc["total_services"] = list_to_dict(doc.get("total_services")) doc["total_subscribers"] = list_to_dict(doc.get("total_subscribers")) if doc["_id"] == self.id: doc["root"] = self.root alarms[doc["_id"]] = doc for doc in alarms.values(): children[doc.get("root")] += [doc] # Get path to from current root upwards to global root # Check for loops, raise Value error if loop detected root_path = get_root_path(self.root) bulk = [] now = datetime.datetime.now() for root in root_path: doc = alarms[root] consequences = children[root] total_objects = get_summary( consequences, "total_objects", {self.managed_object.object_profile.id: 1} ) total_services = get_summary(consequences, "total_services", doc.get("direct_services")) total_subscribers = get_summary( consequences, "total_subscribers", doc.get("direct_subscribers") ) if ( doc["total_objects"] != total_objects or doc["total_services"] != total_services or doc["total_subscribers"] != total_subscribers ): # Changed severity = ServiceSummary.get_severity( { "service": total_services, "subscriber": total_subscribers, "objects": total_objects, } ) op = { "$set": { "severity": severity, "total_objects": dict_to_list(total_objects), "total_services": dict_to_list(total_services), "total_subscribers": dict_to_list(total_subscribers), } } if severity != doc.get("severity"): op["$push"] = { "log": { "timestamp": now, "from_status": "A", "to_status": "A", "message": "Severity changed to %d" % severity, } } bulk += [UpdateOne({"_id": root}, op)] return bulk def set_root(self, root_alarm, rca_type=RCA_OTHER): """ Set root cause """ if self.root: return if self.id == root_alarm.id: raise Exception("Cannot set self as root cause") # Set root self.root = root_alarm.id self.rca_type = rca_type try: bulk = self._get_path_summary_bulk() except ValueError: return # Loop detected bulk += [ UpdateOne({"_id": self.id}, {"$set": {"root": root_alarm.id, "rca_type": rca_type}}) ] self.log_message("Alarm %s has been marked as root cause" % root_alarm.id, bulk=bulk) # self.save() Saved by log_message root_alarm.log_message("Alarm %s has been marked as child" % self.id, bulk=bulk) if self.id: ActiveAlarm._get_collection().bulk_write(bulk, ordered=True) def escalate(self, tt_id, close_tt=False, wait_tt=None): self.escalation_tt = tt_id self.escalation_ts = datetime.datetime.now() self.close_tt = close_tt self.wait_tt = wait_tt self.log_message("Escalated to %s" % tt_id) q = {"_id": self.id} op = { "$set": { "escalation_tt": self.escalation_tt, "escalation_ts": self.escalation_ts, "close_tt": self.close_tt, "wait_tt": self.wait_tt, "escalation_error": None, } } r = ActiveAlarm._get_collection().update_one(q, op) if r.acknowledged and not r.modified_count: # Already closed, update archive ArchivedAlarm._get_collection().update_one(q, op) def set_escalation_error(self, error): self.escalation_error = error self._get_collection().update_one({"_id": self.id}, {"$set": {"escalation_error": error}}) def set_escalation_context(self): current_context, current_span = get_current_span() if current_context or self.escalation_ctx: self.escalation_ctx = current_context self._get_collection().update_one( {"_id": self.id}, {"$set": {"escalation_ctx": current_context}} ) def set_clear_notification(self, notification_group, template): self.clear_notification_group = notification_group self.clear_template = template self.safe_save(save_condition={"managed_object": {"$exists": True}, "id": self.id}) def iter_consequences(self): """ Generator yielding all consequences alarm """ for a in ActiveAlarm.objects.filter(root=self.id): yield a yield from a.iter_consequences() def iter_affected(self): """ Generator yielding all affected managed objects """ seen = {self.managed_object} yield self.managed_object for a in self.iter_consequences(): if a.managed_object not in seen: seen.add(a.managed_object) yield a.managed_object def iter_escalated(self): """ Generator yielding all escalated consequences """ for a in self.iter_consequences(): if a.escalation_tt: yield a def iter_effective_labels(self): return set(self.managed_object.labels or []) | set( self.managed_object.object_profile.labels or [] ) @classmethod def can_set_label(cls, label): return Label.get_effective_setting(label, "enable_alarm")
class ArchivedAlarm(Document): meta = { "collection": "noc.alarms.archived", "strict": False, "auto_create_index": False, "indexes": [ "root", "timestamp", "managed_object", ("managed_object", "discriminator", "control_time"), "escalation_tt", "escalation_ts", ], } status = "C" timestamp = DateTimeField(required=True) clear_timestamp = DateTimeField(required=True) managed_object = ForeignKeyField(ManagedObject) alarm_class = PlainReferenceField(AlarmClass) severity = IntField(required=True) vars = DictField() log = ListField(EmbeddedDocumentField(AlarmLog)) # opening_event = ObjectIdField(required=False) closing_event = ObjectIdField(required=False) # Number of reopens reopens = IntField(required=False) # Copied discriminator discriminator = StringField(required=False) # Manual acknowledgement timestamp ack_ts = DateTimeField(required=False) # Manual acknowledgement user name ack_user = StringField(required=False) # Control time within alarm will be reopen instead # instead of creating the new alarm control_time = DateTimeField(required=False) # RCA # Reference to root cause (Active Alarm or Archived Alarm instance) root = ObjectIdField(required=False) # Escalated TT ID in form # <external system name>:<external tt id> escalation_ts = DateTimeField(required=False) escalation_tt = StringField(required=False) escalation_error = StringField(required=False) escalation_ctx = LongField(required=False) escalation_close_ts = DateTimeField(required=False) escalation_close_error = StringField(required=False) escalation_close_ctx = LongField(required=False) # Directly affected services summary, grouped by profiles # (connected to the same managed object) direct_services = ListField(EmbeddedDocumentField(SummaryItem)) direct_subscribers = ListField(EmbeddedDocumentField(SummaryItem)) # Indirectly affected services summary, groupped by profiles # (covered by this and all inferred alarms) total_objects = ListField(EmbeddedDocumentField(ObjectSummaryItem)) total_services = ListField(EmbeddedDocumentField(SummaryItem)) total_subscribers = ListField(EmbeddedDocumentField(SummaryItem)) # Paths adm_path = ListField(IntField()) segment_path = ListField(ObjectIdField()) container_path = ListField(ObjectIdField()) # Uplinks, for topology_rca only uplinks = ListField(IntField()) # RCA neighbor cache, for topology_rca only rca_neighbors = ListField(IntField()) def __str__(self): return "%s" % self.id def iter_changed_datastream(self, changed_fields=None): if config.datastream.enable_alarm: yield "alarm", self.id def log_message(self, message): self.log += [ AlarmLog( timestamp=datetime.datetime.now(), from_status=self.status, to_status=self.status, message=message, ) ] self.save() def get_template_vars(self): """ Prepare template variables """ vars = self.vars.copy() vars.update({"alarm": self}) return vars @property def subject(self): ctx = Context(self.get_template_vars()) s = Template(self.alarm_class.subject_template).render(ctx) if len(s) >= 255: s = s[:125] + " ... " + s[-125:] return s @property def body(self): ctx = Context(self.get_template_vars()) s = Template(self.alarm_class.body_template).render(ctx) return s @property def duration(self): dt = self.clear_timestamp - self.timestamp return dt.days * 86400 + dt.seconds @property def display_duration(self): duration = self.clear_timestamp - self.timestamp secs = duration.seconds % 60 mins = (duration.seconds / 60) % 60 hours = (duration.seconds / 3600) % 24 days = duration.days if days: return "%dd %02d:%02d:%02d" % (days, hours, mins, secs) else: return "%02d:%02d:%02d" % (hours, mins, secs) @property def effective_style(self): return AlarmSeverity.get_severity(self.severity).style def set_root(self, root_alarm): pass def reopen(self, message): """ Reopen alarm back """ reopens = self.reopens or 0 ts = datetime.datetime.now() log = self.log + [ AlarmLog( timestamp=ts, from_status="C", to_status="A", message=message) ] a = ActiveAlarm( id=self.id, timestamp=self.timestamp, last_update=ts, managed_object=self.managed_object, alarm_class=self.alarm_class, severity=self.severity, vars=self.vars, log=log, root=self.root, escalation_ts=self.escalation_ts, escalation_tt=self.escalation_tt, escalation_error=self.escalation_error, escalation_ctx=self.escalation_ctx, opening_event=self.opening_event, discriminator=self.discriminator, reopens=reopens + 1, direct_services=self.direct_services, direct_subscribers=self.direct_subscribers, total_objects=self.total_objects, total_services=self.total_services, total_subscribers=self.total_subscribers, adm_path=self.adm_path, segment_path=self.segment_path, container_path=self.container_path, uplinks=self.uplinks, ) a.save() # @todo: Clear related correlator jobs self.delete() # Send notifications # Do not set notifications for child and for previously reopened # alarms if not a.root and not reopens: a.managed_object.event( a.managed_object.EV_ALARM_REOPENED, { "alarm": a, "subject": a.subject, "body": a.body, "symptoms": a.alarm_class.symptoms, "recommended_actions": a.alarm_class.recommended_actions, "probable_causes": a.alarm_class.probable_causes, }, ) return a def iter_consequences(self): """ Generator yielding all consequences alarm """ for a in ArchivedAlarm.objects.filter(root=self.id): yield a for ca in a.iter_consequences(): yield ca def iter_affected(self): """ Generator yielding all affected managed objects """ seen = {self.managed_object} yield self.managed_object for a in self.iter_consequences(): if a.managed_object not in seen: seen.add(a.managed_object) yield a.managed_object def set_escalation_close_error(self, error): self.escalation_error = error self._get_collection().update( {"_id": self.id}, {"$set": { "escalation_close_error": error }}) def close_escalation(self): now = datetime.datetime.now() self.escalation_close_ts = now self._get_collection().update({"_id": self.id}, {"$set": { "escalation_close_ts": now }}) def set_escalation_close_ctx(self): current_context, current_span = get_current_span() if current_context or self.escalation_close_ctx: self.escalation_close_ctx = current_context self._get_collection().update( {"_id": self.id}, {"$set": { "escalation_close_ctx": current_context }})
class ActionCommands(Document): meta = { "collection": "noc.actioncommands", "strict": False, "auto_create_index": False, "json_collection": "sa.actioncommands", "json_depends_on": ["sa.actions", "sa.profile"], "json_unique_fields": ["name"], } name = StringField(unique=True) uuid = UUIDField(unique=True) action = ReferenceField(Action) description = StringField() profile = PlainReferenceField(Profile) config_mode = BooleanField(default=False) match = ListField(EmbeddedDocumentField(PlatformMatch)) commands = StringField() preference = IntField(default=1000) timeout = IntField(default=60) def __str__(self): return self.name def get_json_path(self): p = [quote_safe_path(n.strip()) for n in self.name.split("|")] return os.path.join(*p) + ".json" @property def json_data(self): r = { "name": self.name, "$collection": self._meta["json_collection"], "uuid": self.uuid, "action__name": self.action.name, "description": self.description, "profile__name": self.profile.name, "config_mode": self.config_mode, "match": [c.json_data for c in self.match], "commands": self.commands, "preference": self.preference, "timeout": self.timeout, } return r def to_json(self): return to_json( self.json_data, order=[ "name", "$collection", "uuid", "action__name", "description", "profile__name", "config_mode", "preference", "match", "commands", "timeout", ], )
class Cert(Document): """Certificate mongo document model.""" log_id = IntField(primary_key=True) # serial is 20 octets, see: https://tools.ietf.org/html/rfc5280#section-4.1.2.2 serial = StringField(required=True) issuer = StringField(required=True) not_before = DateTimeField(required=True) not_after = DateTimeField(required=True) sct_or_not_before = DateTimeField(required=True) sct_exists = BooleanField(required=True) pem = StringField(required=True) _subjects = ListField(required=True, field=StringField(), db_field="subjects") _trimmed_subjects = ListField(required=True, field=StringField(), db_field="trimmed_subjects") meta = { "collection": "certs", "indexes": [ "+_subjects", "+_trimmed_subjects", { "fields": ("+issuer", "+serial"), "unique": True }, ], } @property def subjects(self): """Getter for subjects.""" return self._subjects @subjects.setter def subjects(self, values): """Subjects setter. Normalizes inputs, and dervices trimmed_subjects """ self._subjects = list({i.lower() for i in values}) self._trimmed_subjects = list(trim_domains(self._subjects)) @property def trimmed_subjects(self): """Read-only property. This is derived from the subjects.""" return self._trimmed_subjects def to_x509(self): """Return an x509 subject for this certificate.""" return x509.load_pem_x509_certificate(bytes(self.pem, "utf-8"), default_backend()) @classmethod def from_pem(cls, pem): """Create a Cert model object from a PEM certificate string. Arguments: pem -- PEM encoded certificate Returns (cert, precert): cert: a Cert model object precert: a boolean, True if this is a precertificate, False otherwise """ xcert = x509.load_pem_x509_certificate(bytes(pem, "utf-8"), default_backend()) dns_names = get_sans_set(xcert) sct_or_not_before, sct_exists = get_earliest_sct(xcert) cert = cls() cert.serial = hex(xcert.serial_number)[2:] cert.issuer = xcert.issuer.rfc4514_string() cert.not_before = xcert.not_valid_before cert.not_after = xcert.not_valid_after cert.sct_or_not_before = sct_or_not_before cert.sct_exists = sct_exists cert.pem = pem cert.subjects = dns_names return cert, is_poisioned(xcert)
class Interface(Document): """ Interfaces """ meta = { "collection": "noc.interfaces", "strict": False, "auto_create_index": False, "indexes": [("managed_object", "name"), "mac", ("managed_object", "ifindex"), "service", "aggregated_interface"] } managed_object = ForeignKeyField(ManagedObject) name = StringField() # Normalized via Profile.convert_interface_name type = StringField(choices=[(x, x) for x in INTERFACE_TYPES]) description = StringField(required=False) ifindex = IntField(required=False) mac = StringField(required=False) aggregated_interface = PlainReferenceField("self", required=False) enabled_protocols = ListField( StringField(choices=[(x, x) for x in INTERFACE_PROTOCOLS]), default=[]) profile = PlainReferenceField(InterfaceProfile, default=InterfaceProfile.get_default_profile) # profile locked on manual user change profile_locked = BooleanField(required=False, default=False) # project = ForeignKeyField(Project) state = ForeignKeyField(ResourceState) vc_domain = ForeignKeyField(VCDomain) # Current status admin_status = BooleanField(required=False) oper_status = BooleanField(required=False) oper_status_change = DateTimeField(required=False, default=datetime.datetime.now) full_duplex = BooleanField(required=False) in_speed = IntField(required=False) # Input speed, kbit/s out_speed = IntField(required=False) # Output speed, kbit/s bandwidth = IntField(required=False) # Configured bandwidth, kbit/s # Coverage coverage = PlainReferenceField(Coverage) technologies = ListField(StringField()) # External NRI interface name nri_name = StringField() # service = ReferenceField(Service) PROFILE_LINK = "profile" def __unicode__(self): return u"%s: %s" % (self.managed_object.name, self.name) def iter_changed_datastream(self): if config.datastream.enable_managedobject: yield "managedobject", self.managed_object.id def save(self, *args, **kwargs): if not hasattr(self, "_changed_fields") or "name" in self._changed_fields: self.name = self.managed_object.get_profile( ).convert_interface_name(self.name) if (not hasattr(self, "_changed_fields") or "mac" in self._changed_fields) and self.mac: self.mac = MACAddressParameter().clean(self.mac) try: super(Interface, self).save(*args, **kwargs) except Exception as e: raise ValueError("%s: %s" % (e.__doc__, e.message)) if not hasattr(self, "_changed_fields") or "service" in self._changed_fields: ServiceSummary.refresh_object(self.managed_object) def on_delete(self): # Remove all subinterfaces for si in self.subinterface_set.all(): si.delete() # Unlink link = self.link if link: self.unlink() # Flush MACDB MACDB.objects.filter(interface=self.id).delete() @property def link(self): """ Return Link instance or None :return: """ if self.type == "aggregated": q = { "interfaces__in": [self.id] + [i.id for i in self.lag_members] } else: q = {"interfaces": self.id} return Link.objects.filter(**q).first() @property def is_linked(self): """ Check interface is linked :returns: True if interface is linked, False otherwise """ if self.type == "aggregated": q = { "interfaces": { "$in": [self.id] + [i.id for i in self.lag_members] } } else: q = {"interfaces": self.id} return bool(Link._get_collection().with_options( read_preference=ReadPreference.SECONDARY_PREFERRED).find_one(q)) def unlink(self): """ Remove existing link. Raise ValueError if interface is not linked """ link = self.link if link is None: raise ValueError("Interface is not linked") if link.is_ptp or link.is_lag: link.delete() else: raise ValueError("Cannot unlink non p-t-p link") def link_ptp(self, other, method=""): """ Create p-t-p link with other interface Raise ValueError if either of interface already connected. :param other: Other Iface for link :param method: Linking method :type other: Interface :returns: Link instance """ def link_mismatched_lag(agg, phy): """ Try to link LAG to physical interface :param agg: :param phy: :return: """ l_members = [i for i in agg.lag_members if i.oper_status] if len(l_members) > 1: raise ValueError("More then one active interface in LAG") link = Link(interfaces=l_members + [phy], discovery_method=method) link.save() return link # Try to check existing LAG el = Link.objects.filter(interfaces=self.id).first() if el and other not in el.interfaces: el = None if (self.is_linked or other.is_linked) and not el: raise ValueError("Already linked") if self.id == other.id: raise ValueError("Cannot link with self") if self.type in ("physical", "management"): if other.type in ("physical", "management"): # Refine LAG if el: left_ifaces = [ i for i in el.interfaces if i not in (self, other) ] if left_ifaces: el.interfaces = left_ifaces el.save() else: el.delete() # link = Link(interfaces=[self, other], discovery_method=method) link.save() return link elif other.type == "aggregated" and other.profile.allow_lag_mismatch: return link_mismatched_lag(other, self) else: raise ValueError("Cannot connect %s interface to %s" % (self.type, other.type)) elif self.type == "aggregated": # LAG if other.type == "aggregated": # Check LAG size match # Skip already linked members l_members = [i for i in self.lag_members if not i.is_linked] r_members = [i for i in other.lag_members if not i.is_linked] if len(l_members) != len(r_members): raise ValueError("LAG size mismatch") # Create link if l_members: link = Link(interfaces=l_members + r_members, discovery_method=method) link.save() return link else: return elif self.profile.allow_lag_mismatch: return link_mismatched_lag(self, other) else: raise ValueError("Cannot connect %s interface to %s" % (self.type, other.type)) raise ValueError("Cannot link") @classmethod def get_interface(cls, s): """ Parse <managed object>@<interface> string and return interface instance or None """ if "@" not in s: raise ValueError("Invalid interface: %s" % s) o, i = s.rsplit("@", 1) # Get managed object try: mo = ManagedObject.objects.get(name=o) except ManagedObject.DoesNotExist: raise ValueError("Invalid manged object: %s" % o) # Normalize interface name i = mo.get_profile().convert_interface_name(i) # Look for interface iface = Interface.objects.filter(managed_object=mo.id, name=i).first() return iface @property def subinterface_set(self): from .subinterface import SubInterface return SubInterface.objects.filter(interface=self.id) @property def lag_members(self): if self.type != "aggregated": raise ValueError( "Cannot net LAG members for not-aggregated interface") return Interface.objects.filter(aggregated_interface=self.id) @property def effective_vc_domain(self): if self.type in ("null", "tunnel", "other", "unknown"): return None if self.vc_domain: return self.vc_domain if self.managed_object.vc_domain: return self.managed_object.vc_domain return VCDomain.get_default() @property def status(self): """ Returns interface status in form of Up/100/Full """ def humanize_speed(speed): if not speed: return "-" for t, n in [(1000000, "G"), (1000, "M"), (1, "k")]: if speed >= t: if speed // t * t == speed: return "%d%s" % (speed // t, n) else: return "%.2f%s" % (float(speed) / t, n) return str(speed) s = [{True: "Up", False: "Down", None: "-"}[self.oper_status]] # Speed if self.oper_status: if self.in_speed and self.in_speed == self.out_speed: s += [humanize_speed(self.in_speed)] else: s += [ humanize_speed(self.in_speed), humanize_speed(self.out_speed) ] s += [{True: "Full", False: "Half", None: "-"}[self.full_duplex]] else: s += ["-", "-"] return "/".join(s) def set_oper_status(self, status): """ Set current oper status """ if self.oper_status == status: return now = datetime.datetime.now() if self.oper_status != status and (not self.oper_status_change or self.oper_status_change < now): self.update(oper_status=status, oper_status_change=now) if self.profile.status_change_notification: logger.debug("Sending status change notification to %s", self.profile.status_change_notification.name) self.profile.status_change_notification.notify( subject="[%s] Interface %s(%s) is %s" % (self.managed_object.name, self.name, self.description or "", "up" if status else "down"), body="Interface %s (%s) is %s" % (self.name, self.description or "", "up" if status else "down")) @property def parent(self): """ Returns aggregated interface for LAG or self for non-aggregated interface """ if self.aggregated_interface: return self.aggregated_interface else: return self def get_profile(self): if self.profile: return self.profile return InterfaceProfile.get_default_profile()
class VPN(Document): meta = {"collection": "vpns", "strict": False, "auto_create_index": False} name = StringField(unique=True) profile = PlainReferenceField(VPNProfile) description = StringField() state = PlainReferenceField(State) # Link to parent overlay parent = PlainReferenceField("self") project = ForeignKeyField(Project) route_target = ListField(EmbeddedDocumentField(RouteTargetItem)) # Labels labels = ListField(StringField()) effective_labels = ListField(StringField()) # Integration with external NRI and TT systems # Reference to remote system object has been imported from remote_system = PlainReferenceField(RemoteSystem) # Object id in remote system remote_id = StringField() # Object id in BI bi_id = LongField(unique=True) # @todo: last_seen # @todo: expired _id_cache = cachetools.TTLCache(maxsize=100, ttl=60) _bi_id_cache = cachetools.TTLCache(maxsize=100, ttl=60) def __str__(self): return self.name @classmethod @cachetools.cachedmethod(operator.attrgetter("_id_cache"), lock=lambda _: id_lock) def get_by_id(cls, id): return VPN.objects.filter(id=id).first() @classmethod @cachetools.cachedmethod(operator.attrgetter("_bi_id_cache"), lock=lambda _: id_lock) def get_by_bi_id(cls, id): return VPN.objects.filter(bi_id=id).first() def clean(self): super().clean() if self.id and "parent" in self._changed_fields and self.has_loop: raise ValidationError("Creating VPN loop") @property def has_loop(self): """ Check if object creates loop """ if not self.id: return False p = self.parent while p: if p.id == self.id: return True p = p.parent return False @classmethod def can_set_label(cls, label): return Label.get_effective_setting(label, "enable_vpn")
class Vendor(Document): """ Equipment vendor """ meta = { "collection": "noc.vendors", "strict": False, "auto_create_index": False, "json_collection": "inv.vendors", "json_unique_fields": ["name", "code"] } # Short vendor name, included as first part of platform name = StringField(unique=True) # Full vendor name full_name = StringField() # Unique id uuid = UUIDField(binary=True) # List of vendor codes to be searched via .get_by_code() code = ListField(StringField(), unique=True) # Vendor's site site = URLField(required=False) # Object id in BI bi_id = LongField(unique=True) _id_cache = cachetools.TTLCache(1000, ttl=60) _bi_id_cache = cachetools.TTLCache(1000, ttl=60) _code_cache = cachetools.TTLCache(1000, ttl=60) _ensure_cache = cachetools.TTLCache(1000, ttl=60) def __unicode__(self): return self.name @classmethod @cachetools.cachedmethod(operator.attrgetter("_id_cache"), lock=lambda _: id_lock) def get_by_id(cls, id): return Vendor.objects.filter(id=id).first() @classmethod @cachetools.cachedmethod(operator.attrgetter("_bi_id_cache"), lock=lambda _: id_lock) def get_by_bi_id(cls, id): return Vendor.objects.filter(bi_id=id).first() @classmethod def _get_by_code(cls, code): """ Uncached version of get_by_code :param code: :return: """ code = code.upper() return Vendor.objects.filter(code=code).first() @classmethod @cachetools.cachedmethod(operator.attrgetter("_code_cache"), lock=lambda _: id_lock) def get_by_code(cls, code): return cls._get_by_code(code) def clean(self): # Convert code to list if isinstance(self.code, six.string_types): self.code = [self.code] # Uppercase code self.code = [c.upper() for c in self.code] # Fill full name if not set if not self.full_name: self.full_name = self.name # super(Vendor, self).clean() def on_save(self): if not hasattr(self, "_changed_fields") or "name" in self._changed_fields: from .platform import Platform for p in Platform.objects.filter(vendor=self.id): p.save() # Rebuild full name def to_json(self): return to_json({ "name": self.name, "$collection": self._meta["json_collection"], "full_name": self.full_name, "code": self.code, "site": self.site, "uuid": self.uuid }, order=["name", "uuid", "full_name", "code", "site"] ) def get_json_path(self): return "%s.json" % self.name @classmethod @cachetools.cachedmethod(operator.attrgetter("_ensure_cache"), lock=lambda _: id_lock) def ensure_vendor(cls, code): """ Get or create vendor by code :param code: :return: """ while True: vendor = Vendor._get_by_code(code) if vendor: return vendor try: vendor = Vendor(name=code, full_name=code, code=[code], uuid=uuid.uuid4()) vendor.save() return vendor except NotUniqueError: pass # Already created by concurrent process, reread
class TranslationGroup(EmbeddedDocument): translationLemmas = ListField(ReferenceField(Lemma)) restriction = EmbeddedDocumentField(Restriction) exampleGroups = ListField(EmbeddedDocumentField(ExampleGroup))
class ConnectionType(Document): """ Equipment vendor """ meta = { "collection": "noc.connectiontypes", "strict": False, "auto_create_index": False, "indexes": ["extend", "data", "c_group"], "json_collection": "inv.connectiontypes", "json_unique_fields": ["name", "uuid"], } name = StringField(unique=True) is_builtin = BooleanField(default=False) description = StringField() # Type extends another type, if not null extend = PlainReferenceField("self", required=False) # List of available genders genders = StringField( choices=[ "s", # Genderless connection "ss", # Genderless connection 2 or more objects "m", # Only male type "f", # Only female type "mmf", # female, 1 or more males "mf", # male-female "mff", # male, 2 or more females ], default="mf", ) # ModelData data = DictField(default={}) # Compatible group # Connection compatible with opposite gender of same type # and all types having any c_group c_group = ListField(StringField()) uuid = UUIDField(binary=True) OPPOSITE_GENDER = {"s": "s", "m": "f", "f": "m"} category = ObjectIdField() def __str__(self): return self.name @property def json_data(self): r = { "name": self.name, "$collection": self._meta["json_collection"], "uuid": self.uuid, "description": self.description, "genders": self.genders, "c_group": self.c_group, } if self.extend: r["extend__name"] = self.extend.name return r def to_json(self): return to_json(self.json_data, order=["name", "$collection", "uuid", "description"]) def get_json_path(self): p = [quote_safe_path(n.strip()) for n in self.name.split("|")] return os.path.join(*p) + ".json" def get_effective_data(self): """ Calculate effective data :return: """ raise NotImplementedError def get_superclasses(self): s = [] c = self while c: c = c.extend if c: s += [c] return s def get_subclasses(self): s = [] for c in ConnectionType.objects.filter(extend=self.id): s += [c] + c.get_subclasses() return s def get_inheritance_path(self, other): s = [] # Upward direction c = self while c: s.insert(0, c) if other.id == c.id: return s c = c.extend # Not found, try downward direction s = [] c = other while c: s.insert(0, c) if self.id == c.id: return s c = c.extend return s def get_by_c_group(self): c_group = self.c_group if not c_group: return [] r = [] for ct in ConnectionType.objects.filter(c_group__in=c_group): if ct.id != self.id: r += [ct] return r def get_compatible_types(self, gender): r = [] og = self.OPPOSITE_GENDER[gender] # Add self type if opposige gender allowed if og in self.genders: r += [self.id] if gender in ["m", "s"]: # Add superclasses for c in self.get_superclasses(): if og in c.genders: r += [c.id] if gender in ["f", "s"]: # Add subclasses for c in self.get_subclasses(): if og in c.genders: r += [c.id] if self.c_group: for c in self.get_by_c_group(): if og in c.genders: r += [c.id] return r
class Contributions(DynamicDocument): project = LazyReferenceField( "Projects", required=True, passthrough=True, reverse_delete_rule=CASCADE ) identifier = StringField(required=True, help_text="material/composition identifier") formula = StringField(help_text="formula (set dynamically if not provided)") is_public = BooleanField( required=True, default=False, help_text="public/private contribution" ) last_modified = DateTimeField( required=True, default=datetime.utcnow, help_text="time of last modification" ) data = DictField( default={}, validation=valid_dict, help_text="simple free-form data" ) structures = ListField(ReferenceField("Structures"), default=list, max_length=10) tables = ListField(ReferenceField("Tables"), default=list, max_length=10) notebook = LazyReferenceField("Notebooks", passthrough=True) meta = { "collection": "contributions", "indexes": [ "project", "identifier", "formula", "is_public", "last_modified", {"fields": [(r"data.$**", 1)]}, "notebook", ] + list(COMPONENTS.keys()), } @classmethod def post_init(cls, sender, document, **kwargs): # replace existing structures/tables with according ObjectIds for component, fields in COMPONENTS.items(): lst = getattr(document, component) if lst and lst[0].id is None: # id is None for incoming POST dmodule = import_module(f"mpcontribs.api.{component}.document") klass = component.capitalize() Docs = getattr(dmodule, klass) vmodule = import_module(f"mpcontribs.api.{component}.views") Resource = getattr(vmodule, f"{klass}Resource") resource = Resource() for i, o in enumerate(lst): d = resource.serialize(o, fields=fields) s = json.dumps(d, sort_keys=True).encode("utf-8") digest = md5(s).hexdigest() obj = Docs.objects(md5=digest).only("id").first() if obj: obj.reload() lst[i] = obj @classmethod def pre_save_post_validation(cls, sender, document, **kwargs): if kwargs.get("skip"): return # set formula field if hasattr(document, "formula") and not document.formula: formulae = current_app.config["FORMULAE"] document.formula = formulae.get(document.identifier, document.identifier) # project is LazyReferenceField project = document.project.fetch() # run data through Pint Quantities and save as dicts def make_quantities(path, key, value): if key in quantity_keys or not isinstance(value, (str, int, float)): return key, value str_value = str(value) if str_value.count(" ") > 1: return key, value q = get_quantity(str_value) if not q: return key, value # silently ignore "nan" if isnan(q.nominal_value): return False # try compact representation qq = q.value.to_compact() q = new_error_units(q, qq) # reduce dimensionality if possible if not q.check(0): qq = q.value.to_reduced_units() q = new_error_units(q, qq) # ensure that the same units are used across contributions field = delimiter.join(["data"] + list(path) + [key]) try: column = project.columns.get(path=field) if column.unit != str(q.value.units): qq = q.value.to(column.unit) q = new_error_units(q, qq) except DoesNotExist: pass # column doesn't exist yet (generated in post_save) except DimensionalityError: raise ValueError(f"Can't convert [{q.units}] to [{column.unit}]!") # significant digits q = truncate_digits(q) # return new value dict display = str(q.value) if isnan(q.std_dev) else str(q) value = { "display": display, "value": q.nominal_value, "error": q.std_dev, "unit": str(q.units), } return key, value document.data = remap(document.data, visit=make_quantities, enter=enter) @classmethod def post_save(cls, sender, document, **kwargs): if kwargs.get("skip"): return # project is LazyReferenceField project = document.project.fetch() # set columns field for project def update_columns(path, key, value): path = delimiter.join(["data"] + list(path) + [key]) is_quantity = isinstance(value, dict) and quantity_keys.issubset( value.keys() ) is_text = bool( not is_quantity and isinstance(value, str) and key not in quantity_keys ) if is_quantity or is_text: project.reload("columns") try: column = project.columns.get(path=path) if is_quantity: v = value["value"] if isnan(column.max) or v > column.max: column.max = v if isnan(column.min) or v < column.min: column.min = v except DoesNotExist: column = {"path": path} if is_quantity: column["unit"] = value["unit"] column["min"] = column["max"] = value["value"] project.columns.create(**column) project.save().reload("columns") ncolumns = len(project.columns) if ncolumns > 50: raise ValueError("Reached maximum number of columns (50)!") return True # run update_columns over document data remap(document.data, visit=update_columns, enter=enter) # add/remove columns for other components for path in COMPONENTS.keys(): try: project.columns.get(path=path) except DoesNotExist: if getattr(document, path): project.columns.create(path=path) project.save().reload("columns") document.last_modified = datetime.utcnow() @classmethod def pre_delete(cls, sender, document, **kwargs): args = ["notebook"] + list(COMPONENTS.keys()) document.reload(*args) # remove reference documents if document.notebook is not None: from mpcontribs.api.notebooks.document import Notebooks Notebooks.objects(id=document.notebook.id).delete() for component in COMPONENTS.keys(): # check if other contributions exist before deletion! for obj in getattr(document, component): q = {component: obj.id} if sender.objects(**q).count() < 2: obj.delete() @classmethod def post_delete(cls, sender, document, **kwargs): if kwargs.get("skip"): return # reset columns field for project project = document.project.fetch() for column in list(project.columns): if not isnan(column.min) and not isnan(column.max): column.min, column.max = get_min_max(sender, column.path) if isnan(column.min) and isnan(column.max): # just deleted last contribution with this column project.update(pull__columns__path=column.path) else: # use wildcard index if available -> single field query field = column.path.replace(delimiter, "__") + "__type" qs = sender.objects(**{field: "string"}).only(column.path) if qs.count() < 1 or qs.filter(project__name=project.name).count() < 1: project.update(pull__columns__path=column.path)
class MapSettings(Document): meta = { "collection": "noc.mapsettings", "strict": False, "auto_create_index": False } # Segment or selector id segment = StringField(unique=True) # Change time changed = DateTimeField() # Changing user user = StringField() # Paper size width = FloatField() height = FloatField() # nodes = ListField(EmbeddedDocumentField(NodeSettings)) links = ListField(EmbeddedDocumentField(LinkSettings)) def __str__(self): return self.segment def get_nodes(self): """ Returns a dict of id -> Node settings """ nodes = {} for n in self.nodes: nodes[n.node] = n return nodes @classmethod def load_json(cls, data, user=None): """ Load json data of: id nodes: id x y links: id vertices: x y """ d = MapSettings.objects.filter(segment=data["id"]).first() if d: logger.info("Updating settings for %s", data["id"]) else: logger.info("Creating new settings for %s", data["id"]) d = MapSettings(segment=data["id"], nodes=[], links=[]) # Update meta if user: d.user = user d.changed = datetime.datetime.now() # Update nodes new_nodes = {} # id -> data for n in data.get("nodes", []): new_nodes[(n["type"], n["id"])] = n nn = [] for n in d.nodes: nd = new_nodes.get((n.type, n.id)) if not nd: continue # Not found n.x = nd["x"] n.y = nd["y"] nn += [n] del new_nodes[(n.type, n.id)] mx = 0.0 my = 0.0 for n in new_nodes: nd = new_nodes[n] nn += [ NodeSettings(type=nd["type"], id=nd["id"], x=nd["x"], y=nd["y"]) ] mx = max(mx, nd["x"]) my = max(my, nd["y"]) d.width = data.get("width", mx) d.height = data.get("height", my) d.nodes = sorted(nn, key=lambda x: (x.type, x.id)) # Update links new_links = {} for l in data.get("links", []): new_links[(l["type"], l["id"])] = l nn = [] for l in d.links: nl = new_links.get((l.type, l.id)) if not nl: continue # Not found l.vertices = [ VertexPosition(x=v["x"], y=v["y"]) for v in nl.get("vertices", []) ] l.connector = nl.get("connector", LC_NORMAL) nn += [l] del new_links[(l.type, l.id)] for l in new_links: nl = new_links[l] nn += [ LinkSettings( type=nl["type"], id=nl["id"], vertices=[ VertexPosition(x=v["x"], y=v["y"]) for v in nl.get("vertices", []) ], connector=nl.get("connector", "normal"), ) ] d.links = [ l for l in sorted(nn, key=lambda x: (x.type, x.id)) if l.vertices or l.connector != LC_NORMAL ] # Finally save d.save() return d
class ServiceSummary(Document): meta = { "collection": "noc.servicesummary", "strict": False, "auto_create_index": False, "indexes": ["managed_object", "interface"], } managed_object = IntField() interface = ObjectIdField() service = ListField(EmbeddedDocumentField(SummaryItem)) subscriber = ListField(EmbeddedDocumentField(SummaryItem)) @classmethod def build_summary_for_object(cls, managed_object): """ Build active services summary for managed object :param managed_object: Managed Object id :return: dict of interface id -> {service: ..., subscriber: ....} interface None means unbound or box-wise services """ from noc.inv.models.interface import Interface from noc.sa.models.service import Service def iter_services(sd): yield sd for cs in Service._get_collection().find( { "parent": sd["_id"], "logical_status": "R" }, { "_id": 1, "subscriber": 1, "profile": 1 }, ): for ns in iter_services(cs): yield ns def add_dict(d1, d2): """ Add all d2 values to d1 :param d1: :param d2: :return: """ for k in d2: d1[k] = d1.get(k, 0) + d2[k] # service -> interface bindings svc_interface = dict(( x["service"], x["_id"] ) for x in Interface._get_collection().find( { "managed_object": managed_object, "service": { "$exists": True } }, { "_id": 1, "service": 1 }, comment= "[servicesummary.build_summary_for_object] Getting services for interfaces", )) # Iterate over object's services # And walk underlying tree ri = {} for svc in Service._get_collection().find( { "managed_object": managed_object, "logical_status": "R" }, { "_id": 1, "subscriber": 1, "profile": 1 }, comment= "[servicesummary.build_summary_for_object] Getting object services for object", ): # All subscribers for underlying tree subscribers = set() # profile_id -> count svc_profiles = defaultdict(int) for s in iter_services(svc): subscribers.add(s["subscriber"]) svc_profiles[s["profile"]] += 1 # Get subscriber profiles count ra = Subscriber._get_collection().aggregate([ { "$match": { "_id": { "$in": list(subscribers) } } }, { "$group": { "_id": "$profile", "total": { "$sum": 1 } } }, ]) subscriber_profiles = dict((x["_id"], x["total"]) for x in ra) # Bind to interface # None for unbound services iface = svc_interface.get(svc["_id"]) if iface in ri: add_dict(ri[iface]["service"], svc_profiles) add_dict(ri[iface]["subscriber"], subscriber_profiles) else: ri[iface] = { "service": dict(svc_profiles), # defaultdict -> dict "subscriber": subscriber_profiles, } return ri @classmethod def refresh_object(cls, managed_object): if hasattr(managed_object, "id"): managed_object = managed_object.id call_later("noc.sa.models.servicesummary.refresh_object", delay=20, managed_object=managed_object) @classmethod def _refresh_object(cls, managed_object): from noc.sa.models.managedobject import ManagedObject from noc.inv.models.networksegment import NetworkSegment def to_dict(v): return dict((r["profile"], r["summary"]) for r in v) def to_list(v): return [{"profile": k, "summary": v[k]} for k in sorted(v)] if hasattr(managed_object, "id"): managed_object = managed_object.id coll = ServiceSummary._get_collection() bulk = [] # Get existing summary old_summary = dict((x["interface"], x) for x in coll.find( {"managed_object": managed_object}, { "_id": 1, "interface": 1, "service": 1, "subscriber": 1 }, comment= "[servicesummary._refresh_object] Refresh summary of services for managed object", )) # Get actual summary new_summary = ServiceSummary.build_summary_for_object(managed_object) # Merge summaries for iface in old_summary: if iface not in new_summary: # Stale, delete bulk += [DeleteOne({"_id": old_summary[iface]["_id"]})] continue oi = old_summary[iface] old_services = to_dict(oi["service"]) old_subs = to_dict(oi["subscriber"]) ni = new_summary[iface] if old_services != ni["service"] or old_subs != ni["subscriber"]: # Changed, update bulk += [ UpdateOne( {"_id": oi["_id"]}, { "$set": { "service": to_list(ni["service"]), "subscriber": to_list(ni["subscriber"]), } }, ) ] # Mark as processed del new_summary[iface] # Process new items bulk += [ InsertOne({ "managed_object": managed_object, "interface": iface, "service": to_list(new_summary[iface]["service"]), "subscriber": to_list(new_summary[iface]["subscriber"]), }) for iface in new_summary ] if bulk: logger.info("Committing changes to database") try: r = coll.bulk_write(bulk, ordered=False) logger.info("Database has been synced") logger.info("Modify: %d, Deleted: %d", r.modified_count, r.deleted_count) except BulkWriteError as e: logger.error("Bulk write error: '%s'", e.details) logger.error("Stopping check") mo = ManagedObject.get_by_id(managed_object) NetworkSegment.update_summary(mo.segment) @classmethod def get_object_summary(cls, managed_object): def to_dict(v): return dict((r["profile"], r["summary"]) for r in v) if hasattr(managed_object, "id"): managed_object = managed_object.id r = {"service": {}, "subscriber": {}, "interface": {}} for ss in ServiceSummary._get_collection().find( {"managed_object": managed_object}, { "interface": 1, "service": 1, "subscriber": 1 }, comment= "[servicesummary.get_object_summary] Getting summary of services for object", ): ds = to_dict(ss["service"]) if ss.get("interface"): r["interface"][ss["interface"]] = {"service": ds} for k, v in ds.items(): if k in r["service"]: r["service"][k] += v else: r["service"][k] = v ds = to_dict(ss["subscriber"]) if ss.get("interface"): r["interface"][ss["interface"]]["subscriber"] = ds for k, v in ds.items(): if k in r["subscriber"]: r["subscriber"][k] += v else: r["subscriber"][k] = v return r @classmethod def get_objects_summary(cls, managed_objects): def to_dict(v): return dict((r["profile"], r["summary"]) for r in v) kk = {} for ss in ServiceSummary._get_collection().find( { "managed_object": { "$in": [getattr(mo, "id", mo) for mo in managed_objects] } }, { "managed_object": 1, "interface": 1, "service": 1, "subscriber": 1 }, comment= "[servicesummary.get_objects_summary] Getting summary of services for objects list", ): r = {"service": {}, "subscriber": {}, "interface": {}} ds = to_dict(ss["service"]) if ss.get("interface"): r["interface"][ss["interface"]] = {"service": ds} for k, v in ds.items(): if k in r["service"]: r["service"][k] += v else: r["service"][k] = v ds = to_dict(ss["subscriber"]) if ss.get("interface"): r["interface"][ss["interface"]]["subscriber"] = ds for k, v in ds.items(): if k in r["subscriber"]: r["subscriber"][k] += v else: r["subscriber"][k] = v kk[ss["managed_object"]] = r return kk @classmethod def get_weight(cls, summary): """ Convert result of *get_object_summary* to alarm weight """ w = 0 subscribers = summary.get("subscriber", {}) for s in subscribers: sp = SubscriberProfile.get_by_id(s) if sp and sp.weight: w += sp.weight * subscribers[s] services = summary.get("service", {}) for s in services: sp = ServiceProfile.get_by_id(s) if sp and sp.weight: w += sp.weight * services[s] objects = summary.get("object", {}) for s in objects: sp = ManagedObjectProfile.get_by_id(s) if sp and sp.weight: w += sp.weight * objects[s] return w @classmethod def get_severity(cls, summary): """ Convert result of *get_object_summary* to alarm severity """ from noc.fm.models.alarmseverity import AlarmSeverity return AlarmSeverity.severity_for_weight(cls.get_weight(summary)) @classmethod def get_direct_summary(cls, managed_objects, summary_all=False): """ ! Method works on mongodb version 3.4 and greater Calculate direct services and profiles for a list of managed objects :param managed_objects: List of managed object instances or ids :param summary_all: Return summary for all services :return: tuple of service and subscriber dicts """ services = {} subscribers = {} pipeline = [] if not summary_all: # Filter managed objects pipeline += [{ "$match": { "managed_object": { "$in": [getattr(mo, "id", mo) for mo in managed_objects] } } }] # Mark service and profile with type field pipeline += [ { "$project": { "_id": 0, "service": { "$map": { "input": "$service", "as": "svc", "in": { "type": "svc", "profile": "$$svc.profile", "summary": "$$svc.summary", }, } }, "subscriber": { "$map": { "input": "$subscriber", "as": "sub", "in": { "type": "sub", "profile": "$$sub.profile", "summary": "$$sub.summary", }, } }, } }, # Concatenate services and profiles { "$project": { "summary": { "$concatArrays": ["$service", "$subscriber"] } } }, # Unwind *summary* array to independed records { "$unwind": "$summary" }, # Group by (type, profile) { "$group": { "_id": { "type": "$summary.type", "profile": "$summary.profile" }, "summary": { "$sum": "$summary.summary" }, } }, ] # noqa try: for doc in ServiceSummary._get_collection().aggregate(pipeline): profile = doc["_id"]["profile"] if doc["_id"]["type"] == "svc": services[profile] = services.get(profile, 0) + doc["summary"] else: subscribers[profile] = subscribers.get(profile, 0) + doc["summary"] except OperationFailure: # for Mongo less 3.4 pass return services, subscribers
class Platform(Document): meta = { "collection": "noc.platforms", "strict": False, "auto_create_index": False, "json_collection": "inv.platforms", "json_unique_fields": [("vendor", "name")], "indexes": [{ "fields": ["vendor", "name"], "unique": True }, ("vendor", "aliases")], } vendor = PlainReferenceField(Vendor) name = StringField() description = StringField(required=False) # Full name, combined from vendor platform full_name = StringField(unique=True) # Platform start of sale date start_of_sale = DateField() # Platform end of sale date end_of_sale = DateField() # Platform end of support date end_of_support = DateField() # End of extended support date (installation local) end_of_xsupport = DateField() # SNMP OID value # sysObjectID.0 snmp_sysobjectid = StringField(regex=r"^1.3.6(\.\d+)+$") # Global ID uuid = UUIDField(binary=True) # Platform aliases aliases = ListField(StringField()) # Labels labels = ListField(StringField()) effective_labels = ListField(StringField()) # Object id in BI bi_id = LongField(unique=True) _id_cache = cachetools.TTLCache(1000, ttl=60) _bi_id_cache = cachetools.TTLCache(1000, ttl=60) _ensure_cache = cachetools.TTLCache(1000, ttl=60) def __str__(self): return self.full_name def clean(self): self.full_name = "%s %s" % (self.vendor.name, self.name) if self.aliases: self.aliases = sorted(a for a in self.aliases if a != self.name) super().clean() def save(self, *args, **kwargs): to_merge_aliases = not hasattr( self, "_changed_fields") or "aliases" in self._changed_fields super().save(*args, **kwargs) if to_merge_aliases: for a in self.aliases: if a == self.name: continue self.merge_platform(a) @classmethod @cachetools.cachedmethod(operator.attrgetter("_id_cache"), lock=lambda _: id_lock) def get_by_id(cls, id): return Platform.objects.filter(id=id).first() @classmethod @cachetools.cachedmethod(operator.attrgetter("_bi_id_cache"), lock=lambda _: id_lock) def get_by_bi_id(cls, id): return Platform.objects.filter(bi_id=id).first() def to_json(self): r = { "$collection": self._meta["json_collection"], "vendor__code": self.vendor.code[0], "name": self.name, "uuid": self.uuid, } if self.aliases: r["aliases"] = [str(x) for x in self.aliases] if self.description: r["description"] = self.description if self.start_of_sale: r["start_of_sale"] = self.start_of_sale.strftime("%Y-%m-%d") if self.end_of_sale: r["end_of_sale"] = self.end_of_sale.strftime("%Y-%m-%d") if self.end_of_support: r["end_of_support"] = self.end_of_support.strftime("%Y-%m-%d") if self.snmp_sysobjectid: r["snmp_sysobjectid"] = self.snmp_sysobjectid if self.labels: r["labels"] = self.labels return to_json( r, order=[ "vendor__code", "name", "$collection", "uuid", "aliases", "description", "start_of_sale", "end_of_sale", "end_of_support", "snmp_sysobjectid", "labels", ], ) def get_json_path(self): return os.path.join(self.vendor.code[0], "%s.json" % self.name.replace("/", "_")) @classmethod @cachetools.cachedmethod( operator.attrgetter("_ensure_cache"), key=lambda v, n, strict=False, labels=None: "%s-%s" % (v.id, n), lock=lambda _: id_lock, ) def ensure_platform(cls, vendor, name, strict=False, labels=None): """ Get or create platform by vendor and code :param vendor: :param name: :param strict: Return None if platform is not found :param labels: List of platform labels :return: """ # Try to find platform q = Q(vendor=vendor.id, name=name) | Q(vendor=vendor.id, aliases=name) platform = Platform.objects.filter(q).first() if platform or strict: return platform # Try to create labels = labels or [] pu = uuid.uuid4() d = Platform._get_collection().find_one_and_update( { "vendor": vendor.id, "name": name }, { "$setOnInsert": { "uuid": pu, "full_name": "%s %s" % (vendor.name, name), "bi_id": Int64(new_bi_id()), "aliases": [], "labels": labels, } }, upsert=True, return_document=ReturnDocument.AFTER, ) d["id"] = d["_id"] del d["_id"] p = Platform(**d) p._changed_fields = [] return p @property def is_end_of_sale(self): """ Check if platform reached end-of-sale mark :return: """ if not self.end_of_sale: return False return datetime.date.today() > self.end_of_sale @property def is_end_of_support(self): """ Check if platform reached end-of-support mark :return: """ deadline = [] if self.end_of_support: deadline += [self.end_of_support] if self.end_of_xsupport: deadline += [self.end_of_xsupport] if deadline: return datetime.date.today() > max(deadline) else: return False def merge_platform(self, alias): """ Merge *alias* platform :param alias: platform name :return: """ ap = Platform.objects.filter(vendor=self.vendor.id, name=alias).first() if not ap: return # Replace ce platform refs = self._on_delete["check"] + self._on_delete[ "clean"] + self._on_delete["delete"] for model_name, field in refs: model = get_model(model_name) for obj in model.objects.filter(**{field: ap.id}): setattr(obj, field, self) obj.save() # Finally delete aliases platform ap.delete() @classmethod def can_set_label(cls, label): return Label.get_effective_setting(label, setting="enable_platform") @classmethod def iter_lazy_labels(cls, platform: "Platform"): yield f"noc::platform::{platform.name}::="
class MeasurementUnits(Document): meta = { "collection": "measurementunits", "strict": False, "auto_create_index": False, "json_collection": "pm.measurementunits", "json_unique_fields": ["name"], } # Unique units name name = StringField(unique=True) # Global ID uuid = UUIDField(binary=True) # Optional description description = StringField() # Short label label = StringField() # Label for dashboards dashboard_label = StringField() # Type of scale (K/M/G prefixes) # * d - decimal scale, 1/1_000/1_000_000/... # * b - binary scale, 1/2^10/2^20/... scale_type = StringField(choices=["d", "b"], default="d") # Alternative units alt_units = ListField(EmbeddedDocumentField(AltUnit)) # Enumerations enum = ListField(EmbeddedDocumentField(EnumValue)) _id_cache = cachetools.TTLCache(maxsize=100, ttl=60) _name_cache = cachetools.TTLCache(maxsize=100, ttl=60) def __str__(self): return self.name @classmethod @cachetools.cachedmethod(operator.attrgetter("_id_cache"), lock=lambda _: id_lock) def get_by_id(cls, id) -> Optional["MeasurementUnits"]: return MeasurementUnits.objects.filter(id=id).first() @classmethod @cachetools.cachedmethod(operator.attrgetter("_name_cache"), lock=lambda _: id_lock) def get_by_name(cls, name: str) -> Optional["MeasurementUnits"]: return MeasurementUnits.objects.filter(name=name).first() @property def json_data(self): r = { "name": self.name, "$collection": self._meta["json_collection"], "uuid": self.uuid, "label": self.label, "dashboard_label": self.dashboard_label, "scale_type": self.scale_type, } if self.description: r["description"] = self.description if self.alt_units: r["alt_units"] = [x.json_data for x in self.alt_units] if self.enum: r["enum"] = [x.json_data for x in self.enum] return r def to_json(self): return to_json( self.json_data, order=[ "name", "$collection", "uuid", "description", "label", "dashboard_label", "scale_type", "alt_units", "enum", ], ) def get_json_path(self): return "%s.json" % quote_safe_path(self.name)