class Reaction(EmbeddedDocument):
pathways = ListField(StringField(required=True))
name = StringField(required=True)
description = StringField()
substrates = ListField(EmbeddedDocumentField(ChemSpecie))
products = ListField(EmbeddedDocumentField(ChemSpecie))
chockepoint = EmbeddedDocumentField(ChockePoint)
def product(self, **kwargs):
filtered = [
x for x in self.products
if all([x[k] == v for k, v in kwargs.items()])
]
if filtered:
return filtered[0]
return None
def substrate(self, **kwargs):
filtered = [
x for x in self.substrates
if all([x[k] == v for k, v in kwargs.items()])
]
if filtered:
return filtered[0]
return None
class Chats(Document):
"""
Chat document saves all words + language settings of a chat.
Most used word object is save here too
"""
language = StringField()
ID = IntField()
settings = EmbeddedDocumentField(Settings)
words = EmbeddedDocumentListField(Word)
topWord = EmbeddedDocumentField(Word)
sentWords = IntField()
sentMessages = IntField()
responses = DictField(default={})
@staticmethod
def getChat(chatID):
"""Will return a chat object when ID is provided"""
try:
return Chats.objects(ID=chatID).get()
except Chats.DoesNotExist:
return None
@staticmethod
def getSettings(chatID):
"""Will return a settings object to simplify querring the db"""
settingsObjects = Chats.objects(ID=chatID).get().settings
return settingsObjects
class Movie(Document):
meta = {'collection': 'movies'}
title = StringField()
type = StringField()
year = StringField()
director = EmbeddedDocumentField(Person)
actors = ListField(EmbeddedDocumentField(Person))
class Title(Document):
endpoint = StringField(max_length=120)
charset = StringField()
page_title = StringField()
primary_meta_tags = ListField(EmbeddedDocumentField(Primary_tags))
facebook_og_tags = ListField(EmbeddedDocumentField(Facebook_Og_Tags))
twitter_tags = ListField(EmbeddedDocumentField(Twitter_tags))
class ModelVersion(SmartDoc):
PK_FIELDS = ProtoModelVersion.PK_FIELDS
FILE_NAME = ProtoModelVersion.FILE_NAME
EMBEDDED_SCHEMA = EmbeddedModelVersion
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
name = StringField(required=True, max_length=DBConst.MAX_NAME_LEN)
model = ReferenceField(Model, required=True, reverse_delete_rule=CASCADE)
train = EmbeddedDocumentField(EmbeddedTraining, default=None)
ds = EmbeddedDocumentField(EmbeddedDataset, default=None)
compressed = BooleanField(default=False)
details = DictField(default={})
pretrained = EmbeddedDocumentField(EmbeddedModelVersion, default=None)
lightweight = BooleanField(default=False)
def to_embedded(self):
emb: EmbeddedModelVersion = super().to_embedded()
if isinstance(self.pretrained, EmbeddedModelVersion):
emb.pretrained = self.pretrained.show()
return emb
class Analytic(Document):
"""
:param str name: The name of the analytic. All characters are valid
:param List[TechniqueMapping] coverage: A list of all of technique, tactic pairs to be detected
:param bool enabled: Enabled/disabled status of the analytic
"""
# define some constants
FIRST_PASS = '******'
SECOND_PASS = '******'
REALTIME = 'realtime' # not-implemented
IDENTIFY_ONLY = 'identify-only'
IDENTIFY_ONLY_NO_BASELINE = 'identify-only-no-baseline'
modes = [FIRST_PASS, SECOND_PASS, REALTIME, IDENTIFY_ONLY, IDENTIFY_ONLY_NO_BASELINE]
meta = {'allow_inheritance': True}
coverage = ListField(EmbeddedDocumentField(TechniqueMapping))
description = StringField()
enabled = BooleanField(default=True)
name = StringField(required=True, unique=True)
mapped_events = ListField(EmbeddedDocumentField(ObjectMapping))
mode = StringField(choices=modes)
@property
def baseline(self):
return AnalyticBaseline.objects(analytic=self).first()
def extract_events(self, result):
raise NotImplementedError()
def __repr__(self):
return '{}(name={})'.format(type(self).__name__, repr(self.name))
__str__ = __repr__
class Excursión(Document):
nombre = StringField(max_length=120, required=True)
descripción = StringField(required=True)
likes = IntField(default=0)
visitas = IntField(default=0)
tags = ListField(StringField(max_length=20))
comentarios = ListField(EmbeddedDocumentField(Comentarios))
fotos = ListField(EmbeddedDocumentField(Fotos))
class Group(Document):
meta = {"collection": "group"}
name = StringField(primary_key=True, required=True)
bio = StringField()
members = ListField(ReferenceField("User"), default=list)
group_portfolio = EmbeddedDocumentField("Portfolio")
chat = ListField(EmbeddedDocumentField("Comment"), default=list)
metrics = EmbeddedDocumentField("GroupMetrics", dbref=True)
class ProfileResultDO(EmbeddedDocument):
"""
Profiling result plain object.
"""
# Static profile result
static_profile_result = EmbeddedDocumentField(StaticProfileResultDO)
# Dynamic profile result
dynamic_profile_results = ListField(EmbeddedDocumentField(DynamicProfileResultDO))
class Quiz(Document):
meta = {
"collection": "quiz",
"strict": False,
"auto_create_index": False,
"json_collection": "dev.quiz"
}
name = StringField(unique=True)
uuid = UUIDField(binary=True)
description = StringField()
revision = IntField(default=1)
changes = ListField(EmbeddedDocumentField(QuizChange))
# Information text before quiz
disclaimer = StringField()
# List of questions
questions = ListField(EmbeddedDocumentField(QuizQuestion))
_id_cache = cachetools.TTLCache(maxsize=100, ttl=60)
_name_cache = cachetools.TTLCache(maxsize=100, ttl=60)
def __unicode__(self):
return self.name
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_id_cache"),
lock=lambda _: id_lock)
def get_by_id(cls, id):
return Quiz.objects.filter(id=id).first()
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_name_cache"),
lock=lambda _: id_lock)
def get_by_name(cls, name):
return Quiz.objects.filter(name=name).first()
@property
def json_data(self):
return {
"name": self.name,
"$collection": self._meta["json_collection"],
"uuid": str(self.uuid),
"description": self.description,
"revision": self.revision,
"disclaimer": self.disclaimer,
"changes": [c.json_data for c in self.changes],
"questions": [c.json_data for c in self.questions]
}
def to_json(self):
return to_json(self.json_data,
order=[
"name", "$collection", "uuid", "description",
"revision", "disclaimer", "changes", "questions"
])
def get_json_path(self):
return "%s.json" % quote_safe_path(self.name)
class Itinerarie(Document):
meta = {
'collection':
'flights.itinerarie',
'strict':
False,
'auto_create_index':
False,
'indexes': [
'_duration', 'default_price',
('source', 'destination', 'departure_time')
]
}
source = StringField(required=True)
destination = StringField(required=True)
departure_time = DateTimeField(required=True)
arrival_time = DateTimeField(required=True)
_duration = IntField()
default_price = FloatField()
flights = ListField(EmbeddedDocumentField(Flight))
prices = ListField(EmbeddedDocumentField(Price))
@queryset_manager
def objects(cls, queryset):
return queryset.order_by('default_price', '_duration')
def __str__(self):
return 'From {} to {} {} by {}'.format(self.source, self.destination,
self.duration,
self.default_price)
def clean(self):
self._duration = (self.arrival_time -
self.departure_time).total_seconds()
price_for_adult = next(
filter(lambda x: x.type == PT_ADULT, self.prices))
self.default_price = price_for_adult.amount if price_for_adult else 0
@property
def duration(self):
return humanize_date(self._duration)
@property
def json_data(self):
return {
'source': self.source,
'destination': self.destination,
'departure_time': self.departure_time,
'arrival_time': self.arrival_time,
'duration': self.duration,
'flights': [_.to_json() for _ in self.flights],
'prices': [_.to_json() for _ in self.prices],
'price': self.default_price
}
def to_json(self):
return self.json_data
class Intent(Document):
name = StringField(max_length=100, required=True, unique=True)
intentId = StringField(required=True)
apiTrigger = BooleanField(required=True)
apiDetails = EmbeddedDocumentField(ApiDetails)
speechResponse = StringField(required=True)
parameters = ListField(EmbeddedDocumentField(Parameter))
labeledSentences = EmbeddedDocumentListField(LabeledSentences)
trainingData = ListField(required=False)
class ValidationRule(Document):
meta = {
"collection": "noc.validationrules",
"strict": False,
"auto_create_index": False
}
name = StringField(unique=True)
is_active = BooleanField(default=True)
description = StringField()
selectors_list = ListField(EmbeddedDocumentField(SelectorItem))
objects_list = ListField(EmbeddedDocumentField(ObjectItem))
handler = StringField()
config = DictField()
def __unicode__(self):
return self.name
def is_applicable_for(self, object):
"""
Check rule is applicable for managed object
"""
if self.objects_list:
actions = set(i.action for i in self.objects_list
if i.object == object)
if A_DISABLE not in actions:
if A_EXCLUDE in actions:
return False
elif A_INCLUDE in actions:
return True
if self.selectors_list:
actions = set(s.action for s in self.selectors_list
if object in s.selector)
if A_DISABLE not in actions:
if A_EXCLUDE in actions:
return False
elif A_INCLUDE in actions:
return True
return not self.selectors_list and not self.objects_list
def get_handler(self):
if self.handler:
return get_handler(self.handler)
else:
return None
@classmethod
def on_delete(cls, sender, document, **kwargs):
from validationpolicy import ValidationPolicy
logger.info("Deleting rule %s", document.name)
for vp in ValidationPolicy.objects.filter(rules__rule=document):
logger.info("Removing rule %s from policy %s", document.name,
vp.name)
vp.rules = [r for r in vp.rules if r.rule.id != document.id]
vp.save()
class Procedure(BDocument):
name = StringField(required=True, unique=True)
code = StringField(required=True, unique=True)
price = FloatField(default=0)
is_refundable = BooleanField()
average_duration = IntField()
length_of_hospitalization = IntField()
NGAP_code = EmbeddedDocumentField(NGAPCode)
CCAM_code = EmbeddedDocumentField(CCAMCode)
modality = ReferenceField(Modality)
tnr = FloatField(default=0)
class User(Document):
username = StringField(required = True, unique = True)
email = StringField(required = True, unique = True)
password = StringField(required = True)
watchList = SortedListField(EmbeddedDocumentField(Stock), ordering = 'symbol')
balence = FloatField(default = 10000.00)
ownedStock = SortedListField(EmbeddedDocumentField(OwnedStock), ordering = 'symbol')
totalStockValue = FloatField(default = 0.00)
meta = {
'ordering': ['+username']
}
class Variant(Document):
organism = StringField(required=True)
contig = StringField(required=True)
pos = IntField(required=True)
gene = StringField(required=False)
prot_ref = ObjectIdField()
ref = StringField(required=False)
sample_alleles = ListField(EmbeddedDocumentField(Allele))
ontologies = ListField(StringField())
search = EmbeddedDocumentField(ProteinDruggabilitySearch, default=None)
class User(Document):
schoolName = StringField(required=True)
forename = StringField(required=True)
surname = StringField(required=True)
email = EmailField(required=True)
passwordHash = StringField(required=True)
# Embedded Documents
schoolAdmin = EmbeddedDocumentField(SchoolAdmin)
staff = EmbeddedDocumentField(Staff)
teacher = EmbeddedDocumentField(Teacher)
student = EmbeddedDocumentField(Student)
class Protein(Sequence):
meta = {'allow_inheritance': True, 'collection': "proteins",
'index_cls': False,
'indexes': [
{"fields": ["organism", "name"]},
{"fields": ["organism", "gene"]},
{"fields": ["seq_collection_id", "keywords"]},
{"fields": ["organism", "keywords"]},
{"fields": ["organism", "alias"]},
'keywords'
]}
organism = StringField()
gene = ListField(StringField(), required=True)
gene_id = ObjectIdField()
seq_collection_id = ReferenceField(SeqCollection)
reactions = ListField(EmbeddedDocumentField(Reaction), default=[])
reactome = ListField(DictField(), default=[])
druggability = FloatField()
tregulation = DictField(required=False)
alias = ListField(StringField(), default=[])
chembl = EmbeddedDocumentField(ChEMBL)
def __init__(self, **kwargs):
'''
'''
super(Sequence, self).__init__(**kwargs)
self._seq_init()
self.size.unit = "aa"
def add_domain(self, name, start, end, source="unknown"):
dn_feature = Feature(source=source, identifier=name, type=SO_TERMS["polypeptide_domain"],
location=Location(base=self.id, start=start, end=end))
self.features.append(dn_feature)
def domains(self):
return [feature for feature in self.features if feature.type == SO_TERMS["polypeptide_domain"]]
def domain_seqs(self):
return self.seqs_from_features(type=SO_TERMS["polypeptide_domain"])
def reaction(self, **kwargs):
filtered_reactions = [x for x in self.reactions if all([x[k] == v for k, v in kwargs.items()])]
if filtered_reactions:
return filtered_reactions[0]
return None
def __str__(self):
return self.name + " " + str(len(self.seq)) + "aa"
def homologous(self):
return [feature for feature in self.features if feature.type == SO_TERMS["homologous"]]
class Post(Document):
title = StringField(max_length=120, required=True)
description = StringField(max_length=120, required=False)
author = ReferenceField(User)
editor = ReferenceField(User)
tags = ListField(StringField(max_length=30))
user_lists = ListField(ReferenceField(User))
sections = ListField(EmbeddedDocumentField(Content))
content = EmbeddedDocumentField(Content)
is_published = BooleanField()
def primary_user(self):
return self.user_lists[0] if self.user_lists else None
class User(Document):
meta = {"collection": "user"}
name = StringField(required=True)
bio = StringField()
email = StringField(primary_key=True, required=True)
password = StringField(required=True)
profile_pic = StringField()
date_joined = DateTimeField(default=datetime.now)
metrics = EmbeddedDocumentField("UserMetrics", dbref=True)
achievements = ListField(ReferenceField("Achievement"), default=list)
personal_portfolio = EmbeddedDocumentField("Portfolio", default=Portfolio)
groups = ListField(ReferenceField("Group"), default=list)
settings = EmbeddedDocumentField("Settings")
class Allele(EmbeddedDocument):
_id = ObjectIdField()
samples = ListField(EmbeddedDocumentField(SampleAllele), default=[])
alt = StringField(required=True)
variant_type = ListField(StringField(), default=[])
aa_ref = StringField(required=False)
aa_alt = StringField(required=False)
aa_pos = IntField(required=False)
feature_ref = ObjectIdField(required=False)
feature = EmbeddedDocumentField(Feature, required=False)
class Session(Document):
__sessions = {}
domain = StringField()
range = EmbeddedDocumentField(DateRange, required=True)
name = StringField(required=True)
state = EmbeddedDocumentField(SessionState)
def __init__(self, *args, **kwargs):
super(Session, self).__init__(*args, **kwargs)
self._queue = None
def query_context(self, user):
return QueryContext(self, user)
@property
def queue(self):
""":rtype SessionStream """
if self._queue is None:
self._queue = SessionStream.get_queue(self)
return self._queue
def get_clusters(self):
events = list(DataModelEvent.objects(sessions=self).no_dereference())
results = list(AnalyticResult.objects(session=self).no_dereference())
event_keys = set(_.id for _ in events)
def get_neighbors(node):
neighbors = []
if isinstance(node, AnalyticResult):
neighbors.extend(event for event in node.events
if event.id in event_keys)
elif isinstance(node, DataModelEvent):
# TODO: Error check to handle for events outside of current session
neighbors.extend(event for event in node.links
if event.id in event_keys)
neighbors.extend(event for event in node.reverse_links
if event.id in event_keys)
return neighbors
uptree = DisjointSet(events + results, get_neighbors)
clusters = []
for cluster in uptree.clusters():
new_cluster = {'events': [], 'results': []}
for item in cluster:
if isinstance(item, AnalyticResult):
new_cluster['results'].append(item)
elif isinstance(item, DataModelEvent):
new_cluster['events'].append(item)
clusters.append(new_cluster)
return clusters
class Storage(Document):
meta = {"collection": "noc.pm.storages"}
ALL = "all"
PRIORITY = "pri"
ROUNDROBIN = "rr"
RANDOM = "rnd"
name = StringField(unique=True)
description = StringField(required=False)
select_policy = StringField(choices=[(ALL, "All"), (PRIORITY, "Priority"),
(ROUNDROBIN, "Round-Robin"),
(RANDOM, "Random")],
default=PRIORITY)
write_concern = IntField(default=1)
collectors = ListField(EmbeddedDocumentField(CollectorProtocol))
access = ListField(EmbeddedDocumentField(AccessProtocol))
def __unicode__(self):
return self.name
@property
def default_collector(self):
"""
Returns URL of first active
collector. Returns None if no default collector set
"""
selectable = [
c for c in self.collectors if c.is_active and c.is_selectable
]
if not selectable:
return None
if len(selectable) <= self.write_concern:
policy = self.ALL
wc = len(selectable)
else:
policy = self.select_policy
wc = self.write_concern
return {
"policy":
policy,
"write_concern":
wc,
"collectors": [{
"proto": c.protocol,
"address": c.address,
"port": c.port
} for c in selectable]
}
class ObjectValidationPolicy(Document):
meta = {
"collection": "objectvalidationpolicies",
"strict": False,
"auto_create_index": False
}
name = StringField(unique=True)
description = StringField()
filter_query = PlainReferenceField(ConfDBQuery)
rules = ListField(EmbeddedDocumentField(ObjectValidationRule))
_id_cache = cachetools.TTLCache(maxsize=100, ttl=60)
def __str__(self):
return self.name
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_id_cache"),
lock=lambda _: id_lock)
def get_by_id(cls, id):
return ObjectValidationPolicy.objects.filter(id=id).first()
def iter_problems(self, engine):
"""
Check rules against ConfDB engine
:param engine: ConfDB Engine instance
:return: List of problems
"""
# Check filter query, if any
if self.filter_query:
if not self.filter_query.any(engine):
raise StopIteration
# Process rules
for rule in self.rules:
if not rule.is_active:
continue
if rule.filter_query:
if not rule.filter_query.any(engine):
continue
for ctx in rule.query.query(engine, **rule.query_params):
if "error" in ctx:
tpl = Template(rule.error_text_template)
path = []
if rule.error_code:
path += [rule.error_code]
problem = {
"alarm_class":
rule.alarm_class.name if rule.alarm_class else None,
"path":
path,
"message":
tpl.render(ctx),
"code":
rule.error_code or None,
}
yield problem
if rule.is_fatal:
raise StopIteration
class DashboardLayout(Document):
meta = {
"collection": "noc.dashboardlayouts",
"strict": False,
"auto_create_index": False,
"json_collection": "bi.dashboardlayouts",
}
name = StringField()
uuid = UUIDField(binary=True)
description = StringField()
# @todo: Add preview
cells = ListField(EmbeddedDocumentField(DashboardCell))
def __str__(self):
return self.name
def to_json(self):
return to_json(
{
"name": self.name,
"$collection": self._meta["json_collection"],
"uuid": self.uuid,
"description": self.description,
"cells": [s.to_json() for s in self.cells],
},
order=["name", "uuid", "description", "cells"],
)
def get_json_path(self):
return "%s.json" % self.name
class Contest(DynamicDocument):
created = DateTimeField()
updated = DateTimeField()
source = StringField(required=True, help_text="Name of data source (preferably from datasource.py). NOTE: this could be a single file among many for a given state, if results are split into different files by reporting level")
election_id = StringField(required=True, help_text="election id, e.g. md-2012-11-06-general")
slug = StringField(required=True, help_text="Slugified office name, plus district and party if relevant")
state = StringField(required=True, choices=STATE_POSTALS)
start_date = DateTimeField(required=True)
end_date = DateTimeField(required=True)
election_type = StringField(help_text="general, primary, etc. from OpenElex metadata")
result_type = StringField(required=True)
special = BooleanField(default=False)
raw_office = StringField(required=True)
raw_district = StringField()
raw_party = StringField(help_text="This should only be assigned for closed primaries, where voters must be registered in party to vote in the contest")
# FIELDS FOR TRANSFORMED/LINKED DATA
office = EmbeddedDocumentField(Office)
district = StringField()
party = StringField(help_text="This should only be assigned for closed primaries, where voters must be registered in party to vote in the contest")
def __unicode__(self):
return u'%s-%s' % self.key
@property
def key(self):
return (self.election_id, self.slug)
class Comment(Document, DocumentHelperMixin):
TYPE_COMMENT = 1
TYPE_INSIGHT = 10
TYPE_ANALYSIS = 20
TYPE_SYNTHESIS = 30
VISIBILITY_PUBLIC = 1
VISIBILITY_GROUP = 10
VISIBILITY_PRIVATE = 20
nature = IntField(default=TYPE_COMMENT)
visibility = IntField(default=VISIBILITY_PUBLIC)
is_synthesis = BooleanField()
is_analysis = BooleanField()
content = StringField()
feedback = EmbeddedDocumentField(FeedbackDocument)
# We don't comment reads. We comment articles.
#read = ReferenceField('Read')
article = ReferenceField('Article', reverse_delete_rule=CASCADE)
# Thus, we must store
user = ReferenceField('User', reverse_delete_rule=CASCADE)
in_reply_to = ReferenceField('Comment', reverse_delete_rule=NULLIFY)
class ProbeConfigMetric(EmbeddedDocument):
metric = StringField()
metric_type = StringField()
thresholds = ListField()
convert = StringField()
scale = FloatField(default=1.0)
collectors = EmbeddedDocumentField(MetricCollectors)
class VPN(Document):
meta = {
"collection": "vpns",
"strict": False,
"auto_create_index": False
}
name = StringField(unique=True)
profile = PlainReferenceField(VPNProfile)
description = StringField()
state = PlainReferenceField(State)
# Link to parent overlay
parent = PlainReferenceField("self")
project = ForeignKeyField(Project)
route_target = ListField(EmbeddedDocumentField(RouteTargetItem))
tags = ListField(StringField())
# Integration with external NRI and TT systems
# Reference to remote system object has been imported from
remote_system = PlainReferenceField(RemoteSystem)
# Object id in remote system
remote_id = StringField()
# Object id in BI
bi_id = LongField(unique=True)
# @todo: last_seen
# @todo: expired
_id_cache = cachetools.TTLCache(maxsize=100, ttl=60)
_bi_id_cache = cachetools.TTLCache(maxsize=100, ttl=60)
def __unicode__(self):
return self.name
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_id_cache"), lock=lambda _: id_lock)
def get_by_id(cls, id):
return VPN.objects.filter(id=id).first()
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_bi_id_cache"), lock=lambda _: id_lock)
def get_by_bi_id(cls, id):
return VPN.objects.filter(bi_id=id).first()
def clean(self):
super(VPN, self).clean()
if self.id and "parent" in self._changed_fields and self.has_loop:
raise ValidationError("Creating VPN loop")
@property
def has_loop(self):
"""
Check if object creates loop
"""
if not self.id:
return False
p = self.parent
while p:
if p.id == self.id:
return True
p = p.parent
return False
class CustomQueryJob(Job):
mode = StringField(default=Analytic.FIRST_PASS, choices=Analytic.modes)
session = ReferenceField(Session)
events = ListField(ReferenceField(DataModelEvent))
event_query = EmbeddedDocumentField(DataModelQuery)
def get_uuid_tuple(self):
return hash(lift_query(self.event_query)), self.mode, self.session.id
def get_query_context(self):
if self.mode == Analytic.SECOND_PASS:
return QueryContext(mongo_query_layer, session=self.session)
else:
return super(CustomQueryJob, self).get_query_context()
def run(self):
self.update_start()
query_context = self.get_query_context()
event_class = self.event_query.object
event_action = self.event_query.action
for result in query_context.query(self.event_query):
event = event_class.update_existing(action=event_action, **result)
self.update(add_to_set__events=event, inc__count=1)
self.add_to_session(self.session, event)
if self.mode in (Analytic.FIRST_PASS, Analytic.SECOND_PASS):
investigate_job = InvestigateJob.update_existing(
event=event, session=self.session, user=self.user)
investigate_job.submit()
