class ProcessedResult(mongoengine.Document):
added_date = mongoengine.DateTimeField(default=datetime.now)
delphin = mongoengine.GenericReferenceField(required=True)
results_raw = mongoengine.GenericReferenceField(required=True)
meta = collections.processed_result_db
class NotificationOverride(me.EmbeddedDocument):
'''
Represents a single notification override.
'''
id = me.StringField(primary_key=True, default=lambda: uuid4().hex)
source = me.StringField(max_length=64, required=True,
default="") # eg "alerts"
channel = me.StringField(max_length=64, required=False,
default="") # eg "InAppNotification"
machine = me.GenericReferenceField(required=False)
tag = me.GenericReferenceField(required=False)
cloud = me.GenericReferenceField(required=False)
value = me.StringField(max_length=7,
required=True,
choices=('ALLOW', 'BLOCK'),
default='BLOCK')
def matches_notification(self, notification):
if self.machine and notification.machine:
if self.machine != notification.machine:
return False
if self.tag and notification.tag:
if self.tag != notification.tag:
return False
if self.cloud and notification.cloud:
if self.cloud != notification.cloud:
return False
return self.source == type(notification).__name__
class Result(mongoengine.Document):
added_date = mongoengine.DateTimeField(default=datetime.now)
delphin = mongoengine.GenericReferenceField(required=True)
results_processed = mongoengine.GenericReferenceField()
log = mongoengine.DictField(required=True)
results = mongoengine.DictField(required=True)
geometry_file = mongoengine.DictField(required=True)
geometry_file_hash = mongoengine.IntField(required=True)
simulation_started = mongoengine.DateTimeField(required=True)
meta = collections.raw_result_db
class Comment(mongoengine.Document):
account = mongoengine.ReferenceField("User", required=True)
subject = mongoengine.GenericReferenceField(required=True)
created = mongoengine.DateTimeField(default=datetime.now)
updated = mongoengine.DateTimeField(default=datetime.now)
text = mongoengine.StringField(required=True)
counter = mongoengine.SequenceField()
meta = {
"alias": "default",
"collection": "comments",
"indexes": [
"subject",
"account",
]
}
def dict(self):
return {
"created": int(datetime.timestamp(self.created)),
"updated": int(datetime.timestamp(self.updated)),
"account": self.account.dict(),
"counter": self.counter,
"text": self.text
}
class Status(mongoengine.Document):
account = mongoengine.ReferenceField("User", required=True)
subject = mongoengine.GenericReferenceField(required=True)
created = mongoengine.DateTimeField(default=datetime.utcnow)
updated = mongoengine.DateTimeField(default=datetime.utcnow)
rating = mongoengine.IntField(min_value=1, max_value=10)
content = mongoengine.IntField(required=True)
position = mongoengine.IntField(default=None)
status = mongoengine.IntField(default=None)
rewatch = mongoengine.IntField(default=0)
time = mongoengine.IntField(default=0)
meta = {
"alias": "default",
"collection": "statuses",
"indexes": [
"subject",
"account",
"rating",
"status",
],
"ordering": ["-created"]
}
def dict(self):
return {
"created": int(datetime.timestamp(self.created)),
"updated": int(datetime.timestamp(self.updated)),
"status": static.slug("statuses", self.status),
"subject": self.subject.dict(),
"position": self.position,
"rewatch": self.rewatch,
"rating": self.rating,
"time": self.time
}
class Log(mon.Document):
object = mon.GenericReferenceField()
action = mon.StringField()
data = mon.DictField()
timestamp = mon.DateTimeField(default=datetime.utcnow)
meta = {'indexes': [('object', 'timestamp')]}
@staticmethod
def add_entry(object, action, data):
return Log.create(object=object, action=action, data=data)
@staticmethod
def record_new(object):
return Log.add_entry(object, 'create', {})
@staticmethod
def record_update(object):
return Log.add_entry(object, 'update', {})
@staticmethod
def record_error(object, errtype, value):
return Log.add_entry(object, 'error', {
'type': errtype,
'value': value
})
@classmethod
def create(cls, **fields):
object = cls(**fields)
object.save()
class Reporter(mongoengine.Document):
meta = {'collection': 'test_reporter'}
id = mongoengine.StringField(primary_key=True)
first_name = mongoengine.StringField(required=True)
last_name = mongoengine.StringField(required=True)
email = mongoengine.EmailField()
awards = mongoengine.ListField(mongoengine.StringField())
articles = mongoengine.ListField(mongoengine.ReferenceField(Article))
embedded_articles = mongoengine.ListField(
mongoengine.EmbeddedDocumentField(EmbeddedArticle))
embedded_list_articles = mongoengine.EmbeddedDocumentListField(
EmbeddedArticle)
id = mongoengine.StringField(primary_key=True)
first_name = mongoengine.StringField(required=True)
last_name = mongoengine.StringField(required=True)
email = mongoengine.EmailField()
awards = mongoengine.ListField(mongoengine.StringField())
articles = mongoengine.ListField(mongoengine.ReferenceField(Article))
embedded_articles = mongoengine.ListField(
mongoengine.EmbeddedDocumentField(EmbeddedArticle))
embedded_list_articles = mongoengine.EmbeddedDocumentListField(
EmbeddedArticle)
generic_reference = mongoengine.GenericReferenceField(choices=[
Article,
Editor,
])
class Vote(mongoengine.Document):
rating = mongoengine.IntField(min_value=1, max_value=10, required=True)
account = mongoengine.ReferenceField("User", required=True)
subject = mongoengine.GenericReferenceField(required=True)
created = mongoengine.DateTimeField(default=datetime.now)
updated = mongoengine.DateTimeField(default=datetime.now)
meta = {
"alias": "default",
"collection": "votes",
"indexes": [
"subject",
"account",
"rating",
]
}
def dict(self):
return {
"created": int(datetime.timestamp(self.created)),
"updated": int(datetime.timestamp(self.updated)),
"account": self.account.dict(),
"subject": self.subject.dict(),
"rating": self.rating
}
class Categories(mongo.Document):
id = mongo.UUIDField(required=False)
name = mongo.StringField(required=True, default='', unique=True)
privacy = mongo.StringField(required=True, default='')
description = mongo.StringField(required=True, default='')
taxonomies = mongo.StringField(required=False, default='')
custom_field_form = mongo.ReferenceField("FormSchema", required=False)
created = mongo.StringField(required=True, default=str(datetime.now()))
created_by = mongo.StringField(required=False)
updated = mongo.StringField(required=True, default=str(datetime.now()))
tagged = mongo.ListField(mongo.GenericReferenceField(), required=False)
meta = {'indexes': [{'fields': ('name', ), 'unique': True}]}
def add_object(self, obj):
self.tagged.append(obj)
self.save()
@property
def group(self):
return "Categories"
@property
def permission_type(self):
return "category"
@property
def number_using(self):
return len(self.tagged)
@classmethod
def objects_with(cls, name):
try:
return Categories.objects.get(name=name).tagged
except Categories.DoesNotExist:
return []
@classmethod
def for_object(cls, obj):
return Categories.objects(tagged__in=[obj])
@classmethod
def for_object_id(cls, objid, cls_name):
#kinda an ugly hack, but global should have most everything loaded
#if your getting a Keyerror, try importing the cls
cls = globals()[cls_name]
obj = cls.objects.get(id=objid)
return Categories.for_object(obj)
@classmethod
def remove_obj(cls, obj):
for cat in Categories.for_object(obj):
Categories.objects(id=str(cat.id)).update_one(pull__tagged=obj)
class Article(mongoengine.Document):
meta = {'collection': 'test_article'}
headline = mongoengine.StringField(required=True, help_text="The article headline.")
pub_date = mongoengine.DateTimeField(
default=datetime.now,
verbose_name="publication date",
help_text="The date of first press.")
editor = mongoengine.ReferenceField(Editor)
reporter = mongoengine.ReferenceField('Reporter')
# Will not convert this field cause no choices
generic_reference = mongoengine.GenericReferenceField()
class Like(BaseDocument):
user = mongoengine.ReferenceField('User')
likeable = mongoengine.GenericReferenceField('Paste')
@classmethod
def post_save(cls, sender, document, **kwargs):
if kwargs.get('created'):
field = document.likeable._cls.lower()
document.user.increase_counter('%s_likes' % field)
document.likeable.increase_counter('likes')
@classmethod
def post_delete(cls, sender, document, **kwargs):
field = document.likeable._cls.lower()
document.user.increase_counter('%s_likes' % field, -1)
document.likeable.increase_counter('likes', -1)
class Tag(me.Document):
owner = me.ReferenceField(Owner, required=True)
key = me.StringField(required=True)
resource_type = me.StringField()
value = me.StringField()
resource = me.GenericReferenceField()
resource_id = me.StringField()
meta = {
'indexes': [
'owner',
{
'fields': ['resource', 'key'],
'sparse': False,
'unique': False,
'cls': False,
},
],
}
class Tag(me.Document):
owner = me.ReferenceField(Owner, required=True)
key = me.StringField(regex=r'^[a-zA-Z0-9_]+(?:[ :.-][a-zA-Z0-9_]+)*$',
required=True)
resource_type = me.StringField(choices=['cloud', 'clouds', 'keys',
'scripts', 'machine', 'machines',
'template', 'stack', 'image',
'network', 'tunnel', 'schedules',
'networks', 'zones', 'records'])
value = me.StringField()
resource = me.GenericReferenceField()
meta = {
'indexes': [
'owner',
{
'fields': ['resource', 'key'],
'sparse': False,
'unique': True,
'cls': False,
},
],
}
def clean(self):
self.resource_type = self.resource._meta["collection"]
def __str__(self):
return 'Tag %s:%s for %s' % (self.key, self.value, self.resource)
def as_dict(self):
return {
'key': self.key,
'value': self.value,
'owner': self.owner.id,
'resource_type': self.resource_type,
'resource': str(self.resource),
}
class DocOnlyA(me.Document):
id = me.StringField(primary_key=True, default='main')
generic = me.GenericReferenceField(choices=[SubDocA])
class Doc(me.Document):
id = me.StringField(primary_key=True, default='main')
generic = me.GenericReferenceField()
class Notification(me.Document):
'''
Represents a notification associated with a
user-organization pair
'''
meta = {'allow_inheritance': True}
id = me.StringField(primary_key=True, default=lambda: uuid4().hex)
created_date = me.DateTimeField(required=False)
expiry_date = me.DateTimeField(required=False)
user = me.ReferenceField(User, required=True)
organization = me.ReferenceField(Organization, required=True)
# content fields
summary = me.StringField(max_length=512, required=False, default="")
body = me.StringField(required=True, default="")
html_body = me.StringField(required=False, default="")
# taxonomy fields
source = me.StringField(max_length=64, required=True, default="")
machine = me.GenericReferenceField(required=False)
tag = me.GenericReferenceField(required=False)
cloud = me.GenericReferenceField(required=False)
action_link = me.URLField(required=False)
unique = me.BooleanField(required=True, default=True)
severity = me.StringField(max_length=7,
required=True,
choices=('LOW', 'DEFAULT', 'HIGH'),
default='DEFAULT')
feedback = me.StringField(max_length=8,
required=True,
choices=('NEGATIVE', 'NEUTRAL', 'POSITIVE'),
default='NEUTRAL')
def __init__(self, *args, **kwargs):
super(Notification, self).__init__(*args, **kwargs)
if not self.created_date:
self.created_date = datetime.now()
def update_from(self, notification):
self.created_date = notification.created_date
self.expiry_date = notification.expiry_date
self.user = notification.user
self.organization = notification.organization
self.summary = notification.summary
self.body = notification.body
self.html_body = notification.html_body
self.source = notification.source
self.machine = notification.machine
self.tag = notification.tag
self.cloud = notification.cloud
self.unique = notification.unique
self.action_link = notification.action_link
self.severity = notification.severity
self.feedback = notification.feedback
class MUserStory(mongo.Document):
"""
Stories read by the user. These are deleted as the mark_read_date for the
UserSubscription passes the UserStory date.
"""
user_id = mongo.IntField()
feed_id = mongo.IntField()
read_date = mongo.DateTimeField()
story_id = mongo.StringField()
story_date = mongo.DateTimeField()
story = mongo.ReferenceField(MStory, dbref=True)
found_story = mongo.GenericReferenceField()
shared = mongo.BooleanField()
meta = {
'collection':
'userstories',
'indexes': [
{
'fields': ('user_id', 'feed_id', 'story_id'),
'unique': True
},
('feed_id', 'story_id'), # Updating stories with new guids
('feed_id', 'story_date'), # Trimming feeds
('feed_id', '-read_date'), # Trimming feeds
],
'allow_inheritance':
False,
'index_drop_dups':
True,
'cascade':
False,
}
def save(self, *args, **kwargs):
self.sync_redis()
super(MUserStory, self).save(*args, **kwargs)
def delete(self, *args, **kwargs):
self.remove_from_redis()
super(MUserStory, self).delete(*args, **kwargs)
@property
def guid_hash(self):
return hashlib.sha1(self.story_id).hexdigest()[:6]
@property
def feed_guid_hash(self):
return "%s:%s" % (self.feed_id, self.guid_hash)
@classmethod
def delete_old_stories(cls, feed_id):
UNREAD_CUTOFF = datetime.datetime.utcnow() - datetime.timedelta(
days=settings.DAYS_OF_UNREAD * 5)
read_stories = cls.objects(feed_id=feed_id,
read_date__lte=UNREAD_CUTOFF)
read_stories_count = read_stories.count()
if read_stories_count:
feed = Feed.objects.get(pk=feed_id)
total = cls.objects(feed_id=feed_id).count()
logging.info(
" ---> ~SN~FCTrimming ~SB%s~SN/~SB%s~SN read stories from %s..."
% (read_stories_count, total, feed.title[:30]))
read_stories.delete()
@classmethod
def delete_marked_as_read_stories(cls,
user_id,
feed_id,
mark_read_date=None):
if not mark_read_date:
usersub = UserSubscription.objects.get(user__pk=user_id,
feed__pk=feed_id)
mark_read_date = usersub.mark_read_date
# Next line forces only old read stories to be removed, just in case newer stories
# come in as unread because they're being shared.
mark_read_date = datetime.datetime.utcnow() - datetime.timedelta(
days=settings.DAYS_OF_UNREAD)
cls.objects(user_id=user_id,
feed_id=feed_id,
read_date__lte=mark_read_date).delete()
@property
def story_db_id(self):
if self.story:
return self.story.id
elif self.found_story:
if '_ref' in self.found_story:
return self.found_story['_ref'].id
elif hasattr(self.found_story, 'id'):
return self.found_story.id
story, found_original = MStory.find_story(self.feed_id, self.story_id)
if story:
if found_original:
self.story = story
else:
self.found_story = story
self.save()
return story.id
def sync_redis(self, r=None, pipeline=None, hashpipe=None):
if pipeline:
r = pipeline
if hashpipe:
h = pipeline
elif not r:
r = redis.Redis(connection_pool=settings.REDIS_STORY_POOL)
h = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL)
if self.story_db_id:
all_read_stories_key = 'RS:%s' % (self.user_id)
r.sadd(all_read_stories_key, self.story_db_id)
r.expire(all_read_stories_key,
settings.DAYS_OF_UNREAD * 24 * 60 * 60)
h.sadd(all_read_stories_key, self.feed_guid_hash)
h.expire(all_read_stories_key,
settings.DAYS_OF_UNREAD * 24 * 60 * 60)
read_story_key = 'RS:%s:%s' % (self.user_id, self.feed_id)
r.sadd(read_story_key, self.story_db_id)
r.expire(read_story_key, settings.DAYS_OF_UNREAD * 24 * 60 * 60)
h.sadd(read_story_key, self.feed_guid_hash)
h.expire(read_story_key, settings.DAYS_OF_UNREAD * 24 * 60 * 60)
def remove_from_redis(self):
r = redis.Redis(connection_pool=settings.REDIS_STORY_POOL)
h = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL)
if self.story_db_id:
r.srem('RS:%s' % self.user_id, self.story_db_id)
r.srem('RS:%s:%s' % (self.user_id, self.feed_id), self.story_db_id)
h.srem('RS:%s' % self.user_id, self.feed_guid_hash)
h.srem('RS:%s:%s' % (self.user_id, self.feed_id),
self.feed_guid_hash)
@classmethod
def sync_all_redis(cls, user_id=None, feed_id=None, force=False):
r = redis.Redis(connection_pool=settings.REDIS_STORY_POOL)
h = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL)
UNREAD_CUTOFF = datetime.datetime.utcnow() - datetime.timedelta(
days=settings.DAYS_OF_UNREAD + 1)
if feed_id and user_id:
read_stories = cls.objects.filter(user_id=user_id,
feed_id=feed_id,
read_date__gte=UNREAD_CUTOFF)
key = "RS:%s:%s" % (user_id, feed_id)
r.delete(key)
h.delete(key)
elif feed_id:
read_stories = cls.objects.filter(feed_id=feed_id,
read_date__gte=UNREAD_CUTOFF)
keys = r.keys("RS:*:%s" % feed_id)
print " ---> Deleting %s redis keys: %s" % (len(keys), keys)
for key in keys:
r.delete(key)
keys = h.keys("RS:*:%s" % feed_id)
print " ---> Deleting %s redis keys: %s" % (len(keys), keys)
for key in keys:
h.delete(key)
elif user_id:
read_stories = cls.objects.filter(user_id=user_id,
read_date__gte=UNREAD_CUTOFF)
keys = r.keys("RS:%s:*" % user_id)
r.delete("RS:%s" % user_id)
print " ---> Deleting %s redis keys: %s" % (len(keys), keys)
for key in keys:
r.delete(key)
keys = h.keys("RS:%s:*" % user_id)
h.delete("RS:%s" % user_id)
print " ---> Deleting %s redis keys: %s" % (len(keys), keys)
for key in keys:
h.delete(key)
elif force:
read_stories = cls.objects.all(read_date__gte=UNREAD_CUTOFF)
else:
raise "Specify user_id, feed_id, or force."
total = read_stories.count()
logging.debug(" ---> ~SN~FMSyncing ~SB%s~SN stories (%s/%s)" %
(total, user_id, feed_id))
pipeline = None
hashpipe = None
for i, read_story in enumerate(read_stories):
if not pipeline:
pipeline = r.pipeline()
hashpipe = h.pipeline()
if (i + 1) % 1000 == 0:
print " ---> %s/%s" % (i + 1, total)
pipeline.execute()
hashpipe.execute()
pipeline = r.pipeline()
hashpipe = h.pipeline()
read_story.sync_redis(r, pipeline=pipeline, hashpipe=hashpipe)
if pipeline:
pipeline.execute()
hashpipe.execute()
class Link(db.Document):
"""
Link between documents, eg: Objective<->Competency
"""
documents = db.ListField(db.GenericReferenceField(), default=[])
owner = db.ReferenceField('User')
class MUserStory(mongo.Document):
"""
Stories read by the user. These are deleted as the mark_read_date for the
UserSubscription passes the UserStory date.
"""
user_id = mongo.IntField()
feed_id = mongo.IntField()
read_date = mongo.DateTimeField()
story_id = mongo.StringField()
story_date = mongo.DateTimeField()
story = mongo.ReferenceField(MStory, dbref=True)
found_story = mongo.GenericReferenceField()
meta = {
'collection':
'userstories',
'indexes': [
{
'fields': ('user_id', 'feed_id', 'story_id'),
'unique': True
},
('feed_id', 'story_id'), # Updating stories with new guids
('feed_id', 'story_date'), # Trimming feeds
],
'allow_inheritance':
False,
'index_drop_dups':
True,
'cascade':
False,
}
def save(self, *args, **kwargs):
self.sync_redis()
super(MUserStory, self).save(*args, **kwargs)
def delete(self, *args, **kwargs):
self.remove_from_redis()
super(MUserStory, self).delete(*args, **kwargs)
@property
def guid_hash(self):
return hashlib.sha1(self.story_id).hexdigest()
@classmethod
def delete_old_stories(cls, feed_id):
UNREAD_CUTOFF = datetime.datetime.utcnow() - datetime.timedelta(
days=settings.DAYS_OF_UNREAD)
cls.objects(feed_id=feed_id, story_date__lte=UNREAD_CUTOFF).delete()
@classmethod
def delete_marked_as_read_stories(cls,
user_id,
feed_id,
mark_read_date=None):
if not mark_read_date:
usersub = UserSubscription.objects.get(user__pk=user_id,
feed__pk=feed_id)
mark_read_date = usersub.mark_read_date
# Next line forces only old read stories to be removed, just in case newer stories
# come in as unread because they're being shared.
mark_read_date = datetime.datetime.utcnow() - datetime.timedelta(
days=settings.DAYS_OF_UNREAD)
cls.objects(user_id=user_id,
feed_id=feed_id,
read_date__lte=mark_read_date).delete()
@property
def story_db_id(self):
if self.story:
return self.story.id
elif self.found_story:
if '_ref' in self.found_story:
return self.found_story['_ref'].id
elif hasattr(self.found_story, 'id'):
return self.found_story.id
story, found_original = MStory.find_story(self.feed_id, self.story_id)
if story:
if found_original:
self.story = story
else:
self.found_story = story
self.save()
return story.id
def sync_redis(self, r=None):
if not r:
r = redis.Redis(connection_pool=settings.REDIS_STORY_POOL)
if self.story_db_id:
all_read_stories_key = 'RS:%s' % (self.user_id)
r.sadd(all_read_stories_key, self.story_db_id)
read_story_key = 'RS:%s:%s' % (self.user_id, self.feed_id)
r.sadd(read_story_key, self.story_db_id)
r.expire(read_story_key, settings.DAYS_OF_UNREAD * 24 * 60 * 60)
def remove_from_redis(self):
r = redis.Redis(connection_pool=settings.REDIS_STORY_POOL)
if self.story_db_id:
r.srem('RS:%s' % self.user_id, self.story_db_id)
r.srem('RS:%s:%s' % (self.user_id, self.feed_id), self.story_db_id)
@classmethod
def sync_all_redis(cls):
read_stories = cls.objects.all()
for read_story in read_stories:
read_story.sync_redis()
