class ProcessedResult(mongoengine.Document): added_date = mongoengine.DateTimeField(default=datetime.now) delphin = mongoengine.GenericReferenceField(required=True) results_raw = mongoengine.GenericReferenceField(required=True) meta = collections.processed_result_db
class NotificationOverride(me.EmbeddedDocument): ''' Represents a single notification override. ''' id = me.StringField(primary_key=True, default=lambda: uuid4().hex) source = me.StringField(max_length=64, required=True, default="") # eg "alerts" channel = me.StringField(max_length=64, required=False, default="") # eg "InAppNotification" machine = me.GenericReferenceField(required=False) tag = me.GenericReferenceField(required=False) cloud = me.GenericReferenceField(required=False) value = me.StringField(max_length=7, required=True, choices=('ALLOW', 'BLOCK'), default='BLOCK') def matches_notification(self, notification): if self.machine and notification.machine: if self.machine != notification.machine: return False if self.tag and notification.tag: if self.tag != notification.tag: return False if self.cloud and notification.cloud: if self.cloud != notification.cloud: return False return self.source == type(notification).__name__
class Result(mongoengine.Document): added_date = mongoengine.DateTimeField(default=datetime.now) delphin = mongoengine.GenericReferenceField(required=True) results_processed = mongoengine.GenericReferenceField() log = mongoengine.DictField(required=True) results = mongoengine.DictField(required=True) geometry_file = mongoengine.DictField(required=True) geometry_file_hash = mongoengine.IntField(required=True) simulation_started = mongoengine.DateTimeField(required=True) meta = collections.raw_result_db
class Comment(mongoengine.Document): account = mongoengine.ReferenceField("User", required=True) subject = mongoengine.GenericReferenceField(required=True) created = mongoengine.DateTimeField(default=datetime.now) updated = mongoengine.DateTimeField(default=datetime.now) text = mongoengine.StringField(required=True) counter = mongoengine.SequenceField() meta = { "alias": "default", "collection": "comments", "indexes": [ "subject", "account", ] } def dict(self): return { "created": int(datetime.timestamp(self.created)), "updated": int(datetime.timestamp(self.updated)), "account": self.account.dict(), "counter": self.counter, "text": self.text }
class Status(mongoengine.Document): account = mongoengine.ReferenceField("User", required=True) subject = mongoengine.GenericReferenceField(required=True) created = mongoengine.DateTimeField(default=datetime.utcnow) updated = mongoengine.DateTimeField(default=datetime.utcnow) rating = mongoengine.IntField(min_value=1, max_value=10) content = mongoengine.IntField(required=True) position = mongoengine.IntField(default=None) status = mongoengine.IntField(default=None) rewatch = mongoengine.IntField(default=0) time = mongoengine.IntField(default=0) meta = { "alias": "default", "collection": "statuses", "indexes": [ "subject", "account", "rating", "status", ], "ordering": ["-created"] } def dict(self): return { "created": int(datetime.timestamp(self.created)), "updated": int(datetime.timestamp(self.updated)), "status": static.slug("statuses", self.status), "subject": self.subject.dict(), "position": self.position, "rewatch": self.rewatch, "rating": self.rating, "time": self.time }
class Log(mon.Document): object = mon.GenericReferenceField() action = mon.StringField() data = mon.DictField() timestamp = mon.DateTimeField(default=datetime.utcnow) meta = {'indexes': [('object', 'timestamp')]} @staticmethod def add_entry(object, action, data): return Log.create(object=object, action=action, data=data) @staticmethod def record_new(object): return Log.add_entry(object, 'create', {}) @staticmethod def record_update(object): return Log.add_entry(object, 'update', {}) @staticmethod def record_error(object, errtype, value): return Log.add_entry(object, 'error', { 'type': errtype, 'value': value }) @classmethod def create(cls, **fields): object = cls(**fields) object.save()
class Reporter(mongoengine.Document): meta = {'collection': 'test_reporter'} id = mongoengine.StringField(primary_key=True) first_name = mongoengine.StringField(required=True) last_name = mongoengine.StringField(required=True) email = mongoengine.EmailField() awards = mongoengine.ListField(mongoengine.StringField()) articles = mongoengine.ListField(mongoengine.ReferenceField(Article)) embedded_articles = mongoengine.ListField( mongoengine.EmbeddedDocumentField(EmbeddedArticle)) embedded_list_articles = mongoengine.EmbeddedDocumentListField( EmbeddedArticle) id = mongoengine.StringField(primary_key=True) first_name = mongoengine.StringField(required=True) last_name = mongoengine.StringField(required=True) email = mongoengine.EmailField() awards = mongoengine.ListField(mongoengine.StringField()) articles = mongoengine.ListField(mongoengine.ReferenceField(Article)) embedded_articles = mongoengine.ListField( mongoengine.EmbeddedDocumentField(EmbeddedArticle)) embedded_list_articles = mongoengine.EmbeddedDocumentListField( EmbeddedArticle) generic_reference = mongoengine.GenericReferenceField(choices=[ Article, Editor, ])
class Vote(mongoengine.Document): rating = mongoengine.IntField(min_value=1, max_value=10, required=True) account = mongoengine.ReferenceField("User", required=True) subject = mongoengine.GenericReferenceField(required=True) created = mongoengine.DateTimeField(default=datetime.now) updated = mongoengine.DateTimeField(default=datetime.now) meta = { "alias": "default", "collection": "votes", "indexes": [ "subject", "account", "rating", ] } def dict(self): return { "created": int(datetime.timestamp(self.created)), "updated": int(datetime.timestamp(self.updated)), "account": self.account.dict(), "subject": self.subject.dict(), "rating": self.rating }
class Categories(mongo.Document): id = mongo.UUIDField(required=False) name = mongo.StringField(required=True, default='', unique=True) privacy = mongo.StringField(required=True, default='') description = mongo.StringField(required=True, default='') taxonomies = mongo.StringField(required=False, default='') custom_field_form = mongo.ReferenceField("FormSchema", required=False) created = mongo.StringField(required=True, default=str(datetime.now())) created_by = mongo.StringField(required=False) updated = mongo.StringField(required=True, default=str(datetime.now())) tagged = mongo.ListField(mongo.GenericReferenceField(), required=False) meta = {'indexes': [{'fields': ('name', ), 'unique': True}]} def add_object(self, obj): self.tagged.append(obj) self.save() @property def group(self): return "Categories" @property def permission_type(self): return "category" @property def number_using(self): return len(self.tagged) @classmethod def objects_with(cls, name): try: return Categories.objects.get(name=name).tagged except Categories.DoesNotExist: return [] @classmethod def for_object(cls, obj): return Categories.objects(tagged__in=[obj]) @classmethod def for_object_id(cls, objid, cls_name): #kinda an ugly hack, but global should have most everything loaded #if your getting a Keyerror, try importing the cls cls = globals()[cls_name] obj = cls.objects.get(id=objid) return Categories.for_object(obj) @classmethod def remove_obj(cls, obj): for cat in Categories.for_object(obj): Categories.objects(id=str(cat.id)).update_one(pull__tagged=obj)
class Article(mongoengine.Document): meta = {'collection': 'test_article'} headline = mongoengine.StringField(required=True, help_text="The article headline.") pub_date = mongoengine.DateTimeField( default=datetime.now, verbose_name="publication date", help_text="The date of first press.") editor = mongoengine.ReferenceField(Editor) reporter = mongoengine.ReferenceField('Reporter') # Will not convert this field cause no choices generic_reference = mongoengine.GenericReferenceField()
class Like(BaseDocument): user = mongoengine.ReferenceField('User') likeable = mongoengine.GenericReferenceField('Paste') @classmethod def post_save(cls, sender, document, **kwargs): if kwargs.get('created'): field = document.likeable._cls.lower() document.user.increase_counter('%s_likes' % field) document.likeable.increase_counter('likes') @classmethod def post_delete(cls, sender, document, **kwargs): field = document.likeable._cls.lower() document.user.increase_counter('%s_likes' % field, -1) document.likeable.increase_counter('likes', -1)
class Tag(me.Document): owner = me.ReferenceField(Owner, required=True) key = me.StringField(required=True) resource_type = me.StringField() value = me.StringField() resource = me.GenericReferenceField() resource_id = me.StringField() meta = { 'indexes': [ 'owner', { 'fields': ['resource', 'key'], 'sparse': False, 'unique': False, 'cls': False, }, ], }
class Tag(me.Document): owner = me.ReferenceField(Owner, required=True) key = me.StringField(regex=r'^[a-zA-Z0-9_]+(?:[ :.-][a-zA-Z0-9_]+)*$', required=True) resource_type = me.StringField(choices=['cloud', 'clouds', 'keys', 'scripts', 'machine', 'machines', 'template', 'stack', 'image', 'network', 'tunnel', 'schedules', 'networks', 'zones', 'records']) value = me.StringField() resource = me.GenericReferenceField() meta = { 'indexes': [ 'owner', { 'fields': ['resource', 'key'], 'sparse': False, 'unique': True, 'cls': False, }, ], } def clean(self): self.resource_type = self.resource._meta["collection"] def __str__(self): return 'Tag %s:%s for %s' % (self.key, self.value, self.resource) def as_dict(self): return { 'key': self.key, 'value': self.value, 'owner': self.owner.id, 'resource_type': self.resource_type, 'resource': str(self.resource), }
class DocOnlyA(me.Document): id = me.StringField(primary_key=True, default='main') generic = me.GenericReferenceField(choices=[SubDocA])
class Doc(me.Document): id = me.StringField(primary_key=True, default='main') generic = me.GenericReferenceField()
class Notification(me.Document): ''' Represents a notification associated with a user-organization pair ''' meta = {'allow_inheritance': True} id = me.StringField(primary_key=True, default=lambda: uuid4().hex) created_date = me.DateTimeField(required=False) expiry_date = me.DateTimeField(required=False) user = me.ReferenceField(User, required=True) organization = me.ReferenceField(Organization, required=True) # content fields summary = me.StringField(max_length=512, required=False, default="") body = me.StringField(required=True, default="") html_body = me.StringField(required=False, default="") # taxonomy fields source = me.StringField(max_length=64, required=True, default="") machine = me.GenericReferenceField(required=False) tag = me.GenericReferenceField(required=False) cloud = me.GenericReferenceField(required=False) action_link = me.URLField(required=False) unique = me.BooleanField(required=True, default=True) severity = me.StringField(max_length=7, required=True, choices=('LOW', 'DEFAULT', 'HIGH'), default='DEFAULT') feedback = me.StringField(max_length=8, required=True, choices=('NEGATIVE', 'NEUTRAL', 'POSITIVE'), default='NEUTRAL') def __init__(self, *args, **kwargs): super(Notification, self).__init__(*args, **kwargs) if not self.created_date: self.created_date = datetime.now() def update_from(self, notification): self.created_date = notification.created_date self.expiry_date = notification.expiry_date self.user = notification.user self.organization = notification.organization self.summary = notification.summary self.body = notification.body self.html_body = notification.html_body self.source = notification.source self.machine = notification.machine self.tag = notification.tag self.cloud = notification.cloud self.unique = notification.unique self.action_link = notification.action_link self.severity = notification.severity self.feedback = notification.feedback
class MUserStory(mongo.Document): """ Stories read by the user. These are deleted as the mark_read_date for the UserSubscription passes the UserStory date. """ user_id = mongo.IntField() feed_id = mongo.IntField() read_date = mongo.DateTimeField() story_id = mongo.StringField() story_date = mongo.DateTimeField() story = mongo.ReferenceField(MStory, dbref=True) found_story = mongo.GenericReferenceField() shared = mongo.BooleanField() meta = { 'collection': 'userstories', 'indexes': [ { 'fields': ('user_id', 'feed_id', 'story_id'), 'unique': True }, ('feed_id', 'story_id'), # Updating stories with new guids ('feed_id', 'story_date'), # Trimming feeds ('feed_id', '-read_date'), # Trimming feeds ], 'allow_inheritance': False, 'index_drop_dups': True, 'cascade': False, } def save(self, *args, **kwargs): self.sync_redis() super(MUserStory, self).save(*args, **kwargs) def delete(self, *args, **kwargs): self.remove_from_redis() super(MUserStory, self).delete(*args, **kwargs) @property def guid_hash(self): return hashlib.sha1(self.story_id).hexdigest()[:6] @property def feed_guid_hash(self): return "%s:%s" % (self.feed_id, self.guid_hash) @classmethod def delete_old_stories(cls, feed_id): UNREAD_CUTOFF = datetime.datetime.utcnow() - datetime.timedelta( days=settings.DAYS_OF_UNREAD * 5) read_stories = cls.objects(feed_id=feed_id, read_date__lte=UNREAD_CUTOFF) read_stories_count = read_stories.count() if read_stories_count: feed = Feed.objects.get(pk=feed_id) total = cls.objects(feed_id=feed_id).count() logging.info( " ---> ~SN~FCTrimming ~SB%s~SN/~SB%s~SN read stories from %s..." % (read_stories_count, total, feed.title[:30])) read_stories.delete() @classmethod def delete_marked_as_read_stories(cls, user_id, feed_id, mark_read_date=None): if not mark_read_date: usersub = UserSubscription.objects.get(user__pk=user_id, feed__pk=feed_id) mark_read_date = usersub.mark_read_date # Next line forces only old read stories to be removed, just in case newer stories # come in as unread because they're being shared. mark_read_date = datetime.datetime.utcnow() - datetime.timedelta( days=settings.DAYS_OF_UNREAD) cls.objects(user_id=user_id, feed_id=feed_id, read_date__lte=mark_read_date).delete() @property def story_db_id(self): if self.story: return self.story.id elif self.found_story: if '_ref' in self.found_story: return self.found_story['_ref'].id elif hasattr(self.found_story, 'id'): return self.found_story.id story, found_original = MStory.find_story(self.feed_id, self.story_id) if story: if found_original: self.story = story else: self.found_story = story self.save() return story.id def sync_redis(self, r=None, pipeline=None, hashpipe=None): if pipeline: r = pipeline if hashpipe: h = pipeline elif not r: r = redis.Redis(connection_pool=settings.REDIS_STORY_POOL) h = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL) if self.story_db_id: all_read_stories_key = 'RS:%s' % (self.user_id) r.sadd(all_read_stories_key, self.story_db_id) r.expire(all_read_stories_key, settings.DAYS_OF_UNREAD * 24 * 60 * 60) h.sadd(all_read_stories_key, self.feed_guid_hash) h.expire(all_read_stories_key, settings.DAYS_OF_UNREAD * 24 * 60 * 60) read_story_key = 'RS:%s:%s' % (self.user_id, self.feed_id) r.sadd(read_story_key, self.story_db_id) r.expire(read_story_key, settings.DAYS_OF_UNREAD * 24 * 60 * 60) h.sadd(read_story_key, self.feed_guid_hash) h.expire(read_story_key, settings.DAYS_OF_UNREAD * 24 * 60 * 60) def remove_from_redis(self): r = redis.Redis(connection_pool=settings.REDIS_STORY_POOL) h = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL) if self.story_db_id: r.srem('RS:%s' % self.user_id, self.story_db_id) r.srem('RS:%s:%s' % (self.user_id, self.feed_id), self.story_db_id) h.srem('RS:%s' % self.user_id, self.feed_guid_hash) h.srem('RS:%s:%s' % (self.user_id, self.feed_id), self.feed_guid_hash) @classmethod def sync_all_redis(cls, user_id=None, feed_id=None, force=False): r = redis.Redis(connection_pool=settings.REDIS_STORY_POOL) h = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL) UNREAD_CUTOFF = datetime.datetime.utcnow() - datetime.timedelta( days=settings.DAYS_OF_UNREAD + 1) if feed_id and user_id: read_stories = cls.objects.filter(user_id=user_id, feed_id=feed_id, read_date__gte=UNREAD_CUTOFF) key = "RS:%s:%s" % (user_id, feed_id) r.delete(key) h.delete(key) elif feed_id: read_stories = cls.objects.filter(feed_id=feed_id, read_date__gte=UNREAD_CUTOFF) keys = r.keys("RS:*:%s" % feed_id) print " ---> Deleting %s redis keys: %s" % (len(keys), keys) for key in keys: r.delete(key) keys = h.keys("RS:*:%s" % feed_id) print " ---> Deleting %s redis keys: %s" % (len(keys), keys) for key in keys: h.delete(key) elif user_id: read_stories = cls.objects.filter(user_id=user_id, read_date__gte=UNREAD_CUTOFF) keys = r.keys("RS:%s:*" % user_id) r.delete("RS:%s" % user_id) print " ---> Deleting %s redis keys: %s" % (len(keys), keys) for key in keys: r.delete(key) keys = h.keys("RS:%s:*" % user_id) h.delete("RS:%s" % user_id) print " ---> Deleting %s redis keys: %s" % (len(keys), keys) for key in keys: h.delete(key) elif force: read_stories = cls.objects.all(read_date__gte=UNREAD_CUTOFF) else: raise "Specify user_id, feed_id, or force." total = read_stories.count() logging.debug(" ---> ~SN~FMSyncing ~SB%s~SN stories (%s/%s)" % (total, user_id, feed_id)) pipeline = None hashpipe = None for i, read_story in enumerate(read_stories): if not pipeline: pipeline = r.pipeline() hashpipe = h.pipeline() if (i + 1) % 1000 == 0: print " ---> %s/%s" % (i + 1, total) pipeline.execute() hashpipe.execute() pipeline = r.pipeline() hashpipe = h.pipeline() read_story.sync_redis(r, pipeline=pipeline, hashpipe=hashpipe) if pipeline: pipeline.execute() hashpipe.execute()
class Link(db.Document): """ Link between documents, eg: Objective<->Competency """ documents = db.ListField(db.GenericReferenceField(), default=[]) owner = db.ReferenceField('User')
class MUserStory(mongo.Document): """ Stories read by the user. These are deleted as the mark_read_date for the UserSubscription passes the UserStory date. """ user_id = mongo.IntField() feed_id = mongo.IntField() read_date = mongo.DateTimeField() story_id = mongo.StringField() story_date = mongo.DateTimeField() story = mongo.ReferenceField(MStory, dbref=True) found_story = mongo.GenericReferenceField() meta = { 'collection': 'userstories', 'indexes': [ { 'fields': ('user_id', 'feed_id', 'story_id'), 'unique': True }, ('feed_id', 'story_id'), # Updating stories with new guids ('feed_id', 'story_date'), # Trimming feeds ], 'allow_inheritance': False, 'index_drop_dups': True, 'cascade': False, } def save(self, *args, **kwargs): self.sync_redis() super(MUserStory, self).save(*args, **kwargs) def delete(self, *args, **kwargs): self.remove_from_redis() super(MUserStory, self).delete(*args, **kwargs) @property def guid_hash(self): return hashlib.sha1(self.story_id).hexdigest() @classmethod def delete_old_stories(cls, feed_id): UNREAD_CUTOFF = datetime.datetime.utcnow() - datetime.timedelta( days=settings.DAYS_OF_UNREAD) cls.objects(feed_id=feed_id, story_date__lte=UNREAD_CUTOFF).delete() @classmethod def delete_marked_as_read_stories(cls, user_id, feed_id, mark_read_date=None): if not mark_read_date: usersub = UserSubscription.objects.get(user__pk=user_id, feed__pk=feed_id) mark_read_date = usersub.mark_read_date # Next line forces only old read stories to be removed, just in case newer stories # come in as unread because they're being shared. mark_read_date = datetime.datetime.utcnow() - datetime.timedelta( days=settings.DAYS_OF_UNREAD) cls.objects(user_id=user_id, feed_id=feed_id, read_date__lte=mark_read_date).delete() @property def story_db_id(self): if self.story: return self.story.id elif self.found_story: if '_ref' in self.found_story: return self.found_story['_ref'].id elif hasattr(self.found_story, 'id'): return self.found_story.id story, found_original = MStory.find_story(self.feed_id, self.story_id) if story: if found_original: self.story = story else: self.found_story = story self.save() return story.id def sync_redis(self, r=None): if not r: r = redis.Redis(connection_pool=settings.REDIS_STORY_POOL) if self.story_db_id: all_read_stories_key = 'RS:%s' % (self.user_id) r.sadd(all_read_stories_key, self.story_db_id) read_story_key = 'RS:%s:%s' % (self.user_id, self.feed_id) r.sadd(read_story_key, self.story_db_id) r.expire(read_story_key, settings.DAYS_OF_UNREAD * 24 * 60 * 60) def remove_from_redis(self): r = redis.Redis(connection_pool=settings.REDIS_STORY_POOL) if self.story_db_id: r.srem('RS:%s' % self.user_id, self.story_db_id) r.srem('RS:%s:%s' % (self.user_id, self.feed_id), self.story_db_id) @classmethod def sync_all_redis(cls): read_stories = cls.objects.all() for read_story in read_stories: read_story.sync_redis()