class ErroneousModel(mongoengine.Document):
meta = {'collection': 'test_colliding_objects_model'}
objects = mongoengine.ListField(mongoengine.StringField())
class Job(common.NamedDocument, common.Runnable, common.LogProxy,
common.AutoDocumentable):
meta = {
'collection': 'jobs',
'indexes': [
'status',
'created',
]
}
status_text = mongoengine.StringField(required=True, default="")
hostname = mongoengine.StringField()
completion = mongoengine.IntField(required=True,
min_value=0,
max_value=100,
default=0)
timeout = mongoengine.IntField(min_value=0, default=43200) # 12 hours
ttl = mongoengine.IntField(min_value=1, default=1)
history = mongoengine.ListField(field=mongoengine.DictField(), default=[])
def __str__(self):
return "%s %s" % (self.name,
job_status_to_icon.get(self.status, self.status))
@classmethod
def default_slot_amount(cls):
"""
Returns the default amount of job that can be run at the same time on the same machine/client.
Override and set to math.inf for no limiting amount.
You can base that how much CPU cores you have or anything else.
Default is 1 job at a time by default.
"""
return 1
@cached_property
def extra_log_arguments(self):
return {
'job_type': self.__class__.__name__,
'job_uuid': self.uuid,
'job_status': self.status,
}
def update_status(self, completion=None, text=None):
if text:
self.status_text = text
if completion:
self.completion = completion
log = self.log_info
if self.status == 'error':
log = self.log_error
log("Progress update : {progress:5.1f}% - {message}".format(
progress=self.completion, message=self.status_text))
self.update(add_to_set__history={
't': datetime.utcnow(),
'm': self.status_text,
'c': self.completion,
's': self.status
},
status=self.status,
details=self.details,
completion=self.completion,
status_text=self.status_text,
started=self.started,
finished=self.finished)
def update_progress(self, completion, text=None):
self.update_status(completion=completion, text=text)
def save_as_successful(self, text="Job Successful"):
self.update_status(100, text=text)
self.save() # Saves a other fields
def save_as_error(self, text="Job Error"):
self.status = 'error'
self.update_status(text=text)
self.save()
class Order(me.Document):
customer = me.ReferenceField('Customer')
dishes = me.ListField(me.ReferenceField('Dish'))
class UserProfile(mongoengine.EmbeddedDocument):
primary_music_account_id = mongoengine.ObjectIdField()
top_tracks = mongoengine.ListField()
top_artists = mongoengine.ListField()
favorite_genre = mongoengine.StringField()
class FakeModel(stormbase.StormBaseDB):
boolean_field = mongoengine.BooleanField()
datetime_field = mongoengine.DateTimeField()
dict_field = mongoengine.DictField()
integer_field = mongoengine.IntField()
list_field = mongoengine.ListField()
class DocumentWithID(InheritableDocument):
title = mongoengine.StringField(max_length=200, required=True)
comments = mongoengine.ListField(
mongoengine.EmbeddedDocumentField(EmbeddedCommentWithID))
class Feed2(Document):
"""
@summary: 所有订阅内容
"""
keywords = StringField(default='')
job_type = StringField(default='') # 工作类型
talent_level = StringField(default='') # 人才级别
expect_area = StringField(default='') # 期望工作地
job_desc = StringField(default='') # 职位描述
deleted = mongoengine.BooleanField(default=False)
add_time = mongoengine.DateTimeField(default=datetime.now())
delete_time = mongoengine.DateTimeField(default=datetime.now())
expire_time = mongoengine.DateTimeField(required=False)
calced = mongoengine.BooleanField(default=False, )
username = StringField(default='')
# 最新增加对于每个订阅职位的理解,更好辅助算法.
remarks = ListField(EmbeddedDocumentField(Remark),
default=[],
required=False)
ignored = mongoengine.BooleanField(default=False) # 忽略用户输入关键词
last_click_time = mongoengine.DateTimeField(default=datetime.now())
feed_expire_time = mongoengine.DateTimeField(default=after7day())
feed_type = mongoengine.IntField(default=1)
title = mongoengine.StringField(default='', )
salary_min = mongoengine.IntField(default=0, )
salary_max = mongoengine.IntField(default=0, )
work_years_min = mongoengine.IntField(default=0, )
work_years_max = mongoengine.IntField(default=0, )
type = mongoengine.StringField(default='', )
category = mongoengine.StringField(default='', )
degree = mongoengine.IntField(default=0, )
key_points = mongoengine.StringField(default='', )
skill_required = mongoengine.StringField(default='', )
recruit_num = mongoengine.IntField(default=0, )
job_tag = mongoengine.StringField(default='', )
report_to = mongoengine.StringField(default='', )
department_to = mongoengine.StringField(default='', )
language = mongoengine.StringField(default='', )
gender = mongoengine.StringField(default='', )
major = mongoengine.StringField(default='', )
display = mongoengine.BooleanField(default=True, )
job_url = mongoengine.StringField(default='', )
is_related = mongoengine.BooleanField(default=True, )
job_domain = mongoengine.ListField(StringField())
job_welfare = mongoengine.ListField(StringField())
company_prefer = mongoengine.ListField(StringField())
update_time = mongoengine.DateTimeField(
default=datetime.now(),
required=False,
)
analyze_titles = mongoengine.StringField(default='', )
meta = {
'collection':
'feed',
'index_background':
True,
'indexes': [
'job_type',
'feed_type',
'talent_level',
'-add_time',
'-expect_area',
'-username',
'-delete_time',
'-expire_time',
'-last_click_time',
'-feed_expire_time',
]
}
def __unicode__(self):
return self.username
class Meta:
app_label = 'Feed'
def get_keywords_area(self):
return self.keywords.lower() + self.expect_area.lower()
def get_area_list(self):
return self.expect_area.split(',')
def get_uniform_keywords(self):
return uniform(self.keywords)
def get_remarks_keywords_list(self):
"""
@summary: 由于对每个订阅进行了人工备注,需要将其转换为,server计算时的格式.
@author: [email protected] 2014-2-17 10:06:28
"""
return [
list(jieba.cut("".join(remark.get_necessary_keywords())))
for remark in self.remarks
]
def get_exclude_job_words(self):
"""
@summary: 获取需要屏蔽的职位词
"""
exclude_job_words = []
for remark in self.remarks:
exclude_job_words.extend(remark.get_exclude_job_keywords())
return exclude_job_words
def get_latent_semantic_keywords(self):
"""
@summary: 获取需要加分加权重的词
"""
latent_semantic_keywords = []
for remark in self.remarks:
latent_semantic_keywords.extend(
remark.get_latent_semantic_keywords())
return latent_semantic_keywords
def get_clean_keywords_list(self, unique=True):
"""
@summary: 获取规则化的关键词列表, 由于用户输入关键词可能含有中文标点等情况.
#统一为半角,小写,用户输入关键词按照1.自然分词 2.用户本身的输入间隔进行
#返回这两种分割非重复的关键词列表.
"""
uniform_keywords = uniform(self.keywords)
# uniform_keywords = replace_punctuation(uniform_keywords)
seg_keywords = jieba.cut(uniform_keywords)
seg_keywords = [
keyword.lower() for keyword in seg_keywords
if not is_other(keyword)
]
user_seg_keywords = uniform_keywords.split()
user_seg_keywords = [
keyword.lower() for keyword in user_seg_keywords
if not is_other(keyword)
]
if unique:
seg_keywords = get_ordered_unique(seg_keywords)
user_seg_keywords = get_ordered_unique(user_seg_keywords)
if seg_keywords == user_seg_keywords:
return [seg_keywords]
else:
return [seg_keywords, user_seg_keywords]
def get_all_eng_words(self, unique=True):
"""
@summary: 获取关键词和职位描述中的所有单词或者单词+数字类型 ,如cocos2d
"""
desc = uniform(self.keywords + ' ' + self.job_desc)
words = jieba.cut(desc)
eng_words = [word for word in words if is_num_word(word)]
if unique:
eng_words = get_ordered_unique(eng_words)
return eng_words
def get_user_remark_keywords(self):
"""
@summary: 获取用户填的和人工备注的关键词.
"""
keywords_list = []
if not self.ignored:
clean_keywords_list = self.get_clean_keywords_list()
keywords_list.extend(clean_keywords_list)
remark_keywords_list = self.get_remarks_keywords_list()
if remark_keywords_list != [[]]:
keywords_list.extend(remark_keywords_list)
return keywords_list
class SpacyDocument(me.EmbeddedDocument):
ents = me.ListField(me.EmbeddedDocumentField(SpacyEntity, required=True))
sents = me.ListField(me.EmbeddedDocumentField(SpacySentence, required=True))
text = me.StringField(required=True)
tokens = me.ListField(me.EmbeddedDocumentField(SpacyToken, required=True))
_created_at = me.DateTimeField(default=datetime.now(), required=True)
class Owner(me.Document):
id = me.StringField(primary_key=True, default=lambda: uuid4().hex)
activation_date = me.FloatField() # TODO: Use datetime object
# this exists for preventing conflicting rule id's
rule_counter = me.IntField(default=0)
total_machine_count = me.IntField()
alerts_email = me.ListField(me.StringField(), default=[])
avatar = me.StringField(default='')
last_active = me.DateTimeField()
meta = {
'allow_inheritance': True,
'ordering': ['-activation_date'],
'strict': False,
}
def count_mon_machines(self):
from mist.api.clouds.models import Cloud
from mist.api.machines.models import Machine
clouds = Cloud.objects(owner=self, deleted=None)
return Machine.objects(cloud__in=clouds,
monitoring__hasmonitoring=True).count()
def get_id(self):
# TODO: This must be deprecated
if not self.id:
self.id = lambda: uuid.uuid4().hex
return self.id
def get_external_id(self, service):
import mist.api.helpers
return mist.api.helpers.encrypt(self.id, key_salt=service, no_iv=True)
def as_dict(self):
# FIXME: Now, this is just silly
return json.loads(self.to_json())
def clean(self):
# TODO: check if these are valid email addresses,
# to avoid possible spam
if self.alerts_email:
if isinstance(self.alerts_email, string_types):
emails = []
for email in self.alerts_email.split(','):
if re.match("[^@]+@[^@]+\.[^@]+", email):
emails.append(email.replace(' ', ''))
self.emails = emails
super(Owner, self).clean()
def get_rules_dict(self):
from mist.api.rules.models import Rule
return {
rule.id: rule.as_dict()
for rule in Rule.objects(owner_id=self.id)
}
def get_metrics_dict(self):
return {
metric.metric_id: {
'name': metric.name,
'unit': metric.unit
}
for metric in Metric.objects(owner=self)
}
class Course(me.Document):
meta = {
'indexes': [
'_keywords',
],
}
# eg. earth121l
id = me.StringField(primary_key=True)
# eg. earth
department_id = me.StringField(required=True)
# eg. 121l
number = me.StringField(required=True)
# eg. Introductory Earth Sciences Laboratory 1
name = me.StringField(required=True)
# Description about the course
description = me.StringField(required=True)
professor_ids = me.ListField(me.StringField())
antireqs = me.StringField()
coreqs = me.StringField()
prereqs = me.StringField()
url = me.StringField()
# NOTE: The word term is overloaded based on where it's used. Here, it mean
# which terms of the year is the course being offered?
# NOTE: THIS FIELD IS ***DEPRECATED***, because the data source we get
# info about this is not reliable. There may not exist such reliable
# data at all -- course offerings are decided on an annual basis.
# TODO(david): Remove this field and replace it with info from sections.
# e.g. ['01', '05', '09']
terms_offered = me.ListField(me.StringField())
# eg. ['earth', '121l', 'earth121l', 'Introductory',
# 'Earth' 'Sciences', 'Laboratory', '1']
_keywords = me.ListField(me.StringField(), required=True)
SORT_MODES = _SORT_MODES
@property
def code(self):
matches = re.findall(r'^([a-z]+)(.*)$', self.id)[0]
department = matches[0]
number = matches[1]
return '%s %s' % (department.upper(), number.upper())
@cached_property
def comments(self):
return [c for c in CourseComment.objects(course_id=self.id)]
@property
def easiness(self):
ratings = [
comment.easiness for comment in self.comments
if comment.easiness is not None
]
average = len(ratings) and sum(ratings) / len(ratings)
return AggregateRating(rating=average, count=len(ratings))
@property
def interest(self):
ratings = [
comment.interest for comment in self.comments
if comment.interest is not None
]
average = len(ratings) and sum(ratings) / len(ratings)
return AggregateRating(rating=average, count=len(ratings))
@property
def usefulness(self):
ratings = [
comment.usefulness for comment in self.comments
if comment.usefulness is not None
]
average = len(ratings) and sum(ratings) / len(ratings)
return AggregateRating(rating=average, count=len(ratings))
@property
def overall(self):
overall_count = (self.easiness.count + self.interest.count +
self.usefulness.count)
overall_rating = (self.easiness.rating * self.easiness.count +
self.interest.rating * self.interest.count +
self.usefulness.rating *
self.usefulness.count) / max(1, overall_count)
return {"rating": overall_rating, "count": overall_count}
def save(self, *args, **kwargs):
if not self.id:
# id should not be set during first save
self.id = self.department_id + self.number
super(Course, self).save(*args, **kwargs)
def get_ratings(self):
# Ordered for consistency with CourseReview.rating_fields; see #109.
return collections.OrderedDict([
('usefulness', self.usefulness.to_dict()),
('easiness', self.easiness.to_dict()),
('interest', self.interest.to_dict()),
])
def get_reviews(self, current_user=None, user_courses=None):
"""Return a list of all user reviews ("tips") about this course.
Does not include professor reviews.
Arguments:
current_user: The current user. Used for revealing more author
information if possible (eg. reviews written by friends who
allow their friends to know that they wrote it).
user_courses: An optional list of all user_courses that's
associated with this course to speed up this function.
"""
if not user_courses:
limit_fields = ['course_id', 'user_id', 'course_review']
user_courses = _user_course.UserCourse.objects(
course_id=self.id).only(*limit_fields)
reviews = []
for uc in user_courses:
if (len(uc.course_review.comment) <
review.CourseReview.MIN_REVIEW_LENGTH):
continue
reviews.append(
uc.course_review.to_dict(current_user, uc.user_id, uc.id))
# Filter out old reviews if we have enough results.
date_getter = lambda review: review['comment_date']
reviews = util.publicly_visible_ratings_and_reviews_filter(
reviews, date_getter, util.MIN_NUM_REVIEWS)
return reviews
# TODO(mack): this function is way too overloaded, even to separate into
# multiple functions based on usage
@classmethod
def get_course_and_user_course_dicts(cls,
courses,
current_user,
include_friends=False,
include_all_users=False,
full_user_courses=False,
include_sections=False):
limited_user_course_fields = [
'program_year_id', 'term_id', 'user_id', 'course_id'
]
course_dicts = [course.to_dict() for course in courses]
course_ids = [c['id'] for c in course_dicts]
if include_sections:
for course_dict in course_dicts:
# By default, we'll send down section info for current and next
# term for each course we return.
sections = section.Section.get_for_course_and_recent_terms(
course_dict['id'])
course_dict['sections'] = [s.to_dict() for s in sections]
ucs = []
if not current_user:
if include_all_users:
ucs = _user_course.UserCourse.objects(course_id__in=course_ids)
if not full_user_courses:
ucs.only(*limited_user_course_fields)
ucs = list(ucs)
uc_dicts = [uc.to_dict() for uc in ucs]
return course_dicts, uc_dicts, ucs
else:
return course_dicts, [], []
uc_dicts = []
if include_all_users or include_friends:
query = {
'course_id__in': course_ids,
}
# If we're just including friends
if not include_all_users:
query['user_id__in'] = current_user.friend_ids
if full_user_courses:
if not include_all_users:
query.setdefault('user_id__in', []).append(current_user.id)
ucs = list(_user_course.UserCourse.objects(**query))
uc_dicts = [uc.to_dict() for uc in ucs]
else:
ucs = list(
_user_course.UserCourse.objects(**query).only(
*limited_user_course_fields))
friend_uc_fields = [
'id', 'user_id', 'course_id', 'term_id', 'term_name'
]
uc_dicts = [uc.to_dict(friend_uc_fields) for uc in ucs]
# TODO(mack): optimize to not always get full user course
# for current_user
current_ucs = list(
_user_course.UserCourse.objects(
user_id=current_user.id,
course_id__in=course_ids,
id__nin=[uc_dict['id'] for uc_dict in uc_dicts],
))
ucs += current_ucs
uc_dicts += [uc.to_dict() for uc in current_ucs]
current_user_course_by_course = {}
friend_user_courses_by_course = {}
current_friends_set = set(current_user.friend_ids)
current_user_course_ids = set(current_user.course_history)
for uc_dict in uc_dicts:
if uc_dict['id'] in current_user_course_ids:
current_user_course_by_course[uc_dict['course_id']] = uc_dict
elif include_friends:
if uc_dict['user_id'] in current_friends_set:
friend_user_courses_by_course.setdefault(
uc_dict['course_id'], []).append(uc_dict)
for course_dict in course_dicts:
current_uc = current_user_course_by_course.get(course_dict['id'])
current_uc_id = current_uc['id'] if current_uc else None
course_dict['user_course_id'] = current_uc_id
if include_friends:
friend_ucs = friend_user_courses_by_course.get(
course_dict['id'], [])
friend_uc_ids = [uc['id'] for uc in friend_ucs]
course_dict['friend_user_course_ids'] = friend_uc_ids
return course_dicts, uc_dicts, ucs
@staticmethod
def code_to_id(course_code):
return "".join(course_code.split()).lower()
@staticmethod
def search(params, current_user=None):
"""Search for courses based on various parameters.
Arguments:
params: Dict of search parameters (all optional):
keywords: Keywords to search on
sort_mode: Name of a sort mode. See Course.SORT_MODES. The
'friends_taken' sort mode defaults to 'popular' if no
current_user.
direction: 1 for ascending, -1 for descending
count: Max items to return (aka. limit)
offset: Index of first search result to return (aka. skip)
exclude_taken_courses: "yes" to exclude courses current_user
has taken.
current_user: The user making the request.
Returns:
A tuple (courses, has_more):
courses: Search results
has_more: Whether there could be more search results
"""
keywords = params.get('keywords')
sort_mode = params.get('sort_mode', 'popular')
default_direction = _SORT_MODES_BY_NAME[sort_mode]['direction']
direction = int(params.get('direction', default_direction))
count = int(params.get('count', 10))
offset = int(params.get('offset', 0))
exclude_taken_courses = (params.get('exclude_taken_courses') == "yes")
# TODO(david): These logging things should be done asynchronously
rmclogger.log_event(rmclogger.LOG_CATEGORY_COURSE_SEARCH,
rmclogger.LOG_EVENT_SEARCH_PARAMS, params)
filters = {}
if keywords:
# Clean keywords to just alphanumeric and space characters
keywords_cleaned = re.sub(r'[^\w ]', ' ', keywords)
def regexify_keywords(keyword):
keyword = keyword.lower()
return re.compile('^%s' % re.escape(keyword))
keyword_regexes = map(regexify_keywords, keywords_cleaned.split())
filters['_keywords__all'] = keyword_regexes
if exclude_taken_courses:
if current_user:
ucs = (current_user.get_user_courses().only(
'course_id', 'term_id'))
filters['id__nin'] = [
uc.course_id for uc in ucs
if not term.Term.is_shortlist_term(uc.term_id)
]
else:
logging.error('Anonymous user tried excluding taken courses')
if sort_mode == 'friends_taken' and current_user:
import user
friends = user.User.objects(
id__in=current_user.friend_ids).only('course_history')
num_friends_by_course = collections.Counter()
for friend in friends:
num_friends_by_course.update(friend.course_ids)
filters['id__in'] = num_friends_by_course.keys()
existing_courses = Course.objects(**filters).only('id')
existing_course_ids = set(c.id for c in existing_courses)
for course_id in num_friends_by_course.keys():
if course_id not in existing_course_ids:
del num_friends_by_course[course_id]
sorted_course_count_tuples = sorted(
num_friends_by_course.items(),
key=lambda (_, total): total,
reverse=direction < 0,
)[offset:offset + count]
sorted_course_ids = [
course_id for (course_id, total) in sorted_course_count_tuples
]
unsorted_courses = Course.objects(id__in=sorted_course_ids)
course_by_id = {course.id: course for course in unsorted_courses}
courses = [course_by_id[cid] for cid in sorted_course_ids]
else:
sort_options = _SORT_MODES_BY_NAME[sort_mode]
if sort_options['is_rating']:
suffix = 'positive' if direction < 0 else 'negative'
order_by = '-%s.sorting_score_%s' % (sort_options['field'],
suffix)
else:
sign = '-' if direction < 0 else ''
order_by = '%s%s' % (sign, sort_options['field'])
unsorted_courses = Course.objects(**filters)
sorted_courses = unsorted_courses.order_by(order_by)
courses = sorted_courses.skip(offset).limit(count)
has_more = len(courses) == count
return courses, has_more
def to_dict(self):
"""Returns information about a course to be sent down an API.
Args:
course: The course object.
"""
return {
'id': self.id,
'code': self.code,
'name': self.name,
'description': self.description,
'easiness': self.easiness.to_dict(),
'interest': self.interest.to_dict(),
'usefulness': self.usefulness.to_dict(),
'overall': self.overall,
'professor_ids': self.professor_ids,
'prereqs': self.prereqs,
}
def __repr__(self):
return "<Course: %s>" % self.code
class Poligono(mongoengine.EmbeddedDocument):
puntos = mongoengine.StringField()
genUid = mongoengine.LongField()
arregloPuntos = mongoengine.ListField(
mongoengine.EmbeddedDocumentField(Punto))
class Order(me.Document):
user_telegram_id_1 = me.IntField(required=True)
products_1 = me.ListField()
phone_number = me.StringField(max_length=12)
address = me.StringField(min_length=2, max_length=128)
class Record(me.Document):
"""This is the class definition for the Mongo Engine Document related to a
DNS record.
"""
id = me.StringField(primary_key=True, default=lambda: uuid.uuid4().hex)
record_id = me.StringField(required=True)
name = me.StringField(required=True)
type = me.StringField(required=True)
rdata = me.ListField(required=True)
extra = me.DictField()
ttl = me.IntField(default=0)
# This ensures that any records that are under a zone are also deleted when
# we delete the zone.
zone = me.ReferenceField(Zone,
required=True,
reverse_delete_rule=me.CASCADE)
owner = me.ReferenceField('Organization', required=True)
deleted = me.DateTimeField()
meta = {
'collection':
'records',
'allow_inheritance':
True,
'indexes': [{
'fields': ['zone', 'record_id', 'deleted'],
'sparse': False,
'unique': True,
'cls': False,
}],
}
_record_type = None
def __init__(self, *args, **kwargs):
super(Record, self).__init__(*args, **kwargs)
self.ctl = RecordController(self)
@classmethod
def add(cls, owner=None, zone=None, id='', **kwargs):
"""Add Record
This is a class method, meaning that it is meant to be called on the
class itself and not on an instance of the class.
You're not meant to be calling this directly, but on a cloud subclass
instead like this:
record = Record.add(zone=zone, **kwargs)
Params:
- zone is a required param
- only provide a custom record id if you're migrating something
- kwargs will be passed to appropriate controller, in most cases these
should match the extra fields of the particular record type.
"""
if not kwargs['name']:
raise RequiredParameterMissingError('name')
if not kwargs['data']:
raise RequiredParameterMissingError('data')
if not kwargs['type']:
raise RequiredParameterMissingError('type')
# If we were not given a zone then we need the owner to try and find
# the best matching domain.
if not zone and kwargs['type'] in ['A', 'AAAA', 'CNAME']:
assert isinstance(owner, Organization)
zone = BaseDNSController.find_best_matching_zone(
owner, kwargs['name'])
assert isinstance(zone, Zone)
record = cls(zone=zone)
if id:
record.id = id
record.ctl.create_record(**kwargs)
return record
def delete(self):
super(Record, self).delete()
Tag.objects(resource=self).delete()
self.zone.owner.mapper.remove(self)
def clean(self):
"""Overriding the default clean method to implement param checking"""
self.type = self._record_type
if not self.owner:
self.owner = self.zone.owner
def __str__(self):
return 'Record %s (name:%s, type:%s) of %s' % (
self.id, self.name, self.type, self.zone.domain)
def as_dict(self):
""" Return a dict with the model values."""
return {
'id': self.id,
'record_id': self.record_id,
'name': self.name,
'type': self.type,
'rdata': self.rdata,
'ttl': self.ttl,
'extra': self.extra,
'zone': self.zone.id
}
class NotificationAction(BaseAlertAction):
"""An action that notifies the users, once a rule has been triggered."""
atype = 'notification'
users = me.ListField(me.StringField(), default=lambda: [])
teams = me.ListField(me.StringField(), default=lambda: [])
emails = me.ListField(me.StringField(), default=lambda: [])
def run(self,
machine,
value,
triggered,
timestamp,
incident_id,
action='',
notification_level=0):
# FIXME Imported here due to circular dependency issues.
from mist.api.notifications.methods import send_alert_email
# TODO Shouldn't be specific to machines.
assert isinstance(machine, Machine)
assert machine.owner == self._instance.owner
emails = set(self.emails)
user_ids = set(self.users)
if not (self.users or self.teams):
emails |= set(self._instance.owner.get_emails())
emails |= set(self._instance.owner.alerts_email)
if user_ids:
user_ids &= set([m.id for m in self._instance.owner.members])
for user in User.objects(id__in=user_ids):
emails.add(user.email)
for team_id in self.teams:
try:
team = self._instance.owner.teams.get(id=team_id)
emails |= set([member.email for member in team.members])
except me.DoesNotExist:
continue
send_alert_email(machine.owner,
self._instance.id,
value,
triggered,
timestamp,
incident_id,
emails,
cloud_id=machine.cloud.id,
machine_id=machine.machine_id,
action=action)
def clean(self):
"""Perform e-mail address validation."""
for email in self.emails:
if not is_email_valid(email):
raise me.ValidationError('Invalid e-mail address: %s' % email)
def as_dict(self):
return {
'type': self.atype,
'emails': self.emails,
'users': self.users,
'teams': self.teams
}
class ContactGroup(InheritableDocument):
contacts = mongoengine.ListField(
mongoengine.ReferenceField(Contact, required=True))
class User(Owner):
email = HtmlSafeStrField()
# NOTE: deprecated. Only still used to migrate old API tokens
mist_api_token = me.StringField()
last_name = HtmlSafeStrField(default='')
feedback = me.EmbeddedDocumentField(Feedback, default=Feedback())
activation_key = me.StringField()
first_name = HtmlSafeStrField(default='')
invitation_accepted = me.FloatField()
invitation_date = me.FloatField()
last_login = me.FloatField()
password = me.StringField()
password_set_token = me.StringField()
password_set_token_created = me.FloatField()
password_set_user_agent = me.StringField()
registration_date = me.FloatField()
registration_method = me.StringField()
requested_demo = me.BooleanField()
demo_request_date = me.FloatField()
role = me.StringField()
status = me.StringField()
# these fields will exists only for org
# when migration from user to org completes
promo_codes = me.ListField()
selected_plan = me.StringField()
enterprise_plan = me.DictField()
open_id_url = HtmlSafeStrField()
password_reset_token_ip_addr = me.StringField()
password_reset_token = me.StringField()
password_reset_token_created = me.FloatField()
whitelist_ip_token_ip_addr = me.StringField()
whitelist_ip_token = me.StringField()
whitelist_ip_token_created = me.FloatField()
user_agent = me.StringField()
username = me.StringField()
can_create_org = me.BooleanField(default=True)
beta_access = me.BooleanField(default=True)
ips = me.EmbeddedDocumentListField(WhitelistIP, default=[])
meta = {
'indexes':
['email', 'first_name', 'last_name', 'username', 'last_login']
}
def __str__(self):
return 'User %s' % self.email
def set_password(self, password):
"""Update user's password."""
# could perform strength measuring first
hashed_pwd = pwd_context.encrypt(password)
self.password = hashed_pwd
self.save()
def check_password(self, password):
"""
Return True if password matches, False otherwise.
This will also update the password if it's using a deprecated scheme.
If user.password is empty because the user registered through SSO then
the password passed as argument should be empty otherwise False will be
returned.
"""
if not self.password or not password:
return False
ok, new_hash = pwd_context.verify_and_update(password, self.password)
if not ok:
return False
if new_hash:
# hashed password was using a deprecated scheme, update it
log.info("Updating user's password.")
self.password = new_hash
self.save()
return True
def __eq__(self, other):
return self.id == other.id
def clean(self):
# make sure user.email is unique - we can't use the unique keyword on
# the field definition because both User and Organization subclass
# Owner but only user has an email field
if User.objects(email=self.email, id__ne=self.id):
raise me.ValidationError("User with email '%s' already exists." %
self.email)
super(User, self).clean()
def get_nice_name(self):
try:
if self.first_name and not self.last_name:
return self.first_name + '(' + self.email + ')'
else:
name = (self.first_name or '') + ' ' + (self.last_name or '')
return name.strip() or self.email
except AttributeError:
return self.email
def get_ownership_mapper(self, org):
"""Return the `OwnershipMapper` in the specified Org context."""
if config.HAS_RBAC:
from mist.rbac.mappings import OwnershipMapper
else:
from mist.api.dummy.mappings import OwnershipMapper
return OwnershipMapper(self, org)
class Board(InheritableDocument):
posts = mongoengine.ListField(
mongoengine.EmbeddedDocumentField(EmbeddedPost))
class Organization(Owner):
name = me.StringField(required=True)
members = me.ListField(me.ReferenceField(User,
reverse_delete_rule=me.PULL),
required=True)
members_count = me.IntField(default=0)
teams = me.EmbeddedDocumentListField(Team, default=_get_default_org_teams)
teams_count = me.IntField(default=0)
clouds_count = me.IntField(default=0)
# These are assigned only to organization from now on
promo_codes = me.ListField()
selected_plan = me.StringField()
enterprise_plan = me.DictField()
enable_r12ns = me.BooleanField(required=True, default=False)
default_monitoring_method = me.StringField(
choices=config.MONITORING_METHODS)
insights_enabled = me.BooleanField(default=config.HAS_INSIGHTS)
ownership_enabled = me.BooleanField(default=True)
created = me.DateTimeField(default=datetime.datetime.now)
registered_by = me.StringField()
# used to allow creation of sub-org
super_org = me.BooleanField(default=False)
parent = me.ReferenceField('Organization',
required=False,
reverse_delete_rule=me.DENY)
poller_updated = me.DateTimeField()
meta = {'indexes': ['name']}
@property
def mapper(self):
"""Returns the `PermissionMapper` for the current Org context."""
if config.HAS_RBAC:
from mist.rbac.tasks import AsyncPermissionMapper
else:
from mist.api.dummy.mappings import AsyncPermissionMapper
return AsyncPermissionMapper(self)
def __str__(self):
return 'Org %s (%d teams - %d members)' % (self.name, len(
self.teams), len(self.members))
def get_email(self):
return self.teams.get(name='Owners').members[0].email
def get_emails(self):
emails = []
for user in self.teams.get(name='Owners').members:
emails.append(user.email)
return emails
def get_team(self, team_name):
try:
return self.teams.get(name=team_name)
except me.DoesNotExist:
raise TeamNotFound("No team found with name '%s'." % team_name)
def get_team_by_id(self, team_id):
try:
return self.teams.get(id=team_id)
except me.DoesNotExist:
raise TeamNotFound("No team found with id '%s'." % team_id)
def add_member_to_team(self, team_name, user):
team = self.get_team(team_name)
if user not in team.members:
team.members.append(user)
if user not in self.members:
self.members.append(user)
def add_member_to_team_by_id(self, team_id, user):
team = self.get_team_by_id(team_id)
if user not in team.members:
team.members.append(user)
if user not in self.members:
self.members.append(user)
def remove_member_from_team(self, team_name, user):
team = self.get_team(team_name)
for i, member in enumerate(team.members):
if user == member:
team.members.pop(i)
break
def remove_member_from_team_by_id(self, team_id, user):
team = self.get_team_by_id(team_id)
for i, member in enumerate(team.members):
if user == member:
team.members.pop(i)
break
def remove_member_from_members(self, user):
for i, member in enumerate(self.members):
if user == member:
self.members.pop(i)
break
def as_dict(self):
view = json.loads(self.to_json())
view_id = view["_id"]
del view["_id"]
del view["_cls"]
view["id"] = view_id
view["members"] = []
for member in self.members:
try:
name = member.get_nice_name()
except AttributeError: # Cannot dereference member
try:
self.members.remove(member)
self.save()
except Exception as e:
log.error("Failed to remove missing member from %s: %r" %
(self.name, e))
continue
view["members"].append({
"id": member.id,
"name": name,
"email": member.email,
"pending": False,
"parent": False
})
team_pending_members = {}
invitations = MemberInvitation.objects(org=self)
for invitation in invitations:
member = invitation.user
name = ""
name = (member.first_name or ' ') + (member.last_name or '')
name = (name.strip() or member.email)
view["members"].append({
"id": member.id,
"name": name,
"email": member.email,
"pending": True,
"parent": False,
})
for team_id in invitation.teams:
if team_id not in team_pending_members:
team_pending_members[team_id] = []
team_pending_members[team_id].append(member.id)
for team in view['teams']:
team["parent"] = False
if team['id'] in team_pending_members:
team['members'].extend(team_pending_members[team['id']])
# handle here the info from parent org
if self.parent:
view["parent_org_name"] = self.parent.name
parent_org = self.parent.as_dict()
parent_members = parent_org['members']
parent_teams = parent_org['teams']
for p_member in parent_members:
p_member['parent'] = True
view['members'].append(p_member)
for p_team in parent_teams:
p_team['parent'] = True
view["teams"].append(p_team)
return view
def clean(self):
# make sure that each team's name is unique
used = set()
for team in self.teams:
if team.name in used:
raise me.ValidationError("Team name exists.")
used.add(team.name)
# make sure that all team members are also org members
for team in self.teams:
for i, member in enumerate(list(team.members)):
if member not in self.members:
team.members.pop(i)
# make sure that owners team is present
try:
owners = self.teams.get(name='Owners')
except me.DoesNotExist:
raise me.ValidationError("Owners team can't be removed.")
# make sure that owners team is not empty
if not owners.members:
raise me.ValidationError("Owners team can't be empty.")
if config.HAS_RBAC:
# make sure owners policy allows all permissions
if owners.policy.operator != 'ALLOW':
owners.policy.operator = 'ALLOW'
log.warning("Owners policy must be set to ALLOW. Updating...")
# make sure owners policy doesn't contain specific rules
if owners.policy.rules:
raise me.ValidationError("Can't set policy rules for Owners.")
# Ensure RBAC Mappings are properly initialized.
for team in self.teams:
mappings = RBACMapping.objects(org=self.id,
team=team.id).only('id')
if not mappings:
team.init_mappings(org=self)
elif team.name == 'Owners':
raise me.ValidationError(
'RBAC Mappings are not intended for Team Owners')
elif len(mappings) is not 2:
raise me.ValidationError(
'RBAC Mappings have not been properly initialized for '
'Team %s' % team)
# make sure org name is unique - we can't use the unique keyword on the
# field definition because both User and Organization subclass Owner
# but only Organization has a name
if self.name and Organization.objects(name=self.name, id__ne=self.id):
raise me.ValidationError("Organization with name '%s' "
"already exists." % self.name)
self.members_count = len(self.members)
self.teams_count = len(self.teams)
# Add schedule for metering.
try:
from mist.api.poller.models import MeteringPollingSchedule
MeteringPollingSchedule.add(self, run_immediately=False)
except Exception as exc:
log.error('Error adding metering schedule for %s: %r', self, exc)
super(Organization, self).clean()
class MUserFeedNotification(mongo.Document):
'''A user's notifications of a single feed.'''
user_id = mongo.IntField()
feed_id = mongo.IntField()
frequency = mongoengine_fields.IntEnumField(NotificationFrequency)
is_focus = mongo.BooleanField()
last_notification_date = mongo.DateTimeField(default=datetime.datetime.now)
is_email = mongo.BooleanField()
is_web = mongo.BooleanField()
is_ios = mongo.BooleanField()
is_android = mongo.BooleanField()
ios_tokens = mongo.ListField(mongo.StringField(max_length=1024))
meta = {
'collection':
'notifications',
'indexes':
['feed_id', {
'fields': ['user_id', 'feed_id'],
'unique': True,
}],
'allow_inheritance':
False,
}
def __unicode__(self):
notification_types = []
if self.is_email: notification_types.append('email')
if self.is_web: notification_types.append('web')
if self.is_ios: notification_types.append('ios')
if self.is_android: notification_types.append('android')
return "%s/%s: %s -> %s" % (
User.objects.get(pk=self.user_id).username,
Feed.get_by_id(self.feed_id),
','.join(notification_types),
self.last_notification_date,
)
@classmethod
def feed_has_users(cls, feed_id):
return cls.users_for_feed(feed_id).count()
@classmethod
def users_for_feed(cls, feed_id):
notifications = cls.objects.filter(feed_id=feed_id)
return notifications
@classmethod
def feeds_for_user(cls, user_id):
notifications = cls.objects.filter(user_id=user_id)
notifications_by_feed = {}
for feed in notifications:
notifications_by_feed[feed.feed_id] = {
'notification_types': [],
'notification_filter': "focus" if feed.is_focus else "unread",
}
if feed.is_email:
notifications_by_feed[
feed.feed_id]['notification_types'].append('email')
if feed.is_web:
notifications_by_feed[
feed.feed_id]['notification_types'].append('web')
if feed.is_ios:
notifications_by_feed[
feed.feed_id]['notification_types'].append('ios')
if feed.is_android:
notifications_by_feed[
feed.feed_id]['notification_types'].append('android')
return notifications_by_feed
@classmethod
def push_feed_notifications(cls, feed_id, new_stories, force=False):
feed = Feed.get_by_id(feed_id)
notifications = MUserFeedNotification.users_for_feed(feed.pk)
logging.debug(
" ---> [%-30s] ~FCPushing out notifications to ~SB%s users~SN for ~FB~SB%s stories"
% (feed, len(notifications), new_stories))
r = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL)
latest_story_hashes = r.zrange("zF:%s" % feed.pk, -1 * new_stories, -1)
mstories = MStory.objects.filter(
story_hash__in=latest_story_hashes).order_by('-story_date')
stories = Feed.format_stories(mstories)
total_sent_count = 0
for user_feed_notification in notifications:
sent_count = 0
last_notification_date = user_feed_notification.last_notification_date
try:
usersub = UserSubscription.objects.get(
user=user_feed_notification.user_id,
feed=user_feed_notification.feed_id)
except UserSubscription.DoesNotExist:
continue
classifiers = user_feed_notification.classifiers(usersub)
if classifiers == None:
logging.debug("Has no usersubs")
continue
for story in stories:
if sent_count >= 3:
logging.debug("Sent too many, ignoring...")
continue
if story['story_date'] <= last_notification_date and not force:
logging.debug(
"Story date older than last notification date: %s <= %s"
% (story['story_date'], last_notification_date))
continue
if story[
'story_date'] > user_feed_notification.last_notification_date:
user_feed_notification.last_notification_date = story[
'story_date']
user_feed_notification.save()
story['story_content'] = HTMLParser().unescape(
story['story_content'])
sent = user_feed_notification.push_story_notification(
story, classifiers, usersub)
if sent:
sent_count += 1
total_sent_count += 1
return total_sent_count, len(notifications)
def classifiers(self, usersub):
classifiers = {}
if usersub.is_trained:
classifiers['feeds'] = list(
MClassifierFeed.objects(user_id=self.user_id,
feed_id=self.feed_id,
social_user_id=0))
classifiers['authors'] = list(
MClassifierAuthor.objects(user_id=self.user_id,
feed_id=self.feed_id))
classifiers['titles'] = list(
MClassifierTitle.objects(user_id=self.user_id,
feed_id=self.feed_id))
classifiers['tags'] = list(
MClassifierTag.objects(user_id=self.user_id,
feed_id=self.feed_id))
return classifiers
def title_and_body(self, story, usersub, notification_title_only=False):
def replace_with_newlines(element):
text = ''
for elem in element.recursiveChildGenerator():
if isinstance(elem, types.StringTypes):
text += elem
elif elem.name == 'br':
text += '\n'
elif elem.name == 'p':
text += '\n\n'
return text
feed_title = usersub.user_title or usersub.feed.feed_title
# title = "%s: %s" % (feed_title, story['story_title'])
title = feed_title
if notification_title_only:
subtitle = None
body = HTMLParser().unescape(story['story_title'])
else:
subtitle = HTMLParser().unescape(story['story_title'])
# body = HTMLParser().unescape(strip_tags(story['story_content']))
soup = BeautifulSoup(story['story_content'].strip())
# print story['story_content']
body = replace_with_newlines(soup)
body = truncate_chars(body.strip(), 600)
if not body:
body = " "
if not usersub.user.profile.is_premium:
body = "Please upgrade to a premium subscription to receive full push notifications."
return title, subtitle, body
def push_story_notification(self, story, classifiers, usersub):
story_score = self.story_score(story, classifiers)
if self.is_focus and story_score <= 0:
logging.debug("Is focus, but story is hidden")
return False
elif story_score < 0:
logging.debug("Is unread, but story is hidden")
return False
user = User.objects.get(pk=self.user_id)
logging.user(
user, "~FCSending push notification: %s/%s (score: %s)" %
(story['story_title'][:40], story['story_hash'], story_score))
self.send_web(story, user)
self.send_ios(story, user, usersub)
self.send_android(story)
self.send_email(story, usersub)
return True
def send_web(self, story, user):
if not self.is_web: return
r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
r.publish(
user.username,
'notification:%s,%s' % (story['story_hash'], story['story_title']))
def send_ios(self, story, user, usersub):
if not self.is_ios: return
apns = APNs(use_sandbox=False,
cert_file='/srv/newsblur/config/certificates/aps.pem',
key_file='/srv/newsblur/config/certificates/aps.p12.pem',
enhanced=True)
tokens = MUserNotificationTokens.get_tokens_for_user(self.user_id)
notification_title_only = is_true(
user.profile.preference_value('notification_title_only'))
title, subtitle, body = self.title_and_body(story, usersub,
notification_title_only)
image_url = None
if len(story['image_urls']):
image_url = story['image_urls'][0]
# print image_url
def response_listener(error_response):
logging.user(
user,
"~FRAPNS client get error-response: " + str(error_response))
apns.gateway_server.register_response_listener(response_listener)
for token in tokens.ios_tokens:
logging.user(
user, '~BMStory notification by iOS: ~FY~SB%s~SN~BM~FY/~SB%s' %
(story['story_title'][:50], usersub.feed.feed_title[:50]))
payload = Payload(alert={
'title': title,
'subtitle': subtitle,
'body': body
},
category="STORY_CATEGORY",
mutable_content=True,
custom={
'story_hash': story['story_hash'],
'story_feed_id': story['story_feed_id'],
'image_url': image_url,
})
apns.gateway_server.send_notification(token, payload)
def send_android(self, story):
if not self.is_android: return
def send_email(self, story, usersub):
if not self.is_email: return
feed = usersub.feed
story_content = self.sanitize_story(story['story_content'])
params = {
"story": story,
"story_content": story_content,
"feed": feed,
"feed_title": usersub.user_title or feed.feed_title,
"favicon_border": feed.favicon_color,
}
from_address = '*****@*****.**'
to_address = '%s <%s>' % (usersub.user.username, usersub.user.email)
text = render_to_string('mail/email_story_notification.txt', params)
html = render_to_string('mail/email_story_notification.xhtml', params)
subject = '%s: %s' % (usersub.user_title
or usersub.feed.feed_title, story['story_title'])
subject = subject.replace('\n', ' ')
msg = EmailMultiAlternatives(subject,
text,
from_email='NewsBlur <%s>' % from_address,
to=[to_address])
msg.attach_alternative(html, "text/html")
try:
msg.send()
except BotoServerError, e:
logging.user(usersub.user,
'~BMStory notification by email error: ~FR%s' % e)
return
logging.user(
usersub.user,
'~BMStory notification by email: ~FY~SB%s~SN~BM~FY/~SB%s' %
(story['story_title'][:50], usersub.feed.feed_title[:50]))
class ListFieldTest(InheritableDocument):
stringlist = mongoengine.ListField(mongoengine.StringField())
intlist = mongoengine.ListField(mongoengine.IntField())
anytype = mongoengine.ListField()
class FeedResult(Document):
"""
@summary: 订阅结果
"""
RECO_FEEDBACK = (
('title_match', '职位名称匹配'),
('title_not_match', '职位名称不匹配'),
('degree_low', '学历偏低'),
('degree_high', '学历偏高'),
('salary_low', '薪资偏低'),
('salary_high', '薪资偏高'),
('level_low', '级别偏低'),
('level_high', '级别偏高'),
('industry_not_match', '行业不匹配'),
)
feed = ReferenceField(Feed2, reverse_delete_rule=DO_NOTHING) # 所属的订阅
resume = ReferenceField(ResumeData,
reverse_delete_rule=DO_NOTHING) # 计算过的简历
resume_score = StringField(default='') # 该简历的这个关键词的得分
is_recommended = mongoengine.BooleanField(default=False) # 是否推荐该简历
calc_time = mongoengine.DateTimeField(default=datetime.now()) # 该订阅计算时间
resume_update_time = mongoengine.DateTimeField(
default=datetime.now()) # 改简历更新时间
resume_grab_time = mongoengine.DateTimeField(
default=datetime.now()) # 改简历的抓取时间
pub_time = mongoengine.DateTimeField(default=datetime.now()) # 简历发布时间
cos = mongoengine.FloatField(default=0.0) # 订阅职位描述和简历之间的余弦值
reco_index = mongoengine.FloatField(default=0.0) # 简历推荐指数
job_related = mongoengine.IntField(default=0) # 职位相关度
algorithm = StringField(default='')
talent_level = StringField(default='') # 人才级别
is_manual = mongoengine.BooleanField(default=False)
manual_ensure_time = mongoengine.DateTimeField() # 人工确认时间,也就是发布时间
published = mongoengine.BooleanField() # 人工确认该数据是否推送给用户
admin = StringField(default='') # 系统管理员才操作记录,积重难返了,这个字段表示操作的人的名称
is_staff = IntegerField(default=1) # 表示操作的账号是否为管理员账号
user_read_status = StringField(default='unread') # 用户阅读状态
admin_read_status = StringField(default='unread') # 管理员阅读状态
user_read_time = mongoengine.DateTimeField(required=False)
user_feedback_time = mongoengine.DateTimeField(required=False)
display_time = mongoengine.DateTimeField(required=False) # 简历显示时间
feed_source = StringField(default='')
tags = EmbeddedDocumentField(Tags, required=False)
publisher = StringField(default='')
resume_source = StringField(default='', required=False)
sync_partner = mongoengine.BooleanField(default=False)
# 展示数
display_count = mongoengine.IntField(default=0)
# 点击查看简历数
click_count = IntField(default=0)
watch = mongoengine.BooleanField(default=False, required=False)
download = mongoengine.BooleanField(default=False, required=False)
feedback_list = mongoengine.ListField(
mongoengine.StringField(choices=RECO_FEEDBACK),
required=False,
)
score = mongoengine.DictField()
meta = {
"collection":
"feed_result",
"index_background":
True,
'indexes': [
'feed',
'resume',
'-calc_time',
'-resume_update_time',
'job_related',
'-manual_ensure_time',
'admin',
'-display_time',
'-display_count',
'-click_count',
'user_read_time',
],
}
class EmbeddedListFieldTest(InheritableDocument):
embeddedlist = mongoengine.ListField(
mongoengine.EmbeddedDocumentField(EmbeddedPerson))
class Job(mongoengine.Document):
email = mongoengine.StringField(required=True)
request = mongoengine.DictField(required=True)
created_at = mongoengine.DateTimeField()
confirmed = mongoengine.BooleanField(required=True, default=False)
# Array where found jobs for this request are kept
# TODO add separate collection for this
found_jobs = mongoengine.ListField(null=False, default=[])
# Options that can be used for filtering
# queries additionally to the standard request
options = mongoengine.DictField(required=False)
# Encrypted fields
password = mongoengine.StringField(required=True)
user = mongoengine.StringField(required=True)
def save(self, *args, **kwargs):
self.__format_request()
self.created_at = datetime.datetime.now()
self.user = encrypt(self.user)
self.password = encrypt(self.password)
return super().save(*args, **kwargs)
def to_dict(self, remove_sensitive_data=False, minimize=False):
dict_value = json.loads(self.to_json())
dict_value['id'] = dict_value['_id']['$oid']
dict_value['created_at'] = dict_value['created_at']['$date']
del dict_value['_id']
if remove_sensitive_data:
del dict_value['user']
del dict_value['password']
if minimize:
dict_value['found_jobs_count'] = len(dict_value['found_jobs'])
del dict_value['found_jobs']
return dict_value
@staticmethod
def get_by_id(job_id: str):
return Job.objects.get(id=job_id)
def __format_request(self):
with open('./app/request_schema.json', 'r') as schema:
schema = json.loads(''.join(schema.readlines()))
Job.__extend_with_default(Draft4Validator)(schema).validate(
self.request)
return self.request
@staticmethod
def __extend_with_default(validator_class):
validate_properties = validator_class.VALIDATORS['properties']
def set_defaults(validator, properties, instance, schema):
for property_, subschema in properties.items():
if 'default' in subschema and not isinstance(instance, list):
instance.setdefault(property_, subschema['default'])
for error in validate_properties(validator, properties, instance,
schema):
yield error
return validators.extend(
validator_class,
{'properties': set_defaults},
)
class ReferencedListFieldTest(InheritableDocument):
referencedlist = mongoengine.ListField(mongoengine.ReferenceField(Person))
class ActionExecutionDB(stormbase.StormFoundationDB):
RESOURCE_TYPE = ResourceType.EXECUTION
UID_FIELDS = ['id']
trigger = stormbase.EscapedDictField()
trigger_type = stormbase.EscapedDictField()
trigger_instance = stormbase.EscapedDictField()
rule = stormbase.EscapedDictField()
action = stormbase.EscapedDictField(required=True)
runner = stormbase.EscapedDictField(required=True)
# Only the diff between the liveaction type and what is replicated
# in the ActionExecutionDB object.
liveaction = stormbase.EscapedDictField(required=True)
status = me.StringField(required=True,
help_text='The current status of the liveaction.')
start_timestamp = ComplexDateTimeField(
default=date_utils.get_datetime_utc_now,
help_text='The timestamp when the liveaction was created.')
end_timestamp = ComplexDateTimeField(
help_text='The timestamp when the liveaction has finished.')
parameters = stormbase.EscapedDynamicField(
default={},
help_text='The key-value pairs passed as to the action runner & action.'
)
result = stormbase.EscapedDynamicField(default={},
help_text='Action defined result.')
context = me.DictField(
default={},
help_text='Contextual information on the action execution.')
parent = me.StringField()
children = me.ListField(field=me.StringField())
log = me.ListField(field=me.DictField())
# Do not use URLField for web_url. If host doesn't have FQDN set, URLField validation blows.
web_url = me.StringField(required=False)
meta = {
'indexes': [{
'fields': ['rule.ref']
}, {
'fields': ['action.ref']
}, {
'fields': ['liveaction.id']
}, {
'fields': ['start_timestamp']
}, {
'fields': ['end_timestamp']
}, {
'fields': ['status']
}, {
'fields': ['parent']
}, {
'fields': ['rule.name']
}, {
'fields': ['runner.name']
}, {
'fields': ['trigger.name']
}, {
'fields': ['trigger_type.name']
}, {
'fields': ['trigger_instance.id']
}, {
'fields': ['context.user']
}, {
'fields': ['-start_timestamp', 'action.ref', 'status']
}]
}
def get_uid(self):
# TODO Construct od from non id field:
uid = [self.RESOURCE_TYPE, str(self.id)]
return ':'.join(uid)
def mask_secrets(self, value):
result = copy.deepcopy(value)
liveaction = result['liveaction']
parameters = {}
# pylint: disable=no-member
parameters.update(value.get('action', {}).get('parameters', {}))
parameters.update(value.get('runner', {}).get('runner_parameters', {}))
secret_parameters = get_secret_parameters(parameters=parameters)
result['parameters'] = mask_secret_parameters(
parameters=result['parameters'],
secret_parameters=secret_parameters)
if 'parameters' in liveaction:
liveaction['parameters'] = mask_secret_parameters(
parameters=liveaction['parameters'],
secret_parameters=secret_parameters)
if liveaction.get('action', '') == 'st2.inquiry.respond':
# Special case to mask parameters for `st2.inquiry.respond` action
# In this case, this execution is just a plain python action, not
# an inquiry, so we don't natively have a handle on the response
# schema.
#
# To prevent leakage, we can just mask all response fields.
#
# Note: The 'string' type in secret_parameters doesn't matter,
# it's just a placeholder to tell mask_secret_parameters()
# that this parameter is indeed a secret parameter and to
# mask it.
result['parameters']['response'] = mask_secret_parameters(
parameters=liveaction['parameters']['response'],
secret_parameters={
p: 'string'
for p in liveaction['parameters']['response']
})
# TODO(mierdin): This logic should be moved to the dedicated Inquiry
# data model once it exists.
if self.runner.get('name') == "inquirer":
schema = result['result'].get('schema', {})
response = result['result'].get('response', {})
# We can only mask response secrets if response and schema exist and are
# not empty
if response and schema:
result['result']['response'] = mask_inquiry_response(
response, schema)
return result
def get_masked_parameters(self):
"""
Retrieve parameters with the secrets masked.
:rtype: ``dict``
"""
serializable_dict = self.to_serializable_dict(mask_secrets=True)
return serializable_dict['parameters']
class EmbeddedListWithFlagFieldTest(InheritableDocument):
embeddedlist = mongoengine.ListField(
mongoengine.EmbeddedDocumentField(EmbeddedPerson))
is_published = mongoengine.BooleanField(default=False, required=True)
class Course(mongo.Document):
meta = {'db_alias': 'uw-tree', 'collection': 'courses'}
###################
# Data calculated #
###################
# The full course code for this course
# subject + ' ' + category e.g. CS 246
name = mongo.StringField(required=True)
# Computer readable prerequisites info evaluated by parser
evaluated_prereq = mongo.DictField(required=True)
# Computer readable prerequisites info evaluated by a teapot
teapot_prereq = mongo.DictField(required=False)
##################################
# Data provided by uwaterloo api #
##################################
# The digit id assigned to the course. e.g. 004380
course_id = mongo.StringField(required=True)
# The subject of the course. e.g. CS
subject = mongo.StringField(required=True)
# The category of the course. e.g. 246
catalog_number = mongo.StringField(required=True)
# The title of the course. e.g. Object-Oriented Software Development
title = mongo.StringField(required=True)
# The unit you can gain by finishing the course. e.g. 0.5
units = mongo.DecimalField(required=True)
# The course description. e.g. Introduction to object-oriented...
description = mongo.StringField(required=False)
# The activities may be involved in this course. e.g. ["LAB", "LEC", "TST", "TUT"]
instructions = mongo.ListField(required=False)
# The requirement you need to satisfy to enroll this course. e.g. CS 145 taken fall...
prerequisites = mongo.StringField(required=False)
# The antirequisite info of this course. e.g. SYDE 322
antirequisites = mongo.StringField(required=False)
# The corequisite info of this course.
corequisites = mongo.StringField(required=False)
# The crosslisting info of this course.
crosslistings = mongo.StringField(required=False)
# The terms our school offer this course. e.g. ["F", "W"]
terms_offered = mongo.ListField(required=False)
# The note of this course. e.g. [Note: Enrolment is restricted; ...
notes = mongo.StringField(required=False)
# The availibility info of this course. e.g. ["online":false, "online_only":false, ...
offerings = mongo.DictField(required=False)
# The needs_department_consent info.
needs_department_consent = mongo.BooleanField(required=False)
# The needs_instructor_consent info.
needs_instructor_consent = mongo.BooleanField(required=False)
# Some extra info.
extra = mongo.ListField(required=False)
# The calendar year of this course. e.g. 1516
calendar_year = mongo.StringField(required=False)
# The url of this course. e.g. http:\/\/www.ucalendar.uwaterloo.ca\/1516\/COURSE\/course-CS.html#CS246
url = mongo.StringField(required=False)
# The academic_level info of this course. e.g. undergraduate
academic_level = mongo.StringField(required=False)
# TODO fields
# keywords: words extract from the course
#
def setup(self, course_dic):
for key in course_dic:
self[key] = course_dic[key]
self.name = self.subject + ' ' + self.catalog_number
try:
self.evaluated_prereq = parse_prereq(self.prerequisites)
except:
self.evaluated_prereq = {}
self.teapot_prereq = {}
class Pipe(InheritableDocument):
name = mongoengine.StringField(required=True, unique=True)
exporters = mongoengine.ListField(
mongoengine.EmbeddedDocumentField(PipeExporterEmbedded))
class Node(mongoengine.Document):
"""
A single node in the state graph for Q-learning. The neighbours attribute
stores Q(n, a) for all actions which can be taken from this node.
"""
pivot = mongoengine.StringField(max_length=1, primary_key=True)
neighbours = mongoengine.ListField(mongoengine.EmbeddedDocumentField(
Neighbour))
n_updates = mongoengine.IntField(default=0, min_value=0)
def at(self, kanji):
"Gets the neighbour described by the given kanji."
for neighbour in self.neighbours:
if neighbour.kanji == kanji:
return neighbour
raise KeyError(kanji)
@classmethod
def build(cls, cache=None):
"Builds the initial graph for Q learning."
print 'Building neighbourhood graph'
n = settings.N_NEIGHBOURS_RECALLED
if cache is None:
cache = Similarity.load(n)
cls.drop_collection()
dist = cls._load_corpus_counts()
for kanji in _get_kanji():
node = Node(pivot=kanji, neighbours=[])
weights = {}
best_n = sorted(cache.get_heap(kanji).get_contents(),
reverse=True)[:n]
for weight, partner in best_n:
weights[partner] = weight * dist.prob(partner)
total_weights = sum(weights.itervalues())
for partner, weight in sorted(weights.iteritems(),
key=lambda p: p[1], reverse=True):
node.neighbours.append(Neighbour(kanji=partner,
weight=weight / total_weights))
node.save()
@classmethod
def _load_corpus_counts(cls):
input_file = os.path.join(settings.DATA_DIR,
'corpus', 'jp_char_corpus_counts.gz')
freq_dist = FreqDist()
with open(input_file, 'r') as istream:
istream = gzip.GzipFile(fileobj=istream)
istream = codecs.getreader('utf8')(istream)
for line in istream:
kanji, count = line.split()
freq_dist.inc(kanji, count=int(count))
return LaplaceProbDist(freq_dist)
@classmethod
def get_coverage(cls):
"Returns the set of kanji for which neighbours are stored."
db = cls.objects._collection
return set(r['_id'] for r in db.find({}, fields=['_id']))
@classmethod
def update(cls, path):
path = cls._remove_cycles(path)
nodes = cls.objects.filter(pivot__in=list(path))
if len(nodes) != len(path):
found_pivots = set(n.pivot for n in nodes)
missing_kanji = [k for k in path if k not in found_pivots][0]
raise ValueError('node not found in database for kanji %s' % \
missing_kanji.encode('utf8'))
# cache Q(s, a) for the subgraph we're interested in
q = cls._cache_subgraph(nodes)
# Calculate Q'(s, a) in reverse order along the path
# Q'(s, a) = (1 - A(s))Q(s, a) + A(s)*(r(a) + G * max_a Q(s', a))
gamma = settings.UPDATE_GAMMA
for i in xrange(len(path) - 2, -1, -1):
s = path[i]
q_s = q[s]
alpha = 1.0 / (4.0 + 0.5 * q_s.n_updates)
# update very action available from state s
for a in sorted(q_s.neighbours, key=lambda n: n.weight,
reverse=True):
q_old = a.weight
r_a = (1 if a.kanji == path[-1] else 0)
q_opt = r_a + gamma * max(q[a.kanji].neighbours).weight
a.weight = (1.0 - alpha) * q_old + alpha * q_opt
q_s.n_updates += 1
q_s.save()
@classmethod
def _cache_subgraph(cls, nodes):
q = {}
missing_neighbours = set()
for node in nodes:
q[node.pivot] = node
missing_neighbours.update(n.kanji for n in node.neighbours)
missing_neighbours.discard(node.pivot for node in nodes)
extra_nodes = cls.objects.filter(pivot__in=missing_neighbours)
if len(extra_nodes) != len(missing_neighbours):
raise ValueError('cannot cache subgraph -- neighbours missing')
for node in extra_nodes:
q[node.pivot] = node
return q
@staticmethod
def _remove_cycles(path):
# an O(n^2) method is good enough
i = 0
while i < len(path):
kanji = path[i]
j = path.find(kanji, i + 1)
if j >= 0:
# a cycle! trim the path to skip it
path = path[:i] + path[j:]
else:
i += 1
return path
def __unicode__(self):
return self.pivot
class ProfessorVector(mongoengine.Document):
meta = {'collection': 'test_professor_vector'}
vec = mongoengine.ListField(mongoengine.FloatField())
metadata = mongoengine.EmbeddedDocumentField(ProfessorMetadata)
