class UserWord(db.Model, util.JSONSerializable): __tablename__ = 'user_words' __table_args__ = {'mysql_collate': 'utf8_bin'} id = db.Column(db.Integer, primary_key=True) word = db.Column(db.String(255), nullable=False, unique=True) language_id = db.Column(db.String(2), db.ForeignKey("language.id")) language = db.relationship("Language") rank_id = db.Column(db.Integer, db.ForeignKey("word_ranks.id"), nullable=True) rank = db.relationship("WordRank") db.UniqueConstraint(word, language_id) IMPORTANCE_LEVEL_STEP = 1000 IMPOSSIBLE_RANK = 1000000 IMPOSSIBLE_IMPORTANCE_LEVEL = IMPOSSIBLE_RANK / IMPORTANCE_LEVEL_STEP def __init__(self, word, language, rank=None): self.word = word self.language = language self.rank = rank def __repr__(self): return '<UserWord %r>' % (self.word) def serialize(self): return self.word # returns a number between def importance_level(self): if self.rank is not None: return max((10 - self.rank.rank / UserWord.IMPORTANCE_LEVEL_STEP), 0) else: return 0 # we use this in the bookmarks.html to show the rank. # for words in which there is no rank info, we don't display anything def importance_level_string(self): if self.rank == None: return "" b = "|" return b * self.importance_level() @classmethod def find(cls, word, language, rank=None): try: return (cls.query.filter(cls.word == word).filter( cls.language == language).one()) except sqlalchemy.orm.exc.NoResultFound: return cls(word, language, rank) @classmethod def find_rank(cls, word, language): return WordRank.find(word, language) @classmethod def find_all(cls): return cls.query.all()
class WatchInteractionEvent(db.Model): __tablename__ = 'watch_interaction_event' __table_args__ = dict(mysql_collate='utf8_bin') id = db.Column(db.Integer, primary_key=True) time = db.Column(db.DateTime) bookmark_id = db.Column(db.Integer, db.ForeignKey('bookmark.id'), nullable=False) bookmark = db.relationship("Bookmark") event_type_id = db.Column(db.Integer, db.ForeignKey('watch_event_type.id'), nullable=False) event_type = db.relationship("WatchEventType") def __init__(self, event_type, bookmark_id, time): self.time = time self.bookmark_id = bookmark_id self.event_type = event_type def data_as_dictionary(self): return dict(user_id=self.bookmark.user_id, bookmark_id=self.bookmark_id, time=self.time.strftime("%Y-%m-%dT%H:%M:%S"), event=self.event_type.name) def is_learned_event(self): return self.event_type.name == "learnedIt" def is_wrong_translation_event(self): return self.event_type.name == "wrongTranslation" def prevents_further_study(self): """ Some events prevent a bookmark for being good for study in the future :return: """ return self.is_learned_event() or self.is_wrong_translation_event() @classmethod def events_for_bookmark(cls, bookmark): return cls.query.filter_by(bookmark_id=bookmark.id).all() @classmethod def events_for_bookmark_id(cls, bookmark_id): return cls.query.filter_by(bookmark_id=bookmark_id).all() @classmethod def events_for_user(cls, user): return cls.query.join(Bookmark).filter( Bookmark.user_id == user.id).all()
class Card(db.Model): id = db.Column(db.Integer, primary_key=True) position = db.Column(db.Integer) contribution_id = db.Column(db.Integer, db.ForeignKey('contribution.id')) contribution = db.relationship("Contribution", backref="card") last_seen = db.Column(db.DateTime) def __init__(self, contribution): self.contribution = contribution self.position = 0 self.reason = "" self.seen() def seen(self): self.last_seen = datetime.datetime.now() def set_reason(self, reason): self.reason = reason def reason(self): return self.reason def is_starred(self): return self.contribution.user.has_starred(self.contribution.origin) def star(self): word = self.contribution.origin self.contribution.user.starred_words.append(word) print "starred the hell out of... " + self.contribution.origin.word def unstar(self): word = self.contribution.origin self.contribution.user.starred_words.remove(word) print "just unstarred ..." + self.contribution.origin.word
class UserActivityData(db.Model): __table_args__ = dict(mysql_collate="utf8_bin") __tablename__ = 'user_activity_data' id = db.Column(db.Integer, primary_key=True) user_id = db.Column(db.Integer, db.ForeignKey("user.id")) user = db.relationship("User") time = db.Column(db.DateTime) event = db.Column(db.String(255)) value = db.Column(db.String(255)) extra_data = db.Column(db.String(4096)) def __init__(self, user, time, event, value, extra_data): self.user = user self.time = time self.event = event self.value = value self.extra_data = extra_data def data_as_dictionary(self): return dict(user_id=self.user_id, time=self.time.strftime("%Y-%m-%dT%H:%M:%S"), event=self.event, value=self.value, extra_data=self.extra_data)
class Card(db.Model): id = db.Column(db.Integer, primary_key=True) position = db.Column(db.Integer) bookmark_id = db.Column(db.Integer, db.ForeignKey('bookmark.id')) bookmark = db.relationship("Bookmark", backref="card") last_seen = db.Column(db.DateTime) def __init__(self, bookmark): self.bookmark = bookmark self.position = 0 self.reason = "" self.seen() def seen(self): self.last_seen = datetime.datetime.now() def set_reason(self, reason): self.reason = reason def reason(self): return self.reason def is_starred(self): return self.bookmark.user.has_starred(self.bookmark.origin) def star(self): word = self.bookmark.origin self.bookmark.user.starred_words.append(word) def unstar(self): word = self.bookmark.origin self.bookmark.user.starred_words.remove(word)
class Url(db.Model): __table_args__ = {'mysql_collate': 'utf8_bin'} id = db.Column(db.Integer, primary_key=True) title = db.Column(db.String(2083)) path = db.Column(db.String(2083)) url = db.Column(db.String(2083)) domain_name_id = db.Column(db.Integer, db.ForeignKey("domain_name.id")) domain = db.relationship("DomainName") def __init__(self, url, title): self.path = Url.get_path(url) self.domain = DomainName.find(Url.get_domain(url)) self.title = title def title_if_available(self): if self.title != "": return self.title return self.url def as_string(self): return self.domain.domain_name + self.path def domain_name(self): return self.domain.domain_name @classmethod def get_domain(self, url): protocol_re = '(.*://)?' domain_re = '([^/?]*)' path_re = '(.*)' domain = re.findall(protocol_re + domain_re, url)[0] return domain[0] + domain[1] @classmethod def get_path(self, url): protocol_re = '(.*://)?' domain_re = '([^/?]*)' path_re = '(.*)' domain = re.findall(protocol_re + domain_re + path_re, url)[0] return domain[2] @classmethod def find(cls, url, title=""): try: d = DomainName.find(Url.get_domain(url)) return (cls.query.filter(cls.path == Url.get_path(url)).filter( cls.domain == d).one()) except sqlalchemy.orm.exc.NoResultFound: return cls(url, title) def render_link(self, link_text): if self.url != "": return '<a href="' + self.url + '">' + link_text + '</a>' else: return ""
class Impression(db.Model): id = db.Column(db.Integer, primary_key=True) user_id = db.Column(db.Integer, db.ForeignKey("user.id")) user = db.relationship("User", backref="impressions") word_id = db.Column(db.Integer, db.ForeignKey("word.id")) word = db.relationship("Word") text_id = db.Column(db.Integer, db.ForeignKey("text.id")) text = db.relationship("Text") count = db.Column(db.Integer) last_search_id = db.Column(db.Integer, db.ForeignKey("search.id")) last_search = db.relationship("Search") def __init__(self, user, word, text=None): self.user = user self.word = word self.text = text def __repr__(self): return '<Impression %r>' % (self.word.word)
class RSSFeedRegistration(db.Model): __table_args__ = {'mysql_collate': 'utf8_bin'} __tablename__ = 'rss_feed_registration' id = db.Column(db.Integer, primary_key=True) user_id = db.Column(db.Integer, db.ForeignKey("user.id")) user = db.relationship("User") rss_feed_id = db.Column(db.Integer, db.ForeignKey("rss_feed.id")) rss_feed = relationship("RSSFeed") def __init__(self, user, feed): self.user = user self.rss_feed = feed @classmethod def find_or_create(cls, user, feed): try: return (cls.query.filter(cls.user == user).filter( cls.rss_feed == feed).one()) except sqlalchemy.orm.exc.NoResultFound: return cls(user, feed) @classmethod def feeds_for_user(cls, user): """ would have been nicer to define a method on the User class get feeds, but that would pollute the user model, and it's not nice. :param user: :return: """ return (cls.query.filter(cls.user == user)) @classmethod def with_id(cls, id): return (cls.query.filter(cls.id == id)).one() @classmethod def with_feed_id(cls, id, user): return (cls.query.filter(cls.rss_feed_id == id))\ .filter(cls.user_id == user.id).one()
class Exercise(db.Model): __table_args__ = {'mysql_collate': 'utf8_bin'} __tablename__ = 'exercise' id = db.Column(db.Integer, primary_key=True) outcome_id = db.Column(db.Integer, db.ForeignKey('exercise_outcome.id'), nullable=False) outcome = db.relationship("ExerciseOutcome", backref="exercise") source_id = db.Column(db.Integer, db.ForeignKey('exercise_source.id'), nullable=False) source = db.relationship("ExerciseSource", backref="exercise") solving_speed = db.Column(db.Integer) time = db.Column(db.DateTime, nullable=False) def __init__(self, outcome, source, solving_speed, time): self.outcome = outcome self.source = source self.solving_speed = solving_speed self.time = time
class Search(db.Model): id = db.Column(db.Integer, primary_key=True) user_id = db.Column(db.Integer, db.ForeignKey("user.id")) user = db.relationship("User", backref="searches") word_id = db.Column(db.Integer, db.ForeignKey("word.id")) word = db.relationship("Word") language_id = db.Column(db.String(2), db.ForeignKey("language.id")) language = db.relationship("Language") text_id = db.Column(db.Integer, db.ForeignKey("text.id")) text = db.relationship("Text") contribution_id = db.Column(db.Integer, db.ForeignKey("contribution.id")) contribution = db.relationship("Contribution", backref="search") def __init__(self, user, word, language, text=None): self.user = user self.word = word self.language = language self.text = text def __repr__(self): return '<Search %r>' % (self.word.word)
class Contribution(db.Model): id = db.Column(db.Integer, primary_key=True) origin_id = db.Column(db.Integer, db.ForeignKey('word.id')) origin = db.relationship("Word", primaryjoin=origin_id == Word.id, backref="translations") translation_id = db.Column(db.Integer, db.ForeignKey('word.id')) translation = db.relationship("Word", primaryjoin=translation_id == Word.id) user_id = db.Column(db.Integer, db.ForeignKey('user.id')) user = db.relationship("User", backref="contributions") text_id = db.Column(db.Integer, db.ForeignKey('text.id')) text = db.relationship("Text", backref="contributions") time = db.Column(db.DateTime) def __init__(self, origin, translation, user, text, time): self.origin = origin self.translation = translation self.user = user self.time = time self.text = text
class Search(db.Model): __table_args__ = {'mysql_collate': 'utf8_bin'} id = db.Column(db.Integer, primary_key=True) user_id = db.Column(db.Integer, db.ForeignKey("user.id")) user = db.relationship("User", backref="searches") word_id = db.Column(db.Integer, db.ForeignKey("user_words.id")) word = db.relationship("UserWord") language_id = db.Column(db.String(2), db.ForeignKey("language.id")) language = db.relationship("Language") text_id = db.Column(db.Integer, db.ForeignKey("text.id")) text = db.relationship("Text") bookmark_id = db.Column(db.Integer, db.ForeignKey("bookmark.id")) bookmark = db.relationship("Bookmark", backref="search") def __init__(self, user, word, language, text=None): self.user = user self.user_word = word self.language = language self.text = text def __repr__(self): return '<Search %r>' % (self.user_word.word)
class RankedWord(db.Model, util.JSONSerializable): __tablename__ = 'ranked_word' __table_args__ = {'mysql_collate': 'utf8_bin'} id = db.Column(db.Integer, primary_key=True) word = db.Column(db.String(255), nullable=False, unique=True, index=True) language_id = db.Column(db.String(2), db.ForeignKey("language.id")) language = db.relationship("Language") rank = db.Column(db.Integer) db.UniqueConstraint(word, language_id) def __init__(self, word, language, rank): self.word = word self.language = language self.rank = rank @classmethod def find(cls, word, language): word = word.lower() try: return (cls.query.filter(cls.word == word).filter( cls.language == language).one()) except sqlalchemy.orm.exc.NoResultFound: return None @classmethod def find_all(cls, language): return cls.query.filter(cls.language == language).all() @classmethod def exists(cls, word, language): word = word.lower() try: (cls.query.filter(cls.word == word).filter( cls.language == language).one()) return True except sqlalchemy.orm.exc.NoResultFound: return False @classmethod def words_list(cls): words_list = [] for word in cls.find_all(): words_list.append(word.word) return words_list
class Session(db.Model): __table_args__ = {'mysql_collate': 'utf8_bin'} id = db.Column(db.Integer, primary_key=True) user_id = db.Column(db.Integer, db.ForeignKey("user.id")) user = db.relationship("User") last_use = db.Column(db.DateTime) def __init__(self, user, id_): self.id = id_ self.user = user self.update_use_date() def update_use_date(self): self.last_use = datetime.datetime.now() @classmethod def for_user(cls, user): while True: id_ = random.randint(0, zeeguu.app.config.get("MAX_SESSION")) if cls.query.get(id_) is None: break return cls(user, id_) @classmethod def find_for_id(cls, session_id): try: return cls.query.filter(cls.id == session_id).one() except: return None # @classmethod # def find_for_user(cls, user): # return cls.query.filter(cls.user == user).order_by(desc(cls.last_use)).first() @classmethod def find_for_user(cls, user): s = cls.query.filter(cls.user == user).\ filter(cls.id < zeeguu.app.config.get("MAX_SESSION")).\ order_by(desc(cls.last_use)).first() if not s: s = cls.for_user(user) return s
class Session(db.Model): id = db.Column(db.Integer, primary_key=True) user_id = db.Column(db.Integer, db.ForeignKey("user.id")) user = db.relationship("User") last_use = db.Column(db.DateTime) def __init__(self, user, id_): self.id = id_ self.user = user self.update_use_date() def update_use_date(self): self.last_use = datetime.datetime.now() @classmethod def for_user(cls, user): while True: id_ = random.randint(0, 1 << 31) if cls.query.get(id_) is None: break return cls(user, id_)
class Text(db.Model): __table_args__ = {'mysql_collate': 'utf8_bin'} id = db.Column(db.Integer, primary_key=True) content = db.Column(db.String(10000)) content_hash = db.Column(db.LargeBinary(32)) language_id = db.Column(db.String(2), db.ForeignKey("language.id")) language = db.relationship("Language") url_id = db.Column(db.Integer, db.ForeignKey('url.id')) url = db.relationship("Url", backref="texts") def __init__(self, content, language, url): self.content = content self.language = language self.url = url self.content_hash = util.text_hash(content) def __repr__(self): return '<Text %r>' % (self.language.short) def words(self): for word in re.split(re.compile(u"[^\\w]+", re.U), self.content): yield UserWord.find(word, self.language) def shorten_word_context(self, given_word, max_word_count): # shorter_text = "" limited_words = [] words = self.content.split( ) # ==> gives me a list of the words ["these", "types", ",", "the"] word_count = len(words) if word_count <= max_word_count: return self.content for i in range(0, max_word_count): limited_words.append( words[i]) # lista cu primele max_length cuvinte shorter_text = ' '.join( limited_words) # string cu primele 'max_word_count' cuv # sometimes the given_word does not exist in the text. # in that case return a text containing max_length words if given_word not in words: return shorter_text if words.index(given_word) <= max_word_count: return shorter_text for i in range(max_word_count + 1, words.index(given_word) + 1): limited_words.append(words[i]) shorter_text = ' '.join(limited_words) return shorter_text @classmethod def find(cls, text, language): try: query = (cls.query.filter(cls.language == language).filter( cls.content_hash == util.text_hash(text))) if query.count() > 0: query = query.filter(cls.content == text) try: return query.one() except sqlalchemy.orm.exc.NoResultFound: pass return cls(text, language) except: import traceback traceback.print_exc()
class Bookmark(db.Model): __table_args__ = {'mysql_collate': 'utf8_bin'} id = db.Column(db.Integer, primary_key=True) origin_id = db.Column(db.Integer, db.ForeignKey('user_words.id')) origin = db.relationship("UserWord", primaryjoin=origin_id == UserWord.id, backref="translations") translations_list = relationship("UserWord", secondary="bookmark_translation_mapping") user_id = db.Column(db.Integer, db.ForeignKey('user.id')) user = db.relationship("User", backref="bookmarks") text_id = db.Column(db.Integer, db.ForeignKey('text.id')) text = db.relationship("Text", backref="bookmarks") time = db.Column(db.DateTime) exercise_log = relationship("Exercise", secondary="bookmark_exercise_mapping") def __init__(self, origin, translation, user, text, time): self.origin = origin self.translations_list.append(translation) self.user = user self.time = time self.text = text def add_new_exercise(self, exercise): self.exercise_log.append(exercise) def translation(self): return self.translations_list[0] def translations_rendered_as_text(self): return ", ".join(self.translation_words_list()) def translation_words_list(self): translation_words = [] for translation in self.translations_list: translation_words.append(translation.word) return translation_words def add_new_translation(self, translation): self.translations_list.append(translation) def remove_translation(self, translation): if translation in self.translations_list: self.translations_list.remove(translation) def add_exercise_outcome(self, exercise_source, exercise_outcome, exercise_solving_speed): new_source = ExerciseSource.query.filter_by( source=exercise_source).first() new_outcome = ExerciseOutcome.query.filter_by( outcome=exercise_outcome).first() exercise = Exercise(new_outcome, new_source, exercise_solving_speed, datetime.datetime.now()) self.add_new_exercise(exercise) db.session.add(exercise) @classmethod def find_all_filtered_by_user(cls): return cls.query.filter_by(user=flask.g.user).all() @classmethod def find(cls, b_id): return cls.query.filter_by(id=b_id).first() @classmethod def is_sorted_exercise_log_after_date_outcome(cls, outcome, bookmark): sorted_exercise_log_after_date = sorted(bookmark.exercise_log, key=lambda x: x.time, reverse=True) if sorted_exercise_log_after_date: if sorted_exercise_log_after_date[0].outcome.outcome == outcome: return True return False
class User(db.Model): id = db.Column(db.Integer, primary_key=True) email = db.Column(db.String(255), unique=True) name = db.Column(db.String(255)) password = db.Column(db.LargeBinary(255)) password_salt = db.Column(db.LargeBinary(255)) learned_language_id = db.Column( db.String(2), db.ForeignKey("language.id") ) learned_language = sqlalchemy.orm.relationship("Language", foreign_keys=[learned_language_id]) starred_words = relationship("Word", secondary="starred_words_association") native_language_id = db.Column( db.String (2), db.ForeignKey("language.id") ) native_language = sqlalchemy.orm.relationship("Language", foreign_keys=[native_language_id]) def __init__(self, email, name, password, learned_language=None, native_language = None): self.email = email self.name = name self.update_password(password) self.learned_language = learned_language or Language.default() self.native_language = native_language or Language.default_native_language() def __repr__(self): return '<User %r>' % (self.email) def has_starred(self,word): return word in self.starred_words def star(self, word): self.starred_words.append(word) print word.word + " is now starred for user " + self.name # TODO: Does this work without a commit here? To double check. def read(self, text): if (Impression.query.filter(Impression.user == self) .filter(Impression.text == text).count()) > 0: return for word in text.words(): self.impressions.append(Impression(self, word, text)) def set_learned_language(self, code): self.learned_language = Language.find(code) def set_native_language(self, code): self.native_language = Language.find(code) @classmethod def find(cls, email): return User.query.filter(User.email == email).one() @classmethod def find_by_id(cls, id): return User.query.filter(User.id == id).one() @sqlalchemy.orm.validates("email") def validate_email(self, col, email): if "@" not in email: raise ValueError("Invalid email address") return email @sqlalchemy.orm.validates("password") def validate_password(self, col, password): if password is None or len(password) == 0: raise ValueError("Invalid password") return password @sqlalchemy.orm.validates("name") def validate_name(self, col, name): if name is None or len(name) == 0: raise ValueError("Invalid username") return name def update_password(self, password): self.password_salt = "".join( chr(random.randint(0, 255)) for i in range(32) ) self.password = util.password_hash(password, self.password_salt) @classmethod def authorize(cls, email, password): try: user = cls.query.filter(cls.email == email).one() if user.password == util.password_hash(password, user.password_salt): return user except sqlalchemy.orm.exc.NoResultFound: return None def contribs_chronologically(self): return Contribution.query.filter_by(user_id=self.id).order_by(Contribution.time.desc()).all() def user_words(self): return map((lambda x: x.origin.word), self.all_contributions()) def all_contributions(self): return Contribution.query.filter_by(user_id=self.id).order_by(Contribution.time.desc()).all() def contribs_by_date(self): def extract_day_from_date(contrib): return (contrib, contrib.time.replace(contrib.time.year, contrib.time.month, contrib.time.day,0,0,0,0)) contribs = self.all_contributions() contribs_by_date = dict() for elem in map(extract_day_from_date, contribs): contribs_by_date.setdefault(elem[1],[]).append(elem[0]) sorted_dates = contribs_by_date.keys() sorted_dates.sort(reverse=True) return contribs_by_date, sorted_dates def unique_urls(self): urls = set() for c in self.all_contributions(): urls.add(c.text.url) return urls def recommended_urls(self): urls_to_words = {} for contrib in self.all_contributions(): if contrib.text.url.url != "undefined": urls_to_words.setdefault(contrib.text.url,0) urls_to_words [contrib.text.url] += contrib.origin.importance_level() return sorted(urls_to_words, key=urls_to_words.get, reverse=True)
class User(db.Model): __table_args__ = {'mysql_collate': 'utf8_bin'} id = db.Column(db.Integer, primary_key=True) email = db.Column(db.String(255), unique=True) name = db.Column(db.String(255)) password = db.Column(db.LargeBinary(255)) password_salt = db.Column(db.LargeBinary(255)) learned_language_id = db.Column(db.String(2), db.ForeignKey("language.id")) learned_language = sqlalchemy.orm.relationship( "Language", foreign_keys=[learned_language_id]) starred_words = relationship("UserWord", secondary="starred_words_association") native_language_id = db.Column(db.String(2), db.ForeignKey("language.id")) native_language = sqlalchemy.orm.relationship( "Language", foreign_keys=[native_language_id]) def __init__(self, email, username, password, learned_language=None, native_language=None): self.email = email self.name = username self.update_password(password) self.learned_language = learned_language or Language.default_learned() self.native_language = native_language or Language.default_native_language( ) def __repr__(self): return '<User %r>' % (self.email) def has_starred(self, word): return word in self.starred_words def star(self, word): self.starred_words.append(word) print word.word + " is now starred for user " + self.name # TODO: Does this work without a commit here? To double check. def set_learned_language(self, code): self.learned_language = Language.find(code) def set_native_language(self, code): self.native_language = Language.find(code) @classmethod def find(cls, email): return User.query.filter(User.email == email).one() @classmethod def find_by_id(cls, id): return User.query.filter(User.id == id).one() @sqlalchemy.orm.validates("email") def validate_email(self, col, email): if "@" not in email: raise ValueError("Invalid email address") return email @sqlalchemy.orm.validates("password") def validate_password(self, col, password): if password is None or len(password) == 0: raise ValueError("Invalid password") return password @sqlalchemy.orm.validates("name") def validate_name(self, col, name): if name is None or len(name) == 0: raise ValueError("Invalid username") return name def update_password(self, password): self.password_salt = "".join( chr(random.randint(0, 255)) for i in range(32)) self.password = util.password_hash(password, self.password_salt) @classmethod def authorize(cls, email, password): try: user = cls.query.filter(cls.email == email).one() if user.password == util.password_hash(password, user.password_salt): return user except sqlalchemy.orm.exc.NoResultFound: return None def bookmarks_chronologically(self): return Bookmark.query.filter_by(user_id=self.id).order_by( Bookmark.time.desc()).all() def user_words(self): return map((lambda x: x.origin.word), self.all_bookmarks()) def all_bookmarks(self): return Bookmark.query.filter_by(user_id=self.id).order_by( Bookmark.time.desc()).all() def bookmark_count(self): return len(self.all_bookmarks()) def word_count(self): return len(self.user_words()) def bookmarks_by_date(self): def extract_day_from_date(bookmark): return (bookmark, bookmark.time.replace(bookmark.time.year, bookmark.time.month, bookmark.time.day, 0, 0, 0, 0)) bookmarks = self.all_bookmarks() bookmarks_by_date = dict() for elem in map(extract_day_from_date, bookmarks): bookmarks_by_date.setdefault(elem[1], []).append(elem[0]) sorted_dates = bookmarks_by_date.keys() sorted_dates.sort(reverse=True) return bookmarks_by_date, sorted_dates def unique_urls(self): urls = set() for b in self.all_bookmarks(): urls.add(b.text.url) return urls def recommended_urls(self): urls_to_words = {} for bookmark in self.all_bookmarks(): if bookmark.text.url.url != "undefined": urls_to_words.setdefault(bookmark.text.url, 0) urls_to_words[ bookmark.text.url] += bookmark.origin.importance_level() return sorted(urls_to_words, key=urls_to_words.get, reverse=True) def get_known_bookmarks(self): bookmarks = Bookmark.find_all_filtered_by_user() i_know_bookmarks = [] for bookmark in bookmarks: if Bookmark.is_sorted_exercise_log_after_date_outcome( ExerciseOutcome.IKNOW, bookmark): i_know_bookmark_dict = {} i_know_bookmark_dict['id'] = bookmark.id i_know_bookmark_dict['origin'] = bookmark.origin.word i_know_bookmark_dict['text'] = bookmark.text.content i_know_bookmark_dict['time'] = bookmark.time.strftime( '%m/%d/%Y') i_know_bookmarks.append(i_know_bookmark_dict.copy()) return i_know_bookmarks def get_known_bookmarks_count(self): return len(self.get_known_bookmarks()) def get_estimated_vocabulary(self, lang): bookmarks = Bookmark.find_all_filtered_by_user() filtered_words_known_from_user_dict_list = [] marked_words_of_user_in_text = [] words_of_all_bookmarks_content = [] filtered_words_known_from_user = [] for bookmark in bookmarks: bookmark_content_words = re.sub("[^\w]", " ", bookmark.text.content).split() words_of_all_bookmarks_content.extend(bookmark_content_words) marked_words_of_user_in_text.append(bookmark.origin.word) words_known_from_user = [ word for word in words_of_all_bookmarks_content if word not in marked_words_of_user_in_text ] for word_known in words_known_from_user: if WordRank.exists(word_known.lower(), lang): filtered_words_known_from_user.append(word_known) zeeguu.db.session.commit() filtered_words_known_from_user = list( set(filtered_words_known_from_user)) for word in filtered_words_known_from_user: filtered_word_known_from_user_dict = {} filtered_word_known_from_user_dict['word'] = word filtered_words_known_from_user_dict_list.append( filtered_word_known_from_user_dict.copy()) return filtered_words_known_from_user_dict_list def get_estimated_vocabulary_for_learned_language(self): return self.get_estimated_vocabulary(self.learned_language) def get_estimated_vocabulary_count(self): return len(self.get_estimated_vocabulary_for_learned_language())
class Word(db.Model, util.JSONSerializable): id = db.Column(db.Integer, primary_key=True) word = db.Column(db.String(255)) language_id = db.Column(db.String(2), db.ForeignKey("language.id")) language = db.relationship("Language") word_rank = db.Column(db.Integer) IMPORTANCE_LEVEL_STEP = 1000 IMPOSSIBLE_RANK = 1000000 IMPOSSIBLE_IMPORTANCE_LEVEL = IMPOSSIBLE_RANK / IMPORTANCE_LEVEL_STEP def __init__(self, word, language): self.word = word self.language = language self.word_rank = self.get_rank_from_file() def __repr__(self): return '<Word %r>' % (self.word) def serialize(self): return self.word # if the word is not found, we assume a rank of 1000000 def get_rank_from_file(self): import codecs try: f=codecs.open(zeeguu.app.config.get("LANGUAGES_FOLDER").decode('utf-8')+self.language.id+".txt", encoding="iso-8859-1") all_words = f.readlines() all_words_without_space = [] for each_word in all_words: each_word_without_space = each_word[:-1] all_words_without_space.append(each_word_without_space) def importance_range(the_word, frequency_list): if the_word in frequency_list: position = frequency_list.index(the_word) return position else: return Word.IMPOSSIBLE_RANK return importance_range(self.word, all_words_without_space) except: return Word.IMPOSSIBLE_RANK def rank(self): if self.word_rank == None: self.word_rank = self.get_rank_from_file() session = sqlalchemy.orm.object_session(self) session.commit() return self.word_rank # returns a number between def importance_level(self): return max((10 - self.rank() / Word.IMPORTANCE_LEVEL_STEP), 0) # we use this in the contributions.html to show the rank. # for words in which there is no rank info, we don't display anything def importance_level_string(self): if self.rank() == Word.IMPOSSIBLE_RANK: return "" b = "|" return b * self.importance_level() @classmethod def find(cls, word, language): try: return (cls.query.filter(cls.word == word) .filter(cls.language == language) .one()) except sqlalchemy.orm.exc.NoResultFound: return cls(word, language) @classmethod def translate(cls, from_lang, term, to_lang): return (cls.query.join(WordAlias, cls.translation_of) .filter(WordAlias.word == term.lower()) .filter(cls.language == to_lang) .filter(WordAlias.language == from_lang) .all())
class KnownWordProbability(db.Model): __table_args__ = {'mysql_collate': 'utf8_bin'} __tablename__ = 'known_word_probability' id = db.Column(db.Integer, primary_key=True) user_id = db.Column(db.Integer, db.ForeignKey("user.id"), nullable=False) user = db.relationship("User") user_word_id = db.Column(db.Integer, db.ForeignKey('user_word.id'), nullable=True) user_word = db.relationship("UserWord") ranked_word_id = db.Column(db.Integer, db.ForeignKey("ranked_word.id"), nullable=True) ranked_word = db.relationship("RankedWord") probability = db.Column(db.DECIMAL(10, 9), nullable=False) db.CheckConstraint('probability>=0', 'probability<=1') def __init__(self, user, user_word, ranked_word, probability): self.user = user self.user_word = user_word self.ranked_word = ranked_word self.probability = probability def word_has_just_beek_bookmarked(self): self.probability /= 2 @classmethod def calculateKnownWordProb(cls, exerciseProb, encounterProb): return 0.8 * float(exerciseProb) + 0.2 * float(encounterProb) @classmethod def find(cls, user, user_word, ranked_word, probability=None): try: return cls.query.filter_by(user=user, user_word=user_word, ranked_word=ranked_word).one() except sqlalchemy.orm.exc.NoResultFound: return cls(user, user_word, ranked_word, probability) @classmethod def find_all_by_user(cls, user): return cls.query.filter_by(user=user).all() @classmethod def find_all_by_user_cached(cls, user): known_probabilities_cache = {} known_probabilities = cls.find_all_by_user(user) for known_probability in known_probabilities: user_word = known_probability.user_word # TODO: Why are there many KnownWordProbabilities with no user word in the database? if user_word is not None: known_probabilities_cache[ user_word.word] = known_probability.probability return known_probabilities_cache @classmethod def find_all_by_user_with_rank(cls, user): known_probs = cls.query.filter_by(user=user).all() for p in known_probs: if p.ranked_word is None: known_probs.remove(p) return known_probs @classmethod def exists(cls, user, user_word, ranked_word): try: cls.query.filter_by(user=user, user_word=user_word, ranked_word=ranked_word).one() return True except sqlalchemy.orm.exc.NoResultFound: return False @classmethod def get_probably_known_words(cls, user): return cls.query.filter(cls.user == user).filter( cls.probability >= 0.9).all()
class User(db.Model): __table_args__ = {'mysql_collate': 'utf8_bin'} id = db.Column(db.Integer, primary_key=True) email = db.Column(db.String(255), unique=True) name = db.Column(db.String(255)) password = db.Column(db.LargeBinary(255)) password_salt = db.Column(db.LargeBinary(255)) learned_language_id = db.Column(db.String(2), db.ForeignKey("language.id")) learned_language = sqlalchemy.orm.relationship( "Language", foreign_keys=[learned_language_id]) starred_words = relationship("UserWord", secondary="starred_words_association") native_language_id = db.Column(db.String(2), db.ForeignKey("language.id")) native_language = sqlalchemy.orm.relationship( "Language", foreign_keys=[native_language_id]) def __init__(self, email, username, password, learned_language=None, native_language=None): self.email = email self.name = username self.update_password(password) self.learned_language = learned_language or Language.default_learned() self.native_language = native_language or Language.default_native_language( ) def __repr__(self): return '<User %r>' % (self.email) def has_starred(self, word): return word in self.starred_words def star(self, word): self.starred_words.append(word) print word.word + " is now starred for user " + self.name # TODO: Does this work without a commit here? To double check. def set_learned_language(self, code): self.learned_language = Language.find(code) def set_native_language(self, code): self.native_language = Language.find(code) @classmethod def find_all(cls): return User.query.all() @classmethod def find(cls, email): return User.query.filter(User.email == email).one() @classmethod def find_by_id(cls, id): return User.query.filter(User.id == id).one() @sqlalchemy.orm.validates("email") def validate_email(self, col, email): if "@" not in email: raise ValueError("Invalid email address") return email @sqlalchemy.orm.validates("password") def validate_password(self, col, password): if password is None or len(password) == 0: raise ValueError("Invalid password") return password @sqlalchemy.orm.validates("name") def validate_name(self, col, name): if name is None or len(name) == 0: raise ValueError("Invalid username") return name def update_password(self, password): self.password_salt = "".join( chr(random.randint(0, 255)) for i in range(32)) self.password = util.password_hash(password, self.password_salt) @classmethod def authorize(cls, email, password): try: user = cls.query.filter(cls.email == email).one() if user.password == util.password_hash(password, user.password_salt): return user except sqlalchemy.orm.exc.NoResultFound: return None def bookmarks_chronologically(self): return Bookmark.query.filter_by(user_id=self.id).order_by( Bookmark.time.desc()).all() def user_words(self): return map((lambda x: x.origin.word), self.all_bookmarks()) def all_bookmarks(self): return Bookmark.query.filter_by(user_id=self.id).order_by( Bookmark.time.desc()).all() def bookmark_count(self): return len(self.all_bookmarks()) def word_count(self): return len(self.user_words()) def bookmarks_by_date(self): def extract_day_from_date(bookmark): return (bookmark, bookmark.time.replace(bookmark.time.year, bookmark.time.month, bookmark.time.day, 0, 0, 0, 0)) bookmarks = self.all_bookmarks() bookmarks_by_date = dict() for elem in map(extract_day_from_date, bookmarks): bookmarks_by_date.setdefault(elem[1], []).append(elem[0]) sorted_dates = bookmarks_by_date.keys() sorted_dates.sort(reverse=True) return bookmarks_by_date, sorted_dates # returns only HTTP domains. in this way we filter # out empty domains, and others like the android: # that we use for internal tracking... # Returns: list of tuples (domain, date) def recent_domains_with_times(self): domains = [] domains_and_times = [] for b in self.bookmarks_chronologically(): if not b.text.url.domain() in domains\ and 'http' in b.text.url.domain(): domains_and_times.append([b.text.url.domain(), b.time]) domains.append(b.text.url.domain()) return domains_and_times def frequent_domains(self): domains = map(lambda b: b.text.url.domain(), self.bookmarks_chronologically()) from collections import Counter counter = Counter(domains) return counter.most_common() def unique_urls(self): urls = set() for b in self.all_bookmarks(): urls.add(b.text.url) return urls def recommended_urls(self): urls_to_words = {} for bookmark in self.all_bookmarks(): if bookmark.text.url.url != "undefined": urls_to_words.setdefault(bookmark.text.url, 0) urls_to_words[ bookmark.text.url] += bookmark.origin.importance_level() return sorted(urls_to_words, key=urls_to_words.get, reverse=True) def get_not_encountered_words(self, lang): not_encountered_words_dict_list = [] all_ranks = RankedWord.find_all(lang) known_word_probs = KnownWordProbability.find_all_by_user_with_rank( self) for p in known_word_probs: if p.ranked_word in all_ranks: all_ranks.remove(p.ranked_word) for rank in all_ranks: not_encountered_word_dict = {} not_encountered_word_dict['word'] = rank.word not_encountered_words_dict_list.append(not_encountered_word_dict) return not_encountered_words_dict_list def get_not_encountered_words_count(self): return len(self.get_not_encountered_words(self.learned_language)) def get_known_bookmarks(self, lang): bookmarks = flask.g.user.all_bookmarks() known_bookmarks = [] for bookmark in bookmarks: if bookmark.check_is_latest_outcome_too_easy( ) and lang == bookmark.origin.language: known_bookmark_dict = { 'id': bookmark.id, 'origin': bookmark.origin.word, 'text': bookmark.text.content, 'time': bookmark.time.strftime('%m/%d/%Y') } known_bookmarks.append(known_bookmark_dict) return known_bookmarks def get_known_bookmarks_count(self): return len(self.get_known_bookmarks(self.learned_language)) def get_not_looked_up_words(self, lang): filtered_words_known_from_user_dict_list = [] enc_probs = EncounterBasedProbability.find_all_by_user(flask.g.user) for enc_prob in enc_probs: if enc_prob.ranked_word.language == lang: filtered_words_known_from_user_dict_list.append( {'word': enc_prob.ranked_word.word}) return filtered_words_known_from_user_dict_list def get_not_looked_up_words_for_learned_language(self): return self.get_not_looked_up_words(self.learned_language) def get_not_looked_up_words_count(self): return len(self.get_not_looked_up_words_for_learned_language()) def get_probably_known_words(self, lang): known_word_prob_of_user = KnownWordProbability.get_probably_known_words( self) probable_known_words_dict_list = [] for known_word_prob in known_word_prob_of_user: probable_known_word_dict = {} if known_word_prob.ranked_word is not None and known_word_prob.ranked_word.language == lang: probable_known_word_dict[ 'word'] = known_word_prob.ranked_word.word else: probable_known_word_dict[ 'word'] = known_word_prob.user_word.word probable_known_words_dict_list.append(probable_known_word_dict) return probable_known_words_dict_list def get_probably_known_words_count(self): return len(self.get_probably_known_words(self.learned_language)) def get_lower_bound_percentage_of_basic_vocabulary(self): high_known_word_prob_of_user = KnownWordProbability.get_probably_known_words( self) count_high_known_word_prob_of_user_ranked = 0 for prob in high_known_word_prob_of_user: if prob.ranked_word is not None and prob.ranked_word.rank <= 3000: count_high_known_word_prob_of_user_ranked += 1 return round( float(count_high_known_word_prob_of_user_ranked) / 3000 * 100, 2) def get_upper_bound_percentage_of_basic_vocabulary(self): count_not_looked_up_words_with_rank = 0 not_looked_up_words = EncounterBasedProbability.find_all_by_user(self) for prob in not_looked_up_words: if prob.ranked_word.rank <= 3000: count_not_looked_up_words_with_rank += 1 return round( float(count_not_looked_up_words_with_rank) / 3000 * 100, 2) def get_lower_bound_percentage_of_extended_vocabulary(self): high_known_word_prob_of_user = KnownWordProbability.get_probably_known_words( self) count_high_known_word_prob_of_user_ranked = 0 for prob in high_known_word_prob_of_user: if prob.ranked_word is not None and prob.ranked_word.rank <= 10000: count_high_known_word_prob_of_user_ranked += 1 return round( float(count_high_known_word_prob_of_user_ranked) / 10000 * 100, 2) def get_upper_bound_percentage_of_extended_vocabulary(self): count_not_looked_up_words_with_rank = 0 not_looked_up_words = EncounterBasedProbability.find_all_by_user(self) for prob in not_looked_up_words: if prob.ranked_word.rank <= 10000: count_not_looked_up_words_with_rank += 1 return round( float(count_not_looked_up_words_with_rank) / 10000 * 100, 2) def get_percentage_of_probably_known_bookmarked_words(self): high_known_word_prob_of_user = KnownWordProbability.get_probably_known_words( self) count_high_known_word_prob_of_user = 0 count_bookmarks_of_user = len(self.all_bookmarks()) for prob in high_known_word_prob_of_user: if prob.user_word is not None: count_high_known_word_prob_of_user += 1 if count_bookmarks_of_user <> 0: return round( float(count_high_known_word_prob_of_user) / count_bookmarks_of_user * 100, 2) else: return 0 # Reading recommendations def recommendations(self): recommendations = { 'de': [[ 'Der Spiegel', 'http://m.spiegel.de', 'news, advanced', 'World News' ]] } try: return recommendations[self.learned_language_id] except: return []
class RSSFeed(db.Model): __table_args__ = {'mysql_collate': 'utf8_bin'} __tablename__ = 'rss_feed' id = db.Column(db.Integer, primary_key=True) title = db.Column(db.String(2083)) description = db.Column(db.String(2083)) language_id = db.Column(db.String(2), db.ForeignKey("language.id")) language = db.relationship("Language") url_id = db.Column(db.Integer, db.ForeignKey("url.id")) url = db.relationship("Url", foreign_keys='RSSFeed.url_id') image_url_id = db.Column(db.Integer, db.ForeignKey("url.id")) image_url = db.relationship("Url", foreign_keys='RSSFeed.image_url_id') def __init__(self, url, title, description, image_url=None, language=None): self.url = url self.image_url = image_url self.title = title self.language = language self.description = description def as_dictionary(self): image_url = "" if self.image_url: image_url = self.image_url.as_string() return dict(id=self.id, title=self.title, url=self.url.as_string(), description=self.description, language=self.language.id, image_url=image_url) def feed_items(self): feed_data = feedparser.parse(self.url.as_string()) feed_items = [ dict(title=item.get("title", ""), url=item.get("link", ""), content=item.get("content", ""), summary=item.get("summary", ""), published=time.strftime("%Y-%m-%dT%H:%M:%S%z", item.published_parsed)) for item in feed_data.entries ] return feed_items @classmethod def find_by_url(cls, url): try: result = (cls.query.filter(cls.url == url).one()) # print "found an existing RSSFeed object" return result except: return None @classmethod def find_or_create(cls, url, title, description, image_url, language): try: result = ( cls.query.filter(cls.url == url).filter( cls.title == title).filter(cls.language == language) # .filter(cls.image_url == image_url) .filter(cls.description == description).one()) # print "found an existing RSSFeed object" return result except sqlalchemy.orm.exc.NoResultFound: # print "creating new feed object for " + title return cls(url, title, description, image_url, language) @classmethod def find_for_language_id(cls, language_id): return cls.query.filter(cls.language_id == language_id).group_by( cls.title).all()
class Bookmark(db.Model): __table_args__ = {'mysql_collate': 'utf8_bin'} id = db.Column(db.Integer, primary_key=True) origin_id = db.Column(db.Integer, db.ForeignKey('user_word.id')) origin = db.relationship("UserWord", primaryjoin=origin_id == UserWord.id, backref="translations") translations_list = relationship("UserWord", secondary="bookmark_translation_mapping") user_id = db.Column(db.Integer, db.ForeignKey('user.id')) user = db.relationship("User", backref="bookmarks") text_id = db.Column(db.Integer, db.ForeignKey('text.id')) text = db.relationship("Text", backref="bookmarks") time = db.Column(db.DateTime) exercise_log = relationship("Exercise", secondary="bookmark_exercise_mapping") def __init__(self, origin, translation, user, text, time): self.origin = origin self.translations_list.append(translation) self.user = user self.time = time self.text = text def add_new_exercise(self, exercise): self.exercise_log.append(exercise) def translation(self): return self.translations_list[0] def translations_rendered_as_text(self): return ", ".join(self.translation_words_list()) def translation_words_list(self): translation_words = [] for translation in self.translations_list: translation_words.append(translation.word) return translation_words def add_new_translation(self, translation): self.translations_list.append(translation) def context_is_not_too_long(self): return len(self.text.content) < 60 def events_prevent_further_study(self): from zeeguu.model.smartwatch.watch_interaction_event import WatchInteractionEvent events_for_self = WatchInteractionEvent.events_for_bookmark(self) return any([x.prevents_further_study() for x in events_for_self]) def good_for_study(self): # ML TODO: Must replace call to check_is_latest_outcome... with has_been_learned! return not self.check_is_latest_outcome_too_easy( ) and not self.events_prevent_further_study() def remove_translation(self, translation): if translation in self.translations_list: self.translations_list.remove(translation) def add_exercise_outcome(self, exercise_source, exercise_outcome, exercise_solving_speed): new_source = ExerciseSource.query.filter_by( source=exercise_source).first() new_outcome = ExerciseOutcome.query.filter_by( outcome=exercise_outcome).first() exercise = Exercise(new_outcome, new_source, exercise_solving_speed, datetime.datetime.now()) self.add_new_exercise(exercise) db.session.add(exercise) def split_words_from_context(self): words_of_bookmark_content = [] bookmark_content_words = re.findall(r'(?u)\w+', self.text.content) words_of_bookmark_content.extend(bookmark_content_words) return words_of_bookmark_content def context_words_with_rank(self): ranked_context_words = self.split_words_from_context() while self.origin.word in ranked_context_words: ranked_context_words.remove(self.origin.word) filtered_words_known_from_user = [] for word_known in ranked_context_words: if RankedWord.exists(word_known.lower(), self.origin.language): filtered_words_known_from_user.append(word_known) return filtered_words_known_from_user def json_serializable_dict(self, with_context=True): result = dict(id=self.id, to=self.translation_words_list(), from_lang=self.origin.language_id, to_lang=self.translation().language.id, title=self.text.url.title, url=self.text.url.as_string(), origin_rank=self.origin.get_rank()) result["from"] = self.origin.word if with_context: result['context'] = self.text.content return result def calculate_probabilities_after_adding_a_bookmark(self, user, language): """ ML: This has to be refactored. It's a mess. The idea is: you've just added a bookmark. There are two things to do: 1. update the probabilities of the context words (they have been encountered, and not translated) 2. update the probabilities of the word itself - :param user: :param language: :return: """ # 1. computations for adding encounter based probability for the context words for word in self.context_words_with_rank(): enc_prob = EncounterBasedProbability.find_or_create( word, user, language) zeeguu.db.session.add(enc_prob) zeeguu.db.session.commit() user_word = None ranked_word = enc_prob.ranked_word if UserWord.exists(word, language): user_word = UserWord.find(word, language) if ExerciseBasedProbability.exists( user, user_word ): #checks if exercise based probability exists for words in context ex_prob = ExerciseBasedProbability.find(user, user_word) known_word_prob = KnownWordProbability.find( user, user_word, ranked_word) known_word_prob.probability = known_word_prob.calculateKnownWordProb( ex_prob.probability, enc_prob.probability ) #updates known word probability as exercise based probability already existed. else: if KnownWordProbability.exists(user, user_word, ranked_word): known_word_prob = KnownWordProbability.find( user, user_word, ranked_word) known_word_prob.probability = enc_prob.probability # updates known word probability as encounter based probability already existed else: known_word_prob = KnownWordProbability.find( user, user_word, ranked_word, enc_prob.probability ) # new known word probability created as it did not exist zeeguu.db.session.add(known_word_prob) # 2. Update the probabilities of the word itself # 2.a) exercise based prob # ML: Should this thing change? # The ex based probability should probably not change after I add a bookmark # Commenting out the following lines: s # ex_prob = ExerciseBasedProbability.find(user, self.origin) # if ex_prob: # ex_prob.update_probability_after_adding_bookmark_with_same_word(self,user) # zeeguu.db.session.add(ex_prob) # 2.b) encounter based prob ranked_word = RankedWord.find(self.origin.word, language) if ranked_word: #checks if ranked_word exists for that looked up word if EncounterBasedProbability.exists( user, ranked_word ): # checks if encounter based probability exists for that looked up word enc_prob = EncounterBasedProbability.find(user, ranked_word) enc_prob.word_has_just_beek_bookmarked() db.session.add(enc_prob) db.session.commit() # 2.c) update known word probability if it exists if KnownWordProbability.exists(user, self.origin, ranked_word): known_word_prob = KnownWordProbability.find( user, self.origin, ranked_word) known_word_prob.word_has_just_beek_bookmarked() db.session.add(known_word_prob) db.session.commit() @classmethod def find_by_specific_user(cls, user): return cls.query.filter_by(user=user).all() @classmethod def find_all(cls): return cls.query.filter().all() @classmethod def find_all_for_text(cls, text): return cls.query.filter(cls.text == text).all() @classmethod def find(cls, b_id): return cls.query.filter_by(id=b_id).first() @classmethod def find_all_by_user_and_word(cls, user, word): return cls.query.filter_by(user=user, origin=word).all() @classmethod def find_all_by_user_word_and_text(cls, user, word, text): return cls.query.filter_by(user=user, origin=word, text=text).all() # @classmethod # def is_sorted_exercise_log_after_date_outcome(cls,outcome, bookmark): # sorted_exercise_log_after_date=sorted(bookmark.exercise_log, key=lambda x: x.time, reverse=True) # if sorted_exercise_log_after_date: # if sorted_exercise_log_after_date[0].outcome.outcome == outcome: # return True # return False def check_is_latest_outcome_too_easy(self, add_to_result_time=False): sorted_exercise_log_by_latest = sorted(self.exercise_log, key=lambda x: x.time, reverse=True) for exercise in sorted_exercise_log_by_latest: if exercise.outcome.outcome == ExerciseOutcome.TOO_EASY: if add_to_result_time: return True, exercise.time return True elif exercise.outcome.outcome == ExerciseOutcome.SHOW_SOLUTION or exercise.outcome.outcome == ExerciseOutcome.WRONG: if add_to_result_time: return False, None return False if add_to_result_time: return False, None return False def check_if_learned_based_on_exercise_outcomes(self, add_to_result_time=False): """ TODO: This should replace check_is_latest_outcome in the future... :param add_to_result_time: :return: """ sorted_exercise_log_by_latest = sorted(self.exercise_log, key=lambda x: x.time, reverse=True) if sorted_exercise_log_by_latest: last_exercise = sorted_exercise_log_by_latest[0] # If last outcome is TOO EASY we know it if last_exercise.outcome.outcome == ExerciseOutcome.TOO_EASY: if add_to_result_time: return True, last_exercise.time return True CORRECTS_IN_A_ROW = 5 if len(sorted_exercise_log_by_latest) > CORRECTS_IN_A_ROW: # If we got it right for the last CORRECTS_IN_A_ROW times, we know it if all(exercise.outcome.outcome == ExerciseOutcome.CORRECT for exercise in sorted_exercise_log_by_latest[0:CORRECTS_IN_A_ROW - 1]): return True, last_exercise.time if add_to_result_time: return False, None return False def events_indicate_its_learned(self): from zeeguu.model.smartwatch.watch_interaction_event import WatchInteractionEvent events_for_self = WatchInteractionEvent.events_for_bookmark(self) for event in events_for_self: if event.is_learned_event(): return True, event.time return False, None def has_been_learned(self, also_return_time=False): """ :param also_return_time: should the function return also the time when the bookmark has been learned? :return: boolean indicating whether the bookmark has already been learned, togetgher with the time when it was learned if also_return_time is set """ # The first case is when we have an exercise outcome set to Too EASY learned, time = self.check_if_learned_based_on_exercise_outcomes(True) if learned: if also_return_time: return True, time else: return True # The second case is when we have an event in the smartwatch event log # that indicates that the word has been learned learned, time = self.events_indicate_its_learned() if learned: return learned, time if also_return_time: return False, None return False
class ExerciseBasedProbability(db.Model): __tablename__ = 'exercise_based_probability' __table_args__ = {'mysql_collate': 'utf8_bin'} id = db.Column(db.Integer, primary_key=True) user_id = db.Column(db.Integer, db.ForeignKey("user.id"), nullable=False) user = db.relationship("User") user_word_id = db.Column(db.Integer, db.ForeignKey('user_word.id'), nullable=False) user_word = db.relationship("UserWord") probability = db.Column(db.DECIMAL(10, 9), nullable=False) db.UniqueConstraint(user_id, user_word_id) db.CheckConstraint('probability>=0', 'probability<=1') DEFAULT_MIN_PROBABILITY = 0.1 DEFAULT_MAX_PROBABILITY = 1.0 def __init__(self, user, user_word, probability): self.user = user self.user_word = user_word self.probability = probability @classmethod def find(cls, user, user_word): try: return cls.query.filter_by(user=user, user_word=user_word).one() except sqlalchemy.orm.exc.NoResultFound: return cls(user, user_word, 0.1) @classmethod def exists(cls, user, user_word): try: cls.query.filter_by(user=user, user_word=user_word).one() return True except sqlalchemy.orm.exc.NoResultFound: return False @classmethod def find_all(cls): return cls.query.all() def wrong_formula(self, count_wrong_after_another): if self.DEFAULT_MIN_PROBABILITY * count_wrong_after_another >= float( self.probability): self.probability = decimal.Decimal('0.1') else: self.probability = ( float(self.probability) - self.DEFAULT_MIN_PROBABILITY * count_wrong_after_another) def correct_formula(self, count_correct_after_another): if float( self.probability ) + self.DEFAULT_MIN_PROBABILITY * count_correct_after_another >= 1.0: self.probability = decimal.Decimal('1.0') else: self.probability = ( float(self.probability) + self.DEFAULT_MIN_PROBABILITY * count_correct_after_another) def update_probability_after_adding_bookmark_with_same_word( self, bookmark, user): count_bookmarks_with_same_word = len( Bookmark.find_all_by_user_and_word(user, bookmark.origin)) self.probability = ( float(self.probability * count_bookmarks_with_same_word) + 0.1) / ( count_bookmarks_with_same_word + 1 ) # compute avg probability of all bookmarks with same word #calculates the probability of knowing a certain bookmark after a exercise_outcome. def calculate_known_bookmark_probability(self, bookmark): count_correct_after_another = 0 count_wrong_after_another = 0 sorted_exercise_log_after_date = sorted(bookmark.exercise_log, key=lambda x: x.time, reverse=False) for exercise in sorted_exercise_log_after_date: if exercise.outcome.outcome == ExerciseOutcome.TOO_EASY: self.probability = decimal.Decimal('1.0') count_wrong_after_another = 0 elif exercise.outcome.outcome == ExerciseOutcome.SHOW_SOLUTION: self.probability //= 2 if float(self.probability) < 0.1: self.probability = decimal.Decimal('0.1') count_correct_after_another = 0 elif exercise.outcome.outcome == ExerciseOutcome.CORRECT: count_correct_after_another += 1 count_wrong_after_another = 0 if float(self.probability) < 1.0: self.correct_formula(count_correct_after_another) else: self.probability = decimal.Decimal('1.0') elif exercise.outcome.outcome == ExerciseOutcome.WRONG: count_wrong_after_another += 1 count_correct_after_another = 0 if float(self.probability) > 0.1: self.wrong_formula(count_wrong_after_another) else: self.probability = decimal.Decimal('0.1') def halfProbability(self): self.probability /= 2 if float(self.probability) < 0.1: self.probability = decimal.Decimal('0.1')
class Bookmark(db.Model): __table_args__ = {'mysql_collate': 'utf8_bin'} id = db.Column(db.Integer, primary_key=True) origin_id = db.Column(db.Integer, db.ForeignKey('user_word.id')) origin = db.relationship("UserWord", primaryjoin=origin_id == UserWord.id, backref="translations") translations_list = relationship("UserWord", secondary="bookmark_translation_mapping") user_id = db.Column(db.Integer, db.ForeignKey('user.id')) user = db.relationship("User", backref="bookmarks") text_id = db.Column(db.Integer, db.ForeignKey('text.id')) text = db.relationship("Text", backref="bookmarks") time = db.Column(db.DateTime) exercise_log = relationship("Exercise", secondary="bookmark_exercise_mapping") def __init__(self, origin, translation, user, text, time): self.origin = origin self.translations_list.append(translation) self.user = user self.time = time self.text = text def add_new_exercise(self, exercise): self.exercise_log.append(exercise) def translation(self): return self.translations_list[0] def translations_rendered_as_text(self): return ", ".join(self.translation_words_list()) def translation_words_list(self): translation_words = [] for translation in self.translations_list: translation_words.append(translation.word) return translation_words def add_new_translation(self, translation): self.translations_list.append(translation) def remove_translation(self, translation): if translation in self.translations_list: self.translations_list.remove(translation) def add_exercise_outcome(self, exercise_source, exercise_outcome, exercise_solving_speed): new_source = ExerciseSource.query.filter_by( source=exercise_source).first() new_outcome = ExerciseOutcome.query.filter_by( outcome=exercise_outcome).first() exercise = Exercise(new_outcome, new_source, exercise_solving_speed, datetime.datetime.now()) self.add_new_exercise(exercise) db.session.add(exercise) def split_words_from_context(self): words_of_bookmark_content = [] bookmark_content_words = re.findall(r'(?u)\w+', self.text.content) words_of_bookmark_content.extend(bookmark_content_words) return words_of_bookmark_content def context_words_with_rank(self): ranked_context_words = self.split_words_from_context() while self.origin.word in ranked_context_words: ranked_context_words.remove(self.origin.word) filtered_words_known_from_user = [] for word_known in ranked_context_words: if RankedWord.exists(word_known.lower(), self.origin.language): filtered_words_known_from_user.append(word_known) return filtered_words_known_from_user def calculate_known_word_probability_after_adding_exercise_based_probability( self, ex_prob, enc_prob, user): if KnownWordProbability.exists( user, self.origin, self.origin.rank ) and enc_prob == None: #checks if only exercise based probability exists known_word_prob = KnownWordProbability.find( user, self.origin, self.origin.rank) known_word_prob.probability = ex_prob.probability elif enc_prob is not None: #checks if encounter based probability also exists known_word_prob = KnownWordProbability.find( user, self.origin, self.origin.rank) known_word_prob.probability = KnownWordProbability.calculateKnownWordProb( ex_prob, enc_prob) else: known_word_prob = KnownWordProbability.find( user, self.origin, self.origin.rank, ex_prob.probability ) # new known word probability created as it did not exist. return known_word_prob def calculate_probabilities_after_adding_a_bookmark(self, user, language): # computations for adding encounter based probability for word in self.context_words_with_rank(): enc_prob = EncounterBasedProbability.find_or_create(word, user) zeeguu.db.session.add(enc_prob) zeeguu.db.session.commit() user_word = None ranked_word = enc_prob.ranked_word if UserWord.exists(word, language): user_word = UserWord.find(word, language) if ExerciseBasedProbability.exists( user, user_word ): #checks if exercise based probability exists for words in context ex_prob = ExerciseBasedProbability.find(user, user_word) known_word_prob_1 = KnownWordProbability.find( user, user_word, ranked_word) known_word_prob_1.probability = known_word_prob_1.calculateKnownWordProb( ex_prob, enc_prob ) #updates known word probability as exercise based probability already existed. else: if KnownWordProbability.exists(user, user_word, ranked_word): known_word_prob_1 = KnownWordProbability.find( user, user_word, ranked_word) known_word_prob_1.probability = enc_prob.probability # updates known word probability as encounter based probability already existed else: known_word_prob_1 = KnownWordProbability.find( user, user_word, ranked_word, enc_prob.probability ) # new known word probability created as it did not exist zeeguu.db.session.add(known_word_prob_1) # computations for adding exercise based probability enc_prob = None ex_prob = ExerciseBasedProbability.find(user, self.origin) if RankedWord.exists( self.origin.word, language ): #checks if ranked_word exists for that looked up word ranked_word = RankedWord.find(self.origin.word, language) if EncounterBasedProbability.exists( user, ranked_word ): # checks if encounter based probability exists for that looked up word enc_prob = EncounterBasedProbability.find(user, ranked_word) enc_prob.reset_prob( ) # reset encounter based probability to 0.5 if ExerciseBasedProbability.exists(user, self.origin): ex_prob.update_probability_after_adding_bookmark_with_same_word( self, user) zeeguu.db.session.add(ex_prob) known_word_prob_2 = self.calculate_known_word_probability_after_adding_exercise_based_probability( ex_prob, enc_prob, user) zeeguu.db.session.add(known_word_prob_2) zeeguu.db.session.commit() @classmethod def find_by_specific_user(cls, user): return cls.query.filter_by(user=user).all() @classmethod def find_all(cls): return cls.query.filter().all() @classmethod def find(cls, b_id): return cls.query.filter_by(id=b_id).first() @classmethod def find_all_by_user_and_word(cls, user, word): return cls.query.filter_by(user=user, origin=word).all() # @classmethod # def is_sorted_exercise_log_after_date_outcome(cls,outcome, bookmark): # sorted_exercise_log_after_date=sorted(bookmark.exercise_log, key=lambda x: x.time, reverse=True) # if sorted_exercise_log_after_date: # if sorted_exercise_log_after_date[0].outcome.outcome == outcome: # return True # return False def check_is_latest_outcome_too_easy(self): sorted_exercise_log_by_latest = sorted(self.exercise_log, key=lambda x: x.time, reverse=True) for exercise in sorted_exercise_log_by_latest: if exercise.outcome.outcome == ExerciseOutcome.TOO_EASY: return True elif exercise.outcome.outcome == ExerciseOutcome.SHOW_SOLUTION or exercise.outcome.outcome == ExerciseOutcome.WRONG: return False return False
class EncounterBasedProbability(db.Model): __tablename__ = 'encounter_based_probability' __table_args__ = {'mysql_collate': 'utf8_bin'} DEFAULT_PROBABILITY = 0.5 id = db.Column(db.Integer, primary_key=True) user_id = db.Column(db.Integer, db.ForeignKey("user.id"), nullable=False) user = db.relationship("User") ranked_word_id = db.Column(db.Integer, db.ForeignKey("ranked_word.id"), nullable=False) ranked_word = db.relationship("RankedWord") not_looked_up_counter = db.Column(db.Integer, nullable=False) probability = db.Column(db.DECIMAL(10, 9), nullable=False) db.UniqueConstraint(user_id, ranked_word_id) db.CheckConstraint('probability>=0', 'probability<=1') def __init__(self, user, ranked_word, not_looked_up_counter, probability): self.user = user self.ranked_word = ranked_word self.not_looked_up_counter = not_looked_up_counter self.probability = probability @classmethod def find(cls, user, ranked_word, default_probability=None): try: return cls.query.filter_by(user=user, ranked_word=ranked_word).one() except sqlalchemy.orm.exc.NoResultFound: return cls(user, ranked_word, 1, default_probability) @classmethod def find_all(cls): return cls.query.all() @classmethod def find_all_by_user(cls, user): return cls.query.filter_by(user=user).all() @classmethod def exists(cls, user, ranked_word): try: cls.query.filter_by(user=user, ranked_word=ranked_word).one() return True except sqlalchemy.orm.exc.NoResultFound: return False @classmethod def find_or_create(cls, word, user, language): ranked_word = RankedWord.find(word.lower(), language) if EncounterBasedProbability.exists(user, ranked_word): enc_prob = EncounterBasedProbability.find(user, ranked_word) enc_prob.not_looked_up_counter += 1 enc_prob.boost_prob() else: enc_prob = EncounterBasedProbability.find( user, ranked_word, EncounterBasedProbability.DEFAULT_PROBABILITY) return enc_prob def reset_prob(self): # Why is this 0.5? Should be lower! I just looked up the word... # ugh... self.probability = 0.5 def word_has_just_beek_bookmarked(self): """ the user can't know this word very well if he's bookmarking it again :return: """ self.probability /= 2 # This function controls if prob is already 1.0, else it adds 0.1. It maximum adds 0.1, therefore cannot exceed 1 def boost_prob(self): if float(self.probability) <> 1.0: self.probability = float(self.probability) + 0.1
class User(db.Model): __table_args__ = {'mysql_collate': 'utf8_bin'} id = db.Column(db.Integer, primary_key=True) email = db.Column(db.String(255), unique=True) name = db.Column(db.String(255)) password = db.Column(db.LargeBinary(255)) password_salt = db.Column(db.LargeBinary(255)) learned_language_id = db.Column( db.String(2), db.ForeignKey("language.id") ) learned_language = sqlalchemy.orm.relationship("Language", foreign_keys=[learned_language_id]) starred_words = relationship("UserWord", secondary="starred_words_association") native_language_id = db.Column( db.String (2), db.ForeignKey("language.id") ) native_language = sqlalchemy.orm.relationship("Language", foreign_keys=[native_language_id]) def __init__(self, email, username, password, learned_language=None, native_language = None): self.email = email self.name = username self.update_password(password) self.learned_language = learned_language or Language.default_learned() self.native_language = native_language or Language.default_native_language() def __repr__(self): return '<User %r>' % (self.email) def has_starred(self,word): return word in self.starred_words def star(self, word): self.starred_words.append(word) print (word.word + " is now starred for user " + self.name) # TODO: Does this work without a commit here? To double check. def details_as_dictionary(self): return dict ( email=self.email, name=self.name, learned_language=self.learned_language_id, native_language=self.native_language_id ) def set_learned_language(self, code): self.learned_language = Language.find(code) def set_native_language(self, code): self.native_language = Language.find(code) @sqlalchemy.orm.validates("email") def validate_email(self, col, email): if "@" not in email: raise ValueError("Invalid email address") return email @sqlalchemy.orm.validates("password") def validate_password(self, col, password): if password is None or len(password) == 0: raise ValueError("Invalid password") return password @sqlalchemy.orm.validates("name") def validate_name(self, col, name): if name is None or len(name) == 0: raise ValueError("Invalid username") return name def update_password(self, password): self.password_salt = "".join( chr(random.randint(0, 255)) for i in range(32) ) self.password = util.password_hash(password, self.password_salt) def all_bookmarks(self, after_date=datetime.datetime(1970,1,1), before_date=datetime.date.today() + datetime.timedelta(days=1)): from zeeguu.model.bookmark import Bookmark return Bookmark.query.\ filter_by(user_id=self.id).\ filter(Bookmark.time >= after_date). \ filter(Bookmark.time <= before_date). \ order_by(Bookmark.time.desc()).all() def bookmarks_chronologically(self): from zeeguu.model.bookmark import Bookmark return Bookmark.query.filter_by(user_id=self.id).order_by(Bookmark.time.desc()).all() def bookmarks_by_date(self, after_date=datetime.datetime(1970,1,1)): """ :param after_date: :return: a pair of 1. a dict with date-> bookmarks and 2. a sorted list of dates """ def extract_day_from_date(bookmark): return (bookmark, bookmark.time.replace(bookmark.time.year, bookmark.time.month, bookmark.time.day,0,0,0,0)) bookmarks = self.all_bookmarks(after_date) bookmarks_by_date = dict() for elem in map(extract_day_from_date, bookmarks): bookmarks_by_date.setdefault(elem[1],[]).append(elem[0]) sorted_dates = bookmarks_by_date.keys() sorted_dates.sort(reverse=True) return bookmarks_by_date, sorted_dates def bookmarks_by_day(self, with_context, after_date=datetime.datetime(2014,1,1)): bookmarks_by_date, sorted_dates = self.bookmarks_by_date(after_date) dates = [] for date in sorted_dates: bookmarks = [] for bookmark in bookmarks_by_date[date]: bookmarks.append(bookmark.json_serializable_dict(with_context)) date_entry = dict( date=date.strftime("%A, %d %B %Y"), bookmarks=bookmarks ) dates.append(date_entry) return dates def bookmarks_to_study(self, bookmark_count): from zeeguu.model.bookmark import Bookmark from zeeguu import RankedWord all_bookmarks_query = Bookmark.query.\ filter_by(user_id=self.id).\ join(UserWord).\ join(RankedWord).\ filter(UserWord.id == Bookmark.origin_id).\ filter(UserWord.rank_id == RankedWord.id).\ order_by(RankedWord.rank.asc()) all_bookmarks = all_bookmarks_query.all() good_for_study=[] size = 0 for b in all_bookmarks: if b.good_for_study(): good_for_study.append(b) size += 1 if size == bookmark_count: break if size < bookmark_count: # we did not find enough words to study which are ranked # add all the non-ranked ones, in chronological order all_bookmarks = Bookmark.query. \ filter_by(user_id=self.id). \ join(UserWord). \ filter(UserWord.id == Bookmark.origin_id). \ filter(UserWord.rank_id == None). \ order_by(Bookmark.time.desc()).all() for b in all_bookmarks: if b.good_for_study(): good_for_study.append(b) size += 1 if size == bookmark_count: break return map(lambda x: x.json_serializable_dict(), good_for_study) # returns array with added bookmark amount per each date for the last year # this function is for the activity_graph, generates data def bookmark_counts_by_date(self): # compute bookmark_counts_by_date year = datetime.date.today().year - 1 # get data from year 2015(if this year is 2016) month = datetime.date.today().month bookmarks_dict, dates = self.bookmarks_by_date(datetime.datetime(year, month, 1)) counts = [] for date in dates: the_date = date.strftime('%Y-%m-%d') the_count = len(bookmarks_dict[date]) counts.append(dict(date=the_date, count=the_count)) bookmark_counts_by_date = json.dumps(counts) return bookmark_counts_by_date # returns array with learned and learning words count per each month for the last year # this function is for the line_graph, generates data def learner_stats_data(self): # compute learner_stats_data from zeeguu.model.learner_stats.learner_stats import compute_learner_stats learner_stats_data = compute_learner_stats(self) return learner_stats_data def user_words(self): return map((lambda x: x.origin.word), self.all_bookmarks()) def bookmark_count(self): return len(self.all_bookmarks()) def word_count(self): return len(self.user_words()) @classmethod def find_all(cls): return User.query.all() @classmethod def find(cls, email): return User.query.filter(User.email == email).one() @classmethod def find_by_id(cls, id): return User.query.filter(User.id == id).one() @classmethod def authorize(cls, email, password): try: user = cls.query.filter(cls.email == email).one() if user.password == util.password_hash(password, user.password_salt): return user except sqlalchemy.orm.exc.NoResultFound: return None