class UserActivityData(db.Model): __table_args__ = dict(mysql_collate="utf8_bin") __tablename__ = 'user_activity_data' id = db.Column(db.Integer, primary_key=True) user_id = db.Column(db.Integer, db.ForeignKey("user.id")) user = db.relationship("User") time = db.Column(db.DateTime) event = db.Column(db.String(255)) value = db.Column(db.String(255)) extra_data = db.Column(db.String(4096)) def __init__(self, user, time, event, value, extra_data): self.user = user self.time = time self.event = event self.value = value self.extra_data = extra_data def data_as_dictionary(self): return dict(user_id=self.user_id, time=self.time.strftime("%Y-%m-%dT%H:%M:%S"), event=self.event, value=self.value, extra_data=self.extra_data)
class UserWord(db.Model, util.JSONSerializable): __tablename__ = 'user_words' __table_args__ = {'mysql_collate': 'utf8_bin'} id = db.Column(db.Integer, primary_key=True) word = db.Column(db.String(255), nullable=False, unique=True) language_id = db.Column(db.String(2), db.ForeignKey("language.id")) language = db.relationship("Language") rank_id = db.Column(db.Integer, db.ForeignKey("word_ranks.id"), nullable=True) rank = db.relationship("WordRank") db.UniqueConstraint(word, language_id) IMPORTANCE_LEVEL_STEP = 1000 IMPOSSIBLE_RANK = 1000000 IMPOSSIBLE_IMPORTANCE_LEVEL = IMPOSSIBLE_RANK / IMPORTANCE_LEVEL_STEP def __init__(self, word, language, rank=None): self.word = word self.language = language self.rank = rank def __repr__(self): return '<UserWord %r>' % (self.word) def serialize(self): return self.word # returns a number between def importance_level(self): if self.rank is not None: return max((10 - self.rank.rank / UserWord.IMPORTANCE_LEVEL_STEP), 0) else: return 0 # we use this in the bookmarks.html to show the rank. # for words in which there is no rank info, we don't display anything def importance_level_string(self): if self.rank == None: return "" b = "|" return b * self.importance_level() @classmethod def find(cls, word, language, rank=None): try: return (cls.query.filter(cls.word == word).filter( cls.language == language).one()) except sqlalchemy.orm.exc.NoResultFound: return cls(word, language, rank) @classmethod def find_rank(cls, word, language): return WordRank.find(word, language) @classmethod def find_all(cls): return cls.query.all()
class Url(db.Model): __table_args__ = {'mysql_collate': 'utf8_bin'} id = db.Column(db.Integer, primary_key=True) url = db.Column(db.String(2083)) title = db.Column(db.String(2083)) def __init__(self, url, title): self.url = url self.title = title def title_if_available(self): if self.title != "": return self.title return self.url @classmethod def find(cls, url, title): try: return (cls.query.filter(cls.url == url).one()) except sqlalchemy.orm.exc.NoResultFound: return cls(url, title) def render_link(self, link_text): if self.url != "": return '<a href="' + self.url + '">' + link_text + '</a>' else: return ""
class Language(db.Model): id = db.Column(db.String(2), primary_key=True) name = db.Column(db.String(255), unique=True) def __init__(self, id, name): self.name = name self.id = id def __repr__(self): return '<Language %r>' % (self.id) @classmethod def default_learned(cls): return cls.find("de") @classmethod def default_native_language(cls): return cls.find("en") @classmethod def native_languages(cls): return [cls.find("en"), cls.find("ro")] @classmethod def available_languages(cls): return cls.all() @classmethod def find(cls, id_): return cls.query.filter(Language.id == id_).one() @classmethod def all(cls): return cls.query.filter().all()
class Url(db.Model): __table_args__ = {'mysql_collate': 'utf8_bin'} id = db.Column(db.Integer, primary_key=True) title = db.Column(db.String(2083)) path = db.Column(db.String(2083)) url = db.Column(db.String(2083)) domain_name_id = db.Column(db.Integer, db.ForeignKey("domain_name.id")) domain = db.relationship("DomainName") def __init__(self, url, title): self.path = Url.get_path(url) self.domain = DomainName.find(Url.get_domain(url)) self.title = title def title_if_available(self): if self.title != "": return self.title return self.url def as_string(self): return self.domain.domain_name + self.path def domain_name(self): return self.domain.domain_name @classmethod def get_domain(self, url): protocol_re = '(.*://)?' domain_re = '([^/?]*)' path_re = '(.*)' domain = re.findall(protocol_re + domain_re, url)[0] return domain[0] + domain[1] @classmethod def get_path(self, url): protocol_re = '(.*://)?' domain_re = '([^/?]*)' path_re = '(.*)' domain = re.findall(protocol_re + domain_re + path_re, url)[0] return domain[2] @classmethod def find(cls, url, title=""): try: d = DomainName.find(Url.get_domain(url)) return (cls.query.filter(cls.path == Url.get_path(url)).filter( cls.domain == d).one()) except sqlalchemy.orm.exc.NoResultFound: return cls(url, title) def render_link(self, link_text): if self.url != "": return '<a href="' + self.url + '">' + link_text + '</a>' else: return ""
class RankedWord(db.Model, util.JSONSerializable): __tablename__ = 'ranked_word' __table_args__ = {'mysql_collate': 'utf8_bin'} id = db.Column(db.Integer, primary_key=True) word = db.Column(db.String(255), nullable=False, unique=True, index=True) language_id = db.Column(db.String(2), db.ForeignKey("language.id")) language = db.relationship("Language") rank = db.Column(db.Integer) db.UniqueConstraint(word, language_id) def __init__(self, word, language, rank): self.word = word self.language = language self.rank = rank @classmethod def find(cls, word, language): word = word.lower() try: return (cls.query.filter(cls.word == word).filter( cls.language == language).one()) except sqlalchemy.orm.exc.NoResultFound: return None @classmethod def find_all(cls, language): return cls.query.filter(cls.language == language).all() @classmethod def exists(cls, word, language): word = word.lower() try: (cls.query.filter(cls.word == word).filter( cls.language == language).one()) return True except sqlalchemy.orm.exc.NoResultFound: return False @classmethod def words_list(cls): words_list = [] for word in cls.find_all(): words_list.append(word.word) return words_list
class ExerciseSource(db.Model): __tablename__ = 'exercise_source' __table_args__ = {'mysql_collate': 'utf8_bin'} id = db.Column(db.Integer, primary_key=True) source = db.Column(db.String(255), nullable=False) def __init__(self, source): self.source = source
class ExerciseSource(db.Model): __tablename__ = 'exercise_source' __table_args__ = {'mysql_collate': 'utf8_bin'} id = db.Column(db.Integer, primary_key=True) source = db.Column(db.String(255), nullable=False) def __init__(self, source): self.source = source @classmethod def find_by_source(cls, source): new_source = ExerciseSource.query.filter_by(source=source).first() return new_source
class UniqueCode(db.Model): __table_args__ = {'mysql_collate': 'utf8_bin'} id = db.Column(db.Integer, primary_key=True) code = db.Column(db.String(4)) email = db.Column(db.String(255)) time = db.Column(db.DateTime) def __init__(self, email): self.code = randint(100,999) self.email = email self.time = datetime.now() def __str__(self): return str(self.code) @classmethod def last_code(cls, email): return (cls.query.filter(cls.email == email).order_by(cls.time.desc()).first()).code @classmethod def all_codes_for(cls, email): return (cls.query.filter(cls.email == email)).all()
class Language(db.Model): __table_args__ = {'mysql_collate': 'utf8_bin'} id = db.Column(db.String(2), primary_key=True) name = db.Column(db.String(255), unique=True) def __init__(self, id, name): self.name = name self.id = id def __repr__(self): return '<Language %r>' % (self.id) @classmethod def default_learned(cls): return cls.find("de") @classmethod def default_native_language(cls): return cls.find("en") @classmethod def native_languages(cls): return [cls.find("en")] @classmethod def available_languages(cls): return list(set(cls.all()) - set([Language.find("en")])) @classmethod def find(cls, id_): return cls.query.filter(Language.id == id_).one() @classmethod def all(cls): return cls.query.filter().all()
class ExerciseOutcome(db.Model): __tablename__ = 'exercise_outcome' __table_args__ = {'mysql_collate': 'utf8_bin'} id = db.Column(db.Integer, primary_key=True) outcome = db.Column(db.String(255), nullable=False) IKNOW = 'I know' NOT_KNOW = 'Do not know' def __init__(self, outcome): self.outcome = outcome @classmethod def find(cls, outcome): try: return cls.query.filter_by(outcome=outcome).first() except sqlalchemy.orm.exc.NoResultFound: return cls(outcome)
class WatchEventType(db.Model): __table_args__ = dict(mysql_collate='utf8_bin') __tablename__ = 'watch_event_type' id = db.Column(db.Integer, primary_key=True) name = db.Column(db.String(255), nullable=False) def __init__(self, name): self.name = name @classmethod def find_by_name(cls, name): """ :param name: :return: the desired object, or None if it does not exist """ try: return WatchEventType.query.filter_by(name=name).first() except: return None
class Search(db.Model): id = db.Column(db.Integer, primary_key=True) user_id = db.Column(db.Integer, db.ForeignKey("user.id")) user = db.relationship("User", backref="searches") word_id = db.Column(db.Integer, db.ForeignKey("word.id")) word = db.relationship("Word") language_id = db.Column(db.String(2), db.ForeignKey("language.id")) language = db.relationship("Language") text_id = db.Column(db.Integer, db.ForeignKey("text.id")) text = db.relationship("Text") contribution_id = db.Column(db.Integer, db.ForeignKey("contribution.id")) contribution = db.relationship("Contribution", backref="search") def __init__(self, user, word, language, text=None): self.user = user self.word = word self.language = language self.text = text def __repr__(self): return '<Search %r>' % (self.word.word)
class ExerciseOutcome(db.Model): __tablename__ = 'exercise_outcome' __table_args__ = {'mysql_collate': 'utf8_bin'} id = db.Column(db.Integer, primary_key=True) outcome = db.Column(db.String(255), nullable=False) TOO_EASY = 'Too easy' SHOW_SOLUTION = 'Show solution' CORRECT = 'Correct' WRONG = 'Wrong' def __init__(self, outcome): self.outcome = outcome @classmethod def find(cls, outcome): try: return cls.query.filter_by(outcome=outcome).one() except sqlalchemy.orm.exc.NoResultFound: return cls(outcome)
class Search(db.Model): __table_args__ = {'mysql_collate': 'utf8_bin'} id = db.Column(db.Integer, primary_key=True) user_id = db.Column(db.Integer, db.ForeignKey("user.id")) user = db.relationship("User", backref="searches") word_id = db.Column(db.Integer, db.ForeignKey("user_words.id")) word = db.relationship("UserWord") language_id = db.Column(db.String(2), db.ForeignKey("language.id")) language = db.relationship("Language") text_id = db.Column(db.Integer, db.ForeignKey("text.id")) text = db.relationship("Text") bookmark_id = db.Column(db.Integer, db.ForeignKey("bookmark.id")) bookmark = db.relationship("Bookmark", backref="search") def __init__(self, user, word, language, text=None): self.user = user self.user_word = word self.language = language self.text = text def __repr__(self): return '<Search %r>' % (self.user_word.word)
class DomainName(db.Model): __table_args__ = {'mysql_collate': 'utf8_bin'} __tablename__ = 'domain_name' id = db.Column(db.Integer, primary_key=True) domain_name = db.Column(db.String(2083)) def __init__(self, url): self.domain_name = self.extract_domain_name(url) def extract_domain_name(self, url): protocol_re = '(.*://)?' domain_re = '([^/?]*)' domain = re.findall(protocol_re + domain_re, url)[0] return domain[0] + domain[1] @classmethod def find(cls, domain_url): try: return (cls.query.filter(cls.domain_name == domain_url).one()) except sqlalchemy.orm.exc.NoResultFound: # print "tried, but didn't find " + domain_url return cls(domain_url)
class Text(db.Model): __table_args__ = {'mysql_collate': 'utf8_bin'} id = db.Column(db.Integer, primary_key=True) content = db.Column(db.String(10000)) content_hash = db.Column(db.LargeBinary(32)) language_id = db.Column(db.String(2), db.ForeignKey("language.id")) language = db.relationship("Language") url_id = db.Column(db.Integer, db.ForeignKey('url.id')) url = db.relationship("Url", backref="texts") def __init__(self, content, language, url): self.content = content self.language = language self.url = url self.content_hash = util.text_hash(content) def __repr__(self): return '<Text %r>' % (self.language.short) def words(self): for word in re.split(re.compile(u"[^\\w]+", re.U), self.content): yield UserWord.find(word, self.language) def shorten_word_context(self, given_word, max_word_count): # shorter_text = "" limited_words = [] words = self.content.split( ) # ==> gives me a list of the words ["these", "types", ",", "the"] word_count = len(words) if word_count <= max_word_count: return self.content for i in range(0, max_word_count): limited_words.append( words[i]) # lista cu primele max_length cuvinte shorter_text = ' '.join( limited_words) # string cu primele 'max_word_count' cuv # sometimes the given_word does not exist in the text. # in that case return a text containing max_length words if given_word not in words: return shorter_text if words.index(given_word) <= max_word_count: return shorter_text for i in range(max_word_count + 1, words.index(given_word) + 1): limited_words.append(words[i]) shorter_text = ' '.join(limited_words) return shorter_text @classmethod def find(cls, text, language): try: query = (cls.query.filter(cls.language == language).filter( cls.content_hash == util.text_hash(text))) if query.count() > 0: query = query.filter(cls.content == text) try: return query.one() except sqlalchemy.orm.exc.NoResultFound: pass return cls(text, language) except: import traceback traceback.print_exc()
class User(db.Model): id = db.Column(db.Integer, primary_key=True) email = db.Column(db.String(255), unique=True) name = db.Column(db.String(255)) password = db.Column(db.LargeBinary(255)) password_salt = db.Column(db.LargeBinary(255)) learned_language_id = db.Column( db.String(2), db.ForeignKey("language.id") ) learned_language = sqlalchemy.orm.relationship("Language", foreign_keys=[learned_language_id]) starred_words = relationship("Word", secondary="starred_words_association") native_language_id = db.Column( db.String (2), db.ForeignKey("language.id") ) native_language = sqlalchemy.orm.relationship("Language", foreign_keys=[native_language_id]) def __init__(self, email, name, password, learned_language=None, native_language = None): self.email = email self.name = name self.update_password(password) self.learned_language = learned_language or Language.default() self.native_language = native_language or Language.default_native_language() def __repr__(self): return '<User %r>' % (self.email) def has_starred(self,word): return word in self.starred_words def star(self, word): self.starred_words.append(word) print word.word + " is now starred for user " + self.name # TODO: Does this work without a commit here? To double check. def read(self, text): if (Impression.query.filter(Impression.user == self) .filter(Impression.text == text).count()) > 0: return for word in text.words(): self.impressions.append(Impression(self, word, text)) def set_learned_language(self, code): self.learned_language = Language.find(code) def set_native_language(self, code): self.native_language = Language.find(code) @classmethod def find(cls, email): return User.query.filter(User.email == email).one() @classmethod def find_by_id(cls, id): return User.query.filter(User.id == id).one() @sqlalchemy.orm.validates("email") def validate_email(self, col, email): if "@" not in email: raise ValueError("Invalid email address") return email @sqlalchemy.orm.validates("password") def validate_password(self, col, password): if password is None or len(password) == 0: raise ValueError("Invalid password") return password @sqlalchemy.orm.validates("name") def validate_name(self, col, name): if name is None or len(name) == 0: raise ValueError("Invalid username") return name def update_password(self, password): self.password_salt = "".join( chr(random.randint(0, 255)) for i in range(32) ) self.password = util.password_hash(password, self.password_salt) @classmethod def authorize(cls, email, password): try: user = cls.query.filter(cls.email == email).one() if user.password == util.password_hash(password, user.password_salt): return user except sqlalchemy.orm.exc.NoResultFound: return None def contribs_chronologically(self): return Contribution.query.filter_by(user_id=self.id).order_by(Contribution.time.desc()).all() def user_words(self): return map((lambda x: x.origin.word), self.all_contributions()) def all_contributions(self): return Contribution.query.filter_by(user_id=self.id).order_by(Contribution.time.desc()).all() def contribs_by_date(self): def extract_day_from_date(contrib): return (contrib, contrib.time.replace(contrib.time.year, contrib.time.month, contrib.time.day,0,0,0,0)) contribs = self.all_contributions() contribs_by_date = dict() for elem in map(extract_day_from_date, contribs): contribs_by_date.setdefault(elem[1],[]).append(elem[0]) sorted_dates = contribs_by_date.keys() sorted_dates.sort(reverse=True) return contribs_by_date, sorted_dates def unique_urls(self): urls = set() for c in self.all_contributions(): urls.add(c.text.url) return urls def recommended_urls(self): urls_to_words = {} for contrib in self.all_contributions(): if contrib.text.url.url != "undefined": urls_to_words.setdefault(contrib.text.url,0) urls_to_words [contrib.text.url] += contrib.origin.importance_level() return sorted(urls_to_words, key=urls_to_words.get, reverse=True)
class Word(db.Model, util.JSONSerializable): id = db.Column(db.Integer, primary_key=True) word = db.Column(db.String(255)) language_id = db.Column(db.String(2), db.ForeignKey("language.id")) language = db.relationship("Language") word_rank = db.Column(db.Integer) IMPORTANCE_LEVEL_STEP = 1000 IMPOSSIBLE_RANK = 1000000 IMPOSSIBLE_IMPORTANCE_LEVEL = IMPOSSIBLE_RANK / IMPORTANCE_LEVEL_STEP def __init__(self, word, language): self.word = word self.language = language self.word_rank = self.get_rank_from_file() def __repr__(self): return '<Word %r>' % (self.word) def serialize(self): return self.word # if the word is not found, we assume a rank of 1000000 def get_rank_from_file(self): import codecs try: f=codecs.open(zeeguu.app.config.get("LANGUAGES_FOLDER").decode('utf-8')+self.language.id+".txt", encoding="iso-8859-1") all_words = f.readlines() all_words_without_space = [] for each_word in all_words: each_word_without_space = each_word[:-1] all_words_without_space.append(each_word_without_space) def importance_range(the_word, frequency_list): if the_word in frequency_list: position = frequency_list.index(the_word) return position else: return Word.IMPOSSIBLE_RANK return importance_range(self.word, all_words_without_space) except: return Word.IMPOSSIBLE_RANK def rank(self): if self.word_rank == None: self.word_rank = self.get_rank_from_file() session = sqlalchemy.orm.object_session(self) session.commit() return self.word_rank # returns a number between def importance_level(self): return max((10 - self.rank() / Word.IMPORTANCE_LEVEL_STEP), 0) # we use this in the contributions.html to show the rank. # for words in which there is no rank info, we don't display anything def importance_level_string(self): if self.rank() == Word.IMPOSSIBLE_RANK: return "" b = "|" return b * self.importance_level() @classmethod def find(cls, word, language): try: return (cls.query.filter(cls.word == word) .filter(cls.language == language) .one()) except sqlalchemy.orm.exc.NoResultFound: return cls(word, language) @classmethod def translate(cls, from_lang, term, to_lang): return (cls.query.join(WordAlias, cls.translation_of) .filter(WordAlias.word == term.lower()) .filter(cls.language == to_lang) .filter(WordAlias.language == from_lang) .all())
class User(db.Model): __table_args__ = {'mysql_collate': 'utf8_bin'} id = db.Column(db.Integer, primary_key=True) email = db.Column(db.String(255), unique=True) name = db.Column(db.String(255)) password = db.Column(db.LargeBinary(255)) password_salt = db.Column(db.LargeBinary(255)) learned_language_id = db.Column(db.String(2), db.ForeignKey("language.id")) learned_language = sqlalchemy.orm.relationship( "Language", foreign_keys=[learned_language_id]) starred_words = relationship("UserWord", secondary="starred_words_association") native_language_id = db.Column(db.String(2), db.ForeignKey("language.id")) native_language = sqlalchemy.orm.relationship( "Language", foreign_keys=[native_language_id]) def __init__(self, email, username, password, learned_language=None, native_language=None): self.email = email self.name = username self.update_password(password) self.learned_language = learned_language or Language.default_learned() self.native_language = native_language or Language.default_native_language( ) def __repr__(self): return '<User %r>' % (self.email) def has_starred(self, word): return word in self.starred_words def star(self, word): self.starred_words.append(word) print word.word + " is now starred for user " + self.name # TODO: Does this work without a commit here? To double check. def set_learned_language(self, code): self.learned_language = Language.find(code) def set_native_language(self, code): self.native_language = Language.find(code) @classmethod def find(cls, email): return User.query.filter(User.email == email).one() @classmethod def find_by_id(cls, id): return User.query.filter(User.id == id).one() @sqlalchemy.orm.validates("email") def validate_email(self, col, email): if "@" not in email: raise ValueError("Invalid email address") return email @sqlalchemy.orm.validates("password") def validate_password(self, col, password): if password is None or len(password) == 0: raise ValueError("Invalid password") return password @sqlalchemy.orm.validates("name") def validate_name(self, col, name): if name is None or len(name) == 0: raise ValueError("Invalid username") return name def update_password(self, password): self.password_salt = "".join( chr(random.randint(0, 255)) for i in range(32)) self.password = util.password_hash(password, self.password_salt) @classmethod def authorize(cls, email, password): try: user = cls.query.filter(cls.email == email).one() if user.password == util.password_hash(password, user.password_salt): return user except sqlalchemy.orm.exc.NoResultFound: return None def bookmarks_chronologically(self): return Bookmark.query.filter_by(user_id=self.id).order_by( Bookmark.time.desc()).all() def user_words(self): return map((lambda x: x.origin.word), self.all_bookmarks()) def all_bookmarks(self): return Bookmark.query.filter_by(user_id=self.id).order_by( Bookmark.time.desc()).all() def bookmark_count(self): return len(self.all_bookmarks()) def word_count(self): return len(self.user_words()) def bookmarks_by_date(self): def extract_day_from_date(bookmark): return (bookmark, bookmark.time.replace(bookmark.time.year, bookmark.time.month, bookmark.time.day, 0, 0, 0, 0)) bookmarks = self.all_bookmarks() bookmarks_by_date = dict() for elem in map(extract_day_from_date, bookmarks): bookmarks_by_date.setdefault(elem[1], []).append(elem[0]) sorted_dates = bookmarks_by_date.keys() sorted_dates.sort(reverse=True) return bookmarks_by_date, sorted_dates def unique_urls(self): urls = set() for b in self.all_bookmarks(): urls.add(b.text.url) return urls def recommended_urls(self): urls_to_words = {} for bookmark in self.all_bookmarks(): if bookmark.text.url.url != "undefined": urls_to_words.setdefault(bookmark.text.url, 0) urls_to_words[ bookmark.text.url] += bookmark.origin.importance_level() return sorted(urls_to_words, key=urls_to_words.get, reverse=True) def get_known_bookmarks(self): bookmarks = Bookmark.find_all_filtered_by_user() i_know_bookmarks = [] for bookmark in bookmarks: if Bookmark.is_sorted_exercise_log_after_date_outcome( ExerciseOutcome.IKNOW, bookmark): i_know_bookmark_dict = {} i_know_bookmark_dict['id'] = bookmark.id i_know_bookmark_dict['origin'] = bookmark.origin.word i_know_bookmark_dict['text'] = bookmark.text.content i_know_bookmark_dict['time'] = bookmark.time.strftime( '%m/%d/%Y') i_know_bookmarks.append(i_know_bookmark_dict.copy()) return i_know_bookmarks def get_known_bookmarks_count(self): return len(self.get_known_bookmarks()) def get_estimated_vocabulary(self, lang): bookmarks = Bookmark.find_all_filtered_by_user() filtered_words_known_from_user_dict_list = [] marked_words_of_user_in_text = [] words_of_all_bookmarks_content = [] filtered_words_known_from_user = [] for bookmark in bookmarks: bookmark_content_words = re.sub("[^\w]", " ", bookmark.text.content).split() words_of_all_bookmarks_content.extend(bookmark_content_words) marked_words_of_user_in_text.append(bookmark.origin.word) words_known_from_user = [ word for word in words_of_all_bookmarks_content if word not in marked_words_of_user_in_text ] for word_known in words_known_from_user: if WordRank.exists(word_known.lower(), lang): filtered_words_known_from_user.append(word_known) zeeguu.db.session.commit() filtered_words_known_from_user = list( set(filtered_words_known_from_user)) for word in filtered_words_known_from_user: filtered_word_known_from_user_dict = {} filtered_word_known_from_user_dict['word'] = word filtered_words_known_from_user_dict_list.append( filtered_word_known_from_user_dict.copy()) return filtered_words_known_from_user_dict_list def get_estimated_vocabulary_for_learned_language(self): return self.get_estimated_vocabulary(self.learned_language) def get_estimated_vocabulary_count(self): return len(self.get_estimated_vocabulary_for_learned_language())
class RSSFeed(db.Model): __table_args__ = {'mysql_collate': 'utf8_bin'} __tablename__ = 'rss_feed' id = db.Column(db.Integer, primary_key=True) title = db.Column(db.String(2083)) description = db.Column(db.String(2083)) language_id = db.Column(db.String(2), db.ForeignKey("language.id")) language = db.relationship("Language") url_id = db.Column(db.Integer, db.ForeignKey("url.id")) url = db.relationship("Url", foreign_keys='RSSFeed.url_id') image_url_id = db.Column(db.Integer, db.ForeignKey("url.id")) image_url = db.relationship("Url", foreign_keys='RSSFeed.image_url_id') def __init__(self, url, title, description, image_url=None, language=None): self.url = url self.image_url = image_url self.title = title self.language = language self.description = description def as_dictionary(self): image_url = "" if self.image_url: image_url = self.image_url.as_string() return dict(id=self.id, title=self.title, url=self.url.as_string(), description=self.description, language=self.language.id, image_url=image_url) def feed_items(self): feed_data = feedparser.parse(self.url.as_string()) feed_items = [ dict(title=item.get("title", ""), url=item.get("link", ""), content=item.get("content", ""), summary=item.get("summary", ""), published=time.strftime("%Y-%m-%dT%H:%M:%S%z", item.published_parsed)) for item in feed_data.entries ] return feed_items @classmethod def find_by_url(cls, url): try: result = (cls.query.filter(cls.url == url).one()) # print "found an existing RSSFeed object" return result except: return None @classmethod def find_or_create(cls, url, title, description, image_url, language): try: result = ( cls.query.filter(cls.url == url).filter( cls.title == title).filter(cls.language == language) # .filter(cls.image_url == image_url) .filter(cls.description == description).one()) # print "found an existing RSSFeed object" return result except sqlalchemy.orm.exc.NoResultFound: # print "creating new feed object for " + title return cls(url, title, description, image_url, language) @classmethod def find_for_language_id(cls, language_id): return cls.query.filter(cls.language_id == language_id).group_by( cls.title).all()
class User(db.Model): __table_args__ = {'mysql_collate': 'utf8_bin'} id = db.Column(db.Integer, primary_key=True) email = db.Column(db.String(255), unique=True) name = db.Column(db.String(255)) password = db.Column(db.LargeBinary(255)) password_salt = db.Column(db.LargeBinary(255)) learned_language_id = db.Column( db.String(2), db.ForeignKey("language.id") ) learned_language = sqlalchemy.orm.relationship("Language", foreign_keys=[learned_language_id]) starred_words = relationship("UserWord", secondary="starred_words_association") native_language_id = db.Column( db.String (2), db.ForeignKey("language.id") ) native_language = sqlalchemy.orm.relationship("Language", foreign_keys=[native_language_id]) def __init__(self, email, username, password, learned_language=None, native_language = None): self.email = email self.name = username self.update_password(password) self.learned_language = learned_language or Language.default_learned() self.native_language = native_language or Language.default_native_language() def __repr__(self): return '<User %r>' % (self.email) def has_starred(self,word): return word in self.starred_words def star(self, word): self.starred_words.append(word) print (word.word + " is now starred for user " + self.name) # TODO: Does this work without a commit here? To double check. def details_as_dictionary(self): return dict ( email=self.email, name=self.name, learned_language=self.learned_language_id, native_language=self.native_language_id ) def set_learned_language(self, code): self.learned_language = Language.find(code) def set_native_language(self, code): self.native_language = Language.find(code) @sqlalchemy.orm.validates("email") def validate_email(self, col, email): if "@" not in email: raise ValueError("Invalid email address") return email @sqlalchemy.orm.validates("password") def validate_password(self, col, password): if password is None or len(password) == 0: raise ValueError("Invalid password") return password @sqlalchemy.orm.validates("name") def validate_name(self, col, name): if name is None or len(name) == 0: raise ValueError("Invalid username") return name def update_password(self, password): self.password_salt = "".join( chr(random.randint(0, 255)) for i in range(32) ) self.password = util.password_hash(password, self.password_salt) def all_bookmarks(self, after_date=datetime.datetime(1970,1,1), before_date=datetime.date.today() + datetime.timedelta(days=1)): from zeeguu.model.bookmark import Bookmark return Bookmark.query.\ filter_by(user_id=self.id).\ filter(Bookmark.time >= after_date). \ filter(Bookmark.time <= before_date). \ order_by(Bookmark.time.desc()).all() def bookmarks_chronologically(self): from zeeguu.model.bookmark import Bookmark return Bookmark.query.filter_by(user_id=self.id).order_by(Bookmark.time.desc()).all() def bookmarks_by_date(self, after_date=datetime.datetime(1970,1,1)): """ :param after_date: :return: a pair of 1. a dict with date-> bookmarks and 2. a sorted list of dates """ def extract_day_from_date(bookmark): return (bookmark, bookmark.time.replace(bookmark.time.year, bookmark.time.month, bookmark.time.day,0,0,0,0)) bookmarks = self.all_bookmarks(after_date) bookmarks_by_date = dict() for elem in map(extract_day_from_date, bookmarks): bookmarks_by_date.setdefault(elem[1],[]).append(elem[0]) sorted_dates = bookmarks_by_date.keys() sorted_dates.sort(reverse=True) return bookmarks_by_date, sorted_dates def bookmarks_by_day(self, with_context, after_date=datetime.datetime(2014,1,1)): bookmarks_by_date, sorted_dates = self.bookmarks_by_date(after_date) dates = [] for date in sorted_dates: bookmarks = [] for bookmark in bookmarks_by_date[date]: bookmarks.append(bookmark.json_serializable_dict(with_context)) date_entry = dict( date=date.strftime("%A, %d %B %Y"), bookmarks=bookmarks ) dates.append(date_entry) return dates def bookmarks_to_study(self, bookmark_count): from zeeguu.model.bookmark import Bookmark from zeeguu import RankedWord all_bookmarks_query = Bookmark.query.\ filter_by(user_id=self.id).\ join(UserWord).\ join(RankedWord).\ filter(UserWord.id == Bookmark.origin_id).\ filter(UserWord.rank_id == RankedWord.id).\ order_by(RankedWord.rank.asc()) all_bookmarks = all_bookmarks_query.all() good_for_study=[] size = 0 for b in all_bookmarks: if b.good_for_study(): good_for_study.append(b) size += 1 if size == bookmark_count: break if size < bookmark_count: # we did not find enough words to study which are ranked # add all the non-ranked ones, in chronological order all_bookmarks = Bookmark.query. \ filter_by(user_id=self.id). \ join(UserWord). \ filter(UserWord.id == Bookmark.origin_id). \ filter(UserWord.rank_id == None). \ order_by(Bookmark.time.desc()).all() for b in all_bookmarks: if b.good_for_study(): good_for_study.append(b) size += 1 if size == bookmark_count: break return map(lambda x: x.json_serializable_dict(), good_for_study) # returns array with added bookmark amount per each date for the last year # this function is for the activity_graph, generates data def bookmark_counts_by_date(self): # compute bookmark_counts_by_date year = datetime.date.today().year - 1 # get data from year 2015(if this year is 2016) month = datetime.date.today().month bookmarks_dict, dates = self.bookmarks_by_date(datetime.datetime(year, month, 1)) counts = [] for date in dates: the_date = date.strftime('%Y-%m-%d') the_count = len(bookmarks_dict[date]) counts.append(dict(date=the_date, count=the_count)) bookmark_counts_by_date = json.dumps(counts) return bookmark_counts_by_date # returns array with learned and learning words count per each month for the last year # this function is for the line_graph, generates data def learner_stats_data(self): # compute learner_stats_data from zeeguu.model.learner_stats.learner_stats import compute_learner_stats learner_stats_data = compute_learner_stats(self) return learner_stats_data def user_words(self): return map((lambda x: x.origin.word), self.all_bookmarks()) def bookmark_count(self): return len(self.all_bookmarks()) def word_count(self): return len(self.user_words()) @classmethod def find_all(cls): return User.query.all() @classmethod def find(cls, email): return User.query.filter(User.email == email).one() @classmethod def find_by_id(cls, id): return User.query.filter(User.id == id).one() @classmethod def authorize(cls, email, password): try: user = cls.query.filter(cls.email == email).one() if user.password == util.password_hash(password, user.password_salt): return user except sqlalchemy.orm.exc.NoResultFound: return None
class User(db.Model): __table_args__ = {'mysql_collate': 'utf8_bin'} id = db.Column(db.Integer, primary_key=True) email = db.Column(db.String(255), unique=True) name = db.Column(db.String(255)) password = db.Column(db.LargeBinary(255)) password_salt = db.Column(db.LargeBinary(255)) learned_language_id = db.Column(db.String(2), db.ForeignKey("language.id")) learned_language = sqlalchemy.orm.relationship( "Language", foreign_keys=[learned_language_id]) starred_words = relationship("UserWord", secondary="starred_words_association") native_language_id = db.Column(db.String(2), db.ForeignKey("language.id")) native_language = sqlalchemy.orm.relationship( "Language", foreign_keys=[native_language_id]) def __init__(self, email, username, password, learned_language=None, native_language=None): self.email = email self.name = username self.update_password(password) self.learned_language = learned_language or Language.default_learned() self.native_language = native_language or Language.default_native_language( ) def __repr__(self): return '<User %r>' % (self.email) def has_starred(self, word): return word in self.starred_words def star(self, word): self.starred_words.append(word) print word.word + " is now starred for user " + self.name # TODO: Does this work without a commit here? To double check. def set_learned_language(self, code): self.learned_language = Language.find(code) def set_native_language(self, code): self.native_language = Language.find(code) @classmethod def find_all(cls): return User.query.all() @classmethod def find(cls, email): return User.query.filter(User.email == email).one() @classmethod def find_by_id(cls, id): return User.query.filter(User.id == id).one() @sqlalchemy.orm.validates("email") def validate_email(self, col, email): if "@" not in email: raise ValueError("Invalid email address") return email @sqlalchemy.orm.validates("password") def validate_password(self, col, password): if password is None or len(password) == 0: raise ValueError("Invalid password") return password @sqlalchemy.orm.validates("name") def validate_name(self, col, name): if name is None or len(name) == 0: raise ValueError("Invalid username") return name def update_password(self, password): self.password_salt = "".join( chr(random.randint(0, 255)) for i in range(32)) self.password = util.password_hash(password, self.password_salt) @classmethod def authorize(cls, email, password): try: user = cls.query.filter(cls.email == email).one() if user.password == util.password_hash(password, user.password_salt): return user except sqlalchemy.orm.exc.NoResultFound: return None def bookmarks_chronologically(self): return Bookmark.query.filter_by(user_id=self.id).order_by( Bookmark.time.desc()).all() def user_words(self): return map((lambda x: x.origin.word), self.all_bookmarks()) def all_bookmarks(self): return Bookmark.query.filter_by(user_id=self.id).order_by( Bookmark.time.desc()).all() def bookmark_count(self): return len(self.all_bookmarks()) def word_count(self): return len(self.user_words()) def bookmarks_by_date(self): def extract_day_from_date(bookmark): return (bookmark, bookmark.time.replace(bookmark.time.year, bookmark.time.month, bookmark.time.day, 0, 0, 0, 0)) bookmarks = self.all_bookmarks() bookmarks_by_date = dict() for elem in map(extract_day_from_date, bookmarks): bookmarks_by_date.setdefault(elem[1], []).append(elem[0]) sorted_dates = bookmarks_by_date.keys() sorted_dates.sort(reverse=True) return bookmarks_by_date, sorted_dates # returns only HTTP domains. in this way we filter # out empty domains, and others like the android: # that we use for internal tracking... # Returns: list of tuples (domain, date) def recent_domains_with_times(self): domains = [] domains_and_times = [] for b in self.bookmarks_chronologically(): if not b.text.url.domain() in domains\ and 'http' in b.text.url.domain(): domains_and_times.append([b.text.url.domain(), b.time]) domains.append(b.text.url.domain()) return domains_and_times def frequent_domains(self): domains = map(lambda b: b.text.url.domain(), self.bookmarks_chronologically()) from collections import Counter counter = Counter(domains) return counter.most_common() def unique_urls(self): urls = set() for b in self.all_bookmarks(): urls.add(b.text.url) return urls def recommended_urls(self): urls_to_words = {} for bookmark in self.all_bookmarks(): if bookmark.text.url.url != "undefined": urls_to_words.setdefault(bookmark.text.url, 0) urls_to_words[ bookmark.text.url] += bookmark.origin.importance_level() return sorted(urls_to_words, key=urls_to_words.get, reverse=True) def get_not_encountered_words(self, lang): not_encountered_words_dict_list = [] all_ranks = RankedWord.find_all(lang) known_word_probs = KnownWordProbability.find_all_by_user_with_rank( self) for p in known_word_probs: if p.ranked_word in all_ranks: all_ranks.remove(p.ranked_word) for rank in all_ranks: not_encountered_word_dict = {} not_encountered_word_dict['word'] = rank.word not_encountered_words_dict_list.append(not_encountered_word_dict) return not_encountered_words_dict_list def get_not_encountered_words_count(self): return len(self.get_not_encountered_words(self.learned_language)) def get_known_bookmarks(self, lang): bookmarks = flask.g.user.all_bookmarks() known_bookmarks = [] for bookmark in bookmarks: if bookmark.check_is_latest_outcome_too_easy( ) and lang == bookmark.origin.language: known_bookmark_dict = { 'id': bookmark.id, 'origin': bookmark.origin.word, 'text': bookmark.text.content, 'time': bookmark.time.strftime('%m/%d/%Y') } known_bookmarks.append(known_bookmark_dict) return known_bookmarks def get_known_bookmarks_count(self): return len(self.get_known_bookmarks(self.learned_language)) def get_not_looked_up_words(self, lang): filtered_words_known_from_user_dict_list = [] enc_probs = EncounterBasedProbability.find_all_by_user(flask.g.user) for enc_prob in enc_probs: if enc_prob.ranked_word.language == lang: filtered_words_known_from_user_dict_list.append( {'word': enc_prob.ranked_word.word}) return filtered_words_known_from_user_dict_list def get_not_looked_up_words_for_learned_language(self): return self.get_not_looked_up_words(self.learned_language) def get_not_looked_up_words_count(self): return len(self.get_not_looked_up_words_for_learned_language()) def get_probably_known_words(self, lang): known_word_prob_of_user = KnownWordProbability.get_probably_known_words( self) probable_known_words_dict_list = [] for known_word_prob in known_word_prob_of_user: probable_known_word_dict = {} if known_word_prob.ranked_word is not None and known_word_prob.ranked_word.language == lang: probable_known_word_dict[ 'word'] = known_word_prob.ranked_word.word else: probable_known_word_dict[ 'word'] = known_word_prob.user_word.word probable_known_words_dict_list.append(probable_known_word_dict) return probable_known_words_dict_list def get_probably_known_words_count(self): return len(self.get_probably_known_words(self.learned_language)) def get_lower_bound_percentage_of_basic_vocabulary(self): high_known_word_prob_of_user = KnownWordProbability.get_probably_known_words( self) count_high_known_word_prob_of_user_ranked = 0 for prob in high_known_word_prob_of_user: if prob.ranked_word is not None and prob.ranked_word.rank <= 3000: count_high_known_word_prob_of_user_ranked += 1 return round( float(count_high_known_word_prob_of_user_ranked) / 3000 * 100, 2) def get_upper_bound_percentage_of_basic_vocabulary(self): count_not_looked_up_words_with_rank = 0 not_looked_up_words = EncounterBasedProbability.find_all_by_user(self) for prob in not_looked_up_words: if prob.ranked_word.rank <= 3000: count_not_looked_up_words_with_rank += 1 return round( float(count_not_looked_up_words_with_rank) / 3000 * 100, 2) def get_lower_bound_percentage_of_extended_vocabulary(self): high_known_word_prob_of_user = KnownWordProbability.get_probably_known_words( self) count_high_known_word_prob_of_user_ranked = 0 for prob in high_known_word_prob_of_user: if prob.ranked_word is not None and prob.ranked_word.rank <= 10000: count_high_known_word_prob_of_user_ranked += 1 return round( float(count_high_known_word_prob_of_user_ranked) / 10000 * 100, 2) def get_upper_bound_percentage_of_extended_vocabulary(self): count_not_looked_up_words_with_rank = 0 not_looked_up_words = EncounterBasedProbability.find_all_by_user(self) for prob in not_looked_up_words: if prob.ranked_word.rank <= 10000: count_not_looked_up_words_with_rank += 1 return round( float(count_not_looked_up_words_with_rank) / 10000 * 100, 2) def get_percentage_of_probably_known_bookmarked_words(self): high_known_word_prob_of_user = KnownWordProbability.get_probably_known_words( self) count_high_known_word_prob_of_user = 0 count_bookmarks_of_user = len(self.all_bookmarks()) for prob in high_known_word_prob_of_user: if prob.user_word is not None: count_high_known_word_prob_of_user += 1 if count_bookmarks_of_user <> 0: return round( float(count_high_known_word_prob_of_user) / count_bookmarks_of_user * 100, 2) else: return 0 # Reading recommendations def recommendations(self): recommendations = { 'de': [[ 'Der Spiegel', 'http://m.spiegel.de', 'news, advanced', 'World News' ]] } try: return recommendations[self.learned_language_id] except: return []