class UserActivityData(db.Model):
__table_args__ = dict(mysql_collate="utf8_bin")
__tablename__ = 'user_activity_data'
id = db.Column(db.Integer, primary_key=True)
user_id = db.Column(db.Integer, db.ForeignKey("user.id"))
user = db.relationship("User")
time = db.Column(db.DateTime)
event = db.Column(db.String(255))
value = db.Column(db.String(255))
extra_data = db.Column(db.String(4096))
def __init__(self, user, time, event, value, extra_data):
self.user = user
self.time = time
self.event = event
self.value = value
self.extra_data = extra_data
def data_as_dictionary(self):
return dict(user_id=self.user_id,
time=self.time.strftime("%Y-%m-%dT%H:%M:%S"),
event=self.event,
value=self.value,
extra_data=self.extra_data)
class UserWord(db.Model, util.JSONSerializable):
__tablename__ = 'user_words'
__table_args__ = {'mysql_collate': 'utf8_bin'}
id = db.Column(db.Integer, primary_key=True)
word = db.Column(db.String(255), nullable=False, unique=True)
language_id = db.Column(db.String(2), db.ForeignKey("language.id"))
language = db.relationship("Language")
rank_id = db.Column(db.Integer,
db.ForeignKey("word_ranks.id"),
nullable=True)
rank = db.relationship("WordRank")
db.UniqueConstraint(word, language_id)
IMPORTANCE_LEVEL_STEP = 1000
IMPOSSIBLE_RANK = 1000000
IMPOSSIBLE_IMPORTANCE_LEVEL = IMPOSSIBLE_RANK / IMPORTANCE_LEVEL_STEP
def __init__(self, word, language, rank=None):
self.word = word
self.language = language
self.rank = rank
def __repr__(self):
return '<UserWord %r>' % (self.word)
def serialize(self):
return self.word
# returns a number between
def importance_level(self):
if self.rank is not None:
return max((10 - self.rank.rank / UserWord.IMPORTANCE_LEVEL_STEP),
0)
else:
return 0
# we use this in the bookmarks.html to show the rank.
# for words in which there is no rank info, we don't display anything
def importance_level_string(self):
if self.rank == None:
return ""
b = "|"
return b * self.importance_level()
@classmethod
def find(cls, word, language, rank=None):
try:
return (cls.query.filter(cls.word == word).filter(
cls.language == language).one())
except sqlalchemy.orm.exc.NoResultFound:
return cls(word, language, rank)
@classmethod
def find_rank(cls, word, language):
return WordRank.find(word, language)
@classmethod
def find_all(cls):
return cls.query.all()
class Url(db.Model):
__table_args__ = {'mysql_collate': 'utf8_bin'}
id = db.Column(db.Integer, primary_key=True)
url = db.Column(db.String(2083))
title = db.Column(db.String(2083))
def __init__(self, url, title):
self.url = url
self.title = title
def title_if_available(self):
if self.title != "":
return self.title
return self.url
@classmethod
def find(cls, url, title):
try:
return (cls.query.filter(cls.url == url).one())
except sqlalchemy.orm.exc.NoResultFound:
return cls(url, title)
def render_link(self, link_text):
if self.url != "":
return '<a href="' + self.url + '">' + link_text + '</a>'
else:
return ""
class Language(db.Model):
id = db.Column(db.String(2), primary_key=True)
name = db.Column(db.String(255), unique=True)
def __init__(self, id, name):
self.name = name
self.id = id
def __repr__(self):
return '<Language %r>' % (self.id)
@classmethod
def default_learned(cls):
return cls.find("de")
@classmethod
def default_native_language(cls):
return cls.find("en")
@classmethod
def native_languages(cls):
return [cls.find("en"), cls.find("ro")]
@classmethod
def available_languages(cls):
return cls.all()
@classmethod
def find(cls, id_):
return cls.query.filter(Language.id == id_).one()
@classmethod
def all(cls):
return cls.query.filter().all()
class Url(db.Model):
__table_args__ = {'mysql_collate': 'utf8_bin'}
id = db.Column(db.Integer, primary_key=True)
title = db.Column(db.String(2083))
path = db.Column(db.String(2083))
url = db.Column(db.String(2083))
domain_name_id = db.Column(db.Integer, db.ForeignKey("domain_name.id"))
domain = db.relationship("DomainName")
def __init__(self, url, title):
self.path = Url.get_path(url)
self.domain = DomainName.find(Url.get_domain(url))
self.title = title
def title_if_available(self):
if self.title != "":
return self.title
return self.url
def as_string(self):
return self.domain.domain_name + self.path
def domain_name(self):
return self.domain.domain_name
@classmethod
def get_domain(self, url):
protocol_re = '(.*://)?'
domain_re = '([^/?]*)'
path_re = '(.*)'
domain = re.findall(protocol_re + domain_re, url)[0]
return domain[0] + domain[1]
@classmethod
def get_path(self, url):
protocol_re = '(.*://)?'
domain_re = '([^/?]*)'
path_re = '(.*)'
domain = re.findall(protocol_re + domain_re + path_re, url)[0]
return domain[2]
@classmethod
def find(cls, url, title=""):
try:
d = DomainName.find(Url.get_domain(url))
return (cls.query.filter(cls.path == Url.get_path(url)).filter(
cls.domain == d).one())
except sqlalchemy.orm.exc.NoResultFound:
return cls(url, title)
def render_link(self, link_text):
if self.url != "":
return '<a href="' + self.url + '">' + link_text + '</a>'
else:
return ""
class RankedWord(db.Model, util.JSONSerializable):
__tablename__ = 'ranked_word'
__table_args__ = {'mysql_collate': 'utf8_bin'}
id = db.Column(db.Integer, primary_key=True)
word = db.Column(db.String(255), nullable=False, unique=True, index=True)
language_id = db.Column(db.String(2), db.ForeignKey("language.id"))
language = db.relationship("Language")
rank = db.Column(db.Integer)
db.UniqueConstraint(word, language_id)
def __init__(self, word, language, rank):
self.word = word
self.language = language
self.rank = rank
@classmethod
def find(cls, word, language):
word = word.lower()
try:
return (cls.query.filter(cls.word == word).filter(
cls.language == language).one())
except sqlalchemy.orm.exc.NoResultFound:
return None
@classmethod
def find_all(cls, language):
return cls.query.filter(cls.language == language).all()
@classmethod
def exists(cls, word, language):
word = word.lower()
try:
(cls.query.filter(cls.word == word).filter(
cls.language == language).one())
return True
except sqlalchemy.orm.exc.NoResultFound:
return False
@classmethod
def words_list(cls):
words_list = []
for word in cls.find_all():
words_list.append(word.word)
return words_list
class ExerciseSource(db.Model):
__tablename__ = 'exercise_source'
__table_args__ = {'mysql_collate': 'utf8_bin'}
id = db.Column(db.Integer, primary_key=True)
source = db.Column(db.String(255), nullable=False)
def __init__(self, source):
self.source = source
class ExerciseSource(db.Model):
__tablename__ = 'exercise_source'
__table_args__ = {'mysql_collate': 'utf8_bin'}
id = db.Column(db.Integer, primary_key=True)
source = db.Column(db.String(255), nullable=False)
def __init__(self, source):
self.source = source
@classmethod
def find_by_source(cls, source):
new_source = ExerciseSource.query.filter_by(source=source).first()
return new_source
class UniqueCode(db.Model):
__table_args__ = {'mysql_collate': 'utf8_bin'}
id = db.Column(db.Integer, primary_key=True)
code = db.Column(db.String(4))
email = db.Column(db.String(255))
time = db.Column(db.DateTime)
def __init__(self, email):
self.code = randint(100,999)
self.email = email
self.time = datetime.now()
def __str__(self):
return str(self.code)
@classmethod
def last_code(cls, email):
return (cls.query.filter(cls.email == email).order_by(cls.time.desc()).first()).code
@classmethod
def all_codes_for(cls, email):
return (cls.query.filter(cls.email == email)).all()
class Language(db.Model):
__table_args__ = {'mysql_collate': 'utf8_bin'}
id = db.Column(db.String(2), primary_key=True)
name = db.Column(db.String(255), unique=True)
def __init__(self, id, name):
self.name = name
self.id = id
def __repr__(self):
return '<Language %r>' % (self.id)
@classmethod
def default_learned(cls):
return cls.find("de")
@classmethod
def default_native_language(cls):
return cls.find("en")
@classmethod
def native_languages(cls):
return [cls.find("en")]
@classmethod
def available_languages(cls):
return list(set(cls.all()) - set([Language.find("en")]))
@classmethod
def find(cls, id_):
return cls.query.filter(Language.id == id_).one()
@classmethod
def all(cls):
return cls.query.filter().all()
class ExerciseOutcome(db.Model):
__tablename__ = 'exercise_outcome'
__table_args__ = {'mysql_collate': 'utf8_bin'}
id = db.Column(db.Integer, primary_key=True)
outcome = db.Column(db.String(255), nullable=False)
IKNOW = 'I know'
NOT_KNOW = 'Do not know'
def __init__(self, outcome):
self.outcome = outcome
@classmethod
def find(cls, outcome):
try:
return cls.query.filter_by(outcome=outcome).first()
except sqlalchemy.orm.exc.NoResultFound:
return cls(outcome)
class WatchEventType(db.Model):
__table_args__ = dict(mysql_collate='utf8_bin')
__tablename__ = 'watch_event_type'
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.String(255), nullable=False)
def __init__(self, name):
self.name = name
@classmethod
def find_by_name(cls, name):
"""
:param name:
:return: the desired object, or None if it does not exist
"""
try:
return WatchEventType.query.filter_by(name=name).first()
except:
return None
class Search(db.Model):
id = db.Column(db.Integer, primary_key=True)
user_id = db.Column(db.Integer, db.ForeignKey("user.id"))
user = db.relationship("User", backref="searches")
word_id = db.Column(db.Integer, db.ForeignKey("word.id"))
word = db.relationship("Word")
language_id = db.Column(db.String(2), db.ForeignKey("language.id"))
language = db.relationship("Language")
text_id = db.Column(db.Integer, db.ForeignKey("text.id"))
text = db.relationship("Text")
contribution_id = db.Column(db.Integer, db.ForeignKey("contribution.id"))
contribution = db.relationship("Contribution", backref="search")
def __init__(self, user, word, language, text=None):
self.user = user
self.word = word
self.language = language
self.text = text
def __repr__(self):
return '<Search %r>' % (self.word.word)
class ExerciseOutcome(db.Model):
__tablename__ = 'exercise_outcome'
__table_args__ = {'mysql_collate': 'utf8_bin'}
id = db.Column(db.Integer, primary_key=True)
outcome = db.Column(db.String(255), nullable=False)
TOO_EASY = 'Too easy'
SHOW_SOLUTION = 'Show solution'
CORRECT = 'Correct'
WRONG = 'Wrong'
def __init__(self, outcome):
self.outcome = outcome
@classmethod
def find(cls, outcome):
try:
return cls.query.filter_by(outcome=outcome).one()
except sqlalchemy.orm.exc.NoResultFound:
return cls(outcome)
class Search(db.Model):
__table_args__ = {'mysql_collate': 'utf8_bin'}
id = db.Column(db.Integer, primary_key=True)
user_id = db.Column(db.Integer, db.ForeignKey("user.id"))
user = db.relationship("User", backref="searches")
word_id = db.Column(db.Integer, db.ForeignKey("user_words.id"))
word = db.relationship("UserWord")
language_id = db.Column(db.String(2), db.ForeignKey("language.id"))
language = db.relationship("Language")
text_id = db.Column(db.Integer, db.ForeignKey("text.id"))
text = db.relationship("Text")
bookmark_id = db.Column(db.Integer, db.ForeignKey("bookmark.id"))
bookmark = db.relationship("Bookmark", backref="search")
def __init__(self, user, word, language, text=None):
self.user = user
self.user_word = word
self.language = language
self.text = text
def __repr__(self):
return '<Search %r>' % (self.user_word.word)
class DomainName(db.Model):
__table_args__ = {'mysql_collate': 'utf8_bin'}
__tablename__ = 'domain_name'
id = db.Column(db.Integer, primary_key=True)
domain_name = db.Column(db.String(2083))
def __init__(self, url):
self.domain_name = self.extract_domain_name(url)
def extract_domain_name(self, url):
protocol_re = '(.*://)?'
domain_re = '([^/?]*)'
domain = re.findall(protocol_re + domain_re, url)[0]
return domain[0] + domain[1]
@classmethod
def find(cls, domain_url):
try:
return (cls.query.filter(cls.domain_name == domain_url).one())
except sqlalchemy.orm.exc.NoResultFound:
# print "tried, but didn't find " + domain_url
return cls(domain_url)
class Text(db.Model):
__table_args__ = {'mysql_collate': 'utf8_bin'}
id = db.Column(db.Integer, primary_key=True)
content = db.Column(db.String(10000))
content_hash = db.Column(db.LargeBinary(32))
language_id = db.Column(db.String(2), db.ForeignKey("language.id"))
language = db.relationship("Language")
url_id = db.Column(db.Integer, db.ForeignKey('url.id'))
url = db.relationship("Url", backref="texts")
def __init__(self, content, language, url):
self.content = content
self.language = language
self.url = url
self.content_hash = util.text_hash(content)
def __repr__(self):
return '<Text %r>' % (self.language.short)
def words(self):
for word in re.split(re.compile(u"[^\\w]+", re.U), self.content):
yield UserWord.find(word, self.language)
def shorten_word_context(self, given_word, max_word_count):
# shorter_text = ""
limited_words = []
words = self.content.split(
) # ==> gives me a list of the words ["these", "types", ",", "the"]
word_count = len(words)
if word_count <= max_word_count:
return self.content
for i in range(0, max_word_count):
limited_words.append(
words[i]) # lista cu primele max_length cuvinte
shorter_text = ' '.join(
limited_words) # string cu primele 'max_word_count' cuv
# sometimes the given_word does not exist in the text.
# in that case return a text containing max_length words
if given_word not in words:
return shorter_text
if words.index(given_word) <= max_word_count:
return shorter_text
for i in range(max_word_count + 1, words.index(given_word) + 1):
limited_words.append(words[i])
shorter_text = ' '.join(limited_words)
return shorter_text
@classmethod
def find(cls, text, language):
try:
query = (cls.query.filter(cls.language == language).filter(
cls.content_hash == util.text_hash(text)))
if query.count() > 0:
query = query.filter(cls.content == text)
try:
return query.one()
except sqlalchemy.orm.exc.NoResultFound:
pass
return cls(text, language)
except:
import traceback
traceback.print_exc()
class User(db.Model):
id = db.Column(db.Integer, primary_key=True)
email = db.Column(db.String(255), unique=True)
name = db.Column(db.String(255))
password = db.Column(db.LargeBinary(255))
password_salt = db.Column(db.LargeBinary(255))
learned_language_id = db.Column(
db.String(2),
db.ForeignKey("language.id")
)
learned_language = sqlalchemy.orm.relationship("Language", foreign_keys=[learned_language_id])
starred_words = relationship("Word", secondary="starred_words_association")
native_language_id = db.Column(
db.String (2),
db.ForeignKey("language.id")
)
native_language = sqlalchemy.orm.relationship("Language", foreign_keys=[native_language_id])
def __init__(self, email, name, password, learned_language=None, native_language = None):
self.email = email
self.name = name
self.update_password(password)
self.learned_language = learned_language or Language.default()
self.native_language = native_language or Language.default_native_language()
def __repr__(self):
return '<User %r>' % (self.email)
def has_starred(self,word):
return word in self.starred_words
def star(self, word):
self.starred_words.append(word)
print word.word + " is now starred for user " + self.name
# TODO: Does this work without a commit here? To double check.
def read(self, text):
if (Impression.query.filter(Impression.user == self)
.filter(Impression.text == text).count()) > 0:
return
for word in text.words():
self.impressions.append(Impression(self, word, text))
def set_learned_language(self, code):
self.learned_language = Language.find(code)
def set_native_language(self, code):
self.native_language = Language.find(code)
@classmethod
def find(cls, email):
return User.query.filter(User.email == email).one()
@classmethod
def find_by_id(cls, id):
return User.query.filter(User.id == id).one()
@sqlalchemy.orm.validates("email")
def validate_email(self, col, email):
if "@" not in email:
raise ValueError("Invalid email address")
return email
@sqlalchemy.orm.validates("password")
def validate_password(self, col, password):
if password is None or len(password) == 0:
raise ValueError("Invalid password")
return password
@sqlalchemy.orm.validates("name")
def validate_name(self, col, name):
if name is None or len(name) == 0:
raise ValueError("Invalid username")
return name
def update_password(self, password):
self.password_salt = "".join(
chr(random.randint(0, 255)) for i in range(32)
)
self.password = util.password_hash(password, self.password_salt)
@classmethod
def authorize(cls, email, password):
try:
user = cls.query.filter(cls.email == email).one()
if user.password == util.password_hash(password,
user.password_salt):
return user
except sqlalchemy.orm.exc.NoResultFound:
return None
def contribs_chronologically(self):
return Contribution.query.filter_by(user_id=self.id).order_by(Contribution.time.desc()).all()
def user_words(self):
return map((lambda x: x.origin.word), self.all_contributions())
def all_contributions(self):
return Contribution.query.filter_by(user_id=self.id).order_by(Contribution.time.desc()).all()
def contribs_by_date(self):
def extract_day_from_date(contrib):
return (contrib, contrib.time.replace(contrib.time.year, contrib.time.month, contrib.time.day,0,0,0,0))
contribs = self.all_contributions()
contribs_by_date = dict()
for elem in map(extract_day_from_date, contribs):
contribs_by_date.setdefault(elem[1],[]).append(elem[0])
sorted_dates = contribs_by_date.keys()
sorted_dates.sort(reverse=True)
return contribs_by_date, sorted_dates
def unique_urls(self):
urls = set()
for c in self.all_contributions():
urls.add(c.text.url)
return urls
def recommended_urls(self):
urls_to_words = {}
for contrib in self.all_contributions():
if contrib.text.url.url != "undefined":
urls_to_words.setdefault(contrib.text.url,0)
urls_to_words [contrib.text.url] += contrib.origin.importance_level()
return sorted(urls_to_words, key=urls_to_words.get, reverse=True)
class Word(db.Model, util.JSONSerializable):
id = db.Column(db.Integer, primary_key=True)
word = db.Column(db.String(255))
language_id = db.Column(db.String(2), db.ForeignKey("language.id"))
language = db.relationship("Language")
word_rank = db.Column(db.Integer)
IMPORTANCE_LEVEL_STEP = 1000
IMPOSSIBLE_RANK = 1000000
IMPOSSIBLE_IMPORTANCE_LEVEL = IMPOSSIBLE_RANK / IMPORTANCE_LEVEL_STEP
def __init__(self, word, language):
self.word = word
self.language = language
self.word_rank = self.get_rank_from_file()
def __repr__(self):
return '<Word %r>' % (self.word)
def serialize(self):
return self.word
# if the word is not found, we assume a rank of 1000000
def get_rank_from_file(self):
import codecs
try:
f=codecs.open(zeeguu.app.config.get("LANGUAGES_FOLDER").decode('utf-8')+self.language.id+".txt", encoding="iso-8859-1")
all_words = f.readlines()
all_words_without_space = []
for each_word in all_words:
each_word_without_space = each_word[:-1]
all_words_without_space.append(each_word_without_space)
def importance_range(the_word, frequency_list):
if the_word in frequency_list:
position = frequency_list.index(the_word)
return position
else:
return Word.IMPOSSIBLE_RANK
return importance_range(self.word, all_words_without_space)
except:
return Word.IMPOSSIBLE_RANK
def rank(self):
if self.word_rank == None:
self.word_rank = self.get_rank_from_file()
session = sqlalchemy.orm.object_session(self)
session.commit()
return self.word_rank
# returns a number between
def importance_level(self):
return max((10 - self.rank() / Word.IMPORTANCE_LEVEL_STEP), 0)
# we use this in the contributions.html to show the rank.
# for words in which there is no rank info, we don't display anything
def importance_level_string(self):
if self.rank() == Word.IMPOSSIBLE_RANK:
return ""
b = "|"
return b * self.importance_level()
@classmethod
def find(cls, word, language):
try:
return (cls.query.filter(cls.word == word)
.filter(cls.language == language)
.one())
except sqlalchemy.orm.exc.NoResultFound:
return cls(word, language)
@classmethod
def translate(cls, from_lang, term, to_lang):
return (cls.query.join(WordAlias, cls.translation_of)
.filter(WordAlias.word == term.lower())
.filter(cls.language == to_lang)
.filter(WordAlias.language == from_lang)
.all())
class User(db.Model):
__table_args__ = {'mysql_collate': 'utf8_bin'}
id = db.Column(db.Integer, primary_key=True)
email = db.Column(db.String(255), unique=True)
name = db.Column(db.String(255))
password = db.Column(db.LargeBinary(255))
password_salt = db.Column(db.LargeBinary(255))
learned_language_id = db.Column(db.String(2), db.ForeignKey("language.id"))
learned_language = sqlalchemy.orm.relationship(
"Language", foreign_keys=[learned_language_id])
starred_words = relationship("UserWord",
secondary="starred_words_association")
native_language_id = db.Column(db.String(2), db.ForeignKey("language.id"))
native_language = sqlalchemy.orm.relationship(
"Language", foreign_keys=[native_language_id])
def __init__(self,
email,
username,
password,
learned_language=None,
native_language=None):
self.email = email
self.name = username
self.update_password(password)
self.learned_language = learned_language or Language.default_learned()
self.native_language = native_language or Language.default_native_language(
)
def __repr__(self):
return '<User %r>' % (self.email)
def has_starred(self, word):
return word in self.starred_words
def star(self, word):
self.starred_words.append(word)
print word.word + " is now starred for user " + self.name
# TODO: Does this work without a commit here? To double check.
def set_learned_language(self, code):
self.learned_language = Language.find(code)
def set_native_language(self, code):
self.native_language = Language.find(code)
@classmethod
def find(cls, email):
return User.query.filter(User.email == email).one()
@classmethod
def find_by_id(cls, id):
return User.query.filter(User.id == id).one()
@sqlalchemy.orm.validates("email")
def validate_email(self, col, email):
if "@" not in email:
raise ValueError("Invalid email address")
return email
@sqlalchemy.orm.validates("password")
def validate_password(self, col, password):
if password is None or len(password) == 0:
raise ValueError("Invalid password")
return password
@sqlalchemy.orm.validates("name")
def validate_name(self, col, name):
if name is None or len(name) == 0:
raise ValueError("Invalid username")
return name
def update_password(self, password):
self.password_salt = "".join(
chr(random.randint(0, 255)) for i in range(32))
self.password = util.password_hash(password, self.password_salt)
@classmethod
def authorize(cls, email, password):
try:
user = cls.query.filter(cls.email == email).one()
if user.password == util.password_hash(password,
user.password_salt):
return user
except sqlalchemy.orm.exc.NoResultFound:
return None
def bookmarks_chronologically(self):
return Bookmark.query.filter_by(user_id=self.id).order_by(
Bookmark.time.desc()).all()
def user_words(self):
return map((lambda x: x.origin.word), self.all_bookmarks())
def all_bookmarks(self):
return Bookmark.query.filter_by(user_id=self.id).order_by(
Bookmark.time.desc()).all()
def bookmark_count(self):
return len(self.all_bookmarks())
def word_count(self):
return len(self.user_words())
def bookmarks_by_date(self):
def extract_day_from_date(bookmark):
return (bookmark,
bookmark.time.replace(bookmark.time.year,
bookmark.time.month,
bookmark.time.day, 0, 0, 0, 0))
bookmarks = self.all_bookmarks()
bookmarks_by_date = dict()
for elem in map(extract_day_from_date, bookmarks):
bookmarks_by_date.setdefault(elem[1], []).append(elem[0])
sorted_dates = bookmarks_by_date.keys()
sorted_dates.sort(reverse=True)
return bookmarks_by_date, sorted_dates
def unique_urls(self):
urls = set()
for b in self.all_bookmarks():
urls.add(b.text.url)
return urls
def recommended_urls(self):
urls_to_words = {}
for bookmark in self.all_bookmarks():
if bookmark.text.url.url != "undefined":
urls_to_words.setdefault(bookmark.text.url, 0)
urls_to_words[
bookmark.text.url] += bookmark.origin.importance_level()
return sorted(urls_to_words, key=urls_to_words.get, reverse=True)
def get_known_bookmarks(self):
bookmarks = Bookmark.find_all_filtered_by_user()
i_know_bookmarks = []
for bookmark in bookmarks:
if Bookmark.is_sorted_exercise_log_after_date_outcome(
ExerciseOutcome.IKNOW, bookmark):
i_know_bookmark_dict = {}
i_know_bookmark_dict['id'] = bookmark.id
i_know_bookmark_dict['origin'] = bookmark.origin.word
i_know_bookmark_dict['text'] = bookmark.text.content
i_know_bookmark_dict['time'] = bookmark.time.strftime(
'%m/%d/%Y')
i_know_bookmarks.append(i_know_bookmark_dict.copy())
return i_know_bookmarks
def get_known_bookmarks_count(self):
return len(self.get_known_bookmarks())
def get_estimated_vocabulary(self, lang):
bookmarks = Bookmark.find_all_filtered_by_user()
filtered_words_known_from_user_dict_list = []
marked_words_of_user_in_text = []
words_of_all_bookmarks_content = []
filtered_words_known_from_user = []
for bookmark in bookmarks:
bookmark_content_words = re.sub("[^\w]", " ",
bookmark.text.content).split()
words_of_all_bookmarks_content.extend(bookmark_content_words)
marked_words_of_user_in_text.append(bookmark.origin.word)
words_known_from_user = [
word for word in words_of_all_bookmarks_content
if word not in marked_words_of_user_in_text
]
for word_known in words_known_from_user:
if WordRank.exists(word_known.lower(), lang):
filtered_words_known_from_user.append(word_known)
zeeguu.db.session.commit()
filtered_words_known_from_user = list(
set(filtered_words_known_from_user))
for word in filtered_words_known_from_user:
filtered_word_known_from_user_dict = {}
filtered_word_known_from_user_dict['word'] = word
filtered_words_known_from_user_dict_list.append(
filtered_word_known_from_user_dict.copy())
return filtered_words_known_from_user_dict_list
def get_estimated_vocabulary_for_learned_language(self):
return self.get_estimated_vocabulary(self.learned_language)
def get_estimated_vocabulary_count(self):
return len(self.get_estimated_vocabulary_for_learned_language())
class RSSFeed(db.Model):
__table_args__ = {'mysql_collate': 'utf8_bin'}
__tablename__ = 'rss_feed'
id = db.Column(db.Integer, primary_key=True)
title = db.Column(db.String(2083))
description = db.Column(db.String(2083))
language_id = db.Column(db.String(2), db.ForeignKey("language.id"))
language = db.relationship("Language")
url_id = db.Column(db.Integer, db.ForeignKey("url.id"))
url = db.relationship("Url", foreign_keys='RSSFeed.url_id')
image_url_id = db.Column(db.Integer, db.ForeignKey("url.id"))
image_url = db.relationship("Url", foreign_keys='RSSFeed.image_url_id')
def __init__(self, url, title, description, image_url=None, language=None):
self.url = url
self.image_url = image_url
self.title = title
self.language = language
self.description = description
def as_dictionary(self):
image_url = ""
if self.image_url:
image_url = self.image_url.as_string()
return dict(id=self.id,
title=self.title,
url=self.url.as_string(),
description=self.description,
language=self.language.id,
image_url=image_url)
def feed_items(self):
feed_data = feedparser.parse(self.url.as_string())
feed_items = [
dict(title=item.get("title", ""),
url=item.get("link", ""),
content=item.get("content", ""),
summary=item.get("summary", ""),
published=time.strftime("%Y-%m-%dT%H:%M:%S%z",
item.published_parsed))
for item in feed_data.entries
]
return feed_items
@classmethod
def find_by_url(cls, url):
try:
result = (cls.query.filter(cls.url == url).one())
# print "found an existing RSSFeed object"
return result
except:
return None
@classmethod
def find_or_create(cls, url, title, description, image_url, language):
try:
result = (
cls.query.filter(cls.url == url).filter(
cls.title == title).filter(cls.language == language)
# .filter(cls.image_url == image_url)
.filter(cls.description == description).one())
# print "found an existing RSSFeed object"
return result
except sqlalchemy.orm.exc.NoResultFound:
# print "creating new feed object for " + title
return cls(url, title, description, image_url, language)
@classmethod
def find_for_language_id(cls, language_id):
return cls.query.filter(cls.language_id == language_id).group_by(
cls.title).all()
class User(db.Model):
__table_args__ = {'mysql_collate': 'utf8_bin'}
id = db.Column(db.Integer, primary_key=True)
email = db.Column(db.String(255), unique=True)
name = db.Column(db.String(255))
password = db.Column(db.LargeBinary(255))
password_salt = db.Column(db.LargeBinary(255))
learned_language_id = db.Column(
db.String(2),
db.ForeignKey("language.id")
)
learned_language = sqlalchemy.orm.relationship("Language", foreign_keys=[learned_language_id])
starred_words = relationship("UserWord", secondary="starred_words_association")
native_language_id = db.Column(
db.String (2),
db.ForeignKey("language.id")
)
native_language = sqlalchemy.orm.relationship("Language", foreign_keys=[native_language_id])
def __init__(self, email, username, password, learned_language=None, native_language = None):
self.email = email
self.name = username
self.update_password(password)
self.learned_language = learned_language or Language.default_learned()
self.native_language = native_language or Language.default_native_language()
def __repr__(self):
return '<User %r>' % (self.email)
def has_starred(self,word):
return word in self.starred_words
def star(self, word):
self.starred_words.append(word)
print (word.word + " is now starred for user " + self.name)
# TODO: Does this work without a commit here? To double check.
def details_as_dictionary(self):
return dict (
email=self.email,
name=self.name,
learned_language=self.learned_language_id,
native_language=self.native_language_id
)
def set_learned_language(self, code):
self.learned_language = Language.find(code)
def set_native_language(self, code):
self.native_language = Language.find(code)
@sqlalchemy.orm.validates("email")
def validate_email(self, col, email):
if "@" not in email:
raise ValueError("Invalid email address")
return email
@sqlalchemy.orm.validates("password")
def validate_password(self, col, password):
if password is None or len(password) == 0:
raise ValueError("Invalid password")
return password
@sqlalchemy.orm.validates("name")
def validate_name(self, col, name):
if name is None or len(name) == 0:
raise ValueError("Invalid username")
return name
def update_password(self, password):
self.password_salt = "".join(
chr(random.randint(0, 255)) for i in range(32)
)
self.password = util.password_hash(password, self.password_salt)
def all_bookmarks(self, after_date=datetime.datetime(1970,1,1), before_date=datetime.date.today() + datetime.timedelta(days=1)):
from zeeguu.model.bookmark import Bookmark
return Bookmark.query.\
filter_by(user_id=self.id).\
filter(Bookmark.time >= after_date). \
filter(Bookmark.time <= before_date). \
order_by(Bookmark.time.desc()).all()
def bookmarks_chronologically(self):
from zeeguu.model.bookmark import Bookmark
return Bookmark.query.filter_by(user_id=self.id).order_by(Bookmark.time.desc()).all()
def bookmarks_by_date(self, after_date=datetime.datetime(1970,1,1)):
"""
:param after_date:
:return: a pair of 1. a dict with date-> bookmarks and 2. a sorted list of dates
"""
def extract_day_from_date(bookmark):
return (bookmark, bookmark.time.replace(bookmark.time.year, bookmark.time.month, bookmark.time.day,0,0,0,0))
bookmarks = self.all_bookmarks(after_date)
bookmarks_by_date = dict()
for elem in map(extract_day_from_date, bookmarks):
bookmarks_by_date.setdefault(elem[1],[]).append(elem[0])
sorted_dates = bookmarks_by_date.keys()
sorted_dates.sort(reverse=True)
return bookmarks_by_date, sorted_dates
def bookmarks_by_day(self, with_context, after_date=datetime.datetime(2014,1,1)):
bookmarks_by_date, sorted_dates = self.bookmarks_by_date(after_date)
dates = []
for date in sorted_dates:
bookmarks = []
for bookmark in bookmarks_by_date[date]:
bookmarks.append(bookmark.json_serializable_dict(with_context))
date_entry = dict(
date=date.strftime("%A, %d %B %Y"),
bookmarks=bookmarks
)
dates.append(date_entry)
return dates
def bookmarks_to_study(self, bookmark_count):
from zeeguu.model.bookmark import Bookmark
from zeeguu import RankedWord
all_bookmarks_query = Bookmark.query.\
filter_by(user_id=self.id).\
join(UserWord).\
join(RankedWord).\
filter(UserWord.id == Bookmark.origin_id).\
filter(UserWord.rank_id == RankedWord.id).\
order_by(RankedWord.rank.asc())
all_bookmarks = all_bookmarks_query.all()
good_for_study=[]
size = 0
for b in all_bookmarks:
if b.good_for_study():
good_for_study.append(b)
size += 1
if size == bookmark_count:
break
if size < bookmark_count:
# we did not find enough words to study which are ranked
# add all the non-ranked ones, in chronological order
all_bookmarks = Bookmark.query. \
filter_by(user_id=self.id). \
join(UserWord). \
filter(UserWord.id == Bookmark.origin_id). \
filter(UserWord.rank_id == None). \
order_by(Bookmark.time.desc()).all()
for b in all_bookmarks:
if b.good_for_study():
good_for_study.append(b)
size += 1
if size == bookmark_count:
break
return map(lambda x: x.json_serializable_dict(), good_for_study)
# returns array with added bookmark amount per each date for the last year
# this function is for the activity_graph, generates data
def bookmark_counts_by_date(self):
# compute bookmark_counts_by_date
year = datetime.date.today().year - 1 # get data from year 2015(if this year is 2016)
month = datetime.date.today().month
bookmarks_dict, dates = self.bookmarks_by_date(datetime.datetime(year, month, 1))
counts = []
for date in dates:
the_date = date.strftime('%Y-%m-%d')
the_count = len(bookmarks_dict[date])
counts.append(dict(date=the_date, count=the_count))
bookmark_counts_by_date = json.dumps(counts)
return bookmark_counts_by_date
# returns array with learned and learning words count per each month for the last year
# this function is for the line_graph, generates data
def learner_stats_data(self):
# compute learner_stats_data
from zeeguu.model.learner_stats.learner_stats import compute_learner_stats
learner_stats_data = compute_learner_stats(self)
return learner_stats_data
def user_words(self):
return map((lambda x: x.origin.word), self.all_bookmarks())
def bookmark_count(self):
return len(self.all_bookmarks())
def word_count(self):
return len(self.user_words())
@classmethod
def find_all(cls):
return User.query.all()
@classmethod
def find(cls, email):
return User.query.filter(User.email == email).one()
@classmethod
def find_by_id(cls, id):
return User.query.filter(User.id == id).one()
@classmethod
def authorize(cls, email, password):
try:
user = cls.query.filter(cls.email == email).one()
if user.password == util.password_hash(password,
user.password_salt):
return user
except sqlalchemy.orm.exc.NoResultFound:
return None
class User(db.Model):
__table_args__ = {'mysql_collate': 'utf8_bin'}
id = db.Column(db.Integer, primary_key=True)
email = db.Column(db.String(255), unique=True)
name = db.Column(db.String(255))
password = db.Column(db.LargeBinary(255))
password_salt = db.Column(db.LargeBinary(255))
learned_language_id = db.Column(db.String(2), db.ForeignKey("language.id"))
learned_language = sqlalchemy.orm.relationship(
"Language", foreign_keys=[learned_language_id])
starred_words = relationship("UserWord",
secondary="starred_words_association")
native_language_id = db.Column(db.String(2), db.ForeignKey("language.id"))
native_language = sqlalchemy.orm.relationship(
"Language", foreign_keys=[native_language_id])
def __init__(self,
email,
username,
password,
learned_language=None,
native_language=None):
self.email = email
self.name = username
self.update_password(password)
self.learned_language = learned_language or Language.default_learned()
self.native_language = native_language or Language.default_native_language(
)
def __repr__(self):
return '<User %r>' % (self.email)
def has_starred(self, word):
return word in self.starred_words
def star(self, word):
self.starred_words.append(word)
print word.word + " is now starred for user " + self.name
# TODO: Does this work without a commit here? To double check.
def set_learned_language(self, code):
self.learned_language = Language.find(code)
def set_native_language(self, code):
self.native_language = Language.find(code)
@classmethod
def find_all(cls):
return User.query.all()
@classmethod
def find(cls, email):
return User.query.filter(User.email == email).one()
@classmethod
def find_by_id(cls, id):
return User.query.filter(User.id == id).one()
@sqlalchemy.orm.validates("email")
def validate_email(self, col, email):
if "@" not in email:
raise ValueError("Invalid email address")
return email
@sqlalchemy.orm.validates("password")
def validate_password(self, col, password):
if password is None or len(password) == 0:
raise ValueError("Invalid password")
return password
@sqlalchemy.orm.validates("name")
def validate_name(self, col, name):
if name is None or len(name) == 0:
raise ValueError("Invalid username")
return name
def update_password(self, password):
self.password_salt = "".join(
chr(random.randint(0, 255)) for i in range(32))
self.password = util.password_hash(password, self.password_salt)
@classmethod
def authorize(cls, email, password):
try:
user = cls.query.filter(cls.email == email).one()
if user.password == util.password_hash(password,
user.password_salt):
return user
except sqlalchemy.orm.exc.NoResultFound:
return None
def bookmarks_chronologically(self):
return Bookmark.query.filter_by(user_id=self.id).order_by(
Bookmark.time.desc()).all()
def user_words(self):
return map((lambda x: x.origin.word), self.all_bookmarks())
def all_bookmarks(self):
return Bookmark.query.filter_by(user_id=self.id).order_by(
Bookmark.time.desc()).all()
def bookmark_count(self):
return len(self.all_bookmarks())
def word_count(self):
return len(self.user_words())
def bookmarks_by_date(self):
def extract_day_from_date(bookmark):
return (bookmark,
bookmark.time.replace(bookmark.time.year,
bookmark.time.month,
bookmark.time.day, 0, 0, 0, 0))
bookmarks = self.all_bookmarks()
bookmarks_by_date = dict()
for elem in map(extract_day_from_date, bookmarks):
bookmarks_by_date.setdefault(elem[1], []).append(elem[0])
sorted_dates = bookmarks_by_date.keys()
sorted_dates.sort(reverse=True)
return bookmarks_by_date, sorted_dates
# returns only HTTP domains. in this way we filter
# out empty domains, and others like the android:
# that we use for internal tracking...
# Returns: list of tuples (domain, date)
def recent_domains_with_times(self):
domains = []
domains_and_times = []
for b in self.bookmarks_chronologically():
if not b.text.url.domain() in domains\
and 'http' in b.text.url.domain():
domains_and_times.append([b.text.url.domain(), b.time])
domains.append(b.text.url.domain())
return domains_and_times
def frequent_domains(self):
domains = map(lambda b: b.text.url.domain(),
self.bookmarks_chronologically())
from collections import Counter
counter = Counter(domains)
return counter.most_common()
def unique_urls(self):
urls = set()
for b in self.all_bookmarks():
urls.add(b.text.url)
return urls
def recommended_urls(self):
urls_to_words = {}
for bookmark in self.all_bookmarks():
if bookmark.text.url.url != "undefined":
urls_to_words.setdefault(bookmark.text.url, 0)
urls_to_words[
bookmark.text.url] += bookmark.origin.importance_level()
return sorted(urls_to_words, key=urls_to_words.get, reverse=True)
def get_not_encountered_words(self, lang):
not_encountered_words_dict_list = []
all_ranks = RankedWord.find_all(lang)
known_word_probs = KnownWordProbability.find_all_by_user_with_rank(
self)
for p in known_word_probs:
if p.ranked_word in all_ranks:
all_ranks.remove(p.ranked_word)
for rank in all_ranks:
not_encountered_word_dict = {}
not_encountered_word_dict['word'] = rank.word
not_encountered_words_dict_list.append(not_encountered_word_dict)
return not_encountered_words_dict_list
def get_not_encountered_words_count(self):
return len(self.get_not_encountered_words(self.learned_language))
def get_known_bookmarks(self, lang):
bookmarks = flask.g.user.all_bookmarks()
known_bookmarks = []
for bookmark in bookmarks:
if bookmark.check_is_latest_outcome_too_easy(
) and lang == bookmark.origin.language:
known_bookmark_dict = {
'id': bookmark.id,
'origin': bookmark.origin.word,
'text': bookmark.text.content,
'time': bookmark.time.strftime('%m/%d/%Y')
}
known_bookmarks.append(known_bookmark_dict)
return known_bookmarks
def get_known_bookmarks_count(self):
return len(self.get_known_bookmarks(self.learned_language))
def get_not_looked_up_words(self, lang):
filtered_words_known_from_user_dict_list = []
enc_probs = EncounterBasedProbability.find_all_by_user(flask.g.user)
for enc_prob in enc_probs:
if enc_prob.ranked_word.language == lang:
filtered_words_known_from_user_dict_list.append(
{'word': enc_prob.ranked_word.word})
return filtered_words_known_from_user_dict_list
def get_not_looked_up_words_for_learned_language(self):
return self.get_not_looked_up_words(self.learned_language)
def get_not_looked_up_words_count(self):
return len(self.get_not_looked_up_words_for_learned_language())
def get_probably_known_words(self, lang):
known_word_prob_of_user = KnownWordProbability.get_probably_known_words(
self)
probable_known_words_dict_list = []
for known_word_prob in known_word_prob_of_user:
probable_known_word_dict = {}
if known_word_prob.ranked_word is not None and known_word_prob.ranked_word.language == lang:
probable_known_word_dict[
'word'] = known_word_prob.ranked_word.word
else:
probable_known_word_dict[
'word'] = known_word_prob.user_word.word
probable_known_words_dict_list.append(probable_known_word_dict)
return probable_known_words_dict_list
def get_probably_known_words_count(self):
return len(self.get_probably_known_words(self.learned_language))
def get_lower_bound_percentage_of_basic_vocabulary(self):
high_known_word_prob_of_user = KnownWordProbability.get_probably_known_words(
self)
count_high_known_word_prob_of_user_ranked = 0
for prob in high_known_word_prob_of_user:
if prob.ranked_word is not None and prob.ranked_word.rank <= 3000:
count_high_known_word_prob_of_user_ranked += 1
return round(
float(count_high_known_word_prob_of_user_ranked) / 3000 * 100, 2)
def get_upper_bound_percentage_of_basic_vocabulary(self):
count_not_looked_up_words_with_rank = 0
not_looked_up_words = EncounterBasedProbability.find_all_by_user(self)
for prob in not_looked_up_words:
if prob.ranked_word.rank <= 3000:
count_not_looked_up_words_with_rank += 1
return round(
float(count_not_looked_up_words_with_rank) / 3000 * 100, 2)
def get_lower_bound_percentage_of_extended_vocabulary(self):
high_known_word_prob_of_user = KnownWordProbability.get_probably_known_words(
self)
count_high_known_word_prob_of_user_ranked = 0
for prob in high_known_word_prob_of_user:
if prob.ranked_word is not None and prob.ranked_word.rank <= 10000:
count_high_known_word_prob_of_user_ranked += 1
return round(
float(count_high_known_word_prob_of_user_ranked) / 10000 * 100, 2)
def get_upper_bound_percentage_of_extended_vocabulary(self):
count_not_looked_up_words_with_rank = 0
not_looked_up_words = EncounterBasedProbability.find_all_by_user(self)
for prob in not_looked_up_words:
if prob.ranked_word.rank <= 10000:
count_not_looked_up_words_with_rank += 1
return round(
float(count_not_looked_up_words_with_rank) / 10000 * 100, 2)
def get_percentage_of_probably_known_bookmarked_words(self):
high_known_word_prob_of_user = KnownWordProbability.get_probably_known_words(
self)
count_high_known_word_prob_of_user = 0
count_bookmarks_of_user = len(self.all_bookmarks())
for prob in high_known_word_prob_of_user:
if prob.user_word is not None:
count_high_known_word_prob_of_user += 1
if count_bookmarks_of_user <> 0:
return round(
float(count_high_known_word_prob_of_user) /
count_bookmarks_of_user * 100, 2)
else:
return 0
# Reading recommendations
def recommendations(self):
recommendations = {
'de': [[
'Der Spiegel', 'http://m.spiegel.de', 'news, advanced',
'World News'
]]
}
try:
return recommendations[self.learned_language_id]
except:
return []
