def nouns():
file = open("nouns.txt", "r")
word_count = 0
for line in file:
if Word.query.filter_by(word=line.rstrip()).count() == 0:
dbWord = Word(word=line.rstrip())
dbWord.save()
word_count += 1
file = open("actions.txt", "r")
action_count = 0
for line in file:
if Action.query.filter_by(action=line.rstrip()).count() == 0:
dbAction = Action(action=line.rstrip())
dbAction.save()
action_count += 1
file = open("categories.txt", "r")
category_count = 0
for line in file:
if Category.query.filter_by(name=line.rstrip()).count() == 0:
dbCategory = Category(name=line.rstrip())
dbCategory.save()
category_count += 1
return jsonify({
"Words Added": word_count,
"Actions Added": action_count,
"Categories Added": category_count
})
def execute_adding_data():
words_count = Word.query.count()
users_count = User.query.count()
if words_count > 0:
print('initial words loaded in database')
else:
for item in lexicon:
term = item.get("term")
meaning = item.get("meaning")
pronunciation = item.get("pronunciation")
category = item.get("category")
mnemonic_phrase = item.get("mnemonic_phrase")
new_word = Word(term, meaning, pronunciation, category,
mnemonic_phrase)
new_word.add()
print('words_added')
if users_count > 0:
print('initial users loaded in database')
else:
for item in users:
name = item.get("name")
lastname = item.get("lastname")
email = item.get("email")
password_user = item.get("password")
password = generate_password_hash(password_user, method='sha256')
avatar = create_avatar(name, lastname)
new_user = User(name, lastname, email, password, avatar)
new_user.add()
print('users_added')
def test_model_unit(self):
"""Test to make sure that Unit is working properly.
"""
unit_type = Property(name="unit_type", value="section")
number = 1
unit = Unit()
unit.properties = [unit_type]
unit.number = number
assert unit.number == number
sentence = Sentence()
sentence.words = [Word(lemma="hello"), Word(lemma="world")]
prop = Property(name="title", value="Hello World")
unit.sentences.append(sentence)
unit.properties.append(prop)
assert unit.sentences == [sentence]
assert unit.properties.all() == [unit_type, prop]
unit.save()
prop.save()
retrieved_prop = Property.query.filter(Property.name=="title").\
filter(Property.value == "Hello World").first()
assert retrieved_prop.unit.type == "unit"
assert retrieved_prop.unit.number == unit.number
def delete(self, user_id, dictionary_id, chaper_id):
json_data = request.get_json(force=True)
if not json_data:
return {'message': 'No input data provided'}, 400
Word.filter_by(id=json_data['id']).delete()
db.session.commit()
return {"status": 'success', 'data': {}}, 204
def search():
if request.method == "GET":
wordlist_collection = current_user.wordlists.all()
return render_template("search.html",
wordlist_collection=wordlist_collection)
elif request.method == "POST":
selected_wl = request.form.get('my_wordlist')
req = request.get_json()
print("my word attributes are:", req)
print(req['name'])
print(req['type'])
print(req['definition'])
print(req['selected_wordlist'])
wl = Wordlist.query.get(int(req['selected_wordlist']))
print(wl)
example1 = 'example1'
if example1 in req.keys():
print(req['example1'])
w = Word(name=req['name'],
part=req['type'],
definition=req['definition'],
example1=req['example1'])
else:
print("key doesnt exist")
w = Word(name=req['name'],
part=req['type'],
definition=req['definition'])
wl.words.append(w)
db.session.commit()
flash("Word successfully added!")
return redirect(url_for("index"))
def fill_out_demo_data():
""" Create demo user, dictionary, words """
app = create_app()
with app.app_context():
words = []
with open('demo.json') as json_file:
data = json.load(json_file)
for word in data['words']:
words.append({
'spelling': word['spelling'],
'definition': word['definition'],
'synonyms': word['synonyms']
})
# User
demo_user = User.query.filter_by(username='******').first()
if not demo_user:
demo_user = User(username='******')
demo_user.set_password(Config.DEMO_PASS)
db.session.add(demo_user)
db.session.commit()
# Dictionary
demo_dict = Dictionary.query.\
filter_by(user_id=demo_user.id).\
filter_by(dictionary_name='Demo').first()
if not demo_dict:
demo_dict = Dictionary(user_id=demo_user.id, dictionary_name='Demo')
db.session.add(demo_dict)
db.session.commit()
# Words
# words = Word.query.filter_by(dictionary_id=demo_dict.id).all()
for word in words:
demo_word = Word.query.\
filter_by(dictionary_id=demo_dict.id,).\
filter_by(spelling=word['spelling']).first()
if not demo_word:
demo_word = Word(
dictionary_id=demo_dict.id,
spelling=word['spelling']
)
demo_word.definition = word['definition']
db.session.add(demo_word)
db.session.commit()
for synonym in word['synonyms']:
demo_synonym = WordSynonyms.query.\
filter_by(word_id=demo_word.id).\
filter_by(synonym=synonym).first()
if not demo_synonym:
demo_synonym = WordSynonyms(word_id=demo_word.id, synonym=synonym)
db.session.add(demo_synonym)
db.session.commit()
def test_follow_words(self):
# create four users
u1 = User(username='******', email='*****@*****.**')
u2 = User(username='******', email='*****@*****.**')
u3 = User(username='******', email='*****@*****.**')
u4 = User(username='******', email='*****@*****.**')
db.session.add_all([u1, u2, u3, u4])
# create four posts
now = datetime.utcnow()
w1 = Word(name="cool")
w2 = Word(name="sadie")
w3 = Word(name="new")
db.session.add_all([w1, w2, w3])
db.session.commit()
# users each have dictionary
d1 = Dictionary(name='john dict', user_id=u1.id)
d2 = Dictionary(name='susan dict', user_id=u2.id)
d3 = Dictionary(name='mary dict', user_id=u3.id)
d4 = Dictionary(name='david dict', user_id=u4.id)
db.session.add_all([d1, d2, d3, d4])
db.session.commit()
# users add words to their dictionaries
uw1 = UserWord(word_id=w1.id, user_id=u1.id, dictionary_id=d1.id)
uw2 = UserWord(word_id=w1.id, user_id=u2.id, dictionary_id=d2.id)
uw3 = UserWord(word_id=w1.id, user_id=u3.id, dictionary_id=d3.id)
uw4 = UserWord(word_id=w2.id, user_id=u4.id, dictionary_id=d4.id)
uw5 = UserWord(word_id=w3.id, user_id=u4.id, dictionary_id=d4.id)
uw6 = UserWord(word_id=w2.id, user_id=u3.id, dictionary_id=d3.id)
db.session.add_all([uw1, uw2, uw3, uw4, uw5, uw6])
db.session.commit()
# setup the followers
u1.follow(u2) # john follows susan
u1.follow(u4) # john follows david
u2.follow(u3) # susan follows mary
u3.follow(u4) # mary follows david
db.session.commit()
# check the followed posts of each user
f1 = u1.friend_highlights().all() # john should see susan's and david dictionary words
f2 = u2.friend_highlights().all() # susan should see mary words
f3 = u3.friend_highlights().all()
f4 = u4.friend_highlights().all()
self.assertEqual(f1, [uw5, uw4, uw2])
self.assertEqual(f2, [uw6, uw3])
self.assertEqual(f3, [uw5, uw4])
self.assertEqual(f4, [])
def init_matches(word_list):
while word_list: # removes words from array until empty
pair = random.sample(word_list, k=2)
word_list.remove(pair[0])
word_list.remove(pair[1])
word_1 = Word(word=pair[0], matched=True)
word_2 = Word(word=pair[1], matched=True)
db.session.add(word_1)
db.session.add(word_2)
match = Match(word_1=word_1.word, word_2=word_2.word)
db.session.add(match)
db.session.commit()
def test_userWords(self):
u = User(username='******', email='*****@*****.**')
w = Word(context="wordtest")
w2 = Word(context="word2test")
db.session.add(u)
db.session.add(w)
db.session.add(w2)
db.session.commit()
uw = UserWord(word_id=w.id, user_id=u.id)
uw2 = UserWord(word_id=w2.id, user_id=u.id)
db.session.add(uw)
db.session.add(uw2)
db.session.commit()
self.assertEqual(u.words.all(), [uw, uw2])
def transcribe_document(
document: Document,
aws_bucket_name: str,
lang: str,
):
filekey = document.filename
job_raw = transcribe(
filekey=filekey,
aws_bucket_name=aws_bucket_name,
lang=lang,
)
full_text = job_raw["results"]["transcripts"][0]["transcript"]
transcription = AWSTranscription(raw=job_raw, full_text=full_text)
document.words = []
for i, item in enumerate(job_raw["results"]["items"]):
if item["type"] == "punctuation":
continue
alternative = item["alternatives"][0]
assert "start_time" in item, item
word = Word(
word=alternative["content"],
order=i,
start_time=timedelta(seconds=float(item["start_time"])),
end_time=timedelta(seconds=float(item["end_time"])),
confidence=float(alternative["confidence"]),
)
document.words.append(word)
document.transcription = transcription
def get(value: str, language, all_translations=False):
"""Returns the User instance, with its token
only, by searching it by its email
"""
try:
if not check_spelling(value, language):
raise ParameterExpected("Misspelled word provided!")
except ValueError as ve_err:
logger.error("Error while checking spell word : %s", str(ve_err))
raise ParameterExpected("Invalid language provided!")
result = None
try:
word_translations = Word.objects(
value=value, language=language).only('translations').first()
# translations contains only the first translation
logger.info(str(word_translations))
translations = None
if word_translations:
translations = list(word_translations.translations)
logger.info(str(translations))
if not all_translations:
translations = [translations[0]]
result = translations
logger.info(str(result))
except LookUpError as lue_err:
logger.error("Error while getting word : %s", str(lue_err))
except TypeError as te_err:
logger.error("Error while getting word : %s", str(te_err))
# pass
return result
def conv_cizu2zi():
words = {}
for w in Word.query.filter(func.length(Word.word) <= 3).all():
words.setdefault((w.user_id, w.word, w.book, w.chapter), w)
print(f"read {len(words)} cizu")
del_ids = []
for w in Word.query.filter(func.length(Word.word) > 3).all():
cizu = w.word
can_delete = True
for zi in cizu:
k = (w.user_id, zi, w.book, w.chapter)
if k in words:
print(f"skip {zi}")
else:
print("adding ", zi, w.book, w.chapter)
w2 = Word(user_id=w.user_id, word = zi, book = w.book,
chapter = w.chapter, study_date = w.study_date,
cur_xpoints = w.cur_xpoints,
tot_xpoints = w.tot_xpoints,
num_pass = w.num_pass,
num_fail = w.num_fail,
streak = w.streak,
istep = w.istep,
next_study = w.next_study)
db.session.add(w2)
can_delete = False
words[k] = w2
db.session.commit()
if can_delete:
print(f"delete {cizu}, {w.id}")
del_ids.append(w.id)
db.session.delete(w)
db.session.commit()
def add_word():
""" Add new word to db """
db_user = User.check_request(request)
if not db_user or not db_user.is_authenticated():
return {'message': 'Demo mode'}
request_data = request.get_json()
synonyms = request_data.get('synonyms')
new_word = Word(spelling=request_data.get('spelling').strip(),
definition=request_data.get('definition').strip(),
dictionary_id=request_data.get('dictionary_id'))
db.session.add(new_word)
# Commit to save new word
db.session.commit()
# Add learning index
learning_index = LearningIndex(word_id=new_word.id, index=0)
db.session.add(learning_index)
# Add synonyms
if synonyms:
for synonym in synonyms:
db_synonym = WordSynonyms(word_id=new_word.id, synonym=synonym)
db.session.add(db_synonym)
db.session.commit()
logger.info(f'Added word: {new_word.spelling}')
return {'new_word_id': new_word.id}
def test_model_word(self):
"""Test to make sure that the atttributes of the Word model can be
properly set.
"""
string_1 = "hello"
string_2 = "world"
word_1 = Word()
word_2 = Word()
word_1.word = string_1
word_2.word = string_2
assert word_1.word == string_1
assert word_2.word == string_2
def test_delete_word(self):
word = Word(chapter=self.chapter, word='хліб', translation='bread')
self.refresh_db(word)
db.session.delete(word)
db.session.commit()
yield db
assert db.session.query(Word).count() == 0
def add_word():
new_word = Word(spelling=request.form['spelling'].strip(),
definition=request.form['definition'].strip(),
dictionary_id=request.form['dictionary_id'])
db.session.add(new_word)
db.session.commit()
return jsonify({'new_word_id': new_word.id})
def create(value, language, translation):
"""Create Word/add translation to existing word for App
It creates a valid Word for the app. If the word exists,
it will add the new translation to the word's translations. If it doens't
it will create a new Word with the translation provided
"""
if not check_spelling(value, language):
raise ParameterExpected("Misspelled word provided!")
result = None
try:
value = validate_string(value, "[^a-z]")
valid_word = Word.objects(value=value, language=language).first()
#word already exist so let's add a new translation
if not valid_word:
valid_word = Word(value=value, language=language)
valid_word.save()
valid_word.update(add_to_set__translations=[translation])
except Exception as ue_err:
logger.error(str(ue_err))
result = True
return result
def save_words(self, form: forms.WordsForm):
words = [
Word(text=text,
translate=translate,
phrase=phrase,
user=self._user)
for text, translate, phrase in form.get_translates()
]
Word.objects.bulk_create(words, batch_size=500)
def index():
form = TranslationForm()
if form.validate_on_submit():
word = Word(word=form.input_text.data.lower(), translate=form.output_text.data.lower(), user_id=int(current_user.get_id()))
db.session.add(word)
db.session.commit()
flash('"{}" added to your dictionary'.format(word.word))
return redirect(url_for('main.index'))
return render_template('index.html', form=form)
def setUpClass(cls):
database.clean()
cls.word1 = Word(lemma="bar")
cls.word2 = Word(lemma="baz")
cls.word3 = Word(lemma="bar")
cls.word4 = Word(lemma="qux")
sequence1 = Sequence(words=[cls.word1, cls.word2])
sequence2 = Sequence(words=[cls.word4])
sequence3 = Sequence(words=[cls.word3])
cls.sequenceset1 = SequenceSet(sequences=[sequence1, sequence2])
cls.sequenceset2 = SequenceSet(sequences=[sequence3])
db.session.add_all([
sequence1, sequence2, sequence3, cls.sequenceset1,
cls.sequenceset2, cls.word1, cls.word2, cls.word3, cls.word4
])
db.session.commit()
def test_model_sentence(self):
"""Test to make sure that Sentence is working properly.
"""
text = "hello world"
sentence = Sentence()
sentence.text = text
assert sentence.text == text
word_1 = Word(lemma="hello")
word_2 = Word(lemma="world")
sentence.words.append(word_1)
sentence.words.append(word_2)
assert sentence.words == [word_1, word_2]
sentence.save()
#Test with Dependencies
dependency1 = Dependency()
dependency2 = Dependency()
sentence.dependencies = [dependency1, dependency2]
db.session.add_all([dependency1, dependency2])
db.session.commit()
#Test with Sequences
sequence1 = Sequence()
sequence2 = Sequence()
sentence.sequences = [sequence1, sequence2]
db.session.add_all([sequence1, sequence2])
db.session.commit()
#Test with Project
project = Project()
sentence.project = project
db.session.add_all([project])
db.session.commit()
def save_word_to_db(res):
word = res['word']
definitions = res['results']
w = Word(context=word)
db.session.add(w)
for d in definitions:
dd = Definition(context=d['definition'], word=w)
db.session.add(dd)
db.session.commit()
return jsonify(success=True)
def test_add_word(self):
"""Test the ``add_word()`` method of ``Sentence``.
"""
sentence = Sentence(text="foo")
word = Word(word="foo")
project = Project()
project.save()
sentence.save()
word.save()
rel = sentence.add_word(word, position=4, space_before=" ",
part_of_speech="ADF", project=project)
assert rel.word == word
assert rel.sentence == sentence
assert rel.position == 4
assert rel.space_before == " "
assert rel.part_of_speech == "ADF"
assert rel.project == project
def getWords():
try:
keywordsFile = open('C:/keywords.json', 'rb')
except:
print 'Keywords.json not found.'
sys.exit()
try:
f = keywordsFile.read()
keywordsDict = jpickle.decode(f)
except:
keywordsDict = []
print 'Keywords list corrupted'
sys.exit()
keywordsFile.close()
#Sorted by average rating
SortedByRating = {}
SortedByRating['keywords'] = []
SortedByRating['ratings'] = []
SortedByRating['usage'] = []
for word in sorted(keywordsDict.items()[10400:], key=lambda k: (k[1][0] / k[1][1]), reverse=True):
try:
ratingSum = word[1][0]
usage = word[1][1]
newWord = Word()
newWord.rating = ratingSum/usage
newWord.value = word[0]
newWord.votes = usage
newWord.save()
except Exception,e:
HttpResponse("ratingsum: " +word[1][0] + " usage:" + word[1][1] + "value:" + word[0] + str(e) )
def test_definitions(self):
w = Word(context="test word")
w2 = Word(context="test word2")
d1w = Definition(context="definition1 of word is definitions", word=w)
d2w = Definition(context="definition2 of word is definitions", word=w)
d1w2 = Definition(context="definition1 of word2 is definitions",
word=w2)
d2w2 = Definition(context="definition2 of word2 is definitions",
word=w2)
db.session.add(w)
db.session.add(d1w)
db.session.add(d2w)
db.session.add(d1w2)
db.session.add(d2w2)
db.session.commit()
self.assertEqual(d1w.word, w)
self.assertEqual(d2w.word, w)
self.assertEqual(d1w2.word, w2)
self.assertEqual(d2w2.word, w2)
self.assertEqual(w.definitions.all(), [d1w, d2w])
self.assertEqual(w2.definitions.all(), [d1w2, d2w2])
def test_add_word(self):
"""Test the ``add_word()`` method of ``Sentence``.
"""
project = Project()
sentence = Sentence(text="foo", project=project)
word = Word(lemma="foo")
project.save()
sentence.save()
word.save()
rel = sentence.add_word(word,
position=4,
space_before=" ",
project=project)
assert rel.word == word
assert rel.sentence == sentence
assert rel.position == 4
assert rel.space_before == " "
assert rel.project == project
def test_add_dependency(self):
"""Test the ``add_dependency()`` method of ``Sentence``.
"""
sentence = Sentence(text="foo")
word = Word(word="foo")
dependency = Dependency(governor=word)
project = Project()
project.save()
sentence.save()
dependency.save()
word.save()
rel = sentence.add_dependency(dependency, governor_index=1,
dependent_index=2, project=project)
assert rel.dependency == dependency
assert rel.sentence == sentence
assert rel.governor_index == 1
assert rel.dependent_index == 2
assert rel.project == project
def wordbook_add():
form = request.form
r = {'data': []}
book_name = form['book_name']
wordbook = Wordbook.query.filter_by(book_name=book_name,
user_id=g.user.id).first()
if wordbook is not None:
r['success'] = False
r['message'] = "单词书已存在"
return json.dumps(r, ensure_ascii=False)
else:
wordbook = Wordbook(book_name, g.user.id)
wordbook.save()
# 此处硬编码
with open('wordbook.json', 'r') as f:
wordbook_data = json.loads(f.read())
for w in wordbook_data:
word = Word(w, wordbook.id)
word.save()
r['success'] = True
return json.dumps(r, ensure_ascii=False)
def create():
identity = get_jwt_identity()
user = User.query.filter_by(username=identity['username']).first()
g.current_user = user
data = request.get_json() or {}
dictionary = Dictionary.query.get(data['dictionary_id'])
if dictionary in g.current_user.dictionaries:
dictionary_id = dictionary.id
else:
abort(403)
if 'name' not in data:
return bad_request('word cannot be blank')
if Word.query.filter_by(name=data['name']).first():
word = Word.query.filter_by(name=data['name']).first()
else:
word = Word(data["name"], created_by=g.current_user.username)
db.session.add(word)
db.session.commit()
user_word = UserWord(word.id, dictionary_id, g.current_user.id)
db.session.add(user_word)
try:
db.session.commit()
except IntegrityError:
return bad_request('word already is in your dictionary')
if 'description' in data:
user_word.description = data['description']
db.session.add(user_word)
db.session.commit()
if 'translations' in data:
for t in data['translations']:
trns = Translation(word.id, dictionary.id, t)
db.session.add(trns)
db.session.commit()
response = jsonify(word.as_json())
response.status_code = 201
response.headers['Location'] = url_for('words.show', id=word.id)
return response
def user(username):
form = WordForm()
if form.validate_on_submit():
word = Word(word=form.word.data,
translation=form.translation.data,
context=form.context.data,
note=form.note.data,
user=current_user)
db.session.add(word)
db.session.commit()
flash('Your word has been added!')
return redirect(url_for('user', username=current_user.username))
user = User.query.filter_by(username=username).first_or_404()
posts = current_user.words.all()
return render_template('user.html', user=user, posts=posts, form=form)
def test_add_dependency(self):
"""Test the ``add_dependency()`` method of ``Sentence``.
"""
project = Project()
sentence = Sentence(text="foo", project=project)
word = Word(lemma="foo")
dependency = Dependency(governor=word)
project.save()
sentence.save()
dependency.save()
word.save()
rel = sentence.add_dependency(dependency,
governor_index=1,
dependent_index=2,
project=project)
assert rel.dependency == dependency
assert rel.sentence == sentence
assert rel.governor_index == 1
assert rel.dependent_index == 2
assert rel.project == project
def add_word():
foreign_language = ""
if current_user.language.lit == request.form['foreign_Language']:
foreign_language = request.form['native_Language']
else:
foreign_language = request.form['foreign_Language']
word = Word(native_language=request.form['text'],
foreign_language=request.form['translation'],
dictionary_id=Dictionary.query.filter_by(
language_id=Language.query.filter_by(
lit=foreign_language).first().id).filter_by(
user_id=current_user.id).first().id)
db.session.add(word)
db.session.commit()
flash(_('The word has been added in the dictionary ! '))
return render_template('search.html', title='Search')
def post(self, user_id, dictionary_id, chapter_id):
json_data = request.get_json(force=True)
if not json_data:
return {'message': 'No input data provided'}, 400
# Validate and deserialize input
data, errors = user_schema.load(json_data)
if errors:
return errors, 422
word = Word.query.filter_by(id=data['id']).first()
if word:
return {'message': 'Word already exists'}, 400
word = Word(chapter=json_data['chapter'],
word=json_data['word'],
translation=json_data['translation'])
db.session.add(word)
db.session.commit()
result = word_schema.dump(word).data
return {"status": 'success', 'data': result}, 201
def test_model_word(self):
"""Test to make sure that the atttributes of the Word model can be
properly set.
"""
string_1 = "hello"
string_2 = "world"
word_1 = Word()
word_2 = Word()
word_1.word = string_1
word_2.word = string_2
assert word_1.word == string_1
assert word_2.word == string_2
def inject_word_classes():
return {'word_classes': [word_class.__mapper_args__['polymorphic_identity']
for word_class in Word.__subclasses__()]}
