def setUp(self):
"""Set up environment for api view test suite"""
self.client = APIClient()
self.take_data = {'location': 'test_location', 'chapter': 5}
self.lang_data = {'lang': 'english', 'code': 'abc'}
self.user_data = {
'name': 'tester',
'agreed': True,
'picture': 'test.pic'
}
self.comment = {'location': 'test_location'}
self.book_data = {'code': 'ex', 'name': 'english', 'booknum': 5}
self.take_object = Take(location='test_location',
id=1,
language_id=1,
book_id=1,
user_id=1)
self.language_object = Language(code='en-demo', name='english', id=1)
self.book_object = Book(code='en-demo',
name='english',
booknum=5,
id=1)
self.user_object = User(name='testy',
agreed=True,
picture='mypic.jpg',
id=1)
self.comment_object = Comment.objects.create(
location='/test-location/', id=1)
def inserts(self):
"""Do some initial inserts for the test"""
en = Language(id=1, language='en')
self.db.session.add(en)
es = Language(id=2, language='es')
self.db.session.add(es)
requested = TranslationStatus(id=1, status='requested')
self.db.session.add(requested)
pending = TranslationStatus(id=2, status='pending')
self.db.session.add(pending)
translated = TranslationStatus(id=3, status='translated')
self.db.session.add(translated)
translation = Translation(uid='336f7c6064',
source_language_id=1,
input_text='hello world',
target_language_id=2,
output_text='hola mundo',
status_id=1)
self.db.session.add(translation)
self.db.session.commit()
self.db.session.close()
def index():
devs = Developer.query.all()
orgs = Organization.query.all()
lang = Language.query.all()
if not lang:
print("lang empty")
Language.fillLangauges()
return render_template("register.html", lang=lang, devs=devs, orgs=orgs)
def populate():
new_langs = [
Language('Python', 'py'),
Language('Ruby', 'rb'),
Language('Common Lisp', 'lisp'),
Language('Objective-C', 'm'),
]
session.add_all(new_langs)
session.commit()
def populate(request):
session = request.db_session
new_langs = [
Language('Python', 'py'),
Language('Ruby', 'rb'),
Language('Common Lisp', 'lisp'),
Language('Objective-C', 'm')
]
session.add_all(new_langs)
session.commit()
def create_languages(doc):
# Create a unique list of all Languages in the parsed document
languages= list()
path = '//@xml:lang'
for result in doc.xpath(path, namespaces=ns):
if result not in languages:
languages.append(result)
for lang in languages:
if len(Language.objects.filter(abbreviation=str(lang))) == 0:
new_language = Language(abbreviation=str(lang))
new_language.save()
def create_languages(doc):
# Create a unique list of all Languages in the parsed document
languages = list()
path = '//@xml:lang'
for result in doc.xpath(path, namespaces=ns):
if result not in languages:
languages.append(result)
for lang in languages:
if len(Language.objects.filter(abbreviation=str(lang))) == 0:
new_language = Language(abbreviation=str(lang))
new_language.save()
def get_languages(lang):
items = []
if lang == 'en':
items = Language.objects().exclude("translation").order_by("name")
items = items2dict(items)
elif lang is not None:
items = Language.objects(
translation__language=lang).exclude("name").exclude("language")
items = items2dict(items)
items = translations_unify(items, lang)
else:
item = []
return items
def language():
Entry = namedtuple('Entry',
['language_name', 'speaking', 'reading', 'writing'])
language_value = Language.load(current_user.id)
if language_value.language_list:
data = {'language_list': []}
for language_dict in language_value.language_list:
data['language_list'].append(
Entry(language_dict['language_name'],
language_dict['speaking'], language_dict['reading'],
language_dict['writing']))
form = LanguageListForm(data=data)
else:
form = LanguageListForm()
if form.validate_on_submit():
if form.add.data:
if bool(form.new_language.data) and (form.speaking.data
or form.reading.data
or form.writing.data):
language_dict = {
'language_name': form.new_language.data,
'speaking': form.speaking.data,
'reading': form.reading.data,
'writing': form.writing.data
}
language_value.add(language_dict)
flash('Language added', 'success')
commit()
else:
flash('Empty language field', 'danger')
return redirect(url_for('language.language'))
else:
for index in range(len(form.language_list)):
if form.language_list[index].save.data:
if bool(form.language_list[index].language_name.data) and (
form.language_list[index].speaking.data
or form.language_list[index].reading.data
or form.language_list[index].writing.data):
language_dict = {
'language_name':
form.language_list[index].language_name.data,
'speaking':
form.language_list[index].speaking.data,
'reading':
form.language_list[index].reading.data,
'writing':
form.language_list[index].writing.data
}
language_value.update(index, language_dict)
commit()
flash('Updated successfully', 'success')
else:
flash('Empty language field', 'danger')
return redirect(url_for('language.language'))
if form.language_list[index].delete.data:
language_value.delete(index)
commit()
flash('Deleted successfully', 'success')
return redirect(url_for('language.language'))
return render_template('language.html', form=form)
def run_task(data_dir, task_name='Language modeling'):
snli_loader = SNLILoader(data_dir)
train_data = snli_loader.train_data
test_data = snli_loader.test_data
lang_config = config(snli_loader, train_data, test_data)
with tf.Session() as sess:
model = Language(lang_config, sess, train_data, test_data)
if len(argvs) < 2:
model.build_model()
model.run(task_name)
else:
model.demo_model()
model.demo(argvs[1])
def edit_vocab_word(word_id):
word = VocabWord.get_by_id(word_id)
form = VocabWordForm(obj=word)
form.source_code.choices = Language.get_all_options()
if form.validate_on_submit():
source_code = form.source_code.data
root = form.root.data
translation = form.translation.data
definition = form.definition.data
synonyms = form.synonyms.data
examples = form.examples.data
notes = form.notes.data
update_vocab_word = word.update(source_code,
root, translation, definition, synonyms, examples, notes)
g.user.update_last_language(source_code)
flash(f"{update_vocab_word.root} edited!", 'success')
return redirect(f"/words/{update_vocab_word.id}")
else:
flash(f"Error updating vocabulary word!", 'warning')
return render_template('edit_vocab_word.html', vocab_word_form=form, word=word)
def get_language_by_id(_id, lang):
item = {}
if lang == 'en':
item = Language.objects(_id=_id).exclude("translation").first()
item = item2dict(item)
elif lang is not None:
item = Language.objects(_id=_id,
translation__language=lang).only("translation")
if item:
item = item[0]
item = item2dict(item)
item = translation_unify(item, lang)
else:
item = {}
return item
def update_user(payload={}, fields=[]):
if 'user_id' in payload:
user_id = payload.pop('user_id')
# print(payload)
user = User.objects(pk=user_id)
if user is not None:
user = user.first()
if 'country' in payload:
payload['country'] = Country(pk=payload['country'])
if 'gender' in payload and payload['gender'] is not None:
payload['gender'] = Gender(pk=int(payload['gender']))
if 'languages' in payload:
payload['languages'] = [
Language(pk=lang) for lang in payload['languages']
]
if 'department' in payload and payload['department'] is not None:
payload['department'] = Department(
pk=int(payload['department']))
if 'chair' in payload and payload['chair'] is not None:
payload['chair'] = Chair(pk=int(payload['chair']))
if 'bio' in payload: payload["bio"] = payload['bio']
updated = user.update(**payload)
print(user)
return updated >= 1, payload
else:
print("Not found")
return False, None
else:
print("Not found")
return False, None
def add_new_variation_by_api():
current_user = get_jwt_identity()
user = User.get_by_id(current_user)
form = VocabWordAndComponentForm()
form.source_code.choices = Language.get_all_options()
if form.validate():
root_id = request.json['root_id']
user_id = user.id
part_of_speech = request.json['part_of_speech']
word = request.json['word']
translation = request.json['translation']
examples = request.json['examples']
new_component = VocabWordComponent.add_variation(
root_id, user_id, part_of_speech, word, translation, examples)
response = {
'component': new_component.serialize(),
'status': 'success'
}
return jsonify(response)
else:
response = {
'errors': form.errors,
'status': 'errors'
}
return jsonify(response)
def view_user_words(source_code):
languages = Language.get_all_options()
sorted_words = db.session.query(VocabWord).filter(
VocabWord.owner_id == g.user.id, VocabWord.source_code == source_code).order_by(VocabWord.root).all()
return render_template('words.html', languages=languages, words=sorted_words, source_code=source_code)
def add_to_data_load(data):
country = Country(
code=data.get('alpha2Code'),
name=data.get('name'),
capital=data.get('capital'),
region=data.get('region') or 'Unknown',
subregion=data.get('subregion'),
population=data.get('population'),
flag=data.get('flag'),
)
if country not in countries:
countries.append(country)
for language_data in data.get('languages'):
language = Language(code=language_data.get('iso639_2'),
name=language_data.get('name'))
if language not in languages:
languages.append(language)
country_language.append(
CountryLanguage(country_code=country.code,
language_code=language.code))
for currency_data in data.get('currencies'):
if currency := validate_currency(currency_data):
if currency not in currencies:
currencies.append(currency)
country_currency.append(
CountryCurrency(country_code=country.code,
currency_code=currency.code))
def admin_dashboard_language_add():
if request.form:
language = Language(name=request.form.get('name'))
db_session.add(language)
db_session.commit()
flash('Language successfully added')
return redirect(url_for('admin_dashboard_languages'))
return render_template('/admins/language_add.html')
def test_language_column_queries(self):
"""
test column queries
"""
tokens = ['%bacon%', '%eggs%']
queries = Language.column_queries(tokens)
for query in queries:
self.assertTrue(
isinstance(query, sqlalchemy.sql.elements.BooleanClauseList))
def add_language():
if request.method == 'POST':
name = request.form['name']
short_name = request.form['short_name']
new_language = Language(name, short_name)
session.add(new_language)
session.commit()
flash(u'Язык добавлен')
return redirect(url_for('words'))
else:
return redirect(url_for('words'))
def show_vocab_word(word_id):
word = VocabWord.get_by_id(word_id)
components = word.components
parts_of_speech = word.components_pos()
print(f"POS: {parts_of_speech}", file=sys.stderr)
vocab_word_form = VocabWordForm(obj=word)
vocab_word_form.source_code.choices = Language.get_all_options()
vocab_component_form = VocabComponentForm()
return render_template('view_vocab_word.html', vocab_word_form=vocab_word_form, vocab_component_form=vocab_component_form, word=word, components=components, parts_of_speech=parts_of_speech)
def setUp(self):
paradigm = Paradigm(name='Functional')
language = Language(name='tstLang',
creator='bob',
type='Interpreted',
firstAppeared=1987,
description='test')
company = Company(name='TestCom',
email='*****@*****.**',
url="test.com/testcom",
avatar_url='test.com/testcom/avatar',
description='test')
contributor = Contributor(username='******',
email='*****@*****.**',
url='test.com/testusr',
avatar_url='test.com/testusr/avatar',
location='Texas')
project = Project(name='testProj',
url='test.com/testproj',
description='test',
createdDate='June, 11',
private=False)
language.id = 1
contributor.id = 2
company.id = 3
project.id = 4
language.paradigms.append(paradigm)
project.languages.append(language)
project.company = company
project.owner_id = company.id
project.contributors.append(contributor)
company.members.append(contributor)
self.paradigm = paradigm
self.language = language
self.contributor = contributor
self.company = company
self.project = project
def insert_tags(company_id, language_tags):
df = pd.DataFrame(language_tags)
for index, row in df.iterrows():
tag_id = None
for country_code in df.columns:
tag_value = row[country_code]
tag_name = TagName.query \
.filter_by(
name=tag_value
) \
.first()
curr_tag_id = tag_name.tag_id if tag_name else False
if tag_id is None:
tag_id = curr_tag_id
else:
assert tag_id == curr_tag_id, \
'prev({}) != curr({}) of {}'.format(
tag_id, curr_tag_id, tag_value)
assert tag_id is not None
if tag_id is False:
tag = Tag()
db.session.add(tag)
db.session.commit()
print('tag', tag.id)
for country_code in df.columns:
tag_value = row[country_code]
language = Language.query \
.filter_by(country_code=country_code) \
.first()
if language is None:
language = Language(country_code)
db.session.add(language)
db.session.commit()
# print('language', language.id)
tag_name = TagName(tag.id, language.id, tag_value)
db.session.add(tag_name)
db.session.commit()
print('tag_name', tag_name.id, tag_value)
tag_id = tag.id
else:
print('already registered tag', tag_id)
company_tag = CompanyTag(company_id, tag_id)
db.session.add(company_tag)
db.session.commit()
def get_specific_exercise(language, name):
known_language = Language.get(language)
if not known_language:
return 404
exercise, artifact = session.query(Exercise, Artifact)\
.filter(Exercise.artifact_id == Artifact.id)\
.filter_by(language=language, name=name)\
.order_by(asc(Exercise.difficulty))\
.first()
results = complete_exercise_to_dict(exercise, artifact)
return jsonify(results)
def post(self):
"""
Creates language
"""
name = request.json.get('name')
value = request.json.get('value')
extension = request.json.get('extension')
new_language = Language(name=name, value=value, extension=extension)
db.session.add(new_language)
db.session.commit()
return {
'id': new_language.id,
'name': new_language.name,
'value': new_language.value,
'extension': new_language.extension
}, 201
def get_exercises_by_language(language):
limit = extract_int_arg(request, 'page_size', default=default_limit)
offset = extract_int_arg(request, 'page', default=0)
known_language = Language.get(language)
if not known_language:
return (404)
exercises = session.query(Exercise)\
.filter_by(language=known_language)\
.order_by(asc(Exercise.difficulty))\
.limit(limit)\
.offset(offset)\
.all()
results = [exercise_to_dict(ex) for ex in exercises]
return jsonify(results)
def get_topics_by_language(language):
limit = extract_int_arg(request, 'page_size', default=default_limit)
offset = extract_int_arg(request, 'page', default=0)
known_language = Language.get(language)
if not known_language:
return (404)
topic = func.unnest(Exercise.topics)
results = session\
.query(topic, Exercise.language)\
.filter_by(language=known_language)\
.group_by(topic, Exercise.language)\
.order_by(asc(topic))\
.limit(limit)\
.offset(offset)\
.all()
return jsonify(pluck_first_column(results))
def add_new_word_by_api():
current_user = get_jwt_identity()
user = User.get_by_id(current_user)
form = VocabWordAndComponentForm()
form.source_code.choices = Language.get_all_options()
if form.validate():
user_id = user.id
source_code = request.json['source_code']
part_of_speech = request.json['part_of_speech']
word = request.json['word']
translation = request.json['translation']
definition = request.json['definition']
synonyms = request.json['synonyms']
examples = request.json['examples']
if definition == '0':
definition = ''
new_word = VocabWord.add_vocab_word(
user_id, source_code, word, translation, definition, synonyms, examples, '')
print(f"ROOT_ID: {new_word.id}", file=sys.stderr)
new_component = VocabWordComponent.add_variation(
new_word.id, user_id, part_of_speech, word, translation, examples)
response = {
'word': new_word.serialize(),
'status': 'success'
}
return jsonify(response)
else:
response = {
'errors': form.errors,
'status': 'errors'
}
return jsonify(response)
def update_locales_db (api_search):
"""
:return:
"""
try:
Language.objects.all().delete()
English = Language(name='English (United Kingdom)', code='en-GB')
English.save()
French = Language(name='Français (France)', code='fr-FR')
French.save()
return True
except Exception as E:
#TODO Proper error management inside logs
print(get_exception_info(E))
return {'status':False, 'message': get_exception_info(E)}
def load_languages():
"""Load language infor from indigenousLanguages.json into
database."""
print('Languages')
Language.query.delete()
lang_file = "seed_data/indigenousLanguages.json"
lang_dict = json_reader(lang_file)
i = 0
for key in lang_dict['features']:
language_name = lang_dict['features'][i]['properties']['Name']
i += 1
language = Language(language_name=language_name)
db.session.add(language)
db.session.commit()
def add_vocab_word():
form = VocabWordForm()
form.source_code.choices = Language.get_all_options()
if form.validate_on_submit():
source_code = form.source_code.data
root = form.root.data
translation = form.translation.data
definition = form.definition.data
synonyms = form.synonyms.data
examples = form.examples.data
notes = form.notes.data
new_vocab_word = VocabWord.add_vocab_word(
session[CURR_USER_ID], source_code, root, translation, definition, synonyms, examples, notes)
g.user.update_last_language(source_code)
flash(f"{new_vocab_word.root} created!", 'success')
return redirect(f"/words/{new_vocab_word.id}")
else:
return render_template('new_vocab_word.html', vocab_word_form=form)
def insert_company(csv_file):
df = pd.read_csv(csv_file)
for index, row in df.iterrows():
company = Company()
db.session.add(company)
db.session.commit()
print('company', company.id)
language_tags = {}
for column in df.columns:
value = row[column]
if isinstance(value, float) and math.isnan(value):
continue
company_name_tag, country_code = column.split('_')
language = Language.query \
.filter_by(country_code=country_code) \
.first()
if language is None:
language = Language(country_code)
db.session.add(language)
db.session.commit()
# print('language', language.id)
if company_name_tag == 'company':
company_name = CompanyName(company.id, language.id, value)
db.session.add(company_name)
db.session.commit()
print('company_name', company_name.id, value)
elif company_name_tag == 'tag':
language_tags[country_code] = value.split('|')
else:
assert False, 'Not supported column title({})'.format(column)
insert_tags(company.id, language_tags)
translationatom = TranslationAtom(client_id=translation_string[key]['client_id'],
parent=translationgist,
content=translation_string[key]['content'],
locale_id=key)
DBSession.add(translationatom)
DBSession.flush()
DBSession.commit()
language_ids = dict()
for language in old_DBSession.query(old_Language).all():
new_language = Language(client_id=language.client_id,
# parent_client_id=language.parent_client_id,
# parent_object_id=language.parent_object_id,
translation_gist_client_id=translation_strings[language.translation_string]['ids'][
'client_id'],
translation_gist_object_id=translation_strings[language.translation_string]['ids'][
'object_id'],
marked_for_deletion=language.marked_for_deletion
)
DBSession.add(new_language)
for group in old_DBSession.query(old_Group).join(old_BaseGroup).filter(
old_Group.subject_client_id == language.client_id,
old_Group.subject_object_id == language.object_id,
old_BaseGroup.subject == 'language').all():
new_group = Group(id=group.id, base_group_id=group.base_group_id,
subject_client_id=new_language.client_id,
subject_object_id=new_language.object_id)
DBSession.add(new_group)
language_ids[str(language.client_id) + '_' + str(language.object_id)] = {'client_id': new_language.client_id,
'object_id': new_language.object_id}
def show_study_material():
languages = Language.get_all_options()
return render_template('study_material.html', languages=languages, parts_of_speech=PARTS_OF_SPEECH)
def _i18n_string(self, token_slug, lang_code=None):
# fetch the persistant app object for this
# app, since the tokens are linked to it
persistant_app = PersistantApp.resolve(self)
if persistant_app is None:
# this shouldn't ever happen, so log and abort
self.warning("PersistantApp does not exist for %r" % (self))
return None
# we accept a few different language-aware
# objects, so resolve this elsewhere. the
# output is always a language code
if lang_code is not None:
language = self._language(
lang_code)
# if the langauage code doesn't exist, (which shouldn't
# happen, since the code passed to this method should be
# sourced from previously-validated Language objects),
# warn and fall back to the system default
if language is None:
self.warning(
"No such language: %r" %\
(lang_code))
# fall back to the system default.
# it's better than nothing at all
language = Language.default()
# or if the lang_code was omitted,
# fall back to the system default
else:
language = Language.default()
# fetch the token object via its slug,
# which should be defined in locale.py
try:
token = persistant_app.token_set.get(
slug=token_slug)
# token objects don't auto-spawn any more, so this
# could happen because of a typo or something. warn,
# but let the calling method deal with it
except Token.DoesNotExist:
self.warning(
"No such token: %s" %\
(token_slug))
return None
# attempt to fetch the translation of the token in the
# requested language, and warn/abort if none were found
string = token.translation(language)
if string is None:
self.warning(
"No strings for token: %s in language: %s (or fallbacks)" %\
(token_slug, lang_code))
return None
return string
l.page = Page.objects.filter(id=request.POST['page']).get()
l.save()
except Exception, e:
print e
return HttpResponse(json.dumps({
'location': l.location,
'id': l.id,
}))
elif request.POST['method'] == 'remove_location':
l = Location.objects.filter(id=request.POST['location']).get()
l.delete()
return HttpResponse(json.dumps({
'id': request.POST['location'],
}))
elif request.POST['method'] == 'add_language':
lang = Language()
lang.code = request.POST['code']
lang.name = request.POST['name']
lang.save()
return HttpResponse('true')
elif request.POST['method'] == 'remove_language':
lang = Language.objects.filter(code=request.POST['code']).get()
lang.delete()
return HttpResponse('true')
elif request.POST['method'] == 'save_page':
translation = PageTranslation.objects.filter(id=request.POST['id']).get()
translation.title = request.POST['title']
translation.content = request.POST['content']
translation.save()
return HttpResponse('true')
elif request.POST['method'] == 'delete_page':
