def create_group(request):
try:
variables = {'auth': request.authenticated_userid}
req = request.json_body
client = DBSession.query(Client).filter_by(id=variables['auth']).first()
if not client:
raise KeyError("Invalid client id (not registered on server). Try to logout and then login.",
variables['auth'])
user = DBSession.query(User).filter_by(id=client.user_id).first()
if not user:
raise CommonException("This client id is orphaned. Try to logout and then login once more.")
if not DBSession.query(Group).filter_by(id=req['id']).first():
group = Group(id=req['id'],
base_group_id=req['base_group_id'],
subject_client_id=req['subject_client_id'],
subject_object_id=req['subject_object_id'])
DBSession.add(group)
for user_id in req['users']:
curr_user = DBSession.query(User).filter_by(id=user_id).first()
if curr_user not in group.users:
group.users.append(curr_user)
return {}
except KeyError as e:
request.response.status = HTTPBadRequest.code
return {'error': str(e)}
except IntegrityError as e:
request.response.status = HTTPInternalServerError.code
return {'error': str(e)}
def create_translationgist(request):
try:
variables = {'auth': request.authenticated_userid}
req = request.json_body
object_id = req.get('object_id', None)
type = req['type']
client = DBSession.query(Client).filter_by(
id=variables['auth']).first()
if not client:
raise KeyError(
"Invalid client id (not registered on server). Try to logout and then login.",
variables['auth'])
user = DBSession.query(User).filter_by(id=client.user_id).first()
if not user:
raise CommonException(
"This client id is orphaned. Try to logout and then login once more."
)
translationgist = TranslationGist(client_id=variables['auth'],
object_id=object_id,
type=type)
DBSession.add(translationgist)
DBSession.flush()
basegroups = []
basegroups += [
DBSession.query(BaseGroup).filter_by(
name="Can delete translationgist").first()
]
if not object_id:
groups = []
for base in basegroups:
group = Group(subject_client_id=translationgist.client_id,
subject_object_id=translationgist.object_id,
parent=base)
groups += [group]
for group in groups:
add_user_to_group(user, group)
request.response.status = HTTPOk.code
return {
'object_id': translationgist.object_id,
'client_id': translationgist.client_id
}
except KeyError as e:
request.response.status = HTTPBadRequest.code
return {'error': str(e)}
except IntegrityError as e:
request.response.status = HTTPInternalServerError.code
return {'error': str(e)}
except CommonException as e:
request.response.status = HTTPConflict.code
return {'error': str(e)}
def create_organization(request): # TODO: test
try:
variables = {'auth': request.authenticated_userid}
req = request.json_body
name = req['name']
about = req['about']
client = DBSession.query(Client).filter_by(
id=variables['auth']).first()
if not client:
raise KeyError(
"Invalid client id (not registered on server). Try to logout and then login."
)
user = DBSession.query(User).filter_by(id=client.user_id).first()
if not user:
raise CommonException(
"This client id is orphaned. Try to logout and then login once more."
)
organization = Organization(name=name, about=about)
if user not in organization.users:
organization.users.append(user)
DBSession.add(organization)
DBSession.flush()
bases = DBSession.query(BaseGroup).filter_by(subject='organization')
for base in bases:
group = Group(parent=base, subject_object_id=organization.id)
add_user_to_group(user, group)
DBSession.add(group)
request.response.status = HTTPOk.code
return {'organization_id': organization.id}
except KeyError as e:
request.response.status = HTTPBadRequest.code
return {'error': str(e)}
except IntegrityError as e:
request.response.status = HTTPInternalServerError.code
return {'error': str(e)}
except CommonException as e:
request.response.status = HTTPConflict.code
return {'error': str(e)}
def convert_five_tiers(dictionary_client_id,
dictionary_object_id,
user_id,
origin_client_id,
origin_object_id,
sqlalchemy_url,
storage,
eaf_url,
sound_url=None):
log = logging.getLogger(__name__)
log.setLevel(logging.DEBUG)
no_sound = True
if sound_url:
no_sound = False
with warnings.catch_warnings():
warnings.filterwarnings('error')
try:
from pydub import AudioSegment
except Warning as e:
no_sound = True
if not no_sound:
with tempfile.NamedTemporaryFile() as temp:
try:
sound_file = request.urlopen(sound_url)
except HTTPError as e:
return {'error': str(e.read().decode("utf8", 'ignore'))}
with open(temp.name, 'wb') as output:
output.write(sound_file.read())
full_audio = AudioSegment.from_wav(temp.name)
temp.flush()
field_ids = {}
with transaction.manager:
client = DBSession.query(Client).filter_by(id=user_id).first()
if not client:
raise KeyError(
"Invalid client id (not registered on server). Try to logout and then login.",
user_id)
user = DBSession.query(User).filter_by(id=client.user_id).first()
all_fieldnames = ("Markup", "Paradigm Markup", "Word", "Transcription",
"Translation", "Sound", "Etymology", "Backref",
"Word of Paradigmatic forms",
"Transcription of Paradigmatic forms",
"Translation of Paradigmatic forms",
"Sounds of Paradigmatic forms")
for name in all_fieldnames:
data_type_query = DBSession.query(Field) \
.join(TranslationGist,
and_(Field.translation_gist_object_id == TranslationGist.object_id,
Field.translation_gist_client_id == TranslationGist.client_id))\
.join(TranslationGist.translationatom)
field = data_type_query.filter(
TranslationAtom.locale_id == 2,
TranslationAtom.content == name).one()
field_ids[name] = (field.client_id, field.object_id)
fp_fields = ("Word", "Transcription", "Translation", "Sound", "Markup",
"Etymology", "Backref")
sp_fields = ("Word of Paradigmatic forms",
"Transcription of Paradigmatic forms",
"Translation of Paradigmatic forms",
"Sounds of Paradigmatic forms", "Paradigm Markup",
"Backref")
fp_structure = set([field_ids[x] for x in fp_fields])
sp_structure = set([field_ids[x] for x in sp_fields])
DBSession.flush()
resp = translation_service_search("WiP")
state_translation_gist_object_id, state_translation_gist_client_id = resp[
'object_id'], resp['client_id']
for base in DBSession.query(BaseGroup).filter_by(
dictionary_default=True):
new_group = Group(parent=base,
subject_object_id=dictionary_object_id,
subject_client_id=dictionary_client_id)
if user not in new_group.users:
new_group.users.append(user)
DBSession.add(new_group)
DBSession.flush()
origin_metadata = {
"origin_client_id": origin_client_id,
"origin_object_id": origin_object_id
}
parent = DBSession.query(Dictionary).filter_by(
client_id=dictionary_client_id,
object_id=dictionary_object_id).first()
if not parent:
return {'error': str("No such dictionary in the system")}
first_perspective = None
second_perspective = None
for perspective in DBSession.query(DictionaryPerspective).filter_by(
parent=parent, marked_for_deletion=False):
structure = set()
fields = DBSession.query(DictionaryPerspectiveToField)\
.filter_by(parent=perspective)\
.all()
DBSession.flush()
for p_to_field in fields:
structure.add(
(p_to_field.field_client_id, p_to_field.field_object_id))
if structure == fp_structure:
first_perspective = perspective
elif structure == sp_structure:
second_perspective = perspective
structure.clear()
lexes = []
if first_perspective:
lexes = DBSession.query(DictionaryPerspective, LexicalEntry, Entity)\
.filter(and_(DictionaryPerspective.object_id==first_perspective.object_id,
DictionaryPerspective.client_id==first_perspective.client_id))\
.join(LexicalEntry, and_( LexicalEntry.parent_object_id==DictionaryPerspective.object_id,
LexicalEntry.parent_client_id==DictionaryPerspective.client_id))\
.join(Entity, and_(LexicalEntry.object_id==Entity.parent_object_id,
LexicalEntry.client_id==Entity.parent_client_id))
p_lexes = []
if second_perspective:
p_lexes = DBSession.query(DictionaryPerspective, LexicalEntry, Entity)\
.filter(and_(DictionaryPerspective.object_id==second_perspective.object_id,
DictionaryPerspective.client_id==second_perspective.client_id))\
.join(LexicalEntry, and_( LexicalEntry.parent_object_id==DictionaryPerspective.object_id,
LexicalEntry.parent_client_id==DictionaryPerspective.client_id))\
.join(Entity, and_(LexicalEntry.object_id==Entity.parent_object_id,
LexicalEntry.client_id==Entity.parent_client_id))
hashes = [
x[2].additional_metadata["hash"] for x in lexes
if x[2].field.data_type == "Sound"
]
hashes = hashes[:] + [
x[2].additional_metadata["hash"]
for x in p_lexes if x[2].field.data_type == "Sound"
]
links = [((x[2].link.client_id, x[2].link.object_id),
(x[1].client_id, x[1].object_id)) for x in lexes
if x[2].field.data_type == "Link"]
links = links[:] + [((x[2].link.client_id, x[2].link.object_id),
(x[1].client_id, x[1].object_id))
for x in p_lexes if x[2].field.data_type == "Link"]
resp = translation_service_search("WiP")
state_translation_gist_object_id, state_translation_gist_client_id = resp[
'object_id'], resp['client_id']
for base in DBSession.query(BaseGroup).filter_by(
dictionary_default=True):
new_group = Group(parent=base,
subject_object_id=dictionary_object_id,
subject_client_id=dictionary_client_id)
if user not in new_group.users:
new_group.users.append(user)
DBSession.add(new_group)
DBSession.flush()
"""
# FIRST PERSPECTIVE
"""
if first_perspective is None:
resp = translation_service_search_all("Lexical Entries")
persp_translation_gist_client_id, persp_translation_gist_object_id = resp[
'client_id'], resp['object_id']
first_perspective = DictionaryPerspective(
client_id=client.id, ###
state_translation_gist_object_id=
state_translation_gist_object_id,
state_translation_gist_client_id=
state_translation_gist_client_id,
parent=parent,
# import_source=req.get('import_source'),
# import_hash=req.get('import_hash'),
additional_metadata=origin_metadata,
translation_gist_client_id=persp_translation_gist_client_id,
translation_gist_object_id=persp_translation_gist_object_id)
first_perspective.additional_metadata = origin_metadata
DBSession.add(first_perspective)
owner_client = DBSession.query(Client).filter_by(
id=parent.client_id).first()
owner = owner_client.user
for base in DBSession.query(BaseGroup).filter_by(
perspective_default=True):
new_group = Group(parent=base,
subject_object_id=first_perspective.object_id,
subject_client_id=first_perspective.client_id)
if user not in new_group.users:
new_group.users.append(user)
if owner not in new_group.users:
new_group.users.append(owner)
DBSession.add(new_group)
DBSession.flush()
first_perspective_client_id = first_perspective.client_id
first_perspective_object_id = first_perspective.object_id
"""
# SECOND PERSPECTIVE
"""
resp = translation_service_search_all("Paradigms")
persp_translation_gist_client_id, persp_translation_gist_object_id = resp[
'client_id'], resp['object_id']
if second_perspective is None:
second_perspective = DictionaryPerspective(
client_id=client.id, ### variables['auth']
state_translation_gist_object_id=
state_translation_gist_object_id,
state_translation_gist_client_id=
state_translation_gist_client_id,
parent=parent,
# import_source=req.get('import_source'),
# import_hash=req.get('import_hash'),
additional_metadata=origin_metadata,
translation_gist_client_id=persp_translation_gist_client_id,
translation_gist_object_id=persp_translation_gist_object_id)
second_perspective.additional_metadata = origin_metadata
# if is_template is not None:
# perspective.is_template = is_template
DBSession.add(second_perspective)
owner_client = DBSession.query(Client).filter_by(
id=parent.client_id).first()
owner = owner_client.user
for base in DBSession.query(BaseGroup).filter_by(
perspective_default=True):
new_group = Group(parent=base,
subject_object_id=second_perspective.object_id,
subject_client_id=second_perspective.client_id)
if user not in new_group.users:
new_group.users.append(user)
if owner not in new_group.users:
new_group.users.append(owner)
DBSession.add(new_group)
second_perspective_client_id = second_perspective.client_id
second_perspective_object_id = second_perspective.object_id
fp_fields_dict = {}
"""
# FIRST PERSPECTIVE FIELDS CREATION
"""
fp_field_names = ("Word", "Transcription", "Translation", "Sound",
"Etymology", "Backref")
fields_list = []
for fieldname in fp_field_names: #
if fieldname == "Backref":
fields_list.append({
"client_id": field_ids[fieldname][0],
"object_id": field_ids[fieldname][1],
"link": {
"client_id": second_perspective_client_id,
"object_id": second_perspective_object_id
}
})
elif fieldname == "Sound":
fields_list.append({
"client_id":
field_ids[fieldname][0],
"object_id":
field_ids[fieldname][1],
"contains": [{
"client_id": field_ids["Markup"][0],
"object_id": field_ids["Markup"][1]
}]
})
else:
fields_list.append({
"client_id": field_ids[fieldname][0],
"object_id": field_ids[fieldname][1]
})
fp_fields_dict[fieldname] = (field_ids[fieldname][0],
field_ids[fieldname][1])
fp_fields_dict["Markup"] = (field_ids["Markup"][0],
field_ids["Markup"][1])
update_perspective_fields(fields_list, first_perspective_client_id,
first_perspective_object_id, client)
"""
# Creating fields of the second perspective
"""
sp_field_names = ("Word of Paradigmatic forms",
"Transcription of Paradigmatic forms",
"Translation of Paradigmatic forms",
"Sounds of Paradigmatic forms", "Backref")
sp_fields_dict = {}
fields_list = []
for fieldname in sp_field_names:
if fieldname == "Backref":
fields_list.append({
"client_id": field_ids[fieldname][0],
"object_id": field_ids[fieldname][1],
"link": {
"client_id": first_perspective_client_id,
"object_id": first_perspective_object_id
}
})
elif fieldname == "Sounds of Paradigmatic forms":
fields_list.append({
"client_id":
field_ids[fieldname][0],
"object_id":
field_ids[fieldname][1],
"contains": [{
"client_id": field_ids["Paradigm Markup"][0],
"object_id": field_ids["Paradigm Markup"][1]
}]
})
else:
fields_list.append({
"client_id": field_ids[fieldname][0],
"object_id": field_ids[fieldname][1]
})
sp_fields_dict[fieldname] = (field_ids[fieldname][0],
field_ids[fieldname][1])
sp_fields_dict["Paradigm Markup"] = (field_ids["Paradigm Markup"][0],
field_ids["Paradigm Markup"][1])
update_perspective_fields(fields_list, second_perspective_client_id,
second_perspective_object_id, client)
dubl = []
try:
eaffile = request.urlopen(eaf_url)
except HTTPError as e:
return {'error': str(e.read().decode("utf8", 'ignore'))}
with tempfile.NamedTemporaryFile() as temp:
temp.write(eaffile.read())
converter = elan_parser.Elan(temp.name)
converter.parse()
final_dicts = converter.proc()
temp.flush()
for phrase in final_dicts:
curr_dict = None
paradigm_words = []
for word_translation in phrase:
if type(word_translation) is not list:
curr_dict = word_translation
mt_words = [
word_translation[i][1].text for i in word_translation
if len(word_translation[i]) > 1
and type(word_translation[i][1].text) is str
]
main_tier_text = " ".join(mt_words)
if main_tier_text:
paradigm_words.append(
elan_parser.Word(text=main_tier_text,
tier="Word of Paradigmatic forms",
time=word.time))
else:
word = word_translation[0]
tier_name = word.tier
new = " ".join([i.text for i in word_translation])
paradigm_words.append(
elan_parser.Word(text=new,
tier=tier_name,
time=word.time))
p_match_dict = defaultdict(list)
for pword in paradigm_words:
match = [x for x in p_lexes
if x[2].content == pword.text] #LEX COUNT OR RANDOM
for t in match:
if field_ids[EAF_TIERS[pword.tier]] == (
t[2].field.client_id, t[2].field.object_id):
p_match_dict[t[1]].append(t)
p_match_dict = {
k: v
for k, v in p_match_dict.items() if len(v) >= 2
}
max_sim = None
for le in p_match_dict:
if max_sim is None:
max_sim = le
else:
if len(p_match_dict[le]) >= len(p_match_dict[max_sim]):
max_sim = le
if max_sim:
sp_lexical_entry_client_id = max_sim.client_id
sp_lexical_entry_object_id = max_sim.object_id
else:
lexentr = LexicalEntry(
client_id=client.id,
parent_object_id=second_perspective_object_id,
parent=second_perspective)
DBSession.add(lexentr)
sp_lexical_entry_client_id = lexentr.client_id
sp_lexical_entry_object_id = lexentr.object_id
for other_word in paradigm_words:
if max_sim:
text_and_field = (other_word.text,
field_ids[EAF_TIERS[other_word.tier]])
sim = [(x[2].content, (x[2].field.client_id,
x[2].field.object_id))
for x in p_match_dict[max_sim]]
if text_and_field not in sim:
create_entity(sp_lexical_entry_client_id,
sp_lexical_entry_object_id,
field_ids[EAF_TIERS[other_word.tier]][0],
field_ids[EAF_TIERS[other_word.tier]][1],
None,
client,
other_word.text,
filename=None,
storage=storage)
else:
create_entity(sp_lexical_entry_client_id,
sp_lexical_entry_object_id,
field_ids[EAF_TIERS[other_word.tier]][0],
field_ids[EAF_TIERS[other_word.tier]][1],
None,
client,
other_word.text,
filename=None,
storage=storage)
if not no_sound:
if word.time[1] < len(full_audio):
with tempfile.NamedTemporaryFile() as temp:
full_audio[word.time[0]:word.time[1]].export(
temp.name, format="wav")
audio_slice = temp.read()
if max_sim:
hash = hashlib.sha224(audio_slice).hexdigest()
if not hash in hashes:
hashes.append(hash)
create_entity(
sp_lexical_entry_client_id,
sp_lexical_entry_object_id,
field_ids["Sounds of Paradigmatic forms"]
[0],
field_ids["Sounds of Paradigmatic forms"]
[1],
None,
client,
filename="%s.wav" % (word.index),
folder_name="sound1",
content=base64.urlsafe_b64encode(
audio_slice).decode(),
storage=storage)
else:
create_entity(
sp_lexical_entry_client_id,
sp_lexical_entry_object_id,
field_ids["Sounds of Paradigmatic forms"][0],
field_ids["Sounds of Paradigmatic forms"][1],
None,
client,
filename="%s.wav" % (word.index),
folder_name="sound1",
content=base64.urlsafe_b64encode(
audio_slice).decode(),
storage=storage)
temp.flush()
p_match_dict.clear()
paradigm_words[:] = []
for word in curr_dict:
column = [word] + curr_dict[word]
match_dict = defaultdict(list)
for crt in tuple(i for i in column):
match = [x for x in lexes if x[2].content == crt.text]
for t in match:
if field_ids[EAF_TIERS[crt.tier]] == (
t[2].field.client_id, t[2].field.object_id):
match_dict[t[1]].append(t)
match_dict = {
k: v
for k, v in match_dict.items() if len(v) >= 2
}
max_sim = None
for le in match_dict:
if max_sim is None:
max_sim = le
else:
if len(match_dict[le]) >= len(match_dict[max_sim]):
max_sim = le
if max_sim:
fp_lexical_entry_client_id = max_sim.client_id
fp_lexical_entry_object_id = max_sim.object_id
else:
lexentr = LexicalEntry(
client_id=client.id,
parent_object_id=first_perspective_object_id,
parent=first_perspective)
DBSession.add(lexentr)
fp_lexical_entry_client_id = lexentr.client_id
fp_lexical_entry_object_id = lexentr.object_id
for other_word in column:
if max_sim:
text_and_field = (
other_word.text,
field_ids[EAF_TIERS[other_word.tier]])
sim = [(x[2].content, (x[2].field.client_id,
x[2].field.object_id))
for x in match_dict[max_sim]]
if text_and_field not in sim:
create_entity(
fp_lexical_entry_client_id,
fp_lexical_entry_object_id,
field_ids[EAF_TIERS[other_word.tier]][0],
field_ids[EAF_TIERS[other_word.tier]][1],
None,
client,
other_word.text,
filename=None,
storage=storage)
else:
create_entity(fp_lexical_entry_client_id,
fp_lexical_entry_object_id,
field_ids[EAF_TIERS[other_word.tier]][0],
field_ids[EAF_TIERS[other_word.tier]][1],
None,
client,
other_word.text,
filename=None,
storage=storage)
if not no_sound:
if word.time[1] < len(full_audio):
with tempfile.NamedTemporaryFile() as temp:
full_audio[word.time[0]:word.time[1]].export(
temp.name, format="wav")
audio_slice = temp.read()
hash = hashlib.sha224(audio_slice).hexdigest()
if max_sim:
if not hash in hashes:
hashes.append(hash)
create_entity(
fp_lexical_entry_client_id,
fp_lexical_entry_object_id,
field_ids["Sound"][0],
field_ids["Sound"][1],
None,
client,
filename="%s.wav" % (word.index),
folder_name="sound1",
content=base64.urlsafe_b64encode(
audio_slice).decode(),
storage=storage)
else:
create_entity(fp_lexical_entry_client_id,
fp_lexical_entry_object_id,
field_ids["Sound"][0],
field_ids["Sound"][1],
None,
client,
filename="%s.wav" % (word.index),
folder_name="sound1",
content=base64.urlsafe_b64encode(
audio_slice).decode(),
storage=storage)
temp.flush()
fp_le_ids = (fp_lexical_entry_client_id,
fp_lexical_entry_object_id)
sp_le_ids = (sp_lexical_entry_client_id,
sp_lexical_entry_object_id)
dubl_tuple = (sp_le_ids, fp_le_ids)
if not dubl_tuple in dubl:
dubl.append(dubl_tuple)
if max_sim:
if not (sp_le_ids, fp_le_ids) in links:
create_entity(
sp_lexical_entry_client_id,
sp_lexical_entry_object_id,
field_ids["Backref"][0],
field_ids["Backref"][1],
None,
client,
filename=None,
link_client_id=fp_lexical_entry_client_id,
link_object_id=fp_lexical_entry_object_id,
storage=storage)
if not (fp_le_ids, sp_le_ids) in links:
create_entity(
fp_lexical_entry_client_id,
fp_lexical_entry_object_id,
field_ids["Backref"][0],
field_ids["Backref"][1],
None,
client,
filename=None,
link_client_id=sp_lexical_entry_client_id,
link_object_id=sp_lexical_entry_object_id,
storage=storage)
else:
create_entity(
sp_lexical_entry_client_id,
sp_lexical_entry_object_id,
field_ids["Backref"][0],
field_ids["Backref"][1],
None,
client,
filename=None,
link_client_id=fp_lexical_entry_client_id,
link_object_id=fp_lexical_entry_object_id,
storage=storage)
create_entity(
fp_lexical_entry_client_id,
fp_lexical_entry_object_id,
field_ids["Backref"][0],
field_ids["Backref"][1],
None,
client,
filename=None,
link_client_id=sp_lexical_entry_client_id,
link_object_id=sp_lexical_entry_object_id,
storage=storage)
column[:] = []
match_dict.clear()
return
def convert_db_new( blob_client_id, blob_object_id, language_client_id, language_object_id, user_id, gist_client_id, gist_object_id, storage,
locale_id=2):
log = logging.getLogger(__name__)
log.setLevel(logging.DEBUG)
time.sleep(4)
field_ids = {}
with transaction.manager:
blob = DBSession.query(UserBlobs).filter_by(client_id=blob_client_id, object_id=blob_object_id).first()
# DBSession.flush()
filename = blob.real_storage_path
log.debug("user_id: %s" % user_id)
log.debug("Starting convert_one")
log.debug("Creating session")
sqconn = sqlite3.connect(filename)
log.debug("Connected to sqlite3 database")
client = DBSession.query(Client).filter_by(id=user_id).first()
if not client:
raise KeyError("Invalid client id (not registered on server). Try to logout and then login.",
user_id)
user = DBSession.query(User).filter_by(id=client.user_id).first()
if not user:
log.debug("ERROR")
all_fieldnames = ("Markup",
"Paradigm Markup",
"Word",
"Transcription",
"Translation",
"Sound",
"Etymology",
"Backref",
"Word of Paradigmatic forms",
"Transcription of Paradigmatic forms",
"Translation of Paradigmatic forms",
"Sounds of Paradigmatic forms"
)
for name in all_fieldnames:
data_type_query = DBSession.query(Field) \
.join(TranslationGist,
and_(Field.translation_gist_object_id == TranslationGist.object_id,
Field.translation_gist_client_id == TranslationGist.client_id))\
.join(TranslationGist.translationatom)
field = data_type_query.filter(TranslationAtom.locale_id == 2,
TranslationAtom.content == name).one() # todo: a way to find this fields if wwe cannot use one
field_ids[name] = (field.client_id, field.object_id)
DBSession.flush()
dict_attributes = get_dict_attributes(sqconn)
"""
dict_attributes = get_dict_attributes(sqconn)
translationgist = TranslationGist(client_id=user_id, type="Dictionary")
DBSession.add(translationgist)
DBSession.flush()
gist_client_id = translationgist.client_id
gist_object_id = translationgist.object_id
"""
parent_client_id = gist_client_id
parent_object_id = gist_object_id
parent = DBSession.query(TranslationGist).filter_by(client_id=parent_client_id, object_id=parent_object_id).first()
"""
translationatom = TranslationAtom(client_id=client.id,
parent=parent,
locale_id=locale_id,
content=dict_attributes["dictionary_name"])
DBSession.add(translationatom)
DBSession.flush()
atom_client_id = translationatom.client_id
atom_object_id = translationatom.object_id
log.debug(dict_attributes["dictionary_name"])
language_client_id = atom_client_id
language_object_id = atom_object_id
"""
lang_parent = DBSession.query(Language).filter_by(client_id=language_client_id, object_id=language_object_id).first()
resp = translation_service_search("WiP")
state_translation_gist_object_id, state_translation_gist_client_id = resp['object_id'], resp['client_id']
dictionary = Dictionary(client_id=user_id,
state_translation_gist_object_id=state_translation_gist_object_id,
state_translation_gist_client_id=state_translation_gist_client_id,
parent=lang_parent,
translation_gist_client_id=gist_client_id,
translation_gist_object_id=gist_object_id
)
#additional_metadata=additional_metadata)
DBSession.add(dictionary)
DBSession.flush()
dictionary_client_id = dictionary.client_id
dictionary_object_id = dictionary.object_id
for base in DBSession.query(BaseGroup).filter_by(dictionary_default=True):
new_group = Group(parent=base,
subject_object_id=dictionary.object_id, subject_client_id=dictionary.client_id)
if user not in new_group.users:
new_group.users.append(user)
DBSession.add(new_group)
DBSession.flush()
"""
# FIRST PERSPECTIVE
"""
resp = translation_service_search_all("Lexical Entries")
persp_translation_gist_client_id, persp_translation_gist_object_id = resp['client_id'], resp['object_id']
parent = DBSession.query(Dictionary).filter_by(client_id=dictionary_client_id, object_id=dictionary_object_id).first()
perspective = DictionaryPerspective(client_id=client.id, ###
state_translation_gist_object_id=state_translation_gist_object_id,
state_translation_gist_client_id=state_translation_gist_client_id,
parent=parent,
import_source="Lingvodoc-0.98",
import_hash=dict_attributes['dialeqt_id'],
# additional_metadata=additional_metadata,
translation_gist_client_id=persp_translation_gist_client_id,
translation_gist_object_id=persp_translation_gist_object_id
)
# if is_template is not None:
# perspective.is_template = is_template
DBSession.add(perspective)
owner_client = DBSession.query(Client).filter_by(id=parent.client_id).first()
owner = owner_client.user
for base in DBSession.query(BaseGroup).filter_by(perspective_default=True):
new_group = Group(parent=base,
subject_object_id=perspective.object_id, subject_client_id=perspective.client_id)
if user not in new_group.users:
new_group.users.append(user)
if owner not in new_group.users:
new_group.users.append(owner)
DBSession.add(new_group)
DBSession.flush()
first_perspective_client_id = perspective.client_id
first_perspective_object_id = perspective.object_id
"""
# SECOND PERSPECTIVE
"""
resp = translation_service_search_all("Paradigms")
persp_translation_gist_client_id, persp_translation_gist_object_id = resp['client_id'], resp['object_id']
parent = DBSession.query(Dictionary).filter_by(client_id=dictionary_client_id, object_id=dictionary_object_id).first()
if not parent:
return {'error': str("No such dictionary in the system")}
perspective = DictionaryPerspective(client_id=client.id, ### variables['auth']
state_translation_gist_object_id=state_translation_gist_object_id,
state_translation_gist_client_id=state_translation_gist_client_id,
parent=parent,
import_source="Lingvodoc-0.98",
import_hash=dict_attributes['dialeqt_id'],
# additional_metadata=additional_metadata,
translation_gist_client_id=persp_translation_gist_client_id,
translation_gist_object_id=persp_translation_gist_object_id
)
# if is_template is not None:
# perspective.is_template = is_template
DBSession.add(perspective)
owner_client = DBSession.query(Client).filter_by(id=parent.client_id).first()
owner = owner_client.user
for base in DBSession.query(BaseGroup).filter_by(perspective_default=True):
new_group = Group(parent=base,
subject_object_id=perspective.object_id, subject_client_id=perspective.client_id)
if user not in new_group.users:
new_group.users.append(user)
if owner not in new_group.users:
new_group.users.append(owner)
DBSession.add(new_group)
second_perspective_client_id = perspective.client_id
second_perspective_object_id = perspective.object_id
get_fp_ids = sqconn.cursor()
get_fp_ids.execute("select id from dictionary where is_a_regular_form=1")
count_cursor = sqconn.cursor()
count_cursor.execute("select count(*) from dictionary where is_a_regular_form=1")
words_count = count_cursor.fetchone()[0]
count_cursor2 = sqconn.cursor()
count_cursor2.execute("select count(*) from dictionary where is_a_regular_form=0")
words_count2 = count_cursor2.fetchone()[0]
ids_dict = dict()
for i in range(words_count):
perspective = DBSession.query(DictionaryPerspective).\
filter_by(client_id=first_perspective_client_id, object_id = first_perspective_object_id).first()
if not perspective:
return {'error': str("No such perspective in the system")}
lexentr = LexicalEntry( client_id=client.id,
parent_object_id=first_perspective_object_id, parent=perspective)
DBSession.add(lexentr)
lexical_entry_client_id = lexentr.client_id
lexical_entry_object_id = lexentr.object_id
ids_dict[i] = (lexical_entry_client_id, lexical_entry_object_id)
DBSession.flush()
ids_dict2 = dict()
for i in range(words_count2):
perspective = DBSession.query(DictionaryPerspective).\
filter_by(client_id=first_perspective_client_id, object_id = second_perspective_object_id).first()
if not perspective:
return {'error': str("No such perspective in the system")}
lexentr = LexicalEntry( client_id=client.id,
parent_object_id=second_perspective_object_id, parent=perspective)
DBSession.add(lexentr)
lexical_entry_client_id = lexentr.client_id
lexical_entry_object_id = lexentr.object_id
ids_dict2[i] = (lexical_entry_client_id, lexical_entry_object_id)
DBSession.flush()
get_sp_ids = sqconn.cursor()
get_sp_ids.execute("select id from dictionary where is_a_regular_form=0")
ids_mapping2 = dict()
i = 0
sp_le_id_dict = {}
for id_cursor in get_sp_ids:
id = id_cursor[0]
sp_le_id_dict[id] = i
client_id = ids_dict2[i][0]
object_id = ids_dict2[i][1]
ids_mapping2[int(id)] = (client_id, object_id)
i += 1
get_fp_ids = sqconn.cursor()
get_fp_ids.execute("select id from dictionary where is_a_regular_form=1")
ids_mapping = dict()
i = 0
fp_le_id_dict = {}
for id_cursor in get_fp_ids:
id = id_cursor[0]
fp_le_id_dict[id] = i
client_id = ids_dict[i][0]
object_id = ids_dict[i][1]
ids_mapping[id] = (client_id, object_id)
i += 1
fp_fields_dict = {}
"""
# FIRST PERSPECTIVE FIELDS CREATION
"""
fp_field_names = ("Word", "Transcription", "Translation", "Sound", "Etymology", "Backref")
fields_list = []
for fieldname in fp_field_names: #
if fieldname == "Backref":
fields_list.append(
{
"client_id": field_ids[fieldname][0],
"object_id": field_ids[fieldname][1],
"link":{
"client_id": second_perspective_client_id,
"object_id": second_perspective_object_id
}
}
)
elif fieldname == "Sound":
fields_list.append(
{
"client_id": field_ids[fieldname][0],
"object_id": field_ids[fieldname][1],
"contains":[{
"client_id": field_ids["Markup"][0],
"object_id": field_ids["Markup"][1]
}
]
}
)
else:
fields_list.append({"client_id": field_ids[fieldname][0], "object_id": field_ids[fieldname][1]})
fp_fields_dict[fieldname] = (field_ids[fieldname][0], field_ids[fieldname][1])
fp_fields_dict["Markup"] = (field_ids["Markup"][0], field_ids["Markup"][1])
update_perspective_fields(fields_list, first_perspective_client_id, first_perspective_object_id, client )
"""
# Creating fields of the second perspective
"""
sp_fields_dict = {}
fields_list = []
sp_field_names = ("Word of Paradigmatic forms", "Transcription of Paradigmatic forms", "Translation of Paradigmatic forms", "Sounds of Paradigmatic forms", "Backref")
for fieldname in sp_field_names: #
if fieldname == "Backref":
fields_list.append(
{
"client_id": field_ids[fieldname][0],
"object_id": field_ids[fieldname][1],
"link":{
"client_id": first_perspective_client_id,
"object_id": first_perspective_object_id
}
}
)
elif fieldname == "Sounds of Paradigmatic forms":
fields_list.append(
{
"client_id": field_ids[fieldname][0],
"object_id": field_ids[fieldname][1],
"contains":[{
"client_id": field_ids["Paradigm Markup"][0],
"object_id": field_ids["Paradigm Markup"][1]
}
]
}
)
else:
fields_list.append({"client_id": field_ids[fieldname][0], "object_id": field_ids[fieldname][1]})
sp_fields_dict[fieldname] = (field_ids[fieldname][0], field_ids[fieldname][1])
sp_fields_dict["Paradigm Markup"] = (field_ids["Paradigm Markup"][0], field_ids["Paradigm Markup"][1])
update_perspective_fields(fields_list, second_perspective_client_id, second_perspective_object_id, client)
columns = ("word", "Transcription", "translation")
# First Perspective entity
sqcursor = sqconn.cursor()
for column in columns:
sqcursor.execute("select id,%s from dictionary where is_a_regular_form=1" % column)
for row in sqcursor:
row_id = int(row[0])
content = row[1]
name = None
if column == "word":
name = "Word"
if column == "Transcription":
name = "Transcription"
if column == "translation":
name = "Translation"
create_entity(ids_dict[fp_le_id_dict[row_id]][0], ids_dict[fp_le_id_dict[row_id]][1], fp_fields_dict[name][0], fp_fields_dict[name][1],
None, client, content, filename=None, storage=storage)
# Second Perspective entity
sqcursor = sqconn.cursor()
for column in columns:
sqcursor.execute("select id,%s from dictionary where is_a_regular_form=0" % column)
for row in sqcursor:
row_id = int(row[0])
content = row[1]
name = None
if column == "word":
name = "Word of Paradigmatic forms"
if column == "Transcription":
name = "Transcription of Paradigmatic forms"
if column == "translation":
name = "Translation of Paradigmatic forms"
create_entity(ids_dict2[sp_le_id_dict[row_id]][0], ids_dict2[sp_le_id_dict[row_id]][1], sp_fields_dict[name][0], sp_fields_dict[name][1],
None, client, content, filename=None, storage=storage)
sqcursor = sqconn.cursor()
sqcursor.execute("select id,regular_form from dictionary where is_a_regular_form=0")
for le_cursor in sqcursor:
fp_id = int(le_cursor[1])
sp_id = int(le_cursor[0])
if fp_id in ids_mapping:
create_entity(ids_dict[fp_le_id_dict[fp_id]][0], ids_dict[fp_le_id_dict[fp_id]][1], fp_fields_dict["Backref"][0], fp_fields_dict["Backref"][1],
None, client, filename=None, link_client_id=ids_dict2[sp_le_id_dict[sp_id]][0], link_object_id=ids_dict2[sp_le_id_dict[sp_id]][1], storage=storage)
create_entity(ids_dict2[sp_le_id_dict[sp_id]][0], ids_dict2[sp_le_id_dict[sp_id]][1], sp_fields_dict["Backref"][0], sp_fields_dict["Backref"][1],
None, client, filename=None, link_client_id=ids_dict[fp_le_id_dict[fp_id]][0], link_object_id=ids_dict[fp_le_id_dict[fp_id]][1], storage=storage)
#DBSession.flush()
# if req.get('is_translatable', None):
# field.is_translatable = bool(req['is_translatable'])
audio_hashes = set()
markup_hashes = set()
DBSession.flush()
"""
Sound and Markup
"""
audio_ids = set()
paradigm_audio_ids = set()
sound_and_markup_word_cursor = sqconn.cursor()
sound_and_markup_word_cursor.execute("""select blobs.id,
blobs.secblob,
blobs.mainblob,
dict_blobs_description.name,
dictionary.id,
dict_blobs_description.type
from blobs, dict_blobs_description, dictionary
where dict_blobs_description.blobid=blobs.id
and dict_blobs_description.wordid=dictionary.id
and dictionary.is_a_regular_form=1;""")
folder_name = "praat_markup"
upload_audio_with_markup(audio_ids, ids_mapping, fp_fields_dict, sound_and_markup_word_cursor, audio_hashes, markup_hashes, folder_name,
user_id, True, client, storage)
sound_and_markup_word_cursor = sqconn.cursor()
sound_and_markup_word_cursor.execute("""select blobs.id,
blobs.secblob,
blobs.mainblob,
dict_blobs_description.name,
dictionary.id,
dict_blobs_description.type
from blobs, dict_blobs_description, dictionary
where dict_blobs_description.blobid=blobs.id
and dict_blobs_description.wordid=dictionary.id
and dictionary.is_a_regular_form=1;""")
upload_audio(audio_ids, ids_mapping, fp_fields_dict, sound_and_markup_word_cursor, audio_hashes, markup_hashes, folder_name,
user_id, True, client, storage)
paradigm_sound_and_markup_cursor = sqconn.cursor()
paradigm_sound_and_markup_cursor.execute("""select blobs.id,
blobs.secblob,
blobs.mainblob,
dict_blobs_description.name,
dictionary.id,
dict_blobs_description.type
from blobs, dict_blobs_description, dictionary
where dict_blobs_description.blobid=blobs.id
and dict_blobs_description.wordid=dictionary.id
and dictionary.is_a_regular_form=0;""")
folder_name = "paradigm_praat_markup"
upload_audio_with_markup(paradigm_audio_ids, ids_mapping2, sp_fields_dict, paradigm_sound_and_markup_cursor, audio_hashes, markup_hashes, folder_name,
user_id, True, client, storage)
paradigm_sound_and_markup_cursor = sqconn.cursor()
paradigm_sound_and_markup_cursor.execute("""select blobs.id,
blobs.secblob,
blobs.mainblob,
dict_blobs_description.name,
dictionary.id,
dict_blobs_description.type
from blobs, dict_blobs_description, dictionary
where dict_blobs_description.blobid=blobs.id
and dict_blobs_description.wordid=dictionary.id
and dictionary.is_a_regular_form=0;""")
upload_audio(paradigm_audio_ids, ids_mapping2, sp_fields_dict, paradigm_sound_and_markup_cursor, audio_hashes, markup_hashes, folder_name,
user_id, True, client, storage)
"""
Etimology_tag
"""
etymology_cursor = sqconn.cursor()
etymology_cursor.execute("""select id, etimology_tag
FROM dictionary
WHERE etimology_tag NOT NULL
and dictionary.is_a_regular_form=1; """)
for cursor in etymology_cursor:
id = int(cursor[0])
client_id = ids_mapping[id][0]
object_id = ids_mapping[id][1]
item = {"entity_type": "Etymology", "tag": cursor[1],
"field_client_id": field_ids["Etymology"][0],
"field_object_id": field_ids["Etymology"][1],
"connections": [{"client_id": client_id, "object_id": object_id}]}
create_group_entity(item, client, user)
# status = session.post(connect_url, json=item)
# log.debug(status.text)
dictionary = {}
return dictionary
def create_language(request): # tested & in docs
try:
variables = {'auth': request.authenticated_userid}
req = request.json_body
try:
parent_client_id = req['parent_client_id']
parent_object_id = req['parent_object_id']
except:
parent_client_id = None
parent_object_id = None
translation_gist_client_id = req['translation_gist_client_id']
translation_gist_object_id = req['translation_gist_object_id']
client = DBSession.query(Client).filter_by(
id=variables['auth']).first()
object_id = req.get('object_id', None)
if not client:
raise KeyError(
"Invalid client id (not registered on server). Try to logout and then login.",
variables['auth'])
user = DBSession.query(User).filter_by(id=client.user_id).first()
if not user:
raise CommonException(
"This client id is orphaned. Try to logout and then login once more."
)
parent = None
if parent_client_id and parent_object_id:
parent = DBSession.query(Language).filter_by(
client_id=parent_client_id,
object_id=parent_object_id).first()
language = Language(
client_id=variables['auth'],
object_id=object_id,
translation_gist_client_id=translation_gist_client_id,
translation_gist_object_id=translation_gist_object_id)
DBSession.add(language)
if parent:
language.parent = parent
DBSession.flush()
basegroups = []
basegroups += [
DBSession.query(BaseGroup).filter_by(
name="Can edit languages").first()
]
basegroups += [
DBSession.query(BaseGroup).filter_by(
name="Can delete languages").first()
]
if not object_id:
groups = []
for base in basegroups:
group = Group(subject_client_id=language.client_id,
subject_object_id=language.object_id,
parent=base)
groups += [group]
for group in groups:
add_user_to_group(user, group)
request.response.status = HTTPOk.code
return {
'object_id': language.object_id,
'client_id': language.client_id
}
except KeyError as e:
request.response.status = HTTPBadRequest.code
return {'error': str(e)}
except IntegrityError as e:
request.response.status = HTTPInternalServerError.code
return {'error': str(e)}
except CommonException as e:
request.response.status = HTTPConflict.code
return {'error': str(e)}