def login_cheat(request): # TODO: test
next = request.params.get('next') or request.route_url('dashboard')
login = request.json_body.get('login', '')
passwordhash = request.json_body.get('passwordhash', '')
log.debug("Logging in with cheat method:" + login)
user = DBSession.query(User).filter_by(login=login).first()
if user and user.password.hash == passwordhash:
log.debug("Login successful")
client = Client(user_id=user.id)
user.clients.append(client)
DBSession.add(client)
DBSession.flush()
headers = remember(request, principal=client.id)
response = Response()
response.headers = headers
locale_id = user.default_locale_id
if not locale_id:
locale_id = 1
response.set_cookie(key='locale_id', value=str(locale_id))
response.set_cookie(key='client_id', value=str(client.id))
headers = remember(request, principal=client.id)
return response
log.debug("Login unsuccessful for " + login)
return HTTPUnauthorized(location=request.route_url('login'))
def signin(request):
req = request.json_body
login = req['login']
password = req['password']
# login = request.POST.get('login', '')
# password = request.POST.get('password', '')
user = DBSession.query(User).filter_by(login=login).first()
if user and user.check_password(password):
client = Client(user_id=user.id)
user.clients.append(client)
DBSession.add(client)
DBSession.flush()
headers = remember(request, principal=client.id)
response = Response()
response.headers = headers
locale_id = user.default_locale_id
if not locale_id:
locale_id = 1
response.set_cookie(key='locale_id', value=str(locale_id))
response.set_cookie(key='client_id', value=str(client.id))
result = dict()
result['client_id'] = client.id
request.response.status = HTTPOk.code
# request.response.headers = headers
# return response
return HTTPOk(headers=response.headers, json_body=result)
# return result
return HTTPUnauthorized(location=request.route_url('login'))
def new_client_server(request):
old_client = DBSession.query(Client).filter_by(
id=authenticated_userid(request)).first()
if old_client:
user = old_client.user
if user:
client = Client(user_id=user.id, is_browser_client=False)
user.clients.append(client)
DBSession.add(client)
DBSession.flush()
headers = remember(request, principal=client.id)
response = Response()
response.headers = headers
locale_id = user.default_locale_id
if not locale_id:
locale_id = 1
response.set_cookie(key='locale_id', value=str(locale_id))
response.set_cookie(key='client_id', value=str(client.id))
result = dict()
result['client_id'] = client.id
request.response.status = HTTPOk.code
# request.response.headers = headers
# return response
return HTTPOk(headers=response.headers, json_body=result)
# return result
return HTTPUnauthorized(location=request.route_url('login'))
def create_group(request):
try:
variables = {'auth': request.authenticated_userid}
req = request.json_body
client = DBSession.query(Client).filter_by(id=variables['auth']).first()
if not client:
raise KeyError("Invalid client id (not registered on server). Try to logout and then login.",
variables['auth'])
user = DBSession.query(User).filter_by(id=client.user_id).first()
if not user:
raise CommonException("This client id is orphaned. Try to logout and then login once more.")
if not DBSession.query(Group).filter_by(id=req['id']).first():
group = Group(id=req['id'],
base_group_id=req['base_group_id'],
subject_client_id=req['subject_client_id'],
subject_object_id=req['subject_object_id'])
DBSession.add(group)
for user_id in req['users']:
curr_user = DBSession.query(User).filter_by(id=user_id).first()
if curr_user not in group.users:
group.users.append(curr_user)
return {}
except KeyError as e:
request.response.status = HTTPBadRequest.code
return {'error': str(e)}
except IntegrityError as e:
request.response.status = HTTPInternalServerError.code
return {'error': str(e)}
def login_cheat(request): # TODO: test
next = request.params.get('next') or request.route_url('dashboard')
login = request.json_body.get('login', '')
passwordhash = request.json_body.get('passwordhash', '')
log.debug("Logging in with cheat method:" + login)
user = DBSession.query(User).filter_by(login=login).first()
if user and user.password.hash == passwordhash:
log.debug("Login successful")
client = Client(user_id=user.id)
user.clients.append(client)
DBSession.add(client)
DBSession.flush()
headers = remember(request, principal=client.id)
response = Response()
response.headers = headers
locale_id = user.default_locale_id
if not locale_id:
locale_id = 1
response.set_cookie(key='locale_id', value=str(locale_id))
response.set_cookie(key='client_id', value=str(client.id))
headers = remember(request, principal=client.id)
return response
log.debug("Login unsuccessful for " + login)
return HTTPUnauthorized(location=request.route_url('login'))
def login_post(request): # tested
next = request.params.get('next') or request.route_url('home')
login = request.POST.get('login', '')
password = request.POST.get('password', '')
# print(login)
log.debug(login)
user = DBSession.query(User).filter_by(login=login).first()
if user and user.check_password(password):
client = Client(user_id=user.id)
user.clients.append(client)
DBSession.add(client)
DBSession.flush()
headers = remember(request, principal=client.id)
response = Response()
response.headers = headers
locale_id = user.default_locale_id
if not locale_id:
locale_id = 1
response.set_cookie(key='locale_id', value=str(locale_id))
response.set_cookie(key='client_id', value=str(client.id))
headers = remember(request, principal=client.id)
# return HTTPFound(location=next, headers=response.headers)
return HTTPOk(headers=response.headers, json_body={})
# return {}
return HTTPUnauthorized(location=request.route_url('login'))
def create_field(translation_gist_client_id, translation_gist_object_id,
data_type_translation_gist_client_id,
data_type_translation_gist_object_id, client):
try:
if not client:
raise KeyError(
"Invalid client id (not registered on server). Try to logout and then login."
)
user = DBSession.query(User).filter_by(id=client.user_id).first()
field = Field(
client_id=client.id, ###
data_type_translation_gist_client_id=
data_type_translation_gist_client_id,
data_type_translation_gist_object_id=
data_type_translation_gist_object_id,
translation_gist_client_id=translation_gist_client_id,
translation_gist_object_id=translation_gist_object_id)
DBSession.add(field)
DBSession.flush()
return {'object_id': field.object_id, 'client_id': field.client_id}
except KeyError as e:
return {'error': str(e)}
def login_post(request): # tested
# next = request.params.get('next') or request.route_url('home')
login = request.POST.get('login', '')
password = request.POST.get('password', '')
# print(login)
log.debug(login)
user = DBSession.query(User).filter_by(login=login).first()
if user and user.check_password(password):
client = Client(user_id=user.id)
user.clients.append(client)
DBSession.add(client)
DBSession.flush()
headers = remember(request, principal=client.id)
response = Response()
response.headers = headers
locale_id = user.default_locale_id
if not locale_id:
locale_id = 1
response.set_cookie(key='locale_id', value=str(locale_id))
response.set_cookie(key='client_id', value=str(client.id))
headers = remember(request, principal=client.id)
# return HTTPFound(location=next, headers=response.headers)
return HTTPOk(headers=response.headers, json_body={})
# return {}
return HTTPUnauthorized(location=request.route_url('login'))
def signin(request):
req = request.json_body
login = req['login']
password = req['password']
# login = request.POST.get('login', '')
# password = request.POST.get('password', '')
desktop = req.get('desktop', False)
user = DBSession.query(User).filter_by(login=login).first()
if user and user.check_password(password):
client = Client(user_id=user.id, is_browser_client=not desktop)
user.clients.append(client)
DBSession.add(client)
DBSession.flush()
headers = remember(request, principal=client.id, max_age=315360000)
response = Response()
response.headers = headers
locale_id = user.default_locale_id
if not locale_id:
locale_id = 1
response.set_cookie(key='locale_id',
value=str(locale_id),
max_age=datetime.timedelta(days=3650))
response.set_cookie(key='client_id',
value=str(client.id),
max_age=datetime.timedelta(days=3650))
result = dict()
result['client_id'] = client.id
request.response.status = HTTPOk.code
# request.response.headers = headers
# return response
return HTTPOk(headers=response.headers, json_body=result)
# return result
return HTTPUnauthorized(location=request.route_url('login'))
def setUp(self):
self.config = testing.setUp()
import webtest
from pyramid import paster
from sqlalchemy import create_engine
engine = create_engine('sqlite://')
myapp = paster.get_app('testing.ini')
self.app = webtest.TestApp(myapp)
from lingvodoc.models import (
Base,
Locale,
User,
Passhash,
Client
)
DBSession.configure(bind=engine)
Base.metadata.create_all(engine)
with transaction.manager:
ru_locale = Locale(id=1, shortcut="ru", intl_name="Русский")
DBSession.add(ru_locale)
en_locale = Locale(id=2, shortcut="en", intl_name="English")
DBSession.add(en_locale)
DBSession.flush()
new_user = User(id=1, login='******', default_locale_id = 1)
new_pass = Passhash(password='******')
DBSession.add(new_pass)
new_user.password = new_pass
DBSession.add(new_user)
new_client = Client(id=1, user=new_user)
DBSession.add(new_client)
def upload_user_blob(request): # TODO: remove blob Object
variables = {'auth': authenticated_userid(request)}
response = dict()
filename = request.POST['blob'].filename
input_file = request.POST['blob'].file
class Object(object):
pass
blob = Object()
blob.client_id = variables['auth']
client = DBSession.query(Client).filter_by(id=variables['auth']).first()
blob.object_id = DBSession.query(UserBlobs).filter_by(client_id=client.id).count() + 1
blob.data_type = request.POST['data_type']
blob.filename = filename
current_user = DBSession.query(User).filter_by(id=client.user_id).first()
blob_object = UserBlobs(object_id=blob.object_id,
client_id=blob.client_id,
name=filename,
data_type=blob.data_type,
user_id=current_user.id)
current_user.userblobs.append(blob_object)
blob_object.real_storage_path, blob_object.content = create_object(request, input_file, blob_object, blob.data_type,
blob.filename, json_input=False)
DBSession.add(blob_object)
DBSession.add(current_user)
DBSession.flush()
request.response.status = HTTPOk.code
response = {"client_id": blob_object.client_id, "object_id": blob_object.object_id, "content": blob_object.content}
return response
def create_group_entity(request): # tested
try:
variables = {'auth': authenticated_userid(request)}
response = dict()
req = request.json_body
client = DBSession.query(Client).filter_by(id=variables['auth']).first()
if not client:
raise KeyError("Invalid client id (not registered on server). Try to logout and then login.")
user = DBSession.query(User).filter_by(id=client.user_id).first()
if not user:
raise CommonException("This client id is orphaned. Try to logout and then login once more.")
tags = []
if 'tag' in req:
tags += [req['tag']]
for par in req['connections']:
parent = DBSession.query(LexicalEntry).\
filter_by(client_id=par['client_id'], object_id=par['object_id']).first()
if not parent:
request.response.status = HTTPNotFound.code
return {'error': str("No such lexical entry in the system")}
# par_tags = DBSession.query(GroupingEntity).\
# filter_by(entity_type=req['entity_type'], parent=parent).all()
par_tags = None
tags += [o.content for o in par_tags]
if not tags:
n = 10 # better read from settings
tag = time.ctime() + ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.digits)
for c in range(n))
tags += [tag]
parents = req['connections']
for par in parents:
parent = DBSession.query(LexicalEntry).\
filter_by(client_id=par['client_id'], object_id=par['object_id']).first()
for tag in tags:
# ent = DBSession.query(GroupingEntity).\
# filter_by(entity_type=req['entity_type'], content=tag, parent=parent).first()
ent = None
if not ent:
# entity = GroupingEntity(client_id=client.id, object_id=DBSession.query(GroupingEntity).filter_by(client_id=client.id).count() + 1,
# entity_type=req['entity_type'], content=tag, parent=parent)
entity = None
DBSession.add(entity)
DBSession.flush()
log.debug('TAGS: %s', tags)
request.response.status = HTTPOk.code
return {}
except KeyError as e:
request.response.status = HTTPBadRequest.code
return {'error': str(e)}
except IntegrityError as e:
request.response.status = HTTPInternalServerError.code
return {'error': str(e)}
except CommonException as e:
request.response.status = HTTPConflict.code
return {'error': str(e)}
def create_language(request): # tested & in docs
try:
variables = {'auth': request.authenticated_userid}
req = request.json_body
try:
parent_client_id = req['parent_client_id']
parent_object_id = req['parent_object_id']
except:
parent_client_id = None
parent_object_id = None
translation_gist_client_id = req['translation_gist_client_id']
translation_gist_object_id = req['translation_gist_object_id']
client = DBSession.query(Client).filter_by(id=variables['auth']).first()
object_id = req.get('object_id', None)
if not client:
raise KeyError("Invalid client id (not registered on server). Try to logout and then login.",
variables['auth'])
user = DBSession.query(User).filter_by(id=client.user_id).first()
if not user:
raise CommonException("This client id is orphaned. Try to logout and then login once more.")
parent = None
if parent_client_id and parent_object_id:
parent = DBSession.query(Language).filter_by(client_id=parent_client_id, object_id=parent_object_id).first()
language = Language(client_id=variables['auth'],
object_id=object_id,
translation_gist_client_id=translation_gist_client_id,
translation_gist_object_id=translation_gist_object_id)
DBSession.add(language)
if parent:
language.parent = parent
DBSession.flush()
basegroups = []
basegroups += [DBSession.query(BaseGroup).filter_by(name="Can edit languages").first()]
basegroups += [DBSession.query(BaseGroup).filter_by(name="Can delete languages").first()]
if not object_id:
groups = []
for base in basegroups:
group = Group(subject_client_id=language.client_id, subject_object_id=language.object_id, parent=base)
groups += [group]
for group in groups:
add_user_to_group(user, group)
request.response.status = HTTPOk.code
return {'object_id': language.object_id,
'client_id': language.client_id}
except KeyError as e:
request.response.status = HTTPBadRequest.code
return {'error': str(e)}
except IntegrityError as e:
request.response.status = HTTPInternalServerError.code
return {'error': str(e)}
except CommonException as e:
request.response.status = HTTPConflict.code
return {'error': str(e)}
def create_lexical_entry_bulk(request): # TODO: test
try:
dictionary_client_id = request.matchdict.get('dictionary_client_id')
dictionary_object_id = request.matchdict.get('dictionary_object_id')
perspective_client_id = request.matchdict.get('perspective_client_id')
perspective_object_id = request.matchdict.get('perspective_object_id')
count = request.json_body.get('count') or 0
variables = {'auth': request.authenticated_userid}
client = DBSession.query(Client).filter_by(
id=variables['auth']).first()
if not client:
raise KeyError(
"Invalid client id (not registered on server). Try to logout and then login.",
variables['auth'])
user = DBSession.query(User).filter_by(id=client.user_id).first()
if not user:
raise CommonException(
"This client id is orphaned. Try to logout and then login once more."
)
perspective = DBSession.query(DictionaryPerspective). \
filter_by(client_id=perspective_client_id, object_id = perspective_object_id).first()
if not perspective:
request.response.status = HTTPNotFound.code
return {'error': str("No such perspective in the system")}
lexes_list = []
for i in range(0, count):
lexentr = LexicalEntry(client_id=variables['auth'],
parent_object_id=perspective_object_id,
parent=perspective)
DBSession.add(lexentr)
lexes_list.append(lexentr)
DBSession.flush()
lexes_ids_list = []
for lexentr in lexes_list:
lexes_ids_list.append({
'client_id': lexentr.client_id,
'object_id': lexentr.object_id
})
request.response.status = HTTPOk.code
return lexes_ids_list
except KeyError as e:
request.response.status = HTTPBadRequest.code
return {'error': str(e)}
except IntegrityError as e:
request.response.status = HTTPInternalServerError.code
return {'error': str(e)}
except CommonException as e:
request.response.status = HTTPConflict.code
return {'error': str(e)}
def signup_post(request): # tested
try:
req = request.json_body
login = req['login']
name = req['name']
email = req['email']
password = req['password']
day = req.get('day')
month = req.get('month')
year = req.get('year')
if day is None or month is None or year is None:
request.response.status = HTTPBadRequest.code
return {'Error': "day, month or year of the birth is missing"}
# birthday = datetime.datetime.strptime(day + month + year, "%d%m%Y").date()
try:
birthday = datetime.date(year, month, day)
except ValueError:
request.response.status = HTTPBadRequest.code
return {'Error': "Invalid birthday"}
if DBSession.query(User).filter_by(login=login).first():
raise CommonException("The user with this login is already registered")
if DBSession.query(Email).filter_by(email=email).first():
raise CommonException("The user with this email is already registered")
new_user = User(login=login, name=name, signup_date=datetime.datetime.utcnow(), intl_name=login, birthday=birthday, is_active=True)
pwd = Passhash(password=password)
email = Email(email=email)
new_user.password = pwd
new_user.email.append(email)
DBSession.add(new_user)
basegroups = []
basegroups += [DBSession.query(BaseGroup).filter_by(name="Can create dictionaries").first()]
basegroups += [DBSession.query(BaseGroup).filter_by(name="Can create languages").first()]
basegroups += [DBSession.query(BaseGroup).filter_by(name="Can create organizations").first()]
basegroups += [DBSession.query(BaseGroup).filter_by(name="Can create translation strings").first()]
groups = []
for base in basegroups:
groups += [DBSession.query(Group).filter_by(subject_override=True, base_group_id=base.id).first()]
for group in groups:
if group not in new_user.groups:
new_user.groups.append(group)
DBSession.flush()
return {}
except KeyError as e:
request.response.status = HTTPBadRequest.code
return {'status': request.response.status, 'error': str(e)}
except CommonException as e:
request.response.status = HTTPConflict.code
return {'status': request.response.status, 'error': str(e)}
except ValueError as e:
request.response.status = HTTPConflict.code
return {'status': request.response.status, 'error': str(e)}
def setUp(self):
self.config = testing.setUp()
from sqlalchemy import create_engine
engine = create_engine('sqlite://')
from lingvodoc.models import (Base, Language)
DBSession.configure(bind=engine)
Base.metadata.create_all(engine)
with transaction.manager:
new_lang = Language(client_id=1,
object_id=1,
translation_string='test')
DBSession.add(new_lang)
def create_translationgist(request):
try:
variables = {'auth': request.authenticated_userid}
req = request.json_body
object_id = req.get('object_id', None)
type = req['type']
client = DBSession.query(Client).filter_by(
id=variables['auth']).first()
if not client:
raise KeyError(
"Invalid client id (not registered on server). Try to logout and then login.",
variables['auth'])
user = DBSession.query(User).filter_by(id=client.user_id).first()
if not user:
raise CommonException(
"This client id is orphaned. Try to logout and then login once more."
)
translationgist = TranslationGist(client_id=variables['auth'],
object_id=object_id,
type=type)
DBSession.add(translationgist)
DBSession.flush()
basegroups = []
basegroups += [
DBSession.query(BaseGroup).filter_by(
name="Can delete translationgist").first()
]
if not object_id:
groups = []
for base in basegroups:
group = Group(subject_client_id=translationgist.client_id,
subject_object_id=translationgist.object_id,
parent=base)
groups += [group]
for group in groups:
add_user_to_group(user, group)
request.response.status = HTTPOk.code
return {
'object_id': translationgist.object_id,
'client_id': translationgist.client_id
}
except KeyError as e:
request.response.status = HTTPBadRequest.code
return {'error': str(e)}
except IntegrityError as e:
request.response.status = HTTPInternalServerError.code
return {'error': str(e)}
except CommonException as e:
request.response.status = HTTPConflict.code
return {'error': str(e)}
def create_translationatom(request):
try:
variables = {'auth': request.authenticated_userid}
req = request.json_body
parent_client_id = req['parent_client_id']
parent_object_id = req['parent_object_id']
locale_id = req['locale_id']
content = req['content']
object_id = req.get('object_id', None)
client = DBSession.query(Client).filter_by(id=variables['auth']).first()
if not client:
raise KeyError("Invalid client id (not registered on server). Try to logout and then login.",
variables['auth'])
user = DBSession.query(User).filter_by(id=client.user_id).first()
if not user:
raise CommonException("This client id is orphaned. Try to logout and then login once more.")
parent = DBSession.query(TranslationGist).filter_by(client_id=parent_client_id, object_id=parent_object_id).first()
if not parent.marked_for_deletion:
translationatom = TranslationAtom(client_id=variables['auth'],
object_id=object_id,
parent=parent,
locale_id=locale_id,
content=content)
DBSession.add(translationatom)
DBSession.flush()
if not object_id:
basegroups = []
basegroups += [DBSession.query(BaseGroup).filter_by(name="Can edit translationatom").first()]
if not object_id:
groups = []
for base in basegroups:
group = Group(subject_client_id=translationatom.client_id, subject_object_id=translationatom.object_id, parent=base)
groups += [group]
for group in groups:
add_user_to_group(user, group)
request.response.status = HTTPOk.code
return {'object_id': translationatom.object_id,
'client_id': translationatom.client_id}
else:
raise KeyError("TranslationGist deleted")
except KeyError as e:
request.response.status = HTTPBadRequest.code
return {'error': str(e)}
except IntegrityError as e:
request.response.status = HTTPInternalServerError.code
return {'error': str(e)}
except CommonException as e:
request.response.status = HTTPConflict.code
return {'error': str(e)}
def upload_user_blob(request): # TODO: remove blob Object
variables = {'auth': authenticated_userid(request)}
response = dict()
filename = request.POST['blob'].filename
input_file = request.POST['blob'].file
class Object(object):
pass
blob = Object()
blob.client_id = variables['auth']
client = DBSession.query(Client).filter_by(id=variables['auth']).first()
blob.data_type = request.POST['data_type']
blob.filename = filename
current_user = DBSession.query(User).filter_by(id=client.user_id).first()
object_id = request.POST.get('object_id', None)
blob_object = UserBlobs(object_id=object_id,
client_id=blob.client_id,
name=filename,
data_type=blob.data_type,
user_id=current_user.id)
current_user.userblobs.append(blob_object)
blob_object.real_storage_path, blob_object.content = create_object(
request,
input_file,
blob_object,
blob.data_type,
blob.filename,
json_input=False)
if blob.data_type == "sociolinguistics":
try:
check_socio(blob_object.real_storage_path)
except Exception as e:
request.response.status = HTTPBadRequest.code
response = {"error": str(e)}
return response
DBSession.add(blob_object)
DBSession.add(current_user)
DBSession.flush()
request.response.status = HTTPOk.code
response = {
"client_id": blob_object.client_id,
"object_id": blob_object.object_id,
"content": blob_object.content
}
return response
def setUp(self):
self.config = testing.setUp()
from sqlalchemy import create_engine
engine = create_engine('sqlite://')
from lingvodoc.models import (
Base,
Language
)
DBSession.configure(bind=engine)
Base.metadata.create_all(engine)
with transaction.manager:
new_lang = Language(client_id=1, object_id=1, translation_string='test')
DBSession.add(new_lang)
def create_lexical_entry_bulk(request): # TODO: test
try:
dictionary_client_id = request.matchdict.get('dictionary_client_id')
dictionary_object_id = request.matchdict.get('dictionary_object_id')
perspective_client_id = request.matchdict.get('perspective_client_id')
perspective_object_id = request.matchdict.get('perspective_object_id')
count = request.json_body.get('count') or 0
variables = {'auth': request.authenticated_userid}
client = DBSession.query(Client).filter_by(id=variables['auth']).first()
if not client:
raise KeyError("Invalid client id (not registered on server). Try to logout and then login.",
variables['auth'])
user = DBSession.query(User).filter_by(id=client.user_id).first()
if not user:
raise CommonException("This client id is orphaned. Try to logout and then login once more.")
perspective = DBSession.query(DictionaryPerspective). \
filter_by(client_id=perspective_client_id, object_id = perspective_object_id).first()
if not perspective:
request.response.status = HTTPNotFound.code
return {'error': str("No such perspective in the system")}
lexes_list = []
for i in range(0, count):
lexentr = LexicalEntry(client_id=variables['auth'],
parent_object_id=perspective_object_id, parent=perspective)
DBSession.add(lexentr)
lexes_list.append(lexentr)
DBSession.flush()
lexes_ids_list = []
for lexentr in lexes_list:
lexes_ids_list.append({'client_id': lexentr.client_id, 'object_id': lexentr.object_id})
request.response.status = HTTPOk.code
return lexes_ids_list
except KeyError as e:
request.response.status = HTTPBadRequest.code
return {'error': str(e)}
except IntegrityError as e:
request.response.status = HTTPInternalServerError.code
return {'error': str(e)}
except CommonException as e:
request.response.status = HTTPConflict.code
return {'error': str(e)}
def testing(request):
with_group = 0
without_group = 0
for group in DBSession.query(Group).filter_by(base_group_id=26).all():
DBSession.delete(group)
for persp in DBSession.query(DictionaryPerspective):
group = DBSession.query(Group).filter_by(base_group_id=22, subject_client_id=persp.client_id,
subject_object_id=persp.object_id).first()
if not group:
without_group +=1
new_group = Group(base_group_id=26, subject_client_id=persp.client_id,
subject_object_id=persp.object_id)
for user in group.users:
new_group.users.append(user)
DBSession.add(new_group)
return {"good": with_group, "bad": without_group}
def upload_user_blob(request): # TODO: remove blob Object
variables = {'auth': authenticated_userid(request)}
response = dict()
filename = request.POST['blob'].filename
input_file = request.POST['blob'].file
class Object(object):
pass
blob = Object()
blob.client_id = variables['auth']
client = DBSession.query(Client).filter_by(id=variables['auth']).first()
blob.data_type = request.POST['data_type']
blob.filename = filename
current_user = DBSession.query(User).filter_by(id=client.user_id).first()
object_id = request.POST.get('object_id', None)
blob_object = UserBlobs(object_id=object_id,
client_id=blob.client_id,
name=filename,
data_type=blob.data_type,
user_id=current_user.id)
current_user.userblobs.append(blob_object)
blob_object.real_storage_path, blob_object.content = create_object(request, input_file, blob_object, blob.data_type,
blob.filename, json_input=False)
if blob.data_type == "sociolinguistics":
try:
check_socio(blob_object.real_storage_path)
except Exception as e:
request.response.status = HTTPBadRequest.code
response = {"error": str(e)}
return response
DBSession.add(blob_object)
DBSession.add(current_user)
DBSession.flush()
request.response.status = HTTPOk.code
response = {"client_id": blob_object.client_id, "object_id": blob_object.object_id, "content": blob_object.content}
return response
def create_organization(request): # TODO: test
try:
variables = {'auth': request.authenticated_userid}
req = request.json_body
name = req['name']
about = req['about']
client = DBSession.query(Client).filter_by(
id=variables['auth']).first()
if not client:
raise KeyError(
"Invalid client id (not registered on server). Try to logout and then login."
)
user = DBSession.query(User).filter_by(id=client.user_id).first()
if not user:
raise CommonException(
"This client id is orphaned. Try to logout and then login once more."
)
organization = Organization(name=name, about=about)
if user not in organization.users:
organization.users.append(user)
DBSession.add(organization)
DBSession.flush()
bases = DBSession.query(BaseGroup).filter_by(subject='organization')
for base in bases:
group = Group(parent=base, subject_object_id=organization.id)
add_user_to_group(user, group)
DBSession.add(group)
request.response.status = HTTPOk.code
return {'organization_id': organization.id}
except KeyError as e:
request.response.status = HTTPBadRequest.code
return {'error': str(e)}
except IntegrityError as e:
request.response.status = HTTPInternalServerError.code
return {'error': str(e)}
except CommonException as e:
request.response.status = HTTPConflict.code
return {'error': str(e)}
def create_organization(request): # TODO: test
try:
variables = {'auth': request.authenticated_userid}
req = request.json_body
name = req['name']
about = req['about']
client = DBSession.query(Client).filter_by(id=variables['auth']).first()
if not client:
raise KeyError("Invalid client id (not registered on server). Try to logout and then login.")
user = DBSession.query(User).filter_by(id=client.user_id).first()
if not user:
raise CommonException("This client id is orphaned. Try to logout and then login once more.")
organization = Organization(name=name,
about=about)
if user not in organization.users:
organization.users.append(user)
DBSession.add(organization)
DBSession.flush()
bases = DBSession.query(BaseGroup).filter_by(subject='organization')
for base in bases:
group = Group(parent=base, subject_object_id=organization.id)
if not user in group.users:
group.users.append(user)
DBSession.add(group)
request.response.status = HTTPOk.code
return {'organization_id': organization.id}
except KeyError as e:
request.response.status = HTTPBadRequest.code
return {'error': str(e)}
except IntegrityError as e:
request.response.status = HTTPInternalServerError.code
return {'error': str(e)}
except CommonException as e:
request.response.status = HTTPConflict.code
return {'error': str(e)}
def create_field(translation_gist_client_id, translation_gist_object_id, data_type_translation_gist_client_id,
data_type_translation_gist_object_id, client):
try:
if not client:
raise KeyError("Invalid client id (not registered on server). Try to logout and then login.")
user = DBSession.query(User).filter_by(id=client.user_id).first()
field = Field(client_id=client.id, ###
data_type_translation_gist_client_id=data_type_translation_gist_client_id,
data_type_translation_gist_object_id=data_type_translation_gist_object_id,
translation_gist_client_id=translation_gist_client_id,
translation_gist_object_id=translation_gist_object_id
)
DBSession.add(field)
DBSession.flush()
return {'object_id': field.object_id,
'client_id': field.client_id}
except KeyError as e:
return {'error': str(e)}
def create_persp_to_field(request):
try:
variables = {'auth': request.authenticated_userid}
req = request.json_body
client = DBSession.query(Client).filter_by(id=variables['auth']).first()
if not client:
raise KeyError("Invalid client id (not registered on server). Try to logout and then login.",
variables['auth'])
user = DBSession.query(User).filter_by(id=client.user_id).first()
if not user:
raise CommonException("This client id is orphaned. Try to logout and then login once more.")
if not DBSession.query(DictionaryPerspectiveToField).filter_by(client_id=req['client_id'], object_id=req['object_id']).first():
field_object = DictionaryPerspectiveToField(client_id=req['client_id'],
object_id=req['object_id'],
parent_client_id=req['parent_client_id'],
parent_object_id=req['parent_object_id'],
field_client_id=req['field_client_id'],
field_object_id=req['field_object_id'],
self_client_id=req['self_client_id'],
self_object_id=req['self_object_id'],
position=req['position'])
DBSession.add(field_object)
return {}
except KeyError as e:
request.response.status = HTTPBadRequest.code
return {'error': str(e)}
except IntegrityError as e:
request.response.status = HTTPInternalServerError.code
return {'error': str(e)}
except CommonException as e:
request.response.status = HTTPConflict.code
return {'error': str(e)}
def setUp(self):
self.config = testing.setUp()
import webtest
from pyramid import paster
from sqlalchemy import create_engine
engine = create_engine('sqlite://')
myapp = paster.get_app('testing.ini')
self.app = webtest.TestApp(myapp)
from lingvodoc.models import (Base, Locale, User, Passhash, Client)
DBSession.configure(bind=engine)
Base.metadata.create_all(engine)
with transaction.manager:
ru_locale = Locale(id=1, shortcut="ru", intl_name="Русский")
DBSession.add(ru_locale)
en_locale = Locale(id=2, shortcut="en", intl_name="English")
DBSession.add(en_locale)
DBSession.flush()
new_user = User(id=1, login='******', default_locale_id=1)
new_pass = Passhash(password='******')
DBSession.add(new_pass)
new_user.password = new_pass
DBSession.add(new_user)
new_client = Client(id=1, user=new_user)
DBSession.add(new_client)
def testing(request):
translationgist=TranslationGist(client_id=1, type='Language')
DBSession.add(translationgist)
translationatom=TranslationAtom(client_id=1, parent=translationgist, locale_id=2, content='testing objecttoc')
DBSession.add(translationatom)
lang = Language(client_id=1, translation_gist_client_id=translationgist.client_id,
translation_gist_object_id=translationgist.object_id)
DBSession.add(lang)
DBSession.rollback()
return {}
def setUp(self):
self.config = testing.setUp()
from lingvodoc.models import (
Base,
Language,
UserEntitiesTranslationString,
Locale
)
from sqlalchemy import create_engine
engine = create_engine('sqlite://')
DBSession.configure(bind=engine)
Base.metadata.create_all(engine)
with transaction.manager:
ru_locale = Locale(id=1, shortcut="ru", intl_name="Русский")
DBSession.add(ru_locale)
DBSession.flush()
new_uets = UserEntitiesTranslationString(object_id = 1, client_id = 1, locale_id=1,
translation_string = 'test', translation = 'working')
DBSession.add(new_uets)
new_lang=Language(client_id=1, object_id=1, translation_string = 'test')
DBSession.add(new_lang)
def setUp(self):
self.config = testing.setUp()
from lingvodoc.models import (Base, Language,
UserEntitiesTranslationString, Locale)
from sqlalchemy import create_engine
engine = create_engine('sqlite://')
DBSession.configure(bind=engine)
Base.metadata.create_all(engine)
with transaction.manager:
ru_locale = Locale(id=1, shortcut="ru", intl_name="Русский")
DBSession.add(ru_locale)
DBSession.flush()
new_uets = UserEntitiesTranslationString(object_id=1,
client_id=1,
locale_id=1,
translation_string='test',
translation='working')
DBSession.add(new_uets)
new_lang = Language(client_id=1,
object_id=1,
translation_string='test')
DBSession.add(new_lang)
def create_l2_entity(request):
try:
variables = {'auth': authenticated_userid(request)}
response = dict()
parent_client_id = request.matchdict.get('level_one_client_id')
parent_object_id = request.matchdict.get('level_one_object_id')
req = request.json_body
client = DBSession.query(Client).filter_by(id=variables['auth']).first()
if not client:
raise KeyError("Invalid client id (not registered on server). Try to logout and then login.")
user = DBSession.query(User).filter_by(id=client.user_id).first()
if not user:
raise CommonException("This client id is orphaned. Try to logout and then login once more.")
# parent = DBSession.query(LevelOneEntity).filter_by(client_id=parent_client_id, object_id=parent_object_id).first()
parent = None
if not parent:
request.response.status = HTTPNotFound.code
return {'error': str("No such level one entity in the system")}
additional_metadata = req.get('additional_metadata')
# entity = LevelTwoEntity(client_id=client.id, object_id=DBSession.query(LevelTwoEntity).filter_by(client_id=client.id).count() + 1, entity_type=req['entity_type'],
# locale_id=req['locale_id'], additional_metadata=additional_metadata,
# parent=parent)
entity = None
DBSession.add(entity)
DBSession.flush()
data_type = req.get('data_type')
filename = req.get('filename')
real_location = None
url = None
if data_type == 'image' or data_type == 'sound' or data_type == 'markup':
real_location, url = create_object(request, req['content'], entity, data_type, filename)
if url and real_location:
entity.content = url
old_meta = entity.additional_metadata
need_hash = True
if old_meta:
new_meta=json.loads(old_meta)
if new_meta.get('hash'):
need_hash = False
if need_hash:
hash = hashlib.sha224(base64.urlsafe_b64decode(req['content'])).hexdigest()
hash_dict = {'hash': hash}
if old_meta:
new_meta = json.loads(old_meta)
new_meta.update(hash_dict)
else:
new_meta = hash_dict
entity.additional_metadata = json.dumps(new_meta)
else:
entity.content = req['content']
DBSession.add(entity)
request.response.status = HTTPOk.code
response['client_id'] = entity.client_id
response['object_id'] = entity.object_id
return response
except KeyError as e:
request.response.status = HTTPBadRequest.code
return {'error': str(e)}
except IntegrityError as e:
request.response.status = HTTPInternalServerError.code
return {'error': str(e)}
except CommonException as e:
request.response.status = HTTPConflict.code
return {'error': str(e)}
def create_entity(request): # tested
try:
variables = {'auth': authenticated_userid(request)}
response = dict()
parent_client_id = request.matchdict.get('lexical_entry_client_id')
parent_object_id = request.matchdict.get('lexical_entry_object_id')
req = request.json_body
client = DBSession.query(Client).filter_by(id=variables['auth']).first()
if not client:
raise KeyError("Invalid client id (not registered on server). Try to logout and then login.")
user = DBSession.query(User).filter_by(id=client.user_id).first()
if not user:
raise CommonException("This client id is orphaned. Try to logout and then login once more.")
parent = DBSession.query(LexicalEntry).filter_by(client_id=parent_client_id, object_id=parent_object_id).first()
if not parent:
request.response.status = HTTPNotFound.code
return {'error': str("No such lexical entry in the system")}
if 'content' not in req is None:
request.response.status = HTTPBadRequest.code
return {'error': 'Missing value: content'}
additional_metadata = req.get('additional_metadata')
parent_entity = None
# import pdb
# pdb.set_trace()
# data_type = DBSession.query(TranslationAtom).filter(TranslationAtom.locale_id == 2).join(TranslationGist, and_(
# TranslationAtom.parent_client_id == TranslationGist.client_id,
# TranslationAtom.parent_object_id == TranslationGist.object_id)).join(Field, and_(
# TranslationGist.client_id == Field.data_type_translation_gist_client_id,
# TranslationGist.object_id == Field.data_type_translation_gist_object_id)).filter(
# Field.client_id == req['field_client_id'], Field.object_id == req['field_object_id']).first()
tr_atom = DBSession.query(TranslationAtom).join(TranslationGist, and_(
TranslationAtom.parent_client_id == TranslationGist.client_id,
TranslationAtom.parent_object_id == TranslationGist.object_id)).join(Field, and_(
TranslationGist.client_id == Field.data_type_translation_gist_client_id,
TranslationGist.object_id == Field.data_type_translation_gist_object_id)).filter(
Field.client_id == req['field_client_id'], Field.object_id == req['field_object_id']).first()
data_type = tr_atom.content
if req.get('entity_client_id') and req.get('entity_object_id'):
parent_entity = DBSession.query(Entity).filter_by(client_id=req['entity_client_id'],
object_id=req['entity_object_id']).first()
if not parent_entity:
return {'error': str("No such parent entity in the system")}
entity = Entity(client_id=client.id,
field_client_id=req['field_client_id'],
field_object_id=req['field_object_id'],
locale_id=req['locale_id'],
additional_metadata=additional_metadata,
parent=parent)
if parent_entity:
entity.parent_entity = parent_entity
filename = req.get('filename')
real_location = None
url = None
if data_type == 'image' or data_type == 'sound' or data_type == 'markup':
real_location, url = create_object(request, req['content'], entity, data_type, filename)
entity.content = url
old_meta = entity.additional_metadata
need_hash = True
if old_meta:
new_meta = json.loads(old_meta)
if new_meta.get('hash'):
need_hash = False
if need_hash:
hash = hashlib.sha224(base64.urlsafe_b64decode(req['content'])).hexdigest()
hash_dict = {'hash': hash}
if old_meta:
new_meta = json.loads(old_meta)
new_meta.update(hash_dict)
else:
new_meta = hash_dict
entity.additional_metadata = json.dumps(new_meta)
elif data_type == 'link':
try:
entity.link_client_id = req['content']['client_id']
entity.link_object_id = req['content']['object_id']
except (KeyError, TypeError):
request.response.status = HTTPBadRequest.code
return {'Error': "The field is of link type. You should provide client_id and object id in the content"}
else:
entity.content = req['content']
return None
DBSession.add(entity)
request.response.status = HTTPOk.code
response['client_id'] = entity.client_id
response['object_id'] = entity.object_id
return response
except IntegrityError as e:
request.response.status = HTTPInternalServerError.code
return {'error': str(e)}
except CommonException as e:
request.response.status = HTTPConflict.code
return {'error': str(e)}
def create_entity(le_client_id, le_object_id, field_client_id, field_object_id,
additional_metadata, client, content= None, filename=None,
link_client_id=None, link_object_id=None, folder_name=None, up_lvl=None, locale_id=2, storage=None): # tested
log = logging.getLogger(__name__)
log.setLevel(logging.DEBUG)
parent = DBSession.query(LexicalEntry).filter_by(client_id=le_client_id, object_id=le_object_id).first()
if not parent:
return {'error': str("No such lexical entry in the system")}
upper_level = None
tr_atom = DBSession.query(TranslationAtom).join(TranslationGist, and_(
TranslationAtom.locale_id == 2,
TranslationAtom.parent_client_id == TranslationGist.client_id,
TranslationAtom.parent_object_id == TranslationGist.object_id)).join(Field, and_(
TranslationGist.client_id == Field.data_type_translation_gist_client_id,
TranslationGist.object_id == Field.data_type_translation_gist_object_id)).filter(
Field.client_id == field_client_id, Field.object_id == field_object_id).first()
data_type = tr_atom.content.lower()
if up_lvl:
upper_level = DBSession.query(Entity).filter_by(client_id=up_lvl[0],
object_id=up_lvl[1]).first()
entity = Entity(client_id=client.id,
field_client_id=field_client_id,
field_object_id=field_object_id,
locale_id=locale_id,
###additional_metadata=additional_metadata,
parent=parent)
if upper_level:
entity.upper_level = upper_level
hash = None
real_location = None
url = None
if data_type == 'image' or data_type == 'sound' or 'markup' in data_type:
##entity.data_type = data_type
real_location, url = create_object(content, entity, data_type, filename, folder_name, storage)
entity.content = url
old_meta = entity.additional_metadata
need_hash = True
if old_meta:
new_meta = old_meta #json.loads(old_meta)
if new_meta.get('hash'):
need_hash = False
if need_hash:
hash = hashlib.sha224(base64.urlsafe_b64decode(content)).hexdigest()
hash_dict = {'hash': hash}
if old_meta:
new_meta = old_meta #json.loads(old_meta)
new_meta.update(hash_dict)
else:
new_meta = hash_dict
entity.additional_metadata = new_meta #json.dumps(new_meta)
old_meta = entity.additional_metadata
if data_type == "markup":
data_type_dict = {"data_type": "praat markup"}
if old_meta:
new_meta = old_meta #json.loads(old_meta)
new_meta.update(data_type_dict)
else:
new_meta = data_type_dict
entity.additional_metadata = new_meta #json.dumps(new_meta)
elif data_type == 'link':
try:
entity.link_client_id = link_client_id
entity.link_object_id = link_object_id
except (KeyError, TypeError):
return {'Error': "The field is of link type. You should provide client_id and object id in the content"}
else:
entity.content = content
entity.publishingentity.accepted = True
DBSession.add(entity)
#log.debug(filename)
return (entity.client_id, entity.object_id)
def convert_five_tiers(
dictionary_client_id,
dictionary_object_id,
user_id,
origin_client_id,
origin_object_id,
sqlalchemy_url,
storage,
eaf_url,
sound_url=None
):
log = logging.getLogger(__name__)
log.setLevel(logging.DEBUG)
no_sound = True
if sound_url:
no_sound = False
with warnings.catch_warnings():
warnings.filterwarnings('error')
try:
from pydub import AudioSegment
except Warning as e:
no_sound = True
if not no_sound:
with tempfile.NamedTemporaryFile() as temp:
try:
sound_file = request.urlopen(sound_url)
except HTTPError as e:
return {'error': str(e.read().decode("utf8", 'ignore'))}
with open(temp.name,'wb') as output:
output.write(sound_file.read())
full_audio = AudioSegment.from_wav(temp.name)
temp.flush()
field_ids = {}
with transaction.manager:
client = DBSession.query(Client).filter_by(id=user_id).first()
if not client:
raise KeyError("Invalid client id (not registered on server). Try to logout and then login.",
user_id)
user = DBSession.query(User).filter_by(id=client.user_id).first()
all_fieldnames = ("Markup",
"Paradigm Markup",
"Word",
"Transcription",
"Translation",
"Sound",
"Etymology",
"Backref",
"Word of Paradigmatic forms",
"Transcription of Paradigmatic forms",
"Translation of Paradigmatic forms",
"Sounds of Paradigmatic forms"
)
for name in all_fieldnames:
data_type_query = DBSession.query(Field) \
.join(TranslationGist,
and_(Field.translation_gist_object_id == TranslationGist.object_id,
Field.translation_gist_client_id == TranslationGist.client_id))\
.join(TranslationGist.translationatom)
field = data_type_query.filter(TranslationAtom.locale_id == 2,
TranslationAtom.content == name).one() # todo: a way to find this fields if wwe cannot use one
field_ids[name] = (field.client_id, field.object_id)
DBSession.flush()
"""
parent_client_id = gist_client_id
parent_object_id = gist_object_id
parent = DBSession.query(TranslationGist).filter_by(client_id=parent_client_id, object_id=parent_object_id).first()
lang_parent = DBSession.query(Language).filter_by(client_id=language_client_id, object_id=language_object_id).first()
resp = translation_service_search("WiP")
state_translation_gist_object_id, state_translation_gist_client_id = resp['object_id'], resp['client_id']
dictionary = Dictionary(client_id=user_id,
state_translation_gist_object_id=state_translation_gist_object_id,
state_translation_gist_client_id=state_translation_gist_client_id,
parent=lang_parent,
translation_gist_client_id=gist_client_id,
translation_gist_object_id=gist_object_id
)
#additional_metadata=additional_metadata)
DBSession.add(dictionary)
DBSession.flush()
dictionary_client_id = dictionary.client_id
dictionary_object_id = dictionary.object_id
"""
resp = translation_service_search("WiP")
state_translation_gist_object_id, state_translation_gist_client_id = resp['object_id'], resp['client_id']
for base in DBSession.query(BaseGroup).filter_by(dictionary_default=True):
new_group = Group(parent=base,
subject_object_id=dictionary_object_id, subject_client_id=dictionary_client_id)
if user not in new_group.users:
new_group.users.append(user)
DBSession.add(new_group)
DBSession.flush()
"""
# FIRST PERSPECTIVE
"""
resp = translation_service_search_all("Lexical Entries")
persp_translation_gist_client_id, persp_translation_gist_object_id = resp['client_id'], resp['object_id']
parent = DBSession.query(Dictionary).filter_by(client_id=dictionary_client_id, object_id=dictionary_object_id).first()
origin_metadata= {"origin_client_id": origin_client_id,
"origin_object_id": origin_object_id
}
perspective = DictionaryPerspective(client_id=client.id, ###
state_translation_gist_object_id=state_translation_gist_object_id,
state_translation_gist_client_id=state_translation_gist_client_id,
parent=parent,
# import_source=req.get('import_source'),
# import_hash=req.get('import_hash'),
additional_metadata=origin_metadata,
translation_gist_client_id=persp_translation_gist_client_id,
translation_gist_object_id=persp_translation_gist_object_id
)
perspective.additional_metadata = origin_metadata
# if is_template is not None:
# perspective.is_template = is_template
DBSession.add(perspective)
owner_client = DBSession.query(Client).filter_by(id=parent.client_id).first()
owner = owner_client.user
for base in DBSession.query(BaseGroup).filter_by(perspective_default=True):
new_group = Group(parent=base,
subject_object_id=perspective.object_id, subject_client_id=perspective.client_id)
if user not in new_group.users:
new_group.users.append(user)
if owner not in new_group.users:
new_group.users.append(owner)
DBSession.add(new_group)
DBSession.flush()
first_perspective_client_id = perspective.client_id
first_perspective_object_id = perspective.object_id
"""
# SECOND PERSPECTIVE
"""
resp = translation_service_search_all("Paradigms")
persp_translation_gist_client_id, persp_translation_gist_object_id = resp['client_id'], resp['object_id']
parent = DBSession.query(Dictionary).filter_by(client_id=dictionary_client_id, object_id=dictionary_object_id).first()
if not parent:
return {'error': str("No such dictionary in the system")}
perspective = DictionaryPerspective(client_id=client.id, ### variables['auth']
state_translation_gist_object_id=state_translation_gist_object_id,
state_translation_gist_client_id=state_translation_gist_client_id,
parent=parent,
# import_source=req.get('import_source'),
# import_hash=req.get('import_hash'),
additional_metadata=origin_metadata,
translation_gist_client_id=persp_translation_gist_client_id,
translation_gist_object_id=persp_translation_gist_object_id
)
perspective.additional_metadata = origin_metadata
# if is_template is not None:
# perspective.is_template = is_template
DBSession.add(perspective)
owner_client = DBSession.query(Client).filter_by(id=parent.client_id).first()
owner = owner_client.user
for base in DBSession.query(BaseGroup).filter_by(perspective_default=True):
new_group = Group(parent=base,
subject_object_id=perspective.object_id, subject_client_id=perspective.client_id)
if user not in new_group.users:
new_group.users.append(user)
if owner not in new_group.users:
new_group.users.append(owner)
DBSession.add(new_group)
second_perspective_client_id = perspective.client_id
second_perspective_object_id = perspective.object_id
fp_fields_dict = {}
"""
# FIRST PERSPECTIVE FIELDS CREATION
"""
fp_field_names = ("Word", "Transcription", "Translation", "Sound", "Etymology", "Backref")
fields_list = []
for fieldname in fp_field_names: #
if fieldname == "Backref":
fields_list.append(
{
"client_id": field_ids[fieldname][0],
"object_id": field_ids[fieldname][1],
"link":{
"client_id": second_perspective_client_id,
"object_id": second_perspective_object_id
}
}
)
elif fieldname == "Sound":
fields_list.append(
{
"client_id": field_ids[fieldname][0],
"object_id": field_ids[fieldname][1],
"contains":[{
"client_id": field_ids["Markup"][0],
"object_id": field_ids["Markup"][1]
}
]
}
)
else:
fields_list.append({"client_id": field_ids[fieldname][0], "object_id": field_ids[fieldname][1]})
fp_fields_dict[fieldname] = (field_ids[fieldname][0], field_ids[fieldname][1])
fp_fields_dict["Markup"] = (field_ids["Markup"][0], field_ids["Markup"][1])
update_perspective_fields(fields_list, first_perspective_client_id, first_perspective_object_id, client )
"""
# Creating fields of the second perspective
"""
sp_fields_dict = {}
fields_list = []
sp_field_names = ("Word of Paradigmatic forms", "Transcription of Paradigmatic forms", "Translation of Paradigmatic forms", "Sounds of Paradigmatic forms", "Backref")
for fieldname in sp_field_names: #
if fieldname == "Backref":
fields_list.append(
{
"client_id": field_ids[fieldname][0],
"object_id": field_ids[fieldname][1],
"link":{
"client_id": first_perspective_client_id,
"object_id": first_perspective_object_id
}
}
)
elif fieldname == "Sounds of Paradigmatic forms":
fields_list.append(
{
"client_id": field_ids[fieldname][0],
"object_id": field_ids[fieldname][1],
"contains":[{
"client_id": field_ids["Paradigm Markup"][0],
"object_id": field_ids["Paradigm Markup"][1]
}
]
}
)
else:
fields_list.append({"client_id": field_ids[fieldname][0], "object_id": field_ids[fieldname][1]})
sp_fields_dict[fieldname] = (field_ids[fieldname][0], field_ids[fieldname][1])
sp_fields_dict["Paradigm Markup"] = (field_ids["Paradigm Markup"][0], field_ids["Paradigm Markup"][1])
update_perspective_fields(fields_list, second_perspective_client_id, second_perspective_object_id, client)
link_dict = defaultdict(list)
dubl = []
log = logging.getLogger(__name__)
try:
eaffile = request.urlopen(eaf_url)
except HTTPError as e:
return {'error': str(e.read().decode("utf8", 'ignore'))}
with tempfile.NamedTemporaryFile() as temp:
temp.write(eaffile.read())
converter = elan_parser.Elan(temp.name)
converter.parse()
final_dicts = converter.proc()
temp.flush()
for phrase in final_dicts:
perspective = DBSession.query(DictionaryPerspective).\
filter_by(client_id=second_perspective_client_id, object_id = second_perspective_object_id).first() #sec?
if not perspective:
return {'error': str("No such perspective in the system")}
lexentr = LexicalEntry(client_id=client.id,
parent_object_id=second_perspective_object_id, parent=perspective)
DBSession.add(lexentr)
sp_lexical_entry_client_id = lexentr.client_id
sp_lexical_entry_object_id = lexentr.object_id
curr_dict = None
for word_translation in phrase:
if type(word_translation) is not list:
curr_dict = word_translation
main_tier_text = " ".join([word_translation[i][1].text for i in word_translation if len(word_translation[i]) > 1 and type(word_translation[i][1].text) is str])
if main_tier_text:
create_entity(sp_lexical_entry_client_id, sp_lexical_entry_object_id, field_ids["Word of Paradigmatic forms"][0], field_ids["Word of Paradigmatic forms"][1],
None, client, main_tier_text, filename=None, storage=storage)
if not no_sound:
if word.time[1] < len(full_audio):
with tempfile.NamedTemporaryFile() as temp:
full_audio[ word.time[0]: word.time[1]].export(temp.name, format="wav")
audio_slice = temp.read()
create_entity(sp_lexical_entry_client_id, sp_lexical_entry_object_id, field_ids["Sounds of Paradigmatic forms"][0], field_ids["Sounds of Paradigmatic forms"][1],
None, client, filename="%s.wav" %(word.index) , folder_name="sound1", content=base64.urlsafe_b64encode(audio_slice).decode(), storage=storage)
temp.flush()
else:
word = word_translation[0]
tier_name = word.tier
new = " ".join([i.text for i in word_translation])
create_entity(sp_lexical_entry_client_id, sp_lexical_entry_object_id, field_ids[EAF_TIERS[tier_name]][0], field_ids[EAF_TIERS[tier_name]][1],
None, client, new, filename=None, storage=storage)
for word in curr_dict:
column = [word] + curr_dict[word]
cort = reversed(tuple(i.text for i in column))
if cort in link_dict:
fp_lexical_entry_client_id, fp_lexical_entry_object_id = link_dict[cort]
else:
perspective = DBSession.query(DictionaryPerspective).\
filter_by(client_id=first_perspective_client_id, object_id = first_perspective_object_id).first()
if not perspective:
return {'error': str("No such perspective in the system")}
lexentr = LexicalEntry(client_id=client.id,
parent_object_id=first_perspective_object_id, parent=perspective)
DBSession.add(lexentr)
fp_lexical_entry_client_id = lexentr.client_id
fp_lexical_entry_object_id = lexentr.object_id
create_entity(fp_lexical_entry_client_id, fp_lexical_entry_object_id, field_ids[EAF_TIERS[word.tier]][0], field_ids[EAF_TIERS[word.tier]][1],
None, client, word.text, filename=None, storage=storage)
link_dict[cort] = (fp_lexical_entry_client_id, fp_lexical_entry_object_id)
for other_word in curr_dict[word]:
create_entity(fp_lexical_entry_client_id, fp_lexical_entry_object_id, field_ids[EAF_TIERS[other_word.tier]][0], field_ids[EAF_TIERS[other_word.tier]][1],
None, client, other_word.text, filename=None, storage=storage)
if not no_sound:
if word.time[1] < len(full_audio):
with tempfile.NamedTemporaryFile() as temp:
full_audio[ word.time[0]: word.time[1]].export(temp.name, format="wav")
audio_slice = temp.read()
create_entity(fp_lexical_entry_client_id, fp_lexical_entry_object_id, field_ids["Sound"][0], field_ids["Sound"][1],
None, client, filename="%s.wav" %(word.index) , folder_name="sound1", content=base64.urlsafe_b64encode(audio_slice).decode(), storage=storage)
temp.flush()
dubl_tuple = ((sp_lexical_entry_client_id, sp_lexical_entry_object_id), (fp_lexical_entry_client_id, fp_lexical_entry_object_id))
if not dubl_tuple in dubl:
dubl.append(dubl_tuple)
create_entity(sp_lexical_entry_client_id, sp_lexical_entry_object_id, field_ids["Backref"][0], field_ids["Backref"][1],
None, client, filename=None, link_client_id=fp_lexical_entry_client_id, link_object_id=fp_lexical_entry_object_id, storage=storage)
create_entity(fp_lexical_entry_client_id, fp_lexical_entry_object_id, field_ids["Backref"][0], field_ids["Backref"][1],
None, client, filename=None, link_client_id=sp_lexical_entry_client_id, link_object_id=sp_lexical_entry_object_id, storage=storage)
return
def create_language(request): # tested & in docs
try:
variables = {'auth': request.authenticated_userid}
req = request.json_body
try:
parent_client_id = req['parent_client_id']
parent_object_id = req['parent_object_id']
except:
parent_client_id = None
parent_object_id = None
translation_gist_client_id = req['translation_gist_client_id']
translation_gist_object_id = req['translation_gist_object_id']
client = DBSession.query(Client).filter_by(
id=variables['auth']).first()
object_id = req.get('object_id', None)
if not client:
raise KeyError(
"Invalid client id (not registered on server). Try to logout and then login.",
variables['auth'])
user = DBSession.query(User).filter_by(id=client.user_id).first()
if not user:
raise CommonException(
"This client id is orphaned. Try to logout and then login once more."
)
parent = None
if parent_client_id and parent_object_id:
parent = DBSession.query(Language).filter_by(
client_id=parent_client_id,
object_id=parent_object_id).first()
language = Language(
client_id=variables['auth'],
object_id=object_id,
translation_gist_client_id=translation_gist_client_id,
translation_gist_object_id=translation_gist_object_id)
DBSession.add(language)
if parent:
language.parent = parent
DBSession.flush()
basegroups = []
basegroups += [
DBSession.query(BaseGroup).filter_by(
name="Can edit languages").first()
]
basegroups += [
DBSession.query(BaseGroup).filter_by(
name="Can delete languages").first()
]
if not object_id:
groups = []
for base in basegroups:
group = Group(subject_client_id=language.client_id,
subject_object_id=language.object_id,
parent=base)
groups += [group]
for group in groups:
add_user_to_group(user, group)
request.response.status = HTTPOk.code
return {
'object_id': language.object_id,
'client_id': language.client_id
}
except KeyError as e:
request.response.status = HTTPBadRequest.code
return {'error': str(e)}
except IntegrityError as e:
request.response.status = HTTPInternalServerError.code
return {'error': str(e)}
except CommonException as e:
request.response.status = HTTPConflict.code
return {'error': str(e)}
def convert_db_new( blob_client_id, blob_object_id, language_client_id, language_object_id, user_id, gist_client_id, gist_object_id, storage,
locale_id=2):
log = logging.getLogger(__name__)
log.setLevel(logging.DEBUG)
time.sleep(4)
field_ids = {}
with transaction.manager:
blob = DBSession.query(UserBlobs).filter_by(client_id=blob_client_id, object_id=blob_object_id).first()
# DBSession.flush()
filename = blob.real_storage_path
log.debug("user_id: %s" % user_id)
log.debug("Starting convert_one")
log.debug("Creating session")
sqconn = sqlite3.connect(filename)
log.debug("Connected to sqlite3 database")
client = DBSession.query(Client).filter_by(id=user_id).first()
if not client:
raise KeyError("Invalid client id (not registered on server). Try to logout and then login.",
user_id)
user = DBSession.query(User).filter_by(id=client.user_id).first()
if not user:
log.debug("ERROR")
all_fieldnames = ("Markup",
"Paradigm Markup",
"Word",
"Transcription",
"Translation",
"Sound",
"Etymology",
"Backref",
"Word of Paradigmatic forms",
"Transcription of Paradigmatic forms",
"Translation of Paradigmatic forms",
"Sounds of Paradigmatic forms"
)
for name in all_fieldnames:
data_type_query = DBSession.query(Field) \
.join(TranslationGist,
and_(Field.translation_gist_object_id == TranslationGist.object_id,
Field.translation_gist_client_id == TranslationGist.client_id))\
.join(TranslationGist.translationatom)
field = data_type_query.filter(TranslationAtom.locale_id == 2,
TranslationAtom.content == name).one() # todo: a way to find this fields if wwe cannot use one
field_ids[name] = (field.client_id, field.object_id)
DBSession.flush()
"""
dict_attributes = get_dict_attributes(sqconn)
translationgist = TranslationGist(client_id=user_id, type="Dictionary")
DBSession.add(translationgist)
DBSession.flush()
gist_client_id = translationgist.client_id
gist_object_id = translationgist.object_id
"""
parent_client_id = gist_client_id
parent_object_id = gist_object_id
parent = DBSession.query(TranslationGist).filter_by(client_id=parent_client_id, object_id=parent_object_id).first()
"""
translationatom = TranslationAtom(client_id=client.id,
parent=parent,
locale_id=locale_id,
content=dict_attributes["dictionary_name"])
DBSession.add(translationatom)
DBSession.flush()
atom_client_id = translationatom.client_id
atom_object_id = translationatom.object_id
log.debug(dict_attributes["dictionary_name"])
language_client_id = atom_client_id
language_object_id = atom_object_id
"""
lang_parent = DBSession.query(Language).filter_by(client_id=language_client_id, object_id=language_object_id).first()
resp = translation_service_search("WiP")
state_translation_gist_object_id, state_translation_gist_client_id = resp['object_id'], resp['client_id']
dictionary = Dictionary(client_id=user_id,
state_translation_gist_object_id=state_translation_gist_object_id,
state_translation_gist_client_id=state_translation_gist_client_id,
parent=lang_parent,
translation_gist_client_id=gist_client_id,
translation_gist_object_id=gist_object_id
)
#additional_metadata=additional_metadata)
DBSession.add(dictionary)
DBSession.flush()
dictionary_client_id = dictionary.client_id
dictionary_object_id = dictionary.object_id
for base in DBSession.query(BaseGroup).filter_by(dictionary_default=True):
new_group = Group(parent=base,
subject_object_id=dictionary.object_id, subject_client_id=dictionary.client_id)
if user not in new_group.users:
new_group.users.append(user)
DBSession.add(new_group)
DBSession.flush()
"""
# FIRST PERSPECTIVE
"""
resp = translation_service_search_all("Lexical Entries")
persp_translation_gist_client_id, persp_translation_gist_object_id = resp['client_id'], resp['object_id']
parent = DBSession.query(Dictionary).filter_by(client_id=dictionary_client_id, object_id=dictionary_object_id).first()
perspective = DictionaryPerspective(client_id=client.id, ###
state_translation_gist_object_id=state_translation_gist_object_id,
state_translation_gist_client_id=state_translation_gist_client_id,
parent=parent,
# import_source=req.get('import_source'),
# import_hash=req.get('import_hash'),
# additional_metadata=additional_metadata,
translation_gist_client_id=persp_translation_gist_client_id,
translation_gist_object_id=persp_translation_gist_object_id
)
# if is_template is not None:
# perspective.is_template = is_template
DBSession.add(perspective)
owner_client = DBSession.query(Client).filter_by(id=parent.client_id).first()
owner = owner_client.user
for base in DBSession.query(BaseGroup).filter_by(perspective_default=True):
new_group = Group(parent=base,
subject_object_id=perspective.object_id, subject_client_id=perspective.client_id)
if user not in new_group.users:
new_group.users.append(user)
if owner not in new_group.users:
new_group.users.append(owner)
DBSession.add(new_group)
DBSession.flush()
first_perspective_client_id = perspective.client_id
first_perspective_object_id = perspective.object_id
"""
# SECOND PERSPECTIVE
"""
resp = translation_service_search_all("Paradigms")
persp_translation_gist_client_id, persp_translation_gist_object_id = resp['client_id'], resp['object_id']
parent = DBSession.query(Dictionary).filter_by(client_id=dictionary_client_id, object_id=dictionary_object_id).first()
if not parent:
return {'error': str("No such dictionary in the system")}
perspective = DictionaryPerspective(client_id=client.id, ### variables['auth']
state_translation_gist_object_id=state_translation_gist_object_id,
state_translation_gist_client_id=state_translation_gist_client_id,
parent=parent,
# import_source=req.get('import_source'),
# import_hash=req.get('import_hash'),
# additional_metadata=additional_metadata,
translation_gist_client_id=persp_translation_gist_client_id,
translation_gist_object_id=persp_translation_gist_object_id
)
# if is_template is not None:
# perspective.is_template = is_template
DBSession.add(perspective)
owner_client = DBSession.query(Client).filter_by(id=parent.client_id).first()
owner = owner_client.user
for base in DBSession.query(BaseGroup).filter_by(perspective_default=True):
new_group = Group(parent=base,
subject_object_id=perspective.object_id, subject_client_id=perspective.client_id)
if user not in new_group.users:
new_group.users.append(user)
if owner not in new_group.users:
new_group.users.append(owner)
DBSession.add(new_group)
second_perspective_client_id = perspective.client_id
second_perspective_object_id = perspective.object_id
get_fp_ids = sqconn.cursor()
get_fp_ids.execute("select id from dictionary where is_a_regular_form=1")
count_cursor = sqconn.cursor()
count_cursor.execute("select count(*) from dictionary where is_a_regular_form=1")
words_count = count_cursor.fetchone()[0]
count_cursor2 = sqconn.cursor()
count_cursor2.execute("select count(*) from dictionary where is_a_regular_form=0")
words_count2 = count_cursor2.fetchone()[0]
ids_dict = dict()
for i in range(words_count):
perspective = DBSession.query(DictionaryPerspective).\
filter_by(client_id=first_perspective_client_id, object_id = first_perspective_object_id).first()
if not perspective:
return {'error': str("No such perspective in the system")}
lexentr = LexicalEntry( client_id=client.id,
parent_object_id=first_perspective_object_id, parent=perspective)
DBSession.add(lexentr)
lexical_entry_client_id = lexentr.client_id
lexical_entry_object_id = lexentr.object_id
ids_dict[i] = (lexical_entry_client_id, lexical_entry_object_id)
DBSession.flush()
ids_dict2 = dict()
for i in range(words_count2):
perspective = DBSession.query(DictionaryPerspective).\
filter_by(client_id=first_perspective_client_id, object_id = second_perspective_object_id).first()
if not perspective:
return {'error': str("No such perspective in the system")}
lexentr = LexicalEntry( client_id=client.id,
parent_object_id=second_perspective_object_id, parent=perspective)
DBSession.add(lexentr)
lexical_entry_client_id = lexentr.client_id
lexical_entry_object_id = lexentr.object_id
ids_dict2[i] = (lexical_entry_client_id, lexical_entry_object_id)
DBSession.flush()
get_sp_ids = sqconn.cursor()
get_sp_ids.execute("select id from dictionary where is_a_regular_form=0")
ids_mapping2 = dict()
i = 0
sp_le_id_dict = {}
for id_cursor in get_sp_ids:
id = id_cursor[0]
sp_le_id_dict[id] = i
client_id = ids_dict2[i][0]
object_id = ids_dict2[i][1]
ids_mapping2[int(id)] = (client_id, object_id)
i += 1
get_fp_ids = sqconn.cursor()
get_fp_ids.execute("select id from dictionary where is_a_regular_form=1")
ids_mapping = dict()
i = 0
fp_le_id_dict = {}
for id_cursor in get_fp_ids:
id = id_cursor[0]
fp_le_id_dict[id] = i
client_id = ids_dict[i][0]
object_id = ids_dict[i][1]
ids_mapping[id] = (client_id, object_id)
i += 1
fp_fields_dict = {}
"""
# FIRST PERSPECTIVE FIELDS CREATION
"""
fp_field_names = ("Word", "Transcription", "Translation", "Sound", "Etymology", "Backref")
fields_list = []
for fieldname in fp_field_names: #
if fieldname == "Backref":
fields_list.append(
{
"client_id": field_ids[fieldname][0],
"object_id": field_ids[fieldname][1],
"link":{
"client_id": second_perspective_client_id,
"object_id": second_perspective_object_id
}
}
)
elif fieldname == "Sound":
fields_list.append(
{
"client_id": field_ids[fieldname][0],
"object_id": field_ids[fieldname][1],
"contains":[{
"client_id": field_ids["Markup"][0],
"object_id": field_ids["Markup"][1]
}
]
}
)
else:
fields_list.append({"client_id": field_ids[fieldname][0], "object_id": field_ids[fieldname][1]})
fp_fields_dict[fieldname] = (field_ids[fieldname][0], field_ids[fieldname][1])
fp_fields_dict["Markup"] = (field_ids["Markup"][0], field_ids["Markup"][1])
update_perspective_fields(fields_list, first_perspective_client_id, first_perspective_object_id, client )
"""
# Creating fields of the second perspective
"""
sp_fields_dict = {}
fields_list = []
sp_field_names = ("Word of Paradigmatic forms", "Transcription of Paradigmatic forms", "Translation of Paradigmatic forms", "Sounds of Paradigmatic forms", "Backref")
for fieldname in sp_field_names: #
if fieldname == "Backref":
fields_list.append(
{
"client_id": field_ids[fieldname][0],
"object_id": field_ids[fieldname][1],
"link":{
"client_id": first_perspective_client_id,
"object_id": first_perspective_object_id
}
}
)
elif fieldname == "Sounds of Paradigmatic forms":
fields_list.append(
{
"client_id": field_ids[fieldname][0],
"object_id": field_ids[fieldname][1],
"contains":[{
"client_id": field_ids["Paradigm Markup"][0],
"object_id": field_ids["Paradigm Markup"][1]
}
]
}
)
else:
fields_list.append({"client_id": field_ids[fieldname][0], "object_id": field_ids[fieldname][1]})
sp_fields_dict[fieldname] = (field_ids[fieldname][0], field_ids[fieldname][1])
sp_fields_dict["Paradigm Markup"] = (field_ids["Paradigm Markup"][0], field_ids["Paradigm Markup"][1])
update_perspective_fields(fields_list, second_perspective_client_id, second_perspective_object_id, client)
columns = ("word", "Transcription", "translation")
# First Perspective entity
sqcursor = sqconn.cursor()
for column in columns:
sqcursor.execute("select id,%s from dictionary where is_a_regular_form=1" % column)
for row in sqcursor:
row_id = int(row[0])
content = row[1]
name = None
if column == "word":
name = "Word"
if column == "Transcription":
name = "Transcription"
if column == "translation":
name = "Translation"
create_entity(ids_dict[fp_le_id_dict[row_id]][0], ids_dict[fp_le_id_dict[row_id]][1], fp_fields_dict[name][0], fp_fields_dict[name][1],
None, client, content, filename=None, storage=storage)
# Second Perspective entity
sqcursor = sqconn.cursor()
for column in columns:
sqcursor.execute("select id,%s from dictionary where is_a_regular_form=0" % column)
for row in sqcursor:
row_id = int(row[0])
content = row[1]
name = None
if column == "word":
name = "Word of Paradigmatic forms"
if column == "Transcription":
name = "Transcription of Paradigmatic forms"
if column == "translation":
name = "Translation of Paradigmatic forms"
create_entity(ids_dict2[sp_le_id_dict[row_id]][0], ids_dict2[sp_le_id_dict[row_id]][1], sp_fields_dict[name][0], sp_fields_dict[name][1],
None, client, content, filename=None, storage=storage)
sqcursor = sqconn.cursor()
sqcursor.execute("select id,regular_form from dictionary where is_a_regular_form=0")
for le_cursor in sqcursor:
fp_id = int(le_cursor[1])
sp_id = int(le_cursor[0])
if fp_id in ids_mapping:
create_entity(ids_dict[fp_le_id_dict[fp_id]][0], ids_dict[fp_le_id_dict[fp_id]][1], fp_fields_dict["Backref"][0], fp_fields_dict["Backref"][1],
None, client, filename=None, link_client_id=ids_dict2[sp_le_id_dict[sp_id]][0], link_object_id=ids_dict2[sp_le_id_dict[sp_id]][1], storage=storage)
create_entity(ids_dict2[sp_le_id_dict[sp_id]][0], ids_dict2[sp_le_id_dict[sp_id]][1], sp_fields_dict["Backref"][0], sp_fields_dict["Backref"][1],
None, client, filename=None, link_client_id=ids_dict[fp_le_id_dict[fp_id]][0], link_object_id=ids_dict[fp_le_id_dict[fp_id]][1], storage=storage)
#DBSession.flush()
# if req.get('is_translatable', None):
# field.is_translatable = bool(req['is_translatable'])
audio_hashes = set()
markup_hashes = set()
DBSession.flush()
"""
Sound and Markup
"""
audio_ids = set()
paradigm_audio_ids = set()
sound_and_markup_word_cursor = sqconn.cursor()
sound_and_markup_word_cursor.execute("""select blobs.id,
blobs.secblob,
blobs.mainblob,
dict_blobs_description.name,
dictionary.id,
dict_blobs_description.type
from blobs, dict_blobs_description, dictionary
where dict_blobs_description.blobid=blobs.id
and dict_blobs_description.wordid=dictionary.id
and dictionary.is_a_regular_form=1;""")
folder_name = "praat_markup"
upload_audio_with_markup(audio_ids, ids_mapping, fp_fields_dict, sound_and_markup_word_cursor, audio_hashes, markup_hashes, folder_name,
user_id, True, client, storage)
upload_audio(audio_ids, ids_mapping, fp_fields_dict, sound_and_markup_word_cursor, audio_hashes, markup_hashes, folder_name,
user_id, True, client, storage)
paradigm_sound_and_markup_cursor = sqconn.cursor()
paradigm_sound_and_markup_cursor.execute("""select blobs.id,
blobs.secblob,
blobs.mainblob,
dict_blobs_description.name,
dictionary.id,
dict_blobs_description.type
from blobs, dict_blobs_description, dictionary
where dict_blobs_description.blobid=blobs.id
and dict_blobs_description.wordid=dictionary.id
and dictionary.is_a_regular_form=0;""")
folder_name = "paradigm_praat_markup"
upload_audio_with_markup(paradigm_audio_ids, ids_mapping2, sp_fields_dict, paradigm_sound_and_markup_cursor, audio_hashes, markup_hashes, folder_name,
user_id, True, client, storage)
upload_audio(paradigm_audio_ids, ids_mapping2, sp_fields_dict, paradigm_sound_and_markup_cursor, audio_hashes, markup_hashes, folder_name,
user_id, True, client, storage)
"""
Etimology_tag
"""
etymology_cursor = sqconn.cursor()
etymology_cursor.execute("""select id, etimology_tag
FROM dictionary
WHERE etimology_tag NOT NULL
and dictionary.is_a_regular_form=1; """)
for cursor in etymology_cursor:
id = int(cursor[0])
client_id = ids_mapping[id][0]
object_id = ids_mapping[id][1]
item = {"entity_type": "Etymology", "tag": cursor[1],
"field_client_id": field_ids["Etymology"][0],
"field_object_id": field_ids["Etymology"][1],
"connections": [{"client_id": client_id, "object_id": object_id}]}
create_group_entity(item, client, user)
# status = session.post(connect_url, json=item)
# log.debug(status.text)
dictionary = {}
return dictionary
def setUp(self):
self.config = testing.setUp()
self.config.testing_securitypolicy(userid='1', permissive=True)
import webtest
from pyramid import paster
from sqlalchemy import create_engine
engine = create_engine('sqlite://')
myapp = paster.get_app('testing.ini')
self.app = webtest.TestApp(myapp)
from lingvodoc.models import (Base, User, Client, Passhash, Locale,
UserEntitiesTranslationString, Language)
DBSession.configure(bind=engine)
Base.metadata.create_all(engine)
with transaction.manager:
ru_locale = Locale(id=1, shortcut="ru", intl_name="Русский")
DBSession.add(ru_locale)
en_locale = Locale(id=2, shortcut="en", intl_name="English")
DBSession.add(en_locale)
DBSession.flush()
new_user = User(id=1, login='******', default_locale_id=1)
new_pass = Passhash(password='******')
DBSession.add(new_pass)
new_user.password = new_pass
DBSession.add(new_user)
new_client = Client(id=1, user=new_user)
DBSession.add(new_client)
new_uets = UserEntitiesTranslationString(
client_id=1,
object_id=1,
locale_id=2,
translation_string='imastring')
DBSession.add(new_uets)
new_lang = Language(client_id=1,
object_id=1,
translation_string='testy')
DBSession.add(new_lang)
def signup_post(request): # tested
try:
req = request.json_body
login = req['login']
name = req['name']
email = req['email']
password = req['password']
day = req.get('day')
month = req.get('month')
year = req.get('year')
if day is None or month is None or year is None:
request.response.status = HTTPBadRequest.code
return {'Error': "day, month or year of the birth is missing"}
# birthday = datetime.datetime.strptime(day + month + year, "%d%m%Y").date()
try:
day = int(day)
month = int(month)
year = int(year)
birthday = datetime.date(year, month, day)
except ValueError:
request.response.status = HTTPBadRequest.code
return {'Error': "Invalid birthday"}
if DBSession.query(User).filter_by(login=login).first():
raise CommonException(
"The user with this login is already registered")
if DBSession.query(Email).filter_by(email=email).first():
raise CommonException(
"The user with this email is already registered")
new_user = User(login=login,
name=name,
created_at=datetime.datetime.utcnow(),
intl_name=login,
birthday=birthday,
is_active=True)
pwd = Passhash(password=password)
email = Email(email=email)
new_user.password = pwd
new_user.email = email
DBSession.add(new_user)
basegroups = []
basegroups += [
DBSession.query(BaseGroup).filter_by(
name="Can create dictionaries").first()
]
basegroups += [
DBSession.query(BaseGroup).filter_by(
name="Can create languages").first()
]
basegroups += [
DBSession.query(BaseGroup).filter_by(
name="Can create organizations").first()
]
basegroups += [
DBSession.query(BaseGroup).filter_by(
name="Can create translation strings").first()
]
groups = []
for base in basegroups:
groups += [
DBSession.query(Group).filter_by(
subject_override=True, base_group_id=base.id).first()
]
for group in groups:
add_user_to_group(new_user, group)
DBSession.flush()
return {}
except KeyError as e:
request.response.status = HTTPBadRequest.code
return {'status': request.response.status, 'error': str(e)}
except CommonException as e:
request.response.status = HTTPConflict.code
return {'status': request.response.status, 'error': str(e)}
except ValueError as e:
request.response.status = HTTPConflict.code
return {'status': request.response.status, 'error': str(e)}
def create_entity(le_client_id,
le_object_id,
field_client_id,
field_object_id,
additional_metadata,
client,
content=None,
filename=None,
link_client_id=None,
link_object_id=None,
folder_name=None,
up_lvl=None,
locale_id=2,
storage=None):
# return ##DBG
log = logging.getLogger(__name__)
log.setLevel(logging.DEBUG)
parent = DBSession.query(LexicalEntry).filter_by(
client_id=le_client_id, object_id=le_object_id).first()
if not parent:
return {'error': str("No such lexical entry in the system")}
upper_level = None
tr_atom = DBSession.query(TranslationAtom).join(
TranslationGist,
and_(TranslationAtom.locale_id == 2,
TranslationAtom.parent_client_id == TranslationGist.client_id,
TranslationAtom.parent_object_id ==
TranslationGist.object_id)).join(
Field,
and_(
TranslationGist.client_id ==
Field.data_type_translation_gist_client_id,
TranslationGist.object_id ==
Field.data_type_translation_gist_object_id)).filter(
Field.client_id == field_client_id,
Field.object_id == field_object_id).first()
data_type = tr_atom.content.lower()
if up_lvl:
upper_level = DBSession.query(Entity).filter_by(
client_id=up_lvl[0], object_id=up_lvl[1]).first()
entity = Entity(
client_id=client.id,
field_client_id=field_client_id,
field_object_id=field_object_id,
locale_id=locale_id,
###additional_metadata=additional_metadata,
parent=parent)
if upper_level:
entity.upper_level = upper_level
hash = None
real_location = None
url = None
if data_type == 'image' or data_type == 'sound' or 'markup' in data_type:
##entity.data_type = data_type
real_location, url = create_object(content, entity, data_type,
filename, folder_name, storage)
entity.content = url
old_meta = entity.additional_metadata
need_hash = True
if old_meta:
new_meta = old_meta #json.loads(old_meta)
if new_meta.get('hash'):
need_hash = False
if need_hash:
hash = hashlib.sha224(
base64.urlsafe_b64decode(content)).hexdigest()
hash_dict = {'hash': hash}
if old_meta:
new_meta = old_meta #json.loads(old_meta)
new_meta.update(hash_dict)
else:
new_meta = hash_dict
entity.additional_metadata = new_meta #json.dumps(new_meta)
old_meta = entity.additional_metadata
if data_type == "markup":
data_type_dict = {"data_type": "praat markup"}
if old_meta:
new_meta = old_meta #json.loads(old_meta)
new_meta.update(data_type_dict)
else:
new_meta = data_type_dict
entity.additional_metadata = new_meta #json.dumps(new_meta)
elif data_type == 'link':
try:
entity.link_client_id = link_client_id
entity.link_object_id = link_object_id
except (KeyError, TypeError):
return {
'Error':
"The field is of link type. You should provide client_id and object id in the content"
}
else:
entity.content = content
entity.publishingentity.accepted = True
DBSession.add(entity)
#log.debug(filename)
return (entity.client_id, entity.object_id)
def edit_user_info(request): # TODO: test
from passlib.hash import bcrypt
response = dict()
req = request.json_body
client_id = req.get('client_id')
user_id = req.get('user_id')
user = None
if client_id:
client = DBSession.query(Client).filter_by(id=client_id).first()
if not client:
request.response.status = HTTPNotFound.code
return {'error': str("No such client in the system")}
user = DBSession.query(User).filter_by(id=client.user_id).first()
user_id = client.user_id
if not user:
request.response.status = HTTPNotFound.code
return {'error': str("No such user in the system")}
else:
user = DBSession.query(User).filter_by(id=user_id).first()
if not user:
request.response.status = HTTPNotFound.code
return {'error': str("No such user in the system")}
new_password = req.get('new_password')
old_password = req.get('old_password')
if new_password:
if not old_password:
request.response.status = HTTPBadRequest.code
return {'error': str("Need old password to confirm")}
old_hash = DBSession.query(Passhash).filter_by(user_id=user_id).first()
if old_hash:
if not user.check_password(old_password):
request.response.status = HTTPBadRequest.code
return {'error': str("Wrong password")}
else:
old_hash.hash = bcrypt.encrypt(new_password)
else:
request.response.status = HTTPInternalServerError.code
return {'error': str("User has no password")}
name = req.get('name')
if name:
user.name = name
default_locale_id = req.get('default_locale_id')
if default_locale_id:
user.default_locale_id = default_locale_id
birthday = req.get('birthday')
if birthday:
try:
year, month, day = birthday.split('-')
user.birthday = datetime.date(int(year), int(month), int(day))
except ValueError:
request.response.status = HTTPBadRequest.code
return {'Error': "Invalid birthday"}
email = req.get('email')
if email:
if user.email:
user.email.email = email
else:
new_email = Email(user=user, email=email)
DBSession.add(new_email)
DBSession.flush()
about = req.get('about')
if about:
meta = dict()
if user.additional_metadata:
meta = user.additional_metadata
meta['about'] = about
user.additional_metadata = meta
# response['is_active']=str(user.is_active)
request.response.status = HTTPOk.code
return response
def convert_db_new( blob_client_id, blob_object_id, language_client_id, language_object_id, user_id, gist_client_id, gist_object_id, storage,
locale_id=2):
log = logging.getLogger(__name__)
log.setLevel(logging.DEBUG)
time.sleep(4)
field_ids = {}
with transaction.manager:
blob = DBSession.query(UserBlobs).filter_by(client_id=blob_client_id, object_id=blob_object_id).first()
# DBSession.flush()
filename = blob.real_storage_path
log.debug("user_id: %s" % user_id)
log.debug("Starting convert_one")
log.debug("Creating session")
sqconn = sqlite3.connect(filename)
log.debug("Connected to sqlite3 database")
client = DBSession.query(Client).filter_by(id=user_id).first()
if not client:
raise KeyError("Invalid client id (not registered on server). Try to logout and then login.",
user_id)
user = DBSession.query(User).filter_by(id=client.user_id).first()
if not user:
log.debug("ERROR")
all_fieldnames = ("Markup",
"Paradigm Markup",
"Word",
"Transcription",
"Translation",
"Sound",
"Etymology",
"Backref",
"Word of Paradigmatic forms",
"Transcription of Paradigmatic forms",
"Translation of Paradigmatic forms",
"Sounds of Paradigmatic forms"
)
for name in all_fieldnames:
data_type_query = DBSession.query(Field) \
.join(TranslationGist,
and_(Field.translation_gist_object_id == TranslationGist.object_id,
Field.translation_gist_client_id == TranslationGist.client_id))\
.join(TranslationGist.translationatom)
field = data_type_query.filter(TranslationAtom.locale_id == 2,
TranslationAtom.content == name).one() # todo: a way to find this fields if wwe cannot use one
field_ids[name] = (field.client_id, field.object_id)
DBSession.flush()
dict_attributes = get_dict_attributes(sqconn)
"""
dict_attributes = get_dict_attributes(sqconn)
translationgist = TranslationGist(client_id=user_id, type="Dictionary")
DBSession.add(translationgist)
DBSession.flush()
gist_client_id = translationgist.client_id
gist_object_id = translationgist.object_id
"""
parent_client_id = gist_client_id
parent_object_id = gist_object_id
parent = DBSession.query(TranslationGist).filter_by(client_id=parent_client_id, object_id=parent_object_id).first()
"""
translationatom = TranslationAtom(client_id=client.id,
parent=parent,
locale_id=locale_id,
content=dict_attributes["dictionary_name"])
DBSession.add(translationatom)
DBSession.flush()
atom_client_id = translationatom.client_id
atom_object_id = translationatom.object_id
log.debug(dict_attributes["dictionary_name"])
language_client_id = atom_client_id
language_object_id = atom_object_id
"""
lang_parent = DBSession.query(Language).filter_by(client_id=language_client_id, object_id=language_object_id).first()
resp = translation_service_search("WiP")
state_translation_gist_object_id, state_translation_gist_client_id = resp['object_id'], resp['client_id']
dictionary = Dictionary(client_id=user_id,
state_translation_gist_object_id=state_translation_gist_object_id,
state_translation_gist_client_id=state_translation_gist_client_id,
parent=lang_parent,
translation_gist_client_id=gist_client_id,
translation_gist_object_id=gist_object_id
)
#additional_metadata=additional_metadata)
DBSession.add(dictionary)
DBSession.flush()
dictionary_client_id = dictionary.client_id
dictionary_object_id = dictionary.object_id
for base in DBSession.query(BaseGroup).filter_by(dictionary_default=True):
new_group = Group(parent=base,
subject_object_id=dictionary.object_id, subject_client_id=dictionary.client_id)
if user not in new_group.users:
new_group.users.append(user)
DBSession.add(new_group)
DBSession.flush()
"""
# FIRST PERSPECTIVE
"""
resp = translation_service_search_all("Lexical Entries")
persp_translation_gist_client_id, persp_translation_gist_object_id = resp['client_id'], resp['object_id']
parent = DBSession.query(Dictionary).filter_by(client_id=dictionary_client_id, object_id=dictionary_object_id).first()
perspective = DictionaryPerspective(client_id=client.id, ###
state_translation_gist_object_id=state_translation_gist_object_id,
state_translation_gist_client_id=state_translation_gist_client_id,
parent=parent,
import_source="Lingvodoc-0.98",
import_hash=dict_attributes['dialeqt_id'],
# additional_metadata=additional_metadata,
translation_gist_client_id=persp_translation_gist_client_id,
translation_gist_object_id=persp_translation_gist_object_id
)
# if is_template is not None:
# perspective.is_template = is_template
DBSession.add(perspective)
owner_client = DBSession.query(Client).filter_by(id=parent.client_id).first()
owner = owner_client.user
for base in DBSession.query(BaseGroup).filter_by(perspective_default=True):
new_group = Group(parent=base,
subject_object_id=perspective.object_id, subject_client_id=perspective.client_id)
if user not in new_group.users:
new_group.users.append(user)
if owner not in new_group.users:
new_group.users.append(owner)
DBSession.add(new_group)
DBSession.flush()
first_perspective_client_id = perspective.client_id
first_perspective_object_id = perspective.object_id
"""
# SECOND PERSPECTIVE
"""
resp = translation_service_search_all("Paradigms")
persp_translation_gist_client_id, persp_translation_gist_object_id = resp['client_id'], resp['object_id']
parent = DBSession.query(Dictionary).filter_by(client_id=dictionary_client_id, object_id=dictionary_object_id).first()
if not parent:
return {'error': str("No such dictionary in the system")}
perspective = DictionaryPerspective(client_id=client.id, ### variables['auth']
state_translation_gist_object_id=state_translation_gist_object_id,
state_translation_gist_client_id=state_translation_gist_client_id,
parent=parent,
import_source="Lingvodoc-0.98",
import_hash=dict_attributes['dialeqt_id'],
# additional_metadata=additional_metadata,
translation_gist_client_id=persp_translation_gist_client_id,
translation_gist_object_id=persp_translation_gist_object_id
)
# if is_template is not None:
# perspective.is_template = is_template
DBSession.add(perspective)
owner_client = DBSession.query(Client).filter_by(id=parent.client_id).first()
owner = owner_client.user
for base in DBSession.query(BaseGroup).filter_by(perspective_default=True):
new_group = Group(parent=base,
subject_object_id=perspective.object_id, subject_client_id=perspective.client_id)
if user not in new_group.users:
new_group.users.append(user)
if owner not in new_group.users:
new_group.users.append(owner)
DBSession.add(new_group)
second_perspective_client_id = perspective.client_id
second_perspective_object_id = perspective.object_id
get_fp_ids = sqconn.cursor()
get_fp_ids.execute("select id from dictionary where is_a_regular_form=1")
count_cursor = sqconn.cursor()
count_cursor.execute("select count(*) from dictionary where is_a_regular_form=1")
words_count = count_cursor.fetchone()[0]
count_cursor2 = sqconn.cursor()
count_cursor2.execute("select count(*) from dictionary where is_a_regular_form=0")
words_count2 = count_cursor2.fetchone()[0]
ids_dict = dict()
for i in range(words_count):
perspective = DBSession.query(DictionaryPerspective).\
filter_by(client_id=first_perspective_client_id, object_id = first_perspective_object_id).first()
if not perspective:
return {'error': str("No such perspective in the system")}
lexentr = LexicalEntry( client_id=client.id,
parent_object_id=first_perspective_object_id, parent=perspective)
DBSession.add(lexentr)
lexical_entry_client_id = lexentr.client_id
lexical_entry_object_id = lexentr.object_id
ids_dict[i] = (lexical_entry_client_id, lexical_entry_object_id)
DBSession.flush()
ids_dict2 = dict()
for i in range(words_count2):
perspective = DBSession.query(DictionaryPerspective).\
filter_by(client_id=first_perspective_client_id, object_id = second_perspective_object_id).first()
if not perspective:
return {'error': str("No such perspective in the system")}
lexentr = LexicalEntry( client_id=client.id,
parent_object_id=second_perspective_object_id, parent=perspective)
DBSession.add(lexentr)
lexical_entry_client_id = lexentr.client_id
lexical_entry_object_id = lexentr.object_id
ids_dict2[i] = (lexical_entry_client_id, lexical_entry_object_id)
DBSession.flush()
get_sp_ids = sqconn.cursor()
get_sp_ids.execute("select id from dictionary where is_a_regular_form=0")
ids_mapping2 = dict()
i = 0
sp_le_id_dict = {}
for id_cursor in get_sp_ids:
id = id_cursor[0]
sp_le_id_dict[id] = i
client_id = ids_dict2[i][0]
object_id = ids_dict2[i][1]
ids_mapping2[int(id)] = (client_id, object_id)
i += 1
get_fp_ids = sqconn.cursor()
get_fp_ids.execute("select id from dictionary where is_a_regular_form=1")
ids_mapping = dict()
i = 0
fp_le_id_dict = {}
for id_cursor in get_fp_ids:
id = id_cursor[0]
fp_le_id_dict[id] = i
client_id = ids_dict[i][0]
object_id = ids_dict[i][1]
ids_mapping[id] = (client_id, object_id)
i += 1
fp_fields_dict = {}
"""
# FIRST PERSPECTIVE FIELDS CREATION
"""
fp_field_names = ("Word", "Transcription", "Translation", "Sound", "Etymology", "Backref")
fields_list = []
for fieldname in fp_field_names: #
if fieldname == "Backref":
fields_list.append(
{
"client_id": field_ids[fieldname][0],
"object_id": field_ids[fieldname][1],
"link":{
"client_id": second_perspective_client_id,
"object_id": second_perspective_object_id
}
}
)
elif fieldname == "Sound":
fields_list.append(
{
"client_id": field_ids[fieldname][0],
"object_id": field_ids[fieldname][1],
"contains":[{
"client_id": field_ids["Markup"][0],
"object_id": field_ids["Markup"][1]
}
]
}
)
else:
fields_list.append({"client_id": field_ids[fieldname][0], "object_id": field_ids[fieldname][1]})
fp_fields_dict[fieldname] = (field_ids[fieldname][0], field_ids[fieldname][1])
fp_fields_dict["Markup"] = (field_ids["Markup"][0], field_ids["Markup"][1])
update_perspective_fields(fields_list, first_perspective_client_id, first_perspective_object_id, client )
"""
# Creating fields of the second perspective
"""
sp_fields_dict = {}
fields_list = []
sp_field_names = ("Word of Paradigmatic forms", "Transcription of Paradigmatic forms", "Translation of Paradigmatic forms", "Sounds of Paradigmatic forms", "Backref")
for fieldname in sp_field_names: #
if fieldname == "Backref":
fields_list.append(
{
"client_id": field_ids[fieldname][0],
"object_id": field_ids[fieldname][1],
"link":{
"client_id": first_perspective_client_id,
"object_id": first_perspective_object_id
}
}
)
elif fieldname == "Sounds of Paradigmatic forms":
fields_list.append(
{
"client_id": field_ids[fieldname][0],
"object_id": field_ids[fieldname][1],
"contains":[{
"client_id": field_ids["Paradigm Markup"][0],
"object_id": field_ids["Paradigm Markup"][1]
}
]
}
)
else:
fields_list.append({"client_id": field_ids[fieldname][0], "object_id": field_ids[fieldname][1]})
sp_fields_dict[fieldname] = (field_ids[fieldname][0], field_ids[fieldname][1])
sp_fields_dict["Paradigm Markup"] = (field_ids["Paradigm Markup"][0], field_ids["Paradigm Markup"][1])
update_perspective_fields(fields_list, second_perspective_client_id, second_perspective_object_id, client)
columns = ("word", "Transcription", "translation")
# First Perspective entity
sqcursor = sqconn.cursor()
for column in columns:
sqcursor.execute("select id,%s from dictionary where is_a_regular_form=1" % column)
for row in sqcursor:
row_id = int(row[0])
content = row[1]
name = None
if column == "word":
name = "Word"
if column == "Transcription":
name = "Transcription"
if column == "translation":
name = "Translation"
create_entity(ids_dict[fp_le_id_dict[row_id]][0], ids_dict[fp_le_id_dict[row_id]][1], fp_fields_dict[name][0], fp_fields_dict[name][1],
None, client, content, filename=None, storage=storage)
# Second Perspective entity
sqcursor = sqconn.cursor()
for column in columns:
sqcursor.execute("select id,%s from dictionary where is_a_regular_form=0" % column)
for row in sqcursor:
row_id = int(row[0])
content = row[1]
name = None
if column == "word":
name = "Word of Paradigmatic forms"
if column == "Transcription":
name = "Transcription of Paradigmatic forms"
if column == "translation":
name = "Translation of Paradigmatic forms"
create_entity(ids_dict2[sp_le_id_dict[row_id]][0], ids_dict2[sp_le_id_dict[row_id]][1], sp_fields_dict[name][0], sp_fields_dict[name][1],
None, client, content, filename=None, storage=storage)
sqcursor = sqconn.cursor()
sqcursor.execute("select id,regular_form from dictionary where is_a_regular_form=0")
for le_cursor in sqcursor:
fp_id = int(le_cursor[1])
sp_id = int(le_cursor[0])
if fp_id in ids_mapping:
create_entity(ids_dict[fp_le_id_dict[fp_id]][0], ids_dict[fp_le_id_dict[fp_id]][1], fp_fields_dict["Backref"][0], fp_fields_dict["Backref"][1],
None, client, filename=None, link_client_id=ids_dict2[sp_le_id_dict[sp_id]][0], link_object_id=ids_dict2[sp_le_id_dict[sp_id]][1], storage=storage)
create_entity(ids_dict2[sp_le_id_dict[sp_id]][0], ids_dict2[sp_le_id_dict[sp_id]][1], sp_fields_dict["Backref"][0], sp_fields_dict["Backref"][1],
None, client, filename=None, link_client_id=ids_dict[fp_le_id_dict[fp_id]][0], link_object_id=ids_dict[fp_le_id_dict[fp_id]][1], storage=storage)
#DBSession.flush()
# if req.get('is_translatable', None):
# field.is_translatable = bool(req['is_translatable'])
audio_hashes = set()
markup_hashes = set()
DBSession.flush()
"""
Sound and Markup
"""
audio_ids = set()
paradigm_audio_ids = set()
sound_and_markup_word_cursor = sqconn.cursor()
sound_and_markup_word_cursor.execute("""select blobs.id,
blobs.secblob,
blobs.mainblob,
dict_blobs_description.name,
dictionary.id,
dict_blobs_description.type
from blobs, dict_blobs_description, dictionary
where dict_blobs_description.blobid=blobs.id
and dict_blobs_description.wordid=dictionary.id
and dictionary.is_a_regular_form=1;""")
folder_name = "praat_markup"
upload_audio_with_markup(audio_ids, ids_mapping, fp_fields_dict, sound_and_markup_word_cursor, audio_hashes, markup_hashes, folder_name,
user_id, True, client, storage)
sound_and_markup_word_cursor = sqconn.cursor()
sound_and_markup_word_cursor.execute("""select blobs.id,
blobs.secblob,
blobs.mainblob,
dict_blobs_description.name,
dictionary.id,
dict_blobs_description.type
from blobs, dict_blobs_description, dictionary
where dict_blobs_description.blobid=blobs.id
and dict_blobs_description.wordid=dictionary.id
and dictionary.is_a_regular_form=1;""")
upload_audio(audio_ids, ids_mapping, fp_fields_dict, sound_and_markup_word_cursor, audio_hashes, markup_hashes, folder_name,
user_id, True, client, storage)
paradigm_sound_and_markup_cursor = sqconn.cursor()
paradigm_sound_and_markup_cursor.execute("""select blobs.id,
blobs.secblob,
blobs.mainblob,
dict_blobs_description.name,
dictionary.id,
dict_blobs_description.type
from blobs, dict_blobs_description, dictionary
where dict_blobs_description.blobid=blobs.id
and dict_blobs_description.wordid=dictionary.id
and dictionary.is_a_regular_form=0;""")
folder_name = "paradigm_praat_markup"
upload_audio_with_markup(paradigm_audio_ids, ids_mapping2, sp_fields_dict, paradigm_sound_and_markup_cursor, audio_hashes, markup_hashes, folder_name,
user_id, True, client, storage)
paradigm_sound_and_markup_cursor = sqconn.cursor()
paradigm_sound_and_markup_cursor.execute("""select blobs.id,
blobs.secblob,
blobs.mainblob,
dict_blobs_description.name,
dictionary.id,
dict_blobs_description.type
from blobs, dict_blobs_description, dictionary
where dict_blobs_description.blobid=blobs.id
and dict_blobs_description.wordid=dictionary.id
and dictionary.is_a_regular_form=0;""")
upload_audio(paradigm_audio_ids, ids_mapping2, sp_fields_dict, paradigm_sound_and_markup_cursor, audio_hashes, markup_hashes, folder_name,
user_id, True, client, storage)
"""
Etimology_tag
"""
etymology_cursor = sqconn.cursor()
etymology_cursor.execute("""select id, etimology_tag
FROM dictionary
WHERE etimology_tag NOT NULL
and dictionary.is_a_regular_form=1; """)
for cursor in etymology_cursor:
id = int(cursor[0])
client_id = ids_mapping[id][0]
object_id = ids_mapping[id][1]
item = {"entity_type": "Etymology", "tag": cursor[1],
"field_client_id": field_ids["Etymology"][0],
"field_object_id": field_ids["Etymology"][1],
"connections": [{"client_id": client_id, "object_id": object_id}]}
create_group_entity(item, client, user)
# status = session.post(connect_url, json=item)
# log.debug(status.text)
dictionary = {}
return dictionary
def merge_dictionaries(request): # TODO: test
try:
req = request.json_body
variables = {'auth': request.authenticated_userid}
client = DBSession.query(Client).filter_by(id=variables['auth']).first()
if not client:
raise KeyError("Invalid client id (not registered on server). Try to logout and then login.",
variables['auth'])
user = DBSession.query(User).filter_by(id=client.user_id).first()
if not user:
raise CommonException("This client id is orphaned. Try to logout and then login once more.")
client_id = req.get('client_id')
object_id = req.get('object_id')
parent_object_id = req['language_object_id']
parent_client_id = req['language_client_id']
translation_string = req['translation_string']
translation = translation_string
if 'translation' in req:
translation = req['translation']
dictionaries = req['dictionaries']
if len(dictionaries) != 2:
raise KeyError("Wrong number of dictionaries to merge.",
len(dictionaries))
new_dicts = []
for dicti in dictionaries:
diction = DBSession.query(Dictionary).filter_by(client_id=dicti['client_id'], object_id=dicti['object_id']).first()
if not diction:
raise KeyError("Dictionary do not exist in the system")
if parent_client_id != diction.parent_client_id or parent_object_id != diction.parent_object_id:
raise KeyError("Both dictionaries should have same language.")
new_dicts += [diction]
dictionaries = new_dicts
base = DBSession.query(BaseGroup).filter_by(subject='merge', action='create').first()
override = DBSession.query(Group).filter_by(base_group_id=base.id, subject_override = True).first()
if user not in override.users:
grps = []
for dict in dictionaries:
gr = DBSession.query(Group).filter_by(base_group_id=base.id,
subject_client_id=dict.client_id,
subject_object_id=dict.object_id).first()
grps += [gr]
if client_id and object_id:
gr = DBSession.query(Group).filter_by(base_group_id=base.id,
subject_client_id=client_id,
subject_object_id=object_id).first()
grps += [gr]
for gr in grps:
if user not in gr.users:
raise KeyError("Not enough permission to do that")
if not client_id or not object_id:
subreq = Request.blank('/dictionary')
subreq.method = 'POST'
subreq.json = {'parent_object_id': parent_object_id, 'parent_client_id': parent_client_id,
'translation_string': translation_string, 'translation': translation}
headers = {'Cookie':request.headers['Cookie']}
subreq.headers = headers
response = request.invoke_subrequest(subreq)
client_id = response.json['client_id']
object_id = response.json['object_id']
new_dict = DBSession.query(Dictionary).filter_by(client_id=client_id, object_id=object_id).first()
perspectives = []
for dicti in dictionaries:
for entry in dicti.dictionaryperspective:
perspectives += [entry]
for entry in perspectives:
if entry in dicti.dictionaryperspective:
dicti.dictionaryperspective.remove(entry)
new_dict.dictionaryperspective.append(entry)
cli_id = dicti.client_id
obj_id = dicti.object_id
if (cli_id == client_id) and (obj_id == object_id):
continue
bases = DBSession.query(BaseGroup).filter_by(dictionary_default=True)
groups = []
for base in bases:
group = DBSession.query(Group).filter_by(base_group_id=base.id,
subject_object_id=obj_id,
subject_client_id=cli_id).first()
groups += [group]
for group in groups:
base = group.parent
existing = DBSession.query(Group).filter_by(parent = base,
subject_object_id=object_id,
subject_client_id=client_id).first()
if existing:
users = []
for user in group.users:
users += [user]
for user in users:
if user in group.users:
group.users.remove(user)
if not user in existing.users:
existing.users.append(user)
else:
new_group = Group(base_group_id=group.base_group_id,
subject_object_id=client_id,
subject_client_id=object_id)
DBSession.add(new_group)
users = []
for user in group.users:
users += [user]
for user in users:
if user in group.users:
group.users.remove(user)
if not user in new_group.users:
new_group.users.append(user)
group.marked_for_deletion = True
dicti.marked_for_deletion = True
request.response.status = HTTPOk.code
return {'object_id': object_id,
'client_id': client_id}
except KeyError as e:
request.response.status = HTTPBadRequest.code
return {'error': str(e)}
except IntegrityError as e:
request.response.status = HTTPInternalServerError.code
return {'error': str(e)}
except CommonException as e:
request.response.status = HTTPConflict.code
return {'error': str(e)}
def merge_perspectives_api(request): # TODO: test
try:
req = request.json_body
variables = {'auth': request.authenticated_userid}
client = DBSession.query(Client).filter_by(id=variables['auth']).first()
if not client:
raise KeyError("Invalid client id (not registered on server). Try to logout and then login.",
variables['auth'])
user = DBSession.query(User).filter_by(id=client.user_id).first()
if not user:
raise CommonException("This client id is orphaned. Try to logout and then login once more.")
client_id = req.get('client_id')
object_id = req.get('object_id')
dictionary_client_id = req['dictionary_client_id']
dictionary_object_id = req['dictionary_object_id']
translation_string = req['translation_string']
translation = translation_string
if 'translation' in req:
translation = req['translation']
persps = req['perspectives']
if len(persps) != 2:
raise KeyError("Wrong number of perspectives to merge.",
len(persps))
for persp in persps:
perspe = DBSession.query(DictionaryPerspective).filter_by(client_id=persp['client_id'],
object_id=persp['object_id']).first()
if not perspe:
raise KeyError("Perspective do not exist in the system")
if dictionary_client_id != perspe.parent_client_id or dictionary_object_id != perspe.parent_object_id:
raise KeyError("Both perspective should from same dictionary.")
base = DBSession.query(BaseGroup).filter_by(subject='merge', action='create').first()
override = DBSession.query(Group).filter_by(base_group_id=base.id, subject_override = True).first()
if user not in override.users:
group = DBSession.query(Group).filter_by(base_group_id=base.id,
subject_client_id=dictionary_client_id,
subject_object_id=dictionary_object_id).first()
if user not in group.users:
raise KeyError("Not enough permission to do that")
if client_id and object_id:
gr = DBSession.query(Group).filter_by(base_group_id=base.id,
subject_client_id=client_id,
subject_object_id=object_id).first()
if user not in gr.users:
raise KeyError("Not enough permission to do that")
if not client_id and not object_id:
subreq = Request.blank('/dictionary/%s/%s/perspective' % (dictionary_client_id, dictionary_object_id))
subreq.method = 'POST'
subreq.json = {'translation_string': translation_string, 'translation': translation}
headers = {'Cookie':request.headers['Cookie']}
subreq.headers = headers
response = request.invoke_subrequest(subreq)
client_id = response.json['client_id']
object_id = response.json['object_id']
new_persp = DBSession.query(DictionaryPerspective).filter_by(client_id=client_id, object_id=object_id).first()
fields = []
for persp in persps:
for entry in persp['fields']:
field = dict(entry)
new_type = field.pop('new_type_name', None)
if new_type:
field['entity_type'] = new_type
field['entity_type_translation'] = new_type
if not field in fields:
entity_type_translation = field['entity_type_translation']
add_need = True
for fi in fields:
if fi['entity_type_translation'] == entity_type_translation:
add_need = False
break
if add_need:
fields.append(field)
subreq = Request.blank('/dictionary/%s/%s/perspective/%s/%s/fields' %
(dictionary_client_id,
dictionary_object_id,
client_id,
object_id))
subreq.method = 'POST'
subreq.json = {'fields': fields}
headers = {'Cookie':request.headers['Cookie']}
subreq.headers = headers
response = request.invoke_subrequest(subreq)
for persp in persps:
obj_id = persp['object_id']
cli_id = persp['client_id']
if (cli_id == client_id) and (obj_id == object_id):
continue
parent = DBSession.query(DictionaryPerspective).filter_by(client_id=cli_id, object_id=obj_id).first()
lexes = DBSession.query(LexicalEntry).filter_by(parent_client_id=cli_id, parent_object_id=obj_id).all()
for lex in lexes:
metadata = dict()
if lex.additional_metadata:
metadata = lex.additional_metadata
metadata['came_from'] = {'client_id': lex.parent_client_id, 'object_id': lex.parent_object_id}
lex.additional_metadata = metadata
lex.parent = new_persp
DBSession.flush()
for ent in lex.leveloneentity:
for field in persp['fields']:
if ent.entity_type == field['entity_type']:
if 'new_type_name' in field:
ent.entity_type = field['new_type_name']
bases = DBSession.query(BaseGroup).filter_by(perspective_default=True)
groups = []
for base in bases:
group = DBSession.query(Group).filter_by(base_group_id=base.id,
subject_object_id=obj_id,
subject_client_id=cli_id).first()
if group:
groups += [group]
for group in groups:
base = group.parent
existing = DBSession.query(Group).filter_by(parent = base,
subject_object_id=object_id,
subject_client_id=client_id).first()
if existing:
users = []
for user in group.users:
users += [user]
for user in users:
if user in group.users:
group.users.remove(user)
if not user in existing.users:
existing.users.append(user)
else:
new_group = Group(base_group_id=group.base_group_id,
subject_object_id=client_id,
subject_client_id=object_id)
DBSession.add(new_group)
users = []
for user in group.users:
users += [user]
for user in users:
if user in group.users:
group.users.remove(user)
if not user in new_group.users:
new_group.users.append(user)
group.marked_for_deletion = True
parent.marked_for_deletion = True
new_persp.marked_for_deletion = False # TODO: check where it is deleted
request.response.status = HTTPOk.code
return {'object_id': object_id,
'client_id': client_id}
except KeyError as e:
request.response.status = HTTPBadRequest.code
return {'error': str(e)}
except IntegrityError as e:
request.response.status = HTTPInternalServerError.code
return {'error': str(e)}
except CommonException as e:
request.response.status = HTTPConflict.code
return {'error': str(e)}
def convert_five_tiers(
dictionary_client_id,
dictionary_object_id,
user_id,
origin_client_id,
origin_object_id,
sqlalchemy_url,
storage,
eaf_url,
sound_url=None
):
log = logging.getLogger(__name__)
log.setLevel(logging.DEBUG)
no_sound = True
if sound_url:
no_sound = False
with warnings.catch_warnings():
warnings.filterwarnings('error')
try:
from pydub import AudioSegment
except Warning as e:
no_sound = True
if not no_sound:
with tempfile.NamedTemporaryFile() as temp:
try:
sound_file = request.urlopen(sound_url)
except HTTPError as e:
return {'error': str(e.read().decode("utf8", 'ignore'))}
with open(temp.name,'wb') as output:
output.write(sound_file.read())
full_audio = AudioSegment.from_wav(temp.name)
temp.flush()
field_ids = {}
with transaction.manager:
client = DBSession.query(Client).filter_by(id=user_id).first()
if not client:
raise KeyError("Invalid client id (not registered on server). Try to logout and then login.",
user_id)
user = DBSession.query(User).filter_by(id=client.user_id).first()
all_fieldnames = ("Markup",
"Paradigm Markup",
"Word",
"Transcription",
"Translation",
"Sound",
"Etymology",
"Backref",
"Word of Paradigmatic forms",
"Transcription of Paradigmatic forms",
"Translation of Paradigmatic forms",
"Sounds of Paradigmatic forms"
)
for name in all_fieldnames:
data_type_query = DBSession.query(Field) \
.join(TranslationGist,
and_(Field.translation_gist_object_id == TranslationGist.object_id,
Field.translation_gist_client_id == TranslationGist.client_id))\
.join(TranslationGist.translationatom)
field = data_type_query.filter(TranslationAtom.locale_id == 2,
TranslationAtom.content == name).one()
field_ids[name] = (field.client_id, field.object_id)
fp_fields = ("Word", "Transcription", "Translation", "Sound", "Markup", "Etymology", "Backref")
sp_fields = ("Word of Paradigmatic forms",
"Transcription of Paradigmatic forms",
"Translation of Paradigmatic forms",
"Sounds of Paradigmatic forms",
"Paradigm Markup",
"Backref")
fp_structure = set([field_ids[x] for x in fp_fields])
sp_structure = set([field_ids[x] for x in sp_fields])
DBSession.flush()
resp = translation_service_search("WiP")
state_translation_gist_object_id, state_translation_gist_client_id = resp['object_id'], resp['client_id']
for base in DBSession.query(BaseGroup).filter_by(dictionary_default=True):
new_group = Group(parent=base,
subject_object_id=dictionary_object_id, subject_client_id=dictionary_client_id)
if user not in new_group.users:
new_group.users.append(user)
DBSession.add(new_group)
DBSession.flush()
origin_metadata= {"origin_client_id": origin_client_id,
"origin_object_id": origin_object_id
}
parent = DBSession.query(Dictionary).filter_by(client_id=dictionary_client_id,
object_id=dictionary_object_id).first()
if not parent:
return {'error': str("No such dictionary in the system")}
first_perspective = None
second_perspective = None
for perspective in DBSession.query(DictionaryPerspective).filter_by(parent=parent, marked_for_deletion=False):
structure = set()
fields = DBSession.query(DictionaryPerspectiveToField)\
.filter_by(parent=perspective)\
.all()
DBSession.flush()
for p_to_field in fields:
structure.add((p_to_field.field_client_id, p_to_field.field_object_id))
if structure == fp_structure:
first_perspective = perspective
elif structure == sp_structure:
second_perspective = perspective
structure.clear()
lexes = []
if first_perspective:
lexes = DBSession.query(DictionaryPerspective, LexicalEntry, Entity)\
.filter(and_(DictionaryPerspective.object_id==first_perspective.object_id,
DictionaryPerspective.client_id==first_perspective.client_id))\
.join(LexicalEntry, and_( LexicalEntry.parent_object_id==DictionaryPerspective.object_id,
LexicalEntry.parent_client_id==DictionaryPerspective.client_id))\
.join(Entity, and_(LexicalEntry.object_id==Entity.parent_object_id,
LexicalEntry.client_id==Entity.parent_client_id))
p_lexes = []
if second_perspective:
p_lexes = DBSession.query(DictionaryPerspective, LexicalEntry, Entity)\
.filter(and_(DictionaryPerspective.object_id==second_perspective.object_id,
DictionaryPerspective.client_id==second_perspective.client_id))\
.join(LexicalEntry, and_( LexicalEntry.parent_object_id==DictionaryPerspective.object_id,
LexicalEntry.parent_client_id==DictionaryPerspective.client_id))\
.join(Entity, and_(LexicalEntry.object_id==Entity.parent_object_id,
LexicalEntry.client_id==Entity.parent_client_id))
hashes = [x[2].additional_metadata["hash"] for x in lexes if x[2].field.data_type == "Sound"]
hashes = hashes[:] + [x[2].additional_metadata["hash"] for x in p_lexes if x[2].field.data_type == "Sound"]
links = [((x[2].link.client_id, x[2].link.object_id), (x[1].client_id, x[1].object_id))
for x in lexes if x[2].field.data_type == "Link"]
links = links[:] + [((x[2].link.client_id, x[2].link.object_id), (x[1].client_id, x[1].object_id))
for x in p_lexes if x[2].field.data_type == "Link"]
resp = translation_service_search("WiP")
state_translation_gist_object_id, state_translation_gist_client_id = resp['object_id'], resp['client_id']
for base in DBSession.query(BaseGroup).filter_by(dictionary_default=True):
new_group = Group(parent=base,
subject_object_id=dictionary_object_id, subject_client_id=dictionary_client_id)
if user not in new_group.users:
new_group.users.append(user)
DBSession.add(new_group)
DBSession.flush()
"""
# FIRST PERSPECTIVE
"""
if first_perspective is None:
resp = translation_service_search_all("Lexical Entries")
persp_translation_gist_client_id, persp_translation_gist_object_id = resp['client_id'], resp['object_id']
first_perspective = DictionaryPerspective(client_id=client.id, ###
state_translation_gist_object_id=state_translation_gist_object_id,
state_translation_gist_client_id=state_translation_gist_client_id,
parent=parent,
# import_source=req.get('import_source'),
# import_hash=req.get('import_hash'),
additional_metadata=origin_metadata,
translation_gist_client_id=persp_translation_gist_client_id,
translation_gist_object_id=persp_translation_gist_object_id
)
first_perspective.additional_metadata = origin_metadata
DBSession.add(first_perspective)
owner_client = DBSession.query(Client).filter_by(id=parent.client_id).first()
owner = owner_client.user
for base in DBSession.query(BaseGroup).filter_by(perspective_default=True):
new_group = Group(parent=base,
subject_object_id=first_perspective.object_id,
subject_client_id=first_perspective.client_id)
if user not in new_group.users:
new_group.users.append(user)
if owner not in new_group.users:
new_group.users.append(owner)
DBSession.add(new_group)
DBSession.flush()
first_perspective_client_id = first_perspective.client_id
first_perspective_object_id = first_perspective.object_id
"""
# SECOND PERSPECTIVE
"""
resp = translation_service_search_all("Paradigms")
persp_translation_gist_client_id, persp_translation_gist_object_id = resp['client_id'], resp['object_id']
if second_perspective is None:
second_perspective = DictionaryPerspective(client_id=client.id, ### variables['auth']
state_translation_gist_object_id=state_translation_gist_object_id,
state_translation_gist_client_id=state_translation_gist_client_id,
parent=parent,
# import_source=req.get('import_source'),
# import_hash=req.get('import_hash'),
additional_metadata=origin_metadata,
translation_gist_client_id=persp_translation_gist_client_id,
translation_gist_object_id=persp_translation_gist_object_id
)
second_perspective.additional_metadata = origin_metadata
# if is_template is not None:
# perspective.is_template = is_template
DBSession.add(second_perspective)
owner_client = DBSession.query(Client).filter_by(id=parent.client_id).first()
owner = owner_client.user
for base in DBSession.query(BaseGroup).filter_by(perspective_default=True):
new_group = Group(parent=base,
subject_object_id=second_perspective.object_id,
subject_client_id=second_perspective.client_id)
if user not in new_group.users:
new_group.users.append(user)
if owner not in new_group.users:
new_group.users.append(owner)
DBSession.add(new_group)
second_perspective_client_id = second_perspective.client_id
second_perspective_object_id = second_perspective.object_id
fp_fields_dict = {}
"""
# FIRST PERSPECTIVE FIELDS CREATION
"""
fp_field_names = ("Word", "Transcription", "Translation", "Sound", "Etymology", "Backref")
fields_list = []
for fieldname in fp_field_names: #
if fieldname == "Backref":
fields_list.append(
{
"client_id": field_ids[fieldname][0],
"object_id": field_ids[fieldname][1],
"link":{
"client_id": second_perspective_client_id,
"object_id": second_perspective_object_id
}
}
)
elif fieldname == "Sound":
fields_list.append(
{
"client_id": field_ids[fieldname][0],
"object_id": field_ids[fieldname][1],
"contains":[{
"client_id": field_ids["Markup"][0],
"object_id": field_ids["Markup"][1]
}
]
}
)
else:
fields_list.append({"client_id": field_ids[fieldname][0], "object_id": field_ids[fieldname][1]})
fp_fields_dict[fieldname] = (field_ids[fieldname][0], field_ids[fieldname][1])
fp_fields_dict["Markup"] = (field_ids["Markup"][0], field_ids["Markup"][1])
update_perspective_fields(fields_list, first_perspective_client_id, first_perspective_object_id, client )
"""
# Creating fields of the second perspective
"""
sp_field_names = ("Word of Paradigmatic forms",
"Transcription of Paradigmatic forms",
"Translation of Paradigmatic forms",
"Sounds of Paradigmatic forms",
"Backref")
sp_fields_dict = {}
fields_list = []
for fieldname in sp_field_names:
if fieldname == "Backref":
fields_list.append(
{
"client_id": field_ids[fieldname][0],
"object_id": field_ids[fieldname][1],
"link":{
"client_id": first_perspective_client_id,
"object_id": first_perspective_object_id
}
}
)
elif fieldname == "Sounds of Paradigmatic forms":
fields_list.append(
{
"client_id": field_ids[fieldname][0],
"object_id": field_ids[fieldname][1],
"contains":[{
"client_id": field_ids["Paradigm Markup"][0],
"object_id": field_ids["Paradigm Markup"][1]
}
]
}
)
else:
fields_list.append({"client_id": field_ids[fieldname][0], "object_id": field_ids[fieldname][1]})
sp_fields_dict[fieldname] = (field_ids[fieldname][0], field_ids[fieldname][1])
sp_fields_dict["Paradigm Markup"] = (field_ids["Paradigm Markup"][0], field_ids["Paradigm Markup"][1])
update_perspective_fields(fields_list, second_perspective_client_id, second_perspective_object_id, client)
dubl = []
try:
eaffile = request.urlopen(eaf_url)
except HTTPError as e:
return {'error': str(e.read().decode("utf8", 'ignore'))}
with tempfile.NamedTemporaryFile() as temp:
temp.write(eaffile.read())
converter = elan_parser.Elan(temp.name)
converter.parse()
final_dicts = converter.proc()
temp.flush()
for phrase in final_dicts:
curr_dict = None
paradigm_words = []
for word_translation in phrase:
if type(word_translation) is not list:
curr_dict = word_translation
mt_words = [word_translation[i][1].text for i in word_translation
if len(word_translation[i]) > 1 and type(word_translation[i][1].text) is str]
main_tier_text = " ".join(mt_words)
if main_tier_text:
paradigm_words.append(elan_parser.Word(text=main_tier_text,
tier="Word of Paradigmatic forms",
time=word.time)
)
else:
word = word_translation[0]
tier_name = word.tier
new = " ".join([i.text for i in word_translation])
paradigm_words.append(elan_parser.Word(text=new, tier=tier_name, time=word.time))
p_match_dict = defaultdict(list)
for pword in paradigm_words:
match = [x for x in p_lexes if x[2].content == pword.text] #LEX COUNT OR RANDOM
for t in match:
if field_ids[EAF_TIERS[pword.tier]] == (t[2].field.client_id, t[2].field.object_id):
p_match_dict[t[1]].append(t)
p_match_dict = { k: v for k, v in p_match_dict.items() if len(v) >= 2 }
max_sim = None
for le in p_match_dict:
if max_sim is None:
max_sim = le
else:
if len(p_match_dict[le]) >= len(p_match_dict[max_sim]):
max_sim = le
if max_sim:
sp_lexical_entry_client_id = max_sim.client_id
sp_lexical_entry_object_id = max_sim.object_id
else:
lexentr = LexicalEntry(client_id=client.id,
parent_object_id=second_perspective_object_id,
parent=second_perspective)
DBSession.add(lexentr)
sp_lexical_entry_client_id = lexentr.client_id
sp_lexical_entry_object_id = lexentr.object_id
for other_word in paradigm_words:
if max_sim:
text_and_field = (other_word.text, field_ids[EAF_TIERS[other_word.tier]])
sim = [(x[2].content, (x[2].field.client_id, x[2].field.object_id)) for x in p_match_dict[max_sim]]
if text_and_field not in sim:
create_entity(sp_lexical_entry_client_id,
sp_lexical_entry_object_id,
field_ids[EAF_TIERS[other_word.tier]][0],
field_ids[EAF_TIERS[other_word.tier]][1],
None,
client,
other_word.text,
filename=None,
storage=storage)
else:
create_entity(sp_lexical_entry_client_id, sp_lexical_entry_object_id, field_ids[EAF_TIERS[other_word.tier]][0], field_ids[EAF_TIERS[other_word.tier]][1],
None, client, other_word.text, filename=None, storage=storage)
if not no_sound:
if word.time[1] < len(full_audio):
with tempfile.NamedTemporaryFile() as temp:
full_audio[ word.time[0]: word.time[1]].export(temp.name, format="wav")
audio_slice = temp.read()
if max_sim:
hash = hashlib.sha224(audio_slice).hexdigest()
if not hash in hashes:
hashes.append(hash)
create_entity(sp_lexical_entry_client_id,
sp_lexical_entry_object_id,
field_ids["Sounds of Paradigmatic forms"][0],
field_ids["Sounds of Paradigmatic forms"][1],
None,
client,
filename="%s.wav" %(word.index) ,
folder_name="sound1",
content=base64.urlsafe_b64encode(audio_slice).decode(),
storage=storage)
else:
create_entity(sp_lexical_entry_client_id,
sp_lexical_entry_object_id,
field_ids["Sounds of Paradigmatic forms"][0],
field_ids["Sounds of Paradigmatic forms"][1],
None,
client,
filename="%s.wav" %(word.index) ,
folder_name="sound1",
content=base64.urlsafe_b64encode(audio_slice).decode(),
storage=storage)
temp.flush()
p_match_dict.clear()
paradigm_words[:] = []
for word in curr_dict:
column = [word] + curr_dict[word]
match_dict = defaultdict(list)
for crt in tuple(i for i in column):
match = [x for x in lexes if x[2].content == crt.text]
for t in match:
if field_ids[EAF_TIERS[crt.tier]] == (t[2].field.client_id, t[2].field.object_id):
match_dict[t[1]].append(t)
match_dict = { k: v for k, v in match_dict.items() if len(v) >= 2 }
max_sim = None
for le in match_dict:
if max_sim is None:
max_sim = le
else:
if len(match_dict[le]) >= len(match_dict[max_sim]):
max_sim = le
if max_sim:
fp_lexical_entry_client_id = max_sim.client_id
fp_lexical_entry_object_id = max_sim.object_id
else:
lexentr = LexicalEntry(client_id=client.id,
parent_object_id=first_perspective_object_id, parent=first_perspective)
DBSession.add(lexentr)
fp_lexical_entry_client_id = lexentr.client_id
fp_lexical_entry_object_id = lexentr.object_id
for other_word in column:
if max_sim:
text_and_field = (other_word.text, field_ids[EAF_TIERS[other_word.tier]])
sim = [(x[2].content, (x[2].field.client_id, x[2].field.object_id)) for x in match_dict[max_sim]]
if text_and_field not in sim:
create_entity(fp_lexical_entry_client_id,
fp_lexical_entry_object_id,
field_ids[EAF_TIERS[other_word.tier]][0],
field_ids[EAF_TIERS[other_word.tier]][1],
None,
client,
other_word.text,
filename=None,
storage=storage)
else:
create_entity(fp_lexical_entry_client_id,
fp_lexical_entry_object_id,
field_ids[EAF_TIERS[other_word.tier]][0],
field_ids[EAF_TIERS[other_word.tier]][1],
None,
client,
other_word.text,
filename=None,
storage=storage)
if not no_sound:
if word.time[1] < len(full_audio):
with tempfile.NamedTemporaryFile() as temp:
full_audio[ word.time[0]: word.time[1]].export(temp.name, format="wav")
audio_slice = temp.read()
hash = hashlib.sha224(audio_slice).hexdigest()
if max_sim:
if not hash in hashes:
hashes.append(hash)
create_entity(fp_lexical_entry_client_id,
fp_lexical_entry_object_id,
field_ids["Sound"][0],
field_ids["Sound"][1],
None,
client,
filename="%s.wav" %(word.index) ,
folder_name="sound1",
content=base64.urlsafe_b64encode(audio_slice).decode(),
storage=storage)
else:
create_entity(fp_lexical_entry_client_id,
fp_lexical_entry_object_id,
field_ids["Sound"][0],
field_ids["Sound"][1],
None,
client,
filename="%s.wav" %(word.index) ,
folder_name="sound1",
content=base64.urlsafe_b64encode(audio_slice).decode(),
storage=storage)
temp.flush()
fp_le_ids = (fp_lexical_entry_client_id, fp_lexical_entry_object_id)
sp_le_ids = (sp_lexical_entry_client_id, sp_lexical_entry_object_id)
dubl_tuple = (sp_le_ids, fp_le_ids)
if not dubl_tuple in dubl:
dubl.append(dubl_tuple)
if max_sim:
if not (sp_le_ids, fp_le_ids) in links :
create_entity(sp_lexical_entry_client_id,
sp_lexical_entry_object_id,
field_ids["Backref"][0],
field_ids["Backref"][1],
None,
client,
filename=None,
link_client_id=fp_lexical_entry_client_id,
link_object_id=fp_lexical_entry_object_id,
storage=storage)
if not (fp_le_ids, sp_le_ids) in links:
create_entity(fp_lexical_entry_client_id,
fp_lexical_entry_object_id,
field_ids["Backref"][0],
field_ids["Backref"][1],
None,
client,
filename=None,
link_client_id=sp_lexical_entry_client_id,
link_object_id=sp_lexical_entry_object_id,
storage=storage)
else:
create_entity(sp_lexical_entry_client_id,
sp_lexical_entry_object_id,
field_ids["Backref"][0],
field_ids["Backref"][1],
None,
client,
filename=None,
link_client_id=fp_lexical_entry_client_id,
link_object_id=fp_lexical_entry_object_id,
storage=storage)
create_entity(fp_lexical_entry_client_id,
fp_lexical_entry_object_id,
field_ids["Backref"][0],
field_ids["Backref"][1],
None,
client,
filename=None,
link_client_id=sp_lexical_entry_client_id,
link_object_id=sp_lexical_entry_object_id,
storage=storage)
column[:] = []
match_dict.clear()
return
def setUp(self):
self.config = testing.setUp()
import webtest
from pyramid import paster
from sqlalchemy import create_engine
engine = create_engine('sqlite://')
myapp = paster.get_app('testing.ini')
self.app = webtest.TestApp(myapp)
from lingvodoc.models import (
Base,
Dictionary,
Language,
Organization,
Locale,
User,
Passhash,
Client,
DictionaryPerspective,
UserEntitiesTranslationString
)
DBSession.configure(bind=engine)
Base.metadata.create_all(engine)
with transaction.manager:
ru_locale = Locale(id=1, shortcut="ru", intl_name="Русский")
DBSession.add(ru_locale)
en_locale = Locale(id=2, shortcut="en", intl_name="English")
DBSession.add(en_locale)
DBSession.flush()
new_user = User(id=1, login='******', default_locale_id = 1)
new_pass = Passhash(password='******')
DBSession.add(new_pass)
new_user.password = new_pass
DBSession.add(new_user)
new_client = Client(id=1, user=new_user)
DBSession.add(new_client)
new_user2 = User(id=2, login='******', default_locale_id = 1)
new_pass2 = Passhash(password='******')
DBSession.add(new_pass)
new_user2.password = new_pass2
DBSession.add(new_user2)
new_client = Client(id=2, user=new_user2)
DBSession.add(new_client)
new_client = Client(id=3, user=new_user)
DBSession.add(new_client)
DBSession.flush()
new_user3 = User(id=3, login='******', default_locale_id = 1)
new_pass3 = Passhash(password='******')
DBSession.add(new_pass3)
new_user3.password = new_pass3
DBSession.add(new_user3)
new_client = Client(id=4, user=new_user3)
DBSession.add(new_client)
new_user4 = User(id=4, login='******', default_locale_id = 1)
new_pass4 = Passhash(password='******')
DBSession.add(new_pass4)
new_user4.password = new_pass4
DBSession.add(new_user4)
new_client = Client(id=5, user=new_user4)
DBSession.add(new_client)
new_lang1 = Language(client_id=1, object_id=1, translation_string='head')
DBSession.add(new_lang1)
new_lang2 = Language(client_id=2, object_id=5, translation_string='left son', parent=new_lang1)
DBSession.add(new_lang2)
new_lang3 = Language(client_id=1, object_id=3, translation_string='right son', parent=new_lang1)
DBSession.add(new_lang3)
new_lang4 = Language(client_id=2, object_id=4, translation_string='first grand son', parent=new_lang3)
DBSession.add(new_lang4)
new_lang5 = Language(client_id=1, object_id=5, translation_string='second grand son', parent=new_lang3)
DBSession.add(new_lang5)
new_lang6 = Language(client_id=1, object_id=6, translation_string='third grand son', parent=new_lang3)
DBSession.add(new_lang6)
new_lang7 = Language(client_id=1, object_id=7, translation_string='grand grand son', parent=new_lang5)
DBSession.add(new_lang7)
new_lang8 = Language(client_id=1, object_id=8, translation_string='second head')
DBSession.add(new_lang8)
new_lang9 = Language(client_id=1, object_id=9, translation_string='second left son', parent=new_lang8)
DBSession.add(new_lang9)
new_lang10 = Language(client_id=1, object_id=10, translation_string='second right son', parent=new_lang8)
DBSession.add(new_lang10)
new_org1 = Organization(name='first')
new_org1.users.append(new_user)
new_org1.users.append(new_user3)
DBSession.add(new_org1)
new_org2 = Organization(name='second')
DBSession.add(new_org2)
new_dict = Dictionary(client_id=1, object_id=1, name='dict')
DBSession.add(new_dict)
DBSession.flush()
new_persp1 = DictionaryPerspective(client_id=1, object_id=1, name='persp', parent=new_dict)
DBSession.add(new_persp1)
new_persp2 = DictionaryPerspective(client_id=2, object_id=2, name='persp', parent=new_dict)
DBSession.add(new_persp2)
uets = UserEntitiesTranslationString(locale_id=1, translation_string='persp', translation='персп')
def setUp(self):
self.config = testing.setUp()
self.config.testing_securitypolicy(userid='1',
permissive=True)
import webtest
from pyramid import paster
from sqlalchemy import create_engine
engine = create_engine('sqlite://')
myapp = paster.get_app('testing.ini')
self.app = webtest.TestApp(myapp)
from lingvodoc.models import (
Base,
User,
Client,
Passhash,
Locale,
UserEntitiesTranslationString,
Language
)
DBSession.configure(bind=engine)
Base.metadata.create_all(engine)
with transaction.manager:
ru_locale = Locale(id=1, shortcut="ru", intl_name="Русский")
DBSession.add(ru_locale)
en_locale = Locale(id=2, shortcut="en", intl_name="English")
DBSession.add(en_locale)
DBSession.flush()
new_user = User(id=1, login='******', default_locale_id = 1)
new_pass = Passhash(password='******')
DBSession.add(new_pass)
new_user.password = new_pass
DBSession.add(new_user)
new_client = Client(id=1, user=new_user)
DBSession.add(new_client)
new_uets= UserEntitiesTranslationString(client_id = 1, object_id = 1, locale_id=2,
translation_string = 'imastring')
DBSession.add(new_uets)
new_lang = Language(client_id=1, object_id=1, translation_string='testy')
DBSession.add(new_lang)
def create_entity(request): # tested
try:
variables = {'auth': authenticated_userid(request)}
response = dict()
parent_client_id = request.matchdict.get('lexical_entry_client_id')
parent_object_id = request.matchdict.get('lexical_entry_object_id')
req = request.json_body
client = DBSession.query(Client).filter_by(
id=variables['auth']).first()
object_id = req.get('object_id', None)
if not client:
raise KeyError(
"Invalid client id (not registered on server). Try to logout and then login."
)
user = DBSession.query(User).filter_by(id=client.user_id).first()
if not user:
raise CommonException(
"This client id is orphaned. Try to logout and then login once more."
)
parent = DBSession.query(LexicalEntry).filter_by(
client_id=parent_client_id, object_id=parent_object_id).first()
if not parent:
request.response.status = HTTPNotFound.code
return {'error': str("No such lexical entry in the system")}
if 'content' not in req is None:
request.response.status = HTTPBadRequest.code
return {'error': 'Missing value: content'}
additional_metadata = req.get('additional_metadata')
upper_level = None
# import pdb
# pdb.set_trace(v
# data_type = DBSession.query(TranslationAtom).filter(TranslationAtom.locale_id == 2).join(TranslationGist, and_(
# TranslationAtom.parent_client_id == TranslationGist.client_id,
# TranslationAtom.parent_object_id == TranslationGist.object_id)).join(Field, and_(
# TranslationGist.client_id == Field.data_type_translation_gist_client_id,
# TranslationGist.object_id == Field.data_type_translation_gist_object_id)).filter(
# Field.client_id == req['field_client_id'], Field.object_id == req['field_object_id']).first()
tr_atom = DBSession.query(TranslationAtom).join(
TranslationGist,
and_(
TranslationAtom.locale_id == 2,
TranslationAtom.parent_client_id == TranslationGist.client_id,
TranslationAtom.parent_object_id ==
TranslationGist.object_id)).join(
Field,
and_(
TranslationGist.client_id ==
Field.data_type_translation_gist_client_id,
TranslationGist.object_id ==
Field.data_type_translation_gist_object_id)).filter(
Field.client_id == req['field_client_id'],
Field.object_id == req['field_object_id']).first()
data_type = tr_atom.content.lower()
if req.get('self_client_id') and req.get('self_object_id'):
upper_level = DBSession.query(Entity).filter_by(
client_id=req['self_client_id'],
object_id=req['self_object_id']).first()
if not upper_level:
return {'error': str("No such upper level in the system")}
entity = Entity(client_id=client.id,
object_id=object_id,
field_client_id=req['field_client_id'],
field_object_id=req['field_object_id'],
locale_id=req.get('locale_id'),
additional_metadata=additional_metadata,
parent=parent)
group = DBSession.query(Group).join(BaseGroup).filter(
BaseGroup.subject == 'lexical_entries_and_entities',
Group.subject_client_id == entity.parent.parent.client_id,
Group.subject_object_id == entity.parent.parent.object_id,
BaseGroup.action == 'create').one()
if user in group.users:
entity.publishingentity.accepted = True
if upper_level:
entity.upper_level = upper_level
filename = req.get('filename')
real_location = None
url = None
if data_type == 'image' or data_type == 'sound' or 'markup' in data_type:
real_location, url = create_object(request, req['content'], entity,
data_type, filename)
entity.content = url
old_meta = entity.additional_metadata
need_hash = True
if old_meta:
if old_meta.get('hash'):
need_hash = False
if need_hash:
hash = hashlib.sha224(base64.urlsafe_b64decode(
req['content'])).hexdigest()
hash_dict = {'hash': hash}
if old_meta:
old_meta.update(hash_dict)
else:
old_meta = hash_dict
entity.additional_metadata = old_meta
if 'markup' in data_type:
name = filename.split('.')
ext = name[len(name) - 1]
if ext.lower() == 'textgrid':
data_type = 'praat markup'
elif ext.lower() == 'eaf':
data_type = 'elan markup'
entity.additional_metadata['data_type'] = data_type
elif data_type == 'link':
try:
entity.link_client_id = req['link_client_id']
entity.link_object_id = req['link_object_id']
except (KeyError, TypeError):
request.response.status = HTTPBadRequest.code
return {
'Error':
"The field is of link type. You should provide client_id and object id in the content"
}
else:
entity.content = req['content']
# return None
DBSession.add(entity)
request.response.status = HTTPOk.code
response['client_id'] = entity.client_id
response['object_id'] = entity.object_id
return response
except IntegrityError as e:
request.response.status = HTTPInternalServerError.code
return {'error': str(e)}
except CommonException as e:
request.response.status = HTTPConflict.code
return {'error': str(e)}
def convert_five_tiers(dictionary_client_id,
dictionary_object_id,
user_id,
origin_client_id,
origin_object_id,
sqlalchemy_url,
storage,
eaf_url,
sound_url=None):
log = logging.getLogger(__name__)
log.setLevel(logging.DEBUG)
no_sound = True
if sound_url:
no_sound = False
with warnings.catch_warnings():
warnings.filterwarnings('error')
try:
from pydub import AudioSegment
except Warning as e:
no_sound = True
if not no_sound:
with tempfile.NamedTemporaryFile() as temp:
try:
sound_file = request.urlopen(sound_url)
except HTTPError as e:
return {'error': str(e.read().decode("utf8", 'ignore'))}
with open(temp.name, 'wb') as output:
output.write(sound_file.read())
full_audio = AudioSegment.from_wav(temp.name)
temp.flush()
field_ids = {}
with transaction.manager:
client = DBSession.query(Client).filter_by(id=user_id).first()
if not client:
raise KeyError(
"Invalid client id (not registered on server). Try to logout and then login.",
user_id)
user = DBSession.query(User).filter_by(id=client.user_id).first()
all_fieldnames = ("Markup", "Paradigm Markup", "Word", "Transcription",
"Translation", "Sound", "Etymology", "Backref",
"Word of Paradigmatic forms",
"Transcription of Paradigmatic forms",
"Translation of Paradigmatic forms",
"Sounds of Paradigmatic forms")
for name in all_fieldnames:
data_type_query = DBSession.query(Field) \
.join(TranslationGist,
and_(Field.translation_gist_object_id == TranslationGist.object_id,
Field.translation_gist_client_id == TranslationGist.client_id))\
.join(TranslationGist.translationatom)
field = data_type_query.filter(
TranslationAtom.locale_id == 2,
TranslationAtom.content == name).one()
field_ids[name] = (field.client_id, field.object_id)
fp_fields = ("Word", "Transcription", "Translation", "Sound", "Markup",
"Etymology", "Backref")
sp_fields = ("Word of Paradigmatic forms",
"Transcription of Paradigmatic forms",
"Translation of Paradigmatic forms",
"Sounds of Paradigmatic forms", "Paradigm Markup",
"Backref")
fp_structure = set([field_ids[x] for x in fp_fields])
sp_structure = set([field_ids[x] for x in sp_fields])
DBSession.flush()
resp = translation_service_search("WiP")
state_translation_gist_object_id, state_translation_gist_client_id = resp[
'object_id'], resp['client_id']
for base in DBSession.query(BaseGroup).filter_by(
dictionary_default=True):
new_group = Group(parent=base,
subject_object_id=dictionary_object_id,
subject_client_id=dictionary_client_id)
if user not in new_group.users:
new_group.users.append(user)
DBSession.add(new_group)
DBSession.flush()
origin_metadata = {
"origin_client_id": origin_client_id,
"origin_object_id": origin_object_id
}
parent = DBSession.query(Dictionary).filter_by(
client_id=dictionary_client_id,
object_id=dictionary_object_id).first()
if not parent:
return {'error': str("No such dictionary in the system")}
first_perspective = None
second_perspective = None
for perspective in DBSession.query(DictionaryPerspective).filter_by(
parent=parent, marked_for_deletion=False):
structure = set()
fields = DBSession.query(DictionaryPerspectiveToField)\
.filter_by(parent=perspective)\
.all()
DBSession.flush()
for p_to_field in fields:
structure.add(
(p_to_field.field_client_id, p_to_field.field_object_id))
if structure == fp_structure:
first_perspective = perspective
elif structure == sp_structure:
second_perspective = perspective
structure.clear()
lexes = []
if first_perspective:
lexes = DBSession.query(DictionaryPerspective, LexicalEntry, Entity)\
.filter(and_(DictionaryPerspective.object_id==first_perspective.object_id,
DictionaryPerspective.client_id==first_perspective.client_id))\
.join(LexicalEntry, and_( LexicalEntry.parent_object_id==DictionaryPerspective.object_id,
LexicalEntry.parent_client_id==DictionaryPerspective.client_id))\
.join(Entity, and_(LexicalEntry.object_id==Entity.parent_object_id,
LexicalEntry.client_id==Entity.parent_client_id))
p_lexes = []
if second_perspective:
p_lexes = DBSession.query(DictionaryPerspective, LexicalEntry, Entity)\
.filter(and_(DictionaryPerspective.object_id==second_perspective.object_id,
DictionaryPerspective.client_id==second_perspective.client_id))\
.join(LexicalEntry, and_( LexicalEntry.parent_object_id==DictionaryPerspective.object_id,
LexicalEntry.parent_client_id==DictionaryPerspective.client_id))\
.join(Entity, and_(LexicalEntry.object_id==Entity.parent_object_id,
LexicalEntry.client_id==Entity.parent_client_id))
hashes = [
x[2].additional_metadata["hash"] for x in lexes
if x[2].field.data_type == "Sound"
]
hashes = hashes[:] + [
x[2].additional_metadata["hash"]
for x in p_lexes if x[2].field.data_type == "Sound"
]
links = [((x[2].link.client_id, x[2].link.object_id),
(x[1].client_id, x[1].object_id)) for x in lexes
if x[2].field.data_type == "Link"]
links = links[:] + [((x[2].link.client_id, x[2].link.object_id),
(x[1].client_id, x[1].object_id))
for x in p_lexes if x[2].field.data_type == "Link"]
resp = translation_service_search("WiP")
state_translation_gist_object_id, state_translation_gist_client_id = resp[
'object_id'], resp['client_id']
for base in DBSession.query(BaseGroup).filter_by(
dictionary_default=True):
new_group = Group(parent=base,
subject_object_id=dictionary_object_id,
subject_client_id=dictionary_client_id)
if user not in new_group.users:
new_group.users.append(user)
DBSession.add(new_group)
DBSession.flush()
"""
# FIRST PERSPECTIVE
"""
if first_perspective is None:
resp = translation_service_search_all("Lexical Entries")
persp_translation_gist_client_id, persp_translation_gist_object_id = resp[
'client_id'], resp['object_id']
first_perspective = DictionaryPerspective(
client_id=client.id, ###
state_translation_gist_object_id=
state_translation_gist_object_id,
state_translation_gist_client_id=
state_translation_gist_client_id,
parent=parent,
# import_source=req.get('import_source'),
# import_hash=req.get('import_hash'),
additional_metadata=origin_metadata,
translation_gist_client_id=persp_translation_gist_client_id,
translation_gist_object_id=persp_translation_gist_object_id)
first_perspective.additional_metadata = origin_metadata
DBSession.add(first_perspective)
owner_client = DBSession.query(Client).filter_by(
id=parent.client_id).first()
owner = owner_client.user
for base in DBSession.query(BaseGroup).filter_by(
perspective_default=True):
new_group = Group(parent=base,
subject_object_id=first_perspective.object_id,
subject_client_id=first_perspective.client_id)
if user not in new_group.users:
new_group.users.append(user)
if owner not in new_group.users:
new_group.users.append(owner)
DBSession.add(new_group)
DBSession.flush()
first_perspective_client_id = first_perspective.client_id
first_perspective_object_id = first_perspective.object_id
"""
# SECOND PERSPECTIVE
"""
resp = translation_service_search_all("Paradigms")
persp_translation_gist_client_id, persp_translation_gist_object_id = resp[
'client_id'], resp['object_id']
if second_perspective is None:
second_perspective = DictionaryPerspective(
client_id=client.id, ### variables['auth']
state_translation_gist_object_id=
state_translation_gist_object_id,
state_translation_gist_client_id=
state_translation_gist_client_id,
parent=parent,
# import_source=req.get('import_source'),
# import_hash=req.get('import_hash'),
additional_metadata=origin_metadata,
translation_gist_client_id=persp_translation_gist_client_id,
translation_gist_object_id=persp_translation_gist_object_id)
second_perspective.additional_metadata = origin_metadata
# if is_template is not None:
# perspective.is_template = is_template
DBSession.add(second_perspective)
owner_client = DBSession.query(Client).filter_by(
id=parent.client_id).first()
owner = owner_client.user
for base in DBSession.query(BaseGroup).filter_by(
perspective_default=True):
new_group = Group(parent=base,
subject_object_id=second_perspective.object_id,
subject_client_id=second_perspective.client_id)
if user not in new_group.users:
new_group.users.append(user)
if owner not in new_group.users:
new_group.users.append(owner)
DBSession.add(new_group)
second_perspective_client_id = second_perspective.client_id
second_perspective_object_id = second_perspective.object_id
fp_fields_dict = {}
"""
# FIRST PERSPECTIVE FIELDS CREATION
"""
fp_field_names = ("Word", "Transcription", "Translation", "Sound",
"Etymology", "Backref")
fields_list = []
for fieldname in fp_field_names: #
if fieldname == "Backref":
fields_list.append({
"client_id": field_ids[fieldname][0],
"object_id": field_ids[fieldname][1],
"link": {
"client_id": second_perspective_client_id,
"object_id": second_perspective_object_id
}
})
elif fieldname == "Sound":
fields_list.append({
"client_id":
field_ids[fieldname][0],
"object_id":
field_ids[fieldname][1],
"contains": [{
"client_id": field_ids["Markup"][0],
"object_id": field_ids["Markup"][1]
}]
})
else:
fields_list.append({
"client_id": field_ids[fieldname][0],
"object_id": field_ids[fieldname][1]
})
fp_fields_dict[fieldname] = (field_ids[fieldname][0],
field_ids[fieldname][1])
fp_fields_dict["Markup"] = (field_ids["Markup"][0],
field_ids["Markup"][1])
update_perspective_fields(fields_list, first_perspective_client_id,
first_perspective_object_id, client)
"""
# Creating fields of the second perspective
"""
sp_field_names = ("Word of Paradigmatic forms",
"Transcription of Paradigmatic forms",
"Translation of Paradigmatic forms",
"Sounds of Paradigmatic forms", "Backref")
sp_fields_dict = {}
fields_list = []
for fieldname in sp_field_names:
if fieldname == "Backref":
fields_list.append({
"client_id": field_ids[fieldname][0],
"object_id": field_ids[fieldname][1],
"link": {
"client_id": first_perspective_client_id,
"object_id": first_perspective_object_id
}
})
elif fieldname == "Sounds of Paradigmatic forms":
fields_list.append({
"client_id":
field_ids[fieldname][0],
"object_id":
field_ids[fieldname][1],
"contains": [{
"client_id": field_ids["Paradigm Markup"][0],
"object_id": field_ids["Paradigm Markup"][1]
}]
})
else:
fields_list.append({
"client_id": field_ids[fieldname][0],
"object_id": field_ids[fieldname][1]
})
sp_fields_dict[fieldname] = (field_ids[fieldname][0],
field_ids[fieldname][1])
sp_fields_dict["Paradigm Markup"] = (field_ids["Paradigm Markup"][0],
field_ids["Paradigm Markup"][1])
update_perspective_fields(fields_list, second_perspective_client_id,
second_perspective_object_id, client)
dubl = []
try:
eaffile = request.urlopen(eaf_url)
except HTTPError as e:
return {'error': str(e.read().decode("utf8", 'ignore'))}
with tempfile.NamedTemporaryFile() as temp:
temp.write(eaffile.read())
converter = elan_parser.Elan(temp.name)
converter.parse()
final_dicts = converter.proc()
temp.flush()
for phrase in final_dicts:
curr_dict = None
paradigm_words = []
for word_translation in phrase:
if type(word_translation) is not list:
curr_dict = word_translation
mt_words = [
word_translation[i][1].text for i in word_translation
if len(word_translation[i]) > 1
and type(word_translation[i][1].text) is str
]
main_tier_text = " ".join(mt_words)
if main_tier_text:
paradigm_words.append(
elan_parser.Word(text=main_tier_text,
tier="Word of Paradigmatic forms",
time=word.time))
else:
word = word_translation[0]
tier_name = word.tier
new = " ".join([i.text for i in word_translation])
paradigm_words.append(
elan_parser.Word(text=new,
tier=tier_name,
time=word.time))
p_match_dict = defaultdict(list)
for pword in paradigm_words:
match = [x for x in p_lexes
if x[2].content == pword.text] #LEX COUNT OR RANDOM
for t in match:
if field_ids[EAF_TIERS[pword.tier]] == (
t[2].field.client_id, t[2].field.object_id):
p_match_dict[t[1]].append(t)
p_match_dict = {
k: v
for k, v in p_match_dict.items() if len(v) >= 2
}
max_sim = None
for le in p_match_dict:
if max_sim is None:
max_sim = le
else:
if len(p_match_dict[le]) >= len(p_match_dict[max_sim]):
max_sim = le
if max_sim:
sp_lexical_entry_client_id = max_sim.client_id
sp_lexical_entry_object_id = max_sim.object_id
else:
lexentr = LexicalEntry(
client_id=client.id,
parent_object_id=second_perspective_object_id,
parent=second_perspective)
DBSession.add(lexentr)
sp_lexical_entry_client_id = lexentr.client_id
sp_lexical_entry_object_id = lexentr.object_id
for other_word in paradigm_words:
if max_sim:
text_and_field = (other_word.text,
field_ids[EAF_TIERS[other_word.tier]])
sim = [(x[2].content, (x[2].field.client_id,
x[2].field.object_id))
for x in p_match_dict[max_sim]]
if text_and_field not in sim:
create_entity(sp_lexical_entry_client_id,
sp_lexical_entry_object_id,
field_ids[EAF_TIERS[other_word.tier]][0],
field_ids[EAF_TIERS[other_word.tier]][1],
None,
client,
other_word.text,
filename=None,
storage=storage)
else:
create_entity(sp_lexical_entry_client_id,
sp_lexical_entry_object_id,
field_ids[EAF_TIERS[other_word.tier]][0],
field_ids[EAF_TIERS[other_word.tier]][1],
None,
client,
other_word.text,
filename=None,
storage=storage)
if not no_sound:
if word.time[1] < len(full_audio):
with tempfile.NamedTemporaryFile() as temp:
full_audio[word.time[0]:word.time[1]].export(
temp.name, format="wav")
audio_slice = temp.read()
if max_sim:
hash = hashlib.sha224(audio_slice).hexdigest()
if not hash in hashes:
hashes.append(hash)
create_entity(
sp_lexical_entry_client_id,
sp_lexical_entry_object_id,
field_ids["Sounds of Paradigmatic forms"]
[0],
field_ids["Sounds of Paradigmatic forms"]
[1],
None,
client,
filename="%s.wav" % (word.index),
folder_name="sound1",
content=base64.urlsafe_b64encode(
audio_slice).decode(),
storage=storage)
else:
create_entity(
sp_lexical_entry_client_id,
sp_lexical_entry_object_id,
field_ids["Sounds of Paradigmatic forms"][0],
field_ids["Sounds of Paradigmatic forms"][1],
None,
client,
filename="%s.wav" % (word.index),
folder_name="sound1",
content=base64.urlsafe_b64encode(
audio_slice).decode(),
storage=storage)
temp.flush()
p_match_dict.clear()
paradigm_words[:] = []
for word in curr_dict:
column = [word] + curr_dict[word]
match_dict = defaultdict(list)
for crt in tuple(i for i in column):
match = [x for x in lexes if x[2].content == crt.text]
for t in match:
if field_ids[EAF_TIERS[crt.tier]] == (
t[2].field.client_id, t[2].field.object_id):
match_dict[t[1]].append(t)
match_dict = {
k: v
for k, v in match_dict.items() if len(v) >= 2
}
max_sim = None
for le in match_dict:
if max_sim is None:
max_sim = le
else:
if len(match_dict[le]) >= len(match_dict[max_sim]):
max_sim = le
if max_sim:
fp_lexical_entry_client_id = max_sim.client_id
fp_lexical_entry_object_id = max_sim.object_id
else:
lexentr = LexicalEntry(
client_id=client.id,
parent_object_id=first_perspective_object_id,
parent=first_perspective)
DBSession.add(lexentr)
fp_lexical_entry_client_id = lexentr.client_id
fp_lexical_entry_object_id = lexentr.object_id
for other_word in column:
if max_sim:
text_and_field = (
other_word.text,
field_ids[EAF_TIERS[other_word.tier]])
sim = [(x[2].content, (x[2].field.client_id,
x[2].field.object_id))
for x in match_dict[max_sim]]
if text_and_field not in sim:
create_entity(
fp_lexical_entry_client_id,
fp_lexical_entry_object_id,
field_ids[EAF_TIERS[other_word.tier]][0],
field_ids[EAF_TIERS[other_word.tier]][1],
None,
client,
other_word.text,
filename=None,
storage=storage)
else:
create_entity(fp_lexical_entry_client_id,
fp_lexical_entry_object_id,
field_ids[EAF_TIERS[other_word.tier]][0],
field_ids[EAF_TIERS[other_word.tier]][1],
None,
client,
other_word.text,
filename=None,
storage=storage)
if not no_sound:
if word.time[1] < len(full_audio):
with tempfile.NamedTemporaryFile() as temp:
full_audio[word.time[0]:word.time[1]].export(
temp.name, format="wav")
audio_slice = temp.read()
hash = hashlib.sha224(audio_slice).hexdigest()
if max_sim:
if not hash in hashes:
hashes.append(hash)
create_entity(
fp_lexical_entry_client_id,
fp_lexical_entry_object_id,
field_ids["Sound"][0],
field_ids["Sound"][1],
None,
client,
filename="%s.wav" % (word.index),
folder_name="sound1",
content=base64.urlsafe_b64encode(
audio_slice).decode(),
storage=storage)
else:
create_entity(fp_lexical_entry_client_id,
fp_lexical_entry_object_id,
field_ids["Sound"][0],
field_ids["Sound"][1],
None,
client,
filename="%s.wav" % (word.index),
folder_name="sound1",
content=base64.urlsafe_b64encode(
audio_slice).decode(),
storage=storage)
temp.flush()
fp_le_ids = (fp_lexical_entry_client_id,
fp_lexical_entry_object_id)
sp_le_ids = (sp_lexical_entry_client_id,
sp_lexical_entry_object_id)
dubl_tuple = (sp_le_ids, fp_le_ids)
if not dubl_tuple in dubl:
dubl.append(dubl_tuple)
if max_sim:
if not (sp_le_ids, fp_le_ids) in links:
create_entity(
sp_lexical_entry_client_id,
sp_lexical_entry_object_id,
field_ids["Backref"][0],
field_ids["Backref"][1],
None,
client,
filename=None,
link_client_id=fp_lexical_entry_client_id,
link_object_id=fp_lexical_entry_object_id,
storage=storage)
if not (fp_le_ids, sp_le_ids) in links:
create_entity(
fp_lexical_entry_client_id,
fp_lexical_entry_object_id,
field_ids["Backref"][0],
field_ids["Backref"][1],
None,
client,
filename=None,
link_client_id=sp_lexical_entry_client_id,
link_object_id=sp_lexical_entry_object_id,
storage=storage)
else:
create_entity(
sp_lexical_entry_client_id,
sp_lexical_entry_object_id,
field_ids["Backref"][0],
field_ids["Backref"][1],
None,
client,
filename=None,
link_client_id=fp_lexical_entry_client_id,
link_object_id=fp_lexical_entry_object_id,
storage=storage)
create_entity(
fp_lexical_entry_client_id,
fp_lexical_entry_object_id,
field_ids["Backref"][0],
field_ids["Backref"][1],
None,
client,
filename=None,
link_client_id=sp_lexical_entry_client_id,
link_object_id=sp_lexical_entry_object_id,
storage=storage)
column[:] = []
match_dict.clear()
return
def edit_user_info(request): # TODO: test
from passlib.hash import bcrypt
response = dict()
req = request.json_body
client_id = req.get('client_id')
user_id = req.get('user_id')
user = None
if client_id:
client = DBSession.query(Client).filter_by(id=client_id).first()
if not client:
request.response.status = HTTPNotFound.code
return {'error': str("No such client in the system")}
user = DBSession.query(User).filter_by(id=client.user_id).first()
user_id = client.user_id
if not user:
request.response.status = HTTPNotFound.code
return {'error': str("No such user in the system")}
else:
user = DBSession.query(User).filter_by(id=user_id).first()
if not user:
request.response.status = HTTPNotFound.code
return {'error': str("No such user in the system")}
new_password = req.get('new_password')
old_password = req.get('old_password')
if new_password:
if not old_password:
request.response.status = HTTPBadRequest.code
return {'error': str("Need old password to confirm")}
old_hash = DBSession.query(Passhash).filter_by(user_id=user_id).first()
if old_hash:
if not user.check_password(old_password):
request.response.status = HTTPBadRequest.code
return {'error': str("Wrong password")}
else:
old_hash.hash = bcrypt.encrypt(new_password)
else:
request.response.status = HTTPInternalServerError.code
return {'error': str("User has no password")}
name = req.get('name')
if name:
user.name = name
default_locale_id = req.get('default_locale_id')
if default_locale_id:
user.default_locale_id = default_locale_id
birthday = req.get('birthday')
if birthday:
try:
year, month, day = birthday.split('-')
user.birthday = datetime.date(int(year), int(month), int(day))
except ValueError:
request.response.status = HTTPBadRequest.code
return {'Error': "Invalid birthday"}
email = req.get('email')
if email:
if user.email:
for em in user.email:
em.email = email
else:
new_email = Email(user=user, email=email)
DBSession.add(new_email)
DBSession.flush()
about = req.get('about')
if about:
if user.about:
for ab in user.about:
ab.content = req['about']
else:
new_about = About(user=user, content=about)
DBSession.add(new_about)
DBSession.flush()
# response['is_active']=str(user.is_active)
request.response.status = HTTPOk.code
return response
