def languoids(request):
if request.params.get('id'):
if '-' not in request.params['id']:
return HTTPNotFound()
m, id_ = request.params['id'].split('-', 1)
model = dict(w=Language, g=Genus, f=Family).get(m)
if not model:
return HTTPNotFound()
obj = model.get(id_, default=None)
if not obj:
return HTTPNotFound()
return HTTPFound(location=request.resource_url(obj))
max_results = 20
qs = request.params.get('q')
if not qs:
return []
query = DBSession.query(Language)\
.filter(icontains(Language.name, qs))\
.order_by(WalsLanguage.ascii_name).limit(max_results)
res = [l for l in query]
if len(res) < max_results:
max_results = max_results - len(res)
# fill up suggestions with matching alternative names:
for l in DBSession.query(Language)\
.join(Language.languageidentifier, LanguageIdentifier.identifier)\
.filter(icontains(Identifier.name, qs))\
.order_by(WalsLanguage.ascii_name).limit(max_results):
if l not in res:
res.append(l)
if len(res) < max_results:
max_results = max_results - len(res)
# fill up with matching genera:
for l in DBSession.query(Genus)\
.filter(icontains(Genus.name, qs))\
.order_by(Genus.name).limit(max_results):
res.append(l)
if len(res) < max_results:
max_results = max_results - len(res)
# fill up with matching families:
for l in DBSession.query(Family)\
.filter(icontains(Family.name, qs))\
.order_by(Family.name).limit(max_results):
res.append(l)
ms = LanguoidSelect(request, None, None, url='x')
return dict(results=list(map(ms.format_result, res)),
context={},
more=False)
def languoids(request):
if request.params.get('id'):
if '-' not in request.params['id']:
return HTTPNotFound()
m, id_ = request.params['id'].split('-', 1)
model = dict(w=Language, g=Genus, f=Family).get(m)
if not model:
return HTTPNotFound()
obj = model.get(id_, default=None)
if not obj:
return HTTPNotFound()
return HTTPFound(location=request.resource_url(obj))
max_results = 20
qs = request.params.get('q')
if not qs:
return []
query = DBSession.query(Language)\
.filter(icontains(Language.name, qs))\
.order_by(WalsLanguage.ascii_name).limit(max_results)
res = [l for l in query]
if len(res) < max_results:
max_results = max_results - len(res)
# fill up suggestions with matching alternative names:
for l in DBSession.query(Language)\
.join(Language.languageidentifier, LanguageIdentifier.identifier)\
.filter(icontains(Identifier.name, qs))\
.order_by(WalsLanguage.ascii_name).limit(max_results):
if l not in res:
res.append(l)
if len(res) < max_results:
max_results = max_results - len(res)
# fill up with matching genera:
for l in DBSession.query(Genus)\
.filter(icontains(Genus.name, qs))\
.order_by(Genus.name).limit(max_results):
res.append(l)
if len(res) < max_results:
max_results = max_results - len(res)
# fill up with matching families:
for l in DBSession.query(Family)\
.filter(icontains(Family.name, qs))\
.order_by(Family.name).limit(max_results):
res.append(l)
ms = LanguoidSelect(request, None, None, url='x')
return dict(results=list(map(ms.format_result, res)), context={}, more=False)
def getRefs(params):
query = DBSession.query(Ref)
filtered = False
for param, value in params['biblio'].items():
if value:
filtered = True
query = query.filter(icontains(getattr(Ref, param), value))
if params.get('languoids'):
filtered = True
lids = DBSession.query(TreeClosureTable.child_pk)\
.filter(TreeClosureTable.parent_pk.in_([l.pk for l in params['languoids']]))\
.subquery()
query = query.join(LanguageSource, LanguageSource.source_pk == Ref.pk)\
.filter(LanguageSource.language_pk.in_(lids))
if params.get('doctypes'):
filtered = True
query = query.join(Refdoctype)\
.filter(Refdoctype.doctype_pk.in_([l.pk for l in params['doctypes']]))
if params.get('macroareas'):
filtered = True
query = query.join(Refmacroarea)\
.filter(Refmacroarea.macroarea_pk.in_([l.pk for l in params['macroareas']]))
if not filtered:
return []
return query.distinct()
def getRefs(params):
query = DBSession.query(Ref)
filtered = False
for param, value in params['biblio'].items():
if value:
filtered = True
query = query.filter(icontains(getattr(Ref, param), value))
if params.get('languoids'):
filtered = True
lids = DBSession.query(TreeClosureTable.child_pk)\
.filter(TreeClosureTable.parent_pk.in_([l.pk for l in params['languoids']]))\
.subquery()
query = query.join(LanguageSource, LanguageSource.source_pk == Ref.pk)\
.filter(LanguageSource.language_pk.in_(lids))
if params.get('doctypes'):
filtered = True
query = query.join(Refdoctype)\
.filter(Refdoctype.doctype_pk.in_([l.pk for l in params['doctypes']]))
if params.get('macroareas'):
filtered = True
query = query.join(Refmacroarea)\
.filter(Refmacroarea.macroarea_pk.in_([l.pk for l in params['macroareas']]))
if not filtered:
return []
return query.distinct()
def test_icontains(self):
from clld.db.util import icontains
from clld.db.models.common import Dataset
from clld.db.meta import DBSession
for qs, count in [('Se', 1), ('^d$', 0), ('^d', 1), ('setä$', 1)]:
q = DBSession.query(Dataset).filter(icontains(Dataset.name, qs))
self.assertEqual(q.count(), count)
def test_icontains(data):
from clld.db.util import icontains
from clld.db.models.common import Dataset
from clld.db.meta import DBSession
for qs, count in [('Se', 1), ('^d$', 0), ('^d', 1), ('setä$', 1), ('\\\\b', 0)]:
q = DBSession.query(Dataset).filter(icontains(Dataset.name, qs))
assert q.count() == count
def test_icontains(data):
from clld.db.util import icontains
from clld.db.models.common import Dataset
from clld.db.meta import DBSession
for qs, count in [('Se', 1), ('^d$', 0), ('^d', 1), ('setä$', 1),
('\\\\b', 0)]:
q = DBSession.query(Dataset).filter(icontains(Dataset.name, qs))
assert q.count() == count
def search(self, qs):
if self.model_col:
if isinstance(self.model_col.property.columns[0].type, (String, Unicode)):
if not hasattr(self, 'choices'):
return icontains(self.model_col, qs)
return self.model_col.__eq__(qs)
if isinstance(self.model_col.property.columns[0].type, (Float, Integer)):
return filter_number(self.model_col, qs)
if isinstance(self.model_col.property.columns[0].type, Boolean):
return self.model_col.__eq__(qs == 'True')
def search(self, qs):
"""Called when collecting the filter criteria of a datatable's search query."""
if isinstance(self.model_col_type, (String, Unicode)):
if getattr(self, 'choices', None):
# make sure select box values match sharp!
return self.model_col.__eq__(qs)
else:
return icontains(self.model_col, qs)
if isinstance(self.model_col_type, (Float, Integer)):
return filter_number(self.model_col, qs)
if isinstance(self.model_col_type, Boolean):
return self.model_col.__eq__(qs == 'True')
def search(self, qs):
"""Called when collecting the filter criteria of a datatable's search query."""
if isinstance(self.model_col_type, (String, Unicode)):
if getattr(self, 'choices', None):
# make sure select box values match sharp!
return self.model_col.__eq__(qs)
else:
return icontains(self.model_col, qs)
if isinstance(self.model_col_type, (Float, Integer)):
return filter_number(self.model_col, qs)
if isinstance(self.model_col_type, Boolean):
return self.model_col.__eq__(qs == 'True')
def childnodes(request):
if request.params.get('t') == 'select2':
query = DBSession.query(Languoid.id, Languoid.name, Languoid.level)\
.filter(icontains(Languoid.name, request.params.get('q')))
total = query.count()
ms = LanguoidsMultiSelect(request, None, None, url='x')
return dict(results=[ms.format_result(l) for l in query.limit(100)],
context={},
more=total > 500)
query = DBSession.query(
Languoid.pk,
Languoid.id,
Languoid.name,
Languoid.level,
func.count(TreeClosureTable.child_pk).label('children'))\
.filter(Language.pk == TreeClosureTable.parent_pk)\
.filter(Language.active == true())
if request.params.get('node'):
query = query.filter(Languoid.father_pk == int(request.params['node']))
else:
# narrow down selection of top-level nodes in the tree:
query = query.filter(Languoid.father_pk == null())
if request.params.get('q'):
query = query.filter(
Language.name.contains(request.params.get('q')))
query = query.group_by(Languoid.pk, Languoid.id, Languoid.name,
Languoid.level).order_by(Language.name)
return [
{
'label': ('%s (%s)' %
(l.name, l.children - 1)) if l.children > 1 else l.name,
'glottocode':
l.id,
'lname':
l.name,
'id':
l.pk,
'level':
l.level.value,
#'children': l.children
'load_on_demand':
l.children > 1
} for l in query
]
def update(args):
codes = {}
for lang in reader(args.data_file(DATA_FILE), namedtuples=True):
codes[lang.LangID] = 1
count = 0
for lang in DBSession.query(Languoid)\
.filter(Languoid.hid != None)\
.filter(not_(icontains(Languoid.hid, 'nocode'))):
if lang.hid in codes:
lang.update_jsondata(ethnologue=LANGUAGE_URL + lang.hid)
else:
lang.update_jsondata(ethnologue=None)
count += 1
print count, 'iso codes have no ethnologue code'
ethnologue = args.json
leafsets = defaultdict(list)
for id_, doc in ethnologue['docs'].items():
for sid, spec in get_classification(id_, doc).items():
leafs = sorted(set([p[0] for p in spec[2]]))
if leafs:
leafsets[tuple(leafs)].append(sid)
all = 0
matched = 0
for family in DBSession.query(Languoid)\
.filter(Languoid.level == LanguoidLevel.family)\
.filter(Language.active == True):
leafs = []
all += 1
for row in DBSession.query(TreeClosureTable.child_pk, Languoid.hid)\
.filter(TreeClosureTable.parent_pk == family.pk)\
.filter(TreeClosureTable.child_pk == Languoid.pk)\
.filter(Languoid.hid != None):
if len(row[1]) == 3:
leafs.append(row[1])
leafs = tuple(sorted(set(leafs)))
for i, subgroup in enumerate(leafsets.get(leafs, [])):
if i == 0:
matched += 1
family.update_jsondata(ethnologue=SUBGROUP_URL + subgroup)
break
print matched, 'of', all, 'families have an exact counterpart in ethnologue!'
def childnodes(request):
if request.params.get('t') == 'select2':
query = DBSession.query(Languoid.id, Languoid.name, Languoid.level)\
.filter(icontains(Languoid.name, request.params.get('q')))
total = query.count()
ms = LanguoidsMultiSelect(request, None, None, url='x')
return dict(
results=[ms.format_result(l) for l in query.limit(100)],
context={},
more=total > 500)
query = DBSession.query(
Languoid.pk,
Languoid.id,
Languoid.name,
Languoid.level,
func.count(TreeClosureTable.child_pk).label('children'))\
.filter(Language.pk == TreeClosureTable.parent_pk)\
.filter(Language.active == true())
if request.params.get('node'):
query = query.filter(Languoid.father_pk == int(request.params['node']))
else:
# narrow down selection of top-level nodes in the tree:
query = query.filter(Languoid.father_pk == null())
if request.params.get('q'):
query = query.filter(Language.name.contains(request.params.get('q')))
query = query.group_by(
Languoid.pk,
Languoid.id,
Languoid.name,
Languoid.level).order_by(Language.name)
return [{
'label': ('%s (%s)' % (l.name, l.children - 1))
if l.children > 1 else l.name,
'glottocode': l.id,
'lname': l.name,
'id': l.pk,
'level': l.level.value,
#'children': l.children
'load_on_demand': l.children > 1} for l in query]
def getLanguoids(name=False,
iso=False,
namequerytype='part',
country=False,
multilingual=False,
inactive=False):
"""return an array of languoids responding to the specified criterion.
"""
if not (name or iso or country):
return []
query = DBSession.query(Languoid)\
.options(joinedload(Languoid.family))\
.order_by(Languoid.name)
if not inactive:
query = query.filter(Language.active == True)
if name:
crit = [Identifier.type == 'name']
ul_iname = func.unaccent(func.lower(Identifier.name))
ul_name = func.unaccent(name.lower())
if namequerytype == 'whole':
crit.append(ul_iname == ul_name)
else:
crit.append(ul_iname.contains(ul_name))
if not multilingual:
crit.append(
func.coalesce(Identifier.lang, '').in_((u'', u'eng', u'en')))
crit = Language.identifiers.any(and_(*crit))
query = query.filter(or_(icontains(Languoid.name, name), crit))
elif country:
return [] # pragma: no cover
else:
query = query.join(LanguageIdentifier, Identifier)\
.filter(Identifier.type == IdentifierType.iso.value)\
.filter(Identifier.name.contains(iso.lower()))
return query
def getRefs(params):
query = DBSession.query(Ref)
filtered = False
for param, value in params['biblio'].items():
if value:
filtered = True
query = query.filter(icontains(getattr(Ref, param), value))
if params.get('languoids'):
filtered = True
subquery = DBSession.query(LanguageSource).filter_by(source_pk=Ref.pk)\
.join(TreeClosureTable,
TreeClosureTable.child_pk == LanguageSource.language_pk)\
.filter(TreeClosureTable.parent_pk.in_(
[l.pk for l in params['languoids']]))
query = query.filter(subquery.exists())
if params.get('doctypes'):
filtered = True
subquery = DBSession.query(Refdoctype).filter_by(ref_pk=Ref.pk)\
.filter(Refdoctype.doctype_pk.in_(
[d.pk for d in params['doctypes']]))
query = query.filter(subquery.exists())
if params.get('macroareas'):
filtered = True
names = [
getattr(ma, 'name', None) or DomainElement.get(ma).name
for ma in params['macroareas']
]
query = query.filter(or_(*[Ref.macroareas.contains(n) for n in names]))
if not filtered:
return DBSession.query(Ref).filter(Ref.pk == -1)
return query
def getLanguoids(name=False,
iso=False,
namequerytype='part',
country=False,
multilingual=False,
inactive=False):
"""return an array of languoids responding to the specified criterion.
"""
if not (name or iso or country):
return []
query = DBSession.query(Languoid)\
.options(joinedload(Languoid.family))\
.order_by(Languoid.name)
if not inactive:
query = query.filter(Language.active == True)
if name:
crit = [Identifier.type == 'name']
ul_iname = func.unaccent(func.lower(Identifier.name))
ul_name = func.unaccent(name.lower())
if namequerytype == 'whole':
crit.append(ul_iname == ul_name)
else:
crit.append(ul_iname.contains(ul_name))
if not multilingual:
crit.append(func.coalesce(Identifier.lang, '').in_((u'', u'eng', u'en')))
crit = Language.identifiers.any(and_(*crit))
query = query.filter(or_(icontains(Languoid.name, name), crit))
elif country:
return [] # pragma: no cover
else:
query = query.join(LanguageIdentifier, Identifier)\
.filter(Identifier.type == IdentifierType.iso.value)\
.filter(Identifier.name.contains(iso.lower()))
return query
def getRefs(params):
query = DBSession.query(Ref)
filtered = False
for param, value in params['biblio'].items():
if value:
filtered = True
query = query.filter(icontains(getattr(Ref, param), value))
if params.get('languoids'):
filtered = True
subquery = DBSession.query(LanguageSource).filter_by(source_pk=Ref.pk)\
.join(TreeClosureTable,
TreeClosureTable.child_pk == LanguageSource.language_pk)\
.filter(TreeClosureTable.parent_pk.in_(
[l.pk for l in params['languoids']]))
query = query.filter(subquery.exists())
if params.get('doctypes'):
filtered = True
subquery = DBSession.query(Refdoctype).filter_by(ref_pk=Ref.pk)\
.filter(Refdoctype.doctype_pk.in_(
[d.pk for d in params['doctypes']]))
query = query.filter(subquery.exists())
if params.get('macroareas'):
filtered = True
subquery = DBSession.query(Refmacroarea).filter_by(ref_pk=Ref.pk)\
.filter(Refmacroarea.macroarea_pk.in_(
[m.pk for m in params['macroareas']]))
query = query.filter(subquery.exists())
if not filtered:
return []
return query
def search(self, qs):
return icontains(Country.name, qs)
def search(self, qs):
return icontains(Counterpart.phonetic, qs)
def search(self, qs):
return icontains(self.alias.description, qs)
def search(self, qs):
return icontains(Feature.id, qs)
def search(self, qs):
if self.dt.parameter and self.dt.parameter.domain:
return DomainElement.name.__eq__(qs)
return icontains(Value.description, qs)
def search(self, qs):
return and_(Identifier.type.__eq__(IdentifierType.iso.value),
icontains(Identifier.name, qs))
def search(self, qs):
return icontains(Refprovider.id, qs)
def lquery():
for lang in DBSession.query(Languoid)\
.filter(Languoid.hid != None)\
.filter(not_(icontains(Languoid.hid, 'nocode'))):
if len(lang.hid) == 3:
yield lang
def search(self, qs):
return WalsLanguage.countries.any(icontains(Country.name, qs))
def search(self, qs):
return icontains(common.DomainElement.name, qs)
def search(self, qs):
return icontains(getattr(amsd.models, self.name).name, qs)
def search(self, qs):
return icontains(Country.name, qs)
def search(self, qs):
return icontains(Language.name, qs)
def search(self, qs):
return icontains(models.Variety.canton, qs)
def search(self, qs):
return icontains(self.dt.vars[self.index].value, qs)
def search(self, qs):
if self.dt.language:
return ValueSet.language_pk == int(qs)
if self.dt.parameter:
return icontains(self.dt.vs_lang.name, qs)
def search(self, qs):
if self.dt.parameter and self.dt.parameter.domain:
return DomainElement.name.__eq__(qs)
return icontains(Value.description, qs)
def search(self, qs):
return icontains(self._col, qs)
def search(self, qs):
return icontains(common.Value.name, qs)
def search(self, qs):
return icontains(Contributor.name, qs)
def search(self, qs):
return icontains(self.dt.top_level_family.name, qs)
def search(self, qs):
return icontains(Genus.description, qs)
def search(self, qs):
return icontains(Conceptlist.source_languages, qs)
def search(self, qs):
return icontains(Genus.name, qs)
def search(self, qs):
return icontains(Value.name, qs)
def search(self, qs):
return or_(
icontains(Word.name, qs),
func.unaccent(Word.name).contains(func.unaccent(qs)))
def search(self, qs):
return icontains(self.alias.value, qs)
def search(self, qs):
return icontains(self.dt.vars[self.name].value, qs)
def search(self, qs):
return icontains(self.model_col, qs)
def search(self, qs):
return icontains(Word.semantic_domain, qs)
def search(self, qs):
return icontains(common.DomainElement.name, qs)
def search(self, qs):
return icontains(Word.name, qs)
def search(self, qs):
return icontains(self.dt.top_level_family.name, qs)
def search(self, qs):
return icontains(common.Parameter.name, qs)
def search(self, qs):
if self.dt.parameter and self.dt.parameter.name == 'Base':
return icontains(DomainElement.name, qs)
else:
return icontains(Value.name, qs)
def search(self, qs):
return icontains(Language.name, qs)
def search(self, qs):
return icontains(Parameter.name, qs)
def search(self, qs):
return icontains(Contribution.name, qs)
