def languoids(request): if request.params.get('id'): if '-' not in request.params['id']: return HTTPNotFound() m, id_ = request.params['id'].split('-', 1) model = dict(w=Language, g=Genus, f=Family).get(m) if not model: return HTTPNotFound() obj = model.get(id_, default=None) if not obj: return HTTPNotFound() return HTTPFound(location=request.resource_url(obj)) max_results = 20 qs = request.params.get('q') if not qs: return [] query = DBSession.query(Language)\ .filter(icontains(Language.name, qs))\ .order_by(WalsLanguage.ascii_name).limit(max_results) res = [l for l in query] if len(res) < max_results: max_results = max_results - len(res) # fill up suggestions with matching alternative names: for l in DBSession.query(Language)\ .join(Language.languageidentifier, LanguageIdentifier.identifier)\ .filter(icontains(Identifier.name, qs))\ .order_by(WalsLanguage.ascii_name).limit(max_results): if l not in res: res.append(l) if len(res) < max_results: max_results = max_results - len(res) # fill up with matching genera: for l in DBSession.query(Genus)\ .filter(icontains(Genus.name, qs))\ .order_by(Genus.name).limit(max_results): res.append(l) if len(res) < max_results: max_results = max_results - len(res) # fill up with matching families: for l in DBSession.query(Family)\ .filter(icontains(Family.name, qs))\ .order_by(Family.name).limit(max_results): res.append(l) ms = LanguoidSelect(request, None, None, url='x') return dict(results=list(map(ms.format_result, res)), context={}, more=False)
def getRefs(params): query = DBSession.query(Ref) filtered = False for param, value in params['biblio'].items(): if value: filtered = True query = query.filter(icontains(getattr(Ref, param), value)) if params.get('languoids'): filtered = True lids = DBSession.query(TreeClosureTable.child_pk)\ .filter(TreeClosureTable.parent_pk.in_([l.pk for l in params['languoids']]))\ .subquery() query = query.join(LanguageSource, LanguageSource.source_pk == Ref.pk)\ .filter(LanguageSource.language_pk.in_(lids)) if params.get('doctypes'): filtered = True query = query.join(Refdoctype)\ .filter(Refdoctype.doctype_pk.in_([l.pk for l in params['doctypes']])) if params.get('macroareas'): filtered = True query = query.join(Refmacroarea)\ .filter(Refmacroarea.macroarea_pk.in_([l.pk for l in params['macroareas']])) if not filtered: return [] return query.distinct()
def test_icontains(self): from clld.db.util import icontains from clld.db.models.common import Dataset from clld.db.meta import DBSession for qs, count in [('Se', 1), ('^d$', 0), ('^d', 1), ('setä$', 1)]: q = DBSession.query(Dataset).filter(icontains(Dataset.name, qs)) self.assertEqual(q.count(), count)
def test_icontains(data): from clld.db.util import icontains from clld.db.models.common import Dataset from clld.db.meta import DBSession for qs, count in [('Se', 1), ('^d$', 0), ('^d', 1), ('setä$', 1), ('\\\\b', 0)]: q = DBSession.query(Dataset).filter(icontains(Dataset.name, qs)) assert q.count() == count
def search(self, qs): if self.model_col: if isinstance(self.model_col.property.columns[0].type, (String, Unicode)): if not hasattr(self, 'choices'): return icontains(self.model_col, qs) return self.model_col.__eq__(qs) if isinstance(self.model_col.property.columns[0].type, (Float, Integer)): return filter_number(self.model_col, qs) if isinstance(self.model_col.property.columns[0].type, Boolean): return self.model_col.__eq__(qs == 'True')
def search(self, qs): """Called when collecting the filter criteria of a datatable's search query.""" if isinstance(self.model_col_type, (String, Unicode)): if getattr(self, 'choices', None): # make sure select box values match sharp! return self.model_col.__eq__(qs) else: return icontains(self.model_col, qs) if isinstance(self.model_col_type, (Float, Integer)): return filter_number(self.model_col, qs) if isinstance(self.model_col_type, Boolean): return self.model_col.__eq__(qs == 'True')
def childnodes(request): if request.params.get('t') == 'select2': query = DBSession.query(Languoid.id, Languoid.name, Languoid.level)\ .filter(icontains(Languoid.name, request.params.get('q'))) total = query.count() ms = LanguoidsMultiSelect(request, None, None, url='x') return dict(results=[ms.format_result(l) for l in query.limit(100)], context={}, more=total > 500) query = DBSession.query( Languoid.pk, Languoid.id, Languoid.name, Languoid.level, func.count(TreeClosureTable.child_pk).label('children'))\ .filter(Language.pk == TreeClosureTable.parent_pk)\ .filter(Language.active == true()) if request.params.get('node'): query = query.filter(Languoid.father_pk == int(request.params['node'])) else: # narrow down selection of top-level nodes in the tree: query = query.filter(Languoid.father_pk == null()) if request.params.get('q'): query = query.filter( Language.name.contains(request.params.get('q'))) query = query.group_by(Languoid.pk, Languoid.id, Languoid.name, Languoid.level).order_by(Language.name) return [ { 'label': ('%s (%s)' % (l.name, l.children - 1)) if l.children > 1 else l.name, 'glottocode': l.id, 'lname': l.name, 'id': l.pk, 'level': l.level.value, #'children': l.children 'load_on_demand': l.children > 1 } for l in query ]
def update(args): codes = {} for lang in reader(args.data_file(DATA_FILE), namedtuples=True): codes[lang.LangID] = 1 count = 0 for lang in DBSession.query(Languoid)\ .filter(Languoid.hid != None)\ .filter(not_(icontains(Languoid.hid, 'nocode'))): if lang.hid in codes: lang.update_jsondata(ethnologue=LANGUAGE_URL + lang.hid) else: lang.update_jsondata(ethnologue=None) count += 1 print count, 'iso codes have no ethnologue code' ethnologue = args.json leafsets = defaultdict(list) for id_, doc in ethnologue['docs'].items(): for sid, spec in get_classification(id_, doc).items(): leafs = sorted(set([p[0] for p in spec[2]])) if leafs: leafsets[tuple(leafs)].append(sid) all = 0 matched = 0 for family in DBSession.query(Languoid)\ .filter(Languoid.level == LanguoidLevel.family)\ .filter(Language.active == True): leafs = [] all += 1 for row in DBSession.query(TreeClosureTable.child_pk, Languoid.hid)\ .filter(TreeClosureTable.parent_pk == family.pk)\ .filter(TreeClosureTable.child_pk == Languoid.pk)\ .filter(Languoid.hid != None): if len(row[1]) == 3: leafs.append(row[1]) leafs = tuple(sorted(set(leafs))) for i, subgroup in enumerate(leafsets.get(leafs, [])): if i == 0: matched += 1 family.update_jsondata(ethnologue=SUBGROUP_URL + subgroup) break print matched, 'of', all, 'families have an exact counterpart in ethnologue!'
def childnodes(request): if request.params.get('t') == 'select2': query = DBSession.query(Languoid.id, Languoid.name, Languoid.level)\ .filter(icontains(Languoid.name, request.params.get('q'))) total = query.count() ms = LanguoidsMultiSelect(request, None, None, url='x') return dict( results=[ms.format_result(l) for l in query.limit(100)], context={}, more=total > 500) query = DBSession.query( Languoid.pk, Languoid.id, Languoid.name, Languoid.level, func.count(TreeClosureTable.child_pk).label('children'))\ .filter(Language.pk == TreeClosureTable.parent_pk)\ .filter(Language.active == true()) if request.params.get('node'): query = query.filter(Languoid.father_pk == int(request.params['node'])) else: # narrow down selection of top-level nodes in the tree: query = query.filter(Languoid.father_pk == null()) if request.params.get('q'): query = query.filter(Language.name.contains(request.params.get('q'))) query = query.group_by( Languoid.pk, Languoid.id, Languoid.name, Languoid.level).order_by(Language.name) return [{ 'label': ('%s (%s)' % (l.name, l.children - 1)) if l.children > 1 else l.name, 'glottocode': l.id, 'lname': l.name, 'id': l.pk, 'level': l.level.value, #'children': l.children 'load_on_demand': l.children > 1} for l in query]
def getLanguoids(name=False, iso=False, namequerytype='part', country=False, multilingual=False, inactive=False): """return an array of languoids responding to the specified criterion. """ if not (name or iso or country): return [] query = DBSession.query(Languoid)\ .options(joinedload(Languoid.family))\ .order_by(Languoid.name) if not inactive: query = query.filter(Language.active == True) if name: crit = [Identifier.type == 'name'] ul_iname = func.unaccent(func.lower(Identifier.name)) ul_name = func.unaccent(name.lower()) if namequerytype == 'whole': crit.append(ul_iname == ul_name) else: crit.append(ul_iname.contains(ul_name)) if not multilingual: crit.append( func.coalesce(Identifier.lang, '').in_((u'', u'eng', u'en'))) crit = Language.identifiers.any(and_(*crit)) query = query.filter(or_(icontains(Languoid.name, name), crit)) elif country: return [] # pragma: no cover else: query = query.join(LanguageIdentifier, Identifier)\ .filter(Identifier.type == IdentifierType.iso.value)\ .filter(Identifier.name.contains(iso.lower())) return query
def getRefs(params): query = DBSession.query(Ref) filtered = False for param, value in params['biblio'].items(): if value: filtered = True query = query.filter(icontains(getattr(Ref, param), value)) if params.get('languoids'): filtered = True subquery = DBSession.query(LanguageSource).filter_by(source_pk=Ref.pk)\ .join(TreeClosureTable, TreeClosureTable.child_pk == LanguageSource.language_pk)\ .filter(TreeClosureTable.parent_pk.in_( [l.pk for l in params['languoids']])) query = query.filter(subquery.exists()) if params.get('doctypes'): filtered = True subquery = DBSession.query(Refdoctype).filter_by(ref_pk=Ref.pk)\ .filter(Refdoctype.doctype_pk.in_( [d.pk for d in params['doctypes']])) query = query.filter(subquery.exists()) if params.get('macroareas'): filtered = True names = [ getattr(ma, 'name', None) or DomainElement.get(ma).name for ma in params['macroareas'] ] query = query.filter(or_(*[Ref.macroareas.contains(n) for n in names])) if not filtered: return DBSession.query(Ref).filter(Ref.pk == -1) return query
def getLanguoids(name=False, iso=False, namequerytype='part', country=False, multilingual=False, inactive=False): """return an array of languoids responding to the specified criterion. """ if not (name or iso or country): return [] query = DBSession.query(Languoid)\ .options(joinedload(Languoid.family))\ .order_by(Languoid.name) if not inactive: query = query.filter(Language.active == True) if name: crit = [Identifier.type == 'name'] ul_iname = func.unaccent(func.lower(Identifier.name)) ul_name = func.unaccent(name.lower()) if namequerytype == 'whole': crit.append(ul_iname == ul_name) else: crit.append(ul_iname.contains(ul_name)) if not multilingual: crit.append(func.coalesce(Identifier.lang, '').in_((u'', u'eng', u'en'))) crit = Language.identifiers.any(and_(*crit)) query = query.filter(or_(icontains(Languoid.name, name), crit)) elif country: return [] # pragma: no cover else: query = query.join(LanguageIdentifier, Identifier)\ .filter(Identifier.type == IdentifierType.iso.value)\ .filter(Identifier.name.contains(iso.lower())) return query
def getRefs(params): query = DBSession.query(Ref) filtered = False for param, value in params['biblio'].items(): if value: filtered = True query = query.filter(icontains(getattr(Ref, param), value)) if params.get('languoids'): filtered = True subquery = DBSession.query(LanguageSource).filter_by(source_pk=Ref.pk)\ .join(TreeClosureTable, TreeClosureTable.child_pk == LanguageSource.language_pk)\ .filter(TreeClosureTable.parent_pk.in_( [l.pk for l in params['languoids']])) query = query.filter(subquery.exists()) if params.get('doctypes'): filtered = True subquery = DBSession.query(Refdoctype).filter_by(ref_pk=Ref.pk)\ .filter(Refdoctype.doctype_pk.in_( [d.pk for d in params['doctypes']])) query = query.filter(subquery.exists()) if params.get('macroareas'): filtered = True subquery = DBSession.query(Refmacroarea).filter_by(ref_pk=Ref.pk)\ .filter(Refmacroarea.macroarea_pk.in_( [m.pk for m in params['macroareas']])) query = query.filter(subquery.exists()) if not filtered: return [] return query
def search(self, qs): return icontains(Country.name, qs)
def search(self, qs): return icontains(Counterpart.phonetic, qs)
def search(self, qs): return icontains(self.alias.description, qs)
def search(self, qs): return icontains(Feature.id, qs)
def search(self, qs): if self.dt.parameter and self.dt.parameter.domain: return DomainElement.name.__eq__(qs) return icontains(Value.description, qs)
def search(self, qs): return and_(Identifier.type.__eq__(IdentifierType.iso.value), icontains(Identifier.name, qs))
def search(self, qs): return icontains(Refprovider.id, qs)
def lquery(): for lang in DBSession.query(Languoid)\ .filter(Languoid.hid != None)\ .filter(not_(icontains(Languoid.hid, 'nocode'))): if len(lang.hid) == 3: yield lang
def search(self, qs): return WalsLanguage.countries.any(icontains(Country.name, qs))
def search(self, qs): return icontains(common.DomainElement.name, qs)
def search(self, qs): return icontains(getattr(amsd.models, self.name).name, qs)
def search(self, qs): return icontains(Language.name, qs)
def search(self, qs): return icontains(models.Variety.canton, qs)
def search(self, qs): return icontains(self.dt.vars[self.index].value, qs)
def search(self, qs): if self.dt.language: return ValueSet.language_pk == int(qs) if self.dt.parameter: return icontains(self.dt.vs_lang.name, qs)
def search(self, qs): return icontains(self._col, qs)
def search(self, qs): return icontains(common.Value.name, qs)
def search(self, qs): return icontains(Contributor.name, qs)
def search(self, qs): return icontains(self.dt.top_level_family.name, qs)
def search(self, qs): return icontains(Genus.description, qs)
def search(self, qs): return icontains(Conceptlist.source_languages, qs)
def search(self, qs): return icontains(Genus.name, qs)
def search(self, qs): return icontains(Value.name, qs)
def search(self, qs): return or_( icontains(Word.name, qs), func.unaccent(Word.name).contains(func.unaccent(qs)))
def search(self, qs): return icontains(self.alias.value, qs)
def search(self, qs): return icontains(self.dt.vars[self.name].value, qs)
def search(self, qs): return icontains(self.model_col, qs)
def search(self, qs): return icontains(Word.semantic_domain, qs)
def search(self, qs): return icontains(Word.name, qs)
def search(self, qs): return icontains(common.Parameter.name, qs)
def search(self, qs): if self.dt.parameter and self.dt.parameter.name == 'Base': return icontains(DomainElement.name, qs) else: return icontains(Value.name, qs)
def search(self, qs): return icontains(Parameter.name, qs)
def search(self, qs): return icontains(Contribution.name, qs)