def issue24(session, timestamp): # pragma: no cover #- Update language cea (name, coords, alternative names, iso code (and name)) #Change name of Cree (Eastern) to Cree (Swampy) #Change coordinates to 56dN, 90dW #Change the Ethnologue name to Cree (Swampy) #Remove the Routledge and Other names #Change the ISO code to csw. glottocode to swam1239 cea = common.Language.get('cea', session=session) cre = common.Language.get('cre', session=session) for i in range(len(cea.languageidentifier)): try: del cea.languageidentifier[i] except IndexError: pass for values in [ ('gc-csw', 'swam1239', 'Swampy Cree', 'glottolog'), ('csw', 'csw', 'Cree (Swampy)', 'iso639-3'), ('ethnologue-csw', 'Cree (Swampy)', 'ethnologue', 'name'), ]: id = common.Identifier( **dict(zip('id name description type'.split(), values))) cea.languageidentifier.append( common.LanguageIdentifier(language=cea, identifier=id)) cea.updated = timestamp cea.name = 'Cree (Swampy)' cea.ascii_name = slug('Cree (Swampy)') cea.latitude = 56.0 cea.longitude = -90.0 for pid in ['81A', '82A', '83A']: vsq = session.query(common.ValueSet)\ .join(common.Parameter)\ .filter(common.Parameter.id == pid) vs1 = vsq.filter(common.ValueSet.language_pk == cea.pk).one() vs2 = vsq.filter(common.ValueSet.language_pk == cre.pk).one() vs2.updated = timestamp vs1.updated = timestamp for ref in vs1.references: if ref.source.id == 'Hive-1948': ref.valueset = vs2 session.flush() #- Delete valueset 85A-cea vs = session.query(common.ValueSet)\ .join(common.Parameter)\ .filter(common.Parameter.id == '85A')\ .filter(common.ValueSet.language_pk == cea.pk).one() session.delete(vs.values[0]) session.delete(vs.references[0]) session.delete(vs) #- delete valueset 131A-cea add 131A-cre vs_switch_lang(session, timestamp, '131A-cea', 'cre')
def update_iso(session, timestamp, lang, *obsolete, **new): # pragma: no cover if isinstance(lang, basestring): lang = common.Language.get(lang, session=session) lang.updated = timestamp for code in obsolete: for li in lang.languageidentifier: if li.identifier.id == code or li.identifier.id == 'ethnologue-' + code: session.delete(li) for code, name in new.items(): iso = common.Identifier.get(code, session=session, default=None) ethnologue = common.Identifier.get('ethnologue-' + code, session=session, default=None) if not iso: iso = common.Identifier(id=code, name=code, description=name, type=common.IdentifierType.iso.value) if not ethnologue: ethnologue = common.Identifier(id='ethnologue-' + code, name=name, description='ethnologue', type='name') if iso.id not in [li.identifier.id for li in lang.languageidentifier]: session.add( common.LanguageIdentifier(language=lang, identifier=iso)) if ethnologue.id not in [ li.identifier.id for li in lang.languageidentifier ]: session.add( common.LanguageIdentifier(language=lang, identifier=ethnologue)) return lang
def add_codes(lang): for attr, prefix in dict(wals='wals_code_', iso='', glottolog='').items(): code = getattr(lang, 'code_' + attr) if code: if attr == 'iso' and not re.match('[a-z]{3}$', code): continue id_ = prefix + code identifier = common.Identifier.get(id_, default=None) if not identifier: identifier = common.Identifier(id=id_, name=code, type=getattr( common.IdentifierType, attr).value) common.LanguageIdentifier(identifier=identifier, language=lang)
def update_language(session, timestamp, lang, keep_old_name=False, **kw): # pragma: no cover if isinstance(lang, basestring): lang = common.Language.get(lang, session=session) if 'name' in kw and keep_old_name: name = common.Identifier(id=str(uuid1()), name=lang.name, description='other', type='name') session.add(common.LanguageIdentifier(language=lang, identifier=name)) return update_obj(session, timestamp, lang, **kw)
def add_new_name(args): # TODO: Check the length of the args array is valid gcode, lang, name, type, desc = \ args.args[0], args.args[1], args.args[2], args.args[3], args.args[4] with_session(args) with transaction.manager: languoid = DBSession.query(common.Language) \ .filter_by(id='{0}'.format(gcode)) \ .first() identifier = common.Identifier( (name, type, desc, lang), id='{0}-{1}-{2}-{3}'.format(slug(name), slug(type), slug(desc or ''), lang), name=name, type=type, description=desc, lang=lang) DBSession.add(identifier) DBSession.add( common.LanguageIdentifier(language=languoid, identifier=identifier))
def update(args): count = 0 assert args.json iid = int( DBSession.execute("select max(cast(id as integer)) from identifier"). fetchone()[0]) + 1 pk = DBSession.execute("select max(pk) from identifier").fetchone()[0] + 1 langs = {} for gid, name in args.json['wikipedia'].items(): if gid not in langs: langs[gid] = Languoid.get(gid) langs[gid].update_jsondata(wikipedia=name.split('/')[-1]) for gid, codes in args.json['multitree'].items(): l = langs[gid] lcodes = [i.name for i in l.identifiers if i.type == 'multitree'] for code in set(codes): if code not in lcodes: identifier = DBSession.query(common.Identifier)\ .filter(common.Identifier.type == 'multitree')\ .filter(common.Identifier.name == code)\ .first() if not identifier: identifier = common.Identifier(pk=pk, id=str(iid), name=code, type='multitree') iid += 1 pk += 1 count += 1 DBSession.add( common.LanguageIdentifier(language=l, identifier=identifier)) print count, 'new multitree identifiers'
def update_glottocode(session, timestamp, lang, gc): # pragma: no cover if isinstance(lang, basestring): lang = common.Language.get(lang, session=session) lang.updated = timestamp for li in lang.languageidentifier: if li.identifier.name == gc and li.identifier.type == common.IdentifierType.glottolog.value: session.delete(li) glottocode = session.query(common.Identifier)\ .filter(common.Identifier.type == common.IdentifierType.glottolog.value)\ .filter(common.Identifier.name == gc)\ .first() if not glottocode: glottocode = common.Identifier( id=gc, name=gc, description=gc, type=common.IdentifierType.glottolog.value) session.add(common.LanguageIdentifier(language=lang, identifier=glottocode)) return lang
def setUp(self): TestWithDb.setUp(self) DBSession.add( common.Dataset(id='dataset', name='dataset', description='desc', domain='clld')) source = common.Source(id='source') contributors = { 'contributor': 'A Name', 'b': 'b Name', 'c': 'c Name', 'd': 'd Name' } for id_, name in contributors.items(): contributors[id_] = common.Contributor(id=id_, name=name) contribution = common.Contribution(id='contribution', name='Contribution') cr = common.ContributionReference(contribution=contribution, source=source) assert common.ContributionContributor( contribution=contribution, primary=True, contributor=contributors['contributor']) assert common.ContributionContributor(contribution=contribution, primary=False, contributor=contributors['b']) assert common.ContributionContributor(contribution=contribution, primary=True, contributor=contributors['c']) assert common.ContributionContributor(contribution=contribution, primary=False, contributor=contributors['d']) DBSession.add(contribution) language = common.Language(id='language', name='Language 1', latitude=10.5, longitude=0.3) language.sources.append(source) identifier = common.Identifier(type='iso639-3', id='iso') li = common.LanguageIdentifier(language=language, identifier=identifier) for i in range(2, 102): _l = common.Language(id='l%s' % i, name='Language %s' % i) _i = common.Identifier(type='iso639-3', id='%.3i' % i, name='%.3i' % i) _li = common.LanguageIdentifier(language=_l, identifier=_i) DBSession.add(_l) param = common.Parameter(id='parameter', name='Parameter') de = common.DomainElement(id='de', name='DomainElement', parameter=param) de2 = common.DomainElement(id='de2', name='DomainElement2', parameter=param) valueset = common.ValueSet(id='valueset', language=language, parameter=param, contribution=contribution) value = common.Value(id='value', domainelement=de, valueset=valueset, frequency=50, confidence='high') DBSession.add(value) paramnd = common.Parameter(id='no-domain', name='Parameter without domain') valueset = common.ValueSet(id='vs2', language=language, parameter=paramnd, contribution=contribution) vr = common.ValueSetReference(valueset=valueset, source=source) value = common.Value(id='v2', valueset=valueset, frequency=50, confidence='high') DBSession.add(value) unit = common.Unit(id='unit', name='Unit', language=language) up = common.UnitParameter(id='unitparameter', name='UnitParameter') DBSession.add(unit) DBSession.add( common.UnitValue(id='unitvalue', name='UnitValue', unit=unit, unitparameter=up)) up2 = common.UnitParameter(id='up2', name='UnitParameter with domain') de = common.UnitDomainElement(id='de', name='de', parameter=up2) DBSession.add( common.UnitValue(id='uv2', name='UnitValue2', unit=unit, unitparameter=up2, unitdomainelement=de)) DBSession.add(common.Source(id='s')) sentence = common.Sentence(id='sentence', name='sentence name', description='sentence description', analyzed='a\tmorpheme\tdoes\tdo', gloss='a\tmorpheme\t1SG\tdo.SG2', source='own', comment='comment', original_script='a morpheme', language=language) sr = common.SentenceReference(sentence=sentence, source=source) DBSession.add(common.Config(key='key', value='value')) DBSession.flush()
def glottolog_url(glottocode): return models.Identifier(name=glottocode, type='glottolog').url()
def populate_test_db(engine): set_alembic_version(engine, '58559d4eea0d') data = TestData() data.add_default(common.Dataset, domain='clld', jsondata={ 'license_icon': 'cc-by', 'license_url': 'http://example.org' }) data.add_default(common.Contributor, name='A Name', email='*****@*****.**') for id_, name in { 'b': 'b Name', 'c': 'c Name', 'd': 'd Name', }.items(): data.add(common.Contributor, id_, id=id_, name=name, url='http://example.org') DBSession.add( common.Editor(dataset=data[common.Dataset], contributor=data[common.Contributor])) data.add_default(common.Source) data.add(common.Source, 'replaced', id='replaced', active=False, jsondata={'__replacement_id__': 'source'}) data.add_default(common.Contribution) common.ContributionReference(contribution=data[common.Contribution], source=data[common.Source]) for primary, c in [(True, 'contributor'), (False, 'b'), (True, 'c'), (False, 'd')]: common.ContributionContributor(contribution=data[common.Contribution], primary=primary, contributor=data['Contributor'][c]) data.add_default(common.Language, latitude=10.5, longitude=0.3) data[common.Language].sources.append(data[common.Source]) for i, type_ in enumerate(common.IdentifierType): common.LanguageIdentifier( language=data[common.Language], identifier=common.Identifier( type=type_.value, id=type_.value + str(i), name='abc' if type_.name == 'iso' else 'glot1234')) common.LanguageIdentifier(language=data[common.Language], identifier=common.Identifier(type='name', id='name', name='a')) for i in range(2, 102): _l = common.Language(id='l%s' % i, name='Language %s' % i) _i = common.Identifier(type='iso639-3', id='%.3i' % i, name='abc') common.LanguageIdentifier(language=_l, identifier=_i) DBSession.add(_l) param = data.add_default(common.Parameter) de = common.DomainElement(id='de', name='DomainElement', parameter=param) de2 = common.DomainElement(id='de2', name='DomainElement2', parameter=param) valueset = data.add_default(common.ValueSet, language=data[common.Language], parameter=param, contribution=data[common.Contribution]) common.ValueSetReference(valueset=valueset, source=data[common.Source], description='10-20') data.add_default(common.Value, domainelement=de, valueset=valueset, frequency=50, confidence='high') data.add(common.Value, 'value2', id='value2', domainelement=de2, valueset=valueset, frequency=50, confidence='high') paramnd = data.add(common.Parameter, 'no-domain', id='no-domain', name='Parameter without domain') valueset = common.ValueSet(id='vs2', language=data[common.Language], parameter=paramnd, contribution=data[common.Contribution]) common.ValueSetReference(valueset=valueset, source=data[common.Source], description='10-20') common.Value(id='v2', valueset=valueset, frequency=50, confidence='high') unit = data.add_default(common.Unit, language=data[common.Language]) up = data.add_default(common.UnitParameter) common.UnitValue(id='unitvalue', name='UnitValue', unit=unit, unitparameter=up) up2 = common.UnitParameter(id='up2', name='UnitParameter with domain') de = common.UnitDomainElement(id='de', name='de', parameter=up2) DBSession.add( common.UnitValue(id='uv2', name='UnitValue2', unit=unit, unitparameter=up2, unitdomainelement=de)) DBSession.add(common.Source(id='s')) sentence = data.add_default(common.Sentence, description='sentence description', analyzed='a\tmorpheme\tdoes\tdo', gloss='a\tmorpheme\t1SG\tdo.SG2', source='own', comment='comment', original_script='a morpheme', language=data[common.Language], jsondata={'alt_translation': 'Spanish: ...'}) common.SentenceReference(sentence=sentence, source=data[common.Source]) DBSession.add(common.Config(key='key', value='value')) common.Config.add_replacement('replaced', 'language', model=common.Language) common.Config.add_replacement('gone', None, model=common.Language) DBSession.flush()
def setUp(self): TestWithDb.setUp(self) DBSession.add( common.Dataset(id='dataset', name='dataset', description='desc', domain='clld', jsondata={'license_icon': 'cc-by'})) DBSession.add( common.Source(id='replaced', active=False, jsondata={'__replacement_id__': 'source'})) source = common.Source(id='source') contributors = { 'contributor': 'A Name', 'b': 'b Name', 'c': 'c Name', 'd': 'd Name' } for id_, name in contributors.items(): contributors[id_] = common.Contributor(id=id_, name=name, url='http://example.org') contribution = common.Contribution(id='contribution', name='Contribution') common.ContributionReference(contribution=contribution, source=source) assert common.ContributionContributor( contribution=contribution, primary=True, contributor=contributors['contributor']) assert common.ContributionContributor(contribution=contribution, primary=False, contributor=contributors['b']) assert common.ContributionContributor(contribution=contribution, primary=True, contributor=contributors['c']) assert common.ContributionContributor(contribution=contribution, primary=False, contributor=contributors['d']) DBSession.add(contribution) language = common.Language(id='language', name='Language 1', latitude=10.5, longitude=0.3) language.sources.append(source) for i, type_ in enumerate(common.IdentifierType): id_ = common.Identifier(type=type_.value, id=type_.value + str(i), name='abc') common.LanguageIdentifier(language=language, identifier=id_) for i in range(2, 102): _l = common.Language(id='l%s' % i, name='Language %s' % i) _i = common.Identifier(type='iso639-3', id='%.3i' % i, name='%.3i' % i) common.LanguageIdentifier(language=_l, identifier=_i) DBSession.add(_l) param = common.Parameter(id='parameter', name='Parameter') de = common.DomainElement(id='de', name='DomainElement', parameter=param) de2 = common.DomainElement(id='de2', name='DomainElement2', parameter=param) valueset = common.ValueSet(id='valueset', language=language, parameter=param, contribution=contribution) value = common.Value(id='value', domainelement=de, valueset=valueset, frequency=50, confidence='high') DBSession.add(value) value2 = common.Value(id='value2', domainelement=de2, valueset=valueset, frequency=50, confidence='high') DBSession.add(value2) paramnd = common.Parameter(id='no-domain', name='Parameter without domain') valueset = common.ValueSet(id='vs2', language=language, parameter=paramnd, contribution=contribution) common.ValueSetReference(valueset=valueset, source=source) value = common.Value(id='v2', valueset=valueset, frequency=50, confidence='high') DBSession.add(value) unit = common.Unit(id='unit', name='Unit', language=language) up = common.UnitParameter(id='unitparameter', name='UnitParameter') DBSession.add(unit) DBSession.add( common.UnitValue(id='unitvalue', name='UnitValue', unit=unit, unitparameter=up)) up2 = common.UnitParameter(id='up2', name='UnitParameter with domain') de = common.UnitDomainElement(id='de', name='de', parameter=up2) DBSession.add( common.UnitValue(id='uv2', name='UnitValue2', unit=unit, unitparameter=up2, unitdomainelement=de)) DBSession.add(common.Source(id='s')) sentence = common.Sentence( id='sentence', name='sentence name', description='sentence description', analyzed='a\tmorpheme\tdoes\tdo', gloss='a\tmorpheme\t1SG\tdo.SG2', source='own', comment='comment', original_script='a morpheme', language=language, jsondata={'alt_translation': 'Spanish: ...'}) common.SentenceReference(sentence=sentence, source=source) DBSession.add(common.Config(key='key', value='value')) common.Config.add_replacement('replaced', 'language', model=common.Language) common.Config.add_replacement('gone', None, model=common.Language) DBSession.flush()
def main(args): # pragma: no cover get_repos() api = Grambank(REPOS['Grambank']) cldf = args.cldf data = Data() dataset = models.Grambank( id=grambank.__name__, name="Grambank", description="Grambank", publisher_name="Max Planck Institute for Evolutionary Anthropology", publisher_place="Leipzig", publisher_url="https://www.eva.mpg.de", license="http://creativecommons.org/licenses/by/4.0/", domain='grambank.clld.org', contact='*****@*****.**', jsondata={ 'license_icon': 'cc-by.png', 'license_name': 'Creative Commons Attribution 4.0 International License' }) contributors = {} for i, contrib in enumerate(api.contributors): contrib = common.Contributor( contrib.id, id=contrib.id, name=contrib.name, ) common.Editor(dataset=dataset, contributor=contrib, ord=i) DBSession.add(contrib) DBSession.flush() contributors[contrib.id] = contrib.pk contributions = {r['ID']: r for r in cldf['LanguageTable']} DBSession.add(dataset) for rec in tqdm(list(Database.from_file(cldf.bibpath, lowercase=True)), desc='sources'): data.add(common.Source, rec.id, _obj=bibtex2source(rec)) DBSession.flush() sources = {k: v.pk for k, v in data['Source'].items()} features, codes = import_features(cldf, contributors) transaction.commit() values_by_sheet = [(lid, list(v)) for lid, v in itertools.groupby( sorted(cldf['ValueTable'], key=lambda r: r['Language_ID']), lambda r: r['Language_ID'], )] for lid, values in tqdm(values_by_sheet, desc='loading values'): transaction.begin() import_values(values, contributions[lid], features, codes, contributors, sources) transaction.commit() transaction.begin() glottolog = Glottolog(REPOS['glottolog']) languoids = {l.id: l for l in glottolog.languoids()} gblangs = DBSession.query(models.GrambankLanguage).all() load_families(data, gblangs, glottolog_repos=REPOS['glottolog'], isolates_icon='dcccccc') # Add isolates for lg in gblangs: gl_language = languoids.get(lg.id) if not gl_language.family: family = data.add( Family, gl_language.id, id=gl_language.id, name=gl_language.name, description=common.Identifier( name=gl_language.id, type=common.IdentifierType.glottolog.value).url(), jsondata={"icon": 'tcccccc'}) lg.family = family coverage.main(glottolog) return
def prime_cache(args): """If data needs to be denormalized for lookup, do that here. This procedure should be separate from the db initialization, because it will have to be run periodically whenever data has been updated. """ print('Parsing markdown intros...') for contrib in DBSession.query(models.Contribution): if contrib.description: contrib.markup_description = markdown(contrib.description) else: contrib.markup_description = None print('... done') print('Retrieving language data from glottolog...') catconf = cldfcatalog.Config.from_file() glottolog_path = catconf.get_clone('glottolog') glottolog = Glottolog(glottolog_path) lang_ids = [lang.id for lang in DBSession.query(common.Language)] languoids = {l.id: l for l in glottolog.languoids(lang_ids)} glottocodes = [(l.id, common.Identifier(id=l.id, name=l.id, type='glottolog')) for l in languoids.values()] glottocodes = OrderedDict(sorted(glottocodes, key=lambda t: t[0])) isocodes = [(l.iso, common.Identifier(id=l.iso, name=l.iso, type='iso639-3')) for l in languoids.values() if l.iso] isocodes = OrderedDict(sorted(isocodes, key=lambda t: t[0])) DBSession.add_all(glottocodes.values()) DBSession.add_all(isocodes.values()) DBSession.flush() for lang in DBSession.query(common.Language): if lang.id not in languoids: continue languoid = languoids[lang.id] lang.name = languoid.name lang.latitude = languoid.latitude lang.longitude = languoid.longitude lang.macroarea = languoid.macroareas[ 0].name if languoid.macroareas else '' DBSession.add( common.LanguageIdentifier( language=lang, identifier_pk=glottocodes[languoid.id].pk)) if languoid.iso in isocodes: DBSession.add( common.LanguageIdentifier( language=lang, identifier_pk=isocodes[languoid.iso].pk)) DBSession.flush() print('... done') print('Making pretty colourful dots for parameter values...') all_icons = [icon.name for icon in ORDERED_ICONS] code_query = DBSession.query(common.DomainElement)\ .order_by(common.DomainElement.parameter_pk, common.DomainElement.id) for _, param_codes in groupby(code_query, lambda c: c.parameter_pk): icons = cycle(all_icons) for code in param_codes: code.update_jsondata(icon=next(icons)) DBSession.flush() print('... done')