def test_Reference(): ref = Reference(Source('book', 'huber2005', author='Herrmann Huber'), '2-5') assert '2-5' in repr(ref) assert '%s' % ref == 'huber2005[2-5]' with pytest.raises(ValueError): Reference(Source('book', 'huber2005', author='Herrmann Huber'), '[2-5]')
def test_from_entry(): e = Entry('book', fields={'title': 'Title'}) assert Source.from_entry('abc', e)['title'] == 'Title' with pytest.raises(ValueError): Source.from_entry('a.b', e) assert Source.from_entry('a.b', e, _check_id=False).id == 'a.b'
def bibdata(sheet, values, e, lgks, unresolved): def clean_key(key): return key.replace(':', '_').replace("'", "") for row in values: if row.Source: row.Source_comment = row.Source refs, sources = collections.OrderedDict(), [] uc = sum(list(unresolved.values())) res = srctok.source_to_refs(row.Source, sheet.glottocode, e, lgks, unresolved) if sum(list(unresolved.values())) > uc: # pragma: no cover row.Source_comment += ' (source not confirmed)' for key, pages in res[0]: typ, fields = e[key] ref = key = clean_key(key) if ref not in refs: refs[ref] = set() refs[ref] = refs[ref].union(pages or []) sources.append(Source(typ, key, **fields)) row.Source = [ '{}{}'.format(r, '[{}]'.format(','.join(sorted(p))) if p else '') for r, p in refs.items() ] for src in sources: yield src
def test_Sources(tmpdir): src = Sources() src.add(BIB, Source( 'book', 'huber2005', author='Herrmann Huber', year='2005', title='y')) for entry in src: assert entry.genre == 'book' break assert len(list(src.items())) == 3 assert len(list(src.keys())) == 3 refs = ['huber2005[1-6]', 'Obrazy', 'Elegie[34]'] assert src.format_refs(*list(src.expand_refs(refs))) == refs assert '%s' % src['huber2005'] == 'Huber, Herrmann. 2005. y.' with pytest.raises(ValueError): src.add(5) with pytest.raises(ValueError): src.add('@misc{a.b,\n author="a.b"\n}') with pytest.raises(ValueError): _ = src['unknown'] assert _ # pragma: no cover with pytest.raises(ValueError): src.parse('a[x') with pytest.raises(ValueError): src.parse('[x]') with pytest.raises(ValueError): src.validate(['x']) bib = str(tmpdir / 'test.bib') src.write(bib) src2 = Sources() src2.read(bib) src2.write(bib, ids=['huber2005']) src = Sources.from_file(bib) assert len(src) == 1
def to_cldf(self, dest, mdname='cldf-metadata.json', coordinate_precision=4): """ Write the data from the db to a CLDF dataset according to the metadata in `self.dataset`. :param dest: :param mdname: :return: path of the metadata file """ dest = pathlib.Path(dest) if not dest.exists(): dest.mkdir() data = self.read() if data[self.source_table_name]: sources = Sources() for src in data[self.source_table_name]: sources.add(Source( src['genre'], src['id'], **{k: v for k, v in src.items() if k not in ['id', 'genre']})) sources.write(dest / self.dataset.properties.get('dc:source', 'sources.bib')) for table_type, items in data.items(): try: table = self.dataset[table_type] items = [ self.round_geocoordinates(item, precision=coordinate_precision) for item in items] table.common_props['dc:extent'] = table.write( [self.retranslate(table, item) for item in items], base=dest) except KeyError: assert table_type == self.source_table_name, table_type return self.dataset.write_metadata(dest / mdname)
def parse_source(src): sid = src.get("source-id") if _get_text(src, "type") == "Informant": kw = dict(howpublished="Informant: {0}".format(_get_text(src, "full-citation"))) else: kw = dict(howpublished=_get_text(src, "full-citation"), note=_get_text(src, "note")) kw["key"] = _get_text(src, "citation-key") return Source("misc", sid, **kw)
def itersources(self, pkmap): for row in self.raw_dir.read_csv('source.csv', dicts=True): jsondata = json.loads(row.pop('jsondata', '{}') or '{}') pkmap['source'][row.pop('pk')] = row['id'] row['title'] = row.pop('description') row['key'] = row.pop('name') if (not row['url']) and jsondata.get('gbs', {}).get('id'): row['url'] = 'https://books.google.de/books?id=' + jsondata[ 'gbs']['id'] yield Source(row.pop('bibtex_type'), row.pop('id'), **row)
def test_field_order(tmpdir): srcs = Sources() src = Source('misc', 'x') # src is an OrderedDict and we add title *after* year. src['year'] = '2018' src['title'] = 'The Title' srcs.add(src) bib = tmpdir / 'test.bib' srcs.write(str(bib)) res = bib.read_text(encoding='utf8') # Still, title should be printed in the BibTeX before year: assert res.index('title =') < res.index('year =')
def test_Source_expand_refs(): sources = Sources() src = Source( 'book', 'Meier2005', author='Hans Meier', year='2005', title='The Book') assert 'Meier2005' in repr(src) sources.add(src) bib = sources._bibdata.to_string(bib_format='bibtex') assert len(bib.split('author')) == 2 assert len(list(sources.expand_refs('Meier2005'))) == 1 bib = sources._bibdata.to_string(bib_format='bibtex') assert len(bib.split('author')) == 2 assert len(list(sources.expand_refs('12345'))) == 1
def get_reference(author, year, title, pages, sources, id_=None, genre='misc'): kw = {'title': title} id_ = id_ or None if author and year: id_ = id_ or slug(author + year) kw.update(author=author, year=year) elif title: id_ = id_ or slug(title) if not id_: return source = sources.get(id_) if source is None: sources[id_] = source = Source(genre, id_, **kw) return Reference(source, pages)
def cmd_makecldf(self, args): data = self.raw_dir.read_csv('raw.tsv', delimiter="\t", dicts=True) # Quite a hack to allow things like "1995.pdfb" as Source IDs: bib = pybtex.database.parse_string(self.raw_dir.read('sources.bib'), bib_format='bibtex') sources = [] for k, e in bib.entries.items(): # Unfortunately, Source.from_entry does not allow any keyword arguments to be passed # to the constructor, see https://github.com/cldf/pycldf/issues/99 e.fields['_check_id'] = False sources.append(Source.from_entry(k, e)) args.writer.add_sources(*sources) language_lookup = args.writer.add_languages(lookup_factory='NameInSource') concept_lookup = args.writer.add_concepts( id_factory=lambda x: x.id.split('-')[-1]+'_'+slug(x.english), lookup_factory='Name' ) lang_sources = {l['NameInSource']: l['Source'].split(",") for l in self.languages} # remap concepts for personal pronouns remap_concepts = { '1SG pronoun': '1sg pronoun', '2SG pronoun': '2sg pronoun', '3SG pronoun': '3sg pronoun', } for line_dict in progressbar(data, desc='cldfify'): concept = line_dict['Meaning'] concept_id = concept_lookup.get(remap_concepts.get(concept, concept)) for language, language_id in language_lookup.items(): value = line_dict[language].strip() if value: args.writer.add_form( Value=value, Form=value, Parameter_ID=concept_id, Language_ID=language_id, Source=lang_sources[language] )
def cldf(dataset, concepticon, **kw): concept_map = { int(c['GLOSS']): c['CONCEPTICON_ID'] or None for c in dataset.concepts } gc_pattern = re.compile('[a-z0-9]{4}[1-9][0-9]{3}$') meta = {} for row in read_csv(dataset, 'META'): meta[(row[5], row[9])] = dict( zip( 'NAME,COUNTRY,ISO,GLOTTO_NAME,GLOTTO_CODE,LG_LINK,AUDIO,SOURCE,NR_SETS,VARIANT' .lower().split(','), row)) sources = {} sid = 0 for spec in meta.values(): if spec['source'] and spec['source'] not in sources: sid += 1 sources[spec['source']] = Source('misc', 's%s' % sid, title=spec['source']) unmapped = Unmapped() with CldfDataset(( 'ID', 'Language_ID', 'Language_name', 'Parameter_ID', 'Parameter_name', 'Value', 'Source', 'Comment', ), dataset) as ds: for key, items in groupby( sorted(read_csv(dataset, 'NUMERAL'), key=lambda r: (r[2], r[3], r[0])), lambda r: (r[2], r[3])): if key not in meta: continue if int(float(key[1])) > 1: continue md = meta[key] source, ref = sources.get(md['source']), None if source: ds.sources.add(source) ref = source.id if gc_pattern.match(md['glotto_code']): for concept, rows in groupby(items, lambda k: k[0]): if not concept.endswith('.0'): continue iconcept = int(float(concept)) if iconcept not in concept_map: unmapped.concepts.add((iconcept, iconcept)) for k, row in enumerate(rows): ds.add_row([ '%s-%s-%s' % (lgid(row[2]), iconcept, k + 1), md['glotto_code'], md['name'], concept_map.get(iconcept), '%s' % iconcept, row[1], ref, row[4] or None, ]) unmapped.pprint()
def to_cldf(self, ds, concepts): """ :param ds: the dataset object :concepts: a dictionary mapping concept labels to concept ids :return: A dataset object, ds. """ source = [] if self.language.source: bib = parse_string(self.language.source, "bibtex") try: ds.add_sources( *[Source.from_entry(k, e) for k, e in bib.entries.items()]) source = list(bib.entries.keys()) except: # noqa: E722 self.log.warning("Invalid citekey for %s" % self.language.id) ds.add_language(ID=self.language.id, Glottocode=self.language.glottocode, ISO639P3code=self.language.iso, Name=self.language.name, author=self.language.author, url=self.url('language.php?id=%s' % self.language.id), typedby=self.language.typedby, checkedby=self.language.checkedby, notes=self.language.notes, source=";".join(source)) for entry in self.entries: if entry.name is None or len( entry.name) == 0: # skip empty entries continue # pragma: no cover # skip entries marked as incorrect word form due to semantics # (x = probably, s = definitely) if entry.cognacy and entry.cognacy.lower() in ('s', 'x'): continue # pragma: no cover # handle concepts cid = concepts.get(entry.word_id) if not cid: self.dataset.unmapped.add_concept(ID=entry.word_id, Name=entry.word) # add it if we don't have it. ds.add_concept(ID=entry.word_id, Name=entry.word) cid = entry.word_id # handle lexemes try: lex = ds.add_forms_from_value( Local_ID=entry.id, Language_ID=self.language.id, Parameter_ID=cid, Value=entry.name, # set source to entry-level sources if they exist, otherwise use # the language level source. Source=[entry.source] if entry.source else source, Cognacy=entry.cognacy, Comment=entry.comment or '', Loan=True if entry.loan and len(entry.loan) else False, ) except: # NOQA: E722; pragma: no cover print("ERROR with %r -- %r" % (entry.id, entry.name)) raise if lex: for cognate_set_id in entry.cognates: match = self.dataset.cognate_pattern.match(cognate_set_id) if not match: # pragma: no cover self.log.warning( 'Invalid cognateset ID for entry {0}: {1}'.format( entry.id, cognate_set_id)) else: # make global cognate set id cs_id = "%s-%s" % (slug(entry.word), match.group('id')) ds.add_cognate(lexeme=lex[0], Cognateset_ID=cs_id, Doubt=bool(match.group('doubt')), Source=['Greenhilletal2008'] if self.section == 'austronesian' else []) return ds
def read_bib(self, fname='sources.bib'): bib = database.parse_string(self.read(fname), bib_format='bibtex') return [Source.from_entry(k, e) for k, e in bib.entries.items()]
def cldf(dataset, concepticon, **kw): data = get_all(dataset) gl_map = {k: v.id for k, v in dataset.glottolog_languoids.items()} gl_map.update(dataset.glottocode_by_iso) swadesh_concepts = { k: v for k, v in data['word'].items() if v['id'] in data['concept_ids'] } def normalized_gloss(gloss): if gloss.startswith('to '): gloss = gloss[3:].strip() if '/' in gloss: gloss = gloss.split('/')[0].strip() if '(' in gloss: gloss = gloss.split('(')[0].strip() if gloss.endswith('?'): gloss = gloss[:-1] return gloss swadesh2concepticon = { 'right (hand)': '2183', 'we incl. (pronoun d:1p, incl)': '1131', 'left (hand)': '2182', 'right (correct, true)': '1725', 'in, inside': '1460', 'to lie down': '215', } for conceptlist in [ 'Swadesh-1960-200', 'Swadesh-1971-100', 'Swadesh-1955-100', 'Swadesh-1950-215', 'Swadesh-1955-215' ]: for d in concepticon.conceptlists[conceptlist].concepts.values(): swadesh2concepticon.setdefault(d.english, d.concepticon_id) concept_map = {} for concept in swadesh_concepts.values(): gloss = normalized_gloss(concept['word']) if gloss in swadesh2concepticon: concept_map[concept['id']] = swadesh2concepticon[gloss] elif concept['word'] in swadesh2concepticon: concept_map[concept['id']] = swadesh2concepticon[concept['word']] else: raise ValueError(concept['word']) assert len(concept_map) == len(set(concept_map.values())) for c in dataset.concepts: if c['CONCEPTICON_ID']: concept_map[int(c['ID'])] = c['CONCEPTICON_ID'] or None uc = Counter() unmapped = Unmapped(lambda r: int(r[0])) for language_url, words in groupby( sorted(data['lexicon'].values(), key=lambda i: i['language']), lambda i: i['language']): contribution = data['language'][language_url] with CldfDataset(( 'ID', 'Language_ID', 'Language_iso', 'Language_name', 'Language_local_ID', 'Parameter_ID', 'Parameter_name', 'Parameter_local_ID', 'Value', 'Source', 'Cognate_Set', 'Comment', 'Loan', ), dataset, subset=contribution['id']) as ds: cname = contribution['language'] if contribution['dialect']: cname += ' (%s Dialect)' % contribution['dialect'] lid = gl_map.get(contribution['glottocode']) if not lid: lid = gl_map.get(contribution['isocode']) if not lid: unmapped.languages.add( (contribution['id'], cname, contribution['isocode'])) if contribution['information']: ds.metadata['dc:description'] = contribution['information'] ds.table.schema.aboutUrl = '%s.csv#{ID}' % ds.name ds.table.schema.columns['Loan'].datatype = 'boolean' ds.table.schema.columns['Parameter_local_ID'].valueUrl = \ '%s/word/{Parameter_local_ID}' % BASE_URL ds.table.schema.columns['Language_local_ID'].valueUrl = \ '%s/language/{Language_local_ID}' % BASE_URL for word in words: concept = data['word'][word['word']] if concept['id'] not in concept_map: unmapped.concepts.add((concept['id'], concept['word'])) uc.update([concept['word']]) src = data['source'].get(word['source']) if src: ds.sources.add( Source('misc', src['slug'], author=src['author'], year=src['year'], transnewguinea_id=BASE_URL + '/source/' + src['slug'], title=src['reference'])) ds.add_row([ word['id'], lid, contribution['isocode'], cname, contribution['slug'], concept_map.get(concept['id']), concept['word'], concept['slug'], word['entry'], src['slug'] if src else None, None, word['annotation'], word['loan'], ]) unmapped.pprint()
def cmd_makecldf(self, args): asjp = ASJP(self.raw_dir) meaning_id_lookup = {v: k for k, v in MEANINGS_ALL.items()} meaning_id_lookup["breasts"] = meaning_id_lookup["breast"] for concept in sorted( self.conceptlists[0].concepts.values(), key=lambda c: meaning_id_lookup[c.label.replace("*", "")], ): args.writer.add_concept( ID=meaning_id_lookup[concept.label.replace("*", "")], Name=concept.label, Concepticon_ID=concept.concepticon_id, Concepticon_Gloss=concept.concepticon_gloss, ) iso2gl = {l.iso: l.id for l in self.glottolog.languoids() if l.iso} # Correct glottocodes (mapping, retired ones, etc.) iso2gl.update({ "gtu": "aghu1254", # from aghu1256, AGHU_THARRNGGALA "xss": "kott1239", # from assa1266, ASSAN "bbz": "char1283", # from baba1273, BABALIA_CREOLE_ARABIC (mapping to a dialect of # Chadian Arabic, as per SIL documentation) "bcb": "bain1259", # from bain1260, BAYNUNK_GUJAXER* "bic": "biso1243", # from bika1251, BIKARU* "bvp": "khan1274", # from bika1247, BUMANG "geg": "kuga1239", # from geng1243, GENGLE "ihi": "emai1241", # from ihie1238, IHIEE "kav": "kana1291", # from nucl1668, KATUKINA "kym": "gbay1278", # from kpat1244, KPATILI_2 "mct": "eton1253", # from meng1263, MENGISA "nxu": "kaur1271", # from nara1265, NARAU "nmj": "gund1247", # from ngom1265, NGOMBE_2 "nom": "cash1251", # from noca1240, NOCAMAN "otk": "oldu1238", # from oldt1247, OLD_TURKIC "rui": "nden1248", # from rufi1234, RUFIJI "sxm": "somr1240", # from samr1245, SAMRE "xin": "xinc1242", # from xinc1247, mapping to what is by far the most plausible source # for Lehmann 1920 (all other varieties were extinct, and the # alternative one is dormant and close enough) }) lids = set() for doculect in sorted(asjp.iter_doculects(), key=lambda dl: dl.id): lid = slug(doculect.id) assert lid not in lids, doculect.id lids.add(lid) sources = asjp.source(doculect) or [] sources = [ src for src in sources if src.author or src.year or src.title_etc ] for src in sources: args.writer.add_sources( Source("misc", str(src.id), author=src.author, year=src.year, title=src.title_etc)) # Obtain the glottocode mapping and fix it for cases with multiple mappings glottocode = iso2gl.get(doculect.code_iso) if lid == "LENCA_EL_SALVADOR": glottocode = "lenc1243" elif lid == "LENCA_HONDURAS": glottocode = "lenc1242" elif lid == "DANANSHAN_HMONG": glottocode = "hmon1332" elif lid == "SHIMENKAN_HMONG": glottocode = "smal1236" elif lid == "SUYONG_HMONG": glottocode = "larg1235" elif lid == "URADHI_ANGKAMUTHI": glottocode = "angg1238" elif lid == "URADHI_ATAMPAYA": glottocode = "atam1239" elif lid == "URADHI_YADHAYKENU": glottocode = "yadh1237" args.writer.add_language( ID=lid, Name=doculect.id, ISO639P3code=doculect.code_iso if re.fullmatch( "[a-z]{3}", doculect.code_iso or "") else None, Glottocode=glottocode, Latitude=doculect.latitude, Longitude=doculect.longitude, classification_wals=doculect.classification_wals, classification_ethnologue=doculect.classification_ethnologue, classification_glottolog=doculect.classification_glottolog, recently_extinct=doculect.recently_extinct, long_extinct=doculect.long_extinct, year_of_extinction=doculect.year_of_extinction, code_wals=doculect.code_wals, code_iso=doculect.code_iso, transcribers=" and ".join( sorted( [tr.name for tr in asjp.transcriber(doculect) or []])), ) args.writer.add_languages() for synset in sorted(doculect.synsets, key=lambda ss: ss.meaning_id): for word in synset.words: args.writer.add_form( Language_ID=lid, Parameter_ID=synset.meaning_id, Value=word.form, Form=word.form, Loan=word.loan, Comment=synset.comment, gloss_in_source=synset.meaning, Source=sorted([str(src.id) for src in sources]), )
def cmd_makecldf(self, args): glottolog = args.glottolog.api contrib = ["""\ # Contributors name | affiliation | orcid | github | role --- | ----------- | ----- | ------ | ----"""] for ed in glottolog.current_editors: contrib.append(' | '.join( [ed.name, ed.affiliation, ed.orcid, '@' + getattr(ed, 'github'), 'maintainer'])) self.dir.joinpath('CONTRIBUTORS.md').write_text('\n'.join(contrib), encoding='utf8') # write metadata.json dump( { "title": glottolog.publication.zenodo.title_format.format( glottolog.publication.zenodo.version) + ' as CLDF', "citation": metadata.citation(glottolog), "url": glottolog.publication.web.url, "description": glottolog.publication.web.description, "id": "glottolog", "license": glottolog.publication.zenodo.license_id, }, self.dir / 'metadata.json', indent=4, ) ds = args.writer.cldf ds.add_provenance( wasGeneratedBy=repos('pyglottolog', version=pyglottolog.__version__), ) t = ds.add_component( 'ParameterTable', { 'name': 'type', 'datatype': {'base': 'string', 'format': 'categorical|sequential'}, 'dc:description': 'Describes the domain of the parameter', 'default': None, }, { 'name': 'infoUrl', 'dc:description': 'URL (relative to `aboutUrl`) of a web page with further ' 'information about the parameter', 'aboutUrl': 'https://glottolog.org/{+infoUrl}', }, { 'name': 'datatype', 'datatype': 'json', 'dc:description': 'CSVW datatype description for values for this parameter. I.e. content of the ' 'Value column of associated rows in ValueTable should be interpreted/parsed ' 'accordingly', } ) t.common_props['dc:description'] = \ "This table lists parameters (or aspects) of languoids that Glottolog assigns values " \ "for, such as the languoid's position on the Glottolog classification or the " \ "descriptive status. Refer to the `Description` column in the table for details, and " \ "to the `datatype` columnn for information how values for the parameter should be " \ "interpreted." ds.add_component('CodeTable') ds.add_columns('ValueTable', 'codeReference') t = ds.add_component( 'LanguageTable', { 'name': 'Countries', 'separator': ';', 'dc:description': 'ISO 3166-1 alpha-2 country codes for countries a language is spoken in.', 'propertyUrl': 'https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2', 'aboutUrl': 'https://en.wikipedia.org/wiki/ISO_3166-2:{Countries}', }, { 'name': 'Family_ID', 'dc:description': 'Glottocode of the top-level genetic unit, the languoid belongs to'}, { 'name': 'Language_ID', 'dc:description': 'Glottocode of the language-level languoid, the languoid belongs to ' '(in case of dialects)'}, ) t.common_props['dc:description'] = \ 'This table lists all Glottolog languoids, i.e. families, languages and dialects ' \ 'which are nodes in the Glottolog classification - including "non-genealogical" ' \ 'trees as described at https://glottolog.org/glottolog/glottologinformation . ' \ 'Thus, assumptions about the properties of a languoid listed here should be made ' \ 'after including associated information from ValueTable, in particular for languoid ' \ 'level and category.' t.aboutUrl = 'https://glottolog.org/meta/glossary#Languoid' ds.add_foreign_key('LanguageTable', 'Family_ID', 'LanguageTable', 'ID') ds.add_foreign_key('LanguageTable', 'Language_ID', 'LanguageTable', 'ID') ds['LanguageTable', 'Macroarea'].separator = ';' ds['ValueTable', 'Value'].null = ['<NA>'] data = args.writer.objects for pid, pinfo in schema.PARAMETER_INFO.items(): data['ParameterTable'].append(dict( ID=pid, Name=pinfo.name, type=pinfo.type, Description=pinfo.description, infoUrl=pinfo.url, datatype=pinfo.dt.asdict() )) for level in glottolog.languoid_levels.values(): data['CodeTable'].append(dict( ID='level-{0}'.format(level.name), Parameter_ID='level', Name=level.name, Description=level.description, )) data['CodeTable'].append(dict( ID='category-{0}'.format(level.name.capitalize()), Parameter_ID='category', Name=level.name.capitalize())) data['CodeTable'].append(dict( ID='category-Pseudo_{0}'.format(level.name.capitalize()), Parameter_ID='category', Name='Pseudo {}'.format(level.name.capitalize()))) for el in sorted(glottolog.language_types.values()): data['CodeTable'].append(dict( ID='category-{0}'.format(el.category.replace(' ', '_')), Parameter_ID='category', Name=el.category)) for el in sorted(glottolog.aes_status.values()): data['CodeTable'].append(dict( ID='aes-{0}'.format(el.name.replace(' ', '_')), Parameter_ID='aes', Name=el.name, Description='EGIDS: {0.egids}; UNESCO: {0.unesco}; ElCat: {0.unesco}'.format(el), )) for el in sorted(glottolog.med_types.values()): data['CodeTable'].append(dict( ID='med-{0}'.format(el.id), Parameter_ID='med', Name=el.name, Description=el.description, )) languoids = collections.OrderedDict((lang.id, lang) for lang in glottolog.languoids()) refs_by_languoid, refs = glottolog.refs_by_languoid(languoids) def get_language_id(lang): if lang.level == glottolog.languoid_levels.dialect: for _, lid, _ in reversed(lang.lineage): if languoids[lid].level == glottolog.languoid_levels.language: return lid def format_ref(ref): return '{0}[{1}]'.format(ref.key, ref.pages.replace(';', ',')) if ref.pages else ref.key for lang in languoids.values(): data['LanguageTable'].append(dict( ID=lang.id, Name=lang.name, Glottocode=lang.id, ISO639P3code=lang.iso, Latitude=lang.latitude, Longitude=lang.longitude, Macroarea=[ma.name for ma in lang.macroareas], Countries=[c.id for c in lang.countries], Family_ID=lang.lineage[0][1] if lang.lineage else None, Language_ID=get_language_id(lang), )) med = sorted(refs_by_languoid[lang.id], reverse=True)[0] \ if lang.id in refs_by_languoid else None if med: ds.add_sources(Source(med.type, med.id, _check_id=False, **med.fields)) clf = lang.classification_comment if clf: for ref in clf.merged_refs('family') + clf.merged_refs('sub'): if ref.key not in refs: args.log.warning('missing reference in classification comment: {0}'.format(ref)) continue e = refs[ref.key] ds.add_sources(Source(e.type, ref.key, _check_id=False, **e.fields)) aes_src = lang.endangerment.source.reference_id if lang.endangerment else None if aes_src: e = refs[aes_src] ds.add_sources(Source(e.type, aes_src, _check_id=False, **e.fields)) data['ValueTable'].extend([ value( lang.id, 'level', lang.level.name, Code_ID='level-{0}'.format(lang.level.name)), value( lang.id, 'category', lang.category.replace(' ', '_'), Code_ID='category-{0}'.format(lang.category.replace(' ', '_')), ), value( lang.id, 'classification', schema.PARAMETER_INFO['classification'].dt.formatted( '/'.join([li[1] for li in lang.lineage])) or None, Source=[format_ref(ref) for ref in clf.merged_refs('family')] if clf else [], Comment=clf.family if clf else None, ), value( lang.id, 'subclassification', lang.newick_node(nodes=languoids, template="{l.id}").newick + ';', Source=[format_ref(ref) for ref in clf.merged_refs('sub')] if clf else [], Comment=clf.sub if clf else None, ), value( lang.id, 'aes', lang.endangerment.status.ordinal if lang.endangerment else None, Comment=lang.endangerment.comment if lang.endangerment else None, Source=[aes_src] if aes_src else [], Code_ID='aes-{0}'.format(lang.endangerment.status.name.replace(' ', '_')) if lang.endangerment else None, ), value( lang.id, 'med', med.med_type.rank if med else None, Source=[med.id] if med else [], Code_ID='med-{0}'.format(med.med_type.id) if med else None, ), ])
def test_Wordlist_2(abvd_dataset): with abvd_dataset.cldf as ds: for wl in abvd_dataset.iter_wordlists({'1': 'bali1278'}, None): wl.to_cldf(ds, {}, citekey='x', source=[Source('a', 'b', **dict(title='t'))])
def to_cldf(self, ds, concept_map, citekey=None, source=None, concept_key=None): if concept_key is None: def concept_key(entry): return entry.word_id ref = None if citekey and source: ref = citekey for r in ref.split(";"): for s in source: if isinstance(s, Source): ds.add_sources(s) else: ds.add_sources(Source('misc', r, title=s)) ds.add_language( ID=self.language.id, Glottocode=self.language.glottocode, ISO639P3code=self.language.iso, Name=self.language.name, author=self.language.author, url=self.url('language.php?id=%s' % self.language.id), typedby=self.language.typedby, checkedby=self.language.checkedby, notes=self.language.notes, ) for entry in self.entries: if entry.name is None or len( entry.name) == 0: # skip empty entries continue # pragma: no cover if entry.cognacy and ('s' == entry.cognacy.lower() or 'x' in entry.cognacy.lower()): # skip entries marked as incorrect word form due to semantics # (x = probably, s = definitely) continue # pragma: no cover if not (citekey and source): src = entry.e.find('source') if (src is not None) and getattr(src, 'text'): ref = slug(str(src.text)) ds.add_sources(Source('misc', ref, title=src.text)) cid = concept_map.get(concept_key(entry)) if not cid: self.dataset.unmapped.add_concept(ID=entry.word_id, Name=entry.word) ds.add_concept(ID=entry.word_id, Name=entry.word, Concepticon_ID=cid) for lex in ds.add_lexemes( Language_ID=self.language.id, Parameter_ID=entry.word_id, Value=entry.name, Source=[ref], Cognacy=entry.cognacy, Comment=entry.comment or '', Loan=True if entry.loan and len(entry.loan) else False, Local_ID=entry.id, ): for cognate_set_id in entry.cognates: match = self.dataset.cognate_pattern.match(cognate_set_id) if not match: # pragma: no cover self.log.warn('Invalid cognateset ID: {0}'.format( cognate_set_id)) else: ds.add_cognate(lexeme=lex, Cognateset_ID=match.group('id'), Doubt=bool(match.group('doubt'))) # when an entry is split into multiple forms, we only assign cognate # sets to the first one! break return ds
def main(args): """ Update cognate codes and alignments of a CLDF dataset from an Edictor file. Parameters ---------- args : Namespace A Namespace object with several properties listed below. edictor : FileType ... cldf : Path, optional Path to the CLDF metadata json file. source-id : str, optional String used to source any changes to cognate codes or alignments. This defaults to "edictor". cogid : str, optional String that specifies the header of the column containing the cognate set id's in the Edictor file. This defaults to "COGID". Notes ----- Once this function is called with the proper arguments, cognates.csv in the CLDF dataset is updated based on the output of Edictor when changes are made to cognate codes or alignments. Sources.bib also gets updated with a new source if specified. """ # Check CLDF argument, in order to fail early if this fails. dataset = pycldf.dataset.Wordlist.from_metadata(args.cldf) # Read new cognate classes and their alignments new_cognateset_assignments = {} alignments = {} for row in csv.DictReader(args.edictor, delimiter="\t"): if row["ID"].startswith("#") or not row["ID"] or not row["REFERENCE"]: # LingPy has comment rows continue new_cognateset_assignments[row["REFERENCE"]] = tuple( row[args.cogid].split()) try: alignments[row["REFERENCE"]] = row["ALIGNMENT"].split() except AttributeError: print(row) raise new_cognatesets = swap(new_cognateset_assignments) # Column names are assumed to follow standard CLDF CognateTable conventions. original_rows = [] data_on_form = {} official_cognateset_assignments = {} max_row_id = 0 for r, row in enumerate(dataset["CognateTable"].iterdicts()): original_rows.append(row) data_on_form[row["Form_ID"]] = row official_cognateset_assignments[row["Form_ID"]] = row["Cognateset_ID"] max_row_id = max(max_row_id, row["ID"]) official_cognatesets = swap(official_cognateset_assignments) # Find changed alignments for form, data in data_on_form.items(): if data.get("Alignment", None) == alignments.get(form, False): del alignments[form] # Construct a set of minimal changes to update cognate sets pairs, overlaps = [], [] # First, sort all pairs of old and new cognate sets by overlap. still_to_match = new_cognatesets.copy() for name, cognateset in list(official_cognatesets.items()): cognateset = cognateset.copy() for new_name, new_cognateset in list(still_to_match.items()): if cognateset == new_cognateset: # This cognate set has not changed, ignore. continue overlap = cognateset & new_cognateset if not overlap: # There is no overlap. continue # This cognate set has changed. official_cognatesets.pop(name, None) # Insert the pair into an ordered table of pair sizes. Ensure # biggest overlaps come first, so actually work with their negatives. index = bisect.bisect(overlaps, -len(overlap)) overlaps.insert(index, -len(overlap)) pairs.insert(index, (name, new_name)) remainder = new_cognateset - overlap if remainder: still_to_match[new_name] = remainder else: # All entries of this set have been accounted for, remove it. del still_to_match[new_name] cognateset -= overlap if not cognateset: # All entries of this set have been accounted for, no need to look further. break # Now greedily assign new cognate class ids based on the old ones. (This # greedy algorithm is not optimal, but that issue should not be too # relevant.) other_seen = set() moved_forms = collections.OrderedDict() for index, (name, other) in enumerate(pairs): if other in other_seen: continue else: new_name = name while new_name in official_cognatesets: new_name = new_name + "X" official_cognatesets[new_name] = set() for form in new_cognatesets[other]: if official_cognateset_assignments.get(form) != new_name: moved_forms[form] = new_name official_cognatesets[new_name].add(form) other_seen.add(other) for new_name, new_cognateset in list(still_to_match.items()): while new_name in official_cognatesets: new_name = new_name + "X" for form in new_cognateset: moved_forms[form] = new_name official_cognatesets[new_name] = new_cognateset try: source = dataset.sources[args.source_id] except ValueError: source = Source('misc', id_=args.source_id, year=str(datetime.date.today().year)) dataset.sources.add(source) dataset.write_sources() def new_rows(defaults, last_row_id, moved_forms_p, realigned_forms, source_p): # TODO: Make a docstring? # What is the benefit of defining this within the main function? t = type(last_row_id) i = -1 # FIXME: Is this assignment necessary? empty = {"Alignment": [], "Source": []} for i, (form_id, new_cognateset_it) in enumerate(moved_forms_p.items()): row_new = defaults.get(form_id, empty).copy() row_new["ID"] = last_row_id + t(i + 1) row_new["Form_ID"] = form_id row_new["Cognateset_ID"] = new_cognateset_it if form_id in realigned_forms: row_new["Alignment"] = realigned_forms[form_id] row_new["Source"] = [source.id] else: row_new["Alignment"] = defaults[form_id]["Alignment"] row_new["Source"] = defaults[form_id]["Source"] + [source_p.id] realigned_forms[form_id] = None print(row_new) yield row_new for j, (form_id, new_alignment) in enumerate(realigned_forms.items()): if not new_alignment: continue try: row_new = defaults[form_id].copy() row_new["ID"] = last_row_id + t(i + j + 2) row_new["Alignment"] = new_alignment if source.id not in row_new["Source"]: row_new["Source"] = row_new["Source"] + [source.id] except KeyError: row_new = empty.copy() row_new["ID"] = last_row_id + t(i + j + 2) row_new["Form_ID"] = form_id row_new["Alignment"] = new_alignment row_new["Source"] = [source.id] print(row_new) yield row_new dataset["CognateTable"].write( itertools.chain( original_rows, new_rows(data_on_form, max_row_id, moved_forms, alignments, source)))
def cmd_makecldf(self, args): self.create_schema(args.writer.cldf) pk2id = collections.defaultdict(dict) sources = parse_string( self.raw_dir.joinpath('source.bib').read_text(encoding='utf8'), 'bibtex') self.read('source', pkmap=pk2id) refs = [] for row in self.raw_dir.read_csv('valuesetreference.csv', dicts=True): if row['source_pk']: refs.append( (row['valueset_pk'], pk2id['source'][row['source_pk']], row['description'])) srcids = set(r[1] for r in refs) args.writer.cldf.add_sources(*[ Source.from_entry(id_, e) for id_, e in sources.entries.items() if id_ in srcids ]) contributors = self.read('contributor', pkmap=pk2id, key=lambda r: r['id']) for row in contributors.values(): args.writer.objects['contributors.csv'].append({ 'ID': row['id'], 'Name': row['name'] }) cc = { fid: [pk2id['contributor'][r['contributor_pk']] for r in rows] for fid, rows in itertools.groupby( self.read('contributioncontributor', key=lambda d: (d['contribution_pk'], d['primary'] == 'f', int(d['ord']))).values(), lambda r: r['contribution_pk']) } areas = self.read('area') chapters = self.read('contribution', extended='chapter') for row in self.read('parameter', extended='feature', pkmap=pk2id, key=lambda d: fid_key(d['id'])).values(): args.writer.objects['ParameterTable'].append({ 'ID': row['id'], 'Name': row['name'], 'Area': areas[chapters[row['contribution_pk']]['area_pk']]['name'], 'Chapter': chapters[row['contribution_pk']]['name'], 'Contributor_ID': cc[row['contribution_pk']], }) for row in self.read( 'domainelement', pkmap=pk2id, key=lambda d: (fid_key(d['id'].split('-')[0]), int(d['number']))).values(): args.writer.objects['CodeTable'].append({ 'ID': row['id'], 'Parameter_ID': pk2id['parameter'][row['parameter_pk']], 'Name': row['name'], 'Description': row['description'], 'Number': int(row['number']), 'icon': json.loads(row['jsondata'])['icon'], }) identifier = self.read('identifier') lang2id = collections.defaultdict( lambda: collections.defaultdict(list)) for row in self.read('languageidentifier').values(): id_ = identifier[row['identifier_pk']] lang2id[row['language_pk']][id_['type']].append( (id_['name'], id_['description'])) families = self.read('family', pkmap=pk2id) genera = self.read('genus', pkmap=pk2id) for row in self.read('language', extended='walslanguage', pkmap=pk2id).values(): id = row['id'] genus = genera[row['genus_pk']] family = families[genus['family_pk']] if row['name'] == genus['name'] == family['name']: # an isolate! genus = family = None iso_codes = set(i[0] for i in lang2id[row['pk']].get('iso639-3', [])) glottocodes = [ i[0] for i in lang2id[row['pk']].get('glottolog', []) ] args.writer.objects['LanguageTable'].append({ 'ID': id, 'Name': row['name'], 'ISO639P3code': list(iso_codes)[0] if len(iso_codes) == 1 else None, 'Glottocode': glottocodes[0] if len(glottocodes) == 1 else None, 'ISO_codes': sorted(iso_codes), 'Latitude': row['latitude'], 'Longitude': row['longitude'], 'Genus': genus['name'] if genus else None, 'Subfamily': genus['subfamily'] if genus else None, 'Family': family['name'] if family else None, 'Samples_100': row['samples_100'] == 't', 'Samples_200': row['samples_200'] == 't', }) args.writer.objects['LanguageTable'].sort(key=lambda d: d['ID']) refs = { dpid: [ str( Reference( source=str(r[1]), desc=r[2].replace('[', ')').replace(']', ')').replace( ';', '.').strip() if r[2] else None)) for r in refs_ ] for dpid, refs_ in itertools.groupby(refs, lambda r: r[0]) } vsdict = self.read('valueset', pkmap=pk2id) examples = self.read('sentence', pkmap=pk2id) igts = {} for ex in examples.values(): if all(ex[k] for k in ['description', 'analyzed', 'gloss']): a, g = ex['analyzed'].split(), ex['gloss'].split() if len(a) != len(g): a, g = [ex['analyzed']], [ex['gloss']] igts[ex['pk']] = ex['id'] args.writer.objects['ExampleTable'].append({ 'ID': ex['id'], 'Language_ID': pk2id['language'][ex['language_pk']], 'Primary_Text': ex['name'], 'Translated_Text': ex['description'], 'Analyzed_Word': a, 'Gloss': g, }) example_by_value = { vpk: [r['sentence_pk'] for r in rows] for vpk, rows in itertools.groupby( self.read('valuesentence', key=lambda d: d['value_pk']).values( ), lambda d: d['value_pk']) } for row in self.read('value').values(): vs = vsdict[row['valueset_pk']] comment = None ex = [examples[spk] for spk in example_by_value.get(row['pk'], [])] if len(ex) == 1 and not any( ex[0][k] for k in ['description', 'analyzed', 'gloss']): comment = ex[0]['name'] del example_by_value[row['pk']] args.writer.objects['ValueTable'].append({ 'ID': vs['id'], 'Language_ID': pk2id['language'][vs['language_pk']], 'Parameter_ID': pk2id['parameter'][vs['parameter_pk']], 'Value': pk2id['domainelement'][row['domainelement_pk']].split('-')[1], 'Code_ID': pk2id['domainelement'][row['domainelement_pk']], 'Comment': comment, 'Source': refs.get(vs['pk'], []), 'Example_ID': sorted(igts[epk] for epk in example_by_value.get(row['pk'], []) if epk in igts), }) args.writer.objects['ValueTable'].sort( key=lambda d: (d['Language_ID'], fid_key(d['Parameter_ID']))) altnames = [] for lpk in lang2id: for type in lang2id[lpk]: if type == 'name': for name, prov in lang2id[lpk][type]: altnames.append((prov, name, pk2id['language'][lpk])) lnid = 0 for (type, name), rows in itertools.groupby(sorted(altnames), lambda t: (t[0], t[1])): lnid += 1 args.writer.objects['language_names.csv'].append({ 'ID': str(lnid), 'Language_ID': [r[2] for r in rows], 'Name': name, 'Provider': type, })
def cmd_makecldf(self, args): self.create_schema(args.writer.cldf) pk2id = collections.defaultdict(dict) skip_source = [ 'Lous-1969', # -> Loos-1969 'Payne-1990', # -> Payne-1990a ] updated_source_keys = { 'Anonymous-nd': 'North-East-Frontier-Agency-1963', } updated_source_names = { 'North-East-Frontier-Agency-1963': 'North East Frontier Agency 1963', } sources = parse_string( self.raw_dir.joinpath('source.bib').read_text(encoding='utf8'), 'bibtex') gbs_lg_refs = collections.defaultdict(set) src_names = {} for s in self.read('source', pkmap=pk2id).values(): if s['id'] in skip_source: continue s['id'] = updated_source_keys.get(s['id'], s['id']) src_names[s['id']] = updated_source_names.get(s['id'], s['name']) try: jsd = json.loads(s['jsondata']) if 'wals_code' in jsd: [gbs_lg_refs[c].add(s['id']) for c in jsd['wals_code']] gbs = jsd['gbs'] if gbs['id'].strip(): sef = sources.entries[s['id']].fields sef['google_book_search_id'] = gbs['id'].strip() sef['google_book_viewability'] = gbs['accessInfo'][ 'viewability'].strip() except (json.decoder.JSONDecodeError, KeyError): continue chapters = self.read('contribution', extended='chapter', pkmap=pk2id) refs = [] crefs = collections.defaultdict(list) for row in self.raw_dir.read_csv('valuesetreference.csv', dicts=True): if row['source_pk']: sid = pk2id['source'][row['source_pk']] if sid not in skip_source: refs.append( (row['valueset_pk'], updated_source_keys.get(sid, sid), row['description'])) srcids = set(r[1] for r in refs) for row in self.raw_dir.read_csv('contributionreference.csv', dicts=True): sid = pk2id['source'][row['source_pk']] if sid not in crefs[pk2id['contribution'][row['contribution_pk']]]: crefs[pk2id['contribution'][row['contribution_pk']]].append( sid) srcids.add(sid) unused_srcids = [] for id_, e in sources.entries.items(): if id_ in skip_source: continue if id_ in srcids: if id_ in src_names: e.fields['wals_ref_name'] = src_names[id_] args.writer.cldf.add_sources(Source.from_entry(id_, e)) else: unused_srcids.append(id_) # add language references out of bibtex tag 'wals_code' # to ensure that nothing was missed in raw/languagesource.csv (37 cases) if 'wals_code' in e.fields: [ gbs_lg_refs[c].add(id_) for c in e.fields['wals_code'].split('; ') ] for id_, e in sources.entries.items(): if id_ in skip_source: continue if id_ in unused_srcids: if id_ in src_names: e.fields['wals_ref_name'] = src_names[id_] args.writer.cldf.add_sources(Source.from_entry(id_, e)) editors = { e['contributor_pk']: int(e['ord']) for e in self.read('editor', key=lambda r: int(r['ord'])).values() } contributors = self.read('contributor', pkmap=pk2id, key=lambda r: r['id']) for row in contributors.values(): args.writer.objects['contributors.csv'].append({ 'ID': row['id'], 'Name': row['name'], 'Url': row['url'], 'Editor_Ord': editors[row['pk']] if row['pk'] in editors else 0, }) cc = { chapters[fid]['id']: [(r['primary'], pk2id['contributor'][r['contributor_pk']]) for r in rows] for fid, rows in itertools.groupby( self.read('contributioncontributor', key=lambda d: (d['contribution_pk'], d['primary'] == 'f', int(d['ord']))).values(), lambda r: r['contribution_pk']) } areas = self.read('area') for row in areas.values(): args.writer.objects['areas.csv'].append({ 'ID': row['id'], 'Name': row['name'], 'dbpedia_url': row['dbpedia_url'], }) for row in self.read('parameter', extended='feature', pkmap=pk2id, key=lambda d: fid_key(d['id'])).values(): args.writer.objects['ParameterTable'].append({ 'ID': row['id'], 'Name': row['name'], 'Chapter_ID': chapters[row['contribution_pk']]['id'], }) for row in self.read( 'domainelement', pkmap=pk2id, key=lambda d: (fid_key(d['id'].split('-')[0]), int(d['number']))).values(): args.writer.objects['CodeTable'].append({ 'ID': row['id'], 'Parameter_ID': pk2id['parameter'][row['parameter_pk']], 'Name': row['name'], 'Description': row['description'], 'Number': int(row['number']), 'icon': json.loads(row['jsondata'])['icon'], }) identifier = self.read('identifier') lang2id = collections.defaultdict( lambda: collections.defaultdict(list)) for row in self.read('languageidentifier').values(): id_ = identifier[row['identifier_pk']] lang2id[row['language_pk']][id_['type']].append( (id_['name'], id_['description'])) families = self.read('family', pkmap=pk2id) genera = self.read('genus', pkmap=pk2id) countries = self.read('country', pkmap=pk2id) lang2country = collections.defaultdict(list) for c in self.read('countrylanguage').values(): lang2country[c['language_pk']].append( pk2id['country'][c['country_pk']]) lrefs = collections.defaultdict(list) for c in self.read('languagesource').values(): sid = pk2id['source'][c['source_pk']] sid = updated_source_keys.get(sid, sid) if sid not in lrefs[c['language_pk']]: lrefs[c['language_pk']].append(sid) for row in self.read('language', extended='walslanguage', pkmap=pk2id).values(): id = row['id'] genus = genera[row['genus_pk']] genus_icon = genus['icon'] if genus else '' family = families[genus['family_pk']] if row['name'] == genus['name'] == family['name']: # an isolate! genus = family = None iso_codes = set(i[0] for i in lang2id[row['pk']].get('iso639-3', [])) glottocodes = [ i[0] for i in lang2id[row['pk']].get('glottolog', []) ] srcs = lrefs[row['pk']] if id in gbs_lg_refs: [srcs.append(s) for s in gbs_lg_refs[id] if s not in srcs] args.writer.objects['LanguageTable'].append({ 'ID': id, 'Name': row['name'].strip(), 'ISO639P3code': list(iso_codes)[0] if len(iso_codes) == 1 else None, 'Glottocode': glottocodes[0] if len(glottocodes) == 1 else None, 'ISO_codes': sorted(iso_codes), 'Latitude': row['latitude'], 'Longitude': row['longitude'], 'Macroarea': row['macroarea'], 'Genus': genus['name'] if genus else None, 'GenusIcon': genus_icon, 'Subfamily': genus['subfamily'] if genus else None, 'Family': family['name'] if family else None, 'Samples_100': row['samples_100'] == 't', 'Samples_200': row['samples_200'] == 't', 'Country_ID': lang2country[row['pk']], 'Source': sorted(srcs), }) args.writer.objects['LanguageTable'].sort(key=lambda d: d['ID']) refs = { dpid: [ str( Reference( source=str(r[1]), desc=r[2].replace('[', ')').replace(']', ')').replace( ';', '.').strip() if r[2] else None)) for r in refs_ ] for dpid, refs_ in itertools.groupby(refs, lambda r: r[0]) } vsdict = self.read('valueset', pkmap=pk2id) examples = self.read('sentence', pkmap=pk2id) igts = {} for ex in examples.values(): if all(ex[k] for k in ['description', 'analyzed', 'gloss']): a, g = ex['analyzed'].split(), ex['gloss'].split() if len(a) != len(g): a, g = [ex['analyzed']], [ex['gloss']] igts[ex['pk']] = ex['id'] args.writer.objects['ExampleTable'].append({ 'ID': ex['id'], 'Language_ID': pk2id['language'][ex['language_pk']], 'Primary_Text': ex['name'], 'Translated_Text': ex['description'], 'Analyzed_Word': a, 'Gloss': g, }) example_by_value = { vpk: [r['sentence_pk'] for r in rows] for vpk, rows in itertools.groupby( self.read('valuesentence', key=lambda d: d['value_pk']).values( ), lambda d: d['value_pk']) } for row in self.read('value').values(): vs = vsdict[row['valueset_pk']] comment = None ex = [examples[spk] for spk in example_by_value.get(row['pk'], [])] if len(ex) == 1 and not any( ex[0][k] for k in ['description', 'analyzed', 'gloss']): comment = re.sub(r'[\r\n]', '', ex[0]['xhtml']) del example_by_value[row['pk']] args.writer.objects['ValueTable'].append({ 'ID': vs['id'], 'Language_ID': pk2id['language'][vs['language_pk']], 'Parameter_ID': pk2id['parameter'][vs['parameter_pk']], 'Value': pk2id['domainelement'][row['domainelement_pk']].split('-')[1], 'Code_ID': pk2id['domainelement'][row['domainelement_pk']], 'Comment': comment, 'Source': refs.get(vs['pk'], []), 'Example_ID': sorted(igts[epk] for epk in example_by_value.get(row['pk'], []) if epk in igts), }) args.writer.objects['ValueTable'].sort( key=lambda d: (d['Language_ID'], fid_key(d['Parameter_ID']))) altnames = [] for lpk in lang2id: for type in lang2id[lpk]: if type == 'name': for name, prov in lang2id[lpk][type]: altnames.append((prov, name, pk2id['language'][lpk])) lnid = 0 for (type, name), rows in itertools.groupby(sorted(altnames), lambda t: (t[0], t[1])): lnid += 1 args.writer.objects['language_names.csv'].append({ 'ID': str(lnid), 'Language_ID': [r[2] for r in rows], 'Name': name.strip(), 'Provider': type, }) for c in sorted(countries.values(), key=lambda x: x['id']): args.writer.objects['countries.csv'].append({ 'ID': c['id'], 'Name': c['name'], }) desc_dir = self.raw_dir / 'descriptions' src_pattern = re.compile( 'src="https?://wals.info/static/descriptions/(?P<sid>s?[0-9]+)/images/(?P<fname>[^"]+)"' ) def repl(m): p = desc_dir.joinpath(m.group('sid'), 'images', m.group('fname')) if p.exists(): return 'src="{0}"'.format(data_url(p)) return m.string[m.start():m.end()] descs = {} docs_dir = self.cldf_dir / 'docs' docs_dir.mkdir(exist_ok=True) for d in desc_dir.iterdir(): if d.is_dir(): descs[d.stem] = src_pattern.sub( repl, d.joinpath('body.xhtml').read_text(encoding='utf8')) for c in sorted(chapters.values(), key=lambda x: int(x['sortkey'])): if c['id'] in descs: fname = docs_dir / 'chapter_{}.html'.format(c['id']) with io.open(fname, 'w', encoding='utf-8') as f: f.write(descs[c['id']]) cid, wcid = [], [] if c['id'] in cc: cid = [co[1] for co in cc[c['id']] if co[0] == 't'] wcid = [co[1] for co in cc[c['id']] if co[0] == 'f'] args.writer.objects['chapters.csv'].append({ 'ID': c['id'], 'Name': c['name'], 'wp_slug': c['wp_slug'], 'Number': c['sortkey'], 'Area_ID': areas[c['area_pk']]['id'] if c['area_pk'] in areas else '', 'Source': crefs.get(c['id'], []), 'Contributor_ID': cid, 'With_Contributor_ID': wcid, })
def cmd_makecldf(self, args): args.writer.add_sources(self.raw_dir.read("Citations.bib")) for src in self._read("Citation_codes"): if src["type"] == "E": args.writer.add_sources( Source("misc", src["ref_abbr"], author=src["original_reference"])) glottocodes = { language["ID"]: language["Glottocode"] for language in self.languages } for language in self._read("Languages"): glottocode = glottocodes.get(language["lgid3"]) if not glottocode: glottocode = self.glottolog.glottocode_by_iso.get( language["ISO-639-3"]) args.writer.add_language( ID=language["lgid3"], Name=language["language"], Glottocode=glottocode, Description=language["Description"], Subgroup=language["Subgroup"], ISO639P3code=language["ISO-639-3"], ) for concept in self.concepts: args.writer.add_concept(**concept) for (cid, cogid), ll in itertools.groupby( sorted(self._read("Data"), key=lambda i: (i["mng_item"], i["cogn_set"])), lambda i: (i["mng_item"], i["cogn_set"]), ): for language in ll: kw = dict( Value=language["item"], Language_ID=language["lgid3"], Parameter_ID=cid, Comment=language["general_notes"], Source=[ slug(rid, lowercase=False) for rid in split_text( language["ref_abbr"], ",", strip=True) ], ) kw.update({ k: language[k] for k in [ "item_UPA", "item_IPA", "form_set", "age_term_pq", "age_term_aq", "borr_source", "borr_qual", "etym_notes", "glossing_notes", ] }) for lex in args.writer.add_lexemes(**kw): if cogid != "?": args.writer.add_cognate(lexeme=lex, Cognateset_ID="{0}-{1}".format( cid, cogid))
def cmd_makecldf(self, args): args.writer.add_sources(self.raw_dir.read("Citations.bib")) bib = parse_string(self.raw_dir.read('Borrowing_references.bib'), 'bibtex') for k, v in bib.entries.items(): args.writer.add_sources( Source.from_entry(slug(k, lowercase=False), v)) args.writer.cldf.add_component( 'BorrowingTable', { 'name': 'Likelihood', 'dc:description': 'Likelihood of borrowing (*possible*, *probable* or *clear*).', 'datatype': { 'base': 'string', 'format': 'possible|clear|probable' } }, { 'name': 'SourceLanguoid', 'dc:description': 'Borrowing source of lexeme.', }) args.writer.cldf['FormTable', 'form'].required = False args.writer.cldf['FormTable', 'value'].null = NULL_ITEMS args.writer.cldf['FormTable', 'value'].required = False args.writer.cldf['FormTable', 'value'].common_props['dc:description'] = \ "Lexeme data. Contains a lexeme or '[No equivalent]': no suitable equivalent for a meaning exists), " \ "'[Form not found]': no suitable equivalent was found, or '[Not reconstructable]': non-recontructable " \ "meanings in Proto-Uralic." for src in self._read("Citation_codes"): if src["type"] == "E": args.writer.add_sources( Source("misc", src["ref_abbr"], author=src["original_reference"])) glottocodes = { language["ID"]: language["Glottocode"] for language in self.languages } for language in self._read("Languages"): glottocode = glottocodes.get(language["lgid3"]) if not glottocode: glottocode = self.glottolog.glottocode_by_iso.get( language["ISO-639-3"]) args.writer.add_language( ID=language["lgid3"], Name=language["language"], Glottocode=glottocode, Description=language["Description"], Subgroup=language["Subgroup"], ISO639P3code=language["ISO-639-3"], ) inlists = {r['mng_item']: r for r in self._read('Meaning_lists')} attrs = [ k for k in attr.fields_dict(UralexConcept).keys() if k != 'LJ_rank' ] for concept in self.concepts: if concept['ID'] in inlists: memberships = { k.replace('-', '_'): v == '1' for k, v in inlists[concept['ID']].items() if k.replace('-', '_') in attrs } concept.update(memberships) args.writer.add_concept(**concept) for (cid, cogid), ll in itertools.groupby( sorted(self._read("Data"), key=lambda i: (i["mng_item"], i["cogn_set"])), lambda i: (i["mng_item"], i["cogn_set"]), ): for language in ll: if language['item'] in NULL_ITEMS: language['etym_notes'] = language['etym_notes'] + language[ 'item'] kw = dict( Value=language["item"], Language_ID=language["lgid3"], Parameter_ID=cid, Comment=language["general_notes"], Source=[ slug(rid, lowercase=False) for rid in split_text( language["ref_abbr"], ",", strip=True) ], ) kw.update({ k: language[k] for k in [ "item_UPA", "item_IPA", "form_set", "etym_notes", "glossing_notes", ] }) for i, lex in enumerate(args.writer.add_lexemes(**kw)): lex['Form'] = None if lex['Form'] in NULL_ITEMS else lex[ 'Form'] if cogid not in ["?", "0"]: args.writer.add_cognate(lexeme=lex, Cognateset_ID="{0}-{1}".format( cid, cogid)) if language['borr_qual']: c = ': borrowed to Pre-Permic' ref = language['ref_borr'] if c in ref: comment = c[1:].strip() ref = ref.replace(c, '') else: comment = None args.writer.objects['BorrowingTable'].append( dict( ID=lex['ID'], Target_Form_ID=lex['ID'], SourceLanguoid=language['borr_source'], Likelihood=language['borr_qual'], Source=bibkeys(ref), Comment=comment, ))
def to_cldf(self, concept_map, unmapped, citekey=None, source=None, concept_key=None): if concept_key is None: concept_key = lambda entry: entry.word_id if not self.language.glottocode: unmapped.languages.add( (self.language.id, self.language.name, self.language.iso)) with CldfDataset(( 'ID', 'Language_ID', 'Language_iso', 'Language_name', 'Language_local_ID', 'Parameter_ID', 'Parameter_name', 'Parameter_local_ID', 'Value', 'Value_in_source', 'Segments', 'Context', 'Source', 'Cognate_Set', 'Comment', 'Loan', ), self.dataset, subset=self.language.id) as ds: ds.metadata['dc:creator'] = self.language.author ds.metadata['dc:identifier'] = self.url('language.php?id=%s' % self.language.id) if self.language.typedby: ds.metadata['dc:contributor'] = self.language.typedby if self.language.checkedby: ds.metadata['dc:contributor'] = self.language.checkedby if self.language.notes: ds.metadata['dc:description'] = self.language.notes ds.table.schema.aboutUrl = '%s.csv#{ID}' % ds.name ds.table.schema.columns['Loan'].datatype = 'boolean' ds.table.schema.columns['Parameter_local_ID'].valueUrl = \ self.url('word.php?v=1{Parameter_local_ID}') ds.table.schema.columns['Language_local_ID'].valueUrl = \ self.url('language.php?id={Language_local_ID}') ref = None if citekey and source: ref = citekey ds.sources.add(Source('misc', citekey, title=source)) for entry in self.entries: if entry.name == '?': continue if not (citekey and source): src = entry.e.find('source') if src and getattr(src, 'text'): ref = slug(text_type(src.text)) ds.sources.add(Source('misc', ref, title=src.text)) cid = concept_map.get(concept_key(entry)) if not cid: unmapped.concepts.add((entry.word_id, entry.word)) for i, (form, context) in enumerate(util.split(entry.name)): ds.add_row([ '{0}-{1}'.format(entry.id, i + 1), self.language.glottocode, self.language.iso, self.language.name, self.language.id, cid, entry.word, entry.word_id, util.clean_form(form), form, '', context, ref, entry.cognacy, entry.comment or '', entry.loan == 'L', ]) segmentize(ds) return ds
def cldf(dataset, concepticon, **kw): concepticon = { c.english: c.concepticon_id for c in dataset.conceptlist.concepts.values() } concepticon['you (sing.)'] = concepticon['you (sing.) (thou)'] concepticon['you (pl.)'] = concepticon['you (pl.) (ye)'] concepticon['to itch/itchy'] = concepticon['to itch/to be itchy'] concepticon['medicine'] = concepticon['medicine/juice'] concepticon['excrement/shit'] = concepticon['feces/excrement/shit'] language_map = { 'Tampuon': 'Tampuan', 'Palaung-Namhsan-Taunggyi': 'Palaung-Namhsan', 'Jru-Laven\u02d0': 'Jru-Laven', 'Pnar-Jaintia': 'Pnar', 'K-Surin': 'Khmer-Surin', } languages = {} words = [] with UnicodeReader(dataset.raw.joinpath('ds.Sheet1.csv')) as reader: for i, row in enumerate(reader): if 3 <= i < 125: languages[row[1]] = row elif i > 334: words.append(row) lids = [int(float(r[0])) for r in languages.values()] assert min(lids) == 1 and max(lids) == 122 glottolog = dataset.glottocode_by_iso glottolog.update( {l['NAME']: l['GLOTTOCODE'] or None for l in dataset.languages}) sources = {} for src, langs in groupby(sorted(languages.values(), key=lambda r: r[6]), lambda r: r[6]): langs = [l[1] for l in langs] src = Source('misc', '_'.join(map(slug, langs)), title=src) for lang in langs: sources[lang] = src sources['cognates'] = getEvoBibAsSource(SOURCE) unmapped = Unmapped() with CldfDataset(( 'ID', 'Language_ID', 'Language_name', 'Language_iso', 'Parameter_ID', 'Parameter_name', 'Value', 'Segments', 'Source', 'Comment', ), dataset) as ds: ds.sources.add(*sources.values()) D = {0: ['lid', 'doculect', 'concept', 'ipa', 'tokens', 'cog']} for i, row in enumerate(words): form = row[4] if not form or form in '*-': continue assert row[1] in concepticon lang = language_map.get(row[3], row[3].strip()) assert lang in languages gc = glottolog.get(glottolog.get(languages[lang][7]), lang) if not gc: unmapped.languages.add(('', lang, languages[lang][7])) # get segments segments = clean_string(form)[0] # get cognate identifier cogid = row[5] if row[5].strip() and row[5].strip() != '*' else ( 'e%s' % i) cogid = row[1] + '-' + cogid lid = '{0}-{1}'.format(ds.name, i + 1) ds.add_row([ lid, glottolog.get(lang, glottolog.get(languages[lang][7])), lang, languages[lang][7], concepticon[row[1]], row[1], form, segments, sources[lang].id, None ]) D[i + 1] = [lid, lang, row[1], form, segments, cogid] wl = lp.Wordlist(D) wl.renumber('cog') alm = lp.Alignments(wl) dataset.cognates.extend( iter_alignments(alm, wordlist2cognates(wl, ds, SOURCE))) unmapped.pprint()
def test_Source_persons(): assert len(list(Source.persons('A. Meier'))) == 1 assert len(list(Source.persons('Meier, A.B.'))) == 1 assert len(list(Source.persons('A. Meier, B. Meier, C.Meier'))) == 3
def test_Source_persons(): assert len(list(Source.persons('A. Meier'))) == 1 assert len(list(Source.persons('Meier, A.B.'))) == 1 assert len(list(Source.persons('A. Meier, B. Meier, C.Meier'))) == 3
def test_Sources_with_None_values(tmpdir): src = Sources() src.add(Source('book', 'huber2005', title=None)) bib = tmpdir / 'test.bib' src.write(str(bib))
def test_Source_from_bibtex(): bibtex = '@' + BIB.split('@')[1] assert Source.from_bibtex(bibtex).entry.fields['title'] == 'Obrazy z Rus'
def test_Source_persons(self): from pycldf.sources import Source self.assertEqual(len(list(Source.persons('A. Meier'))), 1) self.assertEqual(len(list(Source.persons('Meier, A.B.'))), 1) self.assertEqual(len(list(Source.persons('A. Meier, B. Meier, C.Meier'))), 3)
def test_Source_from_bibtex(): bibtex = '@' + BIB.split('@')[1] assert Source.from_bibtex(bibtex).entry.fields['title'] == 'Obrazy z Rus'
def cmd_makecldf(self, args): self.create_schema(args.writer.cldf) rmap = collections.defaultdict(list) for row in self.query('select * from "references"'): rmap[row['language_id']].append(row['id']) kw = {} for key, bkey in { 'authors': 'author', 'year': 'year', 'article_title': 'title', 'editors': 'editor', 'book_title': 'booktitle', 'city': 'address', 'issue': 'issue', 'journal': 'journal', 'pages': 'pages', 'publisher': 'publisher', 'series_title': 'series', 'url': 'url', 'volume': 'volume', 'additional_information': 'note', }.items(): if row[key]: kw[bkey] = row[key] args.writer.add_sources( Source(row['bibtex_type'] or 'misc', str(row['id']), **kw)) all_sources = {rid for lang_refs in rmap.values() for rid in lang_refs} contributors = { r['id']: r['name'] for r in self.raw_dir.read_csv('contributors.csv', dicts=True) } contributions = {} for lid, rows in itertools.groupby( sorted(self.raw_dir.read_csv('contributions.csv', dicts=True), key=lambda d: (d['language_id'], int(d['sort_order_number'] or 0))), lambda d: d['language_id'], ): contributions[int(lid)] = ' and '.join([ contributors[d['person_id']] for d in rows if d['person_id'] in contributors ]) lang2gl = {l['Name']: l['Glottocode'] for l in self.languages} lmap = {} for lang in self.query('select * from languages order by name'): lang['glottolog_code'] = lang2gl.get(lang['name'], lang['glottolog_code']) lmap[lang['id']] = lang['glottolog_code'] name = lang['name'] if lang['variety']: name += ' ({0})'.format(lang['variety']) args.writer.add_language( ID=lang['glottolog_code'], Name=name, Glottocode=lang['glottolog_code'], ISO639P3code=lang['iso_code'], Latitude=lang['latitude'], Longitude=lang['longitude'], Comment=lang['comments'], contributors=contributions[lang['id']], continent=lang['continent'], ) args.writer.add_concepts(id_factory=lambda c: c.attributes[ 'label_for_url'].replace('/', '_')) cmap = {} for meaning in self.query("select * from meanings order by label"): cmap[meaning['id']] = meaning['label_for_url'].replace('/', '_') args.writer.add_concept( ID=meaning['label_for_url'].replace('/', '_'), Name=meaning['label'], typical_context=meaning['typical_context'], ) fmap = {} forms = """\ select v.id, v.verb_form, v.original_script, v.comment, v.simplex_or_complex, v.verb_type, v.language_id, v.coding_frame_id, mv.meaning_id from verbs as v, meanings_verbs as mv where v.id = mv.verb_id order by v.language_id, mv.meaning_id;""" for row in self.query(forms): # # FIXME: flesh out data! # res = args.writer.add_forms_from_value( Value=row['verb_form'], Language_ID=lmap[row['language_id']], Parameter_ID=cmap[row['meaning_id']], Comment=row['comment'], Source=rmap[row['language_id']], verb_type=row['verb_type'], original_script=row['original_script'], simplex_or_complex=row['simplex_or_complex'], Basic_Coding_Frame_ID=row['coding_frame_id'], # # FIXME: mark singular (SG) and plural (PL)? # ) fmap[row['id']] = [r['ID'] for r in res] #CREATE TABLE "examples" ( # "id" integer(8), # "reference_id" integer(8), # "person_id" integer(8), # "original_orthography" text, # "media_file_name" varchar(255), # "media_file_timecode" varchar(255), # "reference_pages" varchar(255), # "number" integer); gloss_fix = { 'find.out-NTR<STV>-OBJ-3.ERG DET= boy': 'find.out-NTR<STV>-OBJ-3.ERG DET=boy', 'scream<STV> DET= PL-child': 'scream<STV> DET=PL-child', 'boy.ERG brother.OBL-INSTR girl.OBL-LAT embrace(IV).ABS IV-fill-CAUS- PST': 'boy.ERG brother.OBL-INSTR girl.OBL-LAT embrace(IV).ABS IV-fill-CAUS-PST', } morphemes_fix = { r'tande-mne brbr-äm y/ram\te': 'tande-mne brbr-äm y/ram_te', } ex2verb = {} for row in self.query("""\ select e.id, group_concat(ev.verb_id, ' ') as vids from examples as e, examples_verbs as ev where e.id = ev.example_id group by e.id"""): ex2verb[row['id']] = [] for vid in set(row['vids'].split()): ex2verb[row['id']].extend(fmap[int(vid)]) def example_id(row): return '{0}-{1}'.format(lmap[row['language_id']], row['number']) def get_source(row): source = row.get('reference_id') if source and source in all_sources: ref_pages = row.get('reference_pages') or '' # occasionally, `reference_pages` seems to contain stuff other # than page numbers -- try to ignore that.. if re.fullmatch(r'\s*\d+([-–]+\d+)?\s*', ref_pages): return ['{}[{}]'.format(source, ref_pages)] else: return [str(source)] else: return None maxnum = {} for row in self.query( "select * from examples order by language_id, number desc"): if row['language_id'] not in maxnum: maxnum[row['language_id']] = row['number'] if row['number'] == 0: row['number'] = maxnum[ row['language_id']] = maxnum[row['language_id']] + 1 row['gloss'] = gloss_fix.get(row['gloss'], row['gloss']) row['analyzed_text'] = morphemes_fix.get(row['analyzed_text'], row['analyzed_text']) args.writer.objects['ExampleTable'].append( dict( ID=example_id(row), Language_ID=lmap[row['language_id']], Primary_Text=row['primary_text'], Analyzed_Word=row['analyzed_text'].split(), Gloss=row['gloss'].split(), Translated_Text=row['translation'], Comment=row['comment'], Original_Orthography=row['original_orthography'], Translation_Other=row['translation_other'], Number=row['number'], Example_Type=row['example_type'], Source=get_source(row), Form_IDs=sorted(ex2verb.get(row['id'], [])), )) for row in self.query( "select * from microroles order by meaning_id, id"): args.writer.objects['microroles.csv'].append( dict( ID=row['id'], Name=row['name'], Parameter_ID=cmap[row['meaning_id']], Role_Letter=row['role_letter'], Original_Or_New=row['original_or_new'], Name_For_URL=row['name_for_url'], )) for row in self.query( "select * from coding_sets order by language_id, id"): lang_id = lmap.get(row['language_id']) if lang_id: args.writer.objects['coding-sets.csv'].append( dict( ID=row['id'], Name=row['name'], Comment=row['comment'], Language_ID=lang_id, )) for row in self.query( 'select * from coding_frames order by language_id, id'): args.writer.objects['coding-frames.csv'].append( dict( ID=row['id'], Language_ID=lmap[row['language_id']], Coding_Frame_Schema=row['coding_frame_schema'], Description=row['description'], Comment=row['comment'], Derived=row['derived'], )) cf_roles = collections.defaultdict(list) for row in self.query( 'SELECT coding_frame_index_number_id AS index_id, microrole_id' ' FROM coding_frame_index_numbers_microroles'): cf_roles[row['index_id']].append(str(row['microrole_id'])) imap = {} coding_frames = { row['ID'] for row in args.writer.objects['coding-frames.csv'] } for row in self.query( 'SELECT i.id, coding_frame_id, index_number, coding_set_id, argument_type' ' FROM coding_frame_index_numbers AS i' ' JOIN argument_types AS a' ' ON a.id = i.argument_type_id' ' ORDER BY coding_frame_id, index_number'): if row['coding_frame_id'] not in coding_frames: continue roles = cf_roles.get(row['id']) new_id = '{}-{}'.format(row['coding_frame_id'], row['index_number']) imap[row['id']] = new_id args.writer.objects['coding-frame-index-numbers.csv'].append( dict( ID=new_id, Coding_Frame_ID=row['coding_frame_id'], Index_Number=row['index_number'], Coding_Set_ID=row['coding_set_id'], Argument_Type=row['argument_type'], Microrole_IDs=roles, )) form_cf_roles = collections.OrderedDict() for row in self.query('SELECT verb_id, coding_frame_id, microrole_id' ' FROM verb_coding_frame_microroles'): pair = (row['verb_id'], row['coding_frame_id']) if pair not in form_cf_roles: form_cf_roles[pair] = [] form_cf_roles[pair].append(row['microrole_id']) args.writer.objects['form-coding-frame-microroles.csv'] = [{ 'ID': '{}-{}'.format(fmap[verb_id][0], cf_id), 'Form_ID': fmap[verb_id][0], 'Coding_Frame_ID': str(cf_id), 'Microrole_IDs': list(map(str, role_ids)), } for (verb_id, cf_id), role_ids in form_cf_roles.items() if verb_id in fmap] cf_examples = collections.OrderedDict() for row in self.query( 'select verb_id,coding_frame_id,language_id,number' ' from coding_frame_examples' ' join examples on example_id = examples.id'): verb_id = fmap.get(row['verb_id']) if verb_id: key = (verb_id[0], row['coding_frame_id']) if key not in cf_examples: cf_examples[key] = [] cf_examples[key].append(example_id(row)) args.writer.objects['coding-frame-examples.csv'] = [{ 'ID': str(index + 1), 'Form_ID': verb_id, 'Coding_Frame_ID': cf_id, 'Example_IDs': ex_ids, } for (index, ((verb_id, cf_id), ex_ids)) in enumerate(cf_examples.items())] for row in self.query( "select * from alternations order by language_id, id"): args.writer.objects['alternations.csv'].append( dict( ID=row['id'], Name=re.sub(r'</?span[^>]*>', '', row['name'], flags=re.I), Description=row['description'], Language_ID=lmap[row['language_id']], Alternation_Type=row['alternation_type'], Coding_Frames_Text=row['coding_frames_text'], Complexity=row['complexity'], )) av_examples = collections.defaultdict(list) for row in self.query( 'select alternation_value_id, language_id, number' ' from alternation_values_examples as ve' ' join examples on ve.example_id = examples.id' ' order by alternation_value_id, language_id, number'): av_examples[row['alternation_value_id']].append(example_id(row)) for row in self.query( 'select * from alternation_values order by verb_id, alternation_id, id' ): vid = int(row['verb_id']) if vid in fmap and fmap[vid]: args.writer.objects['alternation-values.csv'].append( dict( ID=row['id'], Form_ID=fmap[vid][0], Alternation_ID=row['alternation_id'], Alternation_Occurs=row['alternation_occurs'], Comment=row['comment'], Derived_Code_Frame_ID=row['derived_coding_frame_id'], Example_IDs=av_examples.get(row['id']), ))