def test_BibFile_getitem(tmp_path):
bib = tmp_path.joinpath('test.bib')
bib.write_text("""\
@book{ab ,
title={ab}
}
@book{a ,
title={a}
}
""",
encoding='utf8')
bf = BibFile(bib)
assert bf['a'].key == 'a'
def __init__(self, dir_):
self.dir = Path(dir_)
self.datasets = [
Dataset(base_dir=self.dir.joinpath('datasets'), **r) for r in
reader(self.dir.joinpath('datasets', 'index.csv'), dicts=True)]
self.phylogenies = [
Phylogeny(base_dir=self.dir.joinpath('phylogenies'), **r) for r in
reader(self.dir.joinpath('phylogenies', 'index.csv'), dicts=True)]
self.societies = {
s.id: s for s in chain.from_iterable(d.societies for d in self.datasets)
}
self.variables = {
v.id: v for v in chain.from_iterable(d.societies for d in self.datasets)
}
self.sources = BibFile(self.dir.joinpath('datasets', 'sources.bib'))
def load(args):
glottolog = args.repos
fts.index('fts_index', models.Ref.fts, DBSession.bind)
DBSession.execute("CREATE EXTENSION IF NOT EXISTS unaccent WITH SCHEMA public;")
version = assert_release(glottolog.repos)
dataset = common.Dataset(
id='glottolog',
name="{0} {1}".format(glottolog.publication.web.name, version),
publisher_name=glottolog.publication.publisher.name,
publisher_place=glottolog.publication.publisher.place,
publisher_url=glottolog.publication.publisher.url,
license=glottolog.publication.license.url,
domain=purl.URL(glottolog.publication.web.url).domain(),
contact=glottolog.publication.web.contact,
jsondata={'license_icon': 'cc-by.png', 'license_name': glottolog.publication.license.name},
)
data = Data()
for e in glottolog.editors.values():
if e.current:
ed = data.add(common.Contributor, e.id, id=e.id, name=e.name)
common.Editor(dataset=dataset, contributor=ed, ord=int(e.ord))
DBSession.add(dataset)
contrib = data.add(common.Contribution, 'glottolog', id='glottolog', name='Glottolog')
DBSession.add(common.ContributionContributor(
contribution=contrib, contributor=data['Contributor']['hammarstroem']))
#
# Add Parameters:
#
add = functools.partial(add_parameter, data)
add('fc', name='Family classification')
add('sc', name='Subclassification')
add('aes',
args.repos.aes_status.values(),
name=args.repos.aes_status.__defaults__['name'],
pkw=dict(
jsondata=dict(
reference_id=args.repos.aes_status.__defaults__['reference_id'],
sources=[attr.asdict(v) for v in args.repos.aes_sources.values()],
scale=[attr.asdict(v) for v in args.repos.aes_status.values()])),
dekw=lambda de: dict(name=de.name, number=de.ordinal, jsondata=dict(icon=de.icon)),
)
add('med',
args.repos.med_types.values(),
name='Most Extensive Description',
dekw=lambda de: dict(
name=de.name, description=de.description, number=de.rank, jsondata=dict(icon=de.icon)),
)
add('macroarea',
args.repos.macroareas.values(),
pkw=dict(
description=args.repos.macroareas.__defaults__['description'],
jsondata=dict(reference_id=args.repos.macroareas.__defaults__['reference_id'])),
dekw=lambda de: dict(
name=de.name,
description=de.description,
jsondata=dict(geojson=read_macroarea_geojson(args.repos, de.name, de.description)),
),
)
add('ltype',
args.repos.language_types.values(),
name='Language Type',
dekw=lambda de: dict(name=de.category, description=de.description),
delookup='category',
)
add('country',
args.repos.countries,
dekw=lambda de: dict(name=de.id, description=de.name),
)
legacy = jsonlib.load(gc2version(args))
for gc, version in legacy.items():
data.add(models.LegacyCode, gc, id=gc, version=version)
#
# Now load languoid data, keeping track of relations that can only be inserted later.
#
lgsources = defaultdict(list)
# Note: We rely on languoids() yielding languoids in the "right" order, i.e. such that top-level
# nodes will precede nested nodes. This order must be preserved using an `OrderedDict`:
nodemap = OrderedDict([(l.id, l) for l in glottolog.languoids()])
lgcodes = {k: v.id for k, v in args.repos.languoids_by_code(nodemap).items()}
for lang in nodemap.values():
for ref in lang.sources:
lgsources['{0.provider}#{0.bibkey}'.format(ref)].append(lang.id)
load_languoid(glottolog, data, lang, nodemap)
for gc in glottolog.glottocodes:
if gc not in data['Languoid'] and gc not in legacy:
common.Config.add_replacement(gc, None, model=common.Language)
for obj in jsonlib.load(glottolog.references_path('replacements.json')):
common.Config.add_replacement(
'{0}'.format(obj['id']),
'{0}'.format(obj['replacement']) if obj['replacement'] else None,
model=common.Source)
DBSession.flush()
for doctype in glottolog.hhtypes:
data.add(
models.Doctype, doctype.id, id=doctype.id,
name=doctype.name,
description=doctype.description,
abbr=doctype.abbv,
ord=doctype.rank)
for bib in glottolog.bibfiles:
data.add(
models.Provider,
bib.id,
id=bib.id,
name=bib.title,
description=bib.description,
abbr=bib.abbr,
url=bib.url)
DBSession.flush()
s = time()
for i, entry in enumerate(
BibFile(glottolog.build_path('monster-utf8.bib'), api=glottolog).iterentries()):
if i % 10000 == 0:
args.log.info('{0}: {1:.3}'.format(i, time() - s))
s = time()
ref = load_ref(data, entry, lgcodes, lgsources)
if 'macro_area' in entry.fields:
mas = []
for ma in split_text(entry.fields['macro_area'], separators=',;', strip=True):
ma = 'North America' if ma == 'Middle America' else ma
ma = glottolog.macroareas.get('Papunesia' if ma == 'Papua' else ma)
mas.append(ma.name)
ref.macroareas = ', '.join(mas)
def load(args):
fts.index('fts_index', models.Ref.fts, DBSession.bind)
DBSession.execute("CREATE EXTENSION IF NOT EXISTS unaccent WITH SCHEMA public;")
dataset = common.Dataset(
id='glottolog',
name="Glottolog {0}".format(args.args[0]),
publisher_name="Max Planck Institute for the Science of Human History",
publisher_place="Jena",
publisher_url="https://shh.mpg.de",
license="http://creativecommons.org/licenses/by/4.0/",
domain='glottolog.org',
contact='*****@*****.**',
jsondata={
'license_icon': 'cc-by.png',
'license_name': 'Creative Commons Attribution 4.0 International License'})
data = Data()
for i, (id_, name) in enumerate([
('hammarstroem', 'Harald Hammarström'),
('bank', 'Sebastian Bank'),
('forkel', 'Robert Forkel'),
('haspelmath', 'Martin Haspelmath'),
]):
ed = data.add(common.Contributor, id_, id=id_, name=name)
common.Editor(dataset=dataset, contributor=ed, ord=i + 1)
DBSession.add(dataset)
clf = data.add(common.Contribution, 'clf', id='clf', name='Classification')
DBSession.add(common.ContributionContributor(
contribution=clf, contributor=data['Contributor']['hammarstroem']))
for pid, pname in [
('fc', 'Family classification'),
('sc', 'Subclassification'),
('vitality', 'Degree of endangerment'),
]:
data.add(common.Parameter, pid, id=pid, name=pname)
legacy = jsonlib.load(gc2version(args))
for gc, version in legacy.items():
data.add(models.LegacyCode, gc, id=gc, version=version)
glottolog = args.repos
for ma in Macroarea:
data.add(
models.Macroarea,
ma.name,
id=ma.name,
name=ma.value,
description=ma.description)
for country in glottolog.countries:
data.add(models.Country, country.id, id=country.id, name=country.name)
lgcodes, mas, countries, lgsources = {}, {}, {}, defaultdict(list)
languoids = list(glottolog.languoids())
nodemap = {l.id: l for l in languoids}
for lang in languoids:
for ref in lang.sources:
lgsources['{0.provider}#{0.bibkey}'.format(ref)].append(lang.id)
load_languoid(data, lang, nodemap)
mas[lang.id] = [ma.name for ma in lang.macroareas]
countries[lang.id] = [c.id for c in lang.countries]
lgcodes[lang.id] = lang.id
if lang.hid:
lgcodes[lang.hid] = lang.id
if lang.iso:
lgcodes[lang.iso] = lang.id
for gc in glottolog.glottocodes:
if gc not in data['Languoid'] and gc not in legacy:
common.Config.add_replacement(gc, None, model=common.Language)
for obj in jsonlib.load(glottolog.references_path('replacements.json')):
common.Config.add_replacement(
'{0}'.format(obj['id']),
'{0}'.format(obj['replacement']) if obj['replacement'] else None,
model=common.Source)
DBSession.flush()
for lid, maids in mas.items():
for ma in maids:
DBSession.add(models.Languoidmacroarea(
languoid_pk=data['Languoid'][lid].pk,
macroarea_pk=data['Macroarea'][ma].pk))
for lid, cids in countries.items():
for cid in cids:
DBSession.add(models.Languoidcountry(
languoid_pk=data['Languoid'][lid].pk,
country_pk=data['Country'][cid].pk))
for doctype in glottolog.hhtypes:
data.add(
models.Doctype, doctype.id, id=doctype.id,
name=doctype.name,
description=doctype.description,
abbr=doctype.abbv,
ord=doctype.rank)
for bib in glottolog.bibfiles:
data.add(
models.Provider,
bib.id,
id=bib.id,
name=bib.title,
description=bib.description,
abbr=bib.abbr,
url=bib.url)
DBSession.flush()
s = time()
for i, entry in enumerate(
BibFile(glottolog.build_path('monster-utf8.bib')).iterentries()):
if i % 10000 == 0:
args.log.info('{0}: {1:.3}'.format(i, time() - s))
s = time()
ref = load_ref(data, entry, lgcodes, lgsources)
if 'macro_area' in entry.fields:
for ma in split_text(entry.fields['macro_area'], separators=',;', strip=True):
ma = 'North America' if ma == 'Middle America' else ma
ma = Macroarea.get('Papunesia' if ma == 'Papua' else ma)
DBSession.add(models.Refmacroarea(
ref_pk=ref.pk, macroarea_pk=data['Macroarea'][ma.name].pk))
def compile(api, log=None, rebuild=False):
log = log or logging.getLogger('pyglottolog')
previous = api.references_path('monster.csv')
replacements = api.references_path('replacements.json')
monster = BibFile(fname=api.build_path('monster-utf8.bib'),
encoding='utf-8',
sortkey='bibkey')
log.info('%s open/rebuild bibfiles db' % time.ctime())
db = api.bibfiles.to_sqlite(api.build_path('_bibfiles.sqlite3'),
rebuild=rebuild)
log.info('%s compile_monster' % time.ctime())
m = dict(db.merged())
log.info('%s load hh.bib' % time.ctime())
hhbib = api.bibfiles['hh.bib'].load()
# Annotate with macro_area from lgcode when lgcode is assigned manually
log.info('%s macro_area_from_lgcode' % time.ctime())
m = macro_area_from_lgcode(m, api.macroarea_map)
# Annotate with hhtype
log.info('%s annotate hhtype' % time.ctime())
m = markconservative(m,
api.hhtypes.triggers,
hhbib,
api.hhtypes,
api.build_path('monstermark-hht.txt'),
rank=lambda l: api.hhtypes[l])
# Annotate with lgcode
log.info('%s annotate lgcode' % time.ctime())
m = markconservative(m, api.triggers['lgcode'], hhbib, api.hhtypes,
api.build_path('monstermark-lgc.txt'))
# Annotate with inlg
log.info('%s add_inlg_e' % time.ctime())
m = add_inlg_e(m, api.triggers['inlg'])
# Print some statistics
stats = Counter()
log.info(time.ctime())
for t, f in m.values():
stats.update(['entry'])
for field in ['lgcode', 'hhtype', 'macro_area']:
if field in f:
stats.update([field])
log.info("# entries {0}".format(stats['entry']))
for field in ['lgcode', 'hhtype', 'macro_area']:
log.info("with {0}: {1}".format(field, stats[field]))
# Update the CSV with the previous mappings for later reference
log.info('%s update_previous' % time.ctime())
db.to_csvfile(previous)
log.info('%s save_replacements' % time.ctime())
db.to_replacements(replacements)
# Trickling back
log.info('%s trickle' % time.ctime())
db.trickle()
# Save
log.info('%s save as utf8' % time.ctime())
monster.save(m)
log.info('%s done.' % time.ctime())
def sources(self):
return BibFile(self.path('sources.bib'))