def run(args): # pragma: no cover
try:
args.version = assert_release(args.repos.repos)
except AssertionError:
raise ParserError('glottolog-data must be checked out at release tag!')
_, settings = get_env_and_settings(
str(args.pkg_dir.parent / 'development.ini'))
with FreshDB.from_settings(settings, log=args.log):
dbload(args)
dbprime(args)
def mark_new_languages(args):
version = assert_release(args.repos.repos)
cfg = get_release_config()
last = sorted(set(cfg.sections()) - {version}, key=lambda s: float(s[1:]))[-1][1:]
cdb = create_engine('postgresql://postgres@/glottolog3')
ldb = create_engine('postgresql://postgres@/glottolog-{0}'.format(last))
sql = "select id from language as l, languoid as ll where l.pk = ll.pk and ll.level = 'language'"
current = set(r[0] for r in cdb.execute(sql))
last = set(r[0] for r in ldb.execute(sql))
for gc in sorted(current - last):
cdb.execute(
"update language set updated = %s where id = %s", (datetime.utcnow(), gc))
def run(args): # pragma: no cover
version = assert_release(args.repos.repos)
if args.previous:
last = args.previous
else:
rels = get_releases(args)
last = sorted(set(rel.tag for rel in rels) - {version},
key=lambda s: float(s[1:]))[-1][1:]
print('previous version: {0}'.format(last))
cdb = create_engine('postgresql://postgres@/glottolog3')
ldb = create_engine('postgresql://postgres@/glottolog-{0}'.format(last))
sql = "select id from language as l, languoid as ll where l.pk = ll.pk and ll.level = 'language'"
current = set(r[0] for r in cdb.execute(sql))
last = set(r[0] for r in ldb.execute(sql))
for gc in sorted(current - last):
cdb.execute("update language set updated = %s where id = %s",
(datetime.utcnow(), gc))
def load(args):
glottolog = args.repos
fts.index('fts_index', models.Ref.fts, DBSession.bind)
DBSession.execute("CREATE EXTENSION IF NOT EXISTS unaccent WITH SCHEMA public;")
version = assert_release(glottolog.repos)
dataset = common.Dataset(
id='glottolog',
name="{0} {1}".format(glottolog.publication.web.name, version),
publisher_name=glottolog.publication.publisher.name,
publisher_place=glottolog.publication.publisher.place,
publisher_url=glottolog.publication.publisher.url,
license=glottolog.publication.license.url,
domain=purl.URL(glottolog.publication.web.url).domain(),
contact=glottolog.publication.web.contact,
jsondata={'license_icon': 'cc-by.png', 'license_name': glottolog.publication.license.name},
)
data = Data()
for e in glottolog.editors.values():
if e.current:
ed = data.add(common.Contributor, e.id, id=e.id, name=e.name)
common.Editor(dataset=dataset, contributor=ed, ord=int(e.ord))
DBSession.add(dataset)
contrib = data.add(common.Contribution, 'glottolog', id='glottolog', name='Glottolog')
DBSession.add(common.ContributionContributor(
contribution=contrib, contributor=data['Contributor']['hammarstroem']))
#
# Add Parameters:
#
add = functools.partial(add_parameter, data)
add('fc', name='Family classification')
add('sc', name='Subclassification')
add('aes',
args.repos.aes_status.values(),
name=args.repos.aes_status.__defaults__['name'],
pkw=dict(
jsondata=dict(
reference_id=args.repos.aes_status.__defaults__['reference_id'],
sources=[attr.asdict(v) for v in args.repos.aes_sources.values()],
scale=[attr.asdict(v) for v in args.repos.aes_status.values()])),
dekw=lambda de: dict(name=de.name, number=de.ordinal, jsondata=dict(icon=de.icon)),
)
add('med',
args.repos.med_types.values(),
name='Most Extensive Description',
dekw=lambda de: dict(
name=de.name, description=de.description, number=de.rank, jsondata=dict(icon=de.icon)),
)
add('macroarea',
args.repos.macroareas.values(),
pkw=dict(
description=args.repos.macroareas.__defaults__['description'],
jsondata=dict(reference_id=args.repos.macroareas.__defaults__['reference_id'])),
dekw=lambda de: dict(
name=de.name,
description=de.description,
jsondata=dict(geojson=read_macroarea_geojson(args.repos, de.name, de.description)),
),
)
add('ltype',
args.repos.language_types.values(),
name='Language Type',
dekw=lambda de: dict(name=de.category, description=de.description),
delookup='category',
)
add('country',
args.repos.countries,
dekw=lambda de: dict(name=de.id, description=de.name),
)
legacy = jsonlib.load(gc2version(args))
for gc, version in legacy.items():
data.add(models.LegacyCode, gc, id=gc, version=version)
#
# Now load languoid data, keeping track of relations that can only be inserted later.
#
lgsources = defaultdict(list)
# Note: We rely on languoids() yielding languoids in the "right" order, i.e. such that top-level
# nodes will precede nested nodes. This order must be preserved using an `OrderedDict`:
nodemap = OrderedDict([(l.id, l) for l in glottolog.languoids()])
lgcodes = {k: v.id for k, v in args.repos.languoids_by_code(nodemap).items()}
for lang in nodemap.values():
for ref in lang.sources:
lgsources['{0.provider}#{0.bibkey}'.format(ref)].append(lang.id)
load_languoid(glottolog, data, lang, nodemap)
for gc in glottolog.glottocodes:
if gc not in data['Languoid'] and gc not in legacy:
common.Config.add_replacement(gc, None, model=common.Language)
for obj in jsonlib.load(glottolog.references_path('replacements.json')):
common.Config.add_replacement(
'{0}'.format(obj['id']),
'{0}'.format(obj['replacement']) if obj['replacement'] else None,
model=common.Source)
DBSession.flush()
for doctype in glottolog.hhtypes:
data.add(
models.Doctype, doctype.id, id=doctype.id,
name=doctype.name,
description=doctype.description,
abbr=doctype.abbv,
ord=doctype.rank)
for bib in glottolog.bibfiles:
data.add(
models.Provider,
bib.id,
id=bib.id,
name=bib.title,
description=bib.description,
abbr=bib.abbr,
url=bib.url)
DBSession.flush()
s = time()
for i, entry in enumerate(
BibFile(glottolog.build_path('monster-utf8.bib'), api=glottolog).iterentries()):
if i % 10000 == 0:
args.log.info('{0}: {1:.3}'.format(i, time() - s))
s = time()
ref = load_ref(data, entry, lgcodes, lgsources)
if 'macro_area' in entry.fields:
mas = []
for ma in split_text(entry.fields['macro_area'], separators=',;', strip=True):
ma = 'North America' if ma == 'Middle America' else ma
ma = glottolog.macroareas.get('Papunesia' if ma == 'Papua' else ma)
mas.append(ma.name)
ref.macroareas = ', '.join(mas)
def prime(args):
"""If data needs to be denormalized for lookup, do that here.
This procedure should be separate from the db initialization, because
it will have to be run periodically whenever data has been updated.
"""
#
# Now that we loaded all languoids and refs, we can compute the MED values.
#
meds = defaultdict(list)
for lpk, spk, sid, sname, med_type, year, pages in DBSession.execute("""\
select
l.pk, r.pk, s.id, s.name, r.med_type, s.year_int, r.med_pages
from
languagesource as ls,
language as l,
source as s,
ref as r
where
ls.active = TRUE and l.pk = ls.language_pk and s.pk = ls.source_pk and s.pk = r.pk
order by
l.id, r.med_index desc, r.med_pages, coalesce(s.year_int, 0), s.pk
"""):
meds[lpk].append((spk, sid, sname, med_type, year, pages)) # The last one is the overall MED
# Now weed out the "newer but worse" sources:
for lpk, sources in {k: reversed(v) for k, v in meds.items()}.items():
relevant, lastyear = [], 10000
for spk, sid, sname, med_type, year, pages in sources:
if year and year < lastyear: # If year is more recent, this is a "newer but worse" item
relevant.append((spk, sid, sname, med_type, year, pages))
lastyear = year
meds[lpk] = relevant
med_param = common.Parameter.get('med')
med_domain = {de.id: de for de in med_param.domain}
contrib = common.Contribution.get('glottolog')
for l in DBSession.query(common.Language).filter(common.Language.pk.in_(list(meds.keys()))):
l.update_jsondata(meds=[
(sid, med_type, year, pages, sname) for spk, sid, sname, med_type, year, pages in meds[l.pk]])
if not meds[l.pk]:
continue
med = meds[l.pk][0]
# Record the overall MED as value for the 'med' Parameter:
vs = common.ValueSet(
id=idjoin('med', l.id),
contribution=contrib,
parameter=med_param,
language=l,
)
DBSession.add(common.Value(
id=idjoin('med', l.id),
name=getattr(args.repos.med_types, med[3]).name,
domainelement=med_domain[idjoin('med', med[3])],
valueset=vs,
))
DBSession.flush()
DBSession.add(common.ValueSetReference(source_pk=med[0], valueset_pk=vs.pk))
recreate_treeclosure()
macroareas = {r[0]: (r[1], r[2]) for r in DBSession.execute("""\
select de.pk, de.id, de.name
from domainelement as de, parameter as p
where de.parameter_pk = p.pk and p.id = 'macroarea'
""")}
for lid, lpk, cpk, ppk, mas in DBSession.execute("""\
select
l.id, l.pk, vs.contribution_pk, vs.parameter_pk, array_agg(distinct v.domainelement_pk)
from
language as l,
treeclosuretable as t,
parameter as p,
valueset as vs,
value as v
where
l.pk = t.parent_pk and
t.child_pk = vs.language_pk and
vs.parameter_pk = p.pk and
p.id = 'macroarea' and
v.valueset_pk = vs.pk and
l.pk not in (
select language_pk
from valueset as _vs, parameter as _p
where _vs.parameter_pk = _p.pk and _p.id = 'macroarea'
)
group by l.id, l.pk, vs.contribution_pk, vs.parameter_pk"""):
for i, mapk in enumerate(mas):
if i == 0:
vs = common.ValueSet(
id=idjoin('macroarea', lid),
language_pk=lpk,
parameter_pk=ppk,
contribution_pk=cpk)
DBSession.add(common.Value(
id=idjoin(macroareas[mapk][0], lid),
name=macroareas[mapk][1],
domainelement_pk=mapk,
valueset=vs))
for vs in DBSession.query(common.ValueSet)\
.join(common.Language)\
.join(common.Parameter)\
.filter(common.Parameter.id == 'macroarea')\
.options(joinedload(common.ValueSet.values), joinedload(common.ValueSet.language)):
vs.language.macroareas = ', '.join([macroareas[v.domainelement_pk][1] for v in vs.values])
for row in list(DBSession.execute(
"select pk, pages, pages_int, startpage_int from source where pages_int < 0"
)):
raise ValueError(row)
version = assert_release(args.repos.repos)
with jsonlib.update(gc2version(args), indent=4) as legacy:
for lang in DBSession.query(common.Language):
if lang.id not in legacy:
lang.update_jsondata(new=True)
legacy[lang.id] = version
valuesets = {
r[0]: r[1] for r in DBSession.query(common.ValueSet.id, common.ValueSet.pk)}
refs = {
r[0]: r[1]
for r in DBSession.query(models.Refprovider.id, models.Refprovider.ref_pk)}
for vsid, vspk in valuesets.items():
if vsid.startswith('macroarea-'):
DBSession.add(common.ValueSetReference(
source_pk=refs[args.repos.macroareas.__defaults__['reference_id']],
valueset_pk=vspk))
for vs in DBSession.query(common.ValueSet)\
.join(common.Parameter)\
.filter(common.Parameter.id == 'aes'):
if vs.jsondata['reference_id']:
DBSession.add(common.ValueSetReference(
source_pk=refs[vs.jsondata['reference_id']], valueset_pk=vs.pk))
for lang in args.repos.languoids():
if lang.category == args.repos.language_types.bookkeeping.category:
continue
clf = lang.classification_comment
if clf:
for pid, attr_ in [('sc', 'sub'), ('fc', 'family')]:
if getattr(clf, attr_ + 'refs'):
if split_items(lang.cfg['classification'][attr_ + 'refs']) != \
split_items(lang.cfg['classification'].get(attr_)):
vspk = valuesets['{0}-{1}'.format(pid, lang.id)]
for ref in getattr(clf, attr_ + 'refs'):
spk = refs.get(ref.key)
if spk:
DBSession.add(
common.ValueSetReference(source_pk=spk, valueset_pk=vspk))
def load(args):
glottolog = args.repos
fts.index('fts_index', models.Ref.fts, DBSession.bind)
DBSession.execute("CREATE EXTENSION IF NOT EXISTS unaccent WITH SCHEMA public;")
version = assert_release(glottolog.repos)
dataset = common.Dataset(
id='glottolog',
name="Glottolog {0}".format(version),
publisher_name="Max Planck Institute for the Science of Human History",
publisher_place="Jena",
publisher_url="https://shh.mpg.de",
license="http://creativecommons.org/licenses/by/4.0/",
domain='glottolog.org',
contact='*****@*****.**',
jsondata={
'license_icon': 'cc-by.png',
'license_name': 'Creative Commons Attribution 4.0 International License'})
data = Data()
for i, (id_, name) in enumerate([
('hammarstroem', 'Harald Hammarström'),
('forkel', 'Robert Forkel'),
('haspelmath', 'Martin Haspelmath'),
]):
ed = data.add(common.Contributor, id_, id=id_, name=name)
common.Editor(dataset=dataset, contributor=ed, ord=i + 1)
DBSession.add(dataset)
clf = data.add(common.Contribution, 'clf', id='clf', name='Classification')
DBSession.add(common.ContributionContributor(
contribution=clf, contributor=data['Contributor']['hammarstroem']))
for pid, pname in [
('fc', 'Family classification'),
('sc', 'Subclassification'),
('vitality', 'Degree of endangerment'),
]:
data.add(common.Parameter, pid, id=pid, name=pname)
legacy = jsonlib.load(gc2version(args))
for gc, version in legacy.items():
data.add(models.LegacyCode, gc, id=gc, version=version)
for ma in Macroarea:
data.add(
models.Macroarea,
ma.name,
id=ma.name,
name=ma.value,
description=ma.description)
for country in glottolog.countries:
data.add(models.Country, country.id, id=country.id, name=country.name)
lgcodes, mas, countries, lgsources = {}, {}, {}, defaultdict(list)
languoids = list(glottolog.languoids())
nodemap = {l.id: l for l in languoids}
for lang in languoids:
for ref in lang.sources:
lgsources['{0.provider}#{0.bibkey}'.format(ref)].append(lang.id)
load_languoid(data, lang, nodemap)
mas[lang.id] = [ma.name for ma in lang.macroareas]
countries[lang.id] = [c.id for c in lang.countries]
lgcodes[lang.id] = lang.id
if lang.hid:
lgcodes[lang.hid] = lang.id
if lang.iso:
lgcodes[lang.iso] = lang.id
for gc in glottolog.glottocodes:
if gc not in data['Languoid'] and gc not in legacy:
common.Config.add_replacement(gc, None, model=common.Language)
for obj in jsonlib.load(glottolog.references_path('replacements.json')):
common.Config.add_replacement(
'{0}'.format(obj['id']),
'{0}'.format(obj['replacement']) if obj['replacement'] else None,
model=common.Source)
DBSession.flush()
for lid, maids in mas.items():
for ma in maids:
DBSession.add(models.Languoidmacroarea(
languoid_pk=data['Languoid'][lid].pk,
macroarea_pk=data['Macroarea'][ma].pk))
for lid, cids in countries.items():
for cid in cids:
DBSession.add(models.Languoidcountry(
languoid_pk=data['Languoid'][lid].pk,
country_pk=data['Country'][cid].pk))
for doctype in glottolog.hhtypes:
data.add(
models.Doctype, doctype.id, id=doctype.id,
name=doctype.name,
description=doctype.description,
abbr=doctype.abbv,
ord=doctype.rank)
for bib in glottolog.bibfiles:
data.add(
models.Provider,
bib.id,
id=bib.id,
name=bib.title,
description=bib.description,
abbr=bib.abbr,
url=bib.url)
DBSession.flush()
s = time()
for i, entry in enumerate(
BibFile(glottolog.build_path('monster-utf8.bib')).iterentries()):
if i % 10000 == 0:
args.log.info('{0}: {1:.3}'.format(i, time() - s))
s = time()
ref = load_ref(data, entry, lgcodes, lgsources)
if 'macro_area' in entry.fields:
for ma in split_text(entry.fields['macro_area'], separators=',;', strip=True):
ma = 'North America' if ma == 'Middle America' else ma
ma = Macroarea.get('Papunesia' if ma == 'Papua' else ma)
DBSession.add(models.Refmacroarea(
ref_pk=ref.pk, macroarea_pk=data['Macroarea'][ma.name].pk))
def prime(args):
"""If data needs to be denormalized for lookup, do that here.
This procedure should be separate from the db initialization, because
it will have to be run periodically whenever data has been updated.
"""
recreate_treeclosure()
for lpk, mas in DBSession.execute("""\
select
l.pk, array_agg(distinct lma.macroarea_pk)
from
language as l,
treeclosuretable as t,
languoidmacroarea as lma,
macroarea as ma
where
l.pk = t.parent_pk and
t.child_pk = lma.languoid_pk and
lma.macroarea_pk = ma.pk and
l.pk not in (select languoid_pk from languoidmacroarea)
group by l.pk"""):
for mapk in mas:
DBSession.add(models.Languoidmacroarea(languoid_pk=lpk, macroarea_pk=mapk))
for row in list(DBSession.execute(
"select pk, pages, pages_int, startpage_int from source where pages_int < 0"
)):
pk, pages, number, start = row
_start, _end, _number = compute_pages(pages)
if _number > 0 and _number != number:
DBSession.execute(
"update source set pages_int = %s, startpage_int = %s where pk = %s" %
(_number, _start, pk))
DBSession.execute(
"update ref set endpage_int = %s where pk = %s" %
(_end, pk))
version = assert_release(args.repos.repos)
with jsonlib.update(gc2version(args), indent=4) as legacy:
for lang in DBSession.query(common.Language):
if lang.id not in legacy:
lang.update_jsondata(new=True)
legacy[lang.id] = version
def items(s):
if not s:
return set()
r = []
for ss in set(s.strip().split()):
if '**:' in ss:
ss = ss.split('**:')[0] + '**'
if ss.endswith(','):
ss = ss[:-1].strip()
r.append(ss)
return set(r)
refs = {
r[0]: r[1]
for r in DBSession.query(models.Refprovider.id, models.Refprovider.ref_pk)}
valuesets = {
r[0]: r[1] for r in DBSession.query(common.ValueSet.id, common.ValueSet.pk)}
for lang in args.repos.languoids():
if lang.category == models.BOOKKEEPING:
continue
clf = lang.classification_comment
if clf:
if clf.subrefs:
if items(lang.cfg['classification']['subrefs']) != \
items(lang.cfg['classification'].get('sub')):
vspk = valuesets['sc-{0}'.format(lang.id)]
for ref in clf.subrefs:
spk = refs.get(ref.key)
DBSession.add(
common.ValueSetReference(source_pk=spk, valueset_pk=vspk))
if clf.familyrefs:
if items(lang.cfg['classification']['familyrefs']) != \
items(lang.cfg['classification'].get('family')):
vspk = valuesets['fc-{0}'.format(lang.id)]
for ref in clf.familyrefs:
spk = refs.get(ref.key)
if spk:
DBSession.add(
common.ValueSetReference(source_pk=spk, valueset_pk=vspk))