def main(args):
#get_obsolete_refs(args)
with transaction.manager:
#match_obsolete_refs(args)
#
# TODO:
# - create bibtex file containing all refs to be removed!
# -
#
matched = args.data_file(args.version, 'obsolete_refs_matched.json')
if matched.exists():
with open(matched) as fp:
matched = json.load(fp)
else:
matched = {}
for id_, repl in matched.items():
if not repl:
continue
ref = Ref.get(id_, default=None)
if ref is None:
continue
Config.add_replacement(ref, repl, session=DBSession, model=Source)
DBSession.delete(ref)
def upgrade():
conn = Connection(op.get_bind())
# https://github.com/clld/wals-data/issues/50
fpk = conn.pk(Family, 'utoaztecan')
gname = 'California Uto-Aztecan'
gid = slug(gname)
gpk = conn.insert(Genus, id=gid, name=gname, icon='fffff00', family_pk=fpk)
for oid in ['takic', 'tubatulabal']:
opk = conn.pk(Genus, oid)
conn.update(WalsLanguage, dict(genus_pk=gpk), genus_pk=opk)
conn.insert(Config, key=Config.replacement_key(Genus, oid), value=gid)
conn.delete(Genus, id=oid)
# https://github.com/clld/wals-data/issues/49
conn.update_name('aym', 'Aymara (Central)')
conn.update_glottocode('aym', 'cent2142')
conn.update_iso('aym', 'ayr', ayc='Southern Aymara')
# https://github.com/clld/wals-data/issues/48
# The genus Guaymi should be renamed Guaymiic.
conn.update(Genus, dict(name='Guaymiic'), id='guaymi')
# The genus Aruak should be renamed Arhuacic.
conn.update(Genus, dict(name='Arhuacic'), id='aruak')
# The language Motilón should be renamed Barí (while keeping Motilón as the name of
# the genus).
conn.update_name('mti', 'Barí')
# The genus Chibchan Proper should be split into two genera, Chibcha-Duit, containing
# the language Muisca, and Tunebo, containing the language Tunebo.
conn.update_genus('msc', ('chibchaduit', 'Chibcha-Duit', 'fffff00'),
family='chibchan')
conn.update_genus('tnb', ('tunebo', 'Tunebo', 'fffcc00'),
family='chibchan')
conn.insert(Config,
key=Config.replacement_key(Genus, 'chibchanproper'),
value=Config.gone)
conn.delete(Genus, id='chibchanproper')
# https://github.com/clld/wals-data/issues/44
conn.update_name('jlu', 'Luwo', other='Jur Luwo')
# https://github.com/clld/wals-data/issues/43
conn.update_genus('ctw', ('catawban', 'Catawban', 'fffcc00'),
family='siouan')
conn.update(Genus, dict(name='Core Siouan'), id='siouan')
# https://github.com/clld/wals-data/issues/40
conn.update_source('Sumbuk-2002', year='1999', name='Sumbuk 1999')
def main(args):
repls = set((i['id'], i['replacement']) for i in
jsonload(args.data_dir.joinpath('scripts', 'monster-replacements.json')))
with transaction.manager:
for ref_id, repl_id in repls:
ref = Source.get('%s' % ref_id, default=None)
if ref:
Config.add_replacement(
ref, '%s' % repl_id, session=DBSession, model=Source)
# FIXME: "redirect" relations, e.g. from valuesetreference as well!
DBSession.delete(ref)
args.log.info('%s replacements' % len(repls))
def upgrade():
conn = Connection(op.get_bind())
# https://github.com/clld/wals-data/issues/50
fpk = conn.pk(Family, 'utoaztecan')
gname = 'California Uto-Aztecan'
gid = slug(gname)
gpk = conn.insert(Genus, id=gid, name=gname, icon='fffff00', family_pk=fpk)
for oid in ['takic', 'tubatulabal']:
opk = conn.pk(Genus, oid)
conn.update(WalsLanguage, dict(genus_pk=gpk), genus_pk=opk)
conn.insert(Config, key=Config.replacement_key(Genus, oid), value=gid)
conn.delete(Genus, id=oid)
# https://github.com/clld/wals-data/issues/49
conn.update_name('aym', 'Aymara (Central)')
conn.update_glottocode('aym', 'cent2142')
conn.update_iso('aym', 'ayr', ayc='Southern Aymara')
# https://github.com/clld/wals-data/issues/48
# The genus Guaymi should be renamed Guaymiic.
conn.update(Genus, dict(name='Guaymiic'), id='guaymi')
# The genus Aruak should be renamed Arhuacic.
conn.update(Genus, dict(name='Arhuacic'), id='aruak')
# The language Motilón should be renamed Barí (while keeping Motilón as the name of
# the genus).
conn.update_name('mti', 'Barí')
# The genus Chibchan Proper should be split into two genera, Chibcha-Duit, containing
# the language Muisca, and Tunebo, containing the language Tunebo.
conn.update_genus(
'msc', ('chibchaduit', 'Chibcha-Duit', 'fffff00'), family='chibchan')
conn.update_genus(
'tnb', ('tunebo', 'Tunebo', 'fffcc00'), family='chibchan')
conn.insert(
Config, key=Config.replacement_key(Genus, 'chibchanproper'), value=Config.gone)
conn.delete(Genus, id='chibchanproper')
# https://github.com/clld/wals-data/issues/44
conn.update_name('jlu', 'Luwo', other='Jur Luwo')
# https://github.com/clld/wals-data/issues/43
conn.update_genus('ctw', ('catawban', 'Catawban', 'fffcc00'), family='siouan')
conn.update(Genus, dict(name='Core Siouan'), id='siouan')
# https://github.com/clld/wals-data/issues/40
conn.update_source('Sumbuk-2002', year='1999', name='Sumbuk 1999')
def main(args):
with open(args.data_file('2.3', 'obsolete_refs.json')) as fp:
obsolete = json.load(fp)
with transaction.manager:
provider = Provider.get('glottolog20121')
for ref in provider.refs:
if ref.id in obsolete:
Config.add_replacement(ref, None, session=DBSession, model=Source)
DBSession.delete(ref)
else:
assert len(ref.providers) > 1
DBSession.flush()
DBSession.delete(provider)
def upgrade():
conn = Connection(op.get_bind())
# The genus for Yanesha’ needs to be renamed Yanesha’.
conn.update(Genus, dict(name="Yanesha'"), id='westernarawakan')
# Bahuana
conn.update_name('bah', 'Xiriana')
conn.update_glottocode('bah', 'xiri1243')
conn.update_iso('bah', xir='Xiriâna')
coords = Coordinates('2d40N', '62d30W')
conn.update(Language,
dict(latitude=coords.latitude, longitude=coords.longitude),
id='bah')
spk = conn.execute('select max(pk) from source').fetchone()[0] + 1
lpk = conn.pk(Language, 'bah')
spk = conn.insert(
Source,
pk=spk,
id='Ramirez-1992',
name='Ramirez 1992',
description='Bahuana: une nouvelle langue de la famille Arawak',
bibtex_type=EntryType.book,
author='Ramirez, Henri',
year='1992',
title='Bahuana: une nouvelle langue de la famille Arawak',
address='Paris',
publisher='Amerindia')
conn.insert(LanguageSource, language_pk=lpk, source_pk=spk)
vspk = conn.pk(ValueSet, lpk, attr='language_pk')
conn.insert(ValueSetReference,
valueset_pk=vspk,
source_pk=spk,
description='35')
# split northern arawakan
GENERA = {
'Alto-Orinoco': 'bnw mpr'.split(),
'Caribbean Arawakan': 'ara grf goa'.split(),
'Inland Northern Arawakan': 'acg bae cur ppc res tar wrk ycn'.split(),
'Bahuanic': ['bah'],
'Wapishanan': ['wps'],
}
ICONS = ['cdd0000', 'cffcc00', 'cffff00', 'cff6600', 'cffffcc']
fpk = conn.pk(Family, 'arawakan')
for icon, (name, lids) in zip(ICONS, GENERA.items()):
gpk = conn.insert(Genus,
id=slug(name),
name=name,
icon=icon,
family_pk=fpk)
for lid in lids:
conn.update_genus(lid, gpk)
conn.insert(Config,
key=Config.replacement_key(Genus, 'northernarawakan'),
value=Config.gone)
conn.delete(Genus, id='northernarawakan')
def upgrade():
conn = Connection(op.get_bind())
for fname, genera in DATA.items():
fpk = conn.insert(Family, id=slug(fname), name=fname)
for gspec, lnames in genera.items():
if isinstance(gspec, tuple):
if len(gspec) == 3:
# new genus
gpk = conn.insert(Genus,
id=gspec[0],
name=gspec[1],
icon=gspec[2],
family_pk=fpk)
elif len(gspec) == 2:
# rename genus
gpk = conn.pk(Genus, gspec[0])
conn.update(Genus,
dict(family_pk=fpk, name=gspec[1]),
pk=gpk)
else:
raise ValueError()
else:
# just update the family
gpk = conn.pk(Genus, gspec)
conn.update(Genus, dict(family_pk=fpk), pk=gpk)
for lname in lnames:
lpk = conn.pk(Language, lname, attr='name')
conn.update(WalsLanguage, dict(genus_pk=gpk), pk=lpk)
for gid in GONE:
conn.insert(Config,
key=Config.replacement_key(Genus, gid),
value=Config.gone)
conn.delete(Genus, id=gid)
conn.insert(Config,
key=Config.replacement_key(Family, 'australian'),
value=Config.gone)
conn.delete(Family, id='australian')
conn.update_name('mdl', 'Matngele')
def upgrade():
conn = Connection(op.get_bind())
# The genus for Yanesha’ needs to be renamed Yanesha’.
conn.update(Genus, dict(name="Yanesha'"), id='westernarawakan')
# Bahuana
conn.update_name('bah', 'Xiriana')
conn.update_glottocode('bah', 'xiri1243')
conn.update_iso('bah', xir='Xiriâna')
coords = Coordinates('2d40N', '62d30W')
conn.update(
Language, dict(latitude=coords.latitude, longitude=coords.longitude), id='bah')
spk = conn.execute('select max(pk) from source').fetchone()[0] + 1
lpk = conn.pk(Language, 'bah')
spk = conn.insert(
Source,
pk=spk,
id='Ramirez-1992',
name='Ramirez 1992',
description='Bahuana: une nouvelle langue de la famille Arawak',
bibtex_type=EntryType.book,
author='Ramirez, Henri',
year='1992',
title='Bahuana: une nouvelle langue de la famille Arawak',
address='Paris',
publisher='Amerindia')
conn.insert(LanguageSource, language_pk=lpk, source_pk=spk)
vspk = conn.pk(ValueSet, lpk, attr='language_pk')
conn.insert(ValueSetReference, valueset_pk=vspk, source_pk=spk, description='35')
# split northern arawakan
GENERA = {
'Alto-Orinoco': 'bnw mpr'.split(),
'Caribbean Arawakan': 'ara grf goa'.split(),
'Inland Northern Arawakan': 'acg bae cur ppc res tar wrk ycn'.split(),
'Bahuanic': ['bah'],
'Wapishanan': ['wps'],
}
ICONS = ['cdd0000', 'cffcc00', 'cffff00', 'cff6600', 'cffffcc']
fpk = conn.pk(Family, 'arawakan')
for icon, (name, lids) in zip(ICONS, GENERA.items()):
gpk = conn.insert(Genus, id=slug(name), name=name, icon=icon, family_pk=fpk)
for lid in lids:
conn.update_genus(lid, gpk)
conn.insert(
Config,
key=Config.replacement_key(Genus, 'northernarawakan'),
value=Config.gone)
conn.delete(Genus, id='northernarawakan')
def upgrade():
conn = Connection(op.get_bind())
for fname, genera in DATA.items():
fpk = conn.insert(Family, id=slug(fname), name=fname)
for gspec, lnames in genera.items():
if isinstance(gspec, tuple):
if len(gspec) == 3:
# new genus
gpk = conn.insert(
Genus, id=gspec[0], name=gspec[1], icon=gspec[2], family_pk=fpk)
elif len(gspec) == 2:
# rename genus
gpk = conn.pk(Genus, gspec[0])
conn.update(Genus, dict(family_pk=fpk, name=gspec[1]), pk=gpk)
else:
raise ValueError()
else:
# just update the family
gpk = conn.pk(Genus, gspec)
conn.update(Genus, dict(family_pk=fpk), pk=gpk)
for lname in lnames:
lpk = conn.pk(Language, lname, attr='name')
conn.update(WalsLanguage, dict(genus_pk=gpk), pk=lpk)
for gid in GONE:
conn.insert(Config, key=Config.replacement_key(Genus, gid), value=Config.gone)
conn.delete(Genus, id=gid)
conn.insert(
Config, key=Config.replacement_key(Family, 'australian'), value=Config.gone)
conn.delete(Family, id='australian')
conn.update_name('mdl', 'Matngele')
def test_Config(self):
from clld.db.models.common import Config
self.assertEquals(Config.replacement_key('X', 'Y'), '__X_Y__')
self.assertEquals(Config.replacement_key(None, 'Y'), '__NoneType_Y__')
def test_Config_replacement_key():
from clld.db.models.common import Config
assert Config.replacement_key('X', 'Y') == '__X_Y__'
assert Config.replacement_key(None, 'Y') == '__NoneType_Y__'
def test_Config(self):
from clld.db.models.common import Config
self.assertEquals(Config.replacement_key('X', 'Y'), '__X_Y__')
self.assertEquals(Config.replacement_key(None, 'Y'), '__NoneType_Y__')
def upgrade():
conn = Connection(op.get_bind())
# Khoisan needs to be replaced by four families:
# Sandawe (no name change)
# Kxa: two genera: =|Hoan, Ju-Kung (a name change from 'Northern Khoisan')
# Tu (a name change from Southern Khoisan)
# Khoe-Kwadi (a name change from Central Khoisan)
genera = {
'hoan': ('Kxa', None),
'sandawe': ('Sandawe', None),
'centralkhoisan': ('Khoe-Kwadi', 'Khoe-Kwadi'),
'southernkhoisan': ('Tu', 'Tu'),
'northernkhoisan': ('Kxa', 'Ju-Kung'),
}
families = {}
for gid, (fname, gname) in genera.items():
fid = slug(fname)
fpk = families.get(fid)
if fpk is None:
families[fid] = fpk = conn.insert(Family, id=fid, name=fname)
values = dict(family_pk=fpk)
if gname:
values['name'] = gname
conn.update(Genus, values, id=gid)
conn.insert(Config,
key=Config.replacement_key(Family, 'khoisan'),
value=Config.gone)
conn.delete(Family, id='khoisan')
# Nilo-Saharan needs to be removed and replaced by the following families, with
# constituent genera.
# The following genera will now be families containing a single genus with the
# same name:
# Berta Kunama Songhay Gumuz Koman Shabo Kuliak Maban Fur
for gname in 'Berta Kunama Songhay Gumuz Koman Shabo Kuliak Maban Fur'.split(
):
gid = gname.lower()
fpk = conn.insert(Family, id=gid, name=gname)
gpk = conn.pk(Genus, gid)
conn.update(Genus, dict(family_pk=fpk), pk=gpk)
# The following is one family with two genera:
# Saharan: Eastern Saharan, Western Saharan
fpk = conn.insert(Family, id='saharan', name='Saharan')
for gname in ['Eastern Saharan', 'Western Saharan']:
conn.update(Genus,
dict(family_pk=fpk),
pk=conn.pk(Genus, gname, attr='name'))
# What is currently the subfamily Central Sudanic becomes a family with the same name
# and same constituent genera
# and what is currently the subfamily Eastern Sudanic becomes a family with the same
# name and the same constituent genera.
for sfn in ['Central Sudanic', 'Eastern Sudanic']:
fpk = conn.insert(Family, id=slug(sfn), name=sfn)
conn.update(Genus, dict(family_pk=fpk, subfamily=None), subfamily=sfn)
conn.insert(Config,
key=Config.replacement_key(Family, 'nilosaharan'),
value=Config.gone)
conn.delete(Family, id='nilosaharan')
# The following groups need to be removed from Niger-Congo and set up as separate
# families. The last two (Dogon and Ijoid) are families containing one genus.
# - Mande (currently a subfamily) with the following two genera:
# Eastern Mande, Western Mande
# - Kordofanian (currently a subfamily) with the following five genera:
# Katla-Tima, Rashad, Heiban, Talodi (a name change from Talodi Proper),
# and Lafofa (renamed from Tegem)
# - Dogon
# - Ijoid
for sfn in ['Mande', 'Kordofanian']:
fpk = conn.insert(Family, id=slug(sfn), name=sfn)
conn.update(Genus, dict(family_pk=fpk, subfamily=None), subfamily=sfn)
for old, new in {'Talodi Proper': 'Talodi', 'Tegem': 'Lafofa'}.items():
conn.update(Genus, dict(name=new), name=old)
for sfn in ['Dogon', 'Ijoid']:
fpk = conn.insert(Family, id=slug(sfn), name=sfn)
conn.update(Genus, dict(family_pk=fpk), name=sfn)
# A minor change in the name of the family Kadugli to just Kadu.
conn.update(Family, dict(name='Kadu'), name='Kadugli')
def upgrade():
conn = Connection(op.get_bind())
# Khoisan needs to be replaced by four families:
# Sandawe (no name change)
# Kxa: two genera: =|Hoan, Ju-Kung (a name change from 'Northern Khoisan')
# Tu (a name change from Southern Khoisan)
# Khoe-Kwadi (a name change from Central Khoisan)
genera = {
'hoan': ('Kxa', None),
'sandawe': ('Sandawe', None),
'centralkhoisan': ('Khoe-Kwadi', 'Khoe-Kwadi'),
'southernkhoisan': ('Tu', 'Tu'),
'northernkhoisan': ('Kxa', 'Ju-Kung'),
}
families = {}
for gid, (fname, gname) in genera.items():
fid = slug(fname)
fpk = families.get(fid)
if fpk is None:
families[fid] = fpk = conn.insert(Family, id=fid, name=fname)
values = dict(family_pk=fpk)
if gname:
values['name'] = gname
conn.update(Genus, values, id=gid)
conn.insert(Config, key=Config.replacement_key(Family, 'khoisan'), value=Config.gone)
conn.delete(Family, id='khoisan')
# Nilo-Saharan needs to be removed and replaced by the following families, with
# constituent genera.
# The following genera will now be families containing a single genus with the
# same name:
# Berta Kunama Songhay Gumuz Koman Shabo Kuliak Maban Fur
for gname in 'Berta Kunama Songhay Gumuz Koman Shabo Kuliak Maban Fur'.split():
gid = gname.lower()
fpk = conn.insert(Family, id=gid, name=gname)
gpk = conn.pk(Genus, gid)
conn.update(Genus, dict(family_pk=fpk), pk=gpk)
# The following is one family with two genera:
# Saharan: Eastern Saharan, Western Saharan
fpk = conn.insert(Family, id='saharan', name='Saharan')
for gname in ['Eastern Saharan', 'Western Saharan']:
conn.update(Genus, dict(family_pk=fpk), pk=conn.pk(Genus, gname, attr='name'))
# What is currently the subfamily Central Sudanic becomes a family with the same name
# and same constituent genera
# and what is currently the subfamily Eastern Sudanic becomes a family with the same
# name and the same constituent genera.
for sfn in ['Central Sudanic', 'Eastern Sudanic']:
fpk = conn.insert(Family, id=slug(sfn), name=sfn)
conn.update(Genus, dict(family_pk=fpk, subfamily=None), subfamily=sfn)
conn.insert(Config, key=Config.replacement_key(Family, 'nilosaharan'), value=Config.gone)
conn.delete(Family, id='nilosaharan')
# The following groups need to be removed from Niger-Congo and set up as separate
# families. The last two (Dogon and Ijoid) are families containing one genus.
# - Mande (currently a subfamily) with the following two genera:
# Eastern Mande, Western Mande
# - Kordofanian (currently a subfamily) with the following five genera:
# Katla-Tima, Rashad, Heiban, Talodi (a name change from Talodi Proper),
# and Lafofa (renamed from Tegem)
# - Dogon
# - Ijoid
for sfn in ['Mande', 'Kordofanian']:
fpk = conn.insert(Family, id=slug(sfn), name=sfn)
conn.update(Genus, dict(family_pk=fpk, subfamily=None), subfamily=sfn)
for old, new in {'Talodi Proper': 'Talodi', 'Tegem': 'Lafofa'}.items():
conn.update(Genus, dict(name=new), name=old)
for sfn in ['Dogon', 'Ijoid']:
fpk = conn.insert(Family, id=slug(sfn), name=sfn)
conn.update(Genus, dict(family_pk=fpk), name=sfn)
# A minor change in the name of the family Kadugli to just Kadu.
conn.update(Family, dict(name='Kadu'), name='Kadugli')
def test_Config_replacement_key():
from clld.db.models.common import Config
assert Config.replacement_key('X', 'Y') == '__X_Y__'
assert Config.replacement_key(None, 'Y') == '__NoneType_Y__'
