def issue24(session, timestamp): # pragma: no cover
#- Update language cea (name, coords, alternative names, iso code (and name))
#Change name of Cree (Eastern) to Cree (Swampy)
#Change coordinates to 56dN, 90dW
#Change the Ethnologue name to Cree (Swampy)
#Remove the Routledge and Other names
#Change the ISO code to csw. glottocode to swam1239
cea = common.Language.get('cea', session=session)
cre = common.Language.get('cre', session=session)
for i in range(len(cea.languageidentifier)):
try:
del cea.languageidentifier[i]
except IndexError:
pass
for values in [
('gc-csw', 'swam1239', 'Swampy Cree', 'glottolog'),
('csw', 'csw', 'Cree (Swampy)', 'iso639-3'),
('ethnologue-csw', 'Cree (Swampy)', 'ethnologue', 'name'),
]:
id = common.Identifier(
**dict(zip('id name description type'.split(), values)))
cea.languageidentifier.append(
common.LanguageIdentifier(language=cea, identifier=id))
cea.updated = timestamp
cea.name = 'Cree (Swampy)'
cea.ascii_name = slug('Cree (Swampy)')
cea.latitude = 56.0
cea.longitude = -90.0
for pid in ['81A', '82A', '83A']:
vsq = session.query(common.ValueSet)\
.join(common.Parameter)\
.filter(common.Parameter.id == pid)
vs1 = vsq.filter(common.ValueSet.language_pk == cea.pk).one()
vs2 = vsq.filter(common.ValueSet.language_pk == cre.pk).one()
vs2.updated = timestamp
vs1.updated = timestamp
for ref in vs1.references:
if ref.source.id == 'Hive-1948':
ref.valueset = vs2
session.flush()
#- Delete valueset 85A-cea
vs = session.query(common.ValueSet)\
.join(common.Parameter)\
.filter(common.Parameter.id == '85A')\
.filter(common.ValueSet.language_pk == cea.pk).one()
session.delete(vs.values[0])
session.delete(vs.references[0])
session.delete(vs)
#- delete valueset 131A-cea add 131A-cre
vs_switch_lang(session, timestamp, '131A-cea', 'cre')
def update_iso(session, timestamp, lang, *obsolete, **new): # pragma: no cover
if isinstance(lang, basestring):
lang = common.Language.get(lang, session=session)
lang.updated = timestamp
for code in obsolete:
for li in lang.languageidentifier:
if li.identifier.id == code or li.identifier.id == 'ethnologue-' + code:
session.delete(li)
for code, name in new.items():
iso = common.Identifier.get(code, session=session, default=None)
ethnologue = common.Identifier.get('ethnologue-' + code,
session=session,
default=None)
if not iso:
iso = common.Identifier(id=code,
name=code,
description=name,
type=common.IdentifierType.iso.value)
if not ethnologue:
ethnologue = common.Identifier(id='ethnologue-' + code,
name=name,
description='ethnologue',
type='name')
if iso.id not in [li.identifier.id for li in lang.languageidentifier]:
session.add(
common.LanguageIdentifier(language=lang, identifier=iso))
if ethnologue.id not in [
li.identifier.id for li in lang.languageidentifier
]:
session.add(
common.LanguageIdentifier(language=lang,
identifier=ethnologue))
return lang
def add_codes(lang):
for attr, prefix in dict(wals='wals_code_', iso='', glottolog='').items():
code = getattr(lang, 'code_' + attr)
if code:
if attr == 'iso' and not re.match('[a-z]{3}$', code):
continue
id_ = prefix + code
identifier = common.Identifier.get(id_, default=None)
if not identifier:
identifier = common.Identifier(id=id_,
name=code,
type=getattr(
common.IdentifierType,
attr).value)
common.LanguageIdentifier(identifier=identifier, language=lang)
def update_language(session,
timestamp,
lang,
keep_old_name=False,
**kw): # pragma: no cover
if isinstance(lang, basestring):
lang = common.Language.get(lang, session=session)
if 'name' in kw and keep_old_name:
name = common.Identifier(id=str(uuid1()),
name=lang.name,
description='other',
type='name')
session.add(common.LanguageIdentifier(language=lang, identifier=name))
return update_obj(session, timestamp, lang, **kw)
def add_new_name(args):
# TODO: Check the length of the args array is valid
gcode, lang, name, type, desc = \
args.args[0], args.args[1], args.args[2], args.args[3], args.args[4]
with_session(args)
with transaction.manager:
languoid = DBSession.query(common.Language) \
.filter_by(id='{0}'.format(gcode)) \
.first()
identifier = common.Identifier(
(name, type, desc, lang),
id='{0}-{1}-{2}-{3}'.format(slug(name), slug(type),
slug(desc or ''), lang),
name=name,
type=type,
description=desc,
lang=lang)
DBSession.add(identifier)
DBSession.add(
common.LanguageIdentifier(language=languoid,
identifier=identifier))
def update(args):
count = 0
assert args.json
iid = int(
DBSession.execute("select max(cast(id as integer)) from identifier").
fetchone()[0]) + 1
pk = DBSession.execute("select max(pk) from identifier").fetchone()[0] + 1
langs = {}
for gid, name in args.json['wikipedia'].items():
if gid not in langs:
langs[gid] = Languoid.get(gid)
langs[gid].update_jsondata(wikipedia=name.split('/')[-1])
for gid, codes in args.json['multitree'].items():
l = langs[gid]
lcodes = [i.name for i in l.identifiers if i.type == 'multitree']
for code in set(codes):
if code not in lcodes:
identifier = DBSession.query(common.Identifier)\
.filter(common.Identifier.type == 'multitree')\
.filter(common.Identifier.name == code)\
.first()
if not identifier:
identifier = common.Identifier(pk=pk,
id=str(iid),
name=code,
type='multitree')
iid += 1
pk += 1
count += 1
DBSession.add(
common.LanguageIdentifier(language=l,
identifier=identifier))
print count, 'new multitree identifiers'
def update_glottocode(session, timestamp, lang, gc): # pragma: no cover
if isinstance(lang, basestring):
lang = common.Language.get(lang, session=session)
lang.updated = timestamp
for li in lang.languageidentifier:
if li.identifier.name == gc and li.identifier.type == common.IdentifierType.glottolog.value:
session.delete(li)
glottocode = session.query(common.Identifier)\
.filter(common.Identifier.type == common.IdentifierType.glottolog.value)\
.filter(common.Identifier.name == gc)\
.first()
if not glottocode:
glottocode = common.Identifier(
id=gc,
name=gc,
description=gc,
type=common.IdentifierType.glottolog.value)
session.add(common.LanguageIdentifier(language=lang,
identifier=glottocode))
return lang
def setUp(self):
TestWithDb.setUp(self)
DBSession.add(
common.Dataset(id='dataset',
name='dataset',
description='desc',
domain='clld'))
source = common.Source(id='source')
contributors = {
'contributor': 'A Name',
'b': 'b Name',
'c': 'c Name',
'd': 'd Name'
}
for id_, name in contributors.items():
contributors[id_] = common.Contributor(id=id_, name=name)
contribution = common.Contribution(id='contribution',
name='Contribution')
cr = common.ContributionReference(contribution=contribution,
source=source)
assert common.ContributionContributor(
contribution=contribution,
primary=True,
contributor=contributors['contributor'])
assert common.ContributionContributor(contribution=contribution,
primary=False,
contributor=contributors['b'])
assert common.ContributionContributor(contribution=contribution,
primary=True,
contributor=contributors['c'])
assert common.ContributionContributor(contribution=contribution,
primary=False,
contributor=contributors['d'])
DBSession.add(contribution)
language = common.Language(id='language',
name='Language 1',
latitude=10.5,
longitude=0.3)
language.sources.append(source)
identifier = common.Identifier(type='iso639-3', id='iso')
li = common.LanguageIdentifier(language=language,
identifier=identifier)
for i in range(2, 102):
_l = common.Language(id='l%s' % i, name='Language %s' % i)
_i = common.Identifier(type='iso639-3',
id='%.3i' % i,
name='%.3i' % i)
_li = common.LanguageIdentifier(language=_l, identifier=_i)
DBSession.add(_l)
param = common.Parameter(id='parameter', name='Parameter')
de = common.DomainElement(id='de',
name='DomainElement',
parameter=param)
de2 = common.DomainElement(id='de2',
name='DomainElement2',
parameter=param)
valueset = common.ValueSet(id='valueset',
language=language,
parameter=param,
contribution=contribution)
value = common.Value(id='value',
domainelement=de,
valueset=valueset,
frequency=50,
confidence='high')
DBSession.add(value)
paramnd = common.Parameter(id='no-domain',
name='Parameter without domain')
valueset = common.ValueSet(id='vs2',
language=language,
parameter=paramnd,
contribution=contribution)
vr = common.ValueSetReference(valueset=valueset, source=source)
value = common.Value(id='v2',
valueset=valueset,
frequency=50,
confidence='high')
DBSession.add(value)
unit = common.Unit(id='unit', name='Unit', language=language)
up = common.UnitParameter(id='unitparameter', name='UnitParameter')
DBSession.add(unit)
DBSession.add(
common.UnitValue(id='unitvalue',
name='UnitValue',
unit=unit,
unitparameter=up))
up2 = common.UnitParameter(id='up2', name='UnitParameter with domain')
de = common.UnitDomainElement(id='de', name='de', parameter=up2)
DBSession.add(
common.UnitValue(id='uv2',
name='UnitValue2',
unit=unit,
unitparameter=up2,
unitdomainelement=de))
DBSession.add(common.Source(id='s'))
sentence = common.Sentence(id='sentence',
name='sentence name',
description='sentence description',
analyzed='a\tmorpheme\tdoes\tdo',
gloss='a\tmorpheme\t1SG\tdo.SG2',
source='own',
comment='comment',
original_script='a morpheme',
language=language)
sr = common.SentenceReference(sentence=sentence, source=source)
DBSession.add(common.Config(key='key', value='value'))
DBSession.flush()
def glottolog_url(glottocode):
return models.Identifier(name=glottocode, type='glottolog').url()
def populate_test_db(engine):
set_alembic_version(engine, '58559d4eea0d')
data = TestData()
data.add_default(common.Dataset,
domain='clld',
jsondata={
'license_icon': 'cc-by',
'license_url': 'http://example.org'
})
data.add_default(common.Contributor, name='A Name', email='*****@*****.**')
for id_, name in {
'b': 'b Name',
'c': 'c Name',
'd': 'd Name',
}.items():
data.add(common.Contributor,
id_,
id=id_,
name=name,
url='http://example.org')
DBSession.add(
common.Editor(dataset=data[common.Dataset],
contributor=data[common.Contributor]))
data.add_default(common.Source)
data.add(common.Source,
'replaced',
id='replaced',
active=False,
jsondata={'__replacement_id__': 'source'})
data.add_default(common.Contribution)
common.ContributionReference(contribution=data[common.Contribution],
source=data[common.Source])
for primary, c in [(True, 'contributor'), (False, 'b'), (True, 'c'),
(False, 'd')]:
common.ContributionContributor(contribution=data[common.Contribution],
primary=primary,
contributor=data['Contributor'][c])
data.add_default(common.Language, latitude=10.5, longitude=0.3)
data[common.Language].sources.append(data[common.Source])
for i, type_ in enumerate(common.IdentifierType):
common.LanguageIdentifier(
language=data[common.Language],
identifier=common.Identifier(
type=type_.value,
id=type_.value + str(i),
name='abc' if type_.name == 'iso' else 'glot1234'))
common.LanguageIdentifier(language=data[common.Language],
identifier=common.Identifier(type='name',
id='name',
name='a'))
for i in range(2, 102):
_l = common.Language(id='l%s' % i, name='Language %s' % i)
_i = common.Identifier(type='iso639-3', id='%.3i' % i, name='abc')
common.LanguageIdentifier(language=_l, identifier=_i)
DBSession.add(_l)
param = data.add_default(common.Parameter)
de = common.DomainElement(id='de', name='DomainElement', parameter=param)
de2 = common.DomainElement(id='de2',
name='DomainElement2',
parameter=param)
valueset = data.add_default(common.ValueSet,
language=data[common.Language],
parameter=param,
contribution=data[common.Contribution])
common.ValueSetReference(valueset=valueset,
source=data[common.Source],
description='10-20')
data.add_default(common.Value,
domainelement=de,
valueset=valueset,
frequency=50,
confidence='high')
data.add(common.Value,
'value2',
id='value2',
domainelement=de2,
valueset=valueset,
frequency=50,
confidence='high')
paramnd = data.add(common.Parameter,
'no-domain',
id='no-domain',
name='Parameter without domain')
valueset = common.ValueSet(id='vs2',
language=data[common.Language],
parameter=paramnd,
contribution=data[common.Contribution])
common.ValueSetReference(valueset=valueset,
source=data[common.Source],
description='10-20')
common.Value(id='v2', valueset=valueset, frequency=50, confidence='high')
unit = data.add_default(common.Unit, language=data[common.Language])
up = data.add_default(common.UnitParameter)
common.UnitValue(id='unitvalue',
name='UnitValue',
unit=unit,
unitparameter=up)
up2 = common.UnitParameter(id='up2', name='UnitParameter with domain')
de = common.UnitDomainElement(id='de', name='de', parameter=up2)
DBSession.add(
common.UnitValue(id='uv2',
name='UnitValue2',
unit=unit,
unitparameter=up2,
unitdomainelement=de))
DBSession.add(common.Source(id='s'))
sentence = data.add_default(common.Sentence,
description='sentence description',
analyzed='a\tmorpheme\tdoes\tdo',
gloss='a\tmorpheme\t1SG\tdo.SG2',
source='own',
comment='comment',
original_script='a morpheme',
language=data[common.Language],
jsondata={'alt_translation': 'Spanish: ...'})
common.SentenceReference(sentence=sentence, source=data[common.Source])
DBSession.add(common.Config(key='key', value='value'))
common.Config.add_replacement('replaced',
'language',
model=common.Language)
common.Config.add_replacement('gone', None, model=common.Language)
DBSession.flush()
def setUp(self):
TestWithDb.setUp(self)
DBSession.add(
common.Dataset(id='dataset',
name='dataset',
description='desc',
domain='clld',
jsondata={'license_icon': 'cc-by'}))
DBSession.add(
common.Source(id='replaced',
active=False,
jsondata={'__replacement_id__': 'source'}))
source = common.Source(id='source')
contributors = {
'contributor': 'A Name',
'b': 'b Name',
'c': 'c Name',
'd': 'd Name'
}
for id_, name in contributors.items():
contributors[id_] = common.Contributor(id=id_,
name=name,
url='http://example.org')
contribution = common.Contribution(id='contribution',
name='Contribution')
common.ContributionReference(contribution=contribution, source=source)
assert common.ContributionContributor(
contribution=contribution,
primary=True,
contributor=contributors['contributor'])
assert common.ContributionContributor(contribution=contribution,
primary=False,
contributor=contributors['b'])
assert common.ContributionContributor(contribution=contribution,
primary=True,
contributor=contributors['c'])
assert common.ContributionContributor(contribution=contribution,
primary=False,
contributor=contributors['d'])
DBSession.add(contribution)
language = common.Language(id='language',
name='Language 1',
latitude=10.5,
longitude=0.3)
language.sources.append(source)
for i, type_ in enumerate(common.IdentifierType):
id_ = common.Identifier(type=type_.value,
id=type_.value + str(i),
name='abc')
common.LanguageIdentifier(language=language, identifier=id_)
for i in range(2, 102):
_l = common.Language(id='l%s' % i, name='Language %s' % i)
_i = common.Identifier(type='iso639-3',
id='%.3i' % i,
name='%.3i' % i)
common.LanguageIdentifier(language=_l, identifier=_i)
DBSession.add(_l)
param = common.Parameter(id='parameter', name='Parameter')
de = common.DomainElement(id='de',
name='DomainElement',
parameter=param)
de2 = common.DomainElement(id='de2',
name='DomainElement2',
parameter=param)
valueset = common.ValueSet(id='valueset',
language=language,
parameter=param,
contribution=contribution)
value = common.Value(id='value',
domainelement=de,
valueset=valueset,
frequency=50,
confidence='high')
DBSession.add(value)
value2 = common.Value(id='value2',
domainelement=de2,
valueset=valueset,
frequency=50,
confidence='high')
DBSession.add(value2)
paramnd = common.Parameter(id='no-domain',
name='Parameter without domain')
valueset = common.ValueSet(id='vs2',
language=language,
parameter=paramnd,
contribution=contribution)
common.ValueSetReference(valueset=valueset, source=source)
value = common.Value(id='v2',
valueset=valueset,
frequency=50,
confidence='high')
DBSession.add(value)
unit = common.Unit(id='unit', name='Unit', language=language)
up = common.UnitParameter(id='unitparameter', name='UnitParameter')
DBSession.add(unit)
DBSession.add(
common.UnitValue(id='unitvalue',
name='UnitValue',
unit=unit,
unitparameter=up))
up2 = common.UnitParameter(id='up2', name='UnitParameter with domain')
de = common.UnitDomainElement(id='de', name='de', parameter=up2)
DBSession.add(
common.UnitValue(id='uv2',
name='UnitValue2',
unit=unit,
unitparameter=up2,
unitdomainelement=de))
DBSession.add(common.Source(id='s'))
sentence = common.Sentence(
id='sentence',
name='sentence name',
description='sentence description',
analyzed='a\tmorpheme\tdoes\tdo',
gloss='a\tmorpheme\t1SG\tdo.SG2',
source='own',
comment='comment',
original_script='a morpheme',
language=language,
jsondata={'alt_translation': 'Spanish: ...'})
common.SentenceReference(sentence=sentence, source=source)
DBSession.add(common.Config(key='key', value='value'))
common.Config.add_replacement('replaced',
'language',
model=common.Language)
common.Config.add_replacement('gone', None, model=common.Language)
DBSession.flush()
def main(args): # pragma: no cover
get_repos()
api = Grambank(REPOS['Grambank'])
cldf = args.cldf
data = Data()
dataset = models.Grambank(
id=grambank.__name__,
name="Grambank",
description="Grambank",
publisher_name="Max Planck Institute for Evolutionary Anthropology",
publisher_place="Leipzig",
publisher_url="https://www.eva.mpg.de",
license="http://creativecommons.org/licenses/by/4.0/",
domain='grambank.clld.org',
contact='*****@*****.**',
jsondata={
'license_icon':
'cc-by.png',
'license_name':
'Creative Commons Attribution 4.0 International License'
})
contributors = {}
for i, contrib in enumerate(api.contributors):
contrib = common.Contributor(
contrib.id,
id=contrib.id,
name=contrib.name,
)
common.Editor(dataset=dataset, contributor=contrib, ord=i)
DBSession.add(contrib)
DBSession.flush()
contributors[contrib.id] = contrib.pk
contributions = {r['ID']: r for r in cldf['LanguageTable']}
DBSession.add(dataset)
for rec in tqdm(list(Database.from_file(cldf.bibpath, lowercase=True)),
desc='sources'):
data.add(common.Source, rec.id, _obj=bibtex2source(rec))
DBSession.flush()
sources = {k: v.pk for k, v in data['Source'].items()}
features, codes = import_features(cldf, contributors)
transaction.commit()
values_by_sheet = [(lid, list(v)) for lid, v in itertools.groupby(
sorted(cldf['ValueTable'], key=lambda r: r['Language_ID']),
lambda r: r['Language_ID'],
)]
for lid, values in tqdm(values_by_sheet, desc='loading values'):
transaction.begin()
import_values(values, contributions[lid], features, codes,
contributors, sources)
transaction.commit()
transaction.begin()
glottolog = Glottolog(REPOS['glottolog'])
languoids = {l.id: l for l in glottolog.languoids()}
gblangs = DBSession.query(models.GrambankLanguage).all()
load_families(data,
gblangs,
glottolog_repos=REPOS['glottolog'],
isolates_icon='dcccccc')
# Add isolates
for lg in gblangs:
gl_language = languoids.get(lg.id)
if not gl_language.family:
family = data.add(
Family,
gl_language.id,
id=gl_language.id,
name=gl_language.name,
description=common.Identifier(
name=gl_language.id,
type=common.IdentifierType.glottolog.value).url(),
jsondata={"icon": 'tcccccc'})
lg.family = family
coverage.main(glottolog)
return
def prime_cache(args):
"""If data needs to be denormalized for lookup, do that here.
This procedure should be separate from the db initialization, because
it will have to be run periodically whenever data has been updated.
"""
print('Parsing markdown intros...')
for contrib in DBSession.query(models.Contribution):
if contrib.description:
contrib.markup_description = markdown(contrib.description)
else:
contrib.markup_description = None
print('... done')
print('Retrieving language data from glottolog...')
catconf = cldfcatalog.Config.from_file()
glottolog_path = catconf.get_clone('glottolog')
glottolog = Glottolog(glottolog_path)
lang_ids = [lang.id for lang in DBSession.query(common.Language)]
languoids = {l.id: l for l in glottolog.languoids(lang_ids)}
glottocodes = [(l.id,
common.Identifier(id=l.id, name=l.id, type='glottolog'))
for l in languoids.values()]
glottocodes = OrderedDict(sorted(glottocodes, key=lambda t: t[0]))
isocodes = [(l.iso, common.Identifier(id=l.iso,
name=l.iso,
type='iso639-3'))
for l in languoids.values() if l.iso]
isocodes = OrderedDict(sorted(isocodes, key=lambda t: t[0]))
DBSession.add_all(glottocodes.values())
DBSession.add_all(isocodes.values())
DBSession.flush()
for lang in DBSession.query(common.Language):
if lang.id not in languoids:
continue
languoid = languoids[lang.id]
lang.name = languoid.name
lang.latitude = languoid.latitude
lang.longitude = languoid.longitude
lang.macroarea = languoid.macroareas[
0].name if languoid.macroareas else ''
DBSession.add(
common.LanguageIdentifier(
language=lang, identifier_pk=glottocodes[languoid.id].pk))
if languoid.iso in isocodes:
DBSession.add(
common.LanguageIdentifier(
language=lang, identifier_pk=isocodes[languoid.iso].pk))
DBSession.flush()
print('... done')
print('Making pretty colourful dots for parameter values...')
all_icons = [icon.name for icon in ORDERED_ICONS]
code_query = DBSession.query(common.DomainElement)\
.order_by(common.DomainElement.parameter_pk, common.DomainElement.id)
for _, param_codes in groupby(code_query, lambda c: c.parameter_pk):
icons = cycle(all_icons)
for code in param_codes:
code.update_jsondata(icon=next(icons))
DBSession.flush()
print('... done')