def testapp():
from webtest import TestApp
from clld.db.meta import DBSession, VersionedDBSession, Base
from clld.db.models import common
from clld_cognacy_plugin.models import Cognateset, Cognate
def main():
cfg = config.Configurator(settings={
'sqlalchemy.url': 'sqlite://',
'mako.directories': [
'clld:web/templates',
'clld_cognacy_plugin:templates'
]})
cfg.include('clld.web.app')
cfg.include('clld_cognacy_plugin')
return cfg.make_wsgi_app()
DBSession.remove()
VersionedDBSession.remove()
wsgi_app = main()
Base.metadata.bind = DBSession.bind
Base.metadata.create_all()
DBSession.add(common.Dataset(id='1', name='test app', domain='example.org'))
cs = Cognateset(id='1', name='cs: test')
lang = common.Language(id='l', latitude=2, longitude=2)
param = common.Parameter(id='l')
vs = common.ValueSet(id='vs', language=lang, parameter=param)
v = common.Value(id='v', name='abc', valueset=vs)
DBSession.add(Cognate(cognateset=cs, counterpart=v))
yield TestApp(wsgi_app)
def add_values(data, dblang, pid, values, with_de=True, **vskw):
vs = None
for i, (vid, vname) in enumerate(values):
if i == 0:
vs = common.ValueSet(
id=idjoin(pid, dblang.id),
language=dblang,
parameter=data['Parameter'][pid],
contribution=data['Contribution']['glottolog'],
**vskw)
vkw = dict(id=idjoin(pid, slug(vid), dblang.id),
name=vname,
valueset=vs)
if with_de:
vkw['domainelement'] = data['DomainElement'][pid, vid]
DBSession.add(common.Value(**vkw))
def vs_copy_lang(session, timestamp, vs, lang): # pragma: no cover
if isinstance(lang, basestring):
lang = common.Language.get(lang, session=session)
vs1 = get_vs(session, vs)
pid, lid = vs1.id.split('-')
id_ = '-'.join([pid, lang.id])
try:
vs2 = get_vs(session, id_)
vs2.updated = timestamp
raise AssertionError
except NoResultFound:
vs2 = common.ValueSet(id=id_,
description=vs1.description,
language=lang,
parameter=vs1.parameter,
contribution=vs1.contribution,
updated=timestamp,
created=timestamp,
source=vs1.source)
session.add(vs2)
# copy values and references:
session.add(
common.Value(id=vs2.id,
valueset=vs2,
domainelement=vs1.values[0].domainelement,
created=timestamp,
updated=timestamp))
for ref in vs1.references:
session.add(
common.ValueSetReference(valueset=vs2,
source=ref.source,
key=ref.key,
description=ref.description))
def vs_switch_lang(session, timestamp, vs, lang): # pragma: no cover
if isinstance(lang, basestring):
lang = common.Language.get(lang, session=session)
vs1 = get_vs(session, vs)
pid, lid = vs1.id.split('-')
id_ = '-'.join([pid, lang.id])
try:
vs2 = get_vs(session, id_)
vs2.updated = timestamp
except NoResultFound:
vs2 = common.ValueSet(id=id_,
description=vs1.description,
language=lang,
parameter=vs1.parameter,
contribution=vs1.contribution,
updated=timestamp,
created=timestamp,
source=vs1.source)
session.add(vs2)
v1 = vs1.values[0]
if vs2.values:
assert v1.domainelement == vs2.values[0].domainelement
else:
session.add(
common.Value(id=vs2.id,
valueset=vs2,
domainelement=v1.domainelement,
created=timestamp,
updated=timestamp))
delete(session, v1)
for ref in vs1.references:
ref.valueset = vs2
delete(session, vs1)
def add_cultural_data(questionnaire_file_name, parameters, language):
""" Parse the typological questionnaire into the database """
contribution_text, parameter_descriptions, answers = parse_culture_questionnaire(
os.path.join(DBPATH, questionnaire_file_name))
# All ValueSets must be related to a contribution, so generate one from the metadata.
contrib = common.Contribution(id='contrib' + newid(),
name=contribution_text + newid())
for p, parameter in parameter_descriptions.iterrows():
# First, make sure that this parameter exists – either look it up or create it.
pid = p.replace(".", "-")
try:
param, domain = parameters[pid]
except KeyError:
param = common.Parameter(
id='culture' + pid,
name=p,
description=parameter['Question_text_English'],
markup_description=parameter['Question_text_English'])
domain = {}
parameters[pid] = (param, domain)
# Secondly, check whether we are aware that this answer is
# valid already – otherwise we add its value to the domain,
# and use that.
# Note: Once we have a database, we can do better filtering
# and constraining, and don't need to rely on reasonable data.
answer = str(answers["Answer"][p])
try:
domain_element = domain[slug(answer)]
except KeyError:
try:
numerical_value = int(answer)
except ValueError:
numerical_value = (
1 if answer == "Y" or answer == 'True' else
0 if answer == "N" or answer == 'False' else None)
domain_element = common.DomainElement(
id=param.id + slug(answer),
description=answer,
number=numerical_value,
name=answer,
parameter=param,
abbr=answer,
jsondata={'color': color(numerical_value)})
DBSession.add(domain_element)
try:
DBSession.flush()
except:
print(domain, domain_element, language.name, pid, param.name)
domain[slug(answer)] = domain_element
# Now create the ValueSet, representing all values the
# language has for this parameter
vs = common.ValueSet(id='vs' + newid(),
language=language,
parameter=param,
jsondata=domain_element.jsondata,
contribution=contrib)
# and fill in the actual values, which in this case is only
# one. This object, and all objects it depends on, are then
# scheduled for writing into the database.
DBSession.add(
common.Value(id='v' + newid(),
valueset=vs,
frequency=float(100),
jsondata=domain_element.jsondata,
domainelement=domain_element))
# Execute all scheduled database updates.
DBSession.flush()
def main(args):
data = Data()
lotw_conn = sqlite3.connect("lotw_base.sqlite")
lotw_base = lotw_conn.cursor()
contrib = common.Contribution(id="initial_contrib",
name="Initial contribution")
dataset = common.Dataset(id=lotw_dev.__name__,
domain='lotw_dev.clld.org',
name="Languages of the World",
publisher_name="IL RAS",
publisher_place="Moscow",
publisher_url="http://iling-ran.ru/main/",
jsondata={
'license_name': 'Creative Commons Attribution 4.0 International License'}
)
DBSession.add(dataset)
feature_dict = {}
unnamed_feature_count = 0
features = lotw_base.execute("SELECT * FROM Feature").fetchall()
names = [y[2] for y in features]
feat_name_counts = {x[2]: [names.count(x[2]), 0] for x in features if names.count(x[2]) > 1}
# features = [convert_feature(x) for x in features]
for feature in features:
name = feature[2]
# if name == ".О":
# continue
if name in feat_name_counts.keys():
temp_name = name
name += ("_" + str(feat_name_counts[name][1]))
feat_name_counts[temp_name][1] += 1
feature_dict[feature[0]] = TreeFeature(pk=feature[0],
id=feature[0],
name=name,
father_pk=feature[5])
print("Added feature %s" % feature[2])
langs = lotw_base.execute("SELECT * FROM Language").fetchall()
assert len(set([lang[0] for lang in langs])) == len([lang[0] for lang in langs])
for language in langs:
value_sets = []
geodata = lotw_base.execute("SELECT * FROM Geographical_index WHERE Lang_id=?", (str(language[0]), )).fetchone()
famdata = lotw_base.execute("SELECT * FROM Genealogical_index WHERE Lang_id=?", (str(language[0]), )).fetchone()
famname = lotw_base.execute("SELECT * FROM Family where Id=?", (famdata[2], )).fetchone()[1]
branchname =lotw_base.execute("SELECT * FROM Branch where Id=?", (famdata[3], )).fetchone()[1]
if not geodata:
geodata = [0.0 for x in range(7)]
data.add(lotw_devLanguage, language[0],
id=str(language[0]),
iso=language[3],
family=famname,
branch=branchname,
name=language[1],
latitude=geodata[5],
longitude=geodata[6])
print("Added language %s" % language[3])
# Lang_id=language["Lang_id"], Order_of_addition=language["Order_of_addition"],
# Sorting_number=language["Sorting_number"], Code_ISO_639_3=language["Code_ISO_639_3"]
language_features = lotw_base.execute("SELECT * FROM Binary_data WHERE Lang_id=? AND Feature_value=1", (str(language[0]), ))
for l_feat in language_features.fetchall():
feat_id = l_feat[0]
try:
feat_name = feature_dict[l_feat[2]].name
except KeyError:
continue
vs = common.ValueSet(id=feat_id,
language=data["lotw_devLanguage"][language[0]],
parameter=feature_dict[l_feat[2]],
contribution=contrib)
DBSession.add(common.Value(id=feat_id,
name=feat_name,
valueset=vs))
print("Added value %s" % feat_id)
lotw_conn.close()
def prime(args):
"""If data needs to be denormalized for lookup, do that here.
This procedure should be separate from the db initialization, because
it will have to be run periodically whenever data has been updated.
"""
#
# Now that we loaded all languoids and refs, we can compute the MED values.
#
meds = defaultdict(list)
for lpk, spk, sid, sname, med_type, year, pages in DBSession.execute("""\
select
l.pk, r.pk, s.id, s.name, r.med_type, s.year_int, r.med_pages
from
languagesource as ls,
language as l,
source as s,
ref as r
where
ls.active = TRUE and l.pk = ls.language_pk and s.pk = ls.source_pk and s.pk = r.pk
order by
l.id, r.med_index desc, r.med_pages, coalesce(s.year_int, 0), s.pk
"""):
meds[lpk].append((spk, sid, sname, med_type, year, pages)) # The last one is the overall MED
# Now weed out the "newer but worse" sources:
for lpk, sources in {k: reversed(v) for k, v in meds.items()}.items():
relevant, lastyear = [], 10000
for spk, sid, sname, med_type, year, pages in sources:
if year and year < lastyear: # If year is more recent, this is a "newer but worse" item
relevant.append((spk, sid, sname, med_type, year, pages))
lastyear = year
meds[lpk] = relevant
med_param = common.Parameter.get('med')
med_domain = {de.id: de for de in med_param.domain}
contrib = common.Contribution.get('glottolog')
for l in DBSession.query(common.Language).filter(common.Language.pk.in_(list(meds.keys()))):
l.update_jsondata(meds=[
(sid, med_type, year, pages, sname) for spk, sid, sname, med_type, year, pages in meds[l.pk]])
if not meds[l.pk]:
continue
med = meds[l.pk][0]
# Record the overall MED as value for the 'med' Parameter:
vs = common.ValueSet(
id=idjoin('med', l.id),
contribution=contrib,
parameter=med_param,
language=l,
)
DBSession.add(common.Value(
id=idjoin('med', l.id),
name=getattr(args.repos.med_types, med[3]).name,
domainelement=med_domain[idjoin('med', med[3])],
valueset=vs,
))
DBSession.flush()
DBSession.add(common.ValueSetReference(source_pk=med[0], valueset_pk=vs.pk))
recreate_treeclosure()
macroareas = {r[0]: (r[1], r[2]) for r in DBSession.execute("""\
select de.pk, de.id, de.name
from domainelement as de, parameter as p
where de.parameter_pk = p.pk and p.id = 'macroarea'
""")}
for lid, lpk, cpk, ppk, mas in DBSession.execute("""\
select
l.id, l.pk, vs.contribution_pk, vs.parameter_pk, array_agg(distinct v.domainelement_pk)
from
language as l,
treeclosuretable as t,
parameter as p,
valueset as vs,
value as v
where
l.pk = t.parent_pk and
t.child_pk = vs.language_pk and
vs.parameter_pk = p.pk and
p.id = 'macroarea' and
v.valueset_pk = vs.pk and
l.pk not in (
select language_pk
from valueset as _vs, parameter as _p
where _vs.parameter_pk = _p.pk and _p.id = 'macroarea'
)
group by l.id, l.pk, vs.contribution_pk, vs.parameter_pk"""):
for i, mapk in enumerate(mas):
if i == 0:
vs = common.ValueSet(
id=idjoin('macroarea', lid),
language_pk=lpk,
parameter_pk=ppk,
contribution_pk=cpk)
DBSession.add(common.Value(
id=idjoin(macroareas[mapk][0], lid),
name=macroareas[mapk][1],
domainelement_pk=mapk,
valueset=vs))
for vs in DBSession.query(common.ValueSet)\
.join(common.Language)\
.join(common.Parameter)\
.filter(common.Parameter.id == 'macroarea')\
.options(joinedload(common.ValueSet.values), joinedload(common.ValueSet.language)):
vs.language.macroareas = ', '.join([macroareas[v.domainelement_pk][1] for v in vs.values])
for row in list(DBSession.execute(
"select pk, pages, pages_int, startpage_int from source where pages_int < 0"
)):
raise ValueError(row)
version = assert_release(args.repos.repos)
with jsonlib.update(gc2version(args), indent=4) as legacy:
for lang in DBSession.query(common.Language):
if lang.id not in legacy:
lang.update_jsondata(new=True)
legacy[lang.id] = version
valuesets = {
r[0]: r[1] for r in DBSession.query(common.ValueSet.id, common.ValueSet.pk)}
refs = {
r[0]: r[1]
for r in DBSession.query(models.Refprovider.id, models.Refprovider.ref_pk)}
for vsid, vspk in valuesets.items():
if vsid.startswith('macroarea-'):
DBSession.add(common.ValueSetReference(
source_pk=refs[args.repos.macroareas.__defaults__['reference_id']],
valueset_pk=vspk))
for vs in DBSession.query(common.ValueSet)\
.join(common.Parameter)\
.filter(common.Parameter.id == 'aes'):
if vs.jsondata['reference_id']:
DBSession.add(common.ValueSetReference(
source_pk=refs[vs.jsondata['reference_id']], valueset_pk=vs.pk))
for lang in args.repos.languoids():
if lang.category == args.repos.language_types.bookkeeping.category:
continue
clf = lang.classification_comment
if clf:
for pid, attr_ in [('sc', 'sub'), ('fc', 'family')]:
if getattr(clf, attr_ + 'refs'):
if split_items(lang.cfg['classification'][attr_ + 'refs']) != \
split_items(lang.cfg['classification'].get(attr_)):
vspk = valuesets['{0}-{1}'.format(pid, lang.id)]
for ref in getattr(clf, attr_ + 'refs'):
spk = refs.get(ref.key)
if spk:
DBSession.add(
common.ValueSetReference(source_pk=spk, valueset_pk=vspk))
def main(args):
data = Data()
data_path = lambda *cs: args.data_file('concepticon-data',
'concepticondata', *cs)
dataset = common.Dataset(
id=concepticon.__name__,
name="Concepticon 1.0",
publisher_name="Max Planck Institute for the Science of Human History",
publisher_place="Jena",
publisher_url="http://www.shh.mpg.de",
license="http://creativecommons.org/licenses/by/4.0/",
contact='*****@*****.**',
domain='concepticon.clld.org',
jsondata={
'license_icon':
'cc-by.png',
'license_name':
'Creative Commons Attribution 4.0 International License'
})
DBSession.add(dataset)
for i, name in enumerate(
['Johann-Mattis List', 'Michael Cysouw', 'Robert Forkel']):
c = common.Contributor(id=slug(name), name=name)
dataset.editors.append(common.Editor(contributor=c, ord=i))
english = data.add(common.Language, 'eng', id='eng', name='English')
files = {}
for fname in data_path('sources').iterdir():
files[fname.stem] = \
"https://github.com/clld/concepticon-data/blob/master/concepticondata/sources/%s" % fname.name
for rec in Database.from_file(data_path('references', 'references.bib'),
lowercase=True):
source = data.add(common.Source, rec.id, _obj=bibtex2source(rec))
if rec.id in files:
DBSession.flush()
DBSession.add(
common.Source_files(mime_type='application/pdf',
object_pk=source.pk,
jsondata=dict(url=files[rec.id])))
for concept in reader(data_path('concepticon.tsv'), namedtuples=True):
data.add(models.ConceptSet,
concept.ID,
id=concept.ID,
name=concept.GLOSS,
description=concept.DEFINITION,
semanticfield=concept.SEMANTICFIELD,
ontological_category=concept.ONTOLOGICAL_CATEGORY)
for rel in reader(data_path('conceptrelations.tsv'), namedtuples=True):
DBSession.add(
models.Relation(source=data['ConceptSet'][rel.SOURCE],
target=data['ConceptSet'][rel.TARGET],
description=rel.RELATION))
unmapped = Counter()
number_pattern = re.compile('(?P<number>[0-9]+)(?P<suffix>.*)')
for cl in reader(data_path('conceptlists.tsv'), dicts=True):
concepts = data_path('conceptlists', '%(ID)s.tsv' % cl)
if not concepts.exists():
continue
langs = [l.lower() for l in split(cl['SOURCE_LANGUAGE'])]
conceptlist = data.add(
models.Conceptlist,
cl['ID'],
id=cl['ID'],
name=' '.join(cl['ID'].split('-')),
description=cl['NOTE'],
target_languages=cl['TARGET_LANGUAGE'],
source_languages=' '.join(langs),
year=int(cl['YEAR']) if cl['YEAR'] else None,
)
for id_ in split(cl['REFS']):
common.ContributionReference(source=data['Source'][id_],
contribution=conceptlist)
for i, name in enumerate(split(cl['AUTHOR'], sep=' and ')):
name = strip_braces(name)
contrib = data['Contributor'].get(name)
if not contrib:
contrib = data.add(common.Contributor,
name,
id=slug(name),
name=name)
DBSession.add(
common.ContributionContributor(ord=i,
contribution=conceptlist,
contributor=contrib))
for k in 'ID NOTE TARGET_LANGUAGE SOURCE_LANGUAGE YEAR REFS AUTHOR'.split(
):
del cl[k]
DBSession.flush()
for k, v in cl.items():
DBSession.add(
common.Contribution_data(object_pk=conceptlist.pk,
key=k,
value=v))
for concept in reader(concepts, namedtuples=True):
if not concept.ID or not concept.CONCEPTICON_ID or concept.CONCEPTICON_ID == 'NAN':
#print conceptlist.id, getattr(concept, 'ENGLISH', getattr(concept, 'GLOSS', None))
unmapped.update([conceptlist.id])
continue
lgs = {}
for lang in langs:
v = getattr(concept, lang.upper())
if v:
lgs[lang] = v
match = number_pattern.match(concept.NUMBER)
if not match:
print(concept.ID)
raise ValueError
vs = common.ValueSet(
id=concept.ID,
description=getattr(concept, 'GLOSS',
getattr(concept, 'ENGLISH', None)),
language=english,
contribution=conceptlist,
parameter=data['ConceptSet'][concept.CONCEPTICON_ID])
d = {}
for key, value in concept.__dict__.items():
if not key.startswith('CONCEPTICON_') and \
key not in ['NUMBER', 'ID', 'GLOSS'] + [l.upper() for l in langs]:
d[key.lower()] = value
v = models.Concept(
id=concept.ID,
valueset=vs,
description=getattr(concept, 'GLOSS',
None), # our own gloss, if available
name='; '.join('%s [%s]' % (lgs[l], l)
for l in sorted(lgs.keys())),
number=int(match.group('number')),
number_suffix=match.group('suffix'),
jsondata=d)
DBSession.flush()
for key, value in lgs.items():
DBSession.add(
common.Value_data(key='lang_' + key,
value=value,
object_pk=v.pk))
print('Unmapped concepts:')
for clid, no in unmapped.most_common():
print(clid, no)
for fname in data_path('concept_set_meta').iterdir():
if fname.suffix == '.tsv':
md = load(fname.parent.joinpath(fname.name + '-metadata.json'))
provider = models.MetaProvider(id=fname.stem,
name=md['dc:title'],
description=md['dc:description'],
url=md['dc:source'],
jsondata=md)
for meta in reader(fname, dicts=True):
try:
for k, v in meta.items():
if v and k != 'CONCEPTICON_ID':
models.ConceptSetMeta(metaprovider=provider,
conceptset=data['ConceptSet']
[meta['CONCEPTICON_ID']],
key=k,
value=v)
except:
print(fname)
print(meta)
raise
def main(args):
data = Data()
glottocodes, bibtex_keys = {}, defaultdict(set)
for d in reader(
args.data_file('repos', 'mappings',
'InventoryID-ISO-gcode-Bibkey-Source.tsv')):
glottocodes[d['InventoryID']] = d['Glottocode']
bibtex_keys[d['InventoryID']].add(d['BibtexKey'])
glottolog = Glottolog(
Path(phoible.__file__).parent.parent.parent.parent.joinpath(
'glottolog3', 'glottolog'))
languoids = {l.id: l for l in glottolog.languoids()}
phonemes = sorted(list(
reader(args.data_file('repos', 'data', 'phoible-by-phoneme.tsv'))),
key=lambda r: (r['InventoryID'], r['GlyphID']))
inventories = defaultdict(set)
for p in phonemes:
if p['InventoryID'] in glottocodes:
inventories[(languoids[glottocodes[p['InventoryID']]].name,
p['SpecificDialect'], p['Source'].upper())].add(
(p['InventoryID'], p['LanguageName']))
inventory_names = {}
for (glname, dname, source), invids in inventories.items():
if len(invids) == 1:
invid, lname = invids.pop()
inventory_names[invid] = name_in_source(glname,
dname) + ' [%s]' % source
else:
use_lname = len(set(r[1] for r in invids)) == len(invids)
for i, (invid,
lname) in enumerate(sorted(invids,
key=lambda j: int(j[0]))):
disambiguation = ' %s' % (i + 1, )
if use_lname:
disambiguation = ' (%s)' % lname
inventory_names[invid] = name_in_source(
glname, dname) + '%s [%s]' % (disambiguation, source)
for (invid, lname, dname, source), ps in groupby(
phonemes, lambda p: (p['InventoryID'], p['LanguageName'], p[
'SpecificDialect'], p['Source'])):
if invid not in glottocodes:
continue
ps = list(ps)
gc = glottocodes[invid]
lang = data['Variety'].get(gc)
if not lang:
languoid = languoids[gc]
lang = data.add(
models.Variety,
gc,
id=gc,
language_code=ps[0]['LanguageCode'],
name=languoid.name,
level=text_type(languoid.level.name),
latitude=languoid.latitude,
longitude=languoid.longitude,
)
if lang.latitude is None and languoid.level == Level.dialect:
ll = get_language(languoid)
lang.latitude = ll.latitude
lang.longitude = ll.longitude
contrib = data.add(
models.Inventory,
invid,
id=invid,
#language=lang,
source=source,
#source_url=source_urls.get(row.InventoryID),
#internetarchive_url=ia_urls.get(row.InventoryID),
name=inventory_names[invid],
description=name_in_source(lname, dname))
return
# FIXME: read from mappings file!
refs = defaultdict(list)
for row in get_rows(args, 'BibtexKey'):
if row[1] == 'NO SOURCE GIVEN':
refs[row[0]] = []
else:
refs[row[0]].append(row[1])
add_sources(args, data)
dataset = data.add(
common.Dataset,
'phoible',
id='phoible',
name='PHOIBLE Online',
description='PHOIBLE Online',
publisher_name="Max Planck Institute for the Science of Human History",
publisher_place="Jena",
publisher_url="https://www.shh.mpg.de",
domain='phoible.org',
license='http://creativecommons.org/licenses/by-sa/3.0/',
contact='*****@*****.**',
jsondata={
'license_icon':
'http://i.creativecommons.org/l/by-sa/3.0/88x31.png',
'license_name':
'Creative Commons Attribution-ShareAlike 3.0 Unported License'
})
for i, spec in enumerate([
('moran', "Steven Moran"),
('mccloy', "Daniel McCloy"),
('wright', "Richard Wright"),
]):
DBSession.add(
common.Editor(dataset=dataset,
ord=i + 1,
contributor=common.Contributor(id=spec[0],
name=spec[1])))
#squibs = defaultdict(list)
#for row in get_rows(args, 'Squib'):
# squibs[row[0]].append(row[1])
source_urls = dict(get_rows(args, 'URL'))
ia_urls = dict(get_rows(args, 'InternetArchive'))
# FIXME: group phoible-by-phoneme by LanguageCode, Source (make sure this is unique!)
aggregated = list(
reader(args.data_file('phoible-aggregated.tsv'),
delimiter='\t',
namedtuples=True))
inventory_names = {}
for key, items in groupby(sorted(aggregated,
key=lambda t: (t.LanguageCode, t.Source)),
key=lambda t: (t.LanguageCode, t.Source)):
items = list(items)
lname = lnames.get(key[0])
if not lname:
lname = items[0].LanguageName
lnames[key[0]] = lname
if len(items) == 1:
inventory_names[items[0].InventoryID] = '%s (%s)' % (lname, key[1])
else:
for i, item in enumerate(items):
inventory_names[item.InventoryID] = '%s %s (%s)' % (lname, i +
1, key[1])
# pull in Glottolog families instead? or in addition?
family_map = {
("Arawakan", "arwk"): "Arawakan",
("Trans-New Guinea", "trng"): "Trans-New Guinea",
("Moklen", "anes"): "Austronesian",
("Oko", "ncon"): "Niger-Congo",
("Muniche", "saso"): "Muniche",
("Tinigua", "saso"): "Tinigua",
("Vilela", "luvi"): "Vilela",
("Ofayé", "macg"): "Kamakanan",
("Purian", "macg"): "PurianPrint",
("Mixed language", "saml"): "Mixed language",
("Tupian", "tupi"): "Tupian",
("Yuwana", "saun"): "YuwanaPrint",
}
family_code_map = {k[1]: v for k, v in family_map.items()}
for row in aggregated:
lang = data['Variety'].get(row.LanguageCode)
if not lang:
if row.LanguageFamilyGenus == 'UNCLASSIFIED':
genus = None
else:
genus_id = slug(strip_quotes(row.LanguageFamilyGenus))
genus = genera.get(genus_id)
if not genus:
genus = genera.get(row.LanguageCode)
if not genus:
#print(row.LanguageFamilyGenus, row.LanguageFamilyRoot)
family = family_map.get(
(row.LanguageFamilyGenus, row.LanguageFamilyRoot))
genus = genera[genus_id] = data.add(
models.Genus,
genus_id,
id=genus_id,
name=row.LanguageFamilyGenus,
description=family or row.LanguageFamilyRoot,
active=False,
root=row.LanguageFamilyRoot)
if not genus.root:
genus.root = row.LanguageFamilyRoot
if genus.description in family_code_map:
genus.description = family_code_map[genus.description]
if row.LanguageCode in geocoords:
coords = geocoords[row.LanguageCode]
elif row.Latitude != 'NULL' and row.Longitude != 'NULL':
coords = (float(row.Latitude), float(row.Longitude))
lang = data.add(models.Variety,
row.LanguageCode,
id=row.LanguageCode,
name=lnames[row.LanguageCode],
genus=genus,
country=strip_quotes(row.Country),
area=strip_quotes(row.Area),
latitude=coords[0],
longitude=coords[1],
jsondata=dict(inventory_id=row.InventoryID))
add_language_codes(data,
lang,
row.LanguageCode,
glottocodes=glottocodes)
contributor = data['Contributor'].get(row.Source)
if not contributor:
contributor = data.add(common.Contributor,
row.Source,
id=row.Source,
name=SOURCES[row.Source][0],
description=SOURCES[row.Source][2])
for ref in SOURCES[row.Source][1]:
DBSession.add(
models.ContributorReference(source=data['Source'][ref],
contributor=contributor))
contrib = data.add(models.Inventory,
row.InventoryID,
id=row.InventoryID,
language=lang,
source=row.Source,
source_url=source_urls.get(row.InventoryID),
internetarchive_url=ia_urls.get(row.InventoryID),
name=inventory_names[row.InventoryID],
description=row.LanguageName)
DBSession.add(
common.ContributionContributor(contribution=contrib,
contributor=contributor))
#for j, squib in enumerate(squibs.get(row.InventoryID, [])):
# f = common.Contribution_files(
# object=contrib,
# id='squib-%s-%s.pdf' % (contrib.id, j + 1),
# name='Phonological squib',
# description=squib,
# mime_type='application/pdf')
# assert f
# # f.create(files_dir, file(args.data_file('phonological_squibs', src)).read())
DBSession.flush()
unknown_refs = {}
for row in reader(args.data_file('phoible-phonemes.tsv'),
namedtuples=True):
inventory = data['Inventory'][row.InventoryID]
segment = data['Segment'].get(row.Phoneme)
if not segment:
unicode_desc = [(c, unicodedata.name(c)) for c in row.Phoneme]
description = ' - '.join([t[1] for t in unicode_desc])
segment = data.add(
models.Segment,
row.Phoneme,
id=b16encode(md5(description).digest()),
name=row.Phoneme,
description=description,
equivalence_class=''.join([
t[0] for t in unicode_desc
if t[1].split()[0] not in ['COMBINING', 'MODIFIER']
]),
segment_class=row.Class,
combined_class=row.CombinedClass)
DBSession.flush()
vs = common.ValueSet(id=row.PhonemeID,
contribution=inventory,
language=inventory.language,
parameter=segment)
for ref in refs.get(row.InventoryID, []):
if ref not in data['Source']:
if ref not in unknown_refs:
print('-------', ref)
unknown_refs[ref] = 1
continue
DBSession.add(
common.ValueSetReference(source=data['Source'][ref],
valueset=vs))
DBSession.add(
common.Value(
id=row.PhonemeID,
name='%s %s' %
(row.Phoneme, data['Inventory'][row.InventoryID].name),
valueset=vs))
DBSession.flush()
for inventory_id in refs:
for ref in refs[inventory_id]:
if ref not in data['Source']:
continue
data.add(common.ContributionReference,
'%s-%s' % (inventory_id, ref),
source=data['Source'][ref],
contribution=data['Inventory'][inventory_id])
for i, row in enumerate(
reader(args.data_file('phoible-segments-features.tsv'))):
if i == 0:
features = list(map(feature_name, row))
continue
if row[0] not in data['Segment']:
# print('skipping feature vector:', row)
continue
for j, value in enumerate(row):
if j and value != '0':
DBSession.add(
common.Parameter_data(
key=features[j],
value=value,
ord=j,
object_pk=data['Segment'][row[0]].pk))
# FIXME: add allophones!
DBSession.flush()
def setUp(self):
TestWithDb.setUp(self)
DBSession.add(
common.Dataset(id='dataset',
name='dataset',
description='desc',
domain='clld'))
source = common.Source(id='source')
contributors = {
'contributor': 'A Name',
'b': 'b Name',
'c': 'c Name',
'd': 'd Name'
}
for id_, name in contributors.items():
contributors[id_] = common.Contributor(id=id_, name=name)
contribution = common.Contribution(id='contribution',
name='Contribution')
cr = common.ContributionReference(contribution=contribution,
source=source)
assert common.ContributionContributor(
contribution=contribution,
primary=True,
contributor=contributors['contributor'])
assert common.ContributionContributor(contribution=contribution,
primary=False,
contributor=contributors['b'])
assert common.ContributionContributor(contribution=contribution,
primary=True,
contributor=contributors['c'])
assert common.ContributionContributor(contribution=contribution,
primary=False,
contributor=contributors['d'])
DBSession.add(contribution)
language = common.Language(id='language',
name='Language 1',
latitude=10.5,
longitude=0.3)
language.sources.append(source)
identifier = common.Identifier(type='iso639-3', id='iso')
li = common.LanguageIdentifier(language=language,
identifier=identifier)
for i in range(2, 102):
_l = common.Language(id='l%s' % i, name='Language %s' % i)
_i = common.Identifier(type='iso639-3',
id='%.3i' % i,
name='%.3i' % i)
_li = common.LanguageIdentifier(language=_l, identifier=_i)
DBSession.add(_l)
param = common.Parameter(id='parameter', name='Parameter')
de = common.DomainElement(id='de',
name='DomainElement',
parameter=param)
de2 = common.DomainElement(id='de2',
name='DomainElement2',
parameter=param)
valueset = common.ValueSet(id='valueset',
language=language,
parameter=param,
contribution=contribution)
value = common.Value(id='value',
domainelement=de,
valueset=valueset,
frequency=50,
confidence='high')
DBSession.add(value)
paramnd = common.Parameter(id='no-domain',
name='Parameter without domain')
valueset = common.ValueSet(id='vs2',
language=language,
parameter=paramnd,
contribution=contribution)
vr = common.ValueSetReference(valueset=valueset, source=source)
value = common.Value(id='v2',
valueset=valueset,
frequency=50,
confidence='high')
DBSession.add(value)
unit = common.Unit(id='unit', name='Unit', language=language)
up = common.UnitParameter(id='unitparameter', name='UnitParameter')
DBSession.add(unit)
DBSession.add(
common.UnitValue(id='unitvalue',
name='UnitValue',
unit=unit,
unitparameter=up))
up2 = common.UnitParameter(id='up2', name='UnitParameter with domain')
de = common.UnitDomainElement(id='de', name='de', parameter=up2)
DBSession.add(
common.UnitValue(id='uv2',
name='UnitValue2',
unit=unit,
unitparameter=up2,
unitdomainelement=de))
DBSession.add(common.Source(id='s'))
sentence = common.Sentence(id='sentence',
name='sentence name',
description='sentence description',
analyzed='a\tmorpheme\tdoes\tdo',
gloss='a\tmorpheme\t1SG\tdo.SG2',
source='own',
comment='comment',
original_script='a morpheme',
language=language)
sr = common.SentenceReference(sentence=sentence, source=source)
DBSession.add(common.Config(key='key', value='value'))
DBSession.flush()
def update(args):
pid, cid = 'vitality', 'unesco'
count = 0
notfound = {}
contrib = common.Contribution.get(cid, default=None)
if not contrib:
contrib = common.Contribution(
id=cid,
name='Atlas of the World’s Languages in Danger',
description=
'Atlas of the World’s Languages in Danger, © UNESCO, http://www.unesco.org/culture/languages-atlas'
)
param = common.Parameter.get(pid, default=None)
if param is None:
param = common.Parameter(id=pid, name='Degree of endangerment')
domain = {de.name: de for de in param.domain}
for i, spec in enumerate(VITALITY_VALUES):
name, desc = spec
if name not in domain:
number = i + 1
domain[name] = common.DomainElement(id='%s-%s' % (pid, number),
name=name,
description=desc,
number=number,
parameter=param)
valuesets = {vs.id: vs for vs in param.valuesets}
for record in et.parse(args.data_file(DATA_FILE)).findall('.//RECORD'):
item = {}
for attr in [
'ID', 'Name in English', 'Name in French', 'Name in Spanish',
'Countries', 'Country codes alpha 3', 'ISO639-3 codes',
'Degree of endangerment'
]:
item[attr] = record.find(attr.replace(' ', '_')).text
if item['ISO639-3 codes']:
for code in item['ISO639-3 codes'].split(','):
code = code.strip()
lang = Languoid.get(code, key='hid', default=None)
if lang:
count += 1
item[
'url'] = 'http://www.unesco.org/culture/languages-atlas/en/atlasmap/language-iso-%s.html' % code
lang.update_jsondata(unesco=item)
de = domain[item['Degree of endangerment']]
vsid = '%s-%s' % (pid, lang.id)
vs = valuesets.get(vsid)
if not vs:
vs = common.ValueSet(id='vitality-%s' % lang.id,
parameter=param,
contribution=contrib,
language=lang)
DBSession.add(
common.Value(valueset=vs,
name=de.name,
domainelement=de))
valuesets[vsid] = vs
else:
vs.values[0].domainelement = de
else:
notfound[code] = 1
print 'assigned', count, 'unesco urls'
print 'missing iso codes:', notfound
def load_languoid(data, lang, nodemap):
dblang = data.add(
models.Languoid,
lang.id,
id=lang.id,
hid=lang.hid,
name=lang.name,
bookkeeping=lang.category == models.BOOKKEEPING,
newick=lang.newick_node(nodemap).newick,
latitude=lang.latitude,
longitude=lang.longitude,
status=models.LanguoidStatus.get(
lang.endangerment.name if lang.endangerment else 'safe'),
level=models.LanguoidLevel.from_string(lang.level.name),
father=data['Languoid'][lang.lineage[-1][1]] if lang.lineage else None)
if lang.iso:
add_language_codes(data, dblang, lang.iso)
for prov, names in lang.names.items():
for name in names:
l = 'en'
if '[' in name and name.endswith(']'):
name, l = [s.strip() for s in name[:-1].split('[', 1)]
add_identifier(dblang, data, name, 'name', prov, lang=l)
for prov, ids in lang.identifier.items():
for id_ in split_text(ids, separators=',;'):
add_identifier(dblang, data, id_, prov, None)
if not dblang.bookkeeping:
# Languages in Bookkeeping do not have a meaningful classification!
clf = lang.classification_comment
if clf:
for attr, pid in [('sub', 'sc'), ('family', 'fc')]:
val = getattr(clf, attr)
if attr == 'sub' and not val:
# Handle cases with subrefs but no sub comment.
val = getattr(clf, 'subrefs')
if val:
val = ', '.join('{0}'.format(r) for r in val)
if not val:
continue
vs = common.ValueSet(
id='%s-%s' % (pid, lang.id),
description=val,
language=dblang,
parameter=data['Parameter'][pid],
contribution=data['Contribution']['clf'])
DBSession.add(common.Value(id='%s-%s' % (pid, lang.id), valueset=vs))
iso_ret = lang.iso_retirement
if iso_ret:
DBSession.add(models.ISORetirement(
id=iso_ret.code,
name=iso_ret.name,
description=iso_ret.comment,
effective=iso_ret.effective,
reason=iso_ret.reason,
remedy=iso_ret.remedy,
change_request=iso_ret.change_request,
languoid=dblang))
eth_cmt = lang.ethnologue_comment
if eth_cmt:
DBSession.add(models.EthnologueComment(
comment=eth_cmt.comment,
code=eth_cmt.isohid,
type=eth_cmt.comment_type,
affected=eth_cmt.ethnologue_versions,
languoid=dblang))
def populate_test_db(engine):
set_alembic_version(engine, '58559d4eea0d')
data = TestData()
data.add_default(common.Dataset,
domain='clld',
jsondata={
'license_icon': 'cc-by',
'license_url': 'http://example.org'
})
data.add_default(common.Contributor, name='A Name', email='*****@*****.**')
for id_, name in {
'b': 'b Name',
'c': 'c Name',
'd': 'd Name',
}.items():
data.add(common.Contributor,
id_,
id=id_,
name=name,
url='http://example.org')
DBSession.add(
common.Editor(dataset=data[common.Dataset],
contributor=data[common.Contributor]))
data.add_default(common.Source)
data.add(common.Source,
'replaced',
id='replaced',
active=False,
jsondata={'__replacement_id__': 'source'})
data.add_default(common.Contribution)
common.ContributionReference(contribution=data[common.Contribution],
source=data[common.Source])
for primary, c in [(True, 'contributor'), (False, 'b'), (True, 'c'),
(False, 'd')]:
common.ContributionContributor(contribution=data[common.Contribution],
primary=primary,
contributor=data['Contributor'][c])
data.add_default(common.Language, latitude=10.5, longitude=0.3)
data[common.Language].sources.append(data[common.Source])
for i, type_ in enumerate(common.IdentifierType):
common.LanguageIdentifier(
language=data[common.Language],
identifier=common.Identifier(
type=type_.value,
id=type_.value + str(i),
name='abc' if type_.name == 'iso' else 'glot1234'))
common.LanguageIdentifier(language=data[common.Language],
identifier=common.Identifier(type='name',
id='name',
name='a'))
for i in range(2, 102):
_l = common.Language(id='l%s' % i, name='Language %s' % i)
_i = common.Identifier(type='iso639-3', id='%.3i' % i, name='abc')
common.LanguageIdentifier(language=_l, identifier=_i)
DBSession.add(_l)
param = data.add_default(common.Parameter)
de = common.DomainElement(id='de', name='DomainElement', parameter=param)
de2 = common.DomainElement(id='de2',
name='DomainElement2',
parameter=param)
valueset = data.add_default(common.ValueSet,
language=data[common.Language],
parameter=param,
contribution=data[common.Contribution])
common.ValueSetReference(valueset=valueset,
source=data[common.Source],
description='10-20')
data.add_default(common.Value,
domainelement=de,
valueset=valueset,
frequency=50,
confidence='high')
data.add(common.Value,
'value2',
id='value2',
domainelement=de2,
valueset=valueset,
frequency=50,
confidence='high')
paramnd = data.add(common.Parameter,
'no-domain',
id='no-domain',
name='Parameter without domain')
valueset = common.ValueSet(id='vs2',
language=data[common.Language],
parameter=paramnd,
contribution=data[common.Contribution])
common.ValueSetReference(valueset=valueset,
source=data[common.Source],
description='10-20')
common.Value(id='v2', valueset=valueset, frequency=50, confidence='high')
unit = data.add_default(common.Unit, language=data[common.Language])
up = data.add_default(common.UnitParameter)
common.UnitValue(id='unitvalue',
name='UnitValue',
unit=unit,
unitparameter=up)
up2 = common.UnitParameter(id='up2', name='UnitParameter with domain')
de = common.UnitDomainElement(id='de', name='de', parameter=up2)
DBSession.add(
common.UnitValue(id='uv2',
name='UnitValue2',
unit=unit,
unitparameter=up2,
unitdomainelement=de))
DBSession.add(common.Source(id='s'))
sentence = data.add_default(common.Sentence,
description='sentence description',
analyzed='a\tmorpheme\tdoes\tdo',
gloss='a\tmorpheme\t1SG\tdo.SG2',
source='own',
comment='comment',
original_script='a morpheme',
language=data[common.Language],
jsondata={'alt_translation': 'Spanish: ...'})
common.SentenceReference(sentence=sentence, source=data[common.Source])
DBSession.add(common.Config(key='key', value='value'))
common.Config.add_replacement('replaced',
'language',
model=common.Language)
common.Config.add_replacement('gone', None, model=common.Language)
DBSession.flush()
def main(args):
data = Data()
for rec in Database.from_file(
data_path('references.bib'), lowercase=False):
source = data.add(common.Source, rec.id, _obj=bibtex2source(rec))
dataset = common.Dataset(
id=clts.__name__,
name="CLTS",
publisher_name="Max Planck Institute for the Science of Human History",
publisher_place="Jena",
publisher_url="http://www.shh.mpg.de",
license="http://creativecommons.org/licenses/by/4.0/",
contact='*****@*****.**',
domain='clts.clld.org',
jsondata={
'license_icon': 'cc-by.png',
'license_name': 'Creative Commons Attribution 4.0 International License'})
DBSession.add(dataset)
for i, name in enumerate(['Johann-Mattis List', 'Cormac Anderson', 'Tiago Tresoldi',
'Thiago Chacon', 'Robert Forkel']):
c = common.Contributor(id=slug(name), name=name)
dataset.editors.append(common.Editor(contributor=c, ord=i))
for i, line in enumerate(reader(data_path('sounds.tsv'), delimiter='\t',
namedtuples=True)):
if not i % 100:
print('-', end="")
key = line.NAME.replace(' ', '_')
data.add(
models.SoundSegment,
key,
id=key,
name = line.NAME,
grapheme=line.GRAPHEME,
aliases=line.ALIASES,
representation=len(line.REFLEXES.split(',')),
reflexes = line.REFLEXES,
generated = True if line.GENERATED else False,
unicode = line.UNICODE,
)
print('')
english = data.add(
common.Language, 'eng',
id='eng',
name='English')
contributions = {}
for line in reader(data_path('datasets.tsv'),
delimiter='\t', namedtuples=True):
contributions[line.NAME] = data.add(
models.CLTSDataSet,
line.NAME,
id=line.NAME,
name=line.NAME,
description=line.DESCRIPTION,
datatype=line.TYPE
)
for id_ in line.REFS.split(', '):
common.ContributionReference(
source=data['Source'][id_],
contribution=contributions[line.NAME])
visited = set()
for i, line in enumerate(reader(data_path('graphemes.tsv'), delimiter="\t",
namedtuples=True)):
if not i % 100: print('-', end='')
key = line.DATASET + ':' + line.NAME+':'+line.GRAPHEME
if key not in visited:
sound_id = line.NAME.replace(' ', '_')
vs = common.ValueSet(
id=key,
description=line.NAME,
language=english,
contribution=contributions[line.DATASET],
parameter=data['SoundSegment'][sound_id]
)
data.add(
models.Grapheme,
key,
id=key,
grapheme=line.GRAPHEME,
bipa_grapheme=line.BIPA,
name=line.NAME,
dataset=line.DATASET,
datatype=line.DATATYPE,
frequency=line.FREQUENCY or 0,
image=line.IMAGE,
url=line.URL,
valueset=vs
)
visited.add(key)
print('-')
def main(args):
data = Data(created=utc.localize(datetime(2013, 11, 15)),
updated=utc.localize(datetime(2013, 12, 12)))
icons = issues.Icons()
#print icons
DBSession.execute("delete from Language")
DBSession.execute("delete from Unit")
DBSession.execute("delete from featuredomain")
DBSession.execute("delete from family")
DBSession.execute("delete from source")
DBSession.execute("delete from parameter")
DBSession.execute("delete from feature")
DBSession.execute("delete from domainelement")
DBSession.execute("delete from valueset")
DBSession.execute("delete from value")
DBSession.execute("delete from lsivalue")
DBSession.execute("delete from dataset")
DBSession.execute("delete from contributor")
DBSession.execute("delete from lsilanguage")
DBSession.execute("delete from contribution")
DBSession.execute("delete from designer")
DBSession.flush()
dtab = partial(_dtab, args.data_file())
#Languages
#print args.data_file()
#tabfns = ['%s' % fn.basename() for fn in args.data_file().files('nts_*.tab')]
#tabfns = ['nts_18.tab']
##tabfns = os.listdir('/Users/virk/shafqat/postDoc-Swe/project/clld/clld/lsi/lsi/data')[1:]
tabfns = os.listdir(
'/Users/virk/shafqat/postDoc-Swe/project/clld/clld/lsi/lsi/lsi_data'
)[1:]
#print tabfns
args.log.info("Sheets found: %s" % tabfns)
ldps = []
lgs = {}
nfeatures = Counter()
nlgs = Counter()
for fn in tabfns:
for ld in dtab(fn):
if ld['language_id'] == 'qgr' or ld['language_id'] == '---' or ld[
'language_id'] == '': # to exclude languages which do not have an iso-code
continue
if "feature_alphanumid" not in ld:
args.log.info("NO FEATUREID %s %s" % (len(ld), ld))
if not ld["feature_alphanumid"].startswith("DRS") \
and ld["feature_alphanumid"].find(".") == -1:
ldps.append(dp_dict(ld))
##print ld
lgs[ld['language_id']] = unescape(ld['language_name'])
if ld["value"] != "?":
nfeatures.update([ld['language_id']])
nlgs.update([ld['feature_alphanumid']])
ldps = sorted(ldps, key=lambda d: d['feature_alphanumid'])
#lgs["ygr"] = "Hua"
for lgid, lgname in lgs.items():
data.add(models.lsiLanguage,
lgid,
id=lgid,
name=lgname,
representation=nfeatures.get(lgid, 0))
DBSession.flush()
#print "I am here"
#print data['ntsLanguage'].values()[1].id
load_families(
data,
##[(NOCODE_TO_GLOTTOCODE.get(l.id, l.id), l) for l in data['lsiLanguage'].values()],
[(NOCODE_TO_GLOTTOCODE.get(l.id, l.id), l)
for l in data['lsiLanguage'].values()
if l.id != '---' and l.id != ''],
isolates_icon='tcccccc')
#print 'family'
#print data['Family'].get('sino1245').jsondata
#Domains
for domain in set(ld['feature_domain'] for ld in ldps):
data.add(models.FeatureDomain, domain, name=domain)
DBSession.flush()
#Designers
#for i, info in enumerate(dtab("ntscontributions.tab") + dtab("ntscontacts.tab")):
for i, info in enumerate([{
'designer': 'shafqat',
'domain': '',
'pdflink': '',
'citation': ''
}, {
'designer': '-',
'domain': '',
'pdflink': '',
'citation': ''
}]):
designer_id = str(i + 1)
data.add(models.Designer,
info['designer'],
id=designer_id,
name=designer_id,
domain=info["domain"],
contributor=info['designer'],
pdflink=info["pdflink"],
citation=info["citation"])
DBSession.flush()
#Sources
'''for k, (typ, bibdata) in [
ktfbib(bibsource) for ld in ldps
if ld.get(u'bibsources') for bibsource in ld['bibsources'].split(",,,")
]:
if k not in data["Source"]:
data.add(common.Source, k, _obj=bibtex2source(Record(typ, k, **bibdata)))
DBSession.flush()'''
#Features
fs = [(fid, mergeds(lds))
for fid, lds in groupby(ldps, key=lambda d: d['feature_alphanumid'])]
fvdesc = [(fid, [
(ld.get("feature_possible_values"), ld.get("fromfile")) for ld in lds
if ld.get("feature_possible_values")
]) for fid, lds in groupby(ldps, key=lambda d: d['feature_alphanumid'])]
fvdt = [(fid, grp2(vdescs)) for (fid, vdescs) in fvdesc]
fvmis = [(fid, vdescs) for (fid, vdescs) in fvdt if len(vdescs) > 1]
for _, dfsids in groupby(sorted(
(f.get('feature_name', fid), fid) for fid, f in fs),
key=lambda t: t[0]):
##print [(k,v) for (k,v) in list(dfsids)],len(list(dfsids))
assert len(list(dfsids)) == 1
#print 'here is nlgs'
for fid, f in fs:
#print "lang name"
#print ldps
#print f.get('feature_possible_values', ""),
if not fid.isdigit():
args.log.info("NO INT FID %s" % f)
feature = data.add(
models.Feature,
fid,
id=fid,
name=f.get('feature_name', f['feature_alphanumid']),
doc=f.get('feature_information', ""),
vdoc=f.get('feature_possible_values', ""),
representation=nlgs.get(fid, 0),
designer=data["Designer"][f['designer']],
dependson=f.get("depends_on", ""),
abbreviation=f.get("abbreviation", ""),
featuredomain=data['FeatureDomain'][f["feature_domain"]],
name_french=f.get('francais', ""),
clarification=f.get(
"draft of clarifying comments to outsiders (hedvig + dunn + harald + suzanne)",
""),
alternative_id=f.get("old feature number", ""),
jl_relevant_unit=f.get("relevant unit(s)", ""),
jl_function=f.get("function", ""),
jl_formal_means=f.get("formal means", ""),
sortkey_str="",
sortkey_int=int(fid))
vdesclist = [veq.split("==") for veq in feature.vdoc.split("||")]
vdesc = {v.replace(".", "-"): desc for [v, desc] in vdesclist}
##vdesc = {fmly+val2icon(v): desc for ((v,desc),fmly) in itertools.product([(vv,desc) for [vv, desc] in vdesclist],['c','d','f','t'])}
vdesc.setdefault('?', 'Not known')
if 'N/A' not in vdesc and feature.dependson:
vdesc["N/A"] = "Not Applicable"
vi = {v: i for (i, v) in enumerate(sorted(vdesc.keys()))}
##vicons = {f+val2icon(v):f+val2icon(v) for (v,f) in itertools.product(['0','1','2','3'],['c','d','f','t'])}
##vicons['?'] = 'c00ffff'
##vicons['N/A'] = 'c00ffff'
##vicons = icons.iconize(vi.keys())
for (v, desc) in vdesc.items():
#print v,vicons[v]
data.add(common.DomainElement, (fid, v),
id='%s-%s' % (fid, v),
name=v,
description=desc,
jsondata={"icon": Colors[v]},
number=vi[v],
parameter=feature)
DBSession.flush()
for ((f, lg), ixs) in grp2([((ld['feature_alphanumid'], ld['language_id']),
i) for i, ld in enumerate(ldps)]):
ixvs = set([ldps[ix]['value'] for ix in ixs])
if len(ixvs) == 1:
continue
args.log.warn("Dup value %s %s %s" %
(f, lg, [(ldps[ix]['value'], ldps[ix]['fromfile'])
for ix in ixs]))
##print "Dup value %s %s %s" % (f, lg, [(ldps[ix]['value'], ldps[ix]['fromfile'], ldps[ix].get('provenance')) for ix in ixs])
errors = {}
done = set()
glottolog = Glottolog()
for ld in ldps:
############################### for printing different map markers for different familys for features:shafqat
#print data['Family']
language = data['lsiLanguage'][ld['language_id']]
if isinstance(language, (tuple, list)) and len(language) == 2:
code, language = language
else:
code = language.id
if code != '-':
gl_language = glottolog.languoid(code)
if gl_language:
gl_family = gl_language.family
if gl_family:
family = data['Family'].get(gl_family.id)
##ld['value'] = ld['value']+'-'+str(family)
##ld['value'] = combineValueFamily(ld['value'],str(family))
#print family
#####################################
parameter = data['Feature'][ld['feature_alphanumid']]
language = data['lsiLanguage'][ld['language_id']]
id_ = '%s-%s' % (parameter.id, language.id)
if id_ in done:
continue
if (ld['feature_alphanumid'],
ld['value']) not in data['DomainElement']:
if not ld["value"].strip():
continue
info = (ld['feature_alphanumid'],
ld.get('feature_name', "[Feature Name Lacking]"),
ld['language_id'], ld['value'], ld['fromfile'])
msg = u"%s %s %s %s %s not in the set of legal values ({0})" % info
args.log.error(
msg.format(
sorted([
y for (x, y) in data['DomainElement'].keys()
if x == ld['feature_alphanumid']
])))
##print msg.format(sorted(
## [y for (x, y) in data['DomainElement'].keys()
## if x == ld['feature_alphanumid']]))
errors[(ld['feature_alphanumid'], ld['language_id'])] = info
continue
vs = common.ValueSet(
id=id_,
language=language,
parameter=parameter,
source=ld["source"] or None,
##contribution=parameter.designer
)
#print
#print "this one"
#print ld['value'],family
models.lsiValue(
id=id_,
domainelement=data['DomainElement'][(ld['feature_alphanumid'],
ld['value'])],
jsondata={
"icon":
data['DomainElement'][(ld['feature_alphanumid'],
ld['value'])].jsondata,
"family":
FamilyCodes[str(family)]
},
comment=ld["comment"],
valueset=vs,
contributed_datapoint=ld["contributor"])
done.add(id_)
'''if not ld.get('bibsources'):
if 'bibsources' not in ld:
args.log.warn("no bibsource %s" % ld)
continue
for k, _ in [ktfbib(bibsource) for bibsource in ld['bibsources'].split(",,,")]:
common.ValueSetReference(valueset=vs, source=data['Source'][k])'''
DBSession.flush()
#To CLDF
cldf = {}
for ld in ldps:
parameter = data['Feature'][ld['feature_alphanumid']]
language = data['lsiLanguage'][ld['language_id']]
id_ = '%s-%s' % (parameter.id, language.id)
if not id_ in done:
continue
dt = (lgs[ld['language_id']], ld['language_id'],
ld['feature_alphanumid'] + ". " + ld['feature_name'],
ld["value"])
cldf[dt] = None
tab = lambda rows: u''.join([u'\t'.join(row) + u"\n" for row in rows])
savu(tab([("Language", "iso-639-3", "Feature", "Value")] + cldf.keys()),
"lsi.cldf")
args.log.info('%s Errors' % len(errors))
dataset = common.Dataset(
id="LSI",
name='Linguistic Survey of India',
publisher_name="Sprakbanken",
publisher_place="Gothenburg",
publisher_url="to be given",
description="this is to be followed",
domain='http://lsi.clld.org',
published=date(2016, 05, 16),
contact='*****@*****.**',
license='http://creativecommons.org/licenses/by-nc-nd/2.0/de/deed.en',
jsondata={
'license_icon':
'http://wals.info/static/images/cc_by_nc_nd.png',
'license_name':
'Creative Commons Attribution-NonCommercial-NoDerivs 2.0 Germany'
})
# disabled for experimental purposes, names were appearing multiple times
for i, contributor in enumerate([
common.Contributor(id="Lars Borin",
name="Lars Borin",
email="*****@*****.**"),
common.Contributor(id="Shafqat Mumtaz Virk",
name="Shafqat Mumtaz Virk",
email="*****@*****.**"),
common.Contributor(id="Anju Saxena",
name="Anju Saxena",
email="*****@*****.**"),
common.Contributor(id="Harald Hammarstrom",
name="Harald Hammarstrom",
email="*****@*****.**")
]):
#print i
common.Editor(dataset=dataset, contributor=contributor, ord=i)
'''cont1 = common.Contributor(
id="Harald Hammarstrom",
name="Harald Hammarstrom",
email="*****@*****.**")
cont2= common.Contributor(
id="Shafqat Mumtaz Virk",
name="Shafqat Mumtaz Virk",
email="*****@*****.**")
cont3 = common.Contributor(
id="Lars Borin",
name="Lars Borin",
email="*****@*****.**")
for contributor in [cont1,cont2,cont3]:
common.Editor(dataset=dataset, contributor=contributor,ord=1)'''
DBSession.add(dataset)
DBSession.flush()
def main(args):
data = Data()
editors = OrderedDict()
editors['Susanne Maria Michaelis'] = None
editors['Philippe Maurer'] = None
editors['Martin Haspelmath'] = None
editors['Magnus Huber'] = None
for row in read(args, 'People'):
name = row['First name'] + ' ' if row['First name'] else ''
name += row['Last name']
kw = dict(
name=name,
id=slug('%(Last name)s%(First name)s' % row),
url=row['Contact Website'].split()[0]
if row['Contact Website'] else None,
address=row['Comments on database'],
)
contrib = data.add(common.Contributor, row['Author ID'], **kw)
if kw['name'] in editors:
editors[kw['name']] = contrib
DBSession.flush()
dataset = common.Dataset(
id='apics',
name='APiCS Online',
description='Atlas of Pidgin and Creole Language Structures Online',
domain='apics-online.info',
published=date(2013, 11, 4),
license='http://creativecommons.org/licenses/by/3.0/',
contact='*****@*****.**',
jsondata={
'license_icon': 'cc-by.png',
'license_name': 'Creative Commons Attribution 3.0 Unported License'
})
DBSession.add(dataset)
for i, editor in enumerate(editors.values()):
common.Editor(dataset=dataset, contributor=editor, ord=i + 1)
colors = dict(
(row['ID'], row['RGB_code']) for row in read(args, 'Colours'))
abbrs = {}
for id_, name in LGR_ABBRS.items():
DBSession.add(common.GlossAbbreviation(id=id_, name=name))
abbrs[id_] = 1
for id_, name in {
'C**T': 'clitic',
'IMPF': 'imperfect',
'INTERM': 'intermediate',
'NCOMPL': 'noncompletive',
'NONFUT': 'nonfuture',
'NPROX': 'nonproximal',
'NSG': 'nonsingular',
'PP': 'past participle',
'PROP': 'proprietive',
'TMA': 'tense-mood-aspect',
}.items():
DBSession.add(common.GlossAbbreviation(id=id_, name=name))
abbrs[id_] = 1
for row in reader(args.data_file('non-lgr-gloss-abbrs.csv'),
delimiter=',',
namedtuples=True):
for match in GLOSS_ABBR_PATTERN.finditer(row.standard):
if match.group('abbr') not in abbrs:
abbrs[match.group('abbr')] = 1
DBSession.add(
common.GlossAbbreviation(id=match.group('abbr'),
name=row.meaning))
non_bibs = {}
for row in read(args, 'References', 'Reference_ID'):
if row['Reference_type'] == 'Non-bib':
non_bibs[row['Reference_ID']] = row['Reference_name']
continue
if isinstance(row['Year'], int):
year_int = row['Year']
year = str(row['Year'])
elif row['Year']:
year_int = None
for m in re.finditer('(?P<year>(1|2)[0-9]{3})', row['Year']):
year_int = int(m.group('year'))
break
year = row['Year']
else:
year, year_int = None, None
title = row['Article_title'] or row['Book_title']
attrs = {}
jsondata = {}
for attr, field in {
'Additional_information': 'note',
'Article_title': 'title',
'Book_title': 'booktitle',
'City': 'address',
'Editors': 'editor',
'Full_reference': None,
'Issue': None,
'Journal': 'journal',
'Language_codes': None,
'LaTeX_cite_key': None,
'Pages': 'pages',
'Publisher': 'publisher',
'Reference_type': 'type',
'School': 'school',
'Series_title': 'series',
'URL': 'url',
'Volume': 'volume',
}.items():
value = row.get(attr)
if not isinstance(value, int):
value = (value or '').strip()
if attr == 'Issue' and value:
try:
value = str(int(value))
except ValueError:
pass
if value:
if field:
attrs[field] = value
else:
jsondata[attr] = value
p = data.add(common.Source,
row['Reference_ID'],
id=str(row['Reference_ID']),
name=row['Reference_name'],
description=title,
author=row['Authors'],
year=year,
year_int=year_int,
bibtex_type=getattr(EntryType, row['BibTeX_type']
or 'misc'),
jsondata=jsondata,
**attrs)
if p.bibtex_type.value == 'misc' and not p.description:
p.description = p.note
DBSession.flush()
DBSession.flush()
infobox = jsonload(args.data_file('infobox.json'))
glottocodes = jsonload(args.data_file('glottocodes.json'))
for row in read(args, 'Languages', 'Order_number'):
lon, lat = [
float(c.strip()) for c in row['map_coordinates'].split(',')
]
kw = dict(
name=row['Language_name'],
id=str(row['Order_number']),
latitude=lat,
longitude=lon,
region=row['Category_region'],
)
lect = data.add(models.Lect, row['Language_ID'], **kw)
DBSession.flush()
for i, item in enumerate(infobox[lect.id]):
DBSession.add(
common.Language_data(object_pk=lect.pk,
ord=i,
key=item[0],
value=item[1]))
if row["Languages_contribution_documentation::Lect_description_checked_status"] \
!= "Checked":
print 'unchecked! ---', row['Language_name']
desc = row.get(
'Languages_contribution_documentation::Lect description', '')
markup_desc = normalize_markup(row[
'Languages_contribution_documentation::z_calc_GetAsCSS_Lect_description']
)
c = data.add(
models.ApicsContribution,
row['Language_ID'],
id=str(row['Order_number']),
name=row['Language_name'],
description=desc,
markup_description=markup_desc,
survey_reference=data['Source'][row['Survey_reference_ID']],
language=lect)
for ext, label, mtype in [
('pdf', 'Glossed text', 'application/pdf'),
('mp3', 'Glossed text audio', 'audio/mpeg'),
]:
fid = '%s-gt.%s' % (c.id, ext)
if args.data_file('files', 'contribution', c.id, fid).exists():
common.Contribution_files(object=c,
id=fid,
name=label,
mime_type=mtype)
else:
print label, 'missing for:', row['Language_name']
#
# TODO: for michif, 75, add link http://www.youtube.com/watch?v=f0C4cODsSyE
#
iso = None
if row['ISO_code'] and len(row['ISO_code']) == 3:
iso = row['ISO_code'].lower()
if 'iso:%s' % row['ISO_code'] not in data['Identifier']:
data.add(common.Identifier,
'iso:%s' % row['ISO_code'],
id=row['ISO_code'].lower(),
name=row['ISO_code'].lower(),
type=common.IdentifierType.iso.value)
DBSession.add(
common.LanguageIdentifier(
language=data['Lect'][row['Language_ID']],
identifier=data['Identifier']['iso:%s' % row['ISO_code']]))
if lect.id in glottocodes:
identifier = data.add(common.Identifier,
'gc:%s' % glottocodes[lect.id],
id=glottocodes[lect.id],
name=glottocodes[lect.id],
type=common.IdentifierType.glottolog.value)
DBSession.add(
common.LanguageIdentifier(
language=data['Lect'][row['Language_ID']],
identifier=identifier))
if row['Language_name_ethnologue']:
if row['Language_name_ethnologue'] not in data['Identifier']:
data.add(common.Identifier,
row['Language_name_ethnologue'],
id=iso
or 'ethnologue:%s' % row['Language_name_ethnologue'],
name=row['Language_name_ethnologue'],
type='ethnologue')
DBSession.add(
common.LanguageIdentifier(
language=data['Lect'][row['Language_ID']],
identifier=data['Identifier'][
row['Language_name_ethnologue']]))
example_count = {}
for row in read(args, 'Examples', 'Order_number'):
assert row['Language_ID']
lang = data['Lect'][row['Language_ID']]
id_ = '%(Language_ID)s-%(Example_number)s' % row
atext, gloss = igt(row)
example_count[row['Language_ID']] = max(
[example_count.get(row['Language_ID'], 1), row['Example_number']])
p = add_sentence(
args,
data,
id_,
id='%s-%s' % (lang.id, row['Example_number']),
name=row['Text'] or row['Analyzed_text'],
description=row['Translation'],
type=row['Type'].strip().lower() if row['Type'] else None,
comment=row['Comments'],
gloss=gloss,
analyzed=atext,
markup_text=normalize_markup(row['z_calc_Text_CSS']),
markup_gloss=normalize_markup(row['z_calc_Gloss_CSS']),
markup_comment=normalize_markup(row['z_calc_Comments_CSS']),
markup_analyzed=normalize_markup(row['z_calc_Analyzed_text_CSS']),
original_script=row['Original_script'],
jsondata={
'sort': row['Order_number'],
'alt_translation': (row['Translation_other'] or '').strip()
or None
},
language=lang)
if row['Reference_ID']:
if row['Reference_ID'] in data['Source']:
source = data['Source'][row['Reference_ID']]
DBSession.add(
common.SentenceReference(
sentence=p,
source=source,
key=source.id,
description=row['Reference_pages']))
else:
p.source = non_bibs[row['Reference_ID']]
DBSession.flush()
for row in read(args, 'Language_references'):
if row['Reference_ID'] not in data['Source']:
assert row['Reference_ID'] in non_bibs
continue
assert row['Language_ID'] in data['ApicsContribution']
source = data['Source'][row['Reference_ID']]
DBSession.add(
common.ContributionReference(
contribution=data['ApicsContribution'][row['Language_ID']],
source=source,
description=row['Pages'],
key=source.id))
#
# global counter for features - across feature types
#
feature_count = 0
for row in read(args, 'Features', 'Feature_number'):
id_ = str(row['Feature_number'])
if int(id_) > feature_count:
feature_count = int(id_)
wals_id = None
desc = row['Feature_annotation_publication']
if row['WALS_match'] == 'Total':
if isinstance(row['WALS_No.'], int):
wals_id = row['WALS_No.']
else:
wals_id = int(row['WALS_No.'].split('.')[0].strip())
p = data.add(models.Feature,
row['Feature_code'],
name=row['Feature_name'],
id=id_,
description=desc,
markup_description=normalize_markup(
row['z_calc_Feature_annotation_publication_CSS']),
feature_type='primary',
multivalued=row['Value_relation_type'] != 'Single',
area=row['Feature_area'],
wals_id=wals_id)
names = {}
for i in range(1, 10):
if not row['Value%s_publication' % i] \
or not row['Value%s_publication' % i].strip():
continue
name = row['Value%s_publication' % i].strip()
if name in names:
name += ' (%s)' % i
names[name] = 1
de = data.add(
common.DomainElement,
'%s-%s' % (row['Feature_code'], i),
id='%s-%s' % (id_, i),
name=name,
parameter=p,
abbr=row['Value%s_for_book_maps' % i] if p.id != '0' else name,
number=int(row['Value%s_value_number_for_publication' % i]),
jsondata={'color': colors[row['Value_%s_colour_ID' % i]]},
)
assert de
if row['Authors_FeatureArticles']:
authors, _ = row['Authors_FeatureArticles'].split('and the APiCS')
authors = authors.strip()
if authors.endswith(','):
authors = authors[:-1].strip()
for i, name in enumerate(authors.split(',')):
assert name.strip() in editors
p._authors.append(
models.FeatureAuthor(ord=i + 1,
contributor=editors[name.strip()]))
DBSession.flush()
primary_to_segment = {123: 63, 126: 35, 128: 45, 130: 41}
segment_to_primary = dict(
zip(primary_to_segment.values(), primary_to_segment.keys()))
number_map = {}
names = {}
for row in read(args, 'Segment_features', 'Order_number'):
symbol = row['Segment_symbol']
if row['Segment_name'] == 'voiceless dental/alveolar sibilant affricate':
symbol = 't\u0361s'
truth = lambda s: s and s.strip().lower() == 'yes'
name = '%s - %s' % (symbol, row['Segment_name'])
if name in names:
number_map[row['Segment_feature_number']] = names[name]
continue
number_map[
row['Segment_feature_number']] = row['Segment_feature_number']
names[name] = row['Segment_feature_number']
feature_count += 1
if row['Segment_feature_number'] in segment_to_primary:
primary_to_segment[segment_to_primary[row['Segment_feature_number']]]\
= str(feature_count)
p = data.add(models.Feature,
row['Segment_feature_number'],
name=name,
id=str(feature_count),
feature_type='segment',
area='Vowels' if truth(row['Vowel']) else
('Obstruent consonants'
if truth(row['Obstruent']) else 'Sonorant consonants'),
jsondata=dict(
number=int(row['Segment_feature_number']),
vowel=truth(row['Vowel']),
consonant=truth(row['Consonant']),
obstruent=truth(row['Obstruent']),
core_list=truth(row['Core_list_segment']),
symbol=symbol,
))
for i, spec in SEGMENT_VALUES.items():
data.add(common.DomainElement,
'%s-%s' % (row['Segment_feature_number'], spec[0]),
id='%s-%s' % (p.id, i),
name=spec[0],
parameter=p,
jsondata={'color': spec[1]},
number=i)
print '--> remapped:', primary_to_segment
DBSession.flush()
for row in read(args, 'Sociolinguistic_features',
'Sociolinguistic_feature_number'):
feature_count += 1
p = data.add(models.Feature,
row['Sociolinguistic_feature_code'],
name=row['Sociolinguistic_feature_name'],
id='%s' % feature_count,
description=row['Sociolinguistic_feature_annotation'],
area='Sociolinguistic',
feature_type='sociolinguistic')
names = {}
for i in range(1, 10):
id_ = '%s-%s' % (row['Sociolinguistic_feature_code'], i)
if row.get('Value%s' % i) and row['Value%s' % i].strip():
name = row['Value%s' % i].strip()
if name in names:
name += ' (%s)' % i
names[name] = 1
else:
continue
kw = dict(id='%s-%s' % (p.id, i), name=name, parameter=p, number=i)
data.add(common.DomainElement,
id_,
id='%s-%s' % (p.id, i),
name=name,
parameter=p,
number=i,
jsondata={
'color':
colors.get(row['Value%s_colour_ID' % i],
colors.values()[i])
})
sd = {}
for row in read(args, 'Segment_data'):
if row['Segment_feature_number'] not in number_map:
continue
number = number_map[row['Segment_feature_number']]
if not row['Presence_in_the_language']:
continue
lang = data['Lect'][row['Language_ID']]
param = data['Feature'][number]
id_ = '%s-%s' % (lang.id, param.id)
if id_ in sd:
assert row['c_Record_is_a_duplicate'] == 'Yes'
continue
sd[id_] = 1
valueset = data.add(
common.ValueSet,
id_,
id=id_,
parameter=param,
language=lang,
contribution=data['ApicsContribution'][row['Language_ID']],
description=row['Comments'],
markup_description=normalize_markup(row['z_calc_Comments_CSS']),
)
v = data.add(
common.Value,
id_,
id=id_,
frequency=float(100),
valueset=valueset,
domainelement=data['DomainElement'][
'%s-%s' % (number, row['Presence_in_the_language'])],
)
if row['Example_word'] and row['Example_word_gloss']:
example_count[row['Language_ID']] += 1
p = add_sentence(args,
data,
'%s-p%s' % (lang.id, data['Feature'][number].id),
id='%s-%s' %
(lang.id, example_count[row['Language_ID']]),
name=row['Example_word'],
description=row['Example_word_gloss'],
language=lang)
DBSession.add(common.ValueSentence(value=v, sentence=p))
source = data['Source'].get(row['Refers_to_references_Reference_ID'])
if source:
DBSession.add(
common.ValueSetReference(valueset=valueset,
source=source,
key=source.id))
elif row['Refers_to_references_Reference_ID'] in non_bibs:
valueset.source = non_bibs[
row['Refers_to_references_Reference_ID']]
lects = defaultdict(lambda: 1)
lect_map = {}
records = {}
false_values = {}
no_values = {}
wals_value_number = {}
for row in read(args, 'wals'):
if row['z_calc_WALS_value_number']:
wals_value_number[
row['Data_record_id']] = row['z_calc_WALS_value_number']
def prefix(attr, _prefix):
if _prefix:
return '%s_%s' % (_prefix, attr)
return attr.capitalize()
for _prefix, abbr in [('', ''), ('Sociolinguistic', 'sl')]:
num_values = 10
for row in read(args, prefix('data', _prefix)):
if not row[prefix('feature_code', _prefix)]:
print('no associated feature for', prefix('data', _prefix),
row[prefix('data_record_id', _prefix)])
continue
lid = row['Language_ID']
lect_attr = row.get('Lect_attribute', 'my default lect').lower()
if lect_attr != 'my default lect':
if (row['Language_ID'], row['Lect_attribute']) in lect_map:
lid = lect_map[(row['Language_ID'], row['Lect_attribute'])]
else:
lang = data['Lect'][row['Language_ID']]
c = lects[row['Language_ID']]
lid = '%s-%s' % (row['Language_ID'], c)
kw = dict(
name='%s (%s)' % (lang.name, row['Lect_attribute']),
id='%s' % (1000 + 10 * int(lang.id) + c),
latitude=lang.latitude,
longitude=lang.longitude,
description=row['Lect_attribute'],
language=lang,
)
data.add(models.Lect, lid, **kw)
lects[row['Language_ID']] += 1
lect_map[(row['Language_ID'], row['Lect_attribute'])] = lid
id_ = abbr + str(row[prefix('data_record_id', _prefix)])
assert id_ not in records
records[id_] = 1
assert row[prefix('feature_code', _prefix)] in data['Feature']
language = data['Lect'][lid]
parameter = data['Feature'][row[prefix('feature_code', _prefix)]]
valueset = common.ValueSet(
id='%s-%s' % (language.id, parameter.id),
description=row['Comments_on_value_assignment'],
markup_description=normalize_markup(
row.get('z_calc_Comments_on_value_assignment_CSS')),
)
values_found = {}
for i in range(1, num_values):
if not row['Value%s_true_false' % i]:
continue
if row['Value%s_true_false' % i].strip().lower() != 'true':
assert row['Value%s_true_false' %
i].strip().lower() == 'false'
false_values[row[prefix('data_record_id', _prefix)]] = 1
continue
iid = '%s-%s' % (row[prefix('feature_code', _prefix)], i)
if iid not in data['DomainElement']:
print(iid, row[prefix('data_record_id',
_prefix)], '--> no domainelement!')
continue
values_found['%s-%s' % (id_, i)] = dict(
id='%s-%s' % (valueset.id, i),
domainelement=data['DomainElement']['%s-%s' % (row[prefix(
'feature_code', _prefix)], i)],
confidence=row['Value%s_confidence' % i],
frequency=float(row['c_V%s_frequency_normalised' %
i]) if _prefix == '' else 100)
if values_found:
if row[prefix('data_record_id', _prefix)] in wals_value_number:
valueset.jsondata = {
'wals_value_number':
wals_value_number.pop(row[prefix(
'data_record_id', _prefix)])
}
valueset.parameter = parameter
valueset.language = language
valueset.contribution = data['ApicsContribution'][
row['Language_ID']]
valueset = data.add(common.ValueSet, id_, _obj=valueset)
for i, item in enumerate(values_found.items()):
if i > 0 and not parameter.multivalued:
print 'multiple values for single-valued parameter: %s' % id_
break
id_, kw = item
kw['valueset'] = valueset
value = data.add(common.Value, id_, **kw)
#
# store references to additional data for segments which should be reused
# for corresponding primary features!
#
if int(parameter.id) in primary_to_segment:
assert len(values_found) == 1
seg_id = '%s-%s' % (language.id, primary_to_segment[int(
parameter.id)])
seg_valueset = data['ValueSet'][seg_id]
seg_value = data['Value'][seg_id]
if not valueset.description and seg_valueset.description:
valueset.description = seg_valueset.description
for s in seg_value.sentence_assocs:
DBSession.add(
common.ValueSentence(value=value,
sentence=s.sentence))
for r in seg_valueset.references:
DBSession.add(
common.ValueSetReference(valueset=valueset,
source=r.source,
key=r.key))
if not valueset.source and seg_valueset.source:
valueset.source = seg_valueset.source
DBSession.flush()
else:
no_values[id_] = 1
DBSession.flush()
for prefix, abbr, num_values in [
('D', '', 10),
('Sociolinguistic_d', 'sl', 7),
]:
for row in read(args, prefix + 'ata_references'):
assert row['Reference_ID'] in data['Source'] \
or row['Reference_ID'] in non_bibs
try:
vs = data['ValueSet'][abbr +
str(row[prefix + 'ata_record_id'])]
if row['Reference_ID'] in data['Source']:
source = data['Source'][row['Reference_ID']]
DBSession.add(
common.ValueSetReference(
valueset=vs,
source=source,
key=source.id,
description=row['Pages'],
))
else:
if vs.source:
vs.source += '; ' + non_bibs[row['Reference_ID']]
else:
vs.source = non_bibs[row['Reference_ID']]
except KeyError:
continue
DBSession.flush()
missing = 0
for row in read(args, 'Value_examples'):
try:
DBSession.add(
common.ValueSentence(
value=data['Value']['%(Data_record_id)s-%(Value_number)s' %
row],
sentence=data['Sentence'][
'%(Language_ID)s-%(Example_number)s' % row],
description=row['Notes'],
))
except KeyError:
missing += 1
print('%s Value_examples are missing data' % missing)
print('%s data sets with false values' % len(false_values))
print('%s data sets without values' % len(no_values))
for k, v in wals_value_number.items():
print 'unclaimed wals value number:', k, v
for i, row in enumerate(read(args, 'Contributors')):
kw = dict(contribution=data['ApicsContribution'][row['Language ID']],
contributor=data['Contributor'][row['Author ID']])
if row['Order_of_appearance']:
kw['ord'] = int(float(row['Order_of_appearance']))
data.add(common.ContributionContributor, i, **kw)
DBSession.flush()
def main(args):
# determine if we run on a machine where other databases are available for lookup
# locally:
data = Data()
genera = get_genera(data) if astroman else {}
glottocodes, lnames, geocoords = {}, {}, {}
if astroman:
for k, v in glottocodes_by_isocode(
'postgresql://robert@/glottolog3',
cols=['id', 'name', 'latitude', 'longitude']).items():
glottocodes[k] = v[0]
lnames[k] = v[1]
geocoords[k] = (v[2], v[3])
refs = defaultdict(list)
for row in get_rows(args, 'BibtexKey'):
if row[1] == 'NO SOURCE GIVEN':
refs[row[0]] = []
else:
refs[row[0]].append(row[1])
add_sources(args, data)
dataset = data.add(
common.Dataset,
'phoible',
id='phoible',
name='PHOIBLE Online',
description='PHOIBLE Online',
publisher_name="Max Planck Institute for Evolutionary Anthropology",
publisher_place="Leipzig",
publisher_url="http://www.eva.mpg.de",
domain='phoible.org',
license='http://creativecommons.org/licenses/by-sa/3.0/',
contact='*****@*****.**',
jsondata={
'license_icon':
'http://i.creativecommons.org/l/by-sa/3.0/88x31.png',
'license_name':
'Creative Commons Attribution-ShareAlike 3.0 Unported License'
})
for i, spec in enumerate([
('moran', "Steven Moran"),
('mccloy', "Daniel McCloy"),
('wright', "Richard Wright"),
]):
DBSession.add(
common.Editor(dataset=dataset,
ord=i + 1,
contributor=common.Contributor(id=spec[0],
name=spec[1])))
squibs = defaultdict(list)
for row in get_rows(args, 'Squib'):
squibs[row[0]].append(row[1])
source_urls = dict(get_rows(args, 'URL'))
ia_urls = dict(get_rows(args, 'InternetArchive'))
aggregated = list(
reader(args.data_file('phoible-aggregated.tsv'), namedtuples=True))
inventory_names = {}
for key, items in groupby(sorted(aggregated,
key=lambda t: (t.LanguageCode, t.Source)),
key=lambda t: (t.LanguageCode, t.Source)):
items = list(items)
lname = lnames.get(key[0])
if not lname:
lname = items[0].LanguageName
lnames[key[0]] = lname
if len(items) == 1:
inventory_names[items[0].InventoryID] = '%s (%s)' % (lname, key[1])
else:
for i, item in enumerate(items):
inventory_names[item.InventoryID] = '%s %s (%s)' % (lname, i +
1, key[1])
family_map = {
("Arawakan", "arwk"): "Arawakan",
("Trans-New Guinea", "trng"): "Trans-New Guinea",
("Moklen", "anes"): "Austronesian",
("Oko", "ncon"): "Niger-Congo",
("Muniche", "saso"): "Muniche",
("Tinigua", "saso"): "Tinigua",
("Vilela", "luvi"): "Vilela",
("Ofayé", "macg"): "Kamakanan",
("Purian", "macg"): "PurianPrint",
("Mixed language", "saml"): "Mixed language",
("Tupian", "tupi"): "Tupian",
("Yuwana", "saun"): "YuwanaPrint",
}
family_code_map = {k[1]: v for k, v in family_map.items()}
for row in aggregated:
lang = data['Variety'].get(row.LanguageCode)
if not lang:
if row.LanguageFamilyGenus == 'UNCLASSIFIED':
genus = None
else:
genus_id = slug(strip_quotes(row.LanguageFamilyGenus))
genus = genera.get(genus_id)
if not genus:
genus = genera.get(row.LanguageCode)
if not genus:
#print(row.LanguageFamilyGenus, row.LanguageFamilyRoot)
family = family_map.get(
(row.LanguageFamilyGenus, row.LanguageFamilyRoot))
genus = genera[genus_id] = data.add(
models.Genus,
genus_id,
id=genus_id,
name=row.LanguageFamilyGenus,
description=family or row.LanguageFamilyRoot,
active=False,
root=row.LanguageFamilyRoot)
if not genus.root:
genus.root = row.LanguageFamilyRoot
if genus.description in family_code_map:
genus.description = family_code_map[genus.description]
if row.LanguageCode in geocoords:
coords = geocoords[row.LanguageCode]
elif row.Latitude != 'NULL' and row.Longitude != 'NULL':
coords = (float(row.Latitude), float(row.Longitude))
lang = data.add(models.Variety,
row.LanguageCode,
id=row.LanguageCode,
name=lnames[row.LanguageCode],
genus=genus,
country=strip_quotes(row.Country),
area=strip_quotes(row.Area),
latitude=coords[0],
longitude=coords[1],
jsondata=dict(inventory_id=row.InventoryID))
add_language_codes(data,
lang,
row.LanguageCode,
glottocodes=glottocodes)
contributor = data['Contributor'].get(row.Source)
if not contributor:
contributor = data.add(common.Contributor,
row.Source,
id=row.Source,
name=SOURCES[row.Source][0],
description=SOURCES[row.Source][2])
for ref in SOURCES[row.Source][1]:
DBSession.add(
models.ContributorReference(source=data['Source'][ref],
contributor=contributor))
contrib = data.add(models.Inventory,
row.InventoryID,
id=row.InventoryID,
language=lang,
source=row.Source,
source_url=source_urls.get(row.InventoryID),
internetarchive_url=ia_urls.get(row.InventoryID),
name=inventory_names[row.InventoryID],
description=row.LanguageName)
DBSession.add(
common.ContributionContributor(contribution=contrib,
contributor=contributor))
for j, squib in enumerate(squibs.get(row.InventoryID, [])):
f = common.Contribution_files(object=contrib,
id='squib-%s-%s.pdf' %
(contrib.id, j + 1),
name='Phonological squib',
description=squib,
mime_type='application/pdf')
assert f
# f.create(files_dir, file(args.data_file('phonological_squibs', src)).read())
DBSession.flush()
unknown_refs = {}
for row in reader(args.data_file('phoible-phonemes.tsv'),
namedtuples=True):
inventory = data['Inventory'][row.InventoryID]
segment = data['Segment'].get(row.Phoneme)
if not segment:
unicode_desc = [(c, unicodedata.name(c)) for c in row.Phoneme]
description = ' - '.join([t[1] for t in unicode_desc])
segment = data.add(
models.Segment,
row.Phoneme,
id=b16encode(md5(description).digest()),
name=row.Phoneme,
description=description,
equivalence_class=''.join([
t[0] for t in unicode_desc
if t[1].split()[0] not in ['COMBINING', 'MODIFIER']
]),
segment_class=row.Class,
combined_class=row.CombinedClass)
DBSession.flush()
vs = common.ValueSet(id=row.PhonemeID,
contribution=inventory,
language=inventory.language,
parameter=segment)
for ref in refs.get(row.InventoryID, []):
if ref not in data['Source']:
if ref not in unknown_refs:
print('-------', ref)
unknown_refs[ref] = 1
continue
DBSession.add(
common.ValueSetReference(source=data['Source'][ref],
valueset=vs))
DBSession.add(
common.Value(
id=row.PhonemeID,
name='%s %s' %
(row.Phoneme, data['Inventory'][row.InventoryID].name),
valueset=vs))
DBSession.flush()
for inventory_id in refs:
for ref in refs[inventory_id]:
if ref not in data['Source']:
continue
data.add(common.ContributionReference,
'%s-%s' % (inventory_id, ref),
source=data['Source'][ref],
contribution=data['Inventory'][inventory_id])
for i, row in enumerate(
reader(args.data_file('phoible-segments-features.tsv'))):
if i == 0:
features = list(map(feature_name, row))
continue
if row[0] not in data['Segment']:
# print('skipping feature vector:', row)
continue
for j, value in enumerate(row):
if j and value != '0':
DBSession.add(
common.Parameter_data(
key=features[j],
value=value,
ord=j,
object_pk=data['Segment'][row[0]].pk))
DBSession.flush()
def main(args): # pragma: no cover
ds = StructureDataset.from_metadata(DS)
data = Data()
for source in ds.sources:
data.add(common.Source, source.id, _obj=bibtex2source(source))
ext = [
Record.from_string('@' + s, lowercase=True)
for s in nfilter(BIB.split('@'))
]
for rec in ext:
if rec.id not in data['Source']:
data.add(common.Source, rec.id, _obj=bibtex2source(rec))
for contrib in ds['contributors.csv']:
o = data.add(
common.Contributor,
contrib['ID'],
id=contrib['ID'].upper(),
name=contrib['Name'],
description=contrib['Description'],
url=contrib['URL'],
jsondata={
'readme': contrib['Readme'],
'contents': contrib['Contents']
},
)
for src in contrib['Source']:
DBSession.add(
models.ContributorReference(source=data['Source'][src],
contributor=o))
dataset = data.add(
common.Dataset,
'phoible',
id='phoible',
name='PHOIBLE 2.0',
description='PHOIBLE 2.0',
publisher_name="Max Planck Institute for the Science of Human History",
publisher_place="Jena",
publisher_url="https://www.shh.mpg.de",
domain='phoible.org',
license='https://creativecommons.org/licenses/by-sa/3.0/',
contact='*****@*****.**',
jsondata={
'license_icon':
'https://i.creativecommons.org/l/by-sa/3.0/88x31.png',
'license_name':
'Creative Commons Attribution-ShareAlike 3.0 Unported License'
})
for i, (cid, name) in enumerate([
('UZ', "Steven Moran"),
('mccloy', "Daniel McCloy"),
],
start=1):
contrib = data['Contributor'].get(cid)
if not contrib:
contrib = common.Contributor(id=cid, name=name)
DBSession.add(
common.Editor(dataset=dataset, ord=i, contributor=contrib))
glottolog = Glottolog(
Path(phoible.__file__).parent.parent.parent.parent.joinpath(
'glottolog', 'glottolog'))
for lang in ds['LanguageTable']:
l = data.add(
models.Variety,
lang['ID'],
id=lang['ID'],
name=lang['Name'],
)
load_families(data, [(l.id, l)
for l in data['Variety'].values() if len(l.id) == 8],
glottolog.repos)
DBSession.flush()
# assign color codes:
families = defaultdict(list)
for l in data['Variety'].values():
families[l.family_pk].append(l)
colors = color.qualitative_colors(len(families))
for i, langs in enumerate(sorted(families.values(),
key=lambda v: -len(v))):
for l in langs:
l.jsondata = {'color': colors[i]}
for segment in ds['ParameterTable']:
equivalence_class = ''.join([
t[0] for t in [(c, unicodedata.name(c)) for c in segment['Name']]
if t[1].split()[0] not in ['COMBINING', 'MODIFIER']
]),
data.add(models.Segment,
segment['ID'],
id=segment['ID'],
name=segment['Name'],
description=segment['Description'],
segment_class=segment['SegmentClass'],
equivalence_class=equivalence_class)
DBSession.flush()
# Add redirects for old language pages! get relevant ISO codes and map to Glottocode!
for model, repls in load(
Path(phoible.__file__).parent.parent /
'replacements.json').items():
if model == 'Language':
languoids = {l.id: l for l in glottolog.languoids()}
iso_languoids = {l.iso: l for l in languoids.values() if l.iso}
gl_in_phoible = set(data['Variety'].keys())
for oid, nid in repls.items():
gls = descendants_from_nodemap(
iso_languoids.get(oid),
languoids).intersection(gl_in_phoible)
if gls:
nid = gls.pop()
if len(gls) > 1:
print('+++', oid, gls)
else:
print('---', oid)
common.Config.add_replacement(oid, nid, common.Language)
elif model == 'Parameter':
segments_in_phoible = set(data['Segment'].keys())
for oid, nid in repls.items():
id_ = nid if nid in segments_in_phoible else None
common.Config.add_replacement(oid, id_, common.Parameter)
for segment in ds['ParameterTable']:
for i, (k, v) in enumerate(sorted(segment.items())):
if k not in ['ID', 'Name', 'Description', 'SegmentClass']:
DBSession.add(
common.Parameter_data(
key=feature_name(k),
value=v,
ord=i,
object_pk=data['Segment'][segment['ID']].pk))
for inventory in ds['contributions.csv']:
inv = data.add(
models.Inventory,
inventory['ID'],
id=inventory['ID'],
name='{0} ({1} {2})'.format(
inventory['Name'],
inventory['Contributor_ID'].upper(),
inventory['ID'],
),
source_url=inventory['URL'],
count_tone=inventory['count_tones'],
count_vowel=inventory['count_vowels'],
count_consonant=inventory['count_consonants'],
)
DBSession.add(
common.ContributionContributor(
contribution=inv,
contributor=data['Contributor'][
inventory['Contributor_ID'].upper()]))
for src in inventory['Source']:
DBSession.add(
common.ContributionReference(contribution=inv,
source=data['Source'][src]))
for phoneme in ds['ValueTable']:
lang = data['Variety'][phoneme['Language_ID']]
inv = data['Inventory'][phoneme['Contribution_ID']]
if not inv.language:
inv.language = lang
vs = common.ValueSet(
id=phoneme['ID'],
contribution=inv,
language=lang,
parameter=data['Segment'][phoneme['Parameter_ID']])
for ref in phoneme['Source']:
DBSession.add(
common.ValueSetReference(source=data['Source'][ref],
valueset=vs))
DBSession.add(
models.Phoneme(
id=phoneme['ID'],
name='%s %s' %
(phoneme['Value'],
data['Inventory'][phoneme['Contribution_ID']].name),
allophones=' '.join(phoneme['Allophones']),
marginal=phoneme['Marginal'],
valueset=vs))
return
def setUp(self):
TestWithDb.setUp(self)
DBSession.add(
common.Dataset(id='dataset',
name='dataset',
description='desc',
domain='clld',
jsondata={'license_icon': 'cc-by'}))
DBSession.add(
common.Source(id='replaced',
active=False,
jsondata={'__replacement_id__': 'source'}))
source = common.Source(id='source')
contributors = {
'contributor': 'A Name',
'b': 'b Name',
'c': 'c Name',
'd': 'd Name'
}
for id_, name in contributors.items():
contributors[id_] = common.Contributor(id=id_,
name=name,
url='http://example.org')
contribution = common.Contribution(id='contribution',
name='Contribution')
common.ContributionReference(contribution=contribution, source=source)
assert common.ContributionContributor(
contribution=contribution,
primary=True,
contributor=contributors['contributor'])
assert common.ContributionContributor(contribution=contribution,
primary=False,
contributor=contributors['b'])
assert common.ContributionContributor(contribution=contribution,
primary=True,
contributor=contributors['c'])
assert common.ContributionContributor(contribution=contribution,
primary=False,
contributor=contributors['d'])
DBSession.add(contribution)
language = common.Language(id='language',
name='Language 1',
latitude=10.5,
longitude=0.3)
language.sources.append(source)
for i, type_ in enumerate(common.IdentifierType):
id_ = common.Identifier(type=type_.value,
id=type_.value + str(i),
name='abc')
common.LanguageIdentifier(language=language, identifier=id_)
for i in range(2, 102):
_l = common.Language(id='l%s' % i, name='Language %s' % i)
_i = common.Identifier(type='iso639-3',
id='%.3i' % i,
name='%.3i' % i)
common.LanguageIdentifier(language=_l, identifier=_i)
DBSession.add(_l)
param = common.Parameter(id='parameter', name='Parameter')
de = common.DomainElement(id='de',
name='DomainElement',
parameter=param)
de2 = common.DomainElement(id='de2',
name='DomainElement2',
parameter=param)
valueset = common.ValueSet(id='valueset',
language=language,
parameter=param,
contribution=contribution)
value = common.Value(id='value',
domainelement=de,
valueset=valueset,
frequency=50,
confidence='high')
DBSession.add(value)
value2 = common.Value(id='value2',
domainelement=de2,
valueset=valueset,
frequency=50,
confidence='high')
DBSession.add(value2)
paramnd = common.Parameter(id='no-domain',
name='Parameter without domain')
valueset = common.ValueSet(id='vs2',
language=language,
parameter=paramnd,
contribution=contribution)
common.ValueSetReference(valueset=valueset, source=source)
value = common.Value(id='v2',
valueset=valueset,
frequency=50,
confidence='high')
DBSession.add(value)
unit = common.Unit(id='unit', name='Unit', language=language)
up = common.UnitParameter(id='unitparameter', name='UnitParameter')
DBSession.add(unit)
DBSession.add(
common.UnitValue(id='unitvalue',
name='UnitValue',
unit=unit,
unitparameter=up))
up2 = common.UnitParameter(id='up2', name='UnitParameter with domain')
de = common.UnitDomainElement(id='de', name='de', parameter=up2)
DBSession.add(
common.UnitValue(id='uv2',
name='UnitValue2',
unit=unit,
unitparameter=up2,
unitdomainelement=de))
DBSession.add(common.Source(id='s'))
sentence = common.Sentence(
id='sentence',
name='sentence name',
description='sentence description',
analyzed='a\tmorpheme\tdoes\tdo',
gloss='a\tmorpheme\t1SG\tdo.SG2',
source='own',
comment='comment',
original_script='a morpheme',
language=language,
jsondata={'alt_translation': 'Spanish: ...'})
common.SentenceReference(sentence=sentence, source=source)
DBSession.add(common.Config(key='key', value='value'))
common.Config.add_replacement('replaced',
'language',
model=common.Language)
common.Config.add_replacement('gone', None, model=common.Language)
DBSession.flush()
