def main(args):
data = Data()
glottocodes, bibtex_keys = {}, defaultdict(set)
for d in reader(
args.data_file('repos', 'mappings',
'InventoryID-ISO-gcode-Bibkey-Source.tsv')):
glottocodes[d['InventoryID']] = d['Glottocode']
bibtex_keys[d['InventoryID']].add(d['BibtexKey'])
glottolog = Glottolog(
Path(phoible.__file__).parent.parent.parent.parent.joinpath(
'glottolog3', 'glottolog'))
languoids = {l.id: l for l in glottolog.languoids()}
phonemes = sorted(list(
reader(args.data_file('repos', 'data', 'phoible-by-phoneme.tsv'))),
key=lambda r: (r['InventoryID'], r['GlyphID']))
inventories = defaultdict(set)
for p in phonemes:
if p['InventoryID'] in glottocodes:
inventories[(languoids[glottocodes[p['InventoryID']]].name,
p['SpecificDialect'], p['Source'].upper())].add(
(p['InventoryID'], p['LanguageName']))
inventory_names = {}
for (glname, dname, source), invids in inventories.items():
if len(invids) == 1:
invid, lname = invids.pop()
inventory_names[invid] = name_in_source(glname,
dname) + ' [%s]' % source
else:
use_lname = len(set(r[1] for r in invids)) == len(invids)
for i, (invid,
lname) in enumerate(sorted(invids,
key=lambda j: int(j[0]))):
disambiguation = ' %s' % (i + 1, )
if use_lname:
disambiguation = ' (%s)' % lname
inventory_names[invid] = name_in_source(
glname, dname) + '%s [%s]' % (disambiguation, source)
for (invid, lname, dname, source), ps in groupby(
phonemes, lambda p: (p['InventoryID'], p['LanguageName'], p[
'SpecificDialect'], p['Source'])):
if invid not in glottocodes:
continue
ps = list(ps)
gc = glottocodes[invid]
lang = data['Variety'].get(gc)
if not lang:
languoid = languoids[gc]
lang = data.add(
models.Variety,
gc,
id=gc,
language_code=ps[0]['LanguageCode'],
name=languoid.name,
level=text_type(languoid.level.name),
latitude=languoid.latitude,
longitude=languoid.longitude,
)
if lang.latitude is None and languoid.level == Level.dialect:
ll = get_language(languoid)
lang.latitude = ll.latitude
lang.longitude = ll.longitude
contrib = data.add(
models.Inventory,
invid,
id=invid,
#language=lang,
source=source,
#source_url=source_urls.get(row.InventoryID),
#internetarchive_url=ia_urls.get(row.InventoryID),
name=inventory_names[invid],
description=name_in_source(lname, dname))
return
# FIXME: read from mappings file!
refs = defaultdict(list)
for row in get_rows(args, 'BibtexKey'):
if row[1] == 'NO SOURCE GIVEN':
refs[row[0]] = []
else:
refs[row[0]].append(row[1])
add_sources(args, data)
dataset = data.add(
common.Dataset,
'phoible',
id='phoible',
name='PHOIBLE Online',
description='PHOIBLE Online',
publisher_name="Max Planck Institute for the Science of Human History",
publisher_place="Jena",
publisher_url="https://www.shh.mpg.de",
domain='phoible.org',
license='http://creativecommons.org/licenses/by-sa/3.0/',
contact='*****@*****.**',
jsondata={
'license_icon':
'http://i.creativecommons.org/l/by-sa/3.0/88x31.png',
'license_name':
'Creative Commons Attribution-ShareAlike 3.0 Unported License'
})
for i, spec in enumerate([
('moran', "Steven Moran"),
('mccloy', "Daniel McCloy"),
('wright', "Richard Wright"),
]):
DBSession.add(
common.Editor(dataset=dataset,
ord=i + 1,
contributor=common.Contributor(id=spec[0],
name=spec[1])))
#squibs = defaultdict(list)
#for row in get_rows(args, 'Squib'):
# squibs[row[0]].append(row[1])
source_urls = dict(get_rows(args, 'URL'))
ia_urls = dict(get_rows(args, 'InternetArchive'))
# FIXME: group phoible-by-phoneme by LanguageCode, Source (make sure this is unique!)
aggregated = list(
reader(args.data_file('phoible-aggregated.tsv'),
delimiter='\t',
namedtuples=True))
inventory_names = {}
for key, items in groupby(sorted(aggregated,
key=lambda t: (t.LanguageCode, t.Source)),
key=lambda t: (t.LanguageCode, t.Source)):
items = list(items)
lname = lnames.get(key[0])
if not lname:
lname = items[0].LanguageName
lnames[key[0]] = lname
if len(items) == 1:
inventory_names[items[0].InventoryID] = '%s (%s)' % (lname, key[1])
else:
for i, item in enumerate(items):
inventory_names[item.InventoryID] = '%s %s (%s)' % (lname, i +
1, key[1])
# pull in Glottolog families instead? or in addition?
family_map = {
("Arawakan", "arwk"): "Arawakan",
("Trans-New Guinea", "trng"): "Trans-New Guinea",
("Moklen", "anes"): "Austronesian",
("Oko", "ncon"): "Niger-Congo",
("Muniche", "saso"): "Muniche",
("Tinigua", "saso"): "Tinigua",
("Vilela", "luvi"): "Vilela",
("Ofayé", "macg"): "Kamakanan",
("Purian", "macg"): "PurianPrint",
("Mixed language", "saml"): "Mixed language",
("Tupian", "tupi"): "Tupian",
("Yuwana", "saun"): "YuwanaPrint",
}
family_code_map = {k[1]: v for k, v in family_map.items()}
for row in aggregated:
lang = data['Variety'].get(row.LanguageCode)
if not lang:
if row.LanguageFamilyGenus == 'UNCLASSIFIED':
genus = None
else:
genus_id = slug(strip_quotes(row.LanguageFamilyGenus))
genus = genera.get(genus_id)
if not genus:
genus = genera.get(row.LanguageCode)
if not genus:
#print(row.LanguageFamilyGenus, row.LanguageFamilyRoot)
family = family_map.get(
(row.LanguageFamilyGenus, row.LanguageFamilyRoot))
genus = genera[genus_id] = data.add(
models.Genus,
genus_id,
id=genus_id,
name=row.LanguageFamilyGenus,
description=family or row.LanguageFamilyRoot,
active=False,
root=row.LanguageFamilyRoot)
if not genus.root:
genus.root = row.LanguageFamilyRoot
if genus.description in family_code_map:
genus.description = family_code_map[genus.description]
if row.LanguageCode in geocoords:
coords = geocoords[row.LanguageCode]
elif row.Latitude != 'NULL' and row.Longitude != 'NULL':
coords = (float(row.Latitude), float(row.Longitude))
lang = data.add(models.Variety,
row.LanguageCode,
id=row.LanguageCode,
name=lnames[row.LanguageCode],
genus=genus,
country=strip_quotes(row.Country),
area=strip_quotes(row.Area),
latitude=coords[0],
longitude=coords[1],
jsondata=dict(inventory_id=row.InventoryID))
add_language_codes(data,
lang,
row.LanguageCode,
glottocodes=glottocodes)
contributor = data['Contributor'].get(row.Source)
if not contributor:
contributor = data.add(common.Contributor,
row.Source,
id=row.Source,
name=SOURCES[row.Source][0],
description=SOURCES[row.Source][2])
for ref in SOURCES[row.Source][1]:
DBSession.add(
models.ContributorReference(source=data['Source'][ref],
contributor=contributor))
contrib = data.add(models.Inventory,
row.InventoryID,
id=row.InventoryID,
language=lang,
source=row.Source,
source_url=source_urls.get(row.InventoryID),
internetarchive_url=ia_urls.get(row.InventoryID),
name=inventory_names[row.InventoryID],
description=row.LanguageName)
DBSession.add(
common.ContributionContributor(contribution=contrib,
contributor=contributor))
#for j, squib in enumerate(squibs.get(row.InventoryID, [])):
# f = common.Contribution_files(
# object=contrib,
# id='squib-%s-%s.pdf' % (contrib.id, j + 1),
# name='Phonological squib',
# description=squib,
# mime_type='application/pdf')
# assert f
# # f.create(files_dir, file(args.data_file('phonological_squibs', src)).read())
DBSession.flush()
unknown_refs = {}
for row in reader(args.data_file('phoible-phonemes.tsv'),
namedtuples=True):
inventory = data['Inventory'][row.InventoryID]
segment = data['Segment'].get(row.Phoneme)
if not segment:
unicode_desc = [(c, unicodedata.name(c)) for c in row.Phoneme]
description = ' - '.join([t[1] for t in unicode_desc])
segment = data.add(
models.Segment,
row.Phoneme,
id=b16encode(md5(description).digest()),
name=row.Phoneme,
description=description,
equivalence_class=''.join([
t[0] for t in unicode_desc
if t[1].split()[0] not in ['COMBINING', 'MODIFIER']
]),
segment_class=row.Class,
combined_class=row.CombinedClass)
DBSession.flush()
vs = common.ValueSet(id=row.PhonemeID,
contribution=inventory,
language=inventory.language,
parameter=segment)
for ref in refs.get(row.InventoryID, []):
if ref not in data['Source']:
if ref not in unknown_refs:
print('-------', ref)
unknown_refs[ref] = 1
continue
DBSession.add(
common.ValueSetReference(source=data['Source'][ref],
valueset=vs))
DBSession.add(
common.Value(
id=row.PhonemeID,
name='%s %s' %
(row.Phoneme, data['Inventory'][row.InventoryID].name),
valueset=vs))
DBSession.flush()
for inventory_id in refs:
for ref in refs[inventory_id]:
if ref not in data['Source']:
continue
data.add(common.ContributionReference,
'%s-%s' % (inventory_id, ref),
source=data['Source'][ref],
contribution=data['Inventory'][inventory_id])
for i, row in enumerate(
reader(args.data_file('phoible-segments-features.tsv'))):
if i == 0:
features = list(map(feature_name, row))
continue
if row[0] not in data['Segment']:
# print('skipping feature vector:', row)
continue
for j, value in enumerate(row):
if j and value != '0':
DBSession.add(
common.Parameter_data(
key=features[j],
value=value,
ord=j,
object_pk=data['Segment'][row[0]].pk))
# FIXME: add allophones!
DBSession.flush()
def main(args):
data = Data()
data_path = lambda *cs: args.data_file('concepticon-data',
'concepticondata', *cs)
dataset = common.Dataset(
id=concepticon.__name__,
name="Concepticon 1.0",
publisher_name="Max Planck Institute for the Science of Human History",
publisher_place="Jena",
publisher_url="http://www.shh.mpg.de",
license="http://creativecommons.org/licenses/by/4.0/",
contact='*****@*****.**',
domain='concepticon.clld.org',
jsondata={
'license_icon':
'cc-by.png',
'license_name':
'Creative Commons Attribution 4.0 International License'
})
DBSession.add(dataset)
for i, name in enumerate(
['Johann-Mattis List', 'Michael Cysouw', 'Robert Forkel']):
c = common.Contributor(id=slug(name), name=name)
dataset.editors.append(common.Editor(contributor=c, ord=i))
english = data.add(common.Language, 'eng', id='eng', name='English')
files = {}
for fname in data_path('sources').iterdir():
files[fname.stem] = \
"https://github.com/clld/concepticon-data/blob/master/concepticondata/sources/%s" % fname.name
for rec in Database.from_file(data_path('references', 'references.bib'),
lowercase=True):
source = data.add(common.Source, rec.id, _obj=bibtex2source(rec))
if rec.id in files:
DBSession.flush()
DBSession.add(
common.Source_files(mime_type='application/pdf',
object_pk=source.pk,
jsondata=dict(url=files[rec.id])))
for concept in reader(data_path('concepticon.tsv'), namedtuples=True):
data.add(models.ConceptSet,
concept.ID,
id=concept.ID,
name=concept.GLOSS,
description=concept.DEFINITION,
semanticfield=concept.SEMANTICFIELD,
ontological_category=concept.ONTOLOGICAL_CATEGORY)
for rel in reader(data_path('conceptrelations.tsv'), namedtuples=True):
DBSession.add(
models.Relation(source=data['ConceptSet'][rel.SOURCE],
target=data['ConceptSet'][rel.TARGET],
description=rel.RELATION))
unmapped = Counter()
number_pattern = re.compile('(?P<number>[0-9]+)(?P<suffix>.*)')
for cl in reader(data_path('conceptlists.tsv'), dicts=True):
concepts = data_path('conceptlists', '%(ID)s.tsv' % cl)
if not concepts.exists():
continue
langs = [l.lower() for l in split(cl['SOURCE_LANGUAGE'])]
conceptlist = data.add(
models.Conceptlist,
cl['ID'],
id=cl['ID'],
name=' '.join(cl['ID'].split('-')),
description=cl['NOTE'],
target_languages=cl['TARGET_LANGUAGE'],
source_languages=' '.join(langs),
year=int(cl['YEAR']) if cl['YEAR'] else None,
)
for id_ in split(cl['REFS']):
common.ContributionReference(source=data['Source'][id_],
contribution=conceptlist)
for i, name in enumerate(split(cl['AUTHOR'], sep=' and ')):
name = strip_braces(name)
contrib = data['Contributor'].get(name)
if not contrib:
contrib = data.add(common.Contributor,
name,
id=slug(name),
name=name)
DBSession.add(
common.ContributionContributor(ord=i,
contribution=conceptlist,
contributor=contrib))
for k in 'ID NOTE TARGET_LANGUAGE SOURCE_LANGUAGE YEAR REFS AUTHOR'.split(
):
del cl[k]
DBSession.flush()
for k, v in cl.items():
DBSession.add(
common.Contribution_data(object_pk=conceptlist.pk,
key=k,
value=v))
for concept in reader(concepts, namedtuples=True):
if not concept.ID or not concept.CONCEPTICON_ID or concept.CONCEPTICON_ID == 'NAN':
#print conceptlist.id, getattr(concept, 'ENGLISH', getattr(concept, 'GLOSS', None))
unmapped.update([conceptlist.id])
continue
lgs = {}
for lang in langs:
v = getattr(concept, lang.upper())
if v:
lgs[lang] = v
match = number_pattern.match(concept.NUMBER)
if not match:
print(concept.ID)
raise ValueError
vs = common.ValueSet(
id=concept.ID,
description=getattr(concept, 'GLOSS',
getattr(concept, 'ENGLISH', None)),
language=english,
contribution=conceptlist,
parameter=data['ConceptSet'][concept.CONCEPTICON_ID])
d = {}
for key, value in concept.__dict__.items():
if not key.startswith('CONCEPTICON_') and \
key not in ['NUMBER', 'ID', 'GLOSS'] + [l.upper() for l in langs]:
d[key.lower()] = value
v = models.Concept(
id=concept.ID,
valueset=vs,
description=getattr(concept, 'GLOSS',
None), # our own gloss, if available
name='; '.join('%s [%s]' % (lgs[l], l)
for l in sorted(lgs.keys())),
number=int(match.group('number')),
number_suffix=match.group('suffix'),
jsondata=d)
DBSession.flush()
for key, value in lgs.items():
DBSession.add(
common.Value_data(key='lang_' + key,
value=value,
object_pk=v.pk))
print('Unmapped concepts:')
for clid, no in unmapped.most_common():
print(clid, no)
for fname in data_path('concept_set_meta').iterdir():
if fname.suffix == '.tsv':
md = load(fname.parent.joinpath(fname.name + '-metadata.json'))
provider = models.MetaProvider(id=fname.stem,
name=md['dc:title'],
description=md['dc:description'],
url=md['dc:source'],
jsondata=md)
for meta in reader(fname, dicts=True):
try:
for k, v in meta.items():
if v and k != 'CONCEPTICON_ID':
models.ConceptSetMeta(metaprovider=provider,
conceptset=data['ConceptSet']
[meta['CONCEPTICON_ID']],
key=k,
value=v)
except:
print(fname)
print(meta)
raise
def main(args):
# determine if we run on a machine where other databases are available for lookup
# locally:
data = Data()
genera = get_genera(data) if astroman else {}
glottocodes, lnames, geocoords = {}, {}, {}
if astroman:
for k, v in glottocodes_by_isocode(
'postgresql://robert@/glottolog3',
cols=['id', 'name', 'latitude', 'longitude']).items():
glottocodes[k] = v[0]
lnames[k] = v[1]
geocoords[k] = (v[2], v[3])
refs = defaultdict(list)
for row in get_rows(args, 'BibtexKey'):
if row[1] == 'NO SOURCE GIVEN':
refs[row[0]] = []
else:
refs[row[0]].append(row[1])
add_sources(args, data)
dataset = data.add(
common.Dataset,
'phoible',
id='phoible',
name='PHOIBLE Online',
description='PHOIBLE Online',
publisher_name="Max Planck Institute for Evolutionary Anthropology",
publisher_place="Leipzig",
publisher_url="http://www.eva.mpg.de",
domain='phoible.org',
license='http://creativecommons.org/licenses/by-sa/3.0/',
contact='*****@*****.**',
jsondata={
'license_icon':
'http://i.creativecommons.org/l/by-sa/3.0/88x31.png',
'license_name':
'Creative Commons Attribution-ShareAlike 3.0 Unported License'
})
for i, spec in enumerate([
('moran', "Steven Moran"),
('mccloy', "Daniel McCloy"),
('wright', "Richard Wright"),
]):
DBSession.add(
common.Editor(dataset=dataset,
ord=i + 1,
contributor=common.Contributor(id=spec[0],
name=spec[1])))
squibs = defaultdict(list)
for row in get_rows(args, 'Squib'):
squibs[row[0]].append(row[1])
source_urls = dict(get_rows(args, 'URL'))
ia_urls = dict(get_rows(args, 'InternetArchive'))
aggregated = list(
reader(args.data_file('phoible-aggregated.tsv'), namedtuples=True))
inventory_names = {}
for key, items in groupby(sorted(aggregated,
key=lambda t: (t.LanguageCode, t.Source)),
key=lambda t: (t.LanguageCode, t.Source)):
items = list(items)
lname = lnames.get(key[0])
if not lname:
lname = items[0].LanguageName
lnames[key[0]] = lname
if len(items) == 1:
inventory_names[items[0].InventoryID] = '%s (%s)' % (lname, key[1])
else:
for i, item in enumerate(items):
inventory_names[item.InventoryID] = '%s %s (%s)' % (lname, i +
1, key[1])
family_map = {
("Arawakan", "arwk"): "Arawakan",
("Trans-New Guinea", "trng"): "Trans-New Guinea",
("Moklen", "anes"): "Austronesian",
("Oko", "ncon"): "Niger-Congo",
("Muniche", "saso"): "Muniche",
("Tinigua", "saso"): "Tinigua",
("Vilela", "luvi"): "Vilela",
("Ofayé", "macg"): "Kamakanan",
("Purian", "macg"): "PurianPrint",
("Mixed language", "saml"): "Mixed language",
("Tupian", "tupi"): "Tupian",
("Yuwana", "saun"): "YuwanaPrint",
}
family_code_map = {k[1]: v for k, v in family_map.items()}
for row in aggregated:
lang = data['Variety'].get(row.LanguageCode)
if not lang:
if row.LanguageFamilyGenus == 'UNCLASSIFIED':
genus = None
else:
genus_id = slug(strip_quotes(row.LanguageFamilyGenus))
genus = genera.get(genus_id)
if not genus:
genus = genera.get(row.LanguageCode)
if not genus:
#print(row.LanguageFamilyGenus, row.LanguageFamilyRoot)
family = family_map.get(
(row.LanguageFamilyGenus, row.LanguageFamilyRoot))
genus = genera[genus_id] = data.add(
models.Genus,
genus_id,
id=genus_id,
name=row.LanguageFamilyGenus,
description=family or row.LanguageFamilyRoot,
active=False,
root=row.LanguageFamilyRoot)
if not genus.root:
genus.root = row.LanguageFamilyRoot
if genus.description in family_code_map:
genus.description = family_code_map[genus.description]
if row.LanguageCode in geocoords:
coords = geocoords[row.LanguageCode]
elif row.Latitude != 'NULL' and row.Longitude != 'NULL':
coords = (float(row.Latitude), float(row.Longitude))
lang = data.add(models.Variety,
row.LanguageCode,
id=row.LanguageCode,
name=lnames[row.LanguageCode],
genus=genus,
country=strip_quotes(row.Country),
area=strip_quotes(row.Area),
latitude=coords[0],
longitude=coords[1],
jsondata=dict(inventory_id=row.InventoryID))
add_language_codes(data,
lang,
row.LanguageCode,
glottocodes=glottocodes)
contributor = data['Contributor'].get(row.Source)
if not contributor:
contributor = data.add(common.Contributor,
row.Source,
id=row.Source,
name=SOURCES[row.Source][0],
description=SOURCES[row.Source][2])
for ref in SOURCES[row.Source][1]:
DBSession.add(
models.ContributorReference(source=data['Source'][ref],
contributor=contributor))
contrib = data.add(models.Inventory,
row.InventoryID,
id=row.InventoryID,
language=lang,
source=row.Source,
source_url=source_urls.get(row.InventoryID),
internetarchive_url=ia_urls.get(row.InventoryID),
name=inventory_names[row.InventoryID],
description=row.LanguageName)
DBSession.add(
common.ContributionContributor(contribution=contrib,
contributor=contributor))
for j, squib in enumerate(squibs.get(row.InventoryID, [])):
f = common.Contribution_files(object=contrib,
id='squib-%s-%s.pdf' %
(contrib.id, j + 1),
name='Phonological squib',
description=squib,
mime_type='application/pdf')
assert f
# f.create(files_dir, file(args.data_file('phonological_squibs', src)).read())
DBSession.flush()
unknown_refs = {}
for row in reader(args.data_file('phoible-phonemes.tsv'),
namedtuples=True):
inventory = data['Inventory'][row.InventoryID]
segment = data['Segment'].get(row.Phoneme)
if not segment:
unicode_desc = [(c, unicodedata.name(c)) for c in row.Phoneme]
description = ' - '.join([t[1] for t in unicode_desc])
segment = data.add(
models.Segment,
row.Phoneme,
id=b16encode(md5(description).digest()),
name=row.Phoneme,
description=description,
equivalence_class=''.join([
t[0] for t in unicode_desc
if t[1].split()[0] not in ['COMBINING', 'MODIFIER']
]),
segment_class=row.Class,
combined_class=row.CombinedClass)
DBSession.flush()
vs = common.ValueSet(id=row.PhonemeID,
contribution=inventory,
language=inventory.language,
parameter=segment)
for ref in refs.get(row.InventoryID, []):
if ref not in data['Source']:
if ref not in unknown_refs:
print('-------', ref)
unknown_refs[ref] = 1
continue
DBSession.add(
common.ValueSetReference(source=data['Source'][ref],
valueset=vs))
DBSession.add(
common.Value(
id=row.PhonemeID,
name='%s %s' %
(row.Phoneme, data['Inventory'][row.InventoryID].name),
valueset=vs))
DBSession.flush()
for inventory_id in refs:
for ref in refs[inventory_id]:
if ref not in data['Source']:
continue
data.add(common.ContributionReference,
'%s-%s' % (inventory_id, ref),
source=data['Source'][ref],
contribution=data['Inventory'][inventory_id])
for i, row in enumerate(
reader(args.data_file('phoible-segments-features.tsv'))):
if i == 0:
features = list(map(feature_name, row))
continue
if row[0] not in data['Segment']:
# print('skipping feature vector:', row)
continue
for j, value in enumerate(row):
if j and value != '0':
DBSession.add(
common.Parameter_data(
key=features[j],
value=value,
ord=j,
object_pk=data['Segment'][row[0]].pk))
DBSession.flush()
def main(args): # pragma: no cover
license = licenses.find(args.cldf.properties['dc:license'])
assert license and license.id.startswith('CC-')
clts = CLTS(
input('Path to cldf-clts/clts:') or '../../cldf-clts/clts-data')
data = Data()
ds = data.add(
common.Dataset,
vanuatuvoices.__name__,
id=vanuatuvoices.__name__,
name='Vanuatu Voices',
domain='vanuatuvoices.clld.org',
contact="*****@*****.**",
publisher_name="Max Planck Institute for Evolutionary Anthropology",
publisher_place="Leipzig",
publisher_url="https://www.eva.mpg.de",
license=license.url,
jsondata={
'license_icon':
'{}.png'.format('-'.join(
[p.lower() for p in license.id.split('-')[:-1]])),
'license_name':
license.name
},
)
form2audio = audioutil.form2audio(args.cldf, 'audio/mpeg')
r = get_dataset('vanuatuvoices', ep='lexibank.dataset')
authors, _ = r.get_creators_and_contributors()
for ord, author in enumerate(authors):
cid = slug(HumanName(author['name']).last)
img = pathlib.Path(
vanuatuvoices.__file__).parent / 'static' / '{}.jpg'.format(cid)
c = data.add(
common.Contributor,
cid,
id=cid,
name=author['name'],
description=author.get('description'),
jsondata=dict(img=img.name if img.exists() else None),
)
data.add(
common.Contributor,
'forkel',
id='forkel',
name='Robert Forkel',
description='Data curation and website implementation',
jsondata=dict(img=None),
)
for ord, cid in enumerate(['walworth', 'forkel', 'gray']):
DBSession.add(
common.Editor(ord=ord,
dataset=ds,
contributor=data['Contributor'][cid]))
contribs = collections.defaultdict(lambda: collections.defaultdict(list))
for c in args.cldf.iter_rows('contributions.csv'):
for role in ['phonetic_transcriptions', 'recording', 'sound_editing']:
for name in c[role].split(' and '):
if name:
cid = slug(HumanName(name).last)
contribs[c['Language_ID']][cid].append(role)
for lang in args.cldf.iter_rows('LanguageTable', 'id', 'glottocode',
'name', 'latitude', 'longitude'):
contrib = data.add(
common.Contribution,
lang['id'],
id=lang['id'],
name='Wordlist for {}'.format(lang['name']),
)
if lang['id'] in contribs:
for cid, roles in contribs[lang['id']].items():
DBSession.add(
common.ContributionContributor(
contribution=contrib,
contributor=data['Contributor'][cid],
jsondata=dict(roles=roles),
))
data.add(
models.Variety,
lang['id'],
id=lang['id'],
name=lang['name'],
latitude=lang['latitude'],
longitude=lang['longitude'],
glottocode=lang['glottocode'],
description=lang['LongName'],
contribution=contrib,
island=lang['Island'],
)
for rec in bibtex.Database.from_file(args.cldf.bibpath, lowercase=True):
data.add(common.Source, rec.id, _obj=bibtex2source(rec))
refs = collections.defaultdict(list)
for param in args.cldf.iter_rows('ParameterTable', 'id',
'concepticonReference', 'name'):
data.add(
models.Concept,
param['id'],
id=param['id'],
name='{} [{}]'.format(param['name'], param['id'].split('_')[0]),
description=param['Bislama_Gloss'],
concepticon_id=param['concepticonReference'],
concepticon_gloss=param['Concepticon_Gloss'],
)
inventories = collections.defaultdict(collections.Counter)
for form in args.cldf.iter_rows('FormTable', 'id', 'form',
'languageReference', 'parameterReference',
'source'):
inventories[form['languageReference']].update(form['Segments'])
vsid = (form['languageReference'], form['parameterReference'])
vs = data['ValueSet'].get(vsid)
if not vs:
vs = data.add(
common.ValueSet,
vsid,
id='-'.join(vsid),
language=data['Variety'][form['languageReference']],
parameter=data['Concept'][form['parameterReference']],
contribution=data['Contribution'][form['languageReference']],
)
for ref in form.get('source', []):
sid, pages = Sources.parse(ref)
refs[(vsid, sid)].append(pages)
data.add(Counterpart,
form['id'],
id=form['id'],
name=form['form'],
valueset=vs,
audio=form2audio.get(form['id']))
for (vsid, sid), pages in refs.items():
DBSession.add(
common.ValueSetReference(valueset=data['ValueSet'][vsid],
source=data['Source'][sid],
description='; '.join(nfilter(pages))))
for lid, inv in inventories.items():
inv = [clts.bipa[c] for c in inv]
data['Variety'][lid].update_jsondata(
inventory=[(str(c), c.name) for c in inv
if getattr(c, 'name', None)])
def main(args): # pragma: no cover
ds = StructureDataset.from_metadata(DS)
data = Data()
for source in ds.sources:
data.add(common.Source, source.id, _obj=bibtex2source(source))
ext = [
Record.from_string('@' + s, lowercase=True)
for s in nfilter(BIB.split('@'))
]
for rec in ext:
if rec.id not in data['Source']:
data.add(common.Source, rec.id, _obj=bibtex2source(rec))
for contrib in ds['contributors.csv']:
o = data.add(
common.Contributor,
contrib['ID'],
id=contrib['ID'].upper(),
name=contrib['Name'],
description=contrib['Description'],
url=contrib['URL'],
jsondata={
'readme': contrib['Readme'],
'contents': contrib['Contents']
},
)
for src in contrib['Source']:
DBSession.add(
models.ContributorReference(source=data['Source'][src],
contributor=o))
dataset = data.add(
common.Dataset,
'phoible',
id='phoible',
name='PHOIBLE 2.0',
description='PHOIBLE 2.0',
publisher_name="Max Planck Institute for the Science of Human History",
publisher_place="Jena",
publisher_url="https://www.shh.mpg.de",
domain='phoible.org',
license='https://creativecommons.org/licenses/by-sa/3.0/',
contact='*****@*****.**',
jsondata={
'license_icon':
'https://i.creativecommons.org/l/by-sa/3.0/88x31.png',
'license_name':
'Creative Commons Attribution-ShareAlike 3.0 Unported License'
})
for i, (cid, name) in enumerate([
('UZ', "Steven Moran"),
('mccloy', "Daniel McCloy"),
],
start=1):
contrib = data['Contributor'].get(cid)
if not contrib:
contrib = common.Contributor(id=cid, name=name)
DBSession.add(
common.Editor(dataset=dataset, ord=i, contributor=contrib))
glottolog = Glottolog(
Path(phoible.__file__).parent.parent.parent.parent.joinpath(
'glottolog', 'glottolog'))
for lang in ds['LanguageTable']:
l = data.add(
models.Variety,
lang['ID'],
id=lang['ID'],
name=lang['Name'],
)
load_families(data, [(l.id, l)
for l in data['Variety'].values() if len(l.id) == 8],
glottolog.repos)
DBSession.flush()
# assign color codes:
families = defaultdict(list)
for l in data['Variety'].values():
families[l.family_pk].append(l)
colors = color.qualitative_colors(len(families))
for i, langs in enumerate(sorted(families.values(),
key=lambda v: -len(v))):
for l in langs:
l.jsondata = {'color': colors[i]}
for segment in ds['ParameterTable']:
equivalence_class = ''.join([
t[0] for t in [(c, unicodedata.name(c)) for c in segment['Name']]
if t[1].split()[0] not in ['COMBINING', 'MODIFIER']
]),
data.add(models.Segment,
segment['ID'],
id=segment['ID'],
name=segment['Name'],
description=segment['Description'],
segment_class=segment['SegmentClass'],
equivalence_class=equivalence_class)
DBSession.flush()
# Add redirects for old language pages! get relevant ISO codes and map to Glottocode!
for model, repls in load(
Path(phoible.__file__).parent.parent /
'replacements.json').items():
if model == 'Language':
languoids = {l.id: l for l in glottolog.languoids()}
iso_languoids = {l.iso: l for l in languoids.values() if l.iso}
gl_in_phoible = set(data['Variety'].keys())
for oid, nid in repls.items():
gls = descendants_from_nodemap(
iso_languoids.get(oid),
languoids).intersection(gl_in_phoible)
if gls:
nid = gls.pop()
if len(gls) > 1:
print('+++', oid, gls)
else:
print('---', oid)
common.Config.add_replacement(oid, nid, common.Language)
elif model == 'Parameter':
segments_in_phoible = set(data['Segment'].keys())
for oid, nid in repls.items():
id_ = nid if nid in segments_in_phoible else None
common.Config.add_replacement(oid, id_, common.Parameter)
for segment in ds['ParameterTable']:
for i, (k, v) in enumerate(sorted(segment.items())):
if k not in ['ID', 'Name', 'Description', 'SegmentClass']:
DBSession.add(
common.Parameter_data(
key=feature_name(k),
value=v,
ord=i,
object_pk=data['Segment'][segment['ID']].pk))
for inventory in ds['contributions.csv']:
inv = data.add(
models.Inventory,
inventory['ID'],
id=inventory['ID'],
name='{0} ({1} {2})'.format(
inventory['Name'],
inventory['Contributor_ID'].upper(),
inventory['ID'],
),
source_url=inventory['URL'],
count_tone=inventory['count_tones'],
count_vowel=inventory['count_vowels'],
count_consonant=inventory['count_consonants'],
)
DBSession.add(
common.ContributionContributor(
contribution=inv,
contributor=data['Contributor'][
inventory['Contributor_ID'].upper()]))
for src in inventory['Source']:
DBSession.add(
common.ContributionReference(contribution=inv,
source=data['Source'][src]))
for phoneme in ds['ValueTable']:
lang = data['Variety'][phoneme['Language_ID']]
inv = data['Inventory'][phoneme['Contribution_ID']]
if not inv.language:
inv.language = lang
vs = common.ValueSet(
id=phoneme['ID'],
contribution=inv,
language=lang,
parameter=data['Segment'][phoneme['Parameter_ID']])
for ref in phoneme['Source']:
DBSession.add(
common.ValueSetReference(source=data['Source'][ref],
valueset=vs))
DBSession.add(
models.Phoneme(
id=phoneme['ID'],
name='%s %s' %
(phoneme['Value'],
data['Inventory'][phoneme['Contribution_ID']].name),
allophones=' '.join(phoneme['Allophones']),
marginal=phoneme['Marginal'],
valueset=vs))
return
def main(args):
assert args.glottolog, 'The --glottolog option is required!'
license = licenses.find(args.cldf.properties['dc:license'])
assert license and license.id.startswith('CC-')
data = Data()
ds = data.add(
common.Dataset,
mixezoqueanvoices.__name__,
id=mixezoqueanvoices.__name__,
name="Mixe-Zoquean Voices",
domain='mixezoqueanvoices.clld.org',
publisher_name="Max Planck Institute for Evolutionary Anthropology",
publisher_place="Leipzig",
publisher_url="http://www.eva.mpg.de",
license=license.url,
jsondata={
'license_icon':
'{}.png'.format('-'.join(
[p.lower() for p in license.id.split('-')[:-1]])),
'license_name':
license.name
},
)
contrib = data.add(
common.Contribution,
None,
id='cldf',
name=args.cldf.properties.get('dc:title'),
description=args.cldf.properties.get('dc:bibliographicCitation'),
)
data.add(common.Contributor, 'kondic', id='kondic', name='Ana Kondic')
data.add(common.Contributor, 'gray', id='gray', name='Russell Gray')
DBSession.add(
common.ContributionContributor(
contribution=contrib,
contributor=data['Contributor']['kondic'],
))
for i, ed in enumerate(['kondic', 'gray']):
data.add(common.Editor,
ed,
dataset=ds,
contributor=data['Contributor'][ed],
ord=i)
ancestors = collections.defaultdict(list)
gl = Glottolog(args.glottolog)
lnames = {}
for lang in args.cldf.iter_rows('LanguageTable', 'id', 'glottocode',
'name', 'latitude', 'longitude'):
lnames[lang['id']] = lang['name']
glang = None
if lang['glottocode']:
glang = gl.languoid(lang['glottocode'])
lineage = [i[0] for i in glang.lineage]
if 'Mixe-Zoque' in lineage:
ancestors[lang['id']].append('Protomixezoque')
if 'Mixe' in lineage:
ancestors[lang['id']].append('Protomixe')
if 'Oaxaca Mixe' in lineage:
ancestors[lang['id']].append('Protooaxacamixe')
if not glang:
assert lang['name'] == 'Nizaviguiti'
data.add(
models.Variety,
lang['id'],
id=lang['id'],
name=lang['name'],
latitude=lang['latitude'],
longitude=lang['longitude'],
glottocode=lang['glottocode'],
description=lang['LongName'],
subgroup=glang.lineage[1][0]
if glang and len(glang.lineage) > 1 else None,
)
colors = dict(
zip(
set(l.subgroup for l in data['Variety'].values()),
qualitative_colors(
len(set(l.subgroup for l in data['Variety'].values())))))
for l in data['Variety'].values():
l.jsondata = dict(color=colors[l.subgroup].replace('#', ''))
for rec in bibtex.Database.from_file(args.cldf.bibpath, lowercase=True):
data.add(common.Source, rec.id, _obj=bibtex2source(rec))
refs = collections.defaultdict(list)
# Store proto-forms for later lookup:
proto_forms = collections.defaultdict(
lambda: collections.defaultdict(list))
for form in args.cldf.iter_rows('FormTable', 'id', 'form',
'languageReference', 'parameterReference'):
if form['languageReference'].startswith('Proto'):
proto_forms[form['languageReference']][
form['parameterReference']].append(form['form'])
for param in args.cldf.iter_rows('ParameterTable', 'id',
'concepticonReference', 'name'):
proto = collections.OrderedDict()
for lid, forms in proto_forms.items():
f = forms.get(param['id'])
if f:
proto[lnames[lid]] = f
data.add(
models.Concept,
param['id'],
id=param['id'],
name='{} [{}]'.format(param['name'], param['id'].split('_')[0]),
concepticon_id=param['concepticonReference'],
concepticon_gloss=param['Concepticon_Gloss'],
description=param['Spanish_Gloss'],
jsondata=dict(reconstructions=proto),
)
f2a = form2audio(args.cldf)
for form in args.cldf.iter_rows('FormTable', 'id', 'form',
'languageReference', 'parameterReference',
'source'):
assert not (form['form'] == '►' and not f2a.get(form['id']))
vsid = (form['languageReference'], form['parameterReference'])
vs = data['ValueSet'].get(vsid)
if not vs:
vs = data.add(
common.ValueSet,
vsid,
id='-'.join(vsid),
language=data['Variety'][form['languageReference']],
parameter=data['Concept'][form['parameterReference']],
contribution=contrib,
)
for ref in form.get('source', []):
sid, pages = Sources.parse(ref)
refs[(vsid, sid)].append(pages)
proto = collections.OrderedDict()
for lid in ancestors.get(form['languageReference'], []):
f = proto_forms[lid].get(form['parameterReference'])
if f:
proto[lnames[lid]] = f
data.add(
Counterpart,
form['id'],
id=form['id'],
name=form['form'],
valueset=vs,
audio=f2a.get(form['id']),
jsondata=dict(reconstructions=proto),
)
for (vsid, sid), pages in refs.items():
DBSession.add(
common.ValueSetReference(valueset=data['ValueSet'][vsid],
source=data['Source'][sid],
description='; '.join(nfilter(pages))))
def populate_test_db(engine):
set_alembic_version(engine, '58559d4eea0d')
data = TestData()
data.add_default(common.Dataset,
domain='clld',
jsondata={
'license_icon': 'cc-by',
'license_url': 'http://example.org'
})
data.add_default(common.Contributor, name='A Name', email='*****@*****.**')
for id_, name in {
'b': 'b Name',
'c': 'c Name',
'd': 'd Name',
}.items():
data.add(common.Contributor,
id_,
id=id_,
name=name,
url='http://example.org')
DBSession.add(
common.Editor(dataset=data[common.Dataset],
contributor=data[common.Contributor]))
data.add_default(common.Source)
data.add(common.Source,
'replaced',
id='replaced',
active=False,
jsondata={'__replacement_id__': 'source'})
data.add_default(common.Contribution)
common.ContributionReference(contribution=data[common.Contribution],
source=data[common.Source])
for primary, c in [(True, 'contributor'), (False, 'b'), (True, 'c'),
(False, 'd')]:
common.ContributionContributor(contribution=data[common.Contribution],
primary=primary,
contributor=data['Contributor'][c])
data.add_default(common.Language, latitude=10.5, longitude=0.3)
data[common.Language].sources.append(data[common.Source])
for i, type_ in enumerate(common.IdentifierType):
common.LanguageIdentifier(
language=data[common.Language],
identifier=common.Identifier(
type=type_.value,
id=type_.value + str(i),
name='abc' if type_.name == 'iso' else 'glot1234'))
common.LanguageIdentifier(language=data[common.Language],
identifier=common.Identifier(type='name',
id='name',
name='a'))
for i in range(2, 102):
_l = common.Language(id='l%s' % i, name='Language %s' % i)
_i = common.Identifier(type='iso639-3', id='%.3i' % i, name='abc')
common.LanguageIdentifier(language=_l, identifier=_i)
DBSession.add(_l)
param = data.add_default(common.Parameter)
de = common.DomainElement(id='de', name='DomainElement', parameter=param)
de2 = common.DomainElement(id='de2',
name='DomainElement2',
parameter=param)
valueset = data.add_default(common.ValueSet,
language=data[common.Language],
parameter=param,
contribution=data[common.Contribution])
common.ValueSetReference(valueset=valueset,
source=data[common.Source],
description='10-20')
data.add_default(common.Value,
domainelement=de,
valueset=valueset,
frequency=50,
confidence='high')
data.add(common.Value,
'value2',
id='value2',
domainelement=de2,
valueset=valueset,
frequency=50,
confidence='high')
paramnd = data.add(common.Parameter,
'no-domain',
id='no-domain',
name='Parameter without domain')
valueset = common.ValueSet(id='vs2',
language=data[common.Language],
parameter=paramnd,
contribution=data[common.Contribution])
common.ValueSetReference(valueset=valueset,
source=data[common.Source],
description='10-20')
common.Value(id='v2', valueset=valueset, frequency=50, confidence='high')
unit = data.add_default(common.Unit, language=data[common.Language])
up = data.add_default(common.UnitParameter)
common.UnitValue(id='unitvalue',
name='UnitValue',
unit=unit,
unitparameter=up)
up2 = common.UnitParameter(id='up2', name='UnitParameter with domain')
de = common.UnitDomainElement(id='de', name='de', parameter=up2)
DBSession.add(
common.UnitValue(id='uv2',
name='UnitValue2',
unit=unit,
unitparameter=up2,
unitdomainelement=de))
DBSession.add(common.Source(id='s'))
sentence = data.add_default(common.Sentence,
description='sentence description',
analyzed='a\tmorpheme\tdoes\tdo',
gloss='a\tmorpheme\t1SG\tdo.SG2',
source='own',
comment='comment',
original_script='a morpheme',
language=data[common.Language],
jsondata={'alt_translation': 'Spanish: ...'})
common.SentenceReference(sentence=sentence, source=data[common.Source])
DBSession.add(common.Config(key='key', value='value'))
common.Config.add_replacement('replaced',
'language',
model=common.Language)
common.Config.add_replacement('gone', None, model=common.Language)
DBSession.flush()
def main(args):
fts.index('fts_index', Word.fts, DBSession.bind)
DBSession.execute("CREATE EXTENSION IF NOT EXISTS unaccent WITH SCHEMA public;")
data = Data()
dataset = common.Dataset(
id=dictionaria.__name__,
name="Dictionaria",
description="The Dictionary Journal",
published=date(2017, 3, 30),
contact='*****@*****.**',
domain='dictionaria.clld.org',
publisher_name="Max Planck Institute for the Science of Human History",
publisher_place="Jena",
publisher_url="https://shh.mpg.de",
license="http://creativecommons.org/licenses/by/4.0/",
jsondata={
'license_icon': 'cc-by.png',
'license_name': 'Creative Commons Attribution 4.0 International License'})
for i, (id_, name) in enumerate([
('haspelmathmartin', 'Martin Haspelmath'),
('moselulrike', 'Ulrike Mosel'),
('stiebelsbarbara', 'Barbara Stiebels')
]):
ed = data.add(common.Contributor, id_, id=id_, name=name)
common.Editor(dataset=dataset, contributor=ed, ord=i + 1)
DBSession.add(dataset)
for id_, name in LGR_ABBRS.items():
DBSession.add(common.GlossAbbreviation(id=id_, name=name))
comparison_meanings = {}
print('loading concepts ...')
glosses = set()
concepticon = Concepticon(
REPOS.joinpath('..', '..', 'concepticon', 'concepticon-data'))
if not args.no_concepts:
for conceptset in concepticon.conceptsets.values():
if conceptset.gloss in glosses:
continue
glosses.add(conceptset.gloss)
cm = data.add(
ComparisonMeaning,
conceptset.id,
id=conceptset.id,
name=conceptset.gloss.lower(),
description=conceptset.definition,
concepticon_url='http://concepticon.clld.org/parameters/%s' % conceptset.id)
comparison_meanings[cm.id] = cm
DBSession.flush()
print('... done')
comparison_meanings = {k: v.pk for k, v in comparison_meanings.items()}
submissions = []
for submission in REPOS.joinpath(
'submissions-internal' if args.internal else 'submissions').glob('*'):
if not submission.is_dir():
continue
try:
submission = Submission(submission)
except ValueError:
continue
md = submission.md
if md is None:
continue
if not md['date_published']:
continue
id_ = submission.id
if args.dict and args.dict != id_ and args.dict != 'all':
continue
lmd = md['language']
props = md.get('properties', {})
props.setdefault('custom_fields', [])
props['metalanguage_styles'] = {}
for v, s in zip(props.get('metalanguages', {}).values(),
['success', 'info', 'warning', 'important']):
props['metalanguage_styles'][v] = s
props['custom_fields'] = ['lang-' + f if f in props['metalanguage_styles'] else f
for f in props['custom_fields']]
language = data['Variety'].get(lmd['glottocode'])
if not language:
language = data.add(
Variety, lmd['glottocode'], id=lmd['glottocode'], name=lmd['name'])
md['date_published'] = md['date_published'] or date.today().isoformat()
if '-' not in md['date_published']:
md['date_published'] = md['date_published'] + '-01-01'
dictionary = data.add(
Dictionary,
id_,
id=id_,
number=md.get('number'),
name=props.get('title', lmd['name'] + ' dictionary'),
description=submission.description,
language=language,
published=date(*map(int, md['date_published'].split('-'))),
jsondata=props)
for i, spec in enumerate(md['authors']):
if not isinstance(spec, dict):
cname, address = spec, None
spec = {}
else:
cname, address = spec['name'], spec.get('affiliation')
name = HumanName(cname)
cid = slug('%s%s' % (name.last, name.first))
contrib = data['Contributor'].get(cid)
if not contrib:
contrib = data.add(
common.Contributor,
cid,
id=cid,
name=cname,
address=address,
url=spec.get('url'),
email=spec.get('email'))
DBSession.add(common.ContributionContributor(
ord=i + 1,
primary=True,
contributor=contrib,
contribution=dictionary))
submissions.append((dictionary.id, language.id, submission))
transaction.commit()
for did, lid, submission in submissions:
#if submission.id != 'sidaama':
# continue
transaction.begin()
print('loading %s ...' % submission.id)
dictdata = Data()
lang = Variety.get(lid)
submission.load_examples(Dictionary.get(did), dictdata, lang)
submission.dictionary.load(
submission,
dictdata,
Dictionary.get(did),
lang,
comparison_meanings,
OrderedDict(submission.md.get('properties', {}).get('labels', [])))
transaction.commit()
print('... done')
transaction.begin()
load_families(
Data(),
[v for v in DBSession.query(Variety) if re.match('[a-z]{4}[0-9]{4}', v.id)],
glottolog_repos='../../glottolog3/glottolog')
def main(args):
data = Data()
dataset = common.Dataset(
id=abvd.__name__,
name='ABVD',
description='',
domain='abvd.clld.org',
published=date.today(),
license='https://creativecommons.org/licenses/by/4.0/',
contact='',
jsondata={
'doi': args.doi,
'license_icon': 'cc-by.png',
'license_name': 'Creative Commons Attribution 4.0 International License'})
DBSession.add(dataset)
for name in ['Simon Greenhill', 'Robert Blust', 'Russell Gray']:
common.Editor(contributor=contributor(data, name), dataset=dataset)
cnames = Counter()
families = Counter([l['Family'] for l in args.cldf['LanguageTable']])
colors = dict(
zip([i[0] for i in families.most_common()], color.qualitative_colors(len(families))))
cid2l = {}
for lang in args.cldf['LanguageTable']:
lid = (lang['Name'], lang['Glottocode'])
l = data['Language'].get(lid)
if not l:
l = data.add(
common.Language, lid,
id=lang['ID'],
name=lang['Name'],
latitude=lang['Latitude'],
longitude=lang['Longitude'],
jsondata=dict(
family=lang['Family'],
icon='{0}{1}'.format('c' if lang['Family'] else 't', colors[lang['Family']]),
),
)
if lang['Glottocode'] or lang['ISO639P3code']:
add_language_codes(
data, l, isocode=lang['ISO639P3code'], glottocode=lang['Glottocode'])
cid2l[lang['ID']] = l
cname = '{0} ({1})'.format(lang['Name'], lang['author'])
cnames.update([cname])
if cnames[cname] > 1:
cname += ' {0}'.format(cnames[cname])
c = data.add(
models.Wordlist, lang['ID'],
id=lang['ID'],
name=cname,
description=lang['author'],
language=l,
notes=lang['notes'],
)
i = 0
typers = (lang['typedby'] or '').split(' and ')
checkers = (lang['checkedby'] or '').split(' and ')
for name in typers:
i += 1
DBSession.add(common.ContributionContributor(
contribution=c,
contributor=contributor(data, name),
ord=i,
jsondata=dict(type='typedby and checkedby' if name in checkers else 'typedby'),
))
for name in checkers:
if name in typers:
continue
i += 1
DBSession.add(common.ContributionContributor(
contribution=c,
contributor=contributor(data, name),
ord=i,
jsondata=dict(type='checkedby'),
))
for param in args.cldf['ParameterTable']:
data.add(
common.Parameter, param['ID'],
id=param['ID'],
name=param['Name'],
)
#
# FIXME: add sources!
#
vsrs = set()
for row in args.cldf['FormTable']:
vs = data['ValueSet'].get((row['Language_ID'], row['Parameter_ID']))
if not vs:
vs = data.add(
common.ValueSet,
(row['Language_ID'], row['Parameter_ID']),
id='{0}-{1}'.format(row['Language_ID'], row['Parameter_ID']),
language=cid2l[row['Language_ID']],
parameter=data['Parameter'][row['Parameter_ID']],
contribution=data['Wordlist'][row['Language_ID']],
)
v = data.add(
common.Value,
row['ID'],
id=row['ID'],
name=row['Form'],
valueset=vs
)
for row in args.cldf['CognateTable']:
cc = data['Cognateset'].get(row['Cognateset_ID'])
if not cc:
cc = data.add(Cognateset, row['Cognateset_ID'], id=row['Cognateset_ID'])
data.add(
Cognate,
row['ID'],
cognateset=cc,
counterpart=data['Value'][row['Form_ID']],
doubt=row['Doubt'],
)
def main(args):
meta = parse_meta(args)
print(len(meta))
print(sum(len(m.sources) for m in meta.values()))
sources = {}
for m in meta.values():
for s in m.sources:
sources[s] = None
print(len(sources), 'distinct')
for i, s in enumerate(sources):
sources[s] = get_source(s, i + 1)
glottocodes = glottocodes_by_isocode('postgresql://robert@/glottolog3')
data = Data()
wals = create_engine('postgresql://robert@/wals3')
wals_families = {}
for row in wals.execute('select name, id from family'):
wals_families[row[0]] = row[1]
wals_families[row[1]] = row[1]
#for item in reader(args.data_file('WALSFamilyAbbreviations.tab'), namedtuples=True, encoding='latin1'):
# name = item.FAMILY
# if name not in wals_families:
# name = slug(name)
# if name not in wals_families:
# print('missing wals family:', item.FAMILY)
# name = None
# if name:
# wals_families[item.ABBREVIATION] = wals_families[name]
wals_genera = {
row[0]: row[0]
for row in wals.execute('select id from genus')
}
with args.data_file('listss17.txt').open(encoding='latin1') as fp:
wordlists = ['\n'.join(lines) for lines in parse(fp)]
dataset = common.Dataset(
id=asjp.__name__,
name="The ASJP Database",
contact="*****@*****.**",
description="The Automated Similarity Judgment Program",
domain='asjp.clld.org',
publisher_name="Max Planck Institute for the Science of Human History",
publisher_place="Jena",
publisher_url="https://www.shh.mpg.de",
license='http://creativecommons.org/licenses/by/4.0/',
jsondata={
'license_icon':
'cc-by.png',
'license_name':
'Creative Commons Attribution 4.0 International License'
})
DBSession.add(dataset)
transcribers = get_transcriber_map(args)
for i, spec in enumerate([
('SW', "Søren Wichmann"),
('AM', "André Müller"),
('AKW', "Annkathrin Wett"),
('VV', "Viveka Velupillai"),
('JB', "Julia Bischoffberger"),
('CB', "Cecil H. Brown"),
('EH', "Eric W. Holman"),
('SS', "Sebastian Sauppe"),
('ZM', "Zarina Molochieva"),
('PB', "Pamela Brown"),
('HH', "Harald Hammarström"),
('OB', "Oleg Belyaev"),
('JML', "Johann-Mattis List"),
('DBA', "Dik Bakker"),
('DE', "Dmitry Egorov"),
('MU', "Matthias Urban"),
('RM', "Robert Mailhammer"),
('AC', "Agustina Carrizo"),
('MSD', "Matthew S. Dryer"),
('EK', "Evgenia Korovina"),
('DB', "David Beck"),
('HG', "Helen Geyer"),
('PE', "Patience Epps"),
('AG', "Anthony Grant"),
('PS', "Paul Sidwell"), # not in citation
('KTR', "K. Taraka Rama"), # not in citation
('PV', "Pilar Valenzuela"),
('MD', "Mark Donohue"), # not in citation
]):
id_, name = spec
if id_ in transcribers:
assert name == transcribers.pop(id_)
contributor = data.add(common.Contributor, id_, id=id_, name=name)
if id_ in ['SW', 'CB', 'EH']:
DBSession.add(
common.Editor(dataset=dataset,
ord=i + 1,
contributor=contributor))
for id_, name in transcribers.items():
data.add(common.Contributor, id_, id=id_, name=name)
for id_ in sorted(models.MEANINGS_ALL.keys()):
data.add(models.Meaning,
id_,
id=str(id_),
name=models.MEANINGS_ALL[id_],
core=id_ in models.MEANINGS)
for n, l in enumerate(wordlists):
#if n > 100:
# break
lang = models.Doculect.from_txt(l)
if lang.classification_wals:
family, genus = lang.classification_wals.split('.')
lang.wals_family = wals_families.get(family)
lang.wals_genus = wals_genera.get(slug(genus))
lang.code_glottolog = glottocodes.get(lang.code_iso)
add_codes(lang)
data.add(models.Doculect, lang.id, _obj=lang)
DBSession.flush()
md = meta.pop(lang.id, None)
assert md
# associate transcribers and sources
for i, transcriber in enumerate(md.transcribers):
common.ContributionContributor(
contribution=lang.wordlist,
contributor=data['Contributor'][transcriber],
ord=i + 1)
for source in md.sources:
DBSession.add(
common.LanguageSource(language_pk=lang.pk,
source_pk=sources[source].pk))
assert not list(meta.keys())
def main(args):
for (org, repos), recs in itertools.groupby(
sorted(oai.Records('tular'),
key=lambda r: (r.repos.org, r.repos.repos, r.version),
reverse=True),
lambda r: (r.repos.org, r.repos.repos),
):
if org == 'tupian-language-resources' and repos in DATASETS:
DATASETS[repos] = next(recs)
data = Data()
dataset = data.add(
common.Dataset,
'tular',
id=tular.__name__,
domain="tular.clld.org",
name="TuLaR",
description="Tupían Language Resources",
publisher_name="Max-Planck Institute for Evolutionary Anthropology",
publisher_place="Leipzig",
license='https://creativecommons.org/licenses/by-sa/4.0/',
contact="*****@*****.**",
jsondata={
'license_icon':
'cc-by-sa.png',
'license_name':
'Creative Commons Attribution-ShareAlike 4.0 International License'
},
)
rd = pathlib.Path(tular.__file__).parent.parent.parent.resolve()
root = input('Project dir [{}]: '.format(str(rd)))
root = pathlib.Path(root) if root else rd
clts = clts_from_input(rd / '..' / 'cldf-clts' / 'clts-data')
for db, rec in DATASETS.items():
print(db, rec.doi, rec.tag)
dbdir = root.joinpath(db)
assert dbdir.exists()
md = jsonlib.load(dbdir / 'metadata.json')
name = md['title']
if md['description']:
name += ': {}'.format(md['description'])
contribution = data.add(
Database,
db,
id=db,
name=name,
description=rec.citation if rec else None,
doi=rec.doi if rec else None,
)
header, contribs = next(
iter_markdown_tables(
dbdir.joinpath('CONTRIBUTORS.md').read_text(encoding='utf8')))
for i, contrib in enumerate(contribs):
contrib = dict(zip(header, contrib))
cid = slug(HumanName(contrib['Name']).last)
contributor = data['Contributor'].get(cid)
if not contributor:
contributor = data.add(
common.Contributor,
cid,
id=cid,
name=contrib['Name'],
description=contrib.get('Affiliation'),
)
DBSession.add(
common.ContributionContributor(
contribution=contribution,
contributor=contributor,
primary='author' in contrib['Role'].lower(),
ord=i,
))
for i, cid in enumerate(
['gerardi', 'reichert', 'aragon', 'list', 'forkel']):
DBSession.add(
common.Editor(contributor=data['Contributor'][cid],
dataset=dataset,
ord=i))
source_ids = list(add_sources(args.cldf.bibpath, DBSession))
sources = {s.id: s.pk for s in DBSession.query(common.Source)}
subgroups = []
for row in args.cldf['LanguageTable']:
if row['SubGroup'] not in subgroups:
subgroups.append(row['SubGroup'])
family = data['Family'].get(row['Family'])
if (not family) and row['Family']:
family = data.add(Family,
row['Family'],
id=slug(row['Family']),
name=row['Family'])
data.add(
Doculect,
row['ID'],
id=row['ID'],
name=row['Name'].replace('_', ' '),
family=family,
subfamily=row['SubGroup'],
iso_code=row['ISO639P3code'],
glotto_code=row['Glottocode'],
longitude=row['Longitude'],
latitude=row['Latitude'],
jsondata=dict(icon=SUBGROUPS[row['SubGroup']]),
)
tudet = Dataset.from_metadata(root / 'tudet' / 'cldf' /
'Generic-metadata.json')
seen = set()
for row in tudet['ExampleTable']:
if row['ID'] in seen:
print('skipping duplicate sentence ID {}'.format(row['ID']))
continue
seen.add(row['ID'])
DBSession.add(
Example(id=row['ID'],
name=row['Primary_Text'],
description=row['Translated_Text'],
language=data['Doculect'][row['Language_ID']],
conllu=row['conllu']))
contrib = data['Database']['tuled']
for row in args.cldf['ParameterTable']:
data.add(
Concept,
row['ID'],
id=row['ID'].split('_')[0],
name=row['Name'],
portuguese=row['Portuguese_Gloss'],
semantic_field=row['Semantic_Field'],
concepticon_class=row['Concepticon_ID'],
eol=row['EOL_ID'],
)
for (lid, pid), rows in itertools.groupby(
sorted(args.cldf.iter_rows('FormTable', 'languageReference',
'parameterReference'),
key=lambda r: (r['Language_ID'], r['Parameter_ID'])),
lambda r: (r['Language_ID'], r['Parameter_ID']),
):
vsid = '{}-{}'.format(lid, pid)
vs = data.add(
common.ValueSet,
vsid,
id=vsid,
language=data['Doculect'][lid],
parameter=data['Concept'][pid],
contribution=contrib,
)
refs = set()
for row in rows:
data.add(
Word,
row['ID'],
id=row['ID'],
valueset=vs,
name=row['Form'],
tokens=' '.join(row['Segments']),
simple_cognate=int(row['SimpleCognate']),
notes=row['Comment'],
morphemes=' '.join(row['Morphemes']),
partial_cognate=' '.join([k for k in row['PartialCognates']])
if row['PartialCognates'] else None,
)
refs = refs.union(row['Source'])
for ref in refs:
if ref in source_ids:
DBSession.add(
common.ValueSetReference(valueset=vs,
source_pk=sources[slug(
ref, lowercase=False)]))
load_inventories(args.cldf, clts, data['Doculect'])
for row in args.cldf['CognateTable']:
cc = data['Cognateset'].get(row['Cognateset_ID'])
if not cc:
cc = data.add(
Cognateset,
row['Cognateset_ID'],
id=row['Cognateset_ID'],
name=row['Cognateset_ID'],
contribution=contrib,
)
data.add(
Cognate,
row['ID'],
cognateset=cc,
counterpart=data['Word'][row['Form_ID']],
alignment=' '.join(row['Alignment'] or []),
)
def setUp(self):
TestWithDb.setUp(self)
DBSession.add(
common.Dataset(id='dataset',
name='dataset',
description='desc',
domain='clld',
jsondata={'license_icon': 'cc-by'}))
DBSession.add(
common.Source(id='replaced',
active=False,
jsondata={'__replacement_id__': 'source'}))
source = common.Source(id='source')
contributors = {
'contributor': 'A Name',
'b': 'b Name',
'c': 'c Name',
'd': 'd Name'
}
for id_, name in contributors.items():
contributors[id_] = common.Contributor(id=id_,
name=name,
url='http://example.org')
contribution = common.Contribution(id='contribution',
name='Contribution')
common.ContributionReference(contribution=contribution, source=source)
assert common.ContributionContributor(
contribution=contribution,
primary=True,
contributor=contributors['contributor'])
assert common.ContributionContributor(contribution=contribution,
primary=False,
contributor=contributors['b'])
assert common.ContributionContributor(contribution=contribution,
primary=True,
contributor=contributors['c'])
assert common.ContributionContributor(contribution=contribution,
primary=False,
contributor=contributors['d'])
DBSession.add(contribution)
language = common.Language(id='language',
name='Language 1',
latitude=10.5,
longitude=0.3)
language.sources.append(source)
for i, type_ in enumerate(common.IdentifierType):
id_ = common.Identifier(type=type_.value,
id=type_.value + str(i),
name='abc')
common.LanguageIdentifier(language=language, identifier=id_)
for i in range(2, 102):
_l = common.Language(id='l%s' % i, name='Language %s' % i)
_i = common.Identifier(type='iso639-3',
id='%.3i' % i,
name='%.3i' % i)
common.LanguageIdentifier(language=_l, identifier=_i)
DBSession.add(_l)
param = common.Parameter(id='parameter', name='Parameter')
de = common.DomainElement(id='de',
name='DomainElement',
parameter=param)
de2 = common.DomainElement(id='de2',
name='DomainElement2',
parameter=param)
valueset = common.ValueSet(id='valueset',
language=language,
parameter=param,
contribution=contribution)
value = common.Value(id='value',
domainelement=de,
valueset=valueset,
frequency=50,
confidence='high')
DBSession.add(value)
value2 = common.Value(id='value2',
domainelement=de2,
valueset=valueset,
frequency=50,
confidence='high')
DBSession.add(value2)
paramnd = common.Parameter(id='no-domain',
name='Parameter without domain')
valueset = common.ValueSet(id='vs2',
language=language,
parameter=paramnd,
contribution=contribution)
common.ValueSetReference(valueset=valueset, source=source)
value = common.Value(id='v2',
valueset=valueset,
frequency=50,
confidence='high')
DBSession.add(value)
unit = common.Unit(id='unit', name='Unit', language=language)
up = common.UnitParameter(id='unitparameter', name='UnitParameter')
DBSession.add(unit)
DBSession.add(
common.UnitValue(id='unitvalue',
name='UnitValue',
unit=unit,
unitparameter=up))
up2 = common.UnitParameter(id='up2', name='UnitParameter with domain')
de = common.UnitDomainElement(id='de', name='de', parameter=up2)
DBSession.add(
common.UnitValue(id='uv2',
name='UnitValue2',
unit=unit,
unitparameter=up2,
unitdomainelement=de))
DBSession.add(common.Source(id='s'))
sentence = common.Sentence(
id='sentence',
name='sentence name',
description='sentence description',
analyzed='a\tmorpheme\tdoes\tdo',
gloss='a\tmorpheme\t1SG\tdo.SG2',
source='own',
comment='comment',
original_script='a morpheme',
language=language,
jsondata={'alt_translation': 'Spanish: ...'})
common.SentenceReference(sentence=sentence, source=source)
DBSession.add(common.Config(key='key', value='value'))
common.Config.add_replacement('replaced',
'language',
model=common.Language)
common.Config.add_replacement('gone', None, model=common.Language)
DBSession.flush()
def main(args):
old_db = create_engine(DB)
data = Data()
#
# migrate contributor table: complete
#
for row in old_db.execute("select * from contributor"):
data.add(common.Contributor,
row['id'],
id=row['id'],
name='%(firstname)s %(lastname)s' % row,
url=row['homepage'],
description=row['note'],
email=row['email'],
address=row['address'])
data.add(common.Contributor,
'haspelmathmartin',
id='haspelmathmartin',
name="Martin Haspelmath",
url="http://email.eva.mpg.de/~haspelmt/")
DBSession.flush()
dataset = common.Dataset(
id='wold',
name='WOLD',
description='World Loanword Database',
domain='wold.clld.org',
published=date(2009, 8, 15),
license='http://creativecommons.org/licenses/by/3.0/de/',
contact='*****@*****.**',
jsondata={
'license_icon':
'http://i.creativecommons.org/l/by/3.0/de/88x31.png',
'license_name': 'Creative Commons Attribution 3.0 Germany License'
})
DBSession.add(dataset)
for i, editor in enumerate(['haspelmathmartin', 'tadmoruri']):
common.Editor(dataset=dataset,
contributor=data['Contributor'][editor],
ord=i + 1)
#
# migrate semantic_field table: complete
#
for row in old_db.execute("select * from semantic_field"):
if row['id'] != 25:
kw = dict((key, row[key]) for key in ['id', 'name', 'description'])
data.add(models.SemanticField, row['id'], **kw)
#
# migrate language table: complete
# recipient flag is replaced by vocabulary_pk!
#
for row in old_db.execute("select * from language order by id"):
kw = dict((key, row[key]) for key in [
'fm_dl_id', 'name', 'latitude', 'longitude', 'wals_equivalent',
'affiliation', 'family', 'genus', 'countries'
])
data.add(models.WoldLanguage, row['id'], id=str(row['id']), **kw)
#
# migrate language_code table: complete
#
for row in old_db.execute("select * from language_code"):
_id = '%(type)s-%(code)s' % row
data.add(common.Identifier,
_id,
id=_id,
type=row['type'],
name=row['code'])
if row['type'] == 'iso639-3' and row['code'] in glottocodes:
gc = glottocodes[row['code']]
data.add(common.Identifier,
gc,
id=gc,
type=common.IdentifierType.glottolog.value,
name=gc)
DBSession.flush()
#
# migrate language_code_language table: complete
#
for row in old_db.execute("select * from language_code_language"):
_id = '%(type)s-%(code)s' % row
data.add(common.LanguageIdentifier,
'%s-%s' % (_id, row['language_id']),
identifier_pk=data['Identifier'][_id].pk,
language_pk=data['WoldLanguage'][row['language_id']].pk)
if row['type'] == 'iso639-3' and row['code'] in glottocodes:
gc = glottocodes[row['code']]
data.add(common.LanguageIdentifier,
'%s-%s' % (gc, row['language_id']),
identifier_pk=data['Identifier'][gc].pk,
language_pk=data['WoldLanguage'][row['language_id']].pk)
DBSession.flush()
#
# migrate vocabulary table: complete
#
for row in old_db.execute("select * from vocabulary order by id"):
jsondata = {}
for key in row.keys():
if key.startswith('fd_') or key in [
'other_information', 'abbreviations'
]:
jsondata[key] = row[key]
vocab = data.add(models.Vocabulary,
row['id'],
id=str(row['id']),
name=row['name'],
color=row['color'],
jsondata=jsondata)
DBSession.flush()
data['WoldLanguage'][row['language_id']].vocabulary_pk = vocab.pk
DBSession.flush()
#
# migrate contact_situation and age tables: complete
# contact situations and ages are unitdomainelements!
#
contact_situation = common.UnitParameter(id='cs', name='Contact Situation')
age = common.UnitParameter(id='a', name='Age')
DBSession.add(contact_situation)
DBSession.add(age)
DBSession.flush()
for row in old_db.execute("select * from contact_situation"):
if row['vocabulary_id'] is None:
continue
kw = dict((key, row[key]) for key in ['description', 'id', 'name'])
kw['id'] = 'cs-%s' % kw['id']
p = data.add(models.WoldUnitDomainElement, row['id'], **kw)
p.vocabulary = data['Vocabulary'][row['vocabulary_id']]
p.unitparameter_pk = contact_situation.pk
for row in old_db.execute("select * from age"):
id_ = '%(vocabulary_id)s-%(label)s' % row
kw = dict((key, row[key]) for key in ['start_year', 'end_year'])
p = data.add(models.WoldUnitDomainElement,
id_,
id='a-%s' % id_,
name=row['label'],
description=row['description'],
jsondata=kw)
p.vocabulary = data['Vocabulary'][row['vocabulary_id']]
p.unitparameter_pk = age.pk
#
# migrate meaning table: complete
#
for row in old_db.execute("select * from meaning"):
kw = dict((key, row[key]) for key in [
'description', 'core_list', 'ids_code', 'typical_context',
'semantic_category'
])
p = data.add(
models.Meaning,
row['id'],
id=row['id'].replace('.', '-'),
name=row['label'],
sub_code=row['id'].split('.')[1] if '.' in row['id'] else '',
semantic_field=data['SemanticField'][row['semantic_field_id']],
**kw)
DBSession.flush()
for field in ['french', 'spanish', 'german', 'russian']:
DBSession.add(
models.Translation(name=row[field], lang=field, meaning=p))
for key in data['WoldLanguage']:
lang = data['WoldLanguage'][key]
data.add(common.ValueSet,
'%s-%s' % (key, row['id']),
id='%s-%s' % (key, row['id'].replace('.', '-')),
language=lang,
contribution=lang.vocabulary,
parameter=p)
DBSession.flush()
#
# migrate word table:
# TODO: all the other word properties!!
#
fields = [
'age_label',
'original_script',
'grammatical_info',
'comment_on_word_form',
'gloss',
"comment_on_borrowed",
"calqued",
"borrowed_base",
"numeric_frequency",
"relative_frequency",
"effect",
"integration",
"salience",
"reference",
"other_comments",
"register",
"loan_history",
'colonial_word',
'paraphrase_in_dutch',
'word_source',
'paraphrase_in_german',
'lexical_stratum',
'comparison_with_mandarin',
'year',
'comparison_with_korean',
'czech_translation',
'hungarian_translation',
'early_romani_reconstruction',
'etymological_note',
'boretzky_and_igla_etymology',
'manuss_et_al_etymology',
'vekerdi_etymology',
'turner_etymology',
'other_etymologies',
'mayrhofer_etymology',
]
word_to_vocab = {}
for row in old_db.execute("select * from word"):
word_to_vocab[row['id']] = row['vocabulary_id']
kw = dict((key, row[key]) for key in [
'id', 'age_score', 'borrowed', 'borrowed_score', 'analyzability',
'simplicity_score'
])
w = data.add(models.Word,
row['id'],
name=row['form'],
description=row['free_meaning'],
jsondata={k: row[k]
for k in fields},
**kw)
w.language = data['Vocabulary'][row['vocabulary_id']].language
if row['age_label']:
DBSession.add(
common.UnitValue(
id='%(id)s-a' % row,
unit=w,
unitparameter=age,
unitdomainelement=data['WoldUnitDomainElement'][
'%(vocabulary_id)s-%(age_label)s' % row],
contribution=data['Vocabulary'][row['vocabulary_id']]))
if row['contact_situation_id'] and row[
'contact_situation_id'] != '9129144185487768':
DBSession.add(
common.UnitValue(
id='%(id)s-cs' % row,
unit=w,
unitparameter=contact_situation,
unitdomainelement=data['WoldUnitDomainElement'][
row['contact_situation_id']],
contribution=data['Vocabulary'][row['vocabulary_id']]))
DBSession.flush()
#
# migrate word_meaning table: complete
#
for i, row in enumerate(old_db.execute("select * from word_meaning")):
data.add(
models.Counterpart,
i,
id=i,
description='%(relationship)s (%(comment_on_relationship)s)' % row,
name=data['Word'][row['word_id']].name,
valueset=data['ValueSet']['%s-%s' % (word_to_vocab[row['word_id']],
row['meaning_id'])],
word=data['Word'][row['word_id']])
DBSession.flush()
#
# migrate vocabulary_contributor table: complete
#
for row in old_db.execute("select * from vocabulary_contributor"):
DBSession.add(
common.ContributionContributor(
ord=row['ordinal'],
primary=row['primary'],
contributor_pk=data['Contributor'][row['contributor_id']].pk,
contribution_pk=data['Vocabulary'][row['vocabulary_id']].pk))
DBSession.flush()
#
# source words: we have to make sure a word does only belong to one language.
# thus, we have to reassign identifier!
#
# loop over source_word, source_word_donor_language pairs keeping track of source_word ids:
known_ids = {}
for row in old_db.execute(
"select sw.id, sw.meaning, sw.form, dl.language_id from source_word as sw, source_word_donor_language as dl where sw.id = dl.source_word_id"
):
if row['id'] in known_ids:
# source_word was already seen associated to a different donor language!
assert row['language_id'] not in known_ids[row['id']]
known_ids[row['id']].append(row['language_id'])
id_ = '%s-%s' % (row['id'], len(known_ids[row['id']]))
else:
id_ = '%s-%s' % (row['id'], 1)
known_ids[row['id']] = [row['language_id']]
new = data.add(models.Word,
id_,
id=id_,
name=row['form'],
description=row['meaning'])
new.language = data['WoldLanguage'][row['language_id']]
# source words may end up as words without language!
for row in old_db.execute(
"select id, meaning, form from source_word where id not in (select source_word_id from source_word_donor_language)"
):
id_ = '%s-%s' % (row['id'], 1)
new = data.add(models.Word,
id_,
id=id_,
name=row['form'],
description=row['meaning'])
DBSession.flush()
#
# migrate word_source_word relations
# TODO: should be modelled as UnitParameter!
#
j = 0
for row in old_db.execute("select * from word_source_word"):
# there may be more than one word associated with a source_word_id (see above)
source_words = []
for i in range(4): # but we guess no more than 4 :)
id_ = '%s-%s' % (row['source_word_id'], i + 1)
if id_ in data['Word']:
source_words.append(data['Word'][id_])
if not source_words:
j += 1
#print(row['source_word_id'])
#raise ValueError(row['source_word_id'])
for sw in source_words:
DBSession.add(
models.Loan(source_word=sw,
target_word=data['Word'][row['word_id']],
relation=row['relationship'],
certain=len(source_words) == 1))
print('%s source words not migrated because they have no donor language!' %
j)
def main(args):
data = Data()
dataset = common.Dataset(
id=cobl2.__name__,
name="IE-CoR",
publisher_name="Max Planck Institute for the Science of Human History",
publisher_place="Jena",
publisher_url="https://www.eva.mpg.de",
license="https://creativecommons.org/licenses/by/4.0/",
domain='iecor.clld.org',
contact='*****@*****.**',
jsondata={
'license_icon': 'cc-by.png',
'license_name': 'Creative Commons Attribution 4.0 International License'})
DBSession.add(dataset)
editors = OrderedDict([('Heggarty', None), ('Anderson', None), ('Scarborough', None)])
for row in sorted(ds['authors.csv'], key=lambda x: [
x['Last_Name'].lower(), x['First_Name'].lower()]):
if row['Last_Name'] in editors:
editors[row['Last_Name']] = row['ID']
data.add(
models.Author,
row['ID'],
id=row['ID'],
name='{0} {1}'.format(row['First_Name'], row['Last_Name']),
url=row['URL'],
photo=data_uri(photos[row['Last_Name']], 'image/jpg') if row['Last_Name'] in photos else None)
for i, cid in enumerate(editors.values()):
common.Editor(dataset=dataset, contributor=data['Author'][cid], ord=i + 1)
for src in ds.sources.items():
for invalid in ['isbn', 'part', 'institution']:
if invalid in src:
del src[invalid]
data.add(
common.Source,
src.id,
id=src.id,
name=src.get('author', src.get('editor')),
description=src.get('title', src.get('booktitle')),
bibtex_type=getattr(EntryType, src.genre, EntryType.misc),
**src)
re_links = re.compile(r'\[(?P<label>[^\]]+?)\]\((?P<type>.+?)-(?P<id>\d+)\)')
link_map = {
'cog': '/cognatesets/',
'lex': '/values/',
'src': '/sources/',
}
def parse_links(m):
try:
return '<a href="{}{}">{}</a>'.format(
link_map[m.group('type')], m.group('id'), m.group('label'))
except KeyError:
print("parse_links: type error in '{}'".format(":".join(m.groups())))
return '[{}]({}-{})'.format(m.group('label'), m.group('type'), m.group('id'))
for param in ds['ParameterTable']:
data.add(
models.Meaning,
param['ID'],
id=slug(param['Name']),
name=param['Name'],
description_md=param['Description_md'],
concepticon_id=int(param['Concepticon_ID']) if param['Concepticon_ID'] != '0' else None,
)
for row in ds['clades.csv']:
data.add(
models.Clade,
row['ID'],
id=row['ID'],
level0_name=row['level0_name'],
level1_name=row['level1_name'],
level2_name=row['level2_name'],
level3_name=row['level3_name'],
clade_level0=row['clade_level0'],
clade_level1=row['clade_level1'],
clade_level2=row['clade_level2'],
clade_level3=row['clade_level3'],
clade_name=row['clade_name'],
short_name=row['short_name'],
color=row['color'],
)
for row in ds['LanguageTable']:
c = data.add(
common.Contribution,
row['ID'],
id=row['ID'],
name=row['Name'],
)
for i, cid in enumerate(row['Author_ID']):
DBSession.add(common.ContributionContributor(
contribution=c, contributor=data['Author'][cid], ord=i + 1))
data.add(
models.Variety,
row['ID'],
id=slug(row['Name']),
name=row['Name'],
latitude=float(row['Latitude']) if row['Latitude'] is not None else None,
longitude=float(row['Longitude']) if row['Longitude'] is not None else None,
contribution=c,
color=rgb_as_hex(row['Color']),
clade=', '.join(filter(None, row['Clade'])),
clade_name=row['clade_name'],
glottocode=row['Glottocode'],
historical=row['historical'],
distribution=row['distribution'],
logNormalMean=row['logNormalMean'],
logNormalOffset=row['logNormalOffset'],
logNormalStDev=row['logNormalStDev'],
normalMean=row['normalMean'],
normalStDev=row['normalStDev'],
ascii_name=row['ascii_name'],
iso=row['ISO639P3code'],
lang_description=row['Description'],
variety=row['Variety'],
loc_justification=row['loc_justification'] or None,
sort_order=row['sort_order']
)
vsrs = set()
for row in ds['FormTable']:
vs = data['ValueSet'].get((row['Language_ID'], row['Parameter_ID']))
if not vs:
vs = data.add(
common.ValueSet,
(row['Language_ID'], row['Parameter_ID']),
id='{0}-{1}'.format(row['Language_ID'], row['Parameter_ID']),
language=data['Variety'][row['Language_ID']],
parameter=data['Meaning'][row['Parameter_ID']],
contribution=data['Contribution'][row['Language_ID']],
)
v = data.add(
models.Lexeme,
row['ID'],
id=row['ID'],
name=row['Form'],
native_script=row['native_script'],
phonetic=row['phon_form'],
phonemic=row['Phonemic'],
comment=re_links.sub(parse_links, row['Comment'] or ''),
url=row['url'],
gloss=row['Gloss'],
valueset=vs
)
for src in row['Source']:
sid, pages = ds.sources.parse(src)
key = (vs.id, sid, pages)
if pages:
pages = pages.replace('|', ';')
if key not in vsrs:
DBSession.add(common.ValueSetReference(
valueset=vs, source=data['Source'][sid], description=pages))
vsrs.add(key)
for row in ds['CognatesetTable']:
cc = data.add(
models.CognateClass,
row['ID'],
id=row['ID'],
name=row['ID'],
root_form=row['Root_Form_calc'] if row['Root_Form_calc'] is not None and len(row['Root_Form_calc']) else row['Root_Form'],
root_form_calc=row['Root_Form_calc'] or None,
root_gloss=row['Root_Gloss'] or None,
root_language=row['Root_Language_calc'] if row['Root_Language_calc'] is not None and len(row['Root_Language_calc']) else row['Root_Language'],
root_language_calc=row['Root_Language_calc'] or None,
comment=re_links.sub(parse_links, row['Comment'] or ''),
justification=re_links.sub(parse_links, row['Justification'] or ''),
ideophonic=row['Ideophonic'] or None,
parallel_derivation=row['parallelDerivation'] or None,
revised_by=','.join(row['revised_by']) or None,
superset_id=int(row['supersetid']) if row['supersetid'] else None,
)
for src in row['Source']:
sid, pages = ds.sources.parse(src)
if pages:
pages = pages.replace('|', ';')
DBSession.add(clld_cognacy_plugin.models.CognatesetReference(
cognateset=cc, source=data['Source'][sid], description=pages))
DBSession.flush()
cc_id_pk_map = {str(ccid): cc.pk for ccid, cc in data['CognateClass'].items()}
for row in ds['CognatesetTable']:
if row['proposedAsCognateTo_pk']:
DBSession.add(models.ProposedCognates(
cc1_pk=data['CognateClass'][row['ID']].pk,
cc2_pk=cc_id_pk_map[str(row['proposedAsCognateTo_pk'])],
scale=row['proposedAsCognateToScale']
))
DBSession.flush()
loans = {ln['Cognateset_ID']: ln for ln in ds['loans.csv']}
for ccid, cc in data['CognateClass'].items():
if ccid in loans:
le = loans[ccid]
if le['SourceCognateset_ID']:
cc.loan_source_pk = data['CognateClass'][le['SourceCognateset_ID']].pk
else:
cc.loan_source_pk = None
cc.loan_notes = le['Comment']
cc.loan_source_languoid = le['Source_languoid']
cc.loan_source_form = le['Source_form']
cc.parallel_loan_event = le['Parallel_loan_event']
cc.is_loan = True
for row in ds['CognateTable']:
cc = data['CognateClass'][row['Cognateset_ID']]
if cc.meaning_pk is None:
cc.meaning_pk = data['Lexeme'][row['Form_ID']].valueset.parameter_pk
else:
assert data['Lexeme'][row['Form_ID']].valueset.parameter_pk == cc.meaning_pk
data.add(
clld_cognacy_plugin.models.Cognate,
row['ID'],
cognateset=data['CognateClass'][row['Cognateset_ID']],
counterpart=data['Lexeme'][row['Form_ID']],
doubt=row['Doubt'],
)
l_by_gc = {}
for s in DBSession.query(models.Variety):
l_by_gc[s.glottocode] = s.pk
tree = Phylogeny(
id='1',
name='Bouckaert et al.',
description='',
newick=Path.read_text(data_file_path / 'raw' / 'bouckaert_et_al2012' / 'newick.txt'),
)
for k, taxon in enumerate(reader(data_file_path / 'raw' / 'bouckaert_et_al2012' / 'taxa.csv', namedtuples=True)):
label = TreeLabel(
id='{0}-{1}-{2}'.format(tree.id, slug(taxon.taxon), k + 1),
name=taxon.taxon,
phylogeny=tree,
description=taxon.glottocode)
if taxon.glottocode in l_by_gc:
LanguageTreeLabel(language_pk=l_by_gc[taxon.glottocode], treelabel=label)
DBSession.add(tree)
l_by_ascii = {}
for s in DBSession.query(models.Variety):
l_by_ascii[s.ascii_name] = s.pk
tree = Phylogeny(
id='2',
name='CoBL consensu',
description='',
newick=Path.read_text(data_file_path / 'raw' / 'ie122' / 'newick.txt'),
)
for k, taxon in enumerate(reader(data_file_path / 'raw' / 'ie122' / 'taxa.csv', namedtuples=True)):
label = TreeLabel(
id='{0}-{1}-{2}'.format(tree.id, slug(taxon.taxon), k + 1),
name=taxon.taxon,
phylogeny=tree)
if taxon.taxon in l_by_ascii:
LanguageTreeLabel(language_pk=l_by_ascii[taxon.taxon], treelabel=label)
DBSession.add(tree)
def main(args):
data = Data()
print(args.data_file('x'))
dataset = common.Dataset(
id=grammaticon.__name__,
name="Grammaticon",
publisher_name="Max Planck Institute for the Science of Human History",
publisher_place="Jena",
publisher_url="http://www.shh.mpg.de",
license="http://creativecommons.org/licenses/by/4.0/",
domain='grammaticon.clld.org',
jsondata={
'license_icon': 'cc-by.png',
'license_name': 'Creative Commons Attribution 4.0 International License'})
DBSession.add(dataset)
for i, ed in enumerate(['Martin Haspelmath', 'Robert Forkel']):
common.Editor(dataset=dataset, contributor=get_contributor(data, ed), ord=i + 1)
eng = data.add(common.Language, 'eng', name='English')
for obj in reader(args.data_file('Feature_lists.csv'), dicts=True):
contrib = data.add(
models.Featurelist, obj['id'],
id=slug(obj['name']),
name=obj['name'],
year=obj['year'],
number_of_features=int(obj['number of features']) if obj['number of features'] else None,
url=obj['year'])
if obj['authors']:
for i, author in enumerate(obj['authors'].split(',')):
common.ContributionContributor(
contribution=contrib,
contributor=get_contributor(data, author),
ord=i + 1)
#id,name,feature_area
for name, objs in itertools.groupby(
sorted(reader(args.data_file('Metafeatures.csv'), dicts=True), key=lambda i: i['name']),
lambda i: i['name']):
dbobj = None
for obj in objs:
if not dbobj:
dbobj = data.add(
models.Metafeature, obj['id'],
id=slug(obj['id']), name=obj['name'], area=obj['feature_area'])
else:
data['Metafeature'][obj['id']] = dbobj
DBSession.flush()
#feature_ID,feature name,feature description,meta_feature_id,collection_id,collection URL,collection numbers
for obj in reader(args.data_file('Features.csv'), dicts=True):
if int(obj['collection_id']) == 8:
obj['collection_id'] = '1'
if (not obj['meta_feature_id']): #or obj['meta_feature_id'] in ('89'):
print('skipping: {}'.format(obj))
continue
vsid = (data['Featurelist'][obj['collection_id']].pk, data['Metafeature'][obj['meta_feature_id']].pk)
vs = data['ValueSet'].get(vsid)
if not vs:
vs = data.add(
common.ValueSet, vsid,
id='{0}-{1}'.format(*vsid),
contribution=data['Featurelist'][obj['collection_id']],
parameter=data['Metafeature'][obj['meta_feature_id']],
language=eng)
models.Feature(
valueset=vs, id=slug(obj['feature_ID']), name=obj['feature name'], description=obj['feature description'])
for obj in reader(args.data_file('Concepts.csv'), dicts=True):
data.add(
models.Concept, obj['id'],
id=obj.pop('id'), name=obj.pop('label'), description=obj.pop('definition'),
**{k.replace(' ', '_'): v for k, v in obj.items()})
for obj in reader(args.data_file('Concepts_metafeatures.csv'), dicts=True):
if obj['meta_feature__id'] in ('89',):
print('skipping: {}'.format(obj))
continue
if obj['concept_id'] and obj['meta_feature__id']:
models.ConceptMetafeature(
concept=data['Concept'][obj['concept_id']],
metafeature=data['Metafeature'][obj['meta_feature__id']])
for obj in reader(args.data_file('Concepthierarchy.csv'), dicts=True):
child = data['Concept'].get(obj['concept_id'])
if child:
parent = data['Concept'].get(obj['concept_parent_id'])
if parent:
DBSession.add(models.ConceptRelation(parent=parent, child=child))
def load(args):
glottolog = args.repos
fts.index('fts_index', models.Ref.fts, DBSession.bind)
DBSession.execute("CREATE EXTENSION IF NOT EXISTS unaccent WITH SCHEMA public;")
version = assert_release(glottolog.repos)
dataset = common.Dataset(
id='glottolog',
name="{0} {1}".format(glottolog.publication.web.name, version),
publisher_name=glottolog.publication.publisher.name,
publisher_place=glottolog.publication.publisher.place,
publisher_url=glottolog.publication.publisher.url,
license=glottolog.publication.license.url,
domain=purl.URL(glottolog.publication.web.url).domain(),
contact=glottolog.publication.web.contact,
jsondata={'license_icon': 'cc-by.png', 'license_name': glottolog.publication.license.name},
)
data = Data()
for e in glottolog.editors.values():
if e.current:
ed = data.add(common.Contributor, e.id, id=e.id, name=e.name)
common.Editor(dataset=dataset, contributor=ed, ord=int(e.ord))
DBSession.add(dataset)
contrib = data.add(common.Contribution, 'glottolog', id='glottolog', name='Glottolog')
DBSession.add(common.ContributionContributor(
contribution=contrib, contributor=data['Contributor']['hammarstroem']))
#
# Add Parameters:
#
add = functools.partial(add_parameter, data)
add('fc', name='Family classification')
add('sc', name='Subclassification')
add('aes',
args.repos.aes_status.values(),
name=args.repos.aes_status.__defaults__['name'],
pkw=dict(
jsondata=dict(
reference_id=args.repos.aes_status.__defaults__['reference_id'],
sources=[attr.asdict(v) for v in args.repos.aes_sources.values()],
scale=[attr.asdict(v) for v in args.repos.aes_status.values()])),
dekw=lambda de: dict(name=de.name, number=de.ordinal, jsondata=dict(icon=de.icon)),
)
add('med',
args.repos.med_types.values(),
name='Most Extensive Description',
dekw=lambda de: dict(
name=de.name, description=de.description, number=de.rank, jsondata=dict(icon=de.icon)),
)
add('macroarea',
args.repos.macroareas.values(),
pkw=dict(
description=args.repos.macroareas.__defaults__['description'],
jsondata=dict(reference_id=args.repos.macroareas.__defaults__['reference_id'])),
dekw=lambda de: dict(
name=de.name,
description=de.description,
jsondata=dict(geojson=read_macroarea_geojson(args.repos, de.name, de.description)),
),
)
add('ltype',
args.repos.language_types.values(),
name='Language Type',
dekw=lambda de: dict(name=de.category, description=de.description),
delookup='category',
)
add('country',
args.repos.countries,
dekw=lambda de: dict(name=de.id, description=de.name),
)
legacy = jsonlib.load(gc2version(args))
for gc, version in legacy.items():
data.add(models.LegacyCode, gc, id=gc, version=version)
#
# Now load languoid data, keeping track of relations that can only be inserted later.
#
lgsources = defaultdict(list)
# Note: We rely on languoids() yielding languoids in the "right" order, i.e. such that top-level
# nodes will precede nested nodes. This order must be preserved using an `OrderedDict`:
nodemap = OrderedDict([(l.id, l) for l in glottolog.languoids()])
lgcodes = {k: v.id for k, v in args.repos.languoids_by_code(nodemap).items()}
for lang in nodemap.values():
for ref in lang.sources:
lgsources['{0.provider}#{0.bibkey}'.format(ref)].append(lang.id)
load_languoid(glottolog, data, lang, nodemap)
for gc in glottolog.glottocodes:
if gc not in data['Languoid'] and gc not in legacy:
common.Config.add_replacement(gc, None, model=common.Language)
for obj in jsonlib.load(glottolog.references_path('replacements.json')):
common.Config.add_replacement(
'{0}'.format(obj['id']),
'{0}'.format(obj['replacement']) if obj['replacement'] else None,
model=common.Source)
DBSession.flush()
for doctype in glottolog.hhtypes:
data.add(
models.Doctype, doctype.id, id=doctype.id,
name=doctype.name,
description=doctype.description,
abbr=doctype.abbv,
ord=doctype.rank)
for bib in glottolog.bibfiles:
data.add(
models.Provider,
bib.id,
id=bib.id,
name=bib.title,
description=bib.description,
abbr=bib.abbr,
url=bib.url)
DBSession.flush()
s = time()
for i, entry in enumerate(
BibFile(glottolog.build_path('monster-utf8.bib'), api=glottolog).iterentries()):
if i % 10000 == 0:
args.log.info('{0}: {1:.3}'.format(i, time() - s))
s = time()
ref = load_ref(data, entry, lgcodes, lgsources)
if 'macro_area' in entry.fields:
mas = []
for ma in split_text(entry.fields['macro_area'], separators=',;', strip=True):
ma = 'North America' if ma == 'Middle America' else ma
ma = glottolog.macroareas.get('Papunesia' if ma == 'Papua' else ma)
mas.append(ma.name)
ref.macroareas = ', '.join(mas)
def load():
wals = create_engine('postgresql://robert@/wals3')
contributor = common.Contributor(id='gastvolker', name='Volker Gast')
contribution = common.Contribution(
id='tdir', name='Typological Database of Intensifiers and Reflexives')
cc = common.ContributionContributor(
contribution=contribution, contributor=contributor)
DBSession.add(cc)
for row in read('glosses'):
DBSession.add(common.GlossAbbreviation(id=row['gloss'], name=row['explanation']))
params = {}
for id_, name in PARAMS.items():
params[id_] = common.Parameter(id='tdir-' + id_, name=name)
DBSession.add(params[id_])
#
# TODO: domain for sortal restrictions!
#
values = {}
languages = {}
for row in read('languages'):
if row['adn'] and '<br>' in row['adn']:
row['adn'], other = row['adn'].split('<br>', 1)
if not row['otherint']:
row['otherint'] = ''
row['otherint'] = '\n'.join(filter(None, row['otherint'].split('<br>') + other.split('<br>')))
row['sil'] = row['sil'].lower()
row['sil'] = {
'arm': 'hye',
'vmn': 'mig',
'gli': 'gle',
'grk': 'ell',
'hbr': 'heb',
'ltn': 'lat',
'chn': 'cmn',
'ota': 'ote',
'pnj': 'pan',
'pba': 'rap',
'esg': 'kal',
'vla': 'zea',
'lat': 'lav',
}.get(row['sil'], row['sil'])
l = common.Language(id=row['sil'].lower(), name=row['language'])
languages[row['language']] = l
res = wals.execute("select l.latitude, l.longitude from language as l, languageidentifier as li, identifier as i where l.pk = li.language_pk and li.identifier_pk = i.pk and i.id = '%s' and i.type = 'iso639-3';" \
% row['sil']).fetchone()
if not res:
res = wals.execute("select latitude, longitude from language where name = '%s';" % row['language']).fetchone()
if res:
l.latitude, l.longitude = res
else:
print(row['language'], row['sil'])
#(u'Classical Nahuatl', u'nci') ???
#(u'Ancient Greek', u'gko')
for pid in params.keys():
value = row[pid]
if value:
value = common.Value(
id='tdir-%s-%s' % (pid, l.id),
name=unicode(bs(value)),
contribution=contribution,
parameter=params[pid],
language=l)
values['%s-%s' % (pid, row['language'])] = value
DBSession.add(value)
def normalize_ref(ref):
ref = re.sub('\s+', ' ', ref).strip()
return unicode(bs(ref)).replace('<i>', '"').replace('</i>', '"')
"""
Ogawa, A. (1998)
Wali, K. et al. (2000)
Lyutikova. -> Lyutikova,
se-Bertit -> se-Berit
missing refs:
Sengupta, G. (2000). Lexical anaphors and pronouns in Bangla. In Lust et al. (eds.), <i>Lexical Anaphors and Pronouns in Selected South Asian Languages</i>. Berlin: Mouton de Gruyter.
Davison, A. Mistry (2000). Lexical anaphors and pronouns in Hindi/Urdu. In Lust et al. (eds.), <i>Lexical Anaphors and Pronouns in Selected South Asian Languages</i>. Berlin: Mouton de Gruyter.
"""
refs = {}
for row in read('references'):
name = re.sub('\s+', ' ', row['entry'].split(').')[0].strip()) + ')'
src = common.Source(
id=row['ref'].strip(), name=name, description=normalize_ref(row['entry']))
refs[name] = src
DBSession.add(src)
for row in read('examples'):
if row['language'] not in languages:
print('example for unknown language "%s"' % row['language'])
continue
s = common.Sentence(
id=row['Nr'].strip(),
name=fix_example(row['original'], repl=' '),
language=languages[row['language']],
analyzed=fix_example(row['original']),
gloss=fix_example(row['gloss']),
description=row['translation'],
source=row['source'],
comment=row['comments'])
has_refs = False
for ref in refs:
if ref in row['source']:
if normalize_ref(row['source']) != refs[ref].description:
print('-->')
print(row['source'])
has_refs = True
common.SentenceReference(sentence=s, source=refs[ref])
if not has_refs:
print('+++++')
print(row['source'])
pid = EXAMPLE_MAP[row['pov']]
if pid:
# associate with value!
o = common.ValueSentence(value=values['%s-%s' % (pid, row['language'])], sentence=s)
DBSession.add(s)
def setUp(self):
TestWithDb.setUp(self)
DBSession.add(
common.Dataset(id='dataset',
name='dataset',
description='desc',
domain='clld'))
source = common.Source(id='source')
contributors = {
'contributor': 'A Name',
'b': 'b Name',
'c': 'c Name',
'd': 'd Name'
}
for id_, name in contributors.items():
contributors[id_] = common.Contributor(id=id_, name=name)
contribution = common.Contribution(id='contribution',
name='Contribution')
cr = common.ContributionReference(contribution=contribution,
source=source)
assert common.ContributionContributor(
contribution=contribution,
primary=True,
contributor=contributors['contributor'])
assert common.ContributionContributor(contribution=contribution,
primary=False,
contributor=contributors['b'])
assert common.ContributionContributor(contribution=contribution,
primary=True,
contributor=contributors['c'])
assert common.ContributionContributor(contribution=contribution,
primary=False,
contributor=contributors['d'])
DBSession.add(contribution)
language = common.Language(id='language',
name='Language 1',
latitude=10.5,
longitude=0.3)
language.sources.append(source)
identifier = common.Identifier(type='iso639-3', id='iso')
li = common.LanguageIdentifier(language=language,
identifier=identifier)
for i in range(2, 102):
_l = common.Language(id='l%s' % i, name='Language %s' % i)
_i = common.Identifier(type='iso639-3',
id='%.3i' % i,
name='%.3i' % i)
_li = common.LanguageIdentifier(language=_l, identifier=_i)
DBSession.add(_l)
param = common.Parameter(id='parameter', name='Parameter')
de = common.DomainElement(id='de',
name='DomainElement',
parameter=param)
de2 = common.DomainElement(id='de2',
name='DomainElement2',
parameter=param)
valueset = common.ValueSet(id='valueset',
language=language,
parameter=param,
contribution=contribution)
value = common.Value(id='value',
domainelement=de,
valueset=valueset,
frequency=50,
confidence='high')
DBSession.add(value)
paramnd = common.Parameter(id='no-domain',
name='Parameter without domain')
valueset = common.ValueSet(id='vs2',
language=language,
parameter=paramnd,
contribution=contribution)
vr = common.ValueSetReference(valueset=valueset, source=source)
value = common.Value(id='v2',
valueset=valueset,
frequency=50,
confidence='high')
DBSession.add(value)
unit = common.Unit(id='unit', name='Unit', language=language)
up = common.UnitParameter(id='unitparameter', name='UnitParameter')
DBSession.add(unit)
DBSession.add(
common.UnitValue(id='unitvalue',
name='UnitValue',
unit=unit,
unitparameter=up))
up2 = common.UnitParameter(id='up2', name='UnitParameter with domain')
de = common.UnitDomainElement(id='de', name='de', parameter=up2)
DBSession.add(
common.UnitValue(id='uv2',
name='UnitValue2',
unit=unit,
unitparameter=up2,
unitdomainelement=de))
DBSession.add(common.Source(id='s'))
sentence = common.Sentence(id='sentence',
name='sentence name',
description='sentence description',
analyzed='a\tmorpheme\tdoes\tdo',
gloss='a\tmorpheme\t1SG\tdo.SG2',
source='own',
comment='comment',
original_script='a morpheme',
language=language)
sr = common.SentenceReference(sentence=sentence, source=source)
DBSession.add(common.Config(key='key', value='value'))
DBSession.flush()
def load(args):
fts.index('fts_index', models.Ref.fts, DBSession.bind)
DBSession.execute("CREATE EXTENSION IF NOT EXISTS unaccent WITH SCHEMA public;")
dataset = common.Dataset(
id='glottolog',
name="Glottolog {0}".format(args.args[0]),
publisher_name="Max Planck Institute for the Science of Human History",
publisher_place="Jena",
publisher_url="https://shh.mpg.de",
license="http://creativecommons.org/licenses/by/4.0/",
domain='glottolog.org',
contact='*****@*****.**',
jsondata={
'license_icon': 'cc-by.png',
'license_name': 'Creative Commons Attribution 4.0 International License'})
data = Data()
for i, (id_, name) in enumerate([
('hammarstroem', 'Harald Hammarström'),
('bank', 'Sebastian Bank'),
('forkel', 'Robert Forkel'),
('haspelmath', 'Martin Haspelmath'),
]):
ed = data.add(common.Contributor, id_, id=id_, name=name)
common.Editor(dataset=dataset, contributor=ed, ord=i + 1)
DBSession.add(dataset)
clf = data.add(common.Contribution, 'clf', id='clf', name='Classification')
DBSession.add(common.ContributionContributor(
contribution=clf, contributor=data['Contributor']['hammarstroem']))
for pid, pname in [
('fc', 'Family classification'),
('sc', 'Subclassification'),
('vitality', 'Degree of endangerment'),
]:
data.add(common.Parameter, pid, id=pid, name=pname)
legacy = jsonlib.load(gc2version(args))
for gc, version in legacy.items():
data.add(models.LegacyCode, gc, id=gc, version=version)
glottolog = args.repos
for ma in Macroarea:
data.add(
models.Macroarea,
ma.name,
id=ma.name,
name=ma.value,
description=ma.description)
for country in glottolog.countries:
data.add(models.Country, country.id, id=country.id, name=country.name)
lgcodes, mas, countries, lgsources = {}, {}, {}, defaultdict(list)
languoids = list(glottolog.languoids())
nodemap = {l.id: l for l in languoids}
for lang in languoids:
for ref in lang.sources:
lgsources['{0.provider}#{0.bibkey}'.format(ref)].append(lang.id)
load_languoid(data, lang, nodemap)
mas[lang.id] = [ma.name for ma in lang.macroareas]
countries[lang.id] = [c.id for c in lang.countries]
lgcodes[lang.id] = lang.id
if lang.hid:
lgcodes[lang.hid] = lang.id
if lang.iso:
lgcodes[lang.iso] = lang.id
for gc in glottolog.glottocodes:
if gc not in data['Languoid'] and gc not in legacy:
common.Config.add_replacement(gc, None, model=common.Language)
for obj in jsonlib.load(glottolog.references_path('replacements.json')):
common.Config.add_replacement(
'{0}'.format(obj['id']),
'{0}'.format(obj['replacement']) if obj['replacement'] else None,
model=common.Source)
DBSession.flush()
for lid, maids in mas.items():
for ma in maids:
DBSession.add(models.Languoidmacroarea(
languoid_pk=data['Languoid'][lid].pk,
macroarea_pk=data['Macroarea'][ma].pk))
for lid, cids in countries.items():
for cid in cids:
DBSession.add(models.Languoidcountry(
languoid_pk=data['Languoid'][lid].pk,
country_pk=data['Country'][cid].pk))
for doctype in glottolog.hhtypes:
data.add(
models.Doctype, doctype.id, id=doctype.id,
name=doctype.name,
description=doctype.description,
abbr=doctype.abbv,
ord=doctype.rank)
for bib in glottolog.bibfiles:
data.add(
models.Provider,
bib.id,
id=bib.id,
name=bib.title,
description=bib.description,
abbr=bib.abbr,
url=bib.url)
DBSession.flush()
s = time()
for i, entry in enumerate(
BibFile(glottolog.build_path('monster-utf8.bib')).iterentries()):
if i % 10000 == 0:
args.log.info('{0}: {1:.3}'.format(i, time() - s))
s = time()
ref = load_ref(data, entry, lgcodes, lgsources)
if 'macro_area' in entry.fields:
for ma in split_text(entry.fields['macro_area'], separators=',;', strip=True):
ma = 'North America' if ma == 'Middle America' else ma
ma = Macroarea.get('Papunesia' if ma == 'Papua' else ma)
DBSession.add(models.Refmacroarea(
ref_pk=ref.pk, macroarea_pk=data['Macroarea'][ma.name].pk))
def main(args):
data = Data()
ds = Pofatu(
pathlib.Path(pofatu.__file__).parent.parent.parent / 'pofatu-data')
dataset = common.Dataset(
id=pofatu.__name__,
name="POFATU",
publisher_name="Max Planck Institute for Evolutionary Anthropology",
publisher_place="Leipzig",
publisher_url="https://www.eva.mpg.de",
license="https://creativecommons.org/licenses/by/4.0/",
domain='pofatu.clld.org',
contact='*****@*****.**',
jsondata={
'license_icon':
'cc-by.png',
'license_name':
'Creative Commons Attribution 4.0 International License'
})
for i, (id_, name) in enumerate([
('hermannaymeric', 'Aymeric Hermann'),
('forkelrobert', 'Robert Forkel'),
]):
ed = data.add(common.Contributor, id_, id=id_, name=name)
common.Editor(dataset=dataset, contributor=ed, ord=i + 1)
DBSession.add(dataset)
for rec in ds.iterbib():
rec.genre = bibtex.EntryType.from_string(
ENTRY_TYPES.get(rec.genre, rec.genre))
if 'date' in rec:
rec['year'] = rec.pop('date')
data.add(common.Source,
rec.id,
_obj=bibtex2source(rec, lowercase_id=False))
analyses = list(ds.iterdata())
def midpoint(coords):
p = MultiPoint([(lat, lon + 360 if lon < 0 else lon)
for lat, lon in coords]).convex_hull
#geojson = {
# 'type': 'Feature',
# 'properties': {},
# 'geometry': mapping(p)}
c = p.centroid
return c.x, (c.y - 360) if c.y > 180 else c.y
artefacts = collections.defaultdict(dict)
midpoints = {}
for a in analyses:
l = a.sample.location
lid = l.id
if lid not in midpoints:
midpoints[lid] = set()
if l.latitude is not None and l.longitude is not None:
midpoints[lid].add((l.latitude, l.longitude))
art = a.sample.artefact
for attr_ in ['name', 'category', 'collection_type']:
if not artefacts[slug(art.id)].get(attr_):
artefacts[slug(art.id)][attr_] = getattr(art, attr_)
midpoints = {
k: midpoint(v) if v else (None, None)
for k, v in midpoints.items()
}
for analysis in analyses:
loc = analysis.sample.location
if loc.id not in data['Location']:
data.add(
models.Location,
loc.id,
id=valid_id(loc.id),
name=loc.label,
latitude=midpoints[loc.id][0],
longitude=midpoints[loc.id][1],
region=loc.region.replace('_', ' '),
subregion=loc.subregion,
location=loc.locality,
)
# Add contributions
for contrib in ds.itercontributions():
contribution = data.add(
common.Contribution,
contrib.id,
id=valid_id(contrib.id),
name=contrib.label,
description=contrib.description,
)
DBSession.flush()
for i, name in enumerate(contrib.contributors):
cid = slug(name)
co = data['Contributor'].get(cid)
if not co:
co = data.add(common.Contributor, cid, id=cid, name=name)
common.ContributionContributor(ord=i,
contribution=contribution,
contributor=co)
for ref in contrib.source_ids:
DBSession.add(
common.ContributionReference(
contribution=contribution,
source=data['Source'][ref],
))
data['Contribution'][ref] = contribution
methods = collections.defaultdict(list)
for method in ds.itermethods():
m = data.add(
models.Method,
method.id,
id=valid_id(method.id),
name=method.label,
code=method.code,
parameter=method.parameter.strip(),
instrument=method.instrument,
number_of_replicates=method.number_of_replicates,
date=method.date,
comment=method.comment,
detection_limit=method.detection_limit,
detection_limit_unit=method.detection_limit_unit,
total_procedural_blank_value=method.total_procedural_blank_value,
total_procedural_unit=method.total_procedural_unit,
)
methods[(m.code.lower(), m.parameter.lower())].append(m)
for ref in method.references:
DBSession.add(
models.MethodReference(
method=m,
sample_name=ref.sample_name,
sample_measured_value=ref.sample_measured_value,
uncertainty=ref.uncertainty,
uncertainty_unit=ref.uncertainty_unit,
number_of_measurements=ref.number_of_measurements,
))
for ref in method.normalizations:
DBSession.add(
models.Normalization(
method=m,
reference_sample_name=ref.reference_sample_name,
reference_sample_accepted_value=ref.
reference_sample_accepted_value,
citation=ref.citation,
))
parameter = data.add(common.Parameter,
'c',
id='category',
name='Sample category')
for i, opt in enumerate(attr.fields_dict(
pypofatu.models.Sample)['sample_category'].validator.options,
start=1):
data.add(common.DomainElement,
opt,
parameter=parameter,
id=str(i),
name=opt)
DBSession.flush()
assert parameter.pk
# Add Samples and UnitParameters and Measurements
for analysis in analyses:
sample = analysis.sample
vsid = '{0}-{1}'.format(sample.location.id,
data['Contribution'][sample.source_id].id)
vs = data['ValueSet'].get(vsid)
if not vs:
vs = data.add(
common.ValueSet,
vsid,
id=valid_id(vsid),
language_pk=data['Location'][sample.location.id].pk,
parameter_pk=parameter.pk,
contribution_pk=data['Contribution'][sample.source_id].pk,
)
v = data['Sample'].get(sample.id)
if not v:
v = data.add(
models.Sample,
sample.id,
id=valid_id(sample.id),
name=sample.id,
sample_name=sample.sample_name,
sample_comment=sample.sample_comment,
petrography=sample.petrography,
latitude=sample.location.latitude,
longitude=sample.location.longitude,
elevation=sample.location.elevation,
location_comment=sample.location.comment,
site_name=sample.site.name,
site_code=sample.site.code,
site_context=sample.site.context,
site_comment=sample.site.comment,
site_stratigraphic_position=sample.site.stratigraphic_position,
site_stratigraphy_comment=sample.site.stratigraphy_comment,
domainelement=data['DomainElement'][sample.sample_category],
valueset=vs,
artefact_id=sample.artefact.id,
artefact_name=sample.artefact.name,
artefact_category=sample.artefact.category,
artefact_comment=sample.artefact.comment,
artefact_attributes=sample.artefact.attributes,
artefact_collector=sample.artefact.collector,
artefact_collection_type=sample.artefact.collection_type,
artefact_collection_location=sample.artefact.
collection_location,
artefact_collection_comment=sample.artefact.collection_comment,
artefact_fieldwork_date=sample.artefact.fieldwork_date,
)
DBSession.add(
models.SampleReference(
description='sample',
sample=v,
source=data['Source'][sample.source_id]))
for ref in sample.artefact.source_ids:
DBSession.add(
models.SampleReference(description='artefact',
sample=v,
source=data['Source'][ref]))
for ref in sample.site.source_ids:
DBSession.add(
models.SampleReference(description='site',
sample=v,
source=data['Source'][ref]))
a = data.add(
models.Analysis,
analysis.id,
id=better_slug(analysis.id),
name=analysis.id,
sample=v,
)
for i, measurement in enumerate(analysis.measurements):
if i == 0:
method = measurement.method
if method:
a.analyzed_material_1 = method.analyzed_material_1,
a.analyzed_material_2 = method.analyzed_material_2,
a.sample_preparation = method.sample_preparation,
a.chemical_treatment = method.chemical_treatment,
a.technique = method.technique,
a.laboratory = method.laboratory,
a.analyst = method.analyst,
pid = slug(measurement.parameter, lowercase=False)
p = data['Param'].get(pid)
if not p:
p = data.add(models.Param,
pid,
id=pid,
name=measurement.parameter)
data.add(
models.Measurement,
None,
id='{0}-{1}'.format(a.id, p.id),
analysis=a,
method=data['Method'].get(measurement.method.id)
if measurement.method else None,
value=measurement.value,
less=measurement.less,
precision=measurement.value_sd,
sigma=measurement.sd_sigma,
unitparameter=p,
)
def main(args):
data = Data()
doi = input('DOI of the released dataset: ')
dataset = common.Dataset(
id=ewave.__name__,
name='eWAVE',
description='The Electronic World Atlas of Varieties of English',
domain='ewave-atlas.org',
published=date.today(),
license='http://creativecommons.org/licenses/by/3.0/',
contact='*****@*****.**',
jsondata={
'doi': doi,
'license_icon': 'cc-by.png',
'license_name': 'Creative Commons Attribution 3.0 Unported License'})
DBSession.add(dataset)
ed_pattern = re.compile('ed(?P<ord>[0-9]+)$')
for c in args.cldf['contributors.csv']:
contrib = data.add(
models.WaveContributor,
c['ID'],
id=c['ID'],
name=c['Name'],
email=c['Email'],
url=c['URL'],
address=c['Address'],
sortkey=HumanName(c['Name']).last,
)
m = ed_pattern.match(c['ID'])
if m:
common.Editor(dataset=dataset, contributor=contrib, ord=int(m.group('ord')))
for fc in args.cldf['featurecategories.csv']:
data.add(
models.FeatureCategory, fc['ID'],
id=fc['ID'], name=fc['Name'], description=fc['Description'])
for vt in args.cldf['varietytypes.csv']:
data.add(
models.VarietyType, vt['ID'],
id=vt['ID'],
name=vt['Name'],
description=vt['Description'],
jsondata=VARIETY_TYPE_ICONS[vt['ID']],
)
for vt in args.cldf['regions.csv']:
data.add(models.Region, vt['ID'], id=vt['ID'], name=vt['Name'])
for lang in args.cldf['LanguageTable']:
l = data.add(
models.Variety, lang['ID'],
id=lang['ID'],
name=lang['Name'],
latitude=lang['Latitude'],
longitude=lang['Longitude'],
abbr=lang['abbr'],
region=data['Region'][lang['Region_ID']],
type=data['VarietyType'][lang['Type_ID']],
)
if lang['Glottocode']:
add_language_codes(data, l, None, glottocode=lang['Glottocode'])
c = data.add(
models.WaveContribution, lang['ID'],
id=str(lang['ID']),
name=lang['Name'],
description=lang['Description'],
variety=l)
for i, cid in enumerate(lang['Contributor_ID']):
DBSession.add(common.ContributionContributor(
contribution=c,
contributor=data['WaveContributor'][cid],
ord=i+1,
))
for param in args.cldf['ParameterTable']:
data.add(
models.Feature, param['ID'],
id=param['ID'],
category=data['FeatureCategory'][param['Category_ID']],
name=param['Name'],
description=param['Description'],
jsondata={'example_source': param['Example_Source']})
for de in args.cldf['CodeTable']:
data.add(
common.DomainElement, de['ID'],
id=de['ID'],
parameter=data['Feature'][de['Parameter_ID']],
name=de['Name'],
description=de['Description'],
jsondata={'color': CODE_COLORS[de['Name']]},
number=de['Number'])
for rec in bibtex.Database.from_file(args.cldf.bibpath):
data.add(common.Source, slug(rec.id), _obj=bibtex2source(rec))
for example in args.cldf['ExampleTable']:
s = data.add(
common.Sentence, example['ID'],
id=example['ID'],
name=example['Primary_Text'],
gloss='\t'.join(example['Gloss']) if example['Gloss'] else None,
comment=example['Comment'] or None,
description=example['Translated_Text'] or None,
language=data['Variety'][example['Language_ID']])
for ref in example['Source']:
sid, pages = Sources.parse(ref)
DBSession.add(common.SentenceReference(
sentence=s, source=data['Source'][sid], description=pages, key=sid))
for value in args.cldf['ValueTable']:
de = data['DomainElement'][value['Code_ID']]
vs = data.add(
common.ValueSet, value['ID'],
id=value['ID'],
contribution=data['WaveContribution'][value['Language_ID']],
parameter=data['Feature'][value['Parameter_ID']],
jsondata=de.jsondata,
language=data['Variety'][value['Language_ID']])
v = data.add(
common.Value, value['ID'],
id=value['ID'],
domainelement=de,
valueset=vs)
for eid in value['Example_ID']:
DBSession.add(common.ValueSentence(sentence=data['Sentence'][eid], value=v))
