def main(args):
fts.index('fts_index', Word.fts, DBSession.bind)
DBSession.execute("CREATE EXTENSION IF NOT EXISTS unaccent WITH SCHEMA public;")
data = Data()
dataset = common.Dataset(
id=dictionaria.__name__,
name="Dictionaria",
description="The Dictionary Journal",
published=date(2017, 3, 30),
contact='*****@*****.**',
domain='dictionaria.clld.org',
publisher_name="Max Planck Institute for the Science of Human History",
publisher_place="Jena",
publisher_url="https://shh.mpg.de",
license="http://creativecommons.org/licenses/by/4.0/",
jsondata={
'license_icon': 'cc-by.png',
'license_name': 'Creative Commons Attribution 4.0 International License'})
for i, (id_, name) in enumerate([
('haspelmathmartin', 'Martin Haspelmath'),
('moselulrike', 'Ulrike Mosel'),
('stiebelsbarbara', 'Barbara Stiebels')
]):
ed = data.add(common.Contributor, id_, id=id_, name=name)
common.Editor(dataset=dataset, contributor=ed, ord=i + 1)
DBSession.add(dataset)
for id_, name in LGR_ABBRS.items():
DBSession.add(common.GlossAbbreviation(id=id_, name=name))
comparison_meanings = {}
print('loading concepts ...')
glosses = set()
concepticon = Concepticon(
REPOS.joinpath('..', '..', 'concepticon', 'concepticon-data'))
if not args.no_concepts:
for conceptset in concepticon.conceptsets.values():
if conceptset.gloss in glosses:
continue
glosses.add(conceptset.gloss)
cm = data.add(
ComparisonMeaning,
conceptset.id,
id=conceptset.id,
name=conceptset.gloss.lower(),
description=conceptset.definition,
concepticon_url='http://concepticon.clld.org/parameters/%s' % conceptset.id)
comparison_meanings[cm.id] = cm
DBSession.flush()
print('... done')
comparison_meanings = {k: v.pk for k, v in comparison_meanings.items()}
submissions = []
for submission in REPOS.joinpath(
'submissions-internal' if args.internal else 'submissions').glob('*'):
if not submission.is_dir():
continue
try:
submission = Submission(submission)
except ValueError:
continue
md = submission.md
if md is None:
continue
if not md['date_published']:
continue
id_ = submission.id
if args.dict and args.dict != id_ and args.dict != 'all':
continue
lmd = md['language']
props = md.get('properties', {})
props.setdefault('custom_fields', [])
props['metalanguage_styles'] = {}
for v, s in zip(props.get('metalanguages', {}).values(),
['success', 'info', 'warning', 'important']):
props['metalanguage_styles'][v] = s
props['custom_fields'] = ['lang-' + f if f in props['metalanguage_styles'] else f
for f in props['custom_fields']]
language = data['Variety'].get(lmd['glottocode'])
if not language:
language = data.add(
Variety, lmd['glottocode'], id=lmd['glottocode'], name=lmd['name'])
md['date_published'] = md['date_published'] or date.today().isoformat()
if '-' not in md['date_published']:
md['date_published'] = md['date_published'] + '-01-01'
dictionary = data.add(
Dictionary,
id_,
id=id_,
number=md.get('number'),
name=props.get('title', lmd['name'] + ' dictionary'),
description=submission.description,
language=language,
published=date(*map(int, md['date_published'].split('-'))),
jsondata=props)
for i, spec in enumerate(md['authors']):
if not isinstance(spec, dict):
cname, address = spec, None
spec = {}
else:
cname, address = spec['name'], spec.get('affiliation')
name = HumanName(cname)
cid = slug('%s%s' % (name.last, name.first))
contrib = data['Contributor'].get(cid)
if not contrib:
contrib = data.add(
common.Contributor,
cid,
id=cid,
name=cname,
address=address,
url=spec.get('url'),
email=spec.get('email'))
DBSession.add(common.ContributionContributor(
ord=i + 1,
primary=True,
contributor=contrib,
contribution=dictionary))
submissions.append((dictionary.id, language.id, submission))
transaction.commit()
for did, lid, submission in submissions:
#if submission.id != 'sidaama':
# continue
transaction.begin()
print('loading %s ...' % submission.id)
dictdata = Data()
lang = Variety.get(lid)
submission.load_examples(Dictionary.get(did), dictdata, lang)
submission.dictionary.load(
submission,
dictdata,
Dictionary.get(did),
lang,
comparison_meanings,
OrderedDict(submission.md.get('properties', {}).get('labels', [])))
transaction.commit()
print('... done')
transaction.begin()
load_families(
Data(),
[v for v in DBSession.query(Variety) if re.match('[a-z]{4}[0-9]{4}', v.id)],
glottolog_repos='../../glottolog3/glottolog')
"TD": "time depth/ proximality marker",
"TELIC": "telic",
"TEMPRY": "temporary",
"TH": "thematic suffix",
"THM": "theme (i.e. the semantic role)",
"TOD.PST": "today past",
"TRASL": "traslative",
"TRI": "trial",
"UNSP": "unspecified",
"VBLZ": "verbalizer",
"VENT": "ventive",
"VIS": "visual evidential",
"VP": "verb phrase",
}
for k, v in LGR_ABBRS.items():
ABBRS.setdefault(k, v)
def get_source(id): # pragma: no cover
"""retrieve a source record from wals_refdb
"""
field_map = {
"onlineversion": "url",
"gbs_id": "google_book_search_id",
"doi": "jsondata",
"cited": "jsondata",
"conference": "jsondata",
"iso_code": "jsondata",
"olac_field": "jsondata",
"wals_code": "jsondata",
def main(args):
data = Data()
files_dir.rmtree()
files_dir.mkdir()
editors = OrderedDict()
editors['Susanne Maria Michaelis'] = None
editors['Philippe Maurer'] = None
editors['Martin Haspelmath'] = None
editors['Magnus Huber'] = None
for row in read('People'):
name = row['First name'] + ' ' if row['First name'] else ''
name += row['Last name']
kw = dict(
name=name,
id=slug('%(Last name)s%(First name)s' % row),
url=row['Contact Website'].split()[0] if row['Contact Website'] else None,
address=row['Comments on database'],
)
contrib = data.add(common.Contributor, row['Author ID'], **kw)
if kw['name'] in editors:
editors[kw['name']] = contrib
DBSession.flush()
dataset = common.Dataset(
id='apics',
name='APiCS Online',
description='Atlas of Pidgin and Creole Language Structures Online',
domain='apics-online.info',
published=date(2013, 8, 15),
#
# TODO: switch license!
#
license='http://creativecommons.org/licenses/by/3.0/',
contact='*****@*****.**',
jsondata={
'license_icon': 'cc-by.png',
'license_name': 'Creative Commons Attribution 3.0 Unported License'})
DBSession.add(dataset)
for i, editor in enumerate(editors.values()):
common.Editor(dataset=dataset, contributor=editor, ord=i + 1)
colors = dict((row['ID'], row['RGB_code']) for row in read('Colours'))
abbrs = {}
for id_, name in LGR_ABBRS.items():
DBSession.add(common.GlossAbbreviation(id=id_, name=name))
abbrs[id_] = 1
for id_, name in {
'C**T': 'clitic',
'IMPF': 'imperfect',
'INTERM': 'intermediate',
'NCOMPL': 'noncompletive',
'NONFUT': 'nonfuture',
'NPROX': 'nonproximal',
'NSG': 'nonsingular',
'PP': 'past participle',
'PROP': 'proprietive',
'TMA': 'tense-mood-aspect',
}.items():
DBSession.add(common.GlossAbbreviation(id=id_, name=name))
abbrs[id_] = 1
with open(data_dir.joinpath('non-lgr-gloss-abbrs.csv'), 'rb') as csvfile:
for row in csv.reader(csvfile):
for match in GLOSS_ABBR_PATTERN.finditer(row[1]):
if match.group('abbr') not in abbrs:
abbrs[match.group('abbr')] = 1
DBSession.add(
common.GlossAbbreviation(id=match.group('abbr'), name=row[0]))
non_bibs = {}
for row in read('References', 'Reference_ID'):
if row['Reference_type'] == 'Non-bib':
non_bibs[row['Reference_ID']] = row['Reference_name']
continue
if isinstance(row['Year'], int):
year_int = row['Year']
year = str(row['Year'])
elif row['Year']:
year_int = None
for m in re.finditer('(?P<year>(1|2)[0-9]{3})', row['Year']):
year_int = int(m.group('year'))
break
year = row['Year']
else:
year, year_int = None, None
title = row['Article_title'] or row['Book_title']
attrs = {}
jsondata = {}
for attr, field in {
'Additional_information': 'note',
'Article_title': 'title',
'Book_title': 'booktitle',
'City': 'address',
'Editors': 'editor',
'Full_reference': None,
'Issue': None,
'Journal': 'journal',
'Language_codes': None,
'LaTeX_cite_key': None,
'Pages': 'pages',
'Publisher': 'publisher',
'Reference_type': 'type',
'School': 'school',
'Series_title': 'series',
'URL': 'url',
'Volume': 'volume',
}.items():
value = row.get(attr)
if not isinstance(value, int):
value = (value or '').strip()
if attr == 'Issue' and value:
try:
value = str(int(value))
except ValueError:
pass
if value:
if field:
attrs[field] = value
else:
jsondata[attr] = value
p = data.add(
common.Source, row['Reference_ID'],
id=row['Reference_ID'],
name=row['Reference_name'],
description=title,
author=row['Authors'],
year=year,
year_int=year_int,
bibtex_type=getattr(EntryType, row['BibTeX_type'] or 'misc'),
jsondata=jsondata,
**attrs)
if p.bibtex_type.value == 'misc' and not p.description:
p.description = p.note
DBSession.flush()
DBSession.flush()
gt = {}
p = re.compile('[0-9]+\_(?P<name>[^\_]+)\_(GT|Text)')
for d in data_dir.joinpath('gt').files():
m = p.search(unicode(d.basename()))
if m:
for part in m.group('name').split('&'):
# make sure we prefer files named "Text_for_soundfile"
if slug(unicode(part)) not in gt or 'Text_for_' in d.basename():
gt[slug(unicode(part))] = d
gt_audio = {}
p = re.compile('(?P<name>[^\.]+)\.mp3')
for d in data_dir.joinpath('gt', 'audio').files():
m = p.search(unicode(d.basename()))
assert m
for part in m.group('name').split('&'):
gt_audio[slug(unicode(part))] = d
with open(args.data_file('infobox.json')) as fp:
infobox = json.load(fp)
for row in read('Languages', 'Order_number'):
lon, lat = [float(c.strip()) for c in row['map_coordinates'].split(',')]
kw = dict(
name=row['Language_name'],
id=str(row['Order_number']),
latitude=lat,
longitude=lon,
region=row['Category_region'],
#base_language=row['Category_base_language'],
)
lect = data.add(models.Lect, row['Language_ID'], **kw)
DBSession.flush()
for i, item in enumerate(infobox[lect.id]):
DBSession.add(common.Language_data(
object_pk=lect.pk, ord=i, key=item[0], value=item[1]))
if row["Languages_contribution_documentation::Lect_description_checked_status"] == "Checked":
desc = row.get('Languages_contribution_documentation::Lect description', '')
else:
desc = ''
c = data.add(
models.ApicsContribution, row['Language_ID'],
id=row['Order_number'],
name=row['Language_name'],
description=desc,
survey_reference=data['Source'][row['Survey_reference_ID']],
language=lect)
if slug(row['Language_name']) in gt:
f = common.Contribution_files(
object=c, id='%s-gt.pdf' % c.id, name='Glossed text', mime_type='application/pdf')
f.create(files_dir, file(gt[slug(row['Language_name'])]).read())
else:
print '--- no glossed text for:', row['Language_name']
if slug(row['Language_name']) in gt_audio:
f = common.Contribution_files(
object=c, id='%s-gt.mp3' % c.id, name='Glossed text audio', mime_type='audio/mpeg')
f.create(files_dir, file(gt_audio[slug(row['Language_name'])]).read())
else:
print '--- no audio for:', row['Language_name']
#
# TODO: for michif, 75, add link http://www.youtube.com/watch?v=f0C4cODsSyE
#
iso = None
if row['ISO_code'] and len(row['ISO_code']) == 3:
iso = row['ISO_code'].lower()
if 'iso:%s' % row['ISO_code'] not in data['Identifier']:
data.add(
common.Identifier, 'iso:%s' % row['ISO_code'],
id=row['ISO_code'].lower(),
name=row['ISO_code'].lower(),
type='iso639-3')
DBSession.add(common.LanguageIdentifier(
language=data['Lect'][row['Language_ID']],
identifier=data['Identifier']['iso:%s' % row['ISO_code']]))
if row['Language_name_ethnologue']:
if row['Language_name_ethnologue'] not in data['Identifier']:
data.add(
common.Identifier, row['Language_name_ethnologue'],
id=iso or 'ethnologue:%s' % row['Language_name_ethnologue'],
name=row['Language_name_ethnologue'],
type='ethnologue')
DBSession.add(common.LanguageIdentifier(
language=data['Lect'][row['Language_ID']],
identifier=data['Identifier'][row['Language_name_ethnologue']]))
example_count = {}
soundfiles = {}
for p in data_dir.joinpath('Soundfiles_Examples').files():
soundfiles[p.namebase] = p
for row in read('Examples', 'Order_number'):
assert row['Language_ID']
lang = data['Lect'][row['Language_ID']]
id_ = '%(Language_ID)s-%(Example_number)s' % row
atext, gloss = igt(row)
example_count[row['Language_ID']] = max([example_count.get(row['Language_ID'], 1), row['Example_number']])
p = data.add(
common.Sentence, id_,
id='%s-%s' % (lang.id, row['Example_number']),
name=row['Text'] or row['Analyzed_text'],
description=row['Translation'],
type=row['Type'].strip().lower() if row['Type'] else None,
comment=row['Comments'],
gloss=gloss,
analyzed=atext,
markup_text=normalize_markup(row['z_calc_Text_CSS']),
markup_gloss=normalize_markup(row['z_calc_Gloss_CSS']),
markup_comment=normalize_markup(row['z_calc_Comments_CSS']),
markup_analyzed=normalize_markup(row['z_calc_Analyzed_text_CSS']),
original_script=row['Original_script'],
jsondata={'sort': row['Order_number']},
language=lang)
if id_ in soundfiles:
#print '---> sound', id_
f = common.Sentence_files(
object=p, id='%s.mp3' % p.id, name='Audio', mime_type='audio/mpeg')
f.create(files_dir, file(soundfiles[id_]).read())
if row['Reference_ID']:
if row['Reference_ID'] in data['Source']:
source = data['Source'][row['Reference_ID']]
DBSession.add(common.SentenceReference(
sentence=p,
source=source,
key=source.id,
description=row['Reference_pages'],
))
else:
p.source = non_bibs[row['Reference_ID']]
DBSession.flush()
for row in read('Language_references'):
if row['Reference_ID'] not in data['Source']:
assert row['Reference_ID'] in non_bibs
continue
assert row['Language_ID'] in data['ApicsContribution']
source = data['Source'][row['Reference_ID']]
DBSession.add(common.ContributionReference(
contribution=data['ApicsContribution'][row['Language_ID']],
source=source,
description=row['Pages'],
key=source.id))
#
# global counter for features - across feature types
#
feature_count = 0
for row in read('Features', 'Feature_number'):
id_ = str(row['Feature_number'])
if int(id_) > feature_count:
feature_count = int(id_)
wals_id = None
if row['WALS_match'] == 'Total':
if isinstance(row['WALS_No.'], int):
wals_id = row['WALS_No.']
else:
wals_id = int(row['WALS_No.'].split('.')[0].strip())
p = data.add(
models.Feature, row['Feature_code'],
name=row['Feature_name'],
id=id_,
description=row['Feature_annotation_publication'],
markup_description=normalize_markup(row['z_calc_Feature_annotation_publication_CSS']),
feature_type='primary',
multivalued=row['Value_relation_type'] != 'Single',
area=row['Feature_area'],
wals_id=wals_id)
names = {}
for i in range(1, 10):
if not row['Value%s_publication' % i] or not row['Value%s_publication' % i].strip():
continue
name = row['Value%s_publication' % i].strip()
if name in names:
name += ' (%s)' % i
names[name] = 1
de = data.add(
common.DomainElement, '%s-%s' % (row['Feature_code'], i),
id='%s-%s' % (id_, i),
name=name,
parameter=p,
abbr=row['Value%s_for_book_maps' % i] if p.id != '0' else name,
number=int(row['Value%s_value_number_for_publication' % i]),
jsondata={'color': colors[row['Value_%s_colour_ID' % i]]},
)
if row['Authors_FeatureArticles']:
authors, _ = row['Authors_FeatureArticles'].split('and the APiCS')
authors = authors.strip()
if authors.endswith(','):
authors = authors[:-1].strip()
for i, name in enumerate(authors.split(',')):
assert name.strip() in editors
p._authors.append(models.FeatureAuthor(
ord=i + 1, contributor=editors[name.strip()]))
DBSession.flush()
primary_to_segment = {123: 63, 126: 35, 128: 45, 130: 41}
segment_to_primary = dict(zip(
primary_to_segment.values(), primary_to_segment.keys()))
number_map = {}
names = {}
for row in read('Segment_features', 'Order_number'):
symbol = row['Segment_symbol']
if row['Segment_name'] == 'voiceless dental/alveolar sibilant affricate':
symbol = 't\u0361s'
truth = lambda s: s and s.strip().lower() == 'yes'
name = '%s - %s' % (symbol, row['Segment_name'])
if name in names:
number_map[row['Segment_feature_number']] = names[name]
continue
number_map[row['Segment_feature_number']] = row['Segment_feature_number']
names[name] = row['Segment_feature_number']
feature_count += 1
if row['Segment_feature_number'] in segment_to_primary:
primary_to_segment[segment_to_primary[row['Segment_feature_number']]] = str(feature_count)
p = data.add(
models.Feature, row['Segment_feature_number'],
name=name,
id=str(feature_count),
feature_type='segment',
area='Vowels' if truth(row['Vowel']) else (
'Obstruent consonants' if truth(row['Obstruent'])
else 'Sonorant consonants'),
jsondata=dict(
number=int(row['Segment_feature_number']),
vowel=truth(row['Vowel']),
consonant=truth(row['Consonant']),
obstruent=truth(row['Obstruent']),
core_list=truth(row['Core_list_segment']),
symbol=symbol,
))
for i, spec in SEGMENT_VALUES.items():
data.add(
common.DomainElement,
'%s-%s' % (row['Segment_feature_number'], spec[0]),
id='%s-%s' % (p.id, i),
name=spec[0],
parameter=p,
jsondata={'color': spec[1]},
number=i)
print '--> remapped:', primary_to_segment
DBSession.flush()
for row in read('Sociolinguistic_features', 'Sociolinguistic_feature_number'):
feature_count += 1
p = data.add(
models.Feature, row['Sociolinguistic_feature_code'],
name=row['Sociolinguistic_feature_name'],
id='%s' % feature_count,
area='Sociolinguistic',
feature_type='sociolinguistic')
names = {}
for i in range(1, 7):
id_ = '%s-%s' % (row['Sociolinguistic_feature_code'], i)
if row['Value%s' % i] and row['Value%s' % i].strip():
name = row['Value%s' % i].strip()
if name in names:
name += ' (%s)' % i
names[name] = 1
else:
name = '%s - %s' % (row['Sociolinguistic_feature_name'], i)
kw = dict(id='%s-%s' % (p.id, i), name=name, parameter=p, number=i)
de = data.add(
common.DomainElement,
id_,
id='%s-%s' % (p.id, i),
name=name,
parameter=p,
number=i,
jsondata={'color': colors.values()[i]})
sd = {}
soundfiles = {}
for p in data_dir.joinpath('Soundfiles_Segments').files():
soundfiles[p.namebase] = p
for row in read('Segment_data'):
if row['Segment_feature_number'] not in number_map:
continue
number = number_map[row['Segment_feature_number']]
#Language_ID,Segment_feature_number,Comments,Audio_file_name,Example_word,
#Example_word_gloss,Presence_in_the_language,Refers_to_references_Reference_ID
if not row['Presence_in_the_language']:
continue
lang = data['Lect'][row['Language_ID']]
param = data['Feature'][number]
id_ = '%s-%s' % (lang.id, param.id)
if id_ in sd:
assert row['c_Record_is_a_duplicate'] == 'Yes'
continue
sd[id_] = 1
valueset = data.add(
common.ValueSet,
id_,
id=id_,
parameter=param,
language=lang,
contribution=data['ApicsContribution'][row['Language_ID']],
description=row['Comments'],
markup_description=normalize_markup(row['z_calc_Comments_CSS']),
)
v = data.add(
common.Value,
id_,
id=id_,
frequency=float(100),
valueset=valueset,
domainelement=data['DomainElement']['%s-%s' % (
number, row['Presence_in_the_language'])],
)
if row['Example_word'] and row['Example_word_gloss']:
example_count[row['Language_ID']] += 1
p = data.add(
common.Sentence, '%s-p%s' % (lang.id, data['Feature'][number].id),
id='%s-%s' % (lang.id, example_count[row['Language_ID']]),
name=row['Example_word'],
description=row['Example_word_gloss'],
language=lang)
sid = '%(Language_ID)s-%(Segment_feature_number)s' % row
if sid in soundfiles:
print '---> sound', sid
f = common.Sentence_files(
object=p, id='%s.mp3' % p.id, name='Audio', mime_type='audio/mpeg')
f.create(files_dir, file(soundfiles[sid]).read())
DBSession.add(common.ValueSentence(value=v, sentence=p))
source = data['Source'].get(row['Refers_to_references_Reference_ID'])
if source:
DBSession.add(common.ValueSetReference(
valueset=valueset, source=source, key=source.id))
elif row['Refers_to_references_Reference_ID'] in non_bibs:
valueset.source = non_bibs[row['Refers_to_references_Reference_ID']]
lects = defaultdict(lambda: 1)
lect_map = {}
records = {}
false_values = {}
no_values = {}
wals_value_number = {}
for row in read('wals'):
if row['z_calc_WALS_value_number']:
wals_value_number[row['Data_record_id']] = row['z_calc_WALS_value_number']
def prefix(attr, _prefix):
if _prefix:
return '%s_%s' % (_prefix, attr)
return attr.capitalize()
for _prefix, abbr, num_values in [
('', '', 10),
('Sociolinguistic', 'sl', 7),
]:
for row in read(prefix('data', _prefix)):
if not row[prefix('feature_code', _prefix)]:
print 'no associated feature for', prefix('data', _prefix), row[prefix('data_record_id', _prefix)]
continue
lid = row['Language_ID']
lect_attr = row.get('Lect_attribute', 'my default lect').lower()
if lect_attr != 'my default lect':
if (row['Language_ID'], row['Lect_attribute']) in lect_map:
lid = lect_map[(row['Language_ID'], row['Lect_attribute'])]
else:
lang = data['Lect'][row['Language_ID']]
c = lects[row['Language_ID']]
lid = '%s-%s' % (row['Language_ID'], c)
kw = dict(
name='%s (%s)' % (lang.name, row['Lect_attribute']),
id='%s' % (1000 + 10 * int(lang.id) + c),
latitude=lang.latitude,
longitude=lang.longitude,
description=row['Lect_attribute'],
language=lang,
)
data.add(models.Lect, lid, **kw)
lects[row['Language_ID']] += 1
lect_map[(row['Language_ID'], row['Lect_attribute'])] = lid
id_ = abbr + str(row[prefix('data_record_id', _prefix)])
assert id_ not in records
records[id_] = 1
assert row[prefix('feature_code', _prefix)] in data['Feature']
#if row[prefix('feature_code', _prefix)] not in data['Feature']:
# print row[prefix('feature_code', _prefix)]
# print str(row[prefix('data_record_id', _prefix)])
# raise ValueError
language = data['Lect'][lid]
parameter = data['Feature'][row[prefix('feature_code', _prefix)]]
valueset = common.ValueSet(
id='%s-%s' % (language.id, parameter.id),
description=row['Comments_on_value_assignment'],
markup_description=normalize_markup(row.get('z_calc_Comments_on_value_assignment_CSS')),
)
values_found = {}
for i in range(1, num_values):
if not row['Value%s_true_false' % i]:
continue
if row['Value%s_true_false' % i].strip().lower() != 'true':
assert row['Value%s_true_false' % i].strip().lower() == 'false'
false_values[row[prefix('data_record_id', _prefix)]] = 1
continue
values_found['%s-%s' % (id_, i)] = dict(
id='%s-%s' % (valueset.id, i),
#valueset=valueset,
domainelement=data['DomainElement']['%s-%s' % (
row[prefix('feature_code', _prefix)], i)],
confidence=row['Value%s_confidence' % i],
frequency=float(row['c_V%s_frequency_normalised' % i])
if _prefix == '' else 100)
if values_found:
if row[prefix('data_record_id', _prefix)] in wals_value_number:
valueset.jsondata = {'wals_value_number': wals_value_number.pop(row[prefix('data_record_id', _prefix)])}
valueset.parameter = parameter
valueset.language = language
valueset.contribution = data['ApicsContribution'][row['Language_ID']]
valueset = data.add(common.ValueSet, id_, _obj=valueset)
for i, item in enumerate(values_found.items()):
if i > 0 and not parameter.multivalued:
print 'multiple values for single-valued parameter: %s' % id_
break
id_, kw = item
kw['valueset'] = valueset
value = data.add(common.Value, id_, **kw)
#
# store references to additional data for segments which should be reused
# for corresponding primary features!
#
if int(parameter.id) in primary_to_segment:
assert len(values_found) == 1
seg_id = '%s-%s' % (language.id, primary_to_segment[int(parameter.id)])
seg_valueset = data['ValueSet'][seg_id]
seg_value = data['Value'][seg_id]
if not valueset.description and seg_valueset.description:
valueset.description = seg_valueset.description
for s in seg_value.sentence_assocs:
DBSession.add(common.ValueSentence(value=value, sentence=s.sentence))
for r in seg_valueset.references:
DBSession.add(common.ValueSetReference(
valueset=valueset, source=r.source, key=r.key))
if not valueset.source and seg_valueset.source:
valueset.source = seg_valueset.source
DBSession.flush()
else:
no_values[id_] = 1
DBSession.flush()
for prefix, abbr, num_values in [
('D', '', 10),
('Sociolinguistic_d', 'sl', 7),
]:
for row in read(prefix + 'ata_references'):
assert row['Reference_ID'] in data['Source'] or row['Reference_ID'] in non_bibs
try:
vs = data['ValueSet'][abbr + str(row[prefix + 'ata_record_id'])]
if row['Reference_ID'] in data['Source']:
source = data['Source'][row['Reference_ID']]
DBSession.add(common.ValueSetReference(
valueset=vs,
source=source,
key=source.id,
description=row['Pages'],
))
else:
if vs.source:
vs.source += '; ' + non_bibs[row['Reference_ID']]
else:
vs.source = non_bibs[row['Reference_ID']]
except KeyError:
print('Reference for unknown dataset: %s'
% row[prefix + 'ata_record_id'])
continue
DBSession.flush()
missing = 0
for row in read('Value_examples'):
try:
DBSession.add(common.ValueSentence(
value=data['Value']['%(Data_record_id)s-%(Value_number)s' % row],
sentence=data['Sentence']['%(Language_ID)s-%(Example_number)s' % row],
description=row['Notes'],
))
except KeyError:
missing += 1
print('%s Value_examples are missing data' % missing)
print('%s data sets with false values' % len(false_values))
print('%s data sets without values' % len(no_values))
for k, v in wals_value_number.items():
print 'unclaimed wals value number:', k, v
for i, row in enumerate(read('Contributors')):
kw = dict(
contribution=data['ApicsContribution'][row['Language ID']],
contributor=data['Contributor'][row['Author ID']]
)
if row['Order_of_appearance']:
kw['ord'] = int(float(row['Order_of_appearance']))
data.add(common.ContributionContributor, i, **kw)
DBSession.flush()
def main(args):
fts.index('fts_index', Word.fts, DBSession.bind)
DBSession.execute("CREATE EXTENSION IF NOT EXISTS unaccent WITH SCHEMA public;")
if DBSession.bind.dialect.name == 'postgresql':
Index('ducet', collkey(common.Unit.name)).create(DBSession.bind)
data = Data()
dataset = common.Dataset(
id=dictionaria.__name__,
name="Dictionaria",
description="The Dictionary Journal",
published=date(2017, 3, 30),
contact='*****@*****.**',
domain='dictionaria.clld.org',
publisher_name="Max Planck Institute for the Science of Human History",
publisher_place="Jena",
publisher_url="https://shh.mpg.de",
license="http://creativecommons.org/licenses/by/4.0/",
jsondata={
'license_icon': 'cc-by.png',
'license_name': 'Creative Commons Attribution 4.0 International License'})
for i, (id_, name) in enumerate([
('haspelmathmartin', 'Martin Haspelmath'),
('stiebelsbarbara', 'Barbara Stiebels')
]):
ed = data.add(common.Contributor, id_, id=id_, name=name)
common.Editor(dataset=dataset, contributor=ed, ord=i + 1)
DBSession.add(dataset)
for id_, name in LGR_ABBRS.items():
DBSession.add(common.GlossAbbreviation(id=id_, name=name))
comparison_meanings = {}
print('loading concepts ...')
glosses = set()
concepticon = Concepticon(
REPOS.joinpath('..', '..', 'concepticon', 'concepticon-data'))
if not args.no_concepts:
for conceptset in concepticon.conceptsets.values():
if conceptset.gloss in glosses:
continue
glosses.add(conceptset.gloss)
cm = data.add(
ComparisonMeaning,
conceptset.id,
id=conceptset.id,
name=conceptset.gloss.lower(),
description=conceptset.definition,
concepticon_url='http://concepticon.clld.org/parameters/%s' % conceptset.id)
comparison_meanings[cm.id] = cm
DBSession.flush()
print('... done')
comparison_meanings = {k: v.pk for k, v in comparison_meanings.items()}
submissions = []
for submission in REPOS.joinpath(
'submissions-internal' if args.internal else 'submissions').glob('*'):
if not submission.is_dir():
continue
try:
submission = Submission(submission)
except ValueError:
continue
md = submission.md
if md is None:
print('no md', submission.id)
continue
if not md['date_published']:
print('no date', submission.id)
continue
id_ = submission.id
if args.dict and args.dict != id_ and args.dict != 'all':
print('not selected', submission.id)
continue
lmd = md['language']
props = md.get('properties', {})
props.setdefault('custom_fields', [])
props['metalanguage_styles'] = {}
for v, s in zip(props.get('metalanguages', {}).values(),
['success', 'info', 'warning', 'important']):
props['metalanguage_styles'][v] = s
props['custom_fields'] = ['lang-' + f if f in props['metalanguage_styles'] else f
for f in props['custom_fields']]
props.setdefault('choices', {})
language = data['Variety'].get(lmd['glottocode'])
if not language:
language = data.add(
Variety, lmd['glottocode'], id=lmd['glottocode'], name=lmd['name'])
md['date_published'] = md['date_published'] or date.today().isoformat()
if '-' not in md['date_published']:
md['date_published'] = md['date_published'] + '-01-01'
dictionary = data.add(
Dictionary,
id_,
id=id_,
number=md.get('number'),
name=props.get('title', lmd['name'] + ' dictionary'),
description=submission.description,
language=language,
published=date(*map(int, md['date_published'].split('-'))),
doi=md.get('doi'),
jsondata=props)
for i, spec in enumerate(md['authors']):
if not isinstance(spec, dict):
cname, address = spec, None
spec = {}
else:
cname, address = spec['name'], spec.get('affiliation')
name = HumanName(cname)
cid = slug('%s%s' % (name.last, name.first))
contrib = data['Contributor'].get(cid)
if not contrib:
contrib = data.add(
common.Contributor,
cid,
id=cid,
name=cname,
address=address,
url=spec.get('url'),
email=spec.get('email'))
DBSession.add(common.ContributionContributor(
ord=i + 1,
primary=spec.get('primary', True),
contributor=contrib,
contribution=dictionary))
submissions.append((dictionary.id, language.id, submission))
transaction.commit()
for did, lid, submission in submissions:
transaction.begin()
print('loading %s ...' % submission.id)
dictdata = Data()
lang = Variety.get(lid)
submission.load_sources(Dictionary.get(did), dictdata)
submission.load_examples(Dictionary.get(did), dictdata, lang)
submission.dictionary.load(
submission,
dictdata,
Dictionary.get(did),
lang,
comparison_meanings,
OrderedDict(submission.md.get('properties', {}).get('labels', [])))
transaction.commit()
print('... done')
transaction.begin()
load_families(
Data(),
[v for v in DBSession.query(Variety) if re.match('[a-z]{4}[0-9]{4}', v.id)],
glottolog_repos='../../glottolog/glottolog')
"TD": "time depth/ proximality marker",
"TELIC": "telic",
"TEMPRY": "temporary",
"TH": "thematic suffix",
"THM": "theme (i.e. the semantic role)",
"TOD.PST": "today past",
"TRASL": "traslative",
"TRI": "trial",
"UNSP": "unspecified",
"VBLZ": "verbalizer",
"VENT": "ventive",
"VIS": "visual evidential",
"VP": "verb phrase",
}
for k, v in LGR_ABBRS.items():
ABBRS.setdefault(k, v)
def get_source(id): # pragma: no cover
"""retrieve a source record from wals_refdb
"""
field_map = {
'onlineversion': 'url',
'gbs_id': 'google_book_search_id',
'doi': 'jsondata',
'cited': 'jsondata',
'conference': 'jsondata',
'iso_code': 'jsondata',
'olac_field': 'jsondata',
'wals_code': 'jsondata',
def main(args):
data = Data()
editors = OrderedDict()
editors['Susanne Maria Michaelis'] = None
editors['Philippe Maurer'] = None
editors['Martin Haspelmath'] = None
editors['Magnus Huber'] = None
for row in read(args, 'People'):
name = row['First name'] + ' ' if row['First name'] else ''
name += row['Last name']
kw = dict(
name=name,
id=slug('%(Last name)s%(First name)s' % row),
url=row['Contact Website'].split()[0]
if row['Contact Website'] else None,
address=row['Comments on database'],
)
contrib = data.add(common.Contributor, row['Author ID'], **kw)
if kw['name'] in editors:
editors[kw['name']] = contrib
DBSession.flush()
dataset = common.Dataset(
id='apics',
name='APiCS Online',
description='Atlas of Pidgin and Creole Language Structures Online',
domain='apics-online.info',
published=date(2013, 11, 4),
license='http://creativecommons.org/licenses/by/3.0/',
contact='*****@*****.**',
jsondata={
'license_icon': 'cc-by.png',
'license_name': 'Creative Commons Attribution 3.0 Unported License'
})
DBSession.add(dataset)
for i, editor in enumerate(editors.values()):
common.Editor(dataset=dataset, contributor=editor, ord=i + 1)
colors = dict(
(row['ID'], row['RGB_code']) for row in read(args, 'Colours'))
abbrs = {}
for id_, name in LGR_ABBRS.items():
DBSession.add(common.GlossAbbreviation(id=id_, name=name))
abbrs[id_] = 1
for id_, name in {
'C**T': 'clitic',
'IMPF': 'imperfect',
'INTERM': 'intermediate',
'NCOMPL': 'noncompletive',
'NONFUT': 'nonfuture',
'NPROX': 'nonproximal',
'NSG': 'nonsingular',
'PP': 'past participle',
'PROP': 'proprietive',
'TMA': 'tense-mood-aspect',
}.items():
DBSession.add(common.GlossAbbreviation(id=id_, name=name))
abbrs[id_] = 1
for row in reader(args.data_file('non-lgr-gloss-abbrs.csv'),
delimiter=',',
namedtuples=True):
for match in GLOSS_ABBR_PATTERN.finditer(row.standard):
if match.group('abbr') not in abbrs:
abbrs[match.group('abbr')] = 1
DBSession.add(
common.GlossAbbreviation(id=match.group('abbr'),
name=row.meaning))
non_bibs = {}
for row in read(args, 'References', 'Reference_ID'):
if row['Reference_type'] == 'Non-bib':
non_bibs[row['Reference_ID']] = row['Reference_name']
continue
if isinstance(row['Year'], int):
year_int = row['Year']
year = str(row['Year'])
elif row['Year']:
year_int = None
for m in re.finditer('(?P<year>(1|2)[0-9]{3})', row['Year']):
year_int = int(m.group('year'))
break
year = row['Year']
else:
year, year_int = None, None
title = row['Article_title'] or row['Book_title']
attrs = {}
jsondata = {}
for attr, field in {
'Additional_information': 'note',
'Article_title': 'title',
'Book_title': 'booktitle',
'City': 'address',
'Editors': 'editor',
'Full_reference': None,
'Issue': None,
'Journal': 'journal',
'Language_codes': None,
'LaTeX_cite_key': None,
'Pages': 'pages',
'Publisher': 'publisher',
'Reference_type': 'type',
'School': 'school',
'Series_title': 'series',
'URL': 'url',
'Volume': 'volume',
}.items():
value = row.get(attr)
if not isinstance(value, int):
value = (value or '').strip()
if attr == 'Issue' and value:
try:
value = str(int(value))
except ValueError:
pass
if value:
if field:
attrs[field] = value
else:
jsondata[attr] = value
p = data.add(common.Source,
row['Reference_ID'],
id=str(row['Reference_ID']),
name=row['Reference_name'],
description=title,
author=row['Authors'],
year=year,
year_int=year_int,
bibtex_type=getattr(EntryType, row['BibTeX_type']
or 'misc'),
jsondata=jsondata,
**attrs)
if p.bibtex_type.value == 'misc' and not p.description:
p.description = p.note
DBSession.flush()
DBSession.flush()
infobox = jsonload(args.data_file('infobox.json'))
glottocodes = jsonload(args.data_file('glottocodes.json'))
for row in read(args, 'Languages', 'Order_number'):
lon, lat = [
float(c.strip()) for c in row['map_coordinates'].split(',')
]
kw = dict(
name=row['Language_name'],
id=str(row['Order_number']),
latitude=lat,
longitude=lon,
region=row['Category_region'],
)
lect = data.add(models.Lect, row['Language_ID'], **kw)
DBSession.flush()
for i, item in enumerate(infobox[lect.id]):
DBSession.add(
common.Language_data(object_pk=lect.pk,
ord=i,
key=item[0],
value=item[1]))
if row["Languages_contribution_documentation::Lect_description_checked_status"] \
!= "Checked":
print 'unchecked! ---', row['Language_name']
desc = row.get(
'Languages_contribution_documentation::Lect description', '')
markup_desc = normalize_markup(row[
'Languages_contribution_documentation::z_calc_GetAsCSS_Lect_description']
)
c = data.add(
models.ApicsContribution,
row['Language_ID'],
id=str(row['Order_number']),
name=row['Language_name'],
description=desc,
markup_description=markup_desc,
survey_reference=data['Source'][row['Survey_reference_ID']],
language=lect)
for ext, label, mtype in [
('pdf', 'Glossed text', 'application/pdf'),
('mp3', 'Glossed text audio', 'audio/mpeg'),
]:
fid = '%s-gt.%s' % (c.id, ext)
if args.data_file('files', 'contribution', c.id, fid).exists():
common.Contribution_files(object=c,
id=fid,
name=label,
mime_type=mtype)
else:
print label, 'missing for:', row['Language_name']
#
# TODO: for michif, 75, add link http://www.youtube.com/watch?v=f0C4cODsSyE
#
iso = None
if row['ISO_code'] and len(row['ISO_code']) == 3:
iso = row['ISO_code'].lower()
if 'iso:%s' % row['ISO_code'] not in data['Identifier']:
data.add(common.Identifier,
'iso:%s' % row['ISO_code'],
id=row['ISO_code'].lower(),
name=row['ISO_code'].lower(),
type=common.IdentifierType.iso.value)
DBSession.add(
common.LanguageIdentifier(
language=data['Lect'][row['Language_ID']],
identifier=data['Identifier']['iso:%s' % row['ISO_code']]))
if lect.id in glottocodes:
identifier = data.add(common.Identifier,
'gc:%s' % glottocodes[lect.id],
id=glottocodes[lect.id],
name=glottocodes[lect.id],
type=common.IdentifierType.glottolog.value)
DBSession.add(
common.LanguageIdentifier(
language=data['Lect'][row['Language_ID']],
identifier=identifier))
if row['Language_name_ethnologue']:
if row['Language_name_ethnologue'] not in data['Identifier']:
data.add(common.Identifier,
row['Language_name_ethnologue'],
id=iso
or 'ethnologue:%s' % row['Language_name_ethnologue'],
name=row['Language_name_ethnologue'],
type='ethnologue')
DBSession.add(
common.LanguageIdentifier(
language=data['Lect'][row['Language_ID']],
identifier=data['Identifier'][
row['Language_name_ethnologue']]))
example_count = {}
for row in read(args, 'Examples', 'Order_number'):
assert row['Language_ID']
lang = data['Lect'][row['Language_ID']]
id_ = '%(Language_ID)s-%(Example_number)s' % row
atext, gloss = igt(row)
example_count[row['Language_ID']] = max(
[example_count.get(row['Language_ID'], 1), row['Example_number']])
p = add_sentence(
args,
data,
id_,
id='%s-%s' % (lang.id, row['Example_number']),
name=row['Text'] or row['Analyzed_text'],
description=row['Translation'],
type=row['Type'].strip().lower() if row['Type'] else None,
comment=row['Comments'],
gloss=gloss,
analyzed=atext,
markup_text=normalize_markup(row['z_calc_Text_CSS']),
markup_gloss=normalize_markup(row['z_calc_Gloss_CSS']),
markup_comment=normalize_markup(row['z_calc_Comments_CSS']),
markup_analyzed=normalize_markup(row['z_calc_Analyzed_text_CSS']),
original_script=row['Original_script'],
jsondata={
'sort': row['Order_number'],
'alt_translation': (row['Translation_other'] or '').strip()
or None
},
language=lang)
if row['Reference_ID']:
if row['Reference_ID'] in data['Source']:
source = data['Source'][row['Reference_ID']]
DBSession.add(
common.SentenceReference(
sentence=p,
source=source,
key=source.id,
description=row['Reference_pages']))
else:
p.source = non_bibs[row['Reference_ID']]
DBSession.flush()
for row in read(args, 'Language_references'):
if row['Reference_ID'] not in data['Source']:
assert row['Reference_ID'] in non_bibs
continue
assert row['Language_ID'] in data['ApicsContribution']
source = data['Source'][row['Reference_ID']]
DBSession.add(
common.ContributionReference(
contribution=data['ApicsContribution'][row['Language_ID']],
source=source,
description=row['Pages'],
key=source.id))
#
# global counter for features - across feature types
#
feature_count = 0
for row in read(args, 'Features', 'Feature_number'):
id_ = str(row['Feature_number'])
if int(id_) > feature_count:
feature_count = int(id_)
wals_id = None
desc = row['Feature_annotation_publication']
if row['WALS_match'] == 'Total':
if isinstance(row['WALS_No.'], int):
wals_id = row['WALS_No.']
else:
wals_id = int(row['WALS_No.'].split('.')[0].strip())
p = data.add(models.Feature,
row['Feature_code'],
name=row['Feature_name'],
id=id_,
description=desc,
markup_description=normalize_markup(
row['z_calc_Feature_annotation_publication_CSS']),
feature_type='primary',
multivalued=row['Value_relation_type'] != 'Single',
area=row['Feature_area'],
wals_id=wals_id)
names = {}
for i in range(1, 10):
if not row['Value%s_publication' % i] \
or not row['Value%s_publication' % i].strip():
continue
name = row['Value%s_publication' % i].strip()
if name in names:
name += ' (%s)' % i
names[name] = 1
de = data.add(
common.DomainElement,
'%s-%s' % (row['Feature_code'], i),
id='%s-%s' % (id_, i),
name=name,
parameter=p,
abbr=row['Value%s_for_book_maps' % i] if p.id != '0' else name,
number=int(row['Value%s_value_number_for_publication' % i]),
jsondata={'color': colors[row['Value_%s_colour_ID' % i]]},
)
assert de
if row['Authors_FeatureArticles']:
authors, _ = row['Authors_FeatureArticles'].split('and the APiCS')
authors = authors.strip()
if authors.endswith(','):
authors = authors[:-1].strip()
for i, name in enumerate(authors.split(',')):
assert name.strip() in editors
p._authors.append(
models.FeatureAuthor(ord=i + 1,
contributor=editors[name.strip()]))
DBSession.flush()
primary_to_segment = {123: 63, 126: 35, 128: 45, 130: 41}
segment_to_primary = dict(
zip(primary_to_segment.values(), primary_to_segment.keys()))
number_map = {}
names = {}
for row in read(args, 'Segment_features', 'Order_number'):
symbol = row['Segment_symbol']
if row['Segment_name'] == 'voiceless dental/alveolar sibilant affricate':
symbol = 't\u0361s'
truth = lambda s: s and s.strip().lower() == 'yes'
name = '%s - %s' % (symbol, row['Segment_name'])
if name in names:
number_map[row['Segment_feature_number']] = names[name]
continue
number_map[
row['Segment_feature_number']] = row['Segment_feature_number']
names[name] = row['Segment_feature_number']
feature_count += 1
if row['Segment_feature_number'] in segment_to_primary:
primary_to_segment[segment_to_primary[row['Segment_feature_number']]]\
= str(feature_count)
p = data.add(models.Feature,
row['Segment_feature_number'],
name=name,
id=str(feature_count),
feature_type='segment',
area='Vowels' if truth(row['Vowel']) else
('Obstruent consonants'
if truth(row['Obstruent']) else 'Sonorant consonants'),
jsondata=dict(
number=int(row['Segment_feature_number']),
vowel=truth(row['Vowel']),
consonant=truth(row['Consonant']),
obstruent=truth(row['Obstruent']),
core_list=truth(row['Core_list_segment']),
symbol=symbol,
))
for i, spec in SEGMENT_VALUES.items():
data.add(common.DomainElement,
'%s-%s' % (row['Segment_feature_number'], spec[0]),
id='%s-%s' % (p.id, i),
name=spec[0],
parameter=p,
jsondata={'color': spec[1]},
number=i)
print '--> remapped:', primary_to_segment
DBSession.flush()
for row in read(args, 'Sociolinguistic_features',
'Sociolinguistic_feature_number'):
feature_count += 1
p = data.add(models.Feature,
row['Sociolinguistic_feature_code'],
name=row['Sociolinguistic_feature_name'],
id='%s' % feature_count,
description=row['Sociolinguistic_feature_annotation'],
area='Sociolinguistic',
feature_type='sociolinguistic')
names = {}
for i in range(1, 10):
id_ = '%s-%s' % (row['Sociolinguistic_feature_code'], i)
if row.get('Value%s' % i) and row['Value%s' % i].strip():
name = row['Value%s' % i].strip()
if name in names:
name += ' (%s)' % i
names[name] = 1
else:
continue
kw = dict(id='%s-%s' % (p.id, i), name=name, parameter=p, number=i)
data.add(common.DomainElement,
id_,
id='%s-%s' % (p.id, i),
name=name,
parameter=p,
number=i,
jsondata={
'color':
colors.get(row['Value%s_colour_ID' % i],
colors.values()[i])
})
sd = {}
for row in read(args, 'Segment_data'):
if row['Segment_feature_number'] not in number_map:
continue
number = number_map[row['Segment_feature_number']]
if not row['Presence_in_the_language']:
continue
lang = data['Lect'][row['Language_ID']]
param = data['Feature'][number]
id_ = '%s-%s' % (lang.id, param.id)
if id_ in sd:
assert row['c_Record_is_a_duplicate'] == 'Yes'
continue
sd[id_] = 1
valueset = data.add(
common.ValueSet,
id_,
id=id_,
parameter=param,
language=lang,
contribution=data['ApicsContribution'][row['Language_ID']],
description=row['Comments'],
markup_description=normalize_markup(row['z_calc_Comments_CSS']),
)
v = data.add(
common.Value,
id_,
id=id_,
frequency=float(100),
valueset=valueset,
domainelement=data['DomainElement'][
'%s-%s' % (number, row['Presence_in_the_language'])],
)
if row['Example_word'] and row['Example_word_gloss']:
example_count[row['Language_ID']] += 1
p = add_sentence(args,
data,
'%s-p%s' % (lang.id, data['Feature'][number].id),
id='%s-%s' %
(lang.id, example_count[row['Language_ID']]),
name=row['Example_word'],
description=row['Example_word_gloss'],
language=lang)
DBSession.add(common.ValueSentence(value=v, sentence=p))
source = data['Source'].get(row['Refers_to_references_Reference_ID'])
if source:
DBSession.add(
common.ValueSetReference(valueset=valueset,
source=source,
key=source.id))
elif row['Refers_to_references_Reference_ID'] in non_bibs:
valueset.source = non_bibs[
row['Refers_to_references_Reference_ID']]
lects = defaultdict(lambda: 1)
lect_map = {}
records = {}
false_values = {}
no_values = {}
wals_value_number = {}
for row in read(args, 'wals'):
if row['z_calc_WALS_value_number']:
wals_value_number[
row['Data_record_id']] = row['z_calc_WALS_value_number']
def prefix(attr, _prefix):
if _prefix:
return '%s_%s' % (_prefix, attr)
return attr.capitalize()
for _prefix, abbr in [('', ''), ('Sociolinguistic', 'sl')]:
num_values = 10
for row in read(args, prefix('data', _prefix)):
if not row[prefix('feature_code', _prefix)]:
print('no associated feature for', prefix('data', _prefix),
row[prefix('data_record_id', _prefix)])
continue
lid = row['Language_ID']
lect_attr = row.get('Lect_attribute', 'my default lect').lower()
if lect_attr != 'my default lect':
if (row['Language_ID'], row['Lect_attribute']) in lect_map:
lid = lect_map[(row['Language_ID'], row['Lect_attribute'])]
else:
lang = data['Lect'][row['Language_ID']]
c = lects[row['Language_ID']]
lid = '%s-%s' % (row['Language_ID'], c)
kw = dict(
name='%s (%s)' % (lang.name, row['Lect_attribute']),
id='%s' % (1000 + 10 * int(lang.id) + c),
latitude=lang.latitude,
longitude=lang.longitude,
description=row['Lect_attribute'],
language=lang,
)
data.add(models.Lect, lid, **kw)
lects[row['Language_ID']] += 1
lect_map[(row['Language_ID'], row['Lect_attribute'])] = lid
id_ = abbr + str(row[prefix('data_record_id', _prefix)])
assert id_ not in records
records[id_] = 1
assert row[prefix('feature_code', _prefix)] in data['Feature']
language = data['Lect'][lid]
parameter = data['Feature'][row[prefix('feature_code', _prefix)]]
valueset = common.ValueSet(
id='%s-%s' % (language.id, parameter.id),
description=row['Comments_on_value_assignment'],
markup_description=normalize_markup(
row.get('z_calc_Comments_on_value_assignment_CSS')),
)
values_found = {}
for i in range(1, num_values):
if not row['Value%s_true_false' % i]:
continue
if row['Value%s_true_false' % i].strip().lower() != 'true':
assert row['Value%s_true_false' %
i].strip().lower() == 'false'
false_values[row[prefix('data_record_id', _prefix)]] = 1
continue
iid = '%s-%s' % (row[prefix('feature_code', _prefix)], i)
if iid not in data['DomainElement']:
print(iid, row[prefix('data_record_id',
_prefix)], '--> no domainelement!')
continue
values_found['%s-%s' % (id_, i)] = dict(
id='%s-%s' % (valueset.id, i),
domainelement=data['DomainElement']['%s-%s' % (row[prefix(
'feature_code', _prefix)], i)],
confidence=row['Value%s_confidence' % i],
frequency=float(row['c_V%s_frequency_normalised' %
i]) if _prefix == '' else 100)
if values_found:
if row[prefix('data_record_id', _prefix)] in wals_value_number:
valueset.jsondata = {
'wals_value_number':
wals_value_number.pop(row[prefix(
'data_record_id', _prefix)])
}
valueset.parameter = parameter
valueset.language = language
valueset.contribution = data['ApicsContribution'][
row['Language_ID']]
valueset = data.add(common.ValueSet, id_, _obj=valueset)
for i, item in enumerate(values_found.items()):
if i > 0 and not parameter.multivalued:
print 'multiple values for single-valued parameter: %s' % id_
break
id_, kw = item
kw['valueset'] = valueset
value = data.add(common.Value, id_, **kw)
#
# store references to additional data for segments which should be reused
# for corresponding primary features!
#
if int(parameter.id) in primary_to_segment:
assert len(values_found) == 1
seg_id = '%s-%s' % (language.id, primary_to_segment[int(
parameter.id)])
seg_valueset = data['ValueSet'][seg_id]
seg_value = data['Value'][seg_id]
if not valueset.description and seg_valueset.description:
valueset.description = seg_valueset.description
for s in seg_value.sentence_assocs:
DBSession.add(
common.ValueSentence(value=value,
sentence=s.sentence))
for r in seg_valueset.references:
DBSession.add(
common.ValueSetReference(valueset=valueset,
source=r.source,
key=r.key))
if not valueset.source and seg_valueset.source:
valueset.source = seg_valueset.source
DBSession.flush()
else:
no_values[id_] = 1
DBSession.flush()
for prefix, abbr, num_values in [
('D', '', 10),
('Sociolinguistic_d', 'sl', 7),
]:
for row in read(args, prefix + 'ata_references'):
assert row['Reference_ID'] in data['Source'] \
or row['Reference_ID'] in non_bibs
try:
vs = data['ValueSet'][abbr +
str(row[prefix + 'ata_record_id'])]
if row['Reference_ID'] in data['Source']:
source = data['Source'][row['Reference_ID']]
DBSession.add(
common.ValueSetReference(
valueset=vs,
source=source,
key=source.id,
description=row['Pages'],
))
else:
if vs.source:
vs.source += '; ' + non_bibs[row['Reference_ID']]
else:
vs.source = non_bibs[row['Reference_ID']]
except KeyError:
continue
DBSession.flush()
missing = 0
for row in read(args, 'Value_examples'):
try:
DBSession.add(
common.ValueSentence(
value=data['Value']['%(Data_record_id)s-%(Value_number)s' %
row],
sentence=data['Sentence'][
'%(Language_ID)s-%(Example_number)s' % row],
description=row['Notes'],
))
except KeyError:
missing += 1
print('%s Value_examples are missing data' % missing)
print('%s data sets with false values' % len(false_values))
print('%s data sets without values' % len(no_values))
for k, v in wals_value_number.items():
print 'unclaimed wals value number:', k, v
for i, row in enumerate(read(args, 'Contributors')):
kw = dict(contribution=data['ApicsContribution'][row['Language ID']],
contributor=data['Contributor'][row['Author ID']])
if row['Order_of_appearance']:
kw['ord'] = int(float(row['Order_of_appearance']))
data.add(common.ContributionContributor, i, **kw)
DBSession.flush()
def main(args):
data = Data()
dataset = common.Dataset(
id=dictionaria.__name__,
name="Dictionaria",
description="The Dictionary Journal",
published=date(2015, 10, 1),
contact='*****@*****.**',
domain='dictionaria.clld.org',
license="http://creativecommons.org/licenses/by/4.0/",
jsondata={
'license_icon': 'cc-by.png',
'license_name': 'Creative Commons Attribution 4.0 International License'})
ed = data.add(
common.Contributor, 'hartmanniren', id='hartmanniren', name='Iren Hartmann')
common.Editor(dataset=dataset, contributor=ed)
DBSession.add(dataset)
for id_, name in LGR_ABBRS.items():
DBSession.add(common.GlossAbbreviation(id=id_, name=name))
comparison_meanings = {}
comparison_meanings_alt_labels = {}
print('loading concepts ...')
concepticon = Concepticon()
for i, concept_set in enumerate(concepticon.resources('parameter').members):
concept_set = concepticon.resource(concept_set)
cm = ComparisonMeaning(
id=concept_set.id,
name=concept_set.name.lower(),
description=concept_set.description,
concepticon_url='%s' % concept_set.uriref)
DBSession.add(cm)
comparison_meanings[cm.name] = cm
for label in concept_set.alt_labels:
comparison_meanings_alt_labels.setdefault(label.lower(), cm)
DBSession.flush()
print('... done')
comparison_meanings = {k: v.pk for k, v in comparison_meanings.items()}
comparison_meanings_alt_labels = {
k: v.pk for k, v in comparison_meanings_alt_labels.items()}
submissions = []
for submission in REPOS.joinpath('submissions').glob('*'):
if not submission.is_dir():
continue
try:
submission = Submission(submission)
except ValueError:
continue
md = submission.md
id_ = submission.id
lmd = md['language']
language = data['Variety'].get(lmd['glottocode'])
if not language:
language = data.add(
Variety, lmd['glottocode'], id=lmd['glottocode'], name=lmd['name'])
dictionary = data.add(
Dictionary,
id_,
id=id_,
name=lmd['name'] + ' Dictionary',
language=language,
published=date(*map(int, md['published'].split('-'))))
for i, cname in enumerate(md['authors']):
name = HumanName(cname)
cid = slug('%s%s' % (name.last, name.first))
contrib = data['Contributor'].get(cid)
if not contrib:
contrib = data.add(common.Contributor, cid, id=cid, name=cname)
DBSession.add(common.ContributionContributor(
ord=i + 1,
primary=True,
contributor=contrib,
contribution=dictionary))
submissions.append((dictionary.id, language.id, submission))
transaction.commit()
for did, lid, submission in submissions:
try:
mod = __import__(
'dictionaria.loader.' + submission.id, fromlist=['MARKER_MAP'])
marker_map = mod.MARKER_MAP
except ImportError:
marker_map = {}
transaction.begin()
print('loading %s ...' % submission.id)
submission.load(
did,
lid,
comparison_meanings,
comparison_meanings_alt_labels,
marker_map)
transaction.commit()
print('... done')
#('hoocak', 'Hooca\u0328k', 43.5, -88.5, [('hartmanniren', 'Iren Hartmann')]),
#('yakkha', 'Yakkha', 27.37, 87.93, [('schackowdiana', 'Diana Schackow')]),
#('palula', 'Palula', 35.51, 71.84, [('liljegrenhenrik', 'Henrik Liljegren')], {}),
#('daakaka', 'Daakaka', -16.27, 168.01, [('vonprincekilu', 'Kilu von Prince')],
# {'published': date(2015, 9, 30), 'iso': 'bpa', 'glottocode': 'daka1243'}),
#('teop', 'Teop', -5.67, 154.97, [('moselulrike', 'Ulrike Mosel')],
# {'published': date(2015, 9, 30), 'iso': 'tio', 'glottocode': 'teop1238', 'encoding': 'latin1'}),
transaction.begin()
load_families(Data(), DBSession.query(Variety))