def test_Sources_with_None_values(self):
from pycldf.sources import Sources, Source
src = Sources()
src.add(Source('book', 'huber2005', title=None))
bib = self.tmp_path('test.bib')
src.write(bib.name, bib.parent)
def to_cldf(self, dest, mdname='cldf-metadata.json'):
"""
Write the data from the db to a CLDF dataset according to the metadata in `self.dataset`.
:param dest:
:param mdname:
:return: path of the metadata file
"""
dest = Path(dest)
if not dest.exists():
dest.mkdir()
data = self.read()
if data[self.source_table_name]:
sources = Sources()
for src in data[self.source_table_name]:
sources.add(Source(
src['genre'],
src['id'],
**{k: v for k, v in src.items() if k not in ['id', 'genre']}))
sources.write(dest / self.dataset.properties.get('dc:source', 'sources.bib'))
for table_type, items in data.items():
try:
table = self.dataset[table_type]
table.common_props['dc:extent'] = table.write(
[self.retranslate(table, item) for item in items],
base=dest)
except KeyError:
assert table_type == self.source_table_name, table_type
return self.dataset.write_metadata(dest / 'cldf-metadata.json')
def test_Source_expand_refs():
sources = Sources()
src = Source(
'book', 'Meier2005', author='Hans Meier', year='2005', title='The Book')
assert 'Meier2005' in repr(src)
sources.add(src)
bib = sources._bibdata.to_string(bib_format='bibtex')
assert len(bib.split('author')) == 2
assert len(list(sources.expand_refs('Meier2005'))) == 1
bib = sources._bibdata.to_string(bib_format='bibtex')
assert len(bib.split('author')) == 2
assert len(list(sources.expand_refs('12345'))) == 1
def test_Source_expand_refs(self):
from pycldf.sources import Sources, Source
sources = Sources()
src = Source(
'book', 'Meier2005', author='Hans Meier', year='2005', title='The Book')
self.assertIn('Meier2005', repr(src))
sources.add(src)
bib = sources._bibdata.to_string(bib_format='bibtex')
self.assertEqual(len(bib.split('author')), 2)
self.assertEqual(len(list(sources.expand_refs('Meier2005'))), 1)
bib = sources._bibdata.to_string(bib_format='bibtex')
self.assertEqual(len(bib.split('author')), 2)
def cmd_makecldf(self, args):
"""
Convert the raw data to a CLDF dataset.
"""
with self.cldf_writer(args) as writer:
writer.cldf.add_component('CognatesetTable')
writer.add_sources(*self.raw_dir.read_bib('cariban_resolved.bib'))
cmap = writer.add_concepts(lookup_factory=lambda c: c.english)
cmap['you'] = cmap['thou']
cmap['grease/fat'] = cmap['grease']
cmap['breast'] = cmap['breasts']
cmap['son'] = cmap['person']
data = pycldf.Dataset.from_metadata(self.raw_dir /
'cariban_data.json')
for lang in data['LanguageTable']:
writer.add_language(ID=lang['ID'],
Name=lang['Name'],
Glottocode=lang["Glottocode"])
cs_seen = set()
reconstructions = {
tuple(c['ID'].split('-')): c['Form']
for c in self.raw_dir.read_csv(
'cariban_lexical_reconstructions.csv', dicts=True)
}
for lex in self.raw_dir.read_csv('cariban_swadesh_list.csv',
dicts=True):
#"Language_ID","Swadesh_Nr","Feature_ID","Value","Cognateset_ID","Source","Comment","Full_Form"
if lex['Feature_ID'] not in cmap:
print(lex['Feature_ID'])
continue
for form in writer.add_lexemes(
Value=lex['Value'],
Parameter_ID=cmap[lex['Feature_ID']],
Language_ID=lex['Language_ID'],
Source=[
Reference(*d) for d in
[Sources.parse(lex['Source'].replace(';', ','))]
] if lex['Source']
and not lex['Source'].startswith('pc') else [],
):
cs_key = (lex['Feature_ID'], lex['Cognateset_ID'])
cs_id = '{}-{}'.format(cmap[cs_key[0]], cs_key[1])
if cs_key not in cs_seen:
writer.objects['CognatesetTable'].append(
dict(
ID=cs_id,
Description=reconstructions.get(cs_key),
))
cs_seen.add(cs_key)
writer.add_cognate(lexeme=form, Cognateset_ID=cs_id)
# Note: We want to re-use LanguageTable across the two CLDF datasets:
LanguageTable = writer.cldf['LanguageTable']
with self.cldf_writer(args, cldf_spec='structure',
clean=False) as writer:
writer.cldf.add_component(
LanguageTable) # we reuse the one from above!
def to_cldf(self, dest, mdname='cldf-metadata.json', coordinate_precision=4):
"""
Write the data from the db to a CLDF dataset according to the metadata in `self.dataset`.
:param dest:
:param mdname:
:return: path of the metadata file
"""
dest = pathlib.Path(dest)
if not dest.exists():
dest.mkdir()
data = self.read()
if data[self.source_table_name]:
sources = Sources()
for src in data[self.source_table_name]:
sources.add(Source(
src['genre'],
src['id'],
**{k: v for k, v in src.items() if k not in ['id', 'genre']}))
sources.write(dest / self.dataset.properties.get('dc:source', 'sources.bib'))
for table_type, items in data.items():
try:
table = self.dataset[table_type]
items = [
self.round_geocoordinates(item, precision=coordinate_precision)
for item in items]
table.common_props['dc:extent'] = table.write(
[self.retranslate(table, item) for item in items],
base=dest)
except KeyError:
assert table_type == self.source_table_name, table_type
return self.dataset.write_metadata(dest / mdname)
def __init__(self, name):
assert NAME_PATTERN.match(name)
self.name = name
self.sources = Sources()
self.metadata = Metadata()
self._rows = OrderedDict()
# We store the fields (a.k.a. header) as tuple because it must be immutable after
# first assignment (since changing is not well defined when there are already
# rows).
self._fields = ()
self._source_count = None
self._cited_sources = set()
self._table = None
def test_field_order(tmpdir):
srcs = Sources()
src = Source('misc', 'x') # src is an OrderedDict and we add title *after* year.
src['year'] = '2018'
src['title'] = 'The Title'
srcs.add(src)
bib = tmpdir / 'test.bib'
srcs.write(str(bib))
res = bib.read_text(encoding='utf8')
# Still, title should be printed in the BibTeX before year:
assert res.index('title =') < res.index('year =')
def test_Source_expand_refs():
sources = Sources()
src = Source(
'book', 'Meier2005', author='Hans Meier', year='2005', title='The Book')
assert 'Meier2005' in repr(src)
sources.add(src)
bib = sources._bibdata.to_string(bib_format='bibtex')
assert len(bib.split('author')) == 2
assert len(list(sources.expand_refs('Meier2005'))) == 1
bib = sources._bibdata.to_string(bib_format='bibtex')
assert len(bib.split('author')) == 2
assert len(list(sources.expand_refs('12345'))) == 1
def import_values(values, lang, features, codes, contributors,
sources): # pragma: no cover
c = Contribution(
id=lang['ID'],
name='Dataset for {0}'.format(lang['Name']),
)
for i, cid in enumerate(lang['Coders'], start=1):
DBSession.add(
ContributionContributor(
contribution=c,
contributor_pk=contributors[cid],
ord=i,
))
l = GrambankLanguage(
id=lang['ID'],
name=lang['Name'],
macroarea=lang['Macroarea'],
latitude=lang['Latitude'],
longitude=lang['Longitude'],
)
for value in values:
vs = ValueSet(
id=value['ID'],
parameter_pk=features[value['Parameter_ID']],
language=l,
contribution=c,
)
Value(id=value['ID'],
valueset=vs,
name=value['Value'],
description=value['Comment'],
domainelement_pk=codes[value['Code_ID']
or '{}-NA'.format(value['Parameter_ID'])])
if value['Source']:
for ref in value['Source']:
sid, pages = Sources.parse(ref)
ValueSetReference(valueset=vs,
source_pk=sources[sid],
description=pages)
DBSession.add(c)
def test_Sources_roundtrip_latex(tmpdir, bibtex, expected):
src = Sources()
src.add(bibtex)
bib = tmpdir / 'test.bib'
src.write(str(bib))
assert expected in bib.read_text('utf8')
def test_Sources_with_None_values(tmpdir):
src = Sources()
src.add(Source('book', 'huber2005', title=None))
bib = tmpdir / 'test.bib'
src.write(str(bib))
def test_Sources(tmpdir):
src = Sources()
src.add(BIB, Source(
'book', 'huber2005', author='Herrmann Huber', year='2005', title='y'))
for entry in src:
assert entry.genre == 'book'
break
assert len(list(src.items())) == 3
assert len(list(src.keys())) == 3
refs = ['huber2005[1-6]', 'Obrazy', 'Elegie[34]']
assert src.format_refs(*list(src.expand_refs(refs))) == refs
assert '%s' % src['huber2005'] == 'Huber, Herrmann. 2005. y.'
with pytest.raises(ValueError):
src.add(5)
with pytest.raises(ValueError):
src.add('@misc{a.b,\n author="a.b"\n}')
with pytest.raises(ValueError):
_ = src['unknown']
assert _ # pragma: no cover
with pytest.raises(ValueError):
src.parse('a[x')
with pytest.raises(ValueError):
src.parse('[x]')
with pytest.raises(ValueError):
src.validate(['x'])
bib = str(tmpdir / 'test.bib')
src.write(bib)
src2 = Sources()
src2.read(bib)
src2.write(bib, ids=['huber2005'])
src = Sources.from_file(bib)
assert len(src) == 1
def __init__(self, tablegroup):
self.tablegroup = tablegroup
self.auto_constraints()
self.sources = Sources.from_file(self.bibpath)
def __init__(self, tablegroup):
self.tablegroup = tablegroup
self.auto_constraints()
self.sources = Sources.from_file(self.bibpath)
def test_Sources(self):
from pycldf.sources import Sources, Source
src = Sources()
src.add(BIB, Source(
'book', 'huber2005', author='Herrmann Huber', year='2005', title='y'))
self.assertEqual(len(list(src.items())), 3)
self.assertEqual(len(list(src.keys())), 3)
refs = 'huber2005[1-6];Obrazy;Elegie[34]'
self.assertEqual(src.format_refs(*list(src.expand_refs(refs))), refs)
self.assertEqual('%s' % src['huber2005'], 'Huber, Herrmann. 2005. y.')
with self.assertRaises(ValueError):
src.add(5)
with self.assertRaises(ValueError):
src.add('@misc{a.b,\n author="a.b"\n}')
bib = self.tmp_path('test.bib')
src.write(bib.name, bib.parent)
src2 = Sources()
src2.read(bib.name, bib.parent)
bib = self.tmp_path('test.bib')
src2.write(bib.name, bib.parent, ids=['huber2005'])
src = Sources()
src.read(bib.name, bib.parent)
self.assertEqual(len(src), 1)
def test_Sources_roundtrip_latex(tmpdir, bibtex, expected):
src = Sources()
src.add(bibtex)
bib = tmpdir / 'test.bib'
src.write(str(bib))
assert expected in bib.read_text('utf8')
def test_Sources_with_None_values(tmpdir):
src = Sources()
src.add(Source('book', 'huber2005', title=None))
bib = tmpdir / 'test.bib'
src.write(str(bib))
def cmd_makecldf(self, args):
# Add sources
sources = Sources.from_file(self.raw_dir / "sources.bib")
args.writer.cldf.add_sources(*sources)
glottolog = Glottolog(args.glottolog.dir)
clts = CLTS(Config.from_file().get_clone('clts'))
bipa = clts.bipa
clts_saphon = clts.transcriptiondata_dict['saphon']
# Add components
args.writer.cldf.add_columns("ValueTable", {
"name": "Value_in_Source",
"datatype": "string"
})
cltstable = Terms()["cltsReference"].to_column().asdict()
cltstable["datatype"]["format"] = "[a-z_-]+|NA"
args.writer.cldf.add_columns('ParameterTable', cltstable, {
'name': 'CLTS_BIPA',
'datatype': 'string'
}, {
'name': 'CLTS_Name',
'datatype': 'string'
})
args.writer.cldf.add_component("LanguageTable", "Family",
"Glottolog_Name")
languages = []
#all_glottolog = {lng.id: lng for lng in glottolog.languoids()}
#iso2glot = {lng.iso: lng.glottocode for lng in all_glottolog.values()}
#args.log.info("loaded glottolog")
for row in progressbar(
self.etc_dir.read_csv("languages.csv", dicts=True)):
#if row["SAPHON_Code"] in iso2glot:
# glottocode = iso2glot[row["SAPHON_Code"]]
#elif row["SAPHON_Code"][:3] in iso2glot:
# glottocode = iso2glot[row["SAPHON_Code"][:3]]
#else:
# glottocode = ""
#if glottocode and glottocode in all_glottolog:
# lang = all_glottolog[glottocode]
# update = {
# "Family": lang.family.name if lang.family else '',
# "Glottocode": glottocode,
# "Latitude": lang.latitude,
# "Longitude": lang.longitude,
# "Macroarea": lang.macroareas[0].name if lang.macroareas else None,
# "Glottolog_Name": lang.name,
# }
# row.update(update)
languages.append(row)
# Build source map from language
source_map = {
k: v
for k, v in self.raw_dir.read_csv("references.tsv", delimiter="\t")
}
# Parse sources
segments = []
values = []
counter = 1
unknowns = defaultdict(list)
for lid, segment in self.raw_dir.read_csv('inventories.tsv',
delimiter="\t"):
normalized = normalize_grapheme(segment)
if normalized in clts_saphon.grapheme_map:
sound = bipa[clts_saphon.grapheme_map[normalized]]
else:
sound = bipa['<NA>']
unknowns[normalized] += [(lang_key, segment)]
par_id = compute_id(normalized)
if sound.type == 'unknownsound':
bipa_grapheme = ''
desc = ''
else:
bipa_grapheme = str(sound)
desc = sound.name
segments.append((par_id, normalized, bipa_grapheme, desc))
values.append({
"ID": str(counter),
"Language_ID": lid,
"Parameter_ID": par_id,
"Value_in_Source": segment,
"Value": normalized,
"Source": [source_map[lid]]
})
counter += 1
# Build segment data
parameters = [{
"ID": ID,
"Name": normalized,
"Description": '',
"CLTS_ID": desc.replace(' ', '_') if desc.strip() else "NA",
"CLTS_BIPA": bipa_grapheme,
"CLTS_Name": desc
} for ID, normalized, bipa_grapheme, desc in set(segments)]
# Write data and validate
args.writer.write(
**{
"ValueTable": values,
"LanguageTable": languages,
"ParameterTable": parameters,
})
for g, rest in unknowns.items():
print('\t'.join([repr(g), str(len(rest)), g]))
def load(self, ds, args=None, verbose=False):
"""
Load a CLDF dataset into the database.
:param dataset:
:return:
"""
print(ds)
try:
self.fetchone('select ID from dataset')
except sqlite3.OperationalError:
self.create(force=True)
self.unload(ds)
dataset = ds.cldf_reader()
tables, ref_tables = schema(dataset)
# update the DB schema:
for t in tables:
if self._create_table_if_not_exists(t):
continue
db_cols = {k.lower(): v for k, v in self.tables[t.name].items()}
for col in t.columns:
if col.name.lower() not in db_cols:
with self.connection() as conn:
conn.execute(
"ALTER TABLE {0} ADD COLUMN `{1.name}` {1.db_type}".format(
t.name, col))
else:
if db_cols[col.name.lower()] != col.db_type:
raise ValueError(
'column {0}:{1} {2} redefined with new type {3}'.format(
t.name, col.name, db_cols[col.name.lower()], col.db_type))
for t in ref_tables.values():
self._create_table_if_not_exists(t)
self.update_schema()
# then load the data:
with self.connection() as db:
db.execute('PRAGMA foreign_keys = ON;')
insert(
db,
'dataset',
'ID,name,version,metadata_json',
(
ds.id,
'{0}'.format(dataset),
ds.repo.hash() if ds.repo else '',
json.dumps(dataset.metadata_dict)))
insert(
db,
'datasetmeta',
'dataset_ID,key,value',
*[(ds.id, k, '{0}'.format(v)) for k, v in dataset.properties.items()])
# load sources:
rows = []
for src in dataset.sources.items():
values = [ds.id, src.id, src.genre] + [src.get(k) for k in BIBTEX_FIELDS]
values.append(
json.dumps({k: v for k, v in src.items() if k not in BIBTEX_FIELDS}))
rows.append(tuple(values))
insert(
db,
'SourceTable',
['dataset_ID', 'ID', 'bibtex_type'] + BIBTEX_FIELDS + ['extra'],
*rows)
# For regular tables, we extract and keep references to sources.
refs = collections.defaultdict(list)
for t in tables:
# We want to lookup columns by the name used in the CLDF dataset.
cols = {col.cldf_name: col for col in t.columns}
# But we also want to look up primary keys by the database column name.
cols_by_name = {col.name: col for col in t.columns}
ref_table = ref_tables.get(t.name)
rows, keys = [], []
try:
for row in dataset[t.name]:
keys, values = ['dataset_ID'], [ds.id]
for k, v in row.items():
if ref_table and k == ref_table.consumes:
col = cols_by_name[t.primary_key]
refs[ref_table.name].append((row[col.cldf_name], v))
else:
col = cols[k]
if isinstance(v, list):
v = (col.separator or ';').join(
nfilter(col.convert(vv) for vv in v))
else:
v = col.convert(v)
# FIXME: only if non-local!
keys.append("`{0}`".format(col.name))
values.append(v)
keys, values = self.update_row(t.name, keys, values)
rows.append(tuple(values))
insert(db, t.name, keys, *rows, **{'verbose': verbose})
except FileNotFoundError: # pragma: no cover
if t.name != 'CognateTable': # An empty CognateTable is allowed.
raise # pragma: no cover
# Now insert the references, i.e. the associations with sources:
for tname, items in refs.items():
rows = []
for oid, sources in items:
for source in sources:
sid, context = Sources.parse(source)
rows.append([ds.id, oid, sid, context])
oid_col = '{0}_ID'.format(tname.replace('Source', ''))
insert(db, tname, ['dataset_ID', oid_col, 'Source_ID', 'Context'], *rows)
db.commit()
class Dataset(object):
"""
API to access a CLDF dataset.
"""
def __init__(self, name):
assert NAME_PATTERN.match(name)
self.name = name
self.sources = Sources()
self.metadata = Metadata()
self._rows = OrderedDict()
# We store the fields (a.k.a. header) as tuple because it must be immutable after
# first assignment (since changing is not well defined when there are already
# rows).
self._fields = ()
self._source_count = None
self._cited_sources = set()
self._table = None
def __repr__(self):
return '<%s %s>' % (self.__class__.__name__, self.name)
def __len__(self):
"""The length of a dataset is the number of rows in the values file."""
return len(self.rows)
def __getitem__(self, item):
"""
Individual rows can be accessed by integer index or by row ID.
:param item: `int` to access row by index, `str` to access by row ID
:return: `OrderedDict`
"""
if isinstance(item, int):
return self.rows[item]
return self._rows[item]
@property
def fields(self):
"""
Read-only property to access the fields (a.k.a. header) defined for the dataset.
:return: `tuple` of field names
"""
return self._fields
@property
def table(self):
return self._table
@fields.setter
def fields(self, value):
"""
Fields can be assigned (but only once) for a dataset.
:param value: `tuple` of field names.
"""
if self._fields:
raise ValueError('fields can only be assigned once!')
assert isinstance(value, tuple)
assert all(any(field in value for field in variants)
for variants in REQUIRED_FIELDS)
table = self.metadata.get_table()
if table:
assert list(value) == list(table.schema.columns.keys())
else:
table = self.metadata.add_table(
'values',
'',
[{'name': col, 'datatype': 'string'} for col in value])
table.schema.primaryKey = 'ID'
self._table = table
self._fields = value
@property
def rows(self):
return list(self._rows.values())
@property
def stats(self):
return dict(
languages=set(row['Language_ID'] for row in self.rows),
parameters=set(row['Parameter_ID'] for row in self.rows),
rowcount=(
len(self),
sum([1 for row in self.rows
if row['Language_ID'] and row['Parameter_ID']])),
values=Counter(row['Value'] for row in self.rows),
)
def add_row(self, row):
if not row:
return
d = ValuesRow.from_list(self, row)
if d['ID'] in self._rows:
raise ValueError('duplicate row ID: %s' % d['ID'])
for ref in self.sources.expand_refs(d.get('Source', '')):
self._cited_sources.add(ref.source.id)
self._rows[d['ID']] = d
return d
@staticmethod
def filename(fname, type_):
"""
Compute the path for optional CLDF files relative to a given values file.
:param fname: Path of the values file
:param type_: Type of the optional file
:return: name of the optional file
"""
if type_ == 'sources':
return fname.stem + '.bib'
if type_ == 'metadata':
return fname.stem + fname.suffix + MD_SUFFIX
raise ValueError(type_) # pragma: no cover
@staticmethod
def _existing_file(fname):
fname = Path(fname)
assert fname.exists() and fname.is_file()
return fname
@classmethod
def _from(cls, data, container=None, skip_on_error=False):
container = container or data.parent
dataset = cls(data.stem)
dataset.metadata.read(Dataset.filename(data, 'metadata'), container)
dataset._table = dataset.metadata.get_table()
dataset.sources.read(Dataset.filename(data, 'sources'), container)
delimiter = ','
if dataset.table:
delimiter = dataset.table.dialect.delimiter
if data.suffix in TAB_SUFFIXES:
delimiter = '\t'
if isinstance(container, Archive):
rows = container.read_text(data.name).split('\n')
else:
rows = data
for i, row in enumerate(reader(rows, delimiter=delimiter)):
if i == 0:
dataset.fields = tuple(row)
else:
try:
dataset.add_row(row)
except ValueError as e:
if skip_on_error:
log.warn('skipping row in line %s: %s' % (i + 1, e))
else:
raise e
dataset.table.dialect.delimiter = delimiter
dataset.table.url = data.name
return dataset
@classmethod
def from_zip(cls, fname, name=None):
archive = Archive(cls._existing_file(fname))
return cls._from(
Path(archive.metadata_name(prefix=name)[:-len(MD_SUFFIX)]), archive)
@classmethod
def from_metadata(cls, fname, container=None):
fname = Path(fname)
if not fname.name.endswith(MD_SUFFIX):
raise ValueError('metadata file name must end with %s' % MD_SUFFIX)
return cls._from(
fname.parent.joinpath(fname.name[:-len(MD_SUFFIX)]), container=container)
@classmethod
def from_file(cls, fname, skip_on_error=False):
"""
Factory method to create a `Dataset` from a CLDF values file.
:param fname: Path of the CLDF values file.
:return: `Dataset` instance.
"""
return cls._from(cls._existing_file(fname), skip_on_error=skip_on_error)
def write(self, outdir='.', suffix='.csv', cited_sources_only=False, archive=False):
outdir = Path(outdir)
if not outdir.exists():
raise ValueError(outdir.as_posix())
close = False
if archive:
if isinstance(archive, Archive):
container = archive
else:
container = Archive(outdir.joinpath(self.name + '.zip'), mode='w')
close = True
else:
container = outdir
fname = Path(outdir).joinpath(self.name + suffix)
if fname.suffix in TAB_SUFFIXES:
self.table.dialect.delimiter = '\t'
with UnicodeWriter(
None if isinstance(container, Archive) else fname,
delimiter=self.table.dialect.delimiter) as writer:
writer.writerow(self.fields)
for row in self.rows:
writer.writerow(row.to_list())
if isinstance(container, Archive):
container.write_text(writer.read(), fname.name)
self.table.url = fname.name
self.metadata.write(Dataset.filename(fname, 'metadata'), container)
ids = self._cited_sources if cited_sources_only else None
self.sources.write(Dataset.filename(fname, 'sources'), container, ids=ids)
if close:
container.close()
def load(self, dataset):
"""
Load a CLDF dataset into the database.
:param dataset:
:return:
"""
tables, ref_tables = schema(dataset)
# update the DB schema:
for t in tables:
if self._create_table_if_not_exists(t):
continue
db_cols = {
r[1]: r[2]
for r in self.fetchall("PRAGMA table_info({0})".format(t.name))
}
for col in t.columns:
if col.name not in db_cols:
with self.connection() as conn:
conn.execute(
"ALTER TABLE {0} ADD COLUMN \"{1.name}\" {1.db_type}"
.format(t.name, col))
else:
if db_cols[col.name] != col.db_type:
raise ValueError(
'column {0}:{1} {2} redefined with new type {3}'.
format(t.name, col.name, db_cols[col.name],
col.db_type))
for t in ref_tables.values():
self._create_table_if_not_exists(t)
# then load the data:
with self.connection() as db:
db.execute('PRAGMA foreign_keys = ON;')
pk = max([
r[0] for r in self.fetchall("SELECT ID FROM dataset", conn=db)
] or [0]) + 1
insert(db, 'dataset', 'ID,name,module,metadata_json',
(pk, '{0}'.format(dataset), dataset.module,
dumps(dataset.metadata_dict)))
insert(
db, 'datasetmeta', 'dataset_ID,key,value',
*[(pk, k, '{0}'.format(v))
for k, v in dataset.properties.items()])
# load sources:
rows = []
for src in dataset.sources.items():
values = [pk, src.id, src.genre
] + [src.get(k) for k in BIBTEX_FIELDS]
values.append(
dumps({
k: v
for k, v in src.items() if k not in BIBTEX_FIELDS
}))
rows.append(tuple(values))
insert(db, 'SourceTable', ['dataset_ID', 'ID', 'bibtex_type'] +
BIBTEX_FIELDS + ['extra'], *rows)
# For regular tables, we extract and keep references to sources.
refs = defaultdict(list)
for t in tables:
cols = {col.name: col for col in t.columns}
ref_table = ref_tables.get(t.name)
rows, keys = [], []
for row in dataset[t.name]:
keys, values = ['dataset_ID'], [pk]
for k, v in row.items():
if ref_table and k == ref_table.consumes:
refs[ref_table.name].append(
(row[t.primary_key], v))
else:
col = cols[k]
if isinstance(v, list):
v = (col.separator
or ';').join(col.convert(vv) for vv in v)
else:
v = col.convert(v)
keys.append(k)
values.append(v)
rows.append(tuple(values))
insert(db, t.name, keys, *rows)
# Now insert the references, i.e. the associations with sources:
for tname, items in refs.items():
rows = []
for oid, sources in items:
for source in sources:
sid, context = Sources.parse(source)
rows.append([pk, oid, sid, context])
oid_col = '{0}_ID'.format(tname.replace('Source', ''))
insert(db, tname, [
'dataset_ID', '{:}'.format(oid_col), 'Source_ID', 'Context'
], *rows)
db.commit()
def test_Sources(tmpdir):
src = Sources()
src.add(BIB, Source(
'book', 'huber2005', author='Herrmann Huber', year='2005', title='y'))
for entry in src:
assert entry.genre == 'book'
break
assert len(list(src.items())) == 3
assert len(list(src.keys())) == 3
refs = ['huber2005[1-6]', 'Obrazy', 'Elegie[34]']
assert src.format_refs(*list(src.expand_refs(refs))) == refs
assert '%s' % src['huber2005'] == 'Huber, Herrmann. 2005. y.'
with pytest.raises(ValueError):
src.add(5)
with pytest.raises(ValueError):
src.add('@misc{a.b,\n author="a.b"\n}')
with pytest.raises(ValueError):
_ = src['unknown']
assert _ # pragma: no cover
with pytest.raises(ValueError):
src.parse('a[x')
with pytest.raises(ValueError):
src.parse('[x]')
with pytest.raises(ValueError):
src.validate(['x'])
bib = str(tmpdir / 'test.bib')
src.write(bib)
src2 = Sources()
src2.read(bib)
src2.write(bib, ids=['huber2005'])
src = Sources.from_file(bib)
assert len(src) == 1