def test_Dataset_validate(tmpdir, mocker): ds = StructureDataset.in_dir(str(tmpdir / 'new')) ds.write(ValueTable=[]) assert ds.validate() ds['ValueTable'].tableSchema.columns = [] with pytest.raises(ValueError): ds.validate() assert not ds.validate(log=mocker.Mock()) ds.tablegroup.tables = [] with pytest.raises(ValueError): ds.validate() ds = StructureDataset.in_dir(str(tmpdir / 'new')) ds.add_component('LanguageTable') ds.write(ValueTable=[]) ds['LanguageTable'].common_props[ 'dc:conformsTo'] = 'http://cldf.clld.org/404' with pytest.raises(ValueError): ds.validate() ds = StructureDataset.in_dir(str(tmpdir / 'new')) ds['ValueTable'].get_column('Source').propertyUrl = URITemplate( 'http://cldf.clld.org/404') ds.write(ValueTable=[]) with pytest.raises(ValueError): ds.validate()
def test_Dataset_write(tmpdir): ds = StructureDataset.from_metadata(str(tmpdir)) ds.write(ValueTable=[]) assert (tmpdir / 'values.csv').exists() ds.validate() ds.add_sources("@misc{ky,\ntitle={the title}\n}") ds.write(ValueTable=[{ 'ID': '1', 'Language_ID': 'abcd1234', 'Parameter_ID': 'f1', 'Value': 'yes', 'Source': ['key[1-20]', 'ky'], }]) ds2 = StructureDataset.from_metadata( str(tmpdir.join('StructureDataset-metadata.json'))) assert ds2['ValueTable'].common_props['dc:extent'] == 1 assert {s[1]: s[2] for s in ds.stats()}['ValueTable'] == 1 ds['ValueTable'].common_props['dc:extent'] = 3 assert {s[1]: s[2] for s in ds.stats()}['ValueTable'] == 3 with pytest.raises(ValueError): ds.validate() ds.sources.add("@misc{key,\ntitle={the title}\n}") ds.write(ValueTable=({ 'ID': '1', 'Language_ID': 'abcd1234', 'Parameter_ID': 'f1', 'Value': 'yes', 'Source': ['key[1-20]'], } for _ in range(1))) ds.validate() ds.add_component('ExampleTable') ds.write(ValueTable=[{ 'ID': '1', 'Language_ID': 'abcd1234', 'Parameter_ID': 'f1', 'Value': 'yes', 'Source': ['key[1-20]'], }], ExampleTable=[{ 'ID': '1', 'Language_ID': 'abcd1234', 'Primary': 'si', 'Translation': 'yes', 'Analyzed': ['morph1', 'morph2', 'morph3'], 'Gloss': ['gl1', 'gl2'], }]) with pytest.raises(ValueError): ds.validate() ds['ExampleTable'].write([{ 'ID': '1', 'Language_ID': 'abcd1234', 'Primary_Text': 'si', 'Translated_Text': 'yes', 'Analyzed_Word': ['morph1', 'morph2', 'morph3'], 'Gloss': ['gl1', 'gl2', 'gl3'], }]) ds.validate()
def test_Dataset_validate(tmpdir, mocker): ds = StructureDataset.in_dir(str(tmpdir / 'new')) ds.write(ValueTable=[]) values = tmpdir / 'new' / 'values.csv' assert values.check() Path(str(values)).unlink() log = mocker.Mock() assert not ds.validate(log=log) assert log.warn.called ds.write(ValueTable=[]) assert ds.validate() ds['ValueTable'].tableSchema.columns = [] with pytest.raises(ValueError): ds.validate() assert not ds.validate(log=mocker.Mock()) ds.tablegroup.tables = [] with pytest.raises(ValueError): ds.validate() ds = StructureDataset.in_dir(str(tmpdir / 'new')) ds.add_component('LanguageTable') ds.write(ValueTable=[], LanguageTable=[]) assert ds.validate() # test violation of referential integrity: ds.write(ValueTable=[{ 'ID': '1', 'Value': '1', 'Language_ID': 'lid', 'Parameter_ID': 'pid' }], LanguageTable=[]) assert not ds.validate(log=mocker.Mock()) # test an invalid CLDF URL: ds['LanguageTable'].common_props[ 'dc:conformsTo'] = 'http://cldf.clld.org/404' with pytest.raises(ValueError): ds.validate() ds = StructureDataset.in_dir(str(tmpdir / 'new')) ds['ValueTable'].get_column('Source').propertyUrl = URITemplate( 'http://cldf.clld.org/404') ds.write(ValueTable=[]) with pytest.raises(ValueError): ds.validate()
def test_newcol(tmpdir, db): ds = StructureDataset.in_dir(str(tmpdir / 'd')) # We rename the ID column of the ValueTable. Note that the propertyUrl # remains the same: ds['ValueTable', 'ID'].name = 'idx' ds['ValueTable'].tableSchema.columns.extend([ Column(name='col1', datatype='anyURI'), Column(name='col2', datatype='integer'), Column(name='col3'), ]) ds.write(ValueTable=[{ 'idx': '1', 'Language_ID': 'l', 'Parameter_ID': 'p', 'Value': 'v', 'Source': ['meier2015'], 'col2': 5, 'col1': anyURI().to_python('http://example.org') }]) db.create() with pytest.raises(IntegrityError): # A missing source is referenced! db.load(ds) ds.add_sources("@misc{meier2015,\ntitle={title}\n}") db.load(ds) assert db.fetchone("""\ select s.title from SourceTable as s, ValueSource as vs, ValueTable as v where s.ID = vs.Source_ID and vs.Value_ID = v.id and v.id = 1""")[0] == 'title' assert db.fetchone( "select col1 from valuetable")[0] == 'http://example.org' assert db.fetchone("select col2 from valuetable")[0] == 5
def test_Dataset_validate_missing_table(tmpdir, mocker): ds = StructureDataset.from_metadata(str(tmpdir)) ds.tablegroup.tables = [] ds.write() log = mocker.Mock() ds.validate(log=log) assert log.warn.called
def test_Dataset_remove_columns(tmpdir): ds = StructureDataset.in_dir(str(tmpdir / 'new')) ds.add_component('LanguageTable') ds.add_foreign_key('ValueTable', 'Value', 'LanguageTable', 'Name') ds.remove_columns('languages.csv', 'ID') ds.write(ValueTable=[{ 'ID': '1', 'Language_ID': '1', 'Parameter_ID': '1', 'Value': '1' }], LanguageTable=[{ 'Name': 'x' }]) with pytest.raises(ValueError): ds.validate() ds.write(ValueTable=[{ 'ID': '1', 'Language_ID': '1', 'Parameter_ID': '1', 'Value': '1' }], LanguageTable=[{ 'Name': '1' }]) assert ds.validate()
def test_Dataset_validate_missing_table(tmpdir, mocker): ds = StructureDataset.from_metadata(str(tmpdir)) ds.tablegroup.tables = [] ds.write() log = mocker.Mock() ds.validate(log=log) assert log.warn.called
def test_Dataset_auto_foreign_keys(tmpdir): ds = StructureDataset.in_dir(str(tmpdir), empty_tables=True) ds.add_component( { 'url': 'languages.csv', 'dc:conformsTo': 'http://cldf.clld.org/v1.0/terms.rdf#LanguageTable', 'tableSchema': {'primaryKey': 'lid'}}, {'name': 'lid', 'propertyUrl': 'http://cldf.clld.org/v1.0/terms.rdf#id'}) ds.add_component( { 'url': 'values.csv', 'dc:conformsTo': 'http://cldf.clld.org/v1.0/terms.rdf#ValueTable', 'tableSchema': {'primaryKey': 'vid'}}, {'name': 'vid', 'propertyUrl': 'http://cldf.clld.org/v1.0/terms.rdf#id'}, { 'name': 'feature', 'propertyUrl': 'http://cldf.clld.org/v1.0/terms.rdf#parameterReference'}, { 'name': 'language_lid', 'propertyUrl': 'http://cldf.clld.org/v1.0/terms.rdf#languageReference'}, {'name': 'value', 'propertyUrl': 'http://cldf.clld.org/v1.0/terms.rdf#value'}) ds.write( LanguageTable=[{'lid': 'spam'}], ValueTable=[ {'vid': '1', 'feature': 'bing', 'language_lid': 'spam', 'value': 'eggs'}]) ds.validate()
def test_Dataset_remove_table(tmpdir): ds = StructureDataset.in_dir(str(tmpdir / 'new')) ds.add_component('LanguageTable') ds.add_component('ParameterTable') ds.write( ValueTable=[{ 'ID': '1', 'Language_ID': '1', 'Parameter_ID': 1, 'Value': 1 }], LanguageTable=[{ 'ID': '1', 'Name': 'l' }], ParameterTable=[{ 'ID': '1', 'Name': 'l' }], ) assert ds.validate() ds.remove_table('LanguageTable') # Make sure other foreign key constraints are still enforced: ds.write( ValueTable=[{ 'ID': '1', 'Language_ID': '1', 'Parameter_ID': 1, 'Value': 1 }], ParameterTable=[{ 'ID': 'x', 'Name': 'l' }], ) with pytest.raises(ValueError): ds.validate() # But foreign keys into the removed table are not: ds.write( ValueTable=[{ 'ID': '1', 'Language_ID': '1', 'Parameter_ID': 1, 'Value': 1 }], ParameterTable=[{ 'ID': '1', 'Name': 'l' }], ) assert ds.validate()
def test_Dataset_validate(tmpdir, mocker): ds = StructureDataset.in_dir(str(tmpdir / 'new')) ds.write(ValueTable=[]) values = tmpdir / 'new' / 'values.csv' assert values.check() remove(str(values)) log = mocker.Mock() assert not ds.validate(log=log) assert log.warn.called ds.write(ValueTable=[]) assert ds.validate() ds['ValueTable'].tableSchema.columns = [] with pytest.raises(ValueError): ds.validate() assert not ds.validate(log=mocker.Mock()) ds.tablegroup.tables = [] with pytest.raises(ValueError): ds.validate() ds = StructureDataset.in_dir(str(tmpdir / 'new')) ds.add_component('LanguageTable') ds.write(ValueTable=[], LanguageTable=[]) assert ds.validate() # test violation of referential integrity: ds.write(ValueTable=[{'ID': '1', 'Value': '1', 'Language_ID': 'lid', 'Parameter_ID': 'pid'}], LanguageTable=[]) assert not ds.validate(log=mocker.Mock()) # test an invalid CLDF URL: ds['LanguageTable'].common_props['dc:conformsTo'] = 'http://cldf.clld.org/404' with pytest.raises(ValueError): ds.validate() ds = StructureDataset.in_dir(str(tmpdir / 'new')) ds['ValueTable'].get_column('Source').propertyUrl = URITemplate( 'http://cldf.clld.org/404') ds.write(ValueTable=[]) with pytest.raises(ValueError): ds.validate()
def test_Dataset_validate_custom_validator(tmpdir): ds = StructureDataset.in_dir(str(tmpdir / 'new')) ds.write(ValueTable=[ {'ID': '1', 'Value': 'x', 'Language_ID': 'l', 'Parameter_ID': 'p'}]) assert ds.validate() def v(tg, t, c, r): if r[c.name] == 'x': raise ValueError() with pytest.raises(ValueError): ds.validate(validators=[('ValueTable', 'Value', v)])
def test_modules(tmpdir): ds = Dataset(_make_tg(tmpdir)) assert ds.primary_table is None ds = Dataset(_make_tg(tmpdir, {"url": "data.csv"})) assert ds.primary_table is None ds = Dataset(_make_tg(tmpdir, { "url": "data.csv", "dc:conformsTo": "http://cldf.clld.org/v1.0/terms.rdf#ValueTable"})) assert ds.primary_table == 'ValueTable' assert Wordlist.in_dir(str(tmpdir)).primary_table assert Dictionary.in_dir(str(tmpdir)).primary_table assert StructureDataset.in_dir(str(tmpdir)).primary_table
def test_Dataset_write(tmpdir): ds = StructureDataset.from_metadata(str(tmpdir)) ds.write(ValueTable=[]) assert (tmpdir / 'values.csv').exists() ds.validate() ds.add_sources("@misc{ky,\ntitle={the title}\n}") ds.write(ValueTable=[ { 'ID': '1', 'Language_ID': 'abcd1234', 'Parameter_ID': 'f1', 'Value': 'yes', 'Source': ['key[1-20]', 'ky'], }]) ds2 = StructureDataset.from_metadata( str(tmpdir.join('StructureDataset-metadata.json'))) assert ds2['ValueTable'].common_props['dc:extent'] == 1 assert {s[1]: s[2] for s in ds.stats()}['ValueTable'] == 1 ds['ValueTable'].common_props['dc:extent'] = 3 assert {s[1]: s[2] for s in ds.stats()}['ValueTable'] == 3 with pytest.raises(ValueError): ds.validate() ds.sources.add("@misc{key,\ntitle={the title}\n}") ds.write(ValueTable=( { 'ID': '1', 'Language_ID': 'abcd1234', 'Parameter_ID': 'f1', 'Value': 'yes', 'Source': ['key[1-20]'], } for _ in range(1))) ds.validate() ds.add_component('ExampleTable') ds.write( ValueTable=[ { 'ID': '1', 'Language_ID': 'abcd1234', 'Parameter_ID': 'f1', 'Value': 'yes', 'Source': ['key[1-20]'], }], ExampleTable=[ { 'ID': '1', 'Language_ID': 'abcd1234', 'Primary': 'si', 'Translation': 'yes', 'Analyzed': ['morph1', 'morph2', 'morph3'], 'Gloss': ['gl1', 'gl2'], }]) with pytest.raises(ValueError): ds.validate() ds['ExampleTable'].write([ { 'ID': '1', 'Language_ID': 'abcd1234', 'Primary_Text': 'si', 'Translated_Text': 'yes', 'Analyzed_Word': ['morph1', 'morph2', 'morph3'], 'Gloss': ['gl1', 'gl2', 'gl3'], }]) ds.validate()