def test_newcol(tmpdir, db): ds = StructureDataset.in_dir(str(tmpdir / 'd')) # We rename the ID column of the ValueTable. Note that the propertyUrl # remains the same: ds['ValueTable', 'ID'].name = 'idx' ds['ValueTable'].tableSchema.columns.extend([ Column(name='col1', datatype='anyURI'), Column(name='col2', datatype='integer'), Column(name='col3'), ]) ds.write(ValueTable=[{ 'idx': '1', 'Language_ID': 'l', 'Parameter_ID': 'p', 'Value': 'v', 'Source': ['meier2015'], 'col2': 5, 'col1': anyURI().to_python('http://example.org') }]) db.create() with pytest.raises(IntegrityError): # A missing source is referenced! db.load(ds) ds.add_sources("@misc{meier2015,\ntitle={title}\n}") db.load(ds) assert db.fetchone("""\ select s.title from SourceTable as s, ValueSource as vs, ValueTable as v where s.ID = vs.Source_ID and vs.Value_ID = v.id and v.id = 1""")[0] == 'title' assert db.fetchone( "select col1 from valuetable")[0] == 'http://example.org' assert db.fetchone("select col2 from valuetable")[0] == 5
def test_db(tmpdir, dataset, mocker, capsys): db = Database(str(tmpdir.join('lexibank.sqlite'))) db.load(dataset) db.create(exists_ok=True) with pytest.raises(ValueError): db.create() db.create(force=True) db.load(dataset) db.load_glottolog_data(dataset.glottolog) db.load_concepticon_data(mocker.Mock(conceptsets={})) for sql in db.sql: db.fetchall(sql) with db.connection() as conn: db.fetchall('select * from dataset', conn=conn, verbose=True) out, _ = capsys.readouterr() assert 'select' in out db.create(force=True) db.load(dataset) cldf_ds = dataset.cldf_reader() cols = cldf_ds['FormTable'].tableSchema.columns cols.append(Column(name='custom')) cldf_ds.write_metadata() db.load(dataset) cols.pop() cols.append(Column(name='custom', datatype='integer')) cldf_ds.write_metadata() with pytest.raises(ValueError): db.load(dataset) cols.pop() cldf_ds.write_metadata() db.load(dataset)
def make_column(spec): if isinstance(spec, string_types): if spec in TERMS.by_uri: return TERMS.by_uri[spec].to_column() return Column(name=spec, datatype='string') if isinstance(spec, dict): return Column.fromvalue(spec) if isinstance(spec, Column): return spec raise TypeError(spec)
def make_column(spec): if isinstance(spec, string_types): if spec in TERMS.by_uri: return TERMS.by_uri[spec].to_column() return Column(name=spec, datatype='string') if isinstance(spec, dict): return Column.fromvalue(spec) if isinstance(spec, Column): return spec raise TypeError(spec)
def to_column(self): col = Column( name=self.csvw_prop('name') or self.element.find(qname(RDFS, 'label')).text, propertyUrl=self.element.attrib[qname(RDF, 'about')], datatype=self.csvw_prop('datatype') or 'string') for k in ['separator', 'null', 'valueUrl']: v = self.csvw_prop(k) if v: setattr(col, k, v) return col
def __str__(self): # We overwrite the base class' method to fix the order of columns. tg = TableGroup.fromvalue(self.MD) for col in sorted( self.column_labels, key=lambda t: (t == IPA_COLUMN, t.lower()), reverse=True): if col != self.GRAPHEME_COL: tg.tables[0].tableSchema.columns.append( Column.fromvalue({"name": col, "null": self.NULL})) return tg.tables[0].write(self.iteritems(), fname=None).decode('utf8').strip()
def __init__(self, dataset): self._count = defaultdict(int) self._cognate_count = defaultdict(int) self.dataset = dataset md = self.dataset.cldf_dir / MD_NAME if not md.exists(): md = self.dataset.cldf_dir / ALT_MD_NAME if not md.exists(): md = self.dataset.cldf_dir / MD_NAME copy(Path(__file__).parent / MD_NAME, md) self.wl = Wordlist.from_metadata(md) default_cldf = Wordlist.from_metadata( Path(__file__).parent / 'cldf-metadata.json') self.objects = {} self._obj_index = {} for cls in [ self.dataset.lexeme_class, self.dataset.language_class, self.dataset.concept_class, self.dataset.cognate_class, ]: self.objects[cls.__cldf_table__()] = [] self._obj_index[cls.__cldf_table__()] = set() cols = set( col.header for col in self.wl[cls.__cldf_table__()].tableSchema.columns) properties = set( col.propertyUrl.uri for col in self.wl[cls.__cldf_table__()].tableSchema.columns if col.propertyUrl) for field in cls.fieldnames(): try: col = default_cldf[cls.__cldf_table__(), field] # # We added Latitude and Longitude to the default metadata later, and want to # make sure, existing datasets are upgraded silently. # if field in ['Latitude', 'Longitude'] \ and cls.__cldf_table__() == 'LanguageTable': properties.add(col.propertyUrl.uri) self.wl[cls.__cldf_table__(), field].propertyUrl = col.propertyUrl self.wl[cls.__cldf_table__(), field].datatype = col.datatype except KeyError: col = Column(name=field, datatype="string") if (col.propertyUrl and col.propertyUrl.uri not in properties) or \ ((not col.propertyUrl) and (field not in cols)): self.wl[cls.__cldf_table__()].tableSchema.columns.append( col)
def test_update(tmpdir, db): ds = Dictionary.in_dir(str(tmpdir / 'd1')) ds.write(EntryTable=[], SenseTable=[]) ds2 = Dictionary.in_dir(str(tmpdir / 'd2')) ds2.write(EntryTable=[], SenseTable=[]) db.create() db.load(ds) db.load(ds2) ds.tables[0].tableSchema.columns.append( Column(name='newcol', datatype='integer')) db.load(ds) ds.tables[0].tableSchema.columns[-1].datatype.base = 'string' with pytest.raises(ValueError): db.load(ds)
def __enter__(self): super().__enter__() default_cldf = Wordlist.from_metadata( pathlib.Path(__file__).parent / MD_NAME) self._obj_index = {} for cls in [ self.dataset.lexeme_class, self.dataset.language_class, self.dataset.concept_class, self.dataset.cognate_class, ]: self.objects[cls.__cldf_table__()] = [] self._obj_index[cls.__cldf_table__()] = set() cols = set( col.header for col in self.cldf[cls.__cldf_table__()].tableSchema.columns) properties = set( col.propertyUrl.uri for col in self.cldf[cls.__cldf_table__()].tableSchema.columns if col.propertyUrl) for field in cls.fieldnames(): try: col = default_cldf[cls.__cldf_table__(), field] # # We added Latitude and Longitude to the default metadata later, and want to # make sure, existing datasets are upgraded silently. # if field in ['Latitude', 'Longitude'] \ and cls.__cldf_table__() == 'LanguageTable': # pragma: no cover properties.add(col.propertyUrl.uri) self.cldf[cls.__cldf_table__(), field].propertyUrl = col.propertyUrl self.cldf[cls.__cldf_table__(), field].datatype = col.datatype except KeyError: col = Column(name=field, datatype="string") if (col.propertyUrl and col.propertyUrl.uri not in properties) or \ ((not col.propertyUrl) and (field not in cols)): self.cldf[cls.__cldf_table__()].tableSchema.columns.append( col) return self
def create(self, req, filename=None, verbose=True, outfile=None): cldf_cfg = req.registry.getUtility(ICldfConfig) with TemporaryDirectory() as tmpd: cls = getattr(dataset, cldf_cfg.module) ds = cls.in_dir(tmpd) ds.properties['dc:bibliographicCitation '] = text_citation(req, req.dataset) ds.properties['dc:publisher'] = '%s, %s' % ( req.dataset.publisher_name, req.dataset.publisher_place) ds.properties['dc:license'] = req.dataset.license ds.properties['dc:issued'] = req.dataset.published.isoformat() ds.properties['dc:title'] = req.dataset.name ds.properties['dc:creator'] = req.dataset.formatted_editors() ds.properties['dc:identifier'] = req.resource_url(req.dataset) ds.properties['dcat:accessURL'] = req.route_url('download') if DBSession.query(Sentence).count(): ds.add_component('ExampleTable') if DBSession.query(DomainElement).count(): ds.add_component('CodeTable', {'name': 'Number', 'datatype': 'integer'}) ds.add_component('ParameterTable') ds.add_component('LanguageTable') ds.add_table('contributions.csv', 'ID', 'Name', 'Description', 'Contributors') ds.add_columns(ds.primary_table, Column.fromvalue( { 'name': 'Contribution_ID', 'datatype': 'string', 'valueUrl': url_template(req, 'contribution', 'contribution').uri, })) ds.add_foreign_key( ds.primary_table, 'Contribution_ID', 'contributions.csv', 'ID') ds['LanguageTable'].aboutUrl = url_template(req, 'language', 'ID') ds['ParameterTable'].aboutUrl = url_template(req, 'parameter', 'ID') ds[ds.primary_table].aboutUrl = url_template(req, 'value', 'ID') cldf_cfg.custom_schema(req, ds) for src in cldf_cfg.query(Source): ds.sources.add(cldf_cfg.convert(Source, src, req)) fname = outfile or self.abspath(req) transaction.abort() tabledata = defaultdict(list) for table, model in [ ('ParameterTable', Parameter), ('CodeTable', DomainElement), ('LanguageTable', Language), ('ExampleTable', Sentence), ('contributions.csv', Contribution), (ds.primary_table, Value), ]: if verbose: print('exporting {0} ...'.format(model)) transaction.begin() for item in cldf_cfg.query(model): tabledata[table].append(cldf_cfg.convert(model, item, req)) transaction.abort() if verbose: print('... done') transaction.begin() ds.write(**cldf_cfg.custom_tabledata(req, tabledata)) ds.validate() shutil.make_archive( fname.parent.joinpath(fname.stem).as_posix(), 'zip', tmpd.as_posix())
def test_make_column(): assert make_column(term_uri('source')).separator == ';' assert make_column(Column('name')).datatype is None with pytest.raises(TypeError): make_column(5)
def create(self, req, filename=None, verbose=True, outfile=None): cldf_cfg = req.registry.getUtility(ICldfConfig) with TemporaryDirectory() as tmpd: cls = getattr(dataset, cldf_cfg.module) ds = cls.in_dir(tmpd) ds.properties['dc:bibliographicCitation'] = text_citation( req, req.dataset) ds.properties['dc:publisher'] = '%s, %s' % ( req.dataset.publisher_name, req.dataset.publisher_place) ds.properties['dc:license'] = req.dataset.license ds.properties['dc:issued'] = req.dataset.published.isoformat() ds.properties['dc:title'] = req.dataset.name ds.properties['dc:creator'] = req.dataset.formatted_editors() ds.properties['dc:identifier'] = req.resource_url(req.dataset) ds.properties['dcat:accessURL'] = req.route_url('download') if DBSession.query(Sentence).count(): ds.add_component('ExampleTable') if DBSession.query(DomainElement).count(): ds.add_component('CodeTable', { 'name': 'Number', 'datatype': 'integer' }) ds.add_component('ParameterTable') ds.add_component('LanguageTable') ds.add_table('contributions.csv', 'ID', 'Name', 'Description', 'Contributors') ds.add_columns( ds.primary_table, Column.fromvalue({ 'name': 'Contribution_ID', 'datatype': 'string', 'valueUrl': url_template(req, 'contribution', 'contribution').uri, })) ds.add_foreign_key(ds.primary_table, 'Contribution_ID', 'contributions.csv', 'ID') ds['LanguageTable'].aboutUrl = url_template(req, 'language', 'ID') ds['ParameterTable'].aboutUrl = url_template( req, 'parameter', 'ID') ds[ds.primary_table].aboutUrl = url_template(req, 'value', 'ID') cldf_cfg.custom_schema(req, ds) for src in cldf_cfg.query(Source): ds.sources.add(cldf_cfg.convert(Source, src, req)) fname = outfile or self.abspath(req) transaction.abort() tabledata = defaultdict(list) for table, model in [ ('ParameterTable', Parameter), ('CodeTable', DomainElement), ('LanguageTable', Language), ('ExampleTable', Sentence), ('contributions.csv', Contribution), (ds.primary_table, Value), ]: if verbose: print('exporting {0} ...'.format(model)) transaction.begin() for item in cldf_cfg.query(model): tabledata[table].append(cldf_cfg.convert(model, item, req)) transaction.abort() if verbose: print('... done') transaction.begin() ds.write(**cldf_cfg.custom_tabledata(req, tabledata)) ds.validate() shutil.make_archive(str(fname.parent / fname.stem), 'zip', str(tmpd))
def add_metadata(fname: Path, logger: cli.logging.Logger = cli.logger): if fname.name != "forms.csv": cli.Exit.CLI_ARGUMENT_ERROR( "A metadata-free Wordlist must be in a file called 'forms.csv'.") default_wordlist = TableGroup.from_file( pycldf.util.pkg_path("modules", "Wordlist-metadata.json")) default_wordlist._fname = fname.with_name("Wordlist-metadata.json") ds = pycldf.Wordlist(default_wordlist) # `from_data` checks that the reqired columns of the FormTable are present, # but it does not consolidate the columns further. colnames = next(iterrows(fname)) understood_colnames = { c.name for c in ds[ds.primary_table].tableSchema.columns if c.name in colnames } more_columns = { c.propertyUrl.uri: c for c in ds[ds.primary_table].tableSchema.columns if c.name not in understood_colnames } logger.info( "CLDF freely understood the columns %s in your forms.csv.", sorted(understood_colnames), ) # Consider the columns that were not understood. columns_without_metadata = set(colnames) - understood_colnames for column_name in columns_without_metadata: column: Column # Maybe they are known CLDF properties? if column_name in pycldf.terms.TERMS: column = pycldf.TERMS[column_name].to_column() # Maybe they are CLDF default column names? elif column_name in DEFAULT_NAME_COLUMNS: column = DEFAULT_NAME_COLUMNS[column_name] # Maybe they are columns that Lexedata knows to handle? elif column_name in LEXEDATA_COLUMNS: column = LEXEDATA_COLUMNS[column_name] # Maybe they are columns inherited from LingPy? elif column_name.upper() in LINGPY_COLUMNS: column = LINGPY_COLUMNS[column_name.upper()] # Maybe they are some name we have seen before? elif column_name in OTHER_KNOWN_COLUMNS: column = OTHER_KNOWN_COLUMNS[column_name] else: # TODO: Maybe they look like they have a specific type? ... # Otherwise, they are probably just text to be kept. column = Column( datatype=Datatype(base="string"), default="", null=[""], name=column_name, ) column.name = column_name ds[ds.primary_table].tableSchema.columns.append(column) summary = column.propertyUrl or column.datatype logger.info(f"Column {column_name} seems to be a {summary} column.") if column.propertyUrl: to_be_replaced = more_columns.pop(column.propertyUrl.uri, None) if to_be_replaced is not None: ds[ds.primary_table].tableSchema.columns.remove(to_be_replaced) for column in more_columns.values(): logger.info( f"Also added column {column.name}, as expected for a FormTable.") ds[ds.primary_table].tableSchema.columns.sort( key=lambda k: colnames.index(k.name) if k.name in colnames else 1e10) # TODO: Once lexedata is properly published, we can give a better URL. ds.properties["dc:contributor"] = [ "https://github.com/Anaphory/lexedata/blob/master/src/lexedata/edit/add_metadata.py" ] return ds
from csvw.dsv import iterrows from csvw.metadata import Column, Datatype, TableGroup from lexedata import cli DEFAULT_NAME_COLUMNS = { column.name: column for column in (term.to_column() for term in pycldf.TERMS.values()) } LEXEDATA_COLUMNS = { "Status": Column( datatype=Datatype(base="string"), default="", lang="eng", null=[""], name="Value", aboutUrl="...", ), "Orthographic": Column( datatype=Datatype(base="string"), default="", null=[""], name="Orthographic", aboutUrl="...", ), "Phonemic": Column( datatype=Datatype(base="string"), default="",