def __init__(self): r = ElementTree.parse(pkg_path('terms.rdf').as_posix()).getroot() terms = [Term.from_element(e) for e in r.findall(qname(RDF, 'Property'))] for e in r.findall(qname(RDFS, 'Class')): terms.append(Term.from_element(e)) dict.__init__(self, {t.name: t for t in terms}) self.by_uri = {t.uri: t for t in terms}
def from_metadata(cls, fname): fname = pathlib.Path(fname) if fname.is_dir(): name = '{0}{1}'.format(cls.__name__, MD_SUFFIX) tablegroup = TableGroup.from_file(pkg_path('modules', name)) # adapt the path of the metadata file such that paths to tables are resolved # correctly: tablegroup._fname = fname.joinpath(name) else: tablegroup = TableGroup.from_file(fname) comps = collections.Counter() for table in tablegroup.tables: try: dt = Dataset.get_tabletype(table) if dt: comps.update([dt]) except ValueError: pass if comps and comps.most_common(1)[0][1] > 1: raise ValueError('{0}: duplicate components!'.format(fname)) for mod in get_modules(): if mod.match(tablegroup): return mod.cls(tablegroup) return cls(tablegroup)
def add_component(self, component, *cols, **kw): if isinstance(component, str): component = jsonlib.load( pkg_path('components', '{0}{1}'.format(component, MD_SUFFIX))) if isinstance(component, dict): component = Table.fromvalue(component) assert isinstance(component, Table) if kw.get('url'): component.url = Link(kw['url']) for other_table in self.tables: if other_table.url == component.url: raise ValueError('tables must have distinct url properties') self.add_columns(component, *cols) try: table_type = self.get_tabletype(component) except ValueError: table_type = None if table_type: for other_table in self.tables: try: other_table_type = self.get_tabletype(other_table) except ValueError: # pragma: no cover continue if other_table_type == table_type: raise ValueError('components must not be added twice') self.tables.append(component) component._parent = self.tablegroup self.auto_constraints(component) return component
def from_metadata(cls, fname): fname = Path(fname) if fname.is_dir(): name = '{0}{1}'.format(cls.__name__, MD_SUFFIX) tablegroup = TableGroup.from_file(pkg_path('modules', name)) # adapt the path of the metadata file such that paths to tables are resolved # correctly: tablegroup._fname = fname.joinpath(name) else: tablegroup = TableGroup.from_file(fname) for mod in get_modules(): if mod.match(tablegroup): return mod.cls(tablegroup) return cls(tablegroup)
def __attrs_post_init__(self): if self.default_metadata_path: self.default_metadata_path = pathlib.Path( self.default_metadata_path) try: Dataset.from_metadata(self.default_metadata_path) except Exception: raise ValueError('invalid default metadata: {0}'.format( self.default_metadata_path)) else: self.default_metadata_path = pkg_path( 'modules', '{0}{1}'.format(self.module, MD_SUFFIX)) if not self.metadata_fname: self.metadata_fname = self.default_metadata_path.name
def get_modules(): global _modules if not _modules: ds = sys.modules[__name__] for p in pkg_path('modules').glob('*{0}'.format(MD_SUFFIX)): tg = TableGroup.from_file(p) mod = Module( tg.common_props['dc:conformsTo'], tg.tables[0].url.string if tg.tables else None) mod.cls = getattr(ds, mod.id) _modules.append(mod) # prefer Wordlist over ParallelText (forms.csv) _modules = sorted( _modules, key=lambda m: (m.cls in (Wordlist, ParallelText), m.cls is ParallelText)) return _modules
def get_modules(): global _modules if not _modules: ds = sys.modules[__name__] for p in pkg_path('modules').glob('*{0}'.format(MD_SUFFIX)): tg = TableGroup.from_file(p) mod = Module( tg.common_props['dc:conformsTo'], tg.tables[0].url.string if tg.tables else None) mod.cls = getattr(ds, mod.id) _modules.append(mod) # prefer Wordlist over ParallelText (forms.csv) _modules = sorted( _modules, key=lambda m: (m.cls in (Wordlist, ParallelText), m.cls is ParallelText)) return _modules
def component(self, cldf=None): """ :param cldf: `pycldf.Dataset` instance to which the link refers. :return: Name of the CLDF component the link pertains to or `None`. """ name = self.table_or_fname if cldf is None: # If no CLDF dataset is passed as context, we can only detect links using proper # component names as path: return name if name == 'Source' or pkg_path('components', name + MD_SUFFIX).exists() \ else None if name == cldf.bibname or name == 'Source': return 'Source' try: return cldf.get_tabletype(cldf[name]) except (KeyError, ValueError): return None
def add_component(self, component, *cols): if isinstance(component, string_types): component = jsonlib.load( pkg_path('components', '{0}{1}'.format(component, MD_SUFFIX))) if isinstance(component, dict): component = Table.fromvalue(component) assert isinstance(component, Table) self.add_columns(component, *cols) try: table_type = self.get_tabletype(component) except ValueError: table_type = None for other_table in self.tables: try: other_table_type = self.get_tabletype(other_table) except ValueError: continue if other_table_type == table_type: raise ValueError('components must not be added twice') self.tables.append(component) component._parent = self.tablegroup self.auto_constraints(component)
def add_component(self, component, *cols): if isinstance(component, string_types): component = jsonlib.load( pkg_path('components', '{0}{1}'.format(component, MD_SUFFIX))) if isinstance(component, dict): component = Table.fromvalue(component) assert isinstance(component, Table) self.add_columns(component, *cols) try: table_type = self.get_tabletype(component) except ValueError: table_type = None for other_table in self.tables: try: other_table_type = self.get_tabletype(other_table) except ValueError: continue if other_table_type == table_type: raise ValueError('components must not be added twice') self.tables.append(component) component._parent = self.tablegroup self.auto_constraints(component)
def from_metadata(cls, fname): fname = Path(fname) if fname.is_dir(): name = '{0}{1}'.format(cls.__name__, MD_SUFFIX) tablegroup = TableGroup.from_file(pkg_path('modules', name)) # adapt the path of the metadata file such that paths to tables are resolved # correctly: tablegroup._fname = fname.joinpath(name) else: tablegroup = TableGroup.from_file(fname) comps = Counter() for table in tablegroup.tables: try: comps.update([Dataset.get_tabletype(table)]) except ValueError: pass if comps and comps.most_common(1)[0][1] > 1: raise ValueError('{0}: duplicate components!'.format(fname)) for mod in get_modules(): if mod.match(tablegroup): return mod.cls(tablegroup) return cls(tablegroup)
def validate(self, log=None, validators=None): validators = validators or [] validators.extend(VALIDATORS) success = True default_tg = TableGroup.from_file( pkg_path('modules', '{0}{1}'.format(self.module, MD_SUFFIX))) for default_table in default_tg.tables: dtable_uri = default_table.common_props['dc:conformsTo'] try: table = self[dtable_uri] except KeyError: log_or_raise('{0} requires {1}'.format(self.module, dtable_uri), log=log) success = False table = None if table: default_cols = { c.propertyUrl.uri for c in default_table.tableSchema.columns if c.required or c.common_props.get('dc:isRequiredBy')} cols = { c.propertyUrl.uri for c in table.tableSchema.columns if c.propertyUrl} table_uri = table.common_props['dc:conformsTo'] for col in default_cols - cols: log_or_raise('{0} requires column {1}'.format(table_uri, col), log=log) success = False for table in self.tables: type_uri = table.common_props.get('dc:conformsTo') if type_uri: try: TERMS.is_cldf_uri(type_uri) except ValueError: success = False log_or_raise('invalid CLDF URI: {0}'.format(type_uri), log=log) # FIXME: check whether table.common_props['dc:conformsTo'] is in validators! validators_ = [] for col in table.tableSchema.columns: if col.propertyUrl: col_uri = col.propertyUrl.uri try: TERMS.is_cldf_uri(col_uri) except ValueError: success = False log_or_raise('invalid CLDF URI: {0}'.format(col_uri), log=log) for table_, col_, v_ in validators: if (not table_ or table is self.get(table_)) and col is self.get((table, col_)): validators_.append((col, v_)) fname = Path(table.url.resolve(table._parent.base)) if fname.exists(): for fname, lineno, row in table.iterdicts(log=log, with_metadata=True): for col, validate in validators_: try: validate(self, table, col, row) except ValueError as e: log_or_raise( '{0}:{1}:{2} {3}'.format(fname.name, lineno, col.name, e), log=log) success = False if not table.check_primary_key(log=log): success = False if not self.tablegroup.check_referential_integrity(log=log): success = False return success
def validate(self, log=None, validators=None): validators = validators or [] validators.extend(VALIDATORS) success = True default_tg = TableGroup.from_file( pkg_path('modules', '{0}{1}'.format(self.module, MD_SUFFIX))) for default_table in default_tg.tables: dtable_uri = default_table.common_props['dc:conformsTo'] try: table = self[dtable_uri] except KeyError: log_or_raise('{0} requires {1}'.format(self.module, dtable_uri), log=log) success = False table = None if table: default_cols = { c.propertyUrl.uri for c in default_table.tableSchema.columns if c.required or c.common_props.get('dc:isRequiredBy')} cols = { c.propertyUrl.uri for c in table.tableSchema.columns if c.propertyUrl} table_uri = table.common_props['dc:conformsTo'] for col in default_cols - cols: log_or_raise('{0} requires column {1}'.format(table_uri, col), log=log) success = False for table in self.tables: type_uri = table.common_props.get('dc:conformsTo') if type_uri: try: TERMS.is_cldf_uri(type_uri) except ValueError: success = False log_or_raise('invalid CLDF URI: {0}'.format(type_uri), log=log) # FIXME: check whether table.common_props['dc:conformsTo'] is in validators! validators_ = [] for col in table.tableSchema.columns: if col.propertyUrl: col_uri = col.propertyUrl.uri try: TERMS.is_cldf_uri(col_uri) except ValueError: success = False log_or_raise('invalid CLDF URI: {0}'.format(col_uri), log=log) for table_, col_, v_ in validators: if (not table_ or table is self.get(table_)) and col is self.get((table, col_)): validators_.append((col, v_)) fname = Path(table.url.resolve(table._parent.base)) if fname.exists(): for fname, lineno, row in table.iterdicts(log=log, with_metadata=True): for col, validate in validators_: try: validate(self, table, col, row) except ValueError as e: log_or_raise( '{0}:{1}:{2} {3}'.format(fname.name, lineno, col.name, e), log=log) success = False if not table.check_primary_key(log=log): success = False else: log_or_raise('{0} does not exist'.format(fname), log=log) success = False if not self.tablegroup.check_referential_integrity(log=log): success = False return success