class Dataset: """ this will produce the metadata about a dataset following the example laid out here: http://htmlpreview.github.io/? https://github.com/joejimbo/HCLSDatasetDescriptions/blob/master/Overview.html#appendix_1 (mind the wrap) """ def __init__( self, identifier, # name? should be Archive url via Source title, url, ingest_desc=None, license_url=None, data_rights=None, graph_type='rdf_graph', # rdf_graph, streamed_graph file_handle=None): if graph_type is None: self.graph = RDFGraph(None, identifier) elif graph_type == 'streamed_graph': self.graph = StreamedGraph(True, identifier, file_handle=file_handle) elif graph_type == 'rdf_graph': self.graph = RDFGraph(True, identifier) self.model = Model(self.graph) self.globaltt = self.graph.globaltt self.globaltcid = self.graph.globaltcid self.curie_map = self.graph.curie_map # TODO: move hard coded curies to translation table calls self.identifier = identifier if title is None: self.title = identifier else: self.title = title self.version = None self.date_issued = None # The data_accesed value is later used as an literal of properties # such as dcterms:issued, which needs to conform xsd:dateTime format. # TODO ... we need to have a talk about typed literals and SPARQL self.date_accessed = datetime.now().strftime('%Y-%m-%dT%H:%M:%S') self.citation = set() self.license_url = license_url self.model.addType(self.identifier, 'dctypes:Dataset') self.graph.addTriple(self.identifier, 'dcterms:title', title, True) self.graph.addTriple(self.identifier, 'dcterms:identifier', identifier, True) if url is not None: self.graph.addTriple(self.identifier, 'foaf:page', url) # maybe in the future add the logo here: # schemaorg:logo <uri> # TODO add the license info # FIXME:Temporarily making this in IF statement, # can revert after all current resources are updated. if license_url is not None: self.graph.addTriple(self.identifier, 'dcterms:license', license_url) else: LOG.debug('No license provided.') if data_rights is not None: self.graph.addTriple(self.identifier, 'dcterms:rights', data_rights, object_is_literal=True) else: LOG.debug('No rights provided.') if ingest_desc is not None: self.model.addDescription(self.identifier, ingest_desc) return def setVersion(self, date_issued, version_id=None): """ Legacy function... should use the other set_* for version and date as of 2016-10-20 used in: dipper/sources/HPOAnnotations.py 139: dipper/sources/CTD.py 99: dipper/sources/BioGrid.py 100: dipper/sources/MGI.py 255: dipper/sources/EOM.py 93: dipper/sources/Coriell.py 200: dipper/sources/MMRRC.py 77: # TODO set as deprecated :param date_issued: :param version_id: :return: """ if date_issued is not None: self.set_date_issued(date_issued) elif version_id is not None: self.set_version_by_num(version_id) else: LOG.error("date or version not set!") # TODO throw error return if version_id is not None: self.set_version_by_num(version_id) else: LOG.info("set version to %s", self.version) self.set_version_by_date(date_issued) LOG.info("set version to %s", self.version) return def set_date_issued(self, date_issued): self.date_issued = date_issued self.graph.addTriple(self.identifier, 'dcterms:issued', date_issued, object_is_literal=True) LOG.info("setting date to %s", date_issued) return def set_version_by_date(self, date_issued=None): """ This will set the version by the date supplied, the date already stored in the dataset description, or by the download date (today) :param date_issued: :return: """ if date_issued is not None: dat = date_issued elif self.date_issued is not None: dat = self.date_issued else: dat = self.date_accessed LOG.info( "No date supplied, using download timestamp for date_issued") LOG.info("setting version by date to: %s", dat) self.set_version_by_num(dat) return def set_version_by_num(self, version_num): self.version = self.identifier + version_num self.graph.addTriple(self.version, 'dcterms:isVersionOf', self.identifier) self.graph.addTriple(self.version, 'pav:version', version_num, object_is_literal=True) LOG.info("setting version to %s", self.version) # set the monarch-generated-version of the resource-version # TODO sync this up with the ontology version if version_num != self.date_accessed: dipperized_version = ':' + str(self.date_accessed) self.graph.addTriple(dipperized_version, 'dcterms:isVersionOf', "MonarchData:" + self.identifier + ".ttl") # fix suffix self.graph.addTriple(dipperized_version, 'pav:version', self.date_accessed, object_is_literal=True) self.graph.addTriple(dipperized_version, 'dcterms:issued', self.date_accessed, object_is_literal=True, literal_type="xsd:dateTime") return def setFileAccessUrl(self, url, is_object_literal=False): self.graph.addTriple(self.identifier, 'dcat:accessURL', url, is_object_literal) def getGraph(self): return self.graph def set_license(self, license_url): self.license_url = license_url return def get_license(self): return self.license_url def set_citation(self, citation_id): self.citation.add(citation_id) # TODO # model.addTriple(self.identifier, 'cito:citeAsAuthority', citation_id) return
class Dataset: """ this will produce the metadata about a dataset following the example laid out here: http://htmlpreview.github.io/? https://github.com/joejimbo/HCLSDatasetDescriptions/blob/master/Overview.html#appendix_1 (mind the wrap) """ def __init__(self, identifier, title, url, description=None, license_url=None, data_rights=None, graph_type=None, file_handle=None): if graph_type is None: self.graph = RDFGraph() elif graph_type == 'streamed_graph': self.graph = StreamedGraph(True, file_handle=file_handle) elif graph_type == 'rdf_graph': self.graph = RDFGraph() self.model = Model(self.graph) self.identifier = ':' + identifier self.version = None self.date_issued = None # The data_accesed value is later used as an object literal of properties such as dct:issued, which needs to conform xsd:dateTime format. # self.date_accessed = datetime.now().strftime('%Y-%m-%d-%H-%M') self.date_accessed = datetime.now().strftime('%Y-%m-%dT%H:%M:%S') self.citation = set() self.license = license_url self.model.addType(self.identifier, 'dctypes:Dataset') self.graph.addTriple(self.identifier, 'dct:title', title, True) self.graph.addTriple(self.identifier, 'dct:identifier', identifier, object_is_literal=True) self.graph.addTriple(self.identifier, 'foaf:page', url) # maybe in the future add the logo here: # schemaorg:logo <http://www.ebi.ac.uk/rdf/sites/ebi.ac.uk.rdf/files/resize/images/rdf/chembl_service_logo-146x48.gif> . # TODO add the licence info # FIXME:Temporarily making this in IF statement, # can revert after all current resources are updated. if license_url is not None: self.graph.addTriple(self.identifier, 'dct:license', license_url) else: logger.debug('No license provided.') if data_rights is not None: self.graph.addTriple(self.identifier, 'dct:rights', data_rights, object_is_literal=True) else: logger.debug('No rights provided.') if description is not None: self.model.addDescription(self.identifier, description) return def setVersion(self, date_issued, version_id=None): """ Legacy function... should use the other set_* for version and date as of 2016-10-20 used in: dipper/sources/HPOAnnotations.py 139: dipper/sources/CTD.py 99: dipper/sources/BioGrid.py 100: dipper/sources/MGI.py 255: dipper/sources/EOM.py 93: dipper/sources/Coriell.py 200: dipper/sources/MMRRC.py 77: # TODO set as deprecated :param date_issued: :param version_id: :return: """ if date_issued is not None: self.set_date_issued(date_issued) elif version_id is not None: self.set_version_by_num(version_id) else: logger.error("date or version not set!") # TODO throw error return if version_id is not None: self.set_version_by_num(version_id) else: logger.info("set version to %s", self.version) self.set_version_by_date(date_issued) logger.info("set version to %s", self.version) return def set_date_issued(self, date_issued): self.date_issued = date_issued self.graph.addTriple(self.identifier, 'dct:issued', date_issued, object_is_literal=True) logger.info("setting date to %s", date_issued) return def set_version_by_date(self, date_issued=None): """ This will set the version by the date supplied, the date already stored in the dataset description, or by the download date (today) :param date_issued: :return: """ if date_issued is not None: d = date_issued elif self.date_issued is not None: d = self.date_issued else: d = self.date_accessed logger.info("No date supplied for setting version; " "using download timestamp for date_issued") logger.info("setting version by date") self.set_version_by_num(d) return def set_version_by_num(self, version_num): self.version = self.identifier + version_num self.graph.addTriple(self.version, 'dct:isVersionOf', self.identifier) self.graph.addTriple(self.version, 'pav:version', version_num, object_is_literal=True) logger.info("setting version to %s", self.version) # set the monarch-generated-version of the resource-version # TODO sync this up with the ontology version if version_num != self.date_accessed: dipperized_version = ':' + str(self.date_accessed) self.graph.addTriple(dipperized_version, 'dct:isVersionOf', self.version) self.graph.addTriple(dipperized_version, 'pav:version', self.date_accessed, object_is_literal=True) self.graph.addTriple(dipperized_version, 'dct:issued', self.date_accessed, object_is_literal=True, literal_type="xsd:dateTime") return def setFileAccessUrl(self, url, is_object_literal=False): self.graph.addTriple(self.identifier, 'dcat:accessURL', url, is_object_literal) def getGraph(self): return self.graph def set_license(self, license): self.license = license return def get_license(self): return self.license def set_citation(self, citation_id): self.citation.add(citation_id) # TODO # model.addTriple(self.identifier, 'cito:citeAsAuthority', citation_id) return
disease_graph = add_triples_from_bbop(parent_graph, disease_graph) disease_graph = add_triples_from_bbop(child_graph, disease_graph) disease_graph = add_triples_from_bbop(eq_graph, disease_graph) mondo_stub = './output/mondo-stub.xml' mondo_stub_ttl = './output/mondo-stub.ttl' disease_graph.serialize(mondo_stub, 'xml') disease_graph.serialize(mondo_stub_ttl, 'ttl') mondo_stub_lbl = './output/mondo-stub-wlabels.xml' mondo_stub_ttl_lbl = './output/mondo-stub-wlabels.ttl' for node in parent_graph.nodes: disease_graph.addTriple(node.id, 'rdfs:label', node.label, True) for node in child_graph.nodes: disease_graph.addTriple(node.id, 'rdfs:label', node.label, True) for node in eq_graph.nodes: disease_graph.addTriple(node.id, 'rdfs:label', node.label, True) disease_graph.serialize(mondo_stub_lbl, 'xml') disease_graph.serialize(mondo_stub_ttl_lbl, 'ttl') data_graph = RDFGraph() sg = SciGraph(SCIGRAPH_DATA) # Get all children + sickle cell
class Dataset: """ This class produces metadata about a dataset that is compliant with the HCLS dataset specification: https://www.w3.org/TR/2015/NOTE-hcls-dataset-20150514/#s4_4 Summary level: The summary level provides a description of a dataset that is independent of a specific version or format. (e.g. the Monarch ingest of CTD) CURIE for this is something like MonarchData:[SOURCE IDENTIFIER] Version level: The version level captures version-specific characteristics of a dataset. (e.g. the 01-02-2018 ingest of CTD) CURIE for this is something like MonarchData:[SOURCE IDENTIFIER_INGESTTIMESTAMP] Distribution level: The distribution level captures metadata about a specific form and version of a dataset (e.g. turtle file for 01-02-2018 ingest of CTD). There is a [distribution level resource] for each different downloadable file we emit, i.e. one for the TTL file, one for the ntriples file, etc. CURIE for this is like MonarchData:[SOURCE IDENTIFIER_INGESTTIMESTAMP].ttl or MonarchData:[SOURCE IDENTIFIER_INGESTTIMESTAMP].nt or MonarchData:[SOURCE IDENTIFIER_INGESTTIMESTAMP].[whatever file format] We write out at least the following triples: SUMMARY LEVEL TRIPLES: [summary level resource] - rdf:type -> dctypes:Dataset [summary level resource] - dc:title -> title (literal) [summary level resource] - dc:description -> description (literal) (use docstring from Source class) [summary level resource] - dc:source -> [source web page, e.g. omim.org] [summary level resource] - schema:logo -> [source logo IRI] [summary level resource] - dc:publisher -> monarchinitiative.org n.b: about summary level resource triples: -- HCLS spec says we "should" link to our logo and web page, but I'm not, because it would confuse the issue of whether we are pointing to our logo/page or the logo/page of the data source for this ingest. Same below for [version level resource] and [distibution level resource] - I'm not linking to our page/logo down there either. - spec says we "should" include summary level triples describing Update frequency and SPARQL endpoint but I'm omitting this for now, because these are not clearly defined at the moment VERSION LEVEL TRIPLES: [version level resource] - rdf:type -> dctypes:Dataset [version level resource] - dc:title -> version title (literal) [version level resource] - dc:description -> version description (literal) [version level resource] - dc:created -> ingest timestamp [ISO 8601 compliant] [version level resource] - pav:version -> ingest timestamp (same one above) [version level resource] - dc:creator -> monarchinitiative.org [version level resource] - dc:publisher -> monarchinitiative.org [version level resource] - dc:isVersionOf -> [summary level resource] [version level resource] - dc:source -> [source file 1 IRI] [version level resource] - dc:source -> [source file 2 IRI] ... [source file 1 IRI] - pav:retrievedOn -> [download date timestamp] [source file 2 IRI] - pav:version -> [source version (if set, optional)] [source file 2 IRI] - pav:retrievedOn -> [download date timestamp] [source file 2 IRI] - pav:version -> [source version (if set, optional)] ... [version level resource] - pav:createdWith -> [Dipper github URI] [version level resource] - void:dataset -> [distribution level resource] [version level resource] - cito:citesAsAuthoriy -> [citation id 1] [version level resource] - cito:citesAsAuthoriy -> [citation id 2] [version level resource] - cito:citesAsAuthoriy -> [citation id 3] n.b: about version level resource triples: - spec says we "should" include Date of issue/dc:issued triple, but I'm not because it is redundant with this triple above: [version level resource] - dc:created -> time stamp and would introduce ambiguity and confusion if the two disagree. Same below for [distribution level resource] - dc:created -> tgiime stamp below Also omitting: - triples linking to our logo and page, see above. - License/dc:license triple, because we will make this triple via the [distribution level resource] below - Language/dc:language triple b/c it seems superfluous. Same below for [distribution level resource] - no language triple. - [version level resource] - pav:version triple is also a bit redundant with the pav:version triple below, but the spec requires both these triples - I'm omitting the [version level resource] -> pav:previousVersion because Dipper doesn't know this info for certain at run time. Same below for [distribution level resource] - pav:previousVersion. DISTRIBUTION LEVEL TRIPLES: [distribution level resource] - rdf:type -> dctypes:Dataset [distribution level resource] - rdf:type -> dcat:Distribution [distribution level resource] - dc:title -> distribution title (literal) [distribution level resource] - dc:description -> distribution description (lit.) [distribution level resource] - dc:created -> ingest timestamp[ISO 8601 compliant] [distribution level resource] - pav:version -> ingest timestamp (same as above) [distribution level resource] - dc:creator -> monarchinitiative.org [distribution level resource] - dc:publisher -> monarchinitiative.org [distribution level resource] - dc:license -> [license info, if available otherwise indicate unknown] [distribution level resource] - dc:rights -> [data rights IRI] [distribution level resource] - pav:createdWith -> [Dipper github URI] [distribution level resource] - dc:format -> [IRI of ttl|nt|whatever spec] [distribution level resource] - dcat:downloadURL -> [ttl|nt URI] [distribution level resource] - void:triples -> [triples count (literal)] [distribution level resource] - void:entities -> [entities count (literal)] [distribution level resource] - void:distinctSubjects -> [subject count (literal)] [distribution level resource] - void:distinctObjects -> [object count (literal)] [distribution level resource] - void:properties -> [properties count (literal)] ... n.b: about distribution level resource triples: - omitting Vocabularies used/void:vocabulary and Standards used/dc:conformTo triples, because they are described in the ttl file - also omitting Example identifier/idot:exampleIdentifier and Example resource/void:exampleResource, because we don't really have one canonical example of either - they're all very different. - [distribution level resource] - dc:created should have the exact same time stamp as this triple above: [version level resource] - dc:created -> time stamp - this [distribution level resource] - pav:version triple should have the same object as [version level resource] - pav:version triple above - Data source provenance/dc:source triples are above in the [version level resource] - omitting Byte size/dc:byteSize, RDF File URL/void:dataDump, and Linkset/void:subset triples because they probably aren't necessary for MI right now - these triples "should" be emitted, but we will do this in a later iteration: # of classes void:classPartition IRI # of literals void:classPartition IRI # of RDF graphs void:classPartition IRI Note: Do not use blank nodes in the dataset graph. This dataset graph is added to the main Dipper graph in Source.write() like so $ mainGraph = mainGraph + datasetGraph which apparently in theory could lead to blank node ID collisions between the two graphs. Note also that this implementation currently does not support producing metadata for StreamedGraph graphs (see dipper/graph/StreamedGraph.py). StreamedGraph is currently not being used for any ingests, so this isn't a problem. There was talk of using StreamedGraph for a rewrite/refactor of the Clinvar ingest, which would probably require adding support here for StreamedGraph's. """ def __init__( self, identifier, data_release_version, ingest_name, ingest_title, ingest_url, ingest_logo=None, ingest_description=None, license_url=None, data_rights=None, graph_type='rdf_graph', # rdf_graph, streamed_graph file_handle=None, distribution_type='ttl', dataset_curie_prefix='MonarchArchive'): if graph_type is None: self.graph = RDFGraph(None, ":".join([dataset_curie_prefix, identifier])) elif graph_type == 'streamed_graph': self.graph = StreamedGraph(True, ":".join( [dataset_curie_prefix, identifier]), file_handle=file_handle) elif graph_type == 'rdf_graph': self.graph = RDFGraph(True, ':'.join([dataset_curie_prefix, identifier])) if data_release_version is not None: self.data_release_version = data_release_version else: self.data_release_version = datetime.today().strftime("%Y%m%d") self.model = Model(self.graph) self.globaltt = self.graph.globaltt self.globaltcid = self.graph.globaltcid self.curie_map = self.graph.curie_map self.identifier = ':'.join([dataset_curie_prefix, identifier]) self.citation = set() self.ingest_name = ingest_name self.ingest_title = ingest_title if self.ingest_title is None: self.ingest_title = ":".join([dataset_curie_prefix, identifier]) self.ingest_url = ingest_url self.ingest_logo = self.curie_map.get('MonarchLogoRepo') + ingest_logo self.ingest_description = ingest_description self.date_issued = None self.license_url = license_url self.data_rights = data_rights self.distribution_type = distribution_type # set HCLS resource CURIEs self.summary_level_curie = ':'.join( [dataset_curie_prefix, '#' + identifier]) self.version_level_curie = \ dataset_curie_prefix + ':' + \ self.data_release_version + \ '/#' + identifier self.distribution_level_turtle_curie = \ dataset_curie_prefix + ':' + \ self.data_release_version + \ '/rdf/' + \ identifier + "." + self.distribution_type # The following might seem a little odd, but we need to set downloadURLs this # way in order for them to point to where they will end up in archive.MI.org as # of Sept 2019. URL is: # https://archive.MI.org/[release version]/[dist type]/[source].[dist type] self.download_url = \ self.curie_map.get("MonarchArchive") + self.data_release_version + \ "/rdf/" + self.ingest_name + "." + self.distribution_type self._set_summary_level_triples() self._set_version_level_triples() self._set_distribution_level_triples() def _set_summary_level_triples(self): self.model.addType(self.summary_level_curie, self.globaltt['Dataset']) self.graph.addTriple(self.summary_level_curie, self.globaltt['title'], self.ingest_title, True) self.model.addTriple(self.summary_level_curie, self.globaltt['Publisher'], self.curie_map.get("")) self.model.addTriple(self.summary_level_curie, "schema:logo", self.ingest_logo) self.graph.addTriple(self.summary_level_curie, self.globaltt['identifier'], self.summary_level_curie) if self.ingest_url is not None: self.graph.addTriple(self.summary_level_curie, self.globaltt["Source"], self.ingest_url) if self.ingest_description is not None: self.model.addDescription(self.summary_level_curie, self.ingest_description) def _set_version_level_triples(self): self.model.addType(self.version_level_curie, self.globaltt['Dataset']) self.graph.addTriple( self.version_level_curie, self.globaltt['title'], self.ingest_title + " Monarch version " + self.data_release_version, True) if self.ingest_description is not None: self.model.addDescription(self.version_level_curie, self.ingest_description) self.graph.addTriple( self.version_level_curie, self.globaltt['Date Created'], Literal(datetime.today().strftime("%Y%m%d"), datatype=XSD.date)) self.graph.addTriple( self.version_level_curie, self.globaltt['version'], Literal(self.data_release_version, datatype=XSD.date)) self.graph.addTriple(self.version_level_curie, self.globaltt['creator'], self.curie_map.get("")) # eval's to MI.org self.graph.addTriple(self.version_level_curie, self.globaltt['Publisher'], self.curie_map.get("")) # eval's to MI.org self.graph.addTriple(self.version_level_curie, self.globaltt['isVersionOf'], self.summary_level_curie, object_is_literal=False) self.graph.addTriple(self.version_level_curie, self.globaltt['distribution'], self.distribution_level_turtle_curie, object_is_literal=False) def _set_distribution_level_triples(self): self.model.addType(self.distribution_level_turtle_curie, self.globaltt['Dataset']) self.model.addType(self.distribution_level_turtle_curie, self.globaltt['Distribution']) self.graph.addTriple( self.distribution_level_turtle_curie, self.globaltt['title'], self.ingest_title + " distribution " + self.distribution_type, True) if self.ingest_description is not None: self.model.addDescription(self.distribution_level_turtle_curie, self.ingest_description) self.graph.addTriple( self.distribution_level_turtle_curie, self.globaltt['version'], Literal(self.data_release_version, datatype=XSD.date)) self.graph.addTriple( self.distribution_level_turtle_curie, self.globaltt['Date Created'], Literal(datetime.today().strftime("%Y%m%d"), datatype=XSD.date)) self.graph.addTriple(self.distribution_level_turtle_curie, self.globaltt['creator'], self.curie_map.get("")) # eval's to MI.org self.graph.addTriple(self.distribution_level_turtle_curie, self.globaltt['Publisher'], self.curie_map.get("")) # eval's to MI.org self.graph.addTriple(self.distribution_level_turtle_curie, self.globaltt['created_with'], "https://github.com/monarch-initiative/dipper") self.graph.addTriple(self.distribution_level_turtle_curie, self.globaltt['format'], "https://www.w3.org/TR/turtle/") self.graph.addTriple(self.distribution_level_turtle_curie, self.globaltt['downloadURL'], self.download_url) if self.license_url is None: self.graph.addTriple( self.distribution_level_turtle_curie, self.globaltt['license'], 'https://project-open-data.cio.gov/unknown-license/') else: self.graph.addTriple(self.distribution_level_turtle_curie, self.globaltt['license'], self.license_url) if self.data_rights is not None: self.graph.addTriple(self.distribution_level_turtle_curie, self.globaltt['rights'], self.data_rights) self._declare_as_ontology() def set_ingest_source_file_version_num(self, file_iri, version): """ This method sets the version of a remote file or resource that is used in the ingest. It writes this triple: file_iri - 'pav:version' -> version Version is an untyped literal Note: if your version is a date or timestamp, use set_ingest_source_file_version_date() instead :param file_iri: a remote file or resource used in ingest :param version: a number or string (e.g. v1.2.3) that the source (OMIM, CTD) uses to refer to this version of the file/resource used during the ingest :return: None """ self.graph.addTriple(file_iri, self.globaltt['version'], version, object_is_literal=True) def set_ingest_source_file_version_date(self, file_iri, date, datatype=XSD.date): """ This method sets the version that the source (OMIM, CTD, whatever) uses to refer to this version of the remote file/resource that was used in the ingest It writes this triple: file_iri - 'pav:version' -> date or timestamp Version is added as a literal of datatype XSD date Note: if file_iri was retrieved using get_files(), then the following triple was created and you might not need this method: file_iri - 'pav:retrievedOn' -> download date :param file_iri: a remote file or resource used in ingest :param date: a date in YYYYMMDD format that the source (OMIM, CTD). You can add timestamp as a version by using a different datatype (below) :param datatype: an XSD literal datatype, default is XSD.date uses to refer to this version of the file/resource used during the ingest :return: None """ self.graph.addTriple(file_iri, self.globaltt['version'], date, object_is_literal=True, literal_type=datatype) def set_ingest_source_file_version_retrieved_on(self, file_iri, date, datatype=XSD.date): """ This method sets the date on which a remote file/resource (from OMIM, CTD, etc) was retrieved. It writes this triple: file_iri - 'pav:retrievedOn' -> date or timestamp Version is added as a literal of datatype XSD date by default Note: if file_iri was retrieved using get_files(), then the following triple was created and you might not need this method: file_iri - 'pav:retrievedOn' -> download date :param file_iri: a remote file or resource used in ingest :param date: a date in YYYYMMDD format that the source (OMIM, CTD). You can add timestamp as a version by using a different datatype (below) :param datatype: an XSD literal datatype, default is XSD.date uses to refer to this version of the file/resource used during the ingest :return: None """ self.graph.addTriple(file_iri, self.globaltt['retrieved_on'], date, object_is_literal=True, literal_type=datatype) def set_ingest_source(self, url, predicate=None, is_object_literal=False): """ This method writes a triple to the dataset graph indicating that the ingest used a file or resource at [url] during the ingest. Triple emitted is version_level_curie dc:source [url] This triple is likely to be redundant if Source.get_files() is used to retrieve the remote files/resources, since this triple should also be emitted as files/resources are being retrieved. This method is provided as a convenience method for sources that do their own downloading of files. :param url: a remote resource used as a source during ingest :param predicate: the predicate to use for the triple ["dc:source"] from spec (https://www.w3.org/TR/2015/NOTE-hcls-dataset-20150514/) "Use dc:source when the source dataset was used in whole or in part. Use pav:retrievedFrom when the source dataset was used in whole and was not modified from its original distribution. Use prov:wasDerivedFrom when the source dataset was in whole or in part and was modified from its original distribution." :return: None """ if predicate is None: predicate = self.globaltt["Source"] self.graph.addTriple(self.version_level_curie, predicate, url, object_is_literal=is_object_literal, subject_category=blv.terms['DataSetVersion']) def get_graph(self): """ This method returns the dataset graph :param :return: dataset graph """ return self.graph def get_license(self): """ This method returns the license info :param :return: license info """ return self.license_url def set_citation(self, citation_id): """ This method adds [citaton_id] argument to the set of citations, and also adds a triple indicating that version level cito:citesAsAuthority [citation_id] :param: citation_id :return: none """ self.citation.add(citation_id) self.graph.addTriple(self.version_level_curie, self.globaltt['citesAsAuthority'], citation_id) def _declare_as_ontology(self, version_info=None): """ Declare the distribution level IRI as an ontology, and also make triple distribution level IRI - version_iri -> version level IRI TEC: I am not convinced dipper reformatting external data as RDF triples makes an OWL ontology (nor that it should be considered a goal). Proper ontologies are built by ontologists. Dipper reformats data and annotates/decorates it with a minimal set of carefully arranged terms drawn from from multiple proper ontologies. Which allows the whole (dipper's RDF triples and parent ontologies) to function as a single ontology we can reason over when combined in a store such as SciGraph. Including more than the minimal ontological terms in dipper's RDF output constitutes a liability as it allows greater divergence between dipper artifacts and the proper ontologies. :param version_info: a string describing version info for the ontology :return: """ model = Model(self.graph) model.addOntologyDeclaration(self.summary_level_curie) model.addOWLVersionIRI(self.summary_level_curie, self.version_level_curie) if version_info is not None: model.addOWLVersionInfo(self.distribution_level_turtle_curie, version_info) @staticmethod def make_id(long_string, prefix='MONARCH'): """ A method to create DETERMINISTIC identifiers based on a string's digest. currently implemented with sha1 Duplicated from Source.py to avoid circular imports. :param long_string: string to use to generate identifier :param prefix: prefix to prepend to identifier [Monarch] :return: a Monarch identifier """ return ':'.join((prefix, Dataset.hash_id(long_string))) @staticmethod def hash_id(word): # same as graph/GraphUtils.digest_id(wordage) """ Given a string, make a hash Duplicated from Source.py. :param word: str string to be hashed :return: hash of id """ return 'b' + hashlib.sha1(word.encode('utf-8')).hexdigest()[1:20]
class Dataset: """ this will produce the metadata about a dataset following the example laid out here: http://htmlpreview.github.io/? https://github.com/joejimbo/HCLSDatasetDescriptions/blob/master/Overview.html#appendix_1 (mind the wrap) """ def __init__(self, identifier, title, url, description=None, license_url=None, data_rights=None, graph_type=None, file_handle=None): if graph_type is None: self.graph = RDFGraph(None, identifier) # elif graph_type == 'streamed_graph': self.graph = StreamedGraph(True, file_handle=file_handle) elif graph_type == 'rdf_graph': self.graph = RDFGraph() self.model = Model(self.graph) self.identifier = ':' + identifier self.version = None self.date_issued = None # The data_accesed value is later used as an literal of properties # such as dct:issued, which needs to conform xsd:dateTime format. # TODO ... we need to have a talk about typed literals and SPARQL self.date_accessed = datetime.now().strftime('%Y-%m-%dT%H:%M:%S') self.citation = set() self.license = license_url self.model.addType(self.identifier, 'dctypes:Dataset') self.graph.addTriple(self.identifier, 'dct:title', title, True) self.graph.addTriple( self.identifier, 'dct:identifier', identifier, object_is_literal=True) self.graph.addTriple(self.identifier, 'foaf:page', url) # maybe in the future add the logo here: # schemaorg:logo <http://www.ebi.ac.uk/rdf/sites/ebi.ac.uk.rdf/files/resize/images/rdf/chembl_service_logo-146x48.gif> . # TODO add the licence info # FIXME:Temporarily making this in IF statement, # can revert after all current resources are updated. if license_url is not None: self.graph.addTriple( self.identifier, 'dct:license', license_url) else: logger.debug('No license provided.') if data_rights is not None: self.graph.addTriple( self.identifier, 'dct:rights', data_rights, object_is_literal=True) else: logger.debug('No rights provided.') if description is not None: self.model.addDescription(self.identifier, description) return def setVersion(self, date_issued, version_id=None): """ Legacy function... should use the other set_* for version and date as of 2016-10-20 used in: dipper/sources/HPOAnnotations.py 139: dipper/sources/CTD.py 99: dipper/sources/BioGrid.py 100: dipper/sources/MGI.py 255: dipper/sources/EOM.py 93: dipper/sources/Coriell.py 200: dipper/sources/MMRRC.py 77: # TODO set as deprecated :param date_issued: :param version_id: :return: """ if date_issued is not None: self.set_date_issued(date_issued) elif version_id is not None: self.set_version_by_num(version_id) else: logger.error("date or version not set!") # TODO throw error return if version_id is not None: self.set_version_by_num(version_id) else: logger.info("set version to %s", self.version) self.set_version_by_date(date_issued) logger.info("set version to %s", self.version) return def set_date_issued(self, date_issued): self.date_issued = date_issued self.graph.addTriple( self.identifier, 'dct:issued', date_issued, object_is_literal=True) logger.info("setting date to %s", date_issued) return def set_version_by_date(self, date_issued=None): """ This will set the version by the date supplied, the date already stored in the dataset description, or by the download date (today) :param date_issued: :return: """ if date_issued is not None: d = date_issued elif self.date_issued is not None: d = self.date_issued else: d = self.date_accessed logger.info( "No date supplied for setting version; " "using download timestamp for date_issued") logger.info("setting version by date") self.set_version_by_num(d) return def set_version_by_num(self, version_num): self.version = self.identifier+version_num self.graph.addTriple(self.version, 'dct:isVersionOf', self.identifier) self.graph.addTriple(self.version, 'pav:version', version_num, object_is_literal=True) logger.info("setting version to %s", self.version) # set the monarch-generated-version of the resource-version # TODO sync this up with the ontology version if version_num != self.date_accessed: dipperized_version = ':' + str(self.date_accessed) self.graph.addTriple( dipperized_version, 'dct:isVersionOf', self.version) self.graph.addTriple( dipperized_version, 'pav:version', self.date_accessed, object_is_literal=True) self.graph.addTriple( dipperized_version, 'dct:issued', self.date_accessed, object_is_literal=True, literal_type="xsd:dateTime") return def setFileAccessUrl(self, url, is_object_literal=False): self.graph.addTriple(self.identifier, 'dcat:accessURL', url, is_object_literal) def getGraph(self): return self.graph def set_license(self, license): self.license = license return def get_license(self): return self.license def set_citation(self, citation_id): self.citation.add(citation_id) # TODO # model.addTriple(self.identifier, 'cito:citeAsAuthority', citation_id) return
class RDFGraphTestCase(unittest.TestCase): def setUp(self): self.graph = RDFGraph() this_curie_map = curie_map.get() self.cutil = CurieUtil(this_curie_map) # stuff to make test triples self.test_cat_subj = "http://www.google.com" self.test_cat_default_pred = self.cutil.get_uri("biolink:category") self.test_cat_nondefault_pred = self.cutil.get_uri("rdf:type") self.test_cat_default_category = self.cutil.get_uri( "biolink:NamedThing") self.test_cat_nondefault_category = self.cutil.get_uri("biolink:Gene") self.test_cat_type = self.cutil.get_uri("rdf:type") self.test_cat_class = self.cutil.get_uri("rdf:class") def tearDown(self): self.graph = None def test_add_triple_makes_triple(self): """ test that addTriple() makes at least one triple """ self.graph.addTriple(subject_id=self.test_cat_subj, predicate_id="rdf:type", obj="rdf:class") self.assertTrue( len(self.graph) > 0, "addTriples() didn't make >=1 triple") def test_add_triple_subject_category_assignment(self): """ test that addTriple() correctly assigns subject category """ self.graph.addTriple( subject_id=self.test_cat_subj, predicate_id="rdf:comment", obj="website", subject_category=self.test_cat_nondefault_category) triples = list( self.graph.triples((URIRef(self.test_cat_subj), URIRef(self.test_cat_default_pred), None))) self.assertEqual( len(triples), 1, "addTriples() didn't make exactly one triple subject category") self.assertEqual( triples[0][2], URIRef(self.test_cat_nondefault_category), "addTriples() didn't assign the right triple subject category") def test_add_triple_object_category_assignment(self): """ test that addTriple() correctly assigns object category """ self.graph.addTriple(subject_id=self.test_cat_subj, predicate_id=self.test_cat_type, obj=self.test_cat_class, object_category=self.test_cat_nondefault_category) triples = list( self.graph.triples((URIRef(self.test_cat_class), URIRef(self.test_cat_default_pred), None))) self.assertEqual( len(triples), 1, "addTriples() didn't make exactly one triple object category") self.assertEqual( triples[0][2], URIRef(self.test_cat_nondefault_category), "addTriples() didn't assign the right triple object category") def read_graph_from_turtle_file(self, f): """ This will read the specified file into a graph. A simple parsing test. :param f: :return: """ vg = RDFGraph() p = os.path.abspath(f) logger.info("Testing reading turtle file from %s", p) vg.parse(f, format="turtle") logger.info('Found %s graph nodes in %s', len(vg), p) self.assertTrue(len(vg) > 0, "No nodes found in " + p) return def read_graph_into_owl(self, f): """ test if the ttl can be parsed by owlparser this expects owltools to be accessible from commandline :param f: file of ttl :return: """ import subprocess from subprocess import check_call status = check_call(["owltools", f], stderr=subprocess.STDOUT) # returns zero is success! if status != 0: logger.error('finished verifying with owltools with status %s', status) self.assertTrue(status == 0) return def test_make_category_triple_default(self): """ test that method adds category triple to graph correctly (default pred and obj) """ self.graph._make_category_triple(self.test_cat_subj) triples = list(self.graph.triples((None, None, None))) self.assertEqual(len(triples), 1, "method didn't make exactly one triple") self.assertEqual(triples[0][0], URIRef(self.test_cat_subj), "didn't assign correct subject") self.assertEqual(triples[0][1], URIRef(self.test_cat_default_pred), "didn't assign correct predicate") self.assertEqual(triples[0][2], URIRef(self.test_cat_default_category), "didn't assign correct category") def test_make_category_triple_non_default_category(self): """ test that method adds category triple to graph correctly """ self.graph._make_category_triple(self.test_cat_subj, self.test_cat_nondefault_category) triples = list(self.graph.triples((None, None, None))) self.assertEqual(len(triples), 1, "method didn't make exactly one triple") self.assertEqual(URIRef(self.test_cat_nondefault_category), triples[0][2], "didn't assign correct (non-default) category") def test_make_category_triple_non_default_pred(self): """ test that method adds category triple to graph correctly (non default pred) """ self.graph._make_category_triple( self.test_cat_subj, self.test_cat_default_category, predicate=self.test_cat_nondefault_pred) triples = list(self.graph.triples((None, None, None))) self.assertEqual(len(triples), 1, "method didn't make exactly one triple") self.assertEqual(URIRef(self.test_cat_nondefault_pred), triples[0][1], "didn't assign correct (non-default) category") def test_make_category_triple_category_none_should_emit_named_thing(self): """ test that method adds category triple to graph correctly (default pred and obj) """ self.graph._make_category_triple(self.test_cat_subj, category=None) triples = list(self.graph.triples((None, None, None))) self.assertEqual(len(triples), 1, "method didn't make exactly one triple") self.assertEqual(URIRef(self.test_cat_default_category), triples[0][2], "didn't assign correct default category") def test_is_literal(self): """ test that method infers type (either literal or CURIE) correctly """ self.assertTrue(self.graph._is_literal("1")) self.assertTrue(not self.graph._is_literal("foo:bar")) self.assertTrue(not self.graph._is_literal("http://www.zombo.com/")) self.assertTrue(not self.graph._is_literal("https://www.zombo.com/")) self.assertTrue( not self.graph._is_literal("ftp://ftp.1000genomes.ebi.ac.uk/"))