def test_parse(self): if self.source is not None: # don't test the abstract class self.source.parse() """ seems we get a better stack trace by not catching the exception am I missing something? try: self.source.parse() except Exception as ParseException: # tec too broad? logger.error(ParseException) self.assertFalse(True, "Parsing failed") """ try: properties = GraphUtils.get_properties_from_graph( self.source.graph) GraphUtils.add_property_axioms(self.source.graph, properties) self.source.write(format='turtle') except Exception as WriteException: logger.error(WriteException) self.assertFalse(True, "Write failed") return
def test_parse(self): if self.source is not None: # don't test the abstract class self.source.parse() """ seems we get a better stack trace by not catching the exception am I missing something? try: self.source.parse() except Exception as ParseException: # tec too broad? logger.error(ParseException) self.assertFalse(True, "Parsing failed") """ try: properties = GraphUtils.get_properties_from_graph(self.source.graph) GraphUtils.add_property_axioms(self.source.graph, properties) self.source.write() # default to fmt='turtle' #self.source.write(fmt='nt') #self.source.write(fmt='nquads') except Exception as WriteException: logger.error(WriteException) self.assertFalse(True, "Write failed") return
def main(): # TODO this should be generated by looking in the dipper/sources directory source_to_class_map = { # 'facebase_alpha': 'FaceBase_alpha', 'hpoa': 'HPOAnnotations', # ~3 min 'zfin': 'ZFIN', 'omim': 'OMIM', 'biogrid': 'BioGrid', 'mgi': 'MGI', 'impc': 'IMPC', # Panther takes ~1hr to map 7 species-worth of associations 'panther': 'Panther', 'ncbigene': 'NCBIGene', 'ucscbands': 'UCSCBands', 'ctd': 'CTD', 'genereviews': 'GeneReviews', 'eom': 'EOM', 'coriell': 'Coriell', # 'clinvar': 'ClinVar', # needs integrating here 'monochrom': 'Monochrom', 'kegg': 'KEGG', 'animalqtldb': 'AnimalQTLdb', 'ensembl': 'Ensembl', 'hgnc': 'HGNC', 'orphanet': 'Orphanet', 'omia': 'OMIA', 'flybase': 'FlyBase', 'mmrrc': 'MMRRC', 'wormbase': 'WormBase', 'mpd': 'MPD', 'gwascatalog': 'GWASCatalog', 'monarch': 'Monarch', 'go': 'GeneOntology', 'reactome': 'Reactome', 'udp': 'UDP', 'mgi-slim': 'MGISlim', 'zfinslim': 'ZFINSlim', 'bgee': 'Bgee', 'mydrug': 'MyDrug', 'stringdb': 'StringDB', 'rgd': 'RGD', 'sgd': 'SGD', 'mychem': 'MyChem', 'ebi': 'EBIGene2Phen', } parser = argparse.ArgumentParser( description='Dipper: Data Ingestion Pipeline for Monarch', formatter_class=argparse.RawTextHelpFormatter) parser.add_argument( '-g', '--graph', type=str, default="rdf_graph", help='graph type: rdf_graph, streamed_graph') parser.add_argument( '-s', '--sources', type=str, default='?', help='comma separated list of sources') parser.add_argument('-l', '--limit', type=int, help='limit number of rows used') parser.add_argument( '--parse_only', action='store_true', help='parse files without writing RDF') parser.add_argument( '--fetch_only', action='store_true', help='fetch sources without parsing') parser.add_argument( '-f', '--force', action='store_true', help='force re-download of files') parser.add_argument( '--no_verify', help='ignore the verification step', action='store_true') # parser.add_argument( '--query', help='enter in a sparql query', type=str) parser.add_argument( '-q', '--quiet', help='turn off info logging', action="store_true") parser.add_argument( '--debug', help='turn on debug logging', action="store_true") parser.add_argument( '--skip_tests', help='skip any testing', action="store_true") # Blank Nodes can't be visualized in Protege, default to Skolemizing them parser.add_argument( '-b', '--use_bnodes', help="use blank nodes instead of skolemizing", action="store_true", default=False) # parser.add_argument( '-t', '--taxon', type=str, default='9606', help='Constrain Source to supplied taxon identifier(s).\n' 'Please enter comma delimited NCBITaxon numbers:\n' 'Implemented taxa per source\n' 'NCBIGene: 9606,10090,7955\n' 'Panther: 9606,10090,10116,7227,7955,6239,8355\n' 'BioGrid: 9606,10090,10116,7227,7955,6239,8355\n' 'UCSCBands: 9606\n' 'GO: 9606,10090,10116,7227,7955,6239,9615,9823,9031,9913') parser.add_argument( '-o', '--test_only', help='only process and output the pre-configured test subset', action="store_true") parser.add_argument( '--dest_fmt', help='serialization format: [turtle], nt, nquads, rdfxml, n3, raw', type=str) parser.add_argument('-v', '--version', help='version of source', type=str) args = parser.parse_args() tax_ids = None if args.taxon is not None: tax_ids = [str(int(t)) for t in args.taxon.split(',')] taxa_supported = [ # these are not taxa, janky approach 'Panther', 'NCBIGene', 'BioGrid', 'UCSCBands', 'GeneOntology', 'Bgee', 'Ensembl', 'StringDB'] formats_supported = [ 'turtle', 'ttl', 'ntriples', 'nt', 'nquads', 'nq', 'rdfxml', 'xml', 'notation3', 'n3', 'raw'] if args.quiet: logging.getLogger().setLevel(logging.WARNING) else: if args.debug: logging.getLogger().setLevel(logging.DEBUG) else: logging.getLogger().setLevel(logging.INFO) if not args.use_bnodes: LOG.info("Will Skolemize Blank Nodes") # None of these query test utils exist in ./dipper/utils/TestUtils.py # if args.query is not None: # test_query = TestUtils() # for source in args.sources.split(','): # # source = source.lower() # mysource = source_to_class_map[source]() # # # import source lib # module = "dipper.sources.{0}".format(mysource) # imported_module = importlib.import_module(module) # source_class = getattr(imported_module, mysource) # test_query.check_query_syntax(args.query, source_class) # test_query.load_graph_from_turtle(source_class) # # print(test_query.query_graph(args.query, True)) # exit(0) # run initial tests if (args.no_verify or args.skip_tests) is not True: unittest.TextTestRunner(verbosity=2).run(TEST_SUITE) # set serializer if args.dest_fmt is not None: if args.dest_fmt in formats_supported: if args.dest_fmt == 'ttl': args.dest_fmt = 'turtle' elif args.dest_fmt == 'ntriples': args.dest_fmt = 'nt' elif args.dest_fmt == 'nq': args.dest_fmt = 'nquads' elif args.dest_fmt == 'xml': args.dest_fmt = 'rdfxml' elif args.dest_fmt == 'notation3': args.dest_fmt = 'n3' else: LOG.error("You have specified an invalid serializer: %s", args.dest_fmt) exit(0) else: args.dest_fmt = 'turtle' # Provide feedback if we can't proceed if args.sources is None or args.sources.split(',')[0] not in source_to_class_map: LOG.info('Unknown Source %s', args.sources.split(',')[0]) LOG.info('Sources Known are limited to:') for key in sorted(source_to_class_map): LOG.info('\t%s\t%s', key, source_to_class_map[key]) exit(0) # iterate through all the sources for source in args.sources.split(','): LOG.info("\n******* %s *******", source) source = source.lower() src = source_to_class_map[source] # import source lib module = "dipper.sources.{0}".format(src) imported_module = importlib.import_module(module) source_class = getattr(imported_module, src) mysource = None # arg factory source_args = dict( graph_type=args.graph ) source_args['are_bnodes_skolemized'] = not args.use_bnodes # args should be available to source supported (yet) or not if src in taxa_supported: source_args['tax_ids'] = tax_ids if args.version: source_args['version'] = args.version mysource = source_class(**source_args) if args.parse_only is False: start_fetch = time.perf_counter() mysource.fetch(args.force) end_fetch = time.perf_counter() LOG.info("Fetching time: %d sec", end_fetch-start_fetch) mysource.settestonly(args.test_only) # run tests first if (args.no_verify or args.skip_tests) is not True: suite = mysource.getTestSuite() if suite is None: LOG.warning( "No tests configured for this source: %s", source) else: unittest.TextTestRunner(verbosity=2).run(suite) else: LOG.info("Skipping Tests for source: %s", source) if args.test_only is False and args.fetch_only is False: start_parse = time.perf_counter() mysource.parse(args.limit) end_parse = time.perf_counter() LOG.info("Parsing time: %d sec", end_parse-start_parse) if args.graph == 'rdf_graph': LOG.info("Found %d nodes", len(mysource.graph)) # Add property axioms start_axiom_exp = time.perf_counter() LOG.info("Adding property axioms") properties = GraphUtils.get_properties_from_graph(mysource.graph) GraphUtils.add_property_axioms(mysource.graph, properties) end_axiom_exp = time.clock() LOG.info("Property axioms added: %d sec", end_axiom_exp-start_axiom_exp) start_write = time.perf_counter() mysource.write(fmt=args.dest_fmt) end_write = time.perf_counter() LOG.info("Writing time: %d sec", end_write-start_write) # if args.no_verify is not True: # status = mysource.verify() # if status is not True: # LOG.error( # 'Source %s did not pass verification tests.', source) # exit(1) # else: # LOG.info('skipping verification step') LOG.info('***** Finished with %s *****', source) # load configuration parameters # for example, keys LOG.info("All done.")
def main(): # TODO this should be generated by looking in the dipper/sources directory source_to_class_map = { # 'facebase_alpha': 'FaceBase_alpha', 'hpoa': 'HPOAnnotations', 'zfin': 'ZFIN', 'omim': 'OMIM', 'biogrid': 'BioGrid', 'mgi': 'MGI', 'impc': 'IMPC', 'panther': 'Panther', 'ncbigene': 'NCBIGene', 'ucscbands': 'UCSCBands', 'ctd': 'CTD', 'genereviews': 'GeneReviews', 'eom': 'EOM', 'coriell': 'Coriell', # 'clinvar': 'ClinVar', # needs integrating here 'monochrom': 'Monochrom', 'kegg': 'KEGG', 'animalqtldb': 'AnimalQTLdb', 'ensembl': 'Ensembl', 'hgnc': 'HGNC', 'orphanet': 'Orphanet', 'omia': 'OMIA', 'flybase': 'FlyBase', 'mmrrc': 'MMRRC', 'wormbase': 'WormBase', 'mpd': 'MPD', 'gwascatalog': 'GWASCatalog', 'monarch': 'Monarch', 'go': 'GeneOntology', 'reactome': 'Reactome', 'udp': 'UDP', 'mgislim': 'MGISlim', 'zfinslim': 'ZFINSlim', 'bgee': 'Bgee', 'mydrug': 'MyDrug', 'stringdb': 'StringDB', 'rgd': 'RGD', 'sgd': 'SGD', 'mychem': 'MyChem', 'ebi': 'EBIGene2Phen', 'xenbase': 'Xenbase' } parser = argparse.ArgumentParser( description='Dipper: Data Ingestion Pipeline for Monarch', formatter_class=argparse.RawTextHelpFormatter) parser.add_argument('-g', '--graph', type=str, default="rdf_graph", help='graph type: rdf_graph, streamed_graph') parser.add_argument('-s', '--sources', type=str, default='?', help='comma separated list of sources') parser.add_argument('-l', '--limit', type=int, help='limit number of rows used') parser.add_argument('--parse_only', action='store_true', help='parse files without writing RDF') parser.add_argument('--fetch_only', action='store_true', help='fetch sources without parsing') parser.add_argument('-f', '--force', action='store_true', help='force re-download of files') parser.add_argument('--no_verify', help='ignore the verification step', action='store_true') # parser.add_argument( '--query', help='enter in a sparql query', type=str) parser.add_argument('-q', '--quiet', help='turn off info logging', action="store_true") parser.add_argument('--debug', help='turn on debug logging', action="store_true") parser.add_argument('--skip_tests', help='skip any testing', action="store_true") # Blank Nodes can't be visualized in Protege, default to Skolemizing them parser.add_argument('-b', '--use_bnodes', help="use blank nodes instead of skolemizing", action="store_true", default=False) # parser.add_argument( # TODO help needs revisiting, push constraints off the the src '-t', '--taxon', type=str, help=''' Constrain Source to supplied taxon identifier(s). Please enter comma delimited NCBITaxon numbers: Implemented taxa per source NCBIGene: 9606,10090,7955 Panther: 9606,10090,10116,7227,7955,6239,8355 BioGrid: 9606,10090,10116,7227,7955,6239,8355 UCSCBands: 9606 GO: 9606,10090,10116,7227,7955,6239,9615,9823,9031,9913,4896,5782,5052 ''') parser.add_argument( '-o', '--test_only', help='only process and output the pre-configured test subset', action="store_true") parser.add_argument( '--dest_fmt', help='serialization format: [turtle], nt, nquads, rdfxml, n3, raw', type=str) parser.add_argument('-v', '--version', help='version of source (deprecated)', type=str) parser.add_argument('-d', '--data_release_version', help=''' string indicating the version of data release, e.g. '\'201908\' (YYYYMM), used to construct metadata, including version and distribution IRIs and downloadURLs [defaults to date at start of runtime in ISO 8601 format] ''', type=str) args = parser.parse_args() tax_ids = None if args.taxon is not None: tax_ids = [str(t) for t in args.taxon.split(',') if t.isdigit()] species_specific = [ 'Panther', 'NCBIGene', 'BioGrid', 'UCSCBands', 'GeneOntology', 'Bgee', 'StringDB', 'Ensembl' ] formats_supported = [ 'turtle', 'ttl', 'ntriples', 'nt', 'nquads', 'nq', 'rdfxml', 'xml', 'notation3', 'n3', 'raw' ] if args.quiet: logging.getLogger().setLevel(logging.WARNING) else: if args.debug: logging.getLogger().setLevel(logging.DEBUG) else: logging.getLogger().setLevel(logging.INFO) if not args.use_bnodes: LOG.info("Will Skolemize Blank Nodes") # None of these query test utils exist in ./dipper/utils/TestUtils.py # if args.query is not None: # test_query = TestUtils() # for source in args.sources.split(','): # source = source.lower() # mysource = source_to_class_map[source]() # # import source lib # module = "dipper.sources.{0}".format(mysource) # imported_module = importlib.import_module(module) # source_class = getattr(imported_module, mysource) # test_query.check_query_syntax(args.query, source_class) # test_query.load_graph_from_turtle(source_class) # # print(test_query.query_graph(args.query, True)) # exit(0) # run initial tests if (args.no_verify or args.skip_tests) is not True: unittest.TextTestRunner(verbosity=2).run(TEST_SUITE) # set serializer if args.dest_fmt is not None: if args.dest_fmt in formats_supported: if args.dest_fmt == 'ttl': args.dest_fmt = 'turtle' elif args.dest_fmt == 'ntriples': args.dest_fmt = 'nt' elif args.dest_fmt == 'nq': args.dest_fmt = 'nquads' elif args.dest_fmt == 'xml': args.dest_fmt = 'rdfxml' elif args.dest_fmt == 'notation3': args.dest_fmt = 'n3' else: LOG.error("You have specified an invalid serializer: %s", args.dest_fmt) exit(0) else: args.dest_fmt = 'turtle' # Provide feedback if we can't proceed if args.sources is None or args.sources.split( ',')[0] not in source_to_class_map: LOG.info('Unknown Source %s', args.sources.split(',')[0]) LOG.info('Sources Known are limited to:') for key in sorted(source_to_class_map): LOG.info('\t%s\t%s', key, source_to_class_map[key]) exit(0) # iterate through all the sources for source in args.sources.split(','): LOG.info("\n******* %s *******", source) source = source.lower() src = source_to_class_map[source] # import source lib module = "dipper.sources.{0}".format(src) imported_module = importlib.import_module(module) source_class = getattr(imported_module, src) mysource = None LOG.info( 'Command line arguments available to dipper-etl:\n%s', "\n".join(['\t{}: {}'.format(k, v) for k, v in vars(args).items()])) source_args = dict(graph_type=args.graph) source_args['are_bnodes_skolemized'] = not args.use_bnodes if src in species_specific: source_args['tax_ids'] = tax_ids if args.version: source_args['version'] = args.version if args.data_release_version: source_args['data_release_version'] = args.data_release_version mysource = source_class(**source_args) # WIP cli args should be available to source if hasattr(mysource, 'ARGV'): mysource.ARGV = vars(args) else: LOG.error('no where to to put args in %s', mysource.__class__) if args.parse_only is False: start_fetch = time.perf_counter() mysource.fetch(args.force) end_fetch = time.perf_counter() LOG.info("Fetching time: %d sec", end_fetch - start_fetch) mysource.settestonly(args.test_only) # create source ingest graph first (with pristine arguments) if args.test_only is False and args.fetch_only is False: start_parse = time.perf_counter() mysource.parse(args.limit) end_parse = time.perf_counter() LOG.info("Parsing time: %d sec", end_parse - start_parse) if args.graph == 'rdf_graph': LOG.info("Found %d nodes", len(mysource.graph)) # Add property axioms start_axiom_exp = time.perf_counter() LOG.info("Adding property axioms") properties = GraphUtils.get_properties_from_graph( mysource.graph) GraphUtils.add_property_axioms(mysource.graph, properties) LOG.info("Property axioms added: %d sec", time.perf_counter() - start_axiom_exp) start_write = time.perf_counter() mysource.write(fmt=args.dest_fmt) LOG.info("Writing time: %d sec", time.perf_counter() - start_write) # elif args.graph == 'streamed_graph': ... # '*_test.ttl' graphs if requested if (args.no_verify or args.skip_tests) is False: suite = mysource.getTestSuite() if suite is None: LOG.warning("No tests configured for this source: %s", source) else: unittest.TextTestRunner(verbosity=2).run(suite) else: LOG.info("Skipping Tests for source: %s", source) LOG.info('***** Finished with %s *****', source) LOG.info("All done.")
def main(): # TODO this should be generated by looking in the dipper/sources directory # or read from a sources/dataset/config yaml or dir of yamls source_to_class_map = { # 'facebase_alpha': 'FaceBase_alpha', 'hpoa': 'HPOAnnotations', # ~3 min 'zfin': 'ZFIN', 'omim': 'OMIM', # full file takes ~15 min, due to required throttling 'biogrid': 'BioGrid', # interactions file takes <10 minutes 'mgi': 'MGI', 'impc': 'IMPC', # Panther takes ~1hr to map 7 species-worth of associations 'panther': 'Panther', 'oma': 'OMA', 'ncbigene': 'NCBIGene', # takes about 4 minutes to process 2 species 'ucscbands': 'UCSCBands', 'ctd': 'CTD', 'genereviews': 'GeneReviews', 'eom': 'EOM', # Takes about 5 seconds. 'coriell': 'Coriell', # 'clinvar': 'ClinVar', # takes ~ half hour # 'clinvarxml_alpha': 'ClinVarXML_alpha', # takes ~ five minutes 'monochrom': 'Monochrom', 'kegg': 'KEGG', 'animalqtldb': 'AnimalQTLdb', 'ensembl': 'Ensembl', 'hgnc': 'HGNC', 'orphanet': 'Orphanet', 'omia': 'OMIA', 'flybase': 'FlyBase', 'mmrrc': 'MMRRC', 'wormbase': 'WormBase', 'mpd': 'MPD', 'gwascatalog': 'GWASCatalog', 'monarch': 'Monarch', 'go': 'GeneOntology', 'reactome': 'Reactome', 'udp': 'UDP', 'mgi-slim': 'MGISlim', 'zfin-slim': 'ZFINSlim', 'bgee': 'Bgee', 'mydrug': 'MyDrug', 'stringdb': 'StringDB', 'rgd': 'RGD', 'sgd': 'SGD' } logger = logging.getLogger(__name__) parser = argparse.ArgumentParser( description='Dipper: Data Ingestion Pipeline for SciGraph', formatter_class=argparse.RawTextHelpFormatter) parser.add_argument( '-g', '--graph', type=str, default="rdf_graph", help='graph type: rdf_graph, streamed_graph') parser.add_argument( '-s', '--sources', type=str, required=True, help='comma separated list of sources') parser.add_argument( '-l', '--limit', type=int, help='limit number of rows') parser.add_argument( '--parse_only', action='store_true', help='parse files without writing') parser.add_argument( '--fetch_only', action='store_true', help='fetch sources without parsing') parser.add_argument('-f', '--force', action='store_true', help='force re-download of files') parser.add_argument( '--no_verify', help='ignore the verification step', action='store_true') parser.add_argument('--query', help='enter in a sparql query', type=str) parser.add_argument( '-q', '--quiet', help='turn off info logging', action="store_true") parser.add_argument( '--debug', help='turn on debug logging', action="store_true") parser.add_argument( '--skip_tests', help='skip any testing', action="store_true") # Blank Nodes can't be visualized in Protege, default to Skolemizing them parser.add_argument( '-b', '--use_bnodes', help="use blank nodes instead of skolemizing", action="store_true", default=False) # TODO this should live in a global data file # and the same filter be applied to all sources parser.add_argument( '-t', '--taxon', type=str, help='Add a taxon constraint on a source. Enter 1+ NCBITaxon numbers,' ' comma delimited\n' 'Implemented taxa per source\n' 'NCBIGene: 9606,10090,7955\n' 'Panther: 9606,10090,10116,7227,7955,6239,8355\n' 'BioGrid: 9606,10090,10116,7227,7955,6239,8355\n' 'UCSCBands: 9606\n' 'GO: 9606,10090,10116,7227,7955,6239,9615,9823,9031,9913') parser.add_argument( '-o', '--test_only', help='only process and output the pre-configured test subset', action="store_true") parser.add_argument( '--dest_fmt', help='serialization format: [turtle], nt, nquads, rdfxml, n3, raw', type=str) parser.add_argument( '--version', '-v', help='version of source', type=str) args = parser.parse_args() tax_ids = None if args.taxon is not None: tax_ids = [int(t) for t in args.taxon.split(',')] taxa_supported = [ # these are not taxa 'Panther', 'NCBIGene', 'BioGrid', 'UCSCBands', 'GeneOntology', 'Bgee', 'Ensembl', 'StringDB', 'OMA'] formats_supported = [ 'turtle', 'ttl', 'ntriples', 'nt', 'nquads', 'nq', 'rdfxml', 'xml', 'notation3', 'n3', 'raw'] if args.quiet: logging.basicConfig(level=logging.ERROR) else: if args.debug: logging.basicConfig(level=logging.DEBUG) else: logging.basicConfig(level=logging.INFO) if not args.use_bnodes: logger.info("Will Skolemize Blank Nodes") if args.query is not None: test_query = TestUtils() for source in args.sources.split(','): source = source.lower() mysource = source_to_class_map[source]() # import source lib module = "dipper.sources.{0}".format(mysource) imported_module = importlib.import_module(module) source_class = getattr(imported_module, mysource) test_query.check_query_syntax(args.query, source_class) test_query.load_graph_from_turtle(source_class) print(test_query.query_graph(args.query, True)) exit(0) # run initial tests if (args.no_verify or args.skip_tests) is not True: unittest.TextTestRunner(verbosity=2).run(test_suite) # set serializer if args.dest_fmt is not None: if args.dest_fmt in formats_supported: if args.dest_fmt == 'ttl': args.dest_fmt = 'turtle' elif args.dest_fmt == 'ntriples': args.dest_fmt = 'nt' elif args.dest_fmt == 'nq': args.dest_fmt = 'nquads' elif args.dest_fmt == 'xml': args.dest_fmt = 'rdfxml' elif args.dest_fmt == 'notation3': args.dest_fmt = 'n3' else: logger.error( "You have specified an invalid serializer: %s", args.dest_fmt) exit(0) else: args.dest_fmt = 'turtle' # iterate through all the sources for source in args.sources.split(','): logger.info("\n******* %s *******", source) source = source.lower() src = source_to_class_map[source] # import source lib module = "dipper.sources.{0}".format(src) imported_module = importlib.import_module(module) source_class = getattr(imported_module, src) mysource = None # arg factory source_args = dict( graph_type=args.graph ) source_args['are_bnodes_skolemized'] = not args.use_bnodes if src in taxa_supported: source_args['tax_ids'] = tax_ids if args.version: source_args['version'] = args.version mysource = source_class(**source_args) if args.parse_only is False: start_fetch = time.clock() mysource.fetch(args.force) end_fetch = time.clock() logger.info("Fetching time: %d sec", end_fetch-start_fetch) mysource.settestonly(args.test_only) # run tests first if (args.no_verify or args.skip_tests) is not True: suite = mysource.getTestSuite() if suite is None: logger.warning( "No tests configured for this source: %s", source) else: unittest.TextTestRunner(verbosity=2).run(suite) else: logger.info("Skipping Tests for source: %s", source) if args.test_only is False and args.fetch_only is False: start_parse = time.clock() mysource.parse(args.limit) end_parse = time.clock() logger.info("Parsing time: %d sec", end_parse-start_parse) if args.graph == 'rdf_graph': logger.info("Found %d nodes", len(mysource.graph)) # Add property axioms start_axiom_exp = time.clock() logger.info("Adding property axioms") properties = GraphUtils.get_properties_from_graph(mysource.graph) GraphUtils.add_property_axioms(mysource.graph, properties) end_axiom_exp = time.clock() logger.info("Property axioms added: %d sec", end_axiom_exp-start_axiom_exp) start_write = time.clock() mysource.write(fmt=args.dest_fmt) end_write = time.clock() logger.info("Writing time: %d sec", end_write-start_write) # if args.no_verify is not True: # status = mysource.verify() # if status is not True: # logger.error( # 'Source %s did not pass verification tests.', source) # exit(1) # else: # logger.info('skipping verification step') logger.info('***** Finished with %s *****', source) # load configuration parameters # for example, keys logger.info("All done.")