def prepare2test(name):
reset_server()
data = open_csv("test/testcases/testcase_data/testcase_" + name +
"_data.csv")
testcase_id_br = open_csv("test/testcases/testcase_data/indices/" + name +
"/index_id_br_" + name + ".csv")
testcase_id_ra = open_csv("test/testcases/testcase_data/indices/" + name +
"/index_id_ra_" + name + ".csv")
testcase_ar = open_csv("test/testcases/testcase_data/indices/" + name +
"/index_ar_" + name + ".csv")
testcase_re = open_csv("test/testcases/testcase_data/indices/" + name +
"/index_re_" + name + ".csv")
testcase_vi = open_json("test/testcases/testcase_data/indices/" + name +
"/index_vi_" + name + ".json")
testcase_ttl = "test/testcases/testcase_" + name + ".ttl"
creator_info_dir = os.path.join("meta", "tdd", "creator_counter")
creator = Creator(data, SERVER, "https://w3id.org/oc/meta/",
creator_info_dir, "060",
'https://orcid.org/0000-0002-8420-0696', testcase_id_ra,
testcase_id_br, testcase_re, testcase_ar, testcase_vi,
set())
creator_setgraph = creator.creator()
test_graph = Graph()
hack_dates()
test_graph = test_graph.parse(testcase_ttl, format="ttl")
new_graph = Graph()
for g in creator_setgraph.graphs():
new_graph += g
return test_graph, new_graph
def test_creator(self):
reset_server()
data = open_csv(
"test/testcases/testcase_data/resp_agents_curator_output.csv")
creator_info_dir = os.path.join("meta", "tdd", "creator_counter")
testcase_id_ra = open_csv(
"test/testcases/testcase_data/indices/resp_agents_curator_output/index_id_ra.csv"
)
creator = RespAgentsCreator(data, SERVER, "https://w3id.org/oc/meta/",
creator_info_dir, "060",
'https://orcid.org/0000-0002-8420-0696',
testcase_id_ra, set())
creator_graphset = creator.creator()
output_graph = Graph()
for g in creator_graphset.graphs():
output_graph += g
expected_data = '''
<https://w3id.org/oc/meta/ra/0601> <http://purl.org/spar/datacite/hasIdentifier> <https://w3id.org/oc/meta/id/0601> .
<https://w3id.org/oc/meta/ra/0601> <http://xmlns.com/foaf/0.1/givenName> "Ron J." .
<https://w3id.org/oc/meta/ra/0601> <http://xmlns.com/foaf/0.1/familyName> "Deckert" .
<https://w3id.org/oc/meta/ra/0601> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://xmlns.com/foaf/0.1/Agent> .
<https://w3id.org/oc/meta/id/0601> <http://www.essepuntato.it/2010/06/literalreification/hasLiteralValue> "0000-0003-2100-6412" .
<https://w3id.org/oc/meta/id/0601> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://purl.org/spar/datacite/Identifier> .
<https://w3id.org/oc/meta/id/0601> <http://purl.org/spar/datacite/usesIdentifierScheme> <http://purl.org/spar/datacite/orcid> .
<https://w3id.org/oc/meta/ra/0602> <http://xmlns.com/foaf/0.1/givenName> "Juan M." .
<https://w3id.org/oc/meta/ra/0602> <http://xmlns.com/foaf/0.1/familyName> "Ruso" .
<https://w3id.org/oc/meta/ra/0602> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://xmlns.com/foaf/0.1/Agent> .
<https://w3id.org/oc/meta/ra/0602> <http://purl.org/spar/datacite/hasIdentifier> <https://w3id.org/oc/meta/id/0602> .
<https://w3id.org/oc/meta/id/0602> <http://www.essepuntato.it/2010/06/literalreification/hasLiteralValue> "0000-0001-5909-6754" .
<https://w3id.org/oc/meta/id/0602> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://purl.org/spar/datacite/Identifier> .
<https://w3id.org/oc/meta/id/0602> <http://purl.org/spar/datacite/usesIdentifierScheme> <http://purl.org/spar/datacite/orcid> .
<https://w3id.org/oc/meta/ra/0603> <http://xmlns.com/foaf/0.1/familyName> "Sarmiento" .
<https://w3id.org/oc/meta/ra/0603> <http://purl.org/spar/datacite/hasIdentifier> <https://w3id.org/oc/meta/id/0603> .
<https://w3id.org/oc/meta/ra/0603> <http://xmlns.com/foaf/0.1/givenName> "Félix" .
<https://w3id.org/oc/meta/ra/0603> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://xmlns.com/foaf/0.1/Agent> .
<https://w3id.org/oc/meta/id/0603> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://purl.org/spar/datacite/Identifier> .
<https://w3id.org/oc/meta/id/0603> <http://www.essepuntato.it/2010/06/literalreification/hasLiteralValue> "0000-0002-4487-6894" .
<https://w3id.org/oc/meta/id/0603> <http://purl.org/spar/datacite/usesIdentifierScheme> <http://purl.org/spar/datacite/orcid> .
'''
expected_graph = Graph()
expected_graph = expected_graph.parse(data=expected_data, format="nt")
self.assertEqual(compare.isomorphic(output_graph, expected_graph),
True)
def test_provenance(self):
reset_server()
output_folder = os.path.join(BASE_DIR, 'output_3')
meta_process = MetaProcess(
config=os.path.join(BASE_DIR, 'meta_config_3.yaml'))
meta_process.input_csv_dir = os.path.join(BASE_DIR, 'input')
run_meta_process(meta_process)
meta_process.input_csv_dir = os.path.join(BASE_DIR, 'input_2')
run_meta_process(meta_process)
meta_process.input_csv_dir = os.path.join(BASE_DIR, 'input')
run_meta_process(meta_process)
output = dict()
for dirpath, _, filenames in os.walk(os.path.join(
output_folder, 'rdf')):
if dirpath.endswith('prov'):
for filename in filenames:
if filename.endswith('.json'):
filepath = os.path.join(dirpath, filename)
with open(filepath, 'r', encoding='utf-8') as f:
provenance = json.load(f)
essential_provenance = [{
graph: [{
p: set(v[0]['@value'].split(
'INSERT DATA { GRAPH <https://w3id.org/oc/meta/br/> { '
)[1].split(' } }')[0].split('\n'))
if '@value' in v[0] else
v if isinstance(v, list) else v
for p, v in se.items() if p not in {
'http://www.w3.org/ns/prov#generatedAtTime',
'http://purl.org/dc/terms/description',
'@type',
'http://www.w3.org/ns/prov#hadPrimarySource',
'http://www.w3.org/ns/prov#wasAttributedTo',
'http://www.w3.org/ns/prov#invalidatedAtTime'
}
} for se in sorted(ses, key=lambda d: d['@id'])
]
for graph, ses in entity.items()
if graph != '@id'
} for entity in sorted(provenance,
key=lambda x: x['@id'])]
output[dirpath.split(
os.sep)[4]] = essential_provenance
expected_output = {
'ar': [{
'@graph': [{
'@id':
'https://w3id.org/oc/meta/ar/0601/prov/se/1',
'http://www.w3.org/ns/prov#specializationOf': [{
'@id':
'https://w3id.org/oc/meta/ar/0601'
}]
}]
}, {
'@graph': [{
'@id':
'https://w3id.org/oc/meta/ar/0602/prov/se/1',
'http://www.w3.org/ns/prov#specializationOf': [{
'@id':
'https://w3id.org/oc/meta/ar/0602'
}]
}]
}, {
'@graph': [{
'@id':
'https://w3id.org/oc/meta/ar/0603/prov/se/1',
'http://www.w3.org/ns/prov#specializationOf': [{
'@id':
'https://w3id.org/oc/meta/ar/0603'
}]
}]
}],
'br': [{
'@graph': [{
'@id':
'https://w3id.org/oc/meta/br/0601/prov/se/1',
'http://www.w3.org/ns/prov#specializationOf': [{
'@id':
'https://w3id.org/oc/meta/br/0601'
}]
}, {
'@id':
'https://w3id.org/oc/meta/br/0601/prov/se/2',
'http://www.w3.org/ns/prov#specializationOf': [{
'@id':
'https://w3id.org/oc/meta/br/0601'
}],
'http://www.w3.org/ns/prov#wasDerivedFrom': [{
'@id':
'https://w3id.org/oc/meta/br/0601/prov/se/1'
}],
'https://w3id.org/oc/ontology/hasUpdateQuery': {
'<https://w3id.org/oc/meta/br/0601> <http://purl.org/spar/pro/isDocumentContextFor> <https://w3id.org/oc/meta/ar/0601> .',
'<https://w3id.org/oc/meta/br/0601> <http://purl.org/vocab/frbr/core#partOf> <https://w3id.org/oc/meta/br/0603> .',
'<https://w3id.org/oc/meta/br/0601> <http://purl.org/spar/pro/isDocumentContextFor> <https://w3id.org/oc/meta/ar/0603> .',
'<https://w3id.org/oc/meta/br/0601> <http://purl.org/spar/pro/isDocumentContextFor> <https://w3id.org/oc/meta/ar/0602> .',
'<https://w3id.org/oc/meta/br/0601> <http://prismstandard.org/namespaces/basic/2.0/publicationDate> "2015-08-22"^^<http://www.w3.org/2001/XMLSchema#date> .',
'<https://w3id.org/oc/meta/br/0601> <http://purl.org/dc/terms/title> "Some Aspects Of The Evolution Of Chernozems Under The Influence Of Natural And Anthropogenic Factors" .'
}
}]
}, {
'@graph': [{
'@id':
'https://w3id.org/oc/meta/br/0602/prov/se/1',
'http://www.w3.org/ns/prov#specializationOf': [{
'@id':
'https://w3id.org/oc/meta/br/0602'
}]
}]
}, {
'@graph': [{
'@id':
'https://w3id.org/oc/meta/br/0603/prov/se/1',
'http://www.w3.org/ns/prov#specializationOf': [{
'@id':
'https://w3id.org/oc/meta/br/0603'
}]
}]
}],
'id': [{
'@graph': [{
'@id':
'https://w3id.org/oc/meta/id/0601/prov/se/1',
'http://www.w3.org/ns/prov#specializationOf': [{
'@id':
'https://w3id.org/oc/meta/id/0601'
}]
}]
}, {
'@graph': [{
'@id':
'https://w3id.org/oc/meta/id/0602/prov/se/1',
'http://www.w3.org/ns/prov#specializationOf': [{
'@id':
'https://w3id.org/oc/meta/id/0602'
}]
}]
}, {
'@graph': [{
'@id':
'https://w3id.org/oc/meta/id/0603/prov/se/1',
'http://www.w3.org/ns/prov#specializationOf': [{
'@id':
'https://w3id.org/oc/meta/id/0603'
}]
}]
}, {
'@graph': [{
'@id':
'https://w3id.org/oc/meta/id/0604/prov/se/1',
'http://www.w3.org/ns/prov#specializationOf': [{
'@id':
'https://w3id.org/oc/meta/id/0604'
}]
}]
}],
'ra': [{
'@graph': [{
'@id':
'https://w3id.org/oc/meta/ra/0601/prov/se/1',
'http://www.w3.org/ns/prov#specializationOf': [{
'@id':
'https://w3id.org/oc/meta/ra/0601'
}]
}]
}, {
'@graph': [{
'@id':
'https://w3id.org/oc/meta/ra/0602/prov/se/1',
'http://www.w3.org/ns/prov#specializationOf': [{
'@id':
'https://w3id.org/oc/meta/ra/0602'
}]
}]
}],
're': [{
'@graph': [{
'@id':
'https://w3id.org/oc/meta/re/0601/prov/se/1',
'http://www.w3.org/ns/prov#specializationOf': [{
'@id':
'https://w3id.org/oc/meta/re/0601'
}]
}]
}]
}
shutil.rmtree(output_folder)
self.assertEqual(output, expected_output)
def test_generate_csv(self):
reset_server(server=SERVER)
add_data_ts(server=SERVER, data_path=REAL_DATA_RDF)
csv_generator = CSVGenerator(config=CONFIG)
csv_generator.generate_csv()
with open(os.path.join(OUTPUT_DIR, '060', '100', '10.csv'),
'r',
encoding='utf-8') as f:
output = list(DictReader(f))
expected_output = [{
'id': 'meta:br/0601 doi:10.1108/03068299610124298',
'title': 'Ethics And Efficiency In Organizations',
'author': 'Hausken, Kjell [meta:ra/0601]',
'pub_date': '1996-09',
'venue':
'International Journal Of Social Economics [meta:br/06011 issn:0306-8293]',
'volume': '23',
'issue': '9',
'page': '15-40',
'type': 'journal article',
'publisher': 'Emerald [meta:ra/0602 crossref:140]',
'editor': ''
}, {
'id': 'meta:br/0602 doi:10.1108/10610429810209746',
'title': 'Steel Price Determination In The European Community',
'author': 'Richardson, P.K. [meta:ra/0603]',
'pub_date': '1998-02',
'venue':
'Journal Of Product & Brand Management [meta:br/06014 issn:1061-0421]',
'volume': '7',
'issue': '1',
'page': '62-73',
'type': 'journal article',
'publisher': 'Emerald [meta:ra/0602 crossref:140]',
'editor': ''
}, {
'id': 'meta:br/0603 doi:10.1108/14684520210424557',
'title':
'Users, End‐Users, And End‐User Searchers Of Online Information: A Historical Overview',
'author':
'Farber, Miriam [meta:ra/0604]; Shoham, Snunith [meta:ra/0605]',
'pub_date': '2002-04',
'venue':
'Online Information Review [meta:br/06017 issn:1468-4527]',
'volume': '26',
'issue': '2',
'page': '92-100',
'type': 'journal article',
'publisher': 'Emerald [meta:ra/0602 crossref:140]',
'editor': ''
}, {
'id': 'meta:br/0604 doi:10.1080/10357710220147433',
'title':
'The Merger Of The Foreign Affairs And Trade Departments Revisited',
'author': 'Harris, Stuart [meta:ra/0606]',
'pub_date': '2002-07',
'venue':
'Australian Journal Of International Affairs [meta:br/06020 issn:1035-7718 issn:1465-332X]',
'volume': '56',
'issue': '2',
'page': '223-235',
'type': 'journal article',
'publisher': 'Informa Uk Limited [meta:ra/0607 crossref:301]',
'editor': ''
}, {
'id': 'meta:br/0605 doi:10.1149/1.1481718',
'title':
'Preparation Of Cu-Co Alloy Thin Films On n-Si By Galvanostatic DC Electrodeposition',
'author':
'Pattanaik, Gyana R. [meta:ra/0608]; Pandya, Dinesh K. [meta:ra/0609]; Kashyap, Subhash C. [meta:ra/06010]',
'pub_date': '2002',
'venue':
'Journal Of The Electrochemical Society [meta:br/06023 issn:0013-4651]',
'volume': '149',
'issue': '7',
'page': 'C363-C363',
'type': 'journal article',
'publisher':
'The Electrochemical Society [meta:ra/06011 crossref:77]',
'editor': ''
}, {
'id': 'meta:br/0606 doi:10.1108/09544789910214836',
'title':
'Within The Canadian Boundaries: A Close Look At Canadian Industries Implementing Continuous Improvement',
'author':
'Jha, Shelly [meta:ra/06012]; Michela, John [meta:ra/06013]; Noori, Hamid [meta:ra/06014]',
'pub_date': '1999-06',
'venue': 'The TQM Magazine [meta:br/06026 issn:0954-478X]',
'volume': '11',
'issue': '3',
'page': '188-197',
'type': 'journal article',
'publisher': 'Emerald [meta:ra/0602 crossref:140]',
'editor': ''
}, {
'id': 'meta:br/0607 doi:10.1029/2001wr00713',
'title':
'An Approximate Analytical Solution For non-Darcy Flow Toward A Well In Fractured Media',
'author': 'Wu, Yu-Shu [meta:ra/06015]',
'pub_date': '2002-03',
'venue': 'Water Resources Research [meta:br/06029 issn:0043-1397]',
'volume': '38',
'issue': '3',
'page': '5-7',
'type': 'journal article',
'publisher':
'American Geophysical Union (Agu) [meta:ra/06016 crossref:13]',
'editor': ''
}, {
'id': 'meta:br/0608 doi:10.1046/j.1523-1755.2002.00414.x',
'title':
'Proactive Monitoring Of Pediatric Hemodialysis Vascular Access: Effects Of Ultrasound Dilution On Thrombosis Rates',
'author':
'Goldstein, Stuart L. [meta:ra/06017]; Allsteadt, Amelia [meta:ra/06018]; Smith, Carolyn M. [meta:ra/06019]; Currier, Helen [meta:ra/06020]',
'pub_date': '2002-07',
'venue': 'Kidney International [meta:br/06032 issn:0085-2538]',
'volume': '62',
'issue': '1',
'page': '272-275',
'type': 'journal article',
'publisher': 'Elsevier Bv [meta:ra/06021 crossref:78]',
'editor': ''
}, {
'id': 'meta:br/0609 doi:10.1134/1.1493387',
'title':
'Types Of Discrete Symmetries Of Convection In A Plane Fluid Layer',
'author':
'Kistovich, A. V. [meta:ra/06022]; Chashechkin, Yu. D. [meta:ra/06023]',
'pub_date': '2002-06',
'venue':
'Doklady Physics [meta:br/06035 issn:1028-3358 issn:1562-6903]',
'volume': '47',
'issue': '6',
'page': '458-460',
'type': 'journal article',
'publisher':
'Pleiades Publishing Ltd [meta:ra/06024 crossref:137]',
'editor': ''
}, {
'id': 'meta:br/06010 doi:10.1108/14635779910245115',
'title': 'Tools And Supporting Techniques For Design Quality',
'author':
'Franceschini, Fiorenzo [meta:ra/06025]; Rossetto, Sergio [meta:ra/06026]',
'pub_date': '1999-09',
'venue':
'Benchmarking: An International Journal [meta:br/06038 issn:1463-5771]',
'volume': '6',
'issue': '3',
'page': '212-219',
'type': 'journal article',
'publisher': 'Emerald [meta:ra/0602 crossref:140]',
'editor': ''
}]
shutil.rmtree(OUTPUT_DIR)
self.assertEqual(output, expected_output)
def test_run_meta_process(self):
reset_server()
output_folder = os.path.join(BASE_DIR, 'output_1')
meta_process = MetaProcess(
config=os.path.join(BASE_DIR, 'meta_config_1.yaml'))
run_meta_process(meta_process)
output = list()
for dirpath, _, filenames in os.walk(os.path.join(
output_folder, 'csv')):
for file in filenames:
output.extend(get_data(os.path.join(dirpath, file)))
expected_output = [{
'id':
'doi:10.17117/na.2015.08.1067 meta:br/0601',
'title':
'',
'author':
'',
'pub_date':
'',
'venue':
'The Korean Journal Of Food And Nutrition [issn:1225-4339 meta:br/0603]',
'volume':
'26',
'issue':
'',
'page':
'',
'type':
'journal article',
'publisher':
'Consulting Company Ucom [crossref:6623 meta:ra/0601]',
'editor':
'Naimi, Elmehdi [orcid:0000-0002-4126-8519 meta:ra/0602]'
}, {
'id': 'issn:1524-4539 issn:0009-7322 meta:br/0602',
'title': 'Circulation',
'author': '',
'pub_date': '',
'venue': '',
'volume': '',
'issue': '',
'page': '',
'type': 'journal',
'publisher': '',
'editor': ''
}, {
'id':
'doi:10.9799/ksfan.2012.25.1.069 meta:br/0605',
'title':
'Nonthermal Sterilization And Shelf-life Extension Of Seafood Products By Intense Pulsed Light Treatment',
'author':
'Cheigh, Chan-Ick [orcid:0000-0003-2542-5788 meta:ra/0603]; Mun, Ji-Hye [meta:ra/0604]; Chung, Myong-Soo [meta:ra/0605]',
'pub_date':
'2012-03-31',
'venue':
'The Korean Journal Of Food And Nutrition [issn:1225-4339 meta:br/0603]',
'volume':
'25',
'issue':
'1',
'page':
'69-76',
'type':
'journal article',
'publisher':
'The Korean Society Of Food And Nutrition [crossref:4768 meta:ra/0606]',
'editor':
'Chung, Myong-Soo [orcid:0000-0002-9666-2513 meta:ra/0607]'
}, {
'id': 'doi:10.9799/ksfan.2012.25.1.077 meta:br/0606',
'title': 'Properties Of Immature Green Cherry Tomato Pickles',
'author':
'Koh, Jong-Ho [meta:ra/0608]; Shin, Hae-Hun [meta:ra/0609]; Kim, Young-Shik [orcid:0000-0001-5673-6314 meta:ra/06010]; Kook, Moo-Chang [meta:ra/06011]',
'pub_date': '2012-03-31',
'venue':
'The Korean Journal Of Food And Nutrition [issn:1225-4339 meta:br/0603]',
'volume': '',
'issue': '2',
'page': '77-82',
'type': 'journal article',
'publisher':
'The Korean Society Of Food And Nutrition [crossref:4768 meta:ra/0606]',
'editor': ''
}]
output = sorted(sorted(d.items()) for d in output)
expected_output = sorted(sorted(d.items()) for d in expected_output)
shutil.rmtree(output_folder)
self.assertEqual(output, expected_output)