def testAggregateSPARQL():
memStore = plugin.get('IOMemory', Store)()
graph1 = Graph(memStore, URIRef("graph1"))
graph2 = Graph(memStore, URIRef("graph2"))
graph3 = Graph(memStore, URIRef("graph3"))
for n3Str, graph in [(testGraph1N3, graph1), (testGraph2N3, graph2),
(testGraph3N3, graph3)]:
graph.parse(StringIO(n3Str), format='n3')
print '-------------------testAggregateSPARQL()----------------------'
print RDFS.RDFSNS
print '---------------------------------------------------------------'
graph4 = Graph(memStore, RDFS.RDFSNS)
graph4.parse(RDFS.RDFSNS)
#print graph4.serialize()
G = ConjunctiveGraph(memStore)
rt = G.query(sparqlQ)
print '-------------------G.query(sparqlQ)----------------------'
#print rt.serialize(format='xml')
print '---------------------------------------------------------------'
assert len(rt) > 1
#print rt.serialize(format='xml')
LOG_NS = Namespace(u'http://www.w3.org/2000/10/swap/log#')
rt = G.query(sparqlQ2, initBindings={u'?graph': URIRef("graph3")})
#print rt.serialize(format='json')
assert rt.serialize('python')[0] == LOG_NS.N3Document, str(rt)
def testAggregateSPARQL():
memStore = plugin.get('IOMemory',Store)()
graph1 = Graph(memStore,URIRef("graph1"))
graph2 = Graph(memStore,URIRef("graph2"))
graph3 = Graph(memStore,URIRef("graph3"))
for n3Str,graph in [(testGraph1N3,graph1),
(testGraph2N3,graph2),
(testGraph3N3,graph3)]:
graph.parse(StringIO(n3Str),format='n3')
graph4 = Graph(memStore,RDFS.RDFSNS)
graph4.parse(RDFS.RDFSNS)
G = ConjunctiveGraph(memStore)
rt = G.query(sparqlQ)
assert len(rt) > 1
#print rt.serialize(format='xml')
LOG_NS = Namespace(u'http://www.w3.org/2000/10/swap/log#')
rt=G.query(sparqlQ2,initBindings={u'?graph' : URIRef("graph3")})
#print rt.serialize(format='json')
assert rt.serialize('python')[0] == LOG_NS.N3Document,str(rt)
def testDefaultGraph():
memStore = plugin.get('IOMemory',Store)()
graph1 = Graph(memStore,URIRef("graph1"))
graph2 = Graph(memStore,URIRef("graph2"))
graph3 = Graph(memStore,URIRef("graph3"))
for n3Str,graph in [(testGraph1N3,graph1),
(testGraph2N3,graph2),
(testGraph3N3,graph3)]:
graph.parse(StringIO(n3Str),format='n3')
G = ConjunctiveGraph(memStore)
#test that CG includes triples from all 3
assert G.query(sparqlQ3),"CG as default graph should *all* triples"
assert not graph2.query(sparqlQ3),"Graph as default graph should *not* include triples from other graphs"
def testDefaultGraph():
memStore = plugin.get('IOMemory', Store)()
graph1 = Graph(memStore, URIRef("graph1"))
graph2 = Graph(memStore, URIRef("graph2"))
graph3 = Graph(memStore, URIRef("graph3"))
for n3Str, graph in [(testGraph1N3, graph1), (testGraph2N3, graph2),
(testGraph3N3, graph3)]:
graph.parse(StringIO(n3Str), format='n3')
G = ConjunctiveGraph(memStore)
#test that CG includes triples from all 3
assert G.query(sparqlQ3), "CG as default graph should *all* triples"
assert not graph2.query(
sparqlQ3
), "Graph as default graph should *not* include triples from other graphs"
def testQueryingMore(self):
for result in self.results:
uri = result[0]
g = ConjunctiveGraph()
g.parse(uri)
query = Parse("""
SELECT ?person
WHERE {
<%s> foaf:primaryTopic ?person .
?person rdf:type foaf:Person .
}
""" % uri )
queryResults = g.query(query, initNs=NSbindings).serialize('python')
if (len(queryResults)>0):
self.assertEquals(str(queryResults[0]), "http://www.wikier.org/foaf#wikier")
class TestSPARQLToldBNodes(unittest.TestCase):
def setUp(self):
NS = u"http://example.org/"
self.graph = ConjunctiveGraph()
self.graph.parse(StringInputSource("""
@prefix : <http://example.org/> .
@prefix rdf: <%s> .
@prefix rdfs: <%s> .
[ :prop :val ].
[ a rdfs:Class ]."""%(RDF.RDFNS,RDFS.RDFSNS)), format="n3")
def testToldBNode(self):
for s,p,o in self.graph.triples((None,RDF.type,None)):
pass
query = """SELECT ?obj WHERE { %s ?prop ?obj }"""%s.n3()
print query
rt = self.graph.query(query)
self.failUnless(len(rt) == 1,"BGP should only match the 'told' BNode by name (result set size: %s)"%len(rt))
def testQueryingMore(self):
for result in self.results:
uri = result[0]
g = ConjunctiveGraph()
g.parse(uri)
query = Parse("""
SELECT ?person
WHERE {
<%s> foaf:primaryTopic ?person .
?person rdf:type foaf:Person .
}
""" % uri)
queryResults = g.query(query,
initNs=NSbindings).serialize('python')
if (len(queryResults) > 0):
self.assertEquals(str(queryResults[0]),
"http://www.wikier.org/foaf#wikier")
def testSPARQLNotEquals():
NS = u"http://example.org/"
graph = ConjunctiveGraph()
graph.parse(StringInputSource("""
@prefix : <http://example.org/> .
@prefix rdf: <%s> .
:foo rdf:value 1.
:bar rdf:value 2."""%RDF.RDFNS), format="n3")
rt = graph.query("""SELECT ?node
WHERE {
?node rdf:value ?val.
FILTER (?val != 1)
}""",
initNs={'rdf':RDF.RDFNS},
DEBUG=False)
for row in rt:
item = row[0]
assert item == URIRef("http://example.org/bar")
def testSPARQLNotEquals():
NS = u"http://example.org/"
graph = ConjunctiveGraph()
graph.parse(StringInputSource("""
@prefix : <http://example.org/> .
@prefix rdf: <%s> .
:foo rdf:value 1.
:bar rdf:value 2.""" % RDF.RDFNS),
format="n3")
rt = graph.query("""SELECT ?node
WHERE {
?node rdf:value ?val.
FILTER (?val != 1)
}""",
initNs={'rdf': RDF.RDFNS},
DEBUG=False)
for row in rt:
item = row[0]
assert item == URIRef("http://example.org/bar")
test_query = """
PREFIX data: <http://example.org/foaf/>
PREFIX foaf: <http://xmlns.com/foaf/0.1/>
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
SELECT ?mbox ?nick ?ppd
WHERE
{
GRAPH data:aliceFoaf
{
?alice foaf:mbox <mailto:[email protected]> ;
foaf:knows ?whom .
?whom foaf:mbox ?mbox ;
rdfs:seeAlso ?ppd .
?ppd a foaf:PersonalProfileDocument .
} .
GRAPH ?ppd
{
?w foaf:mbox ?mbox ;
foaf:nick ?nick
}
}
"""
print
print 'Value Constraints:'
print
results = graph.query(test_query)
print results.serialize(format='xml')
class sparql_funcs():
def __init__(self):
self.g = Graph('IOMemory')
#self.endpoint = "http://www.opencorrespondence.org/data/endpoint/rdf"
#self.g.bind('geo', geo)
def find_places(self):
'''
Function to get the distinct locations mentioned in the headers of the letters.
These are the locations from which Dickens wrote.
TODO: Parsing the letters to get the places mentioned in them
'''
row = set()
o = OFS()
for b in o.list_buckets():
endpoint = o.get_stream(b, "endpoint")
self.g.parse(endpoint)
for s, _, n in self.g.triples((None, dublin_core['title'], None)):
loc_key = urllib.unquote(
n.replace("http://www.opencorrespondence.org/place/resource/",
"").replace("/rdf", ""))
row.add(self.tidy_location(loc_key))
return row
def tidy_location(self, location):
'''
Function to tidy up some of the places where they refer to the same place
TODO: prob need some language processing to make this scalable
'''
ret_location = ''
if location == 'Office Of "household Words,':
ret_location = "Household Words"
elif location == '"household Words" Office':
ret_location = "Household Words"
elif location == '"household Words"':
ret_location = "Household Words"
elif location == 'H. W. Office':
ret_location = "Household Words"
elif location == '"household Words,':
ret_location = "Household Words"
elif location == '"all The Year Round" Office':
ret_location = "All The Year Round"
elif location == 'Office Of "all The Year Round,':
ret_location = "All The Year Round"
elif location == "Gad's Hill Place":
ret_location = "Gads Hill"
elif location == "Gad's Hill":
ret_location = "Gads Hill"
elif location == "Gad's Hill Place, Higham":
ret_location = "Gads Hill"
elif location == "Tavistock House, Tavistock Square":
ret_location = "Tavistock House"
elif location == "London, Tavistock House":
ret_location = "Tavistock House"
elif location == "Tavistock House, London":
ret_location = "Tavistock House"
else:
if "U.s." in location:
location = str(location).replace("U.s", "")
ret_location = str(location).replace(".", "")
return ret_location
def find_correspondents(self):
'''
Function to get the distinct locations mentioned in the headers of the letters.
These are the locations from which Dickens wrote.
TODO: Parsing the letters to get the places mentioned in them
'''
row = set()
self.g.parse(self.endpoint)
for s, _, n in self.g.triples((None, letter['correspondent'], None)):
loc_key = urllib.unquote(
n.replace(
"http://www.opencorrespondence.org/correspondent/resource/",
"").replace("/rdf", ""))
row.add(loc_key)
return row
def get_abstract(self, resource_id):
self.g.parse('http://dbpedia.org/resource/'.resource_id)
q = '''
SELECT *
WHERE
{
?x dbpedia:abstract ?abstract .
FILTER (lang(?abstract) = 'en')
}
'''
for row in self.g.query(
q,
initNs=dict(dbpedia=Namespace("http://dbpedia.org/ontology/")),
initBindings={}):
return row[1]
def query_dates(self, author):
'''query to identify individual dates to a correspondent'''
q = '''
SELECT ?date
FROM <http://localhost:5000/data/endpoint/rdf>
WHERE {
?r dc:subject \'''' + author + '''\' .
?r dc:date ?date.
}
'''
dates = []
for row in self.g.query(
q,
initNs=dict(letter=Namespace(
"http://www.opencorrespondence.org/schema#"),
dc=Namespace("http://purl.org/dc/elements/1.1/")),
initBindings={}):
date = str(row[0]).split('-')
if date[0][1:].isdigit():
dates.append(date[0])
print dates
dic = {}
for dt in dates:
dic[dt] = dates.count(dt)
return dic
class TriplifierTest(unittest.TestCase):
def setUp(self):
self.values = optparse.Values()
self.values.ensure_value("baseURI", "b")
self.graph = ConjunctiveGraph()
self.t = Triplifier(self.graph, self.values)
self.mox = mox.Mox()
def compareGeneratedTriples(self, expected):
for triple in self.graph.query(ALLTRIPLES):
self.assertTrue(triple in expected, "%s is not in" % str(triple))
def testPushInitialTriples(self):
self.values.ensure_value("distribution", "http://example.com/d")
self.values.ensure_value("distdate", "Not used")
self.values.ensure_value("parsedDistDate", date(1985, 07, 01))
self.t.pushInitialTriples()
self.assertEqual(2, len(self.graph))
uriref = URIRef("http://example.com/d")
expected = [(uriref, RDF.type, DEB['Distribution']),\
(uriref, DEB['releaseDate'], Literal(date(1985, 07, 01)))]
self.compareGeneratedTriples(expected)
def testTriplifyArchitecture(self):
arch = Architecture("testArch")
uriref = URIRef("b/arch/testArch")
self.assertEqual(uriref, self.t.triplifyArchitecture(arch))
self.assertEqual(2, len(self.graph))
expected = [(uriref, RDF.type, DEB['Architecture']),\
(uriref, RDFS.label, Literal("Architecture: testArch", lang='en'))]
self.compareGeneratedTriples(expected)
def testTriplifyArchitectureOntologyInstance(self):
arch = Architecture("all")
self.assertEqual(DEB['all'], self.t.triplifyArchitecture(arch))
def testTriplifyVersionNumberSimple(self):
version = VersionNumber("1.0-1")
uriref = URIRef("b/version/1.0-1")
self.assertEqual(uriref, self.t.triplifyVersionNumber(version))
self.assertEqual(5, len(self.graph))
expected = [(uriref, RDF.type, DEB['VersionNumber']),\
(uriref, RDFS.label, Literal("Version: 1.0-1", lang='en')),\
(uriref, DEB['fullVersion'], Literal("1.0-1")),\
(uriref, DEB['upstreamVersion'], Literal("1.0")),\
(uriref, DEB['debianRevision'], Literal("1"))]
self.compareGeneratedTriples(expected)
def testTriplifyConstraintSimple(self):
constraint = Constraint()
constraint.package = UnversionedBinaryPackage("pkg")
constraint.operator = ">>"
constraint.version = VersionNumber("1.0-1")
self.t.triplifyVersionNumber =\
self.mockTriplifyVersionNumber(constraint.version)
self.t.triplifyUnversionedBinaryPackage =\
self.mockUnversionedBinaryPackage(constraint.package)
uriref = URIRef("b/constraint/pkg StrictlyLater 1.0-1")
self.assertEqual(uriref, self.t.triplifyConstraint(constraint))
self.mox.VerifyAll()
self.assertEqual(5, len(self.graph))
expected = [(uriref, RDF.type, DEB['SimplePackageConstraint']),\
(uriref, RDFS.label, Literal("Constraint: pkg (>> 1.0-1)", lang='en')),\
(uriref, DEB['package'], URIRef("b/binary/pkg")),\
(uriref, DEB['constraintOperator'], Literal(">>")),\
(uriref, DEB['versionNumber'], URIRef("b/version/1.0-1"))]
self.compareGeneratedTriples(expected)
def testTriplifyOrConstraint(self):
orconstraint = OrConstraint()
constraint1 = Constraint()
constraint1.package = UnversionedBinaryPackage("pkg1")
constraint2 = Constraint()
constraint2.package = UnversionedBinaryPackage("pkg2")
orconstraint.add(constraint1)
orconstraint.add(constraint2)
self.t.triplifyConstraint = self.mockTriplifyConstraint([constraint1, constraint2])
self.assertEqual(BNode, self.t.triplifyOrConstraint(orconstraint).__class__)
self.mox.VerifyAll()
self.assertEqual(3, len(self.graph))
def testTriplifyBinaryPackageBuild(self):
bs = BinaryPackage("pkg1", "6.7")
b = BinaryPackageBuild(bs)
b.architecture = Architecture("arch")
b.installedSize = "12345"
self.t.triplifyArchitecture = self.mockTriplifyArchitecture(b.architecture)
uriref = URIRef("b/binary/pkg1/6.7/arch")
self.assertEqual(uriref, self.t.triplifyBinaryPackageBuild(b))
self.assertEqual(4, len(self.graph))
expected = [(uriref, RDF.type, DEB['BinaryBuild']),\
(uriref, RDFS.label, Literal("BinaryBuild: pkg1 (6.7) [arch]", lang='en')),\
(uriref, DEB['installed-size'], Literal(int("12345"))),\
(uriref, DEB['architecture'], URIRef("b/arch/arch"))]
self.compareGeneratedTriples(expected)
def testTriplifyFile(self):
f = File("testname", "hash", "1234", Directory("test/path"))
uriref = URIRef("b/path/test/path/testname")
self.assertEqual(uriref, self.t.triplifyFile(f))
self.assertEqual(8, len(self.graph))
def testTriplifyTag(self):
t = Tag("facet", "tag:tag")
uriref = URIRef("b/tag/facet/tag:tag")
self.assertEqual(uriref, self.t.triplifyTag(t))
self.assertEqual(4, len(self.graph))
expected = [(uriref, RDF.type, TAG['Tag']),\
(uriref, RDFS.label, Literal("Tag: facet::tag:tag", lang='en')),\
(uriref, DEB['facet'], Literal("facet")),\
(uriref, TAG['name'], Literal("tag:tag"))]
self.compareGeneratedTriples(expected)
def testTriplifySection(self):
s = Section("test")
uriref = URIRef("b/section/test")
self.assertEqual(uriref, self.t.triplifySection(s))
self.assertEqual(3, len(self.graph))
expected = [(uriref, RDF.type, DEB['Section']),\
(uriref, RDFS.label, Literal("Section: test", lang='en')),\
(uriref, DEB['sectionName'], Literal("test"))]
self.compareGeneratedTriples(expected)
def testTriplifyPriority(self):
p = PriorityBox.get("optional")
self.assertEqual(DEB["optional"], self.t.triplifyPriority(p))
def testTriplifyContributorHuman(self):
c = Human("Jon Doe", "*****@*****.**")
uriref = URIRef("b/people/[email protected]")
self.assertEqual(uriref, self.t.triplifyContributor(c))
self.assertEqual(4, len(self.graph))
expected = [(uriref, RDF.type, FOAF['Person']),\
(uriref, RDFS.label, Literal("Human: [email protected]", lang='en')),\
(uriref, FOAF['name'], Literal("Jon Doe")),\
(uriref, FOAF['mbox'], Literal("*****@*****.**"))]
self.compareGeneratedTriples(expected)
def testTriplifyContributorHumanNoName(self):
c = Human(None, "*****@*****.**")
uriref = URIRef("b/people/[email protected]")
self.assertEqual(uriref, self.t.triplifyContributor(c))
self.assertEqual(3, len(self.graph))
expected = [(uriref, RDF.type, FOAF['Person']),\
(uriref, RDFS.label, Literal("Human: [email protected]", lang='en')),\
(uriref, FOAF['mbox'], Literal("*****@*****.**"))]
self.compareGeneratedTriples(expected)
def testTriplifyContributorTeam(self):
c = Team("Debian Love Team", "*****@*****.**")
uriref = URIRef("b/team/[email protected]")
self.assertEqual(uriref, self.t.triplifyContributor(c))
self.assertEqual(4, len(self.graph))
expected = [(uriref, RDF.type, FOAF['Group']),\
(uriref, RDFS.label, Literal("Team: [email protected]", lang='en')),\
(uriref, FOAF['name'], Literal("Debian Love Team")),\
(uriref, FOAF['mbox'], Literal("*****@*****.**"))]
self.compareGeneratedTriples(expected)
def testTriplifyTeamAddMemberHumanToGroup(self):
t = Team("Debian Love Team", "*****@*****.**")
h = Human("Jon Doe", "*****@*****.**")
tRef = URIRef("b/team/[email protected]")
hRef = URIRef("b/people/[email protected]")
self.t.triplifyTeamAddMember(t,h)
self.assertEqual(1, len(self.graph))
expected = [(tRef, FOAF['member'], hRef)]
self.compareGeneratedTriples(expected)
def testTriplifyTeamAddMemberGroupToGroup(self):
t = Team("Debian Love Team", "*****@*****.**")
tRef = URIRef("b/team/Debian+Love+Team")
self.t.triplifyTeamAddMember(t,t)
self.assertEqual(0, len(self.graph))
def testTriplifyArea(self):
a = AreaBox.get("main")
self.assertEqual(DEB["main"], self.t.triplifyArea(a))
def testTriplifyHomepage(self):
h = "http://example.org"
uriref = URIRef(h)
self.assertEqual(uriref, self.t.triplifyHomepage(h))
self.assertEqual(1, len(self.graph))
expected = [(uriref, RDF.type, FOAF['Document'])]
self.compareGeneratedTriples(expected)
def testTriplifyUnversionedSourcePackage(self):
us = UnversionedSourcePackage("name")
uriref = URIRef("b/source/name")
self.assertEqual(uriref, self.t.triplifyUnversionedSourcePackage(us))
self.assertEqual(2, len(self.graph))
expected = [(uriref, RDF.type, DEB['UnversionedSource']),\
(uriref, RDFS.label, Literal("Unversioned Source: name", lang='en'))]
self.compareGeneratedTriples(expected)
def testTriplifyUnversionedBinaryPackage(self):
ub = UnversionedBinaryPackage("name")
uriref = URIRef("b/binary/name")
self.assertEqual(uriref, self.t.triplifyUnversionedBinaryPackage(ub))
self.assertEqual(2, len(self.graph))
expected = [(uriref, RDF.type, DEB['UnversionedBinary']),\
(uriref, RDFS.label, Literal("Unversioned Binary: name", lang='en'))]
self.compareGeneratedTriples(expected)
def testTriplifyRepositoryFull(self):
r = GitRepository("http://example.com", "git://git.example.com")
bnode = self.t.triplifyRepository(r)
self.assertEqual(BNode, bnode.__class__)
self.assertEqual(5, len(self.graph))
expected = [(bnode, RDF.type, DOAP['GitRepository']),\
(bnode, RDFS.label, Literal("Repository: git://git.example.com", lang='en')),\
(bnode, DOAP['location'], URIRef("git://git.example.com")),\
(bnode, DOAP['browse'], URIRef("http://example.com")),\
(URIRef("http://example.com"), RDF.type, FOAF['page'])]
self.compareGeneratedTriples(expected)
def testTriplifyRepositoryNoBrowser(self):
r = GitRepository(None, "git://git.example.com")
bnode = self.t.triplifyRepository(r)
self.assertEqual(BNode, bnode.__class__)
self.assertEqual(3, len(self.graph))
expected = [(bnode, RDF.type, DOAP['GitRepository']),\
(bnode, RDFS.label, Literal("Repository: git://git.example.com", lang='en')),\
(bnode, DOAP['location'], URIRef("git://git.example.com"))]
self.compareGeneratedTriples(expected)
def testTriplifyRepositoryNoURI(self):
r = GitRepository("http://example.com", None)
bnode = self.t.triplifyRepository(r)
self.assertEqual(BNode, bnode.__class__)
self.assertEqual(4, len(self.graph))
expected = [(bnode, RDF.type, DOAP['GitRepository']),\
(bnode, RDFS.label, Literal("Repository", lang='en')),\
(bnode, DOAP['browse'], URIRef("http://example.com")),\
(URIRef("http://example.com"), RDF.type, FOAF['page'])]
self.compareGeneratedTriples(expected)
# Mocks
def mockTriplifyVersionNumber(self, version):
classMock = self.mox.CreateMock(Triplifier)
classMock.triplifyVersionNumber(version)\
.AndReturn(URIRef(version.asURI("b")))
self.mox.ReplayAll()
return classMock.triplifyVersionNumber
def mockTriplifyConstraint(self, constraints):
classMock = self.mox.CreateMock(Triplifier)
for constraint in constraints:
classMock.triplifyConstraint(constraint)\
.InAnyOrder()\
.AndReturn(URIRef(constraint.asURI("b")))
self.mox.ReplayAll()
return classMock.triplifyConstraint
def mockTriplifyArchitecture(self, arch):
classMock = self.mox.CreateMock(Triplifier)
classMock.triplifyArchitecture(arch)\
.AndReturn(URIRef(arch.asURI("b")))
self.mox.ReplayAll()
return classMock.triplifyArchitecture
def mockUnversionedBinaryPackage(self, ubinary):
classMock = self.mox.CreateMock(Triplifier)
classMock.triplifyUnversionedBinaryPackage(ubinary)\
.AndReturn(URIRef(ubinary.asURI("b")))
self.mox.ReplayAll()
return classMock.triplifyUnversionedBinaryPackage
def testBasic(DEBUG = False):
from glob import glob
from sre import sub
for testFile in glob('data/examples/*.rq'):#glob('data/*/*.rq'):
store = plugin.get(STORE,Store)()
bootStrapStore(store)
store.commit()
prefix = testFile.split('.rq')[-1]
manifestPath = '/'.join(testFile.split('/')[:-1]+['manifest.n3'])
manifestPath2 = '/'.join(testFile.split('/')[:-1]+['manifest.ttl'])
queryFileName = testFile.split('/')[-1]
store = plugin.get(STORE,Store)()
store.open(configString,create=False)
assert len(store) == 0
manifestG=ConjunctiveGraph(store)
if not os.path.exists(manifestPath):
assert os.path.exists(manifestPath2)
manifestPath = manifestPath2
manifestG.default_context.parse(open(manifestPath),publicID=TEST_BASE,format='n3')
manifestData = \
manifestG.query(
PARSED_MANIFEST_QUERY,
initBindings={'?query' : TEST_BASE[queryFileName]},
initNs=manifestNS,
DEBUG = False)
store.rollback()
store.close()
for source,testCaseName,testCaseComment,expectedRT in manifestData:
if expectedRT:
expectedRT = '/'.join(testFile.split('/')[:-1]+[expectedRT.replace(TEST_BASE,'')])
if source:
source = '/'.join(testFile.split('/')[:-1]+[source.replace(TEST_BASE,'')])
testCaseName = testCaseComment and testCaseComment or testCaseName
print "## Source: %s ##"%source
print "## Test: %s ##"%testCaseName
print "## Result: %s ##"%expectedRT
#Expected results
if expectedRT:
store = plugin.get(STORE,Store)()
store.open(configString,create=False)
resultG=ConjunctiveGraph(store).default_context
# if DEBUG:
# print "###"*10
# print "parsing: ", open(expectedRT).read()
# print "###"*10
assert len(store) == 0
print "## Parsing (%s) ##"%(expectedRT)
if not trialAndErrorRTParse(resultG,expectedRT,DEBUG):
if DEBUG:
print "Unexpected result format (for %s), skipping"%(expectedRT)
store.rollback()
store.close()
continue
if DEBUG:
print "## Done .. ##"
rtVars = [rtVar for rtVar in resultG.objects(None,RESULT_NS.resultVariable)]
bindings = []
resultSetNode = resultG.value(predicate=RESULT_NS.value,object=RESULT_NS.ResultSet)
for solutionNode in resultG.objects(resultSetNode,RESULT_NS.solution):
bindingDict = dict([(key,None) for key in rtVars])
for bindingNode in resultG.objects(solutionNode,RESULT_NS.binding):
value = resultG.value(subject=bindingNode,predicate=RESULT_NS.value)
name = resultG.value(subject=bindingNode,predicate=RESULT_NS.variable)
bindingDict[name] = value
bindings.append(tuple([bindingDict[vName] for vName in rtVars]))
if DEBUG:
print "Expected bindings: ", bindings
print open(expectedRT).read()
store.rollback()
store.close()
if testFile.startswith('data/NegativeSyntax'):
try:
query = open(testFile).read()
p = Parse(query,DEBUG)
except:
continue
else:
raise Exception("Test %s should have failed!"%testFile)
if testFile in tests2Skip:
print "Skipping test (%s)"%testCaseName
continue
query = open(testFile).read()
print "### %s (%s) ###"%(testCaseName,testFile)
print query
p = Parse(query,DEBUG_PARSE)
if DEBUG:
print p
if EVALUATE and source:
if DEBUG:
print "### Source Graph: ###"
print open(source).read()
store = plugin.get(STORE,Store)()
store.open(configString,create=False)
g=ConjunctiveGraph(store)
try:
g.parse(open(source),format='n3')
except:
print "Unexpected data format (for %s), skipping"%(source)
store.rollback()
store.close()
continue
#print store
rt = g.query(p,DEBUG = DEBUG)
if expectedRT:
if rt != bindings and Set([Set(i) for i in rt]) != Set([Set(i) for i in bindings]):#unorderedComparison(rt,bindings):
print "### Expected Result (%s) ###"%expectedRT
pprint(bindings)
print "### Actual Results ###"
pprint(rt)
raise Exception("### TEST FAILED!: %s ###"%testCaseName)
else:
print "### TEST PASSED!: %s ###"%testCaseName
store.rollback()
class TriplesDatabase(object):
"""A database from the defined triples"""
def __init__(self):
self._open = False
def open(self, filename, graphClass=None):
"""
Load existing database at 'filename'.
"""
if filename is None:
if graphClass is None:
self.graph = Graph()
else:
self.graph = graphClass()
else:
assert os.path.exists(filename), (
"%s must be an existing database" % (filename,))
path, filename = os.path.split(filename)
self.graph = sqliteBackedGraph(path, filename)
self._open = True
def query(self, rest, initNs=None, initBindings=None):
"""
Execute a SPARQL query and get the results as a SPARQLResult
{rest} is a string that should begin with "SELECT ", usually
"""
assert self._open
if initNs is None:
initNs = dict(self.graph.namespaces())
if initBindings is None: initBindings = {}
sel = select(self.getBase(), rest)
ret = self.graph.query(sel, initNs=initNs, initBindings=initBindings,
DEBUG=False)
return ret
def getBase(self):
d = dict(self.graph.namespaces())
return d.get('', RDFSNS)
def addTriple(self, s, v, *objects):
"""
Make a statement/arc/triple in the database.
Strings, ints and floats as s or o will automatically be coerced to
RDFLiteral(). It is an error to give a RDFLiteral as v, so no
coercion will be done in that position.
2-tuples will be coerced to bnodes.
If more than one object is given, i.e.
addTriple(a, b, c1, c2, c3)
this is equivalent to:
addTriple(a,b,c1); addTriple(a,b,c2); addTriple(a,b,c3)
"""
assert self._open
assert len(objects) >= 1, "You must provide at least one object"
if canBeLiteral(s):
s = RDFLiteral(s)
bnode = None
for o in objects:
if canBeLiteral(o):
o = RDFLiteral(o)
elif isinstance(o, tuple) and len(o) == 2:
if bnode is None:
bnode = BNode()
self.addTriple(bnode, *o)
o = bnode
assert None not in [s,v,o]
self.graph.add((s, v, o))
def dump(self):
assert self._open
io = StringIO()
try:
self.graph.serialize(destination=io, format='n3')
except Exception, e:
import sys, pdb; pdb.post_mortem(sys.exc_info()[2])
return io.getvalue()
class sparql_funcs():
def __init__(self):
self.g = Graph('IOMemory')
#self.endpoint = "http://www.opencorrespondence.org/data/endpoint/rdf"
#self.g.bind('geo', geo)
def find_places(self):
'''
Function to get the distinct locations mentioned in the headers of the letters.
These are the locations from which Dickens wrote.
TODO: Parsing the letters to get the places mentioned in them
'''
row = set()
o = OFS()
for b in o.list_buckets():
endpoint = o.get_stream(b, "endpoint")
self.g.parse(endpoint)
for s,_,n in self.g.triples((None, dublin_core['title'], None)):
loc_key = urllib.unquote(n.replace("http://www.opencorrespondence.org/place/resource/", "").replace("/rdf",""))
row.add(self.tidy_location(loc_key))
return row
def tidy_location (self, location):
'''
Function to tidy up some of the places where they refer to the same place
TODO: prob need some language processing to make this scalable
'''
ret_location = '';
if location == 'Office Of "household Words,':
ret_location = "Household Words"
elif location== '"household Words" Office':
ret_location = "Household Words"
elif location== '"household Words"':
ret_location = "Household Words"
elif location== 'H. W. Office':
ret_location = "Household Words"
elif location == '"household Words,':
ret_location = "Household Words"
elif location == '"all The Year Round" Office':
ret_location = "All The Year Round"
elif location == 'Office Of "all The Year Round,':
ret_location = "All The Year Round"
elif location == "Gad's Hill Place":
ret_location = "Gads Hill"
elif location == "Gad's Hill":
ret_location = "Gads Hill"
elif location == "Gad's Hill Place, Higham":
ret_location = "Gads Hill"
elif location == "Tavistock House, Tavistock Square":
ret_location = "Tavistock House"
elif location == "London, Tavistock House":
ret_location = "Tavistock House"
elif location == "Tavistock House, London":
ret_location = "Tavistock House"
else:
if "U.s." in location:
location = str(location).replace("U.s", "")
ret_location = str(location).replace(".", "")
return ret_location
def find_correspondents(self):
'''
Function to get the distinct locations mentioned in the headers of the letters.
These are the locations from which Dickens wrote.
TODO: Parsing the letters to get the places mentioned in them
'''
row = set()
self.g.parse(self.endpoint)
for s,_,n in self.g.triples((None, letter['correspondent'], None)):
loc_key = urllib.unquote(n.replace("http://www.opencorrespondence.org/correspondent/resource/", "").replace("/rdf", ""))
row.add(loc_key)
return row
def get_abstract (self, resource_id):
self.g.parse('http://dbpedia.org/resource/'.resource_id)
q = '''
SELECT *
WHERE
{
?x dbpedia:abstract ?abstract .
FILTER (lang(?abstract) = 'en')
}
'''
for row in self.g.query(q,
initNs=dict(dbpedia=Namespace("http://dbpedia.org/ontology/")),
initBindings={}):
return row[1]
def query_dates(self, author):
'''query to identify individual dates to a correspondent'''
q = '''
SELECT ?date
FROM <http://localhost:5000/data/endpoint/rdf>
WHERE {
?r dc:subject \'''' + author + '''\' .
?r dc:date ?date.
}
'''
dates = []
for row in self.g.query(q,
initNs=dict(letter=Namespace("http://www.opencorrespondence.org/schema#"), dc=Namespace("http://purl.org/dc/elements/1.1/")),
initBindings={}):
date = str(row[0]).split('-')
if date[0][1:].isdigit():
dates.append(date[0])
print dates
dic = {}
for dt in dates:
dic[dt] = dates.count(dt)
return dic
class TriplifierTest(unittest.TestCase):
def setUp(self):
self.values = optparse.Values()
self.values.ensure_value("baseURI", "b")
self.graph = ConjunctiveGraph()
self.t = Triplifier(self.graph, self.values)
self.mox = mox.Mox()
def compareGeneratedTriples(self, expected):
for triple in self.graph.query(ALLTRIPLES):
self.assertTrue(triple in expected, "%s is not in" % str(triple))
def testPushInitialTriples(self):
self.values.ensure_value("distribution", "http://example.com/d")
self.values.ensure_value("distdate", "Not used")
self.values.ensure_value("parsedDistDate", date(1985, 07, 01))
self.t.pushInitialTriples()
self.assertEqual(2, len(self.graph))
uriref = URIRef("http://example.com/d")
expected = [(uriref, RDF.type, DEB['Distribution']),\
(uriref, DEB['releaseDate'], Literal(date(1985, 07, 01)))]
self.compareGeneratedTriples(expected)
def testTriplifyArchitecture(self):
arch = Architecture("testArch")
uriref = URIRef("b/arch/testArch")
self.assertEqual(uriref, self.t.triplifyArchitecture(arch))
self.assertEqual(2, len(self.graph))
expected = [(uriref, RDF.type, DEB['Architecture']),\
(uriref, RDFS.label, Literal("Architecture: testArch", lang='en'))]
self.compareGeneratedTriples(expected)
def testTriplifyArchitectureOntologyInstance(self):
arch = Architecture("all")
self.assertEqual(DEB['all'], self.t.triplifyArchitecture(arch))
def testTriplifyVersionNumberSimple(self):
version = VersionNumber("1.0-1")
uriref = URIRef("b/version/1.0-1")
self.assertEqual(uriref, self.t.triplifyVersionNumber(version))
self.assertEqual(5, len(self.graph))
expected = [(uriref, RDF.type, DEB['VersionNumber']),\
(uriref, RDFS.label, Literal("Version: 1.0-1", lang='en')),\
(uriref, DEB['fullVersion'], Literal("1.0-1")),\
(uriref, DEB['upstreamVersion'], Literal("1.0")),\
(uriref, DEB['debianRevision'], Literal("1"))]
self.compareGeneratedTriples(expected)
def testTriplifyConstraintSimple(self):
constraint = Constraint()
constraint.package = UnversionedBinaryPackage("pkg")
constraint.operator = ">>"
constraint.version = VersionNumber("1.0-1")
self.t.triplifyVersionNumber =\
self.mockTriplifyVersionNumber(constraint.version)
self.t.triplifyUnversionedBinaryPackage =\
self.mockUnversionedBinaryPackage(constraint.package)
uriref = URIRef("b/constraint/pkg StrictlyLater 1.0-1")
self.assertEqual(uriref, self.t.triplifyConstraint(constraint))
self.mox.VerifyAll()
self.assertEqual(5, len(self.graph))
expected = [(uriref, RDF.type, DEB['SimplePackageConstraint']),\
(uriref, RDFS.label, Literal("Constraint: pkg (>> 1.0-1)", lang='en')),\
(uriref, DEB['package'], URIRef("b/binary/pkg")),\
(uriref, DEB['constraintOperator'], Literal(">>")),\
(uriref, DEB['versionNumber'], URIRef("b/version/1.0-1"))]
self.compareGeneratedTriples(expected)
def testTriplifyOrConstraint(self):
orconstraint = OrConstraint()
constraint1 = Constraint()
constraint1.package = UnversionedBinaryPackage("pkg1")
constraint2 = Constraint()
constraint2.package = UnversionedBinaryPackage("pkg2")
orconstraint.add(constraint1)
orconstraint.add(constraint2)
self.t.triplifyConstraint = self.mockTriplifyConstraint(
[constraint1, constraint2])
self.assertEqual(BNode,
self.t.triplifyOrConstraint(orconstraint).__class__)
self.mox.VerifyAll()
self.assertEqual(3, len(self.graph))
def testTriplifyBinaryPackageBuild(self):
bs = BinaryPackage("pkg1", "6.7")
b = BinaryPackageBuild(bs)
b.architecture = Architecture("arch")
b.installedSize = "12345"
self.t.triplifyArchitecture = self.mockTriplifyArchitecture(
b.architecture)
uriref = URIRef("b/binary/pkg1/6.7/arch")
self.assertEqual(uriref, self.t.triplifyBinaryPackageBuild(b))
self.assertEqual(4, len(self.graph))
expected = [(uriref, RDF.type, DEB['BinaryBuild']),\
(uriref, RDFS.label, Literal("BinaryBuild: pkg1 (6.7) [arch]", lang='en')),\
(uriref, DEB['installed-size'], Literal(int("12345"))),\
(uriref, DEB['architecture'], URIRef("b/arch/arch"))]
self.compareGeneratedTriples(expected)
def testTriplifyFile(self):
f = File("testname", "hash", "1234", Directory("test/path"))
uriref = URIRef("b/path/test/path/testname")
self.assertEqual(uriref, self.t.triplifyFile(f))
self.assertEqual(8, len(self.graph))
def testTriplifyTag(self):
t = Tag("facet", "tag:tag")
uriref = URIRef("b/tag/facet/tag:tag")
self.assertEqual(uriref, self.t.triplifyTag(t))
self.assertEqual(4, len(self.graph))
expected = [(uriref, RDF.type, TAG['Tag']),\
(uriref, RDFS.label, Literal("Tag: facet::tag:tag", lang='en')),\
(uriref, DEB['facet'], Literal("facet")),\
(uriref, TAG['name'], Literal("tag:tag"))]
self.compareGeneratedTriples(expected)
def testTriplifySection(self):
s = Section("test")
uriref = URIRef("b/section/test")
self.assertEqual(uriref, self.t.triplifySection(s))
self.assertEqual(3, len(self.graph))
expected = [(uriref, RDF.type, DEB['Section']),\
(uriref, RDFS.label, Literal("Section: test", lang='en')),\
(uriref, DEB['sectionName'], Literal("test"))]
self.compareGeneratedTriples(expected)
def testTriplifyPriority(self):
p = PriorityBox.get("optional")
self.assertEqual(DEB["optional"], self.t.triplifyPriority(p))
def testTriplifyContributorHuman(self):
c = Human("Jon Doe", "*****@*****.**")
uriref = URIRef("b/people/[email protected]")
self.assertEqual(uriref, self.t.triplifyContributor(c))
self.assertEqual(4, len(self.graph))
expected = [(uriref, RDF.type, FOAF['Person']),\
(uriref, RDFS.label, Literal("Human: [email protected]", lang='en')),\
(uriref, FOAF['name'], Literal("Jon Doe")),\
(uriref, FOAF['mbox'], Literal("*****@*****.**"))]
self.compareGeneratedTriples(expected)
def testTriplifyContributorHumanNoName(self):
c = Human(None, "*****@*****.**")
uriref = URIRef("b/people/[email protected]")
self.assertEqual(uriref, self.t.triplifyContributor(c))
self.assertEqual(3, len(self.graph))
expected = [(uriref, RDF.type, FOAF['Person']),\
(uriref, RDFS.label, Literal("Human: [email protected]", lang='en')),\
(uriref, FOAF['mbox'], Literal("*****@*****.**"))]
self.compareGeneratedTriples(expected)
def testTriplifyContributorTeam(self):
c = Team("Debian Love Team", "*****@*****.**")
uriref = URIRef("b/team/[email protected]")
self.assertEqual(uriref, self.t.triplifyContributor(c))
self.assertEqual(4, len(self.graph))
expected = [(uriref, RDF.type, FOAF['Group']),\
(uriref, RDFS.label, Literal("Team: [email protected]", lang='en')),\
(uriref, FOAF['name'], Literal("Debian Love Team")),\
(uriref, FOAF['mbox'], Literal("*****@*****.**"))]
self.compareGeneratedTriples(expected)
def testTriplifyTeamAddMemberHumanToGroup(self):
t = Team("Debian Love Team", "*****@*****.**")
h = Human("Jon Doe", "*****@*****.**")
tRef = URIRef("b/team/[email protected]")
hRef = URIRef("b/people/[email protected]")
self.t.triplifyTeamAddMember(t, h)
self.assertEqual(1, len(self.graph))
expected = [(tRef, FOAF['member'], hRef)]
self.compareGeneratedTriples(expected)
def testTriplifyTeamAddMemberGroupToGroup(self):
t = Team("Debian Love Team", "*****@*****.**")
tRef = URIRef("b/team/Debian+Love+Team")
self.t.triplifyTeamAddMember(t, t)
self.assertEqual(0, len(self.graph))
def testTriplifyArea(self):
a = AreaBox.get("main")
self.assertEqual(DEB["main"], self.t.triplifyArea(a))
def testTriplifyHomepage(self):
h = "http://example.org"
uriref = URIRef(h)
self.assertEqual(uriref, self.t.triplifyHomepage(h))
self.assertEqual(1, len(self.graph))
expected = [(uriref, RDF.type, FOAF['Document'])]
self.compareGeneratedTriples(expected)
def testTriplifyUnversionedSourcePackage(self):
us = UnversionedSourcePackage("name")
uriref = URIRef("b/source/name")
self.assertEqual(uriref, self.t.triplifyUnversionedSourcePackage(us))
self.assertEqual(2, len(self.graph))
expected = [(uriref, RDF.type, DEB['UnversionedSource']),\
(uriref, RDFS.label, Literal("Unversioned Source: name", lang='en'))]
self.compareGeneratedTriples(expected)
def testTriplifyUnversionedBinaryPackage(self):
ub = UnversionedBinaryPackage("name")
uriref = URIRef("b/binary/name")
self.assertEqual(uriref, self.t.triplifyUnversionedBinaryPackage(ub))
self.assertEqual(2, len(self.graph))
expected = [(uriref, RDF.type, DEB['UnversionedBinary']),\
(uriref, RDFS.label, Literal("Unversioned Binary: name", lang='en'))]
self.compareGeneratedTriples(expected)
def testTriplifyRepositoryFull(self):
r = GitRepository("http://example.com", "git://git.example.com")
bnode = self.t.triplifyRepository(r)
self.assertEqual(BNode, bnode.__class__)
self.assertEqual(5, len(self.graph))
expected = [(bnode, RDF.type, DOAP['GitRepository']),\
(bnode, RDFS.label, Literal("Repository: git://git.example.com", lang='en')),\
(bnode, DOAP['location'], URIRef("git://git.example.com")),\
(bnode, DOAP['browse'], URIRef("http://example.com")),\
(URIRef("http://example.com"), RDF.type, FOAF['page'])]
self.compareGeneratedTriples(expected)
def testTriplifyRepositoryNoBrowser(self):
r = GitRepository(None, "git://git.example.com")
bnode = self.t.triplifyRepository(r)
self.assertEqual(BNode, bnode.__class__)
self.assertEqual(3, len(self.graph))
expected = [(bnode, RDF.type, DOAP['GitRepository']),\
(bnode, RDFS.label, Literal("Repository: git://git.example.com", lang='en')),\
(bnode, DOAP['location'], URIRef("git://git.example.com"))]
self.compareGeneratedTriples(expected)
def testTriplifyRepositoryNoURI(self):
r = GitRepository("http://example.com", None)
bnode = self.t.triplifyRepository(r)
self.assertEqual(BNode, bnode.__class__)
self.assertEqual(4, len(self.graph))
expected = [(bnode, RDF.type, DOAP['GitRepository']),\
(bnode, RDFS.label, Literal("Repository", lang='en')),\
(bnode, DOAP['browse'], URIRef("http://example.com")),\
(URIRef("http://example.com"), RDF.type, FOAF['page'])]
self.compareGeneratedTriples(expected)
# Mocks
def mockTriplifyVersionNumber(self, version):
classMock = self.mox.CreateMock(Triplifier)
classMock.triplifyVersionNumber(version)\
.AndReturn(URIRef(version.asURI("b")))
self.mox.ReplayAll()
return classMock.triplifyVersionNumber
def mockTriplifyConstraint(self, constraints):
classMock = self.mox.CreateMock(Triplifier)
for constraint in constraints:
classMock.triplifyConstraint(constraint)\
.InAnyOrder()\
.AndReturn(URIRef(constraint.asURI("b")))
self.mox.ReplayAll()
return classMock.triplifyConstraint
def mockTriplifyArchitecture(self, arch):
classMock = self.mox.CreateMock(Triplifier)
classMock.triplifyArchitecture(arch)\
.AndReturn(URIRef(arch.asURI("b")))
self.mox.ReplayAll()
return classMock.triplifyArchitecture
def mockUnversionedBinaryPackage(self, ubinary):
classMock = self.mox.CreateMock(Triplifier)
classMock.triplifyUnversionedBinaryPackage(ubinary)\
.AndReturn(URIRef(ubinary.asURI("b")))
self.mox.ReplayAll()
return classMock.triplifyUnversionedBinaryPackage
