class ClientCustomPrefixes(AioSPARQLTestCase):
client_kwargs = {
"endpoint": "/sparql",
"update_endpoint": "/sparql-update",
"graph": IRI("http://mu.semte.ch/test-application"),
"prefixes": {
"foo": IRI("http://foo#"),
"bar": IRI("http://bar#"),
"baz": IRI("http://baz#"),
},
}
async def get_application(self):
app = web.Application()
app.router.add_post('/sparql', sparql_endpoint)
app.router.add_post('/sparql-update', sparql_endpoint)
return app
@unittest_run_loop
async def test_custom_prefixes(self):
res = await self.client.query("noop")
self.assertEqual(res['post']['query'], dedent("""\
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
PREFIX bar: <http://bar#>
PREFIX baz: <http://baz#>
PREFIX foo: <http://foo#>
noop"""))
class SwarmUI(Namespace):
__iri__ = IRI("http://swarmui.semte.ch/vocabularies/core/")
Down = PrefixedName
Error = PrefixedName
Initializing = PrefixedName
Killed = PrefixedName
Killing = PrefixedName
Pipeline = PrefixedName
Removed = PrefixedName
Removing = PrefixedName
Repository = PrefixedName
Restarting = PrefixedName
Scaling = PrefixedName
Service = PrefixedName
Started = PrefixedName
Starting = PrefixedName
Stopped = PrefixedName
Stopping = PrefixedName
Up = PrefixedName
Updating = PrefixedName
branch = PrefixedName
deleteRequested = PrefixedName
dockerComposeFile = PrefixedName
pipelines = PrefixedName
requestedScaling = PrefixedName
requestedStatus = PrefixedName
restartRequested = PrefixedName
scaling = PrefixedName
services = PrefixedName
status = PrefixedName
updateRequested = PrefixedName
def __init__(self, data):
assert isinstance(data, dict)
assert "s" in data
assert isinstance(data['s'], dict) and data['p']['type'] == "uri"
assert "p" in data
assert isinstance(data['p'], dict) and data['p']['type'] == "uri"
assert "o" in data
assert isinstance(data['o'], dict)
self.s = IRI(data['s']['value'])
self.p = IRI(data['p']['value'])
if data['o']['type'] == "uri":
self.o = IRI(data['o']['value'])
elif data['o']['type'] in ("literal", "typed-literal"):
self.o = Literal(data['o']['value'])
else:
raise NotImplementedError("object type %s" % data['o']['type'])
async def pollAccumulate(count=0, client=None):
"""
Poll the database until it is ready to answer queries
"""
if client is None:
client = SPARQLClient(ENV['MU_SPARQL_ENDPOINT'],
graph=IRI(ENV['MU_APPLICATION_GRAPH']))
if count >= int(ENV['POLL_RETRIES']):
await client.close()
return False
try:
result = await client.query("""
ASK
FROM {{graph}}
WHERE {
?s ?p ?o
}
""")
if not result:
logger.warn('SPARQL endpoint not yet ready')
sleep(randint(1, 5))
return await pollAccumulate(count + 1, client)
else:
logger.info('SPARQL endpoint is ready')
await client.close()
return True
except ClientConnectionError:
sleep(randint(1, 5))
return await pollAccumulate(count + 1, client)
def test_node_in_node(self):
node1 = Node(IRI("john"), [("foo", "bar")])
node2 = Node(IRI("jane"), [("foo", "bar")])
node3 = Node("parent", [("child1", node1), ("child2", node2),
("foo", "bar")])
self.assertEqual(
str(node3),
dedent("""\
parent child1 <john> ;
child2 <jane> ;
foo "bar" .
<jane> foo "bar" .
<john> foo "bar" ."""),
)
async def test_delete(self):
await self.client.delete()
self.assertEqual(self.app['last_request'].method, "DELETE")
self.assertEqual(self.app['last_request'].query_string,
"graph=%s" % self.client_kwargs['graph'].value)
await self.client.delete(IRI("foo"))
self.assertEqual(self.app['last_request'].query_string, "graph=foo")
class DockEventTypes(Namespace):
__iri__ = IRI("http://ontology.aksw.org/dockevent/types/")
container = PrefixedName
event = PrefixedName
network = PrefixedName
plugin = PrefixedName
volume = PrefixedName
class AioSPARQLTestCase(TestSPARQLClient):
client_kwargs = {
"endpoint": "/sparql",
"graph": IRI("http://mu.semte.ch/test-application"),
}
async def get_client(self, server):
return TestSPARQLClient(server, loop=self.loop, **self.client_kwargs)
class DockContainer(Namespace):
__iri__ = IRI("http://ontology.aksw.org/dockcontainer/")
env = PrefixedName
id = PrefixedName
label = PrefixedName
name = PrefixedName
network = PrefixedName
def test_iri(self):
self.assertEqual(str(IRI("http://example.org")),
"<http://example.org>")
self.assertEqual(IRI("http://example.org"), IRI("http://example.org"))
self.assertEqual(IRI("http://example.org"), "http://example.org")
self.assertEqual(
len(set([IRI("http://example.org"),
IRI("http://example.org")])), 1)
self.assertEqual(
IRI("http://example.org/") + "boo", IRI("http://example.org/boo"))
def sparql(self):
"""
The SPARQL client
"""
if not hasattr(self, '_sparql'):
self._sparql = SPARQLClient(ENV['MU_SPARQL_ENDPOINT'],
graph=IRI(ENV['MU_APPLICATION_GRAPH']),
loop=self.loop,
read_timeout=self.sparql_timeout)
return self._sparql
async def test_post(self):
await self.client.post(b"", format="some/format")
self.assertEqual(self.app['last_request'].method, "POST")
self.assertEqual(self.app['last_request'].query_string,
"graph=%s" % self.client_kwargs['graph'].value)
self.assertEqual(self.app['last_request'].headers['Content-Type'],
"some/format")
await self.client.post(b"", format="some/format", graph=IRI("foo"))
self.assertEqual(self.app['last_request'].query_string, "graph=foo")
async def test_delete(self):
await self.client.delete()
self.assertEqual(self.app["state"]["last_request"].method, "DELETE")
self.assertEqual(
self.app["state"]["last_request"].query_string,
"graph=%s" % self.client_kwargs["graph"].value,
)
await self.client.delete(IRI("foo"))
self.assertEqual(self.app["state"]["last_request"].query_string, "graph=foo")
class DockEventActor(Namespace):
__iri__ = IRI("http://ontology.aksw.org/dockeventactor/")
actorId = PrefixedName
image = PrefixedName
name = PrefixedName
nodeId = PrefixedName
nodeIp = PrefixedName
nodeIpPort = PrefixedName
nodeName = PrefixedName
async def test_get(self):
async with self.client.get(format="some/format") as response:
self.assertIsInstance(response, aiohttp.ClientResponse)
self.assertEqual(self.app["state"]["last_request"].method, "GET")
self.assertEqual(self.app["state"]["last_request"].query_string, "default")
self.assertEqual(self.app["state"]["last_request"].headers["Accept"], "some/format")
async with self.client.get(format="some/format", graph=IRI("foo")) as response:
self.assertIsInstance(response, aiohttp.ClientResponse)
self.assertEqual(self.app["state"]["last_request"].method, "GET")
self.assertEqual(self.app["state"]["last_request"].query_string, "graph=foo")
self.assertEqual(self.app["state"]["last_request"].headers["Accept"], "some/format")
async def test_put(self):
await self.client.put(b"", format="some/format")
self.assertEqual(self.app["state"]["last_request"].method, "PUT")
self.assertEqual(
self.app["state"]["last_request"].query_string,
"graph=%s" % self.client_kwargs["graph"].value,
)
self.assertEqual(
self.app["state"]["last_request"].headers["Content-Type"], "some/format"
)
await self.client.put(b"", format="some/format", graph=IRI("foo"))
self.assertEqual(self.app["state"]["last_request"].query_string, "graph=foo")
def virtuoso_client(loop):
_virtuoso_client = SPARQLClient(
"http://localhost:8890/sparql",
update_endpoint=ENV.get("SPARQL_UPDATE_ENDPOINT"),
crud_endpoint="http://localhost:8890/sparql-graph-crud",
graph=IRI("http://aiosparql.org/%s" % uuid.uuid4().hex[:7]))
yield _virtuoso_client
try:
loop.run_until_complete(_virtuoso_client.delete())
except aiohttp.ClientResponseError as exc:
if exc.status != 404:
raise
loop.run_until_complete(_virtuoso_client.close())
async def create_container_log_concept(sparql, container):
"""
Create a container log concept, this is the node that will group all the
log lines together
"""
concept = IRI("docklogs:%s" % container['Id'])
resp = await sparql.query("ASK FROM {{graph}} WHERE { {{}} ?p ?o }",
concept)
if not resp['boolean']:
resp = await sparql.update(
"WITH {{graph}} INSERT DATA { {{}} dct:title {{}} }",
concept, escape_string(container['Name'][1:]))
logger.info("Created logging concept: %s", concept)
return concept
def test_prefixed_name(self):
self.assertEqual(PrefixedName(IRI("foo"), "bar", "baz"), IRI("foobaz"))
self.assertEqual(PrefixedName(IRI("foo"), "bar", "baz"),
PrefixedName(IRI("foo"), "bar", "baz"))
self.assertEqual(
len(
set([
PrefixedName(IRI("foo"), "bar", "baz"),
PrefixedName(IRI("foo"), "bar", "baz")
])), 1)
mapping = {"foobaz": "ok"}
self.assertEqual(
mapping.get(PrefixedName(IRI("foo"), "bar", "baz"), "notok"), "ok")
class SwarmUI(Namespace):
__iri__ = IRI("http://swarmui.semte.ch/vocabularies/core/")
PidsStats = PrefixedName
SectorsRecursive = PrefixedName
CpuUsage = PrefixedName
ThrottlingData = PrefixedName
CpuStats = PrefixedName
CpuUsage = PrefixedName
ThrottlingData = PrefixedName
PrecpuStats = PrefixedName
Stats = PrefixedName
MemoryStats = PrefixedName
Network = PrefixedName
Stats = PrefixedName
def setUp(self):
self.loop = setup_test_loop()
self.db = SPARQLClient(endpoint="http://database:8890/sparql",
graph=IRI(ENV['MU_APPLICATION_GRAPH']),
loop=self.loop,
read_timeout=self.sparql_timeout)
self.loop.run_until_complete(self.prepare_database())
self.app = self.loop.run_until_complete(self.get_application())
self.server = FixedPortTestServer(self.app)
self.client = self.loop.run_until_complete(
self._get_client(self.server))
self.loop.run_until_complete(self.client.start_server())
async def test_get(self):
async with self.client.get(format="some/format") as response:
self.assertIsInstance(response, aiohttp.ClientResponse)
self.assertEqual(self.app['last_request'].method, "GET")
self.assertEqual(self.app['last_request'].query_string,
"graph=%s" % self.client_kwargs['graph'].value)
self.assertEqual(self.app['last_request'].headers['Accept'],
"some/format")
async with self.client.get(format="some/format", graph=IRI("foo")) \
as response:
self.assertIsInstance(response, aiohttp.ClientResponse)
self.assertEqual(self.app['last_request'].method, "GET")
self.assertEqual(self.app['last_request'].query_string, "graph=foo")
self.assertEqual(self.app['last_request'].headers['Accept'],
"some/format")
class DockEventActions(Namespace):
__iri__ = IRI("http://ontology.aksw.org/dockevent/actions/")
attach = PrefixedName
connect = PrefixedName
create = PrefixedName
destroy = PrefixedName
die = PrefixedName
exec_create = PrefixedName
exec_start = PrefixedName
health_status = PrefixedName
resize = PrefixedName
start = PrefixedName
stop = PrefixedName
kill = PrefixedName
restart = PrefixedName
async def get_services(self, project_name):
result = await self.app.sparql.query(
"""
SELECT ?name ?service ?uuid
FROM {{graph}}
WHERE {
?pipeline mu:uuid {{}} ;
swarmui:services ?service .
?service mu:uuid ?uuid ;
dct:title ?name .
}
""", escape_any(project_name))
return {
x['name']['value']:
(IRI(x['service']['value']), x['uuid']['value'])
for x in result['results']['bindings']
}
class DockEvent(Namespace):
__iri__ = IRI("http://ontology.aksw.org/dockevent/")
action = PrefixedName
actionExtra = PrefixedName
actor = PrefixedName
actorId = PrefixedName
container = PrefixedName
dateTime = PrefixedName
eventId = PrefixedName
image = PrefixedName
link = PrefixedName
name = PrefixedName
nodeId = PrefixedName
nodeIp = PrefixedName
nodeIpPort = PrefixedName
nodeName = PrefixedName
source = PrefixedName
time = PrefixedName
timeNano = PrefixedName
type = PrefixedName
async def create_drc_node(self,
repository_iri=_sentinel,
location=_sentinel):
if repository_iri is _sentinel:
repository_iri, repository_id = \
await self.create_repository(location=location)
else:
s_repository_iri = str(repository_iri)
repository_id = s_repository_iri.split('/')[-1][:-1]
drc_text = dedent("""\
version: "2"
services:
service1:
image: busybox
command: "sleep 60"
service2:
image: busybox
command: "sleep 60"
""")
drc_id = self.uuid4()
d_iri = IRI("http://stack-builder.big-data-europe.eu/resources/")
drc_iri = d_iri + "%s/%s" % ("docker-composes", drc_id)
drc_title = "stack_{}_drc_{}".format(repository_id, drc_id)
drc_node = Node(
drc_iri, {
RDF.type: Stackbuilder.DockerCompose,
Mu.uuid: drc_id,
Dct.title: drc_title,
Stackbuilder.text: drc_text
})
await self.insert_triples([
drc_node,
(repository_iri, SwarmUI.dockerComposeFile, drc_node),
])
return (drc_iri, drc_id)
class Client(AioSPARQLTestCase):
client_kwargs = {
"endpoint": "/sparql",
"update_endpoint": "/sparql-update",
"crud_endpoint": "/crud",
"graph": IRI("http://mu.semte.ch/test-application"),
}
async def get_application(self):
app = web.Application()
app.router.add_post('/sparql', sparql_endpoint)
app.router.add_post('/sparql-update', sparql_endpoint)
app.router.add_route('*', '/crud', crud_endpoint)
return app
@unittest_run_loop
async def test_query(self):
triples = Triples([("john", RDF.type, "doe"), ("john", "p", "o")])
res = await self.client.query("""
SELECT *
FROM {{graph}}
WHERE {
{{}}
}
""", triples)
self.assertEqual(res['path'], self.client_kwargs['endpoint'])
self.assertIn('post', res)
self.assertIn('query', res['post'])
self.assertEqual(res['post']['query'], dedent("""\
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
SELECT *
FROM <http://mu.semte.ch/test-application>
WHERE {
john rdf:type "doe" ;
p "o" .
}"""))
with self.assertRaises(SPARQLRequestFailed):
await self.client.query("failure")
@unittest_run_loop
async def test_update(self):
triples = Triples([("john", RDF.type, "doe"), ("john", "p", "o")])
res = await self.client.update("""
WITH {{graph}}
INSERT DATA {
{{}}
}
""", triples)
self.assertEqual(res['path'], self.client_kwargs['update_endpoint'])
self.assertIn('post', res)
self.assertIn('update', res['post'])
self.assertEqual(res['post']['update'], dedent("""\
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
WITH <http://mu.semte.ch/test-application>
INSERT DATA {
john rdf:type "doe" ;
p "o" .
}"""))
with self.assertRaises(SPARQLRequestFailed):
await self.client.update("failure")
@unittest_run_loop
async def test_get(self):
async with self.client.get(format="some/format") as response:
self.assertIsInstance(response, aiohttp.ClientResponse)
self.assertEqual(self.app['last_request'].method, "GET")
self.assertEqual(self.app['last_request'].query_string,
"graph=%s" % self.client_kwargs['graph'].value)
self.assertEqual(self.app['last_request'].headers['Accept'],
"some/format")
async with self.client.get(format="some/format", graph=IRI("foo")) \
as response:
self.assertIsInstance(response, aiohttp.ClientResponse)
self.assertEqual(self.app['last_request'].method, "GET")
self.assertEqual(self.app['last_request'].query_string, "graph=foo")
self.assertEqual(self.app['last_request'].headers['Accept'],
"some/format")
@unittest_run_loop
async def test_put(self):
await self.client.put(b"", format="some/format")
self.assertEqual(self.app['last_request'].method, "PUT")
self.assertEqual(self.app['last_request'].query_string,
"graph=%s" % self.client_kwargs['graph'].value)
self.assertEqual(self.app['last_request'].headers['Content-Type'],
"some/format")
await self.client.put(b"", format="some/format", graph=IRI("foo"))
self.assertEqual(self.app['last_request'].query_string, "graph=foo")
@unittest_run_loop
async def test_delete(self):
await self.client.delete()
self.assertEqual(self.app['last_request'].method, "DELETE")
self.assertEqual(self.app['last_request'].query_string,
"graph=%s" % self.client_kwargs['graph'].value)
await self.client.delete(IRI("foo"))
self.assertEqual(self.app['last_request'].query_string, "graph=foo")
@unittest_run_loop
async def test_post(self):
await self.client.post(b"", format="some/format")
self.assertEqual(self.app['last_request'].method, "POST")
self.assertEqual(self.app['last_request'].query_string,
"graph=%s" % self.client_kwargs['graph'].value)
self.assertEqual(self.app['last_request'].headers['Content-Type'],
"some/format")
await self.client.post(b"", format="some/format", graph=IRI("foo"))
self.assertEqual(self.app['last_request'].query_string, "graph=foo")
def _generate_random_graph(self):
return IRI("http://aiosparql.org/%s" % uuid.uuid4().hex[:7])
async def get_sparql_client():
sparql = SPARQLClient(ENV['MU_SPARQL_ENDPOINT'],
graph=IRI(ENV['MU_APPLICATION_GRAPH']))
yield sparql
await sparql.close()
class Stackbuilder(Namespace):
__iri__ = IRI("http://stackbuilder.semte.ch/vocabularies/core/")
text = PrefixedName
DockerCompose = PrefixedName
