async def start_server(self):
await self._server.start_server(loop=self._loop)
kwargs = dict(self._client_kwargs)
if kwargs.get('endpoint'):
kwargs['endpoint'] = self.make_url(kwargs['endpoint'])
if kwargs.get('update_endpoint'):
kwargs['update_endpoint'] = \
self.make_url(kwargs['update_endpoint'])
if kwargs.get('crud_endpoint'):
kwargs['crud_endpoint'] = self.make_url(kwargs['crud_endpoint'])
self._session = SPARQLClient(loop=self._loop, **kwargs)
def virtuoso_client(loop):
_virtuoso_client = SPARQLClient(
"http://localhost:8890/sparql",
update_endpoint=ENV.get("SPARQL_UPDATE_ENDPOINT"),
crud_endpoint="http://localhost:8890/sparql-graph-crud",
graph=IRI("http://aiosparql.org/%s" % uuid.uuid4().hex[:7]))
yield _virtuoso_client
try:
loop.run_until_complete(_virtuoso_client.delete())
except aiohttp.ClientResponseError as exc:
if exc.status != 404:
raise
loop.run_until_complete(_virtuoso_client.close())
async def test_client_context(loop):
async with SPARQLClient(endpoint="http://example.org",
graph="http://example/graph") as client:
assert not client.session.closed
assert not client.closed
assert client.session.closed
assert client.closed
def setUp(self):
self.loop = setup_test_loop()
self.db = SPARQLClient(endpoint="http://database:8890/sparql",
graph=IRI(ENV['MU_APPLICATION_GRAPH']),
loop=self.loop,
read_timeout=self.sparql_timeout)
self.loop.run_until_complete(self.prepare_database())
self.app = self.loop.run_until_complete(self.get_application())
self.server = FixedPortTestServer(self.app)
self.client = self.loop.run_until_complete(
self._get_client(self.server))
self.loop.run_until_complete(self.client.start_server())
async def pollAccumulate(count=0, client=None):
"""
Poll the database until it is ready to answer queries
"""
if client is None:
client = SPARQLClient(ENV['MU_SPARQL_ENDPOINT'],
graph=IRI(ENV['MU_APPLICATION_GRAPH']))
if count >= int(ENV['POLL_RETRIES']):
await client.close()
return False
try:
result = await client.query("""
ASK
FROM {{graph}}
WHERE {
?s ?p ?o
}
""")
if not result:
logger.warn('SPARQL endpoint not yet ready')
sleep(randint(1, 5))
return await pollAccumulate(count + 1, client)
else:
logger.info('SPARQL endpoint is ready')
await client.close()
return True
except ClientConnectionError:
sleep(randint(1, 5))
return await pollAccumulate(count + 1, client)
async def _create_client(self):
return SPARQLClient(ENV.get("SPARQL_ENDPOINT",
"http://localhost:8890/sparql"),
update_endpoint=ENV.get("SPARQL_UPDATE_ENDPOINT"),
crud_endpoint=ENV.get(
"SPARQL_UPDATE_ENDPOINT",
"http://localhost:8890/sparql-graph-crud"),
graph=self.graph)
def sparql(self):
"""
The SPARQL client
"""
if not hasattr(self, '_sparql'):
self._sparql = SPARQLClient(ENV['MU_SPARQL_ENDPOINT'],
graph=IRI(ENV['MU_APPLICATION_GRAPH']),
loop=self.loop,
read_timeout=self.sparql_timeout)
return self._sparql
async def start_server(self):
await self._server.start_server(loop=self._loop)
kwargs = dict(self._client_kwargs)
if kwargs.get("endpoint"):
kwargs["endpoint"] = self.make_url(kwargs["endpoint"])
if kwargs.get("update_endpoint"):
kwargs["update_endpoint"] = self.make_url(
kwargs["update_endpoint"])
if kwargs.get("crud_endpoint"):
kwargs["crud_endpoint"] = self.make_url(kwargs["crud_endpoint"])
self._session = SPARQLClient(loop=self._loop, **kwargs)
async def run_loop(sparql_endpoint=None, debug=False):
sparql_context = SPARQLClient(sparql_endpoint)
docker_args = kwargs_from_env()
docker_context = APIClient(timeout=5, **docker_args)
with sparql_context as sparql, docker_context as docker:
await run_on_startup_subroutines(docker, sparql)
async for x in docker.events(decode=True):
try:
event = new_event(docker, x)
await asyncio.gather(
*(handler(event, sparql)
for handler in list_handlers(event, reload=debug)))
except Exception:
logger.exception(
"An error occurred during a coroutine execution. "
"The loop will not be interrupted.")
def setUp(self):
self.loop = setup_test_loop()
self.db = SPARQLClient(endpoint="http://database:8890/sparql",
graph=IRI(ENV['MU_APPLICATION_GRAPH']),
loop=self.loop,
read_timeout=self.sparql_timeout)
self.loop.run_until_complete(self.prepare_database())
self.app = self.loop.run_until_complete(self.get_application())
self.server = FixedPortTestServer(self.app)
self.client = self.loop.run_until_complete(
self._get_client(self.server))
self.loop.run_until_complete(self.client.start_server())
async def async_execute_query(self, query, post = False):
"""
Always returns JSON response.
"""
async with SPARQLClient(self.host_name) as client:
complete = False
tries = 0
maxtries = 10
while not complete and tries < maxtries:
try:
result = await client.query(query)
complete = True
except:
time.sleep(0.5)
tries += 1
return result
async def sparql_client(sparql_endpoint: str):
_sparql_client = SPARQLClient(sparql_endpoint)
yield _sparql_client
await _sparql_client.close()
async def get_sparql_client():
sparql = SPARQLClient(ENV['MU_SPARQL_ENDPOINT'],
graph=IRI(ENV['MU_APPLICATION_GRAPH']))
yield sparql
await sparql.close()
async def jena_client(loop):
_jena_client = SPARQLClient("http://localhost:3030/ds")
yield _jena_client
await _jena_client.close()
class IntegrationTestCase(AioHTTPTestCase):
example_repo = \
"https://github.com/big-data-europe/mu-swarm-ui-testing.git"
sparql_timeout = 5
async def get_application(self):
app = copy(muswarmadmin.main.app)
app.sparql_timeout = self.sparql_timeout
return app
async def scheduler_complete(self, key):
if key not in ActionScheduler.executers:
raise KeyError(
"ActionScheduler for key %s does not exist. "
"HINT: the ActionScheduler is removed automatically after "
"calling this function" % key)
await ActionScheduler.executers[key].cancel()
async def wait_scheduler(self, key, timeout=3):
for i in range(timeout * 5):
if key in ActionScheduler.executers:
break
await asyncio.sleep(0.2)
await self.scheduler_complete(key)
def uuid4(self):
return str(uuid.uuid4()).replace("-", "").upper()
def resource(self, type_, id):
return (
muswarmadmin.main.Application.base_resource + "%s/%s" % (type_, id)
)
def project_exists(self, project_name):
return os.path.exists("/data/%s" % project_name)
async def triple_exists(self, s=None, p=None, o=None):
s = escape_any(s) if s is not None else "?s"
p = escape_any(p) if p is not None else "?p"
o = escape_any(o) if o is not None else "?o"
result = await self.app.sparql.query(
"ASK FROM {{graph}} WHERE { {{}} {{}} {{}} }", s, p, o)
return result['boolean']
async def prepare_triples(self, triples):
await self.db.update(
"INSERT DATA { GRAPH {{graph}} { {{}} } }", Triples(triples))
async def insert_triples(self, triples):
await self.app.sparql.update(
"INSERT DATA { GRAPH {{graph}} { {{}} } }", Triples(triples))
async def prepare_node(self, node):
await self.prepare_triples([node])
async def insert_node(self, node):
await self.insert_triples([node])
async def remove_triples(self, s=None, p=None, o=None):
s = escape_any(s) if s is not None else "?s"
p = escape_any(p) if p is not None else "?p"
o = escape_any(o) if o is not None else "?o"
await self.app.sparql.update(
"""
WITH {{graph}}
DELETE {
{{s}} {{p}} {{o}}
}
WHERE {
{{s}} {{p}} {{o}}
}""", s=s, p=p, o=o)
async def describe(self, subject):
return await self.app.sparql.query("DESCRIBE {{}} FROM {{graph}}",
subject)
async def create_drc_node(self, repository_iri=_sentinel,
location=_sentinel):
if repository_iri is _sentinel:
repository_iri, repository_id = \
await self.create_repository(location=location)
else:
s_repository_iri = str(repository_iri)
repository_id = s_repository_iri.split('/')[-1][:-1]
drc_text = dedent("""\
version: "2"
services:
service1:
image: busybox
command: "sleep 60"
service2:
image: busybox
command: "sleep 60"
""")
drc_id = self.uuid4()
d_iri = IRI("http://stack-builder.big-data-europe.eu/resources/")
drc_iri = d_iri + "%s/%s" % ("docker-composes", drc_id)
drc_title = "stack_{}_drc_{}".format(repository_id, drc_id)
drc_node = Node(drc_iri, {
RDF.type: Stackbuilder.DockerCompose,
Mu.uuid: drc_id,
Dct.title: drc_title,
Stackbuilder.text: drc_text
})
await self.insert_triples([
drc_node,
(repository_iri, SwarmUI.dockerComposeFile, drc_node),
])
return (drc_iri, drc_id)
async def create_repository(self, location=_sentinel):
if location is _sentinel:
location = self.example_repo
repository_id = self.uuid4()
repository_iri = self.resource("stacks", repository_id)
await self.insert_node(Node(repository_iri, {
RDF.type: Doap.Stack,
Mu.uuid: repository_id,
Doap.location: location,
}))
return (repository_iri, repository_id)
async def create_pipeline(self, repository_iri=_sentinel,
location=_sentinel):
if repository_iri is _sentinel:
repository_iri, repository_id = \
await self.create_repository(location=location)
pipeline_id = self.uuid4()
pipeline_iri = self.resource("pipeline-instances", pipeline_id)
pipeline_node = Node(pipeline_iri, {
RDF.type: SwarmUI.Pipeline,
Mu.uuid: pipeline_id,
})
await self.insert_triples([
pipeline_node,
(repository_iri, SwarmUI.pipelines, pipeline_node),
])
await self.scheduler_complete(pipeline_id)
return (pipeline_iri, pipeline_id)
async def get_services(self, project_name):
result = await self.app.sparql.query(
"""
SELECT ?name ?service ?uuid
FROM {{graph}}
WHERE {
?pipeline mu:uuid {{}} ;
swarmui:services ?service .
?service mu:uuid ?uuid ;
dct:title ?name .
}
""", escape_any(project_name))
return {
x['name']['value']: (IRI(x['service']['value']),
x['uuid']['value'])
for x in result['results']['bindings']
}
async def prepare_database(self):
await self.db.update("CLEAR GRAPH {{graph}}")
def setUp(self):
self.loop = setup_test_loop()
self.db = SPARQLClient(endpoint="http://database:8890/sparql",
graph=IRI(ENV['MU_APPLICATION_GRAPH']),
loop=self.loop,
read_timeout=self.sparql_timeout)
self.loop.run_until_complete(self.prepare_database())
self.app = self.loop.run_until_complete(self.get_application())
self.server = FixedPortTestServer(self.app)
self.client = self.loop.run_until_complete(
self._get_client(self.server))
self.loop.run_until_complete(self.client.start_server())
def tearDown(self):
self.loop.run_until_complete(self.db.close())
super().tearDown()
for project_name in os.listdir("/data"):
project_path = "/data/%s" % project_name
subprocess.call(["docker-compose", "down"], cwd=project_path)
shutil.rmtree(project_path)
# NOTE: temporary fix, will be fixed with the next aiohttp release
@asyncio.coroutine
def _get_client(self, app):
"""Return a TestClient instance."""
return TestClient(app, loop=self.loop)
async def assertNode(self, subject, values):
result = await self.describe(subject)
self.assertTrue(result and result[subject])
for p, o in values.items():
found_values = [x['value'] for x in result[subject][p]]
self.assertEqual(
len(found_values), 1,
"multiple predicates {} in node's subject {}: {!r}".format(
p, subject, found_values))
self.assertEqual(
found_values[0], o,
"predicate {} in node {} has value {}, expected {}".format(
p, subject, found_values[0], o))
async def assertStatus(self, subject, status):
await self.assertNode(subject, {SwarmUI.status: status})
async def assertExists(self, s=None, p=None, o=None):
self.assertTrue(await self.triple_exists(s, p, o))
async def assertNotExists(self, s=None, p=None, o=None):
self.assertFalse(await self.triple_exists(s, p, o))
async def jena_client(jena_endpoint):
_jena_client = SPARQLClient(jena_endpoint)
yield _jena_client
await _jena_client.close()
class IntegrationTestCase(AioHTTPTestCase):
example_repo = \
"https://github.com/big-data-europe/mu-swarm-ui-testing.git"
sparql_timeout = 5
async def get_application(self):
app = copy(muswarmadmin.main.app)
app.sparql_timeout = self.sparql_timeout
return app
async def scheduler_complete(self, key):
if key not in ActionScheduler.executers:
raise KeyError(
"ActionScheduler for key %s does not exist. "
"HINT: the ActionScheduler is removed automatically after "
"calling this function" % key)
await ActionScheduler.executers[key].cancel()
async def wait_scheduler(self, key, timeout=3):
for i in range(timeout * 5):
if key in ActionScheduler.executers:
break
await asyncio.sleep(0.2)
await self.scheduler_complete(key)
def uuid4(self):
return str(uuid.uuid4()).replace("-", "").upper()
def resource(self, type_, id):
return (muswarmadmin.main.Application.base_resource + "%s/%s" %
(type_, id))
def project_exists(self, project_name):
return os.path.exists("/data/%s" % project_name)
async def triple_exists(self, s=None, p=None, o=None):
s = escape_any(s) if s is not None else "?s"
p = escape_any(p) if p is not None else "?p"
o = escape_any(o) if o is not None else "?o"
result = await self.app.sparql.query(
"ASK FROM {{graph}} WHERE { {{}} {{}} {{}} }", s, p, o)
return result['boolean']
async def prepare_triples(self, triples):
await self.db.update("INSERT DATA { GRAPH {{graph}} { {{}} } }",
Triples(triples))
async def insert_triples(self, triples):
await self.app.sparql.update(
"INSERT DATA { GRAPH {{graph}} { {{}} } }", Triples(triples))
async def prepare_node(self, node):
await self.prepare_triples([node])
async def insert_node(self, node):
await self.insert_triples([node])
async def remove_triples(self, s=None, p=None, o=None):
s = escape_any(s) if s is not None else "?s"
p = escape_any(p) if p is not None else "?p"
o = escape_any(o) if o is not None else "?o"
await self.app.sparql.update("""
WITH {{graph}}
DELETE {
{{s}} {{p}} {{o}}
}
WHERE {
{{s}} {{p}} {{o}}
}""",
s=s,
p=p,
o=o)
async def describe(self, subject):
return await self.app.sparql.query("DESCRIBE {{}} FROM {{graph}}",
subject)
async def create_drc_node(self,
repository_iri=_sentinel,
location=_sentinel):
if repository_iri is _sentinel:
repository_iri, repository_id = \
await self.create_repository(location=location)
else:
s_repository_iri = str(repository_iri)
repository_id = s_repository_iri.split('/')[-1][:-1]
drc_text = dedent("""\
version: "2"
services:
service1:
image: busybox
command: "sleep 60"
service2:
image: busybox
command: "sleep 60"
""")
drc_id = self.uuid4()
d_iri = IRI("http://stack-builder.big-data-europe.eu/resources/")
drc_iri = d_iri + "%s/%s" % ("docker-composes", drc_id)
drc_title = "stack_{}_drc_{}".format(repository_id, drc_id)
drc_node = Node(
drc_iri, {
RDF.type: Stackbuilder.DockerCompose,
Mu.uuid: drc_id,
Dct.title: drc_title,
Stackbuilder.text: drc_text
})
await self.insert_triples([
drc_node,
(repository_iri, SwarmUI.dockerComposeFile, drc_node),
])
return (drc_iri, drc_id)
async def create_repository(self, location=_sentinel):
if location is _sentinel:
location = self.example_repo
repository_id = self.uuid4()
repository_iri = self.resource("stacks", repository_id)
await self.insert_node(
Node(
repository_iri, {
RDF.type: Doap.Stack,
Mu.uuid: repository_id,
Doap.location: location,
}))
return (repository_iri, repository_id)
async def create_pipeline(self,
repository_iri=_sentinel,
location=_sentinel):
if repository_iri is _sentinel:
repository_iri, repository_id = \
await self.create_repository(location=location)
pipeline_id = self.uuid4()
pipeline_iri = self.resource("pipeline-instances", pipeline_id)
pipeline_node = Node(pipeline_iri, {
RDF.type: SwarmUI.Pipeline,
Mu.uuid: pipeline_id,
})
await self.insert_triples([
pipeline_node,
(repository_iri, SwarmUI.pipelines, pipeline_node),
])
await self.scheduler_complete(pipeline_id)
return (pipeline_iri, pipeline_id)
async def get_services(self, project_name):
result = await self.app.sparql.query(
"""
SELECT ?name ?service ?uuid
FROM {{graph}}
WHERE {
?pipeline mu:uuid {{}} ;
swarmui:services ?service .
?service mu:uuid ?uuid ;
dct:title ?name .
}
""", escape_any(project_name))
return {
x['name']['value']:
(IRI(x['service']['value']), x['uuid']['value'])
for x in result['results']['bindings']
}
async def prepare_database(self):
await self.db.update("CLEAR GRAPH {{graph}}")
def setUp(self):
self.loop = setup_test_loop()
self.db = SPARQLClient(endpoint="http://database:8890/sparql",
graph=IRI(ENV['MU_APPLICATION_GRAPH']),
loop=self.loop,
read_timeout=self.sparql_timeout)
self.loop.run_until_complete(self.prepare_database())
self.app = self.loop.run_until_complete(self.get_application())
self.server = FixedPortTestServer(self.app)
self.client = self.loop.run_until_complete(
self._get_client(self.server))
self.loop.run_until_complete(self.client.start_server())
def tearDown(self):
self.loop.run_until_complete(self.db.close())
super().tearDown()
for project_name in os.listdir("/data"):
project_path = "/data/%s" % project_name
subprocess.call(["docker-compose", "down"], cwd=project_path)
shutil.rmtree(project_path)
# NOTE: temporary fix, will be fixed with the next aiohttp release
@asyncio.coroutine
def _get_client(self, app):
"""Return a TestClient instance."""
return TestClient(app, loop=self.loop)
async def assertNode(self, subject, values):
result = await self.describe(subject)
self.assertTrue(result and result[subject])
for p, o in values.items():
found_values = [x['value'] for x in result[subject][p]]
self.assertEqual(
len(found_values), 1,
"multiple predicates {} in node's subject {}: {!r}".format(
p, subject, found_values))
self.assertEqual(
found_values[0], o,
"predicate {} in node {} has value {}, expected {}".format(
p, subject, found_values[0], o))
async def assertStatus(self, subject, status):
await self.assertNode(subject, {SwarmUI.status: status})
async def assertExists(self, s=None, p=None, o=None):
self.assertTrue(await self.triple_exists(s, p, o))
async def assertNotExists(self, s=None, p=None, o=None):
self.assertFalse(await self.triple_exists(s, p, o))
def init_sparql(self, sparql_url, *args, **kwargs):
self.sparql = SPARQLClient(sparql_url)
async def async_get_results(endpoint_url, query):
sparql = SPARQLClient(endpoint_url)
result = await sparql.query(query)
await sparql.close()
return result
def jena_client(loop):
_jena_client = SPARQLClient("http://localhost:3030/ds")
yield _jena_client
loop.run_until_complete(_jena_client.close())
