def test_cache(self):
token1 = yield oauth2.get_token('https://localhost:8007',
'4225f4774d6874a68565a04130001144',
'FMjU7vNIay5HGNABQVTTghOfEJqbet')
assert token1 == self.token1
token2 = yield oauth2.get_token('https://localhost:8007',
'4225f4774d6874a68565a04130001144',
'FMjU7vNIay5HGNABQVTTghOfEJqbet')
assert token2 == token1, 'Should receive a cached token'
def test_cache(self):
token1 = yield oauth2.get_token('https://localhost:8007',
'4225f4774d6874a68565a04130001144',
'FMjU7vNIay5HGNABQVTTghOfEJqbet')
assert token1 == self.token1
token2 = yield oauth2.get_token('https://localhost:8007',
'4225f4774d6874a68565a04130001144',
'FMjU7vNIay5HGNABQVTTghOfEJqbet')
assert token2 == token1, 'Should receive a cached token'
def test_expired_token(self):
self.expiry = self.expiry - oauth2.RequestToken.max_until_expired * 3
token1 = yield oauth2.get_token('https://localhost:8007',
'4225f4774d6874a68565a04130001144',
'FMjU7vNIay5HGNABQVTTghOfEJqbet')
assert token1 == self.token1
token2 = yield oauth2.get_token('https://localhost:8007',
'4225f4774d6874a68565a04130001144',
'FMjU7vNIay5HGNABQVTTghOfEJqbet')
assert token2 == self.token2, 'Should not use an expired cached token'
def test_expired_token(self):
self.expiry = self.expiry - oauth2.RequestToken.max_until_expired * 3
token1 = yield oauth2.get_token('https://localhost:8007',
'4225f4774d6874a68565a04130001144',
'FMjU7vNIay5HGNABQVTTghOfEJqbet')
assert token1 == self.token1
token2 = yield oauth2.get_token('https://localhost:8007',
'4225f4774d6874a68565a04130001144',
'FMjU7vNIay5HGNABQVTTghOfEJqbet')
assert token2 == self.token2, 'Should not use an expired cached token'
def exchange_delegate_token(token, repository_id):
"""
Exchange a token for a delegated token
:param token: a JWT granting the onboarding service access to write on the
client's behalf
:param repository_id: the target repsitory's ID
:returns: a new JWT authorized to write to the repository
:raises: HTTPError
"""
try:
new_token = yield oauth2.get_token(
options.url_auth,
options.service_id,
options.client_secret,
scope=oauth2.Write(repository_id),
jwt=token,
ssl_options=ssl_server_options()
)
except httpclient.HTTPError as exc:
if exc.code in (403, 400):
try:
body = json.loads(exc.response.body)
errors = [x['message'] for x in body['errors']]
except (AttributeError, KeyError):
errors = exc.message
raise exceptions.HTTPError(403, errors, source='authentication')
else:
msg = 'Error authorizing access to the repository'
logging.exception(msg)
raise exceptions.HTTPError(500, msg)
raise Return(new_token)
def exchange_delegate_token(token, repository_id):
"""
Exchange a token for a delegated token
:param token: a JWT granting the onboarding service access to write on the
client's behalf
:param repository_id: the target repsitory's ID
:returns: a new JWT authorized to write to the repository
:raises: HTTPError
"""
try:
new_token = yield oauth2.get_token(options.url_auth,
options.service_id,
options.client_secret,
scope=oauth2.Write(repository_id),
jwt=token,
ssl_options=ssl_server_options())
except httpclient.HTTPError as exc:
if exc.code in (403, 400):
try:
body = json.loads(exc.response.body)
errors = [x['message'] for x in body['errors']]
except (AttributeError, KeyError):
errors = exc.message
raise exceptions.HTTPError(403, errors, source='authentication')
else:
msg = 'Error authorizing access to the repository'
logging.exception(msg)
raise exceptions.HTTPError(500, msg)
raise Return(new_token)
def send_notification(repository, **kwargs):
"""
Send a fire-and-forget notification to the index service
NOTE: all exceptions are unhandled. It's assumed that the function is used
as a callback outside of the request's context using IOLoop.spawn_callback
(see: http://www.tornadoweb.org/en/stable/ioloop.html)
"""
headers = {
'Accept': 'application/json',
'Content-Type': 'application/json'
}
try:
token = yield get_token(options.url_auth,
options.service_id,
options.client_secret,
scope=Write(options.url_index),
ssl_options=ssl_server_options())
client = API(options.url_index,
token=token,
ssl_options=ssl_server_options())
client.index.notifications.prepare_request(
request_timeout=options.request_timeout,
headers=headers,
body=json.dumps({'id': repository.repository_id}))
logging.debug('calling send_notification ' +
str(repository.repository_id))
yield client.index.notifications.post()
except Exception as e:
logging.exception("failed to notify index: " + e.message)
def test_get_jwt_token(self):
token = yield oauth2.get_token('https://localhost:8007',
'4225f4774d6874a68565a04130001144',
'FMjU7vNIay5HGNABQVTTghOfEJqbet',
jwt='the client jwt')
assert token == self.token
body = self.API().auth.token.post.call_args[1]['body']
assert urlparse.parse_qs(body) == {'grant_type': [oauth2.JWT_BEARER],
'assertion': ['the client jwt']}
def test_get_token_with_scope_string(self):
token = yield oauth2.get_token('https://localhost:8007',
'4225f4774d6874a68565a04130001144',
'FMjU7vNIay5HGNABQVTTghOfEJqbet',
scope='read')
assert token == self.token
body = self.API().auth.token.post.call_args[1]['body']
assert urlparse.parse_qs(body) == {
'grant_type': [oauth2.CLIENT_CREDENTIALS],
'scope': ['read']}
def repository_service_client(location):
"""
get an api client for a repository service
:params location: base url of the repository
"""
token = yield get_token(options.url_auth,
options.service_id,
options.client_secret,
scope=Read(),
ssl_options=ssl_server_options())
client = API(location, token=token, ssl_options=ssl_server_options())
raise Return(client)
def repository_service_client(location):
"""
get an api client for a repository service
:params location: base url of the repository
"""
token = yield get_token(
options.url_auth, options.service_id,
options.client_secret, scope=Read(),
ssl_options=ssl_server_options()
)
client = API(location, token=token, ssl_options=ssl_server_options())
raise Return(client)
def transform(data, content_type, r2rml_url):
"""
Transforms source data into RDF triples
:param data: the source data
:param content_type: the http request content type
:param r2rml_url: karma mapping file url
:return: Transformed data and errors
"""
logging.debug('>>> transform')
response = None
http_status = 200
errors = []
try:
token = yield oauth2.get_token(
options.url_auth,
options.service_id,
options.client_secret,
scope=oauth2.Write(options.url_transformation),
ssl_options=ssl_server_options()
)
except httpclient.HTTPError as exc:
logging.exception('Error getting token for the transformation service')
raise exceptions.HTTPError(500, 'Internal Server Error')
headers = {'Accept': 'application/json', 'Content-Type': content_type}
client = API(options.url_transformation,
token=token,
ssl_options=ssl_server_options())
if r2rml_url:
params = urlencode({'r2rml_url': r2rml_url})
client.transformation.assets.path += '?{}'.format(params)
try:
client.transformation.assets.prepare_request(
request_timeout=180,
headers=headers,
body=data
)
response = yield client.transformation.assets.post()
except httpclient.HTTPError as exc:
response = exc.response
logging.exception(
'Transformation service error body:{}'.format(exc.response))
http_status = exc.code
errors = json.loads(exc.response.body)['errors']
logging.debug('<<< transform')
raise Return((response, http_status, errors))
def test_get_jwt_token(self):
token = yield oauth2.get_token('https://localhost:8007',
'4225f4774d6874a68565a04130001144',
'FMjU7vNIay5HGNABQVTTghOfEJqbet',
jwt='the client jwt')
assert token == self.token
body = self.API().auth.token.post.call_args[1]['body']
assert urlparse.parse_qs(body) == {
'grant_type': [oauth2.JWT_BEARER],
'assertion': ['the client jwt']
}
def test_get_token_with_scope_string(self):
token = yield oauth2.get_token('https://localhost:8007',
'4225f4774d6874a68565a04130001144',
'FMjU7vNIay5HGNABQVTTghOfEJqbet',
scope='read')
assert token == self.token
body = self.API().auth.token.post.call_args[1]['body']
assert urlparse.parse_qs(body) == {
'grant_type': [oauth2.CLIENT_CREDENTIALS],
'scope': ['read']
}
def transform(data, content_type, r2rml_url):
"""
Transforms source data into RDF triples
:param data: the source data
:param content_type: the http request content type
:param r2rml_url: karma mapping file url
:return: Transformed data and errors
"""
logging.debug('>>> transform')
response = None
http_status = 200
errors = []
try:
token = yield oauth2.get_token(options.url_auth,
options.service_id,
options.client_secret,
scope=oauth2.Write(
options.url_transformation),
ssl_options=ssl_server_options())
except httpclient.HTTPError as exc:
logging.exception('Error getting token for the transformation service')
raise exceptions.HTTPError(500, 'Internal Server Error')
headers = {'Accept': 'application/json', 'Content-Type': content_type}
client = API(options.url_transformation,
token=token,
ssl_options=ssl_server_options())
if r2rml_url:
params = urlencode({'r2rml_url': r2rml_url})
client.transformation.assets.path += '?{}'.format(params)
try:
client.transformation.assets.prepare_request(request_timeout=180,
headers=headers,
body=data)
response = yield client.transformation.assets.post()
except httpclient.HTTPError as exc:
response = exc.response
logging.exception('Transformation service error body:{}'.format(
exc.response))
http_status = exc.code
errors = json.loads(exc.response.body)['errors']
logging.debug('<<< transform')
raise Return((response, http_status, errors))
def _get_repos_for_source_id(source_id_type, source_id):
"""Get repositories having information about a specific source_id
:param source_id_type: type of the source_id
:param source_id: the id of the asset for which we do the query
:returns: organisation resource
:raises: koi.exceptions.HTTPError
"""
token = yield get_token(
options.url_auth, options.service_id,
options.client_secret, scope=Read(),
ssl_options=ssl_server_options()
)
client = API(options.url_index, token=token, ssl_options=ssl_server_options())
repos = yield client.index['entity-types']['asset']['id-types'][source_id_type].ids[source_id].repositories.get()
raise Return(repos['data']['repositories'])
def test_get_token(self):
token = yield oauth2.get_token('https://localhost:8007',
'4225f4774d6874a68565a04130001144',
'FMjU7vNIay5HGNABQVTTghOfEJqbet')
assert token == self.token
self.API.assert_called_once_with(
'https://localhost:8007',
auth_username='******',
auth_password='******')
self.API().auth.token.post.assert_called_once_with(
body=urllib.urlencode({'grant_type': oauth2.CLIENT_CREDENTIALS}),
request_timeout=60,
headers={'Content-Type': 'application/x-www-form-urlencoded',
'Accept': 'application/json'})
def test_get_token(self):
token = yield oauth2.get_token('https://localhost:8007',
'4225f4774d6874a68565a04130001144',
'FMjU7vNIay5HGNABQVTTghOfEJqbet')
assert token == self.token
self.API.assert_called_once_with(
'https://localhost:8007',
auth_username='******',
auth_password='******')
self.API().auth.token.post.assert_called_once_with(
body=urllib.urlencode({'grant_type': oauth2.CLIENT_CREDENTIALS}),
request_timeout=60,
headers={
'Content-Type': 'application/x-www-form-urlencoded',
'Accept': 'application/json'
})
def delete_from_index(repository, ids, **kwargs):
"""
Send a fire-and-forget delete to the index service
NOTE: all exceptions are unhandled. It's assumed that the function is used
as a callback outside of the request's context using IOLoop.spawn_callback
(see: http://www.tornadoweb.org/en/stable/ioloop.html)
"""
headers = {
'Accept': 'application/json',
'Content-Type': 'application/json'
}
try:
# extract the id list and id_type list from the incoming ids parameter
source_id_types = ','.join(
[urllib.unquote(str(x['source_id_type'])) for x in ids])
source_ids = ','.join(
[urllib.unquote(str(x['source_id'])) for x in ids])
logging.debug('delete_from_index : source_id_types ' + source_id_types)
logging.debug('delete_from_index : source_ids ' + source_ids)
token = yield get_token(options.url_auth,
options.service_id,
options.client_secret,
scope=Write(options.url_index),
ssl_options=ssl_server_options())
client = API(options.url_index,
token=token,
ssl_options=ssl_server_options())
logging.debug('delete_from_index : repo ' +
str(repository.repository_id))
yield client.index['entity-types']['asset']['id-types'][
source_id_types].ids[source_ids].repositories[
repository.repository_id].delete()
except Exception as e:
logging.exception("failed to delete from index: " + e.message)
def _get_ids(repository_id, entity_id):
"""Get ids from the repository service
:param provider_id: str
:returns: organisation resource
:raises: koi.exceptions.HTTPError
"""
repository = yield _get_repository(repository_id)
repository_url = repository['data']['service']['location']
token = yield get_token(
options.url_auth, options.service_id,
options.client_secret, scope=Read(),
ssl_options=ssl_server_options()
)
client = API(repository_url, token=token, ssl_options=ssl_server_options())
try:
res = yield client.repository.repositories[repository_id].assets[entity_id].ids.get()
raise Return(res['data'])
except httpclient.HTTPError as exc:
raise exceptions.HTTPError(exc.code, str(exc), source='repository')
