def send_notification(repository, **kwargs):
"""
Send a fire-and-forget notification to the index service
NOTE: all exceptions are unhandled. It's assumed that the function is used
as a callback outside of the request's context using IOLoop.spawn_callback
(see: http://www.tornadoweb.org/en/stable/ioloop.html)
"""
headers = {
'Accept': 'application/json',
'Content-Type': 'application/json'
}
try:
token = yield get_token(options.url_auth,
options.service_id,
options.client_secret,
scope=Write(options.url_index),
ssl_options=ssl_server_options())
client = API(options.url_index,
token=token,
ssl_options=ssl_server_options())
client.index.notifications.prepare_request(
request_timeout=options.request_timeout,
headers=headers,
body=json.dumps({'id': repository.repository_id}))
logging.debug('calling send_notification ' +
str(repository.repository_id))
yield client.index.notifications.post()
except Exception as e:
logging.exception("failed to notify index: " + e.message)
def repository_service_client(location):
"""
get an api client for a repository service
:params location: base url of the repository
"""
token = yield get_token(options.url_auth,
options.service_id,
options.client_secret,
scope=Read(),
ssl_options=ssl_server_options())
client = API(location, token=token, ssl_options=ssl_server_options())
raise Return(client)
def repository_service_client(location):
"""
get an api client for a repository service
:params location: base url of the repository
"""
token = yield get_token(
options.url_auth, options.service_id,
options.client_secret, scope=Read(),
ssl_options=ssl_server_options()
)
client = API(location, token=token, ssl_options=ssl_server_options())
raise Return(client)
def transform(data, content_type, r2rml_url):
"""
Transforms source data into RDF triples
:param data: the source data
:param content_type: the http request content type
:param r2rml_url: karma mapping file url
:return: Transformed data and errors
"""
logging.debug('>>> transform')
response = None
http_status = 200
errors = []
try:
token = yield oauth2.get_token(
options.url_auth,
options.service_id,
options.client_secret,
scope=oauth2.Write(options.url_transformation),
ssl_options=ssl_server_options()
)
except httpclient.HTTPError as exc:
logging.exception('Error getting token for the transformation service')
raise exceptions.HTTPError(500, 'Internal Server Error')
headers = {'Accept': 'application/json', 'Content-Type': content_type}
client = API(options.url_transformation,
token=token,
ssl_options=ssl_server_options())
if r2rml_url:
params = urlencode({'r2rml_url': r2rml_url})
client.transformation.assets.path += '?{}'.format(params)
try:
client.transformation.assets.prepare_request(
request_timeout=180,
headers=headers,
body=data
)
response = yield client.transformation.assets.post()
except httpclient.HTTPError as exc:
response = exc.response
logging.exception(
'Transformation service error body:{}'.format(exc.response))
http_status = exc.code
errors = json.loads(exc.response.body)['errors']
logging.debug('<<< transform')
raise Return((response, http_status, errors))
def transform(data, content_type, r2rml_url):
"""
Transforms source data into RDF triples
:param data: the source data
:param content_type: the http request content type
:param r2rml_url: karma mapping file url
:return: Transformed data and errors
"""
logging.debug('>>> transform')
response = None
http_status = 200
errors = []
try:
token = yield oauth2.get_token(options.url_auth,
options.service_id,
options.client_secret,
scope=oauth2.Write(
options.url_transformation),
ssl_options=ssl_server_options())
except httpclient.HTTPError as exc:
logging.exception('Error getting token for the transformation service')
raise exceptions.HTTPError(500, 'Internal Server Error')
headers = {'Accept': 'application/json', 'Content-Type': content_type}
client = API(options.url_transformation,
token=token,
ssl_options=ssl_server_options())
if r2rml_url:
params = urlencode({'r2rml_url': r2rml_url})
client.transformation.assets.path += '?{}'.format(params)
try:
client.transformation.assets.prepare_request(request_timeout=180,
headers=headers,
body=data)
response = yield client.transformation.assets.post()
except httpclient.HTTPError as exc:
response = exc.response
logging.exception('Transformation service error body:{}'.format(
exc.response))
http_status = exc.code
errors = json.loads(exc.response.body)['errors']
logging.debug('<<< transform')
raise Return((response, http_status, errors))
def _get_repos_for_source_id(source_id_type, source_id):
"""Get repositories having information about a specific source_id
:param source_id_type: type of the source_id
:param source_id: the id of the asset for which we do the query
:returns: organisation resource
:raises: koi.exceptions.HTTPError
"""
token = yield get_token(
options.url_auth, options.service_id,
options.client_secret, scope=Read(),
ssl_options=ssl_server_options()
)
client = API(options.url_index, token=token, ssl_options=ssl_server_options())
repos = yield client.index['entity-types']['asset']['id-types'][source_id_type].ids[source_id].repositories.get()
raise Return(repos['data']['repositories'])
def exchange_delegate_token(token, repository_id):
"""
Exchange a token for a delegated token
:param token: a JWT granting the onboarding service access to write on the
client's behalf
:param repository_id: the target repsitory's ID
:returns: a new JWT authorized to write to the repository
:raises: HTTPError
"""
try:
new_token = yield oauth2.get_token(options.url_auth,
options.service_id,
options.client_secret,
scope=oauth2.Write(repository_id),
jwt=token,
ssl_options=ssl_server_options())
except httpclient.HTTPError as exc:
if exc.code in (403, 400):
try:
body = json.loads(exc.response.body)
errors = [x['message'] for x in body['errors']]
except (AttributeError, KeyError):
errors = exc.message
raise exceptions.HTTPError(403, errors, source='authentication')
else:
msg = 'Error authorizing access to the repository'
logging.exception(msg)
raise exceptions.HTTPError(500, msg)
raise Return(new_token)
def verify_repository_token(self, token, requested_access, repository_id=None):
"""
Verify access to repository
:param repository_id: id of repository being accessed
:param token: Access token
:param requested_access: the access level the client has requested
:returns: boolean
"""
client = API(options.url_auth,
auth_username=options.service_id,
auth_password=options.client_secret,
ssl_options=ssl_server_options())
headers = {'Content-Type': 'application/x-www-form-urlencoded',
'Accept': 'application/json'}
data = {
'token': token,
'requested_access': requested_access
}
if repository_id:
data['resource_id'] = repository_id
client.auth.verify.prepare_request(headers=headers, request_timeout=180)
try:
result = yield client.auth.verify.post(body=urlencode(data))
except tornado.httpclient.HTTPError as ex:
# Must be converted to a tornado.web.HTTPError for the server
# to handle it correctly
logging.exception(ex.message)
raise HTTPError(500, 'Internal Server Error')
raise Return(result['has_access'])
def exchange_delegate_token(token, repository_id):
"""
Exchange a token for a delegated token
:param token: a JWT granting the onboarding service access to write on the
client's behalf
:param repository_id: the target repsitory's ID
:returns: a new JWT authorized to write to the repository
:raises: HTTPError
"""
try:
new_token = yield oauth2.get_token(
options.url_auth,
options.service_id,
options.client_secret,
scope=oauth2.Write(repository_id),
jwt=token,
ssl_options=ssl_server_options()
)
except httpclient.HTTPError as exc:
if exc.code in (403, 400):
try:
body = json.loads(exc.response.body)
errors = [x['message'] for x in body['errors']]
except (AttributeError, KeyError):
errors = exc.message
raise exceptions.HTTPError(403, errors, source='authentication')
else:
msg = 'Error authorizing access to the repository'
logging.exception(msg)
raise exceptions.HTTPError(500, msg)
raise Return(new_token)
def test_ssl_options():
server_opts = ssl_server_options()
if hasattr(ssl, 'SSLContext'):
assert isinstance(server_opts, ssl.SSLContext)
assert server_opts.verify_mode == 0
else:
for name in ['ca_certs', 'keyfile', 'certfile']:
assert os.path.isfile(server_opts[name])
assert server_opts['cert_reqs'] == 0
def test_ssl_options():
server_opts = ssl_server_options()
if hasattr(ssl, 'SSLContext'):
assert isinstance(server_opts, ssl.SSLContext)
assert server_opts.verify_mode == 0
else:
for name in ['ca_certs', 'keyfile', 'certfile']:
assert os.path.isfile(server_opts[name])
assert server_opts['cert_reqs'] == 0
def __init__(self, repository_dict, api_client=None):
if api_client is None:
api_client = API(options.url_accounts,
ssl_options=ssl_server_options())
self._repository_dict = repository_dict
self._api = api_client
self._endpoint = self._api.accounts.repositories
self._shelf = shelve.open(options.local_db, writeback=True)
self.on_new_repo = None
def __init__(self, repository_dict, api_client=None):
if api_client is None:
api_client = API(options.url_accounts,
ssl_options=ssl_server_options())
self._repository_dict = repository_dict
self._api = api_client
self._endpoint = self._api.accounts.repositories
self._shelf = shelve.open(options.local_db, writeback=True)
self.on_new_repo = None
def delete_from_index(repository, ids, **kwargs):
"""
Send a fire-and-forget delete to the index service
NOTE: all exceptions are unhandled. It's assumed that the function is used
as a callback outside of the request's context using IOLoop.spawn_callback
(see: http://www.tornadoweb.org/en/stable/ioloop.html)
"""
headers = {
'Accept': 'application/json',
'Content-Type': 'application/json'
}
try:
# extract the id list and id_type list from the incoming ids parameter
source_id_types = ','.join(
[urllib.unquote(str(x['source_id_type'])) for x in ids])
source_ids = ','.join(
[urllib.unquote(str(x['source_id'])) for x in ids])
logging.debug('delete_from_index : source_id_types ' + source_id_types)
logging.debug('delete_from_index : source_ids ' + source_ids)
token = yield get_token(options.url_auth,
options.service_id,
options.client_secret,
scope=Write(options.url_index),
ssl_options=ssl_server_options())
client = API(options.url_index,
token=token,
ssl_options=ssl_server_options())
logging.debug('delete_from_index : repo ' +
str(repository.repository_id))
yield client.index['entity-types']['asset']['id-types'][
source_id_types].ids[source_ids].repositories[
repository.repository_id].delete()
except Exception as e:
logging.exception("failed to delete from index: " + e.message)
def get_repository(repository_id):
"""
Lookup the repository location from the accounts service
TODO: cache the response
:param repository_id: the repository's ID
:returns: a URL
"""
api = API(options.url_accounts, ssl_options=ssl_server_options())
response = yield api.accounts.repositories[repository_id].get()
raise Return(response)
def get_repository(repository_id):
"""
Lookup the repository location from the accounts service
TODO: cache the response
:param repository_id: the repository's ID
:returns: a URL
"""
api = API(options.url_accounts, ssl_options=ssl_server_options())
response = yield api.accounts.repositories[repository_id].get()
raise Return(response)
def _get_ids(repository_id, entity_id):
"""Get ids from the repository service
:param provider_id: str
:returns: organisation resource
:raises: koi.exceptions.HTTPError
"""
repository = yield _get_repository(repository_id)
repository_url = repository['data']['service']['location']
token = yield get_token(
options.url_auth, options.service_id,
options.client_secret, scope=Read(),
ssl_options=ssl_server_options()
)
client = API(repository_url, token=token, ssl_options=ssl_server_options())
try:
res = yield client.repository.repositories[repository_id].assets[entity_id].ids.get()
raise Return(res['data'])
except httpclient.HTTPError as exc:
raise exceptions.HTTPError(exc.code, str(exc), source='repository')
def _get_asset_details(hubkey):
""" get the asset details from a hubkey
"""
client = API(options.url_query, ssl_options=ssl_server_options())
try:
res = yield client.query.entities.get(hub_key=hubkey)
raise Return(res['data'])
except httpclient.HTTPError as exc:
if exc.code == 404:
msg = 'No matching asset found'
else:
msg = 'Unexpected error ' + exc.message
raise exceptions.HTTPError(exc.code, msg, source='query')
def get_repository(repository_id):
"""
Get the repository service address from accounts service
for storing data.
:param repository_id: the repository ID
:return: url of the repository url
:raise: HTTPError
"""
client = API(options.url_accounts, ssl_options=ssl_server_options())
try:
response = yield client.accounts.repositories[repository_id].get()
logging.debug(response['data'])
raise Return(response['data'])
except KeyError:
error = 'Cannot find a repository'
raise exceptions.HTTPError(404, error)
def get_repository(repository_id):
"""
Get the repository service address from accounts service
for storing data.
:param repository_id: the repository ID
:return: url of the repository url
:raise: HTTPError
"""
client = API(options.url_accounts, ssl_options=ssl_server_options())
try:
response = yield client.accounts.repositories[repository_id].get()
logging.debug(response['data'])
raise Return(response['data'])
except KeyError:
error = 'Cannot find a repository'
raise exceptions.HTTPError(404, error)
def _get_offers_by_type_and_id(source_id_type, source_id):
""" get asset offers for given type and id
:param source_id_type: str
:param source_id: str
:returns: list of offers json
:raises: koi.exceptions.HTTPError
"""
client = API(options.url_query, ssl_options=ssl_server_options())
try:
req_body = '[{"source_id_type": "' + source_id_type + '", "source_id": "' + source_id + '"}]'
client.query.search.offers.prepare_request(headers={'Content-Type': 'application/json'},
body=req_body.strip())
res = yield client.query.search.offers.post()
raise Return(res['data'])
except httpclient.HTTPError as exc:
msg = 'Unexpected error ' + exc.message
raise exceptions.HTTPError(exc.code, msg, source='query')
def delete(response_trans, repository_url, repository_id, token=None):
"""
Send the rdf N3 content to the repository service for it to be deleted
:param response_trans: transformed data from the transformation service
:param repository_url: url of the repository service
:param repository_id: the repository ID
:param token: an authorization token
:return: Errors
"""
http_status = 200
errors = []
headers = {'Content-Type': 'text/rdf+n3', 'Accept': 'application/json'}
client = API(repository_url, ssl_options=ssl_server_options(), token=token)
endpoint = client.repository.repositories[repository_id].assets
try:
rdf_n3 = response_trans['data']['rdf_n3']
endpoint.prepare_request(request_timeout=180,
headers=headers,
body=rdf_n3,
allow_nonstandard_methods=True)
yield endpoint.delete()
except httpclient.HTTPError as exc:
logging.debug('Repository service error code:{}'.format(exc.code))
logging.debug('Repository service error body:{}'.format(exc.response))
if exc.code in (400, 403):
http_status = exc.code
errors = json.loads(exc.response.body)['errors']
else:
http_status = 500
errors = [{
"message": "Repository service error {}".format(exc.code)
}]
# socket error can occur if repository_url doesn't resolve to anything
# by the dns server
except socket.error as exc:
http_status = 500
message = "Socket error {} from {}".format(exc.args, repository_url)
errors = [{"message": message}]
logging.debug('<<< transform')
raise Return((http_status, errors))
def _get_repository(repository_id):
"""Get a repository from the accounts service
:param repository_id: str
:returns: repository resource
:raises: koi.exceptions.HTTPError
"""
client = API(options.url_accounts, ssl_options=ssl_server_options())
try:
repo = yield client.accounts.repositories[repository_id].get()
raise Return(repo)
except httpclient.HTTPError as exc:
if exc.code == 404:
msg = 'Unknown repository ID'
else:
msg = 'Unexpected error'
raise exceptions.HTTPError(exc.code, msg, source='accounts')
def _get_provider_by_name(provider):
"""Get a provider from the accounts service
:param provider: str
:returns: organisation resource
:raises: koi.exceptions.HTTPError
"""
client = API(options.url_accounts, ssl_options=ssl_server_options())
try:
res = yield client.accounts.organisations.get(name=provider)
raise Return(res['data'][0])
except httpclient.HTTPError as exc:
if exc.code == 404:
msg = 'Unknown provider'
else:
msg = 'Unexpected error ' + exc.message
raise exceptions.HTTPError(exc.code, msg, source='accounts')
def _get_provider(provider_id):
"""Get a provider from the accounts service
:param provider_id: str
:returns: organisation resource
:raises: koi.exceptions.HTTPError
"""
client = API(options.url_accounts, ssl_options=ssl_server_options())
try:
org = yield client.accounts.organisations[provider_id].get()
raise Return(org)
except httpclient.HTTPError as exc:
if exc.code == 404:
msg = 'Unknown provider ID'
else:
msg = 'Unexpected error'
raise exceptions.HTTPError(exc.code, msg, source='accounts')
def delete(response_trans, repository_url, repository_id, token=None):
"""
Send the rdf N3 content to the repository service for it to be deleted
:param response_trans: transformed data from the transformation service
:param repository_url: url of the repository service
:param repository_id: the repository ID
:param token: an authorization token
:return: Errors
"""
http_status = 200
errors = []
headers = {
'Content-Type': 'text/rdf+n3',
'Accept': 'application/json'
}
client = API(repository_url, ssl_options=ssl_server_options(), token=token)
endpoint = client.repository.repositories[repository_id].assets
try:
rdf_n3 = response_trans['data']['rdf_n3']
endpoint.prepare_request(
request_timeout=180, headers=headers, body=rdf_n3, allow_nonstandard_methods=True)
yield endpoint.delete()
except httpclient.HTTPError as exc:
logging.debug('Repository service error code:{}'.format(exc.code))
logging.debug('Repository service error body:{}'.format(exc.response))
if exc.code in (400, 403):
http_status = exc.code
errors = json.loads(exc.response.body)['errors']
else:
http_status = 500
errors = [
{"message": "Repository service error {}".format(exc.code)}]
# socket error can occur if repository_url doesn't resolve to anything
# by the dns server
except socket.error as exc:
http_status = 500
message = "Socket error {} from {}".format(exc.args, repository_url)
errors = [{"message": message}]
logging.debug('<<< transform')
raise Return((http_status, errors))
def _get_providers_by_type_and_id(source_id_type, source_id):
""" get the matching providers for a given source_id_type and source_id
:param source_id_type: str
:param source_id: str
:returns: list of organisations
:raises: koi.exceptsion.HTTPError
"""
client = API(options.url_query, ssl_options=ssl_server_options())
try:
res = yield client.query.licensors.get(source_id_type=source_id_type, source_id=source_id)
raise Return(res['data'])
except httpclient.HTTPError as exc:
if exc.code == 404:
msg = 'No matching providers found'
else:
msg = 'Unexpected error ' + exc.message
raise exceptions.HTTPError(exc.code, msg, source='query')
def get_organisation_id(self, repository_id):
"""
Lookup organisation id of repository
:param repository_id: id of repository being accessed
:returns: organisation_id string
"""
client = API(options.url_accounts,
ssl_options=ssl_server_options())
headers = {'Accept': 'application/json'}
client.accounts.repositories[repository_id].prepare_request(headers=headers, request_timeout=180)
try:
result = yield client.accounts.repositories[repository_id].get()
except tornado.httpclient.HTTPError as ex:
# Must be converted to a tornado.web.HTTPError for the server
# to handle it correctly
logging.exception(ex.message)
if ex.code == 404:
raise HTTPError(404, ex.message)
else:
raise HTTPError(500, 'Internal Server Error')
raise Return(result['data']['organisation']['id'])