def send_notification(repository, **kwargs): """ Send a fire-and-forget notification to the index service NOTE: all exceptions are unhandled. It's assumed that the function is used as a callback outside of the request's context using IOLoop.spawn_callback (see: http://www.tornadoweb.org/en/stable/ioloop.html) """ headers = { 'Accept': 'application/json', 'Content-Type': 'application/json' } try: token = yield get_token(options.url_auth, options.service_id, options.client_secret, scope=Write(options.url_index), ssl_options=ssl_server_options()) client = API(options.url_index, token=token, ssl_options=ssl_server_options()) client.index.notifications.prepare_request( request_timeout=options.request_timeout, headers=headers, body=json.dumps({'id': repository.repository_id})) logging.debug('calling send_notification ' + str(repository.repository_id)) yield client.index.notifications.post() except Exception as e: logging.exception("failed to notify index: " + e.message)
def repository_service_client(location): """ get an api client for a repository service :params location: base url of the repository """ token = yield get_token(options.url_auth, options.service_id, options.client_secret, scope=Read(), ssl_options=ssl_server_options()) client = API(location, token=token, ssl_options=ssl_server_options()) raise Return(client)
def repository_service_client(location): """ get an api client for a repository service :params location: base url of the repository """ token = yield get_token( options.url_auth, options.service_id, options.client_secret, scope=Read(), ssl_options=ssl_server_options() ) client = API(location, token=token, ssl_options=ssl_server_options()) raise Return(client)
def transform(data, content_type, r2rml_url): """ Transforms source data into RDF triples :param data: the source data :param content_type: the http request content type :param r2rml_url: karma mapping file url :return: Transformed data and errors """ logging.debug('>>> transform') response = None http_status = 200 errors = [] try: token = yield oauth2.get_token( options.url_auth, options.service_id, options.client_secret, scope=oauth2.Write(options.url_transformation), ssl_options=ssl_server_options() ) except httpclient.HTTPError as exc: logging.exception('Error getting token for the transformation service') raise exceptions.HTTPError(500, 'Internal Server Error') headers = {'Accept': 'application/json', 'Content-Type': content_type} client = API(options.url_transformation, token=token, ssl_options=ssl_server_options()) if r2rml_url: params = urlencode({'r2rml_url': r2rml_url}) client.transformation.assets.path += '?{}'.format(params) try: client.transformation.assets.prepare_request( request_timeout=180, headers=headers, body=data ) response = yield client.transformation.assets.post() except httpclient.HTTPError as exc: response = exc.response logging.exception( 'Transformation service error body:{}'.format(exc.response)) http_status = exc.code errors = json.loads(exc.response.body)['errors'] logging.debug('<<< transform') raise Return((response, http_status, errors))
def transform(data, content_type, r2rml_url): """ Transforms source data into RDF triples :param data: the source data :param content_type: the http request content type :param r2rml_url: karma mapping file url :return: Transformed data and errors """ logging.debug('>>> transform') response = None http_status = 200 errors = [] try: token = yield oauth2.get_token(options.url_auth, options.service_id, options.client_secret, scope=oauth2.Write( options.url_transformation), ssl_options=ssl_server_options()) except httpclient.HTTPError as exc: logging.exception('Error getting token for the transformation service') raise exceptions.HTTPError(500, 'Internal Server Error') headers = {'Accept': 'application/json', 'Content-Type': content_type} client = API(options.url_transformation, token=token, ssl_options=ssl_server_options()) if r2rml_url: params = urlencode({'r2rml_url': r2rml_url}) client.transformation.assets.path += '?{}'.format(params) try: client.transformation.assets.prepare_request(request_timeout=180, headers=headers, body=data) response = yield client.transformation.assets.post() except httpclient.HTTPError as exc: response = exc.response logging.exception('Transformation service error body:{}'.format( exc.response)) http_status = exc.code errors = json.loads(exc.response.body)['errors'] logging.debug('<<< transform') raise Return((response, http_status, errors))
def _get_repos_for_source_id(source_id_type, source_id): """Get repositories having information about a specific source_id :param source_id_type: type of the source_id :param source_id: the id of the asset for which we do the query :returns: organisation resource :raises: koi.exceptions.HTTPError """ token = yield get_token( options.url_auth, options.service_id, options.client_secret, scope=Read(), ssl_options=ssl_server_options() ) client = API(options.url_index, token=token, ssl_options=ssl_server_options()) repos = yield client.index['entity-types']['asset']['id-types'][source_id_type].ids[source_id].repositories.get() raise Return(repos['data']['repositories'])
def exchange_delegate_token(token, repository_id): """ Exchange a token for a delegated token :param token: a JWT granting the onboarding service access to write on the client's behalf :param repository_id: the target repsitory's ID :returns: a new JWT authorized to write to the repository :raises: HTTPError """ try: new_token = yield oauth2.get_token(options.url_auth, options.service_id, options.client_secret, scope=oauth2.Write(repository_id), jwt=token, ssl_options=ssl_server_options()) except httpclient.HTTPError as exc: if exc.code in (403, 400): try: body = json.loads(exc.response.body) errors = [x['message'] for x in body['errors']] except (AttributeError, KeyError): errors = exc.message raise exceptions.HTTPError(403, errors, source='authentication') else: msg = 'Error authorizing access to the repository' logging.exception(msg) raise exceptions.HTTPError(500, msg) raise Return(new_token)
def verify_repository_token(self, token, requested_access, repository_id=None): """ Verify access to repository :param repository_id: id of repository being accessed :param token: Access token :param requested_access: the access level the client has requested :returns: boolean """ client = API(options.url_auth, auth_username=options.service_id, auth_password=options.client_secret, ssl_options=ssl_server_options()) headers = {'Content-Type': 'application/x-www-form-urlencoded', 'Accept': 'application/json'} data = { 'token': token, 'requested_access': requested_access } if repository_id: data['resource_id'] = repository_id client.auth.verify.prepare_request(headers=headers, request_timeout=180) try: result = yield client.auth.verify.post(body=urlencode(data)) except tornado.httpclient.HTTPError as ex: # Must be converted to a tornado.web.HTTPError for the server # to handle it correctly logging.exception(ex.message) raise HTTPError(500, 'Internal Server Error') raise Return(result['has_access'])
def exchange_delegate_token(token, repository_id): """ Exchange a token for a delegated token :param token: a JWT granting the onboarding service access to write on the client's behalf :param repository_id: the target repsitory's ID :returns: a new JWT authorized to write to the repository :raises: HTTPError """ try: new_token = yield oauth2.get_token( options.url_auth, options.service_id, options.client_secret, scope=oauth2.Write(repository_id), jwt=token, ssl_options=ssl_server_options() ) except httpclient.HTTPError as exc: if exc.code in (403, 400): try: body = json.loads(exc.response.body) errors = [x['message'] for x in body['errors']] except (AttributeError, KeyError): errors = exc.message raise exceptions.HTTPError(403, errors, source='authentication') else: msg = 'Error authorizing access to the repository' logging.exception(msg) raise exceptions.HTTPError(500, msg) raise Return(new_token)
def test_ssl_options(): server_opts = ssl_server_options() if hasattr(ssl, 'SSLContext'): assert isinstance(server_opts, ssl.SSLContext) assert server_opts.verify_mode == 0 else: for name in ['ca_certs', 'keyfile', 'certfile']: assert os.path.isfile(server_opts[name]) assert server_opts['cert_reqs'] == 0
def test_ssl_options(): server_opts = ssl_server_options() if hasattr(ssl, 'SSLContext'): assert isinstance(server_opts, ssl.SSLContext) assert server_opts.verify_mode == 0 else: for name in ['ca_certs', 'keyfile', 'certfile']: assert os.path.isfile(server_opts[name]) assert server_opts['cert_reqs'] == 0
def __init__(self, repository_dict, api_client=None): if api_client is None: api_client = API(options.url_accounts, ssl_options=ssl_server_options()) self._repository_dict = repository_dict self._api = api_client self._endpoint = self._api.accounts.repositories self._shelf = shelve.open(options.local_db, writeback=True) self.on_new_repo = None
def __init__(self, repository_dict, api_client=None): if api_client is None: api_client = API(options.url_accounts, ssl_options=ssl_server_options()) self._repository_dict = repository_dict self._api = api_client self._endpoint = self._api.accounts.repositories self._shelf = shelve.open(options.local_db, writeback=True) self.on_new_repo = None
def delete_from_index(repository, ids, **kwargs): """ Send a fire-and-forget delete to the index service NOTE: all exceptions are unhandled. It's assumed that the function is used as a callback outside of the request's context using IOLoop.spawn_callback (see: http://www.tornadoweb.org/en/stable/ioloop.html) """ headers = { 'Accept': 'application/json', 'Content-Type': 'application/json' } try: # extract the id list and id_type list from the incoming ids parameter source_id_types = ','.join( [urllib.unquote(str(x['source_id_type'])) for x in ids]) source_ids = ','.join( [urllib.unquote(str(x['source_id'])) for x in ids]) logging.debug('delete_from_index : source_id_types ' + source_id_types) logging.debug('delete_from_index : source_ids ' + source_ids) token = yield get_token(options.url_auth, options.service_id, options.client_secret, scope=Write(options.url_index), ssl_options=ssl_server_options()) client = API(options.url_index, token=token, ssl_options=ssl_server_options()) logging.debug('delete_from_index : repo ' + str(repository.repository_id)) yield client.index['entity-types']['asset']['id-types'][ source_id_types].ids[source_ids].repositories[ repository.repository_id].delete() except Exception as e: logging.exception("failed to delete from index: " + e.message)
def get_repository(repository_id): """ Lookup the repository location from the accounts service TODO: cache the response :param repository_id: the repository's ID :returns: a URL """ api = API(options.url_accounts, ssl_options=ssl_server_options()) response = yield api.accounts.repositories[repository_id].get() raise Return(response)
def get_repository(repository_id): """ Lookup the repository location from the accounts service TODO: cache the response :param repository_id: the repository's ID :returns: a URL """ api = API(options.url_accounts, ssl_options=ssl_server_options()) response = yield api.accounts.repositories[repository_id].get() raise Return(response)
def _get_ids(repository_id, entity_id): """Get ids from the repository service :param provider_id: str :returns: organisation resource :raises: koi.exceptions.HTTPError """ repository = yield _get_repository(repository_id) repository_url = repository['data']['service']['location'] token = yield get_token( options.url_auth, options.service_id, options.client_secret, scope=Read(), ssl_options=ssl_server_options() ) client = API(repository_url, token=token, ssl_options=ssl_server_options()) try: res = yield client.repository.repositories[repository_id].assets[entity_id].ids.get() raise Return(res['data']) except httpclient.HTTPError as exc: raise exceptions.HTTPError(exc.code, str(exc), source='repository')
def _get_asset_details(hubkey): """ get the asset details from a hubkey """ client = API(options.url_query, ssl_options=ssl_server_options()) try: res = yield client.query.entities.get(hub_key=hubkey) raise Return(res['data']) except httpclient.HTTPError as exc: if exc.code == 404: msg = 'No matching asset found' else: msg = 'Unexpected error ' + exc.message raise exceptions.HTTPError(exc.code, msg, source='query')
def get_repository(repository_id): """ Get the repository service address from accounts service for storing data. :param repository_id: the repository ID :return: url of the repository url :raise: HTTPError """ client = API(options.url_accounts, ssl_options=ssl_server_options()) try: response = yield client.accounts.repositories[repository_id].get() logging.debug(response['data']) raise Return(response['data']) except KeyError: error = 'Cannot find a repository' raise exceptions.HTTPError(404, error)
def get_repository(repository_id): """ Get the repository service address from accounts service for storing data. :param repository_id: the repository ID :return: url of the repository url :raise: HTTPError """ client = API(options.url_accounts, ssl_options=ssl_server_options()) try: response = yield client.accounts.repositories[repository_id].get() logging.debug(response['data']) raise Return(response['data']) except KeyError: error = 'Cannot find a repository' raise exceptions.HTTPError(404, error)
def _get_offers_by_type_and_id(source_id_type, source_id): """ get asset offers for given type and id :param source_id_type: str :param source_id: str :returns: list of offers json :raises: koi.exceptions.HTTPError """ client = API(options.url_query, ssl_options=ssl_server_options()) try: req_body = '[{"source_id_type": "' + source_id_type + '", "source_id": "' + source_id + '"}]' client.query.search.offers.prepare_request(headers={'Content-Type': 'application/json'}, body=req_body.strip()) res = yield client.query.search.offers.post() raise Return(res['data']) except httpclient.HTTPError as exc: msg = 'Unexpected error ' + exc.message raise exceptions.HTTPError(exc.code, msg, source='query')
def delete(response_trans, repository_url, repository_id, token=None): """ Send the rdf N3 content to the repository service for it to be deleted :param response_trans: transformed data from the transformation service :param repository_url: url of the repository service :param repository_id: the repository ID :param token: an authorization token :return: Errors """ http_status = 200 errors = [] headers = {'Content-Type': 'text/rdf+n3', 'Accept': 'application/json'} client = API(repository_url, ssl_options=ssl_server_options(), token=token) endpoint = client.repository.repositories[repository_id].assets try: rdf_n3 = response_trans['data']['rdf_n3'] endpoint.prepare_request(request_timeout=180, headers=headers, body=rdf_n3, allow_nonstandard_methods=True) yield endpoint.delete() except httpclient.HTTPError as exc: logging.debug('Repository service error code:{}'.format(exc.code)) logging.debug('Repository service error body:{}'.format(exc.response)) if exc.code in (400, 403): http_status = exc.code errors = json.loads(exc.response.body)['errors'] else: http_status = 500 errors = [{ "message": "Repository service error {}".format(exc.code) }] # socket error can occur if repository_url doesn't resolve to anything # by the dns server except socket.error as exc: http_status = 500 message = "Socket error {} from {}".format(exc.args, repository_url) errors = [{"message": message}] logging.debug('<<< transform') raise Return((http_status, errors))
def _get_repository(repository_id): """Get a repository from the accounts service :param repository_id: str :returns: repository resource :raises: koi.exceptions.HTTPError """ client = API(options.url_accounts, ssl_options=ssl_server_options()) try: repo = yield client.accounts.repositories[repository_id].get() raise Return(repo) except httpclient.HTTPError as exc: if exc.code == 404: msg = 'Unknown repository ID' else: msg = 'Unexpected error' raise exceptions.HTTPError(exc.code, msg, source='accounts')
def _get_provider_by_name(provider): """Get a provider from the accounts service :param provider: str :returns: organisation resource :raises: koi.exceptions.HTTPError """ client = API(options.url_accounts, ssl_options=ssl_server_options()) try: res = yield client.accounts.organisations.get(name=provider) raise Return(res['data'][0]) except httpclient.HTTPError as exc: if exc.code == 404: msg = 'Unknown provider' else: msg = 'Unexpected error ' + exc.message raise exceptions.HTTPError(exc.code, msg, source='accounts')
def _get_provider(provider_id): """Get a provider from the accounts service :param provider_id: str :returns: organisation resource :raises: koi.exceptions.HTTPError """ client = API(options.url_accounts, ssl_options=ssl_server_options()) try: org = yield client.accounts.organisations[provider_id].get() raise Return(org) except httpclient.HTTPError as exc: if exc.code == 404: msg = 'Unknown provider ID' else: msg = 'Unexpected error' raise exceptions.HTTPError(exc.code, msg, source='accounts')
def delete(response_trans, repository_url, repository_id, token=None): """ Send the rdf N3 content to the repository service for it to be deleted :param response_trans: transformed data from the transformation service :param repository_url: url of the repository service :param repository_id: the repository ID :param token: an authorization token :return: Errors """ http_status = 200 errors = [] headers = { 'Content-Type': 'text/rdf+n3', 'Accept': 'application/json' } client = API(repository_url, ssl_options=ssl_server_options(), token=token) endpoint = client.repository.repositories[repository_id].assets try: rdf_n3 = response_trans['data']['rdf_n3'] endpoint.prepare_request( request_timeout=180, headers=headers, body=rdf_n3, allow_nonstandard_methods=True) yield endpoint.delete() except httpclient.HTTPError as exc: logging.debug('Repository service error code:{}'.format(exc.code)) logging.debug('Repository service error body:{}'.format(exc.response)) if exc.code in (400, 403): http_status = exc.code errors = json.loads(exc.response.body)['errors'] else: http_status = 500 errors = [ {"message": "Repository service error {}".format(exc.code)}] # socket error can occur if repository_url doesn't resolve to anything # by the dns server except socket.error as exc: http_status = 500 message = "Socket error {} from {}".format(exc.args, repository_url) errors = [{"message": message}] logging.debug('<<< transform') raise Return((http_status, errors))
def _get_providers_by_type_and_id(source_id_type, source_id): """ get the matching providers for a given source_id_type and source_id :param source_id_type: str :param source_id: str :returns: list of organisations :raises: koi.exceptsion.HTTPError """ client = API(options.url_query, ssl_options=ssl_server_options()) try: res = yield client.query.licensors.get(source_id_type=source_id_type, source_id=source_id) raise Return(res['data']) except httpclient.HTTPError as exc: if exc.code == 404: msg = 'No matching providers found' else: msg = 'Unexpected error ' + exc.message raise exceptions.HTTPError(exc.code, msg, source='query')
def get_organisation_id(self, repository_id): """ Lookup organisation id of repository :param repository_id: id of repository being accessed :returns: organisation_id string """ client = API(options.url_accounts, ssl_options=ssl_server_options()) headers = {'Accept': 'application/json'} client.accounts.repositories[repository_id].prepare_request(headers=headers, request_timeout=180) try: result = yield client.accounts.repositories[repository_id].get() except tornado.httpclient.HTTPError as ex: # Must be converted to a tornado.web.HTTPError for the server # to handle it correctly logging.exception(ex.message) if ex.code == 404: raise HTTPError(404, ex.message) else: raise HTTPError(500, 'Internal Server Error') raise Return(result['data']['organisation']['id'])