def fetch_token(self, code, state):
"""
Provided a code and a state (provided by the client once redirected
back to the login callback URL), fetches a token from Hydra.
The token contains ID token, Access token, Refresh token,
among other things.
Use self.get_id_token() to decode the ID token.
:param str code:
:param str state:
:rtype: collections.abc.Mapping
"""
try:
return self.client.fetch_token(
url=HYDRA_TOKEN_ENDPOINT,
grant_type='authorization_code',
code=code,
state=state,
redirect_uri=LOGIN_CALLBACK_URL,
verify=not DEBUG,
)
except json.decoder.JSONDecodeError as e:
logger.exception('JSONDecodeError from Hydra',
extra={'doc': e.doc})
raise
def send_key_to_datahub_service(task, subject, gsrn, session):
"""
:param celery.Task task:
:param str subject:
:param str gsrn:
:param sqlalchemy.orm.Session session:
"""
__log_extra = {
'subject': subject,
'gsrn': gsrn,
'pipeline': 'import_meteringpoints',
'task': 'send_key_to_datahub_service',
}
# Get User from DB
try:
user = UserQuery(session) \
.is_active() \
.has_sub(subject) \
.one()
except orm.exc.NoResultFound:
raise
except Exception as e:
logger.exception('Failed to load User from database, retrying...',
extra=__log_extra)
raise task.retry(exc=e)
# Get MeteringPoint from DB
try:
meteringpoint = MeteringPointQuery(session) \
.has_gsrn(gsrn) \
.one()
except orm.exc.NoResultFound:
raise
except Exception as e:
logger.exception(
'Failed to load MeteringPoint from database, retrying...',
extra=__log_extra)
raise task.retry(exc=e)
# Send key to DataHubService
try:
datahub_service.set_key(
token=user.access_token,
gsrn=meteringpoint.gsrn,
key=meteringpoint.extended_key,
)
except DataHubServiceConnectionError as e:
logger.exception(
f'Failed to establish connection to DataHubService, retrying...',
extra=__log_extra)
raise task.retry(exc=e)
except DataHubServiceError as e:
if e.status_code == 400:
raise
else:
logger.exception('Failed to import MeteringPoints, retrying...',
extra=__log_extra)
raise task.retry(exc=e)
def poll_batch_status(task, subject, batch_id, session):
"""
:param celery.Task task:
:param str subject:
:param int batch_id:
:param sqlalchemy.orm.Session session:
"""
__log_extra = {
'subject': subject,
'batch_id': str(batch_id),
'pipeline': 'submit_batch_to_ledger',
'task': 'poll_batch_status',
}
# Get batch from DB
try:
batch = session \
.query(Batch) \
.filter(Batch.id == batch_id) \
.one()
except orm.exc.NoResultFound:
raise
except Exception as e:
logger.exception('Failed to load Batch from database', extra=__log_extra)
raise task.retry(exc=e)
# Get batch status from ledger
try:
response = ledger.get_batch_status(batch.handle)
except ols.LedgerConnectionError as e:
logger.exception('Failed to poll ledger for batch status, retrying...', extra=__log_extra)
raise task.retry(exc=e)
# Assert status
if response.status == ols.BatchStatus.COMMITTED:
logger.info('Ledger batch status: COMMITTED', extra=__log_extra)
elif response.status == ols.BatchStatus.INVALID:
# Raising exception triggers the ON ERROR task (rollback_batch())
raise InvalidBatch('Invalid batch')
elif response.status == ols.BatchStatus.UNKNOWN:
logger.info('Ledger batch status: UNKNOWN', extra=__log_extra)
raise task.retry()
elif response.status == ols.BatchStatus.PENDING:
logger.info('Ledger batch status: PENDING', extra=__log_extra)
raise task.retry()
else:
raise RuntimeError('Unknown batch status returned, should NOT have happened!')
def submit_batch_to_ledger(task, subject, batch_id, session):
"""
:param celery.Task task:
:param str subject:
:param int batch_id:
:param sqlalchemy.orm.Session session:
"""
__log_extra = {
'subject': subject,
'batch_id': str(batch_id),
'pipeline': 'submit_batch_to_ledger',
'task': 'submit_to_ledger',
}
# Get Batch from DB
try:
batch = session \
.query(Batch) \
.filter(Batch.id == batch_id) \
.one()
except orm.exc.NoResultFound:
raise
except Exception as e:
raise task.retry(exc=e)
# Submit batch to ledger
try:
handle = ledger.execute_batch(batch.build_ledger_batch())
except ols.LedgerConnectionError as e:
logger.exception('Failed to submit batch to ledger, retrying...', extra=__log_extra)
raise task.retry(exc=e)
except ols.LedgerException as e:
if e.code in (15, 17, 18):
logger.exception(f'Ledger validator error (code {e.code}), retrying...', extra=__log_extra)
raise task.retry(exc=e)
elif e.code == 31:
logger.info(f'Ledger queue is full, retrying...', extra=__log_extra)
raise task.retry(exc=e)
else:
raise
logger.info(f'Batch submitted to ledger', extra=__log_extra)
return handle
def invoke_on_forecast_received(task, subject, forecast_id, subscription_id,
session, **logging_kwargs):
"""
:param celery.Task task:
:param str subject:
:param int forecast_id:
:param int subscription_id:
:param sqlalchemy.orm.Session session:
"""
__log_extra = logging_kwargs.copy()
__log_extra.update({
'subject': subject,
'forecast_id': str(forecast_id),
'subscription_id': str(subscription_id),
'pipeline': 'webhooks',
'task': 'invoke_on_forecast_received',
})
# Get Forecast from database
try:
forecast = ForecastQuery(session) \
.has_id(forecast_id) \
.one()
except orm.exc.NoResultFound:
raise
except Exception as e:
logger.exception('Failed to load Forecast from database',
extra=__log_extra)
raise task.retry(exc=e)
# Get webhook subscription from database
try:
subscription = webhook_service.get_subscription(
subscription_id, session)
except orm.exc.NoResultFound:
raise
except Exception as e:
logger.exception('Failed to load WebhookSubscription from database',
extra=__log_extra)
raise task.retry(exc=e)
# Publish event to webhook
try:
webhook_service.on_forecast_received(subscription, forecast)
except WebhookConnectionError as e:
logger.exception(
'Failed to invoke webhook: ON_FORECAST_RECEIVED (Connection error)',
extra=__log_extra)
raise task.retry(exc=e)
except WebhookError as e:
logger.exception('Failed to invoke webhook: ON_FORECAST_RECEIVED',
extra=__log_extra)
raise task.retry(exc=e)
def refresh_token(self, refresh_token):
"""
Fetches a new access token using a refresh token.
Use self.get_id_token() to decode the ID token.
:param str refresh_token:
:rtype: collections.abc.Mapping
"""
try:
return self.client.refresh_token(
url=HYDRA_TOKEN_ENDPOINT,
refresh_token=refresh_token,
verify=not DEBUG,
)
except json.decoder.JSONDecodeError as e:
logger.exception('JSONDecodeError from Hydra',
extra={'doc': e.doc})
raise
def register_login_state(self):
"""
Register a login state. Is used before redirecting the client
to Hydra, to perform login. Returns a tuple of (login_url, state)
where the state is used to identify the client when its redirected
back to the callback URL.
:rtype: (str, str)
:returns: Tuple of (login_url, state)
"""
try:
return self.client.create_authorization_url(
url=HYDRA_AUTH_ENDPOINT,
redirect_uri=LOGIN_CALLBACK_URL,
)
except json.decoder.JSONDecodeError as e:
logger.exception('JSONDecodeError from Hydra',
extra={'doc': e.doc})
raise
def batch_on_rollback(task, subject, batch_id, session):
"""
:param celery.Task task:
:param str subject:
:param int batch_id:
:param sqlalchemy.orm.Session session:
"""
try:
session \
.query(Batch) \
.filter(Batch.id == batch_id) \
.one() \
.on_rollback()
except orm.exc.NoResultFound:
raise
except Exception as e:
logger.exception('Failed to invoke on_rollback() on Batch', extra={
'subject': subject,
'batch_id': str(batch_id),
'pipeline': 'submit_batch_to_ledger',
'task': 'batch_on_rollback',
})
raise task.retry(exc=e)
def import_meteringpoints_and_insert_to_db(task, subject, session):
"""
:param celery.Task task:
:param str subject:
:param sqlalchemy.orm.Session session:
"""
__log_extra = {
'subject': subject,
'pipeline': 'import_meteringpoints',
'task': 'import_meteringpoints_and_insert_to_db',
}
# Get User from DB
try:
user = UserQuery(session) \
.is_active() \
.has_sub(subject) \
.one()
except orm.exc.NoResultFound:
raise
except Exception as e:
logger.exception('Failed to load User from database, retrying...',
extra=__log_extra)
raise task.retry(exc=e)
# Import MeteringPoints from DataHubService
try:
response = datahub_service.get_meteringpoints(user.access_token)
except DataHubServiceConnectionError as e:
logger.exception(
f'Failed to establish connection to DataHubService, retrying...',
extra=__log_extra)
raise task.retry(exc=e)
except DataHubServiceError as e:
if e.status_code == 400:
raise
else:
logger.exception('Failed to import MeteringPoints, retrying...',
extra=__log_extra)
raise task.retry(exc=e)
# Save imported MeteringPoints to database
try:
meteringpoints = save_imported_meteringpoints(user, response)
except Exception as e:
logger.exception(
'Failed to save imported Meteringpoints to database, retrying...',
extra=__log_extra)
raise task.retry(exc=e)
logger.info(
f'Imported {len(meteringpoints)} new MeteringPoints from DataHubService',
extra=__log_extra)
# Send MeteringPoint key to DataHubService for each imported MeteringPoint
tasks = []
for meteringpoint in meteringpoints:
logger.info(f'Imported meteringpoint with GSRN: {meteringpoint.gsrn}',
extra={
'gsrn': meteringpoint.gsrn,
'subject': user.sub,
'pipeline': 'import_meteringpoints',
'task': 'import_meteringpoints_and_insert_to_db',
})
tasks.append(
send_key_to_datahub_service.s(
subject=subject,
gsrn=meteringpoint.gsrn,
))
group(*tasks).apply_async()
def handle_request(self, request, session):
"""
:param VerifyLoginCallbackRequest request:
:param sqlalchemy.orm.Session session:
:rtype: flask.Response
"""
return_url = redis.get(request.state)
if return_url is None:
raise BadRequest('Click back in your browser')
else:
return_url = return_url.decode()
redis.delete(request.state)
# Fetch token
try:
token = backend.fetch_token(request.code, request.state)
except:
logger.exception(f'Failed to fetch token',
extra={
'scope': str(request.scope),
'code': request.code,
'state': request.state,
})
return self.redirect_to_failure(return_url)
# Extract data from token
id_token = backend.get_id_token(token)
# No id_token means the user declined to give consent
if id_token is None:
return self.redirect_to_failure(return_url,
'No ID token from Hydra')
expires = datetime \
.fromtimestamp(token['expires_at']) \
.replace(tzinfo=timezone.utc)
# Lookup user from "subject"
user = UserQuery(session) \
.is_active() \
.has_sub(id_token['sub']) \
.one_or_none()
if user is None:
logger.error(
f'User login: Creating new user and subscribing to webhooks',
extra={
'subject': id_token['sub'],
})
self.create_new_user(token, id_token, expires, session)
self.datahub.webhook_on_meteringpoint_available_subscribe(
token['access_token'])
self.datahub.webhook_on_ggo_issued_subscribe(token['access_token'])
else:
logger.error(f'User login: Updating tokens for existing user',
extra={
'subject': id_token['sub'],
})
self.update_user_attributes(user, token, expires)
# Create HTTP response
response = redirect(f'{return_url}?success=1', code=303)
response.headers[
'Cache-Control'] = 'no-cache, no-store, must-revalidate'
response.headers['Pragma'] = 'no-cache'
response.headers['Expires'] = '0'
response.headers['Cache-Control'] = 'public, max-age=0'
return response
