def process_job(job_id): start = datetime.utcnow() job = dao_get_job_by_id(job_id) if job.job_status != "pending": return service = job.service if __sending_limits_for_job_exceeded(service, job, job_id): return job.job_status = "in progress" dao_update_job(job) template = Template(dao_get_template_by_id(job.template_id, job.template_version).__dict__) for row_number, recipient, personalisation in RecipientCSV( s3.get_job_from_s3(str(service.id), str(job_id)), template_type=template.template_type, placeholders=template.placeholders, ).enumerated_recipients_and_personalisation: encrypted = encryption.encrypt( { "template": str(template.id), "template_version": job.template_version, "job": str(job.id), "to": recipient, "row_number": row_number, "personalisation": dict(personalisation), } ) if template.template_type == SMS_TYPE: send_sms.apply_async( (str(job.service_id), create_uuid(), encrypted, datetime.utcnow().strftime(DATETIME_FORMAT)), queue="db-sms" if not service.research_mode else "research-mode", ) if template.template_type == EMAIL_TYPE: send_email.apply_async( (str(job.service_id), create_uuid(), encrypted, datetime.utcnow().strftime(DATETIME_FORMAT)), queue="db-email" if not service.research_mode else "research-mode", ) finished = datetime.utcnow() job.job_status = "finished" job.processing_started = start job.processing_finished = finished dao_update_job(job) current_app.logger.info( "Job {} created at {} started at {} finished at {}".format(job_id, job.created_at, start, finished) )
def test_should_add_to_retry_queue_if_notification_not_found_in_deliver_email_task(mocker): mocker.patch('app.delivery.send_to_providers.send_email_to_provider') mocker.patch('app.celery.provider_tasks.deliver_email.retry') notification_id = app.create_uuid() deliver_email(notification_id) app.delivery.send_to_providers.send_email_to_provider.assert_not_called() app.celery.provider_tasks.deliver_email.retry.assert_called_with(queue="retry", countdown=10)
def test_should_add_to_retry_queue_if_notification_not_found_in_deliver_sms_task( notify_db_session, mocker): mocker.patch('app.delivery.send_to_providers.send_sms_to_provider') mocker.patch('app.celery.provider_tasks.deliver_sms.retry') notification_id = app.create_uuid() deliver_sms(notification_id) app.delivery.send_to_providers.send_sms_to_provider.assert_not_called() app.celery.provider_tasks.deliver_sms.retry.assert_called_with( queue="retry-tasks", countdown=0)
def test_should_add_to_retry_queue_if_notification_not_found_in_deliver_email_task( mocker): mocker.patch('app.delivery.send_to_providers.send_email_to_provider') mocker.patch('app.celery.provider_tasks.deliver_email.retry') notification_id = app.create_uuid() deliver_email(notification_id) app.delivery.send_to_providers.send_email_to_provider.assert_not_called() app.celery.provider_tasks.deliver_email.retry.assert_called_with( queue="retry-tasks")
def send_email_to_provider(notification): service = notification.service if not service.active: technical_failure(notification=notification) return if notification.status == 'created': provider = provider_to_use(EMAIL_TYPE, notification.id) current_app.logger.debug( "Starting sending EMAIL {} to provider at {}".format( notification.id, datetime.utcnow())) template_dict = dao_get_template_by_id( notification.template_id, notification.template_version).__dict__ html_email = HTMLEmailTemplate(template_dict, values=notification.personalisation, **get_html_email_options(service)) plain_text_email = PlainTextEmailTemplate( template_dict, values=notification.personalisation) if service.research_mode or notification.key_type == KEY_TYPE_TEST: reference = str(create_uuid()) notification.billable_units = 0 notification.reference = reference update_notification(notification, provider) send_email_response(reference, notification.to) else: from_address = '"{}" <{}@{}>'.format( service.name, service.email_from, current_app.config['NOTIFY_EMAIL_DOMAIN']) email_reply_to = notification.reply_to_text reference, status = provider.send_email( from_address, validate_and_format_email_address(notification.to), plain_text_email.subject, body=str(plain_text_email), html_body=str(html_email), reply_to_address=validate_and_format_email_address( email_reply_to) if email_reply_to else None, ) notification.reference = reference update_notification(notification, provider, status=status) current_app.logger.debug("SENT_MAIL: {} -- {}".format( validate_and_format_email_address(notification.to), str(plain_text_email))) current_app.logger.debug("Email {} sent to provider at {}".format( notification.id, notification.sent_at)) delta_milliseconds = (datetime.utcnow() - notification.created_at).total_seconds() * 1000 statsd_client.timing("email.total-time", delta_milliseconds)
def test_should_reject_if_notification_id_cannot_be_found(notify_api): with notify_api.test_request_context(): with notify_api.test_client() as client: auth = create_authorization_header() response = client.post( '/deliver/notification/{}'.format(app.create_uuid()), headers=[auth] ) body = json.loads(response.get_data(as_text=True)) assert response.status_code == 404 assert body['message'] == 'No result found' assert body['result'] == 'error'
def test_should_reject_if_notification_id_cannot_be_found(notify_db, notify_api): with notify_api.test_request_context(): with notify_api.test_client() as client: auth = create_authorization_header() response = client.post( '/deliver/notification/{}'.format(app.create_uuid()), headers=[auth] ) body = json.loads(response.get_data(as_text=True)) assert response.status_code == 404 assert body['message'] == 'No result found' assert body['result'] == 'error'
def send_email_to_provider(notification): service = notification.service if not service.active: technical_failure(notification=notification) return if notification.status == 'created': provider = provider_to_use(EMAIL_TYPE) template_dict = dao_get_template_by_id(notification.template_id, notification.template_version).__dict__ html_email = HTMLEmailTemplate( template_dict, values=notification.personalisation, **get_html_email_options(service) ) plain_text_email = PlainTextEmailTemplate( template_dict, values=notification.personalisation ) if service.research_mode or notification.key_type == KEY_TYPE_TEST: notification.reference = str(create_uuid()) update_notification_to_sending(notification, provider) send_email_response(notification.reference, notification.to) else: from_address = '"{}" <{}@{}>'.format(service.name, service.email_from, current_app.config['NOTIFY_EMAIL_DOMAIN']) email_reply_to = notification.reply_to_text reference = provider.send_email( from_address, validate_and_format_email_address(notification.to), plain_text_email.subject, body=str(plain_text_email), html_body=str(html_email), reply_to_address=validate_and_format_email_address(email_reply_to) if email_reply_to else None, ) notification.reference = reference update_notification_to_sending(notification, provider) delta_seconds = (datetime.utcnow() - notification.created_at).total_seconds() if notification.key_type == KEY_TYPE_TEST: statsd_client.timing("email.test-key.total-time", delta_seconds) else: statsd_client.timing("email.live-key.total-time", delta_seconds) if str(service.id) in current_app.config.get('HIGH_VOLUME_SERVICE'): statsd_client.timing("email.live-key.high-volume.total-time", delta_seconds) else: statsd_client.timing("email.live-key.not-high-volume.total-time", delta_seconds)
def send_sms_to_provider(notification): service = notification.service if not service.active: technical_failure(notification=notification) return if notification.status == 'created': provider = provider_to_use(SMS_TYPE, notification, notification.international) template_model = dao_get_template_by_id(notification.template_id, notification.template_version) template = SMSMessageTemplate( template_model.__dict__, values=notification.personalisation, prefix=service.name, show_prefix=service.prefix_sms, ) if service.research_mode or notification.key_type == KEY_TYPE_TEST: notification.reference = create_uuid() update_notification_to_sending(notification, provider) send_sms_response(provider.get_name(), str(notification.id), notification.to, notification.reference) else: try: reference = provider.send_sms( to=validate_and_format_phone_number( notification.to, international=notification.international), content=str(template), reference=str(notification.id), sender=notification.reply_to_text) except Exception as e: notification.billable_units = template.fragment_count dao_update_notification(notification) dao_toggle_sms_provider(provider.name) raise e else: notification.billable_units = template.fragment_count notification.reference = reference update_notification_to_sending(notification, provider) current_app.logger.info( f"Saved provider reference: {reference} for notification id: {notification.id}" ) delta_milliseconds = (datetime.utcnow() - notification.created_at).total_seconds() * 1000 statsd_client.timing("sms.total-time", delta_milliseconds)
def test_should_add_to_retry_queue_if_notification_not_found_in_deliver_sms_task( notify_db_session, mocker, sms_method, sms_method_name, ): mocker.patch('app.delivery.send_to_providers.send_sms_to_provider') mocker.patch(f'app.celery.provider_tasks.{sms_method_name}.retry') notification_id = app.create_uuid() sms_method(notification_id) app.delivery.send_to_providers.send_sms_to_provider.assert_not_called() getattr(app.celery.provider_tasks, sms_method_name).retry.assert_called_with(queue="retry-tasks", countdown=0)
def send_email_to_provider(notification): service = dao_fetch_service_by_id(notification.service_id) provider = provider_to_use(EMAIL_TYPE, notification.id) if notification.status == 'created': template_dict = dao_get_template_by_id(notification.template_id, notification.template_version).__dict__ html_email = HTMLEmailTemplate( template_dict, values=notification.personalisation, **get_html_email_options(service) ) plain_text_email = PlainTextEmailTemplate( template_dict, values=notification.personalisation ) if service.research_mode or notification.key_type == KEY_TYPE_TEST: reference = str(create_uuid()) send_email_response.apply_async( (provider.get_name(), reference, notification.to), queue='research-mode' ) notification.billable_units = 0 else: from_address = '"{}" <{}@{}>'.format(service.name, service.email_from, current_app.config['NOTIFY_EMAIL_DOMAIN']) reference = provider.send_email( from_address, notification.to, plain_text_email.subject, body=str(plain_text_email), html_body=str(html_email), reply_to_address=service.reply_to_email_address, ) notification.reference = reference notification.sent_at = datetime.utcnow() notification.sent_by = provider.get_name(), notification.status = 'sending' dao_update_notification(notification) current_app.logger.info( "Email {} sent to provider at {}".format(notification.id, notification.sent_at) ) delta_milliseconds = (datetime.utcnow() - notification.created_at).total_seconds() * 1000 statsd_client.timing("email.total-time", delta_milliseconds)
def process_row(row: Row, template: Template, job: Job, service: Service): template_type = template.template_type encrypted = encryption.encrypt({ "api_key": job.api_key_id and str(job.api_key_id), "template": str(template.id), "template_version": job.template_version, "job": str(job.id), "to": row.recipient, "row_number": row.index, "personalisation": dict(row.personalisation), "queue": queue_to_use(job.notification_count), }) sender_id = str(job.sender_id) if job.sender_id else None send_fns = { SMS_TYPE: save_sms, EMAIL_TYPE: save_email, LETTER_TYPE: save_letter } send_fn = send_fns[template_type] task_kwargs = {} if sender_id: task_kwargs["sender_id"] = sender_id send_fn.apply_async( ( str(service.id), create_uuid(), encrypted, ), task_kwargs, queue=QueueNames.DATABASE if not service.research_mode else QueueNames.RESEARCH_MODE, )
class TestProcessSNSDeliveryStatus: @pytest.mark.skip(reason="Endpoint disabled and slated for removal") @pytest.mark.parametrize('data', [ payload_with_missing_message_id(), payload_with_missing_status(), get_sns_delivery_status_payload(create_uuid(), "NOT_A_VALID_STATE"), get_sns_delivery_status_payload("not-uuid", SNS_STATUS_SUCCESS) ]) def test_returns_bad_request_on_schema_validation_errors( self, client, data): response = post(client, data) assert response.status_code == 400 @pytest.mark.skip(reason="Endpoint disabled and slated for removal") def test_loads_notification_by_reference( self, client, mock_notification, mock_dao_get_notification_by_reference, mock_update_notification_status, mock_process_service_callback): mock_dao_get_notification_by_reference.return_value = mock_notification post( client, get_sns_delivery_status_payload(mock_notification.reference, SNS_STATUS_SUCCESS)) mock_dao_get_notification_by_reference.assert_called_with( mock_notification.reference) @pytest.mark.skip(reason="Endpoint disabled and slated for removal") @pytest.mark.parametrize( "exception", [MultipleResultsFound(), NoResultFound()]) def test_returns_404_when_unable_to_load_notification( self, client, mock_notification, mock_dao_get_notification_by_reference, exception): mock_dao_get_notification_by_reference.side_effect = exception response = post( client, get_sns_delivery_status_payload(mock_notification.reference, SNS_STATUS_SUCCESS)) assert response.status_code == 404 @pytest.mark.skip(reason="Endpoint disabled and slated for removal") @pytest.mark.parametrize("sns_status, status", [(SNS_STATUS_SUCCESS, NOTIFICATION_SENT), (SNS_STATUS_FAILURE, NOTIFICATION_FAILED)]) def test_should_update_notification_status( self, client, mock_notification, mock_dao_get_notification_by_reference, mock_update_notification_status, mock_process_service_callback, sns_status, status): mock_dao_get_notification_by_reference.return_value = mock_notification post( client, get_sns_delivery_status_payload(mock_notification.reference, sns_status)) mock_update_notification_status.assert_called_with( mock_notification, status) @pytest.mark.skip(reason="Endpoint disabled and slated for removal") def test_should_process_service_callback( self, client, mock_notification, mock_dao_get_notification_by_reference, mock_update_notification_status, mock_process_service_callback, ): mock_dao_get_notification_by_reference.return_value = mock_notification mock_update_notification_status.return_value = mock_notification post( client, get_sns_delivery_status_payload(mock_notification.reference, SNS_STATUS_SUCCESS)) mock_process_service_callback.assert_called_with(mock_notification) @pytest.mark.skip(reason="Endpoint disabled and slated for removal") def test_should_send_callback_metrics( self, client, mock_notification, mock_dao_get_notification_by_reference, mock_update_notification_status, mock_process_service_callback, mock_send_callback_metrics): mock_dao_get_notification_by_reference.return_value = mock_notification mock_update_notification_status.return_value = mock_notification post( client, get_sns_delivery_status_payload(mock_notification.reference, SNS_STATUS_SUCCESS)) mock_send_callback_metrics.assert_called_with(mock_notification) @pytest.mark.skip(reason="Endpoint disabled and slated for removal") def test_returns_204( self, client, mock_notification, mock_dao_get_notification_by_reference, mock_update_notification_status, mock_process_service_callback, ): mock_dao_get_notification_by_reference.return_value = mock_notification mock_update_notification_status.return_value = mock_notification response = post( client, get_sns_delivery_status_payload(mock_notification.reference, SNS_STATUS_SUCCESS)) assert response.status_code == 204
def mock_notification(mocker): notification = mocker.Mock(Notification) notification.id = create_uuid() notification.reference = create_uuid() notification.sent_at = datetime.utcnow() return notification
def send_email_to_provider(notification): service = notification.service if not service.active: technical_failure(notification=notification) return if notification.status == 'created': provider = provider_to_use(EMAIL_TYPE, notification.id) # Extract any file objects from the personalization file_keys = [ k for k, v in (notification.personalisation or {}).items() if isinstance(v, dict) and 'document' in v ] attachments = [] personalisation_data = notification.personalisation.copy() for key in file_keys: # Check if a MLWR sid exists if (current_app.config["MLWR_HOST"] and 'mlwr_sid' in personalisation_data[key]['document'] and personalisation_data[key]['document']['mlwr_sid'] != "false"): mlwr_result = check_mlwr(personalisation_data[key]['document']['mlwr_sid']) if "state" in mlwr_result and mlwr_result["state"] == "completed": # Update notification that it contains malware if "submission" in mlwr_result and mlwr_result["submission"]['max_score'] >= 500: malware_failure(notification=notification) return else: # Throw error so celery will retry in sixty seconds raise MalwarePendingException try: req = urllib.request.Request(personalisation_data[key]['document']['direct_file_url']) with urllib.request.urlopen(req) as response: buffer = response.read() mime_type = magic.from_buffer(buffer, mime=True) if mime_type == 'application/pdf': attachments.append({"name": "{}.pdf".format(key), "data": buffer}) except Exception: current_app.logger.error( "Could not download and attach {}".format(personalisation_data[key]['document']['direct_file_url']) ) personalisation_data[key] = personalisation_data[key]['document']['url'] template_dict = dao_get_template_by_id(notification.template_id, notification.template_version).__dict__ html_email = HTMLEmailTemplate( template_dict, values=personalisation_data, **get_html_email_options(service) ) plain_text_email = PlainTextEmailTemplate( template_dict, values=personalisation_data ) if current_app.config["SCAN_FOR_PII"]: contains_pii(notification, str(plain_text_email)) if service.research_mode or notification.key_type == KEY_TYPE_TEST: notification.reference = str(create_uuid()) update_notification_to_sending(notification, provider) send_email_response(notification.reference, notification.to) else: if service.sending_domain is None or service.sending_domain.strip() == "": sending_domain = current_app.config['NOTIFY_EMAIL_DOMAIN'] else: sending_domain = service.sending_domain from_address = '"{}" <{}@{}>'.format(service.name, service.email_from, sending_domain) email_reply_to = notification.reply_to_text reference = provider.send_email( from_address, validate_and_format_email_address(notification.to), plain_text_email.subject, body=str(plain_text_email), html_body=str(html_email), reply_to_address=validate_and_format_email_address(email_reply_to) if email_reply_to else None, attachments=attachments ) notification.reference = reference update_notification_to_sending(notification, provider) delta_milliseconds = (datetime.utcnow() - notification.created_at).total_seconds() * 1000 statsd_client.timing("email.total-time", delta_milliseconds)
def process_sms_or_email_notification(*, form, notification_type, api_key, template, service, reply_to_text=None): form_send_to = form["email_address"] if notification_type == EMAIL_TYPE else form["phone_number"] send_to = validate_and_format_recipient( send_to=form_send_to, key_type=api_key.key_type, service=service, notification_type=notification_type, ) # Do not persist or send notification to the queue if it is a simulated recipient simulated = simulated_recipient(send_to, notification_type) personalisation = process_document_uploads(form.get("personalisation"), service, simulated, template.id) notification = { "id": create_uuid(), "template": str(template.id), "template_version": str(template.version), "to": form_send_to, "personalisation": personalisation, "simulated": simulated, "api_key": str(api_key.id), "key_type": str(api_key.key_type), "client_reference": form.get("reference", None), } encrypted_notification_data = encryption.encrypt(notification) scheduled_for = form.get("scheduled_for", None) if scheduled_for: notification = persist_notification( template_id=template.id, template_version=template.version, recipient=form_send_to, service=service, personalisation=personalisation, notification_type=notification_type, api_key_id=api_key.id, key_type=api_key.key_type, client_reference=form.get("reference", None), simulated=simulated, reply_to_text=reply_to_text, ) persist_scheduled_notification(notification.id, form["scheduled_for"]) elif current_app.config["FF_NOTIFICATION_CELERY_PERSISTENCE"] and not simulated: # depending on the type route to the appropriate save task if notification_type == EMAIL_TYPE: current_app.logger.info("calling save email task") save_email.apply_async( (authenticated_service.id, create_uuid(), encrypted_notification_data), queue=QueueNames.DATABASE if not authenticated_service.research_mode else QueueNames.RESEARCH_MODE, ) elif notification_type == SMS_TYPE: save_sms.apply_async( (authenticated_service.id, create_uuid(), encrypted_notification_data), queue=QueueNames.DATABASE if not authenticated_service.research_mode else QueueNames.RESEARCH_MODE, ) else: notification = persist_notification( template_id=template.id, template_version=template.version, recipient=form_send_to, service=service, personalisation=personalisation, notification_type=notification_type, api_key_id=api_key.id, key_type=api_key.key_type, client_reference=form.get("reference", None), simulated=simulated, reply_to_text=reply_to_text, ) if not simulated: send_notification_to_queue( notification=notification, research_mode=service.research_mode, queue=template.queue_to_use(), ) else: current_app.logger.debug("POST simulated notification for id: {}".format(notification.id)) if not isinstance(notification, Notification): notification["template_id"] = notification["template"] notification["api_key_id"] = notification["api_key"] notification["template_version"] = template.version notification["service"] = service notification["service_id"] = service.id notification["reply_to_text"] = reply_to_text del notification["template"] del notification["api_key"] del notification["simulated"] notification = Notification(**notification) return notification
def save_service_inbound_api(service_inbound_api): service_inbound_api.id = create_uuid() service_inbound_api.created_at = datetime.utcnow() db.session.add(service_inbound_api)
def test_should_reject_if_not_authenticated(notify_api): with notify_api.test_request_context(): with notify_api.test_client() as client: response = client.post('/deliver/notification/{}'.format(app.create_uuid())) assert response.status_code == 401
def send_email_to_provider(notification): service = notification.service if not service.active: technical_failure(notification=notification) return # TODO: no else - replace with if statement raising error / logging when not 'created' if notification.status == 'created': provider = provider_to_use(notification) # TODO: remove that code or extract attachment handling to separate method # Extract any file objects from the personalization file_keys = [ k for k, v in (notification.personalisation or {}).items() if isinstance(v, dict) and 'document' in v ] attachments = [] personalisation_data = notification.personalisation.copy() for key in file_keys: # Check if a MLWR sid exists if (current_app.config["MLWR_HOST"] and 'mlwr_sid' in personalisation_data[key]['document'] and personalisation_data[key]['document']['mlwr_sid'] != "false"): mlwr_result = check_mlwr( personalisation_data[key]['document']['mlwr_sid']) if "state" in mlwr_result and mlwr_result[ "state"] == "completed": # Update notification that it contains malware if "submission" in mlwr_result and mlwr_result[ "submission"]['max_score'] >= 500: malware_failure(notification=notification) return else: # Throw error so celery will retry in sixty seconds raise MalwarePendingException try: response = requests.get( personalisation_data[key]['document']['direct_file_url']) if response.headers['Content-Type'] == 'application/pdf': attachments.append({ "name": "{}.pdf".format(key), "data": response.content }) except Exception: current_app.logger.error( "Could not download and attach {}".format( personalisation_data[key]['document'] ['direct_file_url'])) personalisation_data[key] = personalisation_data[key]['document'][ 'url'] template_dict = dao_get_template_by_id( notification.template_id, notification.template_version).__dict__ html_email = HTMLEmailTemplate(template_dict, values=personalisation_data, **get_html_email_options( notification, provider)) plain_text_email = PlainTextEmailTemplate(template_dict, values=personalisation_data) if current_app.config["SCAN_FOR_PII"]: contains_pii(notification, str(plain_text_email)) if service.research_mode or notification.key_type == KEY_TYPE_TEST: notification.reference = str(create_uuid()) update_notification_to_sending(notification, provider) send_email_response(notification.reference, notification.to) else: email_reply_to = notification.reply_to_text reference = provider.send_email( source=compute_source_email_address(service, provider), to_addresses=validate_and_format_email_address( notification.to), subject=plain_text_email.subject, body=str(plain_text_email), html_body=str(html_email), reply_to_address=validate_and_format_email_address( email_reply_to) if email_reply_to else None, attachments=attachments) notification.reference = reference update_notification_to_sending(notification, provider) current_app.logger.info( f"Saved provider reference: {reference} for notification id: {notification.id}" ) delta_milliseconds = (datetime.utcnow() - notification.created_at).total_seconds() * 1000 statsd_client.timing("email.total-time", delta_milliseconds)