def send_email_to_provider(notification): service = notification.service if not service.active: technical_failure(notification=notification) return if notification.status == 'created': provider = provider_to_use(EMAIL_TYPE, notification.id) current_app.logger.debug( "Starting sending EMAIL {} to provider at {}".format(notification.id, datetime.utcnow()) ) template_dict = dao_get_template_by_id(notification.template_id, notification.template_version).__dict__ html_email = HTMLEmailTemplate( template_dict, values=notification.personalisation, **get_html_email_options(service) ) plain_text_email = PlainTextEmailTemplate( template_dict, values=notification.personalisation ) if service.research_mode or notification.key_type == KEY_TYPE_TEST: reference = str(create_uuid()) notification.billable_units = 0 notification.reference = reference update_notification(notification, provider) send_email_response(reference, notification.to) else: from_address = '"{}" <{}@{}>'.format(service.name, service.email_from, current_app.config['NOTIFY_EMAIL_DOMAIN']) email_reply_to = notification.reply_to_text reference, status = provider.send_email( from_address, validate_and_format_email_address(notification.to), plain_text_email.subject, body=str(plain_text_email), html_body=str(html_email), reply_to_address=validate_and_format_email_address(email_reply_to) if email_reply_to else None, ) notification.reference = reference update_notification(notification, provider, status=status) current_app.logger.debug("SENT_MAIL: {} -- {}".format( validate_and_format_email_address(notification.to), str(plain_text_email)) ) current_app.logger.debug( "Email {} sent to provider at {}".format(notification.id, notification.sent_at) ) delta_milliseconds = (datetime.utcnow() - notification.created_at).total_seconds() * 1000 statsd_client.timing("email.total-time", delta_milliseconds)
def send_email_to_provider(notification): service = notification.service if not service.active: technical_failure(notification=notification) return if notification.status == 'created': provider = provider_to_use(EMAIL_TYPE) template_dict = dao_get_template_by_id( notification.template_id, notification.template_version).__dict__ html_email = HTMLEmailTemplate(template_dict, values=notification.personalisation, **get_html_email_options(service)) plain_text_email = PlainTextEmailTemplate( template_dict, values=notification.personalisation) if service.research_mode or notification.key_type == KEY_TYPE_TEST: notification.reference = str(create_uuid()) update_notification_to_sending(notification, provider) send_email_response(notification.reference, notification.to) else: from_address = '"{}" <{}@{}>'.format( service.name, service.email_from, current_app.config['NOTIFY_EMAIL_DOMAIN']) email_reply_to = notification.reply_to_text reference = provider.send_email( from_address, validate_and_format_email_address(notification.to), plain_text_email.subject, body=str(plain_text_email), html_body=str(html_email), reply_to_address=validate_and_format_email_address( email_reply_to) if email_reply_to else None, ) notification.reference = reference update_notification_to_sending(notification, provider) delta_seconds = (datetime.utcnow() - notification.created_at).total_seconds() if notification.key_type == KEY_TYPE_TEST: statsd_client.timing("email.test-key.total-time", delta_seconds) else: statsd_client.timing("email.live-key.total-time", delta_seconds) if str(service.id) in current_app.config.get( 'HIGH_VOLUME_SERVICE'): statsd_client.timing("email.live-key.high-volume.total-time", delta_seconds) else: statsd_client.timing( "email.live-key.not-high-volume.total-time", delta_seconds)
def validate_and_format_recipient(send_to, key_type, service, notification_type, allow_safelisted_recipients=True): if send_to is None: raise BadRequestError(message="Recipient can't be empty") service_can_send_to_recipient(send_to, key_type, service, allow_safelisted_recipients) if notification_type == SMS_TYPE: international_phone_info = get_international_phone_info(send_to) if international_phone_info.international and INTERNATIONAL_SMS_TYPE not in [ p.permission for p in service.permissions ]: raise BadRequestError( message="Cannot send to international mobile numbers") return validate_and_format_phone_number( number=send_to, international=international_phone_info.international) elif notification_type == EMAIL_TYPE: return validate_and_format_email_address(email_address=send_to)
def dao_get_notifications_by_recipient_or_reference( service_id, search_term, notification_type=None, statuses=None, page=1, page_size=None, ): if notification_type == SMS_TYPE: normalised = try_validate_and_format_phone_number(search_term) for character in {'(', ')', ' ', '-'}: normalised = normalised.replace(character, '') normalised = normalised.lstrip('+0') elif notification_type == EMAIL_TYPE: try: normalised = validate_and_format_email_address(search_term) except InvalidEmailError: normalised = search_term.lower() elif notification_type in {LETTER_TYPE, None}: # For letters, we store the address without spaces, so we need # to removes spaces from the search term to match. We also do # this when a notification type isn’t provided (this will # happen if a user doesn’t have permission to see the dashboard) # because email addresses and phone numbers will never be stored # with spaces either. normalised = ''.join(search_term.split()).lower() else: raise TypeError( f'Notification type must be {EMAIL_TYPE}, {SMS_TYPE}, {LETTER_TYPE} or None' ) normalised = escape_special_characters(normalised) search_term = escape_special_characters(search_term) filters = [ Notification.service_id == service_id, or_( Notification.normalised_to.like("%{}%".format(normalised)), Notification.client_reference.ilike("%{}%".format(search_term)), ), Notification.key_type != KEY_TYPE_TEST, ] if statuses: filters.append(Notification.status.in_(statuses)) if notification_type: filters.append(Notification.notification_type == notification_type) results = db.session.query(Notification)\ .filter(*filters)\ .order_by(desc(Notification.created_at))\ .paginate(page=page, per_page=page_size) return results
def validate_and_format_recipient(send_to, key_type, service, notification_type, allow_guest_list_recipients=True): if send_to is None: raise BadRequestError(message="Recipient can't be empty") service_can_send_to_recipient(send_to, key_type, service, allow_guest_list_recipients) if notification_type == SMS_TYPE: international_phone_info = check_if_service_can_send_to_number(service, send_to) return validate_and_format_phone_number( number=send_to, international=international_phone_info.international ) elif notification_type == EMAIL_TYPE: return validate_and_format_email_address(email_address=send_to)
def dao_get_notifications_by_to_field(service_id, search_term, notification_type=None, statuses=None): if notification_type is None: notification_type = guess_notification_type(search_term) if notification_type == SMS_TYPE: normalised = try_validate_and_format_phone_number(search_term) for character in {'(', ')', ' ', '-'}: normalised = normalised.replace(character, '') normalised = normalised.lstrip('+0') elif notification_type == EMAIL_TYPE: try: normalised = validate_and_format_email_address(search_term) except InvalidEmailError: normalised = search_term.lower() else: raise InvalidRequest("Only email and SMS can use search by recipient", 400) normalised = escape_special_characters(normalised) filters = [ Notification.service_id == service_id, Notification.normalised_to.like("%{}%".format(normalised)), Notification.key_type != KEY_TYPE_TEST, ] if statuses: filters.append(Notification.status.in_(statuses)) if notification_type: filters.append(Notification.notification_type == notification_type) results = db.session.query(Notification).filter(*filters).order_by( desc(Notification.created_at)).all() return results
def test_simulated_recipient(notify_api, to_address, notification_type, expected): """ The values where the expected = 'research-mode' are listed in the config['SIMULATED_EMAIL_ADDRESSES'] and config['SIMULATED_SMS_NUMBERS']. These values should result in using the research mode queue. SIMULATED_EMAIL_ADDRESSES = ( '*****@*****.**', '*****@*****.**', '*****@*****.**' ) SIMULATED_SMS_NUMBERS = ('6132532222', '+16132532222', '+16132532223') """ formatted_address = None if notification_type == 'email': formatted_address = validate_and_format_email_address(to_address) else: formatted_address = validate_and_format_phone_number(to_address) is_simulated_address = simulated_recipient(formatted_address, notification_type) assert is_simulated_address == expected
def test_simulated_recipient(notify_api, to_address, notification_type, expected): """ The values where the expected = 'research-mode' are listed in the config['SIMULATED_EMAIL_ADDRESSES'] and config['SIMULATED_SMS_NUMBERS']. These values should result in using the research mode queue. SIMULATED_EMAIL_ADDRESSES = ( '*****@*****.**', '*****@*****.**', '*****@*****.**' ) SIMULATED_SMS_NUMBERS = ('+61400900000', '+61400900111', '+61400900222', '+61426305772', '+61426305773', '+61426305774') """ formatted_address = None if notification_type == 'email': formatted_address = validate_and_format_email_address(to_address) else: formatted_address = validate_and_format_phone_number_and_allow_international( to_address) is_simulated_address = simulated_recipient(formatted_address, notification_type) assert is_simulated_address == expected
def send_email_to_provider(notification): service = notification.service if not service.active: technical_failure(notification=notification) return # TODO: no else - replace with if statement raising error / logging when not 'created' if notification.status == 'created': provider = provider_to_use(notification) # TODO: remove that code or extract attachment handling to separate method # Extract any file objects from the personalization file_keys = [ k for k, v in (notification.personalisation or {}).items() if isinstance(v, dict) and 'document' in v ] attachments = [] personalisation_data = notification.personalisation.copy() for key in file_keys: # Check if a MLWR sid exists if (current_app.config["MLWR_HOST"] and 'mlwr_sid' in personalisation_data[key]['document'] and personalisation_data[key]['document']['mlwr_sid'] != "false"): mlwr_result = check_mlwr( personalisation_data[key]['document']['mlwr_sid']) if "state" in mlwr_result and mlwr_result[ "state"] == "completed": # Update notification that it contains malware if "submission" in mlwr_result and mlwr_result[ "submission"]['max_score'] >= 500: malware_failure(notification=notification) return else: # Throw error so celery will retry in sixty seconds raise MalwarePendingException try: response = requests.get( personalisation_data[key]['document']['direct_file_url']) if response.headers['Content-Type'] == 'application/pdf': attachments.append({ "name": "{}.pdf".format(key), "data": response.content }) except Exception: current_app.logger.error( "Could not download and attach {}".format( personalisation_data[key]['document'] ['direct_file_url'])) personalisation_data[key] = personalisation_data[key]['document'][ 'url'] template_dict = dao_get_template_by_id( notification.template_id, notification.template_version).__dict__ html_email = HTMLEmailTemplate(template_dict, values=personalisation_data, **get_html_email_options( notification, provider)) plain_text_email = PlainTextEmailTemplate(template_dict, values=personalisation_data) if current_app.config["SCAN_FOR_PII"]: contains_pii(notification, str(plain_text_email)) if service.research_mode or notification.key_type == KEY_TYPE_TEST: notification.reference = str(create_uuid()) update_notification_to_sending(notification, provider) send_email_response(notification.reference, notification.to) else: email_reply_to = notification.reply_to_text reference = provider.send_email( source=compute_source_email_address(service, provider), to_addresses=validate_and_format_email_address( notification.to), subject=plain_text_email.subject, body=str(plain_text_email), html_body=str(html_email), reply_to_address=validate_and_format_email_address( email_reply_to) if email_reply_to else None, attachments=attachments) notification.reference = reference update_notification_to_sending(notification, provider) current_app.logger.info( f"Saved provider reference: {reference} for notification id: {notification.id}" ) delta_milliseconds = (datetime.utcnow() - notification.created_at).total_seconds() * 1000 statsd_client.timing("email.total-time", delta_milliseconds)
def send_email_to_provider(notification): service = notification.service if not service.active: technical_failure(notification=notification) return if notification.status == 'created': provider = provider_to_use(EMAIL_TYPE, notification.id) # Extract any file objects from the personalization file_keys = [ k for k, v in (notification.personalisation or {}).items() if isinstance(v, dict) and 'document' in v ] attachments = [] personalisation_data = notification.personalisation.copy() for key in file_keys: # Check if a MLWR sid exists if (current_app.config["MLWR_HOST"] and 'mlwr_sid' in personalisation_data[key]['document'] and personalisation_data[key]['document']['mlwr_sid'] != "false"): mlwr_result = check_mlwr(personalisation_data[key]['document']['mlwr_sid']) if "state" in mlwr_result and mlwr_result["state"] == "completed": # Update notification that it contains malware if "submission" in mlwr_result and mlwr_result["submission"]['max_score'] >= 500: malware_failure(notification=notification) return else: # Throw error so celery will retry in sixty seconds raise MalwarePendingException try: req = urllib.request.Request(personalisation_data[key]['document']['direct_file_url']) with urllib.request.urlopen(req) as response: buffer = response.read() mime_type = magic.from_buffer(buffer, mime=True) if mime_type == 'application/pdf': attachments.append({"name": "{}.pdf".format(key), "data": buffer}) except Exception: current_app.logger.error( "Could not download and attach {}".format(personalisation_data[key]['document']['direct_file_url']) ) personalisation_data[key] = personalisation_data[key]['document']['url'] template_dict = dao_get_template_by_id(notification.template_id, notification.template_version).__dict__ html_email = HTMLEmailTemplate( template_dict, values=personalisation_data, **get_html_email_options(service) ) plain_text_email = PlainTextEmailTemplate( template_dict, values=personalisation_data ) if current_app.config["SCAN_FOR_PII"]: contains_pii(notification, str(plain_text_email)) if service.research_mode or notification.key_type == KEY_TYPE_TEST: notification.reference = str(create_uuid()) update_notification_to_sending(notification, provider) send_email_response(notification.reference, notification.to) else: if service.sending_domain is None or service.sending_domain.strip() == "": sending_domain = current_app.config['NOTIFY_EMAIL_DOMAIN'] else: sending_domain = service.sending_domain from_address = '"{}" <{}@{}>'.format(service.name, service.email_from, sending_domain) email_reply_to = notification.reply_to_text reference = provider.send_email( from_address, validate_and_format_email_address(notification.to), plain_text_email.subject, body=str(plain_text_email), html_body=str(html_email), reply_to_address=validate_and_format_email_address(email_reply_to) if email_reply_to else None, attachments=attachments ) notification.reference = reference update_notification_to_sending(notification, provider) delta_milliseconds = (datetime.utcnow() - notification.created_at).total_seconds() * 1000 statsd_client.timing("email.total-time", delta_milliseconds)
def send_email_to_provider(notification): service = notification.service if not service.active: technical_failure(notification=notification) return if notification.status == "created": provider = provider_to_use(EMAIL_TYPE, notification.id) # Extract any file objects from the personalization file_keys = [ k for k, v in (notification.personalisation or {}).items() if isinstance(v, dict) and "document" in v ] attachments = [] personalisation_data = notification.personalisation.copy() for key in file_keys: sending_method = personalisation_data[key]["document"].get( "sending_method") # Check if a MLWR sid exists if (current_app.config["MLWR_HOST"] and "mlwr_sid" in personalisation_data[key]["document"] and personalisation_data[key]["document"]["mlwr_sid"] != "false"): mlwr_result = check_mlwr( personalisation_data[key]["document"]["mlwr_sid"]) if "state" in mlwr_result and mlwr_result[ "state"] == "completed": # Update notification that it contains malware if "submission" in mlwr_result and mlwr_result[ "submission"]["max_score"] >= 500: malware_failure(notification=notification) return else: # Throw error so celery will retry in sixty seconds raise MalwarePendingException if sending_method == "attach": try: req = urllib.request.Request( personalisation_data[key]["document"] ["direct_file_url"]) with urllib.request.urlopen(req) as response: buffer = response.read() filename = personalisation_data[key]["document"].get( "filename") mime_type = personalisation_data[key]["document"].get( "mime_type") attachments.append({ "name": filename, "data": buffer, "mime_type": mime_type, }) except Exception: current_app.logger.error( "Could not download and attach {}".format( personalisation_data[key]["document"] ["direct_file_url"])) del personalisation_data[key] else: personalisation_data[key] = personalisation_data[key][ "document"]["url"] template_dict = dao_get_template_by_id( notification.template_id, notification.template_version).__dict__ # Local Jinja support - Add USE_LOCAL_JINJA_TEMPLATES=True to .env # Add a folder to the project root called 'jinja_templates' # with a copy of 'email_template.jinja2' from notification-utils repo debug_template_path = ( os.path.dirname(os.path.dirname(os.path.abspath(__file__))) if os.environ.get("USE_LOCAL_JINJA_TEMPLATES") == "True" else None) html_email = HTMLEmailTemplate( template_dict, values=personalisation_data, jinja_path=debug_template_path, **get_html_email_options(service), ) plain_text_email = PlainTextEmailTemplate(template_dict, values=personalisation_data) if current_app.config["SCAN_FOR_PII"]: contains_pii(notification, str(plain_text_email)) if service.research_mode or notification.key_type == KEY_TYPE_TEST: notification.reference = send_email_response(notification.to) update_notification_to_sending(notification, provider) else: if service.sending_domain is None or service.sending_domain.strip( ) == "": sending_domain = current_app.config["NOTIFY_EMAIL_DOMAIN"] else: sending_domain = service.sending_domain from_address = '"{}" <{}@{}>'.format(service.name, service.email_from, sending_domain) email_reply_to = notification.reply_to_text reference = provider.send_email( from_address, validate_and_format_email_address(notification.to), plain_text_email.subject, body=str(plain_text_email), html_body=str(html_email), reply_to_address=validate_and_format_email_address( email_reply_to) if email_reply_to else None, attachments=attachments, ) notification.reference = reference update_notification_to_sending(notification, provider) # Record StatsD stats to compute SLOs statsd_client.timing_with_dates("email.total-time", notification.sent_at, notification.created_at) attachments_category = "with-attachments" if attachments else "no-attachments" statsd_key = f"email.{attachments_category}.process_type-{template_dict['process_type']}" statsd_client.timing_with_dates(statsd_key, notification.sent_at, notification.created_at) statsd_client.incr(statsd_key)