def send_broadcast_event(broadcast_event_id):
broadcast_event = dao_get_broadcast_event_by_id(broadcast_event_id)
if (current_app.config['NOTIFY_ENVIRONMENT'] == 'live' and
broadcast_event.message_type == BroadcastEventMessageType.ALERT):
broadcast_message = broadcast_event.broadcast_message
# raise a P1 to alert team that broadcast is going out.
message = '\n'.join([
'Broadcast Sent',
'',
f'https://www.notifications.service.gov.uk/services/{broadcast_message.service_id}/current-alerts/{broadcast_message.id}', # noqa
'',
f'This broacast has been sent on channel {broadcast_message.service.broadcast_channel}.',
f'This broadcast is targeted at areas {broadcast_message.areas.get("areas")}.',
''
f'This broadcast\'s content starts "{broadcast_message.content[:100]}"'
'',
'If this alert is not expected refer to the runbook for instructions.',
'https://docs.google.com/document/d/1J99yOlfp4nQz6et0w5oJVqi-KywtIXkxrEIyq_g2XUs',
])
zendesk_client.create_ticket(
subject="Live broadcast sent",
message=message,
ticket_type=zendesk_client.TYPE_INCIDENT,
p1=True,
)
for provider in broadcast_event.service.get_available_broadcast_providers(
):
send_broadcast_provider_message.apply_async(
kwargs={
'broadcast_event_id': broadcast_event_id,
'provider': provider
},
queue=QueueNames.BROADCASTS)
def raise_alert_if_letter_notifications_still_sending():
today = datetime.utcnow().date()
# Do nothing on the weekend
if today.isoweekday() in [6, 7]:
return
if today.isoweekday() in [1, 2]:
offset_days = 4
else:
offset_days = 2
still_sending = Notification.query.filter(
Notification.notification_type == LETTER_TYPE,
Notification.status == NOTIFICATION_SENDING,
Notification.key_type == KEY_TYPE_NORMAL,
func.date(Notification.sent_at) <= today - timedelta(days=offset_days),
).count()
if still_sending:
message = "There are {} letters in the 'sending' state from {}".format(
still_sending,
(today - timedelta(days=offset_days)).strftime("%A %d %B"))
# Only send alerts in production
if current_app.config["NOTIFY_ENVIRONMENT"] in [
"live", "production", "test"
]:
zendesk_client.create_ticket(
subject="[{}] Letters still sending".format(
current_app.config["NOTIFY_ENVIRONMENT"]),
message=message,
ticket_type=zendesk_client.TYPE_INCIDENT,
)
else:
current_app.logger.info(message)
def branding_request(service_id):
form = BrandingOptionsEmail(options=current_service['branding'])
if form.validate_on_submit():
zendesk_client.create_ticket(
subject='Email branding request - {}'.format(
current_service['name']),
message=('Organisation: {}\n'
'Service: {}\n'
'{}\n'
'\n---'
'\nBranding requested: {}').format(
AgreementInfo.from_current_user().
as_info_for_branding_request,
current_service['name'],
url_for('main.service_dashboard',
service_id=current_service['id'],
_external=True),
branding_options_dict[form.options.data],
),
ticket_type=zendesk_client.TYPE_QUESTION,
user_email=current_user.email_address,
user_name=current_user.name,
)
flash(('Thanks for your branding request. We’ll get back to you '
'within one working day.'), 'default')
return redirect(url_for('.service_settings', service_id=service_id))
return render_template(
'views/service-settings/branding/email-options.html',
form=form,
)
def letter_raise_alert_if_no_ack_file_for_zip():
# get a list of zip files since yesterday
zip_file_set = set()
today_str = datetime.utcnow().strftime("%Y-%m-%d")
yesterday = datetime.now(tz=pytz.utc) - timedelta(
days=1) # AWS datetime format
for key in s3.get_list_of_files_by_suffix(
bucket_name=current_app.config["LETTERS_PDF_BUCKET_NAME"],
subfolder=today_str + "/zips_sent",
suffix=".TXT",
):
subname = key.split("/")[-1] # strip subfolder in name
zip_file_set.add(subname.upper().replace(".ZIP.TXT", ""))
# get acknowledgement file
ack_file_set = set()
for key in s3.get_list_of_files_by_suffix(
bucket_name=current_app.config["DVLA_RESPONSE_BUCKET_NAME"],
subfolder="root/dispatch",
suffix=".ACK.txt",
last_modified=yesterday,
):
ack_file_set.add(
key.lstrip("root/dispatch").upper().replace(".ACK.TXT", ""))
message = ("Letter ack file does not contain all zip files sent. "
"Missing ack for zip files: {}, "
"pdf bucket: {}, subfolder: {}, "
"ack bucket: {}").format(
str(sorted(zip_file_set - ack_file_set)),
current_app.config["LETTERS_PDF_BUCKET_NAME"],
datetime.utcnow().strftime("%Y-%m-%d") + "/zips_sent",
current_app.config["DVLA_RESPONSE_BUCKET_NAME"],
)
# strip empty element before comparison
ack_file_set.discard("")
zip_file_set.discard("")
if len(zip_file_set - ack_file_set) > 0:
if current_app.config["NOTIFY_ENVIRONMENT"] in [
"live", "production", "test"
]:
zendesk_client.create_ticket(
subject="Letter acknowledge error",
message=message,
ticket_type=zendesk_client.TYPE_INCIDENT,
)
current_app.logger.error(message)
if len(ack_file_set - zip_file_set) > 0:
current_app.logger.info(
"letter ack contains zip that is not for today: {}".format(
ack_file_set - zip_file_set))
def submit_request_to_go_live(service_id):
form = RequestToGoLiveForm()
if form.validate_on_submit():
zendesk_client.create_ticket(
subject='Request to go live - {}'.format(current_service['name']),
message=(
'Service: {}\n'
'{}\n'
'\n---'
'\nOrganisation type: {}'
'\nAgreement signed: {}'
'\nChannel: {}\nStart date: {}\nStart volume: {}'
'\nPeak volume: {}'
'\nFeatures: {}').format(
current_service['name'],
url_for('main.service_dashboard',
service_id=current_service['id'],
_external=True),
current_service['organisation_type'],
AgreementInfo.from_current_user().as_human_readable,
formatted_list(filter(None, (
'email' if form.channel_email.data else None,
'text messages' if form.channel_sms.data else None,
'letters' if form.channel_letter.data else None,
)),
before_each='',
after_each=''), form.start_date.data,
form.start_volume.data, form.peak_volume.data,
formatted_list(filter(None, (
'one off' if form.method_one_off.data else None,
'file upload' if form.method_upload.data else None,
'API' if form.method_api.data else None,
)),
before_each='',
after_each='')),
ticket_type=zendesk_client.TYPE_QUESTION,
user_email=current_user.email_address,
user_name=current_user.name)
flash(
'Thanks for your request to go live. We’ll get back to you within one working day.',
'default')
return redirect(url_for('.service_settings', service_id=service_id))
return render_template(
'views/service-settings/submit-request-to-go-live.html', form=form)
def check_for_services_with_high_failure_rates_or_sending_to_tv_numbers():
start_date = (datetime.utcnow() - timedelta(days=1))
end_date = datetime.utcnow()
message = ""
services_with_failures = dao_find_services_with_high_failure_rates(
start_date=start_date, end_date=end_date)
services_sending_to_tv_numbers = dao_find_services_sending_to_tv_numbers(
start_date=start_date, end_date=end_date)
if services_with_failures:
message += "{} service(s) have had high permanent-failure rates for sms messages in last 24 hours:\n".format(
len(services_with_failures))
for service in services_with_failures:
service_dashboard = "{}/services/{}".format(
current_app.config['ADMIN_BASE_URL'],
str(service.service_id),
)
message += "service: {} failure rate: {},\n".format(
service_dashboard, service.permanent_failure_rate)
elif services_sending_to_tv_numbers:
message += "{} service(s) have sent over 500 sms messages to tv numbers in last 24 hours:\n".format(
len(services_sending_to_tv_numbers))
for service in services_sending_to_tv_numbers:
service_dashboard = "{}/services/{}".format(
current_app.config['ADMIN_BASE_URL'],
str(service.service_id),
)
message += "service: {} count of sms to tv numbers: {},\n".format(
service_dashboard, service.notification_count)
if services_with_failures or services_sending_to_tv_numbers:
current_app.logger.warning(message)
if current_app.config['NOTIFY_ENVIRONMENT'] in [
'live', 'production', 'test'
]:
message += (
"\nYou can find instructions for this ticket in our manual:\n"
"https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#Deal-with-services-with-high-failure-rates-or-sending-sms-to-tv-numbers"
) # noqa
zendesk_client.create_ticket(
subject="[{}] High failure rates for sms spotted for services".
format(current_app.config['NOTIFY_ENVIRONMENT']),
message=message,
ticket_type=zendesk_client.TYPE_INCIDENT)
def raise_alert_if_letter_notifications_still_sending():
still_sending_count, sent_date = get_letter_notifications_still_sending_when_they_shouldnt_be()
if still_sending_count:
message = "There are {} letters in the 'sending' state from {}".format(
still_sending_count,
sent_date.strftime('%A %d %B')
)
# Only send alerts in production
if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']:
message += ". Resolve using https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#deal-with-letters-still-in-sending" # noqa
zendesk_client.create_ticket(
subject="[{}] Letters still sending".format(current_app.config['NOTIFY_ENVIRONMENT']),
message=message,
ticket_type=zendesk_client.TYPE_INCIDENT
)
else:
current_app.logger.info(message)
def check_if_letters_still_pending_virus_check():
letters = dao_precompiled_letters_still_pending_virus_check()
if len(letters) > 0:
letter_ids = [(str(letter.id), letter.reference) for letter in letters]
msg = """{} precompiled letters have been pending-virus-check for over 90 minutes. Follow runbook to resolve:
https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#Deal-with-letter-pending-virus-scan-for-90-minutes.
Notifications: {}""".format(len(letters), sorted(letter_ids))
current_app.logger.warning(msg)
if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']:
zendesk_client.create_ticket(
subject="[{}] Letters still pending virus check".format(current_app.config['NOTIFY_ENVIRONMENT']),
message=msg,
ticket_type=zendesk_client.TYPE_INCIDENT
)
def check_templated_letter_state():
letters = dao_old_letters_with_created_status()
if len(letters) > 0:
letter_ids = [str(letter.id) for letter in letters]
msg = "{} letters were created before 17.30 yesterday and still have 'created' status. " \
"Notifications: {}".format(len(letters), letter_ids)
current_app.logger.exception(msg)
if current_app.config['NOTIFY_ENVIRONMENT'] in [
'live', 'production', 'test'
]:
zendesk_client.create_ticket(
subject="[{}] Letters still in 'created' status".format(
current_app.config['NOTIFY_ENVIRONMENT']),
message=msg,
ticket_type=zendesk_client.TYPE_INCIDENT)
def check_precompiled_letter_state():
letters = dao_precompiled_letters_still_pending_virus_check()
if len(letters) > 0:
letter_ids = [str(letter.id) for letter in letters]
msg = "{} precompiled letters have been pending-virus-check for over 90 minutes. " \
"Notifications: {}".format(len(letters), letter_ids)
current_app.logger.exception(msg)
if current_app.config['NOTIFY_ENVIRONMENT'] in [
'live', 'production', 'test'
]:
zendesk_client.create_ticket(
subject="[{}] Letters still pending virus check".format(
current_app.config['NOTIFY_ENVIRONMENT']),
message=msg,
ticket_type=zendesk_client.TYPE_INCIDENT)
def check_if_letters_still_in_created():
letters = dao_old_letters_with_created_status()
if len(letters) > 0:
msg = "{} letters were created before 17.30 yesterday and still have 'created' status. " \
"Follow runbook to resolve: " \
"https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook" \
"#deal-with-Letters-still-in-created.".format(len(letters))
current_app.logger.warning(msg)
if current_app.config['NOTIFY_ENVIRONMENT'] in [
'live', 'production', 'test'
]:
zendesk_client.create_ticket(
subject="[{}] Letters still in 'created' status".format(
current_app.config['NOTIFY_ENVIRONMENT']),
message=msg,
ticket_type=zendesk_client.TYPE_INCIDENT)
def raise_alert_if_letter_notifications_still_sending():
today = datetime.utcnow().date()
# Do nothing on the weekend
if today.isoweekday() in {6, 7}: # sat, sun
return
if today.isoweekday() in {
1, 2
}: # mon, tues. look for files from before the weekend
offset_days = 4
else:
offset_days = 2
q = Notification.query.filter(
Notification.notification_type == LETTER_TYPE,
Notification.status == NOTIFICATION_SENDING,
Notification.key_type == KEY_TYPE_NORMAL,
func.date(Notification.sent_at) <= today - timedelta(days=offset_days))
if today.isoweekday() in {
2, 4
}: # on tue, thu, we only care about first class letters
q = q.filter(Notification.postage == 'first')
still_sending = q.count()
if still_sending:
message = "There are {} letters in the 'sending' state from {}".format(
still_sending,
(today - timedelta(days=offset_days)).strftime('%A %d %B'))
# Only send alerts in production
if current_app.config['NOTIFY_ENVIRONMENT'] in [
'live', 'production', 'test'
]:
zendesk_client.create_ticket(
subject="[{}] Letters still sending".format(
current_app.config['NOTIFY_ENVIRONMENT']),
message=message,
ticket_type=zendesk_client.TYPE_INCIDENT)
else:
current_app.logger.info(message)
def feedback(ticket_type):
try:
form = {
QUESTION_TICKET_TYPE: Feedback,
PROBLEM_TICKET_TYPE: Problem,
}[ticket_type]()
except KeyError:
abort(404)
if not form.feedback.data:
form.feedback.data = session.pop('feedback_message', '')
if request.args.get('severe') in ['yes', 'no']:
severe = convert_to_boolean(request.args.get('severe'))
else:
severe = None
p1 = False
urgent = False
if in_business_hours():
# if we're in the office, it's urgent (aka we'll get back in 30 mins)
urgent = True
elif ticket_type == PROBLEM_TICKET_TYPE and severe:
# out of hours, it's only a p1 and it's only urgent if it's a p1
urgent = True
p1 = True
anonymous = (
(not form.email_address.data) and
(not current_user.is_authenticated)
)
if needs_triage(ticket_type, severe):
session['feedback_message'] = form.feedback.data
return redirect(url_for('.triage'))
if needs_escalation(ticket_type, severe):
return redirect(url_for('.bat_phone'))
if current_user.is_authenticated:
form.email_address.data = current_user.email_address
form.name.data = current_user.name
if form.validate_on_submit():
user_email = form.email_address.data
user_name = form.name.data or None
if current_service:
service_string = 'Service: "{name}"\n{url}\n'.format(
name=current_service['name'],
url=url_for('main.service_dashboard', service_id=current_service['id'], _external=True)
)
else:
service_string = ''
feedback_msg = '{}\n{}{}'.format(
form.feedback.data,
service_string,
'' if user_email else '{} (no email address supplied)'.format(form.name.data)
)
zendesk_client.create_ticket(
subject='Notify feedback',
message=feedback_msg,
ticket_type=ticket_type,
p1=p1,
user_email=user_email,
user_name=user_name
)
return redirect(url_for('.thanks', urgent=urgent, anonymous=anonymous))
if not form.feedback.data:
form.feedback.data = get_prefilled_message()
return render_template(
'views/support/{}.html'.format(ticket_type),
form=form,
ticket_type=ticket_type,
)
def letter_raise_alert_if_no_ack_file_for_zip():
# get a list of zip files since yesterday
zip_file_set = set()
for key in s3.get_list_of_files_by_suffix(
bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'],
subfolder=datetime.utcnow().strftime('%Y-%m-%d') + '/zips_sent',
suffix='.TXT'):
subname = key.split('/')[-1] # strip subfolder in name
zip_file_set.add(subname.upper().rstrip('.TXT'))
# get acknowledgement file
ack_file_set = set()
yesterday = datetime.now(tz=pytz.utc) - timedelta(
days=1) # AWS datetime format
for key in s3.get_list_of_files_by_suffix(
bucket_name=current_app.config['DVLA_RESPONSE_BUCKET_NAME'],
subfolder='root/dispatch',
suffix='.ACK.txt',
last_modified=yesterday):
ack_file_set.add(key)
today_str = datetime.utcnow().strftime('%Y%m%d')
ack_content_set = set()
for key in ack_file_set:
if today_str in key:
content = s3.get_s3_file(
current_app.config['DVLA_RESPONSE_BUCKET_NAME'], key)
for zip_file in content.split('\n'): # each line
s = zip_file.split('|')
ack_content_set.add(s[0].upper())
message = ("Letter ack file does not contain all zip files sent. "
"Missing ack for zip files: {}, "
"pdf bucket: {}, subfolder: {}, "
"ack bucket: {}").format(
str(sorted(zip_file_set - ack_content_set)),
current_app.config['LETTERS_PDF_BUCKET_NAME'],
datetime.utcnow().strftime('%Y-%m-%d') + '/zips_sent',
current_app.config['DVLA_RESPONSE_BUCKET_NAME'])
# strip empty element before comparison
ack_content_set.discard('')
zip_file_set.discard('')
if len(zip_file_set - ack_content_set) > 0:
if current_app.config['NOTIFY_ENVIRONMENT'] in [
'live', 'production', 'test'
]:
zendesk_client.create_ticket(
subject="Letter acknowledge error",
message=message,
ticket_type=zendesk_client.TYPE_INCIDENT)
current_app.logger.error(message)
if len(ack_content_set - zip_file_set) > 0:
current_app.logger.info(
"letter ack contains zip that is not for today: {}".format(
ack_content_set - zip_file_set))
