def delete_notifications_created_more_than_a_week_ago_by_type(
notification_type):
flexible_data_retention = ServiceDataRetention.query.filter(
ServiceDataRetention.notification_type == notification_type).all()
deleted = 0
for f in flexible_data_retention:
days_of_retention = convert_utc_to_bst(
datetime.utcnow()).date() - timedelta(days=f.days_of_retention)
query = db.session.query(Notification).filter(
func.date(Notification.created_at) < days_of_retention,
Notification.notification_type == f.notification_type,
Notification.service_id == f.service_id)
if notification_type == LETTER_TYPE:
_delete_letters_from_s3(query)
deleted += query.delete(synchronize_session='fetch')
seven_days_ago = convert_utc_to_bst(
datetime.utcnow()).date() - timedelta(days=7)
services_with_data_retention = [
x.service_id for x in flexible_data_retention
]
query = db.session.query(Notification).filter(
func.date(Notification.created_at) < seven_days_ago,
Notification.notification_type == notification_type,
Notification.service_id.notin_(services_with_data_retention))
if notification_type == LETTER_TYPE:
_delete_letters_from_s3(query=query)
deleted += query.delete(synchronize_session='fetch')
return deleted
def format_payload(*,
dataset,
date,
group_name,
group_value,
count,
period='day'):
"""
:param dataset - the name of the overall graph, as referred to in the endpoint.
:param date - the date we're sending stats for
:param group_name - the name of the individual groups of data, eg "channel" or "status"
:param group_value - the value of the group, eg "sms" or "email" for group_name=channel
:param count - the actual numeric value to send
:param period - the period that this data covers - "day", "week", "month", "quarter".
"""
payload = {
'_timestamp': convert_utc_to_bst(date).isoformat(),
'service': 'govuk-notify',
'dataType': dataset,
'period': period,
'count': count,
group_name: group_value,
}
payload['_id'] = PerformancePlatformClient.generate_payload_id(
payload, group_name)
return payload
def increment_template_usage_cache(service_id, template_id, created_at):
key = cache_key_for_service_template_usage_per_day(
service_id, convert_utc_to_bst(created_at))
redis_store.increment_hash_value(key, template_id)
# set key to expire in eight days - we don't know if we've just created the key or not, so must assume that we
# have and reset the expiry. Eight days is longer than any notification is in the notifications table, so we'll
# always capture the full week's numbers
redis_store.expire(key, current_app.config['EXPIRE_CACHE_EIGHT_DAYS'])
def create_empty_monthly_notification_status_stats_dict(year):
utc_month_starts = get_months_for_financial_year(year)
# nested dicts - data[month][template type][status] = count
return {
convert_utc_to_bst(start).strftime('%Y-%m'):
{template_type: defaultdict(int)
for template_type in TEMPLATE_TYPES}
for start in utc_month_starts
}
def get_folder_name(_now, is_test_or_scan_letter=False):
if is_test_or_scan_letter:
folder_name = ''
else:
print_datetime = convert_utc_to_bst(_now)
if print_datetime.time() > current_app.config.get(
'LETTER_PROCESSING_DEADLINE'):
print_datetime += timedelta(days=1)
folder_name = '{}/'.format(print_datetime.date())
return folder_name
def fetch_monthly_billing_for_year(service_id, year):
year_start_date, year_end_date = get_financial_year(year)
utcnow = datetime.utcnow()
today = convert_utc_to_bst(utcnow)
# if year end date is less than today, we are calculating for data in the past and have no need for deltas.
if year_end_date >= today:
yesterday = today - timedelta(days=1)
for day in [yesterday, today]:
data = fetch_billing_data_for_day(process_day=day, service_id=service_id)
for d in data:
update_fact_billing(data=d, process_day=day)
email_and_letters = db.session.query(
func.date_trunc('month', FactBilling.bst_date).cast(Date).label("month"),
func.sum(FactBilling.notifications_sent).label("notifications_sent"),
func.sum(FactBilling.notifications_sent).label("billable_units"),
FactBilling.rate.label('rate'),
FactBilling.notification_type.label('notification_type')
).filter(
FactBilling.service_id == service_id,
FactBilling.bst_date >= year_start_date,
FactBilling.bst_date <= year_end_date,
FactBilling.notification_type.in_([EMAIL_TYPE, LETTER_TYPE])
).group_by(
'month',
FactBilling.rate,
FactBilling.notification_type
)
sms = db.session.query(
func.date_trunc('month', FactBilling.bst_date).cast(Date).label("month"),
func.sum(FactBilling.notifications_sent).label("notifications_sent"),
func.sum(FactBilling.billable_units * FactBilling.rate_multiplier).label("billable_units"),
FactBilling.rate,
FactBilling.notification_type
).filter(
FactBilling.service_id == service_id,
FactBilling.bst_date >= year_start_date,
FactBilling.bst_date <= year_end_date,
FactBilling.notification_type == SMS_TYPE
).group_by(
'month',
FactBilling.rate,
FactBilling.notification_type
)
yearly_data = email_and_letters.union_all(sms).order_by(
'month',
'notification_type',
'rate'
).all()
return yearly_data
def test_fetch_billing_data_for_day_is_grouped_by_provider(notify_db_session):
service = create_service()
template = create_template(service=service)
create_notification(template=template, status='delivered', sent_by='mmg')
create_notification(template=template,
status='delivered',
sent_by='firetext')
today = convert_utc_to_bst(datetime.utcnow())
results = fetch_billing_data_for_day(today)
assert len(results) == 2
assert results[0].notifications_sent == 1
assert results[1].notifications_sent == 1
def test_fetch_billing_data_for_day_is_grouped_by_service(notify_db_session):
service_1 = create_service()
service_2 = create_service(service_name='Service 2')
email_template = create_template(service=service_1)
sms_template = create_template(service=service_2)
create_notification(template=email_template, status='delivered')
create_notification(template=sms_template, status='delivered')
today = convert_utc_to_bst(datetime.utcnow())
results = fetch_billing_data_for_day(today)
assert len(results) == 2
assert results[0].notifications_sent == 1
assert results[1].notifications_sent == 1
def test_fetch_billing_data_for_today_includes_data_with_the_right_key_type(
notify_db_session):
service = create_service()
template = create_template(service=service, template_type="email")
for key_type in ['normal', 'test', 'team']:
create_notification(template=template,
status='delivered',
key_type=key_type)
today = convert_utc_to_bst(datetime.utcnow())
results = fetch_billing_data_for_day(today)
assert len(results) == 1
assert results[0].notifications_sent == 2
def test_fetch_billing_data_for_day_returns_list_for_given_service(
notify_db_session):
service = create_service()
service_2 = create_service(service_name='Service 2')
template = create_template(service=service)
template_2 = create_template(service=service_2)
create_notification(template=template, status='delivered')
create_notification(template=template_2, status='delivered')
today = convert_utc_to_bst(datetime.utcnow())
results = fetch_billing_data_for_day(process_day=today,
service_id=service.id)
assert len(results) == 1
assert results[0].service_id == service.id
def _transform_billing_for_month_sms(billing_for_month):
month_name = datetime.strftime(
convert_utc_to_bst(billing_for_month.start_date), "%B")
billing_units = rate = 0
for total in billing_for_month.monthly_totals:
billing_units += (total['billing_units'] * total['rate_multiplier'])
rate = total['rate']
return {
"month": month_name,
"billing_units": billing_units,
"notification_type": billing_for_month.notification_type,
"rate": rate
}
def test_fetch_billing_data_for_today_includes_data_with_the_right_status(
notify_db_session):
service = create_service()
template = create_template(service=service, template_type="email")
for status in ['created', 'technical-failure']:
create_notification(template=template, status=status)
today = convert_utc_to_bst(datetime.utcnow())
results = fetch_billing_data_for_day(today)
assert results == []
for status in ['delivered', 'sending', 'temporary-failure']:
create_notification(template=template, status=status)
results = fetch_billing_data_for_day(today)
assert len(results) == 1
assert results[0].notifications_sent == 3
def test_fetch_billing_data_for_day_is_grouped_by_international(
notify_db_session):
service = create_service()
template = create_template(service=service)
create_notification(template=template,
status='delivered',
international=True)
create_notification(template=template,
status='delivered',
international=False)
today = convert_utc_to_bst(datetime.utcnow())
results = fetch_billing_data_for_day(today)
assert len(results) == 2
assert results[0].notifications_sent == 1
assert results[1].notifications_sent == 1
def test_fetch_billing_data_for_day_is_grouped_by_notification_type(
notify_db_session):
service = create_service()
sms_template = create_template(service=service, template_type='sms')
email_template = create_template(service=service, template_type='email')
letter_template = create_template(service=service, template_type='letter')
create_notification(template=sms_template, status='delivered')
create_notification(template=sms_template, status='delivered')
create_notification(template=sms_template, status='delivered')
create_notification(template=email_template, status='delivered')
create_notification(template=email_template, status='delivered')
create_notification(template=letter_template, status='delivered')
today = convert_utc_to_bst(datetime.utcnow())
results = fetch_billing_data_for_day(today)
assert len(results) == 3
notification_types = [
x[2] for x in results if x[2] in ['email', 'sms', 'letter']
]
assert len(notification_types) == 3
def _transform_billing_for_month_letters(billing_for_month):
month_name = datetime.strftime(
convert_utc_to_bst(billing_for_month.start_date), "%B")
x = list()
for total in billing_for_month.monthly_totals:
y = {
"month": month_name,
"billing_units":
(total['billing_units'] * total['rate_multiplier']),
"notification_type": billing_for_month.notification_type,
"rate": float(total['rate'])
}
x.append(y)
if len(billing_for_month.monthly_totals) == 0:
x.append({
"month": month_name,
"billing_units": 0,
"notification_type": billing_for_month.notification_type,
"rate": 0
})
return x
def test_fetch_billing_data_for_today_includes_data_with_the_right_date(
notify_db_session):
process_day = datetime(2018, 4, 1, 13, 30, 00)
service = create_service()
template = create_template(service=service, template_type="email")
create_notification(template=template,
status='delivered',
created_at=process_day)
create_notification(template=template,
status='delivered',
created_at=datetime(2018, 3, 31, 23, 23, 23))
create_notification(template=template,
status='delivered',
created_at=datetime(2018, 3, 31, 20, 23, 23))
create_notification(template=template,
status='sending',
created_at=process_day + timedelta(days=1))
day_under_test = convert_utc_to_bst(process_day)
results = fetch_billing_data_for_day(day_under_test)
assert len(results) == 1
assert results[0].notifications_sent == 2
def get_monthly_notification_stats(service_id):
# check service_id validity
dao_fetch_service_by_id(service_id)
try:
year = int(request.args.get('year', 'NaN'))
except ValueError:
raise InvalidRequest('Year must be a number', status_code=400)
start_date, end_date = get_financial_year(year)
data = statistics.create_empty_monthly_notification_status_stats_dict(year)
stats = fetch_notification_status_for_service_by_month(
start_date, end_date, service_id)
statistics.add_monthly_notification_status_stats(data, stats)
now = datetime.utcnow()
if end_date > now:
todays_deltas = fetch_notification_status_for_service_for_day(
convert_utc_to_bst(now), service_id=service_id)
statistics.add_monthly_notification_status_stats(data, todays_deltas)
return jsonify(data=data)
def test_get_utc_in_bst_returns_expected_date(date, expected_date):
ret_date = convert_utc_to_bst(date)
assert ret_date == expected_date
def get_billing_date_in_bst_from_filename(filename):
datetime_string = filename.split('-')[1]
datetime_obj = datetime.strptime(datetime_string, '%Y%m%d%H%M%S')
return convert_utc_to_bst(datetime_obj).date()
def test_fetch_billing_data_for_day_returns_empty_list(notify_db_session):
today = convert_utc_to_bst(datetime.utcnow())
results = fetch_billing_data_for_day(today)
assert results == []