def delete_notifications_created_more_than_a_week_ago_by_type( notification_type): flexible_data_retention = ServiceDataRetention.query.filter( ServiceDataRetention.notification_type == notification_type).all() deleted = 0 for f in flexible_data_retention: days_of_retention = convert_utc_to_bst( datetime.utcnow()).date() - timedelta(days=f.days_of_retention) query = db.session.query(Notification).filter( func.date(Notification.created_at) < days_of_retention, Notification.notification_type == f.notification_type, Notification.service_id == f.service_id) if notification_type == LETTER_TYPE: _delete_letters_from_s3(query) deleted += query.delete(synchronize_session='fetch') seven_days_ago = convert_utc_to_bst( datetime.utcnow()).date() - timedelta(days=7) services_with_data_retention = [ x.service_id for x in flexible_data_retention ] query = db.session.query(Notification).filter( func.date(Notification.created_at) < seven_days_ago, Notification.notification_type == notification_type, Notification.service_id.notin_(services_with_data_retention)) if notification_type == LETTER_TYPE: _delete_letters_from_s3(query=query) deleted += query.delete(synchronize_session='fetch') return deleted
def format_payload(*, dataset, date, group_name, group_value, count, period='day'): """ :param dataset - the name of the overall graph, as referred to in the endpoint. :param date - the date we're sending stats for :param group_name - the name of the individual groups of data, eg "channel" or "status" :param group_value - the value of the group, eg "sms" or "email" for group_name=channel :param count - the actual numeric value to send :param period - the period that this data covers - "day", "week", "month", "quarter". """ payload = { '_timestamp': convert_utc_to_bst(date).isoformat(), 'service': 'govuk-notify', 'dataType': dataset, 'period': period, 'count': count, group_name: group_value, } payload['_id'] = PerformancePlatformClient.generate_payload_id( payload, group_name) return payload
def increment_template_usage_cache(service_id, template_id, created_at): key = cache_key_for_service_template_usage_per_day( service_id, convert_utc_to_bst(created_at)) redis_store.increment_hash_value(key, template_id) # set key to expire in eight days - we don't know if we've just created the key or not, so must assume that we # have and reset the expiry. Eight days is longer than any notification is in the notifications table, so we'll # always capture the full week's numbers redis_store.expire(key, current_app.config['EXPIRE_CACHE_EIGHT_DAYS'])
def create_empty_monthly_notification_status_stats_dict(year): utc_month_starts = get_months_for_financial_year(year) # nested dicts - data[month][template type][status] = count return { convert_utc_to_bst(start).strftime('%Y-%m'): {template_type: defaultdict(int) for template_type in TEMPLATE_TYPES} for start in utc_month_starts }
def get_folder_name(_now, is_test_or_scan_letter=False): if is_test_or_scan_letter: folder_name = '' else: print_datetime = convert_utc_to_bst(_now) if print_datetime.time() > current_app.config.get( 'LETTER_PROCESSING_DEADLINE'): print_datetime += timedelta(days=1) folder_name = '{}/'.format(print_datetime.date()) return folder_name
def fetch_monthly_billing_for_year(service_id, year): year_start_date, year_end_date = get_financial_year(year) utcnow = datetime.utcnow() today = convert_utc_to_bst(utcnow) # if year end date is less than today, we are calculating for data in the past and have no need for deltas. if year_end_date >= today: yesterday = today - timedelta(days=1) for day in [yesterday, today]: data = fetch_billing_data_for_day(process_day=day, service_id=service_id) for d in data: update_fact_billing(data=d, process_day=day) email_and_letters = db.session.query( func.date_trunc('month', FactBilling.bst_date).cast(Date).label("month"), func.sum(FactBilling.notifications_sent).label("notifications_sent"), func.sum(FactBilling.notifications_sent).label("billable_units"), FactBilling.rate.label('rate'), FactBilling.notification_type.label('notification_type') ).filter( FactBilling.service_id == service_id, FactBilling.bst_date >= year_start_date, FactBilling.bst_date <= year_end_date, FactBilling.notification_type.in_([EMAIL_TYPE, LETTER_TYPE]) ).group_by( 'month', FactBilling.rate, FactBilling.notification_type ) sms = db.session.query( func.date_trunc('month', FactBilling.bst_date).cast(Date).label("month"), func.sum(FactBilling.notifications_sent).label("notifications_sent"), func.sum(FactBilling.billable_units * FactBilling.rate_multiplier).label("billable_units"), FactBilling.rate, FactBilling.notification_type ).filter( FactBilling.service_id == service_id, FactBilling.bst_date >= year_start_date, FactBilling.bst_date <= year_end_date, FactBilling.notification_type == SMS_TYPE ).group_by( 'month', FactBilling.rate, FactBilling.notification_type ) yearly_data = email_and_letters.union_all(sms).order_by( 'month', 'notification_type', 'rate' ).all() return yearly_data
def test_fetch_billing_data_for_day_is_grouped_by_provider(notify_db_session): service = create_service() template = create_template(service=service) create_notification(template=template, status='delivered', sent_by='mmg') create_notification(template=template, status='delivered', sent_by='firetext') today = convert_utc_to_bst(datetime.utcnow()) results = fetch_billing_data_for_day(today) assert len(results) == 2 assert results[0].notifications_sent == 1 assert results[1].notifications_sent == 1
def test_fetch_billing_data_for_day_is_grouped_by_service(notify_db_session): service_1 = create_service() service_2 = create_service(service_name='Service 2') email_template = create_template(service=service_1) sms_template = create_template(service=service_2) create_notification(template=email_template, status='delivered') create_notification(template=sms_template, status='delivered') today = convert_utc_to_bst(datetime.utcnow()) results = fetch_billing_data_for_day(today) assert len(results) == 2 assert results[0].notifications_sent == 1 assert results[1].notifications_sent == 1
def test_fetch_billing_data_for_today_includes_data_with_the_right_key_type( notify_db_session): service = create_service() template = create_template(service=service, template_type="email") for key_type in ['normal', 'test', 'team']: create_notification(template=template, status='delivered', key_type=key_type) today = convert_utc_to_bst(datetime.utcnow()) results = fetch_billing_data_for_day(today) assert len(results) == 1 assert results[0].notifications_sent == 2
def test_fetch_billing_data_for_day_returns_list_for_given_service( notify_db_session): service = create_service() service_2 = create_service(service_name='Service 2') template = create_template(service=service) template_2 = create_template(service=service_2) create_notification(template=template, status='delivered') create_notification(template=template_2, status='delivered') today = convert_utc_to_bst(datetime.utcnow()) results = fetch_billing_data_for_day(process_day=today, service_id=service.id) assert len(results) == 1 assert results[0].service_id == service.id
def _transform_billing_for_month_sms(billing_for_month): month_name = datetime.strftime( convert_utc_to_bst(billing_for_month.start_date), "%B") billing_units = rate = 0 for total in billing_for_month.monthly_totals: billing_units += (total['billing_units'] * total['rate_multiplier']) rate = total['rate'] return { "month": month_name, "billing_units": billing_units, "notification_type": billing_for_month.notification_type, "rate": rate }
def test_fetch_billing_data_for_today_includes_data_with_the_right_status( notify_db_session): service = create_service() template = create_template(service=service, template_type="email") for status in ['created', 'technical-failure']: create_notification(template=template, status=status) today = convert_utc_to_bst(datetime.utcnow()) results = fetch_billing_data_for_day(today) assert results == [] for status in ['delivered', 'sending', 'temporary-failure']: create_notification(template=template, status=status) results = fetch_billing_data_for_day(today) assert len(results) == 1 assert results[0].notifications_sent == 3
def test_fetch_billing_data_for_day_is_grouped_by_international( notify_db_session): service = create_service() template = create_template(service=service) create_notification(template=template, status='delivered', international=True) create_notification(template=template, status='delivered', international=False) today = convert_utc_to_bst(datetime.utcnow()) results = fetch_billing_data_for_day(today) assert len(results) == 2 assert results[0].notifications_sent == 1 assert results[1].notifications_sent == 1
def test_fetch_billing_data_for_day_is_grouped_by_notification_type( notify_db_session): service = create_service() sms_template = create_template(service=service, template_type='sms') email_template = create_template(service=service, template_type='email') letter_template = create_template(service=service, template_type='letter') create_notification(template=sms_template, status='delivered') create_notification(template=sms_template, status='delivered') create_notification(template=sms_template, status='delivered') create_notification(template=email_template, status='delivered') create_notification(template=email_template, status='delivered') create_notification(template=letter_template, status='delivered') today = convert_utc_to_bst(datetime.utcnow()) results = fetch_billing_data_for_day(today) assert len(results) == 3 notification_types = [ x[2] for x in results if x[2] in ['email', 'sms', 'letter'] ] assert len(notification_types) == 3
def _transform_billing_for_month_letters(billing_for_month): month_name = datetime.strftime( convert_utc_to_bst(billing_for_month.start_date), "%B") x = list() for total in billing_for_month.monthly_totals: y = { "month": month_name, "billing_units": (total['billing_units'] * total['rate_multiplier']), "notification_type": billing_for_month.notification_type, "rate": float(total['rate']) } x.append(y) if len(billing_for_month.monthly_totals) == 0: x.append({ "month": month_name, "billing_units": 0, "notification_type": billing_for_month.notification_type, "rate": 0 }) return x
def test_fetch_billing_data_for_today_includes_data_with_the_right_date( notify_db_session): process_day = datetime(2018, 4, 1, 13, 30, 00) service = create_service() template = create_template(service=service, template_type="email") create_notification(template=template, status='delivered', created_at=process_day) create_notification(template=template, status='delivered', created_at=datetime(2018, 3, 31, 23, 23, 23)) create_notification(template=template, status='delivered', created_at=datetime(2018, 3, 31, 20, 23, 23)) create_notification(template=template, status='sending', created_at=process_day + timedelta(days=1)) day_under_test = convert_utc_to_bst(process_day) results = fetch_billing_data_for_day(day_under_test) assert len(results) == 1 assert results[0].notifications_sent == 2
def get_monthly_notification_stats(service_id): # check service_id validity dao_fetch_service_by_id(service_id) try: year = int(request.args.get('year', 'NaN')) except ValueError: raise InvalidRequest('Year must be a number', status_code=400) start_date, end_date = get_financial_year(year) data = statistics.create_empty_monthly_notification_status_stats_dict(year) stats = fetch_notification_status_for_service_by_month( start_date, end_date, service_id) statistics.add_monthly_notification_status_stats(data, stats) now = datetime.utcnow() if end_date > now: todays_deltas = fetch_notification_status_for_service_for_day( convert_utc_to_bst(now), service_id=service_id) statistics.add_monthly_notification_status_stats(data, todays_deltas) return jsonify(data=data)
def test_get_utc_in_bst_returns_expected_date(date, expected_date): ret_date = convert_utc_to_bst(date) assert ret_date == expected_date
def get_billing_date_in_bst_from_filename(filename): datetime_string = filename.split('-')[1] datetime_obj = datetime.strptime(datetime_string, '%Y%m%d%H%M%S') return convert_utc_to_bst(datetime_obj).date()
def test_fetch_billing_data_for_day_returns_empty_list(notify_db_session): today = convert_utc_to_bst(datetime.utcnow()) results = fetch_billing_data_for_day(today) assert results == []