def fetch_notification_status_for_day(process_day, notification_type):
start_date = convert_bst_to_utc(datetime.combine(process_day, time.min))
end_date = convert_bst_to_utc(
datetime.combine(process_day + timedelta(days=1), time.min))
current_app.logger.info("Fetch ft_notification_status for {} to {}".format(
start_date, end_date))
all_data_for_process_day = []
services = Service.query.all()
# for each service query notifications or notification_history for the day, depending on their data retention
for service in services:
table = get_notification_table_to_use(service,
notification_type,
process_day,
has_delete_task_run=False)
data_for_service_and_type = query_for_fact_status_data(
table=table,
start_date=start_date,
end_date=end_date,
notification_type=notification_type,
service_id=service.id)
all_data_for_process_day += data_for_service_and_type
return all_data_for_process_day
def get_months_for_financial_year(year):
return [
convert_bst_to_utc(month)
for month in (get_months_for_year(4, 13, year) +
get_months_for_year(1, 4, year + 1))
if convert_bst_to_utc(month) < datetime.now()
]
def fetch_billing_data_for_day(process_day,
service_id=None,
check_permissions=False):
start_date = convert_bst_to_utc(datetime.combine(process_day, time.min))
end_date = convert_bst_to_utc(
datetime.combine(process_day + timedelta(days=1), time.min))
current_app.logger.info("Populate ft_billing for {} to {}".format(
start_date, end_date))
transit_data = []
if not service_id:
services = Service.query.all()
else:
services = [Service.query.get(service_id)]
for service in services:
for notification_type in (SMS_TYPE, EMAIL_TYPE, LETTER_TYPE):
if (not check_permissions
) or service.has_permission(notification_type):
table = get_notification_table_to_use(
service,
notification_type,
process_day,
has_delete_task_run=False)
results = _query_for_billing_data(
table=table,
notification_type=notification_type,
start_date=start_date,
end_date=end_date,
service=service)
transit_data += results
return transit_data
def get_month_start_and_end_date_in_utc(month_year):
"""
This function return the start and date of the month_year as UTC,
:param month_year: the datetime to calculate the start and end date for that month
:return: start_date, end_date, month
"""
import calendar
_, num_days = calendar.monthrange(month_year.year, month_year.month)
first_day = datetime(month_year.year, month_year.month, 1, 0, 0, 0)
last_day = datetime(month_year.year, month_year.month, num_days, 23, 59,
59, 99999)
return convert_bst_to_utc(first_day), convert_bst_to_utc(last_day)
def dao_get_letters_to_be_printed(print_run_deadline,
postage,
query_limit=10000):
"""
Return all letters created before the print run deadline that have not yet been sent. This yields in batches of 10k
to prevent the query taking too long and eating up too much memory. As each 10k batch is yielded, the
get_key_and_size_of_letters_to_be_sent_to_print function will go and fetch the s3 data, andhese start sending off
tasks to the notify-ftp app to send them.
CAUTION! Modify this query with caution. Modifying filters etc is fine, but if we join onto another table, then
there may be undefined behaviour. Essentially we need each ORM object returned for each row to be unique,
and we should avoid modifying state of returned objects.
For more reading:
https://docs.sqlalchemy.org/en/13/orm/query.html?highlight=yield_per#sqlalchemy.orm.query.Query.yield_per
https://www.mail-archive.com/[email protected]/msg12443.html
"""
notifications = db.session.query(
Notification.id,
Notification.created_at,
Notification.reference,
Notification.service_id,
Service.crown,
).join(Notification.service).filter(
Notification.created_at < convert_bst_to_utc(print_run_deadline),
Notification.notification_type == LETTER_TYPE,
Notification.status == NOTIFICATION_CREATED,
Notification.key_type == KEY_TYPE_NORMAL,
Notification.postage == postage).order_by(
Notification.service_id,
Notification.created_at).yield_per(query_limit)
return notifications
def format_mmg_datetime(date):
"""
We expect datetimes in format 2017-05-21+11%3A56%3A11 - ie, spaces replaced with pluses, and URI encoded
(the same as UTC)
"""
orig_date = format_mmg_message(date)
parsed_datetime = iso8601.parse_date(orig_date).replace(tzinfo=None)
return convert_bst_to_utc(parsed_datetime)
def printing_today_or_tomorrow(created_at):
print_cutoff = convert_bst_to_utc(
convert_utc_to_bst(datetime.utcnow()).replace(
hour=17, minute=30)).replace(tzinfo=pytz.utc)
created_at = utc_string_to_aware_gmt_datetime(created_at)
if created_at < print_cutoff:
return 'today'
else:
return 'tomorrow'
def dao_get_letters_and_sheets_volume_by_postage(print_run_deadline):
notifications = db.session.query(
func.count(Notification.id).label('letters_count'),
func.sum(Notification.billable_units).label('sheets_count'),
Notification.postage).filter(
Notification.created_at < convert_bst_to_utc(print_run_deadline),
Notification.notification_type == LETTER_TYPE,
Notification.status == NOTIFICATION_CREATED,
Notification.key_type == KEY_TYPE_NORMAL,
).group_by(Notification.postage).order_by(Notification.postage).all()
return notifications
def dao_get_letters_to_be_printed(print_run_deadline):
"""
Return all letters created before the print run deadline that have not yet been sent
"""
notifications = Notification.query.filter(
Notification.created_at < convert_bst_to_utc(print_run_deadline),
Notification.notification_type == LETTER_TYPE,
Notification.status == NOTIFICATION_CREATED,
Notification.key_type == KEY_TYPE_NORMAL).order_by(
Notification.created_at).all()
return notifications
def dao_old_letters_with_created_status():
yesterday_bst = convert_utc_to_bst(datetime.utcnow()) - timedelta(days=1)
last_processing_deadline = yesterday_bst.replace(hour=17, minute=30, second=0, microsecond=0)
notifications = Notification.query.filter(
Notification.created_at < convert_bst_to_utc(last_processing_deadline),
Notification.notification_type == LETTER_TYPE,
Notification.status == NOTIFICATION_CREATED
).order_by(
Notification.created_at
).all()
return notifications
def test_convert_bst_to_utc():
bst = "2017-05-12 13:15"
bst_datetime = datetime.strptime(bst, "%Y-%m-%d %H:%M")
utc = convert_bst_to_utc(bst_datetime)
assert utc == datetime(2017, 5, 12, 12, 15)
def persist_scheduled_notification(notification_id, scheduled_for):
scheduled_datetime = convert_bst_to_utc(
datetime.strptime(scheduled_for, "%Y-%m-%d %H:%M"))
scheduled_notification = ScheduledNotification(
notification_id=notification_id, scheduled_for=scheduled_datetime)
dao_created_scheduled_notification(scheduled_notification)
