def check_job_status():
"""
every x minutes do this check
select
from jobs
where job_status == 'in progress'
and template_type in ('sms', 'email')
and scheduled_at or created_at is older that 30 minutes.
if any results then
raise error
process the rows in the csv that are missing (in another task) just do the check here.
"""
thirty_minutes_ago = datetime.utcnow() - timedelta(minutes=30)
thirty_five_minutes_ago = datetime.utcnow() - timedelta(minutes=35)
jobs_not_complete_after_30_minutes = Job.query.filter(
Job.job_status == JOB_STATUS_IN_PROGRESS,
and_(thirty_five_minutes_ago < Job.processing_started,
Job.processing_started < thirty_minutes_ago)).order_by(
Job.processing_started).all()
# temporarily mark them as ERROR so that they don't get picked up by future check_job_status tasks
# if they haven't been re-processed in time.
job_ids = []
for job in jobs_not_complete_after_30_minutes:
job.job_status = JOB_STATUS_ERROR
dao_update_job(job)
job_ids.append(str(job.id))
if job_ids:
notify_celery.send_task(name=TaskNames.PROCESS_INCOMPLETE_JOBS,
args=(job_ids, ),
queue=QueueNames.JOBS)
raise JobIncompleteError(
"Job(s) {} have not completed.".format(job_ids))
def process_job(job_id, sender_id=None):
start = datetime.utcnow()
job = dao_get_job_by_id(job_id)
current_app.logger.info("Starting process-job task for job id {} with status: {}".format(job_id, job.job_status))
if job.job_status != JOB_STATUS_PENDING:
return
service = job.service
if not service.active:
job.job_status = JOB_STATUS_CANCELLED
dao_update_job(job)
current_app.logger.warning(
"Job {} has been cancelled, service {} is inactive".format(job_id, service.id))
return
if __sending_limits_for_job_exceeded(service, job, job_id):
return
job.job_status = JOB_STATUS_IN_PROGRESS
job.processing_started = start
dao_update_job(job)
recipient_csv, template, sender_id = get_recipient_csv_and_template_and_sender_id(job)
current_app.logger.info("Starting job {} processing {} notifications".format(job_id, job.notification_count))
for row in recipient_csv.get_rows():
process_row(row, template, job, service, sender_id=sender_id)
job_complete(job, start=start)
def test_update_job(sample_job):
assert sample_job.job_status == 'pending'
sample_job.job_status = 'in progress'
dao_update_job(sample_job)
job_from_db = Job.query.get(sample_job.id)
assert job_from_db.job_status == 'in progress'
def test_update_job(sample_job):
assert sample_job.job_status == 'pending'
sample_job.job_status = 'in progress'
dao_update_job(sample_job)
job_from_db = Job.query.get(sample_job.id)
assert job_from_db.job_status == 'in progress'
def process_job(job_id):
start = datetime.utcnow()
job = dao_get_job_by_id(job_id)
if job.job_status != "pending":
return
service = job.service
if __sending_limits_for_job_exceeded(service, job, job_id):
return
job.job_status = "in progress"
dao_update_job(job)
template = Template(dao_get_template_by_id(job.template_id, job.template_version).__dict__)
for row_number, recipient, personalisation in RecipientCSV(
s3.get_job_from_s3(str(service.id), str(job_id)),
template_type=template.template_type,
placeholders=template.placeholders,
).enumerated_recipients_and_personalisation:
encrypted = encryption.encrypt(
{
"template": str(template.id),
"template_version": job.template_version,
"job": str(job.id),
"to": recipient,
"row_number": row_number,
"personalisation": dict(personalisation),
}
)
if template.template_type == SMS_TYPE:
send_sms.apply_async(
(str(job.service_id), create_uuid(), encrypted, datetime.utcnow().strftime(DATETIME_FORMAT)),
queue="db-sms" if not service.research_mode else "research-mode",
)
if template.template_type == EMAIL_TYPE:
send_email.apply_async(
(str(job.service_id), create_uuid(), encrypted, datetime.utcnow().strftime(DATETIME_FORMAT)),
queue="db-email" if not service.research_mode else "research-mode",
)
finished = datetime.utcnow()
job.job_status = "finished"
job.processing_started = start
job.processing_finished = finished
dao_update_job(job)
current_app.logger.info(
"Job {} created at {} started at {} finished at {}".format(job_id, job.created_at, start, finished)
)
def process_incomplete_jobs(job_ids):
jobs = [dao_get_job_by_id(job_id) for job_id in job_ids]
# reset the processing start time so that the check_job_status scheduled task doesn't pick this job up again
for job in jobs:
job.job_status = JOB_STATUS_IN_PROGRESS
job.processing_started = datetime.utcnow()
dao_update_job(job)
current_app.logger.info("Resuming Job(s) {}".format(job_ids))
for job_id in job_ids:
process_incomplete_job(job_id)
def __sending_limits_for_job_exceeded(service, job, job_id):
total_sent = fetch_todays_total_message_count(service.id)
if total_sent + job.notification_count > service.message_limit:
job.job_status = 'sending limits exceeded'
job.processing_finished = datetime.utcnow()
dao_update_job(job)
current_app.logger.info(
"Job {} size {} error. Sending limits {} exceeded".format(
job_id, job.notification_count, service.message_limit))
return True
return False
def job_complete(job, resumed=False, start=None):
job.job_status = JOB_STATUS_FINISHED
finished = datetime.utcnow()
job.processing_finished = finished
dao_update_job(job)
if resumed:
current_app.logger.info("Resumed Job {} completed at {}".format(
job.id, job.created_at))
else:
current_app.logger.info(
"Job {} created at {} started at {} finished at {}".format(
job.id, job.created_at, start, finished))
def __sending_limits_for_job_exceeded(service, job, job_id):
total_sent = fetch_todays_total_message_count(service.id)
if total_sent + job.notification_count > service.message_limit:
job.job_status = "sending limits exceeded"
job.processing_finished = datetime.utcnow()
dao_update_job(job)
current_app.logger.info(
"Job {} size {} error. Sending limits {} exceeded".format(
job_id, job.notification_count, service.message_limit
)
)
return True
return False
def check_job_status():
"""
every x minutes do this check
select
from jobs
where job_status == 'in progress'
and processing started between 30 and 35 minutes ago
OR where the job_status == 'pending'
and the job scheduled_for timestamp is between 30 and 35 minutes ago.
if any results then
update the job_status to 'error'
process the rows in the csv that are missing (in another task) just do the check here.
"""
thirty_minutes_ago = datetime.utcnow() - timedelta(minutes=30)
thirty_five_minutes_ago = datetime.utcnow() - timedelta(minutes=35)
incomplete_in_progress_jobs = Job.query.filter(
Job.job_status == JOB_STATUS_IN_PROGRESS,
between(Job.processing_started, thirty_five_minutes_ago, thirty_minutes_ago)
)
incomplete_pending_jobs = Job.query.filter(
Job.job_status == JOB_STATUS_PENDING,
Job.scheduled_for.isnot(None),
between(Job.scheduled_for, thirty_five_minutes_ago, thirty_minutes_ago)
)
jobs_not_complete_after_30_minutes = incomplete_in_progress_jobs.union(
incomplete_pending_jobs
).order_by(
Job.processing_started, Job.scheduled_for
).all()
# temporarily mark them as ERROR so that they don't get picked up by future check_job_status tasks
# if they haven't been re-processed in time.
job_ids = []
for job in jobs_not_complete_after_30_minutes:
job.job_status = JOB_STATUS_ERROR
dao_update_job(job)
job_ids.append(str(job.id))
if job_ids:
current_app.logger.info("Job(s) {} have not completed.".format(job_ids))
process_incomplete_jobs.apply_async(
[job_ids],
queue=QueueNames.JOBS
)
def process_job(job_id):
start = datetime.utcnow()
job = dao_get_job_by_id(job_id)
if job.job_status != JOB_STATUS_PENDING:
return
service = job.service
if not service.active:
job.job_status = JOB_STATUS_CANCELLED
dao_update_job(job)
current_app.logger.warning(
"Job {} has been cancelled, service {} is inactive".format(
job_id, service.id))
return
if __sending_limits_for_job_exceeded(service, job, job_id):
return
job.job_status = JOB_STATUS_IN_PROGRESS
job.processing_started = start
dao_update_job(job)
# Record StatsD stats to compute SLOs
job_start = job.scheduled_for or job.created_at
statsd_client.timing_with_dates("job.processing-start-delay",
job.processing_started, job_start)
db_template = dao_get_template_by_id(job.template_id, job.template_version)
TemplateClass = get_template_class(db_template.template_type)
template = TemplateClass(db_template.__dict__)
current_app.logger.debug(
"Starting job {} processing {} notifications".format(
job_id, job.notification_count))
csv = get_recipient_csv(job, template)
for row in csv.get_rows():
process_row(row, template, job, service)
job_complete(job, start=start)
def process_job(job_id, sender_id=None):
start = datetime.utcnow()
job = dao_get_job_by_id(job_id)
if job.job_status != JOB_STATUS_PENDING:
return
service = job.service
if not service.active:
job.job_status = JOB_STATUS_CANCELLED
dao_update_job(job)
current_app.logger.warning(
"Job {} has been cancelled, service {} is inactive".format(
job_id, service.id))
return
if __sending_limits_for_job_exceeded(service, job, job_id):
return
job.job_status = JOB_STATUS_IN_PROGRESS
job.processing_started = start
dao_update_job(job)
db_template = dao_get_template_by_id(job.template_id, job.template_version)
TemplateClass = get_template_class(db_template.template_type)
template = TemplateClass(db_template.__dict__)
current_app.logger.debug(
"Starting job {} processing {} notifications".format(
job_id, job.notification_count))
for row in RecipientCSV(
s3.get_job_from_s3(str(service.id), str(job_id)),
template_type=template.template_type,
placeholders=template.placeholders,
max_rows=get_csv_max_rows(service.id),
).get_rows():
process_row(row, template, job, service, sender_id=sender_id)
job_complete(job, start=start)
def cancel_job(service_id, job_id):
job = dao_get_future_scheduled_job_by_id_and_service_id(job_id, service_id)
job.job_status = JOB_STATUS_CANCELLED
dao_update_job(job)
return get_job_by_service_and_job_id(service_id, job_id)
def cancel_job(service_id, job_id):
job = dao_get_future_scheduled_job_by_id_and_service_id(job_id, service_id)
job.job_status = JOB_STATUS_CANCELLED
dao_update_job(job)
return get_job_by_service_and_job_id(service_id, job_id)
