def process_incomplete_job(job_id):
job = dao_get_job_by_id(job_id)
last_notification_added = dao_get_last_notification_added_for_job_id(job_id)
if last_notification_added:
resume_from_row = last_notification_added.job_row_number
else:
resume_from_row = -1 # The first row in the csv with a number is row 0
current_app.logger.info("Resuming job {} from row {}".format(job_id, resume_from_row))
db_template = dao_get_template_by_id(job.template_id, job.template_version)
TemplateClass = get_template_class(db_template.template_type)
template = TemplateClass(db_template.__dict__)
for row in RecipientCSV(
s3.get_job_from_s3(str(job.service_id), str(job.id)),
template_type=template.template_type,
placeholders=template.placeholders
).rows:
if row.index > resume_from_row:
process_row(row, template, job, job.service)
job_complete(job, resumed=True)
def get_recipient_csv(job: Job, template: Template) -> RecipientCSV:
return RecipientCSV(
s3.get_job_from_s3(str(job.service_id), str(job.id)),
template_type=template.template_type,
placeholders=template.placeholders,
max_rows=get_csv_max_rows(job.service_id),
)
def process_job(job_id):
start = datetime.utcnow()
job = dao_get_job_by_id(job_id)
if job.job_status != "pending":
return
service = job.service
if __sending_limits_for_job_exceeded(service, job, job_id):
return
job.job_status = "in progress"
dao_update_job(job)
template = Template(dao_get_template_by_id(job.template_id, job.template_version).__dict__)
for row_number, recipient, personalisation in RecipientCSV(
s3.get_job_from_s3(str(service.id), str(job_id)),
template_type=template.template_type,
placeholders=template.placeholders,
).enumerated_recipients_and_personalisation:
encrypted = encryption.encrypt(
{
"template": str(template.id),
"template_version": job.template_version,
"job": str(job.id),
"to": recipient,
"row_number": row_number,
"personalisation": dict(personalisation),
}
)
if template.template_type == SMS_TYPE:
send_sms.apply_async(
(str(job.service_id), create_uuid(), encrypted, datetime.utcnow().strftime(DATETIME_FORMAT)),
queue="db-sms" if not service.research_mode else "research-mode",
)
if template.template_type == EMAIL_TYPE:
send_email.apply_async(
(str(job.service_id), create_uuid(), encrypted, datetime.utcnow().strftime(DATETIME_FORMAT)),
queue="db-email" if not service.research_mode else "research-mode",
)
finished = datetime.utcnow()
job.job_status = "finished"
job.processing_started = start
job.processing_finished = finished
dao_update_job(job)
current_app.logger.info(
"Job {} created at {} started at {} finished at {}".format(job_id, job.created_at, start, finished)
)
def process_row_from_job(job_id, job_row_number):
job = dao_get_job_by_id(job_id)
db_template = dao_get_template_by_id(job.template_id, job.template_version)
template = db_template._as_utils_template()
for row in RecipientCSV(s3.get_job_from_s3(str(job.service_id),
str(job.id)),
template_type=template.template_type,
placeholders=template.placeholders).get_rows():
if row.index == job_row_number:
notification_id = process_row(row, template, job, job.service)
current_app.logger.info(
"Process row {} for job {} created notification_id: {}".format(
job_row_number, job_id, notification_id))
def process_job(job_id, sender_id=None):
start = datetime.utcnow()
job = dao_get_job_by_id(job_id)
if job.job_status != JOB_STATUS_PENDING:
return
service = job.service
if not service.active:
job.job_status = JOB_STATUS_CANCELLED
dao_update_job(job)
current_app.logger.warning(
"Job {} has been cancelled, service {} is inactive".format(
job_id, service.id))
return
if __sending_limits_for_job_exceeded(service, job, job_id):
return
job.job_status = JOB_STATUS_IN_PROGRESS
job.processing_started = start
dao_update_job(job)
db_template = dao_get_template_by_id(job.template_id, job.template_version)
TemplateClass = get_template_class(db_template.template_type)
template = TemplateClass(db_template.__dict__)
current_app.logger.debug(
"Starting job {} processing {} notifications".format(
job_id, job.notification_count))
for row in RecipientCSV(
s3.get_job_from_s3(str(service.id), str(job_id)),
template_type=template.template_type,
placeholders=template.placeholders,
max_rows=get_csv_max_rows(service.id),
).get_rows():
process_row(row, template, job, service, sender_id=sender_id)
job_complete(job, start=start)
