def check_job_status(): """ every x minutes do this check select from jobs where job_status == 'in progress' and template_type in ('sms', 'email') and scheduled_at or created_at is older that 30 minutes. if any results then raise error process the rows in the csv that are missing (in another task) just do the check here. """ thirty_minutes_ago = datetime.utcnow() - timedelta(minutes=30) thirty_five_minutes_ago = datetime.utcnow() - timedelta(minutes=35) jobs_not_complete_after_30_minutes = Job.query.filter( Job.job_status == JOB_STATUS_IN_PROGRESS, and_(thirty_five_minutes_ago < Job.processing_started, Job.processing_started < thirty_minutes_ago)).order_by( Job.processing_started).all() # temporarily mark them as ERROR so that they don't get picked up by future check_job_status tasks # if they haven't been re-processed in time. job_ids = [] for job in jobs_not_complete_after_30_minutes: job.job_status = JOB_STATUS_ERROR dao_update_job(job) job_ids.append(str(job.id)) if job_ids: notify_celery.send_task(name=TaskNames.PROCESS_INCOMPLETE_JOBS, args=(job_ids, ), queue=QueueNames.JOBS) raise JobIncompleteError( "Job(s) {} have not completed.".format(job_ids))
def process_job(job_id, sender_id=None): start = datetime.utcnow() job = dao_get_job_by_id(job_id) current_app.logger.info("Starting process-job task for job id {} with status: {}".format(job_id, job.job_status)) if job.job_status != JOB_STATUS_PENDING: return service = job.service if not service.active: job.job_status = JOB_STATUS_CANCELLED dao_update_job(job) current_app.logger.warning( "Job {} has been cancelled, service {} is inactive".format(job_id, service.id)) return if __sending_limits_for_job_exceeded(service, job, job_id): return job.job_status = JOB_STATUS_IN_PROGRESS job.processing_started = start dao_update_job(job) recipient_csv, template, sender_id = get_recipient_csv_and_template_and_sender_id(job) current_app.logger.info("Starting job {} processing {} notifications".format(job_id, job.notification_count)) for row in recipient_csv.get_rows(): process_row(row, template, job, service, sender_id=sender_id) job_complete(job, start=start)
def test_update_job(sample_job): assert sample_job.job_status == 'pending' sample_job.job_status = 'in progress' dao_update_job(sample_job) job_from_db = Job.query.get(sample_job.id) assert job_from_db.job_status == 'in progress'
def process_job(job_id): start = datetime.utcnow() job = dao_get_job_by_id(job_id) if job.job_status != "pending": return service = job.service if __sending_limits_for_job_exceeded(service, job, job_id): return job.job_status = "in progress" dao_update_job(job) template = Template(dao_get_template_by_id(job.template_id, job.template_version).__dict__) for row_number, recipient, personalisation in RecipientCSV( s3.get_job_from_s3(str(service.id), str(job_id)), template_type=template.template_type, placeholders=template.placeholders, ).enumerated_recipients_and_personalisation: encrypted = encryption.encrypt( { "template": str(template.id), "template_version": job.template_version, "job": str(job.id), "to": recipient, "row_number": row_number, "personalisation": dict(personalisation), } ) if template.template_type == SMS_TYPE: send_sms.apply_async( (str(job.service_id), create_uuid(), encrypted, datetime.utcnow().strftime(DATETIME_FORMAT)), queue="db-sms" if not service.research_mode else "research-mode", ) if template.template_type == EMAIL_TYPE: send_email.apply_async( (str(job.service_id), create_uuid(), encrypted, datetime.utcnow().strftime(DATETIME_FORMAT)), queue="db-email" if not service.research_mode else "research-mode", ) finished = datetime.utcnow() job.job_status = "finished" job.processing_started = start job.processing_finished = finished dao_update_job(job) current_app.logger.info( "Job {} created at {} started at {} finished at {}".format(job_id, job.created_at, start, finished) )
def process_incomplete_jobs(job_ids): jobs = [dao_get_job_by_id(job_id) for job_id in job_ids] # reset the processing start time so that the check_job_status scheduled task doesn't pick this job up again for job in jobs: job.job_status = JOB_STATUS_IN_PROGRESS job.processing_started = datetime.utcnow() dao_update_job(job) current_app.logger.info("Resuming Job(s) {}".format(job_ids)) for job_id in job_ids: process_incomplete_job(job_id)
def __sending_limits_for_job_exceeded(service, job, job_id): total_sent = fetch_todays_total_message_count(service.id) if total_sent + job.notification_count > service.message_limit: job.job_status = 'sending limits exceeded' job.processing_finished = datetime.utcnow() dao_update_job(job) current_app.logger.info( "Job {} size {} error. Sending limits {} exceeded".format( job_id, job.notification_count, service.message_limit)) return True return False
def job_complete(job, resumed=False, start=None): job.job_status = JOB_STATUS_FINISHED finished = datetime.utcnow() job.processing_finished = finished dao_update_job(job) if resumed: current_app.logger.info("Resumed Job {} completed at {}".format( job.id, job.created_at)) else: current_app.logger.info( "Job {} created at {} started at {} finished at {}".format( job.id, job.created_at, start, finished))
def __sending_limits_for_job_exceeded(service, job, job_id): total_sent = fetch_todays_total_message_count(service.id) if total_sent + job.notification_count > service.message_limit: job.job_status = "sending limits exceeded" job.processing_finished = datetime.utcnow() dao_update_job(job) current_app.logger.info( "Job {} size {} error. Sending limits {} exceeded".format( job_id, job.notification_count, service.message_limit ) ) return True return False
def check_job_status(): """ every x minutes do this check select from jobs where job_status == 'in progress' and processing started between 30 and 35 minutes ago OR where the job_status == 'pending' and the job scheduled_for timestamp is between 30 and 35 minutes ago. if any results then update the job_status to 'error' process the rows in the csv that are missing (in another task) just do the check here. """ thirty_minutes_ago = datetime.utcnow() - timedelta(minutes=30) thirty_five_minutes_ago = datetime.utcnow() - timedelta(minutes=35) incomplete_in_progress_jobs = Job.query.filter( Job.job_status == JOB_STATUS_IN_PROGRESS, between(Job.processing_started, thirty_five_minutes_ago, thirty_minutes_ago) ) incomplete_pending_jobs = Job.query.filter( Job.job_status == JOB_STATUS_PENDING, Job.scheduled_for.isnot(None), between(Job.scheduled_for, thirty_five_minutes_ago, thirty_minutes_ago) ) jobs_not_complete_after_30_minutes = incomplete_in_progress_jobs.union( incomplete_pending_jobs ).order_by( Job.processing_started, Job.scheduled_for ).all() # temporarily mark them as ERROR so that they don't get picked up by future check_job_status tasks # if they haven't been re-processed in time. job_ids = [] for job in jobs_not_complete_after_30_minutes: job.job_status = JOB_STATUS_ERROR dao_update_job(job) job_ids.append(str(job.id)) if job_ids: current_app.logger.info("Job(s) {} have not completed.".format(job_ids)) process_incomplete_jobs.apply_async( [job_ids], queue=QueueNames.JOBS )
def process_job(job_id): start = datetime.utcnow() job = dao_get_job_by_id(job_id) if job.job_status != JOB_STATUS_PENDING: return service = job.service if not service.active: job.job_status = JOB_STATUS_CANCELLED dao_update_job(job) current_app.logger.warning( "Job {} has been cancelled, service {} is inactive".format( job_id, service.id)) return if __sending_limits_for_job_exceeded(service, job, job_id): return job.job_status = JOB_STATUS_IN_PROGRESS job.processing_started = start dao_update_job(job) # Record StatsD stats to compute SLOs job_start = job.scheduled_for or job.created_at statsd_client.timing_with_dates("job.processing-start-delay", job.processing_started, job_start) db_template = dao_get_template_by_id(job.template_id, job.template_version) TemplateClass = get_template_class(db_template.template_type) template = TemplateClass(db_template.__dict__) current_app.logger.debug( "Starting job {} processing {} notifications".format( job_id, job.notification_count)) csv = get_recipient_csv(job, template) for row in csv.get_rows(): process_row(row, template, job, service) job_complete(job, start=start)
def process_job(job_id, sender_id=None): start = datetime.utcnow() job = dao_get_job_by_id(job_id) if job.job_status != JOB_STATUS_PENDING: return service = job.service if not service.active: job.job_status = JOB_STATUS_CANCELLED dao_update_job(job) current_app.logger.warning( "Job {} has been cancelled, service {} is inactive".format( job_id, service.id)) return if __sending_limits_for_job_exceeded(service, job, job_id): return job.job_status = JOB_STATUS_IN_PROGRESS job.processing_started = start dao_update_job(job) db_template = dao_get_template_by_id(job.template_id, job.template_version) TemplateClass = get_template_class(db_template.template_type) template = TemplateClass(db_template.__dict__) current_app.logger.debug( "Starting job {} processing {} notifications".format( job_id, job.notification_count)) for row in RecipientCSV( s3.get_job_from_s3(str(service.id), str(job_id)), template_type=template.template_type, placeholders=template.placeholders, max_rows=get_csv_max_rows(service.id), ).get_rows(): process_row(row, template, job, service, sender_id=sender_id) job_complete(job, start=start)
def cancel_job(service_id, job_id): job = dao_get_future_scheduled_job_by_id_and_service_id(job_id, service_id) job.job_status = JOB_STATUS_CANCELLED dao_update_job(job) return get_job_by_service_and_job_id(service_id, job_id)