def enabled_job(job_key):
"""Temporarily enabled job."""
all_jobs = Job.get_all(include_disabled=True)
job = next((j for j in all_jobs if j.key == job_key))
Job.update_disabled(job_id=job.id, disable=False)
std_commit(allow_test_environment=True)
try:
yield
finally:
Job.update_disabled(job_id=job.id, disable=True)
std_commit(allow_test_environment=True)
def test_edit_schedule_of_running_job(self, client, admin_session):
"""You cannot edit job schedule if the job is running."""
job = Job.get_job_by_key('admin_emails')
job_history = JobHistory.job_started(job_key=job.key)
self._api_job_update_schedule(
client,
expected_status_code=400,
job_id=Job.get_job_by_key('admin_emails').id,
schedule_type='minutes',
schedule_value=3,
)
JobHistory.job_finished(id_=job_history.id, failed=False)
def test_alert_on_job_failure(self):
admin_uid = app.config['EMAIL_DIABLO_ADMIN_UID']
email_count = _get_email_count(admin_uid)
# No alert on happy job.
CanvasJob(simply_yield).run()
assert _get_email_count(admin_uid) == email_count
# Alert on sad job.
all_jobs = Job.get_all(include_disabled=True)
doomed_job = next(
(j for j in all_jobs if j.key == DoomedToFailure.key()))
# Make sure job is enabled
Job.update_disabled(job_id=doomed_job.id, disable=False)
std_commit(allow_test_environment=True)
DoomedToFailure(simply_yield).run()
# Failure alerts do not go through the queue.
assert _get_email_count(admin_uid) == email_count + 1
def test_edit_schedule_of_enabled_job(self, client, admin_session):
"""You cannot edit job schedule if the job is enabled."""
with enabled_job(job_key=AdminEmailsJob.key()):
self._api_job_update_schedule(
client,
expected_status_code=400,
job_id=Job.get_job_by_key('admin_emails').id,
schedule_type='minutes',
schedule_value=3,
)
def update_schedule():
params = request.get_json()
job_id = params.get('jobId')
schedule_type = params.get('type')
schedule_value = params.get('value')
if not job_id or not schedule_type or not schedule_value:
raise BadRequestError('Required parameters are missing.')
job = Job.get_job(job_id=job_id)
if not job.disabled or JobHistory.is_job_running(job_key=job.key):
raise BadRequestError(
'You cannot edit job schedule if job is either enabled or running.'
)
job = Job.update_schedule(job_id=job_id,
schedule_type=schedule_type,
schedule_value=schedule_value)
background_job_manager.restart()
return tolerant_jsonify(job.to_api_json())
def job_disable():
params = request.get_json()
job_id = params.get('jobId')
disable = params.get('disable')
if not job_id or disable is None:
raise BadRequestError('Required parameters are missing.')
job = Job.update_disabled(job_id=job_id, disable=disable)
background_job_manager.restart()
return tolerant_jsonify(job.to_api_json())
def test_authorized(self, client, admin_session):
"""Admin can access available jobs."""
job = Job.get_job_by_key('admin_emails')
expected_value = not job.disabled
api_json = self._api_job_disable(client,
job_id=job.id,
disable=expected_value)
assert api_json['disabled'] is expected_value
std_commit(allow_test_environment=True)
# Reset the value
expected_value = not expected_value
api_json = self._api_job_disable(client,
job_id=job.id,
disable=expected_value)
assert api_json['disabled'] is expected_value
std_commit(allow_test_environment=True)
def run(self, force_run=False):
with self.app_context():
job = Job.get_job_by_key(self.key())
if job:
current_instance_id = os.environ.get('EC2_INSTANCE_ID')
job_runner_id = fetch_job_runner_id()
if job.disabled and not force_run:
app.logger.warn(
f'Job {self.key()} is disabled. It will not run.')
elif current_instance_id and current_instance_id != job_runner_id:
app.logger.warn(
f'Skipping job because current instance {current_instance_id} is not job runner {job_runner_id}'
)
elif JobHistory.is_job_running(job_key=self.key()):
app.logger.warn(
f'Skipping job {self.key()} because an older instance is still running'
)
else:
app.logger.info(f'Job {self.key()} is starting.')
job_tracker = JobHistory.job_started(job_key=self.key())
try:
self._run()
JobHistory.job_finished(id_=job_tracker.id)
app.logger.info(
f'Job {self.key()} finished successfully.')
except Exception as e:
JobHistory.job_finished(id_=job_tracker.id,
failed=True)
summary = f'Job {self.key()} failed due to {str(e)}'
app.logger.error(summary)
app.logger.exception(e)
send_system_error_email(
message=
f'{summary}\n\n<pre>{traceback.format_exc()}</pre>',
subject=f'{summary[:50]}...'
if len(summary) > 50 else summary,
)
else:
raise BackgroundJobError(
f'Job {self.key()} is not registered in the database')
def job_schedule():
api_json = {
'autoStart': app.config['JOBS_AUTO_START'],
'jobs': [],
'secondsBetweenJobsCheck':
app.config['JOBS_SECONDS_BETWEEN_PENDING_CHECK'],
'startedAt': to_isoformat(background_job_manager.get_started_at()),
}
for job in Job.get_all(include_disabled=True):
job_class = next(
(j for j in BackgroundJobManager.available_job_classes()
if j.key() == job.key), None)
if job_class:
api_json['jobs'].append({
**job.to_api_json(),
**_job_class_to_json(job_class),
})
return tolerant_jsonify(api_json)
def test_authorized(self, client, admin_session):
"""Admin can edit job schedule."""
job = Job.get_job_by_key('admin_emails')
api_json = self._api_job_update_schedule(
client,
job_id=job.id,
schedule_type='minutes',
schedule_value=3,
)
assert api_json['schedule'] == {
'type': 'minutes',
'value': 3,
}
api_json = self._api_job_update_schedule(
client,
job_id=job.id,
schedule_type='day_at',
schedule_value='15:30',
)
assert api_json['schedule'] == {
'type': 'day_at',
'value': '15:30',
}
def _set_up_and_run_jobs():
Job.create(job_schedule_type='day_at',
job_schedule_value='15:00',
key='kaltura')
Job.create(job_schedule_type='day_at',
job_schedule_value='04:30',
key='queued_emails')
Job.create(job_schedule_type='day_at',
job_schedule_value='22:00',
key='house_keeping')
Job.create(job_schedule_type='minutes',
job_schedule_value='120',
key='instructor_emails')
Job.create(job_schedule_type='minutes',
job_schedule_value='120',
key='invitations')
Job.create(disabled=True,
job_schedule_type='minutes',
job_schedule_value='120',
key='admin_emails')
Job.create(job_schedule_type='day_at',
job_schedule_value='16:00',
key='canvas')
Job.create(disabled=True,
job_schedule_type='minutes',
job_schedule_value='5',
key='doomed_to_fail')
background_job_manager.start(app)
HouseKeepingJob(app_context=simply_yield).run()
CanvasJob(app_context=simply_yield).run()
std_commit(allow_test_environment=True)
def start(self, app):
"""Continuously run, executing pending jobs per time interval.
It is intended behavior that ScheduleThread does not run missed jobs. For example, if you register a job that
should run every minute and yet JOBS_SECONDS_BETWEEN_PENDING_CHECK is set to one hour, then your job won't run
60 times at each interval. It will run once.
"""
if self.is_running():
return
else:
self.monitor.notify(is_running=True)
self.started_at = datetime.now()
class JobRunnerThread(threading.Thread):
active = False
@classmethod
def run(cls):
cls.active = True
while self.monitor.is_running():
schedule.run_pending()
time.sleep(interval)
schedule.clear()
cls.active = False
interval = app.config['JOBS_SECONDS_BETWEEN_PENDING_CHECK']
all_jobs = Job.get_all()
app.logger.info(f"""
Starting background job manager.
Seconds between pending jobs check = {interval}
Jobs:
{[job.to_api_json() for job in all_jobs]}
""")
# If running on EC2, tell the database that this instance is the one now running scheduled jobs.
instance_id = os.environ.get('EC2_INSTANCE_ID')
if instance_id:
rds.execute(
'DELETE FROM job_runner; INSERT INTO job_runner (ec2_instance_id) VALUES (%s);',
params=(instance_id, ),
)
# Clean up history for any older jobs that got lost.
JobHistory.fail_orphans()
if all_jobs:
for job_config in all_jobs:
self._load_job(
app=app,
job_key=job_config.key,
schedule_type=job_config.job_schedule_type,
schedule_value=job_config.job_schedule_value,
)
self.continuous_thread = JobRunnerThread(daemon=True)
self.continuous_thread.start()
else:
app.logger.warn('No jobs. Nothing scheduled.')
