def test_edit_schedule_of_running_job(self, client, admin_session):
"""You cannot edit job schedule if the job is running."""
job = Job.get_job_by_key('admin_emails')
job_history = JobHistory.job_started(job_key=job.key)
self._api_job_update_schedule(
client,
expected_status_code=400,
job_id=Job.get_job_by_key('admin_emails').id,
schedule_type='minutes',
schedule_value=3,
)
JobHistory.job_finished(id_=job_history.id, failed=False)
def run_with_app_context(self):
with self.app_context():
app.logger.info(f'Job \'{self.key()}\' is starting.')
job_tracker = JobHistory.job_started(job_key=self.key())
try:
self.run()
JobHistory.job_finished(id_=job_tracker.id)
app.logger.info(f'Job \'{self.key()}\' finished successfully.')
except Exception as e:
app.logger.error(f'Job {self.key()} failed')
app.logger.exception(e)
JobHistory.job_finished(id_=job_tracker.id, failed=True)
def job_history(day_count):
def _raise_error():
raise BadRequestError(f'Invalid day_count: {day_count}')
try:
days = int(day_count)
if days < 1:
_raise_error()
return tolerant_jsonify([
h.to_api_json()
for h in JobHistory.get_job_history_in_past_days(day_count=days)
])
except ValueError:
_raise_error()
def update_schedule():
params = request.get_json()
job_id = params.get('jobId')
schedule_type = params.get('type')
schedule_value = params.get('value')
if not job_id or not schedule_type or not schedule_value:
raise BadRequestError('Required parameters are missing.')
job = Job.get_job(job_id=job_id)
if not job.disabled or JobHistory.is_job_running(job_key=job.key):
raise BadRequestError(
'You cannot edit job schedule if job is either enabled or running.'
)
job = Job.update_schedule(job_id=job_id,
schedule_type=schedule_type,
schedule_value=schedule_value)
background_job_manager.restart()
return tolerant_jsonify(job.to_api_json())
def run(self, force_run=False):
with self.app_context():
job = Job.get_job_by_key(self.key())
if job:
current_instance_id = os.environ.get('EC2_INSTANCE_ID')
job_runner_id = fetch_job_runner_id()
if job.disabled and not force_run:
app.logger.warn(
f'Job {self.key()} is disabled. It will not run.')
elif current_instance_id and current_instance_id != job_runner_id:
app.logger.warn(
f'Skipping job because current instance {current_instance_id} is not job runner {job_runner_id}'
)
elif JobHistory.is_job_running(job_key=self.key()):
app.logger.warn(
f'Skipping job {self.key()} because an older instance is still running'
)
else:
app.logger.info(f'Job {self.key()} is starting.')
job_tracker = JobHistory.job_started(job_key=self.key())
try:
self._run()
JobHistory.job_finished(id_=job_tracker.id)
app.logger.info(
f'Job {self.key()} finished successfully.')
except Exception as e:
JobHistory.job_finished(id_=job_tracker.id,
failed=True)
summary = f'Job {self.key()} failed due to {str(e)}'
app.logger.error(summary)
app.logger.exception(e)
send_system_error_email(
message=
f'{summary}\n\n<pre>{traceback.format_exc()}</pre>',
subject=f'{summary[:50]}...'
if len(summary) > 50 else summary,
)
else:
raise BackgroundJobError(
f'Job {self.key()} is not registered in the database')
def _run(self):
JobHistory.expire_old_rows(app.config['JOB_HISTORY_DAYS_UNTIL_EXPIRE'])
def last_successful_run(job_key):
entry = JobHistory.last_successful_run_of(job_key=job_key)
return tolerant_jsonify(entry and entry.to_api_json())
def job_history():
return tolerant_jsonify(
[h.to_api_json() for h in JobHistory.get_job_history()])
def start(self, app):
"""Continuously run, executing pending jobs per time interval.
It is intended behavior that ScheduleThread does not run missed jobs. For example, if you register a job that
should run every minute and yet JOBS_SECONDS_BETWEEN_PENDING_CHECK is set to one hour, then your job won't run
60 times at each interval. It will run once.
"""
if self.is_running():
return
else:
self.monitor.notify(is_running=True)
self.started_at = datetime.now()
class JobRunnerThread(threading.Thread):
active = False
@classmethod
def run(cls):
cls.active = True
while self.monitor.is_running():
schedule.run_pending()
time.sleep(interval)
schedule.clear()
cls.active = False
interval = app.config['JOBS_SECONDS_BETWEEN_PENDING_CHECK']
all_jobs = Job.get_all()
app.logger.info(f"""
Starting background job manager.
Seconds between pending jobs check = {interval}
Jobs:
{[job.to_api_json() for job in all_jobs]}
""")
# If running on EC2, tell the database that this instance is the one now running scheduled jobs.
instance_id = os.environ.get('EC2_INSTANCE_ID')
if instance_id:
rds.execute(
'DELETE FROM job_runner; INSERT INTO job_runner (ec2_instance_id) VALUES (%s);',
params=(instance_id, ),
)
# Clean up history for any older jobs that got lost.
JobHistory.fail_orphans()
if all_jobs:
for job_config in all_jobs:
self._load_job(
app=app,
job_key=job_config.key,
schedule_type=job_config.job_schedule_type,
schedule_value=job_config.job_schedule_value,
)
self.continuous_thread = JobRunnerThread(daemon=True)
self.continuous_thread.start()
else:
app.logger.warn('No jobs. Nothing scheduled.')