def enabled_job(job_key): """Temporarily enabled job.""" all_jobs = Job.get_all(include_disabled=True) job = next((j for j in all_jobs if j.key == job_key)) Job.update_disabled(job_id=job.id, disable=False) std_commit(allow_test_environment=True) try: yield finally: Job.update_disabled(job_id=job.id, disable=True) std_commit(allow_test_environment=True)
def test_edit_schedule_of_running_job(self, client, admin_session): """You cannot edit job schedule if the job is running.""" job = Job.get_job_by_key('admin_emails') job_history = JobHistory.job_started(job_key=job.key) self._api_job_update_schedule( client, expected_status_code=400, job_id=Job.get_job_by_key('admin_emails').id, schedule_type='minutes', schedule_value=3, ) JobHistory.job_finished(id_=job_history.id, failed=False)
def test_alert_on_job_failure(self): admin_uid = app.config['EMAIL_DIABLO_ADMIN_UID'] email_count = _get_email_count(admin_uid) # No alert on happy job. CanvasJob(simply_yield).run() assert _get_email_count(admin_uid) == email_count # Alert on sad job. all_jobs = Job.get_all(include_disabled=True) doomed_job = next( (j for j in all_jobs if j.key == DoomedToFailure.key())) # Make sure job is enabled Job.update_disabled(job_id=doomed_job.id, disable=False) std_commit(allow_test_environment=True) DoomedToFailure(simply_yield).run() # Failure alerts do not go through the queue. assert _get_email_count(admin_uid) == email_count + 1
def test_edit_schedule_of_enabled_job(self, client, admin_session): """You cannot edit job schedule if the job is enabled.""" with enabled_job(job_key=AdminEmailsJob.key()): self._api_job_update_schedule( client, expected_status_code=400, job_id=Job.get_job_by_key('admin_emails').id, schedule_type='minutes', schedule_value=3, )
def update_schedule(): params = request.get_json() job_id = params.get('jobId') schedule_type = params.get('type') schedule_value = params.get('value') if not job_id or not schedule_type or not schedule_value: raise BadRequestError('Required parameters are missing.') job = Job.get_job(job_id=job_id) if not job.disabled or JobHistory.is_job_running(job_key=job.key): raise BadRequestError( 'You cannot edit job schedule if job is either enabled or running.' ) job = Job.update_schedule(job_id=job_id, schedule_type=schedule_type, schedule_value=schedule_value) background_job_manager.restart() return tolerant_jsonify(job.to_api_json())
def job_disable(): params = request.get_json() job_id = params.get('jobId') disable = params.get('disable') if not job_id or disable is None: raise BadRequestError('Required parameters are missing.') job = Job.update_disabled(job_id=job_id, disable=disable) background_job_manager.restart() return tolerant_jsonify(job.to_api_json())
def test_authorized(self, client, admin_session): """Admin can access available jobs.""" job = Job.get_job_by_key('admin_emails') expected_value = not job.disabled api_json = self._api_job_disable(client, job_id=job.id, disable=expected_value) assert api_json['disabled'] is expected_value std_commit(allow_test_environment=True) # Reset the value expected_value = not expected_value api_json = self._api_job_disable(client, job_id=job.id, disable=expected_value) assert api_json['disabled'] is expected_value std_commit(allow_test_environment=True)
def run(self, force_run=False): with self.app_context(): job = Job.get_job_by_key(self.key()) if job: current_instance_id = os.environ.get('EC2_INSTANCE_ID') job_runner_id = fetch_job_runner_id() if job.disabled and not force_run: app.logger.warn( f'Job {self.key()} is disabled. It will not run.') elif current_instance_id and current_instance_id != job_runner_id: app.logger.warn( f'Skipping job because current instance {current_instance_id} is not job runner {job_runner_id}' ) elif JobHistory.is_job_running(job_key=self.key()): app.logger.warn( f'Skipping job {self.key()} because an older instance is still running' ) else: app.logger.info(f'Job {self.key()} is starting.') job_tracker = JobHistory.job_started(job_key=self.key()) try: self._run() JobHistory.job_finished(id_=job_tracker.id) app.logger.info( f'Job {self.key()} finished successfully.') except Exception as e: JobHistory.job_finished(id_=job_tracker.id, failed=True) summary = f'Job {self.key()} failed due to {str(e)}' app.logger.error(summary) app.logger.exception(e) send_system_error_email( message= f'{summary}\n\n<pre>{traceback.format_exc()}</pre>', subject=f'{summary[:50]}...' if len(summary) > 50 else summary, ) else: raise BackgroundJobError( f'Job {self.key()} is not registered in the database')
def job_schedule(): api_json = { 'autoStart': app.config['JOBS_AUTO_START'], 'jobs': [], 'secondsBetweenJobsCheck': app.config['JOBS_SECONDS_BETWEEN_PENDING_CHECK'], 'startedAt': to_isoformat(background_job_manager.get_started_at()), } for job in Job.get_all(include_disabled=True): job_class = next( (j for j in BackgroundJobManager.available_job_classes() if j.key() == job.key), None) if job_class: api_json['jobs'].append({ **job.to_api_json(), **_job_class_to_json(job_class), }) return tolerant_jsonify(api_json)
def test_authorized(self, client, admin_session): """Admin can edit job schedule.""" job = Job.get_job_by_key('admin_emails') api_json = self._api_job_update_schedule( client, job_id=job.id, schedule_type='minutes', schedule_value=3, ) assert api_json['schedule'] == { 'type': 'minutes', 'value': 3, } api_json = self._api_job_update_schedule( client, job_id=job.id, schedule_type='day_at', schedule_value='15:30', ) assert api_json['schedule'] == { 'type': 'day_at', 'value': '15:30', }
def _set_up_and_run_jobs(): Job.create(job_schedule_type='day_at', job_schedule_value='15:00', key='kaltura') Job.create(job_schedule_type='day_at', job_schedule_value='04:30', key='queued_emails') Job.create(job_schedule_type='day_at', job_schedule_value='22:00', key='house_keeping') Job.create(job_schedule_type='minutes', job_schedule_value='120', key='instructor_emails') Job.create(job_schedule_type='minutes', job_schedule_value='120', key='invitations') Job.create(disabled=True, job_schedule_type='minutes', job_schedule_value='120', key='admin_emails') Job.create(job_schedule_type='day_at', job_schedule_value='16:00', key='canvas') Job.create(disabled=True, job_schedule_type='minutes', job_schedule_value='5', key='doomed_to_fail') background_job_manager.start(app) HouseKeepingJob(app_context=simply_yield).run() CanvasJob(app_context=simply_yield).run() std_commit(allow_test_environment=True)
def start(self, app): """Continuously run, executing pending jobs per time interval. It is intended behavior that ScheduleThread does not run missed jobs. For example, if you register a job that should run every minute and yet JOBS_SECONDS_BETWEEN_PENDING_CHECK is set to one hour, then your job won't run 60 times at each interval. It will run once. """ if self.is_running(): return else: self.monitor.notify(is_running=True) self.started_at = datetime.now() class JobRunnerThread(threading.Thread): active = False @classmethod def run(cls): cls.active = True while self.monitor.is_running(): schedule.run_pending() time.sleep(interval) schedule.clear() cls.active = False interval = app.config['JOBS_SECONDS_BETWEEN_PENDING_CHECK'] all_jobs = Job.get_all() app.logger.info(f""" Starting background job manager. Seconds between pending jobs check = {interval} Jobs: {[job.to_api_json() for job in all_jobs]} """) # If running on EC2, tell the database that this instance is the one now running scheduled jobs. instance_id = os.environ.get('EC2_INSTANCE_ID') if instance_id: rds.execute( 'DELETE FROM job_runner; INSERT INTO job_runner (ec2_instance_id) VALUES (%s);', params=(instance_id, ), ) # Clean up history for any older jobs that got lost. JobHistory.fail_orphans() if all_jobs: for job_config in all_jobs: self._load_job( app=app, job_key=job_config.key, schedule_type=job_config.job_schedule_type, schedule_value=job_config.job_schedule_value, ) self.continuous_thread = JobRunnerThread(daemon=True) self.continuous_thread.start() else: app.logger.warn('No jobs. Nothing scheduled.')