def _should_process_job(self, job): if not super(ETScanner, self)._should_process_job(job): return False et_lsd = job.get(et_last_successful_date=None)['et_last_successful_date'] if et_lsd is not None \ and not self._data_available_for_date(job, next_date_for_string(et_lsd)): return False return True
def _should_process_job(self, job): if not super(ETScanner, self)._should_process_job(job): return False et_lsd = job.get( et_last_successful_date=None)['et_last_successful_date'] if et_lsd is not None \ and not self._data_available_for_date(job, next_date_for_string(et_lsd)): return False return True
def _has_more_load_runs_to_schedule(self): if self.stop_load_due_to_failure: return False if self.load_runs_in_flight != 0: return False # only single load run in flight is possible for redshift if self.scheduled_load_date is None: return self._is_ready_for_load_step(self.start) if self.scheduled_load_date == self.end: return False return self._is_ready_for_load_step( next_date_for_string(self.scheduled_load_date, self.step))
def _has_more_load_runs_to_schedule(self): if self.stop_load_due_to_failure: return False if self.load_runs_in_flight != 0: return False # only single load run in flight is possible for redshift if self.scheduled_load_date is None: return self._is_ready_for_load_step(self.start) if self.scheduled_load_date == self.end: return False return self._is_ready_for_load_step( next_date_for_string(self.scheduled_load_date, self.step) )
def _create_work_for_job(self, job): # Update status = SCHEDULED # iff the status is not SCHEDULED/COMPLETE/ERROR job_dict = job.get(**self.DEFAULT_KEYS_TO_FETCH_FROM_JOB) if not self._update_job_status_conditionally(job): log("ET: Could not conditionally update job to status 'scheduled'" ".Skipping this job this time : " + str(job)) return start_date = self._later_date( job_dict['start_date'], next_date_for_string(job_dict['et_last_successful_date'])) end_date = self._earlier_date(job_dict['end_date'], self._get_max_available_date(job)) self._enqueue_work_in_sqs(start_date, end_date, "ET", job_dict) log("Enqueued ET work item in SQS, for job: " + str(job_dict))
def _create_work_for_job(self, job): # Update status = SCHEDULED # iff the status is not SCHEDULED/COMPLETE/ERROR job_dict = job.get(**self.DEFAULT_KEYS_TO_FETCH_FROM_JOB) if not self._update_job_status_conditionally(job): log( "ET: Could not conditionally update job to status 'scheduled'" ".Skipping this job this time : " + str(job) ) return start_date = self._later_date( job_dict['start_date'], next_date_for_string(job_dict['et_last_successful_date']) ) end_date = self._earlier_date(job_dict['end_date'], self._get_max_available_date(job)) self._enqueue_work_in_sqs(start_date, end_date, "ET", job_dict) log("Enqueued ET work item in SQS, for job: " + str(job_dict))
def test_next_date_for_string(date, result): assert next_date_for_string(date) == result