def _should_process_job(self, job):
if not super(ETScanner, self)._should_process_job(job):
return False
et_lsd = job.get(et_last_successful_date=None)['et_last_successful_date']
if et_lsd is not None \
and not self._data_available_for_date(job, next_date_for_string(et_lsd)):
return False
return True
def _should_process_job(self, job):
if not super(ETScanner, self)._should_process_job(job):
return False
et_lsd = job.get(
et_last_successful_date=None)['et_last_successful_date']
if et_lsd is not None \
and not self._data_available_for_date(job, next_date_for_string(et_lsd)):
return False
return True
def _has_more_load_runs_to_schedule(self):
if self.stop_load_due_to_failure:
return False
if self.load_runs_in_flight != 0:
return False # only single load run in flight is possible for redshift
if self.scheduled_load_date is None:
return self._is_ready_for_load_step(self.start)
if self.scheduled_load_date == self.end:
return False
return self._is_ready_for_load_step(
next_date_for_string(self.scheduled_load_date, self.step))
def _has_more_load_runs_to_schedule(self):
if self.stop_load_due_to_failure:
return False
if self.load_runs_in_flight != 0:
return False # only single load run in flight is possible for redshift
if self.scheduled_load_date is None:
return self._is_ready_for_load_step(self.start)
if self.scheduled_load_date == self.end:
return False
return self._is_ready_for_load_step(
next_date_for_string(self.scheduled_load_date, self.step)
)
def _create_work_for_job(self, job):
# Update status = SCHEDULED
# iff the status is not SCHEDULED/COMPLETE/ERROR
job_dict = job.get(**self.DEFAULT_KEYS_TO_FETCH_FROM_JOB)
if not self._update_job_status_conditionally(job):
log("ET: Could not conditionally update job to status 'scheduled'"
".Skipping this job this time : " + str(job))
return
start_date = self._later_date(
job_dict['start_date'],
next_date_for_string(job_dict['et_last_successful_date']))
end_date = self._earlier_date(job_dict['end_date'],
self._get_max_available_date(job))
self._enqueue_work_in_sqs(start_date, end_date, "ET", job_dict)
log("Enqueued ET work item in SQS, for job: " + str(job_dict))
def _create_work_for_job(self, job):
# Update status = SCHEDULED
# iff the status is not SCHEDULED/COMPLETE/ERROR
job_dict = job.get(**self.DEFAULT_KEYS_TO_FETCH_FROM_JOB)
if not self._update_job_status_conditionally(job):
log(
"ET: Could not conditionally update job to status 'scheduled'"
".Skipping this job this time : " + str(job)
)
return
start_date = self._later_date(
job_dict['start_date'],
next_date_for_string(job_dict['et_last_successful_date'])
)
end_date = self._earlier_date(job_dict['end_date'], self._get_max_available_date(job))
self._enqueue_work_in_sqs(start_date, end_date, "ET", job_dict)
log("Enqueued ET work item in SQS, for job: " + str(job_dict))
def test_next_date_for_string(date, result):
assert next_date_for_string(date) == result
def test_next_date_for_string(date, result):
assert next_date_for_string(date) == result
