def check_job_dependency(dependency, job_schedule_time, settings):
dependency_type = dependency.keys()[0]
dependency_job_id = dependency.get(dependency_type).get('id')
dependency_job_schedule_time_lag = dependency.get(dependency_type).get(
'last_schedule_time_interval_minutes')
dependency_job_schedule_time = (string_to_iso_datetime(job_schedule_time) - timedelta(
minutes=int(dependency_job_schedule_time_lag))).isoformat()
if check_job_succeeded(dependency_job_id, dependency_job_schedule_time, db_name=settings.metadata):
return True
else:
return False
def update_task(job_id, schedule_time, status):
try:
schedule_time_parsed = string_to_iso_datetime(schedule_time).isoformat()
except ValueError:
return ('Invalid date %s. Must match ' % schedule_time + "%Y-%m-%dT%H:%M:%S"), 500
try:
assert status in ['failed', 'ready', 'not_ready', 'succeeded']
except:
return "Invalid status. Must be one of the following: %s" % 'failed, ready, not_ready, succeeded', 500
update_job_status.get(status)(job_id, schedule_time_parsed, db_name=metadata)
return '%s marked as %s for %s' % (job_id, status, schedule_time_parsed)
def sync_job(job_config, settings):
job_id = job_config.get('id')
job_config_start_datetime = string_to_iso_datetime(job_config.get('start_time'))
job_schedule_datetime_end = datetime.datetime.now() + datetime.timedelta(days=settings.metadata_future_days)
if job_config_start_datetime <= job_schedule_datetime_end:
job_schedule_datetime_start = string_to_iso_datetime(max(job_config_start_datetime,
datetime.datetime.now() - datetime.timedelta(
days=settings.metadata_history_days)).strftime('%Y-%m-%d') + 'T' +
job_config_start_datetime.strftime('%H:%M:%S'))
delta_schedule_seconds = (job_schedule_datetime_end - job_schedule_datetime_start).days * 1440
job_schedule_interval_minutes = int(job_config.get('interval_minutes') or 60)
job_schedule = [
(job_schedule_datetime_start + datetime.timedelta(
minutes=i * job_schedule_interval_minutes)).isoformat()
for i in range(int(delta_schedule_seconds / job_schedule_interval_minutes))]
for schedule_time in job_schedule:
if not job_is_present(job_id, schedule_time, db_name=settings.metadata):
add_new_job(job_id, schedule_time, db_name=settings.metadata)
else:
# Ignore all jobs that will start in the future
pass
def dependencies_are_met(job_config, schedule_time, settings):
delay_minutes = int(job_config.get('delay_minutes') or 1440)
if string_to_iso_datetime(schedule_time) + timedelta(minutes=delay_minutes) >= datetime.now():
return False
for dependency in job_config.get('dependencies', []):
dependency_type = dependency.keys()[0]
if dependency_type not in supported_dependencies:
settings.logger.error('Unsupported dependency type "%s"') % dependency_type
return False
else:
if dependency_is_met.get(dependency_type)(dependency, schedule_time, settings):
continue
else:
return False
return True
