def check_job_dependency(dependency, job_schedule_time, settings): dependency_type = dependency.keys()[0] dependency_job_id = dependency.get(dependency_type).get('id') dependency_job_schedule_time_lag = dependency.get(dependency_type).get( 'last_schedule_time_interval_minutes') dependency_job_schedule_time = (string_to_iso_datetime(job_schedule_time) - timedelta( minutes=int(dependency_job_schedule_time_lag))).isoformat() if check_job_succeeded(dependency_job_id, dependency_job_schedule_time, db_name=settings.metadata): return True else: return False
def update_task(job_id, schedule_time, status): try: schedule_time_parsed = string_to_iso_datetime(schedule_time).isoformat() except ValueError: return ('Invalid date %s. Must match ' % schedule_time + "%Y-%m-%dT%H:%M:%S"), 500 try: assert status in ['failed', 'ready', 'not_ready', 'succeeded'] except: return "Invalid status. Must be one of the following: %s" % 'failed, ready, not_ready, succeeded', 500 update_job_status.get(status)(job_id, schedule_time_parsed, db_name=metadata) return '%s marked as %s for %s' % (job_id, status, schedule_time_parsed)
def sync_job(job_config, settings): job_id = job_config.get('id') job_config_start_datetime = string_to_iso_datetime(job_config.get('start_time')) job_schedule_datetime_end = datetime.datetime.now() + datetime.timedelta(days=settings.metadata_future_days) if job_config_start_datetime <= job_schedule_datetime_end: job_schedule_datetime_start = string_to_iso_datetime(max(job_config_start_datetime, datetime.datetime.now() - datetime.timedelta( days=settings.metadata_history_days)).strftime('%Y-%m-%d') + 'T' + job_config_start_datetime.strftime('%H:%M:%S')) delta_schedule_seconds = (job_schedule_datetime_end - job_schedule_datetime_start).days * 1440 job_schedule_interval_minutes = int(job_config.get('interval_minutes') or 60) job_schedule = [ (job_schedule_datetime_start + datetime.timedelta( minutes=i * job_schedule_interval_minutes)).isoformat() for i in range(int(delta_schedule_seconds / job_schedule_interval_minutes))] for schedule_time in job_schedule: if not job_is_present(job_id, schedule_time, db_name=settings.metadata): add_new_job(job_id, schedule_time, db_name=settings.metadata) else: # Ignore all jobs that will start in the future pass
def dependencies_are_met(job_config, schedule_time, settings): delay_minutes = int(job_config.get('delay_minutes') or 1440) if string_to_iso_datetime(schedule_time) + timedelta(minutes=delay_minutes) >= datetime.now(): return False for dependency in job_config.get('dependencies', []): dependency_type = dependency.keys()[0] if dependency_type not in supported_dependencies: settings.logger.error('Unsupported dependency type "%s"') % dependency_type return False else: if dependency_is_met.get(dependency_type)(dependency, schedule_time, settings): continue else: return False return True