def sync_jobs(settings):
remove_old_data(settings.metadata_history_days, db_name=settings.metadata)
remote_jobs = get_config(settings.remote_jobs_config, settings.remote_jobs_config_schema)
aws_jobs = get_config(settings.aws_jobs_config, settings.aws_jobs_config_schema)
for remote_job_config in remote_jobs.get('jobs', {}):
sync_job(remote_job_config, settings)
for aws_job_config in aws_jobs.get('jobs', {}):
sync_job(aws_job_config, settings)
return remote_jobs.get('jobs', []) + aws_jobs.get('jobs', [])
def all_jobs_from_db_to_json(db_result):
remote_jobs = get_config(settings.remote_jobs_config, settings.remote_jobs_config_schema)
aws_jobs = get_config(settings.aws_jobs_config, settings.aws_jobs_config_schema)
all_jobs = remote_jobs.get('jobs', []) + aws_jobs.get('jobs', [])
active_job_ids = map(lambda x: x.get('id'), all_jobs)
def get_status_name(status_id):
return [name for name, id in job_status.items() if status_id == id][0]
def format_row(row):
return {
'job_id': row[0],
'schedule_time': row[1],
'execution_time': row[2],
'status': get_status_name(row[3]),
'runs': row[4]
}
return json.dumps(filter(lambda x: x.get('job_id') in active_job_ids, map(lambda x: format_row(x), db_result)),
indent=4)
def test_get_config(self):
result = get_config(settings.remote_jobs_config, settings.remote_jobs_config_schema)
expected_result = {u'jobs': [{u'delay_minutes': u'',
u'dependencies': [{u'job_completed': {u'id': u'test_remote_action',
u'last_schedule_time_interval_minutes': u'1440'}}],
u'id': u'test_remote_action',
u'interval_minutes': u'1440',
u'remote_action': {u'args': [u'dummy_arg', u'dummy_arg'],
u'script': u'/tmp/dummy.sh'},
u'remote_server_id': u'example',
u'retry': u'3',
u'start_time': u'2015-10-05T00:00:00',
u'type': u'ssh'}]}
self.assertEqual(result, expected_result)
def test_launch_emr_task(self):
config = get_config(settings.aws_jobs_config, settings.aws_jobs_config_schema)
for job_config in config.get('jobs', []):
result = launch_emr_task(job_config, 2, settings)
print(result)
