def sync_jobs(settings): remove_old_data(settings.metadata_history_days, db_name=settings.metadata) remote_jobs = get_config(settings.remote_jobs_config, settings.remote_jobs_config_schema) aws_jobs = get_config(settings.aws_jobs_config, settings.aws_jobs_config_schema) for remote_job_config in remote_jobs.get('jobs', {}): sync_job(remote_job_config, settings) for aws_job_config in aws_jobs.get('jobs', {}): sync_job(aws_job_config, settings) return remote_jobs.get('jobs', []) + aws_jobs.get('jobs', [])
def all_jobs_from_db_to_json(db_result): remote_jobs = get_config(settings.remote_jobs_config, settings.remote_jobs_config_schema) aws_jobs = get_config(settings.aws_jobs_config, settings.aws_jobs_config_schema) all_jobs = remote_jobs.get('jobs', []) + aws_jobs.get('jobs', []) active_job_ids = map(lambda x: x.get('id'), all_jobs) def get_status_name(status_id): return [name for name, id in job_status.items() if status_id == id][0] def format_row(row): return { 'job_id': row[0], 'schedule_time': row[1], 'execution_time': row[2], 'status': get_status_name(row[3]), 'runs': row[4] } return json.dumps(filter(lambda x: x.get('job_id') in active_job_ids, map(lambda x: format_row(x), db_result)), indent=4)
def test_get_config(self): result = get_config(settings.remote_jobs_config, settings.remote_jobs_config_schema) expected_result = {u'jobs': [{u'delay_minutes': u'', u'dependencies': [{u'job_completed': {u'id': u'test_remote_action', u'last_schedule_time_interval_minutes': u'1440'}}], u'id': u'test_remote_action', u'interval_minutes': u'1440', u'remote_action': {u'args': [u'dummy_arg', u'dummy_arg'], u'script': u'/tmp/dummy.sh'}, u'remote_server_id': u'example', u'retry': u'3', u'start_time': u'2015-10-05T00:00:00', u'type': u'ssh'}]} self.assertEqual(result, expected_result)
def test_launch_emr_task(self): config = get_config(settings.aws_jobs_config, settings.aws_jobs_config_schema) for job_config in config.get('jobs', []): result = launch_emr_task(job_config, 2, settings) print(result)