def test_get_job_data_gets_data(self):
data = {
'project': 'banana',
'user': '******',
'parameters': json.dumps({'harry': 'potter'}),
'metrics': self._fast_serialize(('123', 'hermione', 'granger')),
'state': 'dead',
'start_time': '456',
'completed_time': '123',
'creation_time': '789',
'tags': {}
}
job_id = 'the boy who lived'
redis_pipe = RedisPipelineWrapper(self._redis.pipeline())
job_data = JobDataRedis(redis_pipe, job_id)
self._load_data_new_job(job_id, data)
result = job_data.get_job_data()
redis_pipe.execute()
expected_result = {
'project_name': 'banana',
'job_id': job_id,
'user': '******',
'job_parameters': {
'harry': 'potter'
},
'output_metrics': [('123', 'hermione', 'granger')],
'status': 'dead',
'start_time': float('456'),
'completed_time': float('123'),
'creation_time': float('789'),
'tags': {}
}
self.assertDictEqual(expected_result, result.get())
def get_param(self, param_name, wait_seconds=5):
from foundations_contrib.job_data_redis import JobDataRedis
from foundations_contrib.global_state import redis_connection
self.wait_for_deployment_to_complete(wait_seconds=wait_seconds,
log_output=False)
pipe = JobDataRedis._create_redis_pipeline(redis_connection)
param = JobDataRedis(pipe, self.job_name()).get_job_param(param_name)
return param
def test_get_job_data_gets_data_default_data(self):
data = {}
job_id = 'the boy who lived'
redis_pipe = RedisPipelineWrapper(self._redis.pipeline())
job_data = JobDataRedis(redis_pipe, job_id)
self._load_data_new_job(job_id, data)
result = job_data.get_job_data()
redis_pipe.execute()
expected_result = {
'project_name': None,
'job_id': job_id,
'user': None,
'job_parameters': {},
'output_metrics': [],
'status': None,
'start_time': None,
'completed_time': None,
'creation_time': None,
'tags': {}
}
self.assertDictEqual(expected_result, result.get())
def test_produces_proper_data(self):
from foundations_contrib.job_data_redis import JobDataRedis
self._deploy_job_file('acceptance/fixtures/job_data_production', entrypoint='success.py')
all_job_data = JobDataRedis.get_all_jobs_data('job_data_production', self._redis)
job_data = [data for data in all_job_data if data['job_id'] == self.job_id][0]
self.assertEqual('job_data_production', job_data['project_name'])
self.assertEqual(self.user_name, job_data['user'])
self.assertEqual('completed', job_data['status'])
self.assertTrue(isinstance(job_data['start_time'], float))
self.assertTrue(isinstance(job_data['completed_time'], float))
self.assertGreater(len(job_data['output_metrics']), 0)
def test_get_job_data_all_jobs_single_job_different_data(self):
project_name = 'pomme'
data = {
'project': project_name,
'user': '******',
'parameters': json.dumps({'ron': 'potter'}),
'metrics': self._fast_serialize(('123', 'hermione', 'granger')),
'state': 'completed',
'start_time': '1231003123',
'completed_time': '123',
'creation_time': '789',
'tags': {
'bep': 'bip'
}
}
job_id_1 = 'sushine'
self._sadd_redis_project_name(project_name, job_id_1)
self._load_data_new_job(job_id_1, data)
expected_result_1 = {
'project_name': project_name,
'job_id': job_id_1,
'user': '******',
'job_parameters': {
'ron': 'potter'
},
'output_metrics': [('123', 'hermione', 'granger')],
'status': 'completed',
'start_time': float('1231003123'),
'completed_time': float('123'),
'creation_time': float('789'),
'tags': {
'bep': 'bip'
}
}
results = JobDataRedis.get_all_jobs_data(project_name, self._redis)
self.assertDictEqual(results[0], expected_result_1)
def completed_job_data(project_name):
"""
Returns all data for running and completed jobs under a specific project name.
Arguments:
project_name {str} -- Name of project
Returns:
job_data {list} -- List with job data. Each element of the list contains a dictionary
with project_name, job_id, user, job_parameters, input_params, output_metrics, status, start_time, completed_time.
"""
from foundations_contrib.job_data_redis import JobDataRedis
from foundations_contrib.job_data_shaper import JobDataShaper
from foundations_contrib.global_state import redis_connection
jobs_data = JobDataRedis.get_all_jobs_data(project_name, redis_connection)
for job in jobs_data:
job['output_metrics'] = JobDataShaper.shape_output_metrics(job['output_metrics'])
return jobs_data
def test_is_job_completed_returns_false_if_job_does_not_exist(self):
self.assertFalse(
JobDataRedis.is_job_completed(self.fake_job_1, self._redis))
def test_is_job_completed_returns_true_if_job_exists(self):
self._set_redis(self.fake_job_1, 'completed_time', self.fake_timestamp)
self.assertTrue(
JobDataRedis.is_job_completed(self.fake_job_1, self._redis))
def test_list_all_completed_jobs_lists_completed_jobs(self):
self._set_redis(self.fake_job_1, 'completed_time', self.fake_timestamp)
self._set_redis(self.fake_job_2, 'completed_time', self.fake_timestamp)
expected_result = [self.fake_job_1, self.fake_job_2]
self.assertEqual(expected_result,
JobDataRedis.list_all_completed_jobs(self._redis))
def test_get_job_data_all_jobs_two_jobs(self):
project_name = 'apple'
data = {
'project': project_name,
'user': '******',
'parameters': json.dumps({'ron': 'potter'}),
'metrics': self._fast_serialize(('123', 'hermione', 'granger')),
'state': 'completed',
'start_time': '1231003123',
'completed_time': '123',
'creation_time': '789',
'tags': {
'bep': 'bip'
}
}
job_id_1 = 'sushine'
job_id_2 = 'rain'
self._sadd_redis_project_name(project_name, job_id_1)
self._sadd_redis_project_name(project_name, job_id_2)
self._load_data_new_job(job_id_1, data)
self._load_data_new_job(job_id_2, data)
expected_result_1 = {
'project_name': project_name,
'job_id': job_id_1,
'user': '******',
'job_parameters': {
'ron': 'potter'
},
'output_metrics': [('123', 'hermione', 'granger')],
'status': 'completed',
'start_time': float('1231003123'),
'completed_time': float('123'),
'creation_time': float('789'),
'tags': {
'bep': 'bip'
}
}
expected_result_2 = {
'project_name': project_name,
'job_id': job_id_2,
'user': '******',
'job_parameters': {
'ron': 'potter'
},
'output_metrics': [('123', 'hermione', 'granger')],
'status': 'completed',
'start_time': float('1231003123'),
'completed_time': float('123'),
'creation_time': float('789'),
'tags': {
'bep': 'bip'
}
}
results = JobDataRedis.get_all_jobs_data(project_name, self._redis)
six.assertCountEqual(self, results,
[expected_result_1, expected_result_2])