class TethysJobTest(TethysTestCase):
def set_up(self):
self.tz = pytz_timezone('America/Denver')
self.user = User.objects.create_user('tethys_super', '*****@*****.**', 'pass')
self.scheduler = CondorScheduler(
name='test_scheduler',
host='localhost',
)
self.scheduler.save()
self.tethysjob = TethysJob(
name='test_tethysjob',
description='test_description',
user=self.user,
label='test_label',
)
self.tethysjob.save()
self.tethysjob_execute_time = TethysJob(
name='test_tethysjob_execute_time',
description='test_description',
user=self.user,
label='test_label',
execute_time=datetime(year=2018, month=1, day=1, tzinfo=self.tz),
completion_time=datetime(year=2018, month=1, day=1, hour=1, tzinfo=self.tz),
_status='VAR',
_process_results_function=test_function
)
self.tethysjob_execute_time.save()
def tear_down(self):
self.tethysjob.delete()
self.tethysjob_execute_time.delete()
self.scheduler.delete()
def test_type(self):
ret = self.tethysjob.type
self.assertEqual('TethysJob', ret)
def test_update_status_interval_prop(self):
ret = TethysJob.objects.get(name='test_tethysjob').update_status_interval
# Check result
self.assertIsInstance(ret, timedelta)
self.assertEqual(timedelta(0, 10), ret)
def test_last_status_update_prop(self):
ret = TethysJob.objects.get(name='test_tethysjob')
check_date = datetime(year=2018, month=1, day=1, tzinfo=self.tz)
ret._last_status_update = check_date
# Check result
self.assertEqual(check_date, ret.last_status_update)
def test_status_prop(self):
ret = TethysJob.objects.get(name='test_tethysjob').status
# Check result
self.assertEqual('Pending', ret)
def test_status_setter_prop(self):
ret = TethysJob.objects.get(name='test_tethysjob_execute_time').status
# Check result
self.assertEqual('Various', ret)
@mock.patch('tethys_compute.models.tethys_job.TethysJob.update_status')
def test_status_setter(self, mock_update):
tethys_job = TethysJob.objects.get(name='test_tethysjob')
tethys_job.status = 'test_status'
# Check result
mock_update.assert_called_with(status='test_status')
def test_run_time_execute_time_prop(self):
ret = TethysJob.objects.get(name='test_tethysjob_execute_time').run_time
# Check result
self.assertIsInstance(ret, timedelta)
self.assertEqual(timedelta(0, 3600), ret)
def test_run_time_execute_time_prop_with_start_time(self):
ret = TethysJob.objects.get(name='test_tethysjob').run_time
# Check result
self.assertEqual('', ret)
def test_run_time_execute_time_no_start_time_prop(self):
ret = TethysJob.objects.get(name='test_tethysjob').run_time
# Check result
self.assertEqual("", ret)
def test_execute(self):
ret_old = TethysJob.objects.get(name='test_tethysjob_execute_time')
TethysJob.objects.get(name='test_tethysjob_execute_time').execute()
ret_new = TethysJob.objects.get(name='test_tethysjob_execute_time')
self.assertNotEqual(ret_old.execute_time, ret_new.execute_time)
self.assertEqual('Various', ret_old.status)
self.assertEqual('Submitted', ret_new.status)
@mock.patch('tethys_compute.models.tethys_job.log')
def test_update_status_invalid(self, mock_log):
tethysjob = TethysJob(
name='test_tethysjob',
description='test_description',
user=self.user,
label='test_label',
execute_time=django_timezone.now(),
)
tethysjob._status = ''
# Check result
self.assertIsNone(tethysjob.update_status('Test'))
mock_log.error.assert_called_with('Invalid status given: Test')
@mock.patch('tethys_compute.models.tethys_job.TethysJob.is_time_to_update')
@mock.patch('tethys_compute.models.condor.condor_base.CondorBase.condor_object')
def test_update_status_run(self, mock_co, mock_tup):
tethysjob = CondorBase(
name='test_tethysjob',
description='test_description',
user=self.user,
label='test_label',
scheduler=self.scheduler,
execute_time=django_timezone.now(),
)
mock_tup.return_value = True
mock_co.status = 'Running'
tethysjob.update_status()
# Check result
self.assertIsNotNone(tethysjob.last_status_update)
self.assertIsInstance(tethysjob.last_status_update, datetime)
self.assertIsNotNone(tethysjob.start_time)
self.assertIsInstance(tethysjob.start_time, datetime)
@mock.patch('tethys_compute.models.tethys_job.log')
def test_update_bad_status(self, mock_log):
tethys_job = TethysJob.objects.get(name='test_tethysjob')
tethys_job.update_status(status='test')
mock_log.error.assert_called_with('Invalid status given: test')
@mock.patch('django.db.models.base.Model.save')
def test_update_valid_status(self, mock_save):
tethys_job = TethysJob.objects.get(name='test_tethysjob')
tethys_job.update_status(status='PEN')
mock_save.assert_called()
@mock.patch('tethys_compute.models.tethys_job.TethysJob.is_time_to_update')
@mock.patch('tethys_compute.models.condor.condor_base.CondorBase.condor_object')
def test_update_status_com(self, mock_co, mock_tup):
tethysjob = CondorBase(
name='test_tethysjob',
description='test_description',
user=self.user,
label='test_label',
scheduler=self.scheduler,
execute_time=django_timezone.now(),
)
mock_tup.return_value = True
mock_co.status = 'Completed'
tethysjob.update_status()
# Check result
self.assertIsNotNone(tethysjob.last_status_update)
self.assertIsInstance(tethysjob.last_status_update, datetime)
self.assertIsNotNone(tethysjob.completion_time)
self.assertIsInstance(tethysjob.completion_time, datetime)
@mock.patch('tethys_compute.models.tethys_job.TethysJob.is_time_to_update')
@mock.patch('tethys_compute.models.condor.condor_base.CondorBase.condor_object')
def test_update_status_vcp(self, mock_co, mock_tup):
tethysjob = CondorBase(
name='test_tethysjob',
description='test_description',
user=self.user,
label='test_label',
scheduler=self.scheduler,
execute_time=django_timezone.now(),
)
mock_tup.return_value = True
mock_co.status = 'Various-Complete'
tethysjob.update_status()
# Check result
self.assertIsNotNone(tethysjob.last_status_update)
self.assertIsInstance(tethysjob.last_status_update, datetime)
self.assertIsNotNone(tethysjob.completion_time)
self.assertIsInstance(tethysjob.completion_time, datetime)
@mock.patch('tethys_compute.models.tethys_job.TethysJob.is_time_to_update')
@mock.patch('tethys_compute.models.condor.condor_base.CondorBase.condor_object')
def test_update_status_err(self, mock_co, mock_tup):
tethysjob = CondorBase(
name='test_tethysjob',
description='test_description',
user=self.user,
label='test_label',
scheduler=self.scheduler,
execute_time=django_timezone.now(),
)
mock_co.status = 'Held'
mock_tup.return_value = True
tethysjob.update_status()
# Check result
self.assertIsNotNone(tethysjob.last_status_update)
self.assertIsInstance(tethysjob.last_status_update, datetime)
self.assertIsNotNone(tethysjob.completion_time)
self.assertIsInstance(tethysjob.completion_time, datetime)
@mock.patch('tethys_compute.models.tethys_job.TethysJob.is_time_to_update')
@mock.patch('tethys_compute.models.condor.condor_base.CondorBase.condor_object')
def test_update_status_abt(self, mock_co, mock_tup):
tethysjob = CondorBase(
name='test_tethysjob',
description='test_description',
user=self.user,
label='test_label',
scheduler=self.scheduler,
execute_time=django_timezone.now(),
)
mock_tup.return_value = True
mock_co.status = 'Removed'
tethysjob.update_status()
# Check result
self.assertIsNotNone(tethysjob.last_status_update)
self.assertIsInstance(tethysjob.last_status_update, datetime)
self.assertIsNotNone(tethysjob.completion_time)
self.assertIsInstance(tethysjob.completion_time, datetime)
@mock.patch('tethys_compute.models.tethys_job.TethysFunctionExtractor')
def test_process_results_function(self, mock_tfe):
mock_tfe().valid = True
mock_tfe().function = 'test_function_return'
# Setter
TethysJob.objects.get(name='test_tethysjob_execute_time').process_results_function = test_function
# Property
ret = TethysJob.objects.get(name='test_tethysjob_execute_time').process_results_function
# Check result
self.assertEqual(ret, 'test_function_return')
mock_tfe.assert_called_with(str(test_function), None)
@mock.patch('tethys_compute.models.tethys_job.TethysFunctionExtractor')
def test_process_results_function_string_input(self, mock_tfe):
mock_tfe().valid = True
mock_tfe().function = 'test_function_return'
# Setter
TethysJob.objects.get(name='test_tethysjob_execute_time').process_results_function = 'tests.unit_tests.' \
'test_tethys_compute.' \
'test_models.' \
'test_TethysJob.' \
'test_function'
# Property
ret = TethysJob.objects.get(name='test_tethysjob_execute_time').process_results_function
# Check result
self.assertEqual(ret, 'test_function_return')
mock_tfe.assert_called_with(str(test_function), None)
def test_process_results(self):
ret = TethysJob.objects.get(name='test_tethysjob')
ret.process_results('test', name='test_name')
# Check result
self.assertIsInstance(ret.completion_time, datetime)
self.assertIsNotNone(ret.completion_time)
def test_abs_method(self):
# Execute
ret = TethysJob.objects.get(name='test_tethysjob')._execute()
# Check result
self.assertIsNone(ret)
# Update Status
ret = TethysJob.objects.get(name='test_tethysjob')._update_status()
# Check result
self.assertIsNone(ret)
# Execute
ret = TethysJob.objects.get(name='test_tethysjob')._process_results()
# Check result
self.assertIsNone(ret)
self.assertRaises(NotImplementedError, TethysJob.objects.get(name='test_tethysjob').stop)
self.assertRaises(NotImplementedError, TethysJob.objects.get(name='test_tethysjob').pause)
self.assertRaises(NotImplementedError, TethysJob.objects.get(name='test_tethysjob').resume)
def test_is_time_to_update(self):
ret = TethysJob.objects.get(name='test_tethysjob')
ret._update_status_interval = timedelta(seconds=0)
fifteen_ago = django_timezone.now() - timedelta(minutes=15)
ret._last_status_update = fifteen_ago
time_to_update_status = ret.is_time_to_update()
self.assertTrue(time_to_update_status)
def test_is_time_to_update_false(self):
ret = TethysJob.objects.get(name='test_tethysjob')
ret._update_status_interval = timedelta(minutes=15)
fifteen_ago = django_timezone.now() - timedelta(minutes=5)
ret._last_status_update = fifteen_ago
time_to_update_status = ret.is_time_to_update()
self.assertFalse(time_to_update_status)
class CondorJobTest(TethysTestCase):
def set_up(self):
self.user = User.objects.create_user('tethys_super', '*****@*****.**', 'pass')
self.scheduler = CondorScheduler(
name='test_scheduler',
host='localhost',
username='******',
password='******',
)
self.scheduler.save()
path = os.path.dirname(__file__)
self.workspace_dir = os.path.join(path, 'workspace')
self.condorjob = CondorJob(
name='test condorbase',
description='test_description',
user=self.user,
label='test_label',
cluster_id='1',
remote_id='test_machine',
workspace=self.workspace_dir,
scheduler=self.scheduler,
condorpyjob_id='99',
_attributes={'foo': 'bar'},
_remote_input_files=['test_file1.txt', 'test_file2.txt'],
)
self.condorjob.save()
self.id_val = TethysJob.objects.get(name='test condorbase').id
def tear_down(self):
self.scheduler.delete()
if self.condorjob.condorpyjob_ptr_id == 99:
self.condorjob.delete()
if os.path.exists(self.workspace_dir):
shutil.rmtree(self.workspace_dir)
def test_type(self):
ret = self.condorjob.type
self.assertEqual('CondorJob', ret)
def test_condor_object_prop(self):
condorpy_job = self.condorjob._condor_object
# Check result
self.assertEqual('test_condorbase', condorpy_job.name)
self.assertEqual('test_condorbase', condorpy_job.attributes['job_name'])
self.assertEqual('bar', condorpy_job.attributes['foo'])
self.assertIn('test_file1.txt', condorpy_job.remote_input_files)
self.assertIn('test_file2.txt', condorpy_job.remote_input_files)
@mock.patch('tethys_compute.models.condor.condor_job.CondorBase.condor_object')
def test_execute(self, mock_cos):
# TODO: Check if we can mock this or we can provide an executable.
# Mock condor_object.submit()
mock_cos.submit.return_value = 111
self.condorjob._execute(queue=2)
# Check result
self.assertEqual(111, self.condorjob.cluster_id)
self.assertEqual(2, self.condorjob.num_jobs)
@mock.patch('tethys_compute.models.condor.condor_job.CondorPyJob.update_database_fields')
@mock.patch('tethys_compute.models.condor.condor_job.CondorBase.update_database_fields')
def test_update_database_fields(self, mock_cb_update, mock_cj_update):
# Mock condor_object.submit()
self.condorjob.update_database_fields()
# Check result
mock_cb_update.assert_called()
mock_cj_update.assert_called()
def test_condor_job_pre_save(self):
# Check if CondorBase is updated
self.assertIsInstance(CondorBase.objects.get(tethysjob_ptr_id=self.id_val), CondorBase)
# Check if CondorPyJob is updated
self.assertIsInstance(CondorPyJob.objects.get(condorpyjob_id=99), CondorPyJob)
@mock.patch('tethys_compute.models.condor.condor_job.CondorBase.condor_object')
def test_condor_job_pre_delete(self, mock_co):
if not os.path.exists(self.workspace_dir):
os.makedirs(self.workspace_dir)
file_path = os.path.join(self.workspace_dir, 'test_file.txt')
open(file_path, 'a').close()
self.condorjob.delete()
# Check if close_remote is called
mock_co.close_remote.assert_called()
# Check if file has been removed
self.assertFalse(os.path.isfile(file_path))
@mock.patch('tethys_compute.models.condor.condor_job.log')
@mock.patch('tethys_compute.models.condor.condor_job.CondorBase.condor_object')
def test_condor_job_pre_delete_exception(self, mock_co, mock_log):
mock_co.close_remote.side_effect = Exception('test error')
self.condorjob.delete()
# Check if close_remote is called
mock_log.exception.assert_called_with('test error')
class CondorWorkflowTest(TethysTestCase):
def set_up(self):
test_models_dir = os.path.dirname(__file__)
self.workspace_dir = os.path.join(test_models_dir, 'workspace')
self.user = User.objects.create_user('tethys_super',
'*****@*****.**', 'pass')
files_dir = os.path.join(os.path.dirname(test_models_dir), 'files')
self.private_key = os.path.join(files_dir, 'keys', 'testkey')
self.private_key_pass = '******'
self.scheduler = CondorScheduler(
name='test_scheduler',
host='localhost',
username='******',
password='******',
private_key_path=self.private_key,
private_key_pass=self.private_key_pass)
self.scheduler.save()
self.condorworkflow = CondorWorkflow(
_max_jobs={'foo': 10},
_config='test_config',
name='test name',
workspace=self.workspace_dir,
user=self.user,
scheduler=self.scheduler,
)
self.condorworkflow.save()
self.id_value = CondorWorkflow.objects.get(
name='test name').condorpyworkflow_ptr_id
self.condorpyworkflow = CondorPyWorkflow.objects.get(
condorpyworkflow_id=self.id_value)
self.condorworkflowjobnode_child = CondorWorkflowJobNode(
name='Node_child',
workflow=self.condorpyworkflow,
_attributes={'test': 'one'},
_num_jobs=1,
_remote_input_files=['test1.txt'],
)
self.condorworkflowjobnode_child.save()
self.condorworkflowjobnode = CondorWorkflowJobNode(
name='Node_1',
workflow=self.condorpyworkflow,
_attributes={'test': 'one'},
_num_jobs=1,
_remote_input_files=['test1.txt'],
)
self.condorworkflowjobnode.save()
# Django model many to many relationship add method
self.condorworkflowjobnode.parent_nodes.add(
self.condorworkflowjobnode_child)
self.condorbase_id = CondorWorkflow.objects.get(
name='test name').condorbase_ptr_id
self.condorpyworkflow_id = CondorWorkflow.objects.get(
name='test name').condorpyworkflow_ptr_id
def tear_down(self):
self.scheduler.delete()
if self.condorworkflow.condorbase_ptr_id == self.condorbase_id:
self.condorworkflow.delete()
if os.path.exists(self.workspace_dir):
shutil.rmtree(self.workspace_dir)
def test_type(self):
ret = self.condorworkflow.type
self.assertEqual('CondorWorkflow', ret)
def test_condor_object_prop(self):
ret = self.condorworkflow._condor_object
# Check workflow return
self.assertEqual({'foo': 10}, ret.max_jobs)
self.assertEqual('test_config', ret.config)
self.assertEqual('<DAG: test_name>', repr(ret))
@mock.patch(
'tethys_compute.models.condor.condor_workflow.CondorPyWorkflow.load_nodes'
)
@mock.patch(
'tethys_compute.models.condor.condor_workflow.CondorBase.condor_object'
)
def test_execute(self, mock_co, mock_ln):
# Mock submit to return a 111 cluster id
mock_co.submit.return_value = 111
# Execute
self.condorworkflow._execute(options=['foo'])
# We already tested load_nodes in CondorPyWorkflow, just mocked to make sure it's called here.
mock_ln.assert_called()
mock_co.submit.assert_called_with(options=['foo'])
# Check cluster_id from _execute in condorbase
self.assertEqual(111, self.condorworkflow.cluster_id)
def test_get_job(self):
ret = self.condorworkflow.get_job(job_name='Node_1')
# Check result
self.assertIsInstance(ret, CondorWorkflowJobNode)
self.assertEqual('Node_1', ret.name)
def test_get_job_does_not_exist(self):
ret = self.condorworkflow.get_job(job_name='Node_2')
# Check result
self.assertIsNone(ret)
@mock.patch(
'tethys_compute.models.condor.condor_workflow.CondorBase.update_database_fields'
)
@mock.patch(
'tethys_compute.models.condor.condor_workflow.CondorPyWorkflow.update_database_fields'
)
def test_update_database_fieds(self, mock_pw_update, mock_ba_update):
# Execute
self.condorworkflow.update_database_fields()
# Check if mock is called
mock_pw_update.assert_called()
mock_ba_update.assert_called()
@mock.patch(
'tethys_compute.models.condor.condor_workflow.CondorWorkflow.update_database_fields'
)
def test_condor_workflow_presave(self, mock_update):
# Excute
self.condorworkflow.save()
# Check if update_database_fields is called
mock_update.assert_called()
@mock.patch(
'tethys_compute.models.condor.condor_workflow.CondorWorkflow.condor_object'
)
def test_condor_job_pre_delete(self, mock_co):
if not os.path.exists(self.workspace_dir):
os.makedirs(self.workspace_dir)
file_path = os.path.join(self.workspace_dir, 'test_file.txt')
open(file_path, 'a').close()
self.condorworkflow.delete()
# Check if close_remote is called
mock_co.close_remote.assert_called()
# Check if file has been removed
self.assertFalse(os.path.isfile(file_path))
@mock.patch('tethys_compute.models.condor.condor_workflow.log')
@mock.patch(
'tethys_compute.models.condor.condor_workflow.CondorWorkflow.condor_object'
)
def test_condor_job_pre_delete_exception(self, mock_co, mock_log):
mock_co.close_remote.side_effect = Exception('test error')
self.condorworkflow.delete()
# Check if close_remote is called
mock_log.exception.assert_called_with('test error')
def test__update_status_no_execute_time(self):
self.condorworkflow.execute_time = None
ret = self.condorworkflow._update_status()
self.assertEqual('SUB', ret)
@mock.patch(
'tethys_compute.models.condor.condor_workflow.CondorBase.condor_object'
)
def test__update_status_not_Running(self, mock_co):
self.condorworkflow.execute_time = tz.now()
mock_co.status = 'Completed'
self.condorworkflow._update_status()
self.assertEqual('COM', self.condorworkflow._status)
@mock.patch(
'tethys_compute.models.condor.condor_workflow.CondorBase.condor_object'
)
def test__update_status_Running_not_running_statuses(self, mock_co):
self.condorworkflow.execute_time = tz.now()
mock_co.status = 'Running'
mock_co.statuses = {
'Unexpanded': 0,
'Idle': 0,
'Running': 0,
'Completed': 1
}
self.condorworkflow._update_status()
self.assertEqual('VCP', self.condorworkflow._status)
@mock.patch(
'tethys_compute.models.condor.condor_workflow.CondorBase.condor_object'
)
def test__update_status_Running_no_statuses(self, mock_co):
self.condorworkflow.execute_time = tz.now()
mock_co.status = 'Running'
mock_co.statuses = {
'Unexpanded': 0,
'Idle': 0,
'Running': 0,
'Completed': 0
}
self.condorworkflow._update_status()
self.assertEqual('SUB', self.condorworkflow._status)
@mock.patch(
'tethys_compute.models.condor.condor_workflow.CondorBase.condor_object'
)
def test__update_status_exception(self, mock_co):
self.condorworkflow.execute_time = tz.now()
type(mock_co).status = mock.PropertyMock(side_effect=Exception)
self.condorworkflow._update_status()
self.assertEqual('ERR', self.condorworkflow._status)
class CondorPyWorkflowTest(TethysTestCase):
def set_up(self):
path = os.path.dirname(__file__)
self.workspace_dir = os.path.join(path, 'workspace')
self.user = User.objects.create_user('tethys_super',
'*****@*****.**', 'pass')
self.scheduler = CondorScheduler(name='test_scheduler',
host='localhost',
username='******',
password='******',
private_key_path='test_path',
private_key_pass='******')
self.scheduler.save()
self.condorworkflow = CondorWorkflow(
_max_jobs={'foo': 10},
_config='test_config',
name='test name',
workspace=self.workspace_dir,
user=self.user,
scheduler=self.scheduler,
)
self.condorworkflow.save()
self.id_value = CondorWorkflow.objects.get(
name='test name').condorpyworkflow_ptr_id
self.condorpyworkflow = CondorPyWorkflow.objects.get(
condorpyworkflow_id=self.id_value)
self.condorworkflowjobnode_a = CondorWorkflowJobNode(
name='Job1_a',
workflow=self.condorpyworkflow,
_attributes={'foo': 'one'},
_num_jobs=1,
_remote_input_files=['test1.txt'],
)
self.condorworkflowjobnode_a.save()
self.condorworkflowjobnode_a1 = CondorWorkflowJobNode(
name='Job1_a1',
workflow=self.condorpyworkflow,
_attributes={'foo': 'one'},
_num_jobs=1,
_remote_input_files=['test1.txt'],
)
self.condorworkflowjobnode_a1.save()
# Django model many to many relationship add method
# self.condorworkflowjobnode.parent_nodes.add(self.condorworkflowjobnode_job)
def tear_down(self):
self.scheduler.delete()
self.condorworkflow.delete()
self.condorworkflowjobnode_a.delete()
self.condorworkflowjobnode_a1.delete()
# pass
def test_condorpy_workflow_prop(self):
ret = self.condorworkflow.condorpy_workflow
# Check Result
self.assertEqual('<DAG: test_name>', repr(ret))
self.assertEqual(self.workspace_dir, ret._cwd)
self.assertEqual('test_config', ret.config)
@mock.patch('tethys_compute.models.condor.condor_py_workflow.Workflow')
def test_max_jobs(self, mock_wf):
max_jobs = {'foo': 5}
self.condorpyworkflow.name = 'test_name'
self.condorpyworkflow.workspace = 'test_dict'
self.condorpyworkflow.max_jobs = max_jobs
ret = self.condorpyworkflow.max_jobs
# Check result
self.assertEqual(5, ret['foo'])
mock_wf.assert_called_with(config='test_config',
max_jobs={'foo': 10},
name='test_name',
working_directory='test_dict')
@mock.patch(
'tethys_compute.models.condor.condor_py_workflow.CondorPyWorkflow.condorpy_workflow'
)
def test_config(self, mock_cw):
test_config_value = 'test_config2'
# Mock condorpy_workflow.config = test_config_value. We have already tested condorpy_workflow.
mock_cw.config = test_config_value
# Setter
self.condorpyworkflow.config = test_config_value
# Property
ret = self.condorpyworkflow.config
# Check result
self.assertEqual('test_config2', ret)
def test_nodes(self):
ret = self.condorworkflow.nodes
# Check result after loading nodes
# self.assertEqual('Node_1', ret[0].name)
# Check result in CondorPyWorkflow object
self.assertEqual({'foo': 10}, ret[0].workflow.max_jobs)
self.assertEqual('test_config', ret[0].workflow.config)
def test_load_nodes(self):
# Before load nodes. Set should be empty
ret_before = self.condorworkflow.condorpy_workflow.node_set
list_before = []
list_after = []
for e in ret_before:
list_before.append(e)
# Check list_before is empty
self.assertFalse(list_before)
# Add parent
self.condorworkflowjobnode_a1.add_parent(self.condorworkflowjobnode_a)
# Execute load nodes
self.condorworkflow.load_nodes()
# After load nodes, Set should have two elements. One parent and one child
ret_after = self.condorworkflow.condorpy_workflow.node_set
# Convert to list for checking result
for e in ret_after:
list_after.append(e)
# Check list_after is not empty
self.assertTrue(list_after)
# sort list and compare result
list_after.sort(key=lambda node: node.job.name)
self.assertEqual('Job1_a', list_after[0].job.name)
self.assertEqual('Job1_a1', list_after[1].job.name)
def test_add_max_jobs_throttle(self):
# Set max_jobs
self.condorworkflow.add_max_jobs_throttle('foo1', 20)
# Get return value
ret = self.condorworkflow.condorpy_workflow
# Check result
self.assertEqual(20, ret.max_jobs['foo1'])
@mock.patch(
'tethys_compute.models.condor.condor_workflow_job_node.CondorWorkflowJobNode.update_database_fields'
)
def test_update_database_fields(self, mock_update):
# Set attribute for node
self.condorpyworkflow.update_database_fields()
# Check if mock is called twice for node and child node
self.assertTrue(mock_update.call_count == 2)
@mock.patch(
'tethys_compute.models.condor.condor_py_workflow.CondorPyWorkflow.condorpy_workflow'
)
def test_num_jobs(self, mock_condorpy_workflow_prop):
ret = self.condorpyworkflow.num_jobs
self.assertEqual(mock_condorpy_workflow_prop.num_jobs, ret)
class TethysJobTest(TethysTestCase):
def set_up(self):
self.tz = pytz_timezone('America/Denver')
self.user = User.objects.create_user('tethys_super',
'*****@*****.**', 'pass')
self.scheduler = CondorScheduler(
name='test_scheduler',
host='localhost',
)
self.scheduler.save()
self.tethysjob = TethysJob(
name='test_tethysjob',
description='test_description',
user=self.user,
label='test_label',
)
self.tethysjob.save()
self.tethysjob_execute_time = TethysJob(
name='test_tethysjob_execute_time',
description='test_description',
user=self.user,
label='test_label',
execute_time=datetime(year=2018, month=1, day=1, tzinfo=self.tz),
completion_time=datetime(year=2018,
month=1,
day=1,
hour=1,
tzinfo=self.tz),
_status='VAR',
_process_results_function=test_function)
self.tethysjob_execute_time.save()
def tear_down(self):
self.tethysjob.delete()
self.tethysjob_execute_time.delete()
self.scheduler.delete()
def test_type(self):
ret = self.tethysjob.type
self.assertEqual('TethysJob', ret)
def test_update_status_interval_prop(self):
ret = TethysJob.objects.get(
name='test_tethysjob').update_status_interval
# Check result
self.assertIsInstance(ret, timedelta)
self.assertEqual(timedelta(0, 10), ret)
def test_last_status_update_prop(self):
ret = TethysJob.objects.get(name='test_tethysjob')
check_date = datetime(year=2018, month=1, day=1, tzinfo=self.tz)
ret._last_status_update = check_date
# Check result
self.assertEqual(check_date, ret.last_status_update)
def test_status_prop(self):
ret = TethysJob.objects.get(name='test_tethysjob').status
# Check result
self.assertEqual('Pending', ret)
def test_status_setter_prop(self):
ret = TethysJob.objects.get(name='test_tethysjob_execute_time').status
# Check result
self.assertEqual('Various', ret)
@mock.patch('tethys_compute.models.tethys_job.TethysJob.update_status')
def test_status_setter(self, mock_update):
tethys_job = TethysJob.objects.get(name='test_tethysjob')
tethys_job.status = 'test_status'
# Check result
mock_update.assert_called_with(status='test_status')
def test_run_time_execute_time_prop(self):
ret = TethysJob.objects.get(
name='test_tethysjob_execute_time').run_time
# Check result
self.assertIsInstance(ret, timedelta)
self.assertEqual(timedelta(0, 3600), ret)
def test_run_time_execute_time_prop_with_start_time(self):
ret = TethysJob.objects.get(name='test_tethysjob').run_time
# Check result
self.assertEqual('', ret)
def test_run_time_execute_time_no_start_time_prop(self):
ret = TethysJob.objects.get(name='test_tethysjob').run_time
# Check result
self.assertEqual("", ret)
def test_execute(self):
ret_old = TethysJob.objects.get(name='test_tethysjob_execute_time')
TethysJob.objects.get(name='test_tethysjob_execute_time').execute()
ret_new = TethysJob.objects.get(name='test_tethysjob_execute_time')
self.assertNotEqual(ret_old.execute_time, ret_new.execute_time)
self.assertEqual('Various', ret_old.status)
self.assertEqual('Submitted', ret_new.status)
@mock.patch('tethys_compute.models.tethys_job.log')
def test_update_status_invalid(self, mock_log):
tethysjob = TethysJob(
name='test_tethysjob',
description='test_description',
user=self.user,
label='test_label',
execute_time=django_timezone.now(),
)
tethysjob._status = ''
# Check result
self.assertIsNone(tethysjob.update_status('Test'))
mock_log.error.assert_called_with('Invalid status given: Test')
@mock.patch('tethys_compute.models.tethys_job.TethysJob.is_time_to_update')
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase.condor_object')
def test_update_status_run(self, mock_co, mock_tup):
tethysjob = CondorBase(
name='test_tethysjob',
description='test_description',
user=self.user,
label='test_label',
scheduler=self.scheduler,
execute_time=django_timezone.now(),
)
mock_tup.return_value = True
mock_co.status = 'Running'
tethysjob.update_status()
# Check result
self.assertIsNotNone(tethysjob.last_status_update)
self.assertIsInstance(tethysjob.last_status_update, datetime)
self.assertIsNotNone(tethysjob.start_time)
self.assertIsInstance(tethysjob.start_time, datetime)
@mock.patch('tethys_compute.models.tethys_job.log')
def test_update_bad_status(self, mock_log):
tethys_job = TethysJob.objects.get(name='test_tethysjob')
tethys_job.update_status(status='test')
mock_log.error.assert_called_with('Invalid status given: test')
@mock.patch('django.db.models.base.Model.save')
def test_update_valid_status(self, mock_save):
tethys_job = TethysJob.objects.get(name='test_tethysjob')
tethys_job.update_status(status='PEN')
mock_save.assert_called()
@mock.patch('tethys_compute.models.tethys_job.TethysJob.is_time_to_update')
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase.condor_object')
def test_update_status_com(self, mock_co, mock_tup):
tethysjob = CondorBase(
name='test_tethysjob',
description='test_description',
user=self.user,
label='test_label',
scheduler=self.scheduler,
execute_time=django_timezone.now(),
)
mock_tup.return_value = True
mock_co.status = 'Completed'
tethysjob.update_status()
# Check result
self.assertIsNotNone(tethysjob.last_status_update)
self.assertIsInstance(tethysjob.last_status_update, datetime)
self.assertIsNotNone(tethysjob.completion_time)
self.assertIsInstance(tethysjob.completion_time, datetime)
@mock.patch('tethys_compute.models.tethys_job.TethysJob.is_time_to_update')
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase.condor_object')
def test_update_status_vcp(self, mock_co, mock_tup):
tethysjob = CondorBase(
name='test_tethysjob',
description='test_description',
user=self.user,
label='test_label',
scheduler=self.scheduler,
execute_time=django_timezone.now(),
)
mock_tup.return_value = True
mock_co.status = 'Various-Complete'
tethysjob.update_status()
# Check result
self.assertIsNotNone(tethysjob.last_status_update)
self.assertIsInstance(tethysjob.last_status_update, datetime)
self.assertIsNotNone(tethysjob.completion_time)
self.assertIsInstance(tethysjob.completion_time, datetime)
@mock.patch('tethys_compute.models.tethys_job.TethysJob.is_time_to_update')
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase.condor_object')
def test_update_status_err(self, mock_co, mock_tup):
tethysjob = CondorBase(
name='test_tethysjob',
description='test_description',
user=self.user,
label='test_label',
scheduler=self.scheduler,
execute_time=django_timezone.now(),
)
mock_co.status = 'Held'
mock_tup.return_value = True
tethysjob.update_status()
# Check result
self.assertIsNotNone(tethysjob.last_status_update)
self.assertIsInstance(tethysjob.last_status_update, datetime)
self.assertIsNotNone(tethysjob.completion_time)
self.assertIsInstance(tethysjob.completion_time, datetime)
@mock.patch('tethys_compute.models.tethys_job.TethysJob.is_time_to_update')
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase.condor_object')
def test_update_status_abt(self, mock_co, mock_tup):
tethysjob = CondorBase(
name='test_tethysjob',
description='test_description',
user=self.user,
label='test_label',
scheduler=self.scheduler,
execute_time=django_timezone.now(),
)
mock_tup.return_value = True
mock_co.status = 'Removed'
tethysjob.update_status()
# Check result
self.assertIsNotNone(tethysjob.last_status_update)
self.assertIsInstance(tethysjob.last_status_update, datetime)
self.assertIsNotNone(tethysjob.completion_time)
self.assertIsInstance(tethysjob.completion_time, datetime)
@mock.patch('tethys_compute.models.tethys_job.TethysFunctionExtractor')
def test_process_results_function(self, mock_tfe):
mock_tfe().valid = True
mock_tfe().function = 'test_function_return'
# Setter
TethysJob.objects.get(name='test_tethysjob_execute_time'
).process_results_function = test_function
# Property
ret = TethysJob.objects.get(
name='test_tethysjob_execute_time').process_results_function
# Check result
self.assertEqual(ret, 'test_function_return')
mock_tfe.assert_called_with(str(test_function), None)
@mock.patch('tethys_compute.models.tethys_job.TethysFunctionExtractor')
def test_process_results_function_string_input(self, mock_tfe):
mock_tfe().valid = True
mock_tfe().function = 'test_function_return'
# Setter
TethysJob.objects.get(name='test_tethysjob_execute_time').process_results_function = 'tests.unit_tests.' \
'test_tethys_compute.' \
'test_models.' \
'test_TethysJob.' \
'test_function'
# Property
ret = TethysJob.objects.get(
name='test_tethysjob_execute_time').process_results_function
# Check result
self.assertEqual(ret, 'test_function_return')
mock_tfe.assert_called_with(str(test_function), None)
def test_process_results(self):
ret = TethysJob.objects.get(name='test_tethysjob')
ret.process_results('test', name='test_name')
# Check result
self.assertIsInstance(ret.completion_time, datetime)
self.assertIsNotNone(ret.completion_time)
@mock.patch('tethys_compute.models.tethys_job.TethysJob._resubmit')
def test_resubmit(self, mock__resubmit):
ret = TethysJob.objects.get(name='test_tethysjob')
ret.resubmit()
# Check result
mock__resubmit.assert_called()
@mock.patch('tethys_compute.models.tethys_job.TethysJob._get_logs')
def test_get_logs(self, mock__get_logs):
ret = TethysJob.objects.get(name='test_tethysjob')
ret.get_logs()
# Check result
mock__get_logs.assert_called()
def test_abs_method(self):
# Resubmit
ret = TethysJob.objects.get(name='test_tethysjob')._resubmit()
# Check result
self.assertIsNone(ret)
# Execute
ret = TethysJob.objects.get(name='test_tethysjob')._execute()
# Check result
self.assertIsNone(ret)
# Update Status
ret = TethysJob.objects.get(name='test_tethysjob')._update_status()
# Check result
self.assertIsNone(ret)
# Execute
ret = TethysJob.objects.get(name='test_tethysjob')._process_results()
# Check result
self.assertIsNone(ret)
# Check get logs
ret = TethysJob.objects.get(name='test_tethysjob')._get_logs()
# Check result
self.assertIsNone(ret)
self.assertRaises(NotImplementedError,
TethysJob.objects.get(name='test_tethysjob').stop)
self.assertRaises(NotImplementedError,
TethysJob.objects.get(name='test_tethysjob').pause)
self.assertRaises(NotImplementedError,
TethysJob.objects.get(name='test_tethysjob').resume)
def test_is_time_to_update(self):
ret = TethysJob.objects.get(name='test_tethysjob')
ret._update_status_interval = timedelta(seconds=0)
fifteen_ago = django_timezone.now() - timedelta(minutes=15)
ret._last_status_update = fifteen_ago
time_to_update_status = ret.is_time_to_update()
self.assertTrue(time_to_update_status)
def test_is_time_to_update_false(self):
ret = TethysJob.objects.get(name='test_tethysjob')
ret._update_status_interval = timedelta(minutes=15)
fifteen_ago = django_timezone.now() - timedelta(minutes=5)
ret._last_status_update = fifteen_ago
time_to_update_status = ret.is_time_to_update()
self.assertFalse(time_to_update_status)
def test_lt(self):
ret = sorted((self.tethysjob, self.tethysjob_execute_time))
expected_value = [self.tethysjob, self.tethysjob_execute_time]
self.assertListEqual(ret, expected_value)
class CondorBaseTest(TethysTestCase):
def set_up(self):
self.user = User.objects.create_user('tethys_super',
'*****@*****.**', 'pass')
self.scheduler = CondorScheduler(name='test_scheduler',
host='localhost',
username='******',
password='******',
private_key_path='test_path',
private_key_pass='******')
self.scheduler.save()
self.condorbase = CondorBase(name='test_condorbase',
description='test_description',
user=self.user,
label='test_label',
cluster_id='1',
remote_id='test_machine',
scheduler=self.scheduler)
self.condorbase.save()
self.condorbase_exe = CondorBase(name='test_condorbase_exe',
description='test_description',
user=self.user,
label='test_label',
execute_time=timezone.now(),
cluster_id='1',
remote_id='test_machine',
scheduler=self.scheduler)
self.condorbase_exe.save()
def tear_down(self):
self.scheduler.delete()
self.condorbase.delete()
self.condorbase_exe.delete()
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase._condor_object')
def test_condor_object_pro(self, mock_co):
ret = CondorBase.objects.get(name='test_condorbase')
mock_co.return_value = ret
ret.condor_object
# Check result
self.assertEqual(mock_co, ret.condor_object)
self.assertEqual(1, ret.condor_object._cluster_id)
self.assertEqual('test_machine', ret.condor_object._remote_id)
mock_co.set_scheduler.assert_called_with('localhost', 'tethys_super',
'pass', 'test_path',
'test_pass')
def test_condor_obj_abs(self):
ret = CondorBase.objects.get(name='test_condorbase')._condor_object()
# Check result.
self.assertIsNone(ret)
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase.condor_object')
def test_statuses_prop(self, mock_co):
mock_co.statuses = 'test_statuses'
condor_obj = CondorBase.objects.get(name='test_condorbase')
# to set updated inside if statement = False
d = timezone.now() - timedelta(days=1)
condor_obj._last_status_update = d
# Execute
ret = condor_obj.statuses
# Check result
self.assertEqual('test_statuses', ret)
# to set updated inside if statement = True
d = timezone.now()
condor_obj._last_status_update = d
mock_co.statuses = 'test_statuses2'
ret = condor_obj.statuses
# Check result, should not set statuses from condor_object again. Same ret as previous.
self.assertEqual('test_statuses', ret)
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase.condor_object')
def test_execute_abs(self, mock_co):
mock_co.submit.return_value = 111
# Execute
CondorBase.objects.get(name='test_condorbase')._execute()
ret = CondorBase.objects.get(name='test_condorbase')
# Check result
self.assertEqual(111, ret.cluster_id)
def test_update_status_not_execute_time(self):
ret = CondorBase.objects.get(name='test_condorbase')._update_status()
# Check result
self.assertEqual('PEN', ret)
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase.condor_object')
def test_update_status(self, mock_co):
mock_co.status = 'Various'
mock_co.statuses = {'Unexpanded': '', 'Idle': '', 'Running': ''}
CondorBase.objects.get(name='test_condorbase_exe')._update_status()
ret = CondorBase.objects.get(name='test_condorbase_exe')._status
# Check result
self.assertEqual('VCP', ret)
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase.condor_object')
def test_update_status_exception(self, mock_co):
mock_co.status = 'Various'
mock_co.statuses = {}
CondorBase.objects.get(name='test_condorbase_exe')._update_status()
ret = CondorBase.objects.get(name='test_condorbase_exe')._status
# Check result
self.assertEqual('ERR', ret)
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase.condor_object')
def test_process_results(self, mock_co):
CondorBase.objects.get(name='test_condorbase_exe')._process_results()
# Check result
mock_co.sync_remote_output.assert_called()
mock_co.close_remote.assert_called()
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase.condor_object')
def test_stop(self, mock_co):
CondorBase.objects.get(name='test_condorbase_exe').stop()
# Check result
mock_co.remove.assert_called()
@mock.patch('tethys_compute.models.condor.condor_base.CondorBase.save')
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase.condor_object')
def test_resubmit(self, mock_co, _):
CondorBase.objects.get(name='test_condorbase')._resubmit()
# Check result
mock_co.close_remote.assert_called()
def test_get_logs_from_workspace_no_file(self):
logs_file = {
'workflow': 'Model_Run.dag.dagman.out',
'test_job': {
'log': 'test_job/logs/*.log',
'error': 'test_job/logs/*.err',
'output': 'test_job/logs/*.out'
}
}
logs = CondorBase.objects.get(
name='test_condorbase')._get_logs_from_workspace(logs_file)
expected_results = {
'workflow': 'test_machine/Model_Run.dag.dagman.out does not exist',
'test_job': {
'log': 'File does not exist',
'error': 'File does not exist',
'output': 'File does not exist'
}
}
self.assertEqual(expected_results, logs)
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase._get_logs_from_workspace'
)
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase._get_logs_from_remote'
)
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase._log_files')
def test_get_logs_no_remote(self, mock_log_files, mock_log_remote,
mock_log_workspace):
logs_file = {
'workflow': 'Model_Run.dag.dagman.out',
'test_job': {
'log': 'test_job/logs/*.log',
'error': 'test_job/logs/*.err',
'output': 'test_job/logs/*.out'
}
}
logs_file_contents = {
'workspace': '',
'test_job': {
'log': '',
'error': '',
'output': ''
}
}
mock_log_files.return_value = logs_file
mock_log_remote.return_value = logs_file_contents
expected_return = 'log_from_workspace'
mock_log_workspace.return_value = expected_return
logs = CondorBase.objects.get(name='test_condorbase')._get_logs()
self.assertEqual(expected_return, logs)
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase._get_logs_from_workspace'
)
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase._get_logs_from_remote'
)
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase._log_files')
def test_get_logs_remote(self, mock_log_files, mock_log_remote,
mock_log_workspace):
logs_file = {
'workflow': 'Model_Run.dag.dagman.out',
'test_job': {
'log': 'test_job/logs/*.log',
'error': 'test_job/logs/*.err',
'output': 'test_job/logs/*.out'
}
}
logs_file_contents = {
'workspace': 'has_content',
'test_job': {
'log': '',
'error': '',
'output': ''
}
}
mock_log_files.return_value = logs_file
mock_log_remote.return_value = logs_file_contents
mock_log_workspace.return_value = 'log_from_workspace'
logs = CondorBase.objects.get(name='test_condorbase')._get_logs()
self.assertEqual(logs_file_contents, logs)
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase._get_logs_from_workspace'
)
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase._get_logs_from_remote'
)
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase._log_files')
def test_get_logs_remote_check_child(self, mock_log_files, mock_log_remote,
mock_log_workspace):
logs_file = {
'workflow': 'Model_Run.dag.dagman.out',
'test_job': {
'log': 'test_job/logs/*.log',
'error': 'test_job/logs/*.err',
'output': 'test_job/logs/*.out'
}
}
logs_file_contents = {
'workspace': '',
'test_job': {
'log': 'has_content',
'error': '',
'output': ''
}
}
mock_log_files.return_value = logs_file
mock_log_remote.return_value = logs_file_contents
mock_log_workspace.return_value = 'log_from_workspace'
logs = CondorBase.objects.get(name='test_condorbase')._get_logs()
self.assertEqual(logs_file_contents, logs)
@mock.patch('builtins.open')
@mock.patch('tethys_compute.models.condor.condor_base.os.listdir')
@mock.patch('tethys_compute.models.condor.condor_base.os.path.isdir')
@mock.patch('tethys_compute.models.condor.condor_base.os.path.isfile')
def test_get_logs_from_workspace_mock_file(self, mock_isfile, mock_isdir,
mock_list_dir, mock_open):
mock_isfile.return_value = True
mock_isdir.return_value = True
mock_list_dir.return_value = ['file1.log', 'file2.err', 'file3.out']
logs_file = {
'workflow': 'Model_Run.dag.dagman.out',
'test_job': {
'log': 'test_job/logs/*.log',
'error': 'test_job/logs/*.err',
'output': 'test_job/logs/*.out'
}
}
logs = CondorBase.objects.get(
name='test_condorbase')._get_logs_from_workspace(logs_file)
self.assertIsNotNone(logs['workflow'])
self.assertIsNotNone(logs['test_job']['log'])
self.assertIsNotNone(logs['test_job']['error'])
self.assertIsNotNone(logs['test_job']['output'])
def test_pause(self):
ret = CondorBase.objects.get(name='test_condorbase_exe').pause()
# Check result
self.assertIsNone(ret)
def test_resume(self):
ret = CondorBase.objects.get(name='test_condorbase_exe').resume()
# Check result
self.assertIsNone(ret)
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase._condor_object')
def test_update_database_fields(self, mock_co):
mock_co._remote_id = 'test_update_remote_id'
ret = CondorBase.objects.get(name='test_condorbase_exe')
ret.remote_id = None
# _condor_object is an abstract method returning a condorpyjob or condorpyworkflow.
# We'll test condor_object.remote_id in condorpyjob test
ret.update_database_fields()
# Check result
self.assertEqual('test_update_remote_id', ret.remote_id)
def test_abs_method(self):
# Resubmit
ret = CondorBase.objects.get(name='test_condorbase')._log_files()
# Check result
self.assertIsNone(ret)
# Resubmit
ret = CondorBase.objects.get(
name='test_condorbase')._get_logs_from_remote('test')
# Check result
self.assertIsNone(ret)
class CondorPyWorkflowTest(TethysTestCase):
def set_up(self):
path = os.path.dirname(__file__)
self.workspace_dir = os.path.join(path, 'workspace')
self.user = User.objects.create_user('tethys_super', '*****@*****.**', 'pass')
self.scheduler = CondorScheduler(
name='test_scheduler',
host='localhost',
username='******',
password='******',
private_key_path='test_path',
private_key_pass='******'
)
self.scheduler.save()
self.condorworkflow = CondorWorkflow(
_max_jobs={'foo': 10},
_config='test_config',
name='test name',
workspace=self.workspace_dir,
user=self.user,
scheduler=self.scheduler,
)
self.condorworkflow.save()
self.id_value = CondorWorkflow.objects.get(name='test name').condorpyworkflow_ptr_id
self.condorpyworkflow = CondorPyWorkflow.objects.get(condorpyworkflow_id=self.id_value)
self.condorworkflowjobnode_a = CondorWorkflowJobNode(
name='Job1_a',
workflow=self.condorpyworkflow,
_attributes={'foo': 'one'},
_num_jobs=1,
_remote_input_files=['test1.txt'],
)
self.condorworkflowjobnode_a.save()
self.condorworkflowjobnode_a1 = CondorWorkflowJobNode(
name='Job1_a1',
workflow=self.condorpyworkflow,
_attributes={'foo': 'one'},
_num_jobs=1,
_remote_input_files=['test1.txt'],
)
self.condorworkflowjobnode_a1.save()
# Django model many to many relationship add method
# self.condorworkflowjobnode.parent_nodes.add(self.condorworkflowjobnode_job)
def tear_down(self):
self.scheduler.delete()
self.condorworkflow.delete()
self.condorworkflowjobnode_a.delete()
self.condorworkflowjobnode_a1.delete()
# pass
def test_condorpy_workflow_prop(self):
ret = self.condorworkflow.condorpy_workflow
# Check Result
self.assertEqual('<DAG: test_name>', repr(ret))
self.assertEqual(self.workspace_dir, ret._cwd)
self.assertEqual('test_config', ret.config)
@mock.patch('tethys_compute.models.condor.condor_py_workflow.Workflow')
def test_max_jobs(self, mock_wf):
max_jobs = {'foo': 5}
self.condorpyworkflow.name = 'test_name'
self.condorpyworkflow.workspace = 'test_dict'
self.condorpyworkflow.max_jobs = max_jobs
ret = self.condorpyworkflow.max_jobs
# Check result
self.assertEqual(5, ret['foo'])
mock_wf.assert_called_with(config='test_config', max_jobs={'foo': 10},
name='test_name', working_directory='test_dict')
@mock.patch('tethys_compute.models.condor.condor_py_workflow.CondorPyWorkflow.condorpy_workflow')
def test_config(self, mock_cw):
test_config_value = 'test_config2'
# Mock condorpy_workflow.config = test_config_value. We have already tested condorpy_workflow.
mock_cw.config = test_config_value
# Setter
self.condorpyworkflow.config = test_config_value
# Property
ret = self.condorpyworkflow.config
# Check result
self.assertEqual('test_config2', ret)
def test_nodes(self):
ret = self.condorworkflow.nodes
# Check result after loading nodes
# self.assertEqual('Node_1', ret[0].name)
# Check result in CondorPyWorkflow object
self.assertEqual({'foo': 10}, ret[0].workflow.max_jobs)
self.assertEqual('test_config', ret[0].workflow.config)
def test_load_nodes(self):
# Before load nodes. Set should be empty
ret_before = self.condorworkflow.condorpy_workflow.node_set
list_before = []
list_after = []
for e in ret_before:
list_before.append(e)
# Check list_before is empty
self.assertFalse(list_before)
# Add parent
self.condorworkflowjobnode_a1.add_parent(self.condorworkflowjobnode_a)
# Execute load nodes
self.condorworkflow.load_nodes()
# After load nodes, Set should have two elements. One parent and one child
ret_after = self.condorworkflow.condorpy_workflow.node_set
# Convert to list for checking result
for e in ret_after:
list_after.append(e)
# Check list_after is not empty
self.assertTrue(list_after)
# sort list and compare result
list_after.sort(key=lambda node: node.job.name)
self.assertEqual('Job1_a', list_after[0].job.name)
self.assertEqual('Job1_a1', list_after[1].job.name)
def test_add_max_jobs_throttle(self):
# Set max_jobs
self.condorworkflow.add_max_jobs_throttle('foo1', 20)
# Get return value
ret = self.condorworkflow.condorpy_workflow
# Check result
self.assertEqual(20, ret.max_jobs['foo1'])
@mock.patch('tethys_compute.models.condor.condor_workflow_job_node.CondorWorkflowJobNode.update_database_fields')
def test_update_database_fields(self, mock_update):
# Set attribute for node
self.condorpyworkflow.update_database_fields()
# Check if mock is called twice for node and child node
self.assertTrue(mock_update.call_count == 2)
@mock.patch('tethys_compute.models.condor.condor_py_workflow.CondorPyWorkflow.condorpy_workflow')
def test_num_jobs(self, mock_condorpy_workflow_prop):
ret = self.condorpyworkflow.num_jobs
self.assertEqual(mock_condorpy_workflow_prop.num_jobs, ret)
class CondorWorkflowTest(TethysTestCase):
def set_up(self):
path = os.path.dirname(__file__)
self.workspace_dir = os.path.join(path, 'workspace')
self.user = User.objects.create_user('tethys_super', '*****@*****.**', 'pass')
self.scheduler = CondorScheduler(
name='test_scheduler',
host='localhost',
username='******',
password='******',
private_key_path='test_path',
private_key_pass='******'
)
self.scheduler.save()
self.condorworkflow = CondorWorkflow(
_max_jobs={'foo': 10},
_config='test_config',
name='test name',
workspace=self.workspace_dir,
user=self.user,
scheduler=self.scheduler,
)
self.condorworkflow.save()
self.id_value = CondorWorkflow.objects.get(name='test name').condorpyworkflow_ptr_id
self.condorpyworkflow = CondorPyWorkflow.objects.get(condorpyworkflow_id=self.id_value)
self.condorworkflowjobnode_child = CondorWorkflowJobNode(
name='Node_child',
workflow=self.condorpyworkflow,
_attributes={'test': 'one'},
_num_jobs=1,
_remote_input_files=['test1.txt'],
)
self.condorworkflowjobnode_child.save()
self.condorworkflowjobnode = CondorWorkflowJobNode(
name='Node_1',
workflow=self.condorpyworkflow,
_attributes={'test': 'one'},
_num_jobs=1,
_remote_input_files=['test1.txt'],
)
self.condorworkflowjobnode.save()
# Django model many to many relationship add method
self.condorworkflowjobnode.parent_nodes.add(self.condorworkflowjobnode_child)
self.condorbase_id = CondorWorkflow.objects.get(name='test name').condorbase_ptr_id
self.condorpyworkflow_id = CondorWorkflow.objects.get(name='test name').condorpyworkflow_ptr_id
def tear_down(self):
self.scheduler.delete()
if self.condorworkflow.condorbase_ptr_id == self.condorbase_id:
self.condorworkflow.delete()
if os.path.exists(self.workspace_dir):
shutil.rmtree(self.workspace_dir)
def test_type(self):
ret = self.condorworkflow.type
self.assertEqual('CondorWorkflow', ret)
def test_condor_object_prop(self):
ret = self.condorworkflow._condor_object
# Check workflow return
self.assertEqual({'foo': 10}, ret.max_jobs)
self.assertEqual('test_config', ret.config)
self.assertEqual('<DAG: test_name>', repr(ret))
@mock.patch('tethys_compute.models.condor.condor_workflow.CondorPyWorkflow.load_nodes')
@mock.patch('tethys_compute.models.condor.condor_workflow.CondorBase.condor_object')
def test_execute(self, mock_co, mock_ln):
# Mock submit to return a 111 cluster id
mock_co.submit.return_value = 111
# Execute
self.condorworkflow._execute(options=['foo'])
# We already tested load_nodes in CondorPyWorkflow, just mocked to make sure it's called here.
mock_ln.assert_called()
mock_co.submit.assert_called_with(options=['foo'])
# Check cluster_id from _execute in condorbase
self.assertEqual(111, self.condorworkflow.cluster_id)
def test_get_job(self):
ret = self.condorworkflow.get_job(job_name='Node_1')
# Check result
self.assertIsInstance(ret, CondorWorkflowJobNode)
self.assertEqual('Node_1', ret.name)
def test_get_job_does_not_exist(self):
ret = self.condorworkflow.get_job(job_name='Node_2')
# Check result
self.assertIsNone(ret)
@mock.patch('tethys_compute.models.condor.condor_workflow.CondorBase.update_database_fields')
@mock.patch('tethys_compute.models.condor.condor_workflow.CondorPyWorkflow.update_database_fields')
def test_update_database_fieds(self, mock_pw_update, mock_ba_update):
# Execute
self.condorworkflow.update_database_fields()
# Check if mock is called
mock_pw_update.assert_called()
mock_ba_update.assert_called()
@mock.patch('tethys_compute.models.condor.condor_workflow.CondorWorkflow.update_database_fields')
def test_condor_workflow_presave(self, mock_update):
# Excute
self.condorworkflow.save()
# Check if update_database_fields is called
mock_update.assert_called()
@mock.patch('tethys_compute.models.condor.condor_workflow.CondorWorkflow.condor_object')
def test_condor_job_pre_delete(self, mock_co):
if not os.path.exists(self.workspace_dir):
os.makedirs(self.workspace_dir)
file_path = os.path.join(self.workspace_dir, 'test_file.txt')
open(file_path, 'a').close()
self.condorworkflow.delete()
# Check if close_remote is called
mock_co.close_remote.assert_called()
# Check if file has been removed
self.assertFalse(os.path.isfile(file_path))
@mock.patch('tethys_compute.models.condor.condor_workflow.log')
@mock.patch('tethys_compute.models.condor.condor_workflow.CondorWorkflow.condor_object')
def test_condor_job_pre_delete_exception(self, mock_co, mock_log):
mock_co.close_remote.side_effect = Exception('test error')
self.condorworkflow.delete()
# Check if close_remote is called
mock_log.exception.assert_called_with('test error')
def test__update_status_no_execute_time(self):
self.condorworkflow.execute_time = None
ret = self.condorworkflow._update_status()
self.assertEqual('PEN', ret)
@mock.patch('tethys_compute.models.condor.condor_workflow.CondorBase.condor_object')
def test__update_status_not_Running(self, mock_co):
self.condorworkflow.execute_time = tz.now()
mock_co.status = 'Completed'
self.condorworkflow._update_status()
self.assertEqual('COM', self.condorworkflow._status)
@mock.patch('tethys_compute.models.condor.condor_workflow.CondorBase.condor_object')
def test__update_status_Running_not_running_statuses(self, mock_co):
self.condorworkflow.execute_time = tz.now()
mock_co.status = 'Running'
mock_co.statuses = {'Unexpanded': 0, 'Idle': 0, 'Running': 0, 'Completed': 1}
self.condorworkflow._update_status()
self.assertEqual('VCP', self.condorworkflow._status)
@mock.patch('tethys_compute.models.condor.condor_workflow.CondorBase.condor_object')
def test__update_status_Running_no_statuses(self, mock_co):
self.condorworkflow.execute_time = tz.now()
mock_co.status = 'Running'
mock_co.statuses = {'Unexpanded': 0, 'Idle': 0, 'Running': 0, 'Completed': 0}
self.condorworkflow._update_status()
self.assertEqual('SUB', self.condorworkflow._status)
@mock.patch('tethys_compute.models.condor.condor_workflow.CondorBase.condor_object')
def test__update_status_exception(self, mock_co):
self.condorworkflow.execute_time = tz.now()
type(mock_co).status = mock.PropertyMock(side_effect=Exception)
self.condorworkflow._update_status()
self.assertEqual('ERR', self.condorworkflow._status)
class CondorJobTest(TethysTestCase):
def set_up(self):
self.user = User.objects.create_user('tethys_super', '*****@*****.**', 'pass')
self.scheduler = CondorScheduler(
name='test_scheduler',
host='localhost',
username='******',
password='******',
)
self.scheduler.save()
path = os.path.dirname(__file__)
self.workspace_dir = os.path.join(path, 'workspace')
self.condorjob = CondorJob(
name='test condorbase',
description='test_description',
user=self.user,
label='test_label',
cluster_id='1',
remote_id='test_machine',
workspace=self.workspace_dir,
scheduler=self.scheduler,
condorpyjob_id='99',
_attributes={'foo': 'bar'},
_remote_input_files=['test_file1.txt', 'test_file2.txt'],
)
self.condorjob.save()
self.id_val = TethysJob.objects.get(name='test condorbase').id
def tear_down(self):
self.scheduler.delete()
if self.condorjob.condorpyjob_ptr_id == 99:
self.condorjob.delete()
if os.path.exists(self.workspace_dir):
shutil.rmtree(self.workspace_dir)
def test_type(self):
ret = self.condorjob.type
self.assertEqual('CondorJob', ret)
def test_condor_object_prop(self):
condorpy_job = self.condorjob._condor_object
# Check result
self.assertEqual('test_condorbase', condorpy_job.name)
self.assertEqual('test_condorbase', condorpy_job.attributes['job_name'])
self.assertEqual('bar', condorpy_job.attributes['foo'])
self.assertIn('test_file1.txt', condorpy_job.remote_input_files)
self.assertIn('test_file2.txt', condorpy_job.remote_input_files)
@mock.patch('tethys_compute.models.condor.condor_job.CondorBase.condor_object')
def test_execute(self, mock_cos):
# TODO: Check if we can mock this or we can provide an executable.
# Mock condor_object.submit()
mock_cos.submit.return_value = 111
self.condorjob._execute(queue=2)
# Check result
self.assertEqual(111, self.condorjob.cluster_id)
self.assertEqual(2, self.condorjob.num_jobs)
@mock.patch('tethys_compute.models.condor.condor_job.CondorPyJob.update_database_fields')
@mock.patch('tethys_compute.models.condor.condor_job.CondorBase.update_database_fields')
def test_update_database_fields(self, mock_cb_update, mock_cj_update):
# Mock condor_object.submit()
self.condorjob.update_database_fields()
# Check result
mock_cb_update.assert_called()
mock_cj_update.assert_called()
def test_condor_job_pre_save(self):
# Check if CondorBase is updated
self.assertIsInstance(CondorBase.objects.get(tethysjob_ptr_id=self.id_val), CondorBase)
# Check if CondorPyJob is updated
self.assertIsInstance(CondorPyJob.objects.get(condorpyjob_id=99), CondorPyJob)
@mock.patch('tethys_compute.models.condor.condor_job.CondorBase.condor_object')
def test_condor_job_pre_delete(self, mock_co):
if not os.path.exists(self.workspace_dir):
os.makedirs(self.workspace_dir)
file_path = os.path.join(self.workspace_dir, 'test_file.txt')
open(file_path, 'a').close()
self.condorjob.delete()
# Check if close_remote is called
mock_co.close_remote.assert_called()
# Check if file has been removed
self.assertFalse(os.path.isfile(file_path))
@mock.patch('tethys_compute.models.condor.condor_job.log')
@mock.patch('tethys_compute.models.condor.condor_job.CondorBase.condor_object')
def test_condor_job_pre_delete_exception(self, mock_co, mock_log):
mock_co.close_remote.side_effect = Exception('test error')
self.condorjob.delete()
# Check if close_remote is called
mock_log.exception.assert_called_with('test error')
class CondorBaseTest(TethysTestCase):
def set_up(self):
self.user = User.objects.create_user('tethys_super',
'*****@*****.**', 'pass')
self.scheduler = CondorScheduler(name='test_scheduler',
host='localhost',
username='******',
password='******',
private_key_path='test_path',
private_key_pass='******')
self.scheduler.save()
self.condorbase = CondorBase(name='test_condorbase',
description='test_description',
user=self.user,
label='test_label',
cluster_id='1',
remote_id='test_machine',
scheduler=self.scheduler)
self.condorbase.save()
self.condorbase_exe = CondorBase(name='test_condorbase_exe',
description='test_description',
user=self.user,
label='test_label',
execute_time=timezone.now(),
cluster_id='1',
remote_id='test_machine',
scheduler=self.scheduler)
self.condorbase_exe.save()
def tear_down(self):
self.scheduler.delete()
self.condorbase.delete()
self.condorbase_exe.delete()
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase._condor_object')
def test_condor_object_pro(self, mock_co):
ret = CondorBase.objects.get(name='test_condorbase')
mock_co.return_value = ret
ret.condor_object
# Check result
self.assertEqual(mock_co, ret.condor_object)
self.assertEqual(1, ret.condor_object._cluster_id)
self.assertEqual('test_machine', ret.condor_object._remote_id)
mock_co.set_scheduler.assert_called_with('localhost', 'tethys_super',
'pass', 'test_path',
'test_pass')
def test_condor_obj_abs(self):
ret = CondorBase.objects.get(name='test_condorbase')._condor_object()
# Check result.
self.assertIsNone(ret)
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase.condor_object')
def test_statuses_prop(self, mock_co):
mock_co.statuses = 'test_statuses'
condor_obj = CondorBase.objects.get(name='test_condorbase')
# to set updated inside if statement = False
d = timezone.now() - timedelta(days=1)
condor_obj._last_status_update = d
# Execute
ret = condor_obj.statuses
# Check result
self.assertEqual('test_statuses', ret)
# to set updated inside if statement = True
d = timezone.now()
condor_obj._last_status_update = d
mock_co.statuses = 'test_statuses2'
ret = condor_obj.statuses
# Check result, should not set statuses from condor_object again. Same ret as previous.
self.assertEqual('test_statuses', ret)
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase.condor_object')
def test_execute_abs(self, mock_co):
mock_co.submit.return_value = 111
# Execute
CondorBase.objects.get(name='test_condorbase')._execute()
ret = CondorBase.objects.get(name='test_condorbase')
# Check result
self.assertEqual(111, ret.cluster_id)
def test_update_status_not_execute_time(self):
ret = CondorBase.objects.get(name='test_condorbase')._update_status()
# Check result
self.assertEqual('PEN', ret)
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase.condor_object')
def test_update_status(self, mock_co):
mock_co.status = 'Various'
mock_co.statuses = {'Unexpanded': '', 'Idle': '', 'Running': ''}
CondorBase.objects.get(name='test_condorbase_exe')._update_status()
ret = CondorBase.objects.get(name='test_condorbase_exe')._status
# Check result
self.assertEqual('VCP', ret)
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase.condor_object')
def test_update_status_exception(self, mock_co):
mock_co.status = 'Various'
mock_co.statuses = {}
CondorBase.objects.get(name='test_condorbase_exe')._update_status()
ret = CondorBase.objects.get(name='test_condorbase_exe')._status
# Check result
self.assertEqual('ERR', ret)
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase.condor_object')
def test_process_results(self, mock_co):
CondorBase.objects.get(name='test_condorbase_exe')._process_results()
# Check result
mock_co.sync_remote_output.assert_called()
mock_co.close_remote.assert_called()
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase.condor_object')
def test_stop(self, mock_co):
CondorBase.objects.get(name='test_condorbase_exe').stop()
# Check result
mock_co.remove.assert_called()
def test_pause(self):
ret = CondorBase.objects.get(name='test_condorbase_exe').pause()
# Check result
self.assertIsNone(ret)
def test_resume(self):
ret = CondorBase.objects.get(name='test_condorbase_exe').resume()
# Check result
self.assertIsNone(ret)
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase._condor_object')
def test_update_database_fields(self, mock_co):
mock_co._remote_id = 'test_update_remote_id'
ret = CondorBase.objects.get(name='test_condorbase_exe')
ret.remote_id = None
# _condor_object is an abstract method returning a condorpyjob or condorpyworkflow.
# We'll test condor_object.remote_id in condorpyjob test
ret.update_database_fields()
# Check result
self.assertEqual('test_update_remote_id', ret.remote_id)
class CondorBaseTest(TethysTestCase):
def set_up(self):
self.user = User.objects.create_user('tethys_super', '*****@*****.**', 'pass')
self.scheduler = CondorScheduler(
name='test_scheduler',
host='localhost',
username='******',
password='******',
private_key_path='test_path',
private_key_pass='******'
)
self.scheduler.save()
self.condorbase = CondorBase(
name='test_condorbase',
description='test_description',
user=self.user,
label='test_label',
cluster_id='1',
remote_id='test_machine',
scheduler=self.scheduler
)
self.condorbase.save()
self.condorbase_exe = CondorBase(
name='test_condorbase_exe',
description='test_description',
user=self.user,
label='test_label',
execute_time=timezone.now(),
cluster_id='1',
remote_id='test_machine',
scheduler=self.scheduler
)
self.condorbase_exe.save()
def tear_down(self):
self.scheduler.delete()
self.condorbase.delete()
self.condorbase_exe.delete()
@mock.patch('tethys_compute.models.condor.condor_base.CondorBase._condor_object')
def test_condor_object_pro(self, mock_co):
ret = CondorBase.objects.get(name='test_condorbase')
mock_co.return_value = ret
ret.condor_object
# Check result
self.assertEqual(mock_co, ret.condor_object)
self.assertEqual(1, ret.condor_object._cluster_id)
self.assertEqual('test_machine', ret.condor_object._remote_id)
mock_co.set_scheduler.assert_called_with('localhost', 'tethys_super', 'pass', 'test_path', 'test_pass')
def test_condor_obj_abs(self):
ret = CondorBase.objects.get(name='test_condorbase')._condor_object()
# Check result.
self.assertIsNone(ret)
@mock.patch('tethys_compute.models.condor.condor_base.CondorBase.condor_object')
def test_statuses_prop(self, mock_co):
mock_co.statuses = 'test_statuses'
condor_obj = CondorBase.objects.get(name='test_condorbase')
# to set updated inside if statement = False
d = datetime.now() - timedelta(days=1)
condor_obj._last_status_update = d
# Execute
ret = condor_obj.statuses
# Check result
self.assertEqual('test_statuses', ret)
# to set updated inside if statement = True
d = datetime.now()
condor_obj._last_status_update = d
mock_co.statuses = 'test_statuses2'
ret = condor_obj.statuses
# Check result, should not set statuses from condor_object again. Same ret as previous.
self.assertEqual('test_statuses', ret)
@mock.patch('tethys_compute.models.condor.condor_base.CondorBase.condor_object')
def test_execute_abs(self, mock_co):
mock_co.submit.return_value = 111
# Execute
CondorBase.objects.get(name='test_condorbase')._execute()
ret = CondorBase.objects.get(name='test_condorbase')
# Check result
self.assertEqual(111, ret.cluster_id)
def test_update_status_not_execute_time(self):
ret = CondorBase.objects.get(name='test_condorbase')._update_status()
# Check result
self.assertEqual('PEN', ret)
@mock.patch('tethys_compute.models.condor.condor_base.CondorBase.condor_object')
def test_update_status(self, mock_co):
mock_co.status = 'Various'
mock_co.statuses = {'Unexpanded': '', 'Idle': '', 'Running': ''}
CondorBase.objects.get(name='test_condorbase_exe')._update_status()
ret = CondorBase.objects.get(name='test_condorbase_exe')._status
# Check result
self.assertEqual('VCP', ret)
@mock.patch('tethys_compute.models.condor.condor_base.CondorBase.condor_object')
def test_update_status_exception(self, mock_co):
mock_co.status = 'Various'
mock_co.statuses = {}
CondorBase.objects.get(name='test_condorbase_exe')._update_status()
ret = CondorBase.objects.get(name='test_condorbase_exe')._status
# Check result
self.assertEqual('ERR', ret)
@mock.patch('tethys_compute.models.condor.condor_base.CondorBase.condor_object')
def test_process_results(self, mock_co):
CondorBase.objects.get(name='test_condorbase_exe')._process_results()
# Check result
mock_co.sync_remote_output.assert_called()
mock_co.close_remote.assert_called()
@mock.patch('tethys_compute.models.condor.condor_base.CondorBase.condor_object')
def test_stop(self, mock_co):
CondorBase.objects.get(name='test_condorbase_exe').stop()
# Check result
mock_co.remove.assert_called()
def test_pause(self):
ret = CondorBase.objects.get(name='test_condorbase_exe').pause()
# Check result
self.assertIsNone(ret)
def test_resume(self):
ret = CondorBase.objects.get(name='test_condorbase_exe').resume()
# Check result
self.assertIsNone(ret)
@mock.patch('tethys_compute.models.condor.condor_base.CondorBase._condor_object')
def test_update_database_fields(self, mock_co):
mock_co._remote_id = 'test_update_remote_id'
ret = CondorBase.objects.get(name='test_condorbase_exe')
# _condor_object is an abstract method returning a condorpyjob or condorpyworkflow.
# We'll test condor_object.remote_id in condorpyjob test
ret.update_database_fields()
# Check result
self.assertEqual('test_update_remote_id', ret.remote_id)