class CondorBaseTest(TethysTestCase):
def set_up(self):
self.user = User.objects.create_user('tethys_super',
'*****@*****.**', 'pass')
self.scheduler = CondorScheduler(name='test_scheduler',
host='localhost',
username='******',
password='******',
private_key_path='test_path',
private_key_pass='******')
self.scheduler.save()
self.condorbase = CondorBase(name='test_condorbase',
description='test_description',
user=self.user,
label='test_label',
cluster_id='1',
remote_id='test_machine',
scheduler=self.scheduler)
self.condorbase.save()
self.condorbase_exe = CondorBase(name='test_condorbase_exe',
description='test_description',
user=self.user,
label='test_label',
execute_time=timezone.now(),
cluster_id='1',
remote_id='test_machine',
scheduler=self.scheduler)
self.condorbase_exe.save()
def tear_down(self):
self.scheduler.delete()
self.condorbase.delete()
self.condorbase_exe.delete()
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase._condor_object')
def test_condor_object_pro(self, mock_co):
ret = CondorBase.objects.get(name='test_condorbase')
mock_co.return_value = ret
ret.condor_object
# Check result
self.assertEqual(mock_co, ret.condor_object)
self.assertEqual(1, ret.condor_object._cluster_id)
self.assertEqual('test_machine', ret.condor_object._remote_id)
mock_co.set_scheduler.assert_called_with('localhost', 'tethys_super',
'pass', 'test_path',
'test_pass')
def test_condor_obj_abs(self):
ret = CondorBase.objects.get(name='test_condorbase')._condor_object()
# Check result.
self.assertIsNone(ret)
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase.condor_object')
def test_statuses_prop(self, mock_co):
mock_co.statuses = 'test_statuses'
condor_obj = CondorBase.objects.get(name='test_condorbase')
# to set updated inside if statement = False
d = timezone.now() - timedelta(days=1)
condor_obj._last_status_update = d
# Execute
ret = condor_obj.statuses
# Check result
self.assertEqual('test_statuses', ret)
# to set updated inside if statement = True
d = timezone.now()
condor_obj._last_status_update = d
mock_co.statuses = 'test_statuses2'
ret = condor_obj.statuses
# Check result, should not set statuses from condor_object again. Same ret as previous.
self.assertEqual('test_statuses', ret)
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase.condor_object')
def test_execute_abs(self, mock_co):
mock_co.submit.return_value = 111
# Execute
CondorBase.objects.get(name='test_condorbase')._execute()
ret = CondorBase.objects.get(name='test_condorbase')
# Check result
self.assertEqual(111, ret.cluster_id)
def test_update_status_not_execute_time(self):
ret = CondorBase.objects.get(name='test_condorbase')._update_status()
# Check result
self.assertEqual('PEN', ret)
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase.condor_object')
def test_update_status(self, mock_co):
mock_co.status = 'Various'
mock_co.statuses = {'Unexpanded': '', 'Idle': '', 'Running': ''}
CondorBase.objects.get(name='test_condorbase_exe')._update_status()
ret = CondorBase.objects.get(name='test_condorbase_exe')._status
# Check result
self.assertEqual('VCP', ret)
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase.condor_object')
def test_update_status_exception(self, mock_co):
mock_co.status = 'Various'
mock_co.statuses = {}
CondorBase.objects.get(name='test_condorbase_exe')._update_status()
ret = CondorBase.objects.get(name='test_condorbase_exe')._status
# Check result
self.assertEqual('ERR', ret)
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase.condor_object')
def test_process_results(self, mock_co):
CondorBase.objects.get(name='test_condorbase_exe')._process_results()
# Check result
mock_co.sync_remote_output.assert_called()
mock_co.close_remote.assert_called()
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase.condor_object')
def test_stop(self, mock_co):
CondorBase.objects.get(name='test_condorbase_exe').stop()
# Check result
mock_co.remove.assert_called()
def test_pause(self):
ret = CondorBase.objects.get(name='test_condorbase_exe').pause()
# Check result
self.assertIsNone(ret)
def test_resume(self):
ret = CondorBase.objects.get(name='test_condorbase_exe').resume()
# Check result
self.assertIsNone(ret)
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase._condor_object')
def test_update_database_fields(self, mock_co):
mock_co._remote_id = 'test_update_remote_id'
ret = CondorBase.objects.get(name='test_condorbase_exe')
ret.remote_id = None
# _condor_object is an abstract method returning a condorpyjob or condorpyworkflow.
# We'll test condor_object.remote_id in condorpyjob test
ret.update_database_fields()
# Check result
self.assertEqual('test_update_remote_id', ret.remote_id)
class CondorBaseTest(TethysTestCase):
def set_up(self):
self.user = User.objects.create_user('tethys_super',
'*****@*****.**', 'pass')
self.scheduler = CondorScheduler(name='test_scheduler',
host='localhost',
username='******',
password='******',
private_key_path='test_path',
private_key_pass='******')
self.scheduler.save()
self.condorbase = CondorBase(name='test_condorbase',
description='test_description',
user=self.user,
label='test_label',
cluster_id='1',
remote_id='test_machine',
scheduler=self.scheduler)
self.condorbase.save()
self.condorbase_exe = CondorBase(name='test_condorbase_exe',
description='test_description',
user=self.user,
label='test_label',
execute_time=timezone.now(),
cluster_id='1',
remote_id='test_machine',
scheduler=self.scheduler)
self.condorbase_exe.save()
def tear_down(self):
self.scheduler.delete()
self.condorbase.delete()
self.condorbase_exe.delete()
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase._condor_object')
def test_condor_object_pro(self, mock_co):
ret = CondorBase.objects.get(name='test_condorbase')
mock_co.return_value = ret
ret.condor_object
# Check result
self.assertEqual(mock_co, ret.condor_object)
self.assertEqual(1, ret.condor_object._cluster_id)
self.assertEqual('test_machine', ret.condor_object._remote_id)
mock_co.set_scheduler.assert_called_with('localhost', 'tethys_super',
'pass', 'test_path',
'test_pass')
def test_condor_obj_abs(self):
ret = CondorBase.objects.get(name='test_condorbase')._condor_object()
# Check result.
self.assertIsNone(ret)
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase.condor_object')
def test_statuses_prop(self, mock_co):
mock_co.statuses = 'test_statuses'
condor_obj = CondorBase.objects.get(name='test_condorbase')
# to set updated inside if statement = False
d = timezone.now() - timedelta(days=1)
condor_obj._last_status_update = d
# Execute
ret = condor_obj.statuses
# Check result
self.assertEqual('test_statuses', ret)
# to set updated inside if statement = True
d = timezone.now()
condor_obj._last_status_update = d
mock_co.statuses = 'test_statuses2'
ret = condor_obj.statuses
# Check result, should not set statuses from condor_object again. Same ret as previous.
self.assertEqual('test_statuses', ret)
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase.condor_object')
def test_execute_abs(self, mock_co):
mock_co.submit.return_value = 111
# Execute
CondorBase.objects.get(name='test_condorbase')._execute()
ret = CondorBase.objects.get(name='test_condorbase')
# Check result
self.assertEqual(111, ret.cluster_id)
def test_update_status_not_execute_time(self):
ret = CondorBase.objects.get(name='test_condorbase')._update_status()
# Check result
self.assertEqual('PEN', ret)
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase.condor_object')
def test_update_status(self, mock_co):
mock_co.status = 'Various'
mock_co.statuses = {'Unexpanded': '', 'Idle': '', 'Running': ''}
CondorBase.objects.get(name='test_condorbase_exe')._update_status()
ret = CondorBase.objects.get(name='test_condorbase_exe')._status
# Check result
self.assertEqual('VCP', ret)
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase.condor_object')
def test_update_status_exception(self, mock_co):
mock_co.status = 'Various'
mock_co.statuses = {}
CondorBase.objects.get(name='test_condorbase_exe')._update_status()
ret = CondorBase.objects.get(name='test_condorbase_exe')._status
# Check result
self.assertEqual('ERR', ret)
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase.condor_object')
def test_process_results(self, mock_co):
CondorBase.objects.get(name='test_condorbase_exe')._process_results()
# Check result
mock_co.sync_remote_output.assert_called()
mock_co.close_remote.assert_called()
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase.condor_object')
def test_stop(self, mock_co):
CondorBase.objects.get(name='test_condorbase_exe').stop()
# Check result
mock_co.remove.assert_called()
@mock.patch('tethys_compute.models.condor.condor_base.CondorBase.save')
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase.condor_object')
def test_resubmit(self, mock_co, _):
CondorBase.objects.get(name='test_condorbase')._resubmit()
# Check result
mock_co.close_remote.assert_called()
def test_get_logs_from_workspace_no_file(self):
logs_file = {
'workflow': 'Model_Run.dag.dagman.out',
'test_job': {
'log': 'test_job/logs/*.log',
'error': 'test_job/logs/*.err',
'output': 'test_job/logs/*.out'
}
}
logs = CondorBase.objects.get(
name='test_condorbase')._get_logs_from_workspace(logs_file)
expected_results = {
'workflow': 'test_machine/Model_Run.dag.dagman.out does not exist',
'test_job': {
'log': 'File does not exist',
'error': 'File does not exist',
'output': 'File does not exist'
}
}
self.assertEqual(expected_results, logs)
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase._get_logs_from_workspace'
)
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase._get_logs_from_remote'
)
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase._log_files')
def test_get_logs_no_remote(self, mock_log_files, mock_log_remote,
mock_log_workspace):
logs_file = {
'workflow': 'Model_Run.dag.dagman.out',
'test_job': {
'log': 'test_job/logs/*.log',
'error': 'test_job/logs/*.err',
'output': 'test_job/logs/*.out'
}
}
logs_file_contents = {
'workspace': '',
'test_job': {
'log': '',
'error': '',
'output': ''
}
}
mock_log_files.return_value = logs_file
mock_log_remote.return_value = logs_file_contents
expected_return = 'log_from_workspace'
mock_log_workspace.return_value = expected_return
logs = CondorBase.objects.get(name='test_condorbase')._get_logs()
self.assertEqual(expected_return, logs)
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase._get_logs_from_workspace'
)
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase._get_logs_from_remote'
)
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase._log_files')
def test_get_logs_remote(self, mock_log_files, mock_log_remote,
mock_log_workspace):
logs_file = {
'workflow': 'Model_Run.dag.dagman.out',
'test_job': {
'log': 'test_job/logs/*.log',
'error': 'test_job/logs/*.err',
'output': 'test_job/logs/*.out'
}
}
logs_file_contents = {
'workspace': 'has_content',
'test_job': {
'log': '',
'error': '',
'output': ''
}
}
mock_log_files.return_value = logs_file
mock_log_remote.return_value = logs_file_contents
mock_log_workspace.return_value = 'log_from_workspace'
logs = CondorBase.objects.get(name='test_condorbase')._get_logs()
self.assertEqual(logs_file_contents, logs)
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase._get_logs_from_workspace'
)
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase._get_logs_from_remote'
)
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase._log_files')
def test_get_logs_remote_check_child(self, mock_log_files, mock_log_remote,
mock_log_workspace):
logs_file = {
'workflow': 'Model_Run.dag.dagman.out',
'test_job': {
'log': 'test_job/logs/*.log',
'error': 'test_job/logs/*.err',
'output': 'test_job/logs/*.out'
}
}
logs_file_contents = {
'workspace': '',
'test_job': {
'log': 'has_content',
'error': '',
'output': ''
}
}
mock_log_files.return_value = logs_file
mock_log_remote.return_value = logs_file_contents
mock_log_workspace.return_value = 'log_from_workspace'
logs = CondorBase.objects.get(name='test_condorbase')._get_logs()
self.assertEqual(logs_file_contents, logs)
@mock.patch('builtins.open')
@mock.patch('tethys_compute.models.condor.condor_base.os.listdir')
@mock.patch('tethys_compute.models.condor.condor_base.os.path.isdir')
@mock.patch('tethys_compute.models.condor.condor_base.os.path.isfile')
def test_get_logs_from_workspace_mock_file(self, mock_isfile, mock_isdir,
mock_list_dir, mock_open):
mock_isfile.return_value = True
mock_isdir.return_value = True
mock_list_dir.return_value = ['file1.log', 'file2.err', 'file3.out']
logs_file = {
'workflow': 'Model_Run.dag.dagman.out',
'test_job': {
'log': 'test_job/logs/*.log',
'error': 'test_job/logs/*.err',
'output': 'test_job/logs/*.out'
}
}
logs = CondorBase.objects.get(
name='test_condorbase')._get_logs_from_workspace(logs_file)
self.assertIsNotNone(logs['workflow'])
self.assertIsNotNone(logs['test_job']['log'])
self.assertIsNotNone(logs['test_job']['error'])
self.assertIsNotNone(logs['test_job']['output'])
def test_pause(self):
ret = CondorBase.objects.get(name='test_condorbase_exe').pause()
# Check result
self.assertIsNone(ret)
def test_resume(self):
ret = CondorBase.objects.get(name='test_condorbase_exe').resume()
# Check result
self.assertIsNone(ret)
@mock.patch(
'tethys_compute.models.condor.condor_base.CondorBase._condor_object')
def test_update_database_fields(self, mock_co):
mock_co._remote_id = 'test_update_remote_id'
ret = CondorBase.objects.get(name='test_condorbase_exe')
ret.remote_id = None
# _condor_object is an abstract method returning a condorpyjob or condorpyworkflow.
# We'll test condor_object.remote_id in condorpyjob test
ret.update_database_fields()
# Check result
self.assertEqual('test_update_remote_id', ret.remote_id)
def test_abs_method(self):
# Resubmit
ret = CondorBase.objects.get(name='test_condorbase')._log_files()
# Check result
self.assertIsNone(ret)
# Resubmit
ret = CondorBase.objects.get(
name='test_condorbase')._get_logs_from_remote('test')
# Check result
self.assertIsNone(ret)
class CondorBaseTest(TethysTestCase):
def set_up(self):
self.user = User.objects.create_user('tethys_super', '*****@*****.**', 'pass')
self.scheduler = CondorScheduler(
name='test_scheduler',
host='localhost',
username='******',
password='******',
private_key_path='test_path',
private_key_pass='******'
)
self.scheduler.save()
self.condorbase = CondorBase(
name='test_condorbase',
description='test_description',
user=self.user,
label='test_label',
cluster_id='1',
remote_id='test_machine',
scheduler=self.scheduler
)
self.condorbase.save()
self.condorbase_exe = CondorBase(
name='test_condorbase_exe',
description='test_description',
user=self.user,
label='test_label',
execute_time=timezone.now(),
cluster_id='1',
remote_id='test_machine',
scheduler=self.scheduler
)
self.condorbase_exe.save()
def tear_down(self):
self.scheduler.delete()
self.condorbase.delete()
self.condorbase_exe.delete()
@mock.patch('tethys_compute.models.condor.condor_base.CondorBase._condor_object')
def test_condor_object_pro(self, mock_co):
ret = CondorBase.objects.get(name='test_condorbase')
mock_co.return_value = ret
ret.condor_object
# Check result
self.assertEqual(mock_co, ret.condor_object)
self.assertEqual(1, ret.condor_object._cluster_id)
self.assertEqual('test_machine', ret.condor_object._remote_id)
mock_co.set_scheduler.assert_called_with('localhost', 'tethys_super', 'pass', 'test_path', 'test_pass')
def test_condor_obj_abs(self):
ret = CondorBase.objects.get(name='test_condorbase')._condor_object()
# Check result.
self.assertIsNone(ret)
@mock.patch('tethys_compute.models.condor.condor_base.CondorBase.condor_object')
def test_statuses_prop(self, mock_co):
mock_co.statuses = 'test_statuses'
condor_obj = CondorBase.objects.get(name='test_condorbase')
# to set updated inside if statement = False
d = datetime.now() - timedelta(days=1)
condor_obj._last_status_update = d
# Execute
ret = condor_obj.statuses
# Check result
self.assertEqual('test_statuses', ret)
# to set updated inside if statement = True
d = datetime.now()
condor_obj._last_status_update = d
mock_co.statuses = 'test_statuses2'
ret = condor_obj.statuses
# Check result, should not set statuses from condor_object again. Same ret as previous.
self.assertEqual('test_statuses', ret)
@mock.patch('tethys_compute.models.condor.condor_base.CondorBase.condor_object')
def test_execute_abs(self, mock_co):
mock_co.submit.return_value = 111
# Execute
CondorBase.objects.get(name='test_condorbase')._execute()
ret = CondorBase.objects.get(name='test_condorbase')
# Check result
self.assertEqual(111, ret.cluster_id)
def test_update_status_not_execute_time(self):
ret = CondorBase.objects.get(name='test_condorbase')._update_status()
# Check result
self.assertEqual('PEN', ret)
@mock.patch('tethys_compute.models.condor.condor_base.CondorBase.condor_object')
def test_update_status(self, mock_co):
mock_co.status = 'Various'
mock_co.statuses = {'Unexpanded': '', 'Idle': '', 'Running': ''}
CondorBase.objects.get(name='test_condorbase_exe')._update_status()
ret = CondorBase.objects.get(name='test_condorbase_exe')._status
# Check result
self.assertEqual('VCP', ret)
@mock.patch('tethys_compute.models.condor.condor_base.CondorBase.condor_object')
def test_update_status_exception(self, mock_co):
mock_co.status = 'Various'
mock_co.statuses = {}
CondorBase.objects.get(name='test_condorbase_exe')._update_status()
ret = CondorBase.objects.get(name='test_condorbase_exe')._status
# Check result
self.assertEqual('ERR', ret)
@mock.patch('tethys_compute.models.condor.condor_base.CondorBase.condor_object')
def test_process_results(self, mock_co):
CondorBase.objects.get(name='test_condorbase_exe')._process_results()
# Check result
mock_co.sync_remote_output.assert_called()
mock_co.close_remote.assert_called()
@mock.patch('tethys_compute.models.condor.condor_base.CondorBase.condor_object')
def test_stop(self, mock_co):
CondorBase.objects.get(name='test_condorbase_exe').stop()
# Check result
mock_co.remove.assert_called()
def test_pause(self):
ret = CondorBase.objects.get(name='test_condorbase_exe').pause()
# Check result
self.assertIsNone(ret)
def test_resume(self):
ret = CondorBase.objects.get(name='test_condorbase_exe').resume()
# Check result
self.assertIsNone(ret)
@mock.patch('tethys_compute.models.condor.condor_base.CondorBase._condor_object')
def test_update_database_fields(self, mock_co):
mock_co._remote_id = 'test_update_remote_id'
ret = CondorBase.objects.get(name='test_condorbase_exe')
# _condor_object is an abstract method returning a condorpyjob or condorpyworkflow.
# We'll test condor_object.remote_id in condorpyjob test
ret.update_database_fields()
# Check result
self.assertEqual('test_update_remote_id', ret.remote_id)