def test_submit(self):
working_dir = os.path.join(self.base_dir, 'test_files', 'working_dir')
self.job = Job('remote_test',
Templates.vanilla_transfer_files,
host='localhost',
username=os.environ['USER'],
private_key='~/.ssh/id_rsa',
remote_input_files=['../copy_test.py', 'input.txt'],
transfer_input_files='../input.txt',
executable=os.path.join(self.base_dir, 'test_files',
'copy_test.py'),
working_directory=working_dir)
remote_base_path = os.path.expanduser('~/' + self.job._remote_id)
if os.path.exists(remote_base_path):
raise
self.job.submit()
self.assertTrue(os.path.exists(remote_base_path))
self.job.wait()
self.job.sync_remote_output()
local_output = os.path.join(working_dir, self.job.name)
self.assertTrue(os.path.exists(local_output))
output = os.path.join(local_output, 'output.txt')
self.assertTrue(os.path.exists(output))
shutil.rmtree(remote_base_path)
shutil.rmtree(local_output)
def test__init__(self):
attributes = dict()
attributes['job_name'] = self.job_name
self.expected = {'_name': self.job_name,
'_attributes': attributes,
'_num_jobs': 1,
'_cluster_id': 0,
'_job_file': '',
'_remote': None,
'_remote_id': None,
'_remote_input_files': None,
'_cwd': '.'}
self.actual = self.job.__dict__
self.msg = 'testing initialization with default values'
self.assertDictEqual(*self.assert_args)
exe = 'exe'
args = 'args'
num_jobs = '5'
self.job = Job(self.job_name, OrderedDict(), num_jobs, executable=exe, arguments=args)
attributes.update({'executable': exe, 'arguments': args})
self.expected.update({'_name': self.job_name,
'_attributes': attributes,
'_num_jobs': int(num_jobs)})
self.actual = self.job.__dict__
self.actual['_attributes'] = dict(self.actual['_attributes'])
self.msg = 'testing initialization with all values supplied'
self.assertDictEqual(*self.assert_args)
num_jobs = 'five'
self.assertRaises(ValueError, Job, self.job_name, num_jobs=num_jobs)
def test_log_file(self):
self.job = Job(self.job_name, Templates.base)
log_file = '%s/%s/%s.%s.log' % (self.job.initial_dir, self.job.logdir, self.job_name, self.job.cluster_id)
expected = log_file
actual = self.job.log_file
msg = 'checking resolving attribute function for log file'
self.assertEqual(expected, actual, '%s\nExpected: %s\nActual: %s\n' % (msg, expected, actual))
def test_resolve_attribute(self):
job = Job(self.job_name, Templates.vanilla_base)
expected = self.job_name
actual = job._resolve_attribute('initialdir')
msg = 'checking resolving attribute function'
self.assertEqual(
expected, actual,
'%s\nExpected: %s\nActual: %s\n' % (msg, expected, actual))
def test__getattr__(self):
exe = 'exe'
self.job = Job(self.job_name, executable=exe)
expected = exe
actual = self.job.executable
msg = 'testing that existing value is returned'
self.assertEqual(expected, actual, '%s\nExpected: %s\nActual: %s\n' % (msg, expected, actual))
pass
def setUp(self):
"""
:return:
"""
self.job_a = Job('a', Templates.base)
self.job_b = Job('b', Templates.base)
self.job_c = Job('c', Templates.base)
self.job_d = Job('d', Templates.base)
self.node_a = Node(self.job_a)
self.node_b = Node(self.job_b)
self.node_c = Node(self.job_c)
self.node_d = Node(self.job_d)
self.node_a.add_child(self.node_b)
self.node_a.add_child(self.node_c)
self.node_d.add_parent(self.node_b)
self.node_d.add_parent(self.node_c)
def condorpy_job(self):
if not hasattr(self, '_condorpy_job'):
job = Job(name=self.name.replace(' ', '_'),
attributes=self.attributes,
num_jobs=self.num_jobs,
remote_input_files=self.remote_input_files,
working_directory=self.workspace)
self._condorpy_job = job
return self._condorpy_job
def test_condorpy_node(self, mock_job):
mock_job_return = Job(name='test_job',
attributes={'foo': 'bar'},
num_jobs=1,
remote_input_files=['test_file.txt'],
working_directory=self.workspace_dir)
mock_job.return_value = mock_job_return
self.condorworkflownode.job = mock_job_return
ret = self.condorworkflownode.condorpy_node
# Check result
self.assertEqual('<Node: test_job parents() children()>', repr(ret))
def test_get(self):
non_existent_attr = 'not-there'
expected = None
actual = self.job.get(non_existent_attr)
msg = 'testing that None is returned when attribute does not exist'
self.assertIsNone(actual, '%s\nExpected: %s\nActual: %s\n' % (msg, expected, actual))
expected = 'expected'
actual = self.job.get(non_existent_attr, expected)
msg = 'testing that supplied value is returned when attribute does not exist'
self.assertEqual(expected, actual, '%s\nExpected: %s\nActual: %s\n' % (msg, expected, actual))
exe = 'exe'
self.job = Job(self.job_name, executable=exe)
expected = exe
actual = self.job.get('executable')
msg = 'testing that existing value is returned'
self.assertEqual(expected, actual, '%s\nExpected: %s\nActual: %s\n' % (msg, expected, actual))
def condorpy_job(self):
if not hasattr(self, '_condorpy_job'):
if 'executable' in self.attributes.keys():
del self.attributes['executable']
if self.scheduler:
host = self.scheduler.host
username = self.scheduler.username
password = self.scheduler.password
private_key = self.scheduler.private_key_path
private_key_pass = self.scheduler.private_key_pass
else:
host = None
username = None
password = None
private_key = None
private_key_pass = None
attributes = dict()
attributes.update(self.attributes)
attributes.pop('remote_input_files', None)
job = Job(name=self.name.replace(' ', '_'),
attributes=self.condorpy_template,
executable=self.executable,
host=host,
username=username,
password=password,
private_key=private_key,
private_key_pass=private_key_pass,
remote_input_files=self.remote_input_files,
working_directory=self.workspace,
**attributes)
job._cluster_id = self.cluster_id
job._num_jobs = self.num_jobs
if self.remote_id:
job._remote_id = self.remote_id
else:
self.remote_id = job._remote_id
self._condorpy_job = job
return self._condorpy_job
def test__init__(self):
attributes = OrderedDict()
attributes['job_name'] = self.job_name
attributes['executable'] = None
attributes['arguments'] = None
expected = {
'_name': self.job_name,
'_attributes': attributes,
'_num_jobs': 1,
'_cluster_id': 0,
'_job_file': ''
}
actual = self.job.__dict__
msg = 'testing initialization with default values'
self.assertDictEqual(
expected, actual,
'%s\nExpected: %s\nActual: %s\n' % (msg, expected, actual))
exe = 'exe'
args = '-args'
num_jobs = '5'
self.job = Job(self.job_name, OrderedDict(), exe, args, num_jobs)
attributes['executable'] = exe
attributes['arguments'] = args
expected = {
'_name': self.job_name,
'_attributes': attributes,
'_num_jobs': int(num_jobs),
'_cluster_id': 0,
'_job_file': ''
}
actual = self.job.__dict__
msg = 'testing initialization with all values supplied'
self.assertDictEqual(
expected, actual,
'%s\nExpected: %s\nActual: %s\n' % (msg, expected, actual))
num_jobs = 'five'
self.assertRaises(ValueError, Job, self.job_name, num_jobs=num_jobs)
def setUp(self):
self.job_name = 'job_name'
self.job = Job(self.job_name)
