def test_submit(self): working_dir = os.path.join(self.base_dir, 'test_files', 'working_dir') self.job = Job('remote_test', Templates.vanilla_transfer_files, host='localhost', username=os.environ['USER'], private_key='~/.ssh/id_rsa', remote_input_files=['../copy_test.py', 'input.txt'], transfer_input_files='../input.txt', executable=os.path.join(self.base_dir, 'test_files', 'copy_test.py'), working_directory=working_dir) remote_base_path = os.path.expanduser('~/' + self.job._remote_id) if os.path.exists(remote_base_path): raise self.job.submit() self.assertTrue(os.path.exists(remote_base_path)) self.job.wait() self.job.sync_remote_output() local_output = os.path.join(working_dir, self.job.name) self.assertTrue(os.path.exists(local_output)) output = os.path.join(local_output, 'output.txt') self.assertTrue(os.path.exists(output)) shutil.rmtree(remote_base_path) shutil.rmtree(local_output)
def test__init__(self): attributes = dict() attributes['job_name'] = self.job_name self.expected = {'_name': self.job_name, '_attributes': attributes, '_num_jobs': 1, '_cluster_id': 0, '_job_file': '', '_remote': None, '_remote_id': None, '_remote_input_files': None, '_cwd': '.'} self.actual = self.job.__dict__ self.msg = 'testing initialization with default values' self.assertDictEqual(*self.assert_args) exe = 'exe' args = 'args' num_jobs = '5' self.job = Job(self.job_name, OrderedDict(), num_jobs, executable=exe, arguments=args) attributes.update({'executable': exe, 'arguments': args}) self.expected.update({'_name': self.job_name, '_attributes': attributes, '_num_jobs': int(num_jobs)}) self.actual = self.job.__dict__ self.actual['_attributes'] = dict(self.actual['_attributes']) self.msg = 'testing initialization with all values supplied' self.assertDictEqual(*self.assert_args) num_jobs = 'five' self.assertRaises(ValueError, Job, self.job_name, num_jobs=num_jobs)
def test_log_file(self): self.job = Job(self.job_name, Templates.base) log_file = '%s/%s/%s.%s.log' % (self.job.initial_dir, self.job.logdir, self.job_name, self.job.cluster_id) expected = log_file actual = self.job.log_file msg = 'checking resolving attribute function for log file' self.assertEqual(expected, actual, '%s\nExpected: %s\nActual: %s\n' % (msg, expected, actual))
def test_resolve_attribute(self): job = Job(self.job_name, Templates.vanilla_base) expected = self.job_name actual = job._resolve_attribute('initialdir') msg = 'checking resolving attribute function' self.assertEqual( expected, actual, '%s\nExpected: %s\nActual: %s\n' % (msg, expected, actual))
def test__getattr__(self): exe = 'exe' self.job = Job(self.job_name, executable=exe) expected = exe actual = self.job.executable msg = 'testing that existing value is returned' self.assertEqual(expected, actual, '%s\nExpected: %s\nActual: %s\n' % (msg, expected, actual)) pass
def setUp(self): """ :return: """ self.job_a = Job('a', Templates.base) self.job_b = Job('b', Templates.base) self.job_c = Job('c', Templates.base) self.job_d = Job('d', Templates.base) self.node_a = Node(self.job_a) self.node_b = Node(self.job_b) self.node_c = Node(self.job_c) self.node_d = Node(self.job_d) self.node_a.add_child(self.node_b) self.node_a.add_child(self.node_c) self.node_d.add_parent(self.node_b) self.node_d.add_parent(self.node_c)
def condorpy_job(self): if not hasattr(self, '_condorpy_job'): job = Job(name=self.name.replace(' ', '_'), attributes=self.attributes, num_jobs=self.num_jobs, remote_input_files=self.remote_input_files, working_directory=self.workspace) self._condorpy_job = job return self._condorpy_job
def test_condorpy_node(self, mock_job): mock_job_return = Job(name='test_job', attributes={'foo': 'bar'}, num_jobs=1, remote_input_files=['test_file.txt'], working_directory=self.workspace_dir) mock_job.return_value = mock_job_return self.condorworkflownode.job = mock_job_return ret = self.condorworkflownode.condorpy_node # Check result self.assertEqual('<Node: test_job parents() children()>', repr(ret))
def test_get(self): non_existent_attr = 'not-there' expected = None actual = self.job.get(non_existent_attr) msg = 'testing that None is returned when attribute does not exist' self.assertIsNone(actual, '%s\nExpected: %s\nActual: %s\n' % (msg, expected, actual)) expected = 'expected' actual = self.job.get(non_existent_attr, expected) msg = 'testing that supplied value is returned when attribute does not exist' self.assertEqual(expected, actual, '%s\nExpected: %s\nActual: %s\n' % (msg, expected, actual)) exe = 'exe' self.job = Job(self.job_name, executable=exe) expected = exe actual = self.job.get('executable') msg = 'testing that existing value is returned' self.assertEqual(expected, actual, '%s\nExpected: %s\nActual: %s\n' % (msg, expected, actual))
def condorpy_job(self): if not hasattr(self, '_condorpy_job'): if 'executable' in self.attributes.keys(): del self.attributes['executable'] if self.scheduler: host = self.scheduler.host username = self.scheduler.username password = self.scheduler.password private_key = self.scheduler.private_key_path private_key_pass = self.scheduler.private_key_pass else: host = None username = None password = None private_key = None private_key_pass = None attributes = dict() attributes.update(self.attributes) attributes.pop('remote_input_files', None) job = Job(name=self.name.replace(' ', '_'), attributes=self.condorpy_template, executable=self.executable, host=host, username=username, password=password, private_key=private_key, private_key_pass=private_key_pass, remote_input_files=self.remote_input_files, working_directory=self.workspace, **attributes) job._cluster_id = self.cluster_id job._num_jobs = self.num_jobs if self.remote_id: job._remote_id = self.remote_id else: self.remote_id = job._remote_id self._condorpy_job = job return self._condorpy_job
def test__init__(self): attributes = OrderedDict() attributes['job_name'] = self.job_name attributes['executable'] = None attributes['arguments'] = None expected = { '_name': self.job_name, '_attributes': attributes, '_num_jobs': 1, '_cluster_id': 0, '_job_file': '' } actual = self.job.__dict__ msg = 'testing initialization with default values' self.assertDictEqual( expected, actual, '%s\nExpected: %s\nActual: %s\n' % (msg, expected, actual)) exe = 'exe' args = '-args' num_jobs = '5' self.job = Job(self.job_name, OrderedDict(), exe, args, num_jobs) attributes['executable'] = exe attributes['arguments'] = args expected = { '_name': self.job_name, '_attributes': attributes, '_num_jobs': int(num_jobs), '_cluster_id': 0, '_job_file': '' } actual = self.job.__dict__ msg = 'testing initialization with all values supplied' self.assertDictEqual( expected, actual, '%s\nExpected: %s\nActual: %s\n' % (msg, expected, actual)) num_jobs = 'five' self.assertRaises(ValueError, Job, self.job_name, num_jobs=num_jobs)
def setUp(self): self.job_name = 'job_name' self.job = Job(self.job_name)