def test_create_cud_from_task(self, mocked_ComputeUnitDescription,
mocked_Logger,
mocked_get_input_list_from_task,
mocked_get_output_list_from_task,
mocked_resolve_arguments):
mocked_ComputeUnitDescription.name = None
mocked_ComputeUnitDescription.pre_exec = None
mocked_ComputeUnitDescription.executable = None
mocked_ComputeUnitDescription.arguments = None
mocked_ComputeUnitDescription.sandbox = None
mocked_ComputeUnitDescription.post_exec = None
mocked_ComputeUnitDescription.tag = None
mocked_ComputeUnitDescription.cpu_processes = None
mocked_ComputeUnitDescription.cpu_threads = None
mocked_ComputeUnitDescription.cpu_process_type = None
mocked_ComputeUnitDescription.cpu_thread_type = None
mocked_ComputeUnitDescription.gpu_processes = None
mocked_ComputeUnitDescription.gpu_threads = None
mocked_ComputeUnitDescription.gpu_process_type = None
mocked_ComputeUnitDescription.gpu_thread_type = None
mocked_ComputeUnitDescription.lfs_per_process = None
mocked_ComputeUnitDescription.stdout = None
mocked_ComputeUnitDescription.stderr = None
mocked_ComputeUnitDescription.input_staging = None
mocked_ComputeUnitDescription.output_staging = None
task = mock.Mock()
task.uid = 'task.0000'
task.name = 'task.0000'
task.parent_stage = {'uid': 'stage.0000', 'name': 'stage.0000'}
task.parent_pipeline = {'uid': 'pipe.0000', 'name': 'pipe.0000'}
task.pre_exec = 'post_exec'
task.executable = '/bin/date'
task.arguments = 'test_arg'
task.sandbox = 'unit.0000'
task.post_exec = ''
task.cpu_reqs = {
'cpu_processes': 5,
'cpu_threads': 6,
'cpu_process_type': 'POSIX',
'cpu_thread_type': None
}
task.gpu_reqs = {
'gpu_processes': 5,
'gpu_threads': 6,
'gpu_process_type': 'POSIX',
'gpu_thread_type': None
}
task.tag = None
task.lfs_per_process = 235
task.stderr = 'stderr'
task.stdout = 'stdout'
test_cud = create_cud_from_task(task, None)
self.assertEqual(
test_cud.name,
'task.0000,task.0000,stage.0000,stage.0000,pipe.0000,pipe.0000')
self.assertEqual(test_cud.pre_exec, 'post_exec')
self.assertEqual(test_cud.executable, '/bin/date')
self.assertEqual(test_cud.arguments, 'test_args')
self.assertEqual(test_cud.sandbox, 'unit.0000')
self.assertEqual(test_cud.post_exec, '')
self.assertEqual(test_cud.cpu_processes, 5)
self.assertEqual(test_cud.cpu_threads, 6)
self.assertEqual(test_cud.cpu_process_type, 'POSIX')
self.assertIsNone(test_cud.cpu_thread_type)
self.assertEqual(test_cud.gpu_processes, 5)
self.assertEqual(test_cud.gpu_threads, 6)
self.assertEqual(test_cud.gpu_process_type, 'POSIX')
self.assertIsNone(test_cud.gpu_thread_type)
self.assertEqual(test_cud.lfs_per_process, 235)
self.assertEqual(test_cud.stdout, 'stdout')
self.assertEqual(test_cud.stderr, 'stderr')
self.assertEqual(test_cud.input_staging, 'inputs')
self.assertEqual(test_cud.output_staging, 'outputs')
def test_issue_259(self, mocked_ComputeUnitDescription, mocked_Logger):
mocked_ComputeUnitDescription.name = None
mocked_ComputeUnitDescription.pre_exec = None
mocked_ComputeUnitDescription.executable = None
mocked_ComputeUnitDescription.arguments = None
mocked_ComputeUnitDescription.sandbox = None
mocked_ComputeUnitDescription.post_exec = None
mocked_ComputeUnitDescription.tag = None
mocked_ComputeUnitDescription.cpu_processes = None
mocked_ComputeUnitDescription.cpu_threads = None
mocked_ComputeUnitDescription.cpu_process_type = None
mocked_ComputeUnitDescription.cpu_thread_type = None
mocked_ComputeUnitDescription.gpu_processes = None
mocked_ComputeUnitDescription.gpu_threads = None
mocked_ComputeUnitDescription.gpu_process_type = None
mocked_ComputeUnitDescription.gpu_thread_type = None
mocked_ComputeUnitDescription.lfs_per_process = None
mocked_ComputeUnitDescription.stdout = None
mocked_ComputeUnitDescription.stderr = None
mocked_ComputeUnitDescription.input_staging = None
mocked_ComputeUnitDescription.output_staging = None
pipeline_name = 'p1'
stage_name = 's1'
t1_name = 't1'
t2_name = 't2'
placeholders = {
pipeline_name: {
stage_name: {
t1_name: {
'path': '/home/vivek/t1',
'rts_uid': 'unit.0002'
},
t2_name: {
'path': '/home/vivek/t2',
'rts_uid': 'unit.0003'
}
}
}
}
task = mock.MagicMock(spec=radical.entk.Task)
task.uid = 'task.0000'
task.name = 'task.0000'
task.parent_stage = {'uid': 'stage.0000', 'name': 'stage.0000'}
task.parent_pipeline = {'uid': 'pipe.0000', 'name': 'pipe.0000'}
task.pre_exec = 'post_exec'
task.executable = '/bin/date'
task.arguments = [
'$SHARED',
'$Pipeline_%s_Stage_%s_Task_%s' %
(pipeline_name, stage_name, t1_name),
'$Pipeline_%s_Stage_%s_Task_%s' %
(pipeline_name, stage_name, t2_name), '$NODE_LFS_PATH/test.txt'
]
task.sandbox = 'unit.0000'
task.post_exec = ''
task.cpu_reqs = {
'cpu_processes': 5,
'cpu_threads': 6,
'cpu_process_type': 'POSIX',
'cpu_thread_type': None
}
task.gpu_reqs = {
'gpu_processes': 5,
'gpu_threads': 6,
'gpu_process_type': 'POSIX',
'gpu_thread_type': None
}
task.tag = None
task.lfs_per_process = 235
task.stderr = 'stderr'
task.stdout = 'stdout'
input_list = [{
'source': 'test_file',
'target': 'pilot:///test_file',
'action': 'Link'
}, {
'source': 'test_file',
'target': 'pilot:///test_file'
}, {
'source': 'test_file',
'target': 'pilot:///test_file',
'action': 'Copy'
}, {
'source': 'test_file',
'target': 'pilot:///test_file',
'action': 'Move'
}]
task.link_input_data = ['test_file > $SHARED/test_file']
task.upload_input_data = ['test_file > $SHARED/test_file']
task.copy_input_data = ['test_file > $SHARED/test_file']
task.move_input_data = ['test_file > $SHARED/test_file']
output_list = [{
'source': 'test_file',
'target': 'pilot:///test_file',
'action': 'Link'
}, {
'source': 'test_file',
'target': 'pilot:///test_file'
}, {
'source': 'test_file',
'target': 'pilot:///test_file',
'action': 'Copy'
}, {
'source': 'test_file',
'target': 'pilot:///test_file',
'action': 'Move'
}]
task.link_output_data = ['test_file > $SHARED/test_file']
task.download_output_data = ['test_file > $SHARED/test_file']
task.copy_output_data = ['test_file > $SHARED/test_file']
task.move_output_data = ['test_file > $SHARED/test_file']
test_cud = create_cud_from_task(task, placeholders)
self.assertEqual(
test_cud.name,
'task.0000,task.0000,stage.0000,stage.0000,pipe.0000,pipe.0000')
self.assertEqual(test_cud.pre_exec, 'post_exec')
self.assertEqual(test_cud.executable, '/bin/date')
self.assertEqual(test_cud.arguments, [
'$RP_PILOT_STAGING', '/home/vivek/t1', '/home/vivek/t2',
'$NODE_LFS_PATH/test.txt'
])
self.assertEqual(test_cud.sandbox, 'unit.0000')
self.assertEqual(test_cud.post_exec, '')
self.assertEqual(test_cud.cpu_processes, 5)
self.assertEqual(test_cud.cpu_threads, 6)
self.assertEqual(test_cud.cpu_process_type, 'POSIX')
self.assertIsNone(test_cud.cpu_thread_type)
self.assertEqual(test_cud.gpu_processes, 5)
self.assertEqual(test_cud.gpu_threads, 6)
self.assertEqual(test_cud.gpu_process_type, 'POSIX')
self.assertIsNone(test_cud.gpu_thread_type)
self.assertEqual(test_cud.lfs_per_process, 235)
self.assertEqual(test_cud.stdout, 'stdout')
self.assertEqual(test_cud.stderr, 'stderr')
self.assertEqual(test_cud.input_staging, input_list)
self.assertEqual(test_cud.output_staging, output_list)
def test_create_cud_from_task():
"""
**Purpose**: Test if the 'create_cud_from_task' function generates a RP
ComputeUnitDescription with the complete Task description.
"""
pipeline = 'p1'
stage = 's1'
task = 't1'
placeholders = {pipeline: {stage: {task: '/home/vivek/some_file.txt'}}}
t1 = Task()
t1.name = 't1'
t1.pre_exec = ['module load gromacs']
t1.executable = 'grompp'
t1.arguments = ['hello']
t1.cpu_reqs = {
'processes': 4,
'process_type': 'MPI',
'threads_per_process': 1,
'thread_type': 'OpenMP'
}
t1.gpu_reqs = {
'processes': 4,
'process_type': 'MPI',
'threads_per_process': 2,
'thread_type': 'OpenMP'
}
t1.post_exec = ['echo test']
t1.upload_input_data = ['upload_input.dat']
t1.copy_input_data = ['copy_input.dat']
t1.link_input_data = ['link_input.dat']
t1.copy_output_data = ['copy_output.dat']
t1.download_output_data = ['download_output.dat']
p = Pipeline()
p.name = 'p1'
s = Stage()
s.name = 's1'
s.tasks = t1
p.stages = s
cud = create_cud_from_task(t1, placeholders)
assert cud.name == '%s,%s,%s,%s,%s,%s' % (
t1.uid, t1.name, t1.parent_stage['uid'], t1.parent_stage['name'],
t1.parent_pipeline['uid'], t1.parent_pipeline['name'])
assert cud.pre_exec == t1.pre_exec
# rp returns executable as a string regardless of whether assignment was using string or list
assert cud.executable == t1.executable
assert cud.arguments == t1.arguments
assert cud.post_exec == t1.post_exec
assert cud.cpu_processes == t1.cpu_reqs['processes']
assert cud.cpu_threads == t1.cpu_reqs['threads_per_process']
assert cud.cpu_process_type == t1.cpu_reqs['process_type']
assert cud.cpu_thread_type == t1.cpu_reqs['thread_type']
assert cud.gpu_processes == t1.gpu_reqs['processes']
assert cud.gpu_threads == t1.gpu_reqs['threads_per_process']
assert cud.gpu_process_type == t1.gpu_reqs['process_type']
assert cud.gpu_thread_type == t1.gpu_reqs['thread_type']
assert {
'source': 'upload_input.dat',
'target': 'upload_input.dat'
} in cud.input_staging
assert {
'source': 'copy_input.dat',
'action': rp.COPY,
'target': 'copy_input.dat'
} in cud.input_staging
assert {
'source': 'link_input.dat',
'action': rp.LINK,
'target': 'link_input.dat'
} in cud.input_staging
assert {
'source': 'copy_output.dat',
'action': rp.COPY,
'target': 'copy_output.dat'
} in cud.output_staging
assert {
'source': 'download_output.dat',
'target': 'download_output.dat'
} in cud.output_staging