def testWorkerObject(self):
worker_config_dict = MarsWorkersConfig(
4, cpu=2, memory=10 * 1024 ** 3, limit_resources=False,
spill_volumes=['/tmp/spill_vol', EmptyDirVolumeConfig('empty-dir', '/tmp/empty')],
worker_cache_mem='20%', modules='mars.test_mod', mount_shm=True).build()
self.assertEqual(worker_config_dict['metadata']['name'], 'marsworker')
self.assertEqual(worker_config_dict['spec']['replicas'], 4)
container_dict = worker_config_dict['spec']['template']['spec']['containers'][0]
self.assertEqual(int(container_dict['resources']['requests']['memory']), 10 * 1024 ** 3)
container_envs = dict((p['name'], p) for p in container_dict['env'])
self.assertEqual(container_envs['MKL_NUM_THREADS']['value'], '2')
self.assertEqual(container_envs['MARS_CPU_TOTAL']['value'], '2')
self.assertEqual(int(container_envs['MARS_MEMORY_TOTAL']['value']), 10 * 1024 ** 3)
self.assertEqual(container_envs['MARS_LOAD_MODULES']['value'], 'mars.test_mod')
self.assertEqual(set(container_envs['MARS_SPILL_DIRS']['value'].split(':')),
{'/tmp/empty', '/mnt/hostpath0'})
self.assertEqual(container_envs['MARS_CACHE_MEM_SIZE']['value'], '20%')
volume_list = worker_config_dict['spec']['template']['spec']['volumes']
volume_envs = dict((v['name'], v) for v in volume_list)
self.assertIn('empty-dir', volume_envs)
self.assertEqual(volume_envs['shm-volume']['emptyDir']['medium'], 'Memory')
self.assertEqual(volume_envs['host-path-vol-0']['hostPath']['path'], '/tmp/spill_vol')
volume_mounts = dict((v['name'], v) for v in container_dict['volumeMounts'])
self.assertEqual(volume_mounts['empty-dir']['mountPath'], '/tmp/empty')
self.assertEqual(volume_mounts['host-path-vol-0']['mountPath'], '/mnt/hostpath0')
self.assertEqual(volume_mounts['shm-volume']['mountPath'], '/dev/shm')
worker_config_dict = MarsWorkersConfig(
4, cpu=2, memory=10 * 1024 ** 3, limit_resources=False,
spill_volumes=['/tmp/spill_vol', EmptyDirVolumeConfig('empty-dir', '/tmp/empty')],
modules='mars.test_mod', mount_shm=False).build()
volume_list = worker_config_dict['spec']['template']['spec']['volumes']
self.assertNotIn('shm-volume', volume_list)
container_dict = worker_config_dict['spec']['template']['spec']['containers'][0]
volume_mounts = dict((v['name'], v) for v in container_dict['volumeMounts'])
self.assertNotIn('shm-volume', volume_mounts)
def test_worker_object():
worker_config_dict = MarsWorkersConfig(4,
cpu=2,
memory=10 * 1024**3,
limit_resources=True,
memory_limit_ratio=2,
spill_volumes=[
'/tmp/spill_vol',
EmptyDirVolumeConfig(
'empty-dir', '/tmp/empty')
],
worker_cache_mem='20%',
min_cache_mem='10%',
modules='mars.test_mod',
mount_shm=True).build()
assert worker_config_dict['metadata']['name'] == 'marsworker'
assert worker_config_dict['spec']['replicas'] == 4
container_dict = worker_config_dict['spec']['template']['spec'][
'containers'][0]
assert int(
container_dict['resources']['requests']['memory']) == 10 * 1024**3
assert int(container_dict['resources']['limits']['memory']) == 20 * 1024**3
container_envs = dict((p['name'], p) for p in container_dict['env'])
assert container_envs['MKL_NUM_THREADS']['value'] == '2'
assert container_envs['MARS_CPU_TOTAL']['value'] == '2'
assert int(container_envs['MARS_MEMORY_TOTAL']['value']) == 10 * 1024**3
assert container_envs['MARS_LOAD_MODULES']['value'] == 'mars.test_mod'
assert set(container_envs['MARS_SPILL_DIRS']['value'].split(':')) == {
'/tmp/empty', '/mnt/hostpath0'
}
assert container_envs['MARS_CACHE_MEM_SIZE']['value'] == '20%'
volume_list = worker_config_dict['spec']['template']['spec']['volumes']
volume_envs = dict((v['name'], v) for v in volume_list)
assert 'empty-dir' in volume_envs
assert volume_envs['host-path-vol-0']['hostPath'][
'path'] == '/tmp/spill_vol'
volume_mounts = dict(
(v['name'], v) for v in container_dict['volumeMounts'])
assert volume_mounts['empty-dir']['mountPath'] == '/tmp/empty'
assert volume_mounts['host-path-vol-0']['mountPath'] == '/mnt/hostpath0'
worker_config_dict = MarsWorkersConfig(4,
cpu=2,
memory=10 * 1024**3,
limit_resources=False,
spill_volumes=[
'/tmp/spill_vol',
EmptyDirVolumeConfig(
'empty-dir', '/tmp/empty')
],
modules='mars.test_mod',
mount_shm=False).build()
volume_list = worker_config_dict['spec']['template']['spec']['volumes']
assert 'shm-volume' not in volume_list
container_dict = worker_config_dict['spec']['template']['spec'][
'containers'][0]
volume_mounts = dict(
(v['name'], v) for v in container_dict['volumeMounts'])
assert 'shm-volume' not in volume_mounts