def test_spark_defaults_single_core(self):
node = dict(fqdn='hosty.domain.be', network='ib0', pid=1234,
cores=1, totalcores=1, usablecores=range(1), num_nodes=1,
memory=dict(meminfo=dict(memtotal=68719476736), ulimit='unlimited'))
dflts = hcip.spark_defaults(None, node)
self.assertTrue(len(dflts), 3)
self.assertEqual(dflts['spark.executor.instances'], 1)
self.assertEqual(dflts['spark.executor.cores'], 1)
self.assertEqual(hcc.parse_memory(dflts['spark.executor.memory']), hcc.parse_memory('56G'))
def test_mapred_site_xml_defaults(self):
node = dict(fqdn='hosty.domain.be', network='ib0', pid=1234,
cores=24, totalcores=24, usablecores=range(24), num_nodes=1,
memory=dict(meminfo=dict(memtotal=68719476736), ulimit='unlimited'))
d = hca.mapred_site_xml_defaults('/', node)
self.assertEqual(len(d), 7)
# Capped at 8g
self.assertEqual(d['mapreduce.map.memory.mb'], hcc.round_mb(hcc.parse_memory('2G')))
self.assertEqual(d['mapreduce.reduce.memory.mb'], hcc.round_mb(hcc.parse_memory('4G')))
def test_mapred_site_xml_defaults(self):
node = dict(fqdn='hosty.domain.be', network='ib0', pid=1234,
cores=24, totalcores=24, usablecores=range(24), num_nodes=1,
memory=dict(meminfo=dict(memtotal=68719476736), ulimit='unlimited'))
d = hca.mapred_site_xml_defaults('/', node)
self.assertEqual(len(d), 7)
# Capped at 8g
self.assertEqual(d['mapreduce.map.memory.mb'], hcc.round_mb(hcc.parse_memory('2G')))
self.assertEqual(d['mapreduce.reduce.memory.mb'], hcc.round_mb(hcc.parse_memory('4G')))
def test_spark_defaults(self):
node = dict(fqdn='hosty.domain.be', network='ib0', pid=1234,
cores=16, totalcores=16, usablecores=range(16), num_nodes=6,
memory=dict(meminfo=dict(memtotal=68719476736), ulimit='unlimited'))
dflts = hcip.spark_defaults(None, node)
self.assertTrue(len(dflts), 3)
self.assertEqual(dflts['spark.executor.instances'], 47) # 6 nodes * 8 cores -1 core used for driver
self.assertEqual(dflts['spark.executor.cores'], 2)
self.assertEqual(hcc.parse_memory(dflts['spark.executor.memory']), hcc.parse_memory('7G'))
def test_spark_defaults(self):
node = dict(fqdn='hosty.domain.be', network='ib0', pid=1234,
cores=16, totalcores=16, usablecores=range(16), num_nodes=6,
memory=dict(meminfo=dict(memtotal=68719476736), ulimit='unlimited'))
dflts = hcip.spark_defaults(None, node)
self.assertTrue(len(dflts), 3)
self.assertEqual(dflts['spark.executor.instances'], 47) # 6 nodes * 8 cores -1 core used for driver
self.assertEqual(dflts['spark.executor.cores'], 2)
self.assertEqual(hcc.parse_memory(dflts['spark.executor.memory']), hcc.parse_memory('7G'))
def test_spark_defaults_single_core(self):
node = dict(fqdn='hosty.domain.be', network='ib0', pid=1234,
cores=1, totalcores=1, usablecores=range(1), num_nodes=1,
memory=dict(meminfo=dict(memtotal=68719476736), ulimit='unlimited'))
dflts = hcip.spark_defaults(None, node)
self.assertTrue(len(dflts), 3)
self.assertEqual(dflts['spark.executor.instances'], 1)
self.assertEqual(dflts['spark.executor.cores'], 1)
self.assertEqual(hcc.parse_memory(dflts['spark.executor.memory']), hcc.parse_memory('56G'))
def test_mapred_site_xml_defaults(self):
'''Test mapred defaults; note: only using 4 from 24 cores.'''
node = dict(fqdn='hosty.domain.be', network='ib0', pid=1234,
cores=4, totalcores=24, usablecores=[0, 1, 2, 3], num_nodes=1,
memory=dict(meminfo=dict(memtotal=68719476736), ulimit='unlimited'))
d = hca.mapred_site_xml_defaults('/', node)
self.assertEqual(len(d), 9)
self.assertEqual(d['hadoop.ln.cmd'], '/bin/ln')
self.assertEqual(d['lustre.dir'], '$workdir')
self.assertEqual(d['mapreduce.map.memory.mb'], hcc.round_mb(hcc.parse_memory('1G')))
self.assertEqual(d['mapreduce.reduce.memory.mb'], hcc.round_mb(hcc.parse_memory('2G')))
def test_mapred_site_xml_defaults(self):
'''Test mapred defaults; note: only using 4 from 24 cores.'''
node = dict(fqdn='hosty.domain.be', network='ib0', pid=1234,
cores=4, totalcores=24, usablecores=[0, 1, 2, 3], num_nodes=1,
memory=dict(meminfo=dict(memtotal=68719476736), ulimit='unlimited'))
d = hca.mapred_site_xml_defaults('/', node)
self.assertEqual(len(d), 9)
self.assertEqual(d['hadoop.ln.cmd'], '/bin/ln')
self.assertEqual(d['lustre.dir'], '$workdir')
self.assertEqual(d['mapreduce.map.memory.mb'], hcc.round_mb(hcc.parse_memory('1G')))
self.assertEqual(d['mapreduce.reduce.memory.mb'], hcc.round_mb(hcc.parse_memory('2G')))
def min_container_size(totalmem):
'''
Given an amount of memory in bytes, return the amount of memory that
represents the minimum amount of memory (in bytes) to be allocated in a Yarn
container.
'''
if totalmem < parse_memory('4G'):
return 256 * (1024**2)
elif totalmem < parse_memory('8G'):
return 512 * (1024**2)
elif totalmem < parse_memory('24G'):
return 1024**3
else:
return 2 * (1024**3)
def min_container_size(totalmem):
'''
Given an amount of memory in bytes, return the amount of memory that
represents the minimum amount of memory (in bytes) to be allocated in a Yarn
container.
'''
if totalmem < parse_memory('4G'):
return 256 * (1024**2)
elif totalmem < parse_memory('8G'):
return 512 * (1024**2)
elif totalmem < parse_memory('24G'):
return 1024**3
else:
return 2 * (1024**3)
def test_mapred_site_xml_defaults(self):
node = dict(
fqdn="hosty.domain.be",
network="ib0",
pid=1234,
cores=24,
totalcores=24,
usablecores=range(24),
num_nodes=1,
memory=dict(meminfo=dict(memtotal=68719476736), ulimit="unlimited"),
)
d = hca.mapred_site_xml_defaults("/", node)
self.assertEqual(len(d), 7)
# Capped at 8g
self.assertEqual(d["mapreduce.map.memory.mb"], hcc.round_mb(hcc.parse_memory("2G")))
self.assertEqual(d["mapreduce.reduce.memory.mb"], hcc.round_mb(hcc.parse_memory("4G")))
def test_yarn_site_xml_defaults(self):
node = dict(fqdn='hosty.domain.be', network='ib0', pid=1234,
cores=24, totalcores=24, usablecores=range(24), num_nodes=1,
memory=dict(meminfo=dict(memtotal=68719476736), ulimit='unlimited'))
d = hca.yarn_site_xml_defaults('/', node)
self.assertEqual(len(d), 16)
self.assertEqual(d['yarn.nodemanager.resource.memory-mb'], hcc.round_mb(hcc.parse_memory('56G')))
self.assertEqual(d['yarn.resourcemanager.webapp.address'], '$masterhostaddress:8088')
self.assertEqual(d['yarn.resourcemanager.webapp.https.address'], '$masterhostaddress:8090')
self.assertEqual(d['yarn.nodemanager.hostname'], '$dataname')
self.assertEqual(d['yarn.nodemanager.webapp.address'], '$hostaddress:8042')
self.assertEqual(d['yarn.scheduler.minimum-allocation-mb'], hcc.round_mb(hcc.parse_memory('2G')))
self.assertEqual(d['yarn.scheduler.maximum-allocation-mb'], hcc.round_mb(hcc.parse_memory('56G')))
self.assertEqual(d['yarn.scheduler.maximum-allocation-vcores'], '24')
self.assertEqual(d['yarn.scheduler.minimum-allocation-vcores'], '1')
self.assertEqual(d['yarn.nodemanager.resource.cpu-vcores'], '24')
def test_available_memory_entire_machine(self):
total_mem = 68719476736
node = dict(fqdn='hosty.domain.be', network='ib0', pid=1234,
cores=24, totalcores=24, usablecores=range(24), num_nodes=1,
memory=dict(meminfo=dict(memtotal=total_mem), ulimit='unlimited'))
avail = total_mem - hcc.parse_memory('8g')
self.assertEqual(hcc.available_memory(node), avail)
def test_yarn_site_xml_defaults(self):
node = dict(fqdn='hosty.domain.be', network='ib0', pid=1234,
cores=24, totalcores=24, usablecores=range(24), num_nodes=1,
memory=dict(meminfo=dict(memtotal=68719476736), ulimit='unlimited'))
d = hca.yarn_site_xml_defaults('/', node)
self.assertEqual(len(d), 16)
self.assertEqual(d['yarn.nodemanager.resource.memory-mb'], hcc.round_mb(hcc.parse_memory('56G')))
self.assertEqual(d['yarn.resourcemanager.webapp.address'], '$masterhostaddress:8088')
self.assertEqual(d['yarn.resourcemanager.webapp.https.address'], '$masterhostaddress:8090')
self.assertEqual(d['yarn.nodemanager.hostname'], '$dataname')
self.assertEqual(d['yarn.nodemanager.webapp.address'], '$hostaddress:8042')
self.assertEqual(d['yarn.scheduler.minimum-allocation-mb'], hcc.round_mb(hcc.parse_memory('2G')))
self.assertEqual(d['yarn.scheduler.maximum-allocation-mb'], hcc.round_mb(hcc.parse_memory('56G')))
self.assertEqual(d['yarn.scheduler.maximum-allocation-vcores'], '24')
self.assertEqual(d['yarn.scheduler.minimum-allocation-vcores'], '1')
self.assertEqual(d['yarn.nodemanager.resource.cpu-vcores'], '24')
def test_reserved_memory(self):
self.assertEqual(hcc.reserved_memory(1024**0), hcc.parse_memory('2g'))
self.assertEqual(hcc.reserved_memory(1024**1), hcc.parse_memory('2g'))
self.assertEqual(hcc.reserved_memory(1024**2), hcc.parse_memory('2g'))
self.assertEqual(hcc.reserved_memory(1024**2), hcc.parse_memory('2g'))
self.assertEqual(hcc.reserved_memory(1024**3), hcc.parse_memory('2g'))
self.assertEqual(hcc.reserved_memory(1024**4), hcc.parse_memory('64g'))
self.assertEqual(hcc.reserved_memory(1024**5), hcc.parse_memory('64g'))
def test_reserved_memory(self):
self.assertEqual(hcc.reserved_memory(1024**0), hcc.parse_memory('2g'))
self.assertEqual(hcc.reserved_memory(1024**1), hcc.parse_memory('2g'))
self.assertEqual(hcc.reserved_memory(1024**2), hcc.parse_memory('2g'))
self.assertEqual(hcc.reserved_memory(1024**2), hcc.parse_memory('2g'))
self.assertEqual(hcc.reserved_memory(1024**3), hcc.parse_memory('2g'))
self.assertEqual(hcc.reserved_memory(1024**4), hcc.parse_memory('64g'))
self.assertEqual(hcc.reserved_memory(1024**5), hcc.parse_memory('64g'))
def test_available_memory_one_third(self):
total_mem = 68719476736
node = dict(fqdn='hosty.domain.be', network='ib0', pid=1234,
cores=8, totalcores=24, usablecores=range(24), num_nodes=1,
memory=dict(meminfo=dict(memtotal=total_mem), ulimit='unlimited'))
avail = total_mem - hcc.parse_memory('8g')
self.assertEqual(hcc.available_memory(node), int(avail * 1./3))
node['cores'] = 12
self.assertEqual(hcc.available_memory(node), int(avail * 1./2))
node['cores'] = 1
self.assertEqual(hcc.available_memory(node), int(avail * 1./24))
def test_available_memory_entire_machine(self):
total_mem = 68719476736
node = dict(fqdn='hosty.domain.be',
network='ib0',
pid=1234,
cores=24,
totalcores=24,
usablecores=range(24),
num_nodes=1,
memory=dict(meminfo=dict(memtotal=total_mem),
ulimit='unlimited'))
avail = total_mem - hcc.parse_memory('8g')
self.assertEqual(hcc.available_memory(node), avail)
def test_yarn_site_xml_defaults(self):
node = dict(
fqdn="hosty.domain.be",
network="ib0",
pid=1234,
cores=24,
totalcores=24,
usablecores=range(24),
num_nodes=1,
memory=dict(meminfo=dict(memtotal=68719476736), ulimit="unlimited"),
)
d = hca.yarn_site_xml_defaults("/", node)
self.assertEqual(len(d), 16)
self.assertEqual(d["yarn.nodemanager.resource.memory-mb"], hcc.round_mb(hcc.parse_memory("56G")))
self.assertEqual(d["yarn.resourcemanager.webapp.address"], "$masterhostaddress:8088")
self.assertEqual(d["yarn.resourcemanager.webapp.https.address"], "$masterhostaddress:8090")
self.assertEqual(d["yarn.nodemanager.hostname"], "$dataname")
self.assertEqual(d["yarn.nodemanager.webapp.address"], "$hostaddress:8042")
self.assertEqual(d["yarn.scheduler.minimum-allocation-mb"], hcc.round_mb(hcc.parse_memory("2G")))
self.assertEqual(d["yarn.scheduler.maximum-allocation-mb"], hcc.round_mb(hcc.parse_memory("56G")))
self.assertEqual(d["yarn.scheduler.maximum-allocation-vcores"], "24")
self.assertEqual(d["yarn.scheduler.minimum-allocation-vcores"], "1")
self.assertEqual(d["yarn.nodemanager.resource.cpu-vcores"], "24")
def test_available_memory_one_third(self):
total_mem = 68719476736
node = dict(fqdn='hosty.domain.be',
network='ib0',
pid=1234,
cores=8,
totalcores=24,
usablecores=range(24),
num_nodes=1,
memory=dict(meminfo=dict(memtotal=total_mem),
ulimit='unlimited'))
avail = total_mem - hcc.parse_memory('8g')
self.assertEqual(hcc.available_memory(node), int(avail * 1. / 3))
node['cores'] = 12
self.assertEqual(hcc.available_memory(node), int(avail * 1. / 2))
node['cores'] = 1
self.assertEqual(hcc.available_memory(node), int(avail * 1. / 24))
def test_parse_memory(self):
self.assertEqual(hcc.parse_memory('1'), 1 * (1024**0))
self.assertEqual(hcc.parse_memory('1b'), 1 * (1024**0))
self.assertEqual(hcc.parse_memory('1B'), 1 * (1024**0))
self.assertEqual(hcc.parse_memory('2k'), 2 * (1024**1))
self.assertEqual(hcc.parse_memory('2kb'), 2 * (1024**1))
self.assertEqual(hcc.parse_memory('2kB'), 2 * (1024**1))
self.assertEqual(hcc.parse_memory('2KB'), 2 * (1024**1))
self.assertEqual(hcc.parse_memory('2K'), 2 * (1024**1))
self.assertEqual(hcc.parse_memory('3m'), 3 * (1024**2))
self.assertEqual(hcc.parse_memory('3M'), 3 * (1024**2))
self.assertEqual(hcc.parse_memory('4g'), 4 * (1024**3))
self.assertEqual(hcc.parse_memory('4G'), 4 * (1024**3))
self.assertEqual(hcc.parse_memory('5t'), 5 * (1024**4))
self.assertEqual(hcc.parse_memory('5T'), 5 * (1024**4))
self.assertEqual(hcc.parse_memory('5T'), 5 * (1024**4))
self.assertEqual(hcc.parse_memory('2.5G'), 2.5 * (1024**3))
self.assertEqual(hcc.parse_memory('0.5T'), 512 * (1024**3))
self.assertRaises(RuntimeError, hcc.parse_memory, '6p')
self.assertRaises(RuntimeError, hcc.parse_memory, '6P')
def test_parse_memory(self):
self.assertEqual(hcc.parse_memory('1'), 1 * (1024**0))
self.assertEqual(hcc.parse_memory('1b'), 1 * (1024**0))
self.assertEqual(hcc.parse_memory('1B'), 1 * (1024**0))
self.assertEqual(hcc.parse_memory('2k'), 2 * (1024**1))
self.assertEqual(hcc.parse_memory('2kb'), 2 * (1024**1))
self.assertEqual(hcc.parse_memory('2kB'), 2 * (1024**1))
self.assertEqual(hcc.parse_memory('2KB'), 2 * (1024**1))
self.assertEqual(hcc.parse_memory('2K'), 2 * (1024**1))
self.assertEqual(hcc.parse_memory('3m'), 3 * (1024**2))
self.assertEqual(hcc.parse_memory('3M'), 3 * (1024**2))
self.assertEqual(hcc.parse_memory('4g'), 4 * (1024**3))
self.assertEqual(hcc.parse_memory('4G'), 4 * (1024**3))
self.assertEqual(hcc.parse_memory('5t'), 5 * (1024**4))
self.assertEqual(hcc.parse_memory('5T'), 5 * (1024**4))
self.assertEqual(hcc.parse_memory('5T'), 5 * (1024**4))
self.assertEqual(hcc.parse_memory('2.5G'), 2.5 * (1024**3))
self.assertEqual(hcc.parse_memory('0.5T'), 512 * (1024**3))
self.assertRaises(RuntimeError, hcc.parse_memory, '6p')
self.assertRaises(RuntimeError, hcc.parse_memory, '6P')