Python parse_memory Examples

Example #1

File: test_ipython_notebook.py Project: ehiggs/hanythingondemand

 def test_spark_defaults_single_core(self):
     node = dict(fqdn='hosty.domain.be', network='ib0', pid=1234,
                 cores=1, totalcores=1, usablecores=range(1), num_nodes=1,
                 memory=dict(meminfo=dict(memtotal=68719476736), ulimit='unlimited'))
     dflts = hcip.spark_defaults(None, node)
     self.assertTrue(len(dflts), 3)
     self.assertEqual(dflts['spark.executor.instances'], 1)
     self.assertEqual(dflts['spark.executor.cores'], 1)
     self.assertEqual(hcc.parse_memory(dflts['spark.executor.memory']), hcc.parse_memory('56G'))

Example #2

File: test_autogen_hadoop.py Project: ehiggs/hanythingondemand

 def test_mapred_site_xml_defaults(self):
     node = dict(fqdn='hosty.domain.be', network='ib0', pid=1234,
             cores=24, totalcores=24, usablecores=range(24), num_nodes=1,
             memory=dict(meminfo=dict(memtotal=68719476736), ulimit='unlimited'))
     d = hca.mapred_site_xml_defaults('/', node)
     self.assertEqual(len(d), 7)
     # Capped at 8g
     self.assertEqual(d['mapreduce.map.memory.mb'], hcc.round_mb(hcc.parse_memory('2G')))
     self.assertEqual(d['mapreduce.reduce.memory.mb'], hcc.round_mb(hcc.parse_memory('4G')))

Example #3

 def test_mapred_site_xml_defaults(self):
     node = dict(fqdn='hosty.domain.be', network='ib0', pid=1234,
             cores=24, totalcores=24, usablecores=range(24), num_nodes=1,
             memory=dict(meminfo=dict(memtotal=68719476736), ulimit='unlimited'))
     d = hca.mapred_site_xml_defaults('/', node)
     self.assertEqual(len(d), 7)
     # Capped at 8g
     self.assertEqual(d['mapreduce.map.memory.mb'], hcc.round_mb(hcc.parse_memory('2G')))
     self.assertEqual(d['mapreduce.reduce.memory.mb'], hcc.round_mb(hcc.parse_memory('4G')))

Example #4

File: test_ipython_notebook.py Project: MatthieuSala/hanythingondemand

 def test_spark_defaults(self):
     node = dict(fqdn='hosty.domain.be', network='ib0', pid=1234,
                 cores=16, totalcores=16, usablecores=range(16), num_nodes=6,
                 memory=dict(meminfo=dict(memtotal=68719476736), ulimit='unlimited'))
     dflts = hcip.spark_defaults(None, node)
     self.assertTrue(len(dflts), 3)
     self.assertEqual(dflts['spark.executor.instances'], 47) # 6 nodes * 8 cores -1 core used for driver
     self.assertEqual(dflts['spark.executor.cores'], 2)
     self.assertEqual(hcc.parse_memory(dflts['spark.executor.memory']), hcc.parse_memory('7G'))

Example #5

File: test_ipython_notebook.py Project: ehiggs/hanythingondemand

 def test_spark_defaults(self):
     node = dict(fqdn='hosty.domain.be', network='ib0', pid=1234,
                 cores=16, totalcores=16, usablecores=range(16), num_nodes=6,
                 memory=dict(meminfo=dict(memtotal=68719476736), ulimit='unlimited'))
     dflts = hcip.spark_defaults(None, node)
     self.assertTrue(len(dflts), 3)
     self.assertEqual(dflts['spark.executor.instances'], 47) # 6 nodes * 8 cores -1 core used for driver
     self.assertEqual(dflts['spark.executor.cores'], 2)
     self.assertEqual(hcc.parse_memory(dflts['spark.executor.memory']), hcc.parse_memory('7G'))

Example #6

File: test_ipython_notebook.py Project: MatthieuSala/hanythingondemand

 def test_spark_defaults_single_core(self):
     node = dict(fqdn='hosty.domain.be', network='ib0', pid=1234,
                 cores=1, totalcores=1, usablecores=range(1), num_nodes=1,
                 memory=dict(meminfo=dict(memtotal=68719476736), ulimit='unlimited'))
     dflts = hcip.spark_defaults(None, node)
     self.assertTrue(len(dflts), 3)
     self.assertEqual(dflts['spark.executor.instances'], 1)
     self.assertEqual(dflts['spark.executor.cores'], 1)
     self.assertEqual(hcc.parse_memory(dflts['spark.executor.memory']), hcc.parse_memory('56G'))

Example #7

File: test_autogen_hadoop_on_lustre2.py Project: ehiggs/hanythingondemand

 def test_mapred_site_xml_defaults(self):
     '''Test mapred defaults; note: only using 4 from 24 cores.'''
     node = dict(fqdn='hosty.domain.be', network='ib0', pid=1234,
             cores=4, totalcores=24, usablecores=[0, 1, 2, 3], num_nodes=1,
             memory=dict(meminfo=dict(memtotal=68719476736), ulimit='unlimited'))
     d = hca.mapred_site_xml_defaults('/', node)
     self.assertEqual(len(d), 9)
     self.assertEqual(d['hadoop.ln.cmd'], '/bin/ln')
     self.assertEqual(d['lustre.dir'], '$workdir')
     self.assertEqual(d['mapreduce.map.memory.mb'], hcc.round_mb(hcc.parse_memory('1G')))
     self.assertEqual(d['mapreduce.reduce.memory.mb'], hcc.round_mb(hcc.parse_memory('2G')))

Example #8

File: test_autogen_hadoop_on_lustre2.py Project: MatthieuSala/hanythingondemand

 def test_mapred_site_xml_defaults(self):
     '''Test mapred defaults; note: only using 4 from 24 cores.'''
     node = dict(fqdn='hosty.domain.be', network='ib0', pid=1234,
             cores=4, totalcores=24, usablecores=[0, 1, 2, 3], num_nodes=1,
             memory=dict(meminfo=dict(memtotal=68719476736), ulimit='unlimited'))
     d = hca.mapred_site_xml_defaults('/', node)
     self.assertEqual(len(d), 9)
     self.assertEqual(d['hadoop.ln.cmd'], '/bin/ln')
     self.assertEqual(d['lustre.dir'], '$workdir')
     self.assertEqual(d['mapreduce.map.memory.mb'], hcc.round_mb(hcc.parse_memory('1G')))
     self.assertEqual(d['mapreduce.reduce.memory.mb'], hcc.round_mb(hcc.parse_memory('2G')))

Example #9

File: hbase.py Project: boegel/hanythingondemand

def min_container_size(totalmem):
    '''
    Given an amount of memory in bytes, return the amount of memory that
    represents the minimum amount of memory (in bytes) to be allocated in a Yarn
    container.
    '''
    if totalmem < parse_memory('4G'):
        return  256 * (1024**2)
    elif totalmem < parse_memory('8G'):
        return 512 * (1024**2)
    elif totalmem < parse_memory('24G'):
        return 1024**3
    else:
        return 2 * (1024**3)

Example #10

def min_container_size(totalmem):
    '''
    Given an amount of memory in bytes, return the amount of memory that
    represents the minimum amount of memory (in bytes) to be allocated in a Yarn
    container.
    '''
    if totalmem < parse_memory('4G'):
        return 256 * (1024**2)
    elif totalmem < parse_memory('8G'):
        return 512 * (1024**2)
    elif totalmem < parse_memory('24G'):
        return 1024**3
    else:
        return 2 * (1024**3)

Example #11

File: test_autogen_hadoop.py Project: hpcugent/hanythingondemand

 def test_mapred_site_xml_defaults(self):
     node = dict(
         fqdn="hosty.domain.be",
         network="ib0",
         pid=1234,
         cores=24,
         totalcores=24,
         usablecores=range(24),
         num_nodes=1,
         memory=dict(meminfo=dict(memtotal=68719476736), ulimit="unlimited"),
     )
     d = hca.mapred_site_xml_defaults("/", node)
     self.assertEqual(len(d), 7)
     # Capped at 8g
     self.assertEqual(d["mapreduce.map.memory.mb"], hcc.round_mb(hcc.parse_memory("2G")))
     self.assertEqual(d["mapreduce.reduce.memory.mb"], hcc.round_mb(hcc.parse_memory("4G")))

Example #12

File: test_autogen_hadoop.py Project: ehiggs/hanythingondemand

 def test_yarn_site_xml_defaults(self):
     node = dict(fqdn='hosty.domain.be', network='ib0', pid=1234,
             cores=24, totalcores=24, usablecores=range(24), num_nodes=1,
             memory=dict(meminfo=dict(memtotal=68719476736), ulimit='unlimited'))
     d = hca.yarn_site_xml_defaults('/', node)
     self.assertEqual(len(d), 16)
     self.assertEqual(d['yarn.nodemanager.resource.memory-mb'], hcc.round_mb(hcc.parse_memory('56G')))
     self.assertEqual(d['yarn.resourcemanager.webapp.address'], '$masterhostaddress:8088')
     self.assertEqual(d['yarn.resourcemanager.webapp.https.address'], '$masterhostaddress:8090')
     self.assertEqual(d['yarn.nodemanager.hostname'], '$dataname')
     self.assertEqual(d['yarn.nodemanager.webapp.address'], '$hostaddress:8042')
     self.assertEqual(d['yarn.scheduler.minimum-allocation-mb'], hcc.round_mb(hcc.parse_memory('2G')))
     self.assertEqual(d['yarn.scheduler.maximum-allocation-mb'], hcc.round_mb(hcc.parse_memory('56G')))
     self.assertEqual(d['yarn.scheduler.maximum-allocation-vcores'], '24')
     self.assertEqual(d['yarn.scheduler.minimum-allocation-vcores'], '1')
     self.assertEqual(d['yarn.nodemanager.resource.cpu-vcores'], '24')

Example #13

File: test_autogen_common.py Project: ehiggs/hanythingondemand

 def test_available_memory_entire_machine(self):
     total_mem = 68719476736
     node = dict(fqdn='hosty.domain.be', network='ib0', pid=1234,
             cores=24, totalcores=24, usablecores=range(24), num_nodes=1,
             memory=dict(meminfo=dict(memtotal=total_mem), ulimit='unlimited'))
     avail = total_mem - hcc.parse_memory('8g')
     self.assertEqual(hcc.available_memory(node), avail)

Example #14

 def test_yarn_site_xml_defaults(self):
     node = dict(fqdn='hosty.domain.be', network='ib0', pid=1234,
             cores=24, totalcores=24, usablecores=range(24), num_nodes=1,
             memory=dict(meminfo=dict(memtotal=68719476736), ulimit='unlimited'))
     d = hca.yarn_site_xml_defaults('/', node)
     self.assertEqual(len(d), 16)
     self.assertEqual(d['yarn.nodemanager.resource.memory-mb'], hcc.round_mb(hcc.parse_memory('56G')))
     self.assertEqual(d['yarn.resourcemanager.webapp.address'], '$masterhostaddress:8088')
     self.assertEqual(d['yarn.resourcemanager.webapp.https.address'], '$masterhostaddress:8090')
     self.assertEqual(d['yarn.nodemanager.hostname'], '$dataname')
     self.assertEqual(d['yarn.nodemanager.webapp.address'], '$hostaddress:8042')
     self.assertEqual(d['yarn.scheduler.minimum-allocation-mb'], hcc.round_mb(hcc.parse_memory('2G')))
     self.assertEqual(d['yarn.scheduler.maximum-allocation-mb'], hcc.round_mb(hcc.parse_memory('56G')))
     self.assertEqual(d['yarn.scheduler.maximum-allocation-vcores'], '24')
     self.assertEqual(d['yarn.scheduler.minimum-allocation-vcores'], '1')
     self.assertEqual(d['yarn.nodemanager.resource.cpu-vcores'], '24')

Example #15

 def test_reserved_memory(self):
     self.assertEqual(hcc.reserved_memory(1024**0), hcc.parse_memory('2g'))
     self.assertEqual(hcc.reserved_memory(1024**1), hcc.parse_memory('2g'))
     self.assertEqual(hcc.reserved_memory(1024**2), hcc.parse_memory('2g'))
     self.assertEqual(hcc.reserved_memory(1024**2), hcc.parse_memory('2g'))
     self.assertEqual(hcc.reserved_memory(1024**3), hcc.parse_memory('2g'))
     self.assertEqual(hcc.reserved_memory(1024**4), hcc.parse_memory('64g'))
     self.assertEqual(hcc.reserved_memory(1024**5), hcc.parse_memory('64g'))

Example #16

File: test_autogen_common.py Project: ehiggs/hanythingondemand

 def test_reserved_memory(self):
     self.assertEqual(hcc.reserved_memory(1024**0), hcc.parse_memory('2g'))
     self.assertEqual(hcc.reserved_memory(1024**1), hcc.parse_memory('2g'))
     self.assertEqual(hcc.reserved_memory(1024**2), hcc.parse_memory('2g'))
     self.assertEqual(hcc.reserved_memory(1024**2), hcc.parse_memory('2g'))
     self.assertEqual(hcc.reserved_memory(1024**3), hcc.parse_memory('2g'))
     self.assertEqual(hcc.reserved_memory(1024**4), hcc.parse_memory('64g'))
     self.assertEqual(hcc.reserved_memory(1024**5), hcc.parse_memory('64g'))

Example #17

File: test_autogen_common.py Project: ehiggs/hanythingondemand

 def test_available_memory_one_third(self):
     total_mem = 68719476736
     node = dict(fqdn='hosty.domain.be', network='ib0', pid=1234,
             cores=8, totalcores=24, usablecores=range(24), num_nodes=1,
             memory=dict(meminfo=dict(memtotal=total_mem), ulimit='unlimited'))
     avail = total_mem - hcc.parse_memory('8g')
     self.assertEqual(hcc.available_memory(node), int(avail * 1./3))
     node['cores'] = 12
     self.assertEqual(hcc.available_memory(node), int(avail * 1./2))
     node['cores'] = 1 
     self.assertEqual(hcc.available_memory(node), int(avail * 1./24))

Example #18

 def test_available_memory_entire_machine(self):
     total_mem = 68719476736
     node = dict(fqdn='hosty.domain.be',
                 network='ib0',
                 pid=1234,
                 cores=24,
                 totalcores=24,
                 usablecores=range(24),
                 num_nodes=1,
                 memory=dict(meminfo=dict(memtotal=total_mem),
                             ulimit='unlimited'))
     avail = total_mem - hcc.parse_memory('8g')
     self.assertEqual(hcc.available_memory(node), avail)

Example #19

File: test_autogen_hadoop.py Project: hpcugent/hanythingondemand

 def test_yarn_site_xml_defaults(self):
     node = dict(
         fqdn="hosty.domain.be",
         network="ib0",
         pid=1234,
         cores=24,
         totalcores=24,
         usablecores=range(24),
         num_nodes=1,
         memory=dict(meminfo=dict(memtotal=68719476736), ulimit="unlimited"),
     )
     d = hca.yarn_site_xml_defaults("/", node)
     self.assertEqual(len(d), 16)
     self.assertEqual(d["yarn.nodemanager.resource.memory-mb"], hcc.round_mb(hcc.parse_memory("56G")))
     self.assertEqual(d["yarn.resourcemanager.webapp.address"], "$masterhostaddress:8088")
     self.assertEqual(d["yarn.resourcemanager.webapp.https.address"], "$masterhostaddress:8090")
     self.assertEqual(d["yarn.nodemanager.hostname"], "$dataname")
     self.assertEqual(d["yarn.nodemanager.webapp.address"], "$hostaddress:8042")
     self.assertEqual(d["yarn.scheduler.minimum-allocation-mb"], hcc.round_mb(hcc.parse_memory("2G")))
     self.assertEqual(d["yarn.scheduler.maximum-allocation-mb"], hcc.round_mb(hcc.parse_memory("56G")))
     self.assertEqual(d["yarn.scheduler.maximum-allocation-vcores"], "24")
     self.assertEqual(d["yarn.scheduler.minimum-allocation-vcores"], "1")
     self.assertEqual(d["yarn.nodemanager.resource.cpu-vcores"], "24")

Example #20

 def test_available_memory_one_third(self):
     total_mem = 68719476736
     node = dict(fqdn='hosty.domain.be',
                 network='ib0',
                 pid=1234,
                 cores=8,
                 totalcores=24,
                 usablecores=range(24),
                 num_nodes=1,
                 memory=dict(meminfo=dict(memtotal=total_mem),
                             ulimit='unlimited'))
     avail = total_mem - hcc.parse_memory('8g')
     self.assertEqual(hcc.available_memory(node), int(avail * 1. / 3))
     node['cores'] = 12
     self.assertEqual(hcc.available_memory(node), int(avail * 1. / 2))
     node['cores'] = 1
     self.assertEqual(hcc.available_memory(node), int(avail * 1. / 24))

Example #21

File: test_autogen_common.py Project: ehiggs/hanythingondemand

    def test_parse_memory(self):
        self.assertEqual(hcc.parse_memory('1'), 1 * (1024**0))
        self.assertEqual(hcc.parse_memory('1b'), 1 * (1024**0))
        self.assertEqual(hcc.parse_memory('1B'), 1 * (1024**0))
        self.assertEqual(hcc.parse_memory('2k'), 2 * (1024**1))
        self.assertEqual(hcc.parse_memory('2kb'), 2 * (1024**1))
        self.assertEqual(hcc.parse_memory('2kB'), 2 * (1024**1))
        self.assertEqual(hcc.parse_memory('2KB'), 2 * (1024**1))
        self.assertEqual(hcc.parse_memory('2K'), 2 * (1024**1))
        self.assertEqual(hcc.parse_memory('3m'), 3 * (1024**2))
        self.assertEqual(hcc.parse_memory('3M'), 3 * (1024**2))
        self.assertEqual(hcc.parse_memory('4g'), 4 * (1024**3))
        self.assertEqual(hcc.parse_memory('4G'), 4 * (1024**3))
        self.assertEqual(hcc.parse_memory('5t'), 5 * (1024**4))
        self.assertEqual(hcc.parse_memory('5T'), 5 * (1024**4))
        self.assertEqual(hcc.parse_memory('5T'), 5 * (1024**4))

        self.assertEqual(hcc.parse_memory('2.5G'), 2.5 * (1024**3))
        self.assertEqual(hcc.parse_memory('0.5T'), 512 * (1024**3))

        self.assertRaises(RuntimeError, hcc.parse_memory, '6p')
        self.assertRaises(RuntimeError, hcc.parse_memory, '6P')

Example #22

    def test_parse_memory(self):
        self.assertEqual(hcc.parse_memory('1'), 1 * (1024**0))
        self.assertEqual(hcc.parse_memory('1b'), 1 * (1024**0))
        self.assertEqual(hcc.parse_memory('1B'), 1 * (1024**0))
        self.assertEqual(hcc.parse_memory('2k'), 2 * (1024**1))
        self.assertEqual(hcc.parse_memory('2kb'), 2 * (1024**1))
        self.assertEqual(hcc.parse_memory('2kB'), 2 * (1024**1))
        self.assertEqual(hcc.parse_memory('2KB'), 2 * (1024**1))
        self.assertEqual(hcc.parse_memory('2K'), 2 * (1024**1))
        self.assertEqual(hcc.parse_memory('3m'), 3 * (1024**2))
        self.assertEqual(hcc.parse_memory('3M'), 3 * (1024**2))
        self.assertEqual(hcc.parse_memory('4g'), 4 * (1024**3))
        self.assertEqual(hcc.parse_memory('4G'), 4 * (1024**3))
        self.assertEqual(hcc.parse_memory('5t'), 5 * (1024**4))
        self.assertEqual(hcc.parse_memory('5T'), 5 * (1024**4))
        self.assertEqual(hcc.parse_memory('5T'), 5 * (1024**4))

        self.assertEqual(hcc.parse_memory('2.5G'), 2.5 * (1024**3))
        self.assertEqual(hcc.parse_memory('0.5T'), 512 * (1024**3))

        self.assertRaises(RuntimeError, hcc.parse_memory, '6p')
        self.assertRaises(RuntimeError, hcc.parse_memory, '6P')