Python yarn_site_xml_defaults Examples

Example #1

File: hadoop_on_lustre2.py Project: boegel/hanythingondemand

def yarn_site_xml_defaults(workdir, node_info):
    dflts = {
        # Hadoop-on-lustre2 wants to find dirs with
        # $workdir/$(nodemanager-hostname) which in our case is 'dataname'
        'yarn.nodemanager.local-dirs': '$workdir/$dataname',
    }
    base_hadoop_dflts = hcah.yarn_site_xml_defaults(workdir, node_info)
    base_hadoop_dflts.update(dflts)
    return base_hadoop_dflts

Example #2

File: hadoop_on_lustre2.py Project: MatthieuSala/hanythingondemand

def yarn_site_xml_defaults(workdir, node_info):
    dflts = {
        # Hadoop-on-lustre2 wants to find dirs with
        # $workdir/$(nodemanager-hostname) which in our case is 'dataname'
        'yarn.nodemanager.local-dirs': '$workdir/$dataname',
    }
    base_hadoop_dflts = hcah.yarn_site_xml_defaults(workdir, node_info)
    base_hadoop_dflts.update(dflts)
    return base_hadoop_dflts

Example #3

 def test_yarn_site_xml_defaults(self):
     node = dict(fqdn='hosty.domain.be', network='ib0', pid=1234,
             cores=24, totalcores=24, usablecores=range(24), num_nodes=1,
             memory=dict(meminfo=dict(memtotal=68719476736), ulimit='unlimited'))
     d = hca.yarn_site_xml_defaults('/', node)
     self.assertEqual(len(d), 16)
     self.assertEqual(d['yarn.nodemanager.resource.memory-mb'], hcc.round_mb(hcc.parse_memory('56G')))
     self.assertEqual(d['yarn.resourcemanager.webapp.address'], '$masterhostaddress:8088')
     self.assertEqual(d['yarn.resourcemanager.webapp.https.address'], '$masterhostaddress:8090')
     self.assertEqual(d['yarn.nodemanager.hostname'], '$dataname')
     self.assertEqual(d['yarn.nodemanager.webapp.address'], '$hostaddress:8042')
     self.assertEqual(d['yarn.scheduler.minimum-allocation-mb'], hcc.round_mb(hcc.parse_memory('2G')))
     self.assertEqual(d['yarn.scheduler.maximum-allocation-mb'], hcc.round_mb(hcc.parse_memory('56G')))
     self.assertEqual(d['yarn.scheduler.maximum-allocation-vcores'], '24')
     self.assertEqual(d['yarn.scheduler.minimum-allocation-vcores'], '1')
     self.assertEqual(d['yarn.nodemanager.resource.cpu-vcores'], '24')

Example #4

File: test_autogen_hadoop.py Project: ehiggs/hanythingondemand

 def test_yarn_site_xml_defaults(self):
     node = dict(fqdn='hosty.domain.be', network='ib0', pid=1234,
             cores=24, totalcores=24, usablecores=range(24), num_nodes=1,
             memory=dict(meminfo=dict(memtotal=68719476736), ulimit='unlimited'))
     d = hca.yarn_site_xml_defaults('/', node)
     self.assertEqual(len(d), 16)
     self.assertEqual(d['yarn.nodemanager.resource.memory-mb'], hcc.round_mb(hcc.parse_memory('56G')))
     self.assertEqual(d['yarn.resourcemanager.webapp.address'], '$masterhostaddress:8088')
     self.assertEqual(d['yarn.resourcemanager.webapp.https.address'], '$masterhostaddress:8090')
     self.assertEqual(d['yarn.nodemanager.hostname'], '$dataname')
     self.assertEqual(d['yarn.nodemanager.webapp.address'], '$hostaddress:8042')
     self.assertEqual(d['yarn.scheduler.minimum-allocation-mb'], hcc.round_mb(hcc.parse_memory('2G')))
     self.assertEqual(d['yarn.scheduler.maximum-allocation-mb'], hcc.round_mb(hcc.parse_memory('56G')))
     self.assertEqual(d['yarn.scheduler.maximum-allocation-vcores'], '24')
     self.assertEqual(d['yarn.scheduler.minimum-allocation-vcores'], '1')
     self.assertEqual(d['yarn.nodemanager.resource.cpu-vcores'], '24')

Example #5

File: test_autogen_hadoop.py Project: hpcugent/hanythingondemand

 def test_yarn_site_xml_defaults(self):
     node = dict(
         fqdn="hosty.domain.be",
         network="ib0",
         pid=1234,
         cores=24,
         totalcores=24,
         usablecores=range(24),
         num_nodes=1,
         memory=dict(meminfo=dict(memtotal=68719476736), ulimit="unlimited"),
     )
     d = hca.yarn_site_xml_defaults("/", node)
     self.assertEqual(len(d), 16)
     self.assertEqual(d["yarn.nodemanager.resource.memory-mb"], hcc.round_mb(hcc.parse_memory("56G")))
     self.assertEqual(d["yarn.resourcemanager.webapp.address"], "$masterhostaddress:8088")
     self.assertEqual(d["yarn.resourcemanager.webapp.https.address"], "$masterhostaddress:8090")
     self.assertEqual(d["yarn.nodemanager.hostname"], "$dataname")
     self.assertEqual(d["yarn.nodemanager.webapp.address"], "$hostaddress:8042")
     self.assertEqual(d["yarn.scheduler.minimum-allocation-mb"], hcc.round_mb(hcc.parse_memory("2G")))
     self.assertEqual(d["yarn.scheduler.maximum-allocation-mb"], hcc.round_mb(hcc.parse_memory("56G")))
     self.assertEqual(d["yarn.scheduler.maximum-allocation-vcores"], "24")
     self.assertEqual(d["yarn.scheduler.minimum-allocation-vcores"], "1")
     self.assertEqual(d["yarn.nodemanager.resource.cpu-vcores"], "24")