def yarn_site_xml_defaults(workdir, node_info): dflts = { # Hadoop-on-lustre2 wants to find dirs with # $workdir/$(nodemanager-hostname) which in our case is 'dataname' 'yarn.nodemanager.local-dirs': '$workdir/$dataname', } base_hadoop_dflts = hcah.yarn_site_xml_defaults(workdir, node_info) base_hadoop_dflts.update(dflts) return base_hadoop_dflts
def test_yarn_site_xml_defaults(self): node = dict(fqdn='hosty.domain.be', network='ib0', pid=1234, cores=24, totalcores=24, usablecores=range(24), num_nodes=1, memory=dict(meminfo=dict(memtotal=68719476736), ulimit='unlimited')) d = hca.yarn_site_xml_defaults('/', node) self.assertEqual(len(d), 16) self.assertEqual(d['yarn.nodemanager.resource.memory-mb'], hcc.round_mb(hcc.parse_memory('56G'))) self.assertEqual(d['yarn.resourcemanager.webapp.address'], '$masterhostaddress:8088') self.assertEqual(d['yarn.resourcemanager.webapp.https.address'], '$masterhostaddress:8090') self.assertEqual(d['yarn.nodemanager.hostname'], '$dataname') self.assertEqual(d['yarn.nodemanager.webapp.address'], '$hostaddress:8042') self.assertEqual(d['yarn.scheduler.minimum-allocation-mb'], hcc.round_mb(hcc.parse_memory('2G'))) self.assertEqual(d['yarn.scheduler.maximum-allocation-mb'], hcc.round_mb(hcc.parse_memory('56G'))) self.assertEqual(d['yarn.scheduler.maximum-allocation-vcores'], '24') self.assertEqual(d['yarn.scheduler.minimum-allocation-vcores'], '1') self.assertEqual(d['yarn.nodemanager.resource.cpu-vcores'], '24')
def test_yarn_site_xml_defaults(self): node = dict( fqdn="hosty.domain.be", network="ib0", pid=1234, cores=24, totalcores=24, usablecores=range(24), num_nodes=1, memory=dict(meminfo=dict(memtotal=68719476736), ulimit="unlimited"), ) d = hca.yarn_site_xml_defaults("/", node) self.assertEqual(len(d), 16) self.assertEqual(d["yarn.nodemanager.resource.memory-mb"], hcc.round_mb(hcc.parse_memory("56G"))) self.assertEqual(d["yarn.resourcemanager.webapp.address"], "$masterhostaddress:8088") self.assertEqual(d["yarn.resourcemanager.webapp.https.address"], "$masterhostaddress:8090") self.assertEqual(d["yarn.nodemanager.hostname"], "$dataname") self.assertEqual(d["yarn.nodemanager.webapp.address"], "$hostaddress:8042") self.assertEqual(d["yarn.scheduler.minimum-allocation-mb"], hcc.round_mb(hcc.parse_memory("2G"))) self.assertEqual(d["yarn.scheduler.maximum-allocation-mb"], hcc.round_mb(hcc.parse_memory("56G"))) self.assertEqual(d["yarn.scheduler.maximum-allocation-vcores"], "24") self.assertEqual(d["yarn.scheduler.minimum-allocation-vcores"], "1") self.assertEqual(d["yarn.nodemanager.resource.cpu-vcores"], "24")