def setUp(self):
self.rootDir = '/tmp/hod-%s' % getpass.getuser()
self.id = 0
self.desc = None
self.tempDir = os.path.join(self.rootDir, 'test_HadoopCommand_tempDir')
self.pkgDir = os.path.join(self.rootDir, 'test_HadoopCommand_pkgDir')
self.log = logging.getLogger() # TODO Use MockLogger
self.javaHome = '/usr/java/bin/'
self.mrSysDir = '/user/' + getpass.getuser() + '/mapredsystem'
self.attrs = {}
self.finalAttrs = {
'fs.default.name': 'nohost.apache.com:56366',
'mapred.child.java.opts': '-Xmx1024m',
'mapred.compress.map.output': 'false',
}
self.attrs = {
'mapred.userlog.limit': '200',
'mapred.userlog.retain.hours': '10',
'mapred.reduce.parallel.copies': '20',
}
self.desc = CommandDesc(
{
'name': 'dummyHadoop',
'program': 'bin/hadoop',
'pkgdirs': self.pkgDir,
'final-attrs': self.finalAttrs,
'attrs': self.attrs,
}, self.log)
# TODO
# 4th arg to HadoopCommand 'tardir' is not used at all. Instead pkgdir is
# specified through HadoopCommand.run(pkgdir). This could be changed so
# that pkgdir is specified at the time of object creation.
# END OF TODO
self.hadoopCommand = HadoopCommand(self.id,
self.desc,
self.tempDir,
self.pkgDir, (50000, 60000),
self.log,
self.javaHome,
self.mrSysDir,
restart=True)
self.hadoopSite = os.path.join(self.hadoopCommand.confdir,
'hadoop-site.xml')
pass
def setUp(self):
self.rootDir = '/tmp/hod-%s' % getpass.getuser()
self.id = 0
self.desc = None
self.tempDir = os.path.join(self.rootDir,'test_HadoopCommand_tempDir')
self.pkgDir = os.path.join(self.rootDir,'test_HadoopCommand_pkgDir')
self.log = logging.getLogger() # TODO Use MockLogger
self.javaHome = '/usr/java/bin/'
self.mrSysDir = '/user/' + getpass.getuser() + '/mapredsystem'
self.attrs = {}
self.finalAttrs = {
'fs.default.name': 'nohost.apache.com:56366',
'mapred.child.java.opts' : '-Xmx1024m',
'mapred.compress.map.output' : 'false',
}
self.attrs = {
'mapred.userlog.limit' : '200',
'mapred.userlog.retain.hours' : '10',
'mapred.reduce.parallel.copies' : '20',
}
self.desc = CommandDesc(
{
'name' : 'dummyHadoop',
'program' : 'bin/hadoop',
'pkgdirs' : self.pkgDir,
'final-attrs' : self.finalAttrs,
'attrs' : self.attrs,
}, self.log
)
# TODO
# 4th arg to HadoopCommand 'tardir' is not used at all. Instead pkgdir is
# specified through HadoopCommand.run(pkgdir). This could be changed so
# that pkgdir is specified at the time of object creation.
# END OF TODO
self.hadoopCommand = HadoopCommand(self.id, self.desc, self.tempDir,
self.pkgDir, self.log, self.javaHome,
self.mrSysDir, restart=True)
self.hadoopSite = os.path.join( self.hadoopCommand.confdir,
'hadoop-site.xml')
pass
class test_HadoopCommand(unittest.TestCase):
def setUp(self):
self.rootDir = '/tmp/hod-%s' % getpass.getuser()
self.id = 0
self.desc = None
self.tempDir = os.path.join(self.rootDir, 'test_HadoopCommand_tempDir')
self.pkgDir = os.path.join(self.rootDir, 'test_HadoopCommand_pkgDir')
self.log = logging.getLogger() # TODO Use MockLogger
self.javaHome = '/usr/java/bin/'
self.mrSysDir = '/user/' + getpass.getuser() + '/mapredsystem'
self.attrs = {}
self.finalAttrs = {
'fs.default.name': 'nohost.apache.com:56366',
'mapred.child.java.opts': '-Xmx1024m',
'mapred.compress.map.output': 'false',
}
self.attrs = {
'mapred.userlog.limit': '200',
'mapred.userlog.retain.hours': '10',
'mapred.reduce.parallel.copies': '20',
}
self.desc = CommandDesc(
{
'name': 'dummyHadoop',
'program': 'bin/hadoop',
'pkgdirs': self.pkgDir,
'final-attrs': self.finalAttrs,
'attrs': self.attrs,
}, self.log)
# TODO
# 4th arg to HadoopCommand 'tardir' is not used at all. Instead pkgdir is
# specified through HadoopCommand.run(pkgdir). This could be changed so
# that pkgdir is specified at the time of object creation.
# END OF TODO
self.hadoopCommand = HadoopCommand(self.id,
self.desc,
self.tempDir,
self.pkgDir, (50000, 60000),
self.log,
self.javaHome,
self.mrSysDir,
restart=True)
self.hadoopSite = os.path.join(self.hadoopCommand.confdir,
'hadoop-site.xml')
pass
def test_createHadoopSiteXml(self):
self.hadoopCommand._createHadoopSiteXml()
xmldoc = minidom.parse(self.hadoopSite)
xmldoc = xmldoc.childNodes[0] # leave out xml spec
properties = xmldoc.childNodes # children of tag configuration
keyvals = {}
for prop in properties:
if not isinstance(prop, minidom.Comment):
# ---------- tag -------------------- -value elem-- data --
name = prop.getElementsByTagName('name')[0].childNodes[0].data
value = prop.getElementsByTagName(
'value')[0].childNodes[0].data
keyvals[name] = value
# fs.default.name should start with hdfs://
assert (keyvals['fs.default.name'].startswith('hdfs://'))
# TODO other tests
pass
def tearDown(self):
pass
class test_HadoopCommand(unittest.TestCase):
def setUp(self):
self.rootDir = '/tmp/hod-%s' % getpass.getuser()
self.id = 0
self.desc = None
self.tempDir = os.path.join(self.rootDir,'test_HadoopCommand_tempDir')
self.pkgDir = os.path.join(self.rootDir,'test_HadoopCommand_pkgDir')
self.log = logging.getLogger() # TODO Use MockLogger
self.javaHome = '/usr/java/bin/'
self.mrSysDir = '/user/' + getpass.getuser() + '/mapredsystem'
self.attrs = {}
self.finalAttrs = {
'fs.default.name': 'nohost.apache.com:56366',
'mapred.child.java.opts' : '-Xmx1024m',
'mapred.compress.map.output' : 'false',
}
self.attrs = {
'mapred.userlog.limit' : '200',
'mapred.userlog.retain.hours' : '10',
'mapred.reduce.parallel.copies' : '20',
}
self.desc = CommandDesc(
{
'name' : 'dummyHadoop',
'program' : 'bin/hadoop',
'pkgdirs' : self.pkgDir,
'final-attrs' : self.finalAttrs,
'attrs' : self.attrs,
}, self.log
)
# TODO
# 4th arg to HadoopCommand 'tardir' is not used at all. Instead pkgdir is
# specified through HadoopCommand.run(pkgdir). This could be changed so
# that pkgdir is specified at the time of object creation.
# END OF TODO
self.hadoopCommand = HadoopCommand(self.id, self.desc, self.tempDir,
self.pkgDir, self.log, self.javaHome,
self.mrSysDir, restart=True)
self.hadoopSite = os.path.join( self.hadoopCommand.confdir,
'hadoop-site.xml')
pass
def test_createHadoopSiteXml(self):
self.hadoopCommand._createHadoopSiteXml()
xmldoc = minidom.parse(self.hadoopSite)
xmldoc = xmldoc.childNodes[0] # leave out xml spec
properties = xmldoc.childNodes # children of tag configuration
keyvals = {}
for prop in properties:
if not isinstance(prop,minidom.Comment):
# ---------- tag -------------------- -value elem-- data --
name = prop.getElementsByTagName('name')[0].childNodes[0].data
value = prop.getElementsByTagName('value')[0].childNodes[0].data
keyvals[name] = value
# fs.default.name should start with hdfs://
assert(keyvals['fs.default.name'].startswith('hdfs://'))
# TODO other tests
pass
def tearDown(self):
pass
