def test_basic001(self):
# reset list of syscalls
#print '++++++++++ DEBUG: starting test_basic001'
global mycmds
mycmds = []
mgr = ClusterMgr()
c1 = mgr.alloc_new('testsub1',self._defcfg,'vagrant')
utils.syscall_cb = mysyscb
c1.start()
tr = TestRunner()
testsuite = tests.call_by_name('basic001')
result = tr.run(c1, testsuite)
print 'Status:', 'Pass' if result.passfail() else 'Fail'
# now check that the right system calls were executed
callpats = [
'vagrant.*up',
'ssh.*echo',
'ssh.*dbFunctional.sh'
]
self.check_syscalls(callpats)
utils.syscall_cb = None
mgr.destroy(c1)
def testComboConfigMachineAccess(self):
#print '++++++++++ DEBUG: starting testComboConfigMachineAccess'
mgr = ClusterMgr()
s = """{
"name" : "combo-cluster",
"idbversion" : "3.5.1-5",
"boxtype" : "cal-precise64",
"rolespec" : {
"pm" : {
"count" : 2,
"dbroots_per" : 1
}
}
}
"""
cfg = ConfigSpec(s)
c = mgr.alloc_new('combo',cfg,'vagrant')
# this should get aliased to pm1
c.machine('um1')
# this still doesn't exist even after the alias
with self.assertRaisesRegexp(ValueError,"Machine name.*not in.*"):
c.machine('um3')
mgr.destroy(c)
def testClusterFilesUpgrade(self):
'''Test cluster with upgrade version specified'''
#print '++++++++++ DEBUG: starting testClusterFilesUpgrade'
mgr = ClusterMgr()
s = """{
"name" : "a-cluster",
"idbversion" : "3.5.1-5",
"boxtype" : "cal-precise64",
"upgrade" : "Latest",
"rolespec" : {
"pm" : {
"count" : 4,
"memory" : 1024,
"dbroots_per" : 1
},
"um" : {
"count" : 1,
"memory" : 2048
}
}
}
"""
cfg = ConfigSpec(s)
c = mgr.alloc_new('testupg',cfg,'vagrant')
ref_file = '%s/testupg-Vagrantfile' % os.path.dirname(__file__)
self.assertTrue( testutils.file_compare(ref_file, c._vmi._vfile))
ref_file = '%s/testupg-postconfigure.in' % os.path.dirname(__file__)
self.assertTrue( testutils.file_compare(ref_file, c._vmi._pfile))
self.assertEqual( mgr.list_clusters()[0][0], 'testupg' )
c.destroy_files_only()
def testClusterFilesEMinVMpm1(self):
'''Test cluster with EM present'''
#print '++++++++++ DEBUG: starting testClusterFilesEMinVM'
mgr = ClusterMgr()
s = """{
"name" : "em-cluster",
"idbversion" : "4.5.0-1",
"binary" : true,
"boxtype" : "cal-precise64",
"em" : { "present" : true, "invm" : true, "role" : "pm1" },
"rolespec" : {
"pm" : {
"count" : 2,
"memory" : 1024,
"dbroots_per" : 1
}
}
}
"""
cfg = ConfigSpec(s)
c = mgr.alloc_new('testeminvmpm1',cfg,'vagrant')
ref_file = '%s/testeminvmpm1-Vagrantfile' % os.path.dirname(__file__)
self.assertTrue( testutils.file_compare(ref_file, c._vmi._vfile))
self.assertEqual( mgr.list_clusters()[0][0], 'testeminvmpm1' )
c.destroy_files_only()
def test1(self):
#print '++++++++++ DEBUG: starting test1'
s = """{
"name" : "a-cluster",
"idbversion" : "Latest",
"boxtype" : "cal-precise64",
"rolespec" : {
"pm" : {
"count" : 4,
"memory" : 1024,
"dbroots_per" : 1
}
},
"hadoop" : {
"instance-templates" : "1 hadoop-namenode+hadoop-jobtracker,2 hadoop-datanode+hadoop-tasktracker"
}
}
"""
mgr = ClusterMgr()
c = mgr.alloc_new('my-clustername',ConfigSpec(s),'vagrant')
w = WhirrConfigWriter(c)
w.write_config(self._basedir)
ref_file = '%s/testwhirr-props' % os.path.dirname(__file__)
self.assertTrue( testutils.file_compare(ref_file, self._propcmp))
ref_file = '%s/testwhirr-nodes' % os.path.dirname(__file__)
self.assertTrue( testutils.file_compare(ref_file, self._nodecmp))
mgr.destroy(c)
def testClusterFiles22(self):
'''Test cluster with version 2.2'''
#print '++++++++++ DEBUG: starting testClusterFiles22'
mgr = ClusterMgr()
s = """{
"name" : "a-cluster",
"idbversion" : "2.2.11-1",
"boxtype" : "cal-precise64",
"rolespec" : {
"pm" : {
"count" : 2,
"dbroots_per" : 1
}
}
}
"""
cfg = ConfigSpec(s)
c = mgr.alloc_new('test22',cfg,'vagrant')
ref_file = '%s/test22-Vagrantfile' % os.path.dirname(__file__)
self.assertTrue( testutils.file_compare(ref_file, c._vmi._vfile))
ref_file = '%s/test22-postconfigure.in' % os.path.dirname(__file__)
self.assertTrue( testutils.file_compare(ref_file, c._vmi._pfile))
self.assertEqual( mgr.list_clusters()[0][0], 'test22' )
c.destroy_files_only()
def testClusterFilesHadoopValidation(self):
'''Test cluster with hadoop validation error'''
#print '++++++++++ DEBUG: starting testClusterFilesHadoopValidation'
mgr = ClusterMgr()
# test without mandatory config specified
s = """{
"name" : "a-cluster",
"idbversion" : "Latest",
"boxtype" : "cal-precise64",
"rolespec" : {
"pm" : {
"count" : 2,
"memory" : 1024,
"dbroots_per" : 1
},
"um" : {
"count" : 1,
"memory" : 2048
}
},
"hadoop" : {
"version" : "1.2.1"
}
}
"""
cfg = ConfigSpec(s)
with self.assertRaisesRegexp(Exception,"Failed to create cluster: testhadoop.*"):
c = mgr.alloc_new('testhadoop',cfg,'vagrant')
def testDatdup(self):
'''Test cluster with datdup specified'''
#print '++++++++++ DEBUG: starting testDatdup'
mgr = ClusterMgr()
s = """{
"name" : "a-cluster",
"idbversion" : "4.0.0-0",
"boxtype" : "cal-precise64",
"datdup" : true,
"rolespec" : {
"pm" : {
"count" : 4,
"memory" : 1024,
"dbroots_per" : 1
},
"um" : {
"count" : 1,
"memory" : 2048
}
}
}
"""
cfg = ConfigSpec(s)
c = mgr.alloc_new('testdatdup',cfg,'vagrant')
self.assertEqual( mgr.list_clusters()[0][0], 'testdatdup' )
mgr.destroy(c)
def testBasic(self):
#print '++++++++++ DEBUG: starting testBasic'
if True:
# put in a code block so that TestReport closed before we validate
tr = TestReport('/tmp/trtest.log')
mgr = ClusterMgr()
cfg1 = configs.multi_1um_2pm('3.5.1-5')
cfg1['boxtype'] = 'cal-precise64'
c = mgr.alloc_new('test-report',cfg1,'vagrant')
tr.new_config(c)
ts = TestSuite('my001','a unit test test suite')
t = datetime.now()
res = TestResult( t, t, True)
tr.new_result(ts, res)
ts = TestSuite('my002','another test suite')
res = TestResult( t, t, False)
tr.new_result(ts, res)
mgr.destroy(c)
# have to explicitly call this to make sure the file is closed
tr.finish()
ref_file = '%s/test1-testlog' % os.path.dirname(__file__)
self.assertTrue( testutils.file_compare(ref_file, '/tmp/trtest.log'))
def testClusterFilesStandard(self):
'''Test cluster with non-enterprise version specified'''
#print '++++++++++ DEBUG: starting testClusterFilesStandard'
mgr = ClusterMgr()
s = """{
"name" : "a-cluster",
"idbversion" : "4.0.0-1",
"boxtype" : "cal-precise64",
"enterprise" : false,
"rolespec" : {
"pm" : {
"count" : 2,
"memory" : 1024,
"dbroots_per" : 1
},
"um" : {
"count" : 1,
"memory" : 2048
}
}
}
"""
cfg = ConfigSpec(s)
c = mgr.alloc_new('teststd',cfg,'vagrant')
ref_file = '%s/teststd-Vagrantfile' % os.path.dirname(__file__)
self.assertTrue( testutils.file_compare(ref_file, c._vmi._vfile))
self.assertEqual( mgr.list_clusters()[0][0], 'teststd' )
c.destroy_files_only()
def testBinUpgradeNonRoot(self):
#print '++++++++++ DEBUG: starting PlaybookInstallTest testBinUpgradeNonRoot'
s = """{
"name" : "1um_2pm",
"idbversion" : "3.5.1-5",
"upgrade" : "4.0",
"boxtype" : "cal-centos6",
"binary" : true,
"nonrootuser" : true,
"idbuser" : "calpont",
"rolespec" : {
"pm" : {
"count" : 2,
"dbroots_per" : 1
},
"um" : {
"count" : 1
}
}
}
"""
mgr = ClusterMgr()
cfg = ConfigSpec(s)
c = mgr.alloc_new('binUpgradeCluster',cfg,'vagrant',False)
c.run_upgrade_recipe()
c.destroy_files_only()
def testBinInstallHadoop_4(self):
#print '++++++++++ DEBUG: starting PlaybookInstallTest testBinInstallHadoop_4'
s = """{
"name" : "1um_2pm",
"idbversion" : "4.0",
"boxtype" : "cal-centos6",
"binary" : true,
"rolespec" : {
"pm" : {
"count" : 2,
"dbroots_per" : 1
},
"um" : {
"count" : 1
}
},
"hadoop" : {
"instance-templates" : "2 hadoop-datanode+hadoop-tasktracker,1 hadoop-namenode+hadoop-jobtracker",
"templates-namenode" : "um1",
"templates-datanode" : "pm1+pm2"
}
}
"""
mgr = ClusterMgr()
cfg = ConfigSpec(s)
c = mgr.alloc_new('binInstallClusterHadoop_4',cfg,'vagrant',False)
c.run_install_recipe()
c.destroy_files_only()
def testDupName(self):
#print '++++++++++ DEBUG: starting testDupName'
mgr = ClusterMgr()
c1 = mgr.alloc_new('testdup',self._defcfg,'vagrant')
c2 = mgr.alloc_new('testdup',self._defcfg,'vagrant')
mgr.destroy(c1)
mgr.destroy(c2)
def testPmQuery(self):
'''Test cluster with PMQuery'''
#print '++++++++++ DEBUG: starting testPMQuery'
mgr = ClusterMgr()
c = mgr.alloc_new('testpmquery', self._pmquerycfg,'vagrant')
ref_file = '%s/testpmquery-postconfigure.in' % os.path.dirname(__file__)
self.assertTrue( testutils.file_compare(ref_file, c._vmi._pfile))
c.destroy_files_only()
def testSubnetAlloc(self):
#print '++++++++++ DEBUG: starting testSubnetAlloc'
mgr = ClusterMgr()
c1 = mgr.alloc_new('testsub1',self._defcfg,'vagrant')
c2 = mgr.alloc_new('testsub2',self._defcfg,'vagrant')
self.assertEqual( c1.get_vmi()._subnet, '192.168.1' )
self.assertEqual( c2.get_vmi()._subnet, '192.168.2' )
mgr.destroy(c1)
mgr.destroy(c2)
def testClusterFilesNonRoot(self):
#print '++++++++++ DEBUG: starting testClusterFilesNonRoot'
mgr = ClusterMgr()
c = mgr.alloc_new('test1',self._nonrootcfg,'vagrant')
ref_file = '%s/testnonroot-Vagrantfile' % os.path.dirname(__file__)
self.assertTrue( testutils.file_compare(ref_file, c._vmi._vfile))
ref_file = '%s/testnonroot-postconfigure.in' % os.path.dirname(__file__)
self.assertTrue( testutils.file_compare(ref_file, c._vmi._pfile))
self.assertEqual( mgr.list_clusters()[0][0], 'test1' )
c.destroy_files_only()
def test_basic002(self):
# reset list of syscalls
#print '++++++++++ DEBUG: starting test_basic002'
global mycmds
mycmds = []
mgr = ClusterMgr()
c1 = mgr.alloc_new('testbasic002',self._defcfg,'vagrant')
utils.syscall_cb = mysyscb
c1.start()
tr = TestRunner()
testsuite = tests.call_by_name('basic002')
result = tr.run(c1, testsuite)
print 'Status:', 'Pass' if result.passfail() else 'Fail'
# now check that the right system calls were executed
callpats = [
'vagrant.*up',
'ssh.*echo',
'ssh.*upgradeTest2.sh',
'ssh.*calpontConsole shutdown y',
'ssh.*sleep 5',
'ssh.*infinidb_not_running.sh',
'ssh.*calpontConsole startsystem',
'ssh.*sleep 5',
'ssh.*upgradeTest2.sh',
'ssh.*calpontConsole restartsystem y',
'ssh.*sleep 5',
'ssh.*upgradeTest2.sh',
'ssh.*calpontConsole stopsystem y',
'ssh.*sleep 5',
'ssh.*calpontConsole startsystem',
'ssh.*sleep 5',
'ssh.*upgradeTest2.sh',
'ssh.*calpontConsole shutdown y',
'ssh.*sleep 5',
'ssh.*infinidb_not_running.sh',
'vagrant halt',
'vboxmanage startvm.*headless',
'vboxmanage startvm.*headless',
'ssh.*healthcheck',
'ssh.*upgradeTest2.sh',
]
self.check_syscalls(callpats)
utils.syscall_cb = None
mgr.destroy(c1)
def main(argv=None):
try:
opts, args = getopt.getopt(sys.argv[1:], "lfun", [])
except getopt.GetoptError as err:
# print help information and exit:
print str(err) # will print something like "option -a not recognized"
usage()
sys.exit(2)
cfile = False
install = True
for o, a in opts:
if o == "-l":
for c in configs.list_all():
print '%-20s : %s' % (c[0], c[1])
return 0
elif o == "-f":
cfile = True
elif o == "-u":
import autooam.test.test_common
elif o == "-n":
install = False
else:
assert False, "unhandled option"
if len(args) != 4:
usage()
sys.exit(2)
Log.info("Creating new cluster instance %s" % args[0])
name = args[0]
config = args[1]
box = args[2]
version = args[3]
mgr = ClusterMgr()
if cfile:
cfg = ConfigSpec(jsonfile=config,idbversion=version,boxtype=box)
else:
cfg = configs.call_by_name(config, version, boxtype=box)
#dmc control chef vs ansible mode through chefmode flag
c = mgr.alloc_new(name,cfg,'vagrant', chefmode=False)
ret = c.start()
if ret != 0:
Log.error("Cluster start failed...aborting launch")
return ret
if install:
c.run_install_recipe()
def main(argv=None):
try:
opts, args = getopt.getopt(sys.argv[1:], "pu", [])
except getopt.GetoptError as err:
# print help information and exit:
print str(err) # will print something like "option -a not recognized"
usage()
sys.exit(2)
killprocs = False
for o, a in opts:
if o == "-p":
killprocs = True
elif o == "-u":
import autooam.test.test_common
utils.syscall_cb = mysyscb
else:
assert False, "unhandled option"
if killprocs:
kill_autorun_procs()
# first get the list of registered vms from virtualbox
vms = get_vbox_vms()
mgr = ClusterMgr()
ids = Set()
for c in mgr.list_clusters():
ids.add(c[1])
for vm in vms:
print "Checking vm %s" % vm[0],
if vm[0] == "<inaccessible>":
print "...Orphaned, Deleting"
cmd = "vboxmanage unregistervm %s --delete" % vm[1]
print "Issuing: %s" % cmd
utils.syscall_log(cmd)
else:
_id = vm[0].split('_')[0]
if not _id in ids:
print "...Orphaned, Deleting"
cmd = "vboxmanage unregistervm %s --delete" % vm[0]
print "Issuing: %s" % cmd
utils.syscall_log(cmd)
else:
print "...Ok"
def testValidateFailure(self):
#print '++++++++++ DEBUG: starting testValidateFailure'
mgr = ClusterMgr()
# this should fail because datdup not supported in 3.5.0-3
s = """{
"name" : "combo-cluster",
"idbversion" : "3.5.0-3",
"boxtype" : "cal-centos6",
"datdup" : true,
"rolespec" : {
"pm" : {
"count" : 2,
"dbroots_per" : 1
}
}
}
"""
cfg = ConfigSpec(s)
with self.assertRaisesRegexp(Exception,"Failed to create cluster"):
c1 = mgr.alloc_new('testdup',cfg,'vagrant')
# test invalid combination; external storage with datdup
s = """{
"name" : "e-cluster",
"idbversion" : "4.0.0-0",
"boxtype" : "cal-centos6",
"storage" : "external",
"datdup" : true,
"rolespec" : {
"pm" : {
"count" : 2,
"memory" : 1024,
"dbroots_per" : 2
},
"um" : {
"count" : 1,
"memory" : 1024
}
}
}
"""
cfg = ConfigSpec(s)
with self.assertRaisesRegexp(Exception,"Failed to create cluster"):
c1 = mgr.alloc_new('testdupExternal',cfg,'vagrant')
def testAllocFailure(self):
#print '++++++++++ DEBUG: starting testAllocFailure'
mgr = ClusterMgr()
s = """{
"name" : "a-cluster",
"idbversion" : "3.5.1-5",
"rolespec" : {
"pm" : {
"count" : 1,
"dbroots_per" : 1
}
}
}
"""
cfg = ConfigSpec(s)
with self.assertRaisesRegexp(Exception,"Failed to create cluster"):
c1 = mgr.alloc_new('testdup',cfg,'vagrant')
def testDEBInstall_3(self):
#print '++++++++++ DEBUG: starting PlaybookInstallTest testDEBInstall_3'
s = """{
"name" : "1um_2pm",
"idbversion" : "3.5.1-5",
"boxtype" : "cal-precise64",
"rolespec" : {
"pm" : {
"count" : 2,
"dbroots_per" : 1
},
"um" : {
"count" : 1
}
}
}
"""
mgr = ClusterMgr()
cfg = ConfigSpec(s)
c = mgr.alloc_new('debInstallCluster_3',cfg,'vagrant',False)
c.run_install_recipe()
c.destroy_files_only()
def testRPMInstall_4(self):
#print '++++++++++ DEBUG: starting PlaybookInstallTest testRPMInstall_4'
s = """{
"name" : "1um_2pm",
"idbversion" : "4.0.0-0",
"boxtype" : "cal-centos6",
"rolespec" : {
"pm" : {
"count" : 2,
"dbroots_per" : 1
},
"um" : {
"count" : 1
}
}
}
"""
mgr = ClusterMgr()
cfg = ConfigSpec(s)
c = mgr.alloc_new('rpmInstallCluster_4',cfg,'vagrant',False)
c.run_install_recipe()
c.destroy_files_only()
def testClusterFilesHadoop(self):
'''Test cluster with hadoop specified'''
#print '++++++++++ DEBUG: starting testClusterFilesHadoop'
mgr = ClusterMgr()
s = """{
"name" : "a-cluster",
"idbversion" : "Latest",
"boxtype" : "cal-precise64",
"rolespec" : {
"pm" : {
"count" : 2,
"memory" : 1024,
"dbroots_per" : 1
},
"um" : {
"count" : 1,
"memory" : 2048
}
},
"hadoop" : {
"instance-templates" : "1 hadoop-namenode+hadoop-jobtracker,2 hadoop-datanode+hadoop-tasktracker"
}
}
"""
cfg = ConfigSpec(s)
c = mgr.alloc_new('testhadoop',cfg,'vagrant')
ref_file = '%s/testhadoop-Vagrantfile' % os.path.dirname(__file__)
self.assertTrue( testutils.file_compare(ref_file, c._vmi._vfile))
ref_file = '%s/testhadoop-postconfigure.in' % os.path.dirname(__file__)
self.assertTrue( testutils.file_compare(ref_file, c._vmi._pfile))
self.assertEqual( mgr.list_clusters()[0][0], 'testhadoop' )
c.destroy_files_only()
def testAttach(self):
#print '++++++++++ DEBUG: starting testAttach'
mgr = ClusterMgr()
c = mgr.alloc_new('testAttach',self._defcfg,'vagrant')
c1 = mgr.attach( 'testAttach' )
self.assertEquals( c1.id() , c.id() )
with self.assertRaisesRegexp(Exception,"Error attaching to cluster.*"):
c2 = mgr.attach( 'no-cluster' )
mgr.destroy(c)
def main(self, argv=None):
if not argv:
argv = sys.argv
try:
opts, args = getopt.getopt(argv[1:], "u", [])
except getopt.GetoptError as err:
# print help information and exit:
print str(err) # will print something like "option -a not recognized"
self.usage()
sys.exit(2)
for o, a in opts:
if o == "-u":
import autooam.test.test_common
# reinitialize this to pickup debug db
self._mgr = ClusterMgr()
else:
assert False, "unhandled option"
cmdaction = {
"list": self.listclusters,
"start": self.start,
"poweron": self.poweron,
"poweroff": self.poweroff,
"pause": self.pause,
"resume": self.resume,
"destroy": self.destroy,
"shell": self.shell,
"show": self.show,
"forceclean": self.forceclean,
"run": self.run,
"attach": self.attach,
}
if len(args) < 1 or not cmdaction.has_key(args[0]):
self.usage()
return cmdaction[args[0]](args[1:])
class ClusterCmd(object):
def __init__(self):
self._mgr = ClusterMgr()
def main(self, argv=None):
if not argv:
argv = sys.argv
try:
opts, args = getopt.getopt(argv[1:], "u", [])
except getopt.GetoptError as err:
# print help information and exit:
print str(err) # will print something like "option -a not recognized"
self.usage()
sys.exit(2)
for o, a in opts:
if o == "-u":
import autooam.test.test_common
# reinitialize this to pickup debug db
self._mgr = ClusterMgr()
else:
assert False, "unhandled option"
cmdaction = {
"list": self.listclusters,
"start": self.start,
"poweron": self.poweron,
"poweroff": self.poweroff,
"pause": self.pause,
"resume": self.resume,
"destroy": self.destroy,
"shell": self.shell,
"show": self.show,
"forceclean": self.forceclean,
"run": self.run,
"attach": self.attach,
}
if len(args) < 1 or not cmdaction.has_key(args[0]):
self.usage()
return cmdaction[args[0]](args[1:])
def usage(self):
print """usage: clustercmd [-u] command [command options] ...
-u : enable unit-test mode
Supported command options:
list
: list all clusters
start cluster
: start the cluster with name <cluster>
poweron cluster
: powers on the cluster with name <cluster>
poweroff cluster
: powers off the cluster with name <cluster>
pause cluster
: pauses the cluster with name <cluster>
resume cluster
: resumes the cluster with name <cluster>
destroy <regex>
: destroy any clusters with name or ID matching <regex>
shell cluster node cmd
: run 'cmd' on node 'node' in the named cluster
show cluster
: show details for the cluster with name <cluster>
forceclean cluster
: destroy and clean cluster, even if in inconsistent state
run cluster test
: run the test suite named 'test' on <cluster>
attach cluster emhost emport
: attempt an attach from EM server <emhost>:<emport> to <cluster>
"""
sys.exit(1)
def listclusters(self, args):
print "Clusters: (name, id)"
for c in self._mgr.list_clusters():
sync = "ok"
try:
cluster = self._mgr.attach(c[0])
sync = "ok,%s" % cluster.status()
except Exception, err:
sync = "invalid (%s),n/a" % err
print "\t(%s,%s,%s)" % (c[0], c[1], sync)
return 0
def __init__(self):
self._mgr = ClusterMgr()
