def testBasic2(self):
'''Test launchcluster -f operation.'''
(ret, output, outerr) = utils.syscall_log('%s/bin/launchcluster.py -u -f unittest-b2 %s/examples/single.json cal-precise64 3.5.1-5' % (os.environ['AUTOOAM_HOME'],os.environ['AUTOOAM_HOME']))
self.assertEqual(ret,0,output)
(ret, output, outerr) = utils.syscall_log('%s/bin/clustercmd.py -u destroy unittest-b2' % os.environ['AUTOOAM_HOME'])
self.assertEqual(ret,0,output)
pass
def testDebugLoggedSyscall(self):
global myret, mydata, mycmd, myscyscb
utils.syscall_cb = mysyscb
logfile = "loggedsyscall"
os.system("rm -f %s" % logfile)
myret = 0
mydata = "foo.py\n"
ret = utils.syscall_log("ls foo.py", logfile)[0]
self.assertEqual(ret, 0)
myret = 1
mydata = ""
ret = utils.syscall_log("grep notthere setup.py", logfile)[0]
self.assertEqual(ret, 1)
myret = 1
mydata = "cat: /etc/foodog: No such file or directory\n"
ret = utils.syscall_log("cat /etc/foodog", logfile)[0]
self.assertEqual(ret, 1)
ref_file = '%s/test2-loggedsyscall' % os.path.dirname(__file__)
self.assertTrue( testutils.file_compare(ref_file, logfile))
# clean up after ourself
os.system("rm -f %s" % logfile)
utils.syscall_cb = None
def testLoggedSyscall(self):
logfile = "loggedsyscall"
os.system("rm -f %s" % logfile)
ret = utils.syscall_log("ls setup.py", logfile)[0]
self.assertEqual(ret, 0)
ret = utils.syscall_log("grep notthere setup.py", logfile)[0]
self.assertEqual(ret, 1)
ret = utils.syscall_log("cat /etc/foobar", logfile)[0]
self.assertEqual(ret, 1)
ref_file = '%s/test1-loggedsyscall' % os.path.dirname(__file__)
self.assertTrue( testutils.file_compare(ref_file, logfile))
# clean up after ourself
os.system("rm -f %s" % logfile)
def start(self):
if common.props['vmi.vagrantvmi.unit-test']:
# if we are running in unit test mode, simulate a start by creating the .vagrant file
f = open("%s/.vagrant" % self._rundir,'w')
f.write('{"active" : {')
first = True
for m in self._cluster.machines():
if not first:
f.write(',')
f.write('"%s":"0e18cd55-ee19-414d-a874-3cfee1a9df5b"' % m)
first = False
f.write('}}')
f.close()
else:
# if we are not running in unit-test mode here then we want to check to make
# sure the user has exported their AUTOOAM_HOME so that it can be mounted by
# the cluster
cmd = 'grep "%s" /etc/exports' % os.environ['AUTOOAM_HOME']
ret = utils.syscall_log(cmd)
if ret[0] != 0:
Log.error("no NFS export for $AUTOOAM_HOME")
print "You need to execute the following commands:"
print " sudo bash -c \"echo '%s 192.168.0.0/255.255.0.0(rw,sync,all_squash,no_subtree_check,anonuid=%d,anongid=%d)' >> /etc/exports\"" % (os.environ['AUTOOAM_HOME'],os.getuid(),os.getgid())
print " sudo exportfs -a"
raise Exception('Fatal error - NFS export not configured')
# our cal-centos58 is excessively slow to boot
base_timeout = 180 if self._cluster.config()['boxtype'] != 'cal-centos58' else 300
# multiply base times # of nodes since vagrant starts sequentially
ctimeout = base_timeout * len(self._cluster.machines())
return self._vagrant_call('vagrant --parallel up',timeout=ctimeout)
def do_facts(cname, hostnames, keystr, ssh_user, requireInfinidb):
factreq = {
"cluster_name": cname,
"hostnames": hostnames,
"ssh_key": keystr,
"ssh_user": ssh_user
}
frfile = '/tmp/factreq.json'
wf = open(frfile, "w")
wf.write(json.dumps(factreq))
wf.close()
cmd = 'getfacts.py --json %s' % frfile
rc, out, err = syscall_log(cmd)
if rc != 0:
print 'ERROR: getfacts.py failed with return code %s!' % rc
print 'stdout = %s' % out
print 'stderr = %s' % err
sys.exit(1)
factreply = json.loads(out)
if (not factreply['cluster_info']['valid']):
print "ERROR: getfacts.py failed to detect a valid cluster:"
print json.dumps(factreply, sort_keys=True, indent=4)
sys.exit(1)
elif (requireInfinidb
and not factreply['cluster_info']['infinidb_version']):
print "ERROR: emsim.py is not going to install InfiniDB, but an InfiniDB installation was not found, consider using -i:"
sys.exit(1)
else:
print 'INFO: getfacts.py PASSED!'
return factreply
def do_playbook(cname, name, hostspec, extravars=''):
pbreq = {
'cluster_name': cname,
'playbook_info': {
'name': name,
'hostspec': hostspec,
'extravars': extravars
}
}
pbfile = '/tmp/playbook.json'
wf = open(pbfile, "w")
wf.write(json.dumps(pbreq))
wf.close()
cmd = 'runplaybook.py --json %s' % pbfile
rc, out, err = syscall_log(cmd)
if rc != 0:
print 'ERROR: runplaybook.py failed with return code %s!' % rc
print 'stdout = %s' % out
print 'stderr = %s' % err
return 1
pbreply = json.loads(out)
if pbreply['rc'] == 0:
print 'INFO: runplaybook.py PASSED!'
else:
print "INFO: runplaybook.py FAILED:"
print json.dumps(pbreply, sort_keys=True, indent=4)
return pbreply['rc']
def do_console(cname, command):
statreq = {
'cluster_name': cname,
'command': command,
}
statfile = '/tmp/status.json'
wf = open(statfile, "w")
wf.write(json.dumps(statreq))
wf.close()
cmd = 'idbconsole.py --json %s' % statfile
rc, out, err = syscall_log(cmd)
if rc != 0:
print 'ERROR: idbconsole.py failed with return code %s!' % rc
print 'stdout = %s' % out
print 'stderr = %s' % err
return rc
statreply = json.loads(out)
if statreply['rc'] == 0:
print 'INFO: idbconsole.py %s PASSED!' % command
else:
print "INFO: idbconsole.py %s FAILED:" % command
print json.dumps(statreply, sort_keys=True, indent=4)
return statreply['rc']
return 0
def retrieve(self, version, ptype):
'''locates the specified version and package type and caches in the local package directory
@param version - version to retrieve. Must be the name of a directory
in the package structure (ex. a directory name under
//calweb/shared/Iterations)
@param ptype - package type. One of 'bin', 'deb', or 'rpm'
Returns the relative path to the package tarball which is guaranteed to be
located in props['cluster.emvmgr.basedir']
Raises exceptions on failure to locate the specified package (or
other misc. errors such as a bad type, etc.).
'''
# unit test bailout
if common.props['vmi.vagrantvmi.unit-test']:
# this is purely made up. In unit test right now, nobody actually tries to do anything with this return
return( 'build-99999', os.path.join( common.props['cluster.emvmgr.packagedir'], 'build-99999', 'infinidb-entmgr-1.0-1.el6.x86_64.rpm'))
if version == 'Latest':
buildver = self._get_current_build_version()
fname, fsize, fdate = self._get_package_details(ptype)
Log.info('Current build version is %s' % buildver)
pkgpath = os.path.join( self.__packagedir, buildver )
pkgfile = os.path.join( pkgpath, fname )
if not os.path.isdir( pkgpath ):
# we don't have this directory at all so we know we will create and download
utils.mkdir_p( pkgpath )
else:
# the directory already exists - let's see if the file is already current
if os.path.exists(pkgfile) and eval(fsize) == os.path.getsize(pkgfile):
Log.info('Package %s is already up-to-date' % pkgfile)
return (buildver, pkgfile)
Log.info('Fetching %s into %s' % (fname, pkgpath))
cmd = 'wget -O %s --http-user=%s --http-password=%s https://infinidb.atlassian.net/builds/artifact/EM-EM/shared/build-latest/%s/%s?os_authType=basic' %\
( pkgfile, self.__httpuser, self.__httppassword, self.__pkgmap[ptype], fname )
rc, out, err = utils.syscall_log(cmd)
return (buildver, pkgfile)
else:
# in this case we aren't going to contact the bamboo server, but
# rather use a previously downloaded version
pkgpath = os.path.join( self.__packagedir, version )
if not os.path.isdir( pkgpath ):
Log.error('Error retrieving EM version %s, %s path does not exist' % (version, pkgpath))
return (None, None)
fileext = ptype if ptype != 'bin' else '.tar.gz'
pkgs = glob.glob('%s/*%s' % (pkgpath, fileext))
if len(pkgs) == 0:
Log.error('No packages of type %s found in %s' % (ptype, pkgpath))
return (None, None)
else:
if len(pkgs) > 1:
Log.warn('Multiple packages of type %s found in %s, using first : %s' % (ptype, pkgpath, pkgs))
return (version, pkgs[0])
def main(argv=None):
try:
opts, args = getopt.getopt(sys.argv[1:], "pu", [])
except getopt.GetoptError as err:
# print help information and exit:
print str(err) # will print something like "option -a not recognized"
usage()
sys.exit(2)
killprocs = False
for o, a in opts:
if o == "-p":
killprocs = True
elif o == "-u":
import autooam.test.test_common
utils.syscall_cb = mysyscb
else:
assert False, "unhandled option"
if killprocs:
kill_autorun_procs()
# first get the list of registered vms from virtualbox
vms = get_vbox_vms()
mgr = ClusterMgr()
ids = Set()
for c in mgr.list_clusters():
ids.add(c[1])
for vm in vms:
print "Checking vm %s" % vm[0],
if vm[0] == "<inaccessible>":
print "...Orphaned, Deleting"
cmd = "vboxmanage unregistervm %s --delete" % vm[1]
print "Issuing: %s" % cmd
utils.syscall_log(cmd)
else:
_id = vm[0].split('_')[0]
if not _id in ids:
print "...Orphaned, Deleting"
cmd = "vboxmanage unregistervm %s --delete" % vm[0]
print "Issuing: %s" % cmd
utils.syscall_log(cmd)
else:
print "...Ok"
def kill_autorun_procs():
# return from ps |grep will look like this
# infinidb 5882 1 0 08:05 ? 00:00:00 /bin/sh /home/infinidb/Calpont/mysql//bin/mysqld_safe --defaults-file=/home/infinidb/Calpont/mysql//my.cnf --datadir=/home/infinidb/Calpont/mysql/db --pid-file=/home/infinidb/Calpont/mysql/db/pm1.pid
(ret,output,outerr) = utils.syscall_log('ps -ef | grep autorun | grep -v grep')
if ret:
# means grep failed so nothing to do
return
else:
lines = output.split('\n')
for l in lines:
# split on whitespace
fields = l.split()
if len(fields) >= 2:
print 'Found rogue process %s...Killing' % fields[8]
pid = fields[1]
cmd = 'kill -9 %s' % pid
print "Issuing: %s" % cmd
utils.syscall_log(cmd)
def testDebugNonLoggedSyscall(self):
global myret, mydata, mycmd, myscyscb
utils.syscall_cb = mysyscb
myret = 29
mydata = "Hello World!"
(ret,out,outerr) = utils.syscall_log("ls foo.py")
self.assertEqual(ret, 29)
self.assertEqual(out, "Hello World!")
self.assertEqual(mycmd, "ls foo.py")
utils.syscall_cb = None
def get_vbox_vms():
vms = []
(ret,output,outerr) = utils.syscall_log('vboxmanage list vms')
lines = output.split('\n')
for l in lines:
if len(l):
mat = re.search('^\"(.*)\" \{(.*)\}$',l)
if not mat:
sys.stderr.write("Unable to parse vm spec %s " % l)
sys.exit(1)
vms.append((mat.group(1),mat.group(2)))
return vms
def _syscall(self, cmd, retrycnt=1, retrywait=1, timeout=-1, polling=False):
'''issue a system call.'''
if not polling:
Log.info('Executing: %s' % cmd)
ret = 1
attempts = 0
while ret and attempts < retrycnt:
attempts += 1
ret = utils.syscall_log(cmd, self._outfile, timeout=timeout)[0]
if ret:
if not polling:
Log.error('%s returned %d' % (cmd, ret))
if attempts < retrywait:
utils.sleep(retrywait)
return ret
def do_installdb(cname, hostnames, roles, idbversion, storage_type='local'):
roleinfo = None
if roles == None and len(hostnames) == 1:
roleinfo = {'pm1': hostnames[0]}
elif roles == None:
print 'ERROR: do_installdb requires the roles map with more than 1 host!'
return 1
else:
roleinfo = roles
if idbversion == None:
print 'ERROR: do_installdb requires idbversion be set!'
return 1
installreq = {
'cluster_name': cname,
'cluster_info': {
'infinidb_version': idbversion,
'dbroots_per_pm': 1,
'infinidb_user': '******',
'storage_type': storage_type,
'pm_query': False,
'um_replication': False
},
'role_info': roleinfo
}
installpb = '/tmp/installdb.json'
wf = open(installpb, "w")
wf.write(json.dumps(installreq))
wf.close()
cmd = 'installdatabase.py --json %s' % installpb
rc, out, err = syscall_log(cmd)
if rc != 0:
print 'ERROR: installdatabase.py failed with return code %s!' % rc
print 'stdout = %s' % out
print 'stderr = %s' % err
return 1
pbreply = json.loads(out)
if pbreply['rc'] == 0:
print 'INFO: installdatabase.py PASSED!'
else:
print "INFO: installdatabase.py FAILED:"
print json.dumps(pbreply, sort_keys=True, indent=4)
return pbreply['rc']
def _get_current_build_version(self):
cmd = 'wget -qO- --http-user=%s --http-password=%s https://infinidb.atlassian.net/builds/artifact/EM-EM/shared/build-latest/?os_authType=basic' %\
( self.__httpuser, self.__httppassword )
rc, out, err = utils.syscall_log(cmd)
if rc != 0:
Log.error('Failed to retrieve current build version. Command %s return %s:%s' % (cmd, rc, out))
return None
bldidpatt = re.compile('.*(build-[0-9]+).*')
mat = bldidpatt.match(out)
if mat:
return mat.group(1)
else:
Log.error('Retrieve of build-latest folder does not appear to contain a build number: %s' % out)
return None
def testBasic1(self):
'''Test launchcluster and various clustercmd's against a multi node cluster.'''
(ret, output, outerr) = utils.syscall_log('%s/bin/launchcluster.py -u unittest multi_1um_2pm cal-precise64 3.5.1-5' % os.environ['AUTOOAM_HOME'])
self.assertEqual(ret,0,output)
(ret, output, outerr) = utils.syscall_log('%s/bin/clustercmd.py -u poweroff unittest' % os.environ['AUTOOAM_HOME'])
self.assertEqual(ret,0,output)
(ret, output, outerr) = utils.syscall_log('%s/bin/clustercmd.py -u poweron unittest' % os.environ['AUTOOAM_HOME'])
self.assertEqual(ret,0,output)
(ret, output, outerr) = utils.syscall_log('%s/bin/clustercmd.py -u pause unittest' % os.environ['AUTOOAM_HOME'])
self.assertEqual(ret,0,output)
(ret, output, outerr) = utils.syscall_log('%s/bin/clustercmd.py -u resume unittest' % os.environ['AUTOOAM_HOME'])
self.assertEqual(ret,0,output)
(ret, output, outerr) = utils.syscall_log('%s/bin/clustercmd.py -u show unittest' % os.environ['AUTOOAM_HOME'])
self.assertEqual(ret,0,output)
(ret, output, outerr) = utils.syscall_log('%s/bin/clustercmd.py -u run unittest basic001' % os.environ['AUTOOAM_HOME'])
self.assertEqual(ret,0,output)
(ret, output, outerr) = utils.syscall_log('%s/bin/clustercmd.py -u destroy unittest' % os.environ['AUTOOAM_HOME'])
self.assertEqual(ret,0,output)
pass
def do_config(cname, action, set_params=None, check_parm=None, check_val=None):
jsonreq = {
'cluster_name': cname,
'action': action,
}
if set_params:
jsonreq['set_params'] = set_params
jsonfile = '/tmp/config.json'
wf = open(jsonfile, "w")
wf.write(json.dumps(jsonreq))
wf.close()
cmd = 'config.py --json %s' % jsonfile
rc, out, err = syscall_log(cmd)
if rc != 0:
print 'ERROR: config.py %s failed with return code %s!' % (action, rc)
print 'stdout = %s' % out
print 'stderr = %s' % err
return rc
jsonreply = json.loads(out)
if jsonreply.has_key('failed') is False:
print 'INFO: config.py %s PASSED!' % action
else:
print "INFO: config.py %s FAILED:" % action
print json.dumps(jsonreply, sort_keys=True, indent=4)
return jsonreply['rc']
if action == 'get' and check_parm and check_val:
for p in jsonreply['config']:
if p['em_parameter'] == check_parm:
if p['value'] == check_val:
print 'INFO: config.py chetk for %s==%s PASSED!' % (
check_parm, check_val)
return 0
else:
print 'ERROR: config.py chetk for %s==%s FAILED!' % (
check_parm, check_val)
return 1
print 'ERROR: config.py chetk for %s==%s FAILED, parm not found!' % (
check_parm, check_val)
return 1
return 0
def _get_package_details(self, ptype):
cmd = 'wget -qO- --http-user=%s --http-password=%s https://infinidb.atlassian.net/builds/artifact/EM-EM/shared/build-latest/%s?os_authType=basic' %\
( self.__httpuser, self.__httppassword, self.__pkgmap[ptype] )
rc, out, err = utils.syscall_log(cmd)
if rc != 0:
Log.error('Failed to retrieve package details. Command %s return %s:%s' % (cmd, rc, out))
return None
# regex won't work across newlines so we strip them out
out = out.replace('\n','')
# split based on HTTP table cells
cells = out.split('</TD>')
# now fields should be as follows:
# [0] - file name
# [1] - file size
# [2] - file datetime
filepatt = re.compile('.*>(infinidb-.*)</a>.*')
sizepatt = re.compile('.*>([0-9]+) bytes.*')
datepatt = re.compile('<TD>(.*)')
filemat = filepatt.match(cells[0])
sizemat = sizepatt.match(cells[1])
datemat = datepatt.match(cells[2])
if filemat:
fname = filemat.group(1)
else:
Log.error('Retrieve of %s package does not appear to contain a filename: %s' % (ptype, cells[0]) )
return None
if sizemat:
fsize = sizemat.group(1)
else:
Log.error('Retrieve of %s package does not appear to contain a file size: %s' % (ptype, cells[1]) )
return None
if datemat:
fdate = datemat.group(1)
else:
Log.error('Retrieve of %s package does not appear to contain a date: %s' % (ptype, cells[2]) )
return None
return fname, fsize, fdate
def testBasic3(self):
'''Test launchcluster and various clustercmd's against a single node cluster.'''
(ret, output, outerr) = utils.syscall_log('%s/bin/launchcluster.py -u unittest-b3 singlenode cal-precise64 3.5.1-5' % (os.environ['AUTOOAM_HOME']))
self.assertEqual(ret,0,output)
(ret, output, outerr) = utils.syscall_log('%s/bin/clustercmd.py -u poweroff unittest-b3' % os.environ['AUTOOAM_HOME'])
self.assertEqual(ret,0,output)
(ret, output, outerr) = utils.syscall_log('%s/bin/clustercmd.py -u poweron unittest-b3' % os.environ['AUTOOAM_HOME'])
self.assertEqual(ret,0,output)
(ret, output, outerr) = utils.syscall_log('%s/bin/clustercmd.py -u show unittest-b3' % os.environ['AUTOOAM_HOME'])
self.assertEqual(ret,0,output)
(ret, output, outerr) = utils.syscall_log('%s/bin/clustercmd.py -u run unittest-b3 basic001' % os.environ['AUTOOAM_HOME'])
self.assertEqual(ret,0,output)
(ret, output, outerr) = utils.syscall_log('%s/bin/clustercmd.py -u destroy unittest-b3' % os.environ['AUTOOAM_HOME'])
self.assertEqual(ret,0,output)
pass
def do_inv(cname, role_out):
writeinv = {'cluster_name': cname, 'role_info': role_out}
frfile = '/tmp/writeinv.json'
wf = open(frfile, "w")
wf.write(json.dumps(writeinv))
wf.close()
cmd = 'writeinventory.py --json %s' % frfile
rc, out, err = syscall_log(cmd)
if rc != 0:
print 'ERROR: writeinventory.py failed with return code %s!' % rc
print 'stdout = %s' % out
print 'stderr = %s' % err
return rc
invreply = json.loads(out)
if invreply['rc'] == 0:
print 'INFO: writeinventory.py PASSED!'
else:
print "INFO: writeinventory.py FAILED:"
print json.dumps(invreply, sort_keys=True, indent=4)
return invreply['rc']
def testBasic(self):
(ret, output, outerr) = utils.syscall_log('%s/bin/cleanorphans.py -u' % (os.environ['AUTOOAM_HOME']))
self.assertEqual(ret,0,output)
(ret, output, outerr) = utils.syscall_log('%s/bin/cleanorphans.py -u -p' % (os.environ['AUTOOAM_HOME']))
self.assertEqual(ret,0,output)
def run_install_recipe(self, cb=None):
"""Run the proper install recipe from the autooam cookbook."""
ret = 0
# skip whirr hadoop install if we are running in unittest mode
if not common.props['vmi.vagrantvmi.unit-test']:
if self._config['hadoop']:
if cb:
cb('Whirr Hadoop install Step')
Log.info('Executing whirr launch-cluster for Hadoop')
w = WhirrConfigWriter( self )
w.write_config( self._vmi._rundir )
owd = os.getcwd()
os.chdir(self._vmi._rundir)
cmd = '%s/whirr launch-cluster --config hadoop.properties --private-key-file %s/insecure_private_key' %\
(common.props['cluster.cluster.whirrdir'], common.props['vmi.vagrantvmi.vagrantroot'])
ret = utils.syscall_log(cmd, self._vmi._outfile)[0]
os.chdir(owd)
# we also need to install libhdfs before moving on to the InfiniDB install
if ret == 0:
if cb:
cb('Chef autooam::hadoop_postinstall Step')
# must be done on every InfiniDB node!
# TODO: consider moving this to an ansible playbook
for m in self._machines.keys():
if not m == "em1":
if ( vagboxes.get_os_family(self._config['boxtype']) == 'ubuntu' or\
vagboxes.get_os_family(self._config['boxtype']) == 'debian' ):
cmd = 'sudo apt-get -y install libhdfs0'
else:
cmd = 'sudo yum -y install hadoop-libhdfs'
ret = self.shell_command(m, cmd)
if ret != 0:
break
if ret != 0:
Log.error('There were errors during Hadoop installation, did not attempt InfiniDB install')
return ret
if cb:
cb('InfiniDB install Step')
Log.info('Performing InfiniDB install')
recipe = 'autooam::binary_install' if self._config['binary'] else 'autooam::package_install'
# Use EM to install Infinidb
if self._emapi and common.props['cluster.cluster.use_em_for_dbinstall']:
# Install EM (if applicable)
ret = self._em_install(cb)
if ret != 0:
return ret
# install db step
ret = self._em_installdb(cb)
# Install Infinidb through execution of ansible playbooks
else:
if self._chefmode:
ret = self._run_chef_recipe(recipe)
else:
ret = self._run_ansible_playbook(recipe)
if ret != 0:
Log.error('There were errors installing InfiniDB')
return ret
if self._emapi:
# Install and attach to EM (if applicable)
ret = self._em_install(cb)
if ret != 0:
return ret
ret = self._em_attach( cb=cb )
return ret
def testNonLoggedSyscall(self):
(ret, out, outerr) = utils.syscall_log("ls setup.py")
self.assertEqual(ret, 0)
self.assertEqual(out, 'setup.py')
def testSysCallTimeout(self):
(ret,out,outerr) = utils.syscall_log("find /",timeout=2)
self.assertEqual(ret, -9)
def syscall(cmd):
print 'INFO: issuing %s' % cmd
(rc, out, err) = syscall_log(cmd)
if rc != 0:
raise Exception("Command %s failed! rc=%s, stdout=%s, stderr=%s" % (cmd, rc, out, err))
return (rc, out, err)