def __init__(self): ''' Constructor. Ensure that we have a package directory present, then create the cache directory if it doesn't already exist. ''' self._basedir = common.props['vmi.versionmgr.basedir'] if not os.path.exists(self._basedir): raise Exception("Package reference directory %s does not exist!" % self._basedir) self._cachedir = common.props['vmi.versionmgr.packagedir'] if not os.path.exists(self._cachedir): utils.mkdir_p(self._cachedir) self._filepatt = { 'binary' : re.compile('(calpont-|)infinidb-ent-([0-9\-\.]*).x86_64.bin.tar.gz'), 'deb' : re.compile('(calpont-|)infinidb-ent-([0-9\-\.]*).amd64.deb.tar.gz'), 'rpm' : re.compile('(calpont-|)infinidb-ent-([0-9\-\.]*).x86_64.rpm.tar.gz'), 'binary-datdup' : re.compile('calpont-infinidb-datdup-(.*).x86_64.bin.tar.gz'), # debian support is via the binary package for now 'deb-datdup' : re.compile('calpont-infinidb-datdup-(.*).x86_64.bin.tar.gz'), 'rpm-datdup' : re.compile('calpont-datdup-(.*).x86_64.rpm'), 'binary-std' : re.compile('infinidb-([0-9\-\.]*).x86_64.bin.tar.gz'), 'deb-std' : re.compile('infinidb-([0-9\-\.]*).amd64.deb.tar.gz'), 'rpm-std' : re.compile('infinidb-([0-9\-\.]*).x86_64.rpm.tar.gz'), } self._verspatt = re.compile('(calpont-|)infinidb-(ent-|)(.*).(x86_64.bin|amd64.deb|x86_64.rpm).tar.gz')
def _get_autooam_utils(self, version): ''' Installs required autooam utils that are not packaged in standard edition. At present, this is only the healthcheck utility, but in the future it may expand to include others. @param version - version to retrieve. Must be the name of a directory in the package structure (ex. a directory name under //calweb/shared/Iterations) @return - Boolean value indicating success(True)/failure(False) On failure, an error log of some type will be generated ''' if not version == 'Latest' and version[0] < '4': # for versions prior to 4.0 this doesn't make sense - we only # test enterprise and it is always in the package return True binent = self._haspkg(self._basedir, version, 'binary', False, True) if not binent: Log.error("Unable to extract Calpont/bin/healthcheck for version %s, no binary enterprise package found!" % version) return False cwd = os.getcwd() cachepath = '%s/%s' % (self._cachedir,binent[1]) utils.mkdir_p(cachepath) os.chdir(cachepath) ret = [ os.system('tar xvzf %s Calpont/bin/healthcheck > /dev/null 2>&1' % binent[0]) >> 8 ] if ret[0] != 0: Log.error('tar xvzf %s Calpont/bin/healthcheck failed: %s' % (binent[0],ret[0])) return False os.chdir(cwd) if not os.path.exists('%s/Calpont/bin/healthcheck' % cachepath): Log.error("Unable to extract healthcheck to %s/Calpont/bin/healthcheck" % cachepath) return False return True
def retrieve(self, version, ptype): '''locates the specified version and package type and caches in the local package directory @param version - version to retrieve. Must be the name of a directory in the package structure (ex. a directory name under //calweb/shared/Iterations) @param ptype - package type. One of 'bin', 'deb', or 'rpm' Returns the relative path to the package tarball which is guaranteed to be located in props['cluster.emvmgr.basedir'] Raises exceptions on failure to locate the specified package (or other misc. errors such as a bad type, etc.). ''' # unit test bailout if common.props['vmi.vagrantvmi.unit-test']: # this is purely made up. In unit test right now, nobody actually tries to do anything with this return return( 'build-99999', os.path.join( common.props['cluster.emvmgr.packagedir'], 'build-99999', 'infinidb-entmgr-1.0-1.el6.x86_64.rpm')) if version == 'Latest': buildver = self._get_current_build_version() fname, fsize, fdate = self._get_package_details(ptype) Log.info('Current build version is %s' % buildver) pkgpath = os.path.join( self.__packagedir, buildver ) pkgfile = os.path.join( pkgpath, fname ) if not os.path.isdir( pkgpath ): # we don't have this directory at all so we know we will create and download utils.mkdir_p( pkgpath ) else: # the directory already exists - let's see if the file is already current if os.path.exists(pkgfile) and eval(fsize) == os.path.getsize(pkgfile): Log.info('Package %s is already up-to-date' % pkgfile) return (buildver, pkgfile) Log.info('Fetching %s into %s' % (fname, pkgpath)) cmd = 'wget -O %s --http-user=%s --http-password=%s https://infinidb.atlassian.net/builds/artifact/EM-EM/shared/build-latest/%s/%s?os_authType=basic' %\ ( pkgfile, self.__httpuser, self.__httppassword, self.__pkgmap[ptype], fname ) rc, out, err = utils.syscall_log(cmd) return (buildver, pkgfile) else: # in this case we aren't going to contact the bamboo server, but # rather use a previously downloaded version pkgpath = os.path.join( self.__packagedir, version ) if not os.path.isdir( pkgpath ): Log.error('Error retrieving EM version %s, %s path does not exist' % (version, pkgpath)) return (None, None) fileext = ptype if ptype != 'bin' else '.tar.gz' pkgs = glob.glob('%s/*%s' % (pkgpath, fileext)) if len(pkgs) == 0: Log.error('No packages of type %s found in %s' % (ptype, pkgpath)) return (None, None) else: if len(pkgs) > 1: Log.warn('Multiple packages of type %s found in %s, using first : %s' % (ptype, pkgpath, pkgs)) return (version, pkgs[0])
def __init__(self): ''' Constructor ''' self.__packagedir = common.props['cluster.emvmgr.packagedir'] self.__httpuser = common.props['cluster.emvmgr.httpuser'] self.__httppassword = common.props['cluster.emvmgr.httppassword'] self.__pkgmap = { 'rpm' : 'infinidb-em.rpm', 'deb' : 'infinidb-em.deb', 'bin' : 'infinidb-em.tar.gz' } if not os.path.isdir(self.__packagedir): utils.mkdir_p(self.__packagedir)
def _update_cache(self, refloc, path, pfile): ''' Private method that updates the cached file of the specified file. @param refloc - fully specified path to package. @param path - a relative path to where the package will be cached. @param pfile - name of the package file Effective the operation is: cp -p <refloc> <cache-base>/<path>/. @return - relative path of the package file in self._cachedir @raises - Exception if failed to update cache ''' Log.info('Updating local package cache for %s/%s' % (path,pfile)) cachepath = self._cachedir + '/' + path utils.mkdir_p(cachepath) cachefile = cachepath + '/' + pfile cmd = 'cp -p %s %s' % (refloc, cachefile) ret = os.system(cmd) >> 8 if ret != 0: raise Exception('Failed to copy %s to %s' % (refloc, cachefile)) return '%s/%s' % (path, pfile)
def _alloc_construct(self, cluster): '''create a new vmi instance.''' if not cluster.config().has_key('boxtype'): raise Exception("Vagrant cluster creation requires a boxtype in the ConfigSpec") # this hadoop validation check was formerly in configspec, but # moved to here to remove autooam/vagboxes dependency from # emtools/configspec if cluster.config().has_key('hadoop') and cluster.config()['hadoop']: if not vagboxes.hadoop_support(cluster.config()['boxtype']): raise Exception("Hadoop not supported on boxtype %s" % self.jsonmap['boxtype']) self._subnet = self._salloc.alloc(cluster) # first we want to look for our root directory, make sure it # does not already exist and then create it root = common.props['vmi.vagrantvmi.vagrantdir'] utils.mkdir_p(root) self._rundir = '%s/%s_%s' % (root, cluster.name(), str(cluster.id())) os.makedirs(self._rundir) # this is where we will write stdout and stderr for any calls # executed agaist this VMI self._outfile = "%s/%s.out" % (self._rundir, cluster.name()) self._defmemsize = common.props['vmi.vagrantvmi.defmemsize'] self._defcpus = common.props['vmi.vagrantvmi.defcpus'] # do a sanity check to make sure we don't ask for a non-existent package # we only support enterprise=False for versions 4.0 and later entpkg = cluster.config()['enterprise'] if ConfigSpec._version_greaterthan('4.0.0-0',cluster.config()['idbversion']): Log.info('resetting enterprise to True for version %s ' % cluster.config()['idbversion']) entpkg = True # make sure that our package exists vm = VersionManager() if cluster.config()['idbuser'] != 'root' or cluster.config()['binary']: ptype = 'binary' # set this to true in case not already set so that vagrant file writer # can depend on it being accurate cluster.config()['binary'] = True else: ptype = vagboxes.get_default_pkgtype(cluster.config()['boxtype']) self._pkgfile = vm.retrieve(cluster.config()['idbversion'], ptype, enterprise=entpkg) # handle the upgrade version if the user specified it upgfile = None upgvers = None if cluster.config()['upgrade']: upgfile = vm.retrieve(cluster.config()['upgrade'], ptype, enterprise=entpkg) upgvers = vm.get_pkg_version(upgfile) self._upgfile = upgfile # handle datdup package if the user requested it - note that the datdup # package is only relevant prior to version 4.0 datduppkgfile = None if cluster.config()['datdup'] and not ConfigSpec._version_greaterthan(cluster.config()['idbversion'],'4.0.0-0'): datduppkgfile = vm.retrieve(cluster.config()['idbversion'], ptype, datdup=True) self._alloc_machines() h = PostConfigureHelper() self._pfile = '%s/postconfigure.in' % self._rundir h.write_input(self._pfile, cluster, ptype) # @bug 5990: don't need to copy public key. vagrant # public access should already be setup when cluster # was instantiated. # copy public key to shared directory so that vagrant can access #utils.mkdir_p("%s/.ssh" % self._rundir) #shutil.copy( '%s.pub' % common.props['emtools.test.sshkeyfile'], # '%s/.ssh/public_key' % self._rundir) self._vfile = self._rundir + '/Vagrantfile' vfile = VagrantFileWriter( cluster, self._pkgfile, vm.get_pkg_version(self._pkgfile), datduppkgfile, self._upgfile, upgvers, self._subnet, self._rundir) vfile.writeVagrantFile( self._vfile ) cluster.vmi(self) # For external DBRoot storage: delete/recreate dataN directories # locally, to be NFS mounted for use on each PM if cluster.config()['storage'] == 'external': rootCount = cluster.config().total_dbroot_count() for i in range( rootCount ): dbRootDir = '%s/data%d' % (self._rundir, i+1) if os.path.exists( dbRootDir ): shutil.rmtree( dbRootDir ) os.mkdir( dbRootDir )
def main(argv): """ main function """ try: opts, args = getopt.getopt(argv, "hvi", ["json="]) except getopt.GetoptError: usage() sys.exit(2) # defaults use_stdin = False json_file = "" # parse command line arguments for o, a in opts: if o == "-h": usage() sys.exit(2) elif o == "-v": print "installdatabase.py Version: %s" % version sys.exit(1) elif o == "-i": use_stdin = True elif o == "--json": json_file = a else: print "unsupported option: %s" % o usage() sys.exit(2) if (use_stdin and json_file) or (not use_stdin and not json_file): print "ERROR: Must specify exactly one of -i or --json" usage() sys.exit(2) try: # load input into json string jsonstr = None if use_stdin: lines = sys.stdin.readlines() jsonstr = "".join(lines) elif json_file: f = open(json_file) lines = f.readlines() jsonstr = "".join(lines) Log = logutils.getLogger("installdatabase") req = installreq.InstallReq(jsonstr) Log.info("request: %s" % req.json_dumps()) # construct configspec cfgspecbld = ConfigSpecBuilder(req) cfgspec, machines = cfgspecbld.run() # determine the approprate package file to be installed emVM = EmVersionManager() pkgfile = emVM.retrieve(cfgspec["idbversion"], "binary") Log.info("pkgfile: %s" % pkgfile) # create runtime directory root = common.props["emtools.playbookmgr.cluster_base"] rundir = "%s/%s" % (root, cfgspec["name"]) if not os.path.exists(rundir): mkdir_p(rundir) # create the cluster emCluster = EmCluster(cfgspec["name"], cfgspec, rundir, pkgfile, machines) # create the postconfig response file h = PostConfigureHelper() pfile = "%s/postconfigure.in" % rundir h.write_input(pfile, emCluster, "binary") # perform the db install rc, results, out, err = emCluster.run_install_recipe() reply_dict = { "cluster_name": cfgspec["name"], "playbook_info": { "name": emCluster.get_playbook_filename(), "hostspec": emCluster.get_inventory_filename(), "extravars": emCluster.get_extra_vars(), }, "rc": rc, "stdout": out, "stderr": err, "recap_info": results, } # test stub output # reply_dict = { # 'cluster_name' : cfgspec['name'], # 'playbook_info': { # 'name' : 'test_name', # 'hostspec' : 'test_hostspec', # 'extravars': 'test_extravars' # }, # 'rc' : 0, # 'stdout' : 'test_stdout', # 'stderr' : 'test_stderr', # } preply = playbookreply.PlaybookReply_from_dict(reply_dict) Log.info("reply: %s" % preply.json_dumps()) print preply return 0 except: import traceback print errormsg.ErrorMsg_from_parms(msg=json.dumps(traceback.format_exc())) sys.exit(1)
def testBasic(self): # standard list_available with no type spec v = VersionManager() self.assertEqual( sorted(v.list_available()), [ '2.2.11-1', '3.5.1-5', '4.0.0-0' ] ) # standard retrieve from reference with no cache pfile = v.retrieve('3.5.1-5','deb') self.assertEqual( pfile, '3.5.1-5/packages/calpont-infinidb-ent-3.5.1-5.amd64.deb.tar.gz') self.assertEqual( os.path.getmtime('%s/%s' % (self._cachepath, pfile)), os.path.getmtime('%s/%s' % (self._pkgpath, pfile))) # retrieve again - this time should come from cache pfile = v.retrieve('3.5.1-5','deb') self.assertEqual( pfile, '3.5.1-5/packages/calpont-infinidb-ent-3.5.1-5.amd64.deb.tar.gz') self.assertEqual( v.get_pkg_version(pfile), '3.5.1-5' ) # the 'Latest' directory in reference has a bin file self.assertEqual( sorted(v.list_available('binary')), [ '2.2.11-1', '3.5.1-5', '4.0', '4.0.0-0', '4.0.0-1', '4.5.0-1', 'Latest' ] ) with self.assertRaisesRegexp(Exception,"Unsupported package type notype"): v.list_available('notype') with self.assertRaisesRegexp(Exception,"Unsupported package type newtype"): v.retrieve('3.5.1-5','newtype') # now create something that will be in cache only newpath = '%s/Unit-test/packages' % self._cachepath utils.mkdir_p(newpath) os.system('touch %s/calpont-infinidb-ent-19.0.1-1.x86_64.rpm.tar.gz' % newpath) self.assertEqual( sorted(v.list_available('rpm')), [ '2.2.11-1', '3.5.1-5', '4.0.0-0', 'Unit-test' ] ) pfile = v.retrieve('Unit-test','rpm') self.assertEqual( pfile, '19.0.1-1/packages/calpont-infinidb-ent-19.0.1-1.x86_64.rpm.tar.gz') self.assertEqual( v.get_pkg_version(pfile), '19.0.1-1' ) # not try to retrieve some files that don't exist with self.assertRaisesRegexp(Exception,"Unable to locate enterprise package for version Unit-test"): v.retrieve('Unit-test','deb') with self.assertRaisesRegexp(Exception,"Unable to locate enterprise package for version non-existent"): v.retrieve('non-existent','deb') with self.assertRaisesRegexp(Exception,"Unable to locate enterprise package for version Latest"): v.retrieve('Latest','rpm') pfile = v.retrieve('Latest','binary') self.assertEqual( pfile, '9.9-9/packages/infinidb-ent-9.9-9.x86_64.bin.tar.gz') self.assertEqual( v.get_pkg_version(pfile), '9.9-9' ) refpath = 'Latest/packages/infinidb-ent-9.9-9.x86_64.bin.tar.gz' self.assertEqual( os.path.getmtime('%s/%s' % (self._cachepath, pfile)), os.path.getmtime('%s/%s' % (self._pkgpath, refpath))) # now we want to force a situation where cached and local copies appear out of sync os.system('touch %s/%s' % (self._cachepath, pfile)) self.assertNotEqual( os.path.getmtime('%s/%s' % (self._cachepath, pfile)), os.path.getmtime('%s/%s' % (self._pkgpath, refpath))) pfile = v.retrieve('Latest','binary') self.assertEqual( pfile, '9.9-9/packages/infinidb-ent-9.9-9.x86_64.bin.tar.gz') self.assertEqual( os.path.getmtime('%s/%s' % (self._cachepath, pfile)), os.path.getmtime('%s/%s' % (self._pkgpath, refpath))) pfile = v.retrieve('4.0.0-0','binary') self.assertEqual( pfile, '4.0.0-0/packages/infinidb-ent-4.0.0-0.x86_64.bin.tar.gz') self.assertEqual( os.path.getmtime('%s/%s' % (self._cachepath, pfile)), os.path.getmtime('%s/%s' % (self._pkgpath, pfile))) self.assertEqual( v.get_pkg_version(pfile), '4.0.0-0' )
def main(argv): ''' main function ''' try: opts, args = getopt.getopt(argv, "hvi", ['json=']) except getopt.GetoptError: usage() sys.exit(2) # defaults use_stdin = False json_file = '' # parse command line arguments for o, a in opts: if o == '-h': usage() sys.exit(2) elif o == '-v': print 'installdatabase.py Version: %s' % version sys.exit(1) elif o == '-i': use_stdin = True elif o == '--json': json_file = a else: print 'unsupported option: %s' % o usage() sys.exit(2) if (use_stdin and json_file) or (not use_stdin and not json_file): print 'ERROR: Must specify exactly one of -i or --json' usage() sys.exit(2) try: # load input into json string jsonstr = None if use_stdin: lines = sys.stdin.readlines() jsonstr = ''.join(lines) elif json_file: f = open(json_file) lines = f.readlines() jsonstr = ''.join(lines) Log = logutils.getLogger('installdatabase') req = installreq.InstallReq(jsonstr) Log.info('request: %s' % req.json_dumps()) # construct configspec cfgspecbld = ConfigSpecBuilder(req) cfgspec, machines = cfgspecbld.run() # determine the approprate package file to be installed emVM = EmVersionManager() pkgfile = emVM.retrieve(cfgspec['idbversion'], 'binary') Log.info('pkgfile: %s' % pkgfile) # create runtime directory root = common.props['emtools.playbookmgr.cluster_base'] rundir = '%s/%s' % (root, cfgspec['name']) if not os.path.exists(rundir): mkdir_p(rundir) # create the cluster emCluster = EmCluster(cfgspec['name'], cfgspec, rundir, pkgfile, machines) # create the postconfig response file h = PostConfigureHelper() pfile = '%s/postconfigure.in' % rundir h.write_input(pfile, emCluster, 'binary') # perform the db install rc, results, out, err = emCluster.run_install_recipe() reply_dict = { 'cluster_name': cfgspec['name'], 'playbook_info': { 'name': emCluster.get_playbook_filename(), 'hostspec': emCluster.get_inventory_filename(), 'extravars': emCluster.get_extra_vars() }, 'rc': rc, 'stdout': out, 'stderr': err, 'recap_info': results } # test stub output #reply_dict = { # 'cluster_name' : cfgspec['name'], # 'playbook_info': { # 'name' : 'test_name', # 'hostspec' : 'test_hostspec', # 'extravars': 'test_extravars' # }, # 'rc' : 0, # 'stdout' : 'test_stdout', # 'stderr' : 'test_stderr', #} preply = playbookreply.PlaybookReply_from_dict(reply_dict) Log.info('reply: %s' % preply.json_dumps()) print preply return 0 except: import traceback print errormsg.ErrorMsg_from_parms( msg=json.dumps(traceback.format_exc())) sys.exit(1)