Example #1
0
    def execute(self):
        entry = ValidateVerificationEntry(token=self.token).run()
        if not entry['verdone']:
            raise WrongStateError(
                "Only finished verification tasks may be cleaned up.")

        path = os.path.join(get_masterdatadir(), 'pg_verify', self.token)
        Command('cleanup', 'rm -rf %s' % path).run(validateAfter=True)
        #RemoveTree(path).run()

        to_clean = ValidateVerification(content=entry['vercontent'],
                                        primaries_only=False).run()
        pool = WorkerPool(min(len(to_clean), self.batch_default))
        for seg in to_clean:
            host = seg.getSegmentHostName()
            path = os.path.join(seg.getSegmentDataDirectory(), 'pg_verify',
                                "*%s*" % self.token)
            cmd = Command('cleanup', 'rm -f %s' % path, remoteHost=host)
            pool.addCommand(cmd)

        logger.info('Waiting for clean commands to complete...')
        pool.wait_and_printdots(len(to_clean))

        for cmd in pool.getCompletedItems():
            res = cmd.get_results()
            if not res.wasSuccessful():
                logger.error('Failed to send cleanup on %s' % cmd.host)
                logger.error('Error: %s' % res.stderr)
                raise CleanVerificationError()
        RemoveVerificationEntry(token=self.token).run()
        logger.info('Verification %s has been cleaned.' % self.token)
Example #2
0
    def execute(self):
        entry = ValidateVerificationEntry(token = self.token).run()
        if not entry['verdone']:   
            raise WrongStateError("Only finished verification tasks may be cleaned up.")

        path = os.path.join(get_masterdatadir(), 'pg_verify', self.token)
        Command('cleanup', 'rm -rf %s' % path).run(validateAfter=True)
        #RemoveTree(path).run()

        to_clean = ValidateVerification(content = entry['vercontent'],
                                        primaries_only = False).run()
        pool = WorkerPool(min(len(to_clean), self.batch_default))
        for seg in to_clean:
            host = seg.getSegmentHostName()
            path = os.path.join(seg.getSegmentDataDirectory(), 'pg_verify', "*%s*" % self.token)
            cmd = Command('cleanup', 'rm -f %s' % path, remoteHost=host)
            pool.addCommand(cmd)

        logger.info('Waiting for clean commands to complete...')
        pool.wait_and_printdots(len(to_clean))

        for cmd in pool.getCompletedItems():
            res = cmd.get_results()
            if not res.wasSuccessful():
                logger.error('Failed to send cleanup on %s' % cmd.host)
                logger.error('Error: %s' % res.stderr)
                raise CleanVerificationError()
        RemoveVerificationEntry(token = self.token).run()
        logger.info('Verification %s has been cleaned.' % self.token)
Example #3
0
    def __init__(self,
                 masterDataDir,
                 readFromMasterCatalog,
                 timeout=None,
                 retries=None):
        """
        masterDataDir: if None then we try to find it from the system environment
        readFromMasterCatalog: if True then we will connect to the master in utility mode and fetch some more
                               data from there (like collation settings)

        """
        if masterDataDir is None:
            self.__masterDataDir = gp.get_masterdatadir()
        else:
            self.__masterDataDir = masterDataDir

        logger.debug("Obtaining master's port from master data directory")
        pgconf_dict = pgconf.readfile(self.__masterDataDir +
                                      "/postgresql.conf")
        self.__masterPort = pgconf_dict.int('port')
        logger.debug("Read from postgresql.conf port=%s" % self.__masterPort)
        self.__masterMaxConnections = pgconf_dict.int('max_connections')
        logger.debug("Read from postgresql.conf max_connections=%s" %
                     self.__masterMaxConnections)

        self.__gpHome = gp.get_gphome()
        self.__gpVersion = gp.GpVersion.local(
            'local GP software version check', self.__gpHome)
        logger.info("local Greenplum Version: '%s'" % self.__gpVersion)

        # read collation settings from master
        if readFromMasterCatalog:
            dbUrl = dbconn.DbURL(port=self.__masterPort,
                                 dbname='template1',
                                 timeout=timeout,
                                 retries=retries)
            conn = dbconn.connect(dbUrl, utility=True)
            (self.__lcCollate, self.__lcMonetary,
             self.__lcNumeric) = catalog.getCollationSettings(conn)

            # MPP-13807, read/show the master's database version too
            self.__pgVersion = dbconn.execSQLForSingletonRow(
                conn, "select version();")[0]
            logger.info("master Greenplum Version: '%s'" % self.__pgVersion)
            conn.close()

            checkNotNone("lc_collate", self.__lcCollate)
            checkNotNone("lc_monetary", self.__lcMonetary)
            checkNotNone("lc_numeric", self.__lcNumeric)
        else:
            self.__lcCollate = None
            self.__lcMonetary = None
            self.__lcNumeric = None
            self.__pgVersion = None
Example #4
0
    def execute(self):
        entry = ValidateVerificationEntry(token=self.token).run()
        to_inspect = ValidateVerification(content=entry['vercontent']).run()

        state_dict = ValidateCompletion(
            token=self.token,
            to_validate=to_inspect,
            batch_default=self.batch_default).run()
        incomplete = state_dict[VerificationState.RUNNING]
        if len(incomplete) > 0:
            # TODO: --force to consolidate files despite ongoing
            logger.error(
                'One or more content verifications is still in progress: %s' %
                incomplete)
            return entry

        GatherResults(master_datadir=get_masterdatadir(),
                      content=entry['vercontent'],
                      token=self.token,
                      batch_default=self.batch_default).run()

        state = VerificationState.SUCCEEDED
        mismatch = False
        aborted = state_dict[VerificationState.ABORTED]
        failed = state_dict[VerificationState.FAILED]
        if len(failed) > 0:  # any FAILED trumps ABORTED
            state = VerificationState.FAILED
            mismatch = True
            logger.warn(
                'One or more contents for verification %s were marked FAILED: %s'
                % (self.token, failed))
        elif len(aborted) > 0:
            state = VerificationState.ABORTED
            logger.warn(
                'One or more contents for verification %s were marked ABORTED: %s'
                % (self.token, aborted))
        else:
            logger.info('Verification %s completed successfully' % self.token)
        if not entry['verdone']:
            UpdateVerificationEntry(token=self.token,
                                    state=state,
                                    mismatch=mismatch,
                                    done=True).run()
            entry.update({
                'vermismatch': mismatch,
                'verdone': True,
                'verstate': state
            })
        return entry
Example #5
0
    def execute(self):
        entry = ValidateVerificationEntry(token=self.token).run()
        to_inspect = ValidateVerification(content=entry['vercontent']).run()

        state_dict = ValidateCompletion(
            token=self.token,
            to_validate=to_inspect,
            batch_default=self.batch_default).run()
        incomplete = state_dict[VerificationState.RUNNING]
        if len(incomplete) > 0:
            # TODO: --force to consolidate files despite ongoing
            logger.error(
                'One or more content verifications is still in progress: %s' %
                incomplete)
            return entry

        GatherResults(
            master_datadir=get_masterdatadir(),
            content=entry['vercontent'],
            token=self.token,
            batch_default=self.batch_default).run()

        state = VerificationState.SUCCEEDED
        mismatch = False
        aborted = state_dict[VerificationState.ABORTED]
        failed = state_dict[VerificationState.FAILED]
        if len(failed) > 0:  # any FAILED trumps ABORTED
            state = VerificationState.FAILED
            mismatch = True
            logger.warn(
                'One or more contents for verification %s were marked FAILED: %s'
                % (self.token, failed))
        elif len(aborted) > 0:
            state = VerificationState.ABORTED
            logger.warn(
                'One or more contents for verification %s were marked ABORTED: %s'
                % (self.token, aborted))
        else:
            logger.info('Verification %s completed successfully' % self.token)
        if not entry['verdone']:
            UpdateVerificationEntry(
                token=self.token, state=state, mismatch=mismatch,
                done=True).run()
            entry.update({
                'vermismatch': mismatch,
                'verdone': True,
                'verstate': state
            })
        return entry
Example #6
0
    def __init__(self, mainOptions):
        self.pidfilename = mainOptions.get('pidfilename', None)  # the file we're using for locking
        self.parentpidvar = mainOptions.get('parentpidvar', None)  # environment variable holding parent pid
        self.parentpid = None  # parent pid which already has the lock
        self.ppath = None  # complete path to the lock file
        self.pidlockfile = None  # PIDLockFile object
        self.pidfilepid = None  # pid of the process which has the lock
        self.locktorelease = None  # PIDLockFile object we should release when done

        if self.parentpidvar is not None and self.parentpidvar in os.environ:
            self.parentpid = int(os.environ[self.parentpidvar])

        if self.pidfilename is not None:
            self.ppath = os.path.join(gp.get_masterdatadir(), self.pidfilename)
            self.pidlockfile = PIDLockFile(self.ppath)
Example #7
0
 def _do_results(self, token, batch_default):
     if self.token is not None:
         entry = FinalizeVerification(token=token, batch_default=batch_default).run()
         entries = [entry]
     else:
         entries = FinalizeAllVerifications(batch_default).run()
     master_datadir = get_masterdatadir()
     for entry in entries:
         logger.info("---------------------------")
         logger.info("Token:    %s" % entry["vertoken"])
         logger.info("Type:     %s" % VerificationType.lookup[entry["vertype"]])
         logger.info("Content:  %s" % (entry["vercontent"] if entry["vercontent"] > 0 else "ALL"))
         logger.info("Started:  %s" % entry["verstarttime"])
         logger.info("State:    %s" % VerificationState.lookup[entry["verstate"]])
         if entry["verdone"]:
             path = os.path.join(master_datadir, "pg_verify", entry["vertoken"])
             logger.info("Details:  %s" % path)
Example #8
0
 def _do_results(self, token, batch_default):
     if self.token is not None:
         entry = FinalizeVerification(token = token,
                                      batch_default = batch_default).run()
         entries = [entry]
     else:
         entries = FinalizeAllVerifications(batch_default).run()
     master_datadir = get_masterdatadir()
     for entry in entries: 
         logger.info('---------------------------')
         logger.info('Token:    %s' % entry['vertoken'])
         logger.info('Type:     %s' % VerificationType.lookup[entry['vertype']])
         logger.info('Content:  %s' % (entry['vercontent'] if entry['vercontent'] > 0 else "ALL"))
         logger.info('Started:  %s' % entry['verstarttime'])
         logger.info('State:    %s' % VerificationState.lookup[entry['verstate']])
         if entry['verdone']:
             path = os.path.join(master_datadir, 'pg_verify', entry['vertoken'])
             logger.info('Details:  %s' % path)
Example #9
0
    def __init__(self, masterDataDir, readFromMasterCatalog, timeout=None, retries=None, verbose=True):
        """
        masterDataDir: if None then we try to find it from the system environment
        readFromMasterCatalog: if True then we will connect to the master in utility mode and fetch some more
                               data from there (like collation settings)

        """
        if masterDataDir is None:
            self.__masterDataDir = gp.get_masterdatadir()
        else: self.__masterDataDir = masterDataDir

        logger.debug("Obtaining master's port from master data directory")
        pgconf_dict = pgconf.readfile(self.__masterDataDir + "/postgresql.conf")
        self.__masterPort = pgconf_dict.int('port')
        logger.debug("Read from postgresql.conf port=%s" % self.__masterPort)
        self.__masterMaxConnections = pgconf_dict.int('max_connections')
        logger.debug("Read from postgresql.conf max_connections=%s" % self.__masterMaxConnections)

        self.__gpHome = gp.get_gphome()
        self.__gpVersion = gp.GpVersion.local('local GP software version check',self.__gpHome)
        
        if verbose:
            logger.info("local Greenplum Version: '%s'" % self.__gpVersion)

        # read collation settings from master
        if readFromMasterCatalog:
            dbUrl = dbconn.DbURL(port=self.__masterPort, dbname='template1', timeout=timeout, retries=retries)
            conn = dbconn.connect(dbUrl, utility=True)
            (self.__lcCollate, self.__lcMonetary, self.__lcNumeric) = catalog.getCollationSettings(conn)

            # MPP-13807, read/show the master's database version too
            self.__pgVersion = dbconn.execSQLForSingletonRow(conn, "select version();")[0]
            logger.info("master Greenplum Version: '%s'" % self.__pgVersion)
            conn.close()

            checkNotNone("lc_collate", self.__lcCollate)
            checkNotNone("lc_monetary", self.__lcMonetary)
            checkNotNone("lc_numeric", self.__lcNumeric)
        else:
            self.__lcCollate = None
            self.__lcMonetary = None
            self.__lcNumeric = None
            self.__pgVersion = None
Example #10
0
class Context(object):
    filename = os.path.join(gp.get_masterdatadir(), 'gpexpand.status')
    dbname = os.getenv('PGDATABASE', 'postgres')
    dburl = dbconn.DbURL(dbname=dbname)
    conn = dbconn.connect(dburl)
    day = 0
Example #11
0
                cmd.run(validateAfter=True)
                results = cmd.get_results().stdout.strip()
                rpm_version_string = results.split(' ')[-1]

                if not rpm_version_string.startswith('4.'):
                    raise ExceptionNoStackTraceNeeded(
                        'gppkg requires rpm version 4.x')

            except ExecutionError, ex:
                results = ex.cmd.get_results().stderr.strip()
                if len(results) != 0 and 'not found' in results:
                    raise ExceptionNoStackTraceNeeded(
                        'gppkg requires RPM to be available in PATH')

        if self.master_datadir is None:
            self.master_datadir = gp.get_masterdatadir()
        self.master_port = self._get_master_port(self.master_datadir)

        self._get_gpdb_host_list()

        if self.migrate:
            MigratePackages(from_gphome=self.migrate[0],
                            to_gphome=self.migrate[1],
                            standby_host=self.standby_host,
                            segment_host_list=self.segment_host_list).run()
            return

        if self.install:
            pkg = Gppkg.from_package_path(self.install)
            InstallPackage(pkg, self.master_host, self.standby_host,
                           self.segment_host_list).run()
Example #12
0
class GpPkgProgram:
    """ This is the CLI entry point to package management code.  """
    def __init__(self, options, args):
        self.master_datadir = options.masterDataDirectory 

        # TODO: AK: Program logic should not be dictating master, standby, and segment information
        # In other words, the fundamental Operations should have APIs that preclude the need for this.
        self.master_host = None
        self.standby_host = None
        self.segment_host_list = None

        self.query = options.query
        self.build = options.build
        self.install = options.install
        self.remove = options.remove
        self.update = options.update
        self.clean = options.clean
        self.migrate = options.migrate

        # only one of the following may be provided: --install, --remove, --update, --query, --build, --clean, --migrate
        count = sum([1 for opt in ['install', 'remove', 'update', 'query', 'build', 'clean', 'migrate'] if getattr(self, opt)])
        if count != 1:
            raise ExceptionNoStackTraceNeeded('Exactly one of the following must be provided: --install, --remove, -update, --query, --clean, --migrate')

        if self.query:
            # gppkg -q can be supplemented with --info, --list, --all
            count = sum([1 for opt in ['info', 'list', 'all'] if options.__dict__[opt]])
            if count > 1:
                raise ExceptionNoStackTraceNeeded('For --query, at most one of the following can be provided: --info, --list, --all') 
            # for all query options other than --all, a package path must be provided
            if not options.all and len(args) != 1:
                raise ExceptionNoStackTraceNeeded('A package must be specified for -q, -q --info, and -q --list.')

            if options.info:
                self.query = (QueryPackage.INFO, args[0])
            elif options.list:
                self.query = (QueryPackage.LIST, args[0])
            elif options.all:
                self.query = (QueryPackage.ALL, None)
            else:
                self.query = (None, args[0])   
        elif self.migrate:
            if len(args) != 2:
                raise ExceptionNoStackTraceNeeded('Invalid syntax, expecting "gppkg --migrate <from_gphome> <to_gphome>".')
            self.migrate = (args[0], args[1])

    @staticmethod
    def create_parser():
        parser = OptParser(option_class=OptChecker,
            description="Greenplum Package Manager",
            version='%prog version $Revision: #1 $')
        parser.setHelp([])

        addStandardLoggingAndHelpOptions(parser, includeNonInteractiveOption=True)

        parser.remove_option('-q')
        parser.remove_option('-l')
        
        add_to = OptionGroup(parser, 'General Options')
        parser.add_option_group(add_to)

        addMasterDirectoryOptionForSingleClusterProgram(add_to)

        # TODO: AK: Eventually, these options may need to be flexible enough to accept mutiple packages
        # in one invocation. If so, the structure of this parser may need to change.
        add_to.add_option('-i', '--install', help='install the given gppkg', metavar='<package>')
        add_to.add_option('-u', '--update', help='update the given gppkg', metavar='<package>')
        add_to.add_option('-r', '--remove', help='remove the given gppkg', metavar='<name>-<version>')
        add_to.add_option('-q', '--query', help='query the gppkg database or a particular gppkg', action='store_true')
        add_to.add_option('-b', '--build', help='build a gppkg', metavar='<directory>')
        add_to.add_option('-c', '--clean', help='clean the cluster of the given gppkg', action='store_true')
        add_to.add_option('--migrate', help='migrate gppkgs from a separate $GPHOME', metavar='<from_gphome> <to_gphome>', action='store_true', default=False)

        add_to = OptionGroup(parser, 'Query Options')
        parser.add_option_group(add_to)
        add_to.add_option('--info', action='store_true', help='print information about the gppkg including name, version, description')
        add_to.add_option('--list', action='store_true', help='print all the files present in the gppkg')
        add_to.add_option('--all', action='store_true', help='print all the gppkgs installed by gppkg')

        return parser
            
    @staticmethod
    def create_program(options, args):
        """ TODO: AK: This convention may be unnecessary. """
        return GpPkgProgram(options, args)    

    def _get_gpdb_host_list(self):
        """
        TODO: AK: Get rid of this. Program logic should not be driving host list building .
        
            This method gets the host names 
            of all hosts in the gpdb array.
            It sets the following variables 
                GpPkgProgram.master_host to master
                GpPkgProgram.standby_host to standby
                GpPkgProgram.segment_host_list to segment hosts
        """
        
        logger.debug('_get_gpdb_host_list')
        
        #Get host list
        GPHOME = os.getenv('GPHOME')
        if GPHOME == '' or not GPHOME:
            logger.info('GPHOME is not set.')
            sys.exit(1)
        hawq_site = HawqXMLParser(GPHOME)
        hawq_site.get_all_values()
        master_port = hawq_site.hawq_dict['hawq_master_address_port']
        master_host = ""
        standby_host = None
        segment_host_list = []
        host_list = get_hawq_hostname_all(master_port)

        for host, status in host_list['master'].iteritems():
            master_host = host
        for host, status in host_list['standby'].iteritems():
            standby_host = host
        for host, status in host_list['segment'].iteritems():
            segment_host_list.append(host)

        #Deduplicate the hosts so that we 
        #dont install multiple times on the same host
        segment_host_list = list(set(segment_host_list))

        #Segments might exist on the master host. Since we store the 
        #master host separately in self.master_host, storing the master_host
        #in the segment_host_list is redundant.
        for host in segment_host_list:
            if host == master_host or host == standby_host:
                segment_host_list.remove(host)

        self.master_host = master_host
        self.standby_host = standby_host
        self.segment_host_list = segment_host_list

    def _get_master_port(self, datadir):

        '''
            Obtain the master port from the pgconf file
        '''
       
        logger.debug('_get_master_port')
        pgconf_dict = pgconf.readfile(os.path.join(datadir, 'postgresql.conf'))
        return pgconf_dict.int('port')

    def run(self):
        if self.build:
            BuildGppkg(self.build).run()    
            return 

        #Check for RPM and Solaris OS
        if curr_platform == SUNOS:
            raise ExceptionNoStackTraceNeeded('gppkg is not supported on Solaris')
                  
        try:
            cmd = Command(name = 'Check for rpm', cmdStr = 'rpm --version')
            cmd.run(validateAfter = True)
            results = cmd.get_results().stdout.strip()
            rpm_version_string = results.split(' ')[-1]

            if not rpm_version_string.startswith('4.'):
                raise ExceptionNoStackTraceNeeded('gppkg requires rpm version 4.x')

        except ExecutionError, ex: 
            results = ex.cmd.get_results().stderr.strip()
            if len(results) != 0 and 'not found' in results:
                raise ExceptionNoStackTraceNeeded('gppkg requires RPM to be available in PATH') 

        if self.migrate:
            MigratePackages(from_gphome = self.migrate[0],
                            to_gphome = self.migrate[1]).run()
            return

        # MASTER_DATA_DIRECTORY and PGPORT must not need to be set for 
        # --build and --migrate to function properly
        if self.master_datadir is None:
            self.master_datadir = gp.get_masterdatadir()
        self.master_port = self._get_master_port(self.master_datadir)

        # TODO: AK: Program logic should not drive host decisions.
        self._get_gpdb_host_list()

        if self.install:
            pkg = Gppkg.from_package_path(self.install)
            InstallPackage(pkg, self.master_host, self.standby_host, self.segment_host_list).run() 
        elif self.query:
            query_type, package_path = self.query
            QueryPackage(query_type, package_path).run()
        elif self.remove:
            if self.remove.count('-') != 1:
                raise ExceptionNoStackTraceNeeded('Please specify the correct <name>-<version>.')
            pkg_file_list = ListFilesByPattern(GPPKG_ARCHIVE_PATH, self.remove + '-*-*' + GPPKG_EXTENSION).run() 
            if len(pkg_file_list) == 0:
                raise ExceptionNoStackTraceNeeded('Package %s has not been installed.' % self.remove)
            assert len(pkg_file_list) == 1
            pkg_file = pkg_file_list[0]
            pkg = Gppkg.from_package_path(os.path.join(GPPKG_ARCHIVE_PATH, pkg_file))
            UninstallPackage(pkg, self.master_host, self.standby_host, self.segment_host_list).run()
        elif self.update:
            pkg = Gppkg.from_package_path(self.update)
            UpdatePackage(pkg, self.master_host, self.standby_host, self.segment_host_list).run()
        elif self.clean:
            CleanGppkg(self.standby_host, self.segment_host_list).run()
Example #13
0
File: gppkg.py Project: pf-qiu/gpdb
            try:
                cmd = Command(name = 'Check for rpm', cmdStr = 'rpm --version')
                cmd.run(validateAfter = True)
                results = cmd.get_results().stdout.strip()
                rpm_version_string = results.split(' ')[-1]

                if not rpm_version_string.startswith('4.'):
                    raise ExceptionNoStackTraceNeeded('gppkg requires rpm version 4.x')

            except ExecutionError, ex:
                results = ex.cmd.get_results().stderr.strip()
                if len(results) != 0 and 'not found' in results:
                    raise ExceptionNoStackTraceNeeded('gppkg requires RPM to be available in PATH')

        if self.master_datadir is None:
            self.master_datadir = gp.get_masterdatadir()
        self.master_port = self._get_master_port(self.master_datadir)

        self._get_gpdb_host_list()

        if self.migrate:
            MigratePackages(from_gphome = self.migrate[0],
                            to_gphome = self.migrate[1],
                            standby_host = self.standby_host,
                            segment_host_list = self.segment_host_list
                            ).run()
            return

        if self.install:
            pkg = Gppkg.from_package_path(self.install)
            InstallPackage(pkg, self.master_host, self.standby_host, self.segment_host_list).run()
Example #14
0
class GpPkgProgram:
    """ This is the CLI entry point to package management code.  """
    def __init__(self, options, args):
        self.master_datadir = options.masterDataDirectory

        # TODO: AK: Program logic should not be dictating master, standby, and segment information
        # In other words, the fundamental Operations should have APIs that preclude the need for this.
        self.master_host = None
        self.standby_host = None
        self.segment_host_list = None

        self.query = options.query
        self.build = options.build
        self.install = options.install
        self.remove = options.remove
        self.update = options.update
        self.clean = options.clean
        self.migrate = options.migrate
        self.interactive = options.interactive
        self.filename = options.filename

        # only one of the following may be provided: --install, --remove, --update, --query, --build, --clean, --migrate
        count = sum([
            1 for opt in [
                'install', 'remove', 'update', 'query', 'build', 'clean',
                'migrate'
            ] if getattr(self, opt)
        ])
        if count != 1:
            raise ExceptionNoStackTraceNeeded(
                'Exactly one of the following must be provided: --install, --remove, -update, --query, --clean, --migrate'
            )

        if self.query:
            # gppkg -q can be supplemented with --info, --list, --all
            count = sum([
                1 for opt in ['info', 'list', 'all'] if options.__dict__[opt]
            ])
            if count > 1:
                raise ExceptionNoStackTraceNeeded(
                    'For --query, at most one of the following can be provided: --info, --list, --all'
                )
            # for all query options other than --all, a package path must be provided
            if not options.all and len(args) != 1:
                raise ExceptionNoStackTraceNeeded(
                    'A package must be specified for -q, -q --info, and -q --list.'
                )

            if options.info:
                self.query = (QueryPackage.INFO, args[0])
            elif options.list:
                self.query = (QueryPackage.LIST, args[0])
            elif options.all:
                self.query = (QueryPackage.ALL, None)
            else:
                self.query = (None, args[0])
        elif self.migrate:
            if len(args) != 2:
                raise ExceptionNoStackTraceNeeded(
                    'Invalid syntax, expecting "gppkg --migrate <from_gphome> <to_gphome>".'
                )
            self.migrate = (args[0], args[1])

        # gppkg should check gpexpand status
        check_result, msg = gp.conflict_with_gpexpand("gppkg",
                                                      refuse_phase1=True,
                                                      refuse_phase2=False)
        if not check_result:
            raise ExceptionNoStackTraceNeeded(msg)

    @staticmethod
    def create_parser():
        parser = OptParser(option_class=OptChecker,
                           description="Greenplum Package Manager",
                           version='%prog version $Revision: #1 $')
        parser.setHelp([])

        addStandardLoggingAndHelpOptions(parser,
                                         includeNonInteractiveOption=True)

        parser.remove_option('-q')
        parser.remove_option('-l')

        add_to = OptionGroup(parser, 'General Options')
        parser.add_option_group(add_to)

        addMasterDirectoryOptionForSingleClusterProgram(add_to)

        # TODO: AK: Eventually, these options may need to be flexible enough to accept mutiple packages
        # in one invocation. If so, the structure of this parser may need to change.
        add_to.add_option('-i',
                          '--install',
                          help='install the given gppkg',
                          metavar='<package>')
        add_to.add_option('-u',
                          '--update',
                          help='update the given gppkg',
                          metavar='<package>')
        add_to.add_option('-r',
                          '--remove',
                          help='remove the given gppkg',
                          metavar='<name>-<version>')
        add_to.add_option(
            '-q',
            '--query',
            help='query the gppkg database or a particular gppkg',
            action='store_true')
        add_to.add_option('-b',
                          '--build',
                          help='build a gppkg',
                          metavar='<directory>')
        add_to.add_option('-c',
                          '--clean',
                          help='clean the cluster of the given gppkg',
                          action='store_true')
        add_to.add_option('--migrate',
                          help='migrate gppkgs from a separate $GPHOME',
                          metavar='<from_gphome> <to_gphome>',
                          action='store_true',
                          default=False)
        add_to.add_option('-f',
                          '--filename',
                          help='set specific package name',
                          metavar='<name>')

        add_to = OptionGroup(parser, 'Query Options')
        parser.add_option_group(add_to)
        add_to.add_option(
            '--info',
            action='store_true',
            help=
            'print information about the gppkg including name, version, description'
        )
        add_to.add_option('--list',
                          action='store_true',
                          help='print all the files present in the gppkg')
        add_to.add_option('--all',
                          action='store_true',
                          help='print all the gppkgs installed by gppkg')

        return parser

    @staticmethod
    def create_program(options, args):
        """ TODO: AK: This convention may be unnecessary. """
        return GpPkgProgram(options, args)

    def _get_gpdb_host_list(self):
        """
        TODO: Perhaps the host list should be produced by gparray instead of here.

            This method gets the host names
            of all hosts in the gpdb array.
            It sets the following variables
                GpPkgProgram.master_host to master
                GpPkgProgram.standby_host to standby
                GpPkgProgram.segment_host_list to segment hosts
        """

        logger.debug('_get_gpdb_host_list')

        gparr = GpArray.initFromCatalog(dbconn.DbURL(port=self.master_port),
                                        utility=True)
        master_host = None
        standby_host = None
        segment_host_list = []

        segs = gparr.getDbList()

        for seg in segs:
            if seg.isSegmentMaster(current_role=True):
                master_host = seg.getSegmentHostName()
            elif seg.isSegmentStandby(current_role=True):
                standby_host = seg.getSegmentHostName()
            else:
                segment_host_list.append(seg.getSegmentHostName())

        # Deduplicate the hosts so that we
        # dont install multiple times on the same host
        segment_host_list = list(set(segment_host_list))

        # Segments might exist on the master host. Since we store the
        # master host separately in self.master_host, storing the master_host
        # in the segment_host_list is redundant.
        for host in segment_host_list:
            if host == master_host or host == standby_host:
                segment_host_list.remove(host)

        self.master_host = master_host
        self.standby_host = standby_host
        self.segment_host_list = segment_host_list

    def _get_master_port(self, datadir):
        '''
            Obtain the master port from the pgconf file
        '''

        logger.debug('_get_master_port')
        pgconf_dict = pgconf.readfile(os.path.join(datadir, 'postgresql.conf'))
        return pgconf_dict.int('port') or os.getenv('PGPORT')

    def run(self):
        if self.build:
            if self.filename:
                BuildGppkg(self.build, self.filename).run()
            else:
                BuildGppkg(self.build, None).run()
            return

        #Check for RPM and Solaris OS
        if curr_platform == SUNOS:
            raise ExceptionNoStackTraceNeeded(
                'gppkg is not supported on Solaris')

        try:
            if platform.linux_distribution()[0] == 'Ubuntu':
                cmd = Command(name='Check for dpkg', cmdStr='dpkg --version')
                cmd.run(validateAfter=True)
            else:
                cmd = Command(name='Check for rpm', cmdStr='rpm --version')
                cmd.run(validateAfter=True)
                results = cmd.get_results().stdout.strip()
                rpm_version_string = results.split(' ')[-1]

                if not rpm_version_string.startswith('4.'):
                    raise ExceptionNoStackTraceNeeded(
                        'gppkg requires rpm version 4.x')

        except ExecutionError, ex:
            results = ex.cmd.get_results().stderr.strip()
            if len(results) != 0 and 'not found' in results:
                raise ExceptionNoStackTraceNeeded(
                    'gppkg requires RPM to be available in PATH')

        if self.master_datadir is None:
            self.master_datadir = gp.get_masterdatadir()
        self.master_port = self._get_master_port(self.master_datadir)

        self._get_gpdb_host_list()

        if self.migrate:
            MigratePackages(from_gphome=self.migrate[0],
                            to_gphome=self.migrate[1],
                            standby_host=self.standby_host,
                            segment_host_list=self.segment_host_list).run()
            return

        if self.install:
            pkg = Gppkg.from_package_path(self.install)
            InstallPackage(pkg, self.master_host, self.standby_host,
                           self.segment_host_list).run()
        elif self.query:
            query_type, package_path = self.query
            QueryPackage(query_type, package_path).run()
        elif self.remove:
            # Check for exact match first, then use wildcard for what will be removed.
            pkg_file_list = ListFilesByPattern(
                GPPKG_ARCHIVE_PATH, self.remove + GPPKG_EXTENSION).run()
            if len(pkg_file_list) == 0:
                # now try wildcard
                pkg_file_list = ListFilesByPattern(
                    GPPKG_ARCHIVE_PATH,
                    self.remove + '*' + GPPKG_EXTENSION).run()
                if len(pkg_file_list) == 0:
                    raise ExceptionNoStackTraceNeeded(
                        'Package %s has not been installed.' % self.remove)

                # refuse to remove at all if the match is too broad, i.e., > 1
                if len(pkg_file_list) > 1:
                    err_msg = "Remove request '%s' too broad. " \
                              "Multiple packages match remove request: ( %s )." % (self.remove, ", ".join(pkg_file_list))
                    raise ExceptionNoStackTraceNeeded(err_msg)

            pkg_file = pkg_file_list[0]
            pkg = Gppkg.from_package_path(
                os.path.join(GPPKG_ARCHIVE_PATH, pkg_file))
            UninstallPackage(pkg, self.master_host, self.standby_host,
                             self.segment_host_list).run()
        elif self.update:
            logger.warning(
                'WARNING: The process of updating a package includes removing all'
            )
            logger.warning(
                'previous versions of the system objects related to the package. For'
            )
            logger.warning(
                'example, previous versions of shared libraries are removed.')
            logger.warning(
                'After the update process, a database function will fail when it is'
            )
            logger.warning(
                'called if the function references a package file that has been removed.'
            )
            if self.interactive:
                if not ask_yesno(None, 'Do you still want to continue ?', 'N'):
                    logger.info('Skipping update of gppkg based on user input')
                    return
            pkg = Gppkg.from_package_path(self.update)
            UpdatePackage(pkg, self.master_host, self.standby_host,
                          self.segment_host_list).run()
        elif self.clean:
            CleanGppkg(self.standby_host, self.segment_host_list).run()
Example #15
0
    def run(self):
        if self.build:
            if self.filename:
                BuildGppkg(self.build, self.filename).run()
            else:
                BuildGppkg(self.build, None).run()
            return

        if platform.linux_distribution()[0] == 'Ubuntu':
            try:
                cmd = Command(name='Check for dpkg', cmdStr='dpkg --version')
                cmd.run(validateAfter=True)
                cmd = Command(name='Check for fakeroot',
                              cmdStr='fakeroot --version')
                cmd.run(validateAfter=True)
            except Exception as ex:
                raise ExceptionNoStackTraceNeeded(
                    'fakeroot and dpkg are both required by gppkg')
        else:
            try:
                cmd = Command(name='Check for rpm', cmdStr='rpm --version')
                cmd.run(validateAfter=True)
                results = cmd.get_results().stdout.strip()
                rpm_version_string = results.split(' ')[-1]

                if not rpm_version_string.startswith('4.'):
                    raise ExceptionNoStackTraceNeeded(
                        'gppkg requires rpm version 4.x')

            except ExecutionError as ex:
                results = ex.cmd.get_results().stderr.strip()
                if len(results) != 0 and 'not found' in results:
                    raise ExceptionNoStackTraceNeeded(
                        'gppkg requires RPM to be available in PATH')

        if self.master_datadir is None:
            self.master_datadir = gp.get_masterdatadir()
        self.master_port = self._get_master_port(self.master_datadir)

        self._get_gpdb_host_list()

        if self.migrate:
            MigratePackages(from_gphome=self.migrate[0],
                            to_gphome=self.migrate[1],
                            standby_host=self.standby_host,
                            segment_host_list=self.segment_host_list).run()
            return

        if self.install:
            pkg = Gppkg.from_package_path(self.install)
            InstallPackage(pkg, self.master_host, self.standby_host,
                           self.segment_host_list).run()
        elif self.query:
            query_type, package_path = self.query
            QueryPackage(query_type, package_path).run()
        elif self.remove:
            # Check for exact match first, then use wildcard for what will be removed.
            pkg_file_list = ListFilesByPattern(
                GPPKG_ARCHIVE_PATH, self.remove + GPPKG_EXTENSION).run()
            if len(pkg_file_list) == 0:
                # now try wildcard
                pkg_file_list = ListFilesByPattern(
                    GPPKG_ARCHIVE_PATH,
                    self.remove + '*' + GPPKG_EXTENSION).run()
                if len(pkg_file_list) == 0:
                    raise ExceptionNoStackTraceNeeded(
                        'Package %s has not been installed.' % self.remove)

                # refuse to remove at all if the match is too broad, i.e., > 1
                if len(pkg_file_list) > 1:
                    err_msg = "Remove request '%s' too broad. " \
                              "Multiple packages match remove request: ( %s )." % (self.remove, ", ".join(pkg_file_list))
                    raise ExceptionNoStackTraceNeeded(err_msg)

            pkg_file = pkg_file_list[0]
            pkg = Gppkg.from_package_path(
                os.path.join(GPPKG_ARCHIVE_PATH, pkg_file))
            UninstallPackage(pkg, self.master_host, self.standby_host,
                             self.segment_host_list).run()
        elif self.update:
            logger.warning(
                'WARNING: The process of updating a package includes removing all'
            )
            logger.warning(
                'previous versions of the system objects related to the package. For'
            )
            logger.warning(
                'example, previous versions of shared libraries are removed.')
            logger.warning(
                'After the update process, a database function will fail when it is'
            )
            logger.warning(
                'called if the function references a package file that has been removed.'
            )
            if self.interactive:
                if not ask_yesno(None, 'Do you still want to continue ?', 'N'):
                    logger.info('Skipping update of gppkg based on user input')
                    return
            pkg = Gppkg.from_package_path(self.update)
            UpdatePackage(pkg, self.master_host, self.standby_host,
                          self.segment_host_list).run()
        elif self.clean:
            CleanGppkg(self.standby_host, self.segment_host_list).run()