Example #1
0
def parseargs(args):
    global logger

    pguser = os.environ.get("PGUSER") or unix.getUserName()
    pghost = os.environ.get("PGHOST") or unix.getLocalHostname()
    pgport = os.environ.get("PGPORT") or 5432

    parser = OptParser(option_class=OptChecker)
    parser.remove_option('-h')
    parser.add_option('-?', '--help', '-h', action='store_true', default=False)
    parser.add_option('-n', '--host', default=pghost)
    parser.add_option('-p', '--port', default=pgport)
    parser.add_option('-u', '--username', default=pguser)
    parser.add_option('-w', '--password', default=False, action='store_true')
    parser.add_option('-v', '--verbose', default=False, action='store_true')
    parser.add_option('-q', '--quiet', default=True, action='store_true')

    (options, args) = parser.parse_args()

    if options.help:
        print __doc__
        sys.exit(1)
    try:
        options.port = int(options.port)
    except:
        logger.error("Invalid PORT: '%s'" % options.port)
        sys.exit(1)

    if options.verbose:
        gplog.enable_verbose_logging()
    elif options.quiet:
        gplog.quiet_stdout_logging()

    return options
Example #2
0
def parseargs(args):
    global logger

    pguser = os.environ.get("PGUSER") or unix.getUserName()
    pghost = os.environ.get("PGHOST") or unix.getLocalHostname()
    pgport = os.environ.get("PGPORT") or 5432

    parser = OptParser(option_class=OptChecker)
    parser.remove_option('-h')
    parser.add_option('-?', '--help', '-h', action='store_true', default=False)
    parser.add_option('-n', '--host', default=pghost)
    parser.add_option('-p', '--port', default=pgport)
    parser.add_option('-u', '--username', default=pguser)
    parser.add_option('-w', '--password', default=False, action='store_true')
    parser.add_option('-v', '--verbose', default=False, action='store_true')
    parser.add_option('-q', '--quiet', default=True, action='store_true')

    (options, args) = parser.parse_args()

    if options.help:
        print __doc__
        sys.exit(1)
    try:
        options.port = int(options.port)
    except:
        logger.error("Invalid PORT: '%s'" % options.port)
        sys.exit(1)

    if options.verbose:
        gplog.enable_verbose_logging()
    elif options.quiet:
        gplog.quiet_stdout_logging()

    return options
Example #3
0
def test_main( testName, newProgramArgs, createOptionParserFn, createCommandFn, extraOutputGenerators, expectedOutput) :
    global gTestOutput

    # update args
    previousArgs = sys.argv
    sys.argv = []
    sys.argv.append(getProgramName())
    sys.argv.extend(newProgramArgs)

    # register command factory
    setExecutionContextFactory(TestExecutionContextFactory())

    commandObject=None
    parser = None

    startTest(testName)

    try:
        gplog.setup_tool_logging(gProgramName,unix.getLocalHostname(),unix.getUserName(),nonuser=False)

        parser = createOptionParserFn()
        (options, args) = parser.parse_args()
        gplog.enable_verbose_logging()

        commandObject = createCommandFn(options, args)
        exitCode = commandObject.run()

        testOutput("sys.exit %s" % exitCode)

    except ProgramArgumentValidationException, e:
        testOutput( "Validation error: %s" % e.getMessage())
Example #4
0
def test_main(testName, newProgramArgs, createOptionParserFn, createCommandFn,
              extraOutputGenerators, expectedOutput):
    global gTestOutput

    # update args
    previousArgs = sys.argv
    sys.argv = []
    sys.argv.append(getProgramName())
    sys.argv.extend(newProgramArgs)

    # register command factory
    setExecutionContextFactory(TestExecutionContextFactory())

    commandObject = None
    parser = None

    startTest(testName)

    try:
        gplog.setup_tool_logging(gProgramName,
                                 unix.getLocalHostname(),
                                 unix.getUserName(),
                                 nonuser=False)

        parser = createOptionParserFn()
        (options, args) = parser.parse_args()
        gplog.enable_verbose_logging()

        commandObject = createCommandFn(options, args)
        exitCode = commandObject.run()

        testOutput("sys.exit %s" % exitCode)

    except ProgramArgumentValidationException, e:
        testOutput("Validation error: %s" % e.getMessage())
Example #5
0
def main():
    options = parseargs()
    if options.verbose:
        gplog.enable_verbose_logging()
    if prepare(options.dbname, options.sqldir, options.gpdbgen) < 0:
        return
    if not options.nopython:
        queries = QUERIES_PYTHON
        if options.gpdbgen == '5':
            queries += QUERIES_PYTHON_5
        if prepare_funcs("python", options.dbname, options.sqldir,
                         options.gpdbgen) < 0:
            return
        run(options.dbname, options.username, options.time, queries, "python")
    if not options.nopython3:
        queries = QUERIES_PYTHON
        if options.gpdbgen == '5':
            queries += QUERIES_PYTHON_5
        if prepare_funcs("python3", options.dbname, options.sqldir,
                         options.gpdbgen) < 0:
            return
        run(options.dbname, options.username, options.time, queries, "python3")
    if not options.nor:
        queries = QUERIES_R
        if options.gpdbgen == '5':
            queries += QUERIES_R_5
        if prepare_funcs("R", options.dbname, options.sqldir,
                         options.gpdbgen) < 0:
            return
        run(options.dbname, options.username, options.time, queries, "R")
Example #6
0
    def main(self, file_name, get_cmd_list):
        pool = None
        logger = None
        try:
            options = self.parseargs()
            exec_name = os.path.split(file_name)[-1]
            logger = gplog.setup_tool_logging(exec_name,
                                              unix.getLocalHostname(),
                                              unix.getUserName(),
                                              logdir=options.logfileDirectory)

            if not options.confinfo:
                raise Exception('Missing --confinfo argument.')

            if options.batch_size <= 0:
                logger.warn('batch_size was less than zero.  Setting to 1.')
                options.batch_size = 1

            if options.verbose:
                gplog.enable_verbose_logging()

            # TODO: should we output the name of the exact file?
            logger.info("Starting recovery with args: %s" %
                        ' '.join(sys.argv[1:]))

            seg_recovery_info_list = recoveryinfo.deserialize_recovery_info_list(
                options.confinfo)
            if len(seg_recovery_info_list) == 0:
                raise Exception(
                    'No segment configuration values found in --confinfo argument'
                )

            cmd_list = get_cmd_list(seg_recovery_info_list,
                                    options.forceoverwrite, logger)

            pool = WorkerPool(
                numWorkers=min(options.batch_size, len(cmd_list)))
            self.run_cmd_list(cmd_list, logger, options, pool)

            sys.exit(0)
        except Exception as e:
            if logger:
                logger.error(str(e))
            print(e, file=sys.stderr)
            sys.exit(1)
        finally:
            if pool:
                pool.haltWork()
Example #7
0
def simple_main_locked(createOptionParserFn, createCommandFn, mainOptions):
    """
    Not to be called externally -- use simple_main instead
    """
    logger = gplog.get_default_logger()

    configurationInterface.registerConfigurationProvider(
        configurationImplGpdb.GpConfigurationProviderUsingGpdbCatalog())
    fileSystemInterface.registerFileSystemProvider(
        fileSystemImplOs.GpFileSystemProviderUsingOs())
    osInterface.registerOsProvider(osImplNative.GpOsProviderUsingNative())
    faultProberInterface.registerFaultProber(
        faultProberImplGpdb.GpFaultProberImplGpdb())

    commandObject = None
    parser = None

    forceQuiet = mainOptions is not None and mainOptions.get(
        "forceQuietOutput")
    options = None

    if mainOptions is not None and mainOptions.get("programNameOverride"):
        global gProgramName
        gProgramName = mainOptions.get("programNameOverride")
    suppressStartupLogMessage = mainOptions is not None and mainOptions.get(
        "suppressStartupLogMessage")

    useHelperToolLogging = mainOptions is not None and mainOptions.get(
        "useHelperToolLogging")
    nonuser = True if mainOptions is not None and mainOptions.get(
        "setNonuserOnToolLogger") else False
    exit_status = 1

    try:
        execname = getProgramName()
        hostname = unix.getLocalHostname()
        username = unix.getUserName()

        parser = createOptionParserFn()
        (options, args) = parser.parse_args()

        if useHelperToolLogging:
            gplog.setup_helper_tool_logging(execname, hostname, username)
        else:
            gplog.setup_tool_logging(execname,
                                     hostname,
                                     username,
                                     logdir=options.ensure_value(
                                         "logfileDirectory", None),
                                     nonuser=nonuser)

        if forceQuiet:
            gplog.quiet_stdout_logging()
        else:
            if options.ensure_value("verbose", False):
                gplog.enable_verbose_logging()
            if options.ensure_value("quiet", False):
                gplog.quiet_stdout_logging()

        if options.ensure_value("masterDataDirectory", None) is not None:
            options.master_data_directory = os.path.abspath(
                options.masterDataDirectory)

        if not suppressStartupLogMessage:
            logger.info("Starting %s with args: %s" %
                        (gProgramName, ' '.join(sys.argv[1:])))

        commandObject = createCommandFn(options, args)
        exitCode = commandObject.run()
        exit_status = exitCode

    except ProgramArgumentValidationException, e:
        if e.shouldPrintHelp():
            parser.print_help()
        logger.error("%s: error: %s" % (gProgramName, e.getMessage()))
        exit_status = 2
Example #8
0
def simple_main_locked(createOptionParserFn, createCommandFn, mainOptions):
    """
    Not to be called externally -- use simple_main instead
    """
    logger = gplog.get_default_logger()

    configurationInterface.registerConfigurationProvider(
        configurationImplGpdb.GpConfigurationProviderUsingGpdbCatalog())
    fileSystemInterface.registerFileSystemProvider(fileSystemImplOs.GpFileSystemProviderUsingOs())
    osInterface.registerOsProvider(osImplNative.GpOsProviderUsingNative())
    faultProberInterface.registerFaultProber(faultProberImplGpdb.GpFaultProberImplGpdb())

    commandObject = None
    parser = None

    forceQuiet = mainOptions is not None and mainOptions.get("forceQuietOutput")
    options = None

    if mainOptions is not None and mainOptions.get("programNameOverride"):
        global gProgramName
        gProgramName = mainOptions.get("programNameOverride")
    suppressStartupLogMessage = mainOptions is not None and mainOptions.get("suppressStartupLogMessage")

    useHelperToolLogging = mainOptions is not None and mainOptions.get("useHelperToolLogging")
    nonuser = True if mainOptions is not None and mainOptions.get("setNonuserOnToolLogger") else False
    exit_status = 1

    # NOTE: if this logic is changed then also change test_main in testUtils.py
    try:
        execname = getProgramName()
        hostname = unix.getLocalHostname()
        username = unix.getUserName()

        parser = createOptionParserFn()
        (options, args) = parser.parse_args()

        if useHelperToolLogging:
            gplog.setup_helper_tool_logging(execname, hostname, username)
        else:
            gplog.setup_tool_logging(execname, hostname, username,
                                     logdir=options.ensure_value("logfileDirectory", None), nonuser=nonuser)

        if forceQuiet:
            gplog.quiet_stdout_logging()
        else:
            if options.ensure_value("verbose", False):
                gplog.enable_verbose_logging()
            if options.ensure_value("quiet", False):
                gplog.quiet_stdout_logging()

        if options.ensure_value("masterDataDirectory", None) is not None:
            options.master_data_directory = os.path.abspath(options.masterDataDirectory)

        if not suppressStartupLogMessage:
            logger.info("Starting %s with args: %s" % (gProgramName, ' '.join(sys.argv[1:])))

        commandObject = createCommandFn(options, args)
        exitCode = commandObject.run()
        exit_status = exitCode

    except ProgramArgumentValidationException, e:
        if e.shouldPrintHelp():
            parser.print_help()
        logger.error("%s: error: %s" % (gProgramName, e.getMessage()))
        exit_status = 2
# Copyright (c) Greenplum Inc 2008. All Rights Reserved.
#
""" Unittesting for pygres module
"""
import logging
import unittest

import pg
import pgdb

from gppylib import gplog
from gppylib.db.dbconn import *
from gppylib.db.test import skipIfDatabaseDown

logger = gplog.get_default_logger()
gplog.enable_verbose_logging()

#TODO:  incomplete list.
catalog_names = [
    'pg_aggregate', 'pg_amop', 'pg_attrdef', 'pg_auth_members',
    'pg_autovacuum', 'pg_class', 'pg_conversion', 'pg_database',
    'pg_description', 'pg_group', 'pg_indexes', 'pg_language',
    'pg_max_external_files', 'pg_opclass', 'pg_partition'
]

datatypes = {
    'oid': True,
    'int2': True,
    'regproc': True,
    'text': True,
    'bool': True,
Example #10
0
#
# Unit Testing of pg commands
#

import os
import unittest
import tempfile

from gppylib.db import dbconn
from gppylib.db.test import skipIfDatabaseDown
from gppylib import gplog
from gppylib.commands import pg
from gppylib.gparray import GpArray

logger = gplog.get_default_logger()
gplog.enable_verbose_logging()

@skipIfDatabaseDown()
class PgCommandsTestCase(unittest.TestCase):
    def setUp(self):
         pass
     
    def tearDown(self):
        pass
    
    def testReadPostmasterTempFile(self):
        logger.info("testReadPostmasterTempFile")
        url = dbconn.DbURL()
        gpdb = GpArray.initFromCatalog(url)
        
        logger.info("Search for valid master port: %s" % gpdb.master.port)
Example #11
0
def simple_main_locked(createOptionParserFn, createCommandFn, mainOptions):
    """
    Not to be called externally -- use simple_main instead
    """
    logger = gplog.get_default_logger()

    configurationInterface.registerConfigurationProvider(
        configurationImplGpdb.GpConfigurationProviderUsingGpdbCatalog())
    fileSystemInterface.registerFileSystemProvider(fileSystemImplOs.GpFileSystemProviderUsingOs())
    osInterface.registerOsProvider(osImplNative.GpOsProviderUsingNative())
    faultProberInterface.registerFaultProber(faultProberImplGpdb.GpFaultProberImplGpdb())

    commandObject = None
    parser = None

    forceQuiet = mainOptions is not None and mainOptions.get("forceQuietOutput")
    options = None

    if mainOptions is not None and mainOptions.get("programNameOverride"):
        global gProgramName
        gProgramName = mainOptions.get("programNameOverride")
    suppressStartupLogMessage = mainOptions is not None and mainOptions.get("suppressStartupLogMessage")

    useHelperToolLogging = mainOptions is not None and mainOptions.get("useHelperToolLogging")
    nonuser = True if mainOptions is not None and mainOptions.get("setNonuserOnToolLogger") else False
    exit_status = 1

    try:
        execname = getProgramName()
        hostname = unix.getLocalHostname()
        username = unix.getUserName()

        parser = createOptionParserFn()
        (options, args) = parser.parse_args()

        if useHelperToolLogging:
            gplog.setup_helper_tool_logging(execname, hostname, username)
        else:
            gplog.setup_tool_logging(execname, hostname, username,
                                     logdir=options.ensure_value("logfileDirectory", None), nonuser=nonuser)

        if forceQuiet:
            gplog.quiet_stdout_logging()
        else:
            if options.ensure_value("verbose", False):
                gplog.enable_verbose_logging()
            if options.ensure_value("quiet", False):
                gplog.quiet_stdout_logging()

        if options.ensure_value("coordinatorDataDirectory", None) is not None:
            options.coordinator_data_directory = os.path.abspath(options.coordinatorDataDirectory)

        if not suppressStartupLogMessage:
            logger.info("Starting %s with args: %s" % (gProgramName, ' '.join(sys.argv[1:])))

        commandObject = createCommandFn(options, args)
        exitCode = commandObject.run()
        exit_status = exitCode

    except ProgramArgumentValidationException as e:
        if e.shouldPrintHelp():
            parser.print_help()
        logger.error("%s: error: %s" % (gProgramName, e.getMessage()))
        exit_status = 2
    except ExceptionNoStackTraceNeeded as e:
        logger.error("%s error: %s" % (gProgramName, e))
        exit_status = 2
    except UserAbortedException as e:
        logger.info("User abort requested, Exiting...")
        exit_status = 4
    except ExecutionError as e:
        logger.fatal("Error occurred: %s\n Command was: '%s'\n"
                     "rc=%d, stdout='%s', stderr='%s'" % \
                     (e.summary, e.cmd.cmdStr, e.cmd.results.rc, e.cmd.results.stdout,
                      e.cmd.results.stderr))
        exit_status = 2
    except Exception as e:
        if options is None:
            logger.exception("%s failed.  exiting...", gProgramName)
        else:
            if options.ensure_value("verbose", False):
                logger.exception("%s failed.  exiting...", gProgramName)
            else:
                logger.fatal("%s failed. (Reason='%s') exiting..." % (gProgramName, e))
        exit_status = 2
    except KeyboardInterrupt:
        exit_status = 2
    finally:
        if commandObject:
            commandObject.cleanup()
    sys.exit(exit_status)
Example #12
0
    def parseargs(self):
        parser = OptParser(option_class=OptChecker,
                           description=' '.join(self.description.split()),
                           version='%prog version $Revision: $')
        parser.set_usage(
            '%prog is a utility script used by gprecoverseg, and gpaddmirrors and is not intended to be run separately.'
        )
        parser.remove_option('-h')

        #TODO we may not need the verbose flag
        parser.add_option('-v',
                          '--verbose',
                          action='store_true',
                          help='debug output.',
                          default=False)
        parser.add_option('-c', '--confinfo', type='string')
        parser.add_option('-b',
                          '--batch-size',
                          type='int',
                          default=DEFAULT_SEGHOST_NUM_WORKERS,
                          metavar='<batch_size>')
        parser.add_option('-f',
                          '--force-overwrite',
                          dest='forceoverwrite',
                          action='store_true',
                          default=False)
        parser.add_option('-l',
                          '--log-dir',
                          dest="logfileDirectory",
                          type="string")
        parser.add_option(
            '',
            '--era',
            dest="era",
            help="coordinator era",
        )

        # Parse the command line arguments
        self.options, _ = parser.parse_args()

        if not self.options.confinfo:
            raise Exception('Missing --confinfo argument.')
        if not self.options.logfileDirectory:
            raise Exception('Missing --log-dir argument.')

        self.logger = gplog.setup_tool_logging(
            os.path.split(self.file_name)[-1],
            unix.getLocalHostname(),
            unix.getUserName(),
            logdir=self.options.logfileDirectory)

        if self.options.batch_size <= 0:
            self.logger.warn('batch_size was less than zero.  Setting to 1.')
            self.options.batch_size = 1

        if self.options.verbose:
            gplog.enable_verbose_logging()

        self.seg_recovery_info_list = recoveryinfo.deserialize_list(
            self.options.confinfo)
        if len(self.seg_recovery_info_list) == 0:
            raise Exception(
                'No segment configuration values found in --confinfo argument')