def parseargs(args):
global logger
pguser = os.environ.get("PGUSER") or unix.getUserName()
pghost = os.environ.get("PGHOST") or unix.getLocalHostname()
pgport = os.environ.get("PGPORT") or 5432
parser = OptParser(option_class=OptChecker)
parser.remove_option('-h')
parser.add_option('-?', '--help', '-h', action='store_true', default=False)
parser.add_option('-n', '--host', default=pghost)
parser.add_option('-p', '--port', default=pgport)
parser.add_option('-u', '--username', default=pguser)
parser.add_option('-w', '--password', default=False, action='store_true')
parser.add_option('-v', '--verbose', default=False, action='store_true')
parser.add_option('-q', '--quiet', default=True, action='store_true')
(options, args) = parser.parse_args()
if options.help:
print __doc__
sys.exit(1)
try:
options.port = int(options.port)
except:
logger.error("Invalid PORT: '%s'" % options.port)
sys.exit(1)
if options.verbose:
gplog.enable_verbose_logging()
elif options.quiet:
gplog.quiet_stdout_logging()
return options
def parseargs(args):
global logger
pguser = os.environ.get("PGUSER") or unix.getUserName()
pghost = os.environ.get("PGHOST") or unix.getLocalHostname()
pgport = os.environ.get("PGPORT") or 5432
parser = OptParser(option_class=OptChecker)
parser.remove_option('-h')
parser.add_option('-?', '--help', '-h', action='store_true', default=False)
parser.add_option('-n', '--host', default=pghost)
parser.add_option('-p', '--port', default=pgport)
parser.add_option('-u', '--username', default=pguser)
parser.add_option('-w', '--password', default=False, action='store_true')
parser.add_option('-v', '--verbose', default=False, action='store_true')
parser.add_option('-q', '--quiet', default=True, action='store_true')
(options, args) = parser.parse_args()
if options.help:
print __doc__
sys.exit(1)
try:
options.port = int(options.port)
except:
logger.error("Invalid PORT: '%s'" % options.port)
sys.exit(1)
if options.verbose:
gplog.enable_verbose_logging()
elif options.quiet:
gplog.quiet_stdout_logging()
return options
def test_main( testName, newProgramArgs, createOptionParserFn, createCommandFn, extraOutputGenerators, expectedOutput) :
global gTestOutput
# update args
previousArgs = sys.argv
sys.argv = []
sys.argv.append(getProgramName())
sys.argv.extend(newProgramArgs)
# register command factory
setExecutionContextFactory(TestExecutionContextFactory())
commandObject=None
parser = None
startTest(testName)
try:
gplog.setup_tool_logging(gProgramName,unix.getLocalHostname(),unix.getUserName(),nonuser=False)
parser = createOptionParserFn()
(options, args) = parser.parse_args()
gplog.enable_verbose_logging()
commandObject = createCommandFn(options, args)
exitCode = commandObject.run()
testOutput("sys.exit %s" % exitCode)
except ProgramArgumentValidationException, e:
testOutput( "Validation error: %s" % e.getMessage())
def test_main(testName, newProgramArgs, createOptionParserFn, createCommandFn,
extraOutputGenerators, expectedOutput):
global gTestOutput
# update args
previousArgs = sys.argv
sys.argv = []
sys.argv.append(getProgramName())
sys.argv.extend(newProgramArgs)
# register command factory
setExecutionContextFactory(TestExecutionContextFactory())
commandObject = None
parser = None
startTest(testName)
try:
gplog.setup_tool_logging(gProgramName,
unix.getLocalHostname(),
unix.getUserName(),
nonuser=False)
parser = createOptionParserFn()
(options, args) = parser.parse_args()
gplog.enable_verbose_logging()
commandObject = createCommandFn(options, args)
exitCode = commandObject.run()
testOutput("sys.exit %s" % exitCode)
except ProgramArgumentValidationException, e:
testOutput("Validation error: %s" % e.getMessage())
def main():
options = parseargs()
if options.verbose:
gplog.enable_verbose_logging()
if prepare(options.dbname, options.sqldir, options.gpdbgen) < 0:
return
if not options.nopython:
queries = QUERIES_PYTHON
if options.gpdbgen == '5':
queries += QUERIES_PYTHON_5
if prepare_funcs("python", options.dbname, options.sqldir,
options.gpdbgen) < 0:
return
run(options.dbname, options.username, options.time, queries, "python")
if not options.nopython3:
queries = QUERIES_PYTHON
if options.gpdbgen == '5':
queries += QUERIES_PYTHON_5
if prepare_funcs("python3", options.dbname, options.sqldir,
options.gpdbgen) < 0:
return
run(options.dbname, options.username, options.time, queries, "python3")
if not options.nor:
queries = QUERIES_R
if options.gpdbgen == '5':
queries += QUERIES_R_5
if prepare_funcs("R", options.dbname, options.sqldir,
options.gpdbgen) < 0:
return
run(options.dbname, options.username, options.time, queries, "R")
def main(self, file_name, get_cmd_list):
pool = None
logger = None
try:
options = self.parseargs()
exec_name = os.path.split(file_name)[-1]
logger = gplog.setup_tool_logging(exec_name,
unix.getLocalHostname(),
unix.getUserName(),
logdir=options.logfileDirectory)
if not options.confinfo:
raise Exception('Missing --confinfo argument.')
if options.batch_size <= 0:
logger.warn('batch_size was less than zero. Setting to 1.')
options.batch_size = 1
if options.verbose:
gplog.enable_verbose_logging()
# TODO: should we output the name of the exact file?
logger.info("Starting recovery with args: %s" %
' '.join(sys.argv[1:]))
seg_recovery_info_list = recoveryinfo.deserialize_recovery_info_list(
options.confinfo)
if len(seg_recovery_info_list) == 0:
raise Exception(
'No segment configuration values found in --confinfo argument'
)
cmd_list = get_cmd_list(seg_recovery_info_list,
options.forceoverwrite, logger)
pool = WorkerPool(
numWorkers=min(options.batch_size, len(cmd_list)))
self.run_cmd_list(cmd_list, logger, options, pool)
sys.exit(0)
except Exception as e:
if logger:
logger.error(str(e))
print(e, file=sys.stderr)
sys.exit(1)
finally:
if pool:
pool.haltWork()
def simple_main_locked(createOptionParserFn, createCommandFn, mainOptions):
"""
Not to be called externally -- use simple_main instead
"""
logger = gplog.get_default_logger()
configurationInterface.registerConfigurationProvider(
configurationImplGpdb.GpConfigurationProviderUsingGpdbCatalog())
fileSystemInterface.registerFileSystemProvider(
fileSystemImplOs.GpFileSystemProviderUsingOs())
osInterface.registerOsProvider(osImplNative.GpOsProviderUsingNative())
faultProberInterface.registerFaultProber(
faultProberImplGpdb.GpFaultProberImplGpdb())
commandObject = None
parser = None
forceQuiet = mainOptions is not None and mainOptions.get(
"forceQuietOutput")
options = None
if mainOptions is not None and mainOptions.get("programNameOverride"):
global gProgramName
gProgramName = mainOptions.get("programNameOverride")
suppressStartupLogMessage = mainOptions is not None and mainOptions.get(
"suppressStartupLogMessage")
useHelperToolLogging = mainOptions is not None and mainOptions.get(
"useHelperToolLogging")
nonuser = True if mainOptions is not None and mainOptions.get(
"setNonuserOnToolLogger") else False
exit_status = 1
try:
execname = getProgramName()
hostname = unix.getLocalHostname()
username = unix.getUserName()
parser = createOptionParserFn()
(options, args) = parser.parse_args()
if useHelperToolLogging:
gplog.setup_helper_tool_logging(execname, hostname, username)
else:
gplog.setup_tool_logging(execname,
hostname,
username,
logdir=options.ensure_value(
"logfileDirectory", None),
nonuser=nonuser)
if forceQuiet:
gplog.quiet_stdout_logging()
else:
if options.ensure_value("verbose", False):
gplog.enable_verbose_logging()
if options.ensure_value("quiet", False):
gplog.quiet_stdout_logging()
if options.ensure_value("masterDataDirectory", None) is not None:
options.master_data_directory = os.path.abspath(
options.masterDataDirectory)
if not suppressStartupLogMessage:
logger.info("Starting %s with args: %s" %
(gProgramName, ' '.join(sys.argv[1:])))
commandObject = createCommandFn(options, args)
exitCode = commandObject.run()
exit_status = exitCode
except ProgramArgumentValidationException, e:
if e.shouldPrintHelp():
parser.print_help()
logger.error("%s: error: %s" % (gProgramName, e.getMessage()))
exit_status = 2
def simple_main_locked(createOptionParserFn, createCommandFn, mainOptions):
"""
Not to be called externally -- use simple_main instead
"""
logger = gplog.get_default_logger()
configurationInterface.registerConfigurationProvider(
configurationImplGpdb.GpConfigurationProviderUsingGpdbCatalog())
fileSystemInterface.registerFileSystemProvider(fileSystemImplOs.GpFileSystemProviderUsingOs())
osInterface.registerOsProvider(osImplNative.GpOsProviderUsingNative())
faultProberInterface.registerFaultProber(faultProberImplGpdb.GpFaultProberImplGpdb())
commandObject = None
parser = None
forceQuiet = mainOptions is not None and mainOptions.get("forceQuietOutput")
options = None
if mainOptions is not None and mainOptions.get("programNameOverride"):
global gProgramName
gProgramName = mainOptions.get("programNameOverride")
suppressStartupLogMessage = mainOptions is not None and mainOptions.get("suppressStartupLogMessage")
useHelperToolLogging = mainOptions is not None and mainOptions.get("useHelperToolLogging")
nonuser = True if mainOptions is not None and mainOptions.get("setNonuserOnToolLogger") else False
exit_status = 1
# NOTE: if this logic is changed then also change test_main in testUtils.py
try:
execname = getProgramName()
hostname = unix.getLocalHostname()
username = unix.getUserName()
parser = createOptionParserFn()
(options, args) = parser.parse_args()
if useHelperToolLogging:
gplog.setup_helper_tool_logging(execname, hostname, username)
else:
gplog.setup_tool_logging(execname, hostname, username,
logdir=options.ensure_value("logfileDirectory", None), nonuser=nonuser)
if forceQuiet:
gplog.quiet_stdout_logging()
else:
if options.ensure_value("verbose", False):
gplog.enable_verbose_logging()
if options.ensure_value("quiet", False):
gplog.quiet_stdout_logging()
if options.ensure_value("masterDataDirectory", None) is not None:
options.master_data_directory = os.path.abspath(options.masterDataDirectory)
if not suppressStartupLogMessage:
logger.info("Starting %s with args: %s" % (gProgramName, ' '.join(sys.argv[1:])))
commandObject = createCommandFn(options, args)
exitCode = commandObject.run()
exit_status = exitCode
except ProgramArgumentValidationException, e:
if e.shouldPrintHelp():
parser.print_help()
logger.error("%s: error: %s" % (gProgramName, e.getMessage()))
exit_status = 2
# Copyright (c) Greenplum Inc 2008. All Rights Reserved.
#
""" Unittesting for pygres module
"""
import logging
import unittest
import pg
import pgdb
from gppylib import gplog
from gppylib.db.dbconn import *
from gppylib.db.test import skipIfDatabaseDown
logger = gplog.get_default_logger()
gplog.enable_verbose_logging()
#TODO: incomplete list.
catalog_names = [
'pg_aggregate', 'pg_amop', 'pg_attrdef', 'pg_auth_members',
'pg_autovacuum', 'pg_class', 'pg_conversion', 'pg_database',
'pg_description', 'pg_group', 'pg_indexes', 'pg_language',
'pg_max_external_files', 'pg_opclass', 'pg_partition'
]
datatypes = {
'oid': True,
'int2': True,
'regproc': True,
'text': True,
'bool': True,
#
# Unit Testing of pg commands
#
import os
import unittest
import tempfile
from gppylib.db import dbconn
from gppylib.db.test import skipIfDatabaseDown
from gppylib import gplog
from gppylib.commands import pg
from gppylib.gparray import GpArray
logger = gplog.get_default_logger()
gplog.enable_verbose_logging()
@skipIfDatabaseDown()
class PgCommandsTestCase(unittest.TestCase):
def setUp(self):
pass
def tearDown(self):
pass
def testReadPostmasterTempFile(self):
logger.info("testReadPostmasterTempFile")
url = dbconn.DbURL()
gpdb = GpArray.initFromCatalog(url)
logger.info("Search for valid master port: %s" % gpdb.master.port)
def simple_main_locked(createOptionParserFn, createCommandFn, mainOptions):
"""
Not to be called externally -- use simple_main instead
"""
logger = gplog.get_default_logger()
configurationInterface.registerConfigurationProvider(
configurationImplGpdb.GpConfigurationProviderUsingGpdbCatalog())
fileSystemInterface.registerFileSystemProvider(fileSystemImplOs.GpFileSystemProviderUsingOs())
osInterface.registerOsProvider(osImplNative.GpOsProviderUsingNative())
faultProberInterface.registerFaultProber(faultProberImplGpdb.GpFaultProberImplGpdb())
commandObject = None
parser = None
forceQuiet = mainOptions is not None and mainOptions.get("forceQuietOutput")
options = None
if mainOptions is not None and mainOptions.get("programNameOverride"):
global gProgramName
gProgramName = mainOptions.get("programNameOverride")
suppressStartupLogMessage = mainOptions is not None and mainOptions.get("suppressStartupLogMessage")
useHelperToolLogging = mainOptions is not None and mainOptions.get("useHelperToolLogging")
nonuser = True if mainOptions is not None and mainOptions.get("setNonuserOnToolLogger") else False
exit_status = 1
try:
execname = getProgramName()
hostname = unix.getLocalHostname()
username = unix.getUserName()
parser = createOptionParserFn()
(options, args) = parser.parse_args()
if useHelperToolLogging:
gplog.setup_helper_tool_logging(execname, hostname, username)
else:
gplog.setup_tool_logging(execname, hostname, username,
logdir=options.ensure_value("logfileDirectory", None), nonuser=nonuser)
if forceQuiet:
gplog.quiet_stdout_logging()
else:
if options.ensure_value("verbose", False):
gplog.enable_verbose_logging()
if options.ensure_value("quiet", False):
gplog.quiet_stdout_logging()
if options.ensure_value("coordinatorDataDirectory", None) is not None:
options.coordinator_data_directory = os.path.abspath(options.coordinatorDataDirectory)
if not suppressStartupLogMessage:
logger.info("Starting %s with args: %s" % (gProgramName, ' '.join(sys.argv[1:])))
commandObject = createCommandFn(options, args)
exitCode = commandObject.run()
exit_status = exitCode
except ProgramArgumentValidationException as e:
if e.shouldPrintHelp():
parser.print_help()
logger.error("%s: error: %s" % (gProgramName, e.getMessage()))
exit_status = 2
except ExceptionNoStackTraceNeeded as e:
logger.error("%s error: %s" % (gProgramName, e))
exit_status = 2
except UserAbortedException as e:
logger.info("User abort requested, Exiting...")
exit_status = 4
except ExecutionError as e:
logger.fatal("Error occurred: %s\n Command was: '%s'\n"
"rc=%d, stdout='%s', stderr='%s'" % \
(e.summary, e.cmd.cmdStr, e.cmd.results.rc, e.cmd.results.stdout,
e.cmd.results.stderr))
exit_status = 2
except Exception as e:
if options is None:
logger.exception("%s failed. exiting...", gProgramName)
else:
if options.ensure_value("verbose", False):
logger.exception("%s failed. exiting...", gProgramName)
else:
logger.fatal("%s failed. (Reason='%s') exiting..." % (gProgramName, e))
exit_status = 2
except KeyboardInterrupt:
exit_status = 2
finally:
if commandObject:
commandObject.cleanup()
sys.exit(exit_status)
def parseargs(self):
parser = OptParser(option_class=OptChecker,
description=' '.join(self.description.split()),
version='%prog version $Revision: $')
parser.set_usage(
'%prog is a utility script used by gprecoverseg, and gpaddmirrors and is not intended to be run separately.'
)
parser.remove_option('-h')
#TODO we may not need the verbose flag
parser.add_option('-v',
'--verbose',
action='store_true',
help='debug output.',
default=False)
parser.add_option('-c', '--confinfo', type='string')
parser.add_option('-b',
'--batch-size',
type='int',
default=DEFAULT_SEGHOST_NUM_WORKERS,
metavar='<batch_size>')
parser.add_option('-f',
'--force-overwrite',
dest='forceoverwrite',
action='store_true',
default=False)
parser.add_option('-l',
'--log-dir',
dest="logfileDirectory",
type="string")
parser.add_option(
'',
'--era',
dest="era",
help="coordinator era",
)
# Parse the command line arguments
self.options, _ = parser.parse_args()
if not self.options.confinfo:
raise Exception('Missing --confinfo argument.')
if not self.options.logfileDirectory:
raise Exception('Missing --log-dir argument.')
self.logger = gplog.setup_tool_logging(
os.path.split(self.file_name)[-1],
unix.getLocalHostname(),
unix.getUserName(),
logdir=self.options.logfileDirectory)
if self.options.batch_size <= 0:
self.logger.warn('batch_size was less than zero. Setting to 1.')
self.options.batch_size = 1
if self.options.verbose:
gplog.enable_verbose_logging()
self.seg_recovery_info_list = recoveryinfo.deserialize_list(
self.options.confinfo)
if len(self.seg_recovery_info_list) == 0:
raise Exception(
'No segment configuration values found in --confinfo argument')