def parseargs(args):
global logger
pguser = os.environ.get("PGUSER") or unix.getUserName()
pghost = os.environ.get("PGHOST") or unix.getLocalHostname()
pgport = os.environ.get("PGPORT") or 5432
parser = OptParser(option_class=OptChecker)
parser.remove_option('-h')
parser.add_option('-?', '--help', '-h', action='store_true', default=False)
parser.add_option('-n', '--host', default=pghost)
parser.add_option('-p', '--port', default=pgport)
parser.add_option('-u', '--username', default=pguser)
parser.add_option('-w', '--password', default=False, action='store_true')
parser.add_option('-v', '--verbose', default=False, action='store_true')
parser.add_option('-q', '--quiet', default=True, action='store_true')
(options, args) = parser.parse_args()
if options.help:
print __doc__
sys.exit(1)
try:
options.port = int(options.port)
except:
logger.error("Invalid PORT: '%s'" % options.port)
sys.exit(1)
if options.verbose:
gplog.enable_verbose_logging()
elif options.quiet:
gplog.quiet_stdout_logging()
return options
def parseargs(args):
global logger
pguser = os.environ.get("PGUSER") or unix.getUserName()
pghost = os.environ.get("PGHOST") or unix.getLocalHostname()
pgport = os.environ.get("PGPORT") or 5432
parser = OptParser(option_class=OptChecker)
parser.remove_option('-h')
parser.add_option('-?', '--help', '-h', action='store_true', default=False)
parser.add_option('-n', '--host', default=pghost)
parser.add_option('-p', '--port', default=pgport)
parser.add_option('-u', '--username', default=pguser)
parser.add_option('-w', '--password', default=False, action='store_true')
parser.add_option('-v', '--verbose', default=False, action='store_true')
parser.add_option('-q', '--quiet', default=True, action='store_true')
(options, args) = parser.parse_args()
if options.help:
print __doc__
sys.exit(1)
try:
options.port = int(options.port)
except:
logger.error("Invalid PORT: '%s'" % options.port)
sys.exit(1)
if options.verbose:
gplog.enable_verbose_logging()
elif options.quiet:
gplog.quiet_stdout_logging()
return options
def test_main( testName, newProgramArgs, createOptionParserFn, createCommandFn, extraOutputGenerators, expectedOutput) :
global gTestOutput
# update args
previousArgs = sys.argv
sys.argv = []
sys.argv.append(getProgramName())
sys.argv.extend(newProgramArgs)
# register command factory
setExecutionContextFactory(TestExecutionContextFactory())
commandObject=None
parser = None
startTest(testName)
try:
gplog.setup_tool_logging(gProgramName,unix.getLocalHostname(),unix.getUserName(),nonuser=False)
parser = createOptionParserFn()
(options, args) = parser.parse_args()
gplog.enable_verbose_logging()
commandObject = createCommandFn(options, args)
exitCode = commandObject.run()
testOutput("sys.exit %s" % exitCode)
except ProgramArgumentValidationException, e:
testOutput( "Validation error: %s" % e.getMessage())
def test_main(testName, newProgramArgs, createOptionParserFn, createCommandFn,
extraOutputGenerators, expectedOutput):
global gTestOutput
# update args
previousArgs = sys.argv
sys.argv = []
sys.argv.append(getProgramName())
sys.argv.extend(newProgramArgs)
# register command factory
setExecutionContextFactory(TestExecutionContextFactory())
commandObject = None
parser = None
startTest(testName)
try:
gplog.setup_tool_logging(gProgramName,
unix.getLocalHostname(),
unix.getUserName(),
nonuser=False)
parser = createOptionParserFn()
(options, args) = parser.parse_args()
gplog.enable_verbose_logging()
commandObject = createCommandFn(options, args)
exitCode = commandObject.run()
testOutput("sys.exit %s" % exitCode)
except ProgramArgumentValidationException, e:
testOutput("Validation error: %s" % e.getMessage())
def main(self, file_name, get_cmd_list):
pool = None
logger = None
try:
options = self.parseargs()
exec_name = os.path.split(file_name)[-1]
logger = gplog.setup_tool_logging(exec_name,
unix.getLocalHostname(),
unix.getUserName(),
logdir=options.logfileDirectory)
if not options.confinfo:
raise Exception('Missing --confinfo argument.')
if options.batch_size <= 0:
logger.warn('batch_size was less than zero. Setting to 1.')
options.batch_size = 1
if options.verbose:
gplog.enable_verbose_logging()
# TODO: should we output the name of the exact file?
logger.info("Starting recovery with args: %s" %
' '.join(sys.argv[1:]))
seg_recovery_info_list = recoveryinfo.deserialize_recovery_info_list(
options.confinfo)
if len(seg_recovery_info_list) == 0:
raise Exception(
'No segment configuration values found in --confinfo argument'
)
cmd_list = get_cmd_list(seg_recovery_info_list,
options.forceoverwrite, logger)
pool = WorkerPool(
numWorkers=min(options.batch_size, len(cmd_list)))
self.run_cmd_list(cmd_list, logger, options, pool)
sys.exit(0)
except Exception as e:
if logger:
logger.error(str(e))
print(e, file=sys.stderr)
sys.exit(1)
finally:
if pool:
pool.haltWork()
def close(self):
"""
Closes all handles to the logs we're watching.
"""
for h in self.handles.values():
h.close()
self.handles = {}
if __name__ == '__main__':
# setup gpAdminLogs logging
execname = os.path.split(sys.argv[0])[-1]
hostname = unix.getLocalHostname()
username = unix.getUserName()
setup_tool_logging(execname, hostname, username)
logger = get_default_logger()
# watch syncmaster logs
if len(sys.argv) > 2 and sys.argv[2] == 'debug':
logger.info("Checking standby master status")
watcher = SyncmasterWatcher( sys.argv[1] )
rc = watcher.monitor_logs()
watcher.close()
# report final status
# logger.info("exiting with %s" % rc)
sys.exit( rc )
def simple_main_locked(createOptionParserFn, createCommandFn, mainOptions):
"""
Not to be called externally -- use simple_main instead
"""
logger = gplog.get_default_logger()
configurationInterface.registerConfigurationProvider(
configurationImplGpdb.GpConfigurationProviderUsingGpdbCatalog())
fileSystemInterface.registerFileSystemProvider(
fileSystemImplOs.GpFileSystemProviderUsingOs())
osInterface.registerOsProvider(osImplNative.GpOsProviderUsingNative())
faultProberInterface.registerFaultProber(
faultProberImplGpdb.GpFaultProberImplGpdb())
commandObject = None
parser = None
forceQuiet = mainOptions is not None and mainOptions.get(
"forceQuietOutput")
options = None
if mainOptions is not None and mainOptions.get("programNameOverride"):
global gProgramName
gProgramName = mainOptions.get("programNameOverride")
suppressStartupLogMessage = mainOptions is not None and mainOptions.get(
"suppressStartupLogMessage")
useHelperToolLogging = mainOptions is not None and mainOptions.get(
"useHelperToolLogging")
nonuser = True if mainOptions is not None and mainOptions.get(
"setNonuserOnToolLogger") else False
exit_status = 1
try:
execname = getProgramName()
hostname = unix.getLocalHostname()
username = unix.getUserName()
parser = createOptionParserFn()
(options, args) = parser.parse_args()
if useHelperToolLogging:
gplog.setup_helper_tool_logging(execname, hostname, username)
else:
gplog.setup_tool_logging(execname,
hostname,
username,
logdir=options.ensure_value(
"logfileDirectory", None),
nonuser=nonuser)
if forceQuiet:
gplog.quiet_stdout_logging()
else:
if options.ensure_value("verbose", False):
gplog.enable_verbose_logging()
if options.ensure_value("quiet", False):
gplog.quiet_stdout_logging()
if options.ensure_value("masterDataDirectory", None) is not None:
options.master_data_directory = os.path.abspath(
options.masterDataDirectory)
if not suppressStartupLogMessage:
logger.info("Starting %s with args: %s" %
(gProgramName, ' '.join(sys.argv[1:])))
commandObject = createCommandFn(options, args)
exitCode = commandObject.run()
exit_status = exitCode
except ProgramArgumentValidationException, e:
if e.shouldPrintHelp():
parser.print_help()
logger.error("%s: error: %s" % (gProgramName, e.getMessage()))
exit_status = 2
def simple_main_locked(createOptionParserFn, createCommandFn, mainOptions):
"""
Not to be called externally -- use simple_main instead
"""
logger = gplog.get_default_logger()
configurationInterface.registerConfigurationProvider(
configurationImplGpdb.GpConfigurationProviderUsingGpdbCatalog())
fileSystemInterface.registerFileSystemProvider(fileSystemImplOs.GpFileSystemProviderUsingOs())
osInterface.registerOsProvider(osImplNative.GpOsProviderUsingNative())
faultProberInterface.registerFaultProber(faultProberImplGpdb.GpFaultProberImplGpdb())
commandObject = None
parser = None
forceQuiet = mainOptions is not None and mainOptions.get("forceQuietOutput")
options = None
if mainOptions is not None and mainOptions.get("programNameOverride"):
global gProgramName
gProgramName = mainOptions.get("programNameOverride")
suppressStartupLogMessage = mainOptions is not None and mainOptions.get("suppressStartupLogMessage")
useHelperToolLogging = mainOptions is not None and mainOptions.get("useHelperToolLogging")
nonuser = True if mainOptions is not None and mainOptions.get("setNonuserOnToolLogger") else False
exit_status = 1
# NOTE: if this logic is changed then also change test_main in testUtils.py
try:
execname = getProgramName()
hostname = unix.getLocalHostname()
username = unix.getUserName()
parser = createOptionParserFn()
(options, args) = parser.parse_args()
if useHelperToolLogging:
gplog.setup_helper_tool_logging(execname, hostname, username)
else:
gplog.setup_tool_logging(execname, hostname, username,
logdir=options.ensure_value("logfileDirectory", None), nonuser=nonuser)
if forceQuiet:
gplog.quiet_stdout_logging()
else:
if options.ensure_value("verbose", False):
gplog.enable_verbose_logging()
if options.ensure_value("quiet", False):
gplog.quiet_stdout_logging()
if options.ensure_value("masterDataDirectory", None) is not None:
options.master_data_directory = os.path.abspath(options.masterDataDirectory)
if not suppressStartupLogMessage:
logger.info("Starting %s with args: %s" % (gProgramName, ' '.join(sys.argv[1:])))
commandObject = createCommandFn(options, args)
exitCode = commandObject.run()
exit_status = exitCode
except ProgramArgumentValidationException, e:
if e.shouldPrintHelp():
parser.print_help()
logger.error("%s: error: %s" % (gProgramName, e.getMessage()))
exit_status = 2
# syncmaster is running and there are no obvious errors in the log
logger.info("syncmaster appears ok, pid %s" % pid)
return 0
def close(self):
"""
Closes all handles to the logs we're watching.
"""
for h in self.handles.values():
h.close()
self.handles = {}
if __name__ == '__main__':
# setup gpAdminLogs logging
execname = os.path.split(sys.argv[0])[-1]
hostname = unix.getLocalHostname()
username = unix.getUserName()
setup_tool_logging(execname, hostname, username)
logger = get_default_logger()
# watch syncmaster logs
watcher = SyncmasterWatcher(sys.argv[1])
rc = watcher.monitor_logs()
watcher.close()
# report final status
logger.info("exiting with %s" % rc)
sys.exit(rc)
def simple_main_locked(createOptionParserFn, createCommandFn, mainOptions):
"""
Not to be called externally -- use simple_main instead
"""
logger = gplog.get_default_logger()
configurationInterface.registerConfigurationProvider(
configurationImplGpdb.GpConfigurationProviderUsingGpdbCatalog())
fileSystemInterface.registerFileSystemProvider(fileSystemImplOs.GpFileSystemProviderUsingOs())
osInterface.registerOsProvider(osImplNative.GpOsProviderUsingNative())
faultProberInterface.registerFaultProber(faultProberImplGpdb.GpFaultProberImplGpdb())
commandObject = None
parser = None
forceQuiet = mainOptions is not None and mainOptions.get("forceQuietOutput")
options = None
if mainOptions is not None and mainOptions.get("programNameOverride"):
global gProgramName
gProgramName = mainOptions.get("programNameOverride")
suppressStartupLogMessage = mainOptions is not None and mainOptions.get("suppressStartupLogMessage")
useHelperToolLogging = mainOptions is not None and mainOptions.get("useHelperToolLogging")
nonuser = True if mainOptions is not None and mainOptions.get("setNonuserOnToolLogger") else False
exit_status = 1
try:
execname = getProgramName()
hostname = unix.getLocalHostname()
username = unix.getUserName()
parser = createOptionParserFn()
(options, args) = parser.parse_args()
if useHelperToolLogging:
gplog.setup_helper_tool_logging(execname, hostname, username)
else:
gplog.setup_tool_logging(execname, hostname, username,
logdir=options.ensure_value("logfileDirectory", None), nonuser=nonuser)
if forceQuiet:
gplog.quiet_stdout_logging()
else:
if options.ensure_value("verbose", False):
gplog.enable_verbose_logging()
if options.ensure_value("quiet", False):
gplog.quiet_stdout_logging()
if options.ensure_value("coordinatorDataDirectory", None) is not None:
options.coordinator_data_directory = os.path.abspath(options.coordinatorDataDirectory)
if not suppressStartupLogMessage:
logger.info("Starting %s with args: %s" % (gProgramName, ' '.join(sys.argv[1:])))
commandObject = createCommandFn(options, args)
exitCode = commandObject.run()
exit_status = exitCode
except ProgramArgumentValidationException as e:
if e.shouldPrintHelp():
parser.print_help()
logger.error("%s: error: %s" % (gProgramName, e.getMessage()))
exit_status = 2
except ExceptionNoStackTraceNeeded as e:
logger.error("%s error: %s" % (gProgramName, e))
exit_status = 2
except UserAbortedException as e:
logger.info("User abort requested, Exiting...")
exit_status = 4
except ExecutionError as e:
logger.fatal("Error occurred: %s\n Command was: '%s'\n"
"rc=%d, stdout='%s', stderr='%s'" % \
(e.summary, e.cmd.cmdStr, e.cmd.results.rc, e.cmd.results.stdout,
e.cmd.results.stderr))
exit_status = 2
except Exception as e:
if options is None:
logger.exception("%s failed. exiting...", gProgramName)
else:
if options.ensure_value("verbose", False):
logger.exception("%s failed. exiting...", gProgramName)
else:
logger.fatal("%s failed. (Reason='%s') exiting..." % (gProgramName, e))
exit_status = 2
except KeyboardInterrupt:
exit_status = 2
finally:
if commandObject:
commandObject.cleanup()
sys.exit(exit_status)
logger.info('Starting expunge commands...')
for host in hosts:
hostname, port = host[0], host[1]
cmd = Command("Expunging index %s on host %s port %d" % (index, hostname, port),
EXPUNGE_COMMAND % (hostname, port, index))
cmd.run(validateAfter=True)
return
def main():
options = parseargs()
database = find_gptext_schema()
if not database:
logger.error('Could not find schema %s.' % GPTEXT_SCHEMA)
logger.error('Use the --database option to specify the database that')
logger.error('contains the %s schema.' % GPTEXT_SCHEMA)
sys.exit(1)
url = dbconn.DbURL(dbname=database)
validate_solr_up(url)
validate_index_exists(url, options.index)
hosts = get_solr_instances(url)
if not userinput.ask_yesno(None, "\nContinue with GPText index '%s' expunge?" % options.index, 'N'):
logger.error ('User asked for termination')
sys.exit(1)
gptext_index_expunge(options.index, hosts)
logger.info('Done')
logger.info('To check the process use "ps -ef | grep curl" - it will show you all the expunge requests issued')
gplog.setup_tool_logging('gptext_expunge', getLocalHostname(), getUserName())
logger = gplog.get_default_logger()
main()
def parseargs(self):
parser = OptParser(option_class=OptChecker,
description=' '.join(self.description.split()),
version='%prog version $Revision: $')
parser.set_usage(
'%prog is a utility script used by gprecoverseg, and gpaddmirrors and is not intended to be run separately.'
)
parser.remove_option('-h')
#TODO we may not need the verbose flag
parser.add_option('-v',
'--verbose',
action='store_true',
help='debug output.',
default=False)
parser.add_option('-c', '--confinfo', type='string')
parser.add_option('-b',
'--batch-size',
type='int',
default=DEFAULT_SEGHOST_NUM_WORKERS,
metavar='<batch_size>')
parser.add_option('-f',
'--force-overwrite',
dest='forceoverwrite',
action='store_true',
default=False)
parser.add_option('-l',
'--log-dir',
dest="logfileDirectory",
type="string")
parser.add_option(
'',
'--era',
dest="era",
help="coordinator era",
)
# Parse the command line arguments
self.options, _ = parser.parse_args()
if not self.options.confinfo:
raise Exception('Missing --confinfo argument.')
if not self.options.logfileDirectory:
raise Exception('Missing --log-dir argument.')
self.logger = gplog.setup_tool_logging(
os.path.split(self.file_name)[-1],
unix.getLocalHostname(),
unix.getUserName(),
logdir=self.options.logfileDirectory)
if self.options.batch_size <= 0:
self.logger.warn('batch_size was less than zero. Setting to 1.')
self.options.batch_size = 1
if self.options.verbose:
gplog.enable_verbose_logging()
self.seg_recovery_info_list = recoveryinfo.deserialize_list(
self.options.confinfo)
if len(self.seg_recovery_info_list) == 0:
raise Exception(
'No segment configuration values found in --confinfo argument')
queries = QUERIES_PYTHON
if options.gpdbgen == '5':
queries += QUERIES_PYTHON_5
if prepare_funcs("python", options.dbname, options.sqldir,
options.gpdbgen) < 0:
return
run(options.dbname, options.username, options.time, queries, "python")
if not options.nopython3:
queries = QUERIES_PYTHON
if options.gpdbgen == '5':
queries += QUERIES_PYTHON_5
if prepare_funcs("python3", options.dbname, options.sqldir,
options.gpdbgen) < 0:
return
run(options.dbname, options.username, options.time, queries, "python3")
if not options.nor:
queries = QUERIES_R
if options.gpdbgen == '5':
queries += QUERIES_R_5
if prepare_funcs("R", options.dbname, options.sqldir,
options.gpdbgen) < 0:
return
run(options.dbname, options.username, options.time, queries, "R")
if __name__ == '__main__':
execname = os.path.split(__file__)[-1]
gplog.setup_tool_logging(execname, getLocalHostname(), getUserName())
logger = gplog.get_default_logger()
main()
