def parseargs(args): global logger pguser = os.environ.get("PGUSER") or unix.getUserName() pghost = os.environ.get("PGHOST") or unix.getLocalHostname() pgport = os.environ.get("PGPORT") or 5432 parser = OptParser(option_class=OptChecker) parser.remove_option('-h') parser.add_option('-?', '--help', '-h', action='store_true', default=False) parser.add_option('-n', '--host', default=pghost) parser.add_option('-p', '--port', default=pgport) parser.add_option('-u', '--username', default=pguser) parser.add_option('-w', '--password', default=False, action='store_true') parser.add_option('-v', '--verbose', default=False, action='store_true') parser.add_option('-q', '--quiet', default=True, action='store_true') (options, args) = parser.parse_args() if options.help: print __doc__ sys.exit(1) try: options.port = int(options.port) except: logger.error("Invalid PORT: '%s'" % options.port) sys.exit(1) if options.verbose: gplog.enable_verbose_logging() elif options.quiet: gplog.quiet_stdout_logging() return options
def test_main( testName, newProgramArgs, createOptionParserFn, createCommandFn, extraOutputGenerators, expectedOutput) : global gTestOutput # update args previousArgs = sys.argv sys.argv = [] sys.argv.append(getProgramName()) sys.argv.extend(newProgramArgs) # register command factory setExecutionContextFactory(TestExecutionContextFactory()) commandObject=None parser = None startTest(testName) try: gplog.setup_tool_logging(gProgramName,unix.getLocalHostname(),unix.getUserName(),nonuser=False) parser = createOptionParserFn() (options, args) = parser.parse_args() gplog.enable_verbose_logging() commandObject = createCommandFn(options, args) exitCode = commandObject.run() testOutput("sys.exit %s" % exitCode) except ProgramArgumentValidationException, e: testOutput( "Validation error: %s" % e.getMessage())
def test_main(testName, newProgramArgs, createOptionParserFn, createCommandFn, extraOutputGenerators, expectedOutput): global gTestOutput # update args previousArgs = sys.argv sys.argv = [] sys.argv.append(getProgramName()) sys.argv.extend(newProgramArgs) # register command factory setExecutionContextFactory(TestExecutionContextFactory()) commandObject = None parser = None startTest(testName) try: gplog.setup_tool_logging(gProgramName, unix.getLocalHostname(), unix.getUserName(), nonuser=False) parser = createOptionParserFn() (options, args) = parser.parse_args() gplog.enable_verbose_logging() commandObject = createCommandFn(options, args) exitCode = commandObject.run() testOutput("sys.exit %s" % exitCode) except ProgramArgumentValidationException, e: testOutput("Validation error: %s" % e.getMessage())
def main(self, file_name, get_cmd_list): pool = None logger = None try: options = self.parseargs() exec_name = os.path.split(file_name)[-1] logger = gplog.setup_tool_logging(exec_name, unix.getLocalHostname(), unix.getUserName(), logdir=options.logfileDirectory) if not options.confinfo: raise Exception('Missing --confinfo argument.') if options.batch_size <= 0: logger.warn('batch_size was less than zero. Setting to 1.') options.batch_size = 1 if options.verbose: gplog.enable_verbose_logging() # TODO: should we output the name of the exact file? logger.info("Starting recovery with args: %s" % ' '.join(sys.argv[1:])) seg_recovery_info_list = recoveryinfo.deserialize_recovery_info_list( options.confinfo) if len(seg_recovery_info_list) == 0: raise Exception( 'No segment configuration values found in --confinfo argument' ) cmd_list = get_cmd_list(seg_recovery_info_list, options.forceoverwrite, logger) pool = WorkerPool( numWorkers=min(options.batch_size, len(cmd_list))) self.run_cmd_list(cmd_list, logger, options, pool) sys.exit(0) except Exception as e: if logger: logger.error(str(e)) print(e, file=sys.stderr) sys.exit(1) finally: if pool: pool.haltWork()
def close(self): """ Closes all handles to the logs we're watching. """ for h in self.handles.values(): h.close() self.handles = {} if __name__ == '__main__': # setup gpAdminLogs logging execname = os.path.split(sys.argv[0])[-1] hostname = unix.getLocalHostname() username = unix.getUserName() setup_tool_logging(execname, hostname, username) logger = get_default_logger() # watch syncmaster logs if len(sys.argv) > 2 and sys.argv[2] == 'debug': logger.info("Checking standby master status") watcher = SyncmasterWatcher( sys.argv[1] ) rc = watcher.monitor_logs() watcher.close() # report final status # logger.info("exiting with %s" % rc) sys.exit( rc )
def simple_main_locked(createOptionParserFn, createCommandFn, mainOptions): """ Not to be called externally -- use simple_main instead """ logger = gplog.get_default_logger() configurationInterface.registerConfigurationProvider( configurationImplGpdb.GpConfigurationProviderUsingGpdbCatalog()) fileSystemInterface.registerFileSystemProvider( fileSystemImplOs.GpFileSystemProviderUsingOs()) osInterface.registerOsProvider(osImplNative.GpOsProviderUsingNative()) faultProberInterface.registerFaultProber( faultProberImplGpdb.GpFaultProberImplGpdb()) commandObject = None parser = None forceQuiet = mainOptions is not None and mainOptions.get( "forceQuietOutput") options = None if mainOptions is not None and mainOptions.get("programNameOverride"): global gProgramName gProgramName = mainOptions.get("programNameOverride") suppressStartupLogMessage = mainOptions is not None and mainOptions.get( "suppressStartupLogMessage") useHelperToolLogging = mainOptions is not None and mainOptions.get( "useHelperToolLogging") nonuser = True if mainOptions is not None and mainOptions.get( "setNonuserOnToolLogger") else False exit_status = 1 try: execname = getProgramName() hostname = unix.getLocalHostname() username = unix.getUserName() parser = createOptionParserFn() (options, args) = parser.parse_args() if useHelperToolLogging: gplog.setup_helper_tool_logging(execname, hostname, username) else: gplog.setup_tool_logging(execname, hostname, username, logdir=options.ensure_value( "logfileDirectory", None), nonuser=nonuser) if forceQuiet: gplog.quiet_stdout_logging() else: if options.ensure_value("verbose", False): gplog.enable_verbose_logging() if options.ensure_value("quiet", False): gplog.quiet_stdout_logging() if options.ensure_value("masterDataDirectory", None) is not None: options.master_data_directory = os.path.abspath( options.masterDataDirectory) if not suppressStartupLogMessage: logger.info("Starting %s with args: %s" % (gProgramName, ' '.join(sys.argv[1:]))) commandObject = createCommandFn(options, args) exitCode = commandObject.run() exit_status = exitCode except ProgramArgumentValidationException, e: if e.shouldPrintHelp(): parser.print_help() logger.error("%s: error: %s" % (gProgramName, e.getMessage())) exit_status = 2
def simple_main_locked(createOptionParserFn, createCommandFn, mainOptions): """ Not to be called externally -- use simple_main instead """ logger = gplog.get_default_logger() configurationInterface.registerConfigurationProvider( configurationImplGpdb.GpConfigurationProviderUsingGpdbCatalog()) fileSystemInterface.registerFileSystemProvider(fileSystemImplOs.GpFileSystemProviderUsingOs()) osInterface.registerOsProvider(osImplNative.GpOsProviderUsingNative()) faultProberInterface.registerFaultProber(faultProberImplGpdb.GpFaultProberImplGpdb()) commandObject = None parser = None forceQuiet = mainOptions is not None and mainOptions.get("forceQuietOutput") options = None if mainOptions is not None and mainOptions.get("programNameOverride"): global gProgramName gProgramName = mainOptions.get("programNameOverride") suppressStartupLogMessage = mainOptions is not None and mainOptions.get("suppressStartupLogMessage") useHelperToolLogging = mainOptions is not None and mainOptions.get("useHelperToolLogging") nonuser = True if mainOptions is not None and mainOptions.get("setNonuserOnToolLogger") else False exit_status = 1 # NOTE: if this logic is changed then also change test_main in testUtils.py try: execname = getProgramName() hostname = unix.getLocalHostname() username = unix.getUserName() parser = createOptionParserFn() (options, args) = parser.parse_args() if useHelperToolLogging: gplog.setup_helper_tool_logging(execname, hostname, username) else: gplog.setup_tool_logging(execname, hostname, username, logdir=options.ensure_value("logfileDirectory", None), nonuser=nonuser) if forceQuiet: gplog.quiet_stdout_logging() else: if options.ensure_value("verbose", False): gplog.enable_verbose_logging() if options.ensure_value("quiet", False): gplog.quiet_stdout_logging() if options.ensure_value("masterDataDirectory", None) is not None: options.master_data_directory = os.path.abspath(options.masterDataDirectory) if not suppressStartupLogMessage: logger.info("Starting %s with args: %s" % (gProgramName, ' '.join(sys.argv[1:]))) commandObject = createCommandFn(options, args) exitCode = commandObject.run() exit_status = exitCode except ProgramArgumentValidationException, e: if e.shouldPrintHelp(): parser.print_help() logger.error("%s: error: %s" % (gProgramName, e.getMessage())) exit_status = 2
# syncmaster is running and there are no obvious errors in the log logger.info("syncmaster appears ok, pid %s" % pid) return 0 def close(self): """ Closes all handles to the logs we're watching. """ for h in self.handles.values(): h.close() self.handles = {} if __name__ == '__main__': # setup gpAdminLogs logging execname = os.path.split(sys.argv[0])[-1] hostname = unix.getLocalHostname() username = unix.getUserName() setup_tool_logging(execname, hostname, username) logger = get_default_logger() # watch syncmaster logs watcher = SyncmasterWatcher(sys.argv[1]) rc = watcher.monitor_logs() watcher.close() # report final status logger.info("exiting with %s" % rc) sys.exit(rc)
def simple_main_locked(createOptionParserFn, createCommandFn, mainOptions): """ Not to be called externally -- use simple_main instead """ logger = gplog.get_default_logger() configurationInterface.registerConfigurationProvider( configurationImplGpdb.GpConfigurationProviderUsingGpdbCatalog()) fileSystemInterface.registerFileSystemProvider(fileSystemImplOs.GpFileSystemProviderUsingOs()) osInterface.registerOsProvider(osImplNative.GpOsProviderUsingNative()) faultProberInterface.registerFaultProber(faultProberImplGpdb.GpFaultProberImplGpdb()) commandObject = None parser = None forceQuiet = mainOptions is not None and mainOptions.get("forceQuietOutput") options = None if mainOptions is not None and mainOptions.get("programNameOverride"): global gProgramName gProgramName = mainOptions.get("programNameOverride") suppressStartupLogMessage = mainOptions is not None and mainOptions.get("suppressStartupLogMessage") useHelperToolLogging = mainOptions is not None and mainOptions.get("useHelperToolLogging") nonuser = True if mainOptions is not None and mainOptions.get("setNonuserOnToolLogger") else False exit_status = 1 try: execname = getProgramName() hostname = unix.getLocalHostname() username = unix.getUserName() parser = createOptionParserFn() (options, args) = parser.parse_args() if useHelperToolLogging: gplog.setup_helper_tool_logging(execname, hostname, username) else: gplog.setup_tool_logging(execname, hostname, username, logdir=options.ensure_value("logfileDirectory", None), nonuser=nonuser) if forceQuiet: gplog.quiet_stdout_logging() else: if options.ensure_value("verbose", False): gplog.enable_verbose_logging() if options.ensure_value("quiet", False): gplog.quiet_stdout_logging() if options.ensure_value("coordinatorDataDirectory", None) is not None: options.coordinator_data_directory = os.path.abspath(options.coordinatorDataDirectory) if not suppressStartupLogMessage: logger.info("Starting %s with args: %s" % (gProgramName, ' '.join(sys.argv[1:]))) commandObject = createCommandFn(options, args) exitCode = commandObject.run() exit_status = exitCode except ProgramArgumentValidationException as e: if e.shouldPrintHelp(): parser.print_help() logger.error("%s: error: %s" % (gProgramName, e.getMessage())) exit_status = 2 except ExceptionNoStackTraceNeeded as e: logger.error("%s error: %s" % (gProgramName, e)) exit_status = 2 except UserAbortedException as e: logger.info("User abort requested, Exiting...") exit_status = 4 except ExecutionError as e: logger.fatal("Error occurred: %s\n Command was: '%s'\n" "rc=%d, stdout='%s', stderr='%s'" % \ (e.summary, e.cmd.cmdStr, e.cmd.results.rc, e.cmd.results.stdout, e.cmd.results.stderr)) exit_status = 2 except Exception as e: if options is None: logger.exception("%s failed. exiting...", gProgramName) else: if options.ensure_value("verbose", False): logger.exception("%s failed. exiting...", gProgramName) else: logger.fatal("%s failed. (Reason='%s') exiting..." % (gProgramName, e)) exit_status = 2 except KeyboardInterrupt: exit_status = 2 finally: if commandObject: commandObject.cleanup() sys.exit(exit_status)
logger.info('Starting expunge commands...') for host in hosts: hostname, port = host[0], host[1] cmd = Command("Expunging index %s on host %s port %d" % (index, hostname, port), EXPUNGE_COMMAND % (hostname, port, index)) cmd.run(validateAfter=True) return def main(): options = parseargs() database = find_gptext_schema() if not database: logger.error('Could not find schema %s.' % GPTEXT_SCHEMA) logger.error('Use the --database option to specify the database that') logger.error('contains the %s schema.' % GPTEXT_SCHEMA) sys.exit(1) url = dbconn.DbURL(dbname=database) validate_solr_up(url) validate_index_exists(url, options.index) hosts = get_solr_instances(url) if not userinput.ask_yesno(None, "\nContinue with GPText index '%s' expunge?" % options.index, 'N'): logger.error ('User asked for termination') sys.exit(1) gptext_index_expunge(options.index, hosts) logger.info('Done') logger.info('To check the process use "ps -ef | grep curl" - it will show you all the expunge requests issued') gplog.setup_tool_logging('gptext_expunge', getLocalHostname(), getUserName()) logger = gplog.get_default_logger() main()
def parseargs(self): parser = OptParser(option_class=OptChecker, description=' '.join(self.description.split()), version='%prog version $Revision: $') parser.set_usage( '%prog is a utility script used by gprecoverseg, and gpaddmirrors and is not intended to be run separately.' ) parser.remove_option('-h') #TODO we may not need the verbose flag parser.add_option('-v', '--verbose', action='store_true', help='debug output.', default=False) parser.add_option('-c', '--confinfo', type='string') parser.add_option('-b', '--batch-size', type='int', default=DEFAULT_SEGHOST_NUM_WORKERS, metavar='<batch_size>') parser.add_option('-f', '--force-overwrite', dest='forceoverwrite', action='store_true', default=False) parser.add_option('-l', '--log-dir', dest="logfileDirectory", type="string") parser.add_option( '', '--era', dest="era", help="coordinator era", ) # Parse the command line arguments self.options, _ = parser.parse_args() if not self.options.confinfo: raise Exception('Missing --confinfo argument.') if not self.options.logfileDirectory: raise Exception('Missing --log-dir argument.') self.logger = gplog.setup_tool_logging( os.path.split(self.file_name)[-1], unix.getLocalHostname(), unix.getUserName(), logdir=self.options.logfileDirectory) if self.options.batch_size <= 0: self.logger.warn('batch_size was less than zero. Setting to 1.') self.options.batch_size = 1 if self.options.verbose: gplog.enable_verbose_logging() self.seg_recovery_info_list = recoveryinfo.deserialize_list( self.options.confinfo) if len(self.seg_recovery_info_list) == 0: raise Exception( 'No segment configuration values found in --confinfo argument')
queries = QUERIES_PYTHON if options.gpdbgen == '5': queries += QUERIES_PYTHON_5 if prepare_funcs("python", options.dbname, options.sqldir, options.gpdbgen) < 0: return run(options.dbname, options.username, options.time, queries, "python") if not options.nopython3: queries = QUERIES_PYTHON if options.gpdbgen == '5': queries += QUERIES_PYTHON_5 if prepare_funcs("python3", options.dbname, options.sqldir, options.gpdbgen) < 0: return run(options.dbname, options.username, options.time, queries, "python3") if not options.nor: queries = QUERIES_R if options.gpdbgen == '5': queries += QUERIES_R_5 if prepare_funcs("R", options.dbname, options.sqldir, options.gpdbgen) < 0: return run(options.dbname, options.username, options.time, queries, "R") if __name__ == '__main__': execname = os.path.split(__file__)[-1] gplog.setup_tool_logging(execname, getLocalHostname(), getUserName()) logger = gplog.get_default_logger() main()