def __init__(self, name, cmdStr, ctxt=LOCAL, remoteHost=None, stdin=None, nakedExecutionInfo=None, gphome=None): self.name = name self.cmdStr = cmdStr self.exec_context = createExecutionContext(ctxt, remoteHost, stdin=stdin, nakedExecutionInfo=nakedExecutionInfo, gphome=gphome) self.remoteHost = remoteHost self.logger = gplog.get_default_logger()
def main(argv): global systemFSMap global logger logger = gplog.get_default_logger() options = parseargs(argv) dbConnect(options) genServerFsList() findFsDetails() genFSMap() sys.exit(0)
def __init__(self, main): """ Create cached yaml from class doc string of the given object, looking for the %YAML indicating the beginning of the object's YAML plan and parse it. Build the plan stages and tasks for the specified scenario. """ # main - object with yaml scenarios (input) # sy - Stage yaml self.logger = gplog.get_default_logger() self.logfilename = gplog.get_logfile() self.main = main self.y = get_yaml(main.__class__) self.name = main.options.scenario if not self.name: self.name = self.y['Default Scenario'] self.scenario = self.y['Scenarios'][self.name] self.errors = self.y['Errors'] self.Tasks = [self._task(ty) for ty in self.scenario]
def __init__(self, numWorkers=16, items=None, daemonize=False, logger=gplog.get_default_logger()): if numWorkers <= 0: raise Exception("WorkerPool(): numWorkers should be greater than 0.") self.workers = [] self.should_stop = False self.work_queue = Queue() self.completed_queue = Queue() self._assigned = 0 self.daemonize = daemonize self.logger = logger if items is not None: for item in items: self.addCommand(item) for i in range(0, numWorkers): w = Worker("worker%d" % i, self) self.workers.append(w) w.start() self.numWorkers = numWorkers
def simple_main_internal(createOptionParserFn, createCommandFn, mainOptions): """ If caller specifies 'pidfilename' in mainOptions then we manage the specified pid file within the MASTER_DATA_DIRECTORY before proceeding to execute the specified program and we clean up the pid file when we're done. """ sml = None if mainOptions is not None and 'pidfilename' in mainOptions: sml = SimpleMainLock(mainOptions) otherpid = sml.acquire() if otherpid is not None: logger = gplog.get_default_logger() logger.error("An instance of %s is already running (pid %s)" % (getProgramName(), otherpid)) return # at this point we have whatever lock we require try: simple_main_locked(createOptionParserFn, createCommandFn, mainOptions) finally: if sml is not None: sml.release()
This file is a wrapper around figleaf and will start/stop coverage as needed. It also includes a method for generating the HTML reports. """ import os import random import figleaf import pickle from glob import glob from gppylib import gplog from gppylib.commands.base import Command, LOCAL, REMOTE, ExecutionContext, RemoteExecutionContext, WorkerPool from gppylib.commands.unix import RemoveFiles, Scp from gppylib.operations import Operation from gppylib.operations.unix import ListFiles, ListRemoteFiles, MakeDir logger = gplog.get_default_logger() COVERAGE_FILENAME = 'cover.out' #------------------------------------------------------------------------------ class GpWriteFigleafCoverageHtml(Command): """Command to write out figleaf html reports to disk based on the coverage information that has been collected.""" def __init__(self,name,filename, directory,ctxt=LOCAL,remoteHost=None): gphome = os.getenv("GPHOME", None) if not gphome: raise Exception('GPHOME environment variable not set.') cmdStr = "%s -d %s %s" % (os.path.normpath(gphome + '/lib/python/figleaf/figleaf2html'), directory, filename) Command.__init__(self,name,cmdStr,ctxt,remoteHost) @staticmethod
def close(self): """ Closes all handles to the logs we're watching. """ for h in self.handles.values(): h.close() self.handles = {} if __name__ == '__main__': # setup gpAdminLogs logging execname = os.path.split(sys.argv[0])[-1] hostname = unix.getLocalHostname() username = unix.getUserName() setup_tool_logging(execname, hostname, username) logger = get_default_logger() # watch syncmaster logs if len(sys.argv) > 2 and sys.argv[2] == 'debug': logger.info("Checking standby master status") watcher = SyncmasterWatcher( sys.argv[1] ) rc = watcher.monitor_logs() watcher.close() # report final status # logger.info("exiting with %s" % rc) sys.exit( rc )
def simple_main_locked(createOptionParserFn, createCommandFn, mainOptions): """ Not to be called externally -- use simple_main instead """ logger = gplog.get_default_logger() configurationInterface.registerConfigurationProvider( configurationImplGpdb.GpConfigurationProviderUsingGpdbCatalog()) fileSystemInterface.registerFileSystemProvider(fileSystemImplOs.GpFileSystemProviderUsingOs()) osInterface.registerOsProvider(osImplNative.GpOsProviderUsingNative()) faultProberInterface.registerFaultProber(faultProberImplGpdb.GpFaultProberImplGpdb()) commandObject = None parser = None forceQuiet = mainOptions is not None and mainOptions.get("forceQuietOutput") options = None if mainOptions is not None and mainOptions.get("programNameOverride"): global gProgramName gProgramName = mainOptions.get("programNameOverride") suppressStartupLogMessage = mainOptions is not None and mainOptions.get("suppressStartupLogMessage") useHelperToolLogging = mainOptions is not None and mainOptions.get("useHelperToolLogging") nonuser = True if mainOptions is not None and mainOptions.get("setNonuserOnToolLogger") else False exit_status = 1 # NOTE: if this logic is changed then also change test_main in testUtils.py try: execname = getProgramName() hostname = unix.getLocalHostname() username = unix.getUserName() parser = createOptionParserFn() (options, args) = parser.parse_args() if useHelperToolLogging: gplog.setup_helper_tool_logging(execname, hostname, username) else: gplog.setup_tool_logging(execname, hostname, username, logdir=options.ensure_value("logfileDirectory", None), nonuser=nonuser) if forceQuiet: gplog.quiet_stdout_logging() else: if options.ensure_value("verbose", False): gplog.enable_verbose_logging() if options.ensure_value("quiet", False): gplog.quiet_stdout_logging() if options.ensure_value("masterDataDirectory", None) is not None: options.master_data_directory = os.path.abspath(options.masterDataDirectory) if not suppressStartupLogMessage: logger.info("Starting %s with args: %s" % (gProgramName, ' '.join(sys.argv[1:]))) commandObject = createCommandFn(options, args) exitCode = commandObject.run() exit_status = exitCode except ProgramArgumentValidationException, e: if e.shouldPrintHelp(): parser.print_help() logger.error("%s: error: %s" % (gProgramName, e.getMessage())) exit_status = 2
def __init__(self, gpEnv, gpArray): self.gpEnv = gpEnv self.gpArray = gpArray self.logger = gplog.get_default_logger()
def __init__(self): self.logger = get_default_logger()
from gppylib.gpparseopts import OptParser, OptChecker from gppylib.operations.startSegments import * from gppylib.operations.buildMirrorSegments import * from gppylib.operations.rebalanceSegments import GpSegmentRebalanceOperation from gppylib.programs import programIoUtils from gppylib.system import configurationInterface as configInterface from gppylib.system.environment import GpMasterEnvironment from gppylib.parseutils import line_reader, parse_gprecoverseg_line, canonicalize_address from gppylib.utils import ParsedConfigFile, ParsedConfigFileRow, writeLinesToFile, \ normalizeAndValidateInputPath, TableLogger from gppylib.gphostcache import GpInterfaceToHostNameCache from gppylib.operations.utils import ParallelOperation from gppylib.operations.package import SyncPackages from gppylib.heapchecksum import HeapChecksum logger = gplog.get_default_logger() class PortAssigner: """ Used to assign new ports to segments on a host Note that this could be improved so that we re-use ports for segments that are being recovered but this does not seem necessary. """ MAX_PORT_EXCLUSIVE = 65536 def __init__(self, gpArray): #
def simple_main_locked(createOptionParserFn, createCommandFn, mainOptions): """ Not to be called externally -- use simple_main instead """ logger = gplog.get_default_logger() configurationInterface.registerConfigurationProvider( configurationImplGpdb.GpConfigurationProviderUsingGpdbCatalog()) fileSystemInterface.registerFileSystemProvider( fileSystemImplOs.GpFileSystemProviderUsingOs()) osInterface.registerOsProvider(osImplNative.GpOsProviderUsingNative()) faultProberInterface.registerFaultProber( faultProberImplGpdb.GpFaultProberImplGpdb()) commandObject = None parser = None forceQuiet = mainOptions is not None and mainOptions.get( "forceQuietOutput") options = None if mainOptions is not None and mainOptions.get("programNameOverride"): global gProgramName gProgramName = mainOptions.get("programNameOverride") suppressStartupLogMessage = mainOptions is not None and mainOptions.get( "suppressStartupLogMessage") useHelperToolLogging = mainOptions is not None and mainOptions.get( "useHelperToolLogging") nonuser = True if mainOptions is not None and mainOptions.get( "setNonuserOnToolLogger") else False exit_status = 1 try: execname = getProgramName() hostname = unix.getLocalHostname() username = unix.getUserName() parser = createOptionParserFn() (options, args) = parser.parse_args() if useHelperToolLogging: gplog.setup_helper_tool_logging(execname, hostname, username) else: gplog.setup_tool_logging(execname, hostname, username, logdir=options.ensure_value( "logfileDirectory", None), nonuser=nonuser) if forceQuiet: gplog.quiet_stdout_logging() else: if options.ensure_value("verbose", False): gplog.enable_verbose_logging() if options.ensure_value("quiet", False): gplog.quiet_stdout_logging() if options.ensure_value("masterDataDirectory", None) is not None: options.master_data_directory = os.path.abspath( options.masterDataDirectory) if not suppressStartupLogMessage: logger.info("Starting %s with args: %s" % (gProgramName, ' '.join(sys.argv[1:]))) commandObject = createCommandFn(options, args) exitCode = commandObject.run() exit_status = exitCode except ProgramArgumentValidationException, e: if e.shouldPrintHelp(): parser.print_help() logger.error("%s: error: %s" % (gProgramName, e.getMessage())) exit_status = 2
return 0 def close(self): """ Closes all handles to the logs we're watching. """ for h in self.handles.values(): h.close() self.handles = {} if __name__ == '__main__': # setup gpAdminLogs logging execname = os.path.split(sys.argv[0])[-1] hostname = unix.getLocalHostname() username = unix.getUserName() setup_tool_logging(execname, hostname, username) logger = get_default_logger() # watch syncmaster logs if len(sys.argv) > 2 and sys.argv[2] == 'debug': logger.info("Checking standby master status") watcher = SyncmasterWatcher(sys.argv[1]) rc = watcher.monitor_logs() watcher.close() # report final status # logger.info("exiting with %s" % rc) sys.exit(rc)
def simple_main_locked(createOptionParserFn, createCommandFn, mainOptions): """ Not to be called externally -- use simple_main instead """ logger = gplog.get_default_logger() configurationInterface.registerConfigurationProvider( configurationImplGpdb.GpConfigurationProviderUsingGpdbCatalog()) fileSystemInterface.registerFileSystemProvider( fileSystemImplOs.GpFileSystemProviderUsingOs()) osInterface.registerOsProvider(osImplNative.GpOsProviderUsingNative()) faultProberInterface.registerFaultProber( faultProberImplGpdb.GpFaultProberImplGpdb()) commandObject = None parser = None forceQuiet = mainOptions is not None and mainOptions.get( "forceQuietOutput") options = None if mainOptions is not None and mainOptions.get("programNameOverride"): global gProgramName gProgramName = mainOptions.get("programNameOverride") suppressStartupLogMessage = mainOptions is not None and mainOptions.get( "suppressStartupLogMessage") useHelperToolLogging = mainOptions is not None and mainOptions.get( "useHelperToolLogging") nonuser = True if mainOptions is not None and mainOptions.get( "setNonuserOnToolLogger") else False exit_status = 1 try: execname = getProgramName() hostname = unix.getLocalHostname() username = unix.getUserName() parser = createOptionParserFn() (options, args) = parser.parse_args() if useHelperToolLogging: gplog.setup_helper_tool_logging(execname, hostname, username) else: gplog.setup_tool_logging(execname, hostname, username, logdir=options.ensure_value( "logfileDirectory", None), nonuser=nonuser) if forceQuiet: gplog.quiet_stdout_logging() else: if options.ensure_value("verbose", False): gplog.enable_verbose_logging() if options.ensure_value("quiet", False): gplog.quiet_stdout_logging() if options.ensure_value("masterDataDirectory", None) is not None: options.master_data_directory = os.path.abspath( options.masterDataDirectory) if not suppressStartupLogMessage: logger.info("Starting %s with args: %s" % (gProgramName, ' '.join(sys.argv[1:]))) commandObject = createCommandFn(options, args) exitCode = commandObject.run() exit_status = exitCode except ProgramArgumentValidationException as e: if e.shouldPrintHelp(): parser.print_help() logger.error("%s: error: %s" % (gProgramName, e.getMessage())) exit_status = 2 except ExceptionNoStackTraceNeeded as e: logger.error("%s error: %s" % (gProgramName, e)) exit_status = 2 except UserAbortedException as e: logger.info("User abort requested, Exiting...") exit_status = 4 except ExecutionError as e: logger.fatal("Error occurred: %s\n Command was: '%s'\n" "rc=%d, stdout='%s', stderr='%s'" % \ (e.summary, e.cmd.cmdStr, e.cmd.results.rc, e.cmd.results.stdout, e.cmd.results.stderr)) exit_status = 2 except Exception as e: if options is None: logger.exception("%s failed. exiting...", gProgramName) else: if options.ensure_value("verbose", False): logger.exception("%s failed. exiting...", gProgramName) else: logger.fatal("%s failed. (Reason='%s') exiting..." % (gProgramName, e)) exit_status = 2 except KeyboardInterrupt: exit_status = 2 finally: if commandObject: commandObject.cleanup() sys.exit(exit_status)