def __init__(self, main): """ Create cached yaml from class doc string of the given object, looking for the %YAML indicating the beginning of the object's YAML plan and parse it. Build the plan stages and tasks for the specified scenario. """ # main - object with yaml scenarios (input) # sy - Stage yaml self.logger = gplog.get_default_logger() self.logfilename = gplog.get_logfile() self.main = main self.y = get_yaml(main.__class__) self.name = main.options.scenario if not self.name: self.name = self.y['Default Scenario'] self.scenario = self.y['Scenarios'][self.name] self.errors = self.y['Errors'] self.Tasks = [self._task(ty) for ty in self.scenario]
def simple_main_internal(createOptionParserFn, createCommandFn, mainOptions): """ If caller specifies 'pidfilename' in mainOptions then we manage the specified pid file within the MASTER_DATA_DIRECTORY before proceeding to execute the specified program and we clean up the pid file when we're done. """ sml = None if mainOptions is not None and 'pidfilename' in mainOptions: sml = SimpleMainLock(mainOptions) otherpid = sml.acquire() if otherpid is not None: logger = gplog.get_default_logger() logger.error("An instance of %s is already running (pid %s)" % (getProgramName(), otherpid)) return # at this point we have whatever lock we require try: simple_main_locked(createOptionParserFn, createCommandFn, mainOptions) finally: if sml is not None: sml.release()
import os import sys import pickle from qautils.gppylib import gplog from qautils.gppylib.commands.base import OperationWorkerPool, Command, REMOTE from qautils.gppylib.operations import Operation DEFAULT_NUM_WORKERS = 64 logger = gplog.get_default_logger() class RemoteOperation(Operation): # TODO: The Operation that is run remotely cannot return Exceptions. # This can be resolved easily with a class that wraps the exception: ExceptionCapsule. (Thank you, Pyro.) # TODO: Remote traceback is lost. Again, this can be solved by embedding remote traceback in an ExceptionCapsule. """ RemoteOperation communicates w/ gpoperation.py on the remote end, with the following assumptions. 1) gppylib exists 2) gpoperation.py can see gppylib as a top-level module 3) obj is defined at the top level of its module This requirement actually arises out of an underlying pickle issue, which in turn, appears to result from a python class oddity. If class B is defined within class A, it does not consider A to be its module. B is merely a class that is an attribute of A. For this reason, once instantiated, B cannot be rebuilt from its module name and class name alone. Its outer class A is a missing piece of information that gppickle cannot attain from python internals. 4) Most importantly, the operation provided here must be imported into the gppylib... namespace. Otherwise, gpoperation.py will be unable to deserialize and import it on the remote end. In the normal gppylib use case, some bin/ level script will use an absolute import to bring in something from qautils.gppylib. In this manner, any ensuing imports (even if they're relative) will still be imported into the
#!/usr/bin/env python # # from qautils.gppylib.testold.testDriver import TestDriver from qautils.gppylib.programs.clsSystemState import * from qautils.gppylib.mainUtils import * from qautils.gppylib.testold.testUtils import * from qautils.gppylib.system import fileSystemImplTest, fileSystemInterface from qautils.gppylib.gplog import get_default_logger programName = sys.argv[0] parserFn = GpSystemStateProgram.createParser commandFn = GpSystemStateProgram.createProgram driver = TestDriver() logger = get_default_logger() ############### # # Now the tests: # #################################### # # def testGuessSpreadAndMultiHome(args): configStr = args[0] driver.setSegments(configStr, gparray.FAULT_STRATEGY_FILE_REPLICATION) gpArray = configInterface.getConfigurationProvider().initializeProvider(5432).loadSystemConfig(useUtilityMode=False)
def simple_main_locked(createOptionParserFn, createCommandFn, mainOptions): """ Not to be called externally -- use simple_main instead """ logger = gplog.get_default_logger() configurationInterface.registerConfigurationProvider( configurationImplGpdb.GpConfigurationProviderUsingGpdbCatalog()) fileSystemInterface.registerFileSystemProvider( fileSystemImplOs.GpFileSystemProviderUsingOs()) osInterface.registerOsProvider(osImplNative.GpOsProviderUsingNative()) faultProberInterface.registerFaultProber( faultProberImplGpdb.GpFaultProberImplGpdb()) commandObject = None parser = None forceQuiet = mainOptions is not None and mainOptions.get( "forceQuietOutput") options = None if mainOptions is not None and mainOptions.get("programNameOverride"): global gProgramName gProgramName = mainOptions.get("programNameOverride") suppressStartupLogMessage = mainOptions is not None and mainOptions.get( "suppressStartupLogMessage") useHelperToolLogging = mainOptions is not None and mainOptions.get( "useHelperToolLogging") nonuser = True if mainOptions is not None and mainOptions.get( "setNonuserOnToolLogger") else False # NOTE: if this logic is changed then also change test_main in testUtils.py try: execname = getProgramName() hostname = unix.getLocalHostname() username = unix.getUserName() parser = createOptionParserFn() (options, args) = parser.parse_args() if useHelperToolLogging: gplog.setup_helper_tool_logging(execname, hostname, username) else: gplog.setup_tool_logging(execname, hostname, username, logdir=options.ensure_value( "logfileDirectory", None), nonuser=nonuser) if forceQuiet: gplog.quiet_stdout_logging() else: if options.ensure_value("verbose", False): gplog.enable_verbose_logging() if options.ensure_value("quiet", False): gplog.quiet_stdout_logging() if options.ensure_value("masterDataDirectory", None) is not None: options.master_data_directory = os.path.abspath( options.masterDataDirectory) if not suppressStartupLogMessage: logger.info("Starting %s with args: %s" % (gProgramName, ' '.join(sys.argv[1:]))) commandObject = createCommandFn(options, args) exitCode = commandObject.run() sys.exit(exitCode) except ProgramArgumentValidationException, e: if e.shouldPrintHelp(): parser.print_help() logger.error("%s: error: %s" % (gProgramName, e.getMessage())) sys.exit(2)