def __init__(self, name, cmdStr, ctxt=LOCAL, remoteHost=None, stdin=None, nakedExecutionInfo=None, gphome=None):
self.name = name
self.cmdStr = cmdStr
self.exec_context = createExecutionContext(ctxt, remoteHost, stdin=stdin, nakedExecutionInfo=nakedExecutionInfo,
gphome=gphome)
self.remoteHost = remoteHost
self.logger = gplog.get_default_logger()
def main(argv):
global systemFSMap
global logger
logger = gplog.get_default_logger()
options = parseargs(argv)
dbConnect(options)
genServerFsList()
findFsDetails()
genFSMap()
sys.exit(0)
def __init__(self, main):
"""
Create cached yaml from class doc string of the given object,
looking for the %YAML indicating the beginning of the object's YAML plan and parse it.
Build the plan stages and tasks for the specified scenario.
"""
# main - object with yaml scenarios (input)
# sy - Stage yaml
self.logger = gplog.get_default_logger()
self.logfilename = gplog.get_logfile()
self.main = main
self.y = get_yaml(main.__class__)
self.name = main.options.scenario
if not self.name:
self.name = self.y['Default Scenario']
self.scenario = self.y['Scenarios'][self.name]
self.errors = self.y['Errors']
self.Tasks = [self._task(ty) for ty in self.scenario]
def __init__(self, numWorkers=16, items=None, daemonize=False, logger=gplog.get_default_logger()):
if numWorkers <= 0:
raise Exception("WorkerPool(): numWorkers should be greater than 0.")
self.workers = []
self.should_stop = False
self.work_queue = Queue()
self.completed_queue = Queue()
self._assigned = 0
self.daemonize = daemonize
self.logger = logger
if items is not None:
for item in items:
self.addCommand(item)
for i in range(0, numWorkers):
w = Worker("worker%d" % i, self)
self.workers.append(w)
w.start()
self.numWorkers = numWorkers
def simple_main_internal(createOptionParserFn, createCommandFn, mainOptions):
"""
If caller specifies 'pidfilename' in mainOptions then we manage the
specified pid file within the MASTER_DATA_DIRECTORY before proceeding
to execute the specified program and we clean up the pid file when
we're done.
"""
sml = None
if mainOptions is not None and 'pidfilename' in mainOptions:
sml = SimpleMainLock(mainOptions)
otherpid = sml.acquire()
if otherpid is not None:
logger = gplog.get_default_logger()
logger.error("An instance of %s is already running (pid %s)" % (getProgramName(), otherpid))
return
# at this point we have whatever lock we require
try:
simple_main_locked(createOptionParserFn, createCommandFn, mainOptions)
finally:
if sml is not None:
sml.release()
This file is a wrapper around figleaf and will start/stop coverage as
needed. It also includes a method for generating the HTML reports.
"""
import os
import random
import figleaf
import pickle
from glob import glob
from gppylib import gplog
from gppylib.commands.base import Command, LOCAL, REMOTE, ExecutionContext, RemoteExecutionContext, WorkerPool
from gppylib.commands.unix import RemoveFiles, Scp
from gppylib.operations import Operation
from gppylib.operations.unix import ListFiles, ListRemoteFiles, MakeDir
logger = gplog.get_default_logger()
COVERAGE_FILENAME = 'cover.out'
#------------------------------------------------------------------------------
class GpWriteFigleafCoverageHtml(Command):
"""Command to write out figleaf html reports to disk based on the
coverage information that has been collected."""
def __init__(self,name,filename, directory,ctxt=LOCAL,remoteHost=None):
gphome = os.getenv("GPHOME", None)
if not gphome:
raise Exception('GPHOME environment variable not set.')
cmdStr = "%s -d %s %s" % (os.path.normpath(gphome + '/lib/python/figleaf/figleaf2html'), directory, filename)
Command.__init__(self,name,cmdStr,ctxt,remoteHost)
@staticmethod
def close(self):
"""
Closes all handles to the logs we're watching.
"""
for h in self.handles.values():
h.close()
self.handles = {}
if __name__ == '__main__':
# setup gpAdminLogs logging
execname = os.path.split(sys.argv[0])[-1]
hostname = unix.getLocalHostname()
username = unix.getUserName()
setup_tool_logging(execname, hostname, username)
logger = get_default_logger()
# watch syncmaster logs
if len(sys.argv) > 2 and sys.argv[2] == 'debug':
logger.info("Checking standby master status")
watcher = SyncmasterWatcher( sys.argv[1] )
rc = watcher.monitor_logs()
watcher.close()
# report final status
# logger.info("exiting with %s" % rc)
sys.exit( rc )
def simple_main_locked(createOptionParserFn, createCommandFn, mainOptions):
"""
Not to be called externally -- use simple_main instead
"""
logger = gplog.get_default_logger()
configurationInterface.registerConfigurationProvider(
configurationImplGpdb.GpConfigurationProviderUsingGpdbCatalog())
fileSystemInterface.registerFileSystemProvider(fileSystemImplOs.GpFileSystemProviderUsingOs())
osInterface.registerOsProvider(osImplNative.GpOsProviderUsingNative())
faultProberInterface.registerFaultProber(faultProberImplGpdb.GpFaultProberImplGpdb())
commandObject = None
parser = None
forceQuiet = mainOptions is not None and mainOptions.get("forceQuietOutput")
options = None
if mainOptions is not None and mainOptions.get("programNameOverride"):
global gProgramName
gProgramName = mainOptions.get("programNameOverride")
suppressStartupLogMessage = mainOptions is not None and mainOptions.get("suppressStartupLogMessage")
useHelperToolLogging = mainOptions is not None and mainOptions.get("useHelperToolLogging")
nonuser = True if mainOptions is not None and mainOptions.get("setNonuserOnToolLogger") else False
exit_status = 1
# NOTE: if this logic is changed then also change test_main in testUtils.py
try:
execname = getProgramName()
hostname = unix.getLocalHostname()
username = unix.getUserName()
parser = createOptionParserFn()
(options, args) = parser.parse_args()
if useHelperToolLogging:
gplog.setup_helper_tool_logging(execname, hostname, username)
else:
gplog.setup_tool_logging(execname, hostname, username,
logdir=options.ensure_value("logfileDirectory", None), nonuser=nonuser)
if forceQuiet:
gplog.quiet_stdout_logging()
else:
if options.ensure_value("verbose", False):
gplog.enable_verbose_logging()
if options.ensure_value("quiet", False):
gplog.quiet_stdout_logging()
if options.ensure_value("masterDataDirectory", None) is not None:
options.master_data_directory = os.path.abspath(options.masterDataDirectory)
if not suppressStartupLogMessage:
logger.info("Starting %s with args: %s" % (gProgramName, ' '.join(sys.argv[1:])))
commandObject = createCommandFn(options, args)
exitCode = commandObject.run()
exit_status = exitCode
except ProgramArgumentValidationException, e:
if e.shouldPrintHelp():
parser.print_help()
logger.error("%s: error: %s" % (gProgramName, e.getMessage()))
exit_status = 2
def __init__(self, gpEnv, gpArray):
self.gpEnv = gpEnv
self.gpArray = gpArray
self.logger = gplog.get_default_logger()
def __init__(self):
self.logger = get_default_logger()
from gppylib.gpparseopts import OptParser, OptChecker
from gppylib.operations.startSegments import *
from gppylib.operations.buildMirrorSegments import *
from gppylib.operations.rebalanceSegments import GpSegmentRebalanceOperation
from gppylib.programs import programIoUtils
from gppylib.system import configurationInterface as configInterface
from gppylib.system.environment import GpMasterEnvironment
from gppylib.parseutils import line_reader, parse_gprecoverseg_line, canonicalize_address
from gppylib.utils import ParsedConfigFile, ParsedConfigFileRow, writeLinesToFile, \
normalizeAndValidateInputPath, TableLogger
from gppylib.gphostcache import GpInterfaceToHostNameCache
from gppylib.operations.utils import ParallelOperation
from gppylib.operations.package import SyncPackages
from gppylib.heapchecksum import HeapChecksum
logger = gplog.get_default_logger()
class PortAssigner:
"""
Used to assign new ports to segments on a host
Note that this could be improved so that we re-use ports for segments that are being recovered but this
does not seem necessary.
"""
MAX_PORT_EXCLUSIVE = 65536
def __init__(self, gpArray):
#
def simple_main_locked(createOptionParserFn, createCommandFn, mainOptions):
"""
Not to be called externally -- use simple_main instead
"""
logger = gplog.get_default_logger()
configurationInterface.registerConfigurationProvider(
configurationImplGpdb.GpConfigurationProviderUsingGpdbCatalog())
fileSystemInterface.registerFileSystemProvider(
fileSystemImplOs.GpFileSystemProviderUsingOs())
osInterface.registerOsProvider(osImplNative.GpOsProviderUsingNative())
faultProberInterface.registerFaultProber(
faultProberImplGpdb.GpFaultProberImplGpdb())
commandObject = None
parser = None
forceQuiet = mainOptions is not None and mainOptions.get(
"forceQuietOutput")
options = None
if mainOptions is not None and mainOptions.get("programNameOverride"):
global gProgramName
gProgramName = mainOptions.get("programNameOverride")
suppressStartupLogMessage = mainOptions is not None and mainOptions.get(
"suppressStartupLogMessage")
useHelperToolLogging = mainOptions is not None and mainOptions.get(
"useHelperToolLogging")
nonuser = True if mainOptions is not None and mainOptions.get(
"setNonuserOnToolLogger") else False
exit_status = 1
try:
execname = getProgramName()
hostname = unix.getLocalHostname()
username = unix.getUserName()
parser = createOptionParserFn()
(options, args) = parser.parse_args()
if useHelperToolLogging:
gplog.setup_helper_tool_logging(execname, hostname, username)
else:
gplog.setup_tool_logging(execname,
hostname,
username,
logdir=options.ensure_value(
"logfileDirectory", None),
nonuser=nonuser)
if forceQuiet:
gplog.quiet_stdout_logging()
else:
if options.ensure_value("verbose", False):
gplog.enable_verbose_logging()
if options.ensure_value("quiet", False):
gplog.quiet_stdout_logging()
if options.ensure_value("masterDataDirectory", None) is not None:
options.master_data_directory = os.path.abspath(
options.masterDataDirectory)
if not suppressStartupLogMessage:
logger.info("Starting %s with args: %s" %
(gProgramName, ' '.join(sys.argv[1:])))
commandObject = createCommandFn(options, args)
exitCode = commandObject.run()
exit_status = exitCode
except ProgramArgumentValidationException, e:
if e.shouldPrintHelp():
parser.print_help()
logger.error("%s: error: %s" % (gProgramName, e.getMessage()))
exit_status = 2
return 0
def close(self):
"""
Closes all handles to the logs we're watching.
"""
for h in self.handles.values():
h.close()
self.handles = {}
if __name__ == '__main__':
# setup gpAdminLogs logging
execname = os.path.split(sys.argv[0])[-1]
hostname = unix.getLocalHostname()
username = unix.getUserName()
setup_tool_logging(execname, hostname, username)
logger = get_default_logger()
# watch syncmaster logs
if len(sys.argv) > 2 and sys.argv[2] == 'debug':
logger.info("Checking standby master status")
watcher = SyncmasterWatcher(sys.argv[1])
rc = watcher.monitor_logs()
watcher.close()
# report final status
# logger.info("exiting with %s" % rc)
sys.exit(rc)
def __init__(self):
self.logger = get_default_logger()
def simple_main_locked(createOptionParserFn, createCommandFn, mainOptions):
"""
Not to be called externally -- use simple_main instead
"""
logger = gplog.get_default_logger()
configurationInterface.registerConfigurationProvider(
configurationImplGpdb.GpConfigurationProviderUsingGpdbCatalog())
fileSystemInterface.registerFileSystemProvider(
fileSystemImplOs.GpFileSystemProviderUsingOs())
osInterface.registerOsProvider(osImplNative.GpOsProviderUsingNative())
faultProberInterface.registerFaultProber(
faultProberImplGpdb.GpFaultProberImplGpdb())
commandObject = None
parser = None
forceQuiet = mainOptions is not None and mainOptions.get(
"forceQuietOutput")
options = None
if mainOptions is not None and mainOptions.get("programNameOverride"):
global gProgramName
gProgramName = mainOptions.get("programNameOverride")
suppressStartupLogMessage = mainOptions is not None and mainOptions.get(
"suppressStartupLogMessage")
useHelperToolLogging = mainOptions is not None and mainOptions.get(
"useHelperToolLogging")
nonuser = True if mainOptions is not None and mainOptions.get(
"setNonuserOnToolLogger") else False
exit_status = 1
try:
execname = getProgramName()
hostname = unix.getLocalHostname()
username = unix.getUserName()
parser = createOptionParserFn()
(options, args) = parser.parse_args()
if useHelperToolLogging:
gplog.setup_helper_tool_logging(execname, hostname, username)
else:
gplog.setup_tool_logging(execname,
hostname,
username,
logdir=options.ensure_value(
"logfileDirectory", None),
nonuser=nonuser)
if forceQuiet:
gplog.quiet_stdout_logging()
else:
if options.ensure_value("verbose", False):
gplog.enable_verbose_logging()
if options.ensure_value("quiet", False):
gplog.quiet_stdout_logging()
if options.ensure_value("masterDataDirectory", None) is not None:
options.master_data_directory = os.path.abspath(
options.masterDataDirectory)
if not suppressStartupLogMessage:
logger.info("Starting %s with args: %s" %
(gProgramName, ' '.join(sys.argv[1:])))
commandObject = createCommandFn(options, args)
exitCode = commandObject.run()
exit_status = exitCode
except ProgramArgumentValidationException as e:
if e.shouldPrintHelp():
parser.print_help()
logger.error("%s: error: %s" % (gProgramName, e.getMessage()))
exit_status = 2
except ExceptionNoStackTraceNeeded as e:
logger.error("%s error: %s" % (gProgramName, e))
exit_status = 2
except UserAbortedException as e:
logger.info("User abort requested, Exiting...")
exit_status = 4
except ExecutionError as e:
logger.fatal("Error occurred: %s\n Command was: '%s'\n"
"rc=%d, stdout='%s', stderr='%s'" % \
(e.summary, e.cmd.cmdStr, e.cmd.results.rc, e.cmd.results.stdout,
e.cmd.results.stderr))
exit_status = 2
except Exception as e:
if options is None:
logger.exception("%s failed. exiting...", gProgramName)
else:
if options.ensure_value("verbose", False):
logger.exception("%s failed. exiting...", gProgramName)
else:
logger.fatal("%s failed. (Reason='%s') exiting..." %
(gProgramName, e))
exit_status = 2
except KeyboardInterrupt:
exit_status = 2
finally:
if commandObject:
commandObject.cleanup()
sys.exit(exit_status)
def __init__(self, gpEnv, gpArray):
self.gpEnv = gpEnv
self.gpArray = gpArray
self.logger = gplog.get_default_logger()