def UserFacingMainline(main, argv, modulesToInitialize=None, parser=None):
"""Helper function that initializes some modules and then calls main.
Used to centralize error handling for common initialization routines and to set up the
initial component hosts.
"""
if parser is None:
parser = Setup.defaultParser()
setup = Setup.defaultSetup()
parsedArguments = parser.parse_args(argv[1:])
setup.processArgs(parsedArguments)
setup.config.configureLoggingForUserProgram()
with Setup.PushSetup(setup):
initializeModules(modulesToInitialize)
result = main(parsedArguments)
if result is None:
result = 0
sys.stdout.flush()
sys.stderr.flush()
os._exit(result)
def UserFacingMainline(main, argv, modulesToInitialize=None, parser=None):
"""Helper function that initializes some modules and then calls main.
Used to centralize error handling for common initialization routines and to set up the
initial component hosts.
"""
if parser is None:
parser = Setup.defaultParser()
setup = Setup.defaultSetup()
parsedArguments = parser.parse_args(argv[1:])
setup.processArgs(parsedArguments)
setup.config.configureLoggingForUserProgram()
with Setup.PushSetup(setup):
initializeModules(modulesToInitialize)
result = main(parsedArguments)
if result is None:
result = 0
sys.stdout.flush()
sys.stderr.flush()
os._exit(result)
def createParser():
desc = """Utility for validating a stream of LocalSchedulerEvent objects in a test failure. """
parser = Setup.defaultParser(description=desc)
parser.add_argument('file', help='names of file to be read')
parser.add_argument('-n',
'--no_validation',
dest='validation',
action='store_false',
default=True,
required=False,
help="don't validate the response stream")
return parser
def createParser():
parser = Setup.defaultParser(
description='Simulate an Amazon EC2 cluster'
)
parser.add_argument('--testAxiomsPath', default=UNIT_TEST_AXIOMS_PATH)
parser.add_argument('--numRandVals', default=8)
parser.add_argument('--numRelaxations', default=4)
parser.add_argument('--maxForRelax', default=8)
parser.add_argument('--maxForRand', default=16)
parser.add_argument('--seed', default=0)
return parser
def createParser():
parser = Setup.defaultParser(
minimalParser=True,
description = "command line interpreter for FORA",
epilog = epilog,
formatter_class=argparse.RawDescriptionHelpFormatter
)
parser.add_argument(
'-i',
'--interpreter',
dest='alwaysRunInterpreted',
action='store_true',
default=False,
required=False,
help="always run interactively"
)
parser.add_argument(
'-t',
'--time',
dest='shouldPrintEvaluationTime',
action='store_true',
default=False,
required=False,
help="show timestamps between runs"
)
parser.add_argument(
'-e',
'--expression',
dest='expressionsToEvaluate',
action='append',
required=False,
default = [],
help="add an expression to the evaluate list"
)
parser.add_argument(
'-r',
'--repeat_expression',
dest='repeaters',
action='append',
required=False,
default = [],
help="add an expression to be evaluated repeatedly"
)
parser.add_argument(
'files',
nargs='*',
help='names of files to be evaluated'
)
return parser
def createParser():
desc = """Read a stream of CumulusWorkerEvent objects and print them"""
epilog = """
Example:
python extractCumulusWorkerEventLogs.py **/CumulusServiceEvents.*.log \\
--ts 1398362678.66 1398362703.98 \\
--exclusions SplitOrMoveIfNeeded Priority \\
--inclusions 866DEDC48E92 1A255CBEF9E1 WorkerReadyToCompute
will dump all logs below the current directory, with timestamps between the given bounds,
and with inclusions and exclusions given
"""
parser = Setup.defaultParser(
minimalParser=True,
description=desc,
epilog=epilog,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('--summarize',
help='only summarize',
default=False,
action='store_true')
parser.add_argument('--state-changes-only',
help='only show state-changes of computations',
default=False,
action='store_true',
dest='state_changes_only')
parser.add_argument('--ts',
help='timestamp range as a pair of floats',
dest='timestamps',
default=None,
type=float,
nargs=2,
action='store')
parser.add_argument('files', help='files to read', nargs='*')
parser.add_argument(
'--inclusions',
nargs='*',
help='strings to search for and include in log messages')
parser.add_argument(
'--exclusions',
nargs='*',
help='strings to search for and exclude from log messages')
return parser
def createParser():
desc = """Utility for validating a stream of LocalSchedulerEvent objects in a test failure. """
parser = Setup.defaultParser(
description = desc
)
parser.add_argument(
'file',
help='names of file to be read'
)
parser.add_argument(
'-n',
'--no_validation',
dest='validation',
action='store_false',
default=True,
required=False,
help="don't validate the response stream"
)
return parser
def createParser():
parser = Setup.defaultParser(
minimalParser=True,
description="command line interpreter for FORA",
epilog=epilog,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('-i',
'--interpreter',
dest='alwaysRunInterpreted',
action='store_true',
default=False,
required=False,
help="always run interactively")
parser.add_argument('-t',
'--time',
dest='shouldPrintEvaluationTime',
action='store_true',
default=False,
required=False,
help="show timestamps between runs")
parser.add_argument('-e',
'--expression',
dest='expressionsToEvaluate',
action='append',
required=False,
default=[],
help="add an expression to the evaluate list")
parser.add_argument('-r',
'--repeat_expression',
dest='repeaters',
action='append',
required=False,
default=[],
help="add an expression to be evaluated repeatedly")
parser.add_argument('files',
nargs='*',
help='names of files to be evaluated')
return parser
for key,val in data.iteritems():
entries.append(
NativeJson.Json(
{'key': tuple(key[ix] for ix in range(len(key))),
'val': val
}
)
)
toWrite.append({
'keyspace': (k.type, k.name, k.dimension),
'values': entries
})
print >> outputStream, NativeJson.Json(toWrite)
def main(parsedArgs):
sharedStateToHumanReadableFormat(parsedArgs.path, sys.stdout)
if __name__ == "__main__":
parser = Setup.defaultParser(description="Dump the contents of SharedState to standard out")
parser.add_argument('path', help='path to the shared-state directory')
Mainline.UserFacingMainline(main, sys.argv, parser=parser)
def createParser():
parser = Setup.defaultParser(description="Simulate an Amazon EC2 cluster")
def UnitTestMainline(modulesToInitialize=None,
exit=True,
loginConfiguration=LoginConfiguration.LoginConfiguration.
defaultForTesting(),
disableLogCapture=False):
"""Helper function to call the unittest mainline
Constructs a default Setup from the config. Calls
x.initialize(setup)
for all x in modulesToInitialize.
"""
modulesToInitialize = modulesToInitialize or []
import ufora.native
argv = list(sys.argv)
setup = Setup.defaultSetup()
parser = Setup.defaultParser()
unitTestOptions = parser.add_argument_group(
title='UnitTestMainline options',
description='options for the general UnitTest framework')
addNoseVerbosityArgument(unitTestOptions)
unitTestOptions.add_argument('testHarnessArguments',
nargs=argparse.REMAINDER)
parser.add_argument(
'-timeout',
type=float,
nargs=1,
help='fail test if not completed after TIMEOUT seconds',
default=None)
parsedArguments = parser.parse_args(argv[1:])
if parsedArguments.timeout is not None:
UnitTestCommon.startTimeoutThread(parsedArguments.timeout[0])
setup.processArgs(parsedArguments)
testHarnessArguments = ["dummy"] + parsedArguments.testHarnessArguments
testHarnessArguments.append('--verbosity=0')
if parsedArguments.testHarnessVerbose or disableLogCapture:
testHarnessArguments.append('--nocaptureall')
setup.config.configureLoggingForUserProgram()
plugins = nose.plugins.manager.PluginManager([OutputCaptureNosePlugin()])
config = nose.config.Config(plugins=plugins)
config.configure(testHarnessArguments)
with Setup.PushSetup(setup):
result = None
for toInitialize in modulesToInitialize:
toInitialize.initialize(setup)
result = nose.core.TestProgram(exit=False,
defaultTest="__main__",
config=config,
argv=testHarnessArguments).success
sys.stdout.flush()
sys.stderr.flush()
ufora.native.Tests.gcov_flush()
if not exit:
return 0 if result else 1
else:
os._exit(0 if result else 1)
def UnitTestMainline(modulesToInitialize=None,
exit=True,
loginConfiguration=LoginConfiguration.LoginConfiguration.defaultForTesting(),
disableLogCapture=False):
"""Helper function to call the unittest mainline
Constructs a default Setup from the config. Calls
x.initialize(setup)
for all x in modulesToInitialize.
"""
modulesToInitialize = modulesToInitialize or []
import ufora.native
argv = list(sys.argv)
setup = Setup.defaultSetup()
parser = Setup.defaultParser()
unitTestOptions = parser.add_argument_group(
title='UnitTestMainline options',
description='options for the general UnitTest framework'
)
addNoseVerbosityArgument(unitTestOptions)
unitTestOptions.add_argument(
'testHarnessArguments',
nargs=argparse.REMAINDER
)
parser.add_argument('-timeout',
type=float,
nargs=1,
help='fail test if not completed after TIMEOUT seconds',
default=None)
parsedArguments = parser.parse_args(argv[1:])
if parsedArguments.timeout is not None:
UnitTestCommon.startTimeoutThread(parsedArguments.timeout[0])
setup.processArgs(parsedArguments)
testHarnessArguments = ["dummy"] + parsedArguments.testHarnessArguments
testHarnessArguments.append('--verbosity=0')
if parsedArguments.testHarnessVerbose or disableLogCapture:
testHarnessArguments.append('--nocaptureall')
setup.config.configureLoggingForUserProgram()
plugins = nose.plugins.manager.PluginManager([OutputCaptureNosePlugin()])
config = nose.config.Config(plugins=plugins)
config.configure(testHarnessArguments)
with Setup.PushSetup(setup):
result = None
for toInitialize in modulesToInitialize:
toInitialize.initialize(setup)
result = nose.core.TestProgram(
exit=False,
defaultTest="__main__",
config=config,
argv=testHarnessArguments
).success
sys.stdout.flush()
sys.stderr.flush()
ufora.native.Tests.gcov_flush()
if not exit:
return 0 if result else 1
else:
os._exit(0 if result else 1)
def createParser():
desc = """Read a stream of CumulusWorkerEvent objects and print them"""
epilog = """
Example:
python extractCumulusWorkerEventLogs.py **/CumulusServiceEvents.*.log \\
--ts 1398362678.66 1398362703.98 \\
--exclusions SplitOrMoveIfNeeded Priority \\
--inclusions 866DEDC48E92 1A255CBEF9E1 WorkerReadyToCompute
will dump all logs below the current directory, with timestamps between the given bounds,
and with inclusions and exclusions given
"""
parser = Setup.defaultParser(
minimalParser = True,
description = desc,
epilog = epilog,
formatter_class=argparse.RawDescriptionHelpFormatter
)
parser.add_argument(
'--summarize',
help='only summarize',
default=False,
action='store_true'
)
parser.add_argument(
'--state-changes-only',
help='only show state-changes of computations',
default=False,
action='store_true',
dest='state_changes_only'
)
parser.add_argument(
'--ts',
help='timestamp range as a pair of floats',
dest='timestamps',
default=None,
type=float,
nargs=2,
action='store'
)
parser.add_argument(
'files',
help='files to read',
nargs='*'
)
parser.add_argument(
'--inclusions',
nargs='*',
help='strings to search for and include in log messages'
)
parser.add_argument(
'--exclusions',
nargs='*',
help='strings to search for and exclude from log messages'
)
return parser
for k in keyspaces:
storageForKeyspace = storage.storageForKeyspace(k, 0)
data = storageForKeyspace.readKeyValueMap()
entries = []
for key, val in data.iteritems():
entries.append(
NativeJson.Json({
'key': tuple(key[ix] for ix in range(len(key))),
'val': val
}))
toWrite.append({
'keyspace': (k.type, k.name, k.dimension),
'values': entries
})
print >> outputStream, NativeJson.Json(toWrite)
def main(parsedArgs):
sharedStateToHumanReadableFormat(parsedArgs.path, sys.stdout)
if __name__ == "__main__":
parser = Setup.defaultParser(
description="Dump the contents of SharedState to standard out")
parser.add_argument('path', help='path to the shared-state directory')
Mainline.UserFacingMainline(main, sys.argv, parser=parser)
def createParser():
parser = Setup.defaultParser(description='Simulate an Amazon EC2 cluster')
