def cli(verbose: int):
levels = {
0: logging.WARNING,
1: logging.INFO,
2: logging.DEBUG,
}
log.configure(levels.get(verbose, logging.DEBUG))
def main():
'''
Runs a tornado http server that listens for any
integration test json result get/post requests
'''
log.configure(level=logging.DEBUG)
if not os.path.exists(RESULTS_DIRECTORY):
os.makedirs(RESULTS_DIRECTORY)
state_map = {}
# for instance states in stateful processing
state_result_map = {}
application = tornado.web.Application([
(r"/", MainHandler),
(r"^/results/([a-zA-Z0-9_-]+$)", FileHandler),
(r"^/state", MemoryMapGetAllHandler, dict(state_map=state_map)),
(r"^/state/([a-zA-Z0-9_-]+$)", MemoryMapHandler, dict(state_map=state_map)),
(r"^/stateResults/([a-zA-Z0-9_-]+$)", StateResultHandler, dict(result_map=state_result_map)),
])
if len(sys.argv) == 1:
logging.error("Missing argument: port addresss")
sys.exit(1)
try:
port = int(sys.argv[1])
except Exception as e:
logging.error("Exception: %s \nprovide a valid port address", e)
sys.exit(1)
logging.info("Starting server at port " + str(port))
application.listen(port)
tornado.ioloop.IOLoop.current().start()
def main():
'''
Runs a tornado http server that listens for any
integration test json result get/post requests
'''
log.configure(level=logging.DEBUG)
if not os.path.exists(RESULTS_DIRECTORY):
os.makedirs(RESULTS_DIRECTORY)
state_map = {}
# for instance states in stateful processing
state_result_map = {}
application = tornado.web.Application([
(r"/", MainHandler),
(r"^/results/([a-zA-Z0-9_-]+$)", FileHandler),
(r"^/state", MemoryMapGetAllHandler, dict(state_map=state_map)),
(r"^/state/([a-zA-Z0-9_-]+$)", MemoryMapHandler, dict(state_map=state_map)),
(r"^/stateResults/([a-zA-Z0-9_-]+$)", StateResultHandler, dict(result_map=state_result_map)),
])
if len(sys.argv) == 1:
logging.error("Missing argument: port addresss")
sys.exit(1)
try:
port = int(sys.argv[1])
except Exception as e:
logging.error("Exception: %s \nprovide a valid port address", e)
sys.exit(1)
logging.info("Starting server at port " + str(port))
application.listen(port)
tornado.ioloop.IOLoop.instance().start()
def cli(
config_file: str,
stmgr_type: str,
name: str,
rootpath: str,
tunnelhost: str,
hostport: str,
port: int,
verbose: bool,
) -> None:
"""
A HTTP service for serving data about clusters.
The statemanager's config from the given config file can be overrided using
options on this executable.
"""
log.configure(logging.DEBUG if verbose else logging.INFO)
# set Tornado global option
define_options(port, config_file)
stmgr_override = {
"type": stmgr_type,
"name": name,
"rootpath": rootpath,
"tunnelhost": tunnelhost,
"hostport": hostport,
}
config = Config(create_tracker_config(config_file, stmgr_override))
# create Tornado application
application = Application(config)
# pylint: disable=unused-argument
# SIGINT handler:
# 1. stop all the running zkstatemanager and filestatemanagers
# 2. stop the Tornado IO loop
def signal_handler(signum, frame):
# start a new line after ^C character because this looks nice
print('\n', end='')
application.stop()
tornado.ioloop.IOLoop.instance().stop()
# associate SIGINT and SIGTERM with a handler
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)
Log.info("Running on port: %d", port)
if config_file:
Log.info("Using config file: %s", config_file)
Log.info(f"Using state manager:\n{config}")
http_server = tornado.httpserver.HTTPServer(application)
http_server.listen(port)
tornado.ioloop.IOLoop.instance().start()
def cli(host: str, port: int, base_url_option: str, tracker_url_option: str,
verbose: bool) -> None:
"""Start a web UI for heron which renders information from the tracker."""
global base_url, tracker_url
base_url = base_url_option
log.configure(level=logging.DEBUG if verbose else logging.INFO)
tracker.tracker_url = tracker_url_option
uvicorn.run(app, host=host, port=port, log_config=None)
def main():
''' main '''
log.configure(level=logging.DEBUG)
conf_file = DEFAULT_TEST_CONF_FILE
# Read the configuration file from package
conf_string = pkgutil.get_data(__name__, conf_file)
decoder = json.JSONDecoder(strict=False)
# Convert the conf file to a json format
conf = decoder.decode(conf_string)
# Parse the arguments passed via command line
parser = argparse.ArgumentParser(description='This is the heron integration test framework')
parser.add_argument('-hc', '--heron-cli-path', dest='heron_cli_path',
default=conf['heronCliPath'])
parser.add_argument('-tb', '--tests-bin-path', dest='tests_bin_path')
parser.add_argument('-cl', '--cluster', dest='cluster', default=conf['cluster'])
parser.add_argument('-ev', '--env', dest='env', default=conf['env'])
parser.add_argument('-rl', '--role', dest='role', default=conf['role'])
parser.add_argument('-rh', '--http-server-hostname', dest='http_server_hostname')
parser.add_argument('-rp', '--http-server-port', dest='http_server_port', type=int,
default=conf['resultsServerPort'])
parser.add_argument('-tp', '--topologies-path', dest='topologies_path')
parser.add_argument('-ts', '--test-topology-pattern', dest='test_topology_pattern', default=None)
parser.add_argument('-pi', '--release-package-uri', dest='release_package_uri', default=None)
parser.add_argument('-cd', '--cli-config-path', dest='cli_config_path',
default=conf['cliConfigPath'])
parser.add_argument('-ms', '--max-thread-number', dest='max_thread_number', default=1)
#parser.add_argument('-dt', '--disable-topologies', dest='disabledTopologies', default='',
# help='comma separated test case(classpath) name that will not be run')
#parser.add_argument('-et', '--enable-topologies', dest='enableTopologies', default=None,
# help='comma separated test case(classpath) name that will be run only')
args, unknown_args = parser.parse_known_args()
if unknown_args:
logging.error('Unknown argument passed to %s: %s', sys.argv[0], unknown_args[0])
sys.exit(1)
tests_start_time = int(time.time())
run_tests(conf, args)
total = len(failures) + len(successes)
logging.info("Total integration test time = %ss" % (int(time.time()) - tests_start_time))
if not failures:
logging.info("SUCCESS: %s (all) tests passed:", len(successes))
for test in successes:
logging.info(" - %s: %s", ("[%ss]" % test[1]).ljust(8), test[0])
sys.exit(0)
else:
logging.error("FAILURE: %s/%s tests failed:", len(failures), total)
for test in failures:
logging.error(" - %s: %s", ("[%ss]" % test[1]).ljust(8), test[0])
sys.exit(1)
def setup(shardid):
# Redirect stdout and stderr to files in append mode
# The filename format is heron-executor-<container_id>.stdxxx
log.configure(logfile='heron-executor-%s.stdout' % shardid)
Log.info('Set up process group; executor becomes leader')
os.setpgrp() # create new process group, become its leader
Log.info('Register the SIGTERM signal handler')
signal.signal(signal.SIGTERM, signal_handler)
Log.info('Register the atexit clean up')
atexit.register(cleanup)
def setup(shardid):
# Redirect stdout and stderr to files in append mode
# The filename format is heron-executor-<container_id>.stdxxx
log.configure(logfile='heron-executor-%s.stdout' % shardid)
Log.info('Set up process group; executor becomes leader')
os.setpgrp() # create new process group, become its leader
Log.info('Register the SIGTERM signal handler')
signal.signal(signal.SIGTERM, signal_handler)
Log.info('Register the atexit clean up')
atexit.register(cleanup)
def main():
''' main '''
log.configure(level=logging.DEBUG)
conf_file = DEFAULT_TEST_CONF_FILE
# Read the configuration file from package
conf_string = pkgutil.get_data(__name__, conf_file)
decoder = json.JSONDecoder(strict=False)
# Convert the conf file to a json format
conf = decoder.decode(conf_string)
# Parse the arguments passed via command line
parser = argparse.ArgumentParser(description='This is the heron integration test framework')
parser.add_argument('-hc', '--heron-cli-path', dest='heron_cli_path',
default=conf['heronCliPath'])
parser.add_argument('-tb', '--tests-bin-path', dest='tests_bin_path')
parser.add_argument('-cl', '--cluster', dest='cluster', default=conf['cluster'])
parser.add_argument('-ev', '--env', dest='env', default=conf['env'])
parser.add_argument('-rl', '--role', dest='role', default=conf['role'])
parser.add_argument('-rh', '--http-server-hostname', dest='http_server_hostname')
parser.add_argument('-rp', '--http-server-port', dest='http_server_port', type=int,
default=conf['resultsServerPort'])
parser.add_argument('-tp', '--topologies-path', dest='topologies_path')
parser.add_argument('-ts', '--test-topology-pattern', dest='test_topology_pattern', default=None)
parser.add_argument('-pi', '--release-package-uri', dest='release_package_uri', default=None)
parser.add_argument('-cd', '--cli-config-path', dest='cli_config_path',
default=conf['cliConfigPath'])
#parser.add_argument('-dt', '--disable-topologies', dest='disabledTopologies', default='',
# help='comma separated test case(classpath) name that will not be run')
#parser.add_argument('-et', '--enable-topologies', dest='enableTopologies', default=None,
# help='comma separated test case(classpath) name that will be run only')
args, unknown_args = parser.parse_known_args()
if unknown_args:
logging.error('Unknown argument passed to %s: %s', sys.argv[0], unknown_args[0])
sys.exit(1)
(successes, failures) = run_tests(conf, args)
total = len(failures) + len(successes)
if not failures:
logging.info("SUCCESS: %s (all) tests passed:", len(successes))
for test in successes:
logging.info(" - %s: %s", ("[%ss]" % test[1]).ljust(8), test[0])
sys.exit(0)
else:
logging.error("FAILURE: %s/%s tests failed:", len(failures), total)
for test in failures:
logging.error(" - %s: %s", ("[%ss]" % test[1]).ljust(8), test[0])
sys.exit(1)
def main():
"""
main
"""
log.configure(level=logging.DEBUG)
conf_file = DEFAULT_TEST_CONF_FILE
# Read the configuration file from package
conf_string = pkgutil.get_data(__name__, conf_file).decode()
decoder = json.JSONDecoder(strict=False)
# Convert the conf file to a json format
conf = decoder.decode(conf_string)
# Parse the arguments passed via command line
parser = argparse.ArgumentParser(description='This is the heron integration test framework')
parser.add_argument('-hc', '--heron-cli-path', dest='heron_cli_path',
default=conf['heronCliPath'])
parser.add_argument('-tb', '--tests-bin-path', dest='tests_bin_path')
parser.add_argument('-cl', '--cluster', dest='cluster', default=conf['cluster'])
parser.add_argument('-ev', '--env', dest='env', default=conf['env'])
parser.add_argument('-rl', '--role', dest='role', default=conf['role'])
parser.add_argument('-rh', '--http-hostname', dest='http_hostname', default='localhost')
parser.add_argument('-rp', '--http-port', dest='http_port', type=int,
default='8080')
parser.add_argument('-tp', '--topologies-path', dest='topologies_path')
parser.add_argument('-ts', '--test-topology-pattern', dest='test_topology_pattern', default=None)
parser.add_argument('-pi', '--release-package-uri', dest='release_package_uri', default=None)
parser.add_argument('-cd', '--cli-config-path', dest='cli_config_path',
default=conf['cliConfigPath'])
parser.add_argument('-ms', '--max-thread-number', dest='max_thread_number', default=1)
args, unknown_args = parser.parse_known_args()
if unknown_args:
logging.error('Unknown argument passed to %s: %s', sys.argv[0], unknown_args[0])
sys.exit(1)
tests_start_time = int(time.time())
run_topology_tests(conf, args)
total = len(failures) + len(successes)
logging.info("Total integration topology test time = %ss" % (int(time.time()) - tests_start_time))
if not failures:
logging.info("SUCCESS: %s (all) tests passed:", len(successes))
for test in successes:
logging.info(" - %s: %s", ("[%ss]" % test[1]).ljust(8), test[0])
sys.exit(0)
else:
logging.error("FAILURE: %s/%s tests failed:", len(failures), total)
for test in failures:
logging.error(" - %s: %s", ("[%ss]" % test[1]).ljust(8), test[0])
sys.exit(1)
def main():
"""
main
"""
log.configure(level=logging.DEBUG)
conf_file = DEFAULT_TEST_CONF_FILE
# Read the configuration file from package
conf_string = pkgutil.get_data(__name__, conf_file)
decoder = json.JSONDecoder(strict=False)
# Convert the conf file to a json format
conf = decoder.decode(conf_string)
# Parse the arguments passed via command line
parser = argparse.ArgumentParser(description='This is the heron integration test framework')
parser.add_argument('-hc', '--heron-cli-path', dest='heron_cli_path',
default=conf['heronCliPath'])
parser.add_argument('-tb', '--tests-bin-path', dest='tests_bin_path')
parser.add_argument('-cl', '--cluster', dest='cluster', default=conf['cluster'])
parser.add_argument('-ev', '--env', dest='env', default=conf['env'])
parser.add_argument('-rl', '--role', dest='role', default=conf['role'])
parser.add_argument('-rh', '--http-hostname', dest='http_hostname', default='localhost')
parser.add_argument('-rp', '--http-port', dest='http_port', type=int,
default='8080')
parser.add_argument('-tp', '--topologies-path', dest='topologies_path')
parser.add_argument('-ts', '--test-topology-pattern', dest='test_topology_pattern', default=None)
parser.add_argument('-pi', '--release-package-uri', dest='release_package_uri', default=None)
parser.add_argument('-cd', '--cli-config-path', dest='cli_config_path',
default=conf['cliConfigPath'])
args, unknown_args = parser.parse_known_args()
if unknown_args:
logging.error('Unknown argument passed to %s: %s', sys.argv[0], unknown_args[0])
sys.exit(1)
(successes, failures) = run_topology_tests(conf, args)
total = len(failures) + len(successes)
if not failures:
logging.info("SUCCESS: %s (all) tests passed:", len(successes))
for test in successes:
logging.info(" - %s: %s", ("[%ss]" % test[1]).ljust(8), test[0])
sys.exit(0)
else:
logging.error("FAILURE: %s/%s tests failed:", len(failures), total)
for test in failures:
logging.error(" - %s: %s", ("[%ss]" % test[1]).ljust(8), test[0])
sys.exit(1)
def setup(executor):
"""Set up log, process and signal handlers"""
# pylint: disable=unused-argument
def signal_handler(signal_to_handle, frame):
# We would do nothing here but just exit
# Just catch the SIGTERM and then cleanup(), registered with atexit, would invoke
Log.info('signal_handler invoked with signal %s', signal_to_handle)
executor.stop_state_manager_watches()
sys.exit(signal_to_handle)
def cleanup():
"""Handler to trigger when receiving the SIGTERM signal
Do cleanup inside this method, including:
1. Terminate all children processes
"""
Log.info('Executor terminated; exiting all process in executor.')
# Kill child processes first and wait for log collection to finish
for pid in list(executor.processes_to_monitor.keys()):
os.kill(pid, signal.SIGTERM)
time.sleep(5)
# We would not wait or check whether process spawned dead or not
os.killpg(0, signal.SIGTERM)
# Redirect stdout and stderr to files in append mode
# The filename format is heron-executor-<container_id>.stdxxx
shardid = executor.shard
log.configure(logfile='heron-executor-%s.stdout' % shardid)
pid = os.getpid()
sid = os.getsid(pid)
# POSIX prohibits the change of the process group ID of a session leader
if pid != sid:
Log.info('Set up process group; executor becomes leader')
os.setpgrp() # create new process group, become its leader
Log.info('Register the SIGTERM signal handler')
signal.signal(signal.SIGTERM, signal_handler)
Log.info('Register the atexit clean up')
atexit.register(cleanup)
def cli(
config_file: str,
stmgr_type: str,
name: str,
rootpath: str,
tunnelhost: str,
hostport: str,
port: int,
verbose: bool,
) -> None:
"""
A HTTP service for serving data about clusters.
The statemanager's config from the given config file can be overrided using
options on this executable.
"""
log_level = logging.DEBUG if verbose else logging.INFO
log.configure(log_level)
stmgr_override = {
"type": stmgr_type,
"name": name,
"rootpath": rootpath,
"tunnelhost": tunnelhost,
"hostport": hostport,
}
config = Config(create_tracker_config(config_file, stmgr_override))
state.tracker = Tracker(config)
state.tracker.sync_topologies()
# this only returns when interrupted
uvicorn.run(app, host="0.0.0.0", port=port, log_level=log_level)
state.tracker.stop_sync()
# non-daemon threads linger and stop the process for quitting, so signal
# for cleaning up
os.kill(os.getpid(), signal.SIGKILL)
def main():
""" main """
log.configure(level=logging.DEBUG)
# Read the configuration file from package
conf_file = DEFAULT_TEST_CONF_FILE
conf_string = pkgutil.get_data(__name__, conf_file)
decoder = json.JSONDecoder(strict=False)
# Convert the conf file to a json format
conf = decoder.decode(conf_string)
args = dict()
home_directory = os.path.expanduser("~")
args['cluster'] = conf['cluster']
args['topologyName'] = conf['topology']['topologyName']
args['topologyClassPath'] = conf['topology']['topologyClassPath']
args['workingDirectory'] = os.path.join(home_directory, ".herondata",
"topologies", conf['cluster'],
getpass.getuser(),
args['topologyName'])
args['cliPath'] = os.path.expanduser(conf['heronCliPath'])
args['trackerPath'] = os.path.expanduser(conf['heronTrackerPath'])
args['trackerPort'] = _random_port()
args['outputFile'] = os.path.join(args['workingDirectory'],
conf['topology']['outputFile'])
args['readFile'] = os.path.join(args['workingDirectory'],
conf['topology']['readFile'])
args['testJarPath'] = conf['testJarPath']
test_classes = TEST_CLASSES
if len(sys.argv) > 1:
first_arg = sys.argv[1]
class_tokens = first_arg.split(".")
if first_arg == "-h" or len(class_tokens) < 2:
usage()
import importlib
package_tokens = class_tokens[:-1]
test_class = class_tokens[-1]
if len(package_tokens) == 1: # prepend base packages for convenience
test_module = "integration_test.src.python.local_test_runner." + package_tokens[
0]
else:
test_module = '.'.join(package_tokens)
logging.info("test_module %s", test_module)
logging.info("test_class %s", test_class)
test_classes = [
getattr(importlib.import_module(test_module), test_class)
]
start_time = time.time()
(successes, failures) = run_tests(test_classes, args)
elapsed_time = time.time() - start_time
total = len(failures) + len(successes)
if not failures:
logging.info("Success: %s (all) tests passed", len(successes))
logging.info("Elapsed time: %s", elapsed_time)
sys.exit(0)
else:
logging.error("Fail: %s/%s test failed:", len(failures), total)
for test in failures:
logging.error(" - %s", test)
sys.exit(1)
def main():
""" main """
log.configure(level=logging.DEBUG)
# Read the configuration file from package
conf_file = DEFAULT_TEST_CONF_FILE
conf_string = pkgutil.get_data(__name__, conf_file)
decoder = json.JSONDecoder(strict=False)
# Convert the conf file to a json format
conf = decoder.decode(conf_string)
args = dict()
home_directory = os.path.expanduser("~")
args['cluster'] = conf['cluster']
args['topologyName'] = conf['topology']['topologyName']
args['topologyClassPath'] = conf['topology']['topologyClassPath']
args['workingDirectory'] = os.path.join(
home_directory,
".herondata",
"topologies",
conf['cluster'],
getpass.getuser(),
args['topologyName']
)
args['cliPath'] = os.path.expanduser(conf['heronCliPath'])
args['trackerPath'] = os.path.expanduser(conf['heronTrackerPath'])
args['trackerPort'] = _random_port()
args['outputFile'] = os.path.join(args['workingDirectory'], conf['topology']['outputFile'])
args['readFile'] = os.path.join(args['workingDirectory'], conf['topology']['readFile'])
args['testJarPath'] = conf['testJarPath']
test_classes = TEST_CLASSES
if len(sys.argv) > 1:
first_arg = sys.argv[1]
class_tokens = first_arg.split(".")
if first_arg == "-h" or len(class_tokens) < 2:
usage()
import importlib
package_tokens = class_tokens[:-1]
test_class = class_tokens[-1]
if len(package_tokens) == 1: # prepend base packages for convenience
test_module = "integration_test.src.python.local_test_runner." + package_tokens[0]
else:
test_module = '.'.join(package_tokens)
logging.info("test_module %s", test_module)
logging.info("test_class %s", test_class)
test_classes = [getattr(importlib.import_module(test_module), test_class)]
start_time = time.time()
(successes, failures) = run_tests(test_classes, args)
elapsed_time = time.time() - start_time
total = len(failures) + len(successes)
if not failures:
logging.info("Success: %s (all) tests passed", len(successes))
logging.info("Elapsed time: %s", elapsed_time)
sys.exit(0)
else:
logging.error("Fail: %s/%s test failed:", len(failures), total)
for test in failures:
logging.error(" - %s", test)
sys.exit(1)