def cli(verbose: int): levels = { 0: logging.WARNING, 1: logging.INFO, 2: logging.DEBUG, } log.configure(levels.get(verbose, logging.DEBUG))
def main(): ''' Runs a tornado http server that listens for any integration test json result get/post requests ''' log.configure(level=logging.DEBUG) if not os.path.exists(RESULTS_DIRECTORY): os.makedirs(RESULTS_DIRECTORY) state_map = {} # for instance states in stateful processing state_result_map = {} application = tornado.web.Application([ (r"/", MainHandler), (r"^/results/([a-zA-Z0-9_-]+$)", FileHandler), (r"^/state", MemoryMapGetAllHandler, dict(state_map=state_map)), (r"^/state/([a-zA-Z0-9_-]+$)", MemoryMapHandler, dict(state_map=state_map)), (r"^/stateResults/([a-zA-Z0-9_-]+$)", StateResultHandler, dict(result_map=state_result_map)), ]) if len(sys.argv) == 1: logging.error("Missing argument: port addresss") sys.exit(1) try: port = int(sys.argv[1]) except Exception as e: logging.error("Exception: %s \nprovide a valid port address", e) sys.exit(1) logging.info("Starting server at port " + str(port)) application.listen(port) tornado.ioloop.IOLoop.current().start()
def main(): ''' Runs a tornado http server that listens for any integration test json result get/post requests ''' log.configure(level=logging.DEBUG) if not os.path.exists(RESULTS_DIRECTORY): os.makedirs(RESULTS_DIRECTORY) state_map = {} # for instance states in stateful processing state_result_map = {} application = tornado.web.Application([ (r"/", MainHandler), (r"^/results/([a-zA-Z0-9_-]+$)", FileHandler), (r"^/state", MemoryMapGetAllHandler, dict(state_map=state_map)), (r"^/state/([a-zA-Z0-9_-]+$)", MemoryMapHandler, dict(state_map=state_map)), (r"^/stateResults/([a-zA-Z0-9_-]+$)", StateResultHandler, dict(result_map=state_result_map)), ]) if len(sys.argv) == 1: logging.error("Missing argument: port addresss") sys.exit(1) try: port = int(sys.argv[1]) except Exception as e: logging.error("Exception: %s \nprovide a valid port address", e) sys.exit(1) logging.info("Starting server at port " + str(port)) application.listen(port) tornado.ioloop.IOLoop.instance().start()
def cli( config_file: str, stmgr_type: str, name: str, rootpath: str, tunnelhost: str, hostport: str, port: int, verbose: bool, ) -> None: """ A HTTP service for serving data about clusters. The statemanager's config from the given config file can be overrided using options on this executable. """ log.configure(logging.DEBUG if verbose else logging.INFO) # set Tornado global option define_options(port, config_file) stmgr_override = { "type": stmgr_type, "name": name, "rootpath": rootpath, "tunnelhost": tunnelhost, "hostport": hostport, } config = Config(create_tracker_config(config_file, stmgr_override)) # create Tornado application application = Application(config) # pylint: disable=unused-argument # SIGINT handler: # 1. stop all the running zkstatemanager and filestatemanagers # 2. stop the Tornado IO loop def signal_handler(signum, frame): # start a new line after ^C character because this looks nice print('\n', end='') application.stop() tornado.ioloop.IOLoop.instance().stop() # associate SIGINT and SIGTERM with a handler signal.signal(signal.SIGINT, signal_handler) signal.signal(signal.SIGTERM, signal_handler) Log.info("Running on port: %d", port) if config_file: Log.info("Using config file: %s", config_file) Log.info(f"Using state manager:\n{config}") http_server = tornado.httpserver.HTTPServer(application) http_server.listen(port) tornado.ioloop.IOLoop.instance().start()
def cli(host: str, port: int, base_url_option: str, tracker_url_option: str, verbose: bool) -> None: """Start a web UI for heron which renders information from the tracker.""" global base_url, tracker_url base_url = base_url_option log.configure(level=logging.DEBUG if verbose else logging.INFO) tracker.tracker_url = tracker_url_option uvicorn.run(app, host=host, port=port, log_config=None)
def main(): ''' main ''' log.configure(level=logging.DEBUG) conf_file = DEFAULT_TEST_CONF_FILE # Read the configuration file from package conf_string = pkgutil.get_data(__name__, conf_file) decoder = json.JSONDecoder(strict=False) # Convert the conf file to a json format conf = decoder.decode(conf_string) # Parse the arguments passed via command line parser = argparse.ArgumentParser(description='This is the heron integration test framework') parser.add_argument('-hc', '--heron-cli-path', dest='heron_cli_path', default=conf['heronCliPath']) parser.add_argument('-tb', '--tests-bin-path', dest='tests_bin_path') parser.add_argument('-cl', '--cluster', dest='cluster', default=conf['cluster']) parser.add_argument('-ev', '--env', dest='env', default=conf['env']) parser.add_argument('-rl', '--role', dest='role', default=conf['role']) parser.add_argument('-rh', '--http-server-hostname', dest='http_server_hostname') parser.add_argument('-rp', '--http-server-port', dest='http_server_port', type=int, default=conf['resultsServerPort']) parser.add_argument('-tp', '--topologies-path', dest='topologies_path') parser.add_argument('-ts', '--test-topology-pattern', dest='test_topology_pattern', default=None) parser.add_argument('-pi', '--release-package-uri', dest='release_package_uri', default=None) parser.add_argument('-cd', '--cli-config-path', dest='cli_config_path', default=conf['cliConfigPath']) parser.add_argument('-ms', '--max-thread-number', dest='max_thread_number', default=1) #parser.add_argument('-dt', '--disable-topologies', dest='disabledTopologies', default='', # help='comma separated test case(classpath) name that will not be run') #parser.add_argument('-et', '--enable-topologies', dest='enableTopologies', default=None, # help='comma separated test case(classpath) name that will be run only') args, unknown_args = parser.parse_known_args() if unknown_args: logging.error('Unknown argument passed to %s: %s', sys.argv[0], unknown_args[0]) sys.exit(1) tests_start_time = int(time.time()) run_tests(conf, args) total = len(failures) + len(successes) logging.info("Total integration test time = %ss" % (int(time.time()) - tests_start_time)) if not failures: logging.info("SUCCESS: %s (all) tests passed:", len(successes)) for test in successes: logging.info(" - %s: %s", ("[%ss]" % test[1]).ljust(8), test[0]) sys.exit(0) else: logging.error("FAILURE: %s/%s tests failed:", len(failures), total) for test in failures: logging.error(" - %s: %s", ("[%ss]" % test[1]).ljust(8), test[0]) sys.exit(1)
def setup(shardid): # Redirect stdout and stderr to files in append mode # The filename format is heron-executor-<container_id>.stdxxx log.configure(logfile='heron-executor-%s.stdout' % shardid) Log.info('Set up process group; executor becomes leader') os.setpgrp() # create new process group, become its leader Log.info('Register the SIGTERM signal handler') signal.signal(signal.SIGTERM, signal_handler) Log.info('Register the atexit clean up') atexit.register(cleanup)
def setup(shardid): # Redirect stdout and stderr to files in append mode # The filename format is heron-executor-<container_id>.stdxxx log.configure(logfile='heron-executor-%s.stdout' % shardid) Log.info('Set up process group; executor becomes leader') os.setpgrp() # create new process group, become its leader Log.info('Register the SIGTERM signal handler') signal.signal(signal.SIGTERM, signal_handler) Log.info('Register the atexit clean up') atexit.register(cleanup)
def main(): ''' main ''' log.configure(level=logging.DEBUG) conf_file = DEFAULT_TEST_CONF_FILE # Read the configuration file from package conf_string = pkgutil.get_data(__name__, conf_file) decoder = json.JSONDecoder(strict=False) # Convert the conf file to a json format conf = decoder.decode(conf_string) # Parse the arguments passed via command line parser = argparse.ArgumentParser(description='This is the heron integration test framework') parser.add_argument('-hc', '--heron-cli-path', dest='heron_cli_path', default=conf['heronCliPath']) parser.add_argument('-tb', '--tests-bin-path', dest='tests_bin_path') parser.add_argument('-cl', '--cluster', dest='cluster', default=conf['cluster']) parser.add_argument('-ev', '--env', dest='env', default=conf['env']) parser.add_argument('-rl', '--role', dest='role', default=conf['role']) parser.add_argument('-rh', '--http-server-hostname', dest='http_server_hostname') parser.add_argument('-rp', '--http-server-port', dest='http_server_port', type=int, default=conf['resultsServerPort']) parser.add_argument('-tp', '--topologies-path', dest='topologies_path') parser.add_argument('-ts', '--test-topology-pattern', dest='test_topology_pattern', default=None) parser.add_argument('-pi', '--release-package-uri', dest='release_package_uri', default=None) parser.add_argument('-cd', '--cli-config-path', dest='cli_config_path', default=conf['cliConfigPath']) #parser.add_argument('-dt', '--disable-topologies', dest='disabledTopologies', default='', # help='comma separated test case(classpath) name that will not be run') #parser.add_argument('-et', '--enable-topologies', dest='enableTopologies', default=None, # help='comma separated test case(classpath) name that will be run only') args, unknown_args = parser.parse_known_args() if unknown_args: logging.error('Unknown argument passed to %s: %s', sys.argv[0], unknown_args[0]) sys.exit(1) (successes, failures) = run_tests(conf, args) total = len(failures) + len(successes) if not failures: logging.info("SUCCESS: %s (all) tests passed:", len(successes)) for test in successes: logging.info(" - %s: %s", ("[%ss]" % test[1]).ljust(8), test[0]) sys.exit(0) else: logging.error("FAILURE: %s/%s tests failed:", len(failures), total) for test in failures: logging.error(" - %s: %s", ("[%ss]" % test[1]).ljust(8), test[0]) sys.exit(1)
def main(): """ main """ log.configure(level=logging.DEBUG) conf_file = DEFAULT_TEST_CONF_FILE # Read the configuration file from package conf_string = pkgutil.get_data(__name__, conf_file).decode() decoder = json.JSONDecoder(strict=False) # Convert the conf file to a json format conf = decoder.decode(conf_string) # Parse the arguments passed via command line parser = argparse.ArgumentParser(description='This is the heron integration test framework') parser.add_argument('-hc', '--heron-cli-path', dest='heron_cli_path', default=conf['heronCliPath']) parser.add_argument('-tb', '--tests-bin-path', dest='tests_bin_path') parser.add_argument('-cl', '--cluster', dest='cluster', default=conf['cluster']) parser.add_argument('-ev', '--env', dest='env', default=conf['env']) parser.add_argument('-rl', '--role', dest='role', default=conf['role']) parser.add_argument('-rh', '--http-hostname', dest='http_hostname', default='localhost') parser.add_argument('-rp', '--http-port', dest='http_port', type=int, default='8080') parser.add_argument('-tp', '--topologies-path', dest='topologies_path') parser.add_argument('-ts', '--test-topology-pattern', dest='test_topology_pattern', default=None) parser.add_argument('-pi', '--release-package-uri', dest='release_package_uri', default=None) parser.add_argument('-cd', '--cli-config-path', dest='cli_config_path', default=conf['cliConfigPath']) parser.add_argument('-ms', '--max-thread-number', dest='max_thread_number', default=1) args, unknown_args = parser.parse_known_args() if unknown_args: logging.error('Unknown argument passed to %s: %s', sys.argv[0], unknown_args[0]) sys.exit(1) tests_start_time = int(time.time()) run_topology_tests(conf, args) total = len(failures) + len(successes) logging.info("Total integration topology test time = %ss" % (int(time.time()) - tests_start_time)) if not failures: logging.info("SUCCESS: %s (all) tests passed:", len(successes)) for test in successes: logging.info(" - %s: %s", ("[%ss]" % test[1]).ljust(8), test[0]) sys.exit(0) else: logging.error("FAILURE: %s/%s tests failed:", len(failures), total) for test in failures: logging.error(" - %s: %s", ("[%ss]" % test[1]).ljust(8), test[0]) sys.exit(1)
def main(): """ main """ log.configure(level=logging.DEBUG) conf_file = DEFAULT_TEST_CONF_FILE # Read the configuration file from package conf_string = pkgutil.get_data(__name__, conf_file) decoder = json.JSONDecoder(strict=False) # Convert the conf file to a json format conf = decoder.decode(conf_string) # Parse the arguments passed via command line parser = argparse.ArgumentParser(description='This is the heron integration test framework') parser.add_argument('-hc', '--heron-cli-path', dest='heron_cli_path', default=conf['heronCliPath']) parser.add_argument('-tb', '--tests-bin-path', dest='tests_bin_path') parser.add_argument('-cl', '--cluster', dest='cluster', default=conf['cluster']) parser.add_argument('-ev', '--env', dest='env', default=conf['env']) parser.add_argument('-rl', '--role', dest='role', default=conf['role']) parser.add_argument('-rh', '--http-hostname', dest='http_hostname', default='localhost') parser.add_argument('-rp', '--http-port', dest='http_port', type=int, default='8080') parser.add_argument('-tp', '--topologies-path', dest='topologies_path') parser.add_argument('-ts', '--test-topology-pattern', dest='test_topology_pattern', default=None) parser.add_argument('-pi', '--release-package-uri', dest='release_package_uri', default=None) parser.add_argument('-cd', '--cli-config-path', dest='cli_config_path', default=conf['cliConfigPath']) args, unknown_args = parser.parse_known_args() if unknown_args: logging.error('Unknown argument passed to %s: %s', sys.argv[0], unknown_args[0]) sys.exit(1) (successes, failures) = run_topology_tests(conf, args) total = len(failures) + len(successes) if not failures: logging.info("SUCCESS: %s (all) tests passed:", len(successes)) for test in successes: logging.info(" - %s: %s", ("[%ss]" % test[1]).ljust(8), test[0]) sys.exit(0) else: logging.error("FAILURE: %s/%s tests failed:", len(failures), total) for test in failures: logging.error(" - %s: %s", ("[%ss]" % test[1]).ljust(8), test[0]) sys.exit(1)
def setup(executor): """Set up log, process and signal handlers""" # pylint: disable=unused-argument def signal_handler(signal_to_handle, frame): # We would do nothing here but just exit # Just catch the SIGTERM and then cleanup(), registered with atexit, would invoke Log.info('signal_handler invoked with signal %s', signal_to_handle) executor.stop_state_manager_watches() sys.exit(signal_to_handle) def cleanup(): """Handler to trigger when receiving the SIGTERM signal Do cleanup inside this method, including: 1. Terminate all children processes """ Log.info('Executor terminated; exiting all process in executor.') # Kill child processes first and wait for log collection to finish for pid in list(executor.processes_to_monitor.keys()): os.kill(pid, signal.SIGTERM) time.sleep(5) # We would not wait or check whether process spawned dead or not os.killpg(0, signal.SIGTERM) # Redirect stdout and stderr to files in append mode # The filename format is heron-executor-<container_id>.stdxxx shardid = executor.shard log.configure(logfile='heron-executor-%s.stdout' % shardid) pid = os.getpid() sid = os.getsid(pid) # POSIX prohibits the change of the process group ID of a session leader if pid != sid: Log.info('Set up process group; executor becomes leader') os.setpgrp() # create new process group, become its leader Log.info('Register the SIGTERM signal handler') signal.signal(signal.SIGTERM, signal_handler) Log.info('Register the atexit clean up') atexit.register(cleanup)
def cli( config_file: str, stmgr_type: str, name: str, rootpath: str, tunnelhost: str, hostport: str, port: int, verbose: bool, ) -> None: """ A HTTP service for serving data about clusters. The statemanager's config from the given config file can be overrided using options on this executable. """ log_level = logging.DEBUG if verbose else logging.INFO log.configure(log_level) stmgr_override = { "type": stmgr_type, "name": name, "rootpath": rootpath, "tunnelhost": tunnelhost, "hostport": hostport, } config = Config(create_tracker_config(config_file, stmgr_override)) state.tracker = Tracker(config) state.tracker.sync_topologies() # this only returns when interrupted uvicorn.run(app, host="0.0.0.0", port=port, log_level=log_level) state.tracker.stop_sync() # non-daemon threads linger and stop the process for quitting, so signal # for cleaning up os.kill(os.getpid(), signal.SIGKILL)
def main(): """ main """ log.configure(level=logging.DEBUG) # Read the configuration file from package conf_file = DEFAULT_TEST_CONF_FILE conf_string = pkgutil.get_data(__name__, conf_file) decoder = json.JSONDecoder(strict=False) # Convert the conf file to a json format conf = decoder.decode(conf_string) args = dict() home_directory = os.path.expanduser("~") args['cluster'] = conf['cluster'] args['topologyName'] = conf['topology']['topologyName'] args['topologyClassPath'] = conf['topology']['topologyClassPath'] args['workingDirectory'] = os.path.join(home_directory, ".herondata", "topologies", conf['cluster'], getpass.getuser(), args['topologyName']) args['cliPath'] = os.path.expanduser(conf['heronCliPath']) args['trackerPath'] = os.path.expanduser(conf['heronTrackerPath']) args['trackerPort'] = _random_port() args['outputFile'] = os.path.join(args['workingDirectory'], conf['topology']['outputFile']) args['readFile'] = os.path.join(args['workingDirectory'], conf['topology']['readFile']) args['testJarPath'] = conf['testJarPath'] test_classes = TEST_CLASSES if len(sys.argv) > 1: first_arg = sys.argv[1] class_tokens = first_arg.split(".") if first_arg == "-h" or len(class_tokens) < 2: usage() import importlib package_tokens = class_tokens[:-1] test_class = class_tokens[-1] if len(package_tokens) == 1: # prepend base packages for convenience test_module = "integration_test.src.python.local_test_runner." + package_tokens[ 0] else: test_module = '.'.join(package_tokens) logging.info("test_module %s", test_module) logging.info("test_class %s", test_class) test_classes = [ getattr(importlib.import_module(test_module), test_class) ] start_time = time.time() (successes, failures) = run_tests(test_classes, args) elapsed_time = time.time() - start_time total = len(failures) + len(successes) if not failures: logging.info("Success: %s (all) tests passed", len(successes)) logging.info("Elapsed time: %s", elapsed_time) sys.exit(0) else: logging.error("Fail: %s/%s test failed:", len(failures), total) for test in failures: logging.error(" - %s", test) sys.exit(1)
def main(): """ main """ log.configure(level=logging.DEBUG) # Read the configuration file from package conf_file = DEFAULT_TEST_CONF_FILE conf_string = pkgutil.get_data(__name__, conf_file) decoder = json.JSONDecoder(strict=False) # Convert the conf file to a json format conf = decoder.decode(conf_string) args = dict() home_directory = os.path.expanduser("~") args['cluster'] = conf['cluster'] args['topologyName'] = conf['topology']['topologyName'] args['topologyClassPath'] = conf['topology']['topologyClassPath'] args['workingDirectory'] = os.path.join( home_directory, ".herondata", "topologies", conf['cluster'], getpass.getuser(), args['topologyName'] ) args['cliPath'] = os.path.expanduser(conf['heronCliPath']) args['trackerPath'] = os.path.expanduser(conf['heronTrackerPath']) args['trackerPort'] = _random_port() args['outputFile'] = os.path.join(args['workingDirectory'], conf['topology']['outputFile']) args['readFile'] = os.path.join(args['workingDirectory'], conf['topology']['readFile']) args['testJarPath'] = conf['testJarPath'] test_classes = TEST_CLASSES if len(sys.argv) > 1: first_arg = sys.argv[1] class_tokens = first_arg.split(".") if first_arg == "-h" or len(class_tokens) < 2: usage() import importlib package_tokens = class_tokens[:-1] test_class = class_tokens[-1] if len(package_tokens) == 1: # prepend base packages for convenience test_module = "integration_test.src.python.local_test_runner." + package_tokens[0] else: test_module = '.'.join(package_tokens) logging.info("test_module %s", test_module) logging.info("test_class %s", test_class) test_classes = [getattr(importlib.import_module(test_module), test_class)] start_time = time.time() (successes, failures) = run_tests(test_classes, args) elapsed_time = time.time() - start_time total = len(failures) + len(successes) if not failures: logging.info("Success: %s (all) tests passed", len(successes)) logging.info("Elapsed time: %s", elapsed_time) sys.exit(0) else: logging.error("Fail: %s/%s test failed:", len(failures), total) for test in failures: logging.error(" - %s", test) sys.exit(1)