Beispiel #1
0
def cli(verbose: int):
  levels = {
      0: logging.WARNING,
      1: logging.INFO,
      2: logging.DEBUG,
  }
  log.configure(levels.get(verbose, logging.DEBUG))
Beispiel #2
0
def main():
  '''
  Runs a tornado http server that listens for any
  integration test json result get/post requests
  '''

  log.configure(level=logging.DEBUG)

  if not os.path.exists(RESULTS_DIRECTORY):
    os.makedirs(RESULTS_DIRECTORY)

  state_map = {}
  # for instance states in stateful processing
  state_result_map = {}
  application = tornado.web.Application([
    (r"/", MainHandler),
    (r"^/results/([a-zA-Z0-9_-]+$)", FileHandler),
    (r"^/state", MemoryMapGetAllHandler, dict(state_map=state_map)),
    (r"^/state/([a-zA-Z0-9_-]+$)", MemoryMapHandler, dict(state_map=state_map)),
    (r"^/stateResults/([a-zA-Z0-9_-]+$)", StateResultHandler, dict(result_map=state_result_map)),
  ])

  if len(sys.argv) == 1:
    logging.error("Missing argument: port addresss")
    sys.exit(1)

  try:
    port = int(sys.argv[1])
  except Exception as e:
    logging.error("Exception: %s \nprovide a valid port address", e)
    sys.exit(1)

  logging.info("Starting server at port " + str(port))
  application.listen(port)
  tornado.ioloop.IOLoop.current().start()
Beispiel #3
0
def main():
  '''
  Runs a tornado http server that listens for any
  integration test json result get/post requests
  '''

  log.configure(level=logging.DEBUG)

  if not os.path.exists(RESULTS_DIRECTORY):
    os.makedirs(RESULTS_DIRECTORY)

  state_map = {}
  # for instance states in stateful processing
  state_result_map = {}
  application = tornado.web.Application([
      (r"/", MainHandler),
      (r"^/results/([a-zA-Z0-9_-]+$)", FileHandler),
      (r"^/state", MemoryMapGetAllHandler, dict(state_map=state_map)),
      (r"^/state/([a-zA-Z0-9_-]+$)", MemoryMapHandler, dict(state_map=state_map)),
      (r"^/stateResults/([a-zA-Z0-9_-]+$)", StateResultHandler, dict(result_map=state_result_map)),
  ])

  if len(sys.argv) == 1:
    logging.error("Missing argument: port addresss")
    sys.exit(1)

  try:
    port = int(sys.argv[1])
  except Exception as e:
    logging.error("Exception: %s \nprovide a valid port address", e)
    sys.exit(1)

  logging.info("Starting server at port " + str(port))
  application.listen(port)
  tornado.ioloop.IOLoop.instance().start()
Beispiel #4
0
def cli(
    config_file: str,
    stmgr_type: str,
    name: str,
    rootpath: str,
    tunnelhost: str,
    hostport: str,
    port: int,
    verbose: bool,
) -> None:
  """
  A HTTP service for serving data about clusters.

  The statemanager's config from the given config file can be overrided using
  options on this executable.

  """

  log.configure(logging.DEBUG if verbose else logging.INFO)

  # set Tornado global option
  define_options(port, config_file)

  stmgr_override = {
      "type": stmgr_type,
      "name": name,
      "rootpath": rootpath,
      "tunnelhost": tunnelhost,
      "hostport": hostport,
  }
  config = Config(create_tracker_config(config_file, stmgr_override))

  # create Tornado application
  application = Application(config)

  # pylint: disable=unused-argument
  # SIGINT handler:
  # 1. stop all the running zkstatemanager and filestatemanagers
  # 2. stop the Tornado IO loop
  def signal_handler(signum, frame):
    # start a new line after ^C character because this looks nice
    print('\n', end='')
    application.stop()
    tornado.ioloop.IOLoop.instance().stop()

  # associate SIGINT and SIGTERM with a handler
  signal.signal(signal.SIGINT, signal_handler)
  signal.signal(signal.SIGTERM, signal_handler)

  Log.info("Running on port: %d", port)
  if config_file:
    Log.info("Using config file: %s", config_file)
  Log.info(f"Using state manager:\n{config}")

  http_server = tornado.httpserver.HTTPServer(application)
  http_server.listen(port)

  tornado.ioloop.IOLoop.instance().start()
Beispiel #5
0
def cli(host: str, port: int, base_url_option: str, tracker_url_option: str,
        verbose: bool) -> None:
    """Start a web UI for heron which renders information from the tracker."""
    global base_url, tracker_url
    base_url = base_url_option
    log.configure(level=logging.DEBUG if verbose else logging.INFO)
    tracker.tracker_url = tracker_url_option

    uvicorn.run(app, host=host, port=port, log_config=None)
Beispiel #6
0
def main():
  ''' main '''
  log.configure(level=logging.DEBUG)
  conf_file = DEFAULT_TEST_CONF_FILE
  # Read the configuration file from package
  conf_string = pkgutil.get_data(__name__, conf_file)
  decoder = json.JSONDecoder(strict=False)
  # Convert the conf file to a json format
  conf = decoder.decode(conf_string)

  # Parse the arguments passed via command line
  parser = argparse.ArgumentParser(description='This is the heron integration test framework')

  parser.add_argument('-hc', '--heron-cli-path', dest='heron_cli_path',
                      default=conf['heronCliPath'])
  parser.add_argument('-tb', '--tests-bin-path', dest='tests_bin_path')
  parser.add_argument('-cl', '--cluster', dest='cluster', default=conf['cluster'])
  parser.add_argument('-ev', '--env', dest='env', default=conf['env'])
  parser.add_argument('-rl', '--role', dest='role', default=conf['role'])
  parser.add_argument('-rh', '--http-server-hostname', dest='http_server_hostname')
  parser.add_argument('-rp', '--http-server-port', dest='http_server_port', type=int,
                      default=conf['resultsServerPort'])
  parser.add_argument('-tp', '--topologies-path', dest='topologies_path')
  parser.add_argument('-ts', '--test-topology-pattern', dest='test_topology_pattern', default=None)
  parser.add_argument('-pi', '--release-package-uri', dest='release_package_uri', default=None)
  parser.add_argument('-cd', '--cli-config-path', dest='cli_config_path',
                      default=conf['cliConfigPath'])
  parser.add_argument('-ms', '--max-thread-number', dest='max_thread_number', default=1)

  #parser.add_argument('-dt', '--disable-topologies', dest='disabledTopologies', default='',
  #                    help='comma separated test case(classpath) name that will not be run')
  #parser.add_argument('-et', '--enable-topologies', dest='enableTopologies', default=None,
  #                    help='comma separated test case(classpath) name that will be run only')

  args, unknown_args = parser.parse_known_args()
  if unknown_args:
    logging.error('Unknown argument passed to %s: %s', sys.argv[0], unknown_args[0])
    sys.exit(1)

  tests_start_time = int(time.time())
  run_tests(conf, args)
  total = len(failures) + len(successes)
  logging.info("Total integration test time = %ss" % (int(time.time()) - tests_start_time))

  if not failures:
    logging.info("SUCCESS: %s (all) tests passed:", len(successes))
    for test in successes:
      logging.info("  - %s: %s", ("[%ss]" % test[1]).ljust(8), test[0])
    sys.exit(0)
  else:
    logging.error("FAILURE: %s/%s tests failed:", len(failures), total)
    for test in failures:
      logging.error("  - %s: %s", ("[%ss]" % test[1]).ljust(8), test[0])
    sys.exit(1)
Beispiel #7
0
  def setup(shardid):
    # Redirect stdout and stderr to files in append mode
    # The filename format is heron-executor-<container_id>.stdxxx
    log.configure(logfile='heron-executor-%s.stdout' % shardid)

    Log.info('Set up process group; executor becomes leader')
    os.setpgrp() # create new process group, become its leader

    Log.info('Register the SIGTERM signal handler')
    signal.signal(signal.SIGTERM, signal_handler)

    Log.info('Register the atexit clean up')
    atexit.register(cleanup)
Beispiel #8
0
  def setup(shardid):
    # Redirect stdout and stderr to files in append mode
    # The filename format is heron-executor-<container_id>.stdxxx
    log.configure(logfile='heron-executor-%s.stdout' % shardid)

    Log.info('Set up process group; executor becomes leader')
    os.setpgrp() # create new process group, become its leader

    Log.info('Register the SIGTERM signal handler')
    signal.signal(signal.SIGTERM, signal_handler)

    Log.info('Register the atexit clean up')
    atexit.register(cleanup)
Beispiel #9
0
def main():
  ''' main '''
  log.configure(level=logging.DEBUG)
  conf_file = DEFAULT_TEST_CONF_FILE
  # Read the configuration file from package
  conf_string = pkgutil.get_data(__name__, conf_file)
  decoder = json.JSONDecoder(strict=False)
  # Convert the conf file to a json format
  conf = decoder.decode(conf_string)

  # Parse the arguments passed via command line
  parser = argparse.ArgumentParser(description='This is the heron integration test framework')

  parser.add_argument('-hc', '--heron-cli-path', dest='heron_cli_path',
                      default=conf['heronCliPath'])
  parser.add_argument('-tb', '--tests-bin-path', dest='tests_bin_path')
  parser.add_argument('-cl', '--cluster', dest='cluster', default=conf['cluster'])
  parser.add_argument('-ev', '--env', dest='env', default=conf['env'])
  parser.add_argument('-rl', '--role', dest='role', default=conf['role'])
  parser.add_argument('-rh', '--http-server-hostname', dest='http_server_hostname')
  parser.add_argument('-rp', '--http-server-port', dest='http_server_port', type=int,
                      default=conf['resultsServerPort'])
  parser.add_argument('-tp', '--topologies-path', dest='topologies_path')
  parser.add_argument('-ts', '--test-topology-pattern', dest='test_topology_pattern', default=None)
  parser.add_argument('-pi', '--release-package-uri', dest='release_package_uri', default=None)
  parser.add_argument('-cd', '--cli-config-path', dest='cli_config_path',
                      default=conf['cliConfigPath'])

  #parser.add_argument('-dt', '--disable-topologies', dest='disabledTopologies', default='',
  #                    help='comma separated test case(classpath) name that will not be run')
  #parser.add_argument('-et', '--enable-topologies', dest='enableTopologies', default=None,
  #                    help='comma separated test case(classpath) name that will be run only')

  args, unknown_args = parser.parse_known_args()
  if unknown_args:
    logging.error('Unknown argument passed to %s: %s', sys.argv[0], unknown_args[0])
    sys.exit(1)

  (successes, failures) = run_tests(conf, args)
  total = len(failures) + len(successes)

  if not failures:
    logging.info("SUCCESS: %s (all) tests passed:", len(successes))
    for test in successes:
      logging.info("  - %s: %s", ("[%ss]" % test[1]).ljust(8), test[0])
    sys.exit(0)
  else:
    logging.error("FAILURE: %s/%s tests failed:", len(failures), total)
    for test in failures:
      logging.error("  - %s: %s", ("[%ss]" % test[1]).ljust(8), test[0])
    sys.exit(1)
Beispiel #10
0
def main():
  """
  main
  """
  log.configure(level=logging.DEBUG)
  conf_file = DEFAULT_TEST_CONF_FILE
  # Read the configuration file from package
  conf_string = pkgutil.get_data(__name__, conf_file).decode()
  decoder = json.JSONDecoder(strict=False)
  # Convert the conf file to a json format
  conf = decoder.decode(conf_string)

  # Parse the arguments passed via command line
  parser = argparse.ArgumentParser(description='This is the heron integration test framework')

  parser.add_argument('-hc', '--heron-cli-path', dest='heron_cli_path',
    default=conf['heronCliPath'])
  parser.add_argument('-tb', '--tests-bin-path', dest='tests_bin_path')
  parser.add_argument('-cl', '--cluster', dest='cluster', default=conf['cluster'])
  parser.add_argument('-ev', '--env', dest='env', default=conf['env'])
  parser.add_argument('-rl', '--role', dest='role', default=conf['role'])
  parser.add_argument('-rh', '--http-hostname', dest='http_hostname', default='localhost')
  parser.add_argument('-rp', '--http-port', dest='http_port', type=int,
    default='8080')
  parser.add_argument('-tp', '--topologies-path', dest='topologies_path')
  parser.add_argument('-ts', '--test-topology-pattern', dest='test_topology_pattern', default=None)
  parser.add_argument('-pi', '--release-package-uri', dest='release_package_uri', default=None)
  parser.add_argument('-cd', '--cli-config-path', dest='cli_config_path',
    default=conf['cliConfigPath'])
  parser.add_argument('-ms', '--max-thread-number', dest='max_thread_number', default=1)

  args, unknown_args = parser.parse_known_args()
  if unknown_args:
    logging.error('Unknown argument passed to %s: %s', sys.argv[0], unknown_args[0])
    sys.exit(1)

  tests_start_time = int(time.time())
  run_topology_tests(conf, args)
  total = len(failures) + len(successes)
  logging.info("Total integration topology test time = %ss" % (int(time.time()) - tests_start_time))

  if not failures:
    logging.info("SUCCESS: %s (all) tests passed:", len(successes))
    for test in successes:
      logging.info("  - %s: %s", ("[%ss]" % test[1]).ljust(8), test[0])
    sys.exit(0)
  else:
    logging.error("FAILURE: %s/%s tests failed:", len(failures), total)
    for test in failures:
      logging.error("  - %s: %s", ("[%ss]" % test[1]).ljust(8), test[0])
    sys.exit(1)
Beispiel #11
0
def main():
  """
  main
  """
  log.configure(level=logging.DEBUG)
  conf_file = DEFAULT_TEST_CONF_FILE
  # Read the configuration file from package
  conf_string = pkgutil.get_data(__name__, conf_file)
  decoder = json.JSONDecoder(strict=False)
  # Convert the conf file to a json format
  conf = decoder.decode(conf_string)

  # Parse the arguments passed via command line
  parser = argparse.ArgumentParser(description='This is the heron integration test framework')

  parser.add_argument('-hc', '--heron-cli-path', dest='heron_cli_path',
    default=conf['heronCliPath'])
  parser.add_argument('-tb', '--tests-bin-path', dest='tests_bin_path')
  parser.add_argument('-cl', '--cluster', dest='cluster', default=conf['cluster'])
  parser.add_argument('-ev', '--env', dest='env', default=conf['env'])
  parser.add_argument('-rl', '--role', dest='role', default=conf['role'])
  parser.add_argument('-rh', '--http-hostname', dest='http_hostname', default='localhost')
  parser.add_argument('-rp', '--http-port', dest='http_port', type=int,
    default='8080')
  parser.add_argument('-tp', '--topologies-path', dest='topologies_path')
  parser.add_argument('-ts', '--test-topology-pattern', dest='test_topology_pattern', default=None)
  parser.add_argument('-pi', '--release-package-uri', dest='release_package_uri', default=None)
  parser.add_argument('-cd', '--cli-config-path', dest='cli_config_path',
    default=conf['cliConfigPath'])

  args, unknown_args = parser.parse_known_args()
  if unknown_args:
    logging.error('Unknown argument passed to %s: %s', sys.argv[0], unknown_args[0])
    sys.exit(1)

  (successes, failures) = run_topology_tests(conf, args)
  total = len(failures) + len(successes)

  if not failures:
    logging.info("SUCCESS: %s (all) tests passed:", len(successes))
    for test in successes:
      logging.info("  - %s: %s", ("[%ss]" % test[1]).ljust(8), test[0])
    sys.exit(0)
  else:
    logging.error("FAILURE: %s/%s tests failed:", len(failures), total)
    for test in failures:
      logging.error("  - %s: %s", ("[%ss]" % test[1]).ljust(8), test[0])
    sys.exit(1)
Beispiel #12
0
def setup(executor):
  """Set up log, process and signal handlers"""
  # pylint: disable=unused-argument
  def signal_handler(signal_to_handle, frame):
    # We would do nothing here but just exit
    # Just catch the SIGTERM and then cleanup(), registered with atexit, would invoke
    Log.info('signal_handler invoked with signal %s', signal_to_handle)
    executor.stop_state_manager_watches()
    sys.exit(signal_to_handle)

  def cleanup():
    """Handler to trigger when receiving the SIGTERM signal
    Do cleanup inside this method, including:
    1. Terminate all children processes
    """
    Log.info('Executor terminated; exiting all process in executor.')

    # Kill child processes first and wait for log collection to finish
    for pid in list(executor.processes_to_monitor.keys()):
      os.kill(pid, signal.SIGTERM)
    time.sleep(5)

    # We would not wait or check whether process spawned dead or not
    os.killpg(0, signal.SIGTERM)

  # Redirect stdout and stderr to files in append mode
  # The filename format is heron-executor-<container_id>.stdxxx
  shardid = executor.shard
  log.configure(logfile='heron-executor-%s.stdout' % shardid)

  pid = os.getpid()
  sid = os.getsid(pid)

  # POSIX prohibits the change of the process group ID of a session leader
  if pid != sid:
    Log.info('Set up process group; executor becomes leader')
    os.setpgrp() # create new process group, become its leader

  Log.info('Register the SIGTERM signal handler')
  signal.signal(signal.SIGTERM, signal_handler)

  Log.info('Register the atexit clean up')
  atexit.register(cleanup)
Beispiel #13
0
def cli(
    config_file: str,
    stmgr_type: str,
    name: str,
    rootpath: str,
    tunnelhost: str,
    hostport: str,
    port: int,
    verbose: bool,
) -> None:
    """
  A HTTP service for serving data about clusters.

  The statemanager's config from the given config file can be overrided using
  options on this executable.

  """

    log_level = logging.DEBUG if verbose else logging.INFO
    log.configure(log_level)

    stmgr_override = {
        "type": stmgr_type,
        "name": name,
        "rootpath": rootpath,
        "tunnelhost": tunnelhost,
        "hostport": hostport,
    }
    config = Config(create_tracker_config(config_file, stmgr_override))

    state.tracker = Tracker(config)
    state.tracker.sync_topologies()
    # this only returns when interrupted
    uvicorn.run(app, host="0.0.0.0", port=port, log_level=log_level)
    state.tracker.stop_sync()

    # non-daemon threads linger and stop the process for quitting, so signal
    # for cleaning up
    os.kill(os.getpid(), signal.SIGKILL)
Beispiel #14
0
def main():
    """ main """
    log.configure(level=logging.DEBUG)

    # Read the configuration file from package
    conf_file = DEFAULT_TEST_CONF_FILE
    conf_string = pkgutil.get_data(__name__, conf_file)
    decoder = json.JSONDecoder(strict=False)

    # Convert the conf file to a json format
    conf = decoder.decode(conf_string)

    args = dict()
    home_directory = os.path.expanduser("~")
    args['cluster'] = conf['cluster']
    args['topologyName'] = conf['topology']['topologyName']
    args['topologyClassPath'] = conf['topology']['topologyClassPath']
    args['workingDirectory'] = os.path.join(home_directory, ".herondata",
                                            "topologies", conf['cluster'],
                                            getpass.getuser(),
                                            args['topologyName'])
    args['cliPath'] = os.path.expanduser(conf['heronCliPath'])
    args['trackerPath'] = os.path.expanduser(conf['heronTrackerPath'])
    args['trackerPort'] = _random_port()
    args['outputFile'] = os.path.join(args['workingDirectory'],
                                      conf['topology']['outputFile'])
    args['readFile'] = os.path.join(args['workingDirectory'],
                                    conf['topology']['readFile'])
    args['testJarPath'] = conf['testJarPath']

    test_classes = TEST_CLASSES
    if len(sys.argv) > 1:
        first_arg = sys.argv[1]
        class_tokens = first_arg.split(".")
        if first_arg == "-h" or len(class_tokens) < 2:
            usage()

        import importlib
        package_tokens = class_tokens[:-1]
        test_class = class_tokens[-1]
        if len(package_tokens) == 1:  # prepend base packages for convenience
            test_module = "integration_test.src.python.local_test_runner." + package_tokens[
                0]
        else:
            test_module = '.'.join(package_tokens)

        logging.info("test_module %s", test_module)
        logging.info("test_class %s", test_class)
        test_classes = [
            getattr(importlib.import_module(test_module), test_class)
        ]

    start_time = time.time()
    (successes, failures) = run_tests(test_classes, args)
    elapsed_time = time.time() - start_time
    total = len(failures) + len(successes)

    if not failures:
        logging.info("Success: %s (all) tests passed", len(successes))
        logging.info("Elapsed time: %s", elapsed_time)
        sys.exit(0)
    else:
        logging.error("Fail: %s/%s test failed:", len(failures), total)
        for test in failures:
            logging.error("  - %s", test)
        sys.exit(1)
Beispiel #15
0
def main():
  """ main """
  log.configure(level=logging.DEBUG)

  # Read the configuration file from package
  conf_file = DEFAULT_TEST_CONF_FILE
  conf_string = pkgutil.get_data(__name__, conf_file)
  decoder = json.JSONDecoder(strict=False)

  # Convert the conf file to a json format
  conf = decoder.decode(conf_string)

  args = dict()
  home_directory = os.path.expanduser("~")
  args['cluster'] = conf['cluster']
  args['topologyName'] = conf['topology']['topologyName']
  args['topologyClassPath'] = conf['topology']['topologyClassPath']
  args['workingDirectory'] = os.path.join(
      home_directory,
      ".herondata",
      "topologies",
      conf['cluster'],
      getpass.getuser(),
      args['topologyName']
  )
  args['cliPath'] = os.path.expanduser(conf['heronCliPath'])
  args['trackerPath'] = os.path.expanduser(conf['heronTrackerPath'])
  args['trackerPort'] = _random_port()
  args['outputFile'] = os.path.join(args['workingDirectory'], conf['topology']['outputFile'])
  args['readFile'] = os.path.join(args['workingDirectory'], conf['topology']['readFile'])
  args['testJarPath'] = conf['testJarPath']

  test_classes = TEST_CLASSES
  if len(sys.argv) > 1:
    first_arg = sys.argv[1]
    class_tokens = first_arg.split(".")
    if first_arg == "-h" or len(class_tokens) < 2:
      usage()

    import importlib
    package_tokens = class_tokens[:-1]
    test_class = class_tokens[-1]
    if len(package_tokens) == 1: # prepend base packages for convenience
      test_module = "integration_test.src.python.local_test_runner." + package_tokens[0]
    else:
      test_module = '.'.join(package_tokens)

    logging.info("test_module %s", test_module)
    logging.info("test_class %s", test_class)
    test_classes = [getattr(importlib.import_module(test_module), test_class)]

  start_time = time.time()
  (successes, failures) = run_tests(test_classes, args)
  elapsed_time = time.time() - start_time
  total = len(failures) + len(successes)

  if not failures:
    logging.info("Success: %s (all) tests passed", len(successes))
    logging.info("Elapsed time: %s", elapsed_time)
    sys.exit(0)
  else:
    logging.error("Fail: %s/%s test failed:", len(failures), total)
    for test in failures:
      logging.error("  - %s", test)
    sys.exit(1)