Ejemplo n.º 1
0
def get_modules(testfile, tests_to_run, config_overrides):
  """
  Gets modules and objects required to run tests

  :param testfile:
  :param tests_to_run:
  :param config_overrides:
  :return:
  """
  test_dic = _parse_input(testfile)
  master_config, configs = load_configs_from_directory(test_dic["configs_directory"], config_overrides)
  _setup_paths(master_config.mapping.get("additional_paths", []))
  deployment_module = utils.load_module(test_dic["deployment_code"])
  if  "dynamic_configuration_code" in test_dic:
    perf_module = utils.load_module(test_dic["dynamic_configuration_code"])
  else:
    perf_module = utils.load_module(test_dic["perf_code"])
  test_modules = [utils.load_module(testcode) for testcode in test_dic["test_code"]]
  if tests_to_run is not None:
    tests = [test for test in _determine_tests(test_modules) if test.name in tests_to_run]
  else:
    tests = [test for test in _determine_tests(test_modules)]
  tests_with_phases = {}
  for test in tests:
    tests_with_phases[test.phase] = tests_with_phases.get(test.phase, [])+[test]
  serial_tests = tests_with_phases.pop(constants.DEFAULT_TEST_PHASE, [])
  tests = serial_tests+[test_with_phase[1] for test_with_phase in sorted(tests_with_phases.items(), key=lambda pair: pair[0])]


  return deployment_module, perf_module, tests, master_config, configs
Ejemplo n.º 2
0
    def test_determine_tests_read_correct_number_of_tests_in_file(self):
        """
    Test that we can read the correct number of tests from the test file
    """
        test_directory = os.path.join(self.FILE_LOCATION,
                                      "samples/determine_tests")
        test_files = [
            file_name for file_name in os.listdir(test_directory)
            if os.path.splitext(file_name)[1] == ".py"
        ]
        for testfile in test_files:
            test_module = utils.load_module(
                os.path.join(test_directory, testfile))
            tests = [
                test.name
                for test in test_runner_helper._determine_tests([test_module])
            ]

            if ("empty" in testfile) or ("invalid"
                                         in testfile) or ("no_test"
                                                          in testfile):
                self.assertEqual(len(tests), 0)
            elif ("single" in testfile) or ("no_validate"
                                            in testfile) or ("no_match"
                                                             in testfile):
                self.assertEqual(len(tests), 1)
                for i in xrange(0, 1):
                    self.assertTrue("test{0}".format(i) in tests)
            elif "multi" in testfile:
                self.assertEqual(len(tests), 2)
                for i in xrange(0, 2):
                    self.assertTrue("test{0}".format(i) in tests)
            else:
                print testfile + " was not used as test input"
Ejemplo n.º 3
0
def call_main(args):
    # Get output directory.
    try:
        if args.output_dir is not None:
            runtime.set_output_dir(args.output_dir)
    except ValueError as e:
        print str(e)
        raise

    # Set up logging.
    setup_logging(runtime.get_output_dir(), args.log_level, args.console_level)
    logger = logging.getLogger("zopkio")
    logger.info("Starting zopkio")

    try:
        utils.check_file_with_exception(args.testfile)
        utils.check_testfile_dir_structure(args.testfile)
        machines = utils.make_machine_mapping(args.machine_list)
        config_overrides = utils.parse_config_list(args.config_overrides)
    except ValueError as e:
        logger.error(str(e))
        print("Error in processing command line arguments:\n {0}".format(
            traceback.format_exc()))
        raise

    runtime.set_machines(machines)
    if args.user is not None:
        user = args.user
    else:
        user = getpass.getuser()
    if args.nopassword:
        password = ""
    else:
        password = getpass.getpass()
    runtime.set_user(user, password)

    try:
        testmodule = utils.load_module(args.testfile)
        ztestsuites = [
            getattr(testmodule, attr) for attr in dir(testmodule)
            if isinstance(getattr(testmodule, attr), ZTestSuite)
        ]
        if len(
                ztestsuites
        ) > 0:  #TODO(jehrlich) intelligently handle multiple test suites
            test_runner = TestRunner(ztestsuite=ztestsuites[0],
                                     testlist=args.test_list,
                                     config_overrides=config_overrides)
        else:
            test_runner = TestRunner(args.testfile, args.test_list,
                                     config_overrides)
    except BaseException as e:
        print("Error setting up testrunner:\n%s" % traceback.format_exc())
        raise ValueError(e.message)

    test_runner.run()

    logger.info("Exiting zopkio")
    return test_runner.success_count(), test_runner.fail_count()
Ejemplo n.º 4
0
  def test_determine_tests_read_correct_total_number_of_tests_in_files(self):
    """
    Tests that determine_tests works for a list of modules
    """
    test_directory = os.path.join(self.FILE_LOCATION, "samples/determine_tests")
    test_files = [file_name for file_name in os.listdir(test_directory)
        if os.path.splitext(file_name)[1] == ".py"]
    test_modules = [utils.load_module(os.path.join(test_directory, test_file))
        for test_file in test_files]
    tests = test_runner_helper._determine_tests(test_modules)

    self.assertEqual(sum(1 for test in tests), 5)
Ejemplo n.º 5
0
  def test_determine_tests_read_correct_total_number_of_tests_in_files(self):
    """
    Tests that determine_tests works for a list of modules
    """
    test_directory = os.path.join(self.FILE_LOCATION, "samples/determine_tests")
    test_files = [file_name for file_name in os.listdir(test_directory)
        if os.path.splitext(file_name)[1] == ".py"]
    test_modules = [utils.load_module(os.path.join(test_directory, test_file))
        for test_file in test_files]
    tests = test_runner_helper._determine_tests(test_modules)

    self.assertEqual(sum(1 for test in tests), 5)
Ejemplo n.º 6
0
def call_main(args):
  # Get output directory.
  try:
    if args.output_dir is not None:
      runtime.set_output_dir(args.output_dir)
  except ValueError as e:
    print str(e)
    raise

  # Set up logging.
  setup_logging(runtime.get_output_dir(), args.log_level, args.console_level)
  logger = logging.getLogger("zopkio")
  logger.info("Starting zopkio")

  try:
    utils.check_file_with_exception(args.testfile)
    utils.check_testfile_dir_structure(args.testfile)
    machines = utils.make_machine_mapping(args.machine_list)
    config_overrides = utils.parse_config_list(args.config_overrides)
  except ValueError as e:
    logger.error(str(e))
    print("Error in processing command line arguments:\n {0}".format(traceback.format_exc()))
    raise

  runtime.set_machines(machines)
  if args.user is not None:
    user = args.user
  else:
    user = getpass.getuser()
  if args.nopassword:
    password = ""
  else:
    password = getpass.getpass()
  runtime.set_user(user, password)

  try:
    testmodule = utils.load_module(args.testfile)
    ztestsuites = [getattr(testmodule, attr)
               for attr in dir(testmodule)
               if isinstance(getattr(testmodule, attr), ZTestSuite)]
    if len(ztestsuites) > 0: #TODO(jehrlich) intelligently handle multiple test suites
      test_runner = TestRunner(ztestsuite=ztestsuites[0], testlist=args.test_list, config_overrides=config_overrides)
    else:
      test_runner = TestRunner(args.testfile, args.test_list, config_overrides)
  except BaseException as e:
    print("Error setting up testrunner:\n%s" % traceback.format_exc())
    raise ValueError(e.message)

  test_runner.run()

  logger.info("Exiting zopkio")
  return test_runner.success_count(), test_runner.fail_count()
Ejemplo n.º 7
0
  def test_determine_tests_generates_correct_test_object(self):
    """
    Tests that determine_tests generate correct test objects
    """
    test_directory = os.path.join(self.FILE_LOCATION, "samples/determine_tests")
    test_module = utils.load_module(os.path.join(test_directory,
                                                 "meta_test_single.py"))
    single_test_iterator = test_runner_helper._determine_tests([test_module])
    test = next(single_test_iterator)

    self.assertEqual(test.name, "test0")
    self.assertTrue(test.validation_function is not None)
    self.assertEqual(test.validation_function.__name__, "validate0")
Ejemplo n.º 8
0
  def test_determine_tests_generates_correct_test_object(self):
    """
    Tests that determine_tests generate correct test objects
    """
    test_directory = os.path.join(self.FILE_LOCATION, "samples/determine_tests")
    test_module = utils.load_module(os.path.join(test_directory,
                                                 "meta_test_single.py"))
    single_test_iterator = test_runner_helper._determine_tests([test_module])
    test = next(single_test_iterator)

    self.assertEqual(test.name, "test0")
    self.assertTrue(test.validation_function is not None)
    self.assertEqual(test.validation_function.__name__, "validate0")
Ejemplo n.º 9
0
  def test_directory_setup_makes_correct_directories(self):
    """
    Tests that directory_setup makes the correct directories
    """
    perf_module = utils.load_module(os.path.join(self.FILE_LOCATION,
                                                 "samples/sample_perf.py"))
    dir_info = test_runner_helper.directory_setup(
        os.path.join(self.FILE_LOCATION, "samples/sample_input.py"),
        perf_module, configobj.Config("Master", {}))
    self.assertTrue(os.path.isdir(runtime.get_reports_dir()))
    self.assertTrue("sample_input" in dir_info["report_name"])
    self.assertTrue(os.path.isdir(dir_info["results_dir"]))
    self.assertTrue(os.path.isdir(dir_info["logs_dir"]))

    shutil.rmtree(dir_info["results_dir"])
    shutil.rmtree(dir_info["logs_dir"])
Ejemplo n.º 10
0
  def test_directory_setup_makes_correct_directories(self):
    """
    Tests that directory_setup makes the correct directories
    """
    perf_module = utils.load_module(os.path.join(self.FILE_LOCATION,
                                                 "samples/sample_perf.py"))
    dir_info = test_runner_helper.directory_setup(
        os.path.join(self.FILE_LOCATION, "samples/sample_input.py"),
        perf_module, configobj.Config("Master", {}))
    self.assertTrue(os.path.isdir(runtime.get_reports_dir()))
    self.assertTrue("sample_input" in dir_info["report_name"])
    self.assertTrue(os.path.isdir(dir_info["results_dir"]))
    self.assertTrue(os.path.isdir(dir_info["logs_dir"]))

    shutil.rmtree(dir_info["results_dir"])
    shutil.rmtree(dir_info["logs_dir"])
Ejemplo n.º 11
0
def _parse_input(testfile):
  """
  Extract the dictionary from the input file
  """
  ext = os.path.splitext(testfile)[-1].lower()
  if ext == ".py":
    test_dic = utils.load_module(testfile).test
  elif ext == ".json":
    json_data = open(testfile).read()
    test_dic = json.loads(json_data)
  else:
    logger.critical(testfile + " is not supported; currently only supports python and json files")
    raise ValueError("currently only supports python and json files")

  # checking test_dic to see if it has valid keys and values
  if not len(test_dic.keys()) == 4:
    logger.critical("input requires four fields: deployment_code, test_code, perf_code, configs_directory")
    raise ValueError("input requires four fields: deployment_code, test_code, perf_code, configs_directory")

  old_valid_key_list = ["deployment_code", "test_code", "perf_code", "configs_directory"]
  new_valid_key_list = ["deployment_code", "test_code", "dynamic_configuration_code", "configs_directory"]
  if not set(old_valid_key_list) == set(test_dic.keys()) and not set(new_valid_key_list) == set(test_dic.keys()):
    logger.critical("input requires four fields: deployment_code, test_code, dynamic_configuration_code, configs_directory")
    raise ValueError("input requires four fields: deployment_code, test_code, dynamic_configuration_code, configs_directory")

  filename = test_dic["deployment_code"]
  utils.check_file_with_exception(filename)
  filenames = test_dic["test_code"]
  for filename in filenames:
    utils.check_file_with_exception(filename)
  if "dynamic_configuration_code" in test_dic:
    filename = test_dic["dynamic_configuration_code"]
  else:
    filename = test_dic["perf_code"]
  utils.check_file_with_exception(filename)
  dirname = test_dic["configs_directory"]
  utils.check_dir_with_exception(dirname)

  return test_dic
Ejemplo n.º 12
0
  def test_determine_tests_read_correct_number_of_tests_in_file(self):
    """
    Test that we can read the correct number of tests from the test file
    """
    test_directory = os.path.join(self.FILE_LOCATION, "samples/determine_tests")
    test_files = [file_name for file_name in os.listdir(test_directory)
        if os.path.splitext(file_name)[1] == ".py"]
    for testfile in test_files:
      test_module = utils.load_module(os.path.join(test_directory, testfile))
      tests = [test.name for test
          in test_runner_helper._determine_tests([test_module])]

      if ("empty" in testfile) or ("invalid" in testfile) or ("no_test" in testfile):
        self.assertEqual(len(tests), 0)
      elif ("single" in testfile) or ("no_validate" in testfile) or ("no_match" in testfile):
        self.assertEqual(len(tests), 1)
        for i in xrange(0, 1):
          self.assertTrue("test{0}".format(i) in tests)
      elif "multi" in testfile:
        self.assertEqual(len(tests), 2)
        for i in xrange(0, 2):
          self.assertTrue("test{0}".format(i) in tests)
      else:
        print testfile + " was not used as test input"
Ejemplo n.º 13
0
def main():
  """
  Parse command line arguments and then run the test suite
  """
  parser = argparse.ArgumentParser(description='A distributed test framework')
  parser.add_argument('testfile',
      help='The file that is used to determine the test suite run')
  parser.add_argument('--test-only',
      nargs='*',
      dest='test_list',
      help='run only the named tests to help debug broken tests')
  parser.add_argument('--machine-list',
      nargs='*',
      dest='machine_list',
      help='''mapping of logical host names to physical names allowing the same
              test suite to run on different hardware, each argument is a pair
              of logical name and physical name separated by a =''')
  parser.add_argument('--config-overrides',
      nargs='*',
      dest='config_overrides',
      help='''config overrides at execution time, each argument is a config with
              its value separated by a =. This has the highest priority of all
              configs''')
  parser.add_argument('-d', '--output-dir',
      dest='output_dir',
      help='''Directory to write output files and logs. Defaults to the current
              directory.''')
  parser.add_argument("--log-level", dest="log_level", help="Log level (default INFO)", default="INFO")
  parser.add_argument("--console-log-level", dest="console_level", help="Console Log level (default ERROR)",
                      default="ERROR")
  parser.add_argument("--nopassword", action='store_true', dest="nopassword", help="Disable password prompt")
  parser.add_argument("--user", dest="user", help="user to run the test as (defaults to current user)")
  args = parser.parse_args()

  # Get output directory.
  try:
    if args.output_dir is not None:
      runtime.set_output_dir(args.output_dir)
  except ValueError as e:
    print str(e)
    sys.exit(1)

  # Set up logging.
  setup_logging(runtime.get_output_dir(), args.log_level, args.console_level)
  logger = logging.getLogger("zopkio")
  logger.info("Starting zopkio")

  try:
    utils.check_file_with_exception(args.testfile)
    utils.check_testfile_dir_structure(args.testfile)
    machines = utils.make_machine_mapping(args.machine_list)
    config_overrides = utils.parse_config_list(args.config_overrides)
  except ValueError as e:
    logger.error(str(e))
    print("Error in processing command line arguments:\n {0}".format(traceback.format_exc()))
    sys.exit(1)

  runtime.set_machines(machines)
  if args.user is not None:
    user = args.user
  else:
    user = getpass.getuser()
  if args.nopassword:
    password = ""
  else:
    password = getpass.getpass()
  runtime.set_user(user, password)

  try:
    testmodule = utils.load_module(args.testfile)
    ztestsuites = [getattr(testmodule, attr)
               for attr in dir(testmodule)
               if isinstance(getattr(testmodule, attr), ZTestSuite)]
    if len(ztestsuites) > 0: #TODO(jehrlich) intelligently handle multiple test suites
      test_runner = TestRunner(ztestsuite=ztestsuites[0], testlist=args.test_list, config_overrides=config_overrides)
    else:
      test_runner = TestRunner(args.testfile, args.test_list, config_overrides)
  except BaseException as e:
    print("Error setting up testrunner:\n%s" % traceback.format_exc())
    sys.exit(1)

  test_runner.run()

  logger.info("Exiting zopkio")
Ejemplo n.º 14
0
def main():
    """
  Parse command line arguments and then run the test suite
  """
    parser = argparse.ArgumentParser(
        description='A distributed test framework')
    parser.add_argument(
        'testfile',
        help='The file that is used to determine the test suite run')
    parser.add_argument(
        '--test-only',
        nargs='*',
        dest='test_list',
        help='run only the named tests to help debug broken tests')
    parser.add_argument(
        '--machine-list',
        nargs='*',
        dest='machine_list',
        help='''mapping of logical host names to physical names allowing the same
              test suite to run on different hardware, each argument is a pair
              of logical name and physical name separated by a =''')
    parser.add_argument(
        '--config-overrides',
        nargs='*',
        dest='config_overrides',
        help=
        '''config overrides at execution time, each argument is a config with
              its value separated by a =. This has the highest priority of all
              configs''')
    parser.add_argument(
        '-d',
        '--output-dir',
        dest='output_dir',
        help='''Directory to write output files and logs. Defaults to the current
              directory.''')
    parser.add_argument("--log-level",
                        dest="log_level",
                        help="Log level (default INFO)",
                        default="INFO")
    parser.add_argument("--console-log-level",
                        dest="console_level",
                        help="Console Log level (default ERROR)",
                        default="ERROR")
    parser.add_argument("--nopassword",
                        action='store_true',
                        dest="nopassword",
                        help="Disable password prompt")
    parser.add_argument(
        "--user",
        dest="user",
        help="user to run the test as (defaults to current user)")
    args = parser.parse_args()

    # Get output directory.
    try:
        if args.output_dir is not None:
            runtime.set_output_dir(args.output_dir)
    except ValueError as e:
        print str(e)
        sys.exit(1)

    # Set up logging.
    setup_logging(runtime.get_output_dir(), args.log_level, args.console_level)
    logger = logging.getLogger("zopkio")
    logger.info("Starting zopkio")

    try:
        utils.check_file_with_exception(args.testfile)
        utils.check_testfile_dir_structure(args.testfile)
        machines = utils.make_machine_mapping(args.machine_list)
        config_overrides = utils.parse_config_list(args.config_overrides)
    except ValueError as e:
        logger.error(str(e))
        print("Error in processing command line arguments:\n {0}".format(
            traceback.format_exc()))
        sys.exit(1)

    runtime.set_machines(machines)
    if args.user is not None:
        user = args.user
    else:
        user = getpass.getuser()
    if args.nopassword:
        password = ""
    else:
        password = getpass.getpass()
    runtime.set_user(user, password)

    try:
        testmodule = utils.load_module(args.testfile)
        ztestsuites = [
            getattr(testmodule, attr) for attr in dir(testmodule)
            if isinstance(getattr(testmodule, attr), ZTestSuite)
        ]
        if len(
                ztestsuites
        ) > 0:  #TODO(jehrlich) intelligently handle multiple test suites
            test_runner = TestRunner(ztestsuite=ztestsuites[0],
                                     testlist=args.test_list,
                                     config_overrides=config_overrides)
        else:
            test_runner = TestRunner(args.testfile, args.test_list,
                                     config_overrides)
    except BaseException as e:
        print("Error setting up testrunner:\n%s" % traceback.format_exc())
        sys.exit(1)

    test_runner.run()

    logger.info("Exiting zopkio")