예제 #1
0
    def test_read_parameters_from_command_line(self):
        # Legal call
        sys.argv = ["test.py", "--folds", "10", "--fold", "0", "--params", "-x",
                    "3"]
        args, params = benchmark_util.parse_cli()
        self.assertEqual(params, {'x': '3'})
        self.assertEqual(args, {'folds': '10', 'fold': '0'})

        # illegal call, arguments with one minus before --params
        sys.argv = ["test.py", "-folds", "10", "--fold", "0", "--params", "-x",
                    "3"]
        with self.assertRaises(ValueError) as cm1:
            benchmark_util.parse_cli()
        self.assertEqual(cm1.exception.message, "You either try to use arguments"
                         " with only one leading minus or try to specify a "
                         "hyperparameter before the --params argument. test.py"
                         " -folds 10 --fold 0 --params -x 3")

        # illegal call, trying to specify an arguments after --params
        sys.argv = ["test.py", "--folds", "10", "--params", "-x",
                    "'3'", "--fold", "0"]
        with self.assertRaises(ValueError) as cm5:
            benchmark_util.parse_cli()
        self.assertEqual(cm5.exception.message, "You are trying to specify an argument after the "
                             "--params argument. Please change the order.")

        # illegal call, no - in front of parameter name
        sys.argv = ["test_cv.py", "--params", "x", "'5'"]
        with self.assertRaises(ValueError) as cm2:
            benchmark_util.parse_cli()
        self.assertEqual(cm2.exception.message, "Illegal command line string, expected a hyperpara"
                             "meter starting with - but found x")
예제 #2
0
def main():
  starttime = time.time()
  args, params = benchmark_util.parse_cli()
  assert args["folds"] == "1", args
  params_hash = DictHash(params)
  sys.stderr.write("Run hash: {}\n".format(params_hash))
  with open(params_hash + ".out", "w") as stdout_file:
    with open(params_hash + ".err", "w") as stderr_file:
      exit_code = subprocess.call(["../tpe3.py", json.dumps(params), params_hash],
                                  stdout=stdout_file, stderr=stderr_file)
      if exit_code != 0:
        raise subprocess.CalledProcessError(exit_code)
  output = open(params_hash + ".out").readlines()[-1]
  result = float(output.strip().split()[-1])
  duration = time.time() - starttime
  print "Result for ParamILS: SAT, {}, 1, {}, -1, {}".format(
      abs(duration), result, str(__file__))
예제 #3
0
    def test_read_parameters_from_command_line(self):
        # Legal call
        sys.argv = [
            "test.py", "--folds", "10", "--fold", "0", "--params", "-x", "3"
        ]
        args, params = benchmark_util.parse_cli()
        self.assertEqual(params, {'x': '3'})
        self.assertEqual(args, {'folds': '10', 'fold': '0'})

        # illegal call, arguments with one minus before --params
        sys.argv = [
            "test.py", "-folds", "10", "--fold", "0", "--params", "-x", "3"
        ]
        with self.assertRaises(ValueError) as cm1:
            benchmark_util.parse_cli()
        self.assertEqual(
            cm1.exception.message, "You either try to use arguments"
            " with only one leading minus or try to specify a "
            "hyperparameter before the --params argument. test.py"
            " -folds 10 --fold 0 --params -x 3")

        # illegal call, trying to specify an arguments after --params
        sys.argv = [
            "test.py", "--folds", "10", "--params", "-x", "'3'", "--fold", "0"
        ]
        with self.assertRaises(ValueError) as cm5:
            benchmark_util.parse_cli()
        self.assertEqual(
            cm5.exception.message,
            "You are trying to specify an argument after the "
            "--params argument. Please change the order.")

        # illegal call, no - in front of parameter name
        sys.argv = ["test_cv.py", "--params", "x", "'5'"]
        with self.assertRaises(ValueError) as cm2:
            benchmark_util.parse_cli()
        self.assertEqual(
            cm2.exception.message,
            "Illegal command line string, expected a hyperpara"
            "meter starting with - but found x")
    """For a given problem train the metric function and return its loss
    value. Arguments:
      * task_files_list
      * experiment_files_list
      * metalearning_directory


    Sample call: python kNDEvaluateSurrogate.py --task_files_list
    /mhome/feurerm/thesis/experiments/AutoSklearn/metalearning_experiments/2014_09_10_test/tasks.txt
    --experiments_list /mhome/feurerm/thesis/experiments/AutoSklearn
    /metalearning_experiments/2014_09_10_test/experiments.txt
    --metalearning_directory /mhome/feurerm/thesis/experiments/AutoSklearn/ --params -random_state 5
    """

    starttime = time.time()
    args, params = benchmark_util.parse_cli()
    os.chdir(args['metalearning_directory'])
    pyMetaLearn.directory_manager.set_local_directory(
        args['metalearning_directory'])

    with open(args["task_files_list"]) as fh:
        task_files_list = fh.readlines()
    with open(args["experiments_list"]) as fh:
        experiments_list = fh.readlines()

    if 'keep_configurations' in args:
        keep_configurations = args['keep_configurations']
        keep_configurations = keep_configurations.split(',')
        keep_configurations = tuple(
            [tuple(kc.split('=')) for kc in keep_configurations])
    else:
                                                   TerminationCriterionMaxEpoch(200),
                                                   TerminationCriterionDivergenceDetection(),
                                                   TerminationCriterionExternalInBackground(
                                                      external_cmd="python -m pylrpredictor.terminationcriterion --nthreads 3",
                                                      run_every_x_epochs=30)],
                        test_every_x_epoch=0.5,
			num_valid=5000,
			snapshot_prefix="caffnet-run%d" % get_run_num(),
			batch_size_valid=100,
			snapshot_on_exit=1,
			device="GPU",
			device_id=0)
    return caffe


def main(params, **kwargs):
    experiment_dir = os.path.dirname(os.path.realpath(__file__))
    working_dir = os.getcwd()
    return hpolib_experiment_main(params, construct_caffeconvnet,
        experiment_dir=experiment_dir,
        working_dir=working_dir,
        mean_performance_on_last=MEAN_PERFORMANCE_ON_LAST)

if __name__ == "__main__":
    starttime = time.time()
    args, params = benchmark_util.parse_cli()
    result = main(params, **args)
    duration = time.time() - starttime
    print "Result for ParamILS: %s, %f, 1, %f, %d, %s" % \
                    ("SAT", abs(duration), result, -1, str(__file__))