Ejemplo n.º 1
0
def plot_learning_curves(name,
                         figure_dir,
                         learning_curves,
                         y_label,
                         y_limits=(0, +1),
                         **kwargs):
    import matplotlib.pyplot as plt
    plt.figure()
    for label, sizes, scores in learning_curves:
        plot_learning_curve(sizes, scores, name=label, **kwargs)

    if name is not None:
        plt.title(name)  # Could extract from figure_dir
    if y_limits is not None:
        plt.ylim(*y_limits)
    plt.xlabel('# Training Examples')
    plt.ylabel(y_label)
    plt.grid()
    plt.legend(loc='best')
    plt.tight_layout()

    if figure_dir is not None:
        mkdir(figure_dir)
        figure_path = os.path.join(figure_dir, '{}.png'.format(y_label))  # pdf
        plt.savefig(figure_path, transparent=False,
                    bbox_inches='tight')  # dpi=1000,
        print('Saved', figure_path)
    if not is_remote():
        plt.show()
    return plt
Ejemplo n.º 2
0
def save_experiments(experiments_dir, expid, experiments):
    if experiments_dir is None:
        return None
    mkdir(experiments_dir)
    data_path = os.path.join(
        experiments_dir, 'experiments_{}.pk{}'.format(expid,
                                                      get_python_version()))
    write_pickle(data_path, experiments)
    print('Saved', data_path)
    return data_path
Ejemplo n.º 3
0
def save_learner(data_dir, learner):
    if data_dir is None:
        return False
    #domain = learner.func
    #data_dir = os.path.join(MODEL_DIRECTORY, domain.name)
    #name = learner.name
    name = get_label(learner.algorithm)
    mkdir(data_dir)
    learner_path = os.path.join(data_dir, '{}.pk{}'.format(name, get_python_version()))
    print('Saved', learner_path)
    write_pickle(learner_path, learner)
    return True
Ejemplo n.º 4
0
 def save(self, data_dir):
     from learn_tools.analyze_experiment import get_label
     if data_dir is None:
         return False
     # domain = learner.func
     # data_dir = os.path.join(MODEL_DIRECTORY, domain.name)
     # name = learner.name
     name = get_label(self.algorithm)
     mkdir(data_dir)
     learner_path = os.path.join(
         data_dir, '{}.pk{}'.format(name, get_python_version()))
     print('Saved learner:', learner_path)
     write_pickle(learner_path, self)
     return True
Ejemplo n.º 5
0
def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('paths', nargs='*', help='Paths to the data.')
    #parser.add_argument('-a', '--active', type=int, default=0, # None
    #                    help='The number of active samples to collect')
    parser.add_argument('-l', '--learner', default=None,
                        help='Path to the learner that should be used')
    parser.add_argument('-n', '--num_trials', type=int, default=100,
                        help='The number of samples to collect')
    parser.add_argument('-s', '--save', action='store_true',
                        help='Whether to save the learners')
    parser.add_argument('-r', '--num_rounds', type=int, default=1,
                        help='The number of rounds to collect')
    #parser.add_argument('-t', '--num_train', type=int, default=None,
    #                    help='The size of the training set')
    args = parser.parse_args()

    # TODO: be careful that paging isn't altering the data
    # TODO: don't penalize if the learner identifies that it can't make a good prediction
    # TODO: use a different set of randomized parameters for train and test

    include_none = False
    serial = is_darwin()


    #training_sizes = inclusive_range(50, 500, 25)
    #training_sizes = inclusive_range(25, 100, 5)
    #training_sizes = inclusive_range(25, 100, 5)
    training_sizes = inclusive_range(10, 50, 5)
    #training_sizes = inclusive_range(100, 1000, 100)
    #training_sizes = [20]
    #training_sizes = [1500]

    #kernels = ['RBF', 'Matern52', 'MLP']
    kernels = ['MLP']

    #hyperparameters = [None]
    #hyperparameters = [True]
    hyperparameters = [True, None] # None,

    query_type = BEST # BEST | CONFIDENT | REJECTION | ACTIVE # type of query used to evaluate the learner
    is_adaptive = False
    max_test = 50 #
    #alphas = np.linspace(0.0, 0.9, num=5, endpoint=True)
    alphas = [0.0, .8, .9, .99]
    #alphas = [None]  # Use the default (i.e. GP parameters)

    use_vars = [True]
    binary = False
    split = UNIFORM # BALANCED

    # Omitting failed labels is okay because they will never be executed
    algorithms = []
    #algorithms += [(Algorithm(BATCH_GP, kernel=kernel, hyperparameters=hype, use_var=use_var), [num_train])
    #               for num_train, kernel, hype, use_var in product(training_sizes, kernels, hyperparameters, use_vars)]
    algorithms += [(Algorithm(STRADDLE_GP, kernel, hype, use_var), training_sizes)
                   for kernel, hype, use_var in product(kernels, hyperparameters, use_vars)]
    #algorithms += [(Algorithm(rf_model, p_explore=None, use_var=use_var), [num_train])
    #               for rf_model, num_train, use_var in product(RF_MODELS, training_sizes, use_vars)]
    #algorithms += [(Algorithm(nn_model, p_explore=None), [num_train])
    #               for nn_model, num_train in product(NN_MODELS, training_sizes)]
    #algorithms += [(Algorithm(RANDOM), None), (Algorithm(DESIGNED), None)]
    print('Algorithms:', algorithms)
    print('Split:', split)

    trials_per_round = sum(1 if train_sizes is None else
                           (train_sizes[-1] - train_sizes[0] + len(train_sizes))
                           for _, train_sizes in algorithms)
    num_experiments = args.num_rounds*trials_per_round

    date_name = datetime.datetime.now().strftime(DATE_FORMAT)
    size_str = '[{},{}]'.format(training_sizes[0], training_sizes[-1])
    #size_str = '-'.join(map(str, training_sizes))
    experiments_name = '{}_r={}_t={}_n={}'.format(date_name, args.num_rounds, size_str, args.num_trials) #'19-08-09_21-44-58_r=5_t=[10,150]_n=1'#
    #experiments_name = 't={}'.format(args.num_rounds)
    # TODO: could include OS and username if desired

    domain = load_data(args.paths)
    print()
    print(domain)
    X, Y, W = domain.get_data(include_none=include_none)
    print('Total number of examples:', len(X))
    if binary:
        # NN can fit perfectly when binary
        # Binary seems to be outperforming w/o
        Y = threshold_scores(Y)

    max_train = len(X) - max_test #min(max([0] + [active_sizes[0] for _, active_sizes in algorithms
                     #          if active_sizes is not None]), len(X))

    #parameters = {
    #    'include None': include_none,
    #    'binary': binary,
    #    'split': split,
    #}

    print('Name:', experiments_name)
    print('Experiments:', num_experiments)
    print('Max train:', max_train)
    print('Include None:', include_none)
    print('Examples: n={}, d={}'.format(*X.shape))
    print('Binary:', binary)
    print('Estimated hours:', num_experiments * SEC_PER_EXPERIMENT / HOURS_TO_SECS)
    user_input('Begin?')
    # TODO: residual learning for sim to real transfer
    # TODO: can always be conservative and add sim negative examples

    # TODO: combine all data to write in one folder
    data_dir = os.path.join(DATA_DIRECTORY, domain.name) # EXPERIMENT_DIRECTORY
    experiments_dir = os.path.join(data_dir, experiments_name)
    mkdir(experiments_dir)
    start_time = time.time()
    experiments = []
    for round_idx in range(args.num_rounds):
        round_dir = os.path.join(data_dir, experiments_name, str(round_idx))
        mkdir(round_dir)
        seed = hash(time.time())
        train_test_file = os.path.join(round_dir, 'data.pk3')
        if not os.path.exists(train_test_file):
            X_train, Y_train, X_test, Y_test = split_data(X, Y, split, max_train)
            X_test, Y_test = X_test[:max_test], Y_test[:max_test]
            write_pickle(train_test_file, (X_train, Y_train, X_test, Y_test))
        else:
            X_train, Y_train, X_test, Y_test = read_pickle(train_test_file)

        print('Train examples:', X_train.shape)
        print('Test examples:', X_test.shape)
        # TODO: need to be super careful when running with multiple contexts

        for algorithm, active_sizes in algorithms:
            # active_sizes = [first #trainingdata selected from X_train, #active exploration + #trainingdata]
            print(SEPARATOR)
            print('Round: {} | {} | Seed: {} | Sizes: {}'.format(round_idx, algorithm, seed, active_sizes))
            # TODO: allow keyboard interrupt
            if active_sizes is None:
                learner = algorithm.name
                active_size = None
                train_confusion = None
                experiments.append(evaluate_learner(domain, seed, train_confusion, X_test, Y_test, algorithm, learner,
                                                    active_size, args.num_trials, alphas,
                                                    serial))
            else:
                # [10 20 25] take first 10 samples from X_train to train the model, 10 samples chosen actively
                # sequentially + evaluate model, 5 samples chosen actively sequentially + evaluate model
                # Could always keep around all the examples and retrain
                # TODO: segfaults when this runs in parallel
                # TODO: may be able to retrain in parallel if I set OPENBLAS_NUM_THREADS
                learner_prior_nx = 0
                '''
                if algorithm.hyperparameters:
                    if domain.skill == 'pour':
                        learner_file = '/Users/ziw/ltamp_pr2/data/pour_19-06-13_00-59-21/19-08-09_19-30-01_r=10_t=[50,400]_n=1/{}/gp_active_mlp_true_true.pk3'.format(
                            round_idx)
                    elif domain.skill == 'scoop':
                        learner_file = '/Users/ziw/ltamp_pr2/data/scoop_19-06-10_20-16-59_top-diameter/19-08-09_19-34-56_r=10_t=[50,400]_n=1/{}/gp_active_mlp_true_true.pk3'.format(
                            round_idx)
                    learner = read_pickle(learner_file)
                    learner_prior_nx = learner.nx
                    learner.retrain(newx=X_train[:active_sizes[0]], newy=Y_train[:active_sizes[0], None])
                else:
                '''
                learner, train_confusion = create_learner(domain, X_train, Y_train, split, algorithm,
                                                          num_train=active_sizes[0], query_type=query_type,
                                                          is_adaptive=is_adaptive)

                if algorithm.name == STRADDLE_GP:
                    X_select, Y_select = X_train[active_sizes[0]:], Y_train[active_sizes[0]:]

                for active_size in active_sizes:
                    num_active = active_size - learner.nx + learner_prior_nx# learner.nx is len(learner.xx)
                    print('\nRound: {} | {} | Seed: {} | Size: {} | Active: {}'.format(
                        round_idx, algorithm, seed, active_size, num_active))
                    if algorithm.name == STRADDLE_GP:
                        X_select, Y_select = active_learning_discrete(learner, num_active, X_select, Y_select)
                    #if args.save:
                    save_learner(round_dir, learner)
                    experiments.append(evaluate_learner(domain, seed, None, X_test, Y_test,
                                                        algorithm, learner,
                                                        active_size, args.num_trials, alphas,
                                                        serial))
                    save_experiments(experiments_dir, experiments)

    print(SEPARATOR)
    if experiments:
        save_experiments(experiments_dir, experiments)
        plot_experiments(domain, experiments_name, experiments_dir, experiments,
                         include_none=False)
                         #include_none=include_none)
        print('Experiments:', experiments_dir)
    print('Total experiments:', len(experiments))
    print('Total hours:', elapsed_time(start_time) / HOURS_TO_SECS)
Ejemplo n.º 6
0
import os
import glob
from pybullet_tools.utils import read_json, write_json
from pddlstream.utils import mkdir

if __name__ == '__main__':
    SKILL = 'scoop'
    MATERIAL = {'pour': 'red_men', 'scoop': 'chickpeas'}
    all_dirs = glob.glob('data/pr2_{}_*'.format(SKILL))
    all_trials = []
    for dir in all_dirs:
        trial_file = os.path.join(dir, 'trials.json')
        data = read_json(trial_file)
        for d in data:
            if d['policy'] == 'training' and d['material'] == MATERIAL[SKILL] and d['score'] is not None:
                all_trials.append(d)
    newdir = 'data/pr2_{}/'.format(SKILL)
    mkdir(newdir)
    write_json(os.path.join(newdir, 'all_trials.json'), all_trials)
Ejemplo n.º 7
0
def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('trainsize',
                        default=2000,
                        type=int,
                        help='training set size')
    parser.add_argument('expid', default=1, type=int, help='experiment ID')
    parser.add_argument(
        'beta_lambda',
        type=float,
        default=0.9,
        help='lambda parameter for computing beta from best beta')
    parser.add_argument('sample_strategy_id', default=1, type=int)  # 1, 2, 3
    parser.add_argument(
        'paths',
        default=[os.path.join(get_data_dir('pour'), 'trials_n=10000.json')],
        nargs='*',
        help='Paths to the data.')
    parser.add_argument('-u',
                        '--use_hyper',
                        action='store_true',
                        help='When enabled, use existing hyper parameter.')
    parser.add_argument('-o',
                        '--use_obstacle',
                        action='store_true',
                        help='When enabled, no obstacle is used in the scene.')

    args = parser.parse_args()
    beta_lambda = args.beta_lambda
    sample_strategy = SAMPLE_STRATEGIES[args.sample_strategy_id]
    global SEED
    SEED = args.expid
    set_seed(SEED)
    n_train_tasks = 50
    n_test_tasks = 20

    train_tasks_seeds = get_seeds(n_train_tasks)
    test_tasks_seeds = get_seeds(n_test_tasks)
    print('loading data')
    domain = load_data(args.paths)
    data = domain.create_dataset(include_none=True, binary=False)
    data.shuffle()
    X, Y, W = data.get_data()
    print('finished obtaining x y data')

    n_train = args.trainsize
    X = X[:n_train]
    Y = Y[:n_train]

    print('initializing ActiveGP with #datapoints = {}'.format(len(X)))

    hype = None

    if 'pour' in args.paths[0] and args.use_hyper:
        hype = POUR_MLP_HYPERPARAM_3000
    elif 'scoop' in args.paths[0] and args.use_hyper:
        hype = SCOOP_MLP_HYPERPARAM_3000

    learner = ActiveGP(domain,
                       initx=X,
                       inity=Y,
                       hyperparameters=hype,
                       sample_time_limit=60,
                       beta_lambda=beta_lambda)
    learner.retrain(num_restarts=10)

    exp_file = 'tasklengthscale_sampling_trainsize={}_beta_lambdda={}_strategy_{}_obs_{}_expid_{}.pk3'.format(
        len(X), beta_lambda, args.sample_strategy_id, int(args.use_obstacle),
        args.expid)
    exp_dirname = os.path.dirname(args.paths[0])
    if args.use_hyper:
        exp_dirname = os.path.join(exp_dirname, 'default_hyper/')
        mkdir(exp_dirname)

    exp_file = os.path.join(exp_dirname, exp_file)
    print('saving results to ', exp_file)

    results = []

    if sample_strategy != DIVERSELK:
        n_train_tasks = 0
        # no need to train

    prev_tasklengthscale = None
    for i in range(n_train_tasks + 1):
        test_results = []
        print('task_lengthscal = {}'.format(learner.task_lengthscale))
        print('================BEGIN TESTING==============')
        if prev_tasklengthscale is not None and (
                learner.task_lengthscale == prev_tasklengthscale).all():
            test_results = results[-1][1]
        else:
            for j in range(n_test_tasks):
                if sample_strategy == DIVERSELK:
                    test_sample_strategy = DIVERSE
                else:
                    test_sample_strategy = sample_strategy
                seed = test_tasks_seeds[j]
                test_result = eval_task_with_seed(
                    domain,
                    seed,
                    learner,
                    sample_strategy=test_sample_strategy,
                    obstacle=args.use_obstacle)
                test_results.append(test_result)

        prev_tasklengthscale = learner.task_lengthscale.copy()

        if i != n_train_tasks:
            seed = train_tasks_seeds[i]
            train_result = eval_task_with_seed(domain,
                                               seed,
                                               learner,
                                               sample_strategy=sample_strategy,
                                               obstacle=args.use_obstacle)
        else:
            train_result = None

        results.append((train_result, test_results))
        write_pickle(exp_file,
                     (results, SEED, train_tasks_seeds, test_tasks_seeds))
Ejemplo n.º 8
0
def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('paths', nargs='*', help='Paths to the data.')
    #parser.add_argument('-a', '--active', type=int, default=0, # None
    #                    help='The number of active samples to collect')
    parser.add_argument(
        '-d',
        '--deterministic',
        action='store_true',
        help='Whether to deterministically create training splits')
    parser.add_argument('-n',
                        '--num_trials',
                        type=int,
                        default=-1,
                        help='The number of samples to collect')
    parser.add_argument('-s',
                        '--save',
                        action='store_true',
                        help='Whether to save the learners')
    parser.add_argument('-r',
                        '--num_rounds',
                        type=int,
                        default=1,
                        help='The number of rounds to collect')
    parser.add_argument('-t',
                        '--test',
                        action='store_true',
                        help='Whether to save the data')
    parser.add_argument('-v',
                        '--visualize',
                        action='store_true',
                        help='When enabled, visualizes execution.')
    args = parser.parse_args()

    # TODO: be careful that paging isn't altering the data
    # TODO: use a different set of randomized parameters for train and test

    serial = is_darwin()
    visualize = serial and args.visualize
    assert implies(visualize, serial)
    num_trials = get_max_cores(
        serial) if args.num_trials < 0 else args.num_trials

    ##################################################

    #train_sizes = inclusive_range(50, 200, 10) # Best
    #train_sizes = inclusive_range(50, 400, 10) # F1
    #train_sizes = inclusive_range(25, 400, 25)
    #train_sizes = inclusive_range(50, 100, 5) # Real
    #train_sizes = inclusive_range(100, 200, 5)
    #train_sizes = inclusive_range(10, 250, 5)
    #train_sizes = inclusive_range(35, 70, 5)
    #train_sizes = inclusive_range(5, 50, 5)
    #train_sizes = inclusive_range(40, 80, 5)
    #train_sizes = inclusive_range(100, 1000, 100)
    #train_sizes = [50]
    #train_sizes = [250]
    train_sizes = [1000]
    #train_sizes = [327] # train + test
    #train_sizes = inclusive_range(5, 150, 25)
    #train_sizes = [100]

    #kernels = ['RBF', 'Matern52', 'MLP']
    kernels = ['MLP']

    hyperparams = [None]
    #hyperparams = [True]
    #hyperparams = [None, True]

    query_type = BEST  # BEST | CONFIDENT | REJECTION | ACTIVE # type of query used to evaluate the learner

    include_none = False
    binary = False

    # 0 => no transfer
    # 1 => mean transfer
    # 2 => kernel transfer
    # 3 => both transfer
    transfer_weights = [None]
    #transfer_weights = list(range(4))
    #transfer_weights = [0, 1]
    #transfer_weights = [3]
    #transfer_weights = np.around(np.linspace(0.0, 1.0, num=1+5, endpoint=True), decimals=3) # max 10 colors
    #transfer_weights = list(range(1, 1+3))

    #split = UNIFORM # BALANCED
    #print('Split:', split)
    #parameters = {
    #    'include None': include_none,
    #    'binary': binary,
    #    'split': split,
    #}

    # Omitting failed labels is okay because they will never be executed
    algorithms = []
    #algorithms += [(Algorithm(nn_model, label='NN'), [num])
    #              for nn_model, num in product(NN_MODELS, train_sizes)]
    #algorithms += [(Algorithm(RANDOM), None), (Algorithm(DESIGNED), None)]

    #algorithms += [(Algorithm(RF_CLASSIFIER, variance=False, transfer_weight=tw, label='RF'), [num])
    #                for num, tw in product(train_sizes, [None])] # transfer_weights
    #algorithms += [(Algorithm(RF_REGRESSOR, variance=False, transfer_weight=tw, label='RF'), [num])
    #                for num, tw in product(train_sizes, [None])] # transfer_weights
    #algorithms += [(Algorithm(BATCH_RF, variance=True, transfer_weight=tw, label='RF'), [num])
    #                for num, tw in product(train_sizes, [None])] # transfer_weights
    #algorithms += [(Algorithm(BATCH_MAXVAR_RF, variance=True, transfer_weight=tw), train_sizes)
    #                for tw in product(use_vars, [None])] # transfer_weights
    #algorithms += [(Algorithm(BATCH_STRADDLE_RF, variance=True, transfer_weight=tw), train_sizes)
    #                for tw, in product([None])] # transfer_weights

    use_vars = [True]
    # STRADDLE is better than MAXVAR when the learner has a good estimate of uncertainty
    algorithms += [
        (Algorithm(BATCH_GP, kernel, hype, use_var, tw,
                   label='GP'), [num])  # label='GP-{}'.format(kernel)
        for num, kernel, hype, use_var, tw in product(
            train_sizes, kernels, hyperparams, use_vars, transfer_weights)
    ]
    #algorithms += [(Algorithm(BATCH_MAXVAR_GP, kernel, hype, True, tw, label='GP-Var'), train_sizes)
    #                for kernel, hype, tw in product(kernels, hyperparams, transfer_weights)]
    #algorithms += [(Algorithm(BATCH_STRADDLE_GP, kernel, hype, True, tw, label='GP-LSE'), train_sizes)
    #                for kernel, hype, tw in product(kernels, hyperparams, transfer_weights)] # default active
    #algorithms += [(Algorithm(BATCH_STRADDLE_GP, kernel, hype, True, tw, label='GP-LSE2'), train_sizes)
    #                for kernel, hype, tw in product(kernels, hyperparams, transfer_weights)] # active control only

    # algorithms += [(Algorithm(MAXVAR_GP, kernel, hype, use_var), train_sizes)
    #                for kernel, hype, use_var in product(kernels, hyperparams, use_vars)]
    #algorithms += [(Algorithm(STRADDLE_GP, kernel, hype, use_var, tw), train_sizes)
    #                for kernel, hype, use_var, tw in product(kernels, hyperparams, use_vars, transfer_weights)]

    #batch_sizes = inclusive_range(train_sizes[0], 90, 10)
    #step_size = 10 # TODO: extract from train_sizes
    #final_size = train_sizes[-1]
    # Previously didn't have use_var=True
    # algorithms += [(Algorithm(BATCH_STRADDLE_GP, kernel, hyperparameters=batch_size, variance=True, transfer_weight=tw),
    #                 inclusive_range(batch_size, final_size, step_size))
    #                for kernel, tw, batch_size in product(kernels, transfer_weights, batch_sizes)]
    # algorithms += [(Algorithm(BATCH_STRADDLE_RF, hyperparameters=batch_size, variance=True, transfer_weight=tw),
    #                 inclusive_range(batch_size, final_size, step_size))
    #                 for tw, batch_size in product(transfer_weights, batch_sizes)]

    print('Algorithms:', algorithms)

    ##################################################

    real_world = not args.paths
    transfer_domain = load_data(TRANSFER_DATASETS, verbose=False)
    transfer_algorithm = None
    if real_world and transfer_weights != [None]:
        #assert transfer_weights[0] is not None
        transfer_data = transfer_domain.create_dataset(
            include_none=include_none, binary=binary)
        transfer_algorithm = Algorithm(BATCH_GP,
                                       kernel=kernels[0],
                                       variance=use_vars[0])

    validity_learner = None
    #validity_learner = create_validity_classifier(transfer_domain)

    ##################################################

    train_paths = args.paths
    if real_world:
        train_paths = SCOOP_TRAIN_DATASETS  # TRAIN_DATASETS
        #train_paths = TRANSFER_DATASETS
        #train_paths = TRAIN_DATASETS + TRANSFER_DATASETS # Train before transfer
    #scale_paths = TRAIN_DATASETS + TEST_DATASETS
    scale_paths = None
    print(SEPARATOR)
    print('Train paths:', train_paths)
    domain = load_data(train_paths)
    print()
    print(domain)
    all_data = domain.create_dataset(include_none=include_none,
                                     binary=binary,
                                     scale_paths=scale_paths)
    #all_data.results = all_data.results[:1000]

    num_failed = 0
    #num_failed = 100
    failed_domain = transfer_domain if real_world else domain
    failed_results = randomize(
        result for result in failed_domain.results
        if not result.get('success', False))[:num_failed]
    #failed_data = Dataset(domain, failed_results, **all_data.kwargs)

    test_paths = SCOOP_TEST_DATASETS  # TEST_DATASETS | SCOOP_TEST_DATASETS
    #test_paths = None
    if real_world and not (set(train_paths) & set(test_paths)):
        #assert not set(train_paths) & set(test_paths)
        #max_test = 0
        test_data = load_data(test_paths).create_dataset(
            include_none=False, binary=binary, scale_paths=scale_paths)
    else:
        #assert scale_paths is None # TODO: max_train will be too small otherwise
        test_paths = test_data = None
    print(SEPARATOR)
    print('Test paths:', test_paths)

    all_active_data = None
    #if real_world:
    #    all_active_data = load_data(ACTIVE_DATASETS).create_dataset(include_none=True, binary=binary, scale_paths=scale_paths)

    # TODO: could include OS and username if desired
    date_name = datetime.datetime.now().strftime(DATE_FORMAT)
    size_str = '[{},{}]'.format(train_sizes[0], train_sizes[-1])
    #size_str = '-'.join(map(str, train_sizes))
    experiments_name = '{}_r={}_t={}_n={}'.format(date_name, args.num_rounds,
                                                  size_str, num_trials)

    trials_per_round = sum(
        1 if train_sizes is None else (train_sizes[-1] - train_sizes[0] +
                                       len(train_sizes))
        for _, train_sizes in algorithms)
    num_experiments = args.num_rounds * trials_per_round
    max_train = min(
        max([0] + [
            active_sizes[0]
            for _, active_sizes in algorithms if active_sizes is not None
        ]), len(all_data))
    max_test = min(len(all_data) - max_train, 1000)

    ##################################################

    # #features = ['bowl_height']
    # features = ['spoon_height']
    # #features = ['bowl_height', 'spoon_height']
    # X, Y, _ = all_data.get_data()
    # #indices = [domain.inputs.index(feature) for feature in features]
    # #X = X[:,indices]
    # X = [[result[FEATURE][name] for name in features] for result in all_data.results]
    # from sklearn.linear_model import LinearRegression
    # model = LinearRegression(fit_intercept=True, normalize=False)
    # model.fit(X, Y)
    # #print(model.get_params())
    # print(model.coef_.tolist(), model.intercept_)
    # print(model.score(X, Y))

    #data_dir = os.path.join(DATA_DIRECTORY, domain.name) # EXPERIMENT_DIRECTORY
    data_dir = os.path.abspath(os.path.join(domain.name, os.path.pardir))
    experiments_dir, data_path = None, None
    if not args.test or not serial:
        experiments_dir = os.path.join(data_dir, experiments_name)
        data_path = os.path.join(
            experiments_dir, 'experiments.pk{}'.format(get_python_version()))

    ##################################################

    print(SEPARATOR)
    print('Name:', experiments_name)
    print('Experiments:', num_experiments)
    print('Experiment dir:', experiments_dir)
    print('Data path:', data_path)
    print('Examples:', len(all_data))
    print('Valid:',
          sum(result.get('valid', True) for result in all_data.results))
    print('Success:',
          sum(result.get('success', False) for result in all_data.results))
    print(
        'Scored:',
        sum(
            result.get('score', None) is not None
            for result in all_data.results))
    print('Max train:', max_train)
    print('Max test:', max_test)
    print('Include None:', include_none)
    print('Examples: n={}, d={}'.format(len(all_data), domain.dx))
    print('Binary:', binary)
    print('Serial:', serial)
    print('Estimated hours: {:.3f}'.format(num_experiments *
                                           SEC_PER_EXPERIMENT / HOURS_TO_SECS))
    user_input('Begin?')

    ##################################################

    experiments = []
    if experiments_dir is not None:
        mkdir(experiments_dir)
        # if os.path.exists(data_path):
        #     experiments.extend(read_pickle(data_path))

    # TODO: embed in a KeyboardInterrupt to allow early termination
    start_time = time.time()
    for round_idx in range(args.num_rounds):
        seed = round_idx if args.deterministic else hash(
            time.time())  # vs just time.time()?
        random.seed(seed)
        all_data.shuffle()
        if test_paths is None:  # cannot use test_data
            #test_data, train_data = split_data(all_data, max_test)
            train_data = test_data = all_data  # Training performance
        else:
            train_data = all_data

        transfer_learner = None
        if transfer_algorithm is not None:
            round_data, _ = transfer_data.partition(index=1000)
            transfer_learner, _ = create_learner(transfer_domain,
                                                 round_data,
                                                 transfer_algorithm,
                                                 verbose=True)
            transfer_learner.retrain()

        print(SEPARATOR)
        print('Round {} | Train examples: {} | Test examples: {}'.format(
            round_idx, len(train_data), len(test_data)))
        for algorithm, active_sizes in algorithms:
            # active_sizes = [first #trainingdata selected from X_train, #active exploration + #trainingdata]
            print(SEPARATOR)
            print('Round: {} | {} | Seed: {} | Sizes: {}'.format(
                round_idx, algorithm, seed, active_sizes))
            # TODO: allow keyboard interrupt
            if active_sizes is None:
                learner = algorithm.name
                active_size = train_confusion = None
                experiments.append(
                    evaluate_learner(domain, seed, train_confusion, test_data,
                                     algorithm, learner, active_size,
                                     num_trials, serial, args.visualize))
                continue
            # [10 20 25] take first 10 samples from X_train to train the model, 10 samples chosen actively
            # sequentially + evaluate model, 5 samples chosen actively sequentially + evaluate model
            # Could always keep around all the examples and retrain
            # TODO: segfaults when this runs in parallel
            # TODO: may be able to retrain in parallel if I set OPENBLAS_NUM_THREADS
            num_batch = active_sizes[0]
            batch_data, active_data = train_data.partition(num_batch)
            if all_active_data is not None:
                active_data = all_active_data.clone()

            #batch_data.results.extend(failed_results)
            learner, train_confusion = create_learner(
                domain,
                batch_data,
                algorithm,  # alphas,
                query_type=query_type,
                verbose=True)
            learner.validity_learner = validity_learner
            if transfer_learner is not None:
                learner.sim_model = transfer_learner.model
            learner.retrain()
            for active_size in active_sizes:
                num_active = active_size - (learner.nx - len(failed_results))
                print('\nRound: {} | {} | Seed: {} | Size: {} | Active: {}'.
                      format(round_idx, algorithm, seed, active_size,
                             num_active))
                if algorithm.name in CONTINUOUS_ACTIVE_GP:
                    active_learning(learner, num_active, visualize=visualize)
                    #active_learning(learner, num_active, discrete_feature=True, random_feature=False)
                    #active_learning_discrete(learner, active_data, num_active, random_feature=False)
                elif algorithm.name in BATCH_ACTIVE:
                    active_learning_discrete(learner, active_data, num_active)
                    #active_learning(learner, num_active, discrete_feature=True, random_feature=True)
                    #active_learning_discrete(learner, active_data, num_active, random_feature=True)
                #if round_dir is not None:
                #    save_learner(round_dir, learner)
                if args.save:
                    learner.save(data_dir)
                experiments.append(
                    evaluate_learner(domain, seed, train_confusion, test_data,
                                     algorithm, learner, active_size,
                                     num_trials, serial, args.visualize))
                save_experiments(data_path, experiments)

    print(SEPARATOR)
    if experiments:
        save_experiments(data_path, experiments)
        plot_experiments(domain,
                         experiments_name,
                         experiments_dir,
                         experiments,
                         include_none=False)
        print('Experiments: {}'.format(experiments_dir))
    print('Total experiments: {}'.format(len(experiments)))
    print('Total hours: {:.3f}'.format(
        elapsed_time(start_time) / HOURS_TO_SECS))