def main(skill='pour'):
parser = argparse.ArgumentParser()
parser.add_argument('expid', type=int, help='experiment ID')
parser.add_argument(
'beta_lambda',
type=float,
default=0.99,
help='lambda parameter for computing beta from best beta')
args = parser.parse_args()
beta_lambda = args.beta_lambda
expid = args.expid
n_samples = 20
trainsize = 1000
exp_dir = 'sampling_trainsize={}_samples={}_beta_lambdda={}'.format(
trainsize, n_samples, beta_lambda)
if skill is 'pour':
exp_dir = os.path.join('data/pour_19-06-13_00-59-21/', exp_dir)
domain = load_data(['data/pour_19-06-13_00-59-21/trials_n=10000.json'])
elif skill is 'scoop':
exp_dir = os.path.join('data/scoop_19-06-10_20-16-59_top-diameter/',
exp_dir)
domain = load_data(
['data/scoop_19-06-10_20-16-59_top-diameter/trials_n=10000.json'])
data_path = os.path.join(
exp_dir, 'experiments_{}.pk{}'.format(expid, get_python_version()))
if not os.path.exists(data_path):
print('{} does not exist'.format(data_path))
return
data, seed = read_pickle(data_path)
print('sample a new task')
current_wd, trial_wd = start_task()
sim_world, collector, task, feature, evalfunc, saver = sample_task_with_seed(
skill, seed=seed, visualize=True)
samples = data[UNIFORM].samples
scores = data[UNIFORM].scores
good_samples = samples[scores > 0]
scores, plan_results = evaluate_samples(sim_world, collector, task,
feature, domain, good_samples[:5],
evalfunc, saver)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('paths', nargs='*', help='Paths to the data.')
#parser.add_argument('-a', '--active', type=int, default=0, # None
# help='The number of active samples to collect')
parser.add_argument('-l', '--learner', default=None,
help='Path to the learner that should be used')
parser.add_argument('-n', '--num_trials', type=int, default=100,
help='The number of samples to collect')
parser.add_argument('-s', '--save', action='store_true',
help='Whether to save the learners')
parser.add_argument('-r', '--num_rounds', type=int, default=1,
help='The number of rounds to collect')
#parser.add_argument('-t', '--num_train', type=int, default=None,
# help='The size of the training set')
args = parser.parse_args()
# TODO: be careful that paging isn't altering the data
# TODO: don't penalize if the learner identifies that it can't make a good prediction
# TODO: use a different set of randomized parameters for train and test
include_none = False
serial = is_darwin()
#training_sizes = inclusive_range(50, 500, 25)
#training_sizes = inclusive_range(25, 100, 5)
#training_sizes = inclusive_range(25, 100, 5)
training_sizes = inclusive_range(10, 50, 5)
#training_sizes = inclusive_range(100, 1000, 100)
#training_sizes = [20]
#training_sizes = [1500]
#kernels = ['RBF', 'Matern52', 'MLP']
kernels = ['MLP']
#hyperparameters = [None]
#hyperparameters = [True]
hyperparameters = [True, None] # None,
query_type = BEST # BEST | CONFIDENT | REJECTION | ACTIVE # type of query used to evaluate the learner
is_adaptive = False
max_test = 50 #
#alphas = np.linspace(0.0, 0.9, num=5, endpoint=True)
alphas = [0.0, .8, .9, .99]
#alphas = [None] # Use the default (i.e. GP parameters)
use_vars = [True]
binary = False
split = UNIFORM # BALANCED
# Omitting failed labels is okay because they will never be executed
algorithms = []
#algorithms += [(Algorithm(BATCH_GP, kernel=kernel, hyperparameters=hype, use_var=use_var), [num_train])
# for num_train, kernel, hype, use_var in product(training_sizes, kernels, hyperparameters, use_vars)]
algorithms += [(Algorithm(STRADDLE_GP, kernel, hype, use_var), training_sizes)
for kernel, hype, use_var in product(kernels, hyperparameters, use_vars)]
#algorithms += [(Algorithm(rf_model, p_explore=None, use_var=use_var), [num_train])
# for rf_model, num_train, use_var in product(RF_MODELS, training_sizes, use_vars)]
#algorithms += [(Algorithm(nn_model, p_explore=None), [num_train])
# for nn_model, num_train in product(NN_MODELS, training_sizes)]
#algorithms += [(Algorithm(RANDOM), None), (Algorithm(DESIGNED), None)]
print('Algorithms:', algorithms)
print('Split:', split)
trials_per_round = sum(1 if train_sizes is None else
(train_sizes[-1] - train_sizes[0] + len(train_sizes))
for _, train_sizes in algorithms)
num_experiments = args.num_rounds*trials_per_round
date_name = datetime.datetime.now().strftime(DATE_FORMAT)
size_str = '[{},{}]'.format(training_sizes[0], training_sizes[-1])
#size_str = '-'.join(map(str, training_sizes))
experiments_name = '{}_r={}_t={}_n={}'.format(date_name, args.num_rounds, size_str, args.num_trials) #'19-08-09_21-44-58_r=5_t=[10,150]_n=1'#
#experiments_name = 't={}'.format(args.num_rounds)
# TODO: could include OS and username if desired
domain = load_data(args.paths)
print()
print(domain)
X, Y, W = domain.get_data(include_none=include_none)
print('Total number of examples:', len(X))
if binary:
# NN can fit perfectly when binary
# Binary seems to be outperforming w/o
Y = threshold_scores(Y)
max_train = len(X) - max_test #min(max([0] + [active_sizes[0] for _, active_sizes in algorithms
# if active_sizes is not None]), len(X))
#parameters = {
# 'include None': include_none,
# 'binary': binary,
# 'split': split,
#}
print('Name:', experiments_name)
print('Experiments:', num_experiments)
print('Max train:', max_train)
print('Include None:', include_none)
print('Examples: n={}, d={}'.format(*X.shape))
print('Binary:', binary)
print('Estimated hours:', num_experiments * SEC_PER_EXPERIMENT / HOURS_TO_SECS)
user_input('Begin?')
# TODO: residual learning for sim to real transfer
# TODO: can always be conservative and add sim negative examples
# TODO: combine all data to write in one folder
data_dir = os.path.join(DATA_DIRECTORY, domain.name) # EXPERIMENT_DIRECTORY
experiments_dir = os.path.join(data_dir, experiments_name)
mkdir(experiments_dir)
start_time = time.time()
experiments = []
for round_idx in range(args.num_rounds):
round_dir = os.path.join(data_dir, experiments_name, str(round_idx))
mkdir(round_dir)
seed = hash(time.time())
train_test_file = os.path.join(round_dir, 'data.pk3')
if not os.path.exists(train_test_file):
X_train, Y_train, X_test, Y_test = split_data(X, Y, split, max_train)
X_test, Y_test = X_test[:max_test], Y_test[:max_test]
write_pickle(train_test_file, (X_train, Y_train, X_test, Y_test))
else:
X_train, Y_train, X_test, Y_test = read_pickle(train_test_file)
print('Train examples:', X_train.shape)
print('Test examples:', X_test.shape)
# TODO: need to be super careful when running with multiple contexts
for algorithm, active_sizes in algorithms:
# active_sizes = [first #trainingdata selected from X_train, #active exploration + #trainingdata]
print(SEPARATOR)
print('Round: {} | {} | Seed: {} | Sizes: {}'.format(round_idx, algorithm, seed, active_sizes))
# TODO: allow keyboard interrupt
if active_sizes is None:
learner = algorithm.name
active_size = None
train_confusion = None
experiments.append(evaluate_learner(domain, seed, train_confusion, X_test, Y_test, algorithm, learner,
active_size, args.num_trials, alphas,
serial))
else:
# [10 20 25] take first 10 samples from X_train to train the model, 10 samples chosen actively
# sequentially + evaluate model, 5 samples chosen actively sequentially + evaluate model
# Could always keep around all the examples and retrain
# TODO: segfaults when this runs in parallel
# TODO: may be able to retrain in parallel if I set OPENBLAS_NUM_THREADS
learner_prior_nx = 0
'''
if algorithm.hyperparameters:
if domain.skill == 'pour':
learner_file = '/Users/ziw/ltamp_pr2/data/pour_19-06-13_00-59-21/19-08-09_19-30-01_r=10_t=[50,400]_n=1/{}/gp_active_mlp_true_true.pk3'.format(
round_idx)
elif domain.skill == 'scoop':
learner_file = '/Users/ziw/ltamp_pr2/data/scoop_19-06-10_20-16-59_top-diameter/19-08-09_19-34-56_r=10_t=[50,400]_n=1/{}/gp_active_mlp_true_true.pk3'.format(
round_idx)
learner = read_pickle(learner_file)
learner_prior_nx = learner.nx
learner.retrain(newx=X_train[:active_sizes[0]], newy=Y_train[:active_sizes[0], None])
else:
'''
learner, train_confusion = create_learner(domain, X_train, Y_train, split, algorithm,
num_train=active_sizes[0], query_type=query_type,
is_adaptive=is_adaptive)
if algorithm.name == STRADDLE_GP:
X_select, Y_select = X_train[active_sizes[0]:], Y_train[active_sizes[0]:]
for active_size in active_sizes:
num_active = active_size - learner.nx + learner_prior_nx# learner.nx is len(learner.xx)
print('\nRound: {} | {} | Seed: {} | Size: {} | Active: {}'.format(
round_idx, algorithm, seed, active_size, num_active))
if algorithm.name == STRADDLE_GP:
X_select, Y_select = active_learning_discrete(learner, num_active, X_select, Y_select)
#if args.save:
save_learner(round_dir, learner)
experiments.append(evaluate_learner(domain, seed, None, X_test, Y_test,
algorithm, learner,
active_size, args.num_trials, alphas,
serial))
save_experiments(experiments_dir, experiments)
print(SEPARATOR)
if experiments:
save_experiments(experiments_dir, experiments)
plot_experiments(domain, experiments_name, experiments_dir, experiments,
include_none=False)
#include_none=include_none)
print('Experiments:', experiments_dir)
print('Total experiments:', len(experiments))
print('Total hours:', elapsed_time(start_time) / HOURS_TO_SECS)
def main():
"""
./home/demo/catkin_percep/collect_scales.sh (includes scale offset)
Make sure to start the scales with nothing on them (for calibration)
"""
assert (get_python_version() == 2) # ariadne has ROS with python2
parser = argparse.ArgumentParser()
parser.add_argument('-a',
'--active',
action='store_true',
help='Uses active learning queries.')
parser.add_argument('-d',
'--debug',
action='store_true',
help='Disables saving during debugging.')
parser.add_argument(
'-f',
'--fn',
type=str,
default=TRAINING, # DESIGNED | TRAINING
help='The name of or the path to the policy that generates parameters.'
)
parser.add_argument('-m',
'--material',
required=True,
choices=sorted(MATERIALS),
help='The name of the material being used.')
parser.add_argument('-p',
'--problem',
required=True,
choices=sorted(REQUIREMENT_FNS.keys()),
help='The name of the skill to learn.')
parser.add_argument('-s',
'--spoon',
default=None,
choices=SPOONS,
help='The name of the spoon being used.')
parser.add_argument('-r',
'--train',
action='store_true',
help='When enabled, uses the training dataset.')
parser.add_argument(
'-v',
'--visualize_planning',
action='store_true',
help=
'When enabled, visualizes planning rather than the world (for debugging).'
)
args = parser.parse_args()
# TODO: toggle material default based on task
# TODO: label material on the image
assert args.material in MODEL_MASSES
print('Policy:', args.fn)
assert implies(args.problem in ['scoop'], args.spoon is not None)
assert implies(args.active, args.train)
ros_world = ROSWorld(sim_only=False, visualize=not args.visualize_planning)
classes_pub = rospy.Publisher('~collect_classes', String, queue_size=1)
with ros_world:
set_camera_pose(np.array([1.5, -0.5, 1.5]),
target_point=np.array([0.75, 0, 0.75]))
arm, is_open = ACTIVE_ARMS[args.problem]
open_grippers = {arm: is_open}
if args.problem == 'scoop':
ros_world.controller.open_gripper(get_arm_prefix(arm), blocking=True)
ros_world.controller.speak("{:.0f} seconds to attach {}".format(
ATTACH_TIME, format_class(args.spoon)))
rospy.sleep(ATTACH_TIME) # Sleep to have time to set the spoon
move_to_initial_config(ros_world, open_grippers) #get_other_arm
# TODO: cross validation for measuring performance across a few bowls
launch = launch_kinect()
video_time = time.time()
if args.debug:
ros_world.controller.speak("Warning! Data will not be saved.")
time.sleep(1.0)
data_path = None
else:
# TODO: only create directory if examples made
data_path = get_data_path(args.problem, real=True)
# TODO: log camera image after the pour
policy = args.fn
learner_path = None
test_data = None
if isinstance(policy, str) and os.path.isfile(policy):
policy = read_pickle(policy)
assert isinstance(policy, ActiveLearner)
print(policy)
print(policy.algorithm)
#policy.transfer_weight = 0
# print(policy.xx.shape)
# policy.results = policy.results[:-1]
# policy.xx = policy.xx[:-1]
# policy.yy = policy.yy[:-1]
# policy.weights = policy.weights[:-1]
# print(policy.xx.shape)
# write_pickle(args.fn, policy)
# print('Saved', args.fn)
if args.active:
# policy.retrain()
test_domain = load_data(SCOOP_TEST_DATASETS, verbose=False)
test_data = test_domain.create_dataset(include_none=False,
binary=False)
policy.query_type = STRADDLE # VARIANCE
#policy.weights = 0.1*np.ones(policy.yy.shape) # TODO: make this multiplicative
#policy.retrain()
evaluate_confusions(test_data, policy)
else:
policy.query_type = BEST
ensure_dir(LEARNER_DIRECTORY)
date_name = datetime.datetime.now().strftime(DATE_FORMAT)
filename = '{}_{}.pk{}'.format(get_label(policy.algorithm), date_name,
get_python_version())
learner_path = os.path.join(LEARNER_DIRECTORY, filename)
if ACTIVE_FEATURE and args.active:
assert isinstance(policy, ActiveLearner)
generator = create_active_generator(args, policy)
else:
generator = create_random_generator(args)
pair = next(generator)
print('Next pair:', pair)
classes_pub.publish('{},{}'.format(*pair))
for phrase in map(format_class, pair):
ros_world.controller.speak(phrase)
wait_for_user('Press enter to begin')
# TODO: change the name of the directory after additional samples
results = []
num_trials = num_failures = num_scored = 0
while True:
start_time = elapsed_time(video_time)
result = run_loop(args, ros_world, policy)
print('Result:', str_from_object(result))
print('{}\nTrials: {} | Successes: {} | Failures: {} | Time: {:.3f}'.
format(SEPARATOR, num_trials, len(results), num_failures,
elapsed_time(video_time)))
num_trials += 1
if result is None: # TODO: result['execution']
num_failures += 1
print('Error! Trial resulted in an exception')
move_to_initial_config(ros_world, open_grippers)
continue
end_time = elapsed_time(video_time)
print('Elapsed time:', end_time - start_time)
# TODO: record the type of failure (planning, execution, etc...)
scored = result['score'] is not None
num_scored += scored
# TODO: print the score
if isinstance(policy, ActiveLearner) and args.active: # and scored:
update_learner(policy, learner_path, result)
evaluate_confusions(test_data, policy)
# TODO: how to handle failures that require bad annotations?
pair = next(generator)
print('Next pair:', pair)
classes_pub.publish('{},{}'.format(*pair))
for phrase in map(format_class, pair):
ros_world.controller.speak(phrase)
annotation = wait_for_user(
'Enter annotation and press enter to continue: ')
result.update({
# TODO: record the query_type
'policy': args.fn,
'active_feature': ACTIVE_FEATURE,
'trial': num_trials,
'start_time': start_time,
'end_time': end_time,
'annotation': annotation,
})
results.append(result)
if data_path is not None:
write_results(data_path, results)
#if annotation in ['q', 'quit']: # TODO: Ctrl-C to quit
# break
ros_world.controller.speak("Finished")
if launch is not None:
launch.shutdown()
print('Total time:', elapsed_time(video_time))
def main():
parser = argparse.ArgumentParser()
parser.add_argument('trainsize',
default=2000,
type=int,
help='training set size')
parser.add_argument('expid', default=1, type=int, help='experiment ID')
parser.add_argument(
'beta_lambda',
type=float,
default=0.9,
help='lambda parameter for computing beta from best beta')
parser.add_argument('sample_strategy_id', default=1, type=int) # 1, 2, 3
parser.add_argument(
'paths',
default=[os.path.join(get_data_dir('pour'), 'trials_n=10000.json')],
nargs='*',
help='Paths to the data.')
parser.add_argument('-u',
'--use_hyper',
action='store_true',
help='When enabled, use existing hyper parameter.')
parser.add_argument('-o',
'--use_obstacle',
action='store_true',
help='When enabled, no obstacle is used in the scene.')
args = parser.parse_args()
beta_lambda = args.beta_lambda
sample_strategy = SAMPLE_STRATEGIES[args.sample_strategy_id]
global SEED
SEED = args.expid
set_seed(SEED)
n_train_tasks = 50
n_test_tasks = 20
train_tasks_seeds = get_seeds(n_train_tasks)
test_tasks_seeds = get_seeds(n_test_tasks)
print('loading data')
domain = load_data(args.paths)
data = domain.create_dataset(include_none=True, binary=False)
data.shuffle()
X, Y, W = data.get_data()
print('finished obtaining x y data')
n_train = args.trainsize
X = X[:n_train]
Y = Y[:n_train]
print('initializing ActiveGP with #datapoints = {}'.format(len(X)))
hype = None
if 'pour' in args.paths[0] and args.use_hyper:
hype = POUR_MLP_HYPERPARAM_3000
elif 'scoop' in args.paths[0] and args.use_hyper:
hype = SCOOP_MLP_HYPERPARAM_3000
learner = ActiveGP(domain,
initx=X,
inity=Y,
hyperparameters=hype,
sample_time_limit=60,
beta_lambda=beta_lambda)
learner.retrain(num_restarts=10)
exp_file = 'tasklengthscale_sampling_trainsize={}_beta_lambdda={}_strategy_{}_obs_{}_expid_{}.pk3'.format(
len(X), beta_lambda, args.sample_strategy_id, int(args.use_obstacle),
args.expid)
exp_dirname = os.path.dirname(args.paths[0])
if args.use_hyper:
exp_dirname = os.path.join(exp_dirname, 'default_hyper/')
mkdir(exp_dirname)
exp_file = os.path.join(exp_dirname, exp_file)
print('saving results to ', exp_file)
results = []
if sample_strategy != DIVERSELK:
n_train_tasks = 0
# no need to train
prev_tasklengthscale = None
for i in range(n_train_tasks + 1):
test_results = []
print('task_lengthscal = {}'.format(learner.task_lengthscale))
print('================BEGIN TESTING==============')
if prev_tasklengthscale is not None and (
learner.task_lengthscale == prev_tasklengthscale).all():
test_results = results[-1][1]
else:
for j in range(n_test_tasks):
if sample_strategy == DIVERSELK:
test_sample_strategy = DIVERSE
else:
test_sample_strategy = sample_strategy
seed = test_tasks_seeds[j]
test_result = eval_task_with_seed(
domain,
seed,
learner,
sample_strategy=test_sample_strategy,
obstacle=args.use_obstacle)
test_results.append(test_result)
prev_tasklengthscale = learner.task_lengthscale.copy()
if i != n_train_tasks:
seed = train_tasks_seeds[i]
train_result = eval_task_with_seed(domain,
seed,
learner,
sample_strategy=sample_strategy,
obstacle=args.use_obstacle)
else:
train_result = None
results.append((train_result, test_results))
write_pickle(exp_file,
(results, SEED, train_tasks_seeds, test_tasks_seeds))
def main():
parser = argparse.ArgumentParser()
parser.add_argument('trainsize',
default=2000,
type=int,
help='training set size')
parser.add_argument('expid', default=0, type=int, help='experiment ID')
parser.add_argument(
'paths',
default=[os.path.join(get_data_dir('pour'), 'trials_n=10000.json')],
nargs='*',
help='Paths to the data.')
parser.add_argument(
'beta_lambda',
default=0.99,
type=float,
help='lambda parameter for computing beta from best beta')
parser.add_argument('-n',
'--include_none',
action='store_true',
help='include none in the dataset')
args = parser.parse_args()
beta_lambda = args.beta_lambda
n_samples = 20
seed = hash((os.getpid(), args.expid))
set_seed(seed)
print('loading data')
domain = load_data(args.paths)
data = domain.create_dataset(include_none=args.include_none, binary=False)
data.shuffle()
X, Y, W = data.get_data()
print('finished obtaining x y data')
n_train = args.trainsize # len(X)
X = X[:n_train]
Y = Y[:n_train]
print('initializing ActiveGP')
if 'pour' in args.paths[0]:
hype = POUR_MLP_HYPERPARAM_3000
elif 'scoop' in args.paths[0]:
hype = SCOOP_MLP_HYPERPARAM_3000
learner = ActiveGP(domain,
initx=X,
inity=Y,
hyperparameters=hype,
sample_time_limit=60,
beta_lambda=beta_lambda)
learner.retrain(num_restarts=10)
print('sample a new task')
current_wd, trial_wd = start_task()
sim_world, collector, task, feature, evalfunc, saver = sample_task_with_seed(
domain.skill, seed=seed, visualize=False)
context = domain.context_from_feature(feature)
# date_name = datetime.datetime.now().strftime('%y-%m-%d_%H-%M-%S')
exp_dir = 'sampling_trainsize={}_samples={}_beta_lambdda={}_includenone_{}'.format(
len(X), n_samples, beta_lambda, int(args.include_none))
exp_dir = os.path.join(os.path.dirname(args.paths[0]), exp_dir)
results = {}
SAMPLE_STRATEGIES.remove(DIVERSELK)
for sample_strategy in SAMPLE_STRATEGIES:
results[sample_strategy] = get_sample_result(sample_strategy, learner,
context, sim_world,
collector, task, feature,
evalfunc, saver,
n_samples)
complete_task(sim_world, current_wd, trial_wd)
save_experiments(exp_dir, args.expid, (results, seed))
for ss in results:
print(ss, results[ss].precision, results[ss].diversity,
results[ss].diversity_5, results[ss].sample_time)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('paths', nargs='*', help='Paths to the data.')
#parser.add_argument('-a', '--active', type=int, default=0, # None
# help='The number of active samples to collect')
parser.add_argument(
'-d',
'--deterministic',
action='store_true',
help='Whether to deterministically create training splits')
parser.add_argument('-n',
'--num_trials',
type=int,
default=-1,
help='The number of samples to collect')
parser.add_argument('-s',
'--save',
action='store_true',
help='Whether to save the learners')
parser.add_argument('-r',
'--num_rounds',
type=int,
default=1,
help='The number of rounds to collect')
parser.add_argument('-t',
'--test',
action='store_true',
help='Whether to save the data')
parser.add_argument('-v',
'--visualize',
action='store_true',
help='When enabled, visualizes execution.')
args = parser.parse_args()
# TODO: be careful that paging isn't altering the data
# TODO: use a different set of randomized parameters for train and test
serial = is_darwin()
visualize = serial and args.visualize
assert implies(visualize, serial)
num_trials = get_max_cores(
serial) if args.num_trials < 0 else args.num_trials
##################################################
#train_sizes = inclusive_range(50, 200, 10) # Best
#train_sizes = inclusive_range(50, 400, 10) # F1
#train_sizes = inclusive_range(25, 400, 25)
#train_sizes = inclusive_range(50, 100, 5) # Real
#train_sizes = inclusive_range(100, 200, 5)
#train_sizes = inclusive_range(10, 250, 5)
#train_sizes = inclusive_range(35, 70, 5)
#train_sizes = inclusive_range(5, 50, 5)
#train_sizes = inclusive_range(40, 80, 5)
#train_sizes = inclusive_range(100, 1000, 100)
#train_sizes = [50]
#train_sizes = [250]
train_sizes = [1000]
#train_sizes = [327] # train + test
#train_sizes = inclusive_range(5, 150, 25)
#train_sizes = [100]
#kernels = ['RBF', 'Matern52', 'MLP']
kernels = ['MLP']
hyperparams = [None]
#hyperparams = [True]
#hyperparams = [None, True]
query_type = BEST # BEST | CONFIDENT | REJECTION | ACTIVE # type of query used to evaluate the learner
include_none = False
binary = False
# 0 => no transfer
# 1 => mean transfer
# 2 => kernel transfer
# 3 => both transfer
transfer_weights = [None]
#transfer_weights = list(range(4))
#transfer_weights = [0, 1]
#transfer_weights = [3]
#transfer_weights = np.around(np.linspace(0.0, 1.0, num=1+5, endpoint=True), decimals=3) # max 10 colors
#transfer_weights = list(range(1, 1+3))
#split = UNIFORM # BALANCED
#print('Split:', split)
#parameters = {
# 'include None': include_none,
# 'binary': binary,
# 'split': split,
#}
# Omitting failed labels is okay because they will never be executed
algorithms = []
#algorithms += [(Algorithm(nn_model, label='NN'), [num])
# for nn_model, num in product(NN_MODELS, train_sizes)]
#algorithms += [(Algorithm(RANDOM), None), (Algorithm(DESIGNED), None)]
#algorithms += [(Algorithm(RF_CLASSIFIER, variance=False, transfer_weight=tw, label='RF'), [num])
# for num, tw in product(train_sizes, [None])] # transfer_weights
#algorithms += [(Algorithm(RF_REGRESSOR, variance=False, transfer_weight=tw, label='RF'), [num])
# for num, tw in product(train_sizes, [None])] # transfer_weights
#algorithms += [(Algorithm(BATCH_RF, variance=True, transfer_weight=tw, label='RF'), [num])
# for num, tw in product(train_sizes, [None])] # transfer_weights
#algorithms += [(Algorithm(BATCH_MAXVAR_RF, variance=True, transfer_weight=tw), train_sizes)
# for tw in product(use_vars, [None])] # transfer_weights
#algorithms += [(Algorithm(BATCH_STRADDLE_RF, variance=True, transfer_weight=tw), train_sizes)
# for tw, in product([None])] # transfer_weights
use_vars = [True]
# STRADDLE is better than MAXVAR when the learner has a good estimate of uncertainty
algorithms += [
(Algorithm(BATCH_GP, kernel, hype, use_var, tw,
label='GP'), [num]) # label='GP-{}'.format(kernel)
for num, kernel, hype, use_var, tw in product(
train_sizes, kernels, hyperparams, use_vars, transfer_weights)
]
#algorithms += [(Algorithm(BATCH_MAXVAR_GP, kernel, hype, True, tw, label='GP-Var'), train_sizes)
# for kernel, hype, tw in product(kernels, hyperparams, transfer_weights)]
#algorithms += [(Algorithm(BATCH_STRADDLE_GP, kernel, hype, True, tw, label='GP-LSE'), train_sizes)
# for kernel, hype, tw in product(kernels, hyperparams, transfer_weights)] # default active
#algorithms += [(Algorithm(BATCH_STRADDLE_GP, kernel, hype, True, tw, label='GP-LSE2'), train_sizes)
# for kernel, hype, tw in product(kernels, hyperparams, transfer_weights)] # active control only
# algorithms += [(Algorithm(MAXVAR_GP, kernel, hype, use_var), train_sizes)
# for kernel, hype, use_var in product(kernels, hyperparams, use_vars)]
#algorithms += [(Algorithm(STRADDLE_GP, kernel, hype, use_var, tw), train_sizes)
# for kernel, hype, use_var, tw in product(kernels, hyperparams, use_vars, transfer_weights)]
#batch_sizes = inclusive_range(train_sizes[0], 90, 10)
#step_size = 10 # TODO: extract from train_sizes
#final_size = train_sizes[-1]
# Previously didn't have use_var=True
# algorithms += [(Algorithm(BATCH_STRADDLE_GP, kernel, hyperparameters=batch_size, variance=True, transfer_weight=tw),
# inclusive_range(batch_size, final_size, step_size))
# for kernel, tw, batch_size in product(kernels, transfer_weights, batch_sizes)]
# algorithms += [(Algorithm(BATCH_STRADDLE_RF, hyperparameters=batch_size, variance=True, transfer_weight=tw),
# inclusive_range(batch_size, final_size, step_size))
# for tw, batch_size in product(transfer_weights, batch_sizes)]
print('Algorithms:', algorithms)
##################################################
real_world = not args.paths
transfer_domain = load_data(TRANSFER_DATASETS, verbose=False)
transfer_algorithm = None
if real_world and transfer_weights != [None]:
#assert transfer_weights[0] is not None
transfer_data = transfer_domain.create_dataset(
include_none=include_none, binary=binary)
transfer_algorithm = Algorithm(BATCH_GP,
kernel=kernels[0],
variance=use_vars[0])
validity_learner = None
#validity_learner = create_validity_classifier(transfer_domain)
##################################################
train_paths = args.paths
if real_world:
train_paths = SCOOP_TRAIN_DATASETS # TRAIN_DATASETS
#train_paths = TRANSFER_DATASETS
#train_paths = TRAIN_DATASETS + TRANSFER_DATASETS # Train before transfer
#scale_paths = TRAIN_DATASETS + TEST_DATASETS
scale_paths = None
print(SEPARATOR)
print('Train paths:', train_paths)
domain = load_data(train_paths)
print()
print(domain)
all_data = domain.create_dataset(include_none=include_none,
binary=binary,
scale_paths=scale_paths)
#all_data.results = all_data.results[:1000]
num_failed = 0
#num_failed = 100
failed_domain = transfer_domain if real_world else domain
failed_results = randomize(
result for result in failed_domain.results
if not result.get('success', False))[:num_failed]
#failed_data = Dataset(domain, failed_results, **all_data.kwargs)
test_paths = SCOOP_TEST_DATASETS # TEST_DATASETS | SCOOP_TEST_DATASETS
#test_paths = None
if real_world and not (set(train_paths) & set(test_paths)):
#assert not set(train_paths) & set(test_paths)
#max_test = 0
test_data = load_data(test_paths).create_dataset(
include_none=False, binary=binary, scale_paths=scale_paths)
else:
#assert scale_paths is None # TODO: max_train will be too small otherwise
test_paths = test_data = None
print(SEPARATOR)
print('Test paths:', test_paths)
all_active_data = None
#if real_world:
# all_active_data = load_data(ACTIVE_DATASETS).create_dataset(include_none=True, binary=binary, scale_paths=scale_paths)
# TODO: could include OS and username if desired
date_name = datetime.datetime.now().strftime(DATE_FORMAT)
size_str = '[{},{}]'.format(train_sizes[0], train_sizes[-1])
#size_str = '-'.join(map(str, train_sizes))
experiments_name = '{}_r={}_t={}_n={}'.format(date_name, args.num_rounds,
size_str, num_trials)
trials_per_round = sum(
1 if train_sizes is None else (train_sizes[-1] - train_sizes[0] +
len(train_sizes))
for _, train_sizes in algorithms)
num_experiments = args.num_rounds * trials_per_round
max_train = min(
max([0] + [
active_sizes[0]
for _, active_sizes in algorithms if active_sizes is not None
]), len(all_data))
max_test = min(len(all_data) - max_train, 1000)
##################################################
# #features = ['bowl_height']
# features = ['spoon_height']
# #features = ['bowl_height', 'spoon_height']
# X, Y, _ = all_data.get_data()
# #indices = [domain.inputs.index(feature) for feature in features]
# #X = X[:,indices]
# X = [[result[FEATURE][name] for name in features] for result in all_data.results]
# from sklearn.linear_model import LinearRegression
# model = LinearRegression(fit_intercept=True, normalize=False)
# model.fit(X, Y)
# #print(model.get_params())
# print(model.coef_.tolist(), model.intercept_)
# print(model.score(X, Y))
#data_dir = os.path.join(DATA_DIRECTORY, domain.name) # EXPERIMENT_DIRECTORY
data_dir = os.path.abspath(os.path.join(domain.name, os.path.pardir))
experiments_dir, data_path = None, None
if not args.test or not serial:
experiments_dir = os.path.join(data_dir, experiments_name)
data_path = os.path.join(
experiments_dir, 'experiments.pk{}'.format(get_python_version()))
##################################################
print(SEPARATOR)
print('Name:', experiments_name)
print('Experiments:', num_experiments)
print('Experiment dir:', experiments_dir)
print('Data path:', data_path)
print('Examples:', len(all_data))
print('Valid:',
sum(result.get('valid', True) for result in all_data.results))
print('Success:',
sum(result.get('success', False) for result in all_data.results))
print(
'Scored:',
sum(
result.get('score', None) is not None
for result in all_data.results))
print('Max train:', max_train)
print('Max test:', max_test)
print('Include None:', include_none)
print('Examples: n={}, d={}'.format(len(all_data), domain.dx))
print('Binary:', binary)
print('Serial:', serial)
print('Estimated hours: {:.3f}'.format(num_experiments *
SEC_PER_EXPERIMENT / HOURS_TO_SECS))
user_input('Begin?')
##################################################
experiments = []
if experiments_dir is not None:
mkdir(experiments_dir)
# if os.path.exists(data_path):
# experiments.extend(read_pickle(data_path))
# TODO: embed in a KeyboardInterrupt to allow early termination
start_time = time.time()
for round_idx in range(args.num_rounds):
seed = round_idx if args.deterministic else hash(
time.time()) # vs just time.time()?
random.seed(seed)
all_data.shuffle()
if test_paths is None: # cannot use test_data
#test_data, train_data = split_data(all_data, max_test)
train_data = test_data = all_data # Training performance
else:
train_data = all_data
transfer_learner = None
if transfer_algorithm is not None:
round_data, _ = transfer_data.partition(index=1000)
transfer_learner, _ = create_learner(transfer_domain,
round_data,
transfer_algorithm,
verbose=True)
transfer_learner.retrain()
print(SEPARATOR)
print('Round {} | Train examples: {} | Test examples: {}'.format(
round_idx, len(train_data), len(test_data)))
for algorithm, active_sizes in algorithms:
# active_sizes = [first #trainingdata selected from X_train, #active exploration + #trainingdata]
print(SEPARATOR)
print('Round: {} | {} | Seed: {} | Sizes: {}'.format(
round_idx, algorithm, seed, active_sizes))
# TODO: allow keyboard interrupt
if active_sizes is None:
learner = algorithm.name
active_size = train_confusion = None
experiments.append(
evaluate_learner(domain, seed, train_confusion, test_data,
algorithm, learner, active_size,
num_trials, serial, args.visualize))
continue
# [10 20 25] take first 10 samples from X_train to train the model, 10 samples chosen actively
# sequentially + evaluate model, 5 samples chosen actively sequentially + evaluate model
# Could always keep around all the examples and retrain
# TODO: segfaults when this runs in parallel
# TODO: may be able to retrain in parallel if I set OPENBLAS_NUM_THREADS
num_batch = active_sizes[0]
batch_data, active_data = train_data.partition(num_batch)
if all_active_data is not None:
active_data = all_active_data.clone()
#batch_data.results.extend(failed_results)
learner, train_confusion = create_learner(
domain,
batch_data,
algorithm, # alphas,
query_type=query_type,
verbose=True)
learner.validity_learner = validity_learner
if transfer_learner is not None:
learner.sim_model = transfer_learner.model
learner.retrain()
for active_size in active_sizes:
num_active = active_size - (learner.nx - len(failed_results))
print('\nRound: {} | {} | Seed: {} | Size: {} | Active: {}'.
format(round_idx, algorithm, seed, active_size,
num_active))
if algorithm.name in CONTINUOUS_ACTIVE_GP:
active_learning(learner, num_active, visualize=visualize)
#active_learning(learner, num_active, discrete_feature=True, random_feature=False)
#active_learning_discrete(learner, active_data, num_active, random_feature=False)
elif algorithm.name in BATCH_ACTIVE:
active_learning_discrete(learner, active_data, num_active)
#active_learning(learner, num_active, discrete_feature=True, random_feature=True)
#active_learning_discrete(learner, active_data, num_active, random_feature=True)
#if round_dir is not None:
# save_learner(round_dir, learner)
if args.save:
learner.save(data_dir)
experiments.append(
evaluate_learner(domain, seed, train_confusion, test_data,
algorithm, learner, active_size,
num_trials, serial, args.visualize))
save_experiments(data_path, experiments)
print(SEPARATOR)
if experiments:
save_experiments(data_path, experiments)
plot_experiments(domain,
experiments_name,
experiments_dir,
experiments,
include_none=False)
print('Experiments: {}'.format(experiments_dir))
print('Total experiments: {}'.format(len(experiments)))
print('Total hours: {:.3f}'.format(
elapsed_time(start_time) / HOURS_TO_SECS))