def main():
# collect data in parallel, parameters are generated uniformly randomly in a range
# data stored to pour_date/trails_n=10000.json
# TODO: ulimit settings
# https://ss64.com/bash/ulimit.html
# https://stackoverflow.com/questions/938733/total-memory-used-by-python-process
# import psutil
# TODO: resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
assert (get_python_version() == 3)
parser = argparse.ArgumentParser()
parser.add_argument('-f',
'--fn',
default=TRAINING,
help='The parameter function to use.')
parser.add_argument('-n',
'--num',
type=int,
default=10000,
help='The number of samples to collect.')
parser.add_argument('-p',
'--problem',
required=True,
choices=sorted(SKILL_COLLECTORS.keys()),
help='The name of the skill to learn.')
parser.add_argument('-t',
'--time',
type=int,
default=1 * 60,
help='The max planning runtime for each trial.')
parser.add_argument('-v',
'--visualize',
action='store_true',
help='When enabled, visualizes execution.')
args = parser.parse_args()
serial = is_darwin()
assert implies(args.visualize, serial)
trials = get_trials(args.problem,
args.fn,
args.num,
max_time=args.time,
valid=True,
visualize=args.visualize,
verbose=serial)
data_path = None if serial else get_data_path(args.problem, trials)
num_cores = get_num_cores(trials, serial)
user_input('Begin?')
# TODO: store the generating distribution for samples and objects?
print(SEPARATOR)
results = run_trials(trials, data_path, num_cores=num_cores)
def get_data_path(skill_name, trials=[], real=False):
print('Skill:', skill_name)
if is_darwin():
return None
# (system, node, release, version, machine, processor)
# system_name = platform.uname()[0].lower()
# computer_name = platform.uname()[1]
# computer_name = socket.gethostname() # TODO: changes when connected to wifi
# What is really want is the name in PS1
# user_name = getpass.getuser()
prefix = '{}_'.format(REAL_PREFIX) if real else ''
date_name = datetime.datetime.now().strftime(DATE_FORMAT)
directory = os.path.join(DATA_DIRECTORY,
'{}{}_{}/'.format(prefix, skill_name, date_name))
ensure_dir(directory)
suffix = '' if real else '_n={}'.format(len(trials))
data_path = os.path.join(directory, 'trials{}'.format(suffix))
print('Data path:', data_path)
# print('System:', system_name)
# print('Username:', user_name)
return data_path
def main():
parser = argparse.ArgumentParser()
parser.add_argument('paths', nargs='*', help='Paths to the data.')
#parser.add_argument('-a', '--active', type=int, default=0, # None
# help='The number of active samples to collect')
parser.add_argument('-l', '--learner', default=None,
help='Path to the learner that should be used')
parser.add_argument('-n', '--num_trials', type=int, default=100,
help='The number of samples to collect')
parser.add_argument('-s', '--save', action='store_true',
help='Whether to save the learners')
parser.add_argument('-r', '--num_rounds', type=int, default=1,
help='The number of rounds to collect')
#parser.add_argument('-t', '--num_train', type=int, default=None,
# help='The size of the training set')
args = parser.parse_args()
# TODO: be careful that paging isn't altering the data
# TODO: don't penalize if the learner identifies that it can't make a good prediction
# TODO: use a different set of randomized parameters for train and test
include_none = False
serial = is_darwin()
#training_sizes = inclusive_range(50, 500, 25)
#training_sizes = inclusive_range(25, 100, 5)
#training_sizes = inclusive_range(25, 100, 5)
training_sizes = inclusive_range(10, 50, 5)
#training_sizes = inclusive_range(100, 1000, 100)
#training_sizes = [20]
#training_sizes = [1500]
#kernels = ['RBF', 'Matern52', 'MLP']
kernels = ['MLP']
#hyperparameters = [None]
#hyperparameters = [True]
hyperparameters = [True, None] # None,
query_type = BEST # BEST | CONFIDENT | REJECTION | ACTIVE # type of query used to evaluate the learner
is_adaptive = False
max_test = 50 #
#alphas = np.linspace(0.0, 0.9, num=5, endpoint=True)
alphas = [0.0, .8, .9, .99]
#alphas = [None] # Use the default (i.e. GP parameters)
use_vars = [True]
binary = False
split = UNIFORM # BALANCED
# Omitting failed labels is okay because they will never be executed
algorithms = []
#algorithms += [(Algorithm(BATCH_GP, kernel=kernel, hyperparameters=hype, use_var=use_var), [num_train])
# for num_train, kernel, hype, use_var in product(training_sizes, kernels, hyperparameters, use_vars)]
algorithms += [(Algorithm(STRADDLE_GP, kernel, hype, use_var), training_sizes)
for kernel, hype, use_var in product(kernels, hyperparameters, use_vars)]
#algorithms += [(Algorithm(rf_model, p_explore=None, use_var=use_var), [num_train])
# for rf_model, num_train, use_var in product(RF_MODELS, training_sizes, use_vars)]
#algorithms += [(Algorithm(nn_model, p_explore=None), [num_train])
# for nn_model, num_train in product(NN_MODELS, training_sizes)]
#algorithms += [(Algorithm(RANDOM), None), (Algorithm(DESIGNED), None)]
print('Algorithms:', algorithms)
print('Split:', split)
trials_per_round = sum(1 if train_sizes is None else
(train_sizes[-1] - train_sizes[0] + len(train_sizes))
for _, train_sizes in algorithms)
num_experiments = args.num_rounds*trials_per_round
date_name = datetime.datetime.now().strftime(DATE_FORMAT)
size_str = '[{},{}]'.format(training_sizes[0], training_sizes[-1])
#size_str = '-'.join(map(str, training_sizes))
experiments_name = '{}_r={}_t={}_n={}'.format(date_name, args.num_rounds, size_str, args.num_trials) #'19-08-09_21-44-58_r=5_t=[10,150]_n=1'#
#experiments_name = 't={}'.format(args.num_rounds)
# TODO: could include OS and username if desired
domain = load_data(args.paths)
print()
print(domain)
X, Y, W = domain.get_data(include_none=include_none)
print('Total number of examples:', len(X))
if binary:
# NN can fit perfectly when binary
# Binary seems to be outperforming w/o
Y = threshold_scores(Y)
max_train = len(X) - max_test #min(max([0] + [active_sizes[0] for _, active_sizes in algorithms
# if active_sizes is not None]), len(X))
#parameters = {
# 'include None': include_none,
# 'binary': binary,
# 'split': split,
#}
print('Name:', experiments_name)
print('Experiments:', num_experiments)
print('Max train:', max_train)
print('Include None:', include_none)
print('Examples: n={}, d={}'.format(*X.shape))
print('Binary:', binary)
print('Estimated hours:', num_experiments * SEC_PER_EXPERIMENT / HOURS_TO_SECS)
user_input('Begin?')
# TODO: residual learning for sim to real transfer
# TODO: can always be conservative and add sim negative examples
# TODO: combine all data to write in one folder
data_dir = os.path.join(DATA_DIRECTORY, domain.name) # EXPERIMENT_DIRECTORY
experiments_dir = os.path.join(data_dir, experiments_name)
mkdir(experiments_dir)
start_time = time.time()
experiments = []
for round_idx in range(args.num_rounds):
round_dir = os.path.join(data_dir, experiments_name, str(round_idx))
mkdir(round_dir)
seed = hash(time.time())
train_test_file = os.path.join(round_dir, 'data.pk3')
if not os.path.exists(train_test_file):
X_train, Y_train, X_test, Y_test = split_data(X, Y, split, max_train)
X_test, Y_test = X_test[:max_test], Y_test[:max_test]
write_pickle(train_test_file, (X_train, Y_train, X_test, Y_test))
else:
X_train, Y_train, X_test, Y_test = read_pickle(train_test_file)
print('Train examples:', X_train.shape)
print('Test examples:', X_test.shape)
# TODO: need to be super careful when running with multiple contexts
for algorithm, active_sizes in algorithms:
# active_sizes = [first #trainingdata selected from X_train, #active exploration + #trainingdata]
print(SEPARATOR)
print('Round: {} | {} | Seed: {} | Sizes: {}'.format(round_idx, algorithm, seed, active_sizes))
# TODO: allow keyboard interrupt
if active_sizes is None:
learner = algorithm.name
active_size = None
train_confusion = None
experiments.append(evaluate_learner(domain, seed, train_confusion, X_test, Y_test, algorithm, learner,
active_size, args.num_trials, alphas,
serial))
else:
# [10 20 25] take first 10 samples from X_train to train the model, 10 samples chosen actively
# sequentially + evaluate model, 5 samples chosen actively sequentially + evaluate model
# Could always keep around all the examples and retrain
# TODO: segfaults when this runs in parallel
# TODO: may be able to retrain in parallel if I set OPENBLAS_NUM_THREADS
learner_prior_nx = 0
'''
if algorithm.hyperparameters:
if domain.skill == 'pour':
learner_file = '/Users/ziw/ltamp_pr2/data/pour_19-06-13_00-59-21/19-08-09_19-30-01_r=10_t=[50,400]_n=1/{}/gp_active_mlp_true_true.pk3'.format(
round_idx)
elif domain.skill == 'scoop':
learner_file = '/Users/ziw/ltamp_pr2/data/scoop_19-06-10_20-16-59_top-diameter/19-08-09_19-34-56_r=10_t=[50,400]_n=1/{}/gp_active_mlp_true_true.pk3'.format(
round_idx)
learner = read_pickle(learner_file)
learner_prior_nx = learner.nx
learner.retrain(newx=X_train[:active_sizes[0]], newy=Y_train[:active_sizes[0], None])
else:
'''
learner, train_confusion = create_learner(domain, X_train, Y_train, split, algorithm,
num_train=active_sizes[0], query_type=query_type,
is_adaptive=is_adaptive)
if algorithm.name == STRADDLE_GP:
X_select, Y_select = X_train[active_sizes[0]:], Y_train[active_sizes[0]:]
for active_size in active_sizes:
num_active = active_size - learner.nx + learner_prior_nx# learner.nx is len(learner.xx)
print('\nRound: {} | {} | Seed: {} | Size: {} | Active: {}'.format(
round_idx, algorithm, seed, active_size, num_active))
if algorithm.name == STRADDLE_GP:
X_select, Y_select = active_learning_discrete(learner, num_active, X_select, Y_select)
#if args.save:
save_learner(round_dir, learner)
experiments.append(evaluate_learner(domain, seed, None, X_test, Y_test,
algorithm, learner,
active_size, args.num_trials, alphas,
serial))
save_experiments(experiments_dir, experiments)
print(SEPARATOR)
if experiments:
save_experiments(experiments_dir, experiments)
plot_experiments(domain, experiments_name, experiments_dir, experiments,
include_none=False)
#include_none=include_none)
print('Experiments:', experiments_dir)
print('Total experiments:', len(experiments))
print('Total hours:', elapsed_time(start_time) / HOURS_TO_SECS)
from sklearn.metrics import mean_squared_error, confusion_matrix, precision_score, recall_score
sys.path.extend([
os.path.join(
os.getcwd(),
'pddlstream'), # Important to use absolute path when doing chdir
os.path.join(os.getcwd(), 'ss-pybullet'),
])
from collections import namedtuple, OrderedDict, defaultdict
from pybullet_tools.utils import is_darwin
from scipy.stats import rankdata
#has_gui = sys.stdin.isatty()
#has_gui = 'DISPLAY' in os.environ
has_gui = is_darwin()
#print('GUI:', has_gui)
#if not has_gui:
# import matplotlib
# matplotlib.use('Agg')
from learn_tools.active_nn import REGRESSOR, CLASSIFIER, NN
from learn_tools.active_learner import score_prediction
from learn_tools.learnable_skill import LearnableSkill, read_data, is_real, get_skill
from learn_tools.statistics import compare_distributions
from learn_tools.learner import plot_learning_curve, LATENT, SCORE, FEATURE, PARAMETER, threshold_score, THRESHOLD, \
estimate_gaussian, SEPARATOR, SUCCESS, FAILURE, DYNAMICS, threshold_scores, SKILL
from pddlstream.utils import mkdir, find_unique
from pybullet_tools.utils import is_remote, safe_zip, SEPARATOR
Algorithm = namedtuple('Algorithm', [
def train_parallel(args, n=1):
from extrusion.run import plan_extrusion
assert SKIP_PERCENTAGE == 0
initial_time = time.time()
problems = sorted(set(enumerate_problems()) - set(EXCLUDE))
#problems = ['four-frame']
#problems = ['simple_frame', 'topopt-101_tiny', 'topopt-100_S1_03-14-2019_w_layer']
algorithms = list(ALGORITHMS)
if args.disable:
for algorithm in LOOKAHEAD_ALGORITHMS:
if algorithm in algorithms:
algorithms.remove(algorithm)
#algorithms = ['regression']
heuristics = HEURISTICS
#heuristics = DISTANCE_HEURISTICS + COST_HEURISTICS
seeds = list(range(args.num))
if n is None:
n = len(seeds)
groups = list(chunks(seeds, n=n))
print('Chunks: {}'.format(len(groups)))
print('Problems ({}): {}'.format(len(problems), problems))
#problems = [path for path in problems if 'simple_frame' in path]
print('Algorithms ({}): {}'.format(len(algorithms), algorithms))
print('Heuristics ({}): {}'.format(len(heuristics), heuristics))
jobs = [[
Configuration(seed, problem, algorithm, heuristic, args.max_time,
args.cfree, args.disable, args.stiffness, args.motions,
args.ee_only) for seed, algorithm, heuristic in product(
group, algorithms, heuristics)
] for problem, group in product(problems, groups)]
# TODO: separate out the algorithms again
# TODO: print the size per job
print('Jobs: {}'.format(len(jobs)))
serial = is_darwin()
available_cores = cpu_count()
num_cores = max(1, min(1 if serial else available_cores - 4, len(jobs)))
print('Max Cores:', available_cores)
print('Serial:', serial)
print('Using Cores:', num_cores)
date = datetime.datetime.now().strftime(DATE_FORMAT)
filename = '{}.pk{}'.format(date, get_python_version())
path = os.path.join(EXPERIMENTS_DIR, filename)
print('Data path:', path)
user_input('Begin?')
start_time = time.time()
timeouts = 0
pool = Pool(processes=num_cores) # , initializer=mute)
generator = pool.imap_unordered(plan_extrusion, jobs, chunksize=1)
results = []
while True:
# TODO: randomly sort instead
last_time = time.time()
try:
for config, data in generator.next(): # timeout=2 * args.max_time)
results.append((config, data))
print('{}/{} completed | {:.3f} seconds | timeouts: {} | {}'.
format(len(results), len(jobs), elapsed_time(start_time),
timeouts,
datetime.datetime.now().strftime(DATE_FORMAT)))
print(config, data)
if results:
write_pickle(path, results)
print('Saved', path)
except StopIteration:
break
# except TimeoutError:
# # TODO: record this as a failure? Nothing is saved though...
# timeouts += 1
# #traceback.print_exc()
# print('Error! Timed out after {:.3f} seconds'.format(elapsed_time(last_time)))
# break # This kills all jobs
# #continue # This repeats jobs until success
print('Total time:', elapsed_time(initial_time))
return results
def main():
parser = argparse.ArgumentParser()
parser.add_argument('paths', nargs='*', help='Paths to the data.')
#parser.add_argument('-a', '--active', type=int, default=0, # None
# help='The number of active samples to collect')
parser.add_argument(
'-d',
'--deterministic',
action='store_true',
help='Whether to deterministically create training splits')
parser.add_argument('-n',
'--num_trials',
type=int,
default=-1,
help='The number of samples to collect')
parser.add_argument('-s',
'--save',
action='store_true',
help='Whether to save the learners')
parser.add_argument('-r',
'--num_rounds',
type=int,
default=1,
help='The number of rounds to collect')
parser.add_argument('-t',
'--test',
action='store_true',
help='Whether to save the data')
parser.add_argument('-v',
'--visualize',
action='store_true',
help='When enabled, visualizes execution.')
args = parser.parse_args()
# TODO: be careful that paging isn't altering the data
# TODO: use a different set of randomized parameters for train and test
serial = is_darwin()
visualize = serial and args.visualize
assert implies(visualize, serial)
num_trials = get_max_cores(
serial) if args.num_trials < 0 else args.num_trials
##################################################
#train_sizes = inclusive_range(50, 200, 10) # Best
#train_sizes = inclusive_range(50, 400, 10) # F1
#train_sizes = inclusive_range(25, 400, 25)
#train_sizes = inclusive_range(50, 100, 5) # Real
#train_sizes = inclusive_range(100, 200, 5)
#train_sizes = inclusive_range(10, 250, 5)
#train_sizes = inclusive_range(35, 70, 5)
#train_sizes = inclusive_range(5, 50, 5)
#train_sizes = inclusive_range(40, 80, 5)
#train_sizes = inclusive_range(100, 1000, 100)
#train_sizes = [50]
#train_sizes = [250]
train_sizes = [1000]
#train_sizes = [327] # train + test
#train_sizes = inclusive_range(5, 150, 25)
#train_sizes = [100]
#kernels = ['RBF', 'Matern52', 'MLP']
kernels = ['MLP']
hyperparams = [None]
#hyperparams = [True]
#hyperparams = [None, True]
query_type = BEST # BEST | CONFIDENT | REJECTION | ACTIVE # type of query used to evaluate the learner
include_none = False
binary = False
# 0 => no transfer
# 1 => mean transfer
# 2 => kernel transfer
# 3 => both transfer
transfer_weights = [None]
#transfer_weights = list(range(4))
#transfer_weights = [0, 1]
#transfer_weights = [3]
#transfer_weights = np.around(np.linspace(0.0, 1.0, num=1+5, endpoint=True), decimals=3) # max 10 colors
#transfer_weights = list(range(1, 1+3))
#split = UNIFORM # BALANCED
#print('Split:', split)
#parameters = {
# 'include None': include_none,
# 'binary': binary,
# 'split': split,
#}
# Omitting failed labels is okay because they will never be executed
algorithms = []
#algorithms += [(Algorithm(nn_model, label='NN'), [num])
# for nn_model, num in product(NN_MODELS, train_sizes)]
#algorithms += [(Algorithm(RANDOM), None), (Algorithm(DESIGNED), None)]
#algorithms += [(Algorithm(RF_CLASSIFIER, variance=False, transfer_weight=tw, label='RF'), [num])
# for num, tw in product(train_sizes, [None])] # transfer_weights
#algorithms += [(Algorithm(RF_REGRESSOR, variance=False, transfer_weight=tw, label='RF'), [num])
# for num, tw in product(train_sizes, [None])] # transfer_weights
#algorithms += [(Algorithm(BATCH_RF, variance=True, transfer_weight=tw, label='RF'), [num])
# for num, tw in product(train_sizes, [None])] # transfer_weights
#algorithms += [(Algorithm(BATCH_MAXVAR_RF, variance=True, transfer_weight=tw), train_sizes)
# for tw in product(use_vars, [None])] # transfer_weights
#algorithms += [(Algorithm(BATCH_STRADDLE_RF, variance=True, transfer_weight=tw), train_sizes)
# for tw, in product([None])] # transfer_weights
use_vars = [True]
# STRADDLE is better than MAXVAR when the learner has a good estimate of uncertainty
algorithms += [
(Algorithm(BATCH_GP, kernel, hype, use_var, tw,
label='GP'), [num]) # label='GP-{}'.format(kernel)
for num, kernel, hype, use_var, tw in product(
train_sizes, kernels, hyperparams, use_vars, transfer_weights)
]
#algorithms += [(Algorithm(BATCH_MAXVAR_GP, kernel, hype, True, tw, label='GP-Var'), train_sizes)
# for kernel, hype, tw in product(kernels, hyperparams, transfer_weights)]
#algorithms += [(Algorithm(BATCH_STRADDLE_GP, kernel, hype, True, tw, label='GP-LSE'), train_sizes)
# for kernel, hype, tw in product(kernels, hyperparams, transfer_weights)] # default active
#algorithms += [(Algorithm(BATCH_STRADDLE_GP, kernel, hype, True, tw, label='GP-LSE2'), train_sizes)
# for kernel, hype, tw in product(kernels, hyperparams, transfer_weights)] # active control only
# algorithms += [(Algorithm(MAXVAR_GP, kernel, hype, use_var), train_sizes)
# for kernel, hype, use_var in product(kernels, hyperparams, use_vars)]
#algorithms += [(Algorithm(STRADDLE_GP, kernel, hype, use_var, tw), train_sizes)
# for kernel, hype, use_var, tw in product(kernels, hyperparams, use_vars, transfer_weights)]
#batch_sizes = inclusive_range(train_sizes[0], 90, 10)
#step_size = 10 # TODO: extract from train_sizes
#final_size = train_sizes[-1]
# Previously didn't have use_var=True
# algorithms += [(Algorithm(BATCH_STRADDLE_GP, kernel, hyperparameters=batch_size, variance=True, transfer_weight=tw),
# inclusive_range(batch_size, final_size, step_size))
# for kernel, tw, batch_size in product(kernels, transfer_weights, batch_sizes)]
# algorithms += [(Algorithm(BATCH_STRADDLE_RF, hyperparameters=batch_size, variance=True, transfer_weight=tw),
# inclusive_range(batch_size, final_size, step_size))
# for tw, batch_size in product(transfer_weights, batch_sizes)]
print('Algorithms:', algorithms)
##################################################
real_world = not args.paths
transfer_domain = load_data(TRANSFER_DATASETS, verbose=False)
transfer_algorithm = None
if real_world and transfer_weights != [None]:
#assert transfer_weights[0] is not None
transfer_data = transfer_domain.create_dataset(
include_none=include_none, binary=binary)
transfer_algorithm = Algorithm(BATCH_GP,
kernel=kernels[0],
variance=use_vars[0])
validity_learner = None
#validity_learner = create_validity_classifier(transfer_domain)
##################################################
train_paths = args.paths
if real_world:
train_paths = SCOOP_TRAIN_DATASETS # TRAIN_DATASETS
#train_paths = TRANSFER_DATASETS
#train_paths = TRAIN_DATASETS + TRANSFER_DATASETS # Train before transfer
#scale_paths = TRAIN_DATASETS + TEST_DATASETS
scale_paths = None
print(SEPARATOR)
print('Train paths:', train_paths)
domain = load_data(train_paths)
print()
print(domain)
all_data = domain.create_dataset(include_none=include_none,
binary=binary,
scale_paths=scale_paths)
#all_data.results = all_data.results[:1000]
num_failed = 0
#num_failed = 100
failed_domain = transfer_domain if real_world else domain
failed_results = randomize(
result for result in failed_domain.results
if not result.get('success', False))[:num_failed]
#failed_data = Dataset(domain, failed_results, **all_data.kwargs)
test_paths = SCOOP_TEST_DATASETS # TEST_DATASETS | SCOOP_TEST_DATASETS
#test_paths = None
if real_world and not (set(train_paths) & set(test_paths)):
#assert not set(train_paths) & set(test_paths)
#max_test = 0
test_data = load_data(test_paths).create_dataset(
include_none=False, binary=binary, scale_paths=scale_paths)
else:
#assert scale_paths is None # TODO: max_train will be too small otherwise
test_paths = test_data = None
print(SEPARATOR)
print('Test paths:', test_paths)
all_active_data = None
#if real_world:
# all_active_data = load_data(ACTIVE_DATASETS).create_dataset(include_none=True, binary=binary, scale_paths=scale_paths)
# TODO: could include OS and username if desired
date_name = datetime.datetime.now().strftime(DATE_FORMAT)
size_str = '[{},{}]'.format(train_sizes[0], train_sizes[-1])
#size_str = '-'.join(map(str, train_sizes))
experiments_name = '{}_r={}_t={}_n={}'.format(date_name, args.num_rounds,
size_str, num_trials)
trials_per_round = sum(
1 if train_sizes is None else (train_sizes[-1] - train_sizes[0] +
len(train_sizes))
for _, train_sizes in algorithms)
num_experiments = args.num_rounds * trials_per_round
max_train = min(
max([0] + [
active_sizes[0]
for _, active_sizes in algorithms if active_sizes is not None
]), len(all_data))
max_test = min(len(all_data) - max_train, 1000)
##################################################
# #features = ['bowl_height']
# features = ['spoon_height']
# #features = ['bowl_height', 'spoon_height']
# X, Y, _ = all_data.get_data()
# #indices = [domain.inputs.index(feature) for feature in features]
# #X = X[:,indices]
# X = [[result[FEATURE][name] for name in features] for result in all_data.results]
# from sklearn.linear_model import LinearRegression
# model = LinearRegression(fit_intercept=True, normalize=False)
# model.fit(X, Y)
# #print(model.get_params())
# print(model.coef_.tolist(), model.intercept_)
# print(model.score(X, Y))
#data_dir = os.path.join(DATA_DIRECTORY, domain.name) # EXPERIMENT_DIRECTORY
data_dir = os.path.abspath(os.path.join(domain.name, os.path.pardir))
experiments_dir, data_path = None, None
if not args.test or not serial:
experiments_dir = os.path.join(data_dir, experiments_name)
data_path = os.path.join(
experiments_dir, 'experiments.pk{}'.format(get_python_version()))
##################################################
print(SEPARATOR)
print('Name:', experiments_name)
print('Experiments:', num_experiments)
print('Experiment dir:', experiments_dir)
print('Data path:', data_path)
print('Examples:', len(all_data))
print('Valid:',
sum(result.get('valid', True) for result in all_data.results))
print('Success:',
sum(result.get('success', False) for result in all_data.results))
print(
'Scored:',
sum(
result.get('score', None) is not None
for result in all_data.results))
print('Max train:', max_train)
print('Max test:', max_test)
print('Include None:', include_none)
print('Examples: n={}, d={}'.format(len(all_data), domain.dx))
print('Binary:', binary)
print('Serial:', serial)
print('Estimated hours: {:.3f}'.format(num_experiments *
SEC_PER_EXPERIMENT / HOURS_TO_SECS))
user_input('Begin?')
##################################################
experiments = []
if experiments_dir is not None:
mkdir(experiments_dir)
# if os.path.exists(data_path):
# experiments.extend(read_pickle(data_path))
# TODO: embed in a KeyboardInterrupt to allow early termination
start_time = time.time()
for round_idx in range(args.num_rounds):
seed = round_idx if args.deterministic else hash(
time.time()) # vs just time.time()?
random.seed(seed)
all_data.shuffle()
if test_paths is None: # cannot use test_data
#test_data, train_data = split_data(all_data, max_test)
train_data = test_data = all_data # Training performance
else:
train_data = all_data
transfer_learner = None
if transfer_algorithm is not None:
round_data, _ = transfer_data.partition(index=1000)
transfer_learner, _ = create_learner(transfer_domain,
round_data,
transfer_algorithm,
verbose=True)
transfer_learner.retrain()
print(SEPARATOR)
print('Round {} | Train examples: {} | Test examples: {}'.format(
round_idx, len(train_data), len(test_data)))
for algorithm, active_sizes in algorithms:
# active_sizes = [first #trainingdata selected from X_train, #active exploration + #trainingdata]
print(SEPARATOR)
print('Round: {} | {} | Seed: {} | Sizes: {}'.format(
round_idx, algorithm, seed, active_sizes))
# TODO: allow keyboard interrupt
if active_sizes is None:
learner = algorithm.name
active_size = train_confusion = None
experiments.append(
evaluate_learner(domain, seed, train_confusion, test_data,
algorithm, learner, active_size,
num_trials, serial, args.visualize))
continue
# [10 20 25] take first 10 samples from X_train to train the model, 10 samples chosen actively
# sequentially + evaluate model, 5 samples chosen actively sequentially + evaluate model
# Could always keep around all the examples and retrain
# TODO: segfaults when this runs in parallel
# TODO: may be able to retrain in parallel if I set OPENBLAS_NUM_THREADS
num_batch = active_sizes[0]
batch_data, active_data = train_data.partition(num_batch)
if all_active_data is not None:
active_data = all_active_data.clone()
#batch_data.results.extend(failed_results)
learner, train_confusion = create_learner(
domain,
batch_data,
algorithm, # alphas,
query_type=query_type,
verbose=True)
learner.validity_learner = validity_learner
if transfer_learner is not None:
learner.sim_model = transfer_learner.model
learner.retrain()
for active_size in active_sizes:
num_active = active_size - (learner.nx - len(failed_results))
print('\nRound: {} | {} | Seed: {} | Size: {} | Active: {}'.
format(round_idx, algorithm, seed, active_size,
num_active))
if algorithm.name in CONTINUOUS_ACTIVE_GP:
active_learning(learner, num_active, visualize=visualize)
#active_learning(learner, num_active, discrete_feature=True, random_feature=False)
#active_learning_discrete(learner, active_data, num_active, random_feature=False)
elif algorithm.name in BATCH_ACTIVE:
active_learning_discrete(learner, active_data, num_active)
#active_learning(learner, num_active, discrete_feature=True, random_feature=True)
#active_learning_discrete(learner, active_data, num_active, random_feature=True)
#if round_dir is not None:
# save_learner(round_dir, learner)
if args.save:
learner.save(data_dir)
experiments.append(
evaluate_learner(domain, seed, train_confusion, test_data,
algorithm, learner, active_size,
num_trials, serial, args.visualize))
save_experiments(data_path, experiments)
print(SEPARATOR)
if experiments:
save_experiments(data_path, experiments)
plot_experiments(domain,
experiments_name,
experiments_dir,
experiments,
include_none=False)
print('Experiments: {}'.format(experiments_dir))
print('Total experiments: {}'.format(len(experiments)))
print('Total hours: {:.3f}'.format(
elapsed_time(start_time) / HOURS_TO_SECS))
from pddlstream.algorithms.algorithm import reset_globals
from src.command import create_state, iterate_commands
from src.observe import observe_pybullet
from src.world import World
from src.policy import run_policy
from src.task import cook_block, TASKS_FNS
from run_pybullet import create_parser
from multiprocessing import Pool, TimeoutError, cpu_count
EXPERIMENTS_DIRECTORY = 'experiments/'
TEMP_DIRECTORY = 'temp_parallel/'
MAX_TIME = 10 * 60
TIME_BUFFER = 60
SERIAL = is_darwin()
VERBOSE = SERIAL
SERIALIZE_TASK = True
MEAN_TIME_PER_TRIAL = 300 # trial / sec
HOURS_TO_SECS = 60 * 60
N_TRIALS = 1 # 1
MAX_MEMORY = 3.5 * KILOBYTES_PER_GIGABYTE
SPARE_CORES = 4
POLICIES = [
{
'constrain': False,
'defer': False
},