def main(): # collect data in parallel, parameters are generated uniformly randomly in a range # data stored to pour_date/trails_n=10000.json # TODO: ulimit settings # https://ss64.com/bash/ulimit.html # https://stackoverflow.com/questions/938733/total-memory-used-by-python-process # import psutil # TODO: resource.getrusage(resource.RUSAGE_SELF).ru_maxrss assert (get_python_version() == 3) parser = argparse.ArgumentParser() parser.add_argument('-f', '--fn', default=TRAINING, help='The parameter function to use.') parser.add_argument('-n', '--num', type=int, default=10000, help='The number of samples to collect.') parser.add_argument('-p', '--problem', required=True, choices=sorted(SKILL_COLLECTORS.keys()), help='The name of the skill to learn.') parser.add_argument('-t', '--time', type=int, default=1 * 60, help='The max planning runtime for each trial.') parser.add_argument('-v', '--visualize', action='store_true', help='When enabled, visualizes execution.') args = parser.parse_args() serial = is_darwin() assert implies(args.visualize, serial) trials = get_trials(args.problem, args.fn, args.num, max_time=args.time, valid=True, visualize=args.visualize, verbose=serial) data_path = None if serial else get_data_path(args.problem, trials) num_cores = get_num_cores(trials, serial) user_input('Begin?') # TODO: store the generating distribution for samples and objects? print(SEPARATOR) results = run_trials(trials, data_path, num_cores=num_cores)
def get_data_path(skill_name, trials=[], real=False): print('Skill:', skill_name) if is_darwin(): return None # (system, node, release, version, machine, processor) # system_name = platform.uname()[0].lower() # computer_name = platform.uname()[1] # computer_name = socket.gethostname() # TODO: changes when connected to wifi # What is really want is the name in PS1 # user_name = getpass.getuser() prefix = '{}_'.format(REAL_PREFIX) if real else '' date_name = datetime.datetime.now().strftime(DATE_FORMAT) directory = os.path.join(DATA_DIRECTORY, '{}{}_{}/'.format(prefix, skill_name, date_name)) ensure_dir(directory) suffix = '' if real else '_n={}'.format(len(trials)) data_path = os.path.join(directory, 'trials{}'.format(suffix)) print('Data path:', data_path) # print('System:', system_name) # print('Username:', user_name) return data_path
def main(): parser = argparse.ArgumentParser() parser.add_argument('paths', nargs='*', help='Paths to the data.') #parser.add_argument('-a', '--active', type=int, default=0, # None # help='The number of active samples to collect') parser.add_argument('-l', '--learner', default=None, help='Path to the learner that should be used') parser.add_argument('-n', '--num_trials', type=int, default=100, help='The number of samples to collect') parser.add_argument('-s', '--save', action='store_true', help='Whether to save the learners') parser.add_argument('-r', '--num_rounds', type=int, default=1, help='The number of rounds to collect') #parser.add_argument('-t', '--num_train', type=int, default=None, # help='The size of the training set') args = parser.parse_args() # TODO: be careful that paging isn't altering the data # TODO: don't penalize if the learner identifies that it can't make a good prediction # TODO: use a different set of randomized parameters for train and test include_none = False serial = is_darwin() #training_sizes = inclusive_range(50, 500, 25) #training_sizes = inclusive_range(25, 100, 5) #training_sizes = inclusive_range(25, 100, 5) training_sizes = inclusive_range(10, 50, 5) #training_sizes = inclusive_range(100, 1000, 100) #training_sizes = [20] #training_sizes = [1500] #kernels = ['RBF', 'Matern52', 'MLP'] kernels = ['MLP'] #hyperparameters = [None] #hyperparameters = [True] hyperparameters = [True, None] # None, query_type = BEST # BEST | CONFIDENT | REJECTION | ACTIVE # type of query used to evaluate the learner is_adaptive = False max_test = 50 # #alphas = np.linspace(0.0, 0.9, num=5, endpoint=True) alphas = [0.0, .8, .9, .99] #alphas = [None] # Use the default (i.e. GP parameters) use_vars = [True] binary = False split = UNIFORM # BALANCED # Omitting failed labels is okay because they will never be executed algorithms = [] #algorithms += [(Algorithm(BATCH_GP, kernel=kernel, hyperparameters=hype, use_var=use_var), [num_train]) # for num_train, kernel, hype, use_var in product(training_sizes, kernels, hyperparameters, use_vars)] algorithms += [(Algorithm(STRADDLE_GP, kernel, hype, use_var), training_sizes) for kernel, hype, use_var in product(kernels, hyperparameters, use_vars)] #algorithms += [(Algorithm(rf_model, p_explore=None, use_var=use_var), [num_train]) # for rf_model, num_train, use_var in product(RF_MODELS, training_sizes, use_vars)] #algorithms += [(Algorithm(nn_model, p_explore=None), [num_train]) # for nn_model, num_train in product(NN_MODELS, training_sizes)] #algorithms += [(Algorithm(RANDOM), None), (Algorithm(DESIGNED), None)] print('Algorithms:', algorithms) print('Split:', split) trials_per_round = sum(1 if train_sizes is None else (train_sizes[-1] - train_sizes[0] + len(train_sizes)) for _, train_sizes in algorithms) num_experiments = args.num_rounds*trials_per_round date_name = datetime.datetime.now().strftime(DATE_FORMAT) size_str = '[{},{}]'.format(training_sizes[0], training_sizes[-1]) #size_str = '-'.join(map(str, training_sizes)) experiments_name = '{}_r={}_t={}_n={}'.format(date_name, args.num_rounds, size_str, args.num_trials) #'19-08-09_21-44-58_r=5_t=[10,150]_n=1'# #experiments_name = 't={}'.format(args.num_rounds) # TODO: could include OS and username if desired domain = load_data(args.paths) print() print(domain) X, Y, W = domain.get_data(include_none=include_none) print('Total number of examples:', len(X)) if binary: # NN can fit perfectly when binary # Binary seems to be outperforming w/o Y = threshold_scores(Y) max_train = len(X) - max_test #min(max([0] + [active_sizes[0] for _, active_sizes in algorithms # if active_sizes is not None]), len(X)) #parameters = { # 'include None': include_none, # 'binary': binary, # 'split': split, #} print('Name:', experiments_name) print('Experiments:', num_experiments) print('Max train:', max_train) print('Include None:', include_none) print('Examples: n={}, d={}'.format(*X.shape)) print('Binary:', binary) print('Estimated hours:', num_experiments * SEC_PER_EXPERIMENT / HOURS_TO_SECS) user_input('Begin?') # TODO: residual learning for sim to real transfer # TODO: can always be conservative and add sim negative examples # TODO: combine all data to write in one folder data_dir = os.path.join(DATA_DIRECTORY, domain.name) # EXPERIMENT_DIRECTORY experiments_dir = os.path.join(data_dir, experiments_name) mkdir(experiments_dir) start_time = time.time() experiments = [] for round_idx in range(args.num_rounds): round_dir = os.path.join(data_dir, experiments_name, str(round_idx)) mkdir(round_dir) seed = hash(time.time()) train_test_file = os.path.join(round_dir, 'data.pk3') if not os.path.exists(train_test_file): X_train, Y_train, X_test, Y_test = split_data(X, Y, split, max_train) X_test, Y_test = X_test[:max_test], Y_test[:max_test] write_pickle(train_test_file, (X_train, Y_train, X_test, Y_test)) else: X_train, Y_train, X_test, Y_test = read_pickle(train_test_file) print('Train examples:', X_train.shape) print('Test examples:', X_test.shape) # TODO: need to be super careful when running with multiple contexts for algorithm, active_sizes in algorithms: # active_sizes = [first #trainingdata selected from X_train, #active exploration + #trainingdata] print(SEPARATOR) print('Round: {} | {} | Seed: {} | Sizes: {}'.format(round_idx, algorithm, seed, active_sizes)) # TODO: allow keyboard interrupt if active_sizes is None: learner = algorithm.name active_size = None train_confusion = None experiments.append(evaluate_learner(domain, seed, train_confusion, X_test, Y_test, algorithm, learner, active_size, args.num_trials, alphas, serial)) else: # [10 20 25] take first 10 samples from X_train to train the model, 10 samples chosen actively # sequentially + evaluate model, 5 samples chosen actively sequentially + evaluate model # Could always keep around all the examples and retrain # TODO: segfaults when this runs in parallel # TODO: may be able to retrain in parallel if I set OPENBLAS_NUM_THREADS learner_prior_nx = 0 ''' if algorithm.hyperparameters: if domain.skill == 'pour': learner_file = '/Users/ziw/ltamp_pr2/data/pour_19-06-13_00-59-21/19-08-09_19-30-01_r=10_t=[50,400]_n=1/{}/gp_active_mlp_true_true.pk3'.format( round_idx) elif domain.skill == 'scoop': learner_file = '/Users/ziw/ltamp_pr2/data/scoop_19-06-10_20-16-59_top-diameter/19-08-09_19-34-56_r=10_t=[50,400]_n=1/{}/gp_active_mlp_true_true.pk3'.format( round_idx) learner = read_pickle(learner_file) learner_prior_nx = learner.nx learner.retrain(newx=X_train[:active_sizes[0]], newy=Y_train[:active_sizes[0], None]) else: ''' learner, train_confusion = create_learner(domain, X_train, Y_train, split, algorithm, num_train=active_sizes[0], query_type=query_type, is_adaptive=is_adaptive) if algorithm.name == STRADDLE_GP: X_select, Y_select = X_train[active_sizes[0]:], Y_train[active_sizes[0]:] for active_size in active_sizes: num_active = active_size - learner.nx + learner_prior_nx# learner.nx is len(learner.xx) print('\nRound: {} | {} | Seed: {} | Size: {} | Active: {}'.format( round_idx, algorithm, seed, active_size, num_active)) if algorithm.name == STRADDLE_GP: X_select, Y_select = active_learning_discrete(learner, num_active, X_select, Y_select) #if args.save: save_learner(round_dir, learner) experiments.append(evaluate_learner(domain, seed, None, X_test, Y_test, algorithm, learner, active_size, args.num_trials, alphas, serial)) save_experiments(experiments_dir, experiments) print(SEPARATOR) if experiments: save_experiments(experiments_dir, experiments) plot_experiments(domain, experiments_name, experiments_dir, experiments, include_none=False) #include_none=include_none) print('Experiments:', experiments_dir) print('Total experiments:', len(experiments)) print('Total hours:', elapsed_time(start_time) / HOURS_TO_SECS)
from sklearn.metrics import mean_squared_error, confusion_matrix, precision_score, recall_score sys.path.extend([ os.path.join( os.getcwd(), 'pddlstream'), # Important to use absolute path when doing chdir os.path.join(os.getcwd(), 'ss-pybullet'), ]) from collections import namedtuple, OrderedDict, defaultdict from pybullet_tools.utils import is_darwin from scipy.stats import rankdata #has_gui = sys.stdin.isatty() #has_gui = 'DISPLAY' in os.environ has_gui = is_darwin() #print('GUI:', has_gui) #if not has_gui: # import matplotlib # matplotlib.use('Agg') from learn_tools.active_nn import REGRESSOR, CLASSIFIER, NN from learn_tools.active_learner import score_prediction from learn_tools.learnable_skill import LearnableSkill, read_data, is_real, get_skill from learn_tools.statistics import compare_distributions from learn_tools.learner import plot_learning_curve, LATENT, SCORE, FEATURE, PARAMETER, threshold_score, THRESHOLD, \ estimate_gaussian, SEPARATOR, SUCCESS, FAILURE, DYNAMICS, threshold_scores, SKILL from pddlstream.utils import mkdir, find_unique from pybullet_tools.utils import is_remote, safe_zip, SEPARATOR Algorithm = namedtuple('Algorithm', [
def train_parallel(args, n=1): from extrusion.run import plan_extrusion assert SKIP_PERCENTAGE == 0 initial_time = time.time() problems = sorted(set(enumerate_problems()) - set(EXCLUDE)) #problems = ['four-frame'] #problems = ['simple_frame', 'topopt-101_tiny', 'topopt-100_S1_03-14-2019_w_layer'] algorithms = list(ALGORITHMS) if args.disable: for algorithm in LOOKAHEAD_ALGORITHMS: if algorithm in algorithms: algorithms.remove(algorithm) #algorithms = ['regression'] heuristics = HEURISTICS #heuristics = DISTANCE_HEURISTICS + COST_HEURISTICS seeds = list(range(args.num)) if n is None: n = len(seeds) groups = list(chunks(seeds, n=n)) print('Chunks: {}'.format(len(groups))) print('Problems ({}): {}'.format(len(problems), problems)) #problems = [path for path in problems if 'simple_frame' in path] print('Algorithms ({}): {}'.format(len(algorithms), algorithms)) print('Heuristics ({}): {}'.format(len(heuristics), heuristics)) jobs = [[ Configuration(seed, problem, algorithm, heuristic, args.max_time, args.cfree, args.disable, args.stiffness, args.motions, args.ee_only) for seed, algorithm, heuristic in product( group, algorithms, heuristics) ] for problem, group in product(problems, groups)] # TODO: separate out the algorithms again # TODO: print the size per job print('Jobs: {}'.format(len(jobs))) serial = is_darwin() available_cores = cpu_count() num_cores = max(1, min(1 if serial else available_cores - 4, len(jobs))) print('Max Cores:', available_cores) print('Serial:', serial) print('Using Cores:', num_cores) date = datetime.datetime.now().strftime(DATE_FORMAT) filename = '{}.pk{}'.format(date, get_python_version()) path = os.path.join(EXPERIMENTS_DIR, filename) print('Data path:', path) user_input('Begin?') start_time = time.time() timeouts = 0 pool = Pool(processes=num_cores) # , initializer=mute) generator = pool.imap_unordered(plan_extrusion, jobs, chunksize=1) results = [] while True: # TODO: randomly sort instead last_time = time.time() try: for config, data in generator.next(): # timeout=2 * args.max_time) results.append((config, data)) print('{}/{} completed | {:.3f} seconds | timeouts: {} | {}'. format(len(results), len(jobs), elapsed_time(start_time), timeouts, datetime.datetime.now().strftime(DATE_FORMAT))) print(config, data) if results: write_pickle(path, results) print('Saved', path) except StopIteration: break # except TimeoutError: # # TODO: record this as a failure? Nothing is saved though... # timeouts += 1 # #traceback.print_exc() # print('Error! Timed out after {:.3f} seconds'.format(elapsed_time(last_time))) # break # This kills all jobs # #continue # This repeats jobs until success print('Total time:', elapsed_time(initial_time)) return results
def main(): parser = argparse.ArgumentParser() parser.add_argument('paths', nargs='*', help='Paths to the data.') #parser.add_argument('-a', '--active', type=int, default=0, # None # help='The number of active samples to collect') parser.add_argument( '-d', '--deterministic', action='store_true', help='Whether to deterministically create training splits') parser.add_argument('-n', '--num_trials', type=int, default=-1, help='The number of samples to collect') parser.add_argument('-s', '--save', action='store_true', help='Whether to save the learners') parser.add_argument('-r', '--num_rounds', type=int, default=1, help='The number of rounds to collect') parser.add_argument('-t', '--test', action='store_true', help='Whether to save the data') parser.add_argument('-v', '--visualize', action='store_true', help='When enabled, visualizes execution.') args = parser.parse_args() # TODO: be careful that paging isn't altering the data # TODO: use a different set of randomized parameters for train and test serial = is_darwin() visualize = serial and args.visualize assert implies(visualize, serial) num_trials = get_max_cores( serial) if args.num_trials < 0 else args.num_trials ################################################## #train_sizes = inclusive_range(50, 200, 10) # Best #train_sizes = inclusive_range(50, 400, 10) # F1 #train_sizes = inclusive_range(25, 400, 25) #train_sizes = inclusive_range(50, 100, 5) # Real #train_sizes = inclusive_range(100, 200, 5) #train_sizes = inclusive_range(10, 250, 5) #train_sizes = inclusive_range(35, 70, 5) #train_sizes = inclusive_range(5, 50, 5) #train_sizes = inclusive_range(40, 80, 5) #train_sizes = inclusive_range(100, 1000, 100) #train_sizes = [50] #train_sizes = [250] train_sizes = [1000] #train_sizes = [327] # train + test #train_sizes = inclusive_range(5, 150, 25) #train_sizes = [100] #kernels = ['RBF', 'Matern52', 'MLP'] kernels = ['MLP'] hyperparams = [None] #hyperparams = [True] #hyperparams = [None, True] query_type = BEST # BEST | CONFIDENT | REJECTION | ACTIVE # type of query used to evaluate the learner include_none = False binary = False # 0 => no transfer # 1 => mean transfer # 2 => kernel transfer # 3 => both transfer transfer_weights = [None] #transfer_weights = list(range(4)) #transfer_weights = [0, 1] #transfer_weights = [3] #transfer_weights = np.around(np.linspace(0.0, 1.0, num=1+5, endpoint=True), decimals=3) # max 10 colors #transfer_weights = list(range(1, 1+3)) #split = UNIFORM # BALANCED #print('Split:', split) #parameters = { # 'include None': include_none, # 'binary': binary, # 'split': split, #} # Omitting failed labels is okay because they will never be executed algorithms = [] #algorithms += [(Algorithm(nn_model, label='NN'), [num]) # for nn_model, num in product(NN_MODELS, train_sizes)] #algorithms += [(Algorithm(RANDOM), None), (Algorithm(DESIGNED), None)] #algorithms += [(Algorithm(RF_CLASSIFIER, variance=False, transfer_weight=tw, label='RF'), [num]) # for num, tw in product(train_sizes, [None])] # transfer_weights #algorithms += [(Algorithm(RF_REGRESSOR, variance=False, transfer_weight=tw, label='RF'), [num]) # for num, tw in product(train_sizes, [None])] # transfer_weights #algorithms += [(Algorithm(BATCH_RF, variance=True, transfer_weight=tw, label='RF'), [num]) # for num, tw in product(train_sizes, [None])] # transfer_weights #algorithms += [(Algorithm(BATCH_MAXVAR_RF, variance=True, transfer_weight=tw), train_sizes) # for tw in product(use_vars, [None])] # transfer_weights #algorithms += [(Algorithm(BATCH_STRADDLE_RF, variance=True, transfer_weight=tw), train_sizes) # for tw, in product([None])] # transfer_weights use_vars = [True] # STRADDLE is better than MAXVAR when the learner has a good estimate of uncertainty algorithms += [ (Algorithm(BATCH_GP, kernel, hype, use_var, tw, label='GP'), [num]) # label='GP-{}'.format(kernel) for num, kernel, hype, use_var, tw in product( train_sizes, kernels, hyperparams, use_vars, transfer_weights) ] #algorithms += [(Algorithm(BATCH_MAXVAR_GP, kernel, hype, True, tw, label='GP-Var'), train_sizes) # for kernel, hype, tw in product(kernels, hyperparams, transfer_weights)] #algorithms += [(Algorithm(BATCH_STRADDLE_GP, kernel, hype, True, tw, label='GP-LSE'), train_sizes) # for kernel, hype, tw in product(kernels, hyperparams, transfer_weights)] # default active #algorithms += [(Algorithm(BATCH_STRADDLE_GP, kernel, hype, True, tw, label='GP-LSE2'), train_sizes) # for kernel, hype, tw in product(kernels, hyperparams, transfer_weights)] # active control only # algorithms += [(Algorithm(MAXVAR_GP, kernel, hype, use_var), train_sizes) # for kernel, hype, use_var in product(kernels, hyperparams, use_vars)] #algorithms += [(Algorithm(STRADDLE_GP, kernel, hype, use_var, tw), train_sizes) # for kernel, hype, use_var, tw in product(kernels, hyperparams, use_vars, transfer_weights)] #batch_sizes = inclusive_range(train_sizes[0], 90, 10) #step_size = 10 # TODO: extract from train_sizes #final_size = train_sizes[-1] # Previously didn't have use_var=True # algorithms += [(Algorithm(BATCH_STRADDLE_GP, kernel, hyperparameters=batch_size, variance=True, transfer_weight=tw), # inclusive_range(batch_size, final_size, step_size)) # for kernel, tw, batch_size in product(kernels, transfer_weights, batch_sizes)] # algorithms += [(Algorithm(BATCH_STRADDLE_RF, hyperparameters=batch_size, variance=True, transfer_weight=tw), # inclusive_range(batch_size, final_size, step_size)) # for tw, batch_size in product(transfer_weights, batch_sizes)] print('Algorithms:', algorithms) ################################################## real_world = not args.paths transfer_domain = load_data(TRANSFER_DATASETS, verbose=False) transfer_algorithm = None if real_world and transfer_weights != [None]: #assert transfer_weights[0] is not None transfer_data = transfer_domain.create_dataset( include_none=include_none, binary=binary) transfer_algorithm = Algorithm(BATCH_GP, kernel=kernels[0], variance=use_vars[0]) validity_learner = None #validity_learner = create_validity_classifier(transfer_domain) ################################################## train_paths = args.paths if real_world: train_paths = SCOOP_TRAIN_DATASETS # TRAIN_DATASETS #train_paths = TRANSFER_DATASETS #train_paths = TRAIN_DATASETS + TRANSFER_DATASETS # Train before transfer #scale_paths = TRAIN_DATASETS + TEST_DATASETS scale_paths = None print(SEPARATOR) print('Train paths:', train_paths) domain = load_data(train_paths) print() print(domain) all_data = domain.create_dataset(include_none=include_none, binary=binary, scale_paths=scale_paths) #all_data.results = all_data.results[:1000] num_failed = 0 #num_failed = 100 failed_domain = transfer_domain if real_world else domain failed_results = randomize( result for result in failed_domain.results if not result.get('success', False))[:num_failed] #failed_data = Dataset(domain, failed_results, **all_data.kwargs) test_paths = SCOOP_TEST_DATASETS # TEST_DATASETS | SCOOP_TEST_DATASETS #test_paths = None if real_world and not (set(train_paths) & set(test_paths)): #assert not set(train_paths) & set(test_paths) #max_test = 0 test_data = load_data(test_paths).create_dataset( include_none=False, binary=binary, scale_paths=scale_paths) else: #assert scale_paths is None # TODO: max_train will be too small otherwise test_paths = test_data = None print(SEPARATOR) print('Test paths:', test_paths) all_active_data = None #if real_world: # all_active_data = load_data(ACTIVE_DATASETS).create_dataset(include_none=True, binary=binary, scale_paths=scale_paths) # TODO: could include OS and username if desired date_name = datetime.datetime.now().strftime(DATE_FORMAT) size_str = '[{},{}]'.format(train_sizes[0], train_sizes[-1]) #size_str = '-'.join(map(str, train_sizes)) experiments_name = '{}_r={}_t={}_n={}'.format(date_name, args.num_rounds, size_str, num_trials) trials_per_round = sum( 1 if train_sizes is None else (train_sizes[-1] - train_sizes[0] + len(train_sizes)) for _, train_sizes in algorithms) num_experiments = args.num_rounds * trials_per_round max_train = min( max([0] + [ active_sizes[0] for _, active_sizes in algorithms if active_sizes is not None ]), len(all_data)) max_test = min(len(all_data) - max_train, 1000) ################################################## # #features = ['bowl_height'] # features = ['spoon_height'] # #features = ['bowl_height', 'spoon_height'] # X, Y, _ = all_data.get_data() # #indices = [domain.inputs.index(feature) for feature in features] # #X = X[:,indices] # X = [[result[FEATURE][name] for name in features] for result in all_data.results] # from sklearn.linear_model import LinearRegression # model = LinearRegression(fit_intercept=True, normalize=False) # model.fit(X, Y) # #print(model.get_params()) # print(model.coef_.tolist(), model.intercept_) # print(model.score(X, Y)) #data_dir = os.path.join(DATA_DIRECTORY, domain.name) # EXPERIMENT_DIRECTORY data_dir = os.path.abspath(os.path.join(domain.name, os.path.pardir)) experiments_dir, data_path = None, None if not args.test or not serial: experiments_dir = os.path.join(data_dir, experiments_name) data_path = os.path.join( experiments_dir, 'experiments.pk{}'.format(get_python_version())) ################################################## print(SEPARATOR) print('Name:', experiments_name) print('Experiments:', num_experiments) print('Experiment dir:', experiments_dir) print('Data path:', data_path) print('Examples:', len(all_data)) print('Valid:', sum(result.get('valid', True) for result in all_data.results)) print('Success:', sum(result.get('success', False) for result in all_data.results)) print( 'Scored:', sum( result.get('score', None) is not None for result in all_data.results)) print('Max train:', max_train) print('Max test:', max_test) print('Include None:', include_none) print('Examples: n={}, d={}'.format(len(all_data), domain.dx)) print('Binary:', binary) print('Serial:', serial) print('Estimated hours: {:.3f}'.format(num_experiments * SEC_PER_EXPERIMENT / HOURS_TO_SECS)) user_input('Begin?') ################################################## experiments = [] if experiments_dir is not None: mkdir(experiments_dir) # if os.path.exists(data_path): # experiments.extend(read_pickle(data_path)) # TODO: embed in a KeyboardInterrupt to allow early termination start_time = time.time() for round_idx in range(args.num_rounds): seed = round_idx if args.deterministic else hash( time.time()) # vs just time.time()? random.seed(seed) all_data.shuffle() if test_paths is None: # cannot use test_data #test_data, train_data = split_data(all_data, max_test) train_data = test_data = all_data # Training performance else: train_data = all_data transfer_learner = None if transfer_algorithm is not None: round_data, _ = transfer_data.partition(index=1000) transfer_learner, _ = create_learner(transfer_domain, round_data, transfer_algorithm, verbose=True) transfer_learner.retrain() print(SEPARATOR) print('Round {} | Train examples: {} | Test examples: {}'.format( round_idx, len(train_data), len(test_data))) for algorithm, active_sizes in algorithms: # active_sizes = [first #trainingdata selected from X_train, #active exploration + #trainingdata] print(SEPARATOR) print('Round: {} | {} | Seed: {} | Sizes: {}'.format( round_idx, algorithm, seed, active_sizes)) # TODO: allow keyboard interrupt if active_sizes is None: learner = algorithm.name active_size = train_confusion = None experiments.append( evaluate_learner(domain, seed, train_confusion, test_data, algorithm, learner, active_size, num_trials, serial, args.visualize)) continue # [10 20 25] take first 10 samples from X_train to train the model, 10 samples chosen actively # sequentially + evaluate model, 5 samples chosen actively sequentially + evaluate model # Could always keep around all the examples and retrain # TODO: segfaults when this runs in parallel # TODO: may be able to retrain in parallel if I set OPENBLAS_NUM_THREADS num_batch = active_sizes[0] batch_data, active_data = train_data.partition(num_batch) if all_active_data is not None: active_data = all_active_data.clone() #batch_data.results.extend(failed_results) learner, train_confusion = create_learner( domain, batch_data, algorithm, # alphas, query_type=query_type, verbose=True) learner.validity_learner = validity_learner if transfer_learner is not None: learner.sim_model = transfer_learner.model learner.retrain() for active_size in active_sizes: num_active = active_size - (learner.nx - len(failed_results)) print('\nRound: {} | {} | Seed: {} | Size: {} | Active: {}'. format(round_idx, algorithm, seed, active_size, num_active)) if algorithm.name in CONTINUOUS_ACTIVE_GP: active_learning(learner, num_active, visualize=visualize) #active_learning(learner, num_active, discrete_feature=True, random_feature=False) #active_learning_discrete(learner, active_data, num_active, random_feature=False) elif algorithm.name in BATCH_ACTIVE: active_learning_discrete(learner, active_data, num_active) #active_learning(learner, num_active, discrete_feature=True, random_feature=True) #active_learning_discrete(learner, active_data, num_active, random_feature=True) #if round_dir is not None: # save_learner(round_dir, learner) if args.save: learner.save(data_dir) experiments.append( evaluate_learner(domain, seed, train_confusion, test_data, algorithm, learner, active_size, num_trials, serial, args.visualize)) save_experiments(data_path, experiments) print(SEPARATOR) if experiments: save_experiments(data_path, experiments) plot_experiments(domain, experiments_name, experiments_dir, experiments, include_none=False) print('Experiments: {}'.format(experiments_dir)) print('Total experiments: {}'.format(len(experiments))) print('Total hours: {:.3f}'.format( elapsed_time(start_time) / HOURS_TO_SECS))
from pddlstream.algorithms.algorithm import reset_globals from src.command import create_state, iterate_commands from src.observe import observe_pybullet from src.world import World from src.policy import run_policy from src.task import cook_block, TASKS_FNS from run_pybullet import create_parser from multiprocessing import Pool, TimeoutError, cpu_count EXPERIMENTS_DIRECTORY = 'experiments/' TEMP_DIRECTORY = 'temp_parallel/' MAX_TIME = 10 * 60 TIME_BUFFER = 60 SERIAL = is_darwin() VERBOSE = SERIAL SERIALIZE_TASK = True MEAN_TIME_PER_TRIAL = 300 # trial / sec HOURS_TO_SECS = 60 * 60 N_TRIALS = 1 # 1 MAX_MEMORY = 3.5 * KILOBYTES_PER_GIGABYTE SPARE_CORES = 4 POLICIES = [ { 'constrain': False, 'defer': False },