def gen_exp_script(trainsize, n_samples, beta_lambda, nexp, nparallel=15, skill='pour'): with open('run_sample_exp.sh', 'w') as f: for beta_lambda in [.99, .98]: for include_none in [1]: exp_dir = 'sampling_trainsize={}_samples={}_beta_lambdda={}_includenone_{}'.format( trainsize, n_samples, beta_lambda, int(include_none)) skill_dir = get_data_dir(skill) exp_dir = os.path.join(skill_dir, exp_dir) lines = [] for i in range(nexp): expfile = os.path.join( exp_dir, 'experiments_{}.pk{}'.format(i, get_python_version())) outfile = exp_dir + '_experiments_{}.pk{}'.format( i, get_python_version()) if os.path.exists(expfile): continue line = 'python3 -m learn_tools.run_sample {} {} {}trials_n=10000.json {} '.format( trainsize, i, skill_dir, beta_lambda) if include_none: line += '-n ' line += '> {}.out &\n'.format(outfile) lines.append(line) for i in range(nparallel - 1, len(lines), nparallel): lines[i] = lines[i].replace('&', '') f.writelines(lines)
def save_experiments(experiments_dir, experiments): if experiments_dir is None: return None data_path = os.path.join(experiments_dir, 'experiments.pk{}'.format(get_python_version())) write_pickle(data_path, experiments) print('Saved', data_path) return data_path
def save_learner(data_dir, learner): if data_dir is None: return False #domain = learner.func #data_dir = os.path.join(MODEL_DIRECTORY, domain.name) #name = learner.name name = get_label(learner.algorithm) mkdir(data_dir) learner_path = os.path.join(data_dir, '{}.pk{}'.format(name, get_python_version())) print('Saved', learner_path) write_pickle(learner_path, learner) return True
def main(): # collect data in parallel, parameters are generated uniformly randomly in a range # data stored to pour_date/trails_n=10000.json # TODO: ulimit settings # https://ss64.com/bash/ulimit.html # https://stackoverflow.com/questions/938733/total-memory-used-by-python-process # import psutil # TODO: resource.getrusage(resource.RUSAGE_SELF).ru_maxrss assert (get_python_version() == 3) parser = argparse.ArgumentParser() parser.add_argument('-f', '--fn', default=TRAINING, help='The parameter function to use.') parser.add_argument('-n', '--num', type=int, default=10000, help='The number of samples to collect.') parser.add_argument('-p', '--problem', required=True, choices=sorted(SKILL_COLLECTORS.keys()), help='The name of the skill to learn.') parser.add_argument('-t', '--time', type=int, default=1 * 60, help='The max planning runtime for each trial.') parser.add_argument('-v', '--visualize', action='store_true', help='When enabled, visualizes execution.') args = parser.parse_args() serial = is_darwin() assert implies(args.visualize, serial) trials = get_trials(args.problem, args.fn, args.num, max_time=args.time, valid=True, visualize=args.visualize, verbose=serial) data_path = None if serial else get_data_path(args.problem, trials) num_cores = get_num_cores(trials, serial) user_input('Begin?') # TODO: store the generating distribution for samples and objects? print(SEPARATOR) results = run_trials(trials, data_path, num_cores=num_cores)
def save(self, data_dir): from learn_tools.analyze_experiment import get_label if data_dir is None: return False # domain = learner.func # data_dir = os.path.join(MODEL_DIRECTORY, domain.name) # name = learner.name name = get_label(self.algorithm) mkdir(data_dir) learner_path = os.path.join( data_dir, '{}.pk{}'.format(name, get_python_version())) print('Saved learner:', learner_path) write_pickle(learner_path, self) return True
def main(): assert get_python_version() == 3 parser = argparse.ArgumentParser() parser.add_argument('path', help='Analyze an experiment') parser.add_argument('-a', '--all', action='store_true', help='Enables the viewer during planning') parser.add_argument('-d', '--dump', action='store_true', help='Dumps the configuration for each ') args = parser.parse_args() np.set_printoptions(precision=3) if args.dump: enumerate_experiments() else: load_experiment(args.path, overall=args.all)
def main(skill='pour'): parser = argparse.ArgumentParser() parser.add_argument('expid', type=int, help='experiment ID') parser.add_argument( 'beta_lambda', type=float, default=0.99, help='lambda parameter for computing beta from best beta') args = parser.parse_args() beta_lambda = args.beta_lambda expid = args.expid n_samples = 20 trainsize = 1000 exp_dir = 'sampling_trainsize={}_samples={}_beta_lambdda={}'.format( trainsize, n_samples, beta_lambda) if skill is 'pour': exp_dir = os.path.join('data/pour_19-06-13_00-59-21/', exp_dir) domain = load_data(['data/pour_19-06-13_00-59-21/trials_n=10000.json']) elif skill is 'scoop': exp_dir = os.path.join('data/scoop_19-06-10_20-16-59_top-diameter/', exp_dir) domain = load_data( ['data/scoop_19-06-10_20-16-59_top-diameter/trials_n=10000.json']) data_path = os.path.join( exp_dir, 'experiments_{}.pk{}'.format(expid, get_python_version())) if not os.path.exists(data_path): print('{} does not exist'.format(data_path)) return data, seed = read_pickle(data_path) print('sample a new task') current_wd, trial_wd = start_task() sim_world, collector, task, feature, evalfunc, saver = sample_task_with_seed( skill, seed=seed, visualize=True) samples = data[UNIFORM].samples scores = data[UNIFORM].scores good_samples = samples[scores > 0] scores, plan_results = evaluate_samples(sim_world, collector, task, feature, domain, good_samples[:5], evalfunc, saver)
def run_trials(trials, data_path=None, num_cores=False, **kwargs): # https://stackoverflow.com/questions/15314189/python-multiprocessing-pool-hangs-at-join # https://stackoverflow.com/questions/39884898/large-amount-of-multiprocessing-process-causing-deadlock # TODO: multiprocessing still seems to hang on one thread before starting assert (get_python_version() == 3) results = [] if not trials: return results start_time = time.time() serial = (num_cores is False) # None is the max number failures = 0 scored = 0 try: for result in map_general(run_trial, trials, serial, num_cores=num_cores, **kwargs): num_trials = len(results) + failures print( '{}\nTrials: {} | Successes: {} | Failures: {} | Scored: {} | Time: {:.3f}' .format(SEPARATOR, num_trials, len(results), failures, scored, elapsed_time(start_time))) print('Result:', str_from_object(result)) if result is None: failures += 1 print('Error! Trial resulted in an exception') continue scored += int(result.get('score', None) is not None) results.append(result) write_results(data_path, results) # except BaseException as e: # traceback.print_exc() # e finally: print(SEPARATOR) safe_rm_dir(TEMP_DIRECTORY) write_results(data_path, results) print('Hours: {:.3f}'.format(elapsed_time(start_time) / HOURS_TO_SECS)) # TODO: make a generator version of this return results
def get_data_path(stream_name): data_dir = DATA_DIR.format(get_python_version()) file_name = '{}.pkl'.format(stream_name) return os.path.join(data_dir, file_name)
def main(): """ ./home/demo/catkin_percep/collect_scales.sh (includes scale offset) Make sure to start the scales with nothing on them (for calibration) """ assert (get_python_version() == 2) # ariadne has ROS with python2 parser = argparse.ArgumentParser() parser.add_argument('-a', '--active', action='store_true', help='Uses active learning queries.') parser.add_argument('-d', '--debug', action='store_true', help='Disables saving during debugging.') parser.add_argument( '-f', '--fn', type=str, default=TRAINING, # DESIGNED | TRAINING help='The name of or the path to the policy that generates parameters.' ) parser.add_argument('-m', '--material', required=True, choices=sorted(MATERIALS), help='The name of the material being used.') parser.add_argument('-p', '--problem', required=True, choices=sorted(REQUIREMENT_FNS.keys()), help='The name of the skill to learn.') parser.add_argument('-s', '--spoon', default=None, choices=SPOONS, help='The name of the spoon being used.') parser.add_argument('-r', '--train', action='store_true', help='When enabled, uses the training dataset.') parser.add_argument( '-v', '--visualize_planning', action='store_true', help= 'When enabled, visualizes planning rather than the world (for debugging).' ) args = parser.parse_args() # TODO: toggle material default based on task # TODO: label material on the image assert args.material in MODEL_MASSES print('Policy:', args.fn) assert implies(args.problem in ['scoop'], args.spoon is not None) assert implies(args.active, args.train) ros_world = ROSWorld(sim_only=False, visualize=not args.visualize_planning) classes_pub = rospy.Publisher('~collect_classes', String, queue_size=1) with ros_world: set_camera_pose(np.array([1.5, -0.5, 1.5]), target_point=np.array([0.75, 0, 0.75])) arm, is_open = ACTIVE_ARMS[args.problem] open_grippers = {arm: is_open} if args.problem == 'scoop': ros_world.controller.open_gripper(get_arm_prefix(arm), blocking=True) ros_world.controller.speak("{:.0f} seconds to attach {}".format( ATTACH_TIME, format_class(args.spoon))) rospy.sleep(ATTACH_TIME) # Sleep to have time to set the spoon move_to_initial_config(ros_world, open_grippers) #get_other_arm # TODO: cross validation for measuring performance across a few bowls launch = launch_kinect() video_time = time.time() if args.debug: ros_world.controller.speak("Warning! Data will not be saved.") time.sleep(1.0) data_path = None else: # TODO: only create directory if examples made data_path = get_data_path(args.problem, real=True) # TODO: log camera image after the pour policy = args.fn learner_path = None test_data = None if isinstance(policy, str) and os.path.isfile(policy): policy = read_pickle(policy) assert isinstance(policy, ActiveLearner) print(policy) print(policy.algorithm) #policy.transfer_weight = 0 # print(policy.xx.shape) # policy.results = policy.results[:-1] # policy.xx = policy.xx[:-1] # policy.yy = policy.yy[:-1] # policy.weights = policy.weights[:-1] # print(policy.xx.shape) # write_pickle(args.fn, policy) # print('Saved', args.fn) if args.active: # policy.retrain() test_domain = load_data(SCOOP_TEST_DATASETS, verbose=False) test_data = test_domain.create_dataset(include_none=False, binary=False) policy.query_type = STRADDLE # VARIANCE #policy.weights = 0.1*np.ones(policy.yy.shape) # TODO: make this multiplicative #policy.retrain() evaluate_confusions(test_data, policy) else: policy.query_type = BEST ensure_dir(LEARNER_DIRECTORY) date_name = datetime.datetime.now().strftime(DATE_FORMAT) filename = '{}_{}.pk{}'.format(get_label(policy.algorithm), date_name, get_python_version()) learner_path = os.path.join(LEARNER_DIRECTORY, filename) if ACTIVE_FEATURE and args.active: assert isinstance(policy, ActiveLearner) generator = create_active_generator(args, policy) else: generator = create_random_generator(args) pair = next(generator) print('Next pair:', pair) classes_pub.publish('{},{}'.format(*pair)) for phrase in map(format_class, pair): ros_world.controller.speak(phrase) wait_for_user('Press enter to begin') # TODO: change the name of the directory after additional samples results = [] num_trials = num_failures = num_scored = 0 while True: start_time = elapsed_time(video_time) result = run_loop(args, ros_world, policy) print('Result:', str_from_object(result)) print('{}\nTrials: {} | Successes: {} | Failures: {} | Time: {:.3f}'. format(SEPARATOR, num_trials, len(results), num_failures, elapsed_time(video_time))) num_trials += 1 if result is None: # TODO: result['execution'] num_failures += 1 print('Error! Trial resulted in an exception') move_to_initial_config(ros_world, open_grippers) continue end_time = elapsed_time(video_time) print('Elapsed time:', end_time - start_time) # TODO: record the type of failure (planning, execution, etc...) scored = result['score'] is not None num_scored += scored # TODO: print the score if isinstance(policy, ActiveLearner) and args.active: # and scored: update_learner(policy, learner_path, result) evaluate_confusions(test_data, policy) # TODO: how to handle failures that require bad annotations? pair = next(generator) print('Next pair:', pair) classes_pub.publish('{},{}'.format(*pair)) for phrase in map(format_class, pair): ros_world.controller.speak(phrase) annotation = wait_for_user( 'Enter annotation and press enter to continue: ') result.update({ # TODO: record the query_type 'policy': args.fn, 'active_feature': ACTIVE_FEATURE, 'trial': num_trials, 'start_time': start_time, 'end_time': end_time, 'annotation': annotation, }) results.append(result) if data_path is not None: write_results(data_path, results) #if annotation in ['q', 'quit']: # TODO: Ctrl-C to quit # break ros_world.controller.speak("Finished") if launch is not None: launch.shutdown() print('Total time:', elapsed_time(video_time))
def train_parallel(args, n=1): from extrusion.run import plan_extrusion assert SKIP_PERCENTAGE == 0 initial_time = time.time() problems = sorted(set(enumerate_problems()) - set(EXCLUDE)) #problems = ['four-frame'] #problems = ['simple_frame', 'topopt-101_tiny', 'topopt-100_S1_03-14-2019_w_layer'] algorithms = list(ALGORITHMS) if args.disable: for algorithm in LOOKAHEAD_ALGORITHMS: if algorithm in algorithms: algorithms.remove(algorithm) #algorithms = ['regression'] heuristics = HEURISTICS #heuristics = DISTANCE_HEURISTICS + COST_HEURISTICS seeds = list(range(args.num)) if n is None: n = len(seeds) groups = list(chunks(seeds, n=n)) print('Chunks: {}'.format(len(groups))) print('Problems ({}): {}'.format(len(problems), problems)) #problems = [path for path in problems if 'simple_frame' in path] print('Algorithms ({}): {}'.format(len(algorithms), algorithms)) print('Heuristics ({}): {}'.format(len(heuristics), heuristics)) jobs = [[ Configuration(seed, problem, algorithm, heuristic, args.max_time, args.cfree, args.disable, args.stiffness, args.motions, args.ee_only) for seed, algorithm, heuristic in product( group, algorithms, heuristics) ] for problem, group in product(problems, groups)] # TODO: separate out the algorithms again # TODO: print the size per job print('Jobs: {}'.format(len(jobs))) serial = is_darwin() available_cores = cpu_count() num_cores = max(1, min(1 if serial else available_cores - 4, len(jobs))) print('Max Cores:', available_cores) print('Serial:', serial) print('Using Cores:', num_cores) date = datetime.datetime.now().strftime(DATE_FORMAT) filename = '{}.pk{}'.format(date, get_python_version()) path = os.path.join(EXPERIMENTS_DIR, filename) print('Data path:', path) user_input('Begin?') start_time = time.time() timeouts = 0 pool = Pool(processes=num_cores) # , initializer=mute) generator = pool.imap_unordered(plan_extrusion, jobs, chunksize=1) results = [] while True: # TODO: randomly sort instead last_time = time.time() try: for config, data in generator.next(): # timeout=2 * args.max_time) results.append((config, data)) print('{}/{} completed | {:.3f} seconds | timeouts: {} | {}'. format(len(results), len(jobs), elapsed_time(start_time), timeouts, datetime.datetime.now().strftime(DATE_FORMAT))) print(config, data) if results: write_pickle(path, results) print('Saved', path) except StopIteration: break # except TimeoutError: # # TODO: record this as a failure? Nothing is saved though... # timeouts += 1 # #traceback.print_exc() # print('Error! Timed out after {:.3f} seconds'.format(elapsed_time(last_time))) # break # This kills all jobs # #continue # This repeats jobs until success print('Total time:', elapsed_time(initial_time)) return results
def plot(trainsize, n_samples, beta_lambda, includenone=1, ignore_plan_fail=True, show=False, skill='pour'): exp_dir = 'sampling_trainsize={}_samples={}_beta_lambdda={}_includenone_{}'.format( trainsize, n_samples, beta_lambda, includenone) if skill is 'pour': exp_dir = os.path.join(get_data_dir('pour'), exp_dir) elif skill is 'scoop': exp_dir = os.path.join(get_data_dir('scoop'), exp_dir) acc = defaultdict(list) div5 = defaultdict(list) div = defaultdict(list) sample_time = defaultdict(list) num_samples_5 = defaultdict(list) npr = defaultdict(list) for expid in range(50): data_path = os.path.join( exp_dir, 'experiments_{}.pk{}'.format(expid, get_python_version())) if not os.path.exists(data_path): print('{} does not exist'.format(data_path)) continue # print('processing {}'.format(data_path)) data, seed = read_pickle(data_path) for strategy in data: if data[strategy].diversity_5 is None: continue none_res = np.sum([ 1 if pr['score'] is None else 0 for pr in data[strategy].plan_results ]) if data[strategy].precision < 1.: print('low precision - score 0:{}, <0:{}, >0:{}'.format( none_res, np.sum(data[strategy].scores < 0), np.sum(data[strategy].scores > 0))) if ignore_plan_fail: acc[strategy].append( (np.sum(data[strategy].scores > 0) + none_res) / len(data[strategy].scores)) else: acc[strategy].append(data[strategy].precision) npr[strategy].append(none_res / len(data[strategy].scores)) div5[strategy].append(data[strategy].diversity_5) div[strategy].append(data[strategy].diversity) sample_time[strategy].append(data[strategy].sample_time) num_samples_5[strategy].append(data[strategy].num_samples_5) sample_time_plot = [] sample_time_err = [] strategies = [] num_samples_5_plot = [] num_samples_5_err = [] for strategy in sample_time: acc[strategy] = np.array(acc[strategy]) print( '{} fpr = {:.2f} \pm {:.2f} npr = {:.2f} \pm {:.2f} time = {:.2f} \pm {:.2f} n5 = {:.2f} \pm {:.2f} div5 = {:.2f} \pm {:.2f}' .format(strategy, np.mean(1 - acc[strategy]), np.std(1 - acc[strategy]), np.mean(npr[strategy]), np.std(npr[strategy]), np.mean(sample_time[strategy]), np.std(sample_time[strategy]), np.mean(num_samples_5[strategy]), np.std(num_samples_5[strategy]), np.mean(div5[strategy]), np.std(div5[strategy]))) sample_time_plot.append(np.mean(sample_time[strategy])) sample_time_err = np.std(sample_time[strategy]) num_samples_5_plot.append(np.mean(num_samples_5[strategy])) num_samples_5_err.append(np.std(num_samples_5[strategy])) strategies.append(strategy) exp_dir = os.path.join(exp_dir, 'figures') if not os.path.exists(exp_dir): os.mkdir(exp_dir) import matplotlib import matplotlib.pyplot as plt matplotlib.rcParams.update({'font.size': 20, 'legend.fontsize': 15}) markers = ['o', 'v', '<', '>', '1', '2', '3'] colors = ['#1f78b4', '#33a02c', '#e31a1c', '#ff7f00', '#cab2d6', '#fb9a99'] # bar plot for sample time return fig, ax = plt.subplots() ax.bar(range(3), sample_time_plot, yerr=sample_time_err, align='center', alpha=0.5, ecolor='black', capsize=10) ax.set_ylabel('Sample time (seconds)') ax.set_xticks(range(3)) ax.set_xticklabels(strategies) ax.yaxis.grid(True) # Save the figure and show plt.tight_layout() plt.savefig(os.path.join(exp_dir, 'sampletime.pdf')) if show: plt.show() else: plt.clf() # bar plot for num sample 5 fig, ax = plt.subplots() ax.bar(range(3), num_samples_5_plot, yerr=num_samples_5_err, align='center', alpha=0.5, ecolor='black', capsize=10) ax.set_ylabel('Number of samples needed to generate 5 good ones') ax.set_xticks(range(3)) ax.set_xticklabels(strategies) ax.yaxis.grid(True) # Save the figure and show # plt.tight_layout() plt.savefig(os.path.join(exp_dir, 'numsample5.pdf')) if show: plt.show() else: plt.clf() # div-acc using all data points i = 0 for strategy in acc: # plt.scatter(np.mean(div5[strategy]), np.mean(acc[strategy]), marker=markers[i], color=colors[i], label=strategy) plt.scatter(div5[strategy], acc[strategy], marker=markers[i], color=colors[i], label=strategy) plt.xlabel('Diversity of the first 5 samples') plt.ylabel('Accuracy ') i += 1 plt.legend() plt.savefig( os.path.join( exp_dir, 'all_data_div5_ignorePlanFail_{}.pdf'.format(ignore_plan_fail))) if show: plt.show() else: plt.clf() i = 0 for strategy in acc: # plt.scatter(np.mean(div[strategy]), np.mean(acc[strategy]), marker=markers[i], color=colors[i], label=strategy) plt.scatter(div[strategy], acc[strategy], marker=markers[i], color=colors[i], label=strategy) plt.xlabel('Diversity of all 20 samples') plt.ylabel('Accuracy ') i += 1 plt.legend() plt.savefig( os.path.join( exp_dir, 'all_data_div_ignorePlanFail_{}.pdf'.format(ignore_plan_fail))) if show: plt.show() else: plt.clf() # plot the average i = 0 fig, ax = plt.subplots() for strategy in acc: confidence_ellipse(div5[strategy], acc[strategy], ax, alpha=0.5, facecolor=colors[i], edgecolor=colors[i], zorder=0) plt.scatter(np.mean(div5[strategy]), np.mean(acc[strategy]), marker='.', s=10, color=colors[i], label=strategy) plt.xlabel('Diversity of the first 5 samples') plt.ylabel('Accuracy ') i += 1 plt.legend() plt.savefig( os.path.join( exp_dir, 'mean_div5_ignorePlanFail_{}.pdf'.format(ignore_plan_fail))) if show: plt.show() else: plt.clf() i = 0 fig, ax = plt.subplots() for strategy in acc: confidence_ellipse(div[strategy], acc[strategy], ax, alpha=0.5, facecolor=colors[i], edgecolor=colors[i], zorder=0) plt.scatter(np.mean(div[strategy]), np.mean(acc[strategy]), marker='.', s=10, color=colors[i], label=strategy) plt.xlabel('Diversity of all 20 samples') plt.ylabel('Accuracy ') i += 1 plt.legend() plt.savefig( os.path.join( exp_dir, 'mean_div_ignorePlanFail_{}.pdf'.format(ignore_plan_fail))) if show: plt.show() else: plt.clf()
def main(): assert (get_python_version() == 2) # ariadne has ROS with python2 parser = argparse.ArgumentParser() parser.add_argument('-p', '--problem', default='test_pick', help='The name of the problem to solve.') parser.add_argument( '-e', '--execute', action='store_true', help='When enabled, executes the plan using physics simulation.') parser.add_argument( '-c', '--cfree', action='store_true', help='When enabled, disables collision checking (for debugging).') parser.add_argument( '-l', '--learning', action='store_true', help='When enabled, uses learned generators when applicable.') parser.add_argument( '-v', '--visualize_planning', action='store_true', help= 'When enabled, visualizes planning rather than the world (for debugging).' ) args = parser.parse_args() task_fn = get_task_fn(PROBLEMS, args.problem) task = task_fn() print('Task:', task.arms) ros_world = ROSWorld(sim_only=False, visualize=not args.visualize_planning) reset_robot(task, ros_world) ros_world.perception.wait_for_update() if task.use_kitchen: add_table_surfaces(ros_world) ros_world.sync_controllers() # AKA update sim robot joint positions ros_world.perception.update_simulation() add_holding(task, ros_world) print('Surfaces:', ros_world.perception.get_surfaces()) print('Items:', ros_world.perception.get_items()) draw_names(ros_world) draw_forward_reachability(ros_world, task.arms) planning_world = PlanningWorld(task, visualize=args.visualize_planning) planning_world.load(ros_world) print(planning_world) plan = plan_actions(planning_world, collisions=not args.cfree) if plan is None: print('Failed to find a plan') user_input('Finish?') return if not args.execute and not review_plan(task, ros_world, plan): return if args.cfree: print('Aborting execution') return #sparsify_plan(plan) simulator = ros_world.alternate_controller if args.execute else None if simulator is not None: simulator.set_gravity() finished = execute_plan(ros_world, plan, joint_speed=0.25) print('Finished:', finished)
def main(): parser = argparse.ArgumentParser() parser.add_argument('paths', nargs='*', help='Paths to the data.') #parser.add_argument('-a', '--active', type=int, default=0, # None # help='The number of active samples to collect') parser.add_argument( '-d', '--deterministic', action='store_true', help='Whether to deterministically create training splits') parser.add_argument('-n', '--num_trials', type=int, default=-1, help='The number of samples to collect') parser.add_argument('-s', '--save', action='store_true', help='Whether to save the learners') parser.add_argument('-r', '--num_rounds', type=int, default=1, help='The number of rounds to collect') parser.add_argument('-t', '--test', action='store_true', help='Whether to save the data') parser.add_argument('-v', '--visualize', action='store_true', help='When enabled, visualizes execution.') args = parser.parse_args() # TODO: be careful that paging isn't altering the data # TODO: use a different set of randomized parameters for train and test serial = is_darwin() visualize = serial and args.visualize assert implies(visualize, serial) num_trials = get_max_cores( serial) if args.num_trials < 0 else args.num_trials ################################################## #train_sizes = inclusive_range(50, 200, 10) # Best #train_sizes = inclusive_range(50, 400, 10) # F1 #train_sizes = inclusive_range(25, 400, 25) #train_sizes = inclusive_range(50, 100, 5) # Real #train_sizes = inclusive_range(100, 200, 5) #train_sizes = inclusive_range(10, 250, 5) #train_sizes = inclusive_range(35, 70, 5) #train_sizes = inclusive_range(5, 50, 5) #train_sizes = inclusive_range(40, 80, 5) #train_sizes = inclusive_range(100, 1000, 100) #train_sizes = [50] #train_sizes = [250] train_sizes = [1000] #train_sizes = [327] # train + test #train_sizes = inclusive_range(5, 150, 25) #train_sizes = [100] #kernels = ['RBF', 'Matern52', 'MLP'] kernels = ['MLP'] hyperparams = [None] #hyperparams = [True] #hyperparams = [None, True] query_type = BEST # BEST | CONFIDENT | REJECTION | ACTIVE # type of query used to evaluate the learner include_none = False binary = False # 0 => no transfer # 1 => mean transfer # 2 => kernel transfer # 3 => both transfer transfer_weights = [None] #transfer_weights = list(range(4)) #transfer_weights = [0, 1] #transfer_weights = [3] #transfer_weights = np.around(np.linspace(0.0, 1.0, num=1+5, endpoint=True), decimals=3) # max 10 colors #transfer_weights = list(range(1, 1+3)) #split = UNIFORM # BALANCED #print('Split:', split) #parameters = { # 'include None': include_none, # 'binary': binary, # 'split': split, #} # Omitting failed labels is okay because they will never be executed algorithms = [] #algorithms += [(Algorithm(nn_model, label='NN'), [num]) # for nn_model, num in product(NN_MODELS, train_sizes)] #algorithms += [(Algorithm(RANDOM), None), (Algorithm(DESIGNED), None)] #algorithms += [(Algorithm(RF_CLASSIFIER, variance=False, transfer_weight=tw, label='RF'), [num]) # for num, tw in product(train_sizes, [None])] # transfer_weights #algorithms += [(Algorithm(RF_REGRESSOR, variance=False, transfer_weight=tw, label='RF'), [num]) # for num, tw in product(train_sizes, [None])] # transfer_weights #algorithms += [(Algorithm(BATCH_RF, variance=True, transfer_weight=tw, label='RF'), [num]) # for num, tw in product(train_sizes, [None])] # transfer_weights #algorithms += [(Algorithm(BATCH_MAXVAR_RF, variance=True, transfer_weight=tw), train_sizes) # for tw in product(use_vars, [None])] # transfer_weights #algorithms += [(Algorithm(BATCH_STRADDLE_RF, variance=True, transfer_weight=tw), train_sizes) # for tw, in product([None])] # transfer_weights use_vars = [True] # STRADDLE is better than MAXVAR when the learner has a good estimate of uncertainty algorithms += [ (Algorithm(BATCH_GP, kernel, hype, use_var, tw, label='GP'), [num]) # label='GP-{}'.format(kernel) for num, kernel, hype, use_var, tw in product( train_sizes, kernels, hyperparams, use_vars, transfer_weights) ] #algorithms += [(Algorithm(BATCH_MAXVAR_GP, kernel, hype, True, tw, label='GP-Var'), train_sizes) # for kernel, hype, tw in product(kernels, hyperparams, transfer_weights)] #algorithms += [(Algorithm(BATCH_STRADDLE_GP, kernel, hype, True, tw, label='GP-LSE'), train_sizes) # for kernel, hype, tw in product(kernels, hyperparams, transfer_weights)] # default active #algorithms += [(Algorithm(BATCH_STRADDLE_GP, kernel, hype, True, tw, label='GP-LSE2'), train_sizes) # for kernel, hype, tw in product(kernels, hyperparams, transfer_weights)] # active control only # algorithms += [(Algorithm(MAXVAR_GP, kernel, hype, use_var), train_sizes) # for kernel, hype, use_var in product(kernels, hyperparams, use_vars)] #algorithms += [(Algorithm(STRADDLE_GP, kernel, hype, use_var, tw), train_sizes) # for kernel, hype, use_var, tw in product(kernels, hyperparams, use_vars, transfer_weights)] #batch_sizes = inclusive_range(train_sizes[0], 90, 10) #step_size = 10 # TODO: extract from train_sizes #final_size = train_sizes[-1] # Previously didn't have use_var=True # algorithms += [(Algorithm(BATCH_STRADDLE_GP, kernel, hyperparameters=batch_size, variance=True, transfer_weight=tw), # inclusive_range(batch_size, final_size, step_size)) # for kernel, tw, batch_size in product(kernels, transfer_weights, batch_sizes)] # algorithms += [(Algorithm(BATCH_STRADDLE_RF, hyperparameters=batch_size, variance=True, transfer_weight=tw), # inclusive_range(batch_size, final_size, step_size)) # for tw, batch_size in product(transfer_weights, batch_sizes)] print('Algorithms:', algorithms) ################################################## real_world = not args.paths transfer_domain = load_data(TRANSFER_DATASETS, verbose=False) transfer_algorithm = None if real_world and transfer_weights != [None]: #assert transfer_weights[0] is not None transfer_data = transfer_domain.create_dataset( include_none=include_none, binary=binary) transfer_algorithm = Algorithm(BATCH_GP, kernel=kernels[0], variance=use_vars[0]) validity_learner = None #validity_learner = create_validity_classifier(transfer_domain) ################################################## train_paths = args.paths if real_world: train_paths = SCOOP_TRAIN_DATASETS # TRAIN_DATASETS #train_paths = TRANSFER_DATASETS #train_paths = TRAIN_DATASETS + TRANSFER_DATASETS # Train before transfer #scale_paths = TRAIN_DATASETS + TEST_DATASETS scale_paths = None print(SEPARATOR) print('Train paths:', train_paths) domain = load_data(train_paths) print() print(domain) all_data = domain.create_dataset(include_none=include_none, binary=binary, scale_paths=scale_paths) #all_data.results = all_data.results[:1000] num_failed = 0 #num_failed = 100 failed_domain = transfer_domain if real_world else domain failed_results = randomize( result for result in failed_domain.results if not result.get('success', False))[:num_failed] #failed_data = Dataset(domain, failed_results, **all_data.kwargs) test_paths = SCOOP_TEST_DATASETS # TEST_DATASETS | SCOOP_TEST_DATASETS #test_paths = None if real_world and not (set(train_paths) & set(test_paths)): #assert not set(train_paths) & set(test_paths) #max_test = 0 test_data = load_data(test_paths).create_dataset( include_none=False, binary=binary, scale_paths=scale_paths) else: #assert scale_paths is None # TODO: max_train will be too small otherwise test_paths = test_data = None print(SEPARATOR) print('Test paths:', test_paths) all_active_data = None #if real_world: # all_active_data = load_data(ACTIVE_DATASETS).create_dataset(include_none=True, binary=binary, scale_paths=scale_paths) # TODO: could include OS and username if desired date_name = datetime.datetime.now().strftime(DATE_FORMAT) size_str = '[{},{}]'.format(train_sizes[0], train_sizes[-1]) #size_str = '-'.join(map(str, train_sizes)) experiments_name = '{}_r={}_t={}_n={}'.format(date_name, args.num_rounds, size_str, num_trials) trials_per_round = sum( 1 if train_sizes is None else (train_sizes[-1] - train_sizes[0] + len(train_sizes)) for _, train_sizes in algorithms) num_experiments = args.num_rounds * trials_per_round max_train = min( max([0] + [ active_sizes[0] for _, active_sizes in algorithms if active_sizes is not None ]), len(all_data)) max_test = min(len(all_data) - max_train, 1000) ################################################## # #features = ['bowl_height'] # features = ['spoon_height'] # #features = ['bowl_height', 'spoon_height'] # X, Y, _ = all_data.get_data() # #indices = [domain.inputs.index(feature) for feature in features] # #X = X[:,indices] # X = [[result[FEATURE][name] for name in features] for result in all_data.results] # from sklearn.linear_model import LinearRegression # model = LinearRegression(fit_intercept=True, normalize=False) # model.fit(X, Y) # #print(model.get_params()) # print(model.coef_.tolist(), model.intercept_) # print(model.score(X, Y)) #data_dir = os.path.join(DATA_DIRECTORY, domain.name) # EXPERIMENT_DIRECTORY data_dir = os.path.abspath(os.path.join(domain.name, os.path.pardir)) experiments_dir, data_path = None, None if not args.test or not serial: experiments_dir = os.path.join(data_dir, experiments_name) data_path = os.path.join( experiments_dir, 'experiments.pk{}'.format(get_python_version())) ################################################## print(SEPARATOR) print('Name:', experiments_name) print('Experiments:', num_experiments) print('Experiment dir:', experiments_dir) print('Data path:', data_path) print('Examples:', len(all_data)) print('Valid:', sum(result.get('valid', True) for result in all_data.results)) print('Success:', sum(result.get('success', False) for result in all_data.results)) print( 'Scored:', sum( result.get('score', None) is not None for result in all_data.results)) print('Max train:', max_train) print('Max test:', max_test) print('Include None:', include_none) print('Examples: n={}, d={}'.format(len(all_data), domain.dx)) print('Binary:', binary) print('Serial:', serial) print('Estimated hours: {:.3f}'.format(num_experiments * SEC_PER_EXPERIMENT / HOURS_TO_SECS)) user_input('Begin?') ################################################## experiments = [] if experiments_dir is not None: mkdir(experiments_dir) # if os.path.exists(data_path): # experiments.extend(read_pickle(data_path)) # TODO: embed in a KeyboardInterrupt to allow early termination start_time = time.time() for round_idx in range(args.num_rounds): seed = round_idx if args.deterministic else hash( time.time()) # vs just time.time()? random.seed(seed) all_data.shuffle() if test_paths is None: # cannot use test_data #test_data, train_data = split_data(all_data, max_test) train_data = test_data = all_data # Training performance else: train_data = all_data transfer_learner = None if transfer_algorithm is not None: round_data, _ = transfer_data.partition(index=1000) transfer_learner, _ = create_learner(transfer_domain, round_data, transfer_algorithm, verbose=True) transfer_learner.retrain() print(SEPARATOR) print('Round {} | Train examples: {} | Test examples: {}'.format( round_idx, len(train_data), len(test_data))) for algorithm, active_sizes in algorithms: # active_sizes = [first #trainingdata selected from X_train, #active exploration + #trainingdata] print(SEPARATOR) print('Round: {} | {} | Seed: {} | Sizes: {}'.format( round_idx, algorithm, seed, active_sizes)) # TODO: allow keyboard interrupt if active_sizes is None: learner = algorithm.name active_size = train_confusion = None experiments.append( evaluate_learner(domain, seed, train_confusion, test_data, algorithm, learner, active_size, num_trials, serial, args.visualize)) continue # [10 20 25] take first 10 samples from X_train to train the model, 10 samples chosen actively # sequentially + evaluate model, 5 samples chosen actively sequentially + evaluate model # Could always keep around all the examples and retrain # TODO: segfaults when this runs in parallel # TODO: may be able to retrain in parallel if I set OPENBLAS_NUM_THREADS num_batch = active_sizes[0] batch_data, active_data = train_data.partition(num_batch) if all_active_data is not None: active_data = all_active_data.clone() #batch_data.results.extend(failed_results) learner, train_confusion = create_learner( domain, batch_data, algorithm, # alphas, query_type=query_type, verbose=True) learner.validity_learner = validity_learner if transfer_learner is not None: learner.sim_model = transfer_learner.model learner.retrain() for active_size in active_sizes: num_active = active_size - (learner.nx - len(failed_results)) print('\nRound: {} | {} | Seed: {} | Size: {} | Active: {}'. format(round_idx, algorithm, seed, active_size, num_active)) if algorithm.name in CONTINUOUS_ACTIVE_GP: active_learning(learner, num_active, visualize=visualize) #active_learning(learner, num_active, discrete_feature=True, random_feature=False) #active_learning_discrete(learner, active_data, num_active, random_feature=False) elif algorithm.name in BATCH_ACTIVE: active_learning_discrete(learner, active_data, num_active) #active_learning(learner, num_active, discrete_feature=True, random_feature=True) #active_learning_discrete(learner, active_data, num_active, random_feature=True) #if round_dir is not None: # save_learner(round_dir, learner) if args.save: learner.save(data_dir) experiments.append( evaluate_learner(domain, seed, train_confusion, test_data, algorithm, learner, active_size, num_trials, serial, args.visualize)) save_experiments(data_path, experiments) print(SEPARATOR) if experiments: save_experiments(data_path, experiments) plot_experiments(domain, experiments_name, experiments_dir, experiments, include_none=False) print('Experiments: {}'.format(experiments_dir)) print('Total experiments: {}'.format(len(experiments))) print('Total hours: {:.3f}'.format( elapsed_time(start_time) / HOURS_TO_SECS))
def get_data_directory(): return DATA_DIR.format(get_python_version())