def process_jobs(config): common_config = config.common; for job in config.jobs: log.info('Processing job {} with base {}'.format(job.name, job.base)); job_config = merge_params(common_config, config[job.base]); log.debug('job overrides: {}'.format(job.overrides)); job_config = merge_params(job_config, job.overrides); job_config.experiment_root = os.path.join( config.output_root, job_config.type, job.name ); log.debug('experiment root: {}'.format(job_config.experiment_root)); print job_config; # try: if job_config.type == 'cnn': train_convnet(job_config); elif job_config.type == 'fftcnn': train_convnet(job_config); elif job_config.type == 'sda': train_mlp(job_config); else: log.error('unsupported job type {}'.format(job_config.type));
def flatten_yaml(yaml_file_path, base_config=None, hyper_params=None): yaml_template = load_yaml_template(yaml_file_path) params = merge_params(default_params=base_config, override_params=hyper_params) yaml = yaml_template % params return yaml
def flatten_yaml(yaml_file_path, base_config=None, hyper_params=None): yaml_template = load_yaml_template(yaml_file_path) params = merge_params( default_params = base_config, override_params = hyper_params ) yaml = yaml_template % params return yaml
def process_jobs(config): common_config = config.common for job in config.jobs: log.info('Processing job {} with base {}'.format(job.name, job.base)) job_config = merge_params(common_config, config[job.base]) log.debug('job overrides: {}'.format(job.overrides)) job_config = merge_params(job_config, job.overrides) job_config.experiment_root = os.path.join(config.output_root, job_config.type, job.name) log.debug('experiment root: {}'.format(job_config.experiment_root)) print job_config # try: if job_config.type == 'cnn': train_convnet(job_config) elif job_config.type == 'fftcnn': train_convnet(job_config) elif job_config.type == 'sda': train_mlp(job_config) else: log.error('unsupported job type {}'.format(job_config.type))
def run_experiment(config, hyper_params, random_seeds): experiment_root = hyper_params['experiment_root'] best_acc = -1 best_results = [np.NAN, np.NAN, np.NAN] for seed in random_seeds: hyper_params['random_seed'] = seed hyper_params['experiment_root'] = experiment_root + '.' + str(seed) params = merge_params(config, hyper_params) if os.path.exists(os.path.join(params.experiment_root, 'mlp.pkl')): print 'found existing mlp.pkl: {}'.format(params.experiment_root) else: print 'no mlp.pkl found at: {}'.format(params.experiment_root) if not config.get('only_extract_results', False): train_convnet(params) try: values = extract_results(params.experiment_root, mode='misclass') results = np.multiply( 100, [ # 1 - values['test_y_misclass'], # 1 - values['test_wseq_misclass_rate'], # 1 - values['test_wtrial_misclass_rate']]); 1 - values['frame_misclass'], 1 - values['sequence_misclass'], 1 - values['trial_misclass'] ]) # save the best results if np.max(results[2]) > best_acc: best_results = results best_acc = np.max(results[2]) except: print traceback.format_exc() results = [np.NAN, np.NAN, np.NAN] print 'results for seed {}: {}'.format(seed, results) if params.save_output: output = extract_output(params, values['best_epoch']) save(os.path.join(params.experiment_root, 'best_output.pklz'), output) print 'best results: {}'.format(best_results) return best_results
def load_datasets_for_subjects(dataset_params, subjects, suffix=''): datasets = {} for key, params in dataset_params.items(): if not key in dataset_names: continue; params['subjects'] = subjects; params['name'] = params['name']+suffix; dataset_config = merge_params(config, params); dataset, dataset_yaml = load_yaml_file( os.path.join(os.path.dirname(__file__), 'run', 'dataset_template.yaml'), params=dataset_config, ); # log.info('dataset loaded. X={} y={}'.format(dataset.X.shape, dataset.y.shape)); datasets[key+suffix] = dataset; del dataset, dataset_yaml; return datasets;
def run_experiment(config, hyper_params, random_seeds): experiment_root = hyper_params['experiment_root']; best_acc = -1; best_results = [np.NAN, np.NAN, np.NAN]; for seed in random_seeds: hyper_params['random_seed'] = seed; hyper_params['experiment_root'] = experiment_root + '.' + str(seed); params = merge_params(config, hyper_params); if os.path.exists(os.path.join(params.experiment_root, 'mlp.pkl')): print 'found existing mlp.pkl: {}'.format(params.experiment_root); else: print 'no mlp.pkl found at: {}'.format(params.experiment_root); if not config.get('only_extract_results', False): train_convnet(params); try: values = extract_results(params.experiment_root, mode='misclass'); results = np.multiply(100, [ # 1 - values['test_y_misclass'], # 1 - values['test_wseq_misclass_rate'], # 1 - values['test_wtrial_misclass_rate']]); 1 - values['frame_misclass'], 1 - values['sequence_misclass'], 1 - values['trial_misclass']]); # save the best results if np.max(results[2]) > best_acc: best_results = results; best_acc = np.max(results[2]); except: print traceback.format_exc(); results = [np.NAN, np.NAN, np.NAN]; print 'results for seed {}: {}'.format(seed, results); if params.save_output: output = extract_output(params, values['best_epoch']); save(os.path.join(params.experiment_root, 'best_output.pklz'), output); print 'best results: {}'.format(best_results); return best_results;
def load_datasets_for_subjects(dataset_params, subjects, suffix=''): datasets = {} for key, params in dataset_params.items(): if not key in dataset_names: continue params['subjects'] = subjects params['name'] = params['name'] + suffix dataset_config = merge_params(config, params) dataset, dataset_yaml = load_yaml_file( os.path.join(os.path.dirname(__file__), 'run', 'dataset_template.yaml'), params=dataset_config, ) # log.info('dataset loaded. X={} y={}'.format(dataset.X.shape, dataset.y.shape)); datasets[key + suffix] = dataset del dataset, dataset_yaml return datasets
job_config.experiment_root = os.path.join( config.output_root, job_config.type, job.name ); log.debug('experiment root: {}'.format(job_config.experiment_root)); print job_config; # try: if job_config.type == 'cnn': train_convnet(job_config); elif job_config.type == 'fftcnn': train_convnet(job_config); elif job_config.type == 'sda': train_mlp(job_config); else: log.error('unsupported job type {}'.format(job_config.type)); # except: # log.fatal("Unexpected error:", sys.exc_info()); if __name__ == '__main__': default_config = os.path.join(os.path.dirname(__file__), 'batch.cfg'); config = load_config(default_config=default_config, reset_logging=False); config = merge_params(load_config_file(default_config), config); process_jobs(config);
def pair_cross_trial_test(config, pairs=None): if pairs is None: pairs = [ [18, 19], [20, 21], [22, 23], [ 0, 1], [15, 9], [16, 17], [11, 5], [12, 6], [ 2, 3], [10, 4], [13, 7], [14, 8], ]; # config.experiment_root = os.path.join(config.experiment_root, 'cross-trial') ; accuracy = np.zeros(len(pairs)) results = np.zeros([len(pairs),3]); for i in xrange(len(pairs)): test_stimulus_ids = pairs[i]; train_stimulus_ids = set(); for j in xrange(48): if not j in test_stimulus_ids: train_stimulus_ids.add(j); train_stimulus_ids = list(train_stimulus_ids); log.info('training stimuli: {} \t test stimuli: {}'.format(train_stimulus_ids, test_stimulus_ids)); hyper_params = { 'experiment_root' : os.path.join(config.experiment_root, 'pair'+str(pairs[i])), 'remove_train_stimulus_ids' : test_stimulus_ids, 'remove_test_stimulus_ids' : train_stimulus_ids, }; params = merge_params(config, hyper_params); # dummy values for testing # results[i] = [i, i*10, i*100.]; # accuracy[i] = 0; # continue; if os.path.exists(os.path.join(params.experiment_root, 'mlp.pkl')): print 'found existing mlp.pkl: {}'.format(params.experiment_root); else: print 'no mlp.pkl found at: {}'.format(params.experiment_root); if not config.get('only_extract_results', False): train_mlp(params); values = extract_results(params.experiment_root, mode='misclass'); results[i] = np.multiply(100, [ # 1 - values['test_y_misclass'], # 1 - values['test_wseq_misclass_rate'], # 1 - values['test_wtrial_misclass_rate']]); 1 - values['frame_misclass'], 1 - values['sequence_misclass'], 1 - values['trial_misclass']]); accuracy[i] = 100 * (1 - extract_best_result(params.experiment_root, mode='misclass', check_dataset='test')[0]); print results; print results.mean(axis=0); print results.max(axis=1); print accuracy; print accuracy.mean(); return results, accuracy;
log.debug('job overrides: {}'.format(job.overrides)) job_config = merge_params(job_config, job.overrides) job_config.experiment_root = os.path.join(config.output_root, job_config.type, job.name) log.debug('experiment root: {}'.format(job_config.experiment_root)) print job_config # try: if job_config.type == 'cnn': train_convnet(job_config) elif job_config.type == 'fftcnn': train_convnet(job_config) elif job_config.type == 'sda': train_mlp(job_config) else: log.error('unsupported job type {}'.format(job_config.type)) # except: # log.fatal("Unexpected error:", sys.exc_info()); if __name__ == '__main__': default_config = os.path.join(os.path.dirname(__file__), 'batch.cfg') config = load_config(default_config=default_config, reset_logging=False) config = merge_params(load_config_file(default_config), config) process_jobs(config)
config = load_config(default_config=os.path.join(os.path.dirname(__file__), 'train_convnet.cfg'), reset_logging=False) batch_subjects = config.get('batch_subjects', xrange(13)) # per person for i in batch_subjects: hyper_params = { 'experiment_root': os.path.join(config.experiment_root, 'individual', 'subj' + str(i + 1)), 'subjects': [i] } params = merge_params(config, hyper_params) run(params) # # run cross_trial_test # try: # params = merge_params(config, {}); # cross_trial_test(params); # Note: this modifies config! # except: # print "Unexpected error:", sys.exc_info()[0] # # # for slow rhythms # hyper_params = { # 'experiment_root' : os.path.join(config.experiment_root, 'slow'), # 'subjects' : [0,1,2,6,7,8] # };
def pair_cross_trial_test(config, pairs=None): if pairs is None: pairs = [ [18, 19], [20, 21], [22, 23], [0, 1], [15, 9], [16, 17], [11, 5], [12, 6], [2, 3], [10, 4], [13, 7], [14, 8], ] # config.experiment_root = os.path.join(config.experiment_root, 'cross-trial') ; accuracy = np.zeros(len(pairs)) results = np.zeros([len(pairs), 3]) for i in xrange(len(pairs)): test_stimulus_ids = pairs[i] train_stimulus_ids = set() for j in xrange(48): if not j in test_stimulus_ids: train_stimulus_ids.add(j) train_stimulus_ids = list(train_stimulus_ids) log.info('training stimuli: {} \t test stimuli: {}'.format( train_stimulus_ids, test_stimulus_ids)) hyper_params = { 'experiment_root': os.path.join(config.experiment_root, 'pair' + str(pairs[i])), 'remove_train_stimulus_ids': test_stimulus_ids, 'remove_test_stimulus_ids': train_stimulus_ids, } params = merge_params(config, hyper_params) # dummy values for testing # results[i] = [i, i*10, i*100.]; # accuracy[i] = 0; # continue; if os.path.exists(os.path.join(params.experiment_root, 'mlp.pkl')): print 'found existing mlp.pkl: {}'.format(params.experiment_root) else: print 'no mlp.pkl found at: {}'.format(params.experiment_root) if not config.get('only_extract_results', False): train_mlp(params) values = extract_results(params.experiment_root, mode='misclass') results[i] = np.multiply( 100, [ # 1 - values['test_y_misclass'], # 1 - values['test_wseq_misclass_rate'], # 1 - values['test_wtrial_misclass_rate']]); 1 - values['frame_misclass'], 1 - values['sequence_misclass'], 1 - values['trial_misclass'] ]) accuracy[i] = 100 * (1 - extract_best_result( params.experiment_root, mode='misclass', check_dataset='test')[0]) print results print results.mean(axis=0) print results.max(axis=1) print accuracy print accuracy.mean() return results, accuracy
with log_timing(log, 'training MLP'): train.main_loop(); log.info('done'); def get_default_config_path(): return os.path.join(os.path.dirname(__file__),'train_sda_mlp.cfg'); if __name__ == '__main__': # config = load_config(default_config='../../train_sda.cfg', reset_logging=False); config = load_config(default_config=get_default_config_path(), reset_logging=False); hyper_params = { }; params = merge_params(config, hyper_params); if not config.get('only_extract_results', False): train_mlp(params); scan_for_best_performance(params.experiment_root, 'valid_y_misclass'); scan_for_best_performance(params.experiment_root, 'valid_ptrial_misclass_rate') values = extract_results(config.experiment_root, mode='misclass'); print np.multiply(100, [ # 1 - values['test_y_misclass'], # 1 - values['test_wseq_misclass_rate'], # 1 - values['test_wtrial_misclass_rate']]); 1 - values['frame_misclass'],
def run_job(job_id, meta_job_path, yaml_template_file, base_config_path, hyper_params, cache_path=None): # ConstrainedGPEIOptChooser requires NaN or inf to recognize constraints # BAD_SOLUTION_RETURN_VALUE = np.inf; BAD_SOLUTION_RETURN_VALUE = 1 # TODO: nice-to-have: make logging a little nicer init_logging(pylearn2_loglevel=logging.INFO) for key, value in hyper_params.items(): hyper_params[key] = value[0] log.debug('{} = {}'.format(key, hyper_params[key])) base_config = load_config_file(base_config_path) # fix dataset path localizer_class = base_config.get( 'localizer_class', 'deepthought.datasets.rwanda2013rhythms.PathLocalizer') # for compatibility with old code localizer = load_class(localizer_class) base_config = localizer.localize_config(base_config) if not hasattr(base_config, 'random_seed') \ and not hasattr(hyper_params, 'random_seed'): random_seed = random.randint(0, 100) hyper_params['random_seed'] = random_seed log.debug('using random seed {}'.format(random_seed)) param_str = '' for key in sorted(hyper_params.keys()): # deterministic order param_str += '_{}_{}'.format(key, hyper_params[key]) verbose_job_id = str(job_id) + param_str base_config.verbose_job_id = verbose_job_id if cache_path is None: cache_path = os.path.join(meta_job_path, 'cache') job_output_path = os.path.join(meta_job_path, 'output', str(job_id)) output_path = os.path.join(cache_path, convert_to_valid_filename(param_str)) # check whether cached result already exists model = None failed_file = os.path.join(output_path, 'failed') if os.path.exists(output_path): # create a link to job-id symlink(output_path, job_output_path, override=True, ignore_errors=True) # using cached result model_file = os.path.join(output_path, 'mlp.pkl') if os.path.exists(model_file): try: with log_timing( log, 'loading cached model from {}'.format(model_file)): model = serial.load(model_file) channels = model.monitor.channels except Exception as e: log.error('unexpected exception loading model from {}: {} \n{}'\ .format(model_file, e, traceback.format_exc())) else: # if mlp.pkl is missing but mlp-best.pkl is there, then it was a bad configuration if os.path.exists(failed_file): log.info('cache contains \'failed\' flag') return BAD_SOLUTION_RETURN_VALUE if model is None: # output_path = os.path.join( # meta_job_path, # 'output', # convert_to_valid_filename(verbose_job_id) # ); # needs to go here to get the internal reference resolved base_config.output_path = output_path # sanity check of structural parameters: if not structural_param_check( merge_params(base_config, hyper_params), raise_error=False, ): touch(failed_file, mkdirs=True) # set marker return BAD_SOLUTION_RETURN_VALUE ensure_dir_exists(output_path) symlink(output_path, job_output_path, override=True, ignore_errors=True) yaml = flatten_yaml( yaml_file_path=yaml_template_file, base_config=base_config, hyper_params=hyper_params, ) save_yaml_file(yaml_str=yaml, yaml_file_path=os.path.join(output_path, 'train.yaml')) with log_timing(log, 'loading yaml for job {}'.format(job_id)): train = load_yaml(yaml)[0] with log_timing(log, 'running job {} '.format(job_id)): try: train.main_loop() except Exception as e: log.error('unexpected exception during training: {} \n{}'\ .format(e, traceback.format_exc())) touch(failed_file, mkdirs=True) # set marker return BAD_SOLUTION_RETURN_VALUE channels = train.model.monitor.channels # directly analyze the model from the train object best_results = _extract_best_results( channels=channels, mode='misclass', check_dataset='valid', check_channels=['_y_misclass'], ) best_epochs = _get_best_epochs(best_results) best_epoch = best_epochs[-1] # take last entry -> more stable??? datasets = ['train', 'valid', 'test', 'post'] measures = ['_y_misclass', '_objective', '_nll'] print 'results for job {}'.format(job_id) for measure, dataset in itertools.product(measures, datasets): channel = dataset + measure if channel in channels: value = float(channels[channel].val_record[best_epoch]) print '{:>30} : {:.4f}'.format(channel, value) # return float(channels['test_y_misclass'].val_record[best_epoch]); return float(channels['valid_y_misclass'].val_record[best_epoch])
def run_job(job_id, meta_job_path, yaml_template_file, base_config_path, hyper_params, cache_path=None): # ConstrainedGPEIOptChooser requires NaN or inf to recognize constraints # BAD_SOLUTION_RETURN_VALUE = np.inf; BAD_SOLUTION_RETURN_VALUE = 1; # TODO: nice-to-have: make logging a little nicer init_logging(pylearn2_loglevel=logging.INFO); for key, value in hyper_params.items(): hyper_params[key] = value[0]; log.debug('{} = {}'.format(key, hyper_params[key])); base_config = load_config_file(base_config_path); # fix dataset path localizer_class = base_config.get('localizer_class', 'deepthought.datasets.rwanda2013rhythms.PathLocalizer'); # for compatibility with old code localizer = load_class(localizer_class); base_config = localizer.localize_config(base_config); if not hasattr(base_config, 'random_seed') \ and not hasattr(hyper_params, 'random_seed'): random_seed = random.randint(0, 100); hyper_params['random_seed'] = random_seed; log.debug('using random seed {}'.format(random_seed)) param_str = ''; for key in sorted(hyper_params.keys()): # deterministic order param_str += '_{}_{}'.format(key, hyper_params[key]); verbose_job_id = str(job_id) + param_str; base_config.verbose_job_id = verbose_job_id; if cache_path is None: cache_path = os.path.join(meta_job_path, 'cache'); job_output_path = os.path.join(meta_job_path, 'output', str(job_id)); output_path = os.path.join( cache_path, convert_to_valid_filename(param_str) ); # check whether cached result already exists model = None; failed_file = os.path.join(output_path, 'failed'); if os.path.exists(output_path): # create a link to job-id symlink(output_path, job_output_path, override=True, ignore_errors=True); # using cached result model_file = os.path.join(output_path, 'mlp.pkl'); if os.path.exists(model_file): try: with log_timing(log, 'loading cached model from {}'.format(model_file)): model = serial.load(model_file); channels = model.monitor.channels; except Exception as e: log.error('unexpected exception loading model from {}: {} \n{}'\ .format(model_file, e, traceback.format_exc())); else: # if mlp.pkl is missing but mlp-best.pkl is there, then it was a bad configuration if os.path.exists(failed_file): log.info('cache contains \'failed\' flag'); return BAD_SOLUTION_RETURN_VALUE; if model is None: # output_path = os.path.join( # meta_job_path, # 'output', # convert_to_valid_filename(verbose_job_id) # ); # needs to go here to get the internal reference resolved base_config.output_path = output_path; # sanity check of structural parameters: if not structural_param_check( merge_params(base_config, hyper_params), raise_error=False, ): touch(failed_file, mkdirs=True); # set marker return BAD_SOLUTION_RETURN_VALUE; ensure_dir_exists(output_path); symlink(output_path, job_output_path, override=True, ignore_errors=True); yaml = flatten_yaml( yaml_file_path = yaml_template_file, base_config = base_config, hyper_params = hyper_params, ); save_yaml_file( yaml_str = yaml, yaml_file_path = os.path.join(output_path, 'train.yaml') ); with log_timing(log, 'loading yaml for job {}'.format(job_id)): train = load_yaml(yaml)[0]; with log_timing(log, 'running job {} '.format(job_id)): try: train.main_loop(); except Exception as e: log.error('unexpected exception during training: {} \n{}'\ .format(e, traceback.format_exc())); touch(failed_file, mkdirs=True); # set marker return BAD_SOLUTION_RETURN_VALUE; channels = train.model.monitor.channels; # directly analyze the model from the train object best_results = _extract_best_results( channels=channels, mode='misclass', check_dataset='valid', check_channels=['_y_misclass'], ); best_epochs = _get_best_epochs(best_results); best_epoch = best_epochs[-1]; # take last entry -> more stable??? datasets = ['train', 'valid', 'test', 'post']; measures = ['_y_misclass', '_objective', '_nll']; print 'results for job {}'.format(job_id); for measure,dataset in itertools.product(measures,datasets): channel = dataset+measure; if channel in channels: value = float(channels[channel].val_record[best_epoch]); print '{:>30} : {:.4f}'.format(channel, value); # return float(channels['test_y_misclass'].val_record[best_epoch]); return float(channels['valid_y_misclass'].val_record[best_epoch]);