Ejemplo n.º 1
0
    # load train
    train_yaml_file = os.path.join(experiment_root, 'train.yaml');
    train_yaml = load_yaml_template(train_yaml_file);
    
    # fix dataset path
    localizer = PathLocalizer();
    train_yaml = localizer.localize_yaml(train_yaml);
    
    with log_timing(log, 'loading train from {}'.format(train_yaml_file)):      
        train = load_yaml(train_yaml)[0];
    
    return train, model;

if __name__ == '__main__':
    init_logging(pylearn2_loglevel=logging.INFO);
    parser = argparse.ArgumentParser(prog='generate_plots', 
                                     description='generates plots ;-)');
     
    # global options
    parser.add_argument('path', help='root path of the experiment');
     
    args = parser.parse_args();
          
    experiment_root = args.path;

#     experiment_root = '/Users/sstober/git/deepbeat/deepbeat/spearmint/h0_input47/20041_h0_pattern_width_[47]_h0_patterns_[30]_h0_pool_size_[1]_learning_rate_[0.01]'
#     path = '/Users/sstober/git/deepbeat/deepbeat/spearmint/best/h0_1bar_nophase_49bins'

    train, model = load_results(experiment_root);
        
Ejemplo n.º 2
0
                    'condition':
                    'n/a',
                    'channels':
                    channel_ids,
                }

                # save data
                savepath = generate_filepath_from_metadata(metadata)
                save(os.path.join(target_path, savepath),
                     (trial_data, metadata),
                     mkdirs=True)

                # save metadata
                metadb[savepath] = metadata

                log.debug('imported {}={} as {}'.format(
                    label, metadata['meta_label'], savepath))

        save(metadb_file, metadb, mkdirs=True)
    log.info('import finished')


if __name__ == '__main__':
    import deepthought
    from deepthought.util.config_util import init_logging
    init_logging()
    source_path = os.path.join(deepthought.DATA_PATH, 'rwanda2013rhythms',
                               'eeg')
    target_path = os.path.join(deepthought.DATA_PATH, 'rwanda2013rhythms',
                               'multichannel')
    import_dataset(source_path, target_path)
Ejemplo n.º 3
0
'''
Created on Jun 4, 2014

@author: sstober
'''
import itertools
import logging

from pylearn2.utils import serial

from deepthought.util.config_util import init_logging
from deepthought.experiments.ismir2014.extract_results import _extract_best_results, _get_best_epochs

if __name__ == '__main__':

    init_logging(pylearn2_loglevel=logging.INFO)

    # using cached result
    model_file = 'mlp.pkl'
    model = serial.load(model_file)

    channels = model.monitor.channels

    # directly analyze the model from the train object
    best_results = _extract_best_results(
        channels=channels,
        mode='misclass',
        check_dataset='valid',
        check_channels=['_y_misclass'],
    )
    best_epochs = _get_best_epochs(best_results)
Ejemplo n.º 4
0
def run_job(job_id,
            meta_job_path,
            yaml_template_file,
            base_config_path,
            hyper_params,
            cache_path=None):

    # ConstrainedGPEIOptChooser requires NaN or inf to recognize constraints
    #     BAD_SOLUTION_RETURN_VALUE = np.inf;
    BAD_SOLUTION_RETURN_VALUE = 1

    # TODO: nice-to-have: make logging a little nicer
    init_logging(pylearn2_loglevel=logging.INFO)

    for key, value in hyper_params.items():
        hyper_params[key] = value[0]
        log.debug('{} = {}'.format(key, hyper_params[key]))

    base_config = load_config_file(base_config_path)

    # fix dataset path
    localizer_class = base_config.get(
        'localizer_class',
        'deepthought.datasets.rwanda2013rhythms.PathLocalizer')
    # for compatibility with old code
    localizer = load_class(localizer_class)
    base_config = localizer.localize_config(base_config)

    if not hasattr(base_config, 'random_seed') \
                            and not hasattr(hyper_params, 'random_seed'):
        random_seed = random.randint(0, 100)
        hyper_params['random_seed'] = random_seed
        log.debug('using random seed {}'.format(random_seed))

    param_str = ''
    for key in sorted(hyper_params.keys()):  # deterministic order
        param_str += '_{}_{}'.format(key, hyper_params[key])

    verbose_job_id = str(job_id) + param_str
    base_config.verbose_job_id = verbose_job_id

    if cache_path is None:
        cache_path = os.path.join(meta_job_path, 'cache')

    job_output_path = os.path.join(meta_job_path, 'output', str(job_id))
    output_path = os.path.join(cache_path,
                               convert_to_valid_filename(param_str))

    # check whether cached result already exists
    model = None
    failed_file = os.path.join(output_path, 'failed')
    if os.path.exists(output_path):
        # create a link to job-id
        symlink(output_path,
                job_output_path,
                override=True,
                ignore_errors=True)

        # using cached result
        model_file = os.path.join(output_path, 'mlp.pkl')
        if os.path.exists(model_file):
            try:
                with log_timing(
                        log,
                        'loading cached model from {}'.format(model_file)):
                    model = serial.load(model_file)

                    channels = model.monitor.channels
            except Exception as e:
                log.error('unexpected exception loading model from {}: {} \n{}'\
                      .format(model_file, e, traceback.format_exc()))
        else:
            # if mlp.pkl is missing but mlp-best.pkl is there, then it was a bad configuration
            if os.path.exists(failed_file):
                log.info('cache contains \'failed\' flag')
                return BAD_SOLUTION_RETURN_VALUE

    if model is None:

        #     output_path = os.path.join(
        #                                meta_job_path,
        #                                'output',
        #                                convert_to_valid_filename(verbose_job_id)
        #                                );

        # needs to go here to get the internal reference resolved
        base_config.output_path = output_path

        # sanity check of structural parameters:
        if not structural_param_check(
                merge_params(base_config, hyper_params),
                raise_error=False,
        ):
            touch(failed_file, mkdirs=True)
            # set marker
            return BAD_SOLUTION_RETURN_VALUE

        ensure_dir_exists(output_path)
        symlink(output_path,
                job_output_path,
                override=True,
                ignore_errors=True)

        yaml = flatten_yaml(
            yaml_file_path=yaml_template_file,
            base_config=base_config,
            hyper_params=hyper_params,
        )

        save_yaml_file(yaml_str=yaml,
                       yaml_file_path=os.path.join(output_path, 'train.yaml'))

        with log_timing(log, 'loading yaml for job {}'.format(job_id)):
            train = load_yaml(yaml)[0]

        with log_timing(log, 'running job {} '.format(job_id)):
            try:
                train.main_loop()
            except Exception as e:
                log.error('unexpected exception during training: {} \n{}'\
                          .format(e, traceback.format_exc()))
                touch(failed_file, mkdirs=True)
                # set marker
                return BAD_SOLUTION_RETURN_VALUE

        channels = train.model.monitor.channels

    # directly analyze the model from the train object
    best_results = _extract_best_results(
        channels=channels,
        mode='misclass',
        check_dataset='valid',
        check_channels=['_y_misclass'],
    )
    best_epochs = _get_best_epochs(best_results)
    best_epoch = best_epochs[-1]
    # take last entry -> more stable???

    datasets = ['train', 'valid', 'test', 'post']
    measures = ['_y_misclass', '_objective', '_nll']

    print 'results for job {}'.format(job_id)
    for measure, dataset in itertools.product(measures, datasets):
        channel = dataset + measure
        if channel in channels:
            value = float(channels[channel].val_record[best_epoch])
            print '{:>30} : {:.4f}'.format(channel, value)

#     return float(channels['test_y_misclass'].val_record[best_epoch]);
    return float(channels['valid_y_misclass'].val_record[best_epoch])
Ejemplo n.º 5
0
def run_job(job_id, meta_job_path, yaml_template_file, base_config_path, hyper_params, cache_path=None):
    
    # ConstrainedGPEIOptChooser requires NaN or inf to recognize constraints
#     BAD_SOLUTION_RETURN_VALUE = np.inf;
    BAD_SOLUTION_RETURN_VALUE = 1;

    # TODO: nice-to-have: make logging a little nicer
    init_logging(pylearn2_loglevel=logging.INFO);
    
    for key, value in hyper_params.items():
        hyper_params[key] = value[0];
        log.debug('{} = {}'.format(key, hyper_params[key]));
    
    base_config = load_config_file(base_config_path);
    
    # fix dataset path
    localizer_class = base_config.get('localizer_class', 
                                      'deepthought.datasets.rwanda2013rhythms.PathLocalizer'); # for compatibility with old code
    localizer = load_class(localizer_class);
    base_config = localizer.localize_config(base_config);

    if not hasattr(base_config, 'random_seed') \
                            and not hasattr(hyper_params, 'random_seed'):
        random_seed = random.randint(0, 100);
        hyper_params['random_seed'] = random_seed;
        log.debug('using random seed {}'.format(random_seed))

    param_str = '';
    for key in sorted(hyper_params.keys()): # deterministic order
        param_str += '_{}_{}'.format(key, hyper_params[key]);
    
    verbose_job_id = str(job_id) + param_str;
    base_config.verbose_job_id = verbose_job_id;
    

    if cache_path is None:
        cache_path = os.path.join(meta_job_path, 'cache');

    job_output_path = os.path.join(meta_job_path, 'output', str(job_id));
    output_path = os.path.join( 
                               cache_path, 
                               convert_to_valid_filename(param_str)
                               );                            

    # check whether cached result already exists
    model = None;
    failed_file = os.path.join(output_path, 'failed');
    if os.path.exists(output_path):
        # create a link to job-id                    
        symlink(output_path, job_output_path, override=True, ignore_errors=True);
        
        # using cached result
        model_file = os.path.join(output_path, 'mlp.pkl');
        if os.path.exists(model_file):    
            try: 
                with log_timing(log, 'loading cached model from {}'.format(model_file)): 
                    model = serial.load(model_file);
                
        
                    channels = model.monitor.channels;
            except Exception as e:
                log.error('unexpected exception loading model from {}: {} \n{}'\
                      .format(model_file, e, traceback.format_exc())); 
        else:
            # if mlp.pkl is missing but mlp-best.pkl is there, then it was a bad configuration
            if os.path.exists(failed_file):
                log.info('cache contains \'failed\' flag'); 
                return BAD_SOLUTION_RETURN_VALUE;
        
    if model is None:
        
#     output_path = os.path.join(
#                                meta_job_path, 
#                                'output', 
#                                convert_to_valid_filename(verbose_job_id)
#                                );    

        # needs to go here to get the internal reference resolved
        base_config.output_path = output_path;     
        
        # sanity check of structural parameters:
        if not structural_param_check(
                                   merge_params(base_config, hyper_params), 
                                   raise_error=False,
                                   ):
            touch(failed_file, mkdirs=True); # set marker
            return BAD_SOLUTION_RETURN_VALUE;
    
        ensure_dir_exists(output_path);
        symlink(output_path, job_output_path, override=True, ignore_errors=True);
    
        
        yaml = flatten_yaml(
                            yaml_file_path = yaml_template_file, 
                            base_config = base_config, 
                            hyper_params = hyper_params,
                            );
    
        save_yaml_file(
                       yaml_str = yaml, 
                       yaml_file_path = os.path.join(output_path, 'train.yaml')
                       );    
        
        with log_timing(log, 'loading yaml for job {}'.format(job_id)):
            train = load_yaml(yaml)[0];
        
        with log_timing(log, 'running job {} '.format(job_id)):   
            try: 
                train.main_loop();
            except Exception as e:
                log.error('unexpected exception during training: {} \n{}'\
                          .format(e, traceback.format_exc()));
                touch(failed_file, mkdirs=True); # set marker
                return BAD_SOLUTION_RETURN_VALUE;
        
        channels = train.model.monitor.channels;

                 
    # directly analyze the model from the train object   
    best_results = _extract_best_results(
                                         channels=channels,
                                         mode='misclass', 
                                         check_dataset='valid',
                                         check_channels=['_y_misclass'],
                                         );    
    best_epochs = _get_best_epochs(best_results);
    best_epoch = best_epochs[-1]; # take last entry -> more stable???
    
    datasets = ['train', 'valid', 'test', 'post'];
    measures = ['_y_misclass', '_objective', '_nll'];
    
    print 'results for job {}'.format(job_id);
    for measure,dataset in itertools.product(measures,datasets):
        channel = dataset+measure;
        if channel in channels:
            value = float(channels[channel].val_record[best_epoch]);
            print '{:>30} : {:.4f}'.format(channel, value);
     
#     return float(channels['test_y_misclass'].val_record[best_epoch]);
    return float(channels['valid_y_misclass'].val_record[best_epoch]);
Ejemplo n.º 6
0
                    'stimulus'      : stimulus,
                    'stimulus_id'   : stimulus_id,
                    'rhythm_type'   : label_converter.get_label(stimulus_id, 'rhythm'),
                    'tempo'         : label_converter.get_label(stimulus_id, 'tempo'),
                    'audio_file'    : label_converter.get_label(stimulus_id, 'audio_file'),
                    'trial_no'      : 1,
                    'trial_type'    : 'perception',
                    'condition'     : 'n/a',
                    'channels'      : channel_ids,
                }

                # save data
                savepath = generate_filepath_from_metadata(metadata)
                save(os.path.join(target_path, savepath), (trial_data, metadata), mkdirs=True)

                # save metadata
                metadb[savepath] = metadata

                log.debug('imported {}={} as {}'.format(label, metadata['meta_label'], savepath))

        save(metadb_file, metadb, mkdirs=True)
    log.info('import finished')


if __name__ == '__main__':
    import deepthought
    from deepthought.util.config_util import init_logging
    init_logging()
    source_path = os.path.join(deepthought.DATA_PATH, 'rwanda2013rhythms', 'eeg')
    target_path = os.path.join(deepthought.DATA_PATH, 'rwanda2013rhythms', 'multichannel')
    import_dataset(source_path, target_path)