from StrongCNN.IO.config_parser import parse_configfile from StrongCNN.IO.load_images import load_data from StrongCNN.IO.augment_data import augment_methods, augment_data from _tools import build_parameter_grid, grid_search from StrongCNN.pipeline.build_pipeline import build_pipeline import ast, time cfg = parse_configfile(sys.argv[1]) start_time = time.time() # Collect training and testing data X, y = load_data(cfg['filenames']['non_lens_glob'], cfg['filenames']['lens_glob']) if 'augment_train_data' in cfg.keys() : X, y = augment_data( X, y, cfg['augment_train_data']['method_label'], **ast.literal_eval(cfg['augment_train_data']['method_kwargs'])) print "len(X) = ", len(X) print "len(y) = ", len(y) X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.2 ) print "len(X_train) =", len(X_train) print "len(y_train) =", len(y_train) print "len(X_test) =", len(X_test) print "len(y_test) =", len(y_test) print # Build the parameter grid
cfg = parse_configfile(cfgdir) if args['time'] is not None : start_time = time.time() else : print "Time is not on!" sys.exit() assert(set_name in ['test','train']) # Collect testing data X_test, y_test, filenames = load_data(cfg[set_name+'_filenames']['non_lens_glob'], cfg[set_name+'_filenames']['lens_glob']) if 'augment_'+set_name+'_data' in cfg.keys() : X_test, y_test = augment_data( X_test, y_test, cfg['augment_'+set_name+'_data']['method_label'], **ast.literal_eval(cfg['augment_'+set_name+'_data']['method_kwargs'])) print "len(X_test) =", len(X_test) print "len(y_test) =", len(y_test) trained_model = load_model(cfgdir+'/'+cfg['model']['pklfile']) print set_name+' filename glob', cfg[set_name+'_filenames']['non_lens_glob'], cfg[set_name+'_filenames']['lens_glob'] print '' print 'Testing model parameter grid:' for k,v in cfg['param_grid'].iteritems() : print k, v print '' if cfg[set_name+'_filenames']['lens_glob'] != '' and cfg[set_name+'_filenames']['non_lens_glob'] != '' :
set_name = args['set_name'] cfg = parse_configfile(cfgdir) start_time = time.time() assert(set_name in ['test','train']) # Collect testing data X_test, y_test = load_data(cfg[set_name+'_filenames']['non_lens_glob'], cfg[set_name+'_filenames']['lens_glob']) if 'augment_'+set_name+'_data' in cfg.keys() : X_test, y_test = augment_data( X_test, y_test, cfg['augment_'+set_name+'_data']['method_label'], **ast.literal_eval(cfg['augment_'+set_name+'_data']['method_kwargs'])) print "len(X_test) =", len(X_test) print "len(y_test) =", len(y_test) trained_model = load_model(cfgdir+'/'+cfg['model']['pklfile']) X, y, filenames = generate_X_y(cfg[set_name+'_filenames']['non_lens_glob'], cfg[set_name+'_filenames']['lens_glob']) print set_name+' filename glob', cfg[set_name+'_filenames']['non_lens_glob'], cfg[set_name+'_filenames']['lens_glob'] print '' print 'Testing model parameter grid:' for k,v in cfg['param_grid'].iteritems() : print k, v print ''
from StrongCNN.pipeline.build_pipeline import build_pipeline from _tools import train_model, dump_model import time cfgdir = sys.argv[1] cfg = parse_configfile(cfgdir) start_time = time.time() # Collect training data X_train, y_train, _ = load_data(cfg['train_filenames']['non_lens_glob'], cfg['train_filenames']['lens_glob']) if 'augment_train_data' in cfg.keys(): X_train, y_train = augment_data( X_train, y_train, cfg['augment_train_data']['method_label'], **ast.literal_eval(cfg['augment_train_data']['method_kwargs'])) print "len(X_train) =", len(X_train) print "len(y_train) =", len(y_train) print X_train[0].shape print "train glob ", cfg['train_filenames']['non_lens_glob'], cfg[ 'train_filenames']['lens_glob'] # Build the pipeline pipeline = build_pipeline(cfg['image_processing'].values(), cfg['classifier']['label']) params = {k: ast.literal_eval(v) for k, v in cfg['param_grid'].iteritems()}