def generate_X_y(non_lens_glob, lens_glob) : '''Reads in data that will be features and targets, outputs as numpy array data''' import glob filenames = glob.glob(non_lens_glob)+glob.glob(lens_glob) X, y = load_data(non_lens_glob, lens_glob) return X, y, filenames
def generate_X_y(non_lens_glob, lens_glob): '''Reads in data that will be features and targets, outputs as numpy array data''' import glob filenames = glob.glob(non_lens_glob) + glob.glob(lens_glob) X, y = load_data(non_lens_glob, lens_glob) return X, y, filenames
import sys from sklearn.cross_validation import train_test_split from StrongCNN.IO.config_parser import parse_configfile from StrongCNN.IO.load_images import load_data from StrongCNN.IO.augment_data import augment_methods, augment_data from _tools import build_parameter_grid, grid_search from StrongCNN.pipeline.build_pipeline import build_pipeline import ast, time cfg = parse_configfile(sys.argv[1]) start_time = time.time() # Collect training and testing data X, y = load_data(cfg['filenames']['non_lens_glob'], cfg['filenames']['lens_glob']) if 'augment_train_data' in cfg.keys() : X, y = augment_data( X, y, cfg['augment_train_data']['method_label'], **ast.literal_eval(cfg['augment_train_data']['method_kwargs'])) print "len(X) = ", len(X) print "len(y) = ", len(y) X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.2 ) print "len(X_train) =", len(X_train) print "len(y_train) =", len(y_train) print "len(X_test) =", len(X_test) print "len(y_test) =", len(y_test) print
parser.add_argument('-r', '--roc_data_filename', required = False) parser.add_argument('-t', '--tpr_filename', required = False) args = vars(parser.parse_args()) cfgdir = args['cfgdir'] set_name = args['set_name'] cfg = parse_configfile(cfgdir) start_time = time.time() assert(set_name in ['test','train']) # Collect testing data X_test, y_test = load_data(cfg[set_name+'_filenames']['non_lens_glob'], cfg[set_name+'_filenames']['lens_glob']) if 'augment_'+set_name+'_data' in cfg.keys() : X_test, y_test = augment_data( X_test, y_test, cfg['augment_'+set_name+'_data']['method_label'], **ast.literal_eval(cfg['augment_'+set_name+'_data']['method_kwargs'])) print "len(X_test) =", len(X_test) print "len(y_test) =", len(y_test) trained_model = load_model(cfgdir+'/'+cfg['model']['pklfile']) X, y, filenames = generate_X_y(cfg[set_name+'_filenames']['non_lens_glob'], cfg[set_name+'_filenames']['lens_glob']) print set_name+' filename glob', cfg[set_name+'_filenames']['non_lens_glob'], cfg[set_name+'_filenames']['lens_glob']
args = vars(parser.parse_args()) cfgdir = args['cfgdir'] set_name = args['set_name'] cfg = parse_configfile(cfgdir) if args['time'] is not None : start_time = time.time() else : print "Time is not on!" sys.exit() assert(set_name in ['test','train']) # Collect testing data X_test, y_test, filenames = load_data(cfg[set_name+'_filenames']['non_lens_glob'], cfg[set_name+'_filenames']['lens_glob']) if 'augment_'+set_name+'_data' in cfg.keys() : X_test, y_test = augment_data( X_test, y_test, cfg['augment_'+set_name+'_data']['method_label'], **ast.literal_eval(cfg['augment_'+set_name+'_data']['method_kwargs'])) print "len(X_test) =", len(X_test) print "len(y_test) =", len(y_test) trained_model = load_model(cfgdir+'/'+cfg['model']['pklfile']) print set_name+' filename glob', cfg[set_name+'_filenames']['non_lens_glob'], cfg[set_name+'_filenames']['lens_glob'] print '' print 'Testing model parameter grid:'
import sys, ast from StrongCNN.IO.config_parser import parse_configfile from StrongCNN.IO.load_images import load_data from StrongCNN.IO.augment_data import augment_methods, augment_data from StrongCNN.pipeline.build_pipeline import build_pipeline from _tools import train_model, dump_model import time cfgdir = sys.argv[1] cfg = parse_configfile(cfgdir) start_time = time.time() # Collect training data X_train, y_train, _ = load_data(cfg['train_filenames']['non_lens_glob'], cfg['train_filenames']['lens_glob']) if 'augment_train_data' in cfg.keys(): X_train, y_train = augment_data( X_train, y_train, cfg['augment_train_data']['method_label'], **ast.literal_eval(cfg['augment_train_data']['method_kwargs'])) print "len(X_train) =", len(X_train) print "len(y_train) =", len(y_train) print X_train[0].shape print "train glob ", cfg['train_filenames']['non_lens_glob'], cfg[ 'train_filenames']['lens_glob'] # Build the pipeline
import sys from sklearn.cross_validation import train_test_split from StrongCNN.IO.config_parser import parse_configfile from StrongCNN.IO.load_images import load_data from StrongCNN.IO.augment_data import augment_methods, augment_data from _tools import build_parameter_grid, grid_search from StrongCNN.pipeline.build_pipeline import build_pipeline import ast, time cfg = parse_configfile(sys.argv[1]) start_time = time.time() # Collect training and testing data X, y, _ = load_data(cfg['filenames']['non_lens_glob'], cfg['filenames']['lens_glob']) if 'augment_train_data' in cfg.keys(): X, y = augment_data( X, y, cfg['augment_train_data']['method_label'], **ast.literal_eval(cfg['augment_train_data']['method_kwargs'])) print "len(X) = ", len(X) print "len(y) = ", len(y) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2) print "len(X_train) =", len(X_train) print "len(y_train) =", len(y_train) print "len(X_test) =", len(X_test) print "len(y_test) =", len(y_test) print