Exemplo n.º 1
0
def generate_X_y(non_lens_glob, lens_glob) :
    '''Reads in data that will be features and targets, outputs as numpy array data'''
    import glob
    filenames = glob.glob(non_lens_glob)+glob.glob(lens_glob)
    X, y = load_data(non_lens_glob, lens_glob)

    return X, y, filenames
Exemplo n.º 2
0
def generate_X_y(non_lens_glob, lens_glob):
    '''Reads in data that will be features and targets, outputs as numpy array data'''
    import glob
    filenames = glob.glob(non_lens_glob) + glob.glob(lens_glob)
    X, y = load_data(non_lens_glob, lens_glob)

    return X, y, filenames
Exemplo n.º 3
0
import sys
from sklearn.cross_validation import train_test_split
from StrongCNN.IO.config_parser import parse_configfile
from StrongCNN.IO.load_images import load_data
from StrongCNN.IO.augment_data import augment_methods, augment_data
from _tools import build_parameter_grid, grid_search
from StrongCNN.pipeline.build_pipeline import build_pipeline
import ast, time

cfg = parse_configfile(sys.argv[1])
start_time = time.time()
# Collect training and testing data
X, y = load_data(cfg['filenames']['non_lens_glob'], 
                             cfg['filenames']['lens_glob'])

if 'augment_train_data' in cfg.keys() :
    X, y = augment_data( X, y, 
                         cfg['augment_train_data']['method_label'],
                         **ast.literal_eval(cfg['augment_train_data']['method_kwargs']))

print "len(X) = ", len(X)
print "len(y) = ", len(y)

X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.2 )

    
print "len(X_train) =", len(X_train)
print "len(y_train) =", len(y_train)
print "len(X_test) =", len(X_test)
print "len(y_test) =", len(y_test)
print
Exemplo n.º 4
0
parser.add_argument('-r', '--roc_data_filename', required = False)
parser.add_argument('-t', '--tpr_filename', required = False) 

args = vars(parser.parse_args())

cfgdir = args['cfgdir']
set_name = args['set_name']

cfg = parse_configfile(cfgdir)

start_time = time.time()

assert(set_name in ['test','train'])

# Collect testing data
X_test, y_test = load_data(cfg[set_name+'_filenames']['non_lens_glob'], 
                           cfg[set_name+'_filenames']['lens_glob'])


if 'augment_'+set_name+'_data' in cfg.keys() :
    X_test, y_test = augment_data( X_test, y_test, 
                                     cfg['augment_'+set_name+'_data']['method_label'],
                                     **ast.literal_eval(cfg['augment_'+set_name+'_data']['method_kwargs']))  
    
print "len(X_test) =", len(X_test)
print "len(y_test) =", len(y_test)

trained_model = load_model(cfgdir+'/'+cfg['model']['pklfile'])

X, y, filenames = generate_X_y(cfg[set_name+'_filenames']['non_lens_glob'], 
                                         cfg[set_name+'_filenames']['lens_glob']) 
print set_name+' filename glob', cfg[set_name+'_filenames']['non_lens_glob'], cfg[set_name+'_filenames']['lens_glob']
Exemplo n.º 5
0
args = vars(parser.parse_args())

cfgdir = args['cfgdir']
set_name = args['set_name']

cfg = parse_configfile(cfgdir)

if args['time'] is not None :  start_time = time.time()
else :
    print "Time is not on!"
    sys.exit()
assert(set_name in ['test','train'])

# Collect testing data
X_test, y_test, filenames = load_data(cfg[set_name+'_filenames']['non_lens_glob'], 
                                      cfg[set_name+'_filenames']['lens_glob'])


if 'augment_'+set_name+'_data' in cfg.keys() :
    X_test, y_test = augment_data( X_test, y_test, 
                                     cfg['augment_'+set_name+'_data']['method_label'],
                                     **ast.literal_eval(cfg['augment_'+set_name+'_data']['method_kwargs']))  
    
print "len(X_test) =", len(X_test)
print "len(y_test) =", len(y_test)

trained_model = load_model(cfgdir+'/'+cfg['model']['pklfile'])

print set_name+' filename glob', cfg[set_name+'_filenames']['non_lens_glob'], cfg[set_name+'_filenames']['lens_glob']
print ''
print 'Testing model parameter grid:'
Exemplo n.º 6
0
import sys, ast
from StrongCNN.IO.config_parser import parse_configfile
from StrongCNN.IO.load_images import load_data
from StrongCNN.IO.augment_data import augment_methods, augment_data
from StrongCNN.pipeline.build_pipeline import build_pipeline
from _tools import train_model, dump_model
import time

cfgdir = sys.argv[1]
cfg = parse_configfile(cfgdir)

start_time = time.time()

# Collect training data
X_train, y_train, _ = load_data(cfg['train_filenames']['non_lens_glob'],
                                cfg['train_filenames']['lens_glob'])

if 'augment_train_data' in cfg.keys():
    X_train, y_train = augment_data(
        X_train, y_train, cfg['augment_train_data']['method_label'],
        **ast.literal_eval(cfg['augment_train_data']['method_kwargs']))

print "len(X_train) =", len(X_train)
print "len(y_train) =", len(y_train)

print X_train[0].shape

print "train glob ", cfg['train_filenames']['non_lens_glob'], cfg[
    'train_filenames']['lens_glob']

# Build the pipeline
Exemplo n.º 7
0
import sys
from sklearn.cross_validation import train_test_split
from StrongCNN.IO.config_parser import parse_configfile
from StrongCNN.IO.load_images import load_data
from StrongCNN.IO.augment_data import augment_methods, augment_data
from _tools import build_parameter_grid, grid_search
from StrongCNN.pipeline.build_pipeline import build_pipeline
import ast, time

cfg = parse_configfile(sys.argv[1])
start_time = time.time()
# Collect training and testing data
X, y, _ = load_data(cfg['filenames']['non_lens_glob'],
                    cfg['filenames']['lens_glob'])

if 'augment_train_data' in cfg.keys():
    X, y = augment_data(
        X, y, cfg['augment_train_data']['method_label'],
        **ast.literal_eval(cfg['augment_train_data']['method_kwargs']))

print "len(X) = ", len(X)
print "len(y) = ", len(y)

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)

print "len(X_train) =", len(X_train)
print "len(y_train) =", len(y_train)
print "len(X_test) =", len(X_test)
print "len(y_test) =", len(y_test)
print