def train_main():
with open('config.json') as f:
args = json.load(f)
# make save folder
try:
print('Creating checkpoint folder...')
os.makedirs(args['save_folder'])
except OSError as e:
if e.errno == errno.EEXIST:
print('Directory already exists.')
else:
raise
# read and preprocess data
train_data = pd.read_csv(args['train_path'])
preprocessed_train = preprocess(train_data, args)
if args['train_model_weights']: # resume training
model = restore_model(args['save_folder'], args['train_model_weights'])
else:
model = create_model(args)
save_model(model, args['save_folder'])
# split data for CV
train_set, val_set = val_split(*preprocessed_train[0],
preprocessed_train[1])
model, history = fit_model(model, train_set, val_set, args)
plot_model_history(history, args['save_folder'])
def detect():
form = Form(request.form)
if request.method == 'POST': #and form.validate():
test_values = np.array([
[
form.pregnancies.data, form.glucose.data,
form.bloodPressure.data, form.skinThickness.data,
form.insulin.data, form.bmi.data,
form.diabetesPedigreeFunction.data, form.age.data
],
],
dtype=np.float32)
result = restore_model(test_values)
print("************")
if (result):
result = "Time to go Sugar Free"
else:
result = "Time for dessert"
else:
result = None
iframe = 'details.html'
return render_template('detect.html',
form=form,
result=result,
iframe=iframe)
def test_main():
with open('config.json') as f:
args = json.load(f)
# read and preprocess data
test_data = pd.read_csv(args['test_path'])
test_data.drop('id', axis=1, inplace=True)
preprocessed_test = preprocess(test_data, args, False)
# load checkpoint
model = restore_model(args['save_folder'], args['test_model_weights'])
model.compile(loss='mse', optimizer='adam') # only for evaluation
test(model, preprocessed_test, args, True)
def pr_stats(run, image_dir, label_db, connected_components_threshold):
# TODO: a bunch of this can go back into one off init in a class
_train_opts, model = m.restore_model(run)
label_db = LabelDB(label_db_file=label_db)
set_comparison = u.SetComparison()
# use 4 images for debug
debug_imgs = []
for idx, filename in enumerate(sorted(os.listdir(image_dir))):
# load next image
# TODO: this block used in various places, refactor
img = np.array(Image.open(image_dir + "/" +
filename)) # uint8 0->255 (H, W)
img = img.astype(np.float32)
img = (img / 127.5) - 1.0 # -1.0 -> 1.0 # see data.py
# run through model
prediction = expit(model.predict(np.expand_dims(img, 0))[0])
if len(debug_imgs) < 4:
debug_imgs.append(u.side_by_side(rgb=img, bitmap=prediction))
# calc [(x,y), ...] centroids
predicted_centroids = u.centroids_of_connected_components(
prediction, rescale=2.0, threshold=connected_components_threshold)
# compare to true labels
true_centroids = label_db.get_labels(filename)
true_centroids = [(y, x) for (x, y) in true_centroids] # sigh...
tp, fn, fp = set_comparison.compare_sets(true_centroids,
predicted_centroids)
precision, recall, f1 = set_comparison.precision_recall_f1()
return {
"debug_imgs": debug_imgs,
"precision": precision,
"recall": recall,
"f1": f1
}
mask, rain_list, out_list, output = model(input_x, input_detail)
output = output.cpu().data.numpy().transpose((0, 2, 3, 1))[0]
output = np.where(output > 0., output, 0.)
output = np.where(output < 1., output, 1.)
truth = cv2.cvtColor(y, cv2.COLOR_BGR2GRAY)
pred = cv2.cvtColor(output, cv2.COLOR_BGR2GRAY)
psnr.append(
compare_psnr(np.array(truth * 255., dtype='uint8'),
np.array(pred * 255., dtype='uint8')))
ssim.append(
compare_ssim(np.array(truth * 255., dtype='uint8'),
np.array(pred * 255., dtype='uint8')))
id += 1
print("average psnr = " + str(np.average(psnr)) +
", average ssim_255 = " + str(np.average(ssim)))
if __name__ == '__main__':
test = GenerateData(
input_dir='path/to/test/data/',
label_dir='path/to/test/label/',
)
test.read_data()
model = restore_model('model/model.pkl')
validate(test, model)
def main():
# parse command line arguments
parser = argparse.ArgumentParser(
description='An rCNN scene labeling model.',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument(
'--dataset',
type=str,
default=FROM_GAMES,
choices=DATASETS.keys(),
help=
'Type of dataset to use. This determines the expected format of the data directory'
)
parser.add_argument('--data_dir',
type=str,
help='Directory for image and label data')
parser.add_argument('--category_map',
type=str,
help='File that maps colors ')
parser.add_argument('--hidden_size_1',
type=int,
default=25,
help='First Hidden size for CNN model')
parser.add_argument('--hidden_size_2',
type=int,
default=50,
help='Second Hidden size for CNN model')
parser.add_argument('--patch_size',
type=int,
default=67,
help='Patch size for input images')
parser.add_argument('--learning_rate',
type=float,
default=1e-4,
help='Learning rate for training CNN model')
# TODO figure out batch size
parser.add_argument('--batch_size',
type=int,
default=1,
help='Batch size for training CNN model')
parser.add_argument('--num_epochs',
type=int,
default=1,
help='Number of epochs for training CNN model')
parser.add_argument('--use_patches',
action='store_true',
default=False,
help='Whether to train model on individual patches')
parser.add_argument(
'--patches_per_image',
type=int,
default=1000,
help=
'Number of patches to sample for each image during training of CNN model'
)
parser.add_argument(
'--gaussian_sigma',
type=int,
choices=[15, 30],
default=None,
help='Size of gaussian mask to apply to patches. Not used by default.')
parser.add_argument(
'--fix_random_seed',
action='store_true',
default=False,
help='Whether to reset random seed at start, for debugging.')
parser.add_argument('--model_save_path',
type=str,
default=None,
help='Optional location to store saved model in.')
parser.add_argument('--model_load_path',
type=str,
default=None,
help='Optional location to load saved model from.')
parser.add_argument(
'--dry_run',
action='store_true',
default=False,
help=
'If true, only trains on one image, to test the training code quickly.'
)
parser.add_argument(
'--train_fraction',
type=float,
default=0.8,
help=
'Fraction of data to train on. If positive, trains on first X images, otherwise trains on '
'last X images.')
args = parser.parse_args()
if args.fix_random_seed:
random.seed(0)
# load class labels
category_colors, category_names, names_to_ids = read_object_classes(
args.category_map)
num_classes = len(category_names)
# create function that when called, provides iterator to an epoch of the data
dataset_func = DATASETS[args.dataset]
if args.dry_run:
def dataset_epoch_iter():
return dataset_func(args.data_dir, num_train=1)
else:
def dataset_epoch_iter():
return dataset_func(args.data_dir,
train_fraction=args.train_fraction)
model = CNNModel(args.hidden_size_1,
args.hidden_size_2,
args.batch_size,
num_classes,
args.learning_rate,
num_layers=2)
sess = tf.Session()
init = tf.initialize_all_variables()
sess.run(init)
if args.model_load_path is not None:
restore_model(sess, args.model_load_path)
train(sess,
model,
dataset_epoch_iter,
num_epochs=args.num_epochs,
use_patches=args.use_patches,
patches_per_image=args.patches_per_image,
save_path=args.model_save_path,
gaussian_sigma=args.gaussian_sigma)
print "Saving trained model to %s ..." % args.model_save_path
save_model(sess, args.model_save_path)
def main():
"""
Trains or evaluates an rCNN model.
"""
# parse command line arguments
parser = argparse.ArgumentParser(
description='An rCNN scene labeling model.',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument(
'--dataset',
type=str,
default=FROM_GAMES,
choices=DATASETS.keys(),
help=
'Type of dataset to use. This determines the expected format of the data directory'
)
parser.add_argument('--category_map',
type=str,
help='File that maps colors ')
parser.add_argument('--data_dir',
type=str,
help='Directory for image and label data')
parser.add_argument(
'--data_fraction',
type=float,
default=None,
help=
'Fraction of data to train on. If positive, trains on first X images, otherwise trains on '
'last X images.')
parser.add_argument('--hidden_size_1',
type=int,
default=10,
help='First Hidden size for CNN model')
parser.add_argument('--hidden_size_2',
type=int,
default=10,
help='Second Hidden size for CNN model')
parser.add_argument('--learning_rate',
type=float,
default=0.001,
help='Learning rate for training CNN model')
parser.add_argument('--num_epochs',
type=int,
default=1,
help='Number of epochs for training CNN model')
parser.add_argument('--model_save_path',
type=str,
default=None,
help='Optional location to store saved model in.')
parser.add_argument('--model_load_path',
type=str,
default=None,
help='Optional location to load saved model from.')
parser.add_argument('--training',
action='store_true',
default=False,
help='Whether or not to train model.')
parser.add_argument('--output_dir',
type=str,
default=None,
help='Directory in which to save test output images.')
parser.add_argument(
'--dry_run',
action='store_true',
default=False,
help=
'If true, only trains on one image, to test the training code quickly.'
)
parser.add_argument(
'--fix_random_seed',
action='store_true',
default=False,
help='Whether to reset random seed at start, for debugging.')
args = parser.parse_args()
if args.fix_random_seed:
random.seed(0)
# load class labels
category_colors, category_names = read_object_classes(args.category_map)
num_classes = len(category_names)
# create function that when called, provides iterator to an epoch of the data
dataset_func = DATASETS[args.dataset]
if args.dry_run:
def dataset_epoch_iter():
return dataset_func(args.data_dir, num_samples=1)
else:
def dataset_epoch_iter():
return dataset_func(args.data_dir,
data_fraction=args.data_fraction)
model = RCNNModel(args.hidden_size_1,
args.hidden_size_2,
num_classes,
args.learning_rate,
num_layers=2)
sess = tf.Session()
init = tf.initialize_all_variables()
sess.run(init)
if args.model_load_path is not None:
restore_model(sess, args.model_load_path)
run_model(sess,
model,
dataset_epoch_iter,
num_epochs=args.num_epochs,
training=args.training,
save_path=args.model_save_path,
output_dir=args.output_dir,
color_map=category_colors)
def main():
# parse command line arguments
parser = argparse.ArgumentParser(
description='Evaluate an rCNN scene labelling model.',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--model', type=str, help='Filename of saved model')
parser.add_argument('--category_map',
type=str,
help='File that maps colors ')
parser.add_argument(
'--dataset',
type=str,
default=FROM_GAMES,
choices=DATASETS.keys(),
help=
'Type of dataset to use. This determines the expected format of the data directory'
)
parser.add_argument('--data_dir',
type=str,
help='Directory for image and label data')
parser.add_argument(
'--output_dir',
type=str,
default=None,
help=
'Directory to store model output. By default no output is generated.')
parser.add_argument('--patch_size',
type=int,
default=67,
help='Size of input patches')
parser.add_argument('--use_patches',
action='store_true',
default=False,
help='Whether to evaluate model on individual patches')
parser.add_argument(
'--patches_per_image',
type=int,
default=2000,
help=
'Number of patches to sample from each test image. Not used by default.'
)
parser.add_argument(
'--gaussian_sigma',
type=int,
choices=[15, 30],
default=None,
help='Size of gaussian mask to apply to patches. Not used by default.')
parser.add_argument(
'--test_fraction',
type=float,
default=-0.2,
help=
'Fraction of data to test on. If positive, tests on first X images, otherwise tests on '
'last X images.')
parser.add_argument('--layer',
choices=[1, 2],
type=int,
default=2,
help='Number of rCNN layers to use.')
args = parser.parse_args()
# load class labels
category_colors, category_names, names_to_ids = read_object_classes(
args.category_map)
num_classes = len(category_names)
# load dataset
def dataset_func():
return DATASETS[args.dataset](args.data_dir,
train_fraction=args.test_fraction)
# TODO only test?
# TODO don't hardcode these (maybe store them in config file?)
model = CNNModel(25, 50, 1, num_classes, 1e-4, num_layers=2)
sess = tf.Session()
restore_model(sess, args.model)
test_model(sess,
model,
dataset_func,
args.layer,
use_patches=args.use_patches,
patches_per_image=args.patches_per_image,
gaussian_sigma=args.gaussian_sigma,
output_dir=args.output_dir,
color_map=category_colors)
)
parser.add_argument('--output-label-db',
type=str,
default=None,
help='if not set dont write label_db')
parser.add_argument('--run',
type=str,
required=True,
help='model, also used as subdir for export-pngs')
parser.add_argument(
'--export-pngs',
default='',
help='how, if at all, to export pngs {"", "predictions", "centroids"}')
opts = parser.parse_args()
train_opts, model = m.restore_model(opts.run)
print(model.summary())
if opts.output_label_db:
db = LabelDB(label_db_file=opts.output_label_db)
db.create_if_required()
else:
db = None
if opts.export_pngs:
export_dir = "predict_examples/%s" % opts.run
print("exporting prediction samples to [%s]" % export_dir)
if not os.path.exists(export_dir):
os.makedirs(export_dir)
imgs = os.listdir(opts.image_dir)
