def train(config):
"""Trains the ReInspect model using SGD with momentum
and prints out the logging information."""
net = apollocaffe.ApolloNet()
net_config = config["net"]
data_config = config["data"]
solver = config["solver"]
logging = config["logging"]
image_mean = load_data_mean(data_config["idl_mean"],
net_config["img_width"],
net_config["img_height"],
image_scaling=1.0)
input_gen = load_idl_list(data_config["train_idl"], image_mean, net_config)
input_gen_test = load_idl_list(data_config["test_idl"], image_mean,
net_config)
forward(net, input_gen.next(), config["net"])
net.draw_to_file(logging["schematic_path"])
if solver["weights"]:
net.load(solver["weights"])
else:
net.load(googlenet.weights_file())
loss_hist = {"train": [], "test": []}
loggers = [
apollocaffe.loggers.TrainLogger(logging["display_interval"],
logging["log_file"]),
apollocaffe.loggers.TestLogger(solver["test_interval"],
logging["log_file"]),
apollocaffe.loggers.SnapshotLogger(logging["snapshot_interval"],
logging["snapshot_prefix"]),
]
for i in range(solver["start_iter"], solver["max_iter"]):
if i % solver["test_interval"] == 0:
net.phase = 'test'
for _ in range(solver["test_iter"]):
forward(net, input_gen_test.next(), config["net"], False)
loss_hist["test"].append(net.loss)
net.phase = 'train'
forward(net, input_gen.next(), config["net"])
loss_hist["train"].append(net.loss)
net.backward()
learning_rate = (solver["base_lr"] *
(solver["gamma"])**(i // solver["stepsize"]))
net.update(lr=learning_rate,
momentum=solver["momentum"],
clip_gradients=solver["clip_gradients"])
for logger in loggers:
logger.log(
i, {
'train_loss': loss_hist["train"],
'test_loss': loss_hist["test"],
'apollo_net': net,
'start_iter': 0
})
def evaluate(config, test_data):
eval_net = apollocaffe.ApolloNet()
net_config = config["net"]
feat=test_data['feats'][0]
feat2=test_data['feats2'][0]
feat=np.hstack((feat,feat2))
evaluate_forward(eval_net, net_config, feat, test_data['scene_feats'][0])
net_add='models/model_lstm_ord.h5'
eval_net.load(net_add)
residuals=[]
fid=open('result_ord.txt','w')
tar_dir='result_ord/'
if not os.path.isdir(tar_dir): os.mkdir(tar_dir)
for idx in xrange(len(test_data['labels'])):
feat=test_data['feats'][idx]
feat2=test_data['feats2'][idx]
feat=np.hstack((feat,feat2))
scene_feat=test_data['scene_feats'][idx]
label=test_data['labels'][idx]
predictions=[]
max_len=min(8, feat.shape[0])
all_permu=list(itertools.permutations(range(max_len),max_len))
some_permu=all_permu[0:min(10,len(all_permu))]
for idx in some_permu:
prediction = evaluate_forward(eval_net, net_config, feat[idx,:],scene_feat)
prediction = np.array(prediction).sum()
predictions.append(prediction)
print 'File: %s, Predicted: %0.5f' % (label, np.array(predictions).mean())
fid.write('%s %0.5f\n' % (label, np.array(predictions).mean()))
with open(tar_dir+label.replace('.jpg', '.txt').replace('.jpeg', '.txt').replace('.png', '.txt'),'w') as ff: ff.write('%0.5f\n'%np.array(predictions).mean())
print "%d images processed using %s" % (len(test_data['labels']),net_add)
fid.close()
def main():
"""Sets up all the configurations for apollocaffe, and ReInspect
and runs the trainer."""
parser = apollocaffe.base_parser()
parser.add_argument('--config', required=True)
args = parser.parse_args()
config = json.load(open(args.config, 'r'))
if args.weights is not None:
config["solver"]["weights"] = args.weights
apollocaffe.set_random_seed(config["solver"]["random_seed"])
apollocaffe.set_device(args.gpu)
apollocaffe.set_cpp_loglevel(args.loglevel)
net = apollocaffe.ApolloNet()
image_mean = load_image_mean_from_binproto(config['data']["idl_mean"])
fake_input_en = {"image": np.zeros((1, 3, 227, 227))}
forward(net, fake_input_en, deploy=True)
if config["solver"]["weights"]:
net.load(config["solver"]["weights"])
else:
raise Exception('weights file is not provided!')
run_socket(net, 13502, image_mean)
def __init__(self, config, opt_config):
self.config = config
self.opt_config = opt_config
self.opt_state = adadelta.State()
self.nmns = dict()
self.apollo_net = apollocaffe.ApolloNet()
def __init__(self, config, opt_config):
self.config = config
self.opt_config = opt_config
self.nets = dict()
self.current_net = None
self.apollo_net = apollocaffe.ApolloNet()
self.sq_grads = dict()
self.sq_updates = dict()
if hasattr(config, "load_lstm"):
self.apollo_net.load(self.config.load_lstm)
def train(config):
"""Trains the ReInspect model using SGD with momentum
and prints out the logging information."""
net = apollocaffe.ApolloNet()
net_config = config["net"]
data_config = config["data"]
solver = config["solver"]
logging = config["logging"]
image_mean = load_data_mean(data_config["idl_mean"],
net_config["img_width"],
net_config["img_height"],
image_scaling=1.0)
input_gen_test = load_idl(data_config["test_idl"],
image_mean,
net_config,
jitter=False)
forward(net, input_gen_test.next(), config["net"])
net.draw_to_file(logging["schematic_path"])
if solver["weights"]:
net.load(solver["weights"])
else:
raise Exception('weights not specified!')
i = '1'
output_file = open('data/new1_lstm_hidden' + i + '_mean.m', 'w')
output_file.write('new1_lstm_hidden' + i + ' = [')
for _ in range(20):
input_gen_test.next()
for input_en in input_gen_test:
# plt.imshow(input_en['raw'])
# plt.show()
forward(net, input_en, config["net"])
lstm_hidden0 = net.blobs['lstm_hidden' + i].data # shape: (300, 1024)
box_flags = net.blobs['box_flags'].data # shape: (300, 1, 5, 1)
N = lstm_hidden0.shape[0]
for n in range(N):
output_file.write(str(box_flags[n, 0, int(i), 0]) + ' ')
for c in range(lstm_hidden0.shape[1]):
output_file.write(str(lstm_hidden0[n, c]) + ' ')
output_file.write('\n')
break
output_file.write('];')
output_file.close()
def setup(config, device_gpu):
apollocaffe.set_device(device_gpu)
net = apollocaffe.ApolloNet()
image_mean = load_image_mean_from_binproto(config["idl_mean"])
fake_input_en = {"image":
np.zeros((config['new_width'], config['new_height']))}
forward(net, fake_input_en, deploy=True)
net.draw_to_file(logging["schematic_path"])
if solver["weights"]:
net.load(config["weights"])
else:
raise Exception('weights file is not provided!')
return net
def __init__(self, input_image, head_boxes, net=None):
self.net = apollocaffe.ApolloNet()
self.head_boxes = np.copy(head_boxes)
self.output_ready = False
self.final_maps = []
self.predictions = []
# preparing inputs
self.inputs = input_gazenet(input_image, self.head_boxes)
# define network structure
# input with some dummy data first, it doesn't matter the result
self.forward(img=self.inputs.input_image_resize, \
eye_img=self.inputs.eye_images_resize[0], \
eye_grid=self.inputs.eyes_grids_flat[0])
# init the net with parameters trained in binary_w.caffemodel
self.net.load(self.weights_file())
def evaluate(config):
eval_net = apollocaffe.ApolloNet()
# evaluate the net once to set up structure before loading parameters
net_config = config["net"]
evaluate_forward(eval_net, net_config)
eval_net.load("%s_%d.h5" % (config["logging"]["snapshot_prefix"],
config["solver"]["max_iter"] - 1))
targets, predictions, residuals = evaluate_forward(eval_net, net_config)
targets = ',\t'.join([('%.2f' % x) for x in targets])
predictions = ',\t'.join([('%.2f' % x) for x in predictions])
residuals = ',\t'.join([('%.2f' % x) for x in residuals])
print 'The target values were:'
print targets
print ''
print "The network predicted:"
print predictions
print ''
print "The residuals are:"
print residuals
def deploy(config):
"""Trains the ReInspect model using SGD with momentum
and prints out the logging information."""
net = apollocaffe.ApolloNet()
data_config = config["data"]
solver = config["solver"]
logging = config["logging"]
net_config = config['net']
image_mean = load_image_mean_from_binproto(data_config["idl_mean"])
input_gen = load_txt(data_config["deploy_idl"], net_config, image_mean)
classes_names = [str(name.strip()) for name in open(logging['classes_file'])]
print classes_names
forward(net, input_gen.next(), deploy=True)
net.draw_to_file(logging["schematic_path"])
if solver["weights"]:
net.load(solver["weights"])
else:
raise Exception('weights file is not provided!')
for input_en in input_gen:
start = time.time()
net.phase = 'test'
test_accuracy = []
probs = forward(net, input_en, deploy=True)
pred_class, value = get_max_index(probs[0,:])
print input_en['imname'], pred_class
# plt.imshow(input_en['raw'])
# plt.show()
if value < 0.5:
class_name = 'unknown'
else:
class_name = classes_names[pred_class]
save_image(input_en['raw'],
os.path.join(logging['deploy_out_dir'], class_name),
input_en['imname'].split('/')[-1])
def train(config):
net = apollocaffe.ApolloNet()
net_config = config["net"]
solver = config["solver"]
logging = config["logging"]
loggers = [
apollocaffe.loggers.TrainLogger(logging["display_interval"]),
apollocaffe.loggers.SnapshotLogger(logging["snapshot_interval"],
logging["snapshot_prefix"]),
]
train_loss = []
for i in range(solver["max_iter"]):
forward(net, net_config)
train_loss.append(net.loss)
net.backward()
lr = (solver["base_lr"] * (solver["gamma"])**(i // solver["stepsize"]))
net.update(lr=lr,
momentum=solver["momentum"],
clip_gradients=solver["clip_gradients"])
for logger in loggers:
logger.log(i, {"train_loss": train_loss, "apollo_net": net})
def train(config):
"""Trains the ReInspect model using SGD with momentum
and prints out the logging information."""
# # # init arguments # # #
net_config = config["net"]
data_config = config["data"]
solver = config["solver"]
logging = config["logging"]
MMD_config = config["MMD"]
image_mean = load_data_mean(data_config["idl_mean"],
net_config["img_width"],
net_config["img_height"],
image_scaling=1.0)
# # # load image data # # #
re_train_gen = load_idl(data_config["reinspect_train_idl"], image_mean,
net_config)
re_test_gen = load_idl(data_config["reinspect_test_idl"],
image_mean,
net_config,
jitter=False,
if_random=False)
boost_test_gen = load_idl(data_config["boost_test_idl"],
image_mean,
net_config,
jitter=False,
if_random=False)
boost_imname_list = load_imname_list(data_config['boost_idl'])
# # # init apollocaffe # # #
# source net
src_net_ = apollocaffe.ApolloNet()
net_config["ignore_label"] = -1
forward(src_net_, re_test_gen.next(), net_config, enable_ip_split=False)
if solver["weights"]:
src_net_.load(solver["weights"])
else:
src_net_.load(googlenet.weights_file())
# transform inner product layer of src_net
src_net = apollocaffe.ApolloNet()
net_convert_ip_2_ip_split(src_net_, src_net, re_test_gen.next(),
net_config)
# boost net with MMD loss layer
boost_net = apollocaffe.ApolloNet()
net_config["ignore_label"] = 0
forward(boost_net, boost_test_gen.next(), net_config)
add_MMD_loss_layer(boost_net, src_net, MMD_config)
boost_net.copy_params_from(src_net)
# reinspect net with MMD loss layer
re_net = apollocaffe.ApolloNet()
net_config["ignore_label"] = 1
forward(re_net, re_test_gen.next(), net_config)
add_MMD_loss_layer(re_net, src_net, MMD_config)
# # # init log # # #
loss_hist = {"train": [], "test": []}
loggers = [
apollocaffe.loggers.TrainLogger(logging["display_interval"],
logging["log_file"]),
apollocaffe.loggers.TestLogger(solver["test_interval"],
logging["log_file"]),
apollocaffe.loggers.SnapshotLogger(logging["snapshot_interval"],
logging["snapshot_prefix"]),
]
# # # boost # # #
for i in range(solver["start_iter"], solver["max_iter"]):
# # test and evaluation
if i % solver["test_interval"] == 0:
boost_net.phase = 'test'
test_loss = []
test_loss2 = []
cc_list = []
ce_list = []
ca_list = []
cp_list = []
for _ in range(solver["test_iter"]):
input_en = boost_test_gen.next()
mmd_stat(boost_net, re_test_gen, net_config, MMD_config)
exit()
(cc, ce, ca, cp) = get_accuracy(boost_net, input_en,
net_config)
add_MMD_loss_layer(boost_net, src_net, MMD_config)
test_loss.append(boost_net.loss)
cc_list.append(cc)
ce_list.append(ce)
ca_list.append(ca)
cp_list.append(cp)
loss_hist["test"].append(np.mean(test_loss))
precision = np.sum(cc_list) / np.sum(cp_list)
recall = np.sum(cc_list) / np.sum(ca_list)
print 'hungarian loss:', np.mean(test_loss)
print input_en['imname']
for layers, loss_weight in zip(MMD_config['layers'],
MMD_config['loss_weights']):
if loss_weight == 0:
continue
print boost_net.blobs[layers[0] + '_loss'].diff[0], '*',
for layer in layers:
print boost_net.blobs[layer + '_loss'].data[0],
print ''
print 'iterate: %6.d error, recall, F1: (%.3f %.3f) -> %.3f' % (
i, 1 - precision, recall, 2 * precision * recall /
(precision + recall))
# # deploy for subsequent training, select boost_iter images from boost_iter_max images
if i % solver["boost_interval"] == 0:
boost_deploy_list = []
random.shuffle(boost_imname_list)
for imname in boost_imname_list[:solver[
"boost_iter_max"]]: # not all images are needed for boost training
input_en = generate_input_en(imname, image_mean, net_config)
(bbox, conf) = forward(boost_net,
input_en,
net_config,
deploy=True)
add_MMD_loss_layer(boost_net, src_net, MMD_config)
mmd_losses = []
for layers, loss_weight in zip(MMD_config['layers'],
MMD_config['loss_weights']):
if loss_weight == 0:
continue
mmd_losses += [
loss_weight * boost_net.blobs[x + '_loss'].data[0]
for x in layers
]
boost_deploy_list.append({
'imname': imname,
'bbox': bbox,
'conf': conf,
'MMDLoss': np.mean(mmd_losses)
})
boost_deploy_list = sorted(
boost_deploy_list,
key=lambda x: x['MMDLoss'],
reverse=solver['reverse'])[:solver['boost_iter']]
thres = 0.9
boot_train_gen = convert_deploy_2_train(
boost_deploy_list,
image_mean,
net_config,
max_heads=solver['boost_iter_max_heads'],
threshold=thres,
if_random=solver['random'])
boot_train_num = boot_train_gen.next()['num']
if i % solver["boost_interval"] >= boot_train_num:
continue
# # train # #
learning_rate = (solver["base_lr"] *
(solver["gamma"])**(i // solver["stepsize"]))
# feed new data to source net to aid "add_MMD_loss_layer"
forward(src_net, re_test_gen.next(), net_config)
# train on reinspect dataset
re_net.phase = "train"
re_net.copy_params_from(boost_net)
for _ in range(solver['Old_over_New']):
forward(re_net, re_train_gen.next(), net_config)
add_MMD_loss_layer(re_net, src_net, MMD_config)
if not math.isnan(re_net.loss):
re_net.backward()
re_net.update(lr=learning_rate,
momentum=solver["momentum"],
clip_gradients=solver["clip_gradients"])
boost_net.copy_params_from(re_net)
# train on boost dataset
boost_net.phase = 'train'
forward(boost_net, boot_train_gen.next(), net_config)
add_MMD_loss_layer(boost_net, src_net, MMD_config)
loss_hist["train"].append(boost_net.loss)
if not math.isnan(
boost_net.loss): # loss may be "nan", caused by ignore label.
boost_net.backward()
boost_net.update(lr=learning_rate,
momentum=solver["momentum"],
clip_gradients=solver["clip_gradients"])
for logger in loggers:
logger.log(
i, {
'train_loss': loss_hist["train"],
'test_loss': loss_hist["test"],
'apollo_net': boost_net,
'start_iter': 0
})
def train(config):
"""Trains the ReInspect model using SGD with momentum
and prints out the logging information."""
net = apollocaffe.ApolloNet()
data_config = config["data"]
solver = config["solver"]
logging = config["logging"]
net_config = config['net']
image_mean = load_image_mean_from_binproto(data_config["idl_mean"])
input_gen = load_txt(data_config["train_idl"], net_config, image_mean)
input_gen_test = load_txt(data_config["test_idl"],
net_config,
image_mean,
jitter=False)
forward(net, input_gen.next())
net.draw_to_file(logging["schematic_path"])
if solver["weights"]:
net.load(solver["weights"])
else:
net.load(alexnet.weights_file())
loss_hist = {"train": [], "test": []}
loggers = [
apollocaffe.loggers.TrainLogger(logging["display_interval"],
logging["log_file"]),
apollocaffe.loggers.TestLogger(solver["test_interval"],
logging["log_file"]),
apollocaffe.loggers.SnapshotLogger(logging["snapshot_interval"],
logging["snapshot_prefix"]),
]
for i in range(solver["start_iter"], solver["max_iter"]):
if i % solver["test_interval"] == 0:
net.phase = 'test'
test_loss = []
test_accuracy = []
for _ in range(solver["test_iter"]):
input_en = input_gen_test.next()
forward(net, input_en, False)
test_loss.append(net.loss)
test_accuracy.append(get_accuracy(net, input_en))
loss_hist["test"].append(np.mean(test_loss))
print 'accuracy', np.mean(test_accuracy)
net.phase = 'train'
forward(net, input_gen.next())
loss_hist["train"].append(net.loss)
net.backward()
learning_rate = (solver["base_lr"] *
(solver["gamma"])**(i // solver["stepsize"]))
net.update(lr=learning_rate, momentum=solver["momentum"])
for logger in loggers:
logger.log(
i, {
'train_loss': loss_hist["train"],
'test_loss': loss_hist["test"],
'apollo_net': net,
'start_iter': 0
})
def train(config):
"""Trains the ReInspect model using SGD with momentum
and prints out the logging information."""
net = apollocaffe.ApolloNet()
net_config = config["net"]
data_config = config["data"]
solver = config["solver"]
logging = config["logging"]
image_mean = load_data_mean(
data_config["idl_mean"], net_config["img_width"],
net_config["img_height"], image_scaling=1.0)
input_gen = load_idl(data_config["train_idl"],
image_mean, net_config, jitter=False)
input_gen_test = load_idl(data_config["test_idl"],
image_mean, net_config, jitter=False)
forward(net, input_gen.next(), config["net"])
try:
net.load(solver["weights"])
except:
pass
loss_hist = {"train": [], "test": []}
loggers = [
apollocaffe.loggers.TrainLogger(logging["display_interval"],
logging["log_file"]),
apollocaffe.loggers.TestLogger(solver["test_interval"],
logging["log_file"]),
apollocaffe.loggers.SnapshotLogger(logging["snapshot_interval"],
logging["snapshot_prefix"]),
]
for i in range(solver["start_iter"], solver["max_iter"]):
if i % solver["test_interval"] == 0:
net.phase = 'test'
test_loss = []
#save the weights
net.save(solver["weights"])
for x in range(solver["test_iter"]):
test_input_data = input_gen_test.next()
tic = time.time()
forward(net, test_input_data, config["net"], False)
print "Forward pass", time.time() - tic
test_loss.append(net.loss)
loss_hist["test"].append(np.mean(test_loss))
net.phase = 'train'
forward(net, input_gen.next(), config["net"])
loss_hist["train"].append(net.loss)
net.backward()
learning_rate = (solver["base_lr"] *
(solver["gamma"])**(i // solver["stepsize"]))
net.update(lr=learning_rate, momentum=solver["momentum"],
clip_gradients=solver["clip_gradients"])
for logger in loggers:
logger.log(i, {'train_loss': loss_hist["train"],
'test_loss': loss_hist["test"],
'apollo_net': net, 'start_iter': 0})
net.f(
InnerProduct('ip',
vocab_size,
bottoms=['dropout'],
param_names=['softmax_ip_weights',
'softmax_ip_bias']))
net.blobs['ip'].data[:] *= i_temperature
net.f(Softmax('softmax', bottoms=['ip']))
net.blobs['lstm_hidden_prev'].data_tensor.copy_from(
net.blobs['lstm_hidden_next'].data_tensor)
net.blobs['lstm_mem_prev'].data_tensor.copy_from(
net.blobs['lstm_mem_next'].data_tensor)
print ''.join([chr(x) for x in output_words])
net = apollocaffe.ApolloNet()
sentence_batches = get_data_batch()
forward(net, sentence_batches)
train_loss_hist = []
display = 20
loggers = [
apollocaffe.loggers.TrainLogger(display),
apollocaffe.loggers.SnapshotLogger(1000, '/tmp/char')
]
for i in range(10000):
forward(net, sentence_batches)
train_loss_hist.append(net.loss)
net.backward()
def test(config):
"""Test the model and output to test/output."""
net = apollocaffe.ApolloNet()
net_config = config["net"]
data_config = config["data"]
solver = config["solver"]
image_mean = load_data_mean(
data_config["idl_mean"], net_config["img_width"],
net_config["img_height"], image_scaling=1.0)
input_gen_test = load_idl(data_config["test_idl"],
image_mean, net_config, jitter=False)
forward(net, input_gen_test.next(), config["net"])
try:
net.load(solver["weights"])
except:
pass
net.phase = 'test'
test_loss = []
for i in range(solver["test_iter"]):
input_test = input_gen_test.next()
image = input_test["raw"]
tic = time.time()
bbox, conf = forward(net, input_test, config["net"], True)
print "forward deploy time", time.time() - tic
bbox_list = bbox
conf_list = conf
pix_per_w = 32
pix_per_h = 32
all_rects = [[[] for x in range(net_config['grid_width'])] for y in range(net_config['grid_height'])]
for n in range(len(bbox_list)):
for k in range(net_config['grid_width'] * net_config['grid_height']):
y = int(k / net_config['grid_width'])
x = int(k % net_config['grid_width'])
bbox = bbox_list[n][k]
conf = conf_list[n][k,1].flatten()[0]
abs_cx = pix_per_w/2 + pix_per_w*x + int(bbox[0,0,0])
abs_cy = pix_per_h/2 + pix_per_h*y+int(bbox[1,0,0])
w = bbox[2,0,0]
h = bbox[3,0,0]
all_rects[y][x].append(Rect(abs_cx,abs_cy,w,h,conf))
acc_rects = stitch_rects(all_rects)
#print acc_rects
for idx, rect in enumerate(acc_rects):
if rect.true_confidence < 0.9:
print 'rejected', rect.true_confidence
continue
else:
print 'found', rect.true_confidence
cv2.rectangle(image, (rect.cx-int(rect.width/2), rect.cy-int(rect.height/2)),
(rect.cx+int(rect.width/2), rect.cy+int(rect.height/2)),
(255,0,0),
2)
cv2.imwrite("test_output/img_out%s.jpg" % i, image)
def train(config):
"""Trains the ReInspect model using SGD with momentum
and prints out the logging information."""
net = apollocaffe.ApolloNet()
net_config = config["net"]
data_config = config["data"]
solver = config["solver"]
logging = config["logging"]
input_gen = load_indice_file(data_config["train_file"], net_config)
input_gen_test = load_indice_file(data_config["test_file"], net_config)
forward(net, input_gen.next(), config["net"])
# net.draw_to_file(logging["schematic_path"]) # !!!!
if solver["weights"]:
net.load(solver["weights"])
# else:
# net.load(googlenet.weights_file())
loss_hist = {"train": [], "test": [], "test_accuracy": []}
loggers = [
apollocaffe.loggers.TrainLogger(logging["display_interval"],
logging["log_file"]),
apollocaffe.loggers.TestLogger(solver["test_interval"],
logging["log_file"]),
apollocaffe.loggers.SnapshotLogger(logging["snapshot_interval"],
logging["snapshot_prefix"]),
]
for i in range(solver["start_iter"], solver["max_iter"]):
if i % solver["test_interval"] == 0:
# test loss
net.phase = 'test'
test_loss = []
test_accuracy = []
for _ in range(solver["test_iter"]):
input_entity = input_gen_test.next()
forward(net, input_entity, config["net"], False)
test_loss.append(net.loss)
c_accuracy = get_net_accuracy(net, input_entity, config['net'])
test_accuracy.append(c_accuracy)
loss_hist["test"].append(np.mean(test_loss))
loss_hist["test_accuracy"].append(np.mean(test_accuracy))
print 'test accuracy:', loss_hist["test_accuracy"][-1]
net.phase = 'train'
forward(net, input_gen.next(), config["net"])
loss_hist["train"].append(net.loss)
net.backward()
learning_rate = (solver["base_lr"] *
(solver["gamma"])**(i // solver["stepsize"]))
net.update(lr=learning_rate,
momentum=solver["momentum"],
clip_gradients=solver["clip_gradients"])
for logger in loggers:
logger.log(
i, {
'train_loss': loss_hist["train"],
'test_loss': loss_hist["test"],
'test_accuracy': loss_hist["test_accuracy"],
'apollo_net': net,
'start_iter': 0
})
def __init__(self, config, opt_config):
self.config = config
self.opt_config = opt_config
self.apollo_net = apollocaffe.ApolloNet()
def train(config):
"""Trains the ReInspect model using SGD with momentum
and prints out the logging information."""
net = apollocaffe.ApolloNet()
net_config = config["net"]
data_config = config["data"]
solver = config["solver"]
logging = config["logging"]
image_mean = load_data_mean(data_config["idl_mean"],
net_config["img_width"],
net_config["img_height"],
image_scaling=1.0)
input_gen = load_idl(data_config["train_idl"], image_mean, net_config)
input_gen_test = load_idl(data_config["test_idl"],
image_mean,
net_config,
jitter=False)
forward(net, input_gen.next(), config["net"])
net.draw_to_file(logging["schematic_path"])
if solver["weights"]:
net.load(solver["weights"])
else:
net.load(googlenet.weights_file())
loss_hist = {"train": [], "test": []}
loggers = [
apollocaffe.loggers.TrainLogger(logging["display_interval"],
logging["log_file"]),
apollocaffe.loggers.TestLogger(solver["test_interval"],
logging["log_file"]),
apollocaffe.loggers.SnapshotLogger(logging["snapshot_interval"],
logging["snapshot_prefix"]),
]
for i in range(solver["start_iter"], solver["max_iter"]):
# # test and evaluation
if i % solver["test_interval"] == 0:
net.phase = 'test'
test_loss = []
test_loss2 = []
cc_list = []
ce_list = []
ca_list = []
cp_list = []
for _ in range(solver["test_iter"]):
input_en = input_gen_test.next()
forward(net, input_en, net_config, False)
test_loss.append(net.loss)
(count_cover, count_error, count_anno,
count_pred) = get_accuracy(net, input_en, net_config)
cc_list.append(count_cover)
ce_list.append(count_error)
ca_list.append(count_anno)
cp_list.append(count_pred)
loss_hist["test"].append(np.mean(test_loss))
precision = np.sum(cc_list) / np.sum(cp_list)
recall = np.sum(cc_list) / np.sum(ca_list)
print 'hungarian loss:', np.mean(test_loss)
print 'iterate: %6.d error, recall, F1: (%.3f %.3f) -> %.3f' % (
i, 1 - precision, recall, 2 * precision * recall /
(precision + recall))
net.phase = 'train'
forward(net, input_gen.next(), config["net"])
loss_hist["train"].append(net.loss)
net.backward()
learning_rate = (solver["base_lr"] *
(solver["gamma"])**(i // solver["stepsize"]))
net.update(lr=learning_rate,
momentum=solver["momentum"],
clip_gradients=solver["clip_gradients"])
for logger in loggers:
logger.log(
i, {
'train_loss': loss_hist["train"],
'test_loss': loss_hist["test"],
'apollo_net': net,
'start_iter': 0
})
def test(config):
""" Takes the config, run test program
"""
data_mean = load_data_mean(config["data"]["idl_mean"],
config["net"]["img_width"],
config["net"]["img_height"],
image_scaling=1.0)
num_test_images = 599
# Warning: load_idl returns an infinite generator. Calling list() before islice() will hang.
test_list = list(
itertools.islice(
load_idl(config["data"]["test_idl"], data_mean, config["net"],
False), 0, num_test_images))
img = np.copy(test_list[-1]["raw"])
# plt.imshow(img)
net = apollocaffe.ApolloNet()
net.phase = 'test'
forward(net, test_list[0], config["net"], True)
net.load("data/snapshot/reinspect_hcs_800000.h5")
annolist = al.AnnoList()
net_config = config["net"]
pix_per_w = net_config["img_width"] / net_config["grid_width"]
pix_per_h = net_config["img_height"] / net_config["grid_height"]
if config.has_key("conf_th"):
conf_th = config["conf_th"]
else:
conf_th = 0.6
mae = 0.
for i in range(num_test_images):
inputs = test_list[i]
bbox_list, conf_list = forward(net, inputs, net_config, True)
img = np.copy(inputs["raw"])
all_rects = [[[] for x in range(net_config["grid_width"])]
for y in range(net_config["grid_height"])]
for n in range(len(bbox_list)):
for k in range(net_config["grid_height"] *
net_config["grid_width"]):
y = int(k / net_config["grid_width"])
x = int(k % net_config["grid_width"])
bbox = bbox_list[n][k]
conf = conf_list[n][k, 1].flatten()[0]
# notice the output rect [cx, cy, w, h]
# cx means center x-cord
abs_cx = pix_per_w / 2 + pix_per_w * x + int(bbox[0, 0, 0])
abs_cy = pix_per_h / 2 + pix_per_h * y + int(bbox[1, 0, 0])
w = bbox[2, 0, 0]
h = bbox[3, 0, 0]
all_rects[y][x].append(Rect(abs_cx, abs_cy, w, h, conf))
acc_rects = stitch_rects(all_rects)
display = True
if display:
for rect in acc_rects:
if rect.true_confidence < conf_th:
continue
cv2.rectangle(img, (rect.cx - int(rect.width / 2),
rect.cy - int(rect.height / 2)),
(rect.cx + int(rect.width / 2),
rect.cy + int(rect.height / 2)), (255, 0, 0), 2)
# cv2.circle(img,
# (rect.cx, rect.cy),
# ((rect.width + rect.height)/4),
# (255,0,0),
# 2)
img_name = './data/tmp/%05d.jpg' % i
plt.imsave(img_name, img)
plt.figure(figsize=(15, 10))
plt.imshow(img)
anno = al.Annotation()
anno.imageName = inputs["imname"]
# count
number = 0
for rect in acc_rects:
r = al.AnnoRect()
r.x1 = rect.cx - rect.width / 2.
r.x2 = rect.cx + rect.width / 2.
r.y1 = rect.cy - rect.height / 2.
r.y2 = rect.cy + rect.height / 2.
r.score = rect.true_confidence
anno.rects.append(r)
if r.score > conf_th:
number += 1
annolist.append(anno)
mae += abs(number - len(inputs["rects"]))
print anno.imageName, number, len(
inputs["rects"]), abs(number - len(inputs["rects"]))
print mae / num_test_images
