res = {'status':'SUCCESS', 'image_label':self.options.learn_image}
else:
res = GoogleGoggles.learn(filename,self.options.learn_image)
self.total_tests += 1
if res['status'] == 'SUCCESS':
print 'success! label is {0}'.format(res['image_label'])
results_string = format_results('LEARN','SUCCESS',self.options.learn_image,res['image_label'],stamp,save_file_name)
else:
print 'failure :('
results_string = format_results('LEARN','FAILURE',self.options.learn_image,'~',stamp,save_file_name)
elif not self.options.no_google:
print "testing...",
if self.options.fake_google:
res = {'status':'SUCCESS', 'image_label':self.options.fake_google}
else:
res = GoogleGoggles.match(filename)
#print res
res_label = res['image_label']
if res_label:
self.label_total[res_label] += 1
else:
self.label_total['FAILURE'] += 1
self.total_tests += 1
if self.options.label:
test_label = self.options.label
success = res_label == test_label
if success:
print 'successful recognition of {0}!'.format(test_label)
self.total_successes += 1
#self.label_successes[test_label] = self.label_successes[test_label] + 1
def _execute(self):
num_training_images = len(self.images_not_learned)
print 'Testing'
round_num = 0
start_time = time.time()
filename_base = 'results_'
if self.options.name:
filename_base = filename_base + self.options.name + '_'
filename_base = filename_base + time.strftime(TIME_FORMAT,time.localtime(start_time))
if self.options.test:
filename_base = 'test_' + filename_base
filename = os.path.join(self.options.save_dir,filename_base + '.txt')
f = open(filename,'w')
err_f = None
pickle_filename = os.path.join(self.options.save_dir,filename_base + '.h5')
hdf = pd.HDFStore(pickle_filename)
pkld = {}
#pkl.dump('start_time'); pkl.dump(start_time)
#pkl.dump('root_dir'); pkl.dump(os.path.abspath(root_dir))
#pkl.dump('random_seed'); pkl.dump(random_seed)
pkld['cmd'] = ' '.join(sys.argv)
pkld['start_time'] = start_time
#pkld['root_dir'] = os.path.abspath(root_dir)
if not self.options.validate_only:
pkld['training_dir'] = self.options.training_dir
pkld['training_dir_abs'] = [os.path.abspath(d) for d in self.options.training_dir]
pkld['validation_dir'] = self.options.validation_dir
pkld['validation_dir_abs'] = os.path.abspath(self.options.validation_dir)
pkld['random_seed'] = self.random_seed
f.write('Command: %s\n' % ' '.join(sys.argv))
if not self.options.validate_only:
if len(self.training_images) == 1:
f.write('Training dir: %s (%s)\n' % (self.options.training_dir[0], os.path.abspath(self.options.training_dir[0])))
else:
f.write('Training dir: %s (%s)\n' % (self.options.training_dir, [os.path.abspath(d) for d in self.options.training_dir]))
f.write('Validation dir: %s (%s)\n' % (self.options.validation_dir, os.path.abspath(self.options.validation_dir)))
f.write('Random seed: %s\n' % self.random_seed)
if not self.options.validate_only:
f.write('Training images:\n')
if len(self.training_images) == 1:
[f.write(img[0]+'\n') for img in self.training_images[0]]
else:
for idx,t_imgs in enumerate(self.training_images):
f.write('Round #%d\n' % (idx+1))
[f.write(img[0]+'\n') for img in t_imgs]
f.write('Validation images:\n')
[f.write(img[0]+'\n') for img in self.validation_images]
learn_responses = []
match_responses = []
learned_images_to_this_round = []
false_detects = []
round_times = []
try:
while self.images_not_learned or self.options.validate_only: #and not rospy.is_shutdown():
round_num += 1
if self.options.num_rounds >= 1:
print "Round #%d/%d" % (round_num,self.options.num_rounds)
f.write('Round %d/%d:\n' % (round_num,self.options.num_rounds))
else:
print "Round #%d" % round_num
f.write('Round %d:\n' % (round_num))
round_start_time = time.time()
this_round_false_detects = []
if not self.options.validate_only:
print 'Learning images...'
learn_responses.append([])
images_for_this_round = []
if len(self.training_images) == 1:
images_for_this_round[:] = self.images_not_learned
else:
images_for_this_round[:] = self.training_images[round_num-1]
images_learned_this_round = []
if self.options.train_all:
print 'Learning all %d images' % len(images_for_this_round)
imgs_to_learn = images_for_this_round
elif self.options.sample_all:
num_samples = min(self.options.num_samples,len(images_for_this_round))
print 'Sampling %d images from %d across all objects' % (num_samples,len(images_for_this_round))
imgs_to_learn = random.sample(images_for_this_round,min(self.options.num_samples,len(images_for_this_round)))
else:
print 'Sampling %d images for each object' % self.options.num_samples
imgs_to_learn = []
for object_name in sorted(self.objects):
#if rospy.is_shutdown(): break
#print object_name
object_images = [img for img in images_for_this_round if img[1] == object_name]
if not object_images: continue
imgs_to_learn += random.sample(object_images,min(self.options.num_samples,len(object_images)))
for img_idx,img_to_learn in enumerate(imgs_to_learn):
print "Learning %d/%d" % (img_idx+1,len(imgs_to_learn)),img_to_learn[1:],
if len(self.training_images) == 1:
t_dir = self.options.training_dir[0]
else:
t_dir = self.options.training_dir[round_num-1]
if not self.options.test:
try:
res = GoogleGoggles.learn(os.path.join(t_dir,img_to_learn[0]),img_to_learn[1])
except Exception, e:
print 'Exception occured during call to learn:',e
res = {'status':'EXCEPTION','exception':e}
if not err_f:
err_filename = os.path.join(self.options.save_dir,filename_base + '_ERRORS.txt')
err_f = open(err_filename,'w')
err_f.write("LEARNING EXCEPTION [%s]: %s\n" % (os.path.join(t_dir,img_to_learn[0]),e))
else:
res = {'status':'SUCCESS'}
learn_responses[-1].append((t_dir,img_to_learn,res))
print res['status']
if res['status'] != 'SUCCESS':
#print 'failed!'
continue
#print 'done'
image_index = (t_dir,img_to_learn[0])
img_id = res['image_id']
self.training = self.training.append(pd.DataFrame(
{'status':res['status'], 'id':img_id},index=[image_index]))
self.image_db.set_value(image_index,'id', img_id)
self.schedule.set_value(image_index,'learn', round_num)
self.img_labels = self.img_labels.set_value(img_id,img_to_learn[1])
self.images_not_learned.remove(img_to_learn)
images_learned_this_round.append(img_to_learn)
self.images_learned.append(images_learned_this_round)
f.write('Learned:\n')
[f.write(img[0]+'\n') for img in self.images_learned[round_num-1]]
learned_images_to_this_round = [item for sublist in self.images_learned[:round_num] for item in sublist]
#learned
for object_name in sorted(self.objects):
f.write('%s:\n' % object_name)
for fieldidx,field_name in enumerate(self.field_names):
if fieldidx == 0: continue
f.write(' %s:\n' % field_name)
learned_fields = defaultdict(int)
for imgidx,img in enumerate(learned_images_to_this_round):
if img[1] != object_name: continue
field_val = img[fieldidx+1]
learned_fields[field_val] = learned_fields[field_val] + 1
for key,val in learned_fields.iteritems():
f.write(" %s: %d\n" % (key,val))
for idx,field_name in enumerate(self.field_names):
if idx == 0: continue
f.write('%s:\n' % field_name)
learned_fields = defaultdict(int)
for img in learned_images_to_this_round:
field_val = img[idx+1]
learned_fields[field_val] = learned_fields[field_val] + 1
for key,val in learned_fields.iteritems():
f.write(" %s: %d\n" % (key,val))
print 'Testing images...'
match_responses.append([])
data_this_round = np.zeros((len(self.validation_images),1),dtype=int)
confidence_this_round = np.zeros((len(self.validation_images),1),dtype=float)
if self.options.dont_validate_training_images:
num_images_to_validate = len(self.validation_images) - len(learned_images_to_this_round)
else:
num_images_to_validate = len(self.validation_images)
actually_tested_idx = -1
for idx,img in enumerate(self.validation_images):
#if rospy.is_shutdown(): break
actually_tested_idx+=1
#print 'testing %d/%d' % (actually_tested_idx+1,num_images_to_validate) , img[1:],
print 'testing %d/%d' % (idx+1,len(self.validation_images)) , img[1:],
if self.options.dont_validate_training_images and \
[li for li in learned_images_to_this_round if os.path.basename(img[0]) == os.path.basename(li[0])]:
res = {'status':'SUCCESS',
'matches':[{'image_label':img[1]}],
'skipped':True}
success = True
elif not self.options.test:
try:
res = GoogleGoggles.match(os.path.join(self.options.validation_dir,img[0]))
except Exception, e:
print 'Exception occured during call to learn:',e
res = {'status':'EXCEPTION','exception':e,'matches':[]}
if not err_f:
err_filename = os.path.join(self.options.save_dir,filename_base + '_ERRORS.txt')
err_f = open(err_filename,'w')
err_f.write("MATCHING EXCEPTION [%s]: %s\n" % (os.path.join(self.options.validation_dir,img[0]),e))
else:
if self.data_table is not None and self.data_table[idx,-1]:
success = True
else:
choice = random.random()
if choice < 0.5:
res = {'status':'SUCCESS',
'matches':[{'image_label':img[1]}]}
else:
choice = random.random()
if choice < 0.25:
#false detection
res = {'status':'SUCCESS',
'matches':[{'image_label':random.choice([o for o in self.objects if o != img[1]])}]}
else:
res = {'status':'FAILURE','matches':[]}
match_responses[-1].append((self.options.validation_dir,img,res))
success = res['status'] == 'SUCCESS' and res['matches'] and res['matches'][0]['image_label'] == img[1]
image_index = (self.options.validation_dir,img[0])
test_index = tuple([round_num] + list(image_index))
if not res.get('skipped'):
self.tests = self.tests.append(pd.DataFrame({'image':[image_index], 'label': [img[1]], 'status':[res['status']]},index=[test_index]))
self.schedule.set_value(image_index,'test', round_num)
self.matches[test_index] = np.nan
for match in res['matches']:
img_id = match.get('image_id',-1)
label = match['image_label']
confidence = float(match.get('match_score',np.nan))
self.matches = self.matches.set_value(img_id,test_index,confidence)
self.img_labels = self.img_labels.set_value(img_id,label)
if res.get('skipped'):
print '[skipped]'
elif res['matches'] and res['matches'][0]['image_label'] and not success:
print 'FALSE DETECT:',res['matches'][0]['image_label']
this_round_false_detects.append((img,res['matches'][0]['image_label']))
elif success or not res['matches']:
print res['status']
else:
print res['status'], res['matches'][0]['image_label']
#print 'done', success
data_this_round[idx,0] = int(success)
confidence_this_round[idx,0] = int(success)
num_successes_this_round = np.sum(data_this_round[:,0])
total_this_round = len(self.validation_images)
print ' %d/%d successes' % (num_successes_this_round,total_this_round)
if self.data_table is None:
self.data_table = data_this_round
else:
self.data_table = np.hstack((self.data_table,data_this_round))
f.write('Test results:\n')
f.write('Summary: %d/%d\n' % (np.sum(self.data_table[:,round_num-1].flatten()),len(self.validation_images)))
for object_name in sorted(self.objects):
successes = 0
total = 0
for imgidx,img in enumerate(self.validation_images):
if img[1] == object_name:
successes += self.data_table[imgidx,round_num-1]
total += 1
f.write('%s: %d/%d\n' % (object_name,successes,total))
for fieldidx,field_name in enumerate(self.field_names):
if fieldidx == 0: continue
f.write(' %s:\n' % field_name)
tested_fields = {}
for imgidx,img in enumerate(self.validation_images):
if img[1] != object_name: continue
field_val = img[fieldidx+1]
if not tested_fields.has_key(field_val):
tested_fields[field_val] = [0,0]
tested_fields[field_val][0] = tested_fields[field_val][0] + self.data_table[imgidx,round_num-1]
tested_fields[field_val][1] = tested_fields[field_val][1] + 1
for key,val in tested_fields.iteritems():
f.write(" %s: %d/%d\n" % (key,val[0],val[1]))
for fieldidx,field_name in enumerate(self.field_names):
f.write('%s:\n' % field_name)
tested_fields = {}
for imgidx,img in enumerate(self.validation_images):
field_val = img[fieldidx+1]
if not tested_fields.has_key(field_val):
tested_fields[field_val] = [0,0]
tested_fields[field_val][0] = tested_fields[field_val][0] + self.data_table[imgidx,round_num-1]
tested_fields[field_val][1] = tested_fields[field_val][1] + 1
for key,val in tested_fields.iteritems():
f.write(" %s: %d/%d\n" % (key,val[0],val[1]))
if this_round_false_detects:
f.write('False detects:\n')
for img,label in this_round_false_detects:
f.write('%s: %s\n' % (img[0],label))
else:
f.write('No false detects\n')
false_detects.append(this_round_false_detects)
f.write('Raw data:\n')
f.write(str(self.data_table[:,round_num-1].flatten().tolist())+'\n')
round_end_time = time.time()
round_time = (round_end_time - round_start_time)
print 'Round complete, took %f seconds' % round_time
f.write('Round time: %fs\n' % round_time)
round_times.append(round_time)
if (not self.options.train_only and np.all(self.data_table[:,-1])) \
or round_num == self.options.num_rounds \
or not self.images_not_learned:
break
if idx != len(learnfiles)-1: time.sleep(5)
if results_file: results_file.close()
if testfiles:
results_file = None
if options.save_results:
results_file_name = 'results_' + time.strftime(TIME_FORMAT,time.localtime()) + '.txt'
results_file = open(results_file_name,'w')
total_successes = 0
total_tests = 0
label_successes = defaultdict(int)
label_total = defaultdict(int)
for (idx,testfile) in enumerate(testfiles):
label = (testfile.split("/")[-1]).split("_")[0]
print "testing file {0} ({1}/{2})".format(testfile,idx+1,len(testfiles))
res = GoogleGoggles.match(testfile)
print '\n\n', res, '\n\n'
result_label = ''
if res.has_key('image_label'):
result_label = res['image_label']
elif res.has_key('matches') and res['matches']:
result_label = res['matches'][0]['image_label']
success = res['status'] == 'SUCCESS' and result_label == label
if success:
print 'success!'
total_successes += 1
label_successes[label] = label_successes[label] + 1
results_string = format_results('TEST','SUCCESS',label,result_label,time.localtime(),testfile)
else:
print 'failure :('
results_string = format_results('TEST','FAILURE',label,result_label,time.localtime(),testfile)