res = {'status':'SUCCESS', 'image_label':self.options.learn_image}
					else:
						res = GoogleGoggles.learn(filename,self.options.learn_image)
					self.total_tests += 1
					if res['status'] == 'SUCCESS':
						print 'success! label is {0}'.format(res['image_label'])
						results_string = format_results('LEARN','SUCCESS',self.options.learn_image,res['image_label'],stamp,save_file_name)
					else:
						print 'failure :('
						results_string = format_results('LEARN','FAILURE',self.options.learn_image,'~',stamp,save_file_name)
				elif not self.options.no_google:
					print "testing...",
					if self.options.fake_google:
						res = {'status':'SUCCESS', 'image_label':self.options.fake_google}
					else:
						res = GoogleGoggles.match(filename)
					#print res
					res_label = res['image_label']
					if res_label:
						self.label_total[res_label] += 1
					else:
						self.label_total['FAILURE'] += 1
					self.total_tests += 1
					
					if self.options.label:
						test_label = self.options.label
						success = res_label == test_label
						if success:
							print 'successful recognition of {0}!'.format(test_label)
							self.total_successes += 1
							#self.label_successes[test_label] = self.label_successes[test_label] + 1
	def _execute(self):
		num_training_images = len(self.images_not_learned)
		
		print 'Testing'
		round_num = 0
		
		start_time = time.time()
		
		filename_base = 'results_'
	
		if self.options.name:
			filename_base = filename_base + self.options.name + '_'
		
		filename_base = filename_base + time.strftime(TIME_FORMAT,time.localtime(start_time))
		if self.options.test:
			filename_base = 'test_' + filename_base
		
		filename = os.path.join(self.options.save_dir,filename_base + '.txt')
		
		f = open(filename,'w')
		
		err_f = None
		
		pickle_filename = os.path.join(self.options.save_dir,filename_base + '.h5')
		hdf = pd.HDFStore(pickle_filename)
		pkld = {}
		
		#pkl.dump('start_time'); pkl.dump(start_time)
		#pkl.dump('root_dir'); pkl.dump(os.path.abspath(root_dir))
		#pkl.dump('random_seed'); pkl.dump(random_seed)
		
		pkld['cmd'] = ' '.join(sys.argv)
		pkld['start_time'] = start_time
		#pkld['root_dir'] = os.path.abspath(root_dir)
		if not self.options.validate_only:
			pkld['training_dir'] = self.options.training_dir
			pkld['training_dir_abs'] = [os.path.abspath(d) for d in self.options.training_dir]
		pkld['validation_dir'] = self.options.validation_dir
		pkld['validation_dir_abs'] = os.path.abspath(self.options.validation_dir)
		pkld['random_seed'] = self.random_seed
		
		f.write('Command: %s\n' % ' '.join(sys.argv))
		
		if not self.options.validate_only:
			if len(self.training_images) == 1:
				f.write('Training dir: %s (%s)\n' % (self.options.training_dir[0], os.path.abspath(self.options.training_dir[0])))
			else:
				f.write('Training dir: %s (%s)\n' % (self.options.training_dir, [os.path.abspath(d) for d in self.options.training_dir]))
		f.write('Validation dir: %s (%s)\n' % (self.options.validation_dir, os.path.abspath(self.options.validation_dir)))
		f.write('Random seed: %s\n' % self.random_seed)
		
		if not self.options.validate_only:
			f.write('Training images:\n')
			if len(self.training_images) == 1:
				[f.write(img[0]+'\n') for img in self.training_images[0]]
			else:
				for idx,t_imgs in enumerate(self.training_images):
					f.write('Round #%d\n' % (idx+1))
					[f.write(img[0]+'\n') for img in t_imgs]
		
		f.write('Validation images:\n')
		[f.write(img[0]+'\n') for img in self.validation_images]
		
		learn_responses = []
		match_responses = []
		
		learned_images_to_this_round = []
		
		false_detects = []
		
		round_times = []
		
		try:
			while self.images_not_learned or self.options.validate_only: #and not rospy.is_shutdown():
				round_num += 1
				if self.options.num_rounds >= 1:
					print "Round #%d/%d" % (round_num,self.options.num_rounds)
					f.write('Round %d/%d:\n' % (round_num,self.options.num_rounds))
				else:
					print "Round #%d" % round_num
					f.write('Round %d:\n' % (round_num))
				round_start_time = time.time()
				
				this_round_false_detects = []
				
				if not self.options.validate_only:
					print 'Learning images...'
					learn_responses.append([])
					images_for_this_round = []
					if len(self.training_images) == 1:
						images_for_this_round[:] = self.images_not_learned
					else:
						images_for_this_round[:] = self.training_images[round_num-1]
					images_learned_this_round = []
					
					if self.options.train_all:
						print 'Learning all %d images' % len(images_for_this_round)
						imgs_to_learn = images_for_this_round
					elif self.options.sample_all:
						num_samples = min(self.options.num_samples,len(images_for_this_round))
						print 'Sampling %d images from %d across all objects' % (num_samples,len(images_for_this_round))
						imgs_to_learn = random.sample(images_for_this_round,min(self.options.num_samples,len(images_for_this_round)))
					else:
						print 'Sampling %d images for each object' % self.options.num_samples
						imgs_to_learn = []
						for object_name in sorted(self.objects):
							#if rospy.is_shutdown(): break
							#print object_name
							object_images = [img for img in images_for_this_round if img[1] == object_name]
							if not object_images: continue
							
							imgs_to_learn += random.sample(object_images,min(self.options.num_samples,len(object_images)))
					
					for img_idx,img_to_learn in enumerate(imgs_to_learn):
						print "Learning %d/%d" % (img_idx+1,len(imgs_to_learn)),img_to_learn[1:],
						if len(self.training_images) == 1:
							t_dir = self.options.training_dir[0]
						else:
							t_dir = self.options.training_dir[round_num-1]
						if not self.options.test:
							try:
								res = GoogleGoggles.learn(os.path.join(t_dir,img_to_learn[0]),img_to_learn[1])
							except Exception, e:
								print 'Exception occured during call to learn:',e
								res = {'status':'EXCEPTION','exception':e}
								if not err_f:
									err_filename = os.path.join(self.options.save_dir,filename_base + '_ERRORS.txt')
									err_f = open(err_filename,'w')
								err_f.write("LEARNING EXCEPTION [%s]: %s\n" % (os.path.join(t_dir,img_to_learn[0]),e))
						else:
							res = {'status':'SUCCESS'}

						
						learn_responses[-1].append((t_dir,img_to_learn,res))
						print res['status']
						if res['status'] != 'SUCCESS':
							#print 'failed!'
							continue
						#print 'done'
					
						image_index = (t_dir,img_to_learn[0])
						img_id = res['image_id']
						self.training = self.training.append(pd.DataFrame(
							{'status':res['status'], 'id':img_id},index=[image_index]))
						
						self.image_db.set_value(image_index,'id', img_id)
						self.schedule.set_value(image_index,'learn', round_num)
						
						self.img_labels = self.img_labels.set_value(img_id,img_to_learn[1])
						
						self.images_not_learned.remove(img_to_learn)
						images_learned_this_round.append(img_to_learn)
					
					self.images_learned.append(images_learned_this_round)
					
					f.write('Learned:\n')
					[f.write(img[0]+'\n') for img in self.images_learned[round_num-1]]
					learned_images_to_this_round = [item for sublist in self.images_learned[:round_num] for item in sublist]
					
					#learned 
					for object_name in sorted(self.objects):
						f.write('%s:\n' % object_name)
						
						for fieldidx,field_name in enumerate(self.field_names):
							if fieldidx == 0: continue
							f.write('  %s:\n' % field_name)
							learned_fields = defaultdict(int)
							for imgidx,img in enumerate(learned_images_to_this_round):
								if img[1] != object_name: continue
								field_val = img[fieldidx+1]
								learned_fields[field_val] = learned_fields[field_val] + 1
							
							for key,val in learned_fields.iteritems():
								f.write("	%s: %d\n" % (key,val))
					
					for idx,field_name in enumerate(self.field_names):
						if idx == 0: continue
						f.write('%s:\n' % field_name)
						learned_fields = defaultdict(int)
						for img in learned_images_to_this_round:
							field_val = img[idx+1]
							learned_fields[field_val] = learned_fields[field_val] + 1
						
						for key,val in learned_fields.iteritems():
							f.write("  %s: %d\n" % (key,val))
				
				print 'Testing images...'
				match_responses.append([])
				data_this_round = np.zeros((len(self.validation_images),1),dtype=int)
				confidence_this_round = np.zeros((len(self.validation_images),1),dtype=float)
				
				if self.options.dont_validate_training_images:
					num_images_to_validate = len(self.validation_images) - len(learned_images_to_this_round)
				else:
					num_images_to_validate = len(self.validation_images)
				
				actually_tested_idx = -1
				for idx,img in enumerate(self.validation_images):
					#if rospy.is_shutdown(): break
					
					actually_tested_idx+=1
					#print 'testing %d/%d' % (actually_tested_idx+1,num_images_to_validate) , img[1:],
					print 'testing %d/%d' % (idx+1,len(self.validation_images)) , img[1:],
					
					if self.options.dont_validate_training_images and \
							[li for li in learned_images_to_this_round if os.path.basename(img[0]) == os.path.basename(li[0])]:
						res = {'status':'SUCCESS',
								'matches':[{'image_label':img[1]}],
								'skipped':True}
						success = True
					elif not self.options.test:
						try:
							res = GoogleGoggles.match(os.path.join(self.options.validation_dir,img[0]))
						except Exception, e:
							print 'Exception occured during call to learn:',e
							res = {'status':'EXCEPTION','exception':e,'matches':[]}
							if not err_f:
								err_filename = os.path.join(self.options.save_dir,filename_base + '_ERRORS.txt')
								err_f = open(err_filename,'w')
							err_f.write("MATCHING EXCEPTION [%s]: %s\n" % (os.path.join(self.options.validation_dir,img[0]),e))
						
					else:
						if self.data_table is not None and self.data_table[idx,-1]:
							success = True
						else:
							choice = random.random()
							if choice < 0.5:
								res = {'status':'SUCCESS',
										'matches':[{'image_label':img[1]}]}
							else:
								choice = random.random()
								if choice < 0.25:
									#false detection
									res = {'status':'SUCCESS',
											'matches':[{'image_label':random.choice([o for o in self.objects if o != img[1]])}]}
								else:
									res = {'status':'FAILURE','matches':[]}
					
					match_responses[-1].append((self.options.validation_dir,img,res))
					success = res['status'] == 'SUCCESS' and res['matches'] and res['matches'][0]['image_label'] == img[1]
					
					image_index = (self.options.validation_dir,img[0])
					
					test_index = tuple([round_num] + list(image_index))
					
					
					
					if not res.get('skipped'):
						self.tests = self.tests.append(pd.DataFrame({'image':[image_index], 'label': [img[1]], 'status':[res['status']]},index=[test_index]))
						
						self.schedule.set_value(image_index,'test', round_num)
						
						self.matches[test_index] = np.nan
					
						for match in res['matches']:
							img_id = match.get('image_id',-1)
							label = match['image_label']
							confidence = float(match.get('match_score',np.nan))
							self.matches = self.matches.set_value(img_id,test_index,confidence)
							self.img_labels = self.img_labels.set_value(img_id,label)
					
					if res.get('skipped'):
						print '[skipped]'
					elif res['matches'] and res['matches'][0]['image_label'] and not success:
						print 'FALSE DETECT:',res['matches'][0]['image_label']
						this_round_false_detects.append((img,res['matches'][0]['image_label']))
					elif success or not res['matches']:
						print res['status']
					else:
						print res['status'], res['matches'][0]['image_label']
					#print 'done', success
						
					
					data_this_round[idx,0] = int(success)
					confidence_this_round[idx,0] = int(success)
				
				num_successes_this_round = np.sum(data_this_round[:,0])
				total_this_round = len(self.validation_images)
				
				print '  %d/%d successes' % (num_successes_this_round,total_this_round)
				
				if self.data_table is None:
					self.data_table = data_this_round
				else:
					self.data_table = np.hstack((self.data_table,data_this_round))
				
				f.write('Test results:\n')
				f.write('Summary: %d/%d\n' % (np.sum(self.data_table[:,round_num-1].flatten()),len(self.validation_images)))
				for object_name in sorted(self.objects):
					successes = 0
					total = 0
					for imgidx,img in enumerate(self.validation_images):
						if img[1] == object_name:
							successes += self.data_table[imgidx,round_num-1]
							total += 1
					f.write('%s: %d/%d\n' % (object_name,successes,total))
					
					for fieldidx,field_name in enumerate(self.field_names):
						if fieldidx == 0: continue
						f.write('  %s:\n' % field_name)
						tested_fields = {}
						for imgidx,img in enumerate(self.validation_images):
							if img[1] != object_name: continue
							field_val = img[fieldidx+1]
							if not tested_fields.has_key(field_val):
								tested_fields[field_val] = [0,0]
							tested_fields[field_val][0] = tested_fields[field_val][0] + self.data_table[imgidx,round_num-1]
							tested_fields[field_val][1] = tested_fields[field_val][1] + 1
						
						for key,val in tested_fields.iteritems():
							f.write("	%s: %d/%d\n" % (key,val[0],val[1]))
				
				for fieldidx,field_name in enumerate(self.field_names):
					f.write('%s:\n' % field_name)
					tested_fields = {}
					for imgidx,img in enumerate(self.validation_images):
						field_val = img[fieldidx+1]
						if not tested_fields.has_key(field_val):
							tested_fields[field_val] = [0,0]
						tested_fields[field_val][0] = tested_fields[field_val][0] + self.data_table[imgidx,round_num-1]
						tested_fields[field_val][1] = tested_fields[field_val][1] + 1
					
					for key,val in tested_fields.iteritems():
						f.write("  %s: %d/%d\n" % (key,val[0],val[1]))
			
				if this_round_false_detects:
					f.write('False detects:\n')
					for img,label in this_round_false_detects:
						f.write('%s: %s\n' % (img[0],label))
				else:
					f.write('No false detects\n')
				false_detects.append(this_round_false_detects)
				
				f.write('Raw data:\n')
				f.write(str(self.data_table[:,round_num-1].flatten().tolist())+'\n')
				
				round_end_time = time.time()
				round_time = (round_end_time - round_start_time)
				print 'Round complete, took %f seconds' % round_time
				f.write('Round time: %fs\n' % round_time)
				round_times.append(round_time)
				
				if (not self.options.train_only and np.all(self.data_table[:,-1])) \
						or round_num == self.options.num_rounds \
						or not self.images_not_learned:
					break
			if idx != len(learnfiles)-1: time.sleep(5)
		if results_file: results_file.close()
	
	if testfiles:
		results_file = None
		if options.save_results:
			results_file_name = 'results_' + time.strftime(TIME_FORMAT,time.localtime()) + '.txt'
			results_file = open(results_file_name,'w')
		total_successes = 0
		total_tests = 0
		label_successes = defaultdict(int)
		label_total = defaultdict(int)
		for (idx,testfile) in enumerate(testfiles):
			label = (testfile.split("/")[-1]).split("_")[0]
			print "testing file {0} ({1}/{2})".format(testfile,idx+1,len(testfiles))
			res = GoogleGoggles.match(testfile)
			print '\n\n', res, '\n\n'
			result_label = ''
			if res.has_key('image_label'):
				result_label = res['image_label']
			elif res.has_key('matches') and res['matches']:
				result_label = res['matches'][0]['image_label']
			success = res['status'] == 'SUCCESS' and result_label == label
			if success:
				print 'success!'
				total_successes += 1
				label_successes[label] = label_successes[label] + 1
				results_string = format_results('TEST','SUCCESS',label,result_label,time.localtime(),testfile)
			else:
				print 'failure :('
				results_string = format_results('TEST','FAILURE',label,result_label,time.localtime(),testfile)