def main(): script_dirname = os.path.abspath(os.path.dirname(__file__)) output_patches = False fold = utils.TRAINING if output_patches else utils.VALIDATION #only use this path to get the names of the files you want to use in_path = utils.get_path(in_or_out=utils.IN, data_fold=fold) in_path_selective = script_dirname+'/' #this is where the files actually live img_names = [img for img in os.listdir(in_path) if img.endswith('jpg')] image_filenames = [in_path_selective+img for img in os.listdir(in_path) if img.endswith('jpg')] #get the proposals k, scale = get_parameters() sim = 'all' color = 'hsv' cmd = 'selective_search' if cmd == 'selective_search': folder_name = 'k{}_scale{}_sim{}_color{}_FIXING/'.format(k, scale, sim, color) else: folder_name = 'selectiveRCNN/' print 'Folder name is: {}'.format(folder_name) with Timer() as t: boxes = get_windows(image_filenames, script_dirname, cmd=cmd, k=k, scale=scale) print 'Time to process {}'.format(t.secs) detections = Detections() detections.total_time = t.secs out_path = utils.get_path(selective=True, in_or_out=utils.OUT, data_fold=fold, out_folder_name=folder_name) evaluation = Evaluation(#use_corrected_roofs=True, report_name='report.txt', method='windows', folder_name=folder_name, out_path=out_path, detections=detections, in_path=in_path) #score the proposals for img, proposals in zip(img_names, boxes): print 'Evaluating {}'.format(img) print("Found {} windows".format(len(proposals))) proposals = selectionboxes2polygons(proposals) detections.set_detections(detection_list=proposals,roof_type='metal', img_name=img) detections.set_detections(detection_list=proposals,roof_type='thatch', img_name=img) print 'Evaluating...' evaluation.score_img(img, (1200,2000)) evaluation.save_images(img) save_training_TP_FP_using_voc(evaluation, img_names, in_path_selective, out_folder_name=folder_name, neg_thresh=0.3) evaluation.print_report() with open(out_path+'evaluation.pickle', 'wb') as f: pickle.dump(evaluation, f)
class Pipeline(object): def __init__(self, groupThres=0, groupBounds=False, erosion=0, suppress=False, overlapThresh=0.3, pickle_viola=None, single_detector=True, in_path=None, out_path=None, neural=None, viola=None, pipe=None, out_folder_name=None): ''' Parameters: ------------------ groupThres bool Decides if we should do grouping on neural detections ''' self.groupThreshold = int(groupThres) self.groupBounds = groupBounds self.erosion = erosion self.suppress = suppress self.overlapThresh = overlapThresh self.single_detector = single_detector self.in_path = in_path self.img_names = [img_name for img_name in os.listdir(self.in_path) if img_name.endswith('jpg')] self.out_path = out_path #create report file #self.report_path = self.out_path+'report_pipe.txt' #open(self.report_path, 'w').close() #Setup Viola: if we are given an evaluation directly, don't bother running viola self.pickle_viola = pickle_viola if self.pickle_viola is None: self.viola = ViolaDetector(pipeline=True, out_path=out_path, in_path=in_path, folder_name=out_folder_name, save_imgs=True, **viola) else: with open(pickle_viola, 'rb') as f: self.viola_evaluation = pickle.load(f) self.viola_evaluation.in_path = self.in_path self.viola_evaluation.out_path = self.out_path #Setup Neural network(s) if self.single_detector: self.net = Experiment(pipeline=True, **neural['metal']) else: self.net = dict() self.net['metal'] = Experiment(pipeline=True, **neural['metal']) self.net['thatch'] = Experiment(pipeline=True, **neural['thatch']) #we keep track of detections before and after neural network #so we can evaluate by how much the neural network is helping us improve #self.detections_before_neural = Detections() self.detections_after_neural = Detections() #self.evaluation_before_neural = Evaluation(detections=self.detections_before_neural, #method='pipeline', save_imgs=False, out_path=self.out_path, #report_name='before_neural.txt', folder_name=out_folder_name, #in_path=self.in_path, detector_names=viola['detector_names']) self.evaluation_after_neural = Evaluation(detections=self.detections_after_neural, method='pipeline', save_imgs=True, out_path=self.out_path, folder_name=out_folder_name, in_path=self.in_path, detector_names=viola['detector_names']) def run(self): ''' 1. Find proposals using ViolaJones 2. Resize the window and classify it 3. Net returns a list of the roof coordinates of each type - saved in roof_coords ''' neural_time = 0 for i, img_name in enumerate(self.img_names): print '***************** Image {0}: {1}/{2} *****************'.format(img_name, i, len(self.img_names)-1) #VIOLA if self.pickle_viola is None: self.viola.detect_roofs(img_name=img_name) #this next line will fail because you dont get the image shape! self.viola.evaluation.score_img(img_name, img_shape[:2]) self.viola.evaluation.save_images(img_name, fname='beforeNeural') current_viola_detections = self.viola.viola_detections viola_time = self.viola.evaluation.detections.total_time else:#use the pickled detections for speed in testing the neural network current_viola_detections = self.viola_evaluation.detections viola_time = self.viola_evaluation.detections.total_time proposal_patches, proposal_coords, img_shape = self.find_viola_proposals(current_viola_detections, img_name=img_name) #NEURALNET with Timer() as t: classified_detections = self.neural_classification(proposal_patches, proposal_coords) #set detections and score for roof_type in utils.ROOF_TYPES: if self.groupThreshold > 0 and roof_type == 'metal': #need to covert to rectangles boxes = utils.get_bounding_boxes(np.array(classified_detections[roof_type])) grouped_boxes, weights = cv2.groupRectangles(np.array(boxes).tolist(), self.groupThreshold) classified_detections[roof_type] = utils.convert_detections_to_polygons(grouped_boxes) #convert back to polygons elif self.groupBounds and roof_type == 'metal': #grouping with the minimal bound of all overlapping rects classified_detections[roof_type] = self.group_min_bound(classified_detections[roof_type], img_shape[:2], erosion=self.erosion) elif self.suppress and roof_type == 'metal': #proper non max suppression from Felzenszwalb et al. classified_detections[roof_type] = self.non_max_suppression(classified_detections[roof_type]) self.detections_after_neural.set_detections(img_name=img_name, roof_type=roof_type, detection_list=classified_detections[roof_type]) neural_time += t.secs self.evaluation_after_neural.score_img(img_name, img_shape[:2], contours=self.groupBounds) self.evaluation_after_neural.save_images(img_name, 'posNeural') if self.pickle_viola is None: self.viola.evaluation.print_report(print_header=True, stage='viola') else: self.viola_evaluation.print_report(print_header=True, stage='viola') self.evaluation_after_neural.detections.total_time = (neural_time) self.evaluation_after_neural.print_report(print_header=False, stage='neural') #mark roofs on image #evaluate predictions #filter the thatched and metal roofs #compare the predictions made by viola and by viola+neural network def non_max_suppression(self,polygons): #we start with polygons, get the bounding box of it rects = utils.get_bounding_boxes(np.array(polygons)) #covert the bounding box to what's requested by the non_max_suppression boxes = utils.rects2boxes(rects) boxes_suppressed = suppression.non_max_suppression(boxes, overlapThresh=self.overlapThresh) polygons_suppressed = utils.boxes2polygons(boxes_suppressed) return polygons_suppressed def group_min_bound(self, polygons, img_shape, erosion=0): ''' Attempt at finding the minbound of all overlapping rects and merging them to a single detection. This unfortunately will merge nearby roofs. ''' bitmap = np.zeros(img_shape, dtype='uint8') utils.draw_detections(np.array(polygons), bitmap, fill=True, color=1) if erosion>0: kernel = np.ones((5,5),np.uint8) bitmap = cv2.erode(bitmap,kernel,iterations = erosion) #get contours contours, hierarchy = cv2.findContours(bitmap, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) #get the min bounding rect for the rects min_area_conts = [np.int0(cv2.cv.BoxPoints(cv2.minAreaRect(cnt))) for cnt in contours] return min_area_conts def find_viola_proposals(self, viola_detections, img_name=None): '''Call viola to find coordinates of candidate roofs. Extract those patches from the image, tranform them so they can be fed to neural network. Return both the coordinates and the patches. ''' try: img_full = cv2.imread(self.in_path+img_name, flags=cv2.IMREAD_COLOR) img_shape = img_full.shape except IOError as e: print e sys.exit(-1) #if DEBUG: # self.viola.evaluation.save_images(img_name) all_proposal_patches = dict() all_proposal_coords = dict() #extract patches for neural network classification for roof_type in ['metal', 'thatch']: all_proposal_coords[roof_type] = viola_detections.get_detections(img_name=img_name, roof_type=roof_type) #all_proposal_coords[roof_type] = self.viola.viola_detections.get_detections(img_name=img_name, roof_type=roof_type) patches = np.empty((len(all_proposal_coords[roof_type]), 3, utils.PATCH_W, utils.PATCH_H)) for i, detection in enumerate(all_proposal_coords[roof_type]): #extract the patch from the image using utils code img = utils.four_point_transform(img_full, detection) #transform the patch using utils code patch = utils.cv2_to_neural(img) patches[i, :, :,:] = patch all_proposal_patches[roof_type] = patches return all_proposal_patches, all_proposal_coords, img_shape def process_viola(self, rows, cols, img_path=None, verbose=False): #Find candidate roof contours using Viola for all types of roof #returns list with as many lists of detections as the detectors we have passed self.viola.detect_roofs(img_name=self.img_name, img_path=self.test_img_path+self.img_name) print 'Detected {0} candidate roofs'.format(len(self.viola.roofs_detected[self.img_name])) if verbose: self.viola.mark_detections_on_img(img=self.image, img_name=self.img_name) #get the mask and the contours for the detections detection_mask, _ = self.viola.get_patch_mask(img_name=self.img_name, rows=rows, cols=cols) patch_location = self.out_path+self.img_name+'_mask.jpg' misc.imsave(patch_location, detection_mask) self.all_contours[self.img_name] = self.viola.get_detection_contours(patch_location, self.img_name) def neural_classification(self, proposal_patches, proposal_coords): classified_detections = defaultdict(list) for roof_type in utils.ROOF_TYPES: #classify with neural network if proposal_patches[roof_type].shape[0] > 1: if self.single_detector: #we have a single net classes = self.net.test(proposal_patches[roof_type]) #filter according to classification for detection, classification in zip(proposal_coords[roof_type], classes): if classification == utils.NON_ROOF: classified_detections['background'].append(detection) elif classification == utils.METAL: classified_detections['metal'].append(detection) elif classification == utils.THATCH: classified_detections['thatch'].append(detection) else: #we have one net per roof type specific_net = self.net[roof_type] classes = specific_net.test(proposal_patches[roof_type]) #filter according to classification for detection, classification in zip(proposal_coords[roof_type], classes): if classification == 0: classified_detections['background'].append(detection) elif classification == 1: classified_detections[roof_type].append(detection) else: raise ValueError('Unknown classification of patch') else: print 'No {0} detections'.format(roof_type) return classified_detections def save_img_detections(self, img_name, proposal_coords, predictions): raise ValueError('Incorrect method') img = cv2.imread(self.in_path+img_name) roofs = DataLoader().get_roofs(self.in_path+img_name[:-3]+'xml', img_name) for roof in roofs: cv2.rectangle(img, (roof.xmin, roof.ymin), (roof.xmin+roof.width, roof.ymin+roof.height), (0,255,0), 2) for (x,y,w,h), accept in zip(proposal_coords['metal'], predictions[img_name]['metal']): color = (0,0,0) if accept==1 else (0,0,255) cv2.rectangle(img, (x,y), (x+w, y+h), color, 2) cv2.imwrite(self.out_path+img_name, img)
class SlidingWindowNeural(object): def __init__(self, data_fold=None, full_dataset=False, out_path=None, in_path=None, output_patches=False, scale=1.5, minSize=(200,200), windowSize=(40,40), stepSize=15): self.scale = scale self.minSize = minSize self.windowSize = windowSize self.output_patches = output_patches self.stepSize = stepSize if self.output_patches == False else 30 self.total_window_num = 0 if data_fold is None: self.data_fold = utils.TRAINING if self.output_patches or full_dataset else utils.VALIDATION else: self.data_fold = data_fold self.in_path = in_path if in_path is not None else utils.get_path(in_or_out=utils.IN, data_fold=self.data_fold, full_dataset=full_dataset) self.img_names = [img_name for img_name in os.listdir(self.in_path) if img_name.endswith('.jpg')] self.img_names = self.img_names[:20] if DEBUG else self.img_names self.detections = Detections() folder_name = 'scale{}_minSize{}-{}_windowSize{}-{}_stepSize{}_dividedSizes/'.format(self.scale, self.minSize[0], self.minSize[1], self.windowSize[0], self.windowSize[1], self.stepSize) self.out_path = out_path if out_path is not None else '{}'.format(utils.get_path(full_dataset=True, in_or_out=utils.OUT, slide=True, data_fold=self.data_fold, out_folder_name=folder_name)) self.evaluation = Evaluation(full_dataset=full_dataset, method='slide',folder_name=folder_name, save_imgs=False, out_path=self.out_path, detections=self.detections, in_path=self.in_path) def get_windows_in_folder(self, folder=None): folder = folder if folder is not None else self.in_path self.all_coordinates = dict() with Timer() as t: for i, img_name in enumerate(self.img_names): print 'Getting windows for image: {}/{}'.format(i+1, len(self.img_names)) self.all_coordinates[img_name] = dict() polygons, rects = self.get_windows(img_name) self.all_coordinates[img_name] = rects print t.secs self.detections.total_time = t.secs print self.total_window_num def is_small_image(self, image): w, h = image.shape[:2] return (w<2000 and h<1000) def get_windows(self, img_name, in_path=None): in_path = in_path if in_path is not None else self.in_path try: image = cv2.imread(in_path+img_name) except IOError: print 'Could not open file' sys.exit(-1) if self.is_small_image(image):# or self.output_patches: return self.detect(img_name, image) else: stepSize = 15 if self.output_patches==False else 50 return self.detect(img_name, image, stepSize=stepSize, windowSize=(40,40), scale=1.5, minSize=(200,200)) def detect(self, img_name, image, stepSize=None, windowSize=None, scale=None, minSize=None): windowSize = windowSize if windowSize is not None else self.windowSize stepSize = stepSize if stepSize is not None else self.stepSize scale = scale if scale is not None else self.scale minSize = minSize if minSize is not None else self.minSize window_num = 0 polygons_metal = list() polygons_thatch = list() rects_metal = list() rects_thatch = list() #loop through pyramid for level, resized in enumerate(utils.pyramid(image, scale=scale, minSize=minSize)): for (x, y, window) in utils.sliding_window(resized, stepSize=stepSize, windowSize=windowSize): #self.debug_scaling(image, img_name, resized, x, y, level): # if the window does not meet our desired window size, ignore it if window.shape[0] != windowSize[0] or window.shape[1] != windowSize[1]: continue window_num += 1 #save the correctly translated coordinates of this window polygon, rectangle = self.get_translated_coords(x, y, level, scale, windowSize) polygons_metal.append(polygon) rects_metal.append(rectangle) polygons_thatch.append(polygon) rects_thatch.append(rectangle) self.total_window_num += window_num rects = {'thatch': rects_thatch, 'metal': rects_metal} polygons = {'thatch': polygons_thatch, 'metal': polygons_metal} return polygons, rects def get_translated_coords(self, x, y, pyramid_level, scale, windowSize): scale_factor = math.pow(scale, pyramid_level) x = x*scale_factor y = y*scale_factor w = int(scale_factor*windowSize[1]) #int(scale_factor*self.windowSize[1]) h = int(scale_factor*windowSize[0]) #int(scale_factor*self.windowSize[0]) rect = Rectangle(int(x), int(y), int(x+w), int(y+h)) return utils.convert_rect_to_polygon(rect), rect def debug_scaling(self, image, img_name, x, y, level): '''Not working ''' clone = resized.copy() cv2.rectangle(clone, (x, y), (x + self.windowSize[1], y + self.windowSize[0]), (0, 255, 0), 2) clone = image.copy() scale_factor = math.pow(self.scale, level) x = (x*scale_factor) y = (y*scale_factor) w = scale_factor*self.windowSize[1] h = scale_factor*self.windowSize[0] start = (int(x), int(y)) end = (int(x + w), int(y + h) ) cv2.rectangle(clone, start, end, (0, 255, 0), 2) b,g,r = cv2.split(clone) img2 = cv2.merge([r,g,b]) plt.subplots(2) subplot(2) plt.imshow(img2) plt.show() def run_evaluation(self): #EVALUATION for i, img_name in enumerate(self.img_names): print 'Evaluating image {}/{}'.format(i+1, len(self.img_names)) #set the detections self.detections.set_detections(roof_type='thatch', detection_list=self.all_coordinates[img_name]['thatch'], img_name=img_name) self.detections.set_detections(roof_type='metal', detection_list=self.all_coordinates[img_name]['metal'], img_name=img_name) #score the image self.evaluation.score_img(img_name=img_name, img_shape=(1200,2000), fast_scoring=True) self.evaluation.print_report()
class SlidingWindowNeural(object): def __init__(self, data_fold=None, full_dataset=False, out_path=None, in_path=None, output_patches=False, scale=1.5, minSize=(200, 200), windowSize=(40, 40), stepSize=15): self.scale = scale self.minSize = minSize self.windowSize = windowSize self.output_patches = output_patches self.stepSize = stepSize if self.output_patches == False else 30 self.total_window_num = 0 if data_fold is None: self.data_fold = utils.TRAINING if self.output_patches or full_dataset else utils.VALIDATION else: self.data_fold = data_fold self.in_path = in_path if in_path is not None else utils.get_path( in_or_out=utils.IN, data_fold=self.data_fold, full_dataset=full_dataset) self.img_names = [ img_name for img_name in os.listdir(self.in_path) if img_name.endswith('.jpg') ] self.img_names = self.img_names[:20] if DEBUG else self.img_names self.detections = Detections() folder_name = 'scale{}_minSize{}-{}_windowSize{}-{}_stepSize{}_dividedSizes/'.format( self.scale, self.minSize[0], self.minSize[1], self.windowSize[0], self.windowSize[1], self.stepSize) self.out_path = out_path if out_path is not None else '{}'.format( utils.get_path(full_dataset=True, in_or_out=utils.OUT, slide=True, data_fold=self.data_fold, out_folder_name=folder_name)) self.evaluation = Evaluation(full_dataset=full_dataset, method='slide', folder_name=folder_name, save_imgs=False, out_path=self.out_path, detections=self.detections, in_path=self.in_path) def get_windows_in_folder(self, folder=None): folder = folder if folder is not None else self.in_path self.all_coordinates = dict() with Timer() as t: for i, img_name in enumerate(self.img_names): print 'Getting windows for image: {}/{}'.format( i + 1, len(self.img_names)) self.all_coordinates[img_name] = dict() polygons, rects = self.get_windows(img_name) self.all_coordinates[img_name] = rects print t.secs self.detections.total_time = t.secs print self.total_window_num def is_small_image(self, image): w, h = image.shape[:2] return (w < 2000 and h < 1000) def get_windows(self, img_name, in_path=None): in_path = in_path if in_path is not None else self.in_path try: image = cv2.imread(in_path + img_name) except IOError: print 'Could not open file' sys.exit(-1) if self.is_small_image(image): # or self.output_patches: return self.detect(img_name, image) else: stepSize = 15 if self.output_patches == False else 50 return self.detect(img_name, image, stepSize=stepSize, windowSize=(40, 40), scale=1.5, minSize=(200, 200)) def detect(self, img_name, image, stepSize=None, windowSize=None, scale=None, minSize=None): windowSize = windowSize if windowSize is not None else self.windowSize stepSize = stepSize if stepSize is not None else self.stepSize scale = scale if scale is not None else self.scale minSize = minSize if minSize is not None else self.minSize window_num = 0 polygons_metal = list() polygons_thatch = list() rects_metal = list() rects_thatch = list() #loop through pyramid for level, resized in enumerate( utils.pyramid(image, scale=scale, minSize=minSize)): for (x, y, window) in utils.sliding_window(resized, stepSize=stepSize, windowSize=windowSize): #self.debug_scaling(image, img_name, resized, x, y, level): # if the window does not meet our desired window size, ignore it if window.shape[0] != windowSize[0] or window.shape[ 1] != windowSize[1]: continue window_num += 1 #save the correctly translated coordinates of this window polygon, rectangle = self.get_translated_coords( x, y, level, scale, windowSize) polygons_metal.append(polygon) rects_metal.append(rectangle) polygons_thatch.append(polygon) rects_thatch.append(rectangle) self.total_window_num += window_num rects = {'thatch': rects_thatch, 'metal': rects_metal} polygons = {'thatch': polygons_thatch, 'metal': polygons_metal} return polygons, rects def get_translated_coords(self, x, y, pyramid_level, scale, windowSize): scale_factor = math.pow(scale, pyramid_level) x = x * scale_factor y = y * scale_factor w = int(scale_factor * windowSize[1]) #int(scale_factor*self.windowSize[1]) h = int(scale_factor * windowSize[0]) #int(scale_factor*self.windowSize[0]) rect = Rectangle(int(x), int(y), int(x + w), int(y + h)) return utils.convert_rect_to_polygon(rect), rect def debug_scaling(self, image, img_name, x, y, level): '''Not working ''' clone = resized.copy() cv2.rectangle(clone, (x, y), (x + self.windowSize[1], y + self.windowSize[0]), (0, 255, 0), 2) clone = image.copy() scale_factor = math.pow(self.scale, level) x = (x * scale_factor) y = (y * scale_factor) w = scale_factor * self.windowSize[1] h = scale_factor * self.windowSize[0] start = (int(x), int(y)) end = (int(x + w), int(y + h)) cv2.rectangle(clone, start, end, (0, 255, 0), 2) b, g, r = cv2.split(clone) img2 = cv2.merge([r, g, b]) plt.subplots(2) subplot(2) plt.imshow(img2) plt.show() def run_evaluation(self): #EVALUATION for i, img_name in enumerate(self.img_names): print 'Evaluating image {}/{}'.format(i + 1, len(self.img_names)) #set the detections self.detections.set_detections( roof_type='thatch', detection_list=self.all_coordinates[img_name]['thatch'], img_name=img_name) self.detections.set_detections( roof_type='metal', detection_list=self.all_coordinates[img_name]['metal'], img_name=img_name) #score the image self.evaluation.score_img(img_name=img_name, img_shape=(1200, 2000), fast_scoring=True) self.evaluation.print_report()
class TemplateMatcher(object): def __init__(self, out_path=utils.get_path(template=True, data_fold=utils.TRAINING, in_or_out=utils.OUT), group_detections=True, contour_detections=True, min_neighbors=1, eps=1, in_path=utils.get_path(data_fold=utils.TRAINING, template=True, in_or_out=utils.IN)): self.in_path = in_path out_folder = utils.time_stamped( '') if group_detections == False else utils.time_stamped( 'grouped_neigh{0}_eps{1}'.format(min_neighbors, eps)) self.out_path = out_path + out_folder utils.mkdir(self.out_path) self.group_detections = group_detections self.contour_detections = contour_detections self.min_neighbors = min_neighbors self.eps = eps self.detections = Detections() self.detector_names = dict() self.detector_names['metal'] = '5 templates' self.detector_names['thatch'] = '1 template' self.evaluation = Evaluation(method='template', folder_name=out_folder, out_path=self.out_path, detections=self.detections, in_path=self.in_path, detector_names=self.detector_names) self.templates = dict() self.set_metal_templates() self.get_thatch_template() self.set_thatch_templates() self.threshold = 0.5 def set_metal_templates(self, shape=None): templates = list() for i in range(5): if shape == 0 or shape == 1 or shape == 2: template_height, template_width = (40, 60) else: # shape == 3 or shape == 4: template_height, template_width = (60, 60) template = np.zeros((template_height + 20, template_width + 20)) template[20:20 + template_height, 20:20 + template_width] = 255 #white central region if shape == 1 or shape == 3: #rotate 45 degrees template = ndimage.rotate(template, 45) if shape == 2 or shape == 4: #rotate 90 degrees template = ndimage.rotate(template, 90) template_name = 'template{0}.jpg'.format(shape) cv2.imwrite(template_name, template) template = cv2.imread(template_name, cv2.IMREAD_GRAYSCALE) templates.append(template) self.templates['metal'] = templates def get_thatch_template(self): #get some thatch roof roof = None for img_name in listdir(self.in_path): if img_name.endswith('.jpg'): roofs = DataLoader().get_roofs( self.in_path + img_name[:-3] + 'xml', '') for r in roofs: if r.roof_type == 'thatch': roof = r break if roof is not None: break #extract patch img = cv2.imread(self.in_path + img_name) template = img[roof.ymin:roof.ymin + roof.height, roof.xmin:roof.xmin + roof.width] img = cv2.imwrite('thatch_template.jpg', template) def set_thatch_templates(self): template = cv2.imread('thatch_template.jpg', cv2.IMREAD_GRAYSCALE) self.templates['thatch'] = [template] def detect_score_roofs(self): for img_name in listdir(self.in_path): if img_name.endswith('.jpg') == False: continue detections_all = defaultdict(list) img_rgb = cv2.imread('{0}{1}'.format(self.in_path, img_name), flags=cv2.IMREAD_COLOR) img_gray = cv2.imread('{0}{1}'.format(self.in_path, img_name), cv2.IMREAD_GRAYSCALE) img_gray = cv2.equalizeHist(img_gray) print '-------------------- Matching.....{0} ------------------------ '.format( img_name) group_detected_roofs = dict() for roof_type, templates in self.templates.iteritems(): with Timer() as t: #MATCHING for template in templates: #get detections for each template, keep those over a threshold res = cv2.matchTemplate(img_gray, template, cv2.TM_CCOEFF_NORMED) detections = np.where(res >= self.threshold) w, h = template.shape[::-1] boxes = [(pt[0], pt[1], w, h) for pt in zip(*detections[::-1])] if len(boxes) > 0: detections_all[roof_type].extend(boxes) print 'Done matching {0}'.format(roof_type) #GROUPING if self.group_detections and self.contour_detections == False: print 'Grouping....' detections_all[ roof_type], weights = cv2.groupRectangles( np.array(detections_all[roof_type]).tolist(), self.min_neighbors, self.eps) print 'Done grouping' elif self.contour_detections == True: #do contour detection instead of grouprects detections_all[ roof_type] = self.evaluation.get_bounding_rects( img_name=img_name, rows=img_gray.shape[0], cols=img_gray.shape[1], detections=detections_all[roof_type]) print '{0} detections: {1}'.format( roof_type, len(detections_all[roof_type])) self.detections.total_time += t.secs print 'Time {0}'.format(t.secs) self.detections.set_detections(img_name=img_name, detection_list=detections_all) self.evaluation.score_img(img_name) self.evaluation.print_report() self.evaluation.pickle_detections() open(self.out_path + 'DONE', 'w').close() def detect_roofs(self): for img_name in listdir(self.in_path): if img_name.endswith('.jpg') == False: continue detections_all = defaultdict(list) img_rgb = cv2.imread('{0}{1}'.format(self.in_path, img_name), flags=cv2.IMREAD_COLOR) img_gray = cv2.imread('{0}{1}'.format(self.in_path, img_name), cv2.IMREAD_GRAYSCALE) print '-------------------- Matching.....{0} ------------------------ '.format( img_name) group_detected_roofs = dict() for roof_type, templates in self.templates.iteritems(): with Timer() as t: #MATCHING for template in templates: #get detections for each template, keep those over a threshold res = cv2.matchTemplate(img_gray, template, cv2.TM_CCOEFF_NORMED) detections = np.where(res >= self.threshold) w, h = template.shape[::-1] boxes = [(pt[0], pt[1], w, h) for pt in zip(*detections[::-1])] if len(boxes) > 0: detections_all[roof_type].extend(boxes) print 'Done matching {0}'.format(roof_type) #GROUPING if self.group_detections: print 'Grouping....' detections_all[ roof_type], weights = cv2.groupRectangles( np.array(detections_all[roof_type]).tolist(), self.min_neighbors, self.eps) print 'Done grouping' print '{0} detections: {1}'.format( roof_type, len(detections_all[roof_type])) self.detections.total_time += t.secs print 'Time {0}'.format(t.secs) self.detections.set_detections(img_name=img_name, detection_list=detections_all) def hist_curve(im): h = np.zeros((300, 256, 3)) if len(im.shape) == 2: color = [(255, 255, 255)] elif im.shape[2] == 3: color = [(255, 0, 0), (0, 255, 0), (0, 0, 255)] for ch, col in enumerate(color): hist_item = cv2.calcHist([im], [ch], None, [256], [0, 256]) cv2.normalize(hist_item, hist_item, 0, 255, cv2.NORM_MINMAX) hist = np.int32(np.around(hist_item)) pts = np.int32(np.column_stack((bins, hist))) cv2.polylines(h, [pts], False, col) y = np.flipud(h) return y def hist_lines(im): h = np.zeros((300, 256, 3)) if len(im.shape) != 2: print "hist_lines applicable only for grayscale images" #print "so converting image to grayscale for representation" im = cv2.cvtColor(im, cv2.COLOR_BGR2GRAY) hist_item = cv2.calcHist([im], [0], None, [256], [0, 256]) cv2.normalize(hist_item, hist_item, 0, 255, cv2.NORM_MINMAX) hist = np.int32(np.around(hist_item)) for x, y in enumerate(hist): cv2.line(h, (x, 0), (x, y), (255, 255, 255)) y = np.flipud(h) return y
class ViolaDetector(object): def __init__(self, TESTING=False, #you can delete this parameter! pipeline=False, in_path=None, out_path=None, folder_name=None, save_imgs=False, detector_names=None, group=None, #minNeighbors, scale... overlapThresh=None, downsized=False, min_neighbors=3, scale=1.1, rotate=True, rotateRectOnly=False, fullAngles=False, removeOff=True, output_patches=True, strict=True, negThres=0.3, mergeFalsePos=False, separateDetections=True, vocGood=0.1, pickled_evaluation=False ): ''' Class used to do preliminary detection of metal and thatch roofs on images Parameters: -------------- pipeline: bool If True, the report file is not created. The pipeline will create its own report file in_path: string path from which images are read out_path: string path in which we create an output folder folder_name: string folder to be created in out_path, into which the content is actually saved detector_names: list(string) names of detectors to be used group: boolean decides whether grouping of detections should occur for each roof type separately downsized: decides if images are downsized by a factor of 2 to perform detection min_neighbors: int parameter for the detectmultiscale method: determines how many neighbours a detection must have in order to keep it scale: float parameter for the detectmultiscale method rotate: boolean whether we should rotate the image rotateRectOnly:boolean rotate only rectangular detectors, instead of all metal detectors removeOff: boolean whether the detections that fall partially off the image should be removed. Relevant in particular to the rotations output_patches: boolean whether good and bad detections should be saved. These can then be used to train other models strict: boolean whether the patches saved should be strictly the true and false detections neg_thres: float the threshold voc score under which a detection is considered a negative example for neural training mergeFalsePos: boolean whether the bad detections of the metal and thatch roofs should be saved together or separately. If it is true, then only bad detections that are bad for both metal and thatch fall into the bad detection category. If it is false, then we consider each roof type separately. For example, if a detection is bad for the metal detector, but it contains a thatch roof, it will be classified as bad for metal. In the other case, any detection can only be classified as bad if it contains neither a metal or a thatch roof ''' self.pipeline = pipeline assert in_path is not None self.in_path = in_path assert out_path is not None self.out_folder = out_path self.out_folder_name = folder_name print 'Viola will output evaluation to: {0}'.format(self.out_folder) self.img_names = [f for f in os.listdir(in_path) if f.endswith('.jpg')] self.save_imgs = save_imgs self.output_patches = output_patches self.strict = strict self.negThres = negThres self.mergeFalsePos = mergeFalsePos self.rotateRectOnly = rotateRectOnly self.viola_detections = Detections(mergeFalsePos = self.mergeFalsePos) self.setup_detectors(detector_names) #parameters for detection self.scale = scale self.min_neighbors = int(min_neighbors) self.group = group self.overlapThresh = overlapThresh if rotate: self.angles = utils.VIOLA_ANGLES else: self.angles = [0] self.remove_off_img = removeOff self.downsized = downsized self.pickled_evaluation = pickled_evaluation if pickled_evaluation == False: self.evaluation = Evaluation(full_dataset=False, negThres=self.negThres, method='viola', folder_name=folder_name, out_path=self.out_folder, detections=self.viola_detections, in_path=self.in_path, detector_names=detector_names, mergeFalsePos=mergeFalsePos, vocGood=vocGood) else: with open(self.out_folder+'evaluation.pickle', 'rb') as f: self.evaluation = pickle.load(f) def setup_detectors(self, detector_names=None, old_detector=False): '''Given a list of detector names, get the detectors specified ''' #get the detectors assert detector_names is not None self.roof_detectors = defaultdict(list) self.detector_names = detector_names self.rotate_detectors = list() rectangular_detector = 'cascade_metal_rect_augm1_singlesize_original_pad0_num872_w40_h20_FA0.4_LBP' for roof_type in utils.ROOF_TYPES: for i, path in enumerate(detector_names[roof_type]): if rectangular_detector in path or self.rotateRectOnly == False: self.rotate_detectors.append(True) else: self.rotate_detectors.append(False) if path.startswith('cascade'): start = '../viola_jones/cascades/' self.roof_detectors[roof_type].append(cv2.CascadeClassifier(start+path+'/cascade.xml')) assert self.roof_detectors[roof_type][-1].empty() == False else: self.roof_detectors[roof_type].append(cv2.CascadeClassifier('../viola_jones/cascade_'+path+'/cascade.xml')) def detect_roofs_in_img_folder(self): '''Compare detections to ground truth roofs for set of images in a folder ''' for i, img_name in enumerate(self.img_names): print '************************ Processing image {0}/{1}\t{2} ************************'.format(i, len(self.img_names), img_name) if self.group: img = self.detect_roofs_group(img_name) else: img = self.detect_roofs(img_name) ''' self.evaluation.score_img(img_name, img.shape) self.evaluation.print_report() with open('{0}evaluation.pickle'.format(self.out_folder), 'wb') as f: pickle.dump(self.evaluation, f) if self.output_patches: self.evaluation.save_training_TP_FP_using_voc() open(self.out_folder+'DONE', 'w').close() ''' def detect_roofs(self, img_name, in_path=None): in_path = self.in_path if in_path is None else in_path try: rgb_unrotated = cv2.imread(in_path+img_name, flags=cv2.IMREAD_COLOR) gray = cv2.cvtColor(rgb_unrotated, cv2.COLOR_BGR2GRAY) gray = cv2.equalizeHist(gray) if self.downsized: rgb_unrotated = utils.resize_rgb(rgb_unrotated, h=rgb_unrotated.shape[0]/2, w=rgb_unrotated.shape[1]/2) gray = utils.resize_grayscale(gray, w=gray.shape[1]/2, h=gray.shape[0]/2) except IOError as e: print e sys.exit(-1) else: for roof_type, detectors in self.roof_detectors.iteritems(): for i, detector in enumerate(detectors): for angle in self.angles: #for thatch we only need one angle if self.rotate_detectors[i] == False and angle>0 or (roof_type=='thatch' and angle>0):#roof_type == 'thatch' and angle>0: continue print 'Detecting with detector: '+str(i) print 'ANGLE '+str(angle) with Timer() as t: rotated_image = utils.rotate_image(gray, angle) if angle>0 else gray delete_image = utils.rotate_image_RGB(rgb_unrotated, angle) if angle>0 else gray detections, _ = self.detect_and_rectify(detector, rotated_image, angle, rgb_unrotated.shape[:2], rgb_rotated=delete_image) if self.downsized: detections = detections*2 self.viola_detections.set_detections(roof_type=roof_type, img_name=img_name, angle=angle, detection_list=detections, img=rotated_image) print 'Time detection: {0}'.format(t.secs) self.viola_detections.total_time += t.secs if DEBUG: rgb_to_write = cv2.imread(in_path+img_name, flags=cv2.IMREAD_COLOR) utils.draw_detections(detections, rgb_to_write, color=(255,0,0)) cv2.imwrite('{0}{3}{1}_{2}.jpg'.format('', img_name[:-4], angle, roof_type), rgb_to_write) return rgb_unrotated def detect_and_rectify(self, detector, image, angle, dest_img_shape, rgb_rotated=None): #do the detection detections = detector.detectMultiScale(image, scaleFactor=self.scale, minNeighbors=self.min_neighbors) ''' print 'were about to save the rotated images, if you dont want this, quit and remove this from like 232 in viola_detector.py' pdb.set_trace() for d in detections: cv2.rectangle(rgb_rotated, (d[0], d[1]), (d[0]+d[2], d[1]+d[3]), (255,255,255), 4) cv2.imwrite('rotated.jpg', rgb_rotated) pdb.set_trace() ''' #convert to proper coordinate system polygons = utils.convert_detections_to_polygons(detections) if angle > 0: #rotate back to original image coordinates print 'rotating...' rectified_detections = utils.rotate_detection_polygons(polygons, image, angle, dest_img_shape, remove_off_img=self.remove_off_img) else: rectified_detections = polygons print 'done rotating' if self.group: bounding_boxes = utils.get_bounding_boxes(np.array(rectified_detections)) else: bounding_boxes = None return rectified_detections, bounding_boxes ''' def detect_roofs_group(self, img_name): try: rgb_unrotated = cv2.imread(self.in_path+img_name, flags=cv2.IMREAD_COLOR) gray = cv2.cvtColor(rgb_unrotated, cv2.COLOR_BGR2GRAY) gray = cv2.equalizeHist(gray) except IOError as e: print e sys.exit(-1) else: for roof_type, detectors in self.roof_detectors.iteritems(): all_detections = list() for i, detector in enumerate(detectors): for angle in self.angles: #for thatch we only need one angle if roof_type == 'thatch' and angle>0: continue print 'Detecting with detector: '+str(i) print 'ANGLE '+str(angle) with Timer() as t: rotated_image = utils.rotate_image(gray, angle) if angle>0 else gray detections, bounding_boxes = self.detect_and_rectify(detector, rotated_image, angle, rgb_unrotated.shape[:2]) all_detections.append(list(bounding_boxes)) print 'Time detection: {0}'.format(t.secs) self.viola_detections.total_time += t.secs #grouping all_detections = [d for detections in all_detections for d in detections] grouped_detections, rects_grouped = cv2.groupRectangles(all_detections, 1) print "GROUPING DOWN BY:" print len(all_detections)-len(grouped_detections) grouped_polygons = utils.convert_detections_to_polygons(grouped_detections) #merge the detections from all angles self.viola_detections.set_detections(roof_type=roof_type, img_name=img_name, detection_list=grouped_polygons, img=rotated_image) print 'Detections for {0}'.format(roof_type) print len(self.viola_detections.get_detections(roof_type=roof_type, img_name=img_name)) return rgb_unrotated ''' def mark_save_current_rotation(self, img_name, img, detections, angle, out_folder=None): out_folder = self.out_folder if out_folder is None else out_folder polygons = np.zeros((len(detections), 4, 2)) for i, d in enumerate(detections): polygons[i, :] = utils.convert_rect_to_polygon(d) img = self.evaluation.mark_roofs_on_img(img_name=img_name, img=img, roofs=polygons, color=(0,0,255)) path = '{0}_angle{1}.jpg'.format(out_folder+img_name[:-4], angle) print path cv2.imwrite(path, img)
class ViolaDetector(object): def __init__( self, TESTING=False, #you can delete this parameter! pipeline=False, in_path=None, out_path=None, folder_name=None, save_imgs=False, detector_names=None, group=None, #minNeighbors, scale... overlapThresh=None, downsized=False, min_neighbors=3, scale=1.1, rotate=True, rotateRectOnly=False, fullAngles=False, removeOff=True, output_patches=True, strict=True, negThres=0.3, mergeFalsePos=False, separateDetections=True, vocGood=0.1, pickled_evaluation=False): ''' Class used to do preliminary detection of metal and thatch roofs on images Parameters: -------------- pipeline: bool If True, the report file is not created. The pipeline will create its own report file in_path: string path from which images are read out_path: string path in which we create an output folder folder_name: string folder to be created in out_path, into which the content is actually saved detector_names: list(string) names of detectors to be used group: boolean decides whether grouping of detections should occur for each roof type separately downsized: decides if images are downsized by a factor of 2 to perform detection min_neighbors: int parameter for the detectmultiscale method: determines how many neighbours a detection must have in order to keep it scale: float parameter for the detectmultiscale method rotate: boolean whether we should rotate the image rotateRectOnly:boolean rotate only rectangular detectors, instead of all metal detectors removeOff: boolean whether the detections that fall partially off the image should be removed. Relevant in particular to the rotations output_patches: boolean whether good and bad detections should be saved. These can then be used to train other models strict: boolean whether the patches saved should be strictly the true and false detections neg_thres: float the threshold voc score under which a detection is considered a negative example for neural training mergeFalsePos: boolean whether the bad detections of the metal and thatch roofs should be saved together or separately. If it is true, then only bad detections that are bad for both metal and thatch fall into the bad detection category. If it is false, then we consider each roof type separately. For example, if a detection is bad for the metal detector, but it contains a thatch roof, it will be classified as bad for metal. In the other case, any detection can only be classified as bad if it contains neither a metal or a thatch roof ''' self.pipeline = pipeline assert in_path is not None self.in_path = in_path assert out_path is not None self.out_folder = out_path self.out_folder_name = folder_name print 'Viola will output evaluation to: {0}'.format(self.out_folder) self.img_names = [f for f in os.listdir(in_path) if f.endswith('.jpg')] self.save_imgs = save_imgs self.output_patches = output_patches self.strict = strict self.negThres = negThres self.mergeFalsePos = mergeFalsePos self.rotateRectOnly = rotateRectOnly self.viola_detections = Detections(mergeFalsePos=self.mergeFalsePos) self.setup_detectors(detector_names) #parameters for detection self.scale = scale self.min_neighbors = int(min_neighbors) self.group = group self.overlapThresh = overlapThresh if rotate: self.angles = utils.VIOLA_ANGLES else: self.angles = [0] self.remove_off_img = removeOff self.downsized = downsized self.pickled_evaluation = pickled_evaluation if pickled_evaluation == False: self.evaluation = Evaluation(full_dataset=False, negThres=self.negThres, method='viola', folder_name=folder_name, out_path=self.out_folder, detections=self.viola_detections, in_path=self.in_path, detector_names=detector_names, mergeFalsePos=mergeFalsePos, vocGood=vocGood) else: with open(self.out_folder + 'evaluation.pickle', 'rb') as f: self.evaluation = pickle.load(f) def setup_detectors(self, detector_names=None, old_detector=False): '''Given a list of detector names, get the detectors specified ''' #get the detectors assert detector_names is not None self.roof_detectors = defaultdict(list) self.detector_names = detector_names self.rotate_detectors = list() rectangular_detector = 'cascade_metal_rect_augm1_singlesize_original_pad0_num872_w40_h20_FA0.4_LBP' for roof_type in utils.ROOF_TYPES: for i, path in enumerate(detector_names[roof_type]): if rectangular_detector in path or self.rotateRectOnly == False: self.rotate_detectors.append(True) else: self.rotate_detectors.append(False) if path.startswith('cascade'): start = '../viola_jones/cascades/' self.roof_detectors[roof_type].append( cv2.CascadeClassifier(start + path + '/cascade.xml')) assert self.roof_detectors[roof_type][-1].empty() == False else: self.roof_detectors[roof_type].append( cv2.CascadeClassifier('../viola_jones/cascade_' + path + '/cascade.xml')) def detect_roofs_in_img_folder(self): '''Compare detections to ground truth roofs for set of images in a folder ''' for i, img_name in enumerate(self.img_names): print '************************ Processing image {0}/{1}\t{2} ************************'.format( i, len(self.img_names), img_name) if self.group: img = self.detect_roofs_group(img_name) else: img = self.detect_roofs(img_name) ''' self.evaluation.score_img(img_name, img.shape) self.evaluation.print_report() with open('{0}evaluation.pickle'.format(self.out_folder), 'wb') as f: pickle.dump(self.evaluation, f) if self.output_patches: self.evaluation.save_training_TP_FP_using_voc() open(self.out_folder+'DONE', 'w').close() ''' def detect_roofs(self, img_name, in_path=None): in_path = self.in_path if in_path is None else in_path try: rgb_unrotated = cv2.imread(in_path + img_name, flags=cv2.IMREAD_COLOR) gray = cv2.cvtColor(rgb_unrotated, cv2.COLOR_BGR2GRAY) gray = cv2.equalizeHist(gray) if self.downsized: rgb_unrotated = utils.resize_rgb(rgb_unrotated, h=rgb_unrotated.shape[0] / 2, w=rgb_unrotated.shape[1] / 2) gray = utils.resize_grayscale(gray, w=gray.shape[1] / 2, h=gray.shape[0] / 2) except IOError as e: print e sys.exit(-1) else: for roof_type, detectors in self.roof_detectors.iteritems(): for i, detector in enumerate(detectors): for angle in self.angles: #for thatch we only need one angle if self.rotate_detectors[i] == False and angle > 0 or ( roof_type == 'thatch' and angle > 0 ): #roof_type == 'thatch' and angle>0: continue print 'Detecting with detector: ' + str(i) print 'ANGLE ' + str(angle) with Timer() as t: rotated_image = utils.rotate_image( gray, angle) if angle > 0 else gray delete_image = utils.rotate_image_RGB( rgb_unrotated, angle) if angle > 0 else gray detections, _ = self.detect_and_rectify( detector, rotated_image, angle, rgb_unrotated.shape[:2], rgb_rotated=delete_image) if self.downsized: detections = detections * 2 self.viola_detections.set_detections( roof_type=roof_type, img_name=img_name, angle=angle, detection_list=detections, img=rotated_image) print 'Time detection: {0}'.format(t.secs) self.viola_detections.total_time += t.secs if DEBUG: rgb_to_write = cv2.imread(in_path + img_name, flags=cv2.IMREAD_COLOR) utils.draw_detections(detections, rgb_to_write, color=(255, 0, 0)) cv2.imwrite( '{0}{3}{1}_{2}.jpg'.format( '', img_name[:-4], angle, roof_type), rgb_to_write) return rgb_unrotated def detect_and_rectify(self, detector, image, angle, dest_img_shape, rgb_rotated=None): #do the detection detections = detector.detectMultiScale(image, scaleFactor=self.scale, minNeighbors=self.min_neighbors) ''' print 'were about to save the rotated images, if you dont want this, quit and remove this from like 232 in viola_detector.py' pdb.set_trace() for d in detections: cv2.rectangle(rgb_rotated, (d[0], d[1]), (d[0]+d[2], d[1]+d[3]), (255,255,255), 4) cv2.imwrite('rotated.jpg', rgb_rotated) pdb.set_trace() ''' #convert to proper coordinate system polygons = utils.convert_detections_to_polygons(detections) if angle > 0: #rotate back to original image coordinates print 'rotating...' rectified_detections = utils.rotate_detection_polygons( polygons, image, angle, dest_img_shape, remove_off_img=self.remove_off_img) else: rectified_detections = polygons print 'done rotating' if self.group: bounding_boxes = utils.get_bounding_boxes( np.array(rectified_detections)) else: bounding_boxes = None return rectified_detections, bounding_boxes ''' def detect_roofs_group(self, img_name): try: rgb_unrotated = cv2.imread(self.in_path+img_name, flags=cv2.IMREAD_COLOR) gray = cv2.cvtColor(rgb_unrotated, cv2.COLOR_BGR2GRAY) gray = cv2.equalizeHist(gray) except IOError as e: print e sys.exit(-1) else: for roof_type, detectors in self.roof_detectors.iteritems(): all_detections = list() for i, detector in enumerate(detectors): for angle in self.angles: #for thatch we only need one angle if roof_type == 'thatch' and angle>0: continue print 'Detecting with detector: '+str(i) print 'ANGLE '+str(angle) with Timer() as t: rotated_image = utils.rotate_image(gray, angle) if angle>0 else gray detections, bounding_boxes = self.detect_and_rectify(detector, rotated_image, angle, rgb_unrotated.shape[:2]) all_detections.append(list(bounding_boxes)) print 'Time detection: {0}'.format(t.secs) self.viola_detections.total_time += t.secs #grouping all_detections = [d for detections in all_detections for d in detections] grouped_detections, rects_grouped = cv2.groupRectangles(all_detections, 1) print "GROUPING DOWN BY:" print len(all_detections)-len(grouped_detections) grouped_polygons = utils.convert_detections_to_polygons(grouped_detections) #merge the detections from all angles self.viola_detections.set_detections(roof_type=roof_type, img_name=img_name, detection_list=grouped_polygons, img=rotated_image) print 'Detections for {0}'.format(roof_type) print len(self.viola_detections.get_detections(roof_type=roof_type, img_name=img_name)) return rgb_unrotated ''' def mark_save_current_rotation(self, img_name, img, detections, angle, out_folder=None): out_folder = self.out_folder if out_folder is None else out_folder polygons = np.zeros((len(detections), 4, 2)) for i, d in enumerate(detections): polygons[i, :] = utils.convert_rect_to_polygon(d) img = self.evaluation.mark_roofs_on_img(img_name=img_name, img=img, roofs=polygons, color=(0, 0, 255)) path = '{0}_angle{1}.jpg'.format(out_folder + img_name[:-4], angle) print path cv2.imwrite(path, img)