def main(): parser = argparse.ArgumentParser() parser.add_argument("infile", type=str, help="List of images for which to produce masks") parser.add_argument("outfile", type=str, help="Path to write predictions") args = parser.parse_args() params = vars(args) # --- Load some necessary helpers ---------------------------------------------------------------------------------- image_id_index = get_image_id_info_index() attr_id_to_name, attr_id_to_idx = load_attributes() # Load image_ids --------------------------------------------------------------------------------------------------- image_id_set = set() with open(params['infile']) as f: for _line in f: _, image_name = osp.split(_line.strip()) image_id, ext = osp.splitext(image_name) image_id_set.add(image_id) # Process these ids ------------------------------------------------------------------------------------------------ predictions = [] n_files = len(image_id_set) for idx, image_id in enumerate(image_id_set): sys.stdout.write("Processing %d/%d (%.2f%% done) \r" % (idx, n_files, (idx * 100.0) / n_files)) sys.stdout.flush() # Load extra-annotation for this image_id fold = image_id_index[image_id]['fold'] image_path = image_id_index[image_id]['image_path'] extra_anno_path = osp.join(EXTRA_ANNO_PATH, fold, image_id + '.json') image_width, image_height = get_image_size(image_path) with open(extra_anno_path) as jf: eanno = json.load(jf) for face_entry in eanno.get('faceAnnotations', []): this_poly = [bb_to_verts(face_entry['fdBoundingPoly']), ] rles = mask.frPyObjects(this_poly, image_height, image_width) rle = mask.merge(rles) predictions.append({ 'image_id': image_id, 'attr_id': 'a105_face_all', 'segmentation': rle, 'score': face_entry['detectionConfidence'], }) # Dump predictions ------------------------------------------------------------------------------------------------- print 'Writing {} predictions to file: {}'.format(len(predictions), params['outfile']) with open(params['outfile'], 'wb') as wf: json.dump(predictions, wf, indent=2)
def dct_to_mask_list(filename_to_probs, fname_index, idx_to_attr_id, attr_set_use, image_id_to_saliency=None): prediction_list = [] n_files = len(filename_to_probs) for idx, (filename, probs) in enumerate(filename_to_probs.iteritems()): sys.stdout.write("Processing %d/%d (%.2f%% done) \r" % (idx, n_files, (idx * 100.0) / n_files)) sys.stdout.flush() image_path = fname_index[filename] image_id, ext = osp.splitext(filename) w, h = get_image_size(image_path) if image_id_to_saliency.get(image_id, None) is not None: # Use saliency mask instead lowres_saliency = image_id_to_saliency[image_id] # Resize mask (originally is 321 x 321) highres_saliency = scipy.misc.imresize(lowres_saliency, [h, w], interp='bilinear', mode='F') # Binarize it bimask = ( highres_saliency > (SALIENCY_THRESH * np.max(highres_saliency))).astype('uint8') bimask = np.asfortranarray(bimask) del lowres_saliency del highres_saliency else: bimask = np.ones((h, w), order='F', dtype='uint8') rle = mask.encode(bimask) del bimask for this_attr_idx, this_attr_prob in enumerate(probs): this_attr_id = idx_to_attr_id[this_attr_idx] if this_attr_id in attr_set_use and this_attr_prob > ATTR_THRESH: score_dct = { 'image_id': image_id, 'attr_id': this_attr_id, 'segmentation': rle, 'score': this_attr_prob, } prediction_list.append(score_dct) return prediction_list
def main(): parser = argparse.ArgumentParser() parser.add_argument("infile", type=str, help="List of images for which to produce masks") parser.add_argument("outfile", type=str, help="Path to write predictions") parser.add_argument("regions", type=str, choices=['block', 'paragraph', 'word', 'symbol', 'all'], help="Which element to extract?") parser.add_argument("-g", "--gt_labels", action='store_true', default=False, help="Create GT-label based predictions instead") args = parser.parse_args() params = vars(args) # --- Load some necessary helpers ---------------------------------------------------------------------------------- image_id_index = get_image_id_info_index() _, attr_id_to_idx_v1 = load_attributes(v1_attributes=True) idx_to_attr_id_v1 = {v: k for k, v in attr_id_to_idx_v1.iteritems()} _, attr_id_to_idx = load_attributes() idx_to_attr_id = {v: k for k, v in attr_id_to_idx.iteritems()} # Load image_ids --------------------------------------------------------------------------------------------------- image_id_set = set() with open(params['infile']) as f: for _line in f: _, image_name = osp.split(_line.strip()) image_id, ext = osp.splitext(image_name) image_id_set.add(image_id) # Load GT attributes too anno = json.load(open(image_id_index[image_id]['anno_path'])) gt_vec_v1 = labels_to_vec(anno['labels'], attr_id_to_idx_v1) gt_vec = prev_to_new_attr_vec(gt_vec_v1, attr_id_to_idx_v1, attr_id_to_idx) image_id_index[image_id]['gt_attr_id_list'] = vec_to_labels(gt_vec, idx_to_attr_id) # Process these ids ------------------------------------------------------------------------------------------------ region_to_predict = params['regions'] predictions = [] n_files = len(image_id_set) for idx, image_id in enumerate(image_id_set): sys.stdout.write("Processing %d/%d (%.2f%% done) \r" % (idx, n_files, (idx * 100.0) / n_files)) sys.stdout.flush() # Load extra-annotation for this image_id fold = image_id_index[image_id]['fold'] image_path = image_id_index[image_id]['image_path'] extra_anno_path = osp.join(EXTRA_ANNO_PATH, fold, image_id + '.json') if params['gt_labels']: gt_attr_id_list = image_id_index[image_id]['gt_attr_id_list'] else: gt_attr_id_list = attr_id_to_idx.keys() image_width, image_height = get_image_size(image_path) with open(extra_anno_path) as jf: eanno = json.load(jf) if 'fullTextAnnotation' not in eanno: continue for page in eanno['fullTextAnnotation']['pages']: page['width'] = page['width'] page['height'] = page['height'] # ------- Block for block in page['blocks']: block_poly = [bb_to_verts(block['boundingBox']), ] block_rles = mask.frPyObjects(block_poly, image_height, image_width) block_rle = mask.merge(block_rles) if region_to_predict in {'block', 'all'}: predictions += get_predictions(image_id, block_rle, gt_attr_id_list=gt_attr_id_list) if 'all' != region_to_predict: continue # ------- Paragraph for paragraph in block['paragraphs']: paragraph_poly = [bb_to_verts(paragraph['boundingBox']), ] paragraph_rles = mask.frPyObjects(paragraph_poly, image_height, image_width) paragraph_rle = mask.merge(paragraph_rles) if region_to_predict in {'paragraph', 'all'}: predictions += get_predictions(image_id, paragraph_rle, gt_attr_id_list=gt_attr_id_list) if 'all' != region_to_predict: continue # ------- Word for word in paragraph['words']: word_poly = [bb_to_verts(word['boundingBox']), ] word_rles = mask.frPyObjects(word_poly, image_height, image_width) word_rle = mask.merge(word_rles) if region_to_predict in {'word', 'all'}: predictions += get_predictions(image_id, word_rle, gt_attr_id_list=gt_attr_id_list) if 'all' != region_to_predict: continue # ------- Symbols for symbol in word['symbols']: symbol_poly = [bb_to_verts(symbol['boundingBox']), ] symbol_rles = mask.frPyObjects(symbol_poly, image_height, image_width) symbol_rle = mask.merge(symbol_rles) if region_to_predict in {'symbol', 'all'}: predictions += get_predictions(image_id, symbol_rle, gt_attr_id_list=gt_attr_id_list) # Dump predictions ------------------------------------------------------------------------------------------------- print 'Writing {} predictions to file: {}'.format(len(predictions), params['outfile']) with open(params['outfile'], 'wb') as wf: json.dump(predictions, wf, indent=2)
def main(): parser = argparse.ArgumentParser() parser.add_argument( "infile", type=str, help="Image Paths (example line: images/val2017/2017_25057208.jpg)") parser.add_argument("outfile", type=str, help="Path to write predictions") parser.add_argument("weights", type=str, help="Path to .caffemodel") parser.add_argument("deploy", type=str, help="Path to deploy.prototxt") parser.add_argument("-r", "--DS_ROOT", type=str, default=DS_ROOT, help="Override VISPR root") parser.add_argument("-d", "--device", type=int, choices=[0, 1, 2, 3], default=0, help="GPU device id") parser.add_argument("-b", "--batch_size", type=int, default=64, help="Batch size") parser.add_argument("-u", "--use_attributes", type=str, default=None, help="Use only these attributes") args = parser.parse_args() params = vars(args) # Initialize Network ----------------------------------------------------------------------------------------------- caffe.set_device(params['device']) caffe.set_mode_gpu() model_def = params['deploy'] model_weights = params['weights'] net = caffe.Net( model_def, # defines the structure of the model model_weights, # contains the trained weights caffe.TEST) # use test mode (e.g., don't perform dropout) # Set up transformer ----------------------------------------------------------------------------------------------- # load the mean ImageNet image (as distributed with Caffe) for subtraction mu = np.load( osp.join(CAFFE_ROOT, 'python/caffe/imagenet/ilsvrc_2012_mean.npy')) mu = mu.mean(1).mean( 1) # average over pixels to obtain the mean (BGR) pixel values # mu = np.asarray([111.0, 102.0, 116.0]) # mu = np.asarray([104.0, 117.0, 123.0]) # print 'mean-subtracted values:', zip('BGR', mu) # create transformer for the input called 'data' transformer = caffe.io.Transformer({'data': net.blobs['data'].data.shape}) transformer.set_transpose( 'data', (2, 0, 1)) # move image channels to outermost dimension transformer.set_mean('data', mu) # subtract the dataset-mean value in each channel transformer.set_raw_scale('data', 255) # rescale from [0, 1] to [0, 255] transformer.set_channel_swap('data', (2, 1, 0)) # swap channels from RGB to BGR # Classify --------------------------------------------------------------------------------------------------------- filename_to_probs = classify_paths(net, transformer, params['infile'], batch_size=params['batch_size'], dataset_root=params['DS_ROOT']) # Convert to new attribute format ---------------------------------------------------------------------------------- # PAP was designed on 68 attributes, but now we have 74 attributes # So, convert previous attribute vectors to new vectors _, attr_id_to_idx_v1 = load_attributes(v1_attributes=True) idx_to_attr_id_v1 = {v: k for k, v in attr_id_to_idx_v1.iteritems()} _, attr_id_to_idx = load_attributes() idx_to_attr_id = {v: k for k, v in attr_id_to_idx.iteritems()} for filename, probs in filename_to_probs.iteritems(): if '254777' in filename: top_10_preds = np.argsort(-probs)[:10] for this_attr_idx in top_10_preds: print idx_to_attr_id_v1[this_attr_idx], probs[this_attr_idx] print for file_id in filename_to_probs: filename_to_probs[file_id] = prev_to_new_attr_vec( filename_to_probs[file_id], attr_id_to_idx_v1, attr_id_to_idx) # Predict masks from attributes ------------------------------------------------------------------------------------ # Create a mask spanning the entire image for each predicted attribute # Required format in: privacy_filters/tools/scripts/evaluate.py if params['use_attributes'] is None: attr_set_use = set(attr_id_to_idx.keys()) else: attr_set_use = set( map(lambda x: x.strip(), open(params['use_attributes']).readlines())) prediction_list = [] n_files = len(filename_to_probs) n_attr = len(attr_set_use) fname_index = get_image_filename_index() print 'Writing masks for {} attributes and {} files'.format( n_attr, n_files) for filename, probs in filename_to_probs.iteritems(): image_path = fname_index[filename] file_id, ext = osp.splitext(filename) w, h = get_image_size(image_path) bimask = np.ones((h, w), order='F', dtype='uint8') rle = mask.encode(bimask) del bimask if '254777' in filename: top_10_preds = np.argsort(-probs)[:10] for this_attr_idx in top_10_preds: print idx_to_attr_id[this_attr_idx], probs[this_attr_idx] for this_attr_idx, this_attr_prob in enumerate(probs): this_attr_id = idx_to_attr_id[this_attr_idx] if this_attr_id in attr_set_use and this_attr_prob > THRESH: score_dct = { 'image_id': file_id, 'attr_id': this_attr_id, 'segmentation': rle, 'score': this_attr_prob, } prediction_list.append(score_dct) # Write scores ----------------------------------------------------------------------------------------------------- with open(params['outfile'], 'w') as wf: json.dump(prediction_list, wf, indent=2)
def main(): parser = argparse.ArgumentParser() parser.add_argument("anno_list", type=str, help="Path to Annotation file (from VISPR-2017)") parser.add_argument( "red_dir", type=str, help="Looks for reduced file-size images in this directory") parser.add_argument( "output_dir", type=str, help="Places annotation files in this directory as <filename>.json") parser.add_argument("-i", "--skip_if_exists", action='store_true', default=True, help="Skip image if it exists") args = parser.parse_args() params = vars(args) anno_files = [x.strip() for x in open(params['anno_list']).readlines()] print '# Files to process = ', len(anno_files) n_processed = 0 n_skipped = 0 for idx, line in enumerate(anno_files): _, file_id_ext = osp.split(line) file_id, _ = osp.splitext(file_id_ext) out_path = osp.join(params['output_dir'], file_id + '-extra.json') if osp.exists(out_path) and params['skip_if_exists']: n_skipped += 1 continue sys.stdout.write("Processing %d/%d (%.2f%% done) \r" % (idx, len(anno_files), idx * 100.0 / len(anno_files))) sys.stdout.flush() # ------------- Get Image Path anno_path = osp.abspath(osp.join(DS_ROOT, line)) with open(anno_path) as jf: anno = json.load(jf) img_path_partial = anno['image_path'] _, img_filename = osp.split(img_path_partial) # Check if a reduced version of this image exists org_img_path = osp.abspath(osp.join(DS_ROOT, img_path_partial)) org_w, org_h = get_image_size(org_img_path) reduced_img_path = osp.join(params['red_dir'], img_filename) if osp.exists(reduced_img_path): image_path = osp.abspath(reduced_img_path) new_w, new_h = get_image_size(image_path) else: image_path = org_img_path new_w, new_h = org_w, org_h # ------------- Process Response vision_response_json = get_annotation(image_path) vision_response_dct = json.loads(vision_response_json) if len(vision_response_dct['responses']) != 1: print 'WARNING: Got {} responses for request: {}'.format( len(vision_response_dct['responses']), file_id) # Only use the first response (which is always the case since we query only a single image) vision_response_dct = vision_response_dct['responses'][0] # Add some additional details vision_response_dct['file_info'] = { 'image_path_prev': image_path, 'img_size_prev': (new_w, new_h), 'image_path': org_img_path, 'size': (org_w, org_h), 'anno_path': anno_path, } if org_img_path != image_path: vision_response_dct = fix_sizes(vision_response_dct, (org_w, org_h), (new_w, new_h)) # ------------- Write Response with open(out_path, 'wb') as wf: json.dump(vision_response_dct, wf, indent=2) n_processed += 1 print '# Processed: ', n_processed print '# Skipped: ', n_skipped
def clean_via_annotations(anno_path, img_fname_index=None, return_all=True): """ Clean and add some additional info to via annotations. Example pre-cleaned annotation: https://pastebin.com/cP3RCS3i Example post-cleaned annotation file: https://pastebin.com/8ifs3RxM :param anno_path: :param img_fname_index: :param return_all: If true, returns all entries of the annotation file. Otherwise, drops certain entries based on file-level attributes (e.g., crowds or discards) :return: """ if img_fname_index is None: img_fname_index = get_image_filename_index() with open(anno_path) as jf: via_anno = json.load(jf) via_cleaned_anno = dict() # The annotations are indexed by <filename><file_size>. Fix this to just <filename>. # Additionally, add filepath to entry for key, entry in via_anno.iteritems(): this_img_filename = entry['filename'] this_file_level_attr = set(entry['file_attributes'].keys()) if len(this_file_level_attr & IGNORE_IF_FILE_ATTR) > 0: ignore_file = True # This annotation entry contains one of the ignore attributes n_regions = 0 if override_if_regions_exist: n_regions += len(entry['regions']) if 'full_scan' in entry['file_attributes']: n_regions += 1 if n_regions > 0: ignore_file = False if not ignore_file: pass elif not return_all: # Simply ignore this entry continue # I sometimes add attr_id to beginning of filename for readability. Strip it if exists # For example: a0_safe_2017_15285423.jpg if not this_img_filename.startswith('2017'): prefix = this_img_filename.split('2017')[0] this_img_filename = this_img_filename.replace(prefix, '') entry['filename'] = this_img_filename if 'filepath' not in entry: this_img_filepath = img_fname_index[this_img_filename] entry['filepath'] = this_img_filepath via_cleaned_anno[this_img_filename] = entry # Convert file-level attribute 'full_scan' to a polygon spanning entire region if 'full_scan' in entry['file_attributes']: # What's the image dimension? w, h = get_image_size(entry['filepath']) # Use this margin (in pixels) scan_margin = 1 # Construct region full_region_dct = { "shape_attributes": { "name": "polygon", "all_points_x": [], "all_points_y": [], "all_ts": [] }, "region_attributes": {} } full_region_dct["shape_attributes"]["all_points_x"] = [ scan_margin, w - scan_margin, w - scan_margin, scan_margin, scan_margin ] full_region_dct["shape_attributes"]["all_points_y"] = [ scan_margin, scan_margin, h - scan_margin, h - scan_margin, scan_margin ] # Add this region to the list of existing regions if len(entry['regions'].keys()) > 0: next_region_id = max(map(int, entry['regions'].keys())) + 1 else: next_region_id = 0 entry['regions'][str(next_region_id)] = full_region_dct shapes_in_anno = set([ region_dct['shape_attributes']['name'] for k, region_dct in entry['regions'].iteritems() ]) # This is of the format: {u'cy': 484, u'cx': 1078, u'r': 38, u'name': u'circle'} if 'circle' in shapes_in_anno: for k, region_dct in entry['regions'].iteritems(): if region_dct['shape_attributes']['name'] == 'circle': new_region_dct = { "shape_attributes": { "name": "polygon", "all_points_x": [], "all_points_y": [], "all_ts": [] }, "region_attributes": region_dct['region_attributes'] } _cx, _cy = region_dct['shape_attributes'][ 'cx'], region_dct['shape_attributes']['cy'] _r = region_dct['shape_attributes']['r'] # No. of polygons to represent this circle n_circle_vertices = 16 for i in range(n_circle_vertices): angle = (2.0 * np.pi / n_circle_vertices) * i _x = _cx + _r * np.cos(angle) _y = _cy + _r * np.sin(angle) new_region_dct['shape_attributes'][ 'all_points_x'].append(_x) new_region_dct['shape_attributes'][ 'all_points_y'].append(_y) # Close the loop new_region_dct['shape_attributes']['all_points_x'].append( new_region_dct['shape_attributes']['all_points_x'][0]) new_region_dct['shape_attributes']['all_points_y'].append( new_region_dct['shape_attributes']['all_points_y'][0]) entry['regions'][k] = new_region_dct # This is of the format: {u'y': 273, u'x': 300, u'height': 13, u'name': u'rect', u'width': 77} if 'rect' in shapes_in_anno: for k, region_dct in entry['regions'].iteritems(): if region_dct['shape_attributes']['name'] == 'rect': new_region_dct = { "shape_attributes": { "name": "polygon", "all_points_x": [], "all_points_y": [], "all_ts": [] }, "region_attributes": region_dct['region_attributes'] } _x, _y = region_dct['shape_attributes']['x'], region_dct[ 'shape_attributes']['y'] _h, _w = region_dct['shape_attributes'][ 'height'], region_dct['shape_attributes']['width'] new_region_dct['shape_attributes']['all_points_x'] = [ _x, _x + _w, _x + _w, _x, _x ] new_region_dct['shape_attributes']['all_points_y'] = [ _y, _y, _y + _h, _y + _h, _y ] entry['regions'][k] = new_region_dct # Add an instance id to each region # Each region either: (a) contains an instance_id attribute OR (b) does not contain it # In case of (b), we need to assign it a random id # Generate some random IDs rand_ids = range(500) # Mapping to tag instance "p_N" to an instance id tag_to_instance_id = dict() for k, region_dct in entry['regions'].iteritems(): if 'instance_id' in region_dct['region_attributes']: # This region has been tagged with a "p_N" if region_dct['region_attributes'][ 'instance_id'] in tag_to_instance_id: this_instance_id = tag_to_instance_id[ region_dct['region_attributes']['instance_id']] else: this_instance_id = rand_ids.pop(0) tag_to_instance_id[region_dct['region_attributes'] ['instance_id']] = this_instance_id else: this_instance_id = rand_ids.pop(0) region_dct['assigned_instance_id'] = this_instance_id # Remove spurious polygons i.e., with just 2 points for key, entry in via_cleaned_anno.iteritems(): for k in entry['regions'].keys(): region_dct = entry['regions'][k] if region_dct['shape_attributes']['name'] == 'polygon': if len(region_dct['shape_attributes']['all_points_x']) < 3: del via_cleaned_anno[key]['regions'][k] # Sometimes VIA uses negative values for points close to the border. Replace them with 0s for key, entry in via_cleaned_anno.iteritems(): for k, region_dct in entry['regions'].iteritems(): if region_dct['shape_attributes']['name'] == 'polygon': all_x = region_dct['shape_attributes']['all_points_x'] all_y = region_dct['shape_attributes']['all_points_y'] if any([z < 0. for z in all_x + all_y]): all_pos_x = [max(0., x) for x in all_x] all_pos_y = [max(0., y) for y in all_y] region_dct['shape_attributes']['all_points_x'] = all_pos_x region_dct['shape_attributes']['all_points_y'] = all_pos_y # Remove file attributes with blank values for key, entry in via_cleaned_anno.iteritems(): emtpy_k_list = [] for k, v in entry['file_attributes'].iteritems(): if v == '': emtpy_k_list.append(k) for k in emtpy_k_list: del via_cleaned_anno[key]['file_attributes'][k] # FIX: Annotator used '[6-10]' and '[10+]' as file labels. Replace them if '[6-10]' in entry['file_attributes'].keys(): entry['file_attributes']['crowd_6-10'] = entry[ 'file_attributes'].pop('[6-10]') if '[10+]' in entry['file_attributes'].keys(): entry['file_attributes']['crowd_10+'] = entry[ 'file_attributes'].pop('[10+]') if '[Unsure]' in entry['file_attributes'].keys(): entry['file_attributes']['unsure'] = entry['file_attributes'].pop( '[Unsure]') return via_cleaned_anno
def collate(fold_name, snapshot_name): # --- Setup paths -------------------------------------------------------------------------------------------------- # Location of annotated batches - Persons phase2_batch_dir = osp.join(SEG_ROOT, 'phase2', 'annotations', fold_name) # Location of annotated batches - Other Attributes phase4_batch_dir = osp.join(SEG_ROOT, 'phase4', 'annotations', fold_name) # Filename -> Filepath img_filename_index = get_image_filename_index() # Out directory final_out_dir = osp.join(SEG_ROOT, 'annotations', snapshot_name) final_out_path = osp.join(final_out_dir, '{}.json'.format(fold_name)) assert not osp.exists( final_out_path ), 'Output path {} exists. Delete it and try again.'.format(final_out_path) # --- Get Person Annotations --------------------------------------------------------------------------------------- # Create a mapping of file_name -> ImageAnnotation file_id_to_img_anno = dict() n_written = 0 n_skipped = 0 n_dupl = 0 batch_anno_filenames = os.listdir(phase2_batch_dir) print 'Processing attribute "Persons"... ' for batch_idx, batch_fname in enumerate( sorted(batch_anno_filenames, key=lambda x: int(osp.splitext(x)[0]))): # Iterate over each batch batch_filepath = osp.join(phase2_batch_dir, batch_fname) via_list = clean_via_annotations(batch_filepath, img_fname_index=img_filename_index, return_all=True) for file_name, entry in via_list.iteritems(): img_path = entry['filepath'] try: w, h = get_image_size(img_path) except ZeroDivisionError: print file_name raise file_attr_dct = entry['file_attributes'] file_id, ext = osp.splitext(file_name) # Skip this image if: a) it contains crowd attributes b) contains an unsure tag c) does not contain regions skip_image = False skip_file_attr = {'crowd_6-10', 'crowd_10+', 'unsure'} if len(set(file_attr_dct.keys()) & skip_file_attr) > 0: skip_image = True if len(entry['regions']) < 1: skip_image = True if skip_image: n_skipped += 1 continue # -- At this point, this anno blob *should* contain regions ainst_id_to_attr_anno = dict() # Iterate over each anno region for region in entry['regions'].values(): all_points_x = region['shape_attributes']['all_points_x'] all_points_y = region['shape_attributes']['all_points_y'] assigned_instance_id = region['assigned_instance_id'] # Squish x and y into [x1 y1 x2 y2 ...] polygon = [ z for xy_tup in zip(all_points_x, all_points_y) for z in xy_tup ] if assigned_instance_id in ainst_id_to_attr_anno: ainst_id_to_attr_anno[assigned_instance_id].add_polygon( polygon) else: try: this_attr_anno = AttributeAnnotation( assigned_instance_id, PERSON_ATTR_ID, [ polygon, ]) except AssertionError: print file_name, batch_filepath raise ainst_id_to_attr_anno[ assigned_instance_id] = this_attr_anno # Create an ImageAnnotation object for this image this_img_anno = ImageAnnotation(file_id, img_path, h, w) for attr_anno in ainst_id_to_attr_anno.values(): this_img_anno.add_attribute_annotation(attr_anno) assert file_name not in file_id_to_img_anno file_id_to_img_anno[file_id] = this_img_anno # --- Get Annotations for other Attributes ------------------------------------------------------------------------- # Walk through the remaining attributes and add these annotations to batch of ImageAnnotations attr_list = os.listdir(phase4_batch_dir) for attr_id in attr_list: attr_batch_dir = osp.join(phase4_batch_dir, attr_id) batch_anno_filenames = os.listdir(attr_batch_dir) if len(batch_anno_filenames) > 0: # Skip batches which were used for consensus (e.g., 0_abc.json) batch_anno_filenames = filter( lambda x: re.search('^[0-9]+$', osp.splitext(x)[0]), batch_anno_filenames) if attr_id not in RENAME_RULES: print 'Processing attribute "{}" (# Batches = {})... '.format( attr_id, len(batch_anno_filenames)) else: print 'Processing attribute "{}" (renamed as {}) (# Batches = {})... '.format( attr_id, RENAME_RULES[attr_id], len(batch_anno_filenames)) for batch_idx, batch_fname in enumerate( sorted(batch_anno_filenames, key=lambda x: int(osp.splitext(x)[0]))): # Iterate over each batch batch_filepath = osp.join(phase4_batch_dir, attr_id, batch_fname) via_list = clean_via_annotations( batch_filepath, img_fname_index=img_filename_index, return_all=True) for file_name, entry in via_list.iteritems(): img_path = entry['filepath'] w, h = get_image_size(img_path) file_attr_dct = entry['file_attributes'] file_id, ext = osp.splitext(file_name) # Skip this image if: a) contains an unsure tag c) does not contain regions skip_image = False # skip_file_attr = {'crowd_6-10', 'crowd_10+', 'unsure'} skip_file_attr = { 'unsure', } if len(set(file_attr_dct.keys()) & skip_file_attr) > 0: skip_image = True if len(entry['regions']) < 1: skip_image = True if skip_image: n_skipped += 1 continue # -- At this point, this anno blob *should* contain regions ainst_id_to_attr_anno = dict() # Iterate over each anno region for region in entry['regions'].values(): try: all_points_x = region['shape_attributes'][ 'all_points_x'] all_points_y = region['shape_attributes'][ 'all_points_y'] except KeyError: print file_name raise assigned_instance_id = region['assigned_instance_id'] # Squish x and y into [x1 y1 x2 y2 ...] polygon = [ z for xy_tup in zip(all_points_x, all_points_y) for z in xy_tup ] if assigned_instance_id in ainst_id_to_attr_anno: ainst_id_to_attr_anno[ assigned_instance_id].add_polygon(polygon) else: try: new_attr_id = RENAME_RULES.get(attr_id, attr_id) this_attr_anno = AttributeAnnotation( assigned_instance_id, new_attr_id, [ polygon, ]) except AssertionError: print file_name, batch_filepath, polygon raise ainst_id_to_attr_anno[ assigned_instance_id] = this_attr_anno if file_id in file_id_to_img_anno: # Retrieve the ImageAnnotation if it was created previously this_img_anno = file_id_to_img_anno[file_id] else: this_img_anno = ImageAnnotation(file_id, img_path, h, w) for attr_anno in ainst_id_to_attr_anno.values(): this_img_anno.add_attribute_annotation(attr_anno) file_id_to_img_anno[file_id] = this_img_anno # --- Complete other instance statistics (eg., rle, area) ---------------------------------------------------------- print 'Inferring instance statistics...' for img_anno in file_id_to_img_anno.values(): img_anno.finalize() # --- Write Annotations -------------------------------------------------------------------------------------------- anno_to_write = { 'annotations': file_id_to_img_anno, 'created_at': str(datetime.datetime.now()), 'stats': anno_stats(file_id_to_img_anno) } if not osp.exists(final_out_dir): print '{} does not exist. Creating it...'.format(final_out_dir) os.makedirs(final_out_dir) with open(final_out_path, 'wb') as wjf: json.dump(anno_to_write, wjf, indent=2, cls=AnnoEncoder)
def main(): parser = argparse.ArgumentParser() parser.add_argument("infile", type=str, help="List of images for which to produce masks") parser.add_argument( "pkl_file", type=str, help="Pickled file containing {image_id -> (localization_masks, " "attr_probs)} information") parser.add_argument("outfile", type=str, help="Path to write predictions") parser.add_argument("-u", "--use_attributes", type=str, default=None, help="Use only these attributes") parser.add_argument( "-g", "--gt_labels", action='store_true', default=False, help= "Create GT-label based predictions instead (i.e., ignore attribute predictions from " "CAM PAP model)") parser.add_argument("-i", "--instances", action='store_true', default=False, help="Predict instances in each image") args = parser.parse_args() params = vars(args) # image_id -> {image_path, anno_path, fold} image_id_index = get_image_id_info_index() _, attr_id_to_idx_v1 = load_attributes(v1_attributes=True) idx_to_attr_id_v1 = {v: k for k, v in attr_id_to_idx_v1.iteritems()} _, attr_id_to_idx = load_attributes() idx_to_attr_id = {v: k for k, v in attr_id_to_idx.iteritems()} if params['use_attributes'] is None: attr_set_use = set(attr_id_to_idx.keys()) else: attr_set_use = set( map(lambda x: x.strip(), open(params['use_attributes']).readlines())) # Load image_ids --------------------------------------------------------------------------------------------------- image_id_set = set() with open(params['infile']) as f: for _line in f: _, image_name = osp.split(_line.strip()) image_id, ext = osp.splitext(image_name) image_id_set.add(image_id) # Load GT attributes too anno = json.load(open(image_id_index[image_id]['anno_path'])) gt_vec_v1 = labels_to_vec(anno['labels'], attr_id_to_idx_v1) gt_vec = prev_to_new_attr_vec(gt_vec_v1, attr_id_to_idx_v1, attr_id_to_idx) image_id_index[image_id]['gt_vec'] = gt_vec # Load weakly supervised predictions ------------------------------------------------------------------------------- print 'Loading weakly supervised masks...' # {image_id -> (localization_masks, attr_probs)} # where localization_masks = 68x41x41 matrix # attr_probs = 68 vector image_id_to_info = pickle.load(open(params['pkl_file'])) predictions = [] print 'Creating predictions from masks...' # Weakly supervised CAM masks -> instance predictions -------------------------------------------------------------- n_files = len(image_id_set) for idx, image_id in enumerate(image_id_set): sys.stdout.write("Processing %d/%d (%.2f%% done) \r" % (idx, n_files, (idx * 100.0) / n_files)) sys.stdout.flush() loc_masks_v1, pred_probs_v1 = image_id_to_info[image_id] pred_probs = prev_to_new_attr_vec(pred_probs_v1, attr_id_to_idx_v1, attr_id_to_idx) gt_labels = image_id_index[image_id]['gt_vec'] w, h = get_image_size(image_id_index[image_id]['image_path']) # loc_masks is a 68x41x41 mask. Infer and append masks for 6 new attributes loc_masks = prev_to_new_masks(loc_masks_v1, attr_id_to_idx_v1, attr_id_to_idx) for this_attr_idx, this_attr_prob in enumerate(pred_probs): this_attr_id = idx_to_attr_id[this_attr_idx] prob_score = gt_labels[this_attr_idx] if params[ 'gt_labels'] else this_attr_prob if (this_attr_id not in attr_set_use) or (prob_score < ATTR_THRESH): continue lowres_mask = loc_masks[this_attr_idx] # 41 x 41 mask # Binarize this mask lowres_bimask = (lowres_mask > CAM_THRESH * np.max(lowres_mask)).astype('uint8') lowres_prediction_list = [] if params['instances']: # Predict attribute for each contiguous blob (c) using connected components labeled_mask, nr_objects = ndimage.label(lowres_bimask > 0) for inst_id in range( 1, nr_objects): # inst_id = 0 indicates background instance_mask = (labeled_mask == inst_id).astype('uint8') lowres_prediction_list.append(instance_mask) else: lowres_prediction_list.append(lowres_bimask) del lowres_mask for lowres_inst_mask in lowres_prediction_list: # Resize mask highres_inst_mask = imresize(lowres_inst_mask, [h, w], interp='bilinear', mode='F') highres_inst_bimask = np.asfortranarray( (highres_inst_mask > (CAM_THRESH * np.max(highres_inst_mask))).astype('uint8')) rle = mask.encode(highres_inst_bimask) predictions.append({ 'image_id': image_id, 'attr_id': this_attr_id, 'segmentation': rle, 'score': prob_score, }) del highres_inst_mask del highres_inst_bimask del lowres_prediction_list # Dump predictions ------------------------------------------------------------------------------------------------- print 'Writing {} predictions to file: {}'.format(len(predictions), params['outfile']) with open(params['outfile'], 'wb') as wf: json.dump(predictions, wf, indent=2)