def prev_to_new_attr_vec(attr_vec_v1,
attr_id_to_idx_v1=None,
attr_id_to_idx_v2=None):
if attr_id_to_idx_v1 is None:
_, attr_id_to_idx_v1 = load_attributes(v1_attributes=True)
idx_to_attr_id_v1 = {v: k for k, v in attr_id_to_idx_v1.iteritems()}
if attr_id_to_idx_v2 is None:
_, attr_id_to_idx_v2 = load_attributes()
idx_to_attr_id_v2 = {v: k for k, v in attr_id_to_idx_v2.iteritems()}
n_attr_v1 = len(idx_to_attr_id_v1)
n_attr_v2 = len(idx_to_attr_id_v2)
attr_vec_v2 = np.zeros(n_attr_v2)
attr_vec_v2[:
n_attr_v1] = attr_vec_v1 # New attributes have been *appended* to the previous list
# a105_face_all = a9_face_complete + a10_face_partial
attr_vec_v2[attr_id_to_idx_v2['a105_face_all']] = max(
attr_vec_v1[attr_id_to_idx_v1['a9_face_complete']],
attr_vec_v1[attr_id_to_idx_v1['a10_face_partial']])
# a106_address_current_all = a74_address_current_complete + a75_address_current_partial
attr_vec_v2[attr_id_to_idx_v2['a106_address_current_all']] = max(
attr_vec_v1[attr_id_to_idx_v1['a74_address_current_complete']],
attr_vec_v1[attr_id_to_idx_v1['a75_address_current_partial']])
# a107_address_home_all = a78_address_home_complete + a79_address_home_partial
attr_vec_v2[attr_id_to_idx_v2['a107_address_home_all']] = max(
attr_vec_v1[attr_id_to_idx_v1['a78_address_home_complete']],
attr_vec_v1[attr_id_to_idx_v1['a79_address_home_partial']])
# a108_license_plate_all = a103_license_plate_complete + a104_license_plate_partial
attr_vec_v2[attr_id_to_idx_v2['a108_license_plate_all']] = max(
attr_vec_v1[attr_id_to_idx_v1['a103_license_plate_complete']],
attr_vec_v1[attr_id_to_idx_v1['a104_license_plate_partial']])
# a109_person_body = a1_age_approx + a2_weight_approx + a3_height_approx + a4_gender + a16_race + a17_color
attr_vec_v2[attr_id_to_idx_v2['a109_person_body']] = max(
attr_vec_v1[attr_id_to_idx_v1['a1_age_approx']],
attr_vec_v1[attr_id_to_idx_v1['a2_weight_approx']],
attr_vec_v1[attr_id_to_idx_v1['a3_height_approx']],
attr_vec_v1[attr_id_to_idx_v1['a4_gender']],
attr_vec_v1[attr_id_to_idx_v1['a16_race']],
attr_vec_v1[attr_id_to_idx_v1['a17_color']])
# a110_nudity_all = a12_semi_nudity + a13_full_nudity
attr_vec_v2[attr_id_to_idx_v2['a110_nudity_all']] = max(
attr_vec_v1[attr_id_to_idx_v1['a12_semi_nudity']],
attr_vec_v1[attr_id_to_idx_v1['a13_full_nudity']])
return attr_vec_v2
def phase4_merge_attr(fold_name, attr_ids_to_merge, new_attr_id):
# --- Setup paths --------------------------------------------------------------------------------------------------
# Filename -> Filepath
img_filename_index = get_image_filename_index()
# Load attributes
attr_id_to_name, attr_id_to_idx = load_attributes()
# Source phase4 images
anno_attr_out_dir = osp.join(SEG_ROOT, 'phase4', 'images', fold_name)
# Output directory
attr_out_dir = osp.join(anno_attr_out_dir, new_attr_id)
# --- Gather images ------------------------------------------------------------------------------------------------
full_image_set = set()
for attr_id in attr_ids_to_merge:
attr_in_dir = osp.join(anno_attr_out_dir, attr_id)
if not osp.exists(attr_in_dir):
raise ValueError('Attribute {} does not exist'.format(attr_in_dir))
for root, dirs, files in os.walk(attr_in_dir):
for image_name in files:
this_img_path = img_filename_index[image_name]
full_image_set.add(this_img_path)
# --- Get existing images ------------------------------------------------------------------------------------------
existing_image_set = set()
if not osp.exists(attr_out_dir):
# If this is the first time the script is being run
print '{} does not exist. Creating it...'.format(attr_out_dir)
os.makedirs(osp.join(attr_out_dir, '0'))
else:
# Walk and get a list of existing images
for root, dirs, files in os.walk(attr_out_dir):
for image_name in files:
this_img_path = img_filename_index[image_name]
existing_image_set.add(this_img_path)
print 'Found {} existing images'.format(len(existing_image_set))
print '# Duplicate images = {}'.format(
len(full_image_set & existing_image_set))
print '# Images to write = {}'.format(
len(full_image_set - existing_image_set))
out_image_set = full_image_set - existing_image_set
# --- Write images -------------------------------------------------------------------------------------------------
n_written = 0
for img_path in out_image_set:
# Which batch to write in?
last_batch_id = max(map(int, os.listdir(attr_out_dir)))
if len(os.listdir(osp.join(attr_out_dir,
str(last_batch_id)))) >= BATCH_SIZE:
# If full, create the next directory
last_batch_id += 1
os.mkdir(osp.join(attr_out_dir, str(last_batch_id)))
this_img_attr_out_dir = osp.join(attr_out_dir, str(last_batch_id))
_, img_filename = osp.split(img_path)
img_dst = osp.join(this_img_attr_out_dir, img_filename)
os.symlink(img_path, img_dst)
n_written += 1
print '# Written = ', n_written
def main():
parser = argparse.ArgumentParser()
parser.add_argument("infile", type=str, help="List of images for which to produce masks")
parser.add_argument("outfile", type=str, help="Path to write predictions")
args = parser.parse_args()
params = vars(args)
# --- Load some necessary helpers ----------------------------------------------------------------------------------
image_id_index = get_image_id_info_index()
attr_id_to_name, attr_id_to_idx = load_attributes()
# Load image_ids ---------------------------------------------------------------------------------------------------
image_id_set = set()
with open(params['infile']) as f:
for _line in f:
_, image_name = osp.split(_line.strip())
image_id, ext = osp.splitext(image_name)
image_id_set.add(image_id)
# Process these ids ------------------------------------------------------------------------------------------------
predictions = []
n_files = len(image_id_set)
for idx, image_id in enumerate(image_id_set):
sys.stdout.write("Processing %d/%d (%.2f%% done) \r" % (idx, n_files,
(idx * 100.0) / n_files))
sys.stdout.flush()
# Load extra-annotation for this image_id
fold = image_id_index[image_id]['fold']
image_path = image_id_index[image_id]['image_path']
extra_anno_path = osp.join(EXTRA_ANNO_PATH, fold, image_id + '.json')
image_width, image_height = get_image_size(image_path)
with open(extra_anno_path) as jf:
eanno = json.load(jf)
for face_entry in eanno.get('faceAnnotations', []):
this_poly = [bb_to_verts(face_entry['fdBoundingPoly']), ]
rles = mask.frPyObjects(this_poly, image_height, image_width)
rle = mask.merge(rles)
predictions.append({
'image_id': image_id,
'attr_id': 'a105_face_all',
'segmentation': rle,
'score': face_entry['detectionConfidence'],
})
# Dump predictions -------------------------------------------------------------------------------------------------
print 'Writing {} predictions to file: {}'.format(len(predictions), params['outfile'])
with open(params['outfile'], 'wb') as wf:
json.dump(predictions, wf, indent=2)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("gt_file", type=str, help="GT File")
parser.add_argument("pred_file", type=str, help="Predicted file")
parser.add_argument("out_dir",
type=str,
help="Output directory to place output files")
parser.add_argument("-s",
"--skip",
action='store_true',
default=False,
help="Skip images that were not predicted")
args = parser.parse_args()
params = vars(args)
# Load necessary files ---------------------------------------------------------------------------------------------
vispr = VISPRSegEval(params['gt_file'], params['pred_file'])
vispr.evaluate()
vispr.accumulate()
vispr.summarize()
image_id_index = get_image_id_info_index()
attr_id_to_name, attr_id_to_idx = load_attributes()
attr_ids = vispr.params.attrIds
image_ids = vispr.params.imgIds
n_imgs = len(image_ids)
out_dir = params['out_dir']
out_skip_dir = osp.join(out_dir, 'skipped')
if not osp.exists(out_dir):
print 'Directory {} does not exist. Creating it...'.format(out_dir)
os.mkdir(out_dir)
if not osp.exists(out_skip_dir):
print 'Directory {} does not exist. Creating it...'.format(
out_skip_dir)
os.mkdir(out_skip_dir)
# Setup colors
np.random.seed(42)
colors = [(np.random.random(size=3) * 255).astype(int) for i in range(40)]
np.random.shuffle(image_ids)
# What are the image IDs of the ones predicted?
predicted_image_ids = set()
for pd in vispr.vispr_pred:
predicted_image_ids.add(pd['image_id'])
# Visualize --------------------------------------------------------------------------------------------------------
for img_idx, image_id in enumerate(image_ids):
sys.stdout.write("Processing %d/%d (%.2f%% done) \r" %
(img_idx, n_imgs, (img_idx * 100.0) / n_imgs))
sys.stdout.flush()
if image_id in predicted_image_ids:
out_path = osp.join(out_dir, image_id + '.jpg')
else:
out_path = osp.join(out_skip_dir, image_id + '.jpg')
if params['skip']:
continue
# For how many attributes do we need to print image? ----------------------------
# Union of predicted + GT attributes in this image
this_image_attr = set()
for attr_id in attr_ids:
key = (image_id, attr_id)
if len(vispr._gts[key]) > 0 or len(vispr._pds[key]) > 0:
this_image_attr.add(attr_id)
this_image_attr = sorted(list(this_image_attr))
n_attr = len(this_image_attr)
nrows = n_attr + 1
ncols = 2
fig_width = ncols * 6
fig_height = nrows * 4
fig, axarr = plt.subplots(nrows=nrows,
ncols=ncols,
sharex=True,
sharey=True,
figsize=(fig_width, fig_height))
im = Image.open(image_id_index[image_id]['image_path'])
w, h = im.size
# Disable axis everywhere ---------------------------
for i in range(axarr.shape[0]):
for j in range(axarr.shape[1]):
axarr[i, j].axis('off')
# First row = image ---------------------------------
ax = axarr[0, 0]
ax.imshow(im)
for _idx, attr_id in enumerate(this_image_attr):
row_idx = _idx + 1
key = (image_id, attr_id)
# Plot GT --------------------------------------
col_idx = 0
ax = axarr[row_idx, col_idx]
if len(vispr._gts[key]) > 0:
ax.imshow(im, alpha=0.5)
if _idx == 0:
ax.set_title('---- GT ----\n{}'.format(
attr_id_to_name[attr_id]))
else:
ax.set_title('{}'.format(attr_id_to_name[attr_id]))
for inst_idx, det in enumerate(
sorted(vispr._gts[key], key=lambda x: -x['area'])):
rle = det['segmentation']
bimask = mask_utils.decode(rle)
inst_mask = bimask_to_rgba(bimask, colors[inst_idx])
ax.imshow(inst_mask, alpha=0.8)
del bimask
del inst_mask
# Plot Pred --------------------------------------
col_idx = 1
ax = axarr[row_idx, col_idx]
if len(vispr._pds[key]) > 0:
ax.imshow(im, alpha=0.5)
if _idx == 0:
ax.set_title('---- Pred ----\n{}'.format(
attr_id_to_name[attr_id]))
else:
ax.set_title('{}'.format(attr_id_to_name[attr_id]))
for inst_idx, det in enumerate(
sorted(vispr._pds[key], key=lambda x: -x['area'])):
rle = det['segmentation']
bimask = mask_utils.decode(rle)
# inst_mask = bimask_to_rgba(bimask, colors[inst_idx])
if inst_idx >= len(colors):
colr_idx = (len(colors) % (inst_idx + 1)) - 1
inst_mask = bimask_to_rgba(bimask, colors[colr_idx])
else:
inst_mask = bimask_to_rgba(bimask, colors[inst_idx])
ax.imshow(inst_mask, alpha=0.8)
del bimask
del inst_mask
plt.tight_layout()
plt.savefig(out_path, bbox_inches='tight')
plt.close()
def main():
parser = argparse.ArgumentParser()
parser.add_argument("infile", type=str, help="List of images for which to produce masks")
parser.add_argument("outfile", type=str, help="Path to write predictions")
parser.add_argument("regions", type=str, choices=['block', 'paragraph', 'word', 'symbol', 'all'],
help="Which element to extract?")
parser.add_argument("-g", "--gt_labels", action='store_true', default=False,
help="Create GT-label based predictions instead")
args = parser.parse_args()
params = vars(args)
# --- Load some necessary helpers ----------------------------------------------------------------------------------
image_id_index = get_image_id_info_index()
_, attr_id_to_idx_v1 = load_attributes(v1_attributes=True)
idx_to_attr_id_v1 = {v: k for k, v in attr_id_to_idx_v1.iteritems()}
_, attr_id_to_idx = load_attributes()
idx_to_attr_id = {v: k for k, v in attr_id_to_idx.iteritems()}
# Load image_ids ---------------------------------------------------------------------------------------------------
image_id_set = set()
with open(params['infile']) as f:
for _line in f:
_, image_name = osp.split(_line.strip())
image_id, ext = osp.splitext(image_name)
image_id_set.add(image_id)
# Load GT attributes too
anno = json.load(open(image_id_index[image_id]['anno_path']))
gt_vec_v1 = labels_to_vec(anno['labels'], attr_id_to_idx_v1)
gt_vec = prev_to_new_attr_vec(gt_vec_v1, attr_id_to_idx_v1, attr_id_to_idx)
image_id_index[image_id]['gt_attr_id_list'] = vec_to_labels(gt_vec, idx_to_attr_id)
# Process these ids ------------------------------------------------------------------------------------------------
region_to_predict = params['regions']
predictions = []
n_files = len(image_id_set)
for idx, image_id in enumerate(image_id_set):
sys.stdout.write("Processing %d/%d (%.2f%% done) \r" % (idx, n_files,
(idx * 100.0) / n_files))
sys.stdout.flush()
# Load extra-annotation for this image_id
fold = image_id_index[image_id]['fold']
image_path = image_id_index[image_id]['image_path']
extra_anno_path = osp.join(EXTRA_ANNO_PATH, fold, image_id + '.json')
if params['gt_labels']:
gt_attr_id_list = image_id_index[image_id]['gt_attr_id_list']
else:
gt_attr_id_list = attr_id_to_idx.keys()
image_width, image_height = get_image_size(image_path)
with open(extra_anno_path) as jf:
eanno = json.load(jf)
if 'fullTextAnnotation' not in eanno:
continue
for page in eanno['fullTextAnnotation']['pages']:
page['width'] = page['width']
page['height'] = page['height']
# ------- Block
for block in page['blocks']:
block_poly = [bb_to_verts(block['boundingBox']), ]
block_rles = mask.frPyObjects(block_poly, image_height, image_width)
block_rle = mask.merge(block_rles)
if region_to_predict in {'block', 'all'}:
predictions += get_predictions(image_id, block_rle, gt_attr_id_list=gt_attr_id_list)
if 'all' != region_to_predict:
continue
# ------- Paragraph
for paragraph in block['paragraphs']:
paragraph_poly = [bb_to_verts(paragraph['boundingBox']), ]
paragraph_rles = mask.frPyObjects(paragraph_poly, image_height, image_width)
paragraph_rle = mask.merge(paragraph_rles)
if region_to_predict in {'paragraph', 'all'}:
predictions += get_predictions(image_id, paragraph_rle, gt_attr_id_list=gt_attr_id_list)
if 'all' != region_to_predict:
continue
# ------- Word
for word in paragraph['words']:
word_poly = [bb_to_verts(word['boundingBox']), ]
word_rles = mask.frPyObjects(word_poly, image_height, image_width)
word_rle = mask.merge(word_rles)
if region_to_predict in {'word', 'all'}:
predictions += get_predictions(image_id, word_rle, gt_attr_id_list=gt_attr_id_list)
if 'all' != region_to_predict:
continue
# ------- Symbols
for symbol in word['symbols']:
symbol_poly = [bb_to_verts(symbol['boundingBox']), ]
symbol_rles = mask.frPyObjects(symbol_poly, image_height, image_width)
symbol_rle = mask.merge(symbol_rles)
if region_to_predict in {'symbol', 'all'}:
predictions += get_predictions(image_id, symbol_rle, gt_attr_id_list=gt_attr_id_list)
# Dump predictions -------------------------------------------------------------------------------------------------
print 'Writing {} predictions to file: {}'.format(len(predictions), params['outfile'])
with open(params['outfile'], 'wb') as wf:
json.dump(predictions, wf, indent=2)
def create_phase5b_fold(fold_name):
# --- Setup paths --------------------------------------------------------------------------------------------------
# Location of annotated batches
anno_batch_dir = osp.join(SEG_ROOT, 'phase5a', 'annotations', fold_name)
# Annotation list
mlc_anno_list_path = osp.join(DS_ROOT, fold_name + '.txt')
# Filename -> Filepath
img_filename_index = get_image_filename_index()
# Load attributes
attr_id_to_name, attr_id_to_idx = load_attributes()
# Where to place the phase4 images
anno_attr_out_dir = osp.join(SEG_ROOT, 'phase5b', 'images', fold_name)
# --- Create a mapping of attr_id <-> [list of images, ...] --------------------------------------------------------
attr_id_to_img = dd(list)
img_to_attr_id = dd(list)
print 'Creating attr_id->[img_path, ] mapping ...'
with open(mlc_anno_list_path) as f:
for line_idx, _line in enumerate(f):
anno_path = osp.join(DS_ROOT, _line.strip())
with open(anno_path) as jf:
anno = json.load(jf)
image_path = osp.join(DS_ROOT, anno['image_path'])
for attr_id in anno['labels']:
attr_id_to_img[attr_id].append(image_path)
img_to_attr_id[image_path].append(attr_id)
# --- Scan anno_batch_dir and detect which images are already present ----------------------------------------------
# This prevents copying duplicates
attr_id_to_existing_img = dd(set)
for attr_id in attr_id_to_idx.keys():
for root, dirs, files in os.walk(osp.join(anno_attr_out_dir, attr_id)):
for image_name in files:
this_img_path = img_filename_index[image_name]
# Sanity check. Not image should be repeated for the same attribute
assert (this_img_path not in attr_id_to_existing_img[attr_id])
attr_id_to_existing_img[attr_id].add(this_img_path)
if len(attr_id_to_existing_img[attr_id]) == 0:
attr_id_to_existing_img[attr_id] = set()
# --- Iterate through VIA annotations and place them accordingly ---------------------------------------------------
n_written = 0
n_skipped = 0
n_dupl = 0
batch_anno_filenames = os.listdir(anno_batch_dir)
batch_anno_filenames = filter(lambda x: osp.splitext(x)[0].isdigit(),
batch_anno_filenames)
for batch_fname in sorted(batch_anno_filenames,
key=lambda x: int(osp.splitext(x)[0])):
# Iterate over each batch
batch_filepath = osp.join(anno_batch_dir, batch_fname)
via_list = clean_via_annotations(batch_filepath,
img_fname_index=img_filename_index,
return_all=True)
via_fname_set = set([e['filename'] for k, e in via_list.iteritems()])
for file_id, entry in via_list.iteritems():
img_path = entry['filepath']
file_attr_dct = entry['file_attributes']
# For each attribute present in this image, symlink the image to the respective phase5a directory
img_attrs = img_to_attr_id[img_path]
for attr_id in img_attrs:
if img_path in attr_id_to_existing_img[attr_id]:
# Skip image if it's already present/annotated in phase 5a
n_dupl += 1
continue
attr_out_dir = osp.join(anno_attr_out_dir, attr_id)
if not osp.exists(attr_out_dir):
print '{} does not exist. Creating it...'.format(
attr_out_dir)
os.makedirs(attr_out_dir)
# ---- Which batch to place it in?
if len(os.listdir(attr_out_dir)) == 0:
# If this is the first time this is being run, create the first directory
os.mkdir(osp.join(attr_out_dir, '0'))
# Get the last batch_id
last_batch_id = max(map(int, os.listdir(attr_out_dir)))
if len(os.listdir(osp.join(attr_out_dir,
str(last_batch_id)))) >= BATCH_SIZE:
# If full, create the next directory
last_batch_id += 1
os.mkdir(osp.join(attr_out_dir, str(last_batch_id)))
this_img_attr_out_dir = osp.join(attr_out_dir,
str(last_batch_id))
_, img_filename = osp.split(img_path)
img_dst = osp.join(this_img_attr_out_dir, img_filename)
os.symlink(img_path, img_dst)
n_written += 1
print '# Written = ', n_written
print '# Skipped = ', n_skipped
print '# Duplicates skipped = ', n_dupl
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
"infile",
type=str,
help="Image Paths (example line: images/val2017/2017_25057208.jpg)")
parser.add_argument("outfile", type=str, help="Path to write predictions")
parser.add_argument("weights", type=str, help="Path to .caffemodel")
parser.add_argument("deploy", type=str, help="Path to deploy.prototxt")
parser.add_argument("-r",
"--DS_ROOT",
type=str,
default=DS_ROOT,
help="Override VISPR root")
parser.add_argument("-d",
"--device",
type=int,
choices=[0, 1, 2, 3],
default=0,
help="GPU device id")
parser.add_argument("-b",
"--batch_size",
type=int,
default=64,
help="Batch size")
parser.add_argument("-u",
"--use_attributes",
type=str,
default=None,
help="Use only these attributes")
args = parser.parse_args()
params = vars(args)
# Initialize Network -----------------------------------------------------------------------------------------------
caffe.set_device(params['device'])
caffe.set_mode_gpu()
model_def = params['deploy']
model_weights = params['weights']
net = caffe.Net(
model_def, # defines the structure of the model
model_weights, # contains the trained weights
caffe.TEST) # use test mode (e.g., don't perform dropout)
# Set up transformer -----------------------------------------------------------------------------------------------
# load the mean ImageNet image (as distributed with Caffe) for subtraction
mu = np.load(
osp.join(CAFFE_ROOT, 'python/caffe/imagenet/ilsvrc_2012_mean.npy'))
mu = mu.mean(1).mean(
1) # average over pixels to obtain the mean (BGR) pixel values
# mu = np.asarray([111.0, 102.0, 116.0])
# mu = np.asarray([104.0, 117.0, 123.0])
# print 'mean-subtracted values:', zip('BGR', mu)
# create transformer for the input called 'data'
transformer = caffe.io.Transformer({'data': net.blobs['data'].data.shape})
transformer.set_transpose(
'data', (2, 0, 1)) # move image channels to outermost dimension
transformer.set_mean('data',
mu) # subtract the dataset-mean value in each channel
transformer.set_raw_scale('data', 255) # rescale from [0, 1] to [0, 255]
transformer.set_channel_swap('data',
(2, 1, 0)) # swap channels from RGB to BGR
# Classify ---------------------------------------------------------------------------------------------------------
filename_to_probs = classify_paths(net,
transformer,
params['infile'],
batch_size=params['batch_size'],
dataset_root=params['DS_ROOT'])
# Convert to new attribute format ----------------------------------------------------------------------------------
# PAP was designed on 68 attributes, but now we have 74 attributes
# So, convert previous attribute vectors to new vectors
_, attr_id_to_idx_v1 = load_attributes(v1_attributes=True)
idx_to_attr_id_v1 = {v: k for k, v in attr_id_to_idx_v1.iteritems()}
_, attr_id_to_idx = load_attributes()
idx_to_attr_id = {v: k for k, v in attr_id_to_idx.iteritems()}
for filename, probs in filename_to_probs.iteritems():
if '254777' in filename:
top_10_preds = np.argsort(-probs)[:10]
for this_attr_idx in top_10_preds:
print idx_to_attr_id_v1[this_attr_idx], probs[this_attr_idx]
print
for file_id in filename_to_probs:
filename_to_probs[file_id] = prev_to_new_attr_vec(
filename_to_probs[file_id], attr_id_to_idx_v1, attr_id_to_idx)
# Predict masks from attributes ------------------------------------------------------------------------------------
# Create a mask spanning the entire image for each predicted attribute
# Required format in: privacy_filters/tools/scripts/evaluate.py
if params['use_attributes'] is None:
attr_set_use = set(attr_id_to_idx.keys())
else:
attr_set_use = set(
map(lambda x: x.strip(),
open(params['use_attributes']).readlines()))
prediction_list = []
n_files = len(filename_to_probs)
n_attr = len(attr_set_use)
fname_index = get_image_filename_index()
print 'Writing masks for {} attributes and {} files'.format(
n_attr, n_files)
for filename, probs in filename_to_probs.iteritems():
image_path = fname_index[filename]
file_id, ext = osp.splitext(filename)
w, h = get_image_size(image_path)
bimask = np.ones((h, w), order='F', dtype='uint8')
rle = mask.encode(bimask)
del bimask
if '254777' in filename:
top_10_preds = np.argsort(-probs)[:10]
for this_attr_idx in top_10_preds:
print idx_to_attr_id[this_attr_idx], probs[this_attr_idx]
for this_attr_idx, this_attr_prob in enumerate(probs):
this_attr_id = idx_to_attr_id[this_attr_idx]
if this_attr_id in attr_set_use and this_attr_prob > THRESH:
score_dct = {
'image_id': file_id,
'attr_id': this_attr_id,
'segmentation': rle,
'score': this_attr_prob,
}
prediction_list.append(score_dct)
# Write scores -----------------------------------------------------------------------------------------------------
with open(params['outfile'], 'w') as wf:
json.dump(prediction_list, wf, indent=2)
def get_phase2_stats(fold_name, csv_out_path):
# --- Setup paths --------------------------------------------------------------------------------------------------
# Location of annotated batches
anno_batch_dir = osp.join(SEG_ROOT, 'phase2', 'annotations', fold_name)
# Annotation list
mlc_anno_list_path = osp.join(DS_ROOT, fold_name + '.txt')
# Filename -> Filepath
img_filename_index = get_image_filename_index()
# Load attributes
attr_id_to_name, attr_id_to_idx = load_attributes()
# --- Create a mapping of attr_id <-> [list of images, ...] --------------------------------------------------------
attr_id_to_img = dd(list)
img_to_attr_id = dd(list)
print 'Creating attr_id->[img_path, ] mapping ...'
with open(mlc_anno_list_path) as f:
for line_idx, _line in enumerate(f):
anno_path = osp.join(DS_ROOT, _line.strip())
with open(anno_path) as jf:
anno = json.load(jf)
image_path = osp.join(DS_ROOT, anno['image_path'])
for attr_id in anno['labels']:
attr_id_to_img[attr_id].append(image_path)
img_to_attr_id[image_path].append(attr_id)
# --- Create a mapping of image_path -> {crowd_6-10, crowd_10+, person, None} --------------------------------------
# Also, perform some sanity checks
img_to_label_type = dict()
batch_anno_filenames = os.listdir(anno_batch_dir)
for batch_fname in sorted(batch_anno_filenames,
key=lambda x: int(osp.splitext(x)[0])):
# Iterate over each batch
batch_filepath = osp.join(anno_batch_dir, batch_fname)
via_list = clean_via_annotations(batch_filepath,
img_fname_index=img_filename_index,
return_all=True)
via_fname_set = set([e['filename'] for k, e in via_list.iteritems()])
for file_id, entry in via_list.iteritems():
img_path = entry['filepath']
file_attr_dct = entry['file_attributes']
if len(entry['regions']) > 0:
img_to_label_type[img_path] = 'person'
if len(file_attr_dct) > 0:
print 'Warning: {} contains regions and tags ({})'.format(
file_id, file_attr_dct)
elif len(file_attr_dct) > 0:
# Sanity check
if len(file_attr_dct) > 1:
print 'Warning: {} contains multiple file attributes ({})'.format(
file_id, file_attr_dct)
# if 'crowd_6-10' in file_attr_dct.keys() or 'crowd_10+' in file_attr_dct.keys():
img_to_label_type[img_path] = file_attr_dct.keys()[0]
else:
img_to_label_type[img_path] = 'none'
# --- Write stats --------------------------------------------------------------------------------------------------
# A) label_type -> # images
print
tot_images = len(img_to_label_type)
print '# Total images = ', tot_images
for label_type in sorted(set(img_to_label_type.values())):
this_count = img_to_label_type.values().count(label_type)
print '# {} = \t{} ({:.2f} %)'.format(
label_type, this_count, (this_count * 100.0) / tot_images)
# B) attr -> # images
attr_stats_dct = dict()
fieldnames = set()
anno_img_set = set(img_to_label_type.keys())
for attr_id, attr_img_list in attr_id_to_img.iteritems():
attr_img_set = set(attr_img_list)
this_attr_stats = dd(int)
this_attr_stats['attr_id'] = attr_id
this_attr_stats['attr_name'] = attr_id_to_name[attr_id]
this_attr_stats['n_vispr'] = len(attr_img_list)
this_attr_stats['n_common'] = len(
anno_img_set.intersection(attr_img_set))
for img in (anno_img_set.intersection(attr_img_set)):
this_attr_stats[img_to_label_type[img]] += 1
attr_stats_dct[attr_id] = this_attr_stats
for k in this_attr_stats.keys():
fieldnames.add(k)
with open(csv_out_path, 'w') as wf:
writer = csv.DictWriter(wf, fieldnames=sorted(list(fieldnames)))
attr_list = sorted(attr_stats_dct.keys(),
key=lambda x: int(x.split('_')[0][1:]))
writer.writeheader()
for k in attr_list:
writer.writerow(attr_stats_dct[k])
def main():
parser = argparse.ArgumentParser()
parser.add_argument("infile",
type=str,
help="List of images for which to produce masks")
parser.add_argument("pap_file", type=str, help="PAP file")
parser.add_argument("outfile", type=str, help="Path to write predictions")
parser.add_argument("-u",
"--use_attributes",
type=str,
default=None,
help="Use only these attributes")
parser.add_argument("-g",
"--gt_preds",
type=str,
default=None,
help="Path to write GT-based predictions")
parser.add_argument("-s",
"--saliency",
type=str,
default=None,
help="Additionally use saliency masks (.pkl file)")
args = parser.parse_args()
params = vars(args)
# Load image_ids ---------------------------------------------------------------------------------------------------
image_id_set = set()
with open(params['infile']) as f:
for _line in f:
_, image_name = osp.split(_line.strip())
image_id, ext = osp.splitext(image_name)
image_id_set.add(image_id)
filename_to_probs = {}
filename_to_gt = {}
# (Optionally) Load saliency mask ----------------------------------------------------------------------------------
image_id_to_saliency = dict()
if params['saliency'] is not None:
print 'Loading saliency map...'
image_id_to_saliency = pickle.load(open(params['saliency']))
print 'Processing masks...'
# Load PAP results -------------------------------------------------------------------------------------------------
_, attr_id_to_idx_v1 = load_attributes(v1_attributes=True)
idx_to_attr_id_v1 = {v: k for k, v in attr_id_to_idx_v1.iteritems()}
_, attr_id_to_idx = load_attributes()
idx_to_attr_id = {v: k for k, v in attr_id_to_idx.iteritems()}
with open(params['pap_file']) as f:
for line in f:
jf = json.loads(line.strip())
anno_file = jf['anno_path']
anno = json.load(open(anno_file))
image_id = anno['id']
_, filename = osp.split(anno['image_path'])
if image_id in image_id_set:
pred_probs = np.asarray(jf['pred_probs'])
gt_labels = anno['labels']
gt_vec = labels_to_vec(gt_labels, attr_id_to_idx_v1)
# Convert to new format and write
filename_to_probs[filename] = prev_to_new_attr_vec(
pred_probs, attr_id_to_idx_v1, attr_id_to_idx)
filename_to_gt[filename] = prev_to_new_attr_vec(
gt_vec, attr_id_to_idx_v1, attr_id_to_idx)
# Predict masks from attributes ------------------------------------------------------------------------------------
# Create a mask spanning the entire image for each predicted attribute
# Required format in: privacy_filters/tools/scripts/evaluate.py
if params['use_attributes'] is None:
attr_set_use = set(attr_id_to_idx.keys())
else:
attr_set_use = set(
map(lambda x: x.strip(),
open(params['use_attributes']).readlines()))
n_files = len(filename_to_probs)
n_attr = len(attr_set_use)
fname_index = get_image_filename_index()
# Write PAP Predicted mask -----------------------------------------------------------------------------------------
print 'Writing masks for {} attributes and {} files...'.format(
n_attr, n_files)
prediction_list = dct_to_mask_list(filename_to_probs, fname_index,
idx_to_attr_id, attr_set_use,
image_id_to_saliency)
with open(params['outfile'], 'w') as wf:
json.dump(prediction_list, wf, indent=2)
# Optionally, write GT Predicted mask ------------------------------------------------------------------------------
if params['gt_preds'] is not None:
print 'Writing GT masks for {} attributes and {} files...'.format(
n_attr, n_files)
prediction_list = dct_to_mask_list(filename_to_gt, fname_index,
idx_to_attr_id, attr_set_use,
image_id_to_saliency)
with open(params['gt_preds'], 'w') as wf:
json.dump(prediction_list, wf, indent=2)
def prev_to_new_masks(masks_v1,
attr_id_to_idx_v1=None,
attr_id_to_idx_v2=None):
"""
Infer and append new masks
:param masks: a 68 x X x Y matrix
:param attr_id_to_idx_v1:
:param attr_id_to_idx_v2:
:return:
"""
if attr_id_to_idx_v1 is None:
_, attr_id_to_idx_v1 = load_attributes(v1_attributes=True)
idx_to_attr_id_v1 = {v: k for k, v in attr_id_to_idx_v1.iteritems()}
if attr_id_to_idx_v2 is None:
_, attr_id_to_idx_v2 = load_attributes()
idx_to_attr_id_v2 = {v: k for k, v in attr_id_to_idx_v2.iteritems()}
n_attr_v1 = len(idx_to_attr_id_v1)
n_attr_v2 = len(idx_to_attr_id_v2)
n_new_attr = n_attr_v2 - n_attr_v1
masks_v2 = np.concatenate(
(masks_v1, np.zeros(
(n_new_attr, masks_v1.shape[1], masks_v1.shape[2]))))
# a105_face_all = a9_face_complete + a10_face_partial
masks_v2[attr_id_to_idx_v2['a105_face_all']] = np.maximum(
masks_v1[attr_id_to_idx_v1['a9_face_complete']],
masks_v1[attr_id_to_idx_v1['a10_face_partial']])
# a106_address_current_all = a74_address_current_complete + a75_address_current_partial
masks_v2[attr_id_to_idx_v2['a106_address_current_all']] = np.maximum(
masks_v1[attr_id_to_idx_v1['a74_address_current_complete']],
masks_v1[attr_id_to_idx_v1['a75_address_current_partial']])
# a107_address_home_all = a78_address_home_complete + a79_address_home_partial
masks_v2[attr_id_to_idx_v2['a107_address_home_all']] = np.maximum(
masks_v1[attr_id_to_idx_v1['a78_address_home_complete']],
masks_v1[attr_id_to_idx_v1['a79_address_home_partial']])
# a108_license_plate_all = a103_license_plate_complete + a104_license_plate_partial
masks_v2[attr_id_to_idx_v2['a108_license_plate_all']] = np.maximum(
masks_v1[attr_id_to_idx_v1['a103_license_plate_complete']],
masks_v1[attr_id_to_idx_v1['a104_license_plate_partial']])
# a109_person_body = a1_age_approx + a2_weight_approx + a3_height_approx + a4_gender + a16_race + a17_color
# np.maximum() allows comparison only between two arrays. So, iteratively cover required attributes
# and write them in-place
masks_v2[attr_id_to_idx_v2['a109_person_body']] = masks_v1[
attr_id_to_idx_v1['a1_age_approx']].copy()
for old_attr_id in [
'a2_weight_approx', 'a3_height_approx', 'a4_gender', 'a16_race',
'a17_color'
]:
np.maximum(masks_v2[attr_id_to_idx_v2['a109_person_body']],
masks_v1[attr_id_to_idx_v1[old_attr_id]],
out=masks_v2[attr_id_to_idx_v2['a109_person_body']])
# a110_nudity_all = a12_semi_nudity + a13_full_nudity
masks_v2[attr_id_to_idx_v2['a110_nudity_all']] = np.maximum(
masks_v1[attr_id_to_idx_v1['a12_semi_nudity']],
masks_v1[attr_id_to_idx_v1['a13_full_nudity']])
return masks_v2
def main():
parser = argparse.ArgumentParser()
parser.add_argument("infile",
type=str,
help="List of images for which to produce masks")
parser.add_argument(
"pkl_file",
type=str,
help="Pickled file containing {image_id -> (localization_masks, "
"attr_probs)} information")
parser.add_argument("outfile", type=str, help="Path to write predictions")
parser.add_argument("-u",
"--use_attributes",
type=str,
default=None,
help="Use only these attributes")
parser.add_argument(
"-g",
"--gt_labels",
action='store_true',
default=False,
help=
"Create GT-label based predictions instead (i.e., ignore attribute predictions from "
"CAM PAP model)")
parser.add_argument("-i",
"--instances",
action='store_true',
default=False,
help="Predict instances in each image")
args = parser.parse_args()
params = vars(args)
# image_id -> {image_path, anno_path, fold}
image_id_index = get_image_id_info_index()
_, attr_id_to_idx_v1 = load_attributes(v1_attributes=True)
idx_to_attr_id_v1 = {v: k for k, v in attr_id_to_idx_v1.iteritems()}
_, attr_id_to_idx = load_attributes()
idx_to_attr_id = {v: k for k, v in attr_id_to_idx.iteritems()}
if params['use_attributes'] is None:
attr_set_use = set(attr_id_to_idx.keys())
else:
attr_set_use = set(
map(lambda x: x.strip(),
open(params['use_attributes']).readlines()))
# Load image_ids ---------------------------------------------------------------------------------------------------
image_id_set = set()
with open(params['infile']) as f:
for _line in f:
_, image_name = osp.split(_line.strip())
image_id, ext = osp.splitext(image_name)
image_id_set.add(image_id)
# Load GT attributes too
anno = json.load(open(image_id_index[image_id]['anno_path']))
gt_vec_v1 = labels_to_vec(anno['labels'], attr_id_to_idx_v1)
gt_vec = prev_to_new_attr_vec(gt_vec_v1, attr_id_to_idx_v1,
attr_id_to_idx)
image_id_index[image_id]['gt_vec'] = gt_vec
# Load weakly supervised predictions -------------------------------------------------------------------------------
print 'Loading weakly supervised masks...'
# {image_id -> (localization_masks, attr_probs)}
# where localization_masks = 68x41x41 matrix
# attr_probs = 68 vector
image_id_to_info = pickle.load(open(params['pkl_file']))
predictions = []
print 'Creating predictions from masks...'
# Weakly supervised CAM masks -> instance predictions --------------------------------------------------------------
n_files = len(image_id_set)
for idx, image_id in enumerate(image_id_set):
sys.stdout.write("Processing %d/%d (%.2f%% done) \r" %
(idx, n_files, (idx * 100.0) / n_files))
sys.stdout.flush()
loc_masks_v1, pred_probs_v1 = image_id_to_info[image_id]
pred_probs = prev_to_new_attr_vec(pred_probs_v1, attr_id_to_idx_v1,
attr_id_to_idx)
gt_labels = image_id_index[image_id]['gt_vec']
w, h = get_image_size(image_id_index[image_id]['image_path'])
# loc_masks is a 68x41x41 mask. Infer and append masks for 6 new attributes
loc_masks = prev_to_new_masks(loc_masks_v1, attr_id_to_idx_v1,
attr_id_to_idx)
for this_attr_idx, this_attr_prob in enumerate(pred_probs):
this_attr_id = idx_to_attr_id[this_attr_idx]
prob_score = gt_labels[this_attr_idx] if params[
'gt_labels'] else this_attr_prob
if (this_attr_id
not in attr_set_use) or (prob_score < ATTR_THRESH):
continue
lowres_mask = loc_masks[this_attr_idx] # 41 x 41 mask
# Binarize this mask
lowres_bimask = (lowres_mask >
CAM_THRESH * np.max(lowres_mask)).astype('uint8')
lowres_prediction_list = []
if params['instances']:
# Predict attribute for each contiguous blob (c) using connected components
labeled_mask, nr_objects = ndimage.label(lowres_bimask > 0)
for inst_id in range(
1, nr_objects): # inst_id = 0 indicates background
instance_mask = (labeled_mask == inst_id).astype('uint8')
lowres_prediction_list.append(instance_mask)
else:
lowres_prediction_list.append(lowres_bimask)
del lowres_mask
for lowres_inst_mask in lowres_prediction_list:
# Resize mask
highres_inst_mask = imresize(lowres_inst_mask, [h, w],
interp='bilinear',
mode='F')
highres_inst_bimask = np.asfortranarray(
(highres_inst_mask >
(CAM_THRESH * np.max(highres_inst_mask))).astype('uint8'))
rle = mask.encode(highres_inst_bimask)
predictions.append({
'image_id': image_id,
'attr_id': this_attr_id,
'segmentation': rle,
'score': prob_score,
})
del highres_inst_mask
del highres_inst_bimask
del lowres_prediction_list
# Dump predictions -------------------------------------------------------------------------------------------------
print 'Writing {} predictions to file: {}'.format(len(predictions),
params['outfile'])
with open(params['outfile'], 'wb') as wf:
json.dump(predictions, wf, indent=2)
