def show_relations(file_input,
output=None,
class_file='classes.cfg',
rels_file='relations.cfg',
keep_names=False):
if not output:
fname, _ = splitext(basename(file_input))
output = join(dirname(file_input), fname + '_list.txt')
# Load classes for objects from dict {0: 'rel0', 1: 'rel1'}
do = fh.ConfigFile(class_file).load_classes(cnames=True)
logger.info('Loaded dictionary with {} objects.'.format(len(do)))
dr = fh.ConfigFile(rels_file).load_classes(cnames=True)
logger.info('Loaded dictionary with {} relations.'.format(len(dr)))
# Check whether the file contains names or ids
with open(file_input) as fin:
for line in fin:
if line[0].isdigit():
break
arr = line.strip().split('-')
if len(arr) == 5:
#0-15-o1-r-o2
handler = fh.CompressedFile(file_input)
else:
#0-o1,r,o2
handler = fh.DecompressedFile(file_input)
list_rels = handler.list_relations(as_set=True)
with open(output, 'w') as fout:
for o1, r, o2 in sorted(list_rels):
fout.write('{} {} {}\n'.format(o1, r, o2))
logger.info('File saved at: %s' % output)
def compress_relations(file_input,
output=None,
file_types='types.pddl',
class_file='classes.cfg',
rels_file='relations.cfg',
keep_names=False):
if not output:
fname, _ = splitext(basename(file_input))
output = join(dirname(file_input), fname + '_predicates.txt')
# Load classes for objects from dict {0: 'rel0', 1: 'rel1'}
do = fh.ConfigFile(class_file).load_classes(cnames=True)
logger.info('Loaded dictionary with {} objects.'.format(len(do)))
dr = fh.ConfigFile(rels_file).load_classes(cnames=True)
logger.info('Loaded dictionary with {} relations.'.format(len(dr)))
dp = fh.PddlTypes(file_types)
df = fh.DecompressedFile(file_input)
rels = df.list_relations()
relations = []
for s, r, o in rels:
s = s.replace('_', '-')
o = o.replace('_', '-')
relations.append((r, dp[s], dp[o]))
relations.append((r, (s, dp[s][1]), (o, dp[o][1])))
with open(output, 'w') as fout:
fout.write('(:predicates\n')
for r, s, o in sorted(set(relations)):
# (on ?f - food ?o - object)
fout.write(' (%s ?%s - %s ?%s - %s)\n' %
(r, s[1], s[0], o[1], o[0]))
fout.write(')')
logger.info('File saved at: %s' % output)
def main(inputfile,
output=None,
class_file='classes.cfg',
rels_file='relations.cfg'):
"""
Create a `so_prior.pkl` file containing the relationship between objects.
"""
if not output:
output = join(dirname(inputfile), 'so_prior.pkl')
# Load classes for objects from dict {0: 'rel0', 1: 'rel1'}
# DO NOT LOAD `__background__`, thus id_person=0
do = fh.ConfigFile(class_file, background=False).load_classes(cnames=True)
logger.info('Loaded dictionary with {} objects.'.format(len(do)))
dr = fh.ConfigFile(rels_file).load_classes(cnames=True)
logger.info('Loaded dictionary with {} relations.'.format(len(dr)))
so_prior = np.zeros((len(do), len(do), len(dr)), dtype='float64')
objsub = np.zeros((len(do), len(do)), dtype='float64')
logger.info('Matrix of objects and relations with shape: {}'.format(
so_prior.shape))
logger.info('Matrix of only objects with shape: {}'.format(objsub.shape))
filerels = fh.DecompressedFile(inputfile)
logger.info('Loading information from file: {}'.format(inputfile))
nb_lines = filerels.nb_lines()
pb = pbar.ProgressBar(nb_lines)
logger.info('Processing {} lines...'.format(nb_lines))
with filerels as frels:
for arr in frels:
fr, o1, r, o2 = arr[0], arr[1], arr[2], arr[3]
idsub = do[o1]
idrel = dr[r]
idobj = do[o2]
so_prior[idsub][idobj][idrel] += 1
objsub[idsub][idobj] += 1
pb.update()
print
for i in range(so_prior.shape[2]):
so_prior[:, :, i] = np.divide(so_prior[:, :, i],
objsub,
out=np.zeros_like(so_prior[:, :, i]),
where=objsub != 0)
fout = open(output, 'wb')
cPickle.dump(so_prior, fout)
fout.close()
logger.info('Saved content in file: {}'.format(output))
def convert_to_lis(file_input,
output=None,
class_file='classes.cfg',
home_path=None):
if not output:
fname, _ = splitext(basename(file_input))
output = join(dirname(file_input), fname + '_lis.txt')
# Load classes for objects from dict {0: 'rel0', 1: 'rel1'}
fpred = fh.PredictionFile(file_input)
do = fh.ConfigFile(class_file).load_classes(cnames=False)
logger.info('Loaded dictionary with {} objects.'.format(len(do)))
logger.info('Converting objects to LIS annotation...')
pb = pbar.ProgressBar(fpred.nb_lines())
with open(output, 'w') as fout, fpred as fp:
fout.write('Frame:\tLabel:\tPoints:\tBounding Box ID:\tFrame path:\n')
for frame, xmin, ymin, xmax, ymax, id_class, _ in fp:
points = (xmin, ymin, xmax - xmin, ymax - ymin)
path = join(home_path, str(frame) + '.jpg')
fout.write('{}\t{}\t{}\t{}\t{}\n'.format(frame, do[id_class],
points, id_class, path))
pb.update()
print
logger.info('Saved {} objects'.format(fpred.nb_lines))
def compress_relations(file_input,
output=None,
class_file='classes.cfg',
rels_file='relations.cfg',
keep_names=False):
if not output:
fname, _ = splitext(basename(file_input))
output = join(dirname(file_input), fname + '_compressed.txt')
# Load classes for objects from dict {0: 'rel0', 1: 'rel1'}
do = fh.ConfigFile(class_file).load_classes(cnames=True)
logger.info('Loaded dictionary with {} objects.'.format(len(do)))
dr = fh.ConfigFile(rels_file).load_classes(cnames=True)
logger.info('Loaded dictionary with {} relations.'.format(len(dr)))
df = fh.DecompressedFile(file_input)
dcomp = df.group_relations()
logger.info(
'Found {} relations spread on {} lines of the input file.'.format(
len(df.start_frames), df.nb_line - 1))
logger.info('Compressed to {} lines in output file.'.format(
len(df.start_frames)))
logger.info('Saving output file...')
with open(output, 'w') as fout:
if keep_names:
fout.write('Initial_frame-Final_frame Subject Relation Object\n')
else:
fout.write('Initial_frame-Final_frame-Subject-Relation-Object\n')
for _, key in sorted(df.start_frames):
start, end = dcomp[key]['contiguous'].pop(0)
subj, rel, obj = key
if keep_names:
fout.write('%d-%d %s %s %s\n' % (start, end, subj, rel, obj))
else:
fout.write('%d-%d-%d-%d-%d\n' %
(start, end, do[subj], dr[rel], do[obj]))
logger.info('File saved at: %s' % output)
def merge_annotation(folder_input, output=None, class_file='classes.cfg', rels_file='relations.cfg'):
if not output:
output = join(folder_input, 'merged_relations.txt')
# Load classes for objects from dict {0: 'rel0', 1: 'rel1'}
do = fh.ConfigFile(class_file).load_classes(cnames=True)
logger.info('Loaded dictionary with {} objects.'.format(len(do)))
dr = fh.ConfigFile(rels_file).load_classes(cnames=True)
logger.info('Loaded dictionary with {} relations.'.format(len(dr)))
files = fh.FolderHandler(folder_input)
with open(output, 'w') as fout:
fout.write('Frame\tSubject\tRelation\tObject\tPath\n')
for path in files:
logger.info('Processing file: %s' % path)
filerels = fh.DecompressedFile(path)
with filerels as frels:
for fr, o1, r, o2 in frels:
check_error(do, o1, frels.nb_lines)
check_error(do, o2, frels.nb_lines)
check_error(dr, r, frels.nb_lines)
fout.write('%d\t%s\t%s\t%s\t%s\n' % (fr, o1, r, o2, filerels.path))
logger.info('Saved relations in file: %s' % output)
def merge_annotation(folder_input, output=None, class_file='classes.cfg'):
if not output:
output = join(folder_input, 'merged_bboxes.txt')
# Load classes for objects from dict {0: 'rel0', 1: 'rel1'}
do = fh.ConfigFile(class_file).load_classes(cnames=True)
logger.info('Loaded dictionary with {} objects.'.format(len(do)))
files = fh.FolderHandler(folder_input)
with open(output, 'w') as fout:
fout.write('Frame\tLabel\tPoints\tBounding Box ID\tFrame path:\n')
for path in files:
logger.info('Processing file: %s' % path)
filelis = fh.LisFile(path)
with filelis as flis:
for arr in flis:
check_error(do, flis.obj, flis.nb_line)
# 0 table (0,112,101,142) 29 0.jpg
fout.write('%s\t%s\t%s\t%s\t%s\n' %
(arr[0], arr[1], arr[2], arr[3],
join(flis.path, arr[4])))
logger.info('Saved bounding boxes in file: %s' % output)
def create_video_from_file(inputfile, outputfile, file_classes='classes.cfg'):
do = fh.ConfigFile(file_classes).load_classes(cnames=True)
img_array = []
if not outputfile:
fname, _ = splitext(basename(inputfile))
fnameout = join(dirname(inputfile), fname + '.avi')
fann = fh.LisFile(inputfile)
pb = pbar.ProgressBar(fann.nb_frames())
metadata = False
with fann as flis:
for fname, objs in flis.iterate_frames():
if exists(fname):
img = cv2.imread(fname)
if not metadata:
height, width, layers = img.shape
size = (width, height)
metadata = True
for label, xmin, ymin, w, h in objs:
id = do[label]
cname = colors.cnames.keys()[id]
xmax = xmin + w
ymax = ymin + h
cv2.rectangle(img, (xmin, ymin), (xmax, ymax), BBOX_COLOR,
1)
cv2.putText(img, label, (xmin - 10, ymin - 10),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, BBOX_COLOR, 1)
img_array.append(img)
else:
logger.info('{} not a file.'.format(fname))
pb.update()
out = cv2.VideoWriter(fnameout, cv2.VideoWriter_fourcc(*'DIVX'), 30, size)
for i in range(len(img_array)):
out.write(img_array[i])
out.release()
def stats_iou(lis_1, lis_2, output=None, classes='classes.cfg'):
""" Check statistics for IoU objects. """
if not output:
output = dirname(lis_1)
dclasses = fh.ConfigFile(classes).load_classes()
accumulated_iou = 0.0
nb_objects = 0
dic_obj = defaultdict(list)
all_iou = []
# create dictionary with iou for each object
list_iou = agreement_iou(lis_1, lis_2)
for _, idobj, iou in list_iou:
dic_obj[idobj].append(iou)
accumulated_iou += iou
all_iou.append(iou)
nb_objects += 1
# convert to numpy arrays
for idobj in dic_obj:
dic_obj[idobj] = np.array(dic_obj[idobj])
all_iou = np.array(all_iou)
# plot distribution for all iou
plt.hist(all_iou, bins=100)
plt.axis(xmin=0, xmax=1.0)
plt.title('Distribution of IoU for all objects')
plt.xlabel('IoU scores')
plt.ylabel('Number of instances')
plt.savefig(join(output, 'iou_all.svg'))
plt.clf()
# plot distribution for each object
for idobj in dic_obj:
iou_obj = dic_obj[idobj]
plt.hist(iou_obj, bins=100)
plt.axis(xmin=0, xmax=1.0)
plt.title('Distribution of IoU for {}'.format(dclasses[idobj]))
plt.xlabel('IoU scores')
plt.ylabel('Number of instances')
plt.savefig(join(output, 'iou_' + dclasses[idobj] + '.svg'))
plt.clf()
# save stats of IoU in a file
with open(join(output, 'stats_iou.txt'), 'w') as fout:
fout.write('Statistics for intersection over union - IoU\n')
fout.write('============================================\n')
fout.write('Input file 1: {}\n'.format(basename(lis_1)))
fout.write('Input file 2: {}\n\n'.format(basename(lis_2)))
fout.write('Statistics for Objects\n')
fout.write('----------------------\n')
fout.write('Total number of object: {}\n'.format(len(all_iou)))
for idobj in dic_obj:
fout.write('Object {}: {}\n'.format(dclasses[idobj],
len(dic_obj[idobj])))
fout.write('\n')
fout.write('General Intersection over Union (IoU)\n')
fout.write('-------------------------------------\n')
fout.write('Mean Iou: {}\n'.format(np.mean(all_iou)))
fout.write('Std Iou: {}\n'.format(np.std(all_iou)))
fout.write('\n')
agree_05 = all_iou[all_iou >= 0.5]
agree_07 = all_iou[all_iou >= 0.7]
fout.write('Correct bboxes IoU>=0.5: {}\n'.format(len(agree_05)))
fout.write('Ratio correct bboxes IoU>=0.5: {}\n'.format(
len(agree_05) / float(len(all_iou))))
fout.write('Correct bboxes IoU>=0.7: {}\n'.format(len(agree_07)))
fout.write('Ratio correct bboxes IoU>=0.7: {}\n\n'.format(
len(agree_07) / float(len(all_iou))))
fout.write('Intersection over Union (IoU) for objects\n')
fout.write('-----------------------------------------\n')
for idobj in dic_obj:
obj_iou = dic_obj[idobj]
agree_05 = obj_iou[obj_iou >= 0.5]
agree_07 = obj_iou[obj_iou >= 0.7]
fout.write('Object {}:\n'.format(dclasses[idobj]))
fout.write('- Correct bboxes IoU>=0.5: {}\n'.format(len(agree_05)))
fout.write('- Ratio correct bboxes IoU>=0.5: {}\n'.format(
len(agree_05) / float(len(obj_iou))))
fout.write('- Correct bboxes IoU>=0.7: {}\n'.format(len(agree_07)))
fout.write('- Ratio correct bboxes IoU>=0.7: {}\n'.format(
len(agree_07) / float(len(obj_iou))))
def main(fileobj,
filerel,
output=None,
class_file='classes.cfg',
rels_file='relations.cfg',
map_paths='map_paths.txt'):
"""
Create a `train.pkl` or 'test.pkl` file containing the relationship between objects.
TODO: Implement relations for two objects of the same class in the same image
"""
if not output:
output = join(dirname(fileobj), 'train.pkl')
fdicobj = join(dirname(output), 'obj.txt')
fdicrel = join(dirname(output), 'rel.txt')
if map_paths:
fmap = fh.MapFile(map_paths)
dmap = fmap.load_dictionary(key='kscgr')
logger.info('Loaded map file containing {} entries.'.format(len(dmap)))
home = fmap.path
# Load classes for objects from dict {0: 'rel0', 1: 'rel1'}
# DO NOT LOAD `__background__`. Thus, id_person=0
do = fh.ConfigFile(class_file, background=False).load_classes(cnames=True)
logger.info('Loaded dictionary with {} objects.'.format(len(do)))
dr = fh.ConfigFile(rels_file).load_classes(cnames=True)
logger.info('Loaded dictionary with {} relations.'.format(len(dr)))
dic_rels = defaultdict(list) # relations for each image
logger.info('Loading information from file: {}'.format(filerel))
filerls = fh.DecompressedFile(filerel)
pb = pbar.ProgressBar(filerls.nb_lines())
with filerls as frels:
for fr, o1, r, o2, path in frels:
idsub = do[o1]
idrel = dr[r]
idobj = do[o2]
pathimg = join(path, str(fr) + '.jpg')
if map_paths:
pathimg = dmap[join(home, pathimg)]
dic_rels[pathimg].append((idsub, idrel, idobj))
pb.update()
print
info = []
# Load objects
logger.info('Loading information from file: {}'.format(fileobj))
flis = fh.LisFile(fileobj)
nb_frames = filerls.nb_frames()
pb = pbar.ProgressBar(nb_frames)
logger.info('Processing {} frames.'.format(nb_frames))
with flis as fin:
for imgname, arr in flis.iterate_frames():
filepath = dmap[join(home, imgname)]
classes, boxes = [], []
vsub, vobj, vrel = [], [], []
dor = {}
for i in range(len(arr)):
obj, x, y, w, h = arr[i]
iobj = do[obj]
dor[iobj] = i
classes.append(iobj)
boxes.append([x, y, x + w, y + w]) # [xmin,ymin,xmax,ymax]
for idsub, idrel, idobj in dic_rels[filepath]:
vsub.append(dor[idsub])
vobj.append(dor[idobj])
vrel.append([idrel])
info.append({
'img_path': filepath,
'classes': np.array(classes),
'boxes': np.array(boxes),
'ix1': np.array(vsub),
'ix2': np.array(vobj),
'rel_classes': vrel
})
pb.update()
logger.info('Saving pickle file...')
fout = open(output, 'wb')
cPickle.dump(info, fout)
fout.close()
logger.info('Saved content in file: {}'.format(output))
save_dictionary(fdicobj, do)
save_dictionary(fdicrel, dr)
def create_gt_pickle(fileobj,
filerel,
output=None,
class_file='classes.cfg',
rels_file='relations.cfg'):
"""
Create a `gt.pkl` file containing the relationship between subjects and objects.
TODO: Implement relations for two objects of the same class in the same image
"""
if not output:
output = join(dirname(fileobj), 'gt.pkl')
dgt = {'sub_bboxes': [], 'obj_bboxes': [], 'tuple_label': []}
# Load classes for objects from dict {0: 'rel0', 1: 'rel1'}
# DO NOT LOAD `__background__`. Thus, id_person=0
do = fh.ConfigFile(class_file, background=False).load_classes(cnames=True)
logger.info('Loaded dictionary with {} objects.'.format(len(do)))
dr = fh.ConfigFile(rels_file).load_classes(cnames=True)
logger.info('Loaded dictionary with {} relations.'.format(len(dr)))
'''
dic_rels = defaultdict(list) # relations for each image
logger.info('Loading information from file: {}'.format(filerel))
filerls = fh.DecompressedFile(filerel)
pb = pbar.ProgressBar(filerls.nb_lines())
with filerls as frels:
for fr, o1, r, o2, path in frels:
idsub = do[o1]
idrel = dr[r]
idobj = do[o2]
pathimg = join(path, str(fr)+'.jpg')
dic_rels[pathimg].append((idsub, idrel, idobj))
pb.update()
'''
dic_rels = load_relations(filerel, do, dr)
print
# Load objects
logger.info('Loading information from file: {}'.format(fileobj))
flis = fh.LisFile(fileobj)
nb_frames = flis.nb_frames()
pb = pbar.ProgressBar(nb_frames)
logger.info('Processing {} frames.'.format(nb_frames))
with flis as fin:
for pathimg, arr in flis.iterate_frames():
sub_boxes, obj_boxes, vrels = [], [], []
for idsub, idrel, idobj in dic_rels[pathimg]:
relation = np.array([idsub, idrel, idobj])
for i in range(len(arr)):
obj, x, y, w, h = arr[i]
iobj = do[obj]
if iobj == idsub:
bbox_sub = np.array([x, y, x + w, y + h])
if iobj == idobj:
bbox_obj = np.array([x, y, x + w, y + h])
sub_boxes = add_element(sub_boxes, bbox_sub)
obj_boxes = add_element(obj_boxes, bbox_obj)
vrels = add_element(vrels, relation)
dgt['sub_bboxes'].append(sub_boxes)
dgt['obj_bboxes'].append(obj_boxes)
dgt['tuple_label'].append(vrels)
pb.update()
dgt['sub_bboxes'] = np.array(dgt['sub_bboxes'])
dgt['obj_bboxes'] = np.array(dgt['obj_bboxes'])
dgt['tuple_label'] = np.array(dgt['tuple_label'])
logger.info('Saving pickle file...')
fout = open(output, 'wb')
cPickle.dump(dgt, fout)
fout.close()
logger.info('Saved content in file: {}'.format(output))
