matching_cutoff=matching_cutoff,
iou_cutoff=iou_cutoff,
conf_cutoff=conf_cutoff,
srr=srr,
ber=ber,
mask_others=mask_others,
PLOT=SHOW)
pred_path = write_kitti_output(preds,
str(id).zfill(4), output_directory)
print("wrote output file {}".format(id))
# get results at the end
mapping_path = "./data/tracking/evaluate_tracking.seqmap"
file_id = "evaluations_{}".format(
id) # give a unique tag to the output files
address = None
# create mail messenger and debug output object
if address:
mail = mailpy.Mail(address)
else:
mail = mailpy.Mail("")
# evaluate results and send notification email to user
success = evaluate(output_directory,
label_dir,
mapping_path,
mail,
file_id=file_id)
#########################################################################
# entry point of evaluation script
# input:
# - result_sha (unique key of results)
# - user_sha (key of user who submitted the results, optional)
# - user_sha (email of user who submitted the results, optional)
if __name__ == "__main__":
# check for correct number of arguments. if user_sha and email are not supplied,
# no notification email is sent (this option is used for auto-updates)
#if len(sys.argv)!=2 and len(sys.argv)!=4:
# print "Usage: python eval_tracking.py result_sha [user_sha email]"
# sys.exit(1);
# get unique sha key of submitted results
result_sha = "sha_key"
# create mail messenger and debug output object
if len(sys.argv) == 4:
mail = mailpy.Mail(sys.argv[3])
else:
mail = mailpy.Mail("")
# evaluate results and send notification email to user
success = evaluate(result_sha, mail)
if len(sys.argv) == 4:
mail.finalize(success, "tracking", result_sha, sys.argv[2])
else:
mail.finalize(success, "tracking", result_sha, "")
# entry point of evaluation script
# input:
# - result_sha (unique key of results)
# - user_sha (key of user who submitted the results, optional)
# - user_sha (email of user who submitted the results, optional)
if __name__ == "__main__":
# check for correct number of arguments. if user_sha and email are not supplied,
# no notification email is sent (this option is used for auto-updates)
# if len(sys.argv)!=2 and len(sys.argv)!=4:
# print("Usage: python eval_tracking.py result_sha [user_sha email]")
# sys.exit(1);
# get unique sha key of submitted results
result_sha = sys.argv[1]
split_version = sys.argv[2] if len(sys.argv) >= 3 else ''
import random
uuid = sys.argv[3] if len(sys.argv) >= 4 else str(random.randint(0, 1e63))
mail = mailpy.Mail("")
# create mail messenger and debug output object
# if len(sys.argv)==4:
# mail = mailpy.Mail(sys.argv[3])
# else:
# mail = mailpy.Mail("")
# evaluate results and send notification email to user
success = evaluate(result_sha,mail,split_version=split_version, uuid=uuid)
if len(sys.argv)==4: mail.finalize(success,"tracking",result_sha,split_version)
else: mail.finalize(success,"tracking",result_sha,"")
def run(*argv):
"""
Parameters:
argv = [signture, dir ,"3D/2D","Baseline","Your model*", subfolder]
signture:
3D/2D:
Baseline: Name of basline
must match the folder where the results are stored.
tracked obejects are not in different
subfolders
Your model/*: name of your model
must match the folder where the results are stored.
Add * at the end if tracked obejects are not in different
subfolders
subfolder: (optional)
to store in a subfoler
"""
num_sample_pts = 41.0
# check for correct number of arguments. if user_sha and email are not supplied,
# no notification email is sent (this option is used for auto-updates)
if len(argv) < 5:
print("Usage: python eval_kitti3dmot.py result_sha ?D(e.g. 2D or 3D)")
sys.exit(1)
# get unique sha key of submitted results
result_sha = argv[0]
obj_tracked = result_sha.split("_")[0]
dir = argv[1]
dt_typ = result_sha.split("_")[3]
baseline_name = argv[3]
mail = mailpy.Mail("")
D = argv[2]
#
if argv[2] == '2D':
eval_3diou, eval_2diou = False, True # eval 2d
elif argv[2] == '3D':
eval_3diou, eval_2diou = True, False # eval 3d
else:
print("Usage: python eval_kitti3dmot.py result_sha ?D(e.g. 2D or 3D)")
sys.exit(1)
# evaluate results
if len(argv) == 6:
table_name = 'results/{}/{}/results_{}_{}_table_{}.csv'.format(
dir, argv[5], obj_tracked, dt_typ, D)
else:
table_name = 'results/{}/results_{}_{}_table_{}.csv'.format(
dir, obj_tracked, dt_typ, D)
if os.path.exists(table_name):
df = pandas.read_csv(table_name)
if not (df["Model"] == baseline_name).any():
print("Evaluating baseline")
success, baseline, base_avgs = evaluate(result_sha, dir,
baseline_name, mail,
eval_3diou, eval_2diou)
df.loc[len(df.index)] = [
baseline_name, baseline.sMOTA, baseline.MOTA, baseline.MOTP,
baseline.MT, baseline.ML, baseline.id_switches,
baseline.fragments, baseline.F1, baseline.precision,
baseline.recall, baseline.FAR, baseline.tp, baseline.fp,
baseline.fn, base_avgs[0], base_avgs[1], base_avgs[2]
]
else:
print("Evaluating baseline :")
success, baseline, base_avgs = evaluate(result_sha, dir, baseline_name,
mail, eval_3diou, eval_2diou)
# basline_data = [[baseline_name],[baseline.sMOTA], [baseline.MOTA], [baseline.MOTP], [baseline.MT], [baseline.ML], [baseline.id_switches], [baseline.fragments],
# [baseline.F1], [baseline.precision], [baseline.recall], [baseline.FAR], [baseline.tp], [baseline.fp], [baseline.fn],
# [base_avgs[0]], [base_avgs[1]], [base_avgs[2]]]
cols = [
"Model", "sMOTA", "MOTA", "MOTP", "MT", "ML", "IDS", "FRAG", "F1",
"Prec", "Recall", "FAR", "TP", "FP", "FN", "sAMOTA", "AMOTA",
"AMOTP"
]
df = pandas.DataFrame(columns=cols)
df.loc[len(df.index)] = [
baseline_name, baseline.sMOTA, baseline.MOTA, baseline.MOTP,
baseline.MT, baseline.ML, baseline.id_switches, baseline.fragments,
baseline.F1, baseline.precision, baseline.recall, baseline.FAR,
baseline.tp, baseline.fp, baseline.fn, base_avgs[0], base_avgs[1],
base_avgs[2]
]
other_name = argv[4]
mail = mailpy.Mail("")
print("Evaluating " + other_name[:-1] + " :")
success, other_model, om_avgs = evaluate(result_sha, dir, other_name, mail,
eval_3diou, eval_2diou)
new_row = [other_name[:-1],other_model.sMOTA, other_model.MOTA, other_model.MOTP, other_model.MT, other_model.ML, other_model.id_switches, other_model.fragments, \
other_model.F1, other_model.precision, other_model.recall, other_model.FAR, other_model.tp, other_model.fp, other_model.fn,\
om_avgs[0], om_avgs[1], om_avgs[2]]
df.loc[len(df.index)] = new_row
print(df.loc[(df['Model'] == baseline_name) |
(df['Model'] == other_name[:-1])])
string_format = df.to_latex(index=False)
if len(argv) == 6:
table_name = '.results/{}/{}/comparison_{}_{}_latex_{}.txt'.format(
dir, argv[5], obj_tracked, dt_typ, D)
else:
table_name = 'results/{}/comparison_{}_{}_latex_{}.txt'.format(
dir, obj_tracked, dt_typ, D)
print(string_format, file=open(table_name, 'w'))
if len(argv) == 6:
table_name = 'results/{}/{}/results_{}_{}_table_{}.csv'.format(
dir, argv[5], obj_tracked, dt_typ, D)
else:
table_name = 'results/{}/results_{}_{}_table_{}.csv'.format(
dir, obj_tracked, dt_typ, D)
file_ = open(table_name, "w")
df.to_csv(file_, index=False, header=True)
return other_model.MOTA, other_model.MOTP
def get_meas_target_set(score_intervals, det_method="lsvm", obj_class="car"):
"""
Input:
- doctor_clutter_probs: if True, replace 0 probabilities with .0000001/float(20+num_zero_probs) and extend
clutter probability list with 20 values of .0000001/20 and subtract .0000001 from element 0
- doctor_birth_probs: if True then if any birth probability is 0 subtract .0000001 from element 0
of its score interval's birth probability list and replacing zero elements with .0000001/(number of
zero elements in the score interval's birth probability list)
"""
# if USE_PICKLED_DATA:
# if not os.path.exists(PICKELD_DATA_DIRECTORY):
# os.makedirs(PICKELD_DATA_DIRECTORY)
#
# data_filename = PICKELD_DATA_DIRECTORY + "/meas_targ_set_scores_%s_det_method_%s_obj_class_%s_include_ignored_gt_%s_include_dontcare_gt_%s_include_ignored_det_%s.pickle" % \
# (str(score_intervals), det_method, obj_class, include_ignored_gt, include_dontcare_in_gt, include_ignored_detections)
# lock_filename = data_filename + "_lock"
#
# if os.path.isfile(data_filename) and (not os.path.isfile(lock_filename)):
# f = open(data_filename, 'r')
# (measurementTargetSetsBySequence, target_emission_probs, clutter_probabilities, birth_probabilities, meas_noise_covs) = pickle.load(f)
# f.close()
# return (measurementTargetSetsBySequence, target_emission_probs, clutter_probabilities, birth_probabilities, meas_noise_covs)
mail = mailpy.Mail("")
print score_intervals
(det_objects) = get_det_objs1(score_intervals[0],
det_method,
mail,
obj_class="car")
measurementTargetSetsBySequence = []
for seq_idx in range(len(det_objects)):
cur_seq_meas_target_set = TargetSet()
for frame_idx in range(len(det_objects[seq_idx])):
cur_frame_measurements = Measurement()
cur_frame_measurements.time = frame_idx * .1 #frames are .1 seconds apart
cur_fram_meas_unsorted = []
for meas_idx in range(len(det_objects[seq_idx][frame_idx])):
cur_meas = det_objects[seq_idx][frame_idx][meas_idx]
meas_pos = np.array([cur_meas.x, cur_meas.y])
meas_width = cur_meas.x2 - cur_meas.x1
meas_height = cur_meas.y2 - cur_meas.y1
# cur_frame_measurements.val.append(meas_pos)
# cur_frame_measurements.widths.append(meas_width)
# cur_frame_measurements.heights.append(meas_height)
# cur_frame_measurements.scores.append(cur_meas.score)
cur_fram_meas_unsorted.append(
(cur_meas.score, (meas_pos, meas_width, meas_height)))
cur_frame_meas_sorted = sorted(cur_fram_meas_unsorted,
key=lambda tup: tup[0],
reverse=True)
for i in range(len(cur_frame_meas_sorted)):
cur_frame_measurements.val.append(
cur_frame_meas_sorted[i][1][0])
cur_frame_measurements.widths.append(
cur_frame_meas_sorted[i][1][1])
cur_frame_measurements.heights.append(
cur_frame_meas_sorted[i][1][2])
cur_frame_measurements.scores.append(
cur_frame_meas_sorted[i][0])
cur_seq_meas_target_set.measurements.append(cur_frame_measurements)
measurementTargetSetsBySequence.append(cur_seq_meas_target_set)
return (measurementTargetSetsBySequence)
def run(*argv):
"""
Parameters:
argv = [signture, dir ,"3D/2D","Baseline","Your model*", subfolder, sequence]
signture:
3D/2D:
df: pandas table
into which results are going to be inserted.
Your model/*: name of your model
must match the folder where the results are stored.
Add * at the end if tracked obejects are not in different
subfolders
subfolder: (optional)
to store in a subfoler
"""
num_sample_pts = 41.0
# check for correct number of arguments. if user_sha and email are not supplied,
# no notification email is sent (this option is used for auto-updates)
if len(argv) < 5:
print("Usage: python eval_kitti3dmot.py result_sha ?D(e.g. 2D or 3D)")
sys.exit(1)
# get unique sha key of submitted results
result_sha = argv[0]
obj_tracked = result_sha.split("_")[0]
dir = argv[1]
dt_typ = result_sha.split("_")[3]
df = argv[3]
mail = mailpy.Mail("")
D = argv[2]
#
if argv[2] == '2D':
eval_3diou, eval_2diou = False, True # eval 2d
elif argv[2] == '3D':
eval_3diou, eval_2diou = True, False # eval 3d
else:
print("Usage: python eval_kitti3dmot.py result_sha ?D(e.g. 2D or 3D)")
sys.exit(1)
# evaluate results
other_name = argv[4]
mail = mailpy.Mail("")
print("Evaluating " + dir + " :")
if len(argv) > 5:
last_arg = argv[-1]
else:
last_arg = None
success, other_model, om_avgs = evaluate(result_sha, dir, other_name, mail,
eval_3diou, eval_2diou, last_arg)
new_row = [dir,other_model.sMOTA, other_model.MOTA, other_model.MOTP, other_model.MT, other_model.ML, other_model.id_switches, other_model.fragments, \
other_model.F1, other_model.precision, other_model.recall, other_model.FAR, other_model.tp, other_model.fp, other_model.fn,\
om_avgs[0], om_avgs[1], om_avgs[2]]
df.loc[len(df.index)] = new_row
return other_model.MOTA, other_model.MOTP, df
