def getFinalRank(gt):
print "Examining ground truth:\n", gt
pname = multiprocessing.current_process().name
metricTable = []
for submissionFiles in submissionFilesTable:
# Search for match to this ground truth in submissions list
subm = None
for f in submissionFiles:
q = getSubjectKey(f)
if os.path.basename(gt).find(q) >= 0:
subm = f
# Handle missing submissions
if subm is None:
print "WARNING: Cannot find any submissions for", gt, \
"in path", os.path.dirname(submissionFiles[0])
metricTable.append( [np.nan] * len(metricNames) )
continue
textout = os.path.join(tempfile.gettempdir(),
pname + "_" + os.path.basename(gt) + ".out")
tempFileSet.add(textout)
# Evaluate each metric on gt and perturbed submission
metricValues = []
for j in range(numMetricBinaries):
# Run validation app and obtain list of metrics from stdout
command = [metricBinaryList[j], gt, subm, textout]
#print "Running", command
p = subprocess.Popen(args=command, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
metricValues += getMetricValues(textout, numObjects)
if debugAddRandom:
metricValues += [random.uniform(0,100)]
metricTable.append(metricValues)
metricTable = np.array(metricTable)
rankagg = UnsupervisedLearningRankAggregator()
rankagg.set_weights(averageWeights)
return rankagg.get_aggregated_rank(metricTable, metricOrder)
