def validateCorrectness( self, outfile ):
correctness = ParallelTest.validateCorrectness( self, outfile )
with Multitmp( len( outfile ) ) as tmp:
Multitmp.check_call(
[ "sed", "-e", "/^#im_vips2ppm/d", outfile ],
stdout = tmp
)
Multitmp.check_call( [ "mv", tmp, outfile ] )
if self.options.error:
with Multitmp( len( outfile ) ) as result:
with open( "/dev/null", 'w' ) as null:
golden = self.getGolden()
diffimg = os.path.join( root, "bin", "diff-img")
Multitmp.check_call(
[ diffimg,
golden,
outfile
],
stdout = result, stderr = null,
verbose = self.options.verbose,
)
errors = list()
for fname in result:
with open( fname ) as fh:
error = 0
for line in fh:
if line.startswith( "total" ):
error += float( line.split()[ 2 ] )
errors.append( 1 / ( error + 1 ) )
self.error = errors
return True
else:
return correctness
def validateCorrectness( self, outfile ):
correctness = ParallelTest.validateCorrectness( self, outfile )
if self.options.error:
with Multitmp( len( outfile ) ) as outdir:
Multitmp.check_call( [ "rm", "-rf", outdir ], verbose = self.options.verbose )
Multitmp.check_call( [ "mkdir", outdir ], verbose = self.options.verbose )
Multitmp.check_call(
[ "avconv",
"-i", outfile,
"-r", "25",
"-loglevel", "panic",
"%03d.png"
],
verbose = self.options.verbose,
cwd = outdir
)
if not ParallelTest.validateCorrectness( self, outdir ):
return False
with Multitmp( len( outfile ) ) as result:
with open( "/dev/null", 'w' ) as null:
golden = self.getGolden()
diffimg = os.path.join( root, "bin", "diff-img.sh")
Multitmp.check_call(
[ diffimg,
golden,
outdir
],
stdout = result, stderr = null,
verbose = self.options.verbose,
)
errors = list()
for fname in result:
with open( fname ) as fh:
error = 0
for line in fh:
if line.startswith( "total" ):
error += float( line.split()[ 2 ] )
errors.append( 1 / ( error + 1 ) )
self.error = errors
return True
else:
return correctness
def validateCorrectness( self, outfile ):
if self.options.error:
with Multitmp( len( outfile ) ) as scratch:
Multitmp.check_call(
[ "mv", outfile, scratch ], verbose = self.options.verbose
)
Multitmp.check_call(
[ "mkdir", outfile ], verbose = self.options.verbose
)
Multitmp.check_call(
[ "avconv",
"-i", scratch,
"-r", "25",
"-loglevel", "panic",
"%03d.png"
],
verbose = self.options.verbose,
cwd = outfile
)
return ParallelTest.validateCorrectness( self, outfile )
else:
return ParallelTest.validateCorrectness( self, outfile )
def validateCorrectness( self, outfile ):
correctness = ParallelTest.validateCorrectness( self, outfile )
if not correctness:
return False
if self.options.error:
golden = self.getGolden()
gold_queries = self.readFile( golden )
errors = list()
# Error function for missing/extra things. Max good is 0, max bad is 1
def errorFun( missing, extra ):
return 1 - ( 1 / ( 2 + ( 2 * missing ) ) ) - ( 1 / ( 2 + ( 2 * extra ) ) )
for fname in outfile:
# No output file is max error
if not os.path.isfile(fname):
return False
test_queries = dict()
test_queries = self.readFile( fname )
t1 = 0 # Kendall tau penalty for weights
t2 = 0 # Kendall tau penalty for rank output order
t3 = 0 # Kendall tau penalty for query output order
w = 0 # Weighting error penalty
r = 0 # Penalty for missing or extra ranks
extra_queries, test_queries_int, gold_queries_int, missing_queries = \
self.comm( test_queries, gold_queries )
# Penalty for missing or extra queries
q = errorFun( len( missing_queries ), len( extra_queries ) )
if len( gold_queries_int ) < 2:
t3 = 1
else:
t3, _ = stats.kendalltau( [ x for x, _ in test_queries_int ],
[ x for x, _ in gold_queries_int ] )
t3 = (.5) - (.5 * t3) # Change tau scale to [0-1] where 0 is good
if isclose( t3, 0 ):
t3 = 0.0
for test_query, gold_query in zip( test_queries_int, gold_queries_int ):
extra_ranks, test_ranks_int, gold_ranks_int, missing_ranks = \
self.comm( test_query[1], gold_query[1] )
r += errorFun( len( missing_ranks ), len( extra_ranks ) )
if len( gold_ranks_int ) < 2:
tau1 = 1
else:
tau1, _ = stats.kendalltau( [x for _, x in gold_ranks_int],
[x for _, x in test_ranks_int] )
tau1 = (.5) - (.5 * tau1) # Change tau scale to [0-1] where 0 is good
if isclose( tau1, 0 ):
tau1 = 0.0
absolute_error = sum( [ abs( a[1] - b[1] ) for a, b in
zip( gold_ranks_int, test_ranks_int ) ] )
if isnan( absolute_error ) or isinf( absolute_error ):
w += 1
else:
w += absolute_error / ( absolute_error + 1 )
t1 += tau1
if len( gold_ranks_int ) < 2:
tau2 = 1
else:
tau2, _ = stats.kendalltau( [x for x, _ in gold_ranks_int],
[x for x, _ in test_ranks_int] )
tau2 = (.5) - (.5 * tau2) # Change tau scale to [0-1] where 0 is good
if isclose( tau2, 0 ):
tau2 = 0.0
t2 += tau2
t1 = t1 / len( gold_queries_int )
t2 = t2 / len( gold_queries_int )
w = w / len( gold_queries_int )
r = r / len( gold_queries_int )
error = 1000 * q + 100 * r + 10 * t1 + 10 * t2 + 5 * t3 + w
errors.append( 1 / ( error + 1 ) )
self.error = errors
return True
else:
return correctness
def validateCorrectness(self, outfile):
Multitmp.check_call(["rm", "-rf", "Face_Data", "Storytelling/output/log.txt"], cwd=outfile)
return ParallelTest.validateCorrectness(self, outfile)
