def validateCorrectness( self, outfile ): correctness = ParallelTest.validateCorrectness( self, outfile ) with Multitmp( len( outfile ) ) as tmp: Multitmp.check_call( [ "sed", "-e", "/^#im_vips2ppm/d", outfile ], stdout = tmp ) Multitmp.check_call( [ "mv", tmp, outfile ] ) if self.options.error: with Multitmp( len( outfile ) ) as result: with open( "/dev/null", 'w' ) as null: golden = self.getGolden() diffimg = os.path.join( root, "bin", "diff-img") Multitmp.check_call( [ diffimg, golden, outfile ], stdout = result, stderr = null, verbose = self.options.verbose, ) errors = list() for fname in result: with open( fname ) as fh: error = 0 for line in fh: if line.startswith( "total" ): error += float( line.split()[ 2 ] ) errors.append( 1 / ( error + 1 ) ) self.error = errors return True else: return correctness
def validateCorrectness( self, outfile ): correctness = ParallelTest.validateCorrectness( self, outfile ) if self.options.error: with Multitmp( len( outfile ) ) as outdir: Multitmp.check_call( [ "rm", "-rf", outdir ], verbose = self.options.verbose ) Multitmp.check_call( [ "mkdir", outdir ], verbose = self.options.verbose ) Multitmp.check_call( [ "avconv", "-i", outfile, "-r", "25", "-loglevel", "panic", "%03d.png" ], verbose = self.options.verbose, cwd = outdir ) if not ParallelTest.validateCorrectness( self, outdir ): return False with Multitmp( len( outfile ) ) as result: with open( "/dev/null", 'w' ) as null: golden = self.getGolden() diffimg = os.path.join( root, "bin", "diff-img.sh") Multitmp.check_call( [ diffimg, golden, outdir ], stdout = result, stderr = null, verbose = self.options.verbose, ) errors = list() for fname in result: with open( fname ) as fh: error = 0 for line in fh: if line.startswith( "total" ): error += float( line.split()[ 2 ] ) errors.append( 1 / ( error + 1 ) ) self.error = errors return True else: return correctness
def validateCorrectness( self, outfile ): if self.options.error: with Multitmp( len( outfile ) ) as scratch: Multitmp.check_call( [ "mv", outfile, scratch ], verbose = self.options.verbose ) Multitmp.check_call( [ "mkdir", outfile ], verbose = self.options.verbose ) Multitmp.check_call( [ "avconv", "-i", scratch, "-r", "25", "-loglevel", "panic", "%03d.png" ], verbose = self.options.verbose, cwd = outfile ) return ParallelTest.validateCorrectness( self, outfile ) else: return ParallelTest.validateCorrectness( self, outfile )
def validateCorrectness( self, outfile ): correctness = ParallelTest.validateCorrectness( self, outfile ) if not correctness: return False if self.options.error: golden = self.getGolden() gold_queries = self.readFile( golden ) errors = list() # Error function for missing/extra things. Max good is 0, max bad is 1 def errorFun( missing, extra ): return 1 - ( 1 / ( 2 + ( 2 * missing ) ) ) - ( 1 / ( 2 + ( 2 * extra ) ) ) for fname in outfile: # No output file is max error if not os.path.isfile(fname): return False test_queries = dict() test_queries = self.readFile( fname ) t1 = 0 # Kendall tau penalty for weights t2 = 0 # Kendall tau penalty for rank output order t3 = 0 # Kendall tau penalty for query output order w = 0 # Weighting error penalty r = 0 # Penalty for missing or extra ranks extra_queries, test_queries_int, gold_queries_int, missing_queries = \ self.comm( test_queries, gold_queries ) # Penalty for missing or extra queries q = errorFun( len( missing_queries ), len( extra_queries ) ) if len( gold_queries_int ) < 2: t3 = 1 else: t3, _ = stats.kendalltau( [ x for x, _ in test_queries_int ], [ x for x, _ in gold_queries_int ] ) t3 = (.5) - (.5 * t3) # Change tau scale to [0-1] where 0 is good if isclose( t3, 0 ): t3 = 0.0 for test_query, gold_query in zip( test_queries_int, gold_queries_int ): extra_ranks, test_ranks_int, gold_ranks_int, missing_ranks = \ self.comm( test_query[1], gold_query[1] ) r += errorFun( len( missing_ranks ), len( extra_ranks ) ) if len( gold_ranks_int ) < 2: tau1 = 1 else: tau1, _ = stats.kendalltau( [x for _, x in gold_ranks_int], [x for _, x in test_ranks_int] ) tau1 = (.5) - (.5 * tau1) # Change tau scale to [0-1] where 0 is good if isclose( tau1, 0 ): tau1 = 0.0 absolute_error = sum( [ abs( a[1] - b[1] ) for a, b in zip( gold_ranks_int, test_ranks_int ) ] ) if isnan( absolute_error ) or isinf( absolute_error ): w += 1 else: w += absolute_error / ( absolute_error + 1 ) t1 += tau1 if len( gold_ranks_int ) < 2: tau2 = 1 else: tau2, _ = stats.kendalltau( [x for x, _ in gold_ranks_int], [x for x, _ in test_ranks_int] ) tau2 = (.5) - (.5 * tau2) # Change tau scale to [0-1] where 0 is good if isclose( tau2, 0 ): tau2 = 0.0 t2 += tau2 t1 = t1 / len( gold_queries_int ) t2 = t2 / len( gold_queries_int ) w = w / len( gold_queries_int ) r = r / len( gold_queries_int ) error = 1000 * q + 100 * r + 10 * t1 + 10 * t2 + 5 * t3 + w errors.append( 1 / ( error + 1 ) ) self.error = errors return True else: return correctness
def validateCorrectness(self, outfile): Multitmp.check_call(["rm", "-rf", "Face_Data", "Storytelling/output/log.txt"], cwd=outfile) return ParallelTest.validateCorrectness(self, outfile)