Ejemplo n.º 1
0
def evaluate( contender, runset, printVerbose=False ):
    if type(runset) is dict and not runset['runset'] is None\
        and isinstance(runset['runset'], cvac.RunSet):
        runset = runset['runset']
    if not runset or not isinstance(runset, cvac.RunSet) or not runset.purposedLists:
        raise RuntimeError("no proper runset")
    evalset = runset

    print( "---- evaluation:" )
    easy.printRunSetInfo( evalset, printArtifacts=printVerbose )
    detector = contender.getDetector()
    detections = easy.detect( detector, contender.detectorData, evalset,
                              detectorProperties=contender.detectorProps )
    ct = getConfusionTable( detections, origSet=evalset, foundMap=contender.foundMap )

    # create result structure
    r = EvaluationResult( 0, ct[0], nores=ct[1],
                          detail=None, name=contender.name )
    return r
Ejemplo n.º 2
0
'''
Easy!  mini tutorial
Create a RunSet in several ways, use it for detection and evaluation
matz 6/18/2013
'''

import easy
import cvac

# a simple RunSet with just one unlabeled image;
# remember that paths are relative to CVAC.DataDir
rs1 = easy.createRunSet( "testImg/italia.jpg" )
print("=== RunSet 1: ===");
easy.printRunSetInfo( rs1, printLabels=True )

# to give samples a purpose, state the purpose:
rs2 = easy.createRunSet( "testImg/italia.jpg", "POSITIVE" )
print("\n=== RunSet 2: ===");
easy.printRunSetInfo( rs2, printLabels=True )

# add more samples to a runset; anything starting with "pos"
# will be added into the POSITIVE sequence of labeled items
easy.addToRunSet( rs2, "testImg/TestKrFlag.jpg", "POS" )
easy.addToRunSet( rs2, "testImg/TestCaFlag.jpg", "neg" )
easy.addToRunSet( rs2, "testImg/TestUsFlag.jpg", "0" )
print("\n=== RunSet 2, after appending: ===");
easy.printRunSetInfo( rs2, printLabels=True )

# create a runset from a folder with sub-folders
rs3 = easy.createRunSet( "trainImg" )
print("\n=== RunSet 3: ===");
Ejemplo n.º 3
0
import time #for computing computation time
import os   #for chdir
stime = time.clock()

'''
Making training data
'''
#trainImg_roc_simple
#trainImg_roc
trainsetPos = easy.createRunSet( 'corporate_logos' )
trainsetNeg = easy.createRunSet( 'trainImg' )

runset = cvac.RunSet()
easy.addToRunSet(runset, trainsetPos, 'pos')
easy.addToRunSet(runset, trainsetNeg, 'neg')
easy.printRunSetInfo( runset, printLabels=True, )

strTrainer = "BOW_Trainer"
strDetector = "BOW_Detector"

list_nWord = [5,10,15,20]


doWithNegativeSample = True

if doWithNegativeSample:
    ###############################################################
    # With background data
    '''
    Execute jousting for generating ROC points
    '''
Ejemplo n.º 4
0
print("\n=== Corpus 2: ===");
print('Obtained {0} labeled artifact{1} from trainImg directory:'.format(
    len(lablist2), ("s","")[len(lablist2)==1]));
easy.printCategoryInfo( categories2 )

# Note how both corpora contain flag images, but they have different
# labels.  To use them for evaluation, let's assign the same purpose
# to syntactically different but semantically identical labels.
# Because we don't specify it, this guesses the specific Purpose that
# is assigned to the labels.
# Also obtain this mapping from Purpose to label name, called "classmap."
rs1 = easy.createRunSet( categories1['CA_flag']+categories2['ca'], "0" )
easy.addToRunSet( rs1, categories1['KO_flag']+categories2['kr'], "1" )
easy.addToRunSet( rs1, categories1['US_flag']+categories2['us'], "2" )
print("\n=== The Corpora combined into one RunSet: ===");
easy.printRunSetInfo( rs1 )

# A runset can be used for training or for testing
print("------- Bag of Words results for corporate logos: -------")
detector = easy.getDetector( "BOW_Detector" )
modelfile = "detectors/bowUSKOCA.zip"
results1 = easy.detect( detector, modelfile, rs1 )
print("Note that both original and found labels are printed:")
easy.printResults( results1 )

# Print again, this time replacing the found labels with a double
# mapping from foundLabel -> guessed Purpose -> classmap label;
# Note that this fails if multiple original labels mapped to the same
# Purpose.
wait()
print("------- Same results, but found labels replaced with guessed original labels: -------")
Ejemplo n.º 5
0
else:
    c = ev.Contender("DPM")
    c.detectorString = "DPM_Detector"
    c.detectorData = "detectors/dpmStarbucksLogo.zip"
    c.foundMap = {'Positive':easy.getPurpose('pos'), 'Negative':easy.getPurpose('neg')}
    contenders.append( c );

# OpenCVCascade, with special settings for anticipated poor performance
if (easy.getTrainer("OpenCVCascadeTrainer")==None):
    print("Cascade service(s) are insufficiently configured, skipping.")
else:
    c = ev.Contender("cascade")
    c.trainerString = "OpenCVCascadeTrainer"
    c.detectorString = "OpenCVCascadeDetector"
    # c.foundMap = {'any':easy.getPurpose('pos')}
    c.foundMap = {'positive':posPurpose, 'negative':negPurpose}
    detector = easy.getDetector(c.detectorString)
    detectorProps = easy.getDetectorProperties(detector)
    c.detectorProps = detectorProps;
    c.detectorProps.props["maxRectangles"] = "200"
    c.detectorProps.minNeighbors = 0; # This prevents hang up in evaluator when training has too few samples
    contenders.append( c );

runset = easy.createRunSet( "trainImg/kr", "pos" )
easy.addToRunSet( runset, "trainImg/ca", "neg" )
easy.printRunSetInfo( runset, printArtifacts=False, printLabels=True )

perfdata = ev.joust( contenders, runset, folds=3 )
ev.printEvaluationResults(perfdata[0])

Ejemplo n.º 6
0
def crossValidate( contender, runset, folds=10, printVerbose=False ):
    '''Returns summary statistics tp, fp, tn, fn, recall, trueNegRate,
    and a detailed matrix of results with one row for
    each fold, and one column each for true positive, false
    positive, true negative, and false negative counts'''

    # sanity checks:
    # only positive and negative purposes,
    # count number of entries for each purpose
    runset_pos = asList( runset, purpose="pos" )
    runset_neg = asList( runset, purpose="neg" )
    num_items = ( len(runset_pos), len(runset_neg) )
    # check that there are no other purposes
    all_items = len( asList( runset ) )
    if sum(num_items)!=all_items:
        raise RuntimeError("crossValidate can only handle Positive and Negative purposes")
    if min(num_items)<2:
        raise RuntimeError("need more than 1 labeled item per purpose to cross validate")

    # make sure there are enough items for xval to make sense
    if folds>min(num_items):
        print("warning: cannot do "+folds+"-fold validation with only "+str(num_items)+" labeled items")
        folds = min(num_items)

    # calculate the size of the training and evaluation sets.
    # if the number of labeled items in the runset divided
    # by the number of folds isn't an even
    # division, use more items for the evaluation
    chunksize = (int(math.floor( num_items[0]/folds )), int(math.floor( num_items[1]/folds )))
    trainsize = (chunksize[0] * (folds-1), chunksize[1] * (folds-1))
    evalsize  = (num_items[0]-trainsize[0], num_items[1]-trainsize[1])
    print( "Will perform a {0}-fold cross-validation with {1} training samples and "
           "{2} evaluation samples".format( folds, trainsize, evalsize ) )

    # randomize the order of the elements in the runset, once and for all folds
    rndidx = ( range( num_items[0] ), range( num_items[1] ) )
    random.shuffle( rndidx[0] ) # shuffles items in place
    random.shuffle( rndidx[1] ) # shuffles items in place

    #confusionTables = numpy.empty( [folds, 5], dtype=int )
    confusionTables = []
    
    for fold in range( folds ):
        # split the runset
        trainset, evalset = splitRunSet( runset_pos, runset_neg, fold, chunksize, evalsize, rndidx )
        print( "-------- fold number {0} --------".format(fold) )

        # training
        print( "---- training:" )
        easy.printRunSetInfo( trainset, printArtifacts=printVerbose )
        trainer = contender.getTrainer()
        
        model = easy.train( trainer, trainset,
                            trainerProperties=contender.trainerProps)

        # detection
        print( "---- evaluation:" )
        easy.printRunSetInfo( evalset, printArtifacts=printVerbose )
        detector = contender.getDetector()
        detections = easy.detect( detector, model, evalset,
                                  detectorProperties=contender.detectorProps )
        confusionTables.append( \
            getConfusionTable( detections, origSet=evalset, foundMap=contender.foundMap ))

    # calculate statistics of our tuple TestResult,nores
    
    sumTestResult = TestResult()
    sumNoRes = 0;
    for entry in confusionTables:
        sumTestResult.tp += entry[0].tp
        sumTestResult.tn += entry[0].tn
        sumTestResult.fp += entry[0].fp
        sumTestResult.fn += entry[0].fn
        sumNoRes += entry[1]
    r = EvaluationResult(folds, sumTestResult, sumNoRes, detail=None, name=contender.name)
    return r
Ejemplo n.º 7
0
# pick a subset: all license plates
license_plates = categories['license plate']
print("There are {0} license plate labels.".format( len(license_plates) ))

# another subset: all labels starting with "car"
cars = []
for key in categories.keys():
    if key.startswith("car"):
        cars.append( categories[key] )
print("There are {0} car-related labels.".format( len(cars) ))

# Note that Labels are cached in the CorpusServer, but the corpus currently
# needs to re-mirror if the CorpusServer is restarted because Labels are
# not stored to disk.  Images are stored to disk.

quit()  # done for now

# Train a detector on license plates
trainer = easy.getTrainer( "BOW_Trainer:default -p 10103 ")
trainset = easy.createRunSet( license_plates, "pos" )
easy.printRunSetInfo( trainset )
licenseplateModel = easy.train( trainer, trainset )

# test the license plate detector on the known locations of cars;
# this will only try to detect within the boundaries of cars.
testset = easy.createRunSet( cars, "unpurposed" )
detector = easy.getDetector( "BOW_Detector:default -p 10104" )
results = easy.detect( detector, licenseplateModel, testset )

printResults( results )
Ejemplo n.º 8
0
    c.detectorData = "detectors/dpmStarbucksLogo.zip"
    c.foundMap = {
        'Positive': easy.getPurpose('pos'),
        'Negative': easy.getPurpose('neg')
    }
    contenders.append(c)

# OpenCVCascade, with special settings for anticipated poor performance
if (easy.getTrainer("OpenCVCascadeTrainer") == None):
    print("Cascade service(s) are insufficiently configured, skipping.")
else:
    c = ev.Contender("cascade")
    c.trainerString = "OpenCVCascadeTrainer"
    c.detectorString = "OpenCVCascadeDetector"
    # c.foundMap = {'any':easy.getPurpose('pos')}
    c.foundMap = {'positive': posPurpose, 'negative': negPurpose}
    detector = easy.getDetector(c.detectorString)
    detectorProps = easy.getDetectorProperties(detector)
    c.detectorProps = detectorProps
    c.detectorProps.props["maxRectangles"] = "200"
    c.detectorProps.minNeighbors = 0
    # This prevents hang up in evaluator when training has too few samples
    contenders.append(c)

runset = easy.createRunSet("trainImg/kr", "pos")
easy.addToRunSet(runset, "trainImg/ca", "neg")
easy.printRunSetInfo(runset, printArtifacts=False, printLabels=True)

perfdata = ev.joust(contenders, runset, folds=3)
ev.printEvaluationResults(perfdata[0])
Ejemplo n.º 9
0
import os
import easy

#
# Create a training set from one sample each of 9 corporate logos
#
trainset1 = easy.createRunSet( "corporate_logos" )

# train, round 1
trainer = easy.getTrainer( "BOW_Trainer")
model1 = easy.train( trainer, trainset1 )

# evaluate the model on a separate test set, images and videos
# in DataDir/testdata1
testset1 = easy.createRunSet( "testImg", "UNPURPOSED" )
easy.printRunSetInfo( testset1 )
detector = easy.getDetector( "BOW_Detector" )
result1 = easy.detect( detector, model1, testset1 )
easy.printResults(result1)

# sort the images from the testdata1 folder into subfolders of
# "testresults1" corresponding to the found labels;
# if multiple labels were found per original, consider only
# the label with the highest confidence
easy.sortIntoFolders( result1, outfolder="testresults1", multi="highest")

# Now manually sort through the created testresults1 and move
# _incorrectly_ classified samples into correctly labeled subfolders
# of a new folder "corporate_logos_round2".  Found labels on locations
# that are not one of the 9 logos have to be sorted into a 10th class
# (subfolder), generally called the "reject" class.
Ejemplo n.º 10
0
import os
import easy

#
# Create a training set from one sample each of 9 corporate logos
#
trainset1 = easy.createRunSet("corporate_logos")

# train, round 1
trainer = easy.getTrainer("BOW_Trainer")
model1 = easy.train(trainer, trainset1)

# evaluate the model on a separate test set, images and videos
# in DataDir/testImg
testset1 = easy.createRunSet("testImg", "UNPURPOSED")
easy.printRunSetInfo(testset1)
detector = easy.getDetector("BOW_Detector")
result1 = easy.detect(detector, model1, testset1)
easy.printResults(result1)

# sort the images from the testdata1 folder into subfolders of
# "testresults1" corresponding to the found labels;
# if multiple labels were found per original, consider only
# the label with the highest confidence
easy.sortIntoFolders(result1, outfolder="testresults1", multi="highest")

# Now manually sort through the created testresults1 and move
# _incorrectly_ classified samples into correctly labeled subfolders
# of a new folder "corporate_logos_round2".  Found labels on locations
# that are not one of the 9 logos have to be sorted into a 10th class
# (subfolder), generally called the "reject" class.
Ejemplo n.º 11
0
# pick a subset: all license plates
license_plates = categories['license plate']
print("There are {0} license plate labels.".format(len(license_plates)))

# another subset: all labels starting with "car"
cars = []
for key in categories.keys():
    if key.startswith("car"):
        cars.append(categories[key])
print("There are {0} car-related labels.".format(len(cars)))

# Note that Labels are cached in the CorpusServer, but the corpus currently
# needs to re-mirror if the CorpusServer is restarted because Labels are
# not stored to disk.  Images are stored to disk.

quit()  # done for now

# Train a detector on license plates
trainer = easy.getTrainer("BOW_Trainer:default -p 10103 ")
trainset = easy.createRunSet(license_plates, "pos")
easy.printRunSetInfo(trainset)
licenseplateModel = easy.train(trainer, trainset)

# test the license plate detector on the known locations of cars;
# this will only try to detect within the boundaries of cars.
testset = easy.createRunSet(cars, "unpurposed")
detector = easy.getDetector("BOW_Detector:default -p 10104")
results = easy.detect(detector, licenseplateModel, testset)

printResults(results)
Ejemplo n.º 12
0
import time  #for computing computation time
import os  #for chdir
stime = time.clock()
'''
Making training data
'''
#trainImg_roc_simple
#trainImg_roc
trainsetPos = easy.createRunSet('corporate_logos')
trainsetNeg = easy.createRunSet('trainImg')

runset = cvac.RunSet()
easy.addToRunSet(runset, trainsetPos, 'pos')
easy.addToRunSet(runset, trainsetNeg, 'neg')
easy.printRunSetInfo(
    runset,
    printLabels=True,
)

strTrainer = "BOW_Trainer"
strDetector = "BOW_Detector"

list_nWord = [5, 10, 15, 20]

doWithNegativeSample = True

if doWithNegativeSample:
    ###############################################################
    # With background data
    '''
    Execute jousting for generating ROC points
    '''
Ejemplo n.º 13
0
print("\n=== Corpus 2: ===")
print('Obtained {0} labeled artifact{1} from trainImg directory:'.format(
    len(lablist2), ("s", "")[len(lablist2) == 1]))
easy.printCategoryInfo(categories2)

# Note how both corpora contain flag images, but they have different
# labels.  To use them for evaluation, let's assign the same purpose
# to syntactically different but semantically identical labels.
# Because we don't specify it, this guesses the specific Purpose that
# is assigned to the labels.
# Also obtain this mapping from Purpose to label name, called "classmap."
rs1 = easy.createRunSet(categories1['CA_flag'] + categories2['ca'], "0")
easy.addToRunSet(rs1, categories1['KO_flag'] + categories2['kr'], "1")
easy.addToRunSet(rs1, categories1['US_flag'] + categories2['us'], "2")
print("\n=== The Corpora combined into one RunSet: ===")
easy.printRunSetInfo(rs1)

# A runset can be used for training or for testing
print("------- Bag of Words results for corporate logos: -------")
detector = easy.getDetector("BOW_Detector")
modelfile = "detectors/bowUSKOCA.zip"
results1 = easy.detect(detector, modelfile, rs1)
print("Note that both original and found labels are printed:")
easy.printResults(results1)

# Print again, this time replacing the found labels with a double
# mapping from foundLabel -> guessed Purpose -> classmap label;
# Note that this fails if multiple original labels mapped to the same
# Purpose.
wait()
print(