def splitRunSet( runset_pos, runset_neg, fold, chunksize, evalsize, rndidx ):
'''Take parts of runset_pos and runset_neg and re-combine into
a training set and an evaluation set. For use by crossValidate().
'''
num_items = ( len(runset_pos), len(runset_neg) )
evalidx_pos = range( fold*chunksize[0], fold*chunksize[0]+evalsize[0] )
evalidx_neg = range( fold*chunksize[1], fold*chunksize[1]+evalsize[1] )
trainidx_pos = range( 0, fold*chunksize[0] ) + range( fold*chunksize[0]+evalsize[0], num_items[0] )
trainidx_neg = range( 0, fold*chunksize[1] ) + range( fold*chunksize[1]+evalsize[1], num_items[1] )
# The following line selects those elements from runset_pos
# that correspond to the randomized indices for the current
# evaluation chunk. Think of this, conceptually:
# evalset_pos = runset_pos[ rndidx[0][evalidx_pos] ]
# Subsequent lines: equivalently, for runset_neg, and trainset pos/neg
evalset_pos = list( runset_pos[i] for i in list( rndidx[0][j] for j in evalidx_pos) )
evalset_neg = list( runset_neg[i] for i in list( rndidx[1][j] for j in evalidx_neg) )
trainset_pos = list( runset_pos[i] for i in list( rndidx[0][j] for j in trainidx_pos) )
trainset_neg = list( runset_neg[i] for i in list( rndidx[1][j] for j in trainidx_neg) )
# create a RunSet with proper purposes
trainset = cvac.RunSet()
trainset.purposedLists = (cvac.PurposedLabelableSeq(easy.getPurpose("pos"), trainset_pos),
cvac.PurposedLabelableSeq(easy.getPurpose("neg"), trainset_neg))
evalset = cvac.RunSet()
evalset.purposedLists = (cvac.PurposedLabelableSeq(easy.getPurpose("pos"), evalset_pos),
cvac.PurposedLabelableSeq(easy.getPurpose("neg"), evalset_neg))
return trainset, evalset
def test_getLabelableList(self):
print("testing getLabelableList")
sys.stdout.flush()
labelList = easy.getLabelableList("easyTestData")
'''verify that the labels are correct
They should be , the root directory "easyTestData",
"kitchen" and "MITmountain"
'''
for entry in labelList:
print("label name " + entry.lab.name)
if entry.lab.name != "easyTestData" and \
entry.lab.name != "kitchen" and \
entry.lab.name != "MITmountain":
raise RuntimeError(
"Not a valid label name in test getLabelableList")
if entry.lab.name == "kitchen":
# should find easyTestData, inside, house properties
if "easyTestData" not in entry.lab.properties:
raise RuntimeError("easyTestData not in properties")
if "inside" not in entry.lab.properties:
raise RuntimeError("inside not in properties")
if "house" not in entry.lab.properties:
raise RuntimeError("house not in properties")
''' verify that we can make a runset and use it with the labelableList
'''
runset = cvac.RunSet()
posPurpose = easy.getPurpose('pos')
easy.addToRunSet(runset, labelList, posPurpose)
if not easy.isProperRunSet(runset):
raise RuntimeError(
"test getLabelableList failed with an invalid runset")
labelList = easy.getLabelableList("easyTestData", recursive=False)
''' should only have one label "easyTestData"
'''
for entry in labelList:
if entry.lab.name != "easyTestData":
raise RuntimeError(
"recursive option failed in test getLabelableList")
if len(entry.lab.properties) > 0:
raise RuntimeError("labelable should not have any properties")
if not easy.isProperRunSet(runset):
raise RuntimeError(
"test getLabelableList failed with an invalid runset with non-recursive call"
)
def test_getLabelableList(self):
print("testing getLabelableList")
sys.stdout.flush()
labelList = easy.getLabelableList("easyTestData")
'''verify that the labels are correct
They should be , the root directory "easyTestData",
"kitchen" and "MITmountain"
'''
for entry in labelList:
print ("label name " + entry.lab.name)
if entry.lab.name != "easyTestData" and \
entry.lab.name != "kitchen" and \
entry.lab.name != "MITmountain":
raise RuntimeError("Not a valid label name in test getLabelableList")
if entry.lab.name == "kitchen":
# should find easyTestData, inside, house properties
if "easyTestData" not in entry.lab.properties:
raise RuntimeError("easyTestData not in properties")
if "inside" not in entry.lab.properties:
raise RuntimeError("inside not in properties")
if "house" not in entry.lab.properties:
raise RuntimeError("house not in properties")
''' verify that we can make a runset and use it with the labelableList
'''
runset = cvac.RunSet()
posPurpose = easy.getPurpose('pos')
easy.addToRunSet(runset, labelList, posPurpose)
if not easy.isProperRunSet(runset):
raise RuntimeError("test getLabelableList failed with an invalid runset")
labelList = easy.getLabelableList("easyTestData", recursive=False)
''' should only have one label "easyTestData"
'''
for entry in labelList:
if entry.lab.name != "easyTestData":
raise RuntimeError("recursive option failed in test getLabelableList")
if len(entry.lab.properties) > 0:
raise RuntimeError("labelable should not have any properties")
if not easy.isProperRunSet(runset):
raise RuntimeError("test getLabelableList failed with an invalid runset with non-recursive call")
def asList( runset, purpose=None ):
'''You can pass in an actual cvac.RunSet or a dictionary with
the runset and a classmap, as returned by createRunSet.'''
if type(runset) is dict and not runset['runset'] is None\
and isinstance(runset['runset'], cvac.RunSet):
runset = runset['runset']
if not runset or not isinstance(runset, cvac.RunSet) or not runset.purposedLists:
raise RuntimeError("no proper runset")
if isinstance(purpose, str):
purpose = easy.getPurpose( purpose )
rsList = []
for plist in runset.purposedLists:
if purpose and not plist.pur==purpose:
# not interested in this purpose
continue
if isinstance(plist, cvac.PurposedDirectory):
print("warning: runset contains directory; will treat as one for folds")
elif isinstance(plist, cvac.PurposedLabelableSeq):
rsList = rsList + plist.labeledArtifacts
else:
raise RuntimeError("unexpected plist type "+type(plist))
return rsList
if doWithNegativeSample:
###############################################################
# With background data
'''
Execute jousting for generating ROC points
'''
contenders = []
for nWord in list_nWord:
c1 = evaluate.Contender("bowROC_binary_"+str(nWord))
c1.trainerString = strTrainer
c1.detectorString = strDetector
trainer = easy.getTrainer(c1.trainerString)
trainerProps = easy.getTrainerProperties(trainer)
trainerProps.props["NumWords"] = str(nWord)
c1.trainerProps = trainerProps
c1.foundMap = {'1':easy.getPurpose('pos'), '0':easy.getPurpose('neg')}
contenders.append(c1)
sortedperfdata, perfdata = evaluate.joust( contenders, runset, folds=3 )
easy.showROCPlot(perfdata)
'''
Extract only optimal ROC points
'''
rocData_full,optimalIndices = easy.discardSuboptimal(perfdata)
#rocData_full,optimalIndices = easy.discardSuboptimal(perfdata,"sboxes/BOW_Trainer_127_0_0_1")
'''
Prepare detector data of optimal ROC points
'''
rocData_optimal = []
'''
Easy! mini tutorial
Compare the performance of several trainers and/or detectors
matz 11/26/2013
'''
import easy
import easy.evaluate as ev
# perform a comparative detector evaluation including
# training of a model
# easy.CVAC_ClientVerbosity = 4
posPurpose = easy.getPurpose('pos')
negPurpose = easy.getPurpose('neg')
contenders = []
# Bag of Words
if (easy.getTrainer("BOW_Trainer")==None):
print("BOW service(s) are insufficiently configured, skipping.")
else:
c = ev.Contender("BOW")
c.trainerString = "BOW_Trainer"
c.detectorString = "BOW_Detector"
c.foundMap = {'1':posPurpose, '0':negPurpose}
contenders.append( c );
# Histogram of Oriented Gradients
if (easy.getTrainer("HOG_Trainer")==None):
print("HOG service(s) are insufficiently configured, skipping.")
else:
c = ev.Contender("HOG")
c.trainerString = "HOG_Trainer"
'''
Easy! mini tutorial
Compare the performance of several trainers and/or detectors
matz 11/26/2013
'''
import easy
import easy.evaluate as ev
# perform a comparative detector evaluation including
# training of a model
# easy.CVAC_ClientVerbosity = 4
posPurpose = easy.getPurpose('pos')
negPurpose = easy.getPurpose('neg')
contenders = []
# Bag of Words
if (easy.getTrainer("BOW_Trainer") == None):
print("BOW service(s) are insufficiently configured, skipping.")
else:
c = ev.Contender("BOW")
c.trainerString = "BOW_Trainer"
c.detectorString = "BOW_Detector"
c.foundMap = {'1': posPurpose, '0': negPurpose}
contenders.append(c)
# Histogram of Oriented Gradients
if (easy.getTrainer("HOG_Trainer") == None):
print("HOG service(s) are insufficiently configured, skipping.")
else:
c = ev.Contender("HOG")
c.trainerString = "HOG_Trainer"
detector = easy.getDetector( "StruckTracker" )
# a test video; the location is relative to the "CVAC.DataDir"
vfile = "tracks/VTS_01_2.mpg"
runset = cvac.RunSet()
# The tracker needs a labeled location that defines the location
# within the first frame of the video of the item we want to track.
# In this we define a bounding box around the toy soldiers initial location.
vlab = easy.getLabelable( vfile, labelText="car" )
labloc = cvac.BBox(290, 280,60,30)
lab = cvac.LabeledLocation()
lab.loc = labloc
lab.confidence = 0
lab.sub = vlab.sub
easy.addToRunSet(runset, lab, easy.getPurpose('pos'))
# Setting the verbosity level gives us some feedback while the tracking is done.
props = easy.getDetectorProperties(detector)
props.verbosity = 7
# A model file is not required for this detector
modelfile = None
class MyDetectorCallbackReceiverI(easy.DetectorCallbackReceiverI):
def __init__(self):
import easy
easy.DetectorCallbackReceiverI.__init__(self)
self.video = cv2.VideoWriter("results.mpg", cv2.cv.CV_FOURCC('P','I','M','1'),25, (720, 480))
if self.video.isOpened() == False:
print("Could not create video file!")
def foundNewResults(self, r2, current=None):
for root, dirnames, filenames in os.walk("testresults1"):
for filename in filenames:
if nologo == filename:
fname = os.path.join(root, filename)
newf = reject_folder + "/" + nologo
if (os.path.isfile(newf)):
os.unlink(newf) #If file exists delete it (required for some OS's)
os.rename( fname, newf)
# Create the new training set and combine it with the trainset1.
# (Alternatively, in the previous step, manually sort the wrong
# classifications into the original corporate_logos subfolders.) Note
# that the original label->purpose assignment needs to be retained, or
# else the createRunSet method will assign arbitrary purposes.
# Also, some trainers treat the "reject" class differently. Omit the line of
# code that assigns the NEGATIVE purpose in case the trainer does not
# distinguish the "reject" class.
mapwithreject = trainset1['classmap']
easy.addToClassmap( mapwithreject, 'reject', easy.getPurpose('neg') )
trainset2 = easy.createRunSet( "corporate_logos_round2", classmap=mapwithreject )
trainset2['runset'].purposedLists.extend( trainset1['runset'].purposedLists )
easy.printRunSetInfo( trainset2, printLabels=True )
# train, round 2
model2 = easy.train( trainer, trainset2 )
# repeat the evaluation (this time with model2), the manual sorting etc.
# with new testsets until the performance is satisfactory.
result1 = easy.detect( detector, model2, testset1 )
easy.printResults(result1)
# Take a look at corpus/LabelMeCarsTest.properties and pay particular
# attention to the following properties:
# LMFolders and LMObjectNames
cs = easy.getCorpusServer( "PythonCorpusService:default -p 10021")
corpus = easy.openCorpus( corpus_fname, corpusServer=cs )
categories, lablist = easy.getDataSet( corpus, corpusServer=cs )
print('Obtained {0} labeled artifact{1} from corpus "{2}":'.format(
len(lablist), ("s","")[len(lablist)==1], corpus.name ));
easy.printCategoryInfo( categories )
# if desired, you can draw the images and their annotations,
# one image at a time, at a given maximum size (width, height)
#easy.drawLabelables( lablist, (512, 512) )
print("==== Training runset: ====")
posPurpose = easy.getPurpose('pos')
negPurpose = easy.getPurpose('neg')
trainset = cvac.RunSet()
easy.addToRunSet(trainset, categories[objname], posPurpose);
#trainset = easy.createRunSet( categories, purpose=posPurpose );
correct = easy.isProperRunSet(trainset, deleteInvalid=True)
if not correct:
print("failed Integrity test!!!!")
exit()
#trainset = easy.createRunSet( categories, purpose=posPurpose );
#easy.printRunSetInfo( trainset, printLabels=True )
easy.printRunSetInfo( trainset )
#
# Connect to the trainer for a Bag of Words algorithm, then
# train with the given runset
runset = cvac.RunSet()
easy.addToRunSet(runset, "trainImg/ca", "neg")
easy.addToRunSet(runset, "trainImg/kr", "pos")
#easy.printRunSetInfo( runset, printArtifacts=False, printLabels=True )
contenders = []
# Bag of Words
if (easy.getDetector("BOW_Detector") == None):
print("BOW detector service is insufficiently configured, skipping.")
else:
c = ev.Contender("BOW")
c.detectorString = "BOW_Detector"
c.detectorData = "detectors/bowUSKOCA.zip"
c.foundMap = {
'kr': easy.getPurpose('pos'),
'ca': easy.getPurpose('neg'),
'us': easy.getPurpose('neg'),
'unlabeled': easy.getPurpose('neg')
}
contenders.append(c)
# OpenCV Cascade detector
if (easy.getDetector("OpenCVCascadeDetector") == None):
print(
"OpenCVCascadeDetector service is insufficiently configured, skipping."
)
else:
c = ev.Contender("Faces")
c.detectorString = "OpenCVCascadeDetector"
c.detectorData = "detectors/OpencvFaces.zip"
runset = cvac.RunSet()
easy.addToRunSet( runset, "trainImg/ca", "neg" )
easy.addToRunSet( runset, "trainImg/kr", "pos" )
#easy.printRunSetInfo( runset, printArtifacts=False, printLabels=True )
contenders = []
# Bag of Words
if (easy.getDetector("BOW_Detector")==None):
print("BOW detector service is insufficiently configured, skipping.")
else:
c = ev.Contender("BOW")
c.detectorString = "BOW_Detector"
c.detectorData = "detectors/bowUSKOCA.zip"
c.foundMap = { 'kr':easy.getPurpose('pos'),
'ca':easy.getPurpose('neg'),
'us':easy.getPurpose('neg'),
'unlabeled':easy.getPurpose('neg')}
contenders.append( c );
# OpenCV Cascade detector
if (easy.getDetector("OpenCVCascadeDetector")==None):
print("OpenCVCascadeDetector service is insufficiently configured, skipping.")
else:
c = ev.Contender("Faces")
c.detectorString = "OpenCVCascadeDetector"
c.detectorData = "detectors/OpencvFaces.zip"
c.foundMap = { 'positive':easy.getPurpose('pos'),
'negative':easy.getPurpose('neg')}
contenders.append( c );
if nologo == filename:
fname = os.path.join(root, filename)
newf = reject_folder + "/" + nologo
if (os.path.isfile(newf)):
os.unlink(
newf
) #If file exists delete it (required for some OS's)
os.rename(fname, newf)
# Create the new training set and combine it with the trainset1.
# (Alternatively, in the previous step, manually sort the wrong
# classifications into the original corporate_logos subfolders.) Note
# that the original label->purpose assignment needs to be retained, or
# else the createRunSet method will assign arbitrary purposes.
# Also, some trainers treat the "reject" class differently. Omit the line of
# code that assigns the NEGATIVE purpose in case the trainer does not
# distinguish the "reject" class.
mapwithreject = trainset1['classmap']
easy.addToClassmap(mapwithreject, 'reject', easy.getPurpose('neg'))
trainset2 = easy.createRunSet("corporate_logos_round2", classmap=mapwithreject)
trainset2['runset'].purposedLists.extend(trainset1['runset'].purposedLists)
easy.printRunSetInfo(trainset2, printLabels=True)
# train, round 2
model2 = easy.train(trainer, trainset2)
# repeat the evaluation (this time with model2), the manual sorting etc.
# with new testsets until the performance is satisfactory.
result1 = easy.detect(detector, model2, testset1)
easy.printResults(result1)
###############################################################
# With background data
'''
Execute jousting for generating ROC points
'''
contenders = []
for nWord in list_nWord:
c1 = evaluate.Contender("bowROC_binary_" + str(nWord))
c1.trainerString = strTrainer
c1.detectorString = strDetector
trainer = easy.getTrainer(c1.trainerString)
trainerProps = easy.getTrainerProperties(trainer)
trainerProps.props["NumWords"] = str(nWord)
c1.trainerProps = trainerProps
c1.foundMap = {
'1': easy.getPurpose('pos'),
'0': easy.getPurpose('neg')
}
contenders.append(c1)
sortedperfdata, perfdata = evaluate.joust(contenders, runset, folds=3)
easy.showROCPlot(perfdata)
'''
Extract only optimal ROC points
'''
rocData_full, optimalIndices = easy.discardSuboptimal(perfdata)
#rocData_full,optimalIndices = easy.discardSuboptimal(perfdata,"sboxes/BOW_Trainer_127_0_0_1")
'''
Prepare detector data of optimal ROC points
'''
rocData_optimal = []
# Take a look at corpus/LabelMeCarsTest.properties and pay particular
# attention to the following properties:
# LMFolders and LMObjectNames
cs = easy.getCorpusServer("PythonCorpusService:default -p 10021")
corpus = easy.openCorpus(corpus_fname, corpusServer=cs)
categories, lablist = easy.getDataSet(corpus, corpusServer=cs)
print('Obtained {0} labeled artifact{1} from corpus "{2}":'.format(
len(lablist), ("s", "")[len(lablist) == 1], corpus.name))
easy.printCategoryInfo(categories)
# if desired, you can draw the images and their annotations,
# one image at a time, at a given maximum size (width, height)
#easy.drawLabelables( lablist, (512, 512) )
print("==== Training runset: ====")
posPurpose = easy.getPurpose('pos')
negPurpose = easy.getPurpose('neg')
trainset = cvac.RunSet()
easy.addToRunSet(trainset, categories[objname], posPurpose)
#trainset = easy.createRunSet( categories, purpose=posPurpose );
correct = easy.isProperRunSet(trainset, deleteInvalid=True)
if not correct:
print("failed Integrity test!!!!")
exit()
#trainset = easy.createRunSet( categories, purpose=posPurpose );
#easy.printRunSetInfo( trainset, printLabels=True )
easy.printRunSetInfo(trainset)
#
# Connect to the trainer for a Bag of Words algorithm, then
# train with the given runset