def train(self):
interactions = config.geti('trainInteractions')
minEpsilon = config.getf('minTrainingEpsilon')
epochSize = len(self.environment.db.images) / 2
epsilon = 1.0
self.controller.setEpsilonGreedy(epsilon)
print 'Epoch 0: Exploration'
self.runEpoch(interactions, len(self.environment.db.images))
self.learner = QLearning()
self.agent.learner = self.learner
epoch = 1
egEpochs = config.geti('epsilonGreedyEpochs')
while epoch <= egEpochs:
epsilon = epsilon - (1.0 - minEpsilon) / float(egEpochs)
if epsilon < minEpsilon: epsilon = minEpsilon
self.controller.setEpsilonGreedy(epsilon)
print 'Epoch', epoch, '(epsilon-greedy:{:5.3f})'.format(epsilon)
self.runEpoch(interactions, epochSize)
epoch += 1
epoch = 1
maxEpochs = config.geti('exploitLearningEpochs')
while epoch <= maxEpochs:
print 'Epoch', epoch + egEpochs, '(exploitation mode: epsilon={:5.3f})'.format(
epsilon)
self.runEpoch(interactions, epochSize)
epoch += 1
def mergeDatasetAndRecords(self, train, val):
numTrain = len(self.dataset) * (1 - config.getf('percentOfValidation'))
numVal = len(self.dataset) * config.getf('percentOfValidation')
random.shuffle(self.dataset)
for i in range(len(self.dataset)):
imgPath = config.get('imageDir') + self.dataset[i]['img'] + '.jpg'
# record format: Action, reward, discountedMaxQ, x1, y1, x2, y2,
record = [self.dataset[i]['A'], self.dataset[i]['R'], 0.0
] + self.dataset[i]['box']
record += [self.dataset[i]['Sp']] + self.dataset[i]['Xp'] + [
self.dataset[i]['Sc']
] + self.dataset[i]['Xc']
record += [self.dataset[i]['Ap']]
if i < numTrain:
try:
train[imgPath].append(record)
except:
train[imgPath] = [record]
else:
try:
val[imgPath].append(record)
except:
val[imgPath] = [record]
return train, val
def train(self):
interactions = config.geti('trainInteractions')
minEpsilon = config.getf('minTrainingEpsilon')
epochSize = len(self.environment.db.images)/2
epsilon = 1.0
self.controller.setEpsilonGreedy(epsilon)
print 'Epoch 0: Exploration'
self.runEpoch(interactions, len(self.environment.db.images))
self.learner = QLearning()
self.agent.learner = self.learner
epoch = 1
egEpochs = config.geti('epsilonGreedyEpochs')
while epoch <= egEpochs:
epsilon = epsilon - (1.0-minEpsilon)/float(egEpochs)
if epsilon < minEpsilon: epsilon = minEpsilon
self.controller.setEpsilonGreedy(epsilon)
print 'Epoch',epoch ,'(epsilon-greedy:{:5.3f})'.format(epsilon)
self.runEpoch(interactions, epochSize)
epoch += 1
epoch = 1
maxEpochs = config.geti('exploitLearningEpochs')
while epoch <= maxEpochs:
print 'Epoch',epoch+egEpochs,'(exploitation mode: epsilon={:5.3f})'.format(epsilon)
self.runEpoch(interactions, epochSize)
epoch += 1
def loadNetwork(self):
modelFile = config.get('networkDir') + 'deploy.prototxt'
meanImage = config.get('meanImagePickle')
self.net = imagenet.ImageNetClassifier(modelFile, self.networkFile, IMAGE_DIM=config.geti('imageSize'), CROPPED_DIM=config.geti('cropSize'), MEAN_IMAGE=meanImage)
self.net.caffenet.set_phase_test()
self.net.caffenet.set_mode_gpu()
self.meanImage = self.net._IMAGENET_MEAN.swapaxes(1, 2).swapaxes(0, 1).astype('float32')
def learn(self, memory, controller):
print '# Identify memory records stored by the agent',memory.O.shape, memory.A.shape,memory.R.shape
totalMemorySize = memory.usableRecords
replayMemorySize = config.geti('trainingIterationsPerBatch')*config.geti('trainingBatchSize')
print '# Select a random sample of records'
recordsToPull = [random.randint(0,totalMemorySize-2) for i in range(replayMemorySize)]
samples = np.zeros( (replayMemorySize, memory.O.shape[1], 1, 1), np.float32 )
targets = np.zeros( (replayMemorySize, 3, 1, 1), np.float32 )
nextStates = np.zeros( (replayMemorySize, memory.O.shape[1], 1, 1), np.float32 )
trainingSet = []
terminalStates = []
for i in range(len(recordsToPull)):
r = recordsToPull[i]
# Make sure that next state belongs to the same image
if memory.I[r] != memory.I[r+1]:
terminalStates.append(i)
samples[i,:,0,0] = memory.O[r,:]
nextStates[i,:,0,0] = memory.O[r+1,:]
action = memory.A[r,0]
reward = memory.R[r,0]
targets[i,:,0,0] = np.array([action, reward, 0.0], np.float32)
if controller.net != None:
controller.loadNetwork(definition='deploy.maxq.prototxt')
discountedMaxNextQ = self.gamma*np.max( controller.getActivations(nextStates), axis=1 )
discountedMaxNextQ[terminalStates] = 0.0
targets[:,2,0,0] = discountedMaxNextQ
print '# Update network'
self.netManager.doNetworkTraining(samples, targets)
def doValidation(self, epoch):
if epoch % config.geti('validationEpochs') != 0:
return
auxRL = BoxSearchRunner('test')
auxRL.run()
indexType = config.get('evaluationIndexType')
category = config.get('category')
if indexType == 'pascal':
categories, catIndex = bse.get20Categories()
elif indexType == 'relations':
categories, catIndex = bse.getCategories()
elif indexType == 'finetunedRelations':
categories, catIndex = bse.getRelationCategories()
if category in categories:
catI = categories.index(category)
else:
catI = -1
scoredDetections = bse.loadScores(config.get('testMemory'), catI)
groundTruthFile = config.get('testGroundTruth')
#ps,rs = bse.evaluateCategory(scoredDetections, 'scores', groundTruthFile)
pl, rl = bse.evaluateCategory(scoredDetections, 'landmarks',
groundTruthFile)
line = lambda x, y, z: x + '\t{:5.3f}\t{:5.3f}\n'.format(y, z)
#print line('Validation Scores:',ps,rs)
print line('Validation Landmarks:', pl, rl)
def runNetworkTuning(self, pretrained):
my_env = os.environ.copy()
my_env['GLOG_logtostderr']='1'
my_env['GLOG_minloglevel']='0'
args = [config.get('tools') + '/finetune_net.bin', config.get('solverFile'), pretrained]
p = subprocess.Popen(args, env=my_env, cwd=self.directory)
p.wait()
return
def run(self):
if self.mode == 'train':
self.agent.persistMemory = True
self.agent.startReplayMemory(len(self.environment.db.images),
config.geti('trainInteractions'),
config.geti('stateFeatures'))
self.train()
elif self.mode == 'test':
self.agent.persistMemory = False
self.test()
def doNetworkTraining(self):
if self.checkpoint == 0:
# launch finetuning
self.runNetworkTuning(config.get('pretrainedModel'))
else:
# Resume training
self.runNetworkTuning(config.get('snapshotPrefix')+'_iter_'+str(self.step))
self.checkpoint += self.step
self.writeCheckpoint()
def __init__(self, workingDir):
self.directory = workingDir
self.writeSolverFile()
self.solver = caffe.SGDSolver(self.directory + 'solver.prototxt')
self.iter = 0
self.itersPerEpisode = config.geti('trainingIterationsPerBatch')
self.lr = config.getf('learningRate')
self.stepSize = config.geti('stepSize')
self.gamma = config.getf('gamma')
print 'CAFFE SOLVER INITALIZED'
def __init__(self, mode):
self.mode = mode
cu.mem('Reinforcement Learning Started')
self.environment = BoxSearchEnvironment(config.get(mode+'Database'), mode, config.get(mode+'GroundTruth'))
self.controller = QNetwork()
cu.mem('QNetwork controller created')
self.learner = None
self.agent = BoxSearchAgent(self.controller, self.learner)
self.task = BoxSearchTask(self.environment, config.get(mode+'GroundTruth'))
self.experiment = Experiment(self.task, self.agent)
def __init__(self, workingDir):
self.directory = workingDir
self.writeSolverFile()
self.solver = caffe.SGDSolver(self.directory + 'solver.prototxt')
self.iter = 0
self.itersPerEpisode = config.geti('trainingIterationsPerBatch')
self.lr = config.getf('learningRate')
self.stepSize = config.geti('stepSize')
self.gamma = config.getf('gamma')
print 'CAFFE SOLVER INITALIZED'
class QNetwork(ActionValueInterface):
networkFile = config.get('networkDir') + config.get(
'snapshotPrefix') + '_iter_' + config.get(
'trainingIterationsPerBatch') + '.caffemodel'
def __init__(self):
self.net = None
print 'QNetwork::Init. Loading ', self.networkFile
if os.path.exists(self.networkFile):
self.loadNetwork()
self.sampler = defaultSampler
def releaseNetwork(self):
if self.net != None:
del self.net
self.net = None
def loadNetwork(self, definition='deploy.prototxt'):
if os.path.isfile(self.networkFile):
modelFile = config.get('networkDir') + definition
self.net = caffe.Net(modelFile, self.networkFile)
self.net.set_phase_test()
self.net.set_mode_gpu()
else:
self.net = None
def getMaxAction(self, state):
values = self.getActionValues(state)
return np.argmax(values, 1)
def getActionValues(self, state):
if self.net == None or self.exploreOrExploit() == EXPLORE:
return self.sampler()
else:
return self.getActivations(state)
def getActivations(self, state):
out = self.net.forward_all(
**{
self.net.inputs[0]:
state.reshape((state.shape[0], state.shape[1], 1, 1))
})
return out['qvalues'].squeeze(axis=(2, 3))
def setEpsilonGreedy(self, epsilon, sampler=None):
if sampler is not None:
self.sampler = sampler
self.epsilon = epsilon
def exploreOrExploit(self):
if self.epsilon > 0:
if random.random() < self.epsilon:
return EXPLORE
return EXPLOIT
def __init__(self, mode):
self.mode = mode
cu.mem('Reinforcement Learning Started')
self.environment = RegionFilteringEnvironment(
config.get(mode + 'Database'), mode)
self.controller = QNetwork()
cu.mem('QNetwork controller created')
self.learner = None
self.agent = RegionFilteringAgent(self.controller, self.learner)
self.task = RegionFilteringTask(self.environment,
config.get(mode + 'GroundTruth'))
self.experiment = Experiment(self.task, self.agent)
def runNetworkTraining(self, args):
my_env = os.environ.copy()
my_env['GLOG_logtostderr']='1'
my_env['GLOG_minloglevel']='0'
monitorFile = self.directory + '/' + config.get('snapshotPrefix')+'_iter_'+str(self.checkpoint+self.step)+'.solverstate'
p = subprocess.Popen(args, env=my_env, cwd=self.directory)
while not os.path.isfile(monitorFile):
time.sleep(10)
p.terminate()
if self.checkpoint > 0:
os.remove(self.directory + '/' + config.get('snapshotPrefix')+'_iter_'+str(self.checkpoint)+'.solverstate')
os.remove(self.directory + '/' + config.get('snapshotPrefix')+'_iter_'+str(self.checkpoint))
return
def getSensors(self):
# Make a vector represenation of the action that brought the agent to this state (9 features)
prevAction = np.zeros( (bs.NUM_ACTIONS) )
prevAction[self.state.actionChosen] = 1.0
# Compute features of visible region (4096 + 21)
activations = self.cnn.getActivations(self.state.box)
# Concatenate all info in the state representation vector
print activations[config.get('convnetLayer')].shape, prevAction.shape, self.startingActivations[config.get('convnetLayer')].shape
state = np.hstack( (activations[config.get('convnetLayer')], self.startingActivations[config.get('convnetLayer')], prevAction) )
self.scores = activations['prob'].tolist()
return {'image':self.imageList[self.idx], 'state':state, 'negEpisode':self.negativeEpisode}
def loadNextEpisode(self):
self.getExample()
if self.imgName == None:
print 'All episodes done'
return
self.visibleImage = Image.open(self.imageDir + '/' + self.imgName + '.jpg')
size = self.visibleImage.size
self.state = [ SingleObjectLocalizer(size, box[0:4], box[4]) for box in self.state ]
if config.geti('maxCandidatesPerImage') != 0 and self.mode == "Training":
random.shuffle(self.state)
self.state = self.state[0: config.geti('maxCandidatesPerImage')]
print 'Episode done:',self.imgName,'Boxes:',len(self.state),'Terminals:',self.terminalCounts,'Moves:',self.episodeMoves
def loadNetwork(self):
modelFile = config.get('networkDir') + 'deploy.prototxt'
meanImage = config.get('meanImagePickle')
self.net = imagenet.ImageNetClassifier(
modelFile,
self.networkFile,
IMAGE_DIM=config.geti('imageSize'),
CROPPED_DIM=config.geti('cropSize'),
MEAN_IMAGE=meanImage)
self.net.caffenet.set_phase_test()
self.net.caffenet.set_mode_gpu()
self.meanImage = self.net._IMAGENET_MEAN.swapaxes(1, 2).swapaxes(
0, 1).astype('float32')
def loadNextEpisode(self):
self.episodeDone = False
self.negativeEpisode = False
if self.selectNegativeSample(): return
# Save actions performed during this episode
if self.mode == 'test' and self.testRecord != None:
with open(config.get('testMemory') + self.imageList[self.idx] + '.txt', 'w') as outfile:
json.dump(self.testRecord, outfile)
# Load a new episode
self.idx += 1
if self.idx < len(self.imageList):
# Initialize state
previousImageName = str(int(self.imageList[self.idx])-1)
print 'Preparing starting image {}'.format(previousImageName)
self.cnn.prepareImage(previousImageName)
print 'Initial box for {} at {}'.format(previousImageName, self.groundTruth[previousImageName])
self.startingActivations = self.cnn.getActivations( self.groundTruth[previousImageName][0])
self.cnn.prepareImage(self.imageList[self.idx])
self.state = bs.BoxSearchState(self.imageList[self.idx], groundTruth=self.groundTruth)
print 'Environment::LoadNextEpisode => Image',self.idx,self.imageList[self.idx],'('+str(self.state.visibleImage.size[0])+','+str(self.state.visibleImage.size[1])+')'
else:
if self.mode == 'train':
random.shuffle(self.imageList)
self.idx = -1
self.loadNextEpisode()
else:
print 'No more images available'
# Restart record for new episode
if self.mode == 'test':
self.testRecord = {'boxes':[], 'actions':[], 'values':[], 'rewards':[], 'scores':[]}
def __init__(self, imageList, mode, groundTruthFile=None):
self.mode = mode
self.cnn = cn.ConvNet()
self.testRecord = None
self.idx = -1
self.imageList = [x.strip() for x in open(imageList)]
self.groundTruth = cu.loadBoxIndexFile(groundTruthFile)
#self.imageList = self.rankImages()
#self.imageList = self.imageList[0:10]
allImgs = set([x.strip() for x in open(config.get('allImagesList'))])
self.negativeSamples = list(allImgs.difference(set(self.groundTruth.keys())))
self.negativeEpisode = False
if self.mode == 'train':
self.negativeProbability = config.getf('negativeEpisodeProb')
random.shuffle(self.imageList)
self.loadNextEpisode()
def dropRecords(self, rec, total, new):
if total > config.geti('replayMemorySize'):
drop = 0
while drop < new:
for k in rec.keys():
rec[k].pop(0)
drop += 1
return rec
def coverRegion(self, box, otherImg=None):
boxes = [map(int,box)]
if otherImg is not None:
self.net.caffenet.CoverRegions(boxes, config.get('imageDir') + otherImg + '.jpg', self.id)
else:
self.net.caffenet.CoverRegions(boxes, '', self.id)
self.id += 1
return True
def getActionValues(self, state):
imgName = state[0]
if self.net == None or self.exploreOrExploit() == EXPLORE:
return np.random.random([len(state[1]), config.geti('outputActions')])
else:
boxes = []
stateFeatures = np.zeros( (len(state[1]), 20, 1, 1), dtype=np.float32)
for i in range(len(state[1])):
s = state[1][i]
boxes.append( map(int, s.nextBox) )
stateFeatures[i,0,0,0] = s.prevScore
stateFeatures[i,1:5,0,0] = np.asarray( s.normPrevBox() )
stateFeatures[i,5,0,0] = s.currScore
stateFeatures[i,6:10,0,0] = np.asarray( s.normCurrBox() )
if s.prevAction() >= 0:
stateFeatures[i,10 + s.prevAction(), 0,0] = 1.0
return self.getActivations(config.get('imageDir') + '/' + imgName + '.jpg', boxes, stateFeatures)
def doNetworkTraining(self, samples, labels):
self.solver.net.set_input_arrays(samples, labels)
self.solver.solve()
self.iter += config.geti('trainingIterationsPerBatch')
if self.iter % self.stepSize == 0:
newLR = self.lr * (self.gamma**int(self.iter / self.stepSize))
print 'Changing LR to:', newLR
self.solver.change_lr(newLR)
def loadNetwork(self, definition='deploy.prototxt'):
if os.path.isfile(self.networkFile):
modelFile = config.get('networkDir') + definition
self.net = caffe.Net(modelFile, self.networkFile)
self.net.set_phase_test()
self.net.set_mode_gpu()
else:
self.net = None
def run(self):
if self.mode == 'train':
self.agent.persistMemory = True
self.agent.startReplayMemory(len(self.environment.imageList), config.geti('trainInteractions'))
self.train()
elif self.mode == 'test':
self.agent.persistMemory = False
self.test()
def __init__(self, imageList, mode, groundTruthFile=None):
self.mode = mode
self.cnn = cn.ConvNet()
self.testRecord = None
self.idx = -1
self.imageList = [x.strip() for x in open(imageList)]
self.groundTruth = cu.loadBoxIndexFile(groundTruthFile)
#self.imageList = self.rankImages()
#self.imageList = self.imageList[0:10]
allImgs = set([x.strip() for x in open(config.get('allImagesList'))])
self.negativeSamples = list(
allImgs.difference(set(self.groundTruth.keys())))
self.negativeEpisode = False
if self.mode == 'train':
self.negativeProbability = config.getf('negativeEpisodeProb')
random.shuffle(self.imageList)
self.loadNextEpisode()
def doNetworkTraining(self, samples, labels):
self.solver.net.set_input_arrays(samples, labels)
self.solver.solve()
self.iter += config.geti('trainingIterationsPerBatch')
if self.iter % self.stepSize == 0:
newLR = self.lr * ( self.gamma** int(self.iter/self.stepSize) )
print 'Changing LR to:',newLR
self.solver.change_lr(newLR)
def loadNetwork(self, definition='deploy.prototxt'):
if os.path.isfile(self.networkFile):
modelFile = config.get('networkDir') + definition
self.net = caffe.Net(modelFile, self.networkFile)
self.net.set_phase_test()
self.net.set_mode_gpu()
else:
self.net = None
def coverRegion(self, box, otherImg=None):
boxes = [map(int, box)]
if otherImg is not None:
self.net.caffenet.CoverRegions(
boxes,
config.get('imageDir') + otherImg + '.jpg', self.id)
else:
self.net.caffenet.CoverRegions(boxes, '', self.id)
self.id += 1
return True
def loadNextEpisode(self):
self.getExample()
if self.imgName == None:
print 'All episodes done'
return
self.visibleImage = Image.open(self.imageDir + '/' + self.imgName +
'.jpg')
size = self.visibleImage.size
self.state = [
SingleObjectLocalizer(size, box[0:4], box[4]) for box in self.state
]
if config.geti(
'maxCandidatesPerImage') != 0 and self.mode == "Training":
random.shuffle(self.state)
self.state = self.state[0:config.geti('maxCandidatesPerImage')]
print 'Episode done:', self.imgName, 'Boxes:', len(
self.state
), 'Terminals:', self.terminalCounts, 'Moves:', self.episodeMoves
def getSensors(self):
# Make a vector represenation of the action that brought the agent to this state (9 features)
prevAction = np.zeros((bs.NUM_ACTIONS))
prevAction[self.state.actionChosen] = 1.0
# Compute features of visible region (4096 + 21)
activations = self.cnn.getActivations(self.state.box)
# Concatenate all info in the state representation vector
print activations[config.get(
'convnetLayer')].shape, prevAction.shape, self.startingActivations[
config.get('convnetLayer')].shape
state = np.hstack(
(activations[config.get('convnetLayer')],
self.startingActivations[config.get('convnetLayer')], prevAction))
self.scores = activations['prob'].tolist()
return {
'image': self.imageList[self.idx],
'state': state,
'negEpisode': self.negativeEpisode
}
def getActionValues(self, state):
imgName = state[0]
if self.net == None or self.exploreOrExploit() == EXPLORE:
return np.random.random(
[len(state[1]), config.geti('outputActions')])
else:
boxes = []
stateFeatures = np.zeros((len(state[1]), 20, 1, 1),
dtype=np.float32)
for i in range(len(state[1])):
s = state[1][i]
boxes.append(map(int, s.nextBox))
stateFeatures[i, 0, 0, 0] = s.prevScore
stateFeatures[i, 1:5, 0, 0] = np.asarray(s.normPrevBox())
stateFeatures[i, 5, 0, 0] = s.currScore
stateFeatures[i, 6:10, 0, 0] = np.asarray(s.normCurrBox())
if s.prevAction() >= 0:
stateFeatures[i, 10 + s.prevAction(), 0, 0] = 1.0
return self.getActivations(
config.get('imageDir') + '/' + imgName + '.jpg', boxes,
stateFeatures)
def loadNetwork(self):
self.imgDim = config.geti('imageDim')
self.cropSize = config.geti('cropSize')
self.contextPad = config.geti('contextPad')
modelFile = config.get('convnetDir') + config.get('convNetDef')
networkFile = config.get('convnetDir') + config.get('trainedConvNet')
self.net = wrapperv0.ImageNetClassifier(modelFile, networkFile, IMAGE_DIM=self.imgDim, CROPPED_DIM=self.cropSize, MEAN_IMAGE=config.get('meanImage'))
self.net.caffenet.set_mode_gpu()
self.net.caffenet.set_phase_test()
self.imageMean = self.net._IMAGENET_MEAN.swapaxes(1, 2).swapaxes(0, 1).astype('float32')
def getActivations(self, imagePath, boxes, state):
n = len(boxes)
activations = cu.emptyMatrix( [n, config.geti('outputActions')] )
numBatches = (n + config.geti('deployBatchSize') - 1) / config.geti('deployBatchSize')
boxes += [ [0,0,0,0] for x in range(numBatches * config.geti('deployBatchSize') - n) ]
stateFeatures = np.zeros( (len(boxes), 20, 1, 1), dtype=np.float32)
stateFeatures[0:n,:,:,:] = state
dims = self.net.caffenet.InitializeImage(imagePath, config.geti('imageSize'), self.meanImage, config.geti('cropSize'))
for k in range(numBatches):
s, f = k * config.geti('deployBatchSize'), (k + 1) * config.geti('deployBatchSize')
e = config.geti('deployBatchSize') if f <= n else n - s
# Forward this batch
#self.net.caffenet.ForwardRegions(boxes[s:f], config.geti('contextPad'))
self.net.caffenet.ForwardRegionsAndState(boxes[s:f], config.geti('contextPad'), [stateFeatures[s:f,:,:,:]])
outputs = self.net.caffenet.blobs
f = n if f > n else f
# Collect outputs
activations[s:f,:] = outputs['prob'].data[0:e,:,:,:].reshape([e,config.geti('outputActions')])
# Release image data
self.net.caffenet.ReleaseImageData()
return activations
def loadNetwork(self):
self.imgDim = config.geti('imageDim')
self.cropSize = config.geti('cropSize')
self.contextPad = config.geti('contextPad')
modelFile = config.get('convnetDir') + config.get('convNetDef')
networkFile = config.get('convnetDir') + config.get('trainedConvNet')
self.net = wrapperv0.ImageNetClassifier(
modelFile,
networkFile,
IMAGE_DIM=self.imgDim,
CROPPED_DIM=self.cropSize,
MEAN_IMAGE=config.get('meanImage'))
self.net.caffenet.set_mode_gpu()
self.net.caffenet.set_phase_test()
self.imageMean = self.net._IMAGENET_MEAN.swapaxes(1, 2).swapaxes(
0, 1).astype('float32')
def __init__(self, imageName, groundTruth=None):
self.imageName = imageName
self.visibleImage = Image.open(config.get('imageDir') + '/' + self.imageName + '.jpg')
self.box = [0,0,0,0]
self.landmarkIndex = {}
self.actionChosen = 2
self.actionValue = 0
self.groundTruth = groundTruth
self.reset()
if self.groundTruth is not None:
self.task = bst.BoxSearchTask()
self.task.groundTruth = self.groundTruth
self.task.loadGroundTruth(self.imageName)
self.stepsWithoutLandmark = 0
def learn(self, memory, controller):
print '# Identify memory records stored by the agent', memory.O.shape, memory.A.shape, memory.R.shape
totalMemorySize = memory.usableRecords
replayMemorySize = config.geti(
'trainingIterationsPerBatch') * config.geti('trainingBatchSize')
print '# Select a random sample of records'
recordsToPull = [
random.randint(0, totalMemorySize - 2)
for i in range(replayMemorySize)
]
samples = np.zeros((replayMemorySize, memory.O.shape[1], 1, 1),
np.float32)
targets = np.zeros((replayMemorySize, 3, 1, 1), np.float32)
nextStates = np.zeros((replayMemorySize, memory.O.shape[1], 1, 1),
np.float32)
trainingSet = []
terminalStates = []
for i in range(len(recordsToPull)):
r = recordsToPull[i]
# Make sure that next state belongs to the same image
if memory.I[r] != memory.I[r + 1]:
terminalStates.append(i)
samples[i, :, 0, 0] = memory.O[r, :]
nextStates[i, :, 0, 0] = memory.O[r + 1, :]
action = memory.A[r, 0]
reward = memory.R[r, 0]
targets[i, :, 0, 0] = np.array([action, reward, 0.0], np.float32)
if controller.net != None:
controller.loadNetwork(definition='deploy.maxq.prototxt')
discountedMaxNextQ = self.gamma * np.max(
controller.getActivations(nextStates), axis=1)
discountedMaxNextQ[terminalStates] = 0.0
targets[:, 2, 0, 0] = discountedMaxNextQ
print '# Update network'
self.netManager.doNetworkTraining(samples, targets)
def doValidation(self, epoch):
if epoch % config.geti('validationEpochs') != 0:
return
auxRL = BoxSearchRunner('test')
auxRL.run()
indexType = config.get('evaluationIndexType')
category = config.get('category')
if indexType == 'pascal':
categories, catIndex = bse.get20Categories()
elif indexType == 'relations':
categories, catIndex = bse.getCategories()
elif indexType == 'finetunedRelations':
categories, catIndex = bse.getRelationCategories()
if category in categories:
catI = categories.index(category)
else:
catI = -1
scoredDetections = bse.loadScores(config.get('testMemory'), catI)
groundTruthFile = config.get('testGroundTruth')
#ps,rs = bse.evaluateCategory(scoredDetections, 'scores', groundTruthFile)
pl,rl = bse.evaluateCategory(scoredDetections, 'landmarks', groundTruthFile)
line = lambda x,y,z: x + '\t{:5.3f}\t{:5.3f}\n'.format(y,z)
#print line('Validation Scores:',ps,rs)
print line('Validation Landmarks:',pl,rl)
def __init__(self, imageName, groundTruth=None):
self.imageName = imageName
self.visibleImage = Image.open(
config.get('imageDir') + '/' + self.imageName + '.jpg')
self.box = [0, 0, 0, 0]
self.landmarkIndex = {}
self.actionChosen = 2
self.actionValue = 0
self.groundTruth = groundTruth
self.reset()
if self.groundTruth is not None:
self.task = bst.BoxSearchTask()
self.task.groundTruth = self.groundTruth
self.task.loadGroundTruth(self.imageName)
self.stepsWithoutLandmark = 0
def train(self):
networkFile = config.get('networkDir') + config.get(
'snapshotPrefix') + '_iter_' + config.get(
'trainingIterationsPerBatch') + '.caffemodel'
interactions = config.geti('trainInteractions')
minEpsilon = config.getf('minTrainingEpsilon')
epochSize = len(self.environment.imageList) / 1
epsilon = 1.0
self.controller.setEpsilonGreedy(epsilon,
self.environment.sampleAction)
epoch = 1
exEpochs = config.geti('explorationEpochs')
while epoch <= exEpochs:
s = cu.tic()
print 'Epoch', epoch, ': Exploration (epsilon=1.0)'
self.runEpoch(interactions, len(self.environment.imageList))
self.task.flushStats()
self.doValidation(epoch)
s = cu.toc('Epoch done in ', s)
epoch += 1
self.learner = QLearning()
self.agent.learner = self.learner
egEpochs = config.geti('epsilonGreedyEpochs')
while epoch <= egEpochs + exEpochs:
s = cu.tic()
epsilon = epsilon - (1.0 - minEpsilon) / float(egEpochs)
if epsilon < minEpsilon: epsilon = minEpsilon
self.controller.setEpsilonGreedy(epsilon,
self.environment.sampleAction)
print 'Epoch', epoch, '(epsilon-greedy:{:5.3f})'.format(epsilon)
self.runEpoch(interactions, epochSize)
self.task.flushStats()
self.doValidation(epoch)
s = cu.toc('Epoch done in ', s)
epoch += 1
maxEpochs = config.geti('exploitLearningEpochs') + exEpochs + egEpochs
while epoch <= maxEpochs:
s = cu.tic()
print 'Epoch', epoch, '(exploitation mode: epsilon={:5.3f})'.format(
epsilon)
self.runEpoch(interactions, epochSize)
self.task.flushStats()
self.doValidation(epoch)
s = cu.toc('Epoch done in ', s)
shutil.copy(networkFile, networkFile + '.' + str(epoch))
epoch += 1
def writeSolverFile(self):
out = open(self.directory + '/solver.prototxt','w')
out.write('train_net: "' + self.directory + 'train.prototxt"\n')
out.write('base_lr: ' + config.get('learningRate') + '\n')
out.write('lr_policy: "step"\n')
out.write('gamma: ' + config.get('gamma') + '\n')
out.write('stepsize: ' + config.get('stepSize') + '\n')
out.write('display: 1\n')
out.write('max_iter: ' + config.get('trainingIterationsPerBatch') + '\n')
out.write('momentum: ' + config.get('momentum') + '\n')
out.write('weight_decay: ' + config.get('weightDecay') + '\n')
out.write('snapshot: ' + config.get('trainingIterationsPerBatch') + '\n')
out.write('snapshot_prefix: "' + self.directory + 'multilayer_qlearner"\n')
out.close()
def loadNextEpisode(self):
# Save actions performed during this episode
if self.mode == 'test' and self.testRecord != None:
with open(config.get('testMemory') + self.db.image + '.txt', 'w') as outfile:
json.dump(self.testRecord, outfile)
# Load a new episode from the database
self.db.loadNext()
if self.db.image != '':
print 'Environment::LoadNextEpisode => Image',self.db.image,'({:4} boxes)'.format(self.db.boxes.shape[0])
# Initialize state
self.state = lh.LayoutHandler(self.db.boxes)
self.performAction([lh.STAY,0])
else:
print 'No more images available'
# Restart record for new episode
if self.mode == 'test':
self.testRecord = {'boxes':[], 'actions':[], 'values':[], 'rewards':[]}
def balanceTrainingExamples(self, candidates):
pos, neg = {},{}
psc, ngc = 0,0
for k in candidates.keys():
for box in candidates[k]:
if box[5] > config.getf('minPositiveOverlap'): # and box[6] >= 1.0:
try: pos[k].append(box)
except: pos[k] = [box]
psc += 1
else:
try: neg[k].append(box)
except: neg[k] = [box]
ngc += 1
self.pos = ImageBoxIndex(pos,True)
self.neg = ImageBoxIndex(neg,True)
self.probNeg = max(float(psc)/float(ngc),0.05)
print 'Positives:',psc,'Negatives:',ngc
def loadNextEpisode(self):
self.episodeDone = False
self.negativeEpisode = False
if self.selectNegativeSample(): return
# Save actions performed during this episode
if self.mode == 'test' and self.testRecord != None:
with open(
config.get('testMemory') + self.imageList[self.idx] +
'.txt', 'w') as outfile:
json.dump(self.testRecord, outfile)
# Load a new episode
self.idx += 1
if self.idx < len(self.imageList):
# Initialize state
previousImageName = str(int(self.imageList[self.idx]) - 1)
print 'Preparing starting image {}'.format(previousImageName)
self.cnn.prepareImage(previousImageName)
print 'Initial box for {} at {}'.format(
previousImageName, self.groundTruth[previousImageName])
self.startingActivations = self.cnn.getActivations(
self.groundTruth[previousImageName][0])
self.cnn.prepareImage(self.imageList[self.idx])
self.state = bs.BoxSearchState(self.imageList[self.idx],
groundTruth=self.groundTruth)
print 'Environment::LoadNextEpisode => Image', self.idx, self.imageList[
self.idx], '(' + str(
self.state.visibleImage.size[0]) + ',' + str(
self.state.visibleImage.size[1]) + ')'
else:
if self.mode == 'train':
random.shuffle(self.imageList)
self.idx = -1
self.loadNextEpisode()
else:
print 'No more images available'
# Restart record for new episode
if self.mode == 'test':
self.testRecord = {
'boxes': [],
'actions': [],
'values': [],
'rewards': [],
'scores': []
}
def writeSolverFile(self):
out = open(self.directory + '/solver.prototxt', 'w')
out.write('train_net: "' + self.directory + 'train.prototxt"\n')
out.write('base_lr: ' + config.get('learningRate') + '\n')
out.write('lr_policy: "step"\n')
out.write('gamma: ' + config.get('gamma') + '\n')
out.write('stepsize: ' + config.get('stepSize') + '\n')
out.write('display: 1\n')
out.write('max_iter: ' + config.get('trainingIterationsPerBatch') +
'\n')
out.write('momentum: ' + config.get('momentum') + '\n')
out.write('weight_decay: ' + config.get('weightDecay') + '\n')
out.write('snapshot: ' + config.get('trainingIterationsPerBatch') +
'\n')
out.write('snapshot_prefix: "' + self.directory +
'multilayer_qlearner"\n')
out.close()
def train(self):
networkFile = config.get('networkDir') + config.get('snapshotPrefix') + '_iter_' + config.get('trainingIterationsPerBatch') + '.caffemodel'
interactions = config.geti('trainInteractions')
minEpsilon = config.getf('minTrainingEpsilon')
epochSize = len(self.environment.imageList)/1
epsilon = 1.0
self.controller.setEpsilonGreedy(epsilon, self.environment.sampleAction)
epoch = 1
exEpochs = config.geti('explorationEpochs')
while epoch <= exEpochs:
s = cu.tic()
print 'Epoch',epoch,': Exploration (epsilon=1.0)'
self.runEpoch(interactions, len(self.environment.imageList))
self.task.flushStats()
self.doValidation(epoch)
s = cu.toc('Epoch done in ',s)
epoch += 1
self.learner = QLearning()
self.agent.learner = self.learner
egEpochs = config.geti('epsilonGreedyEpochs')
while epoch <= egEpochs + exEpochs:
s = cu.tic()
epsilon = epsilon - (1.0-minEpsilon)/float(egEpochs)
if epsilon < minEpsilon: epsilon = minEpsilon
self.controller.setEpsilonGreedy(epsilon, self.environment.sampleAction)
print 'Epoch',epoch ,'(epsilon-greedy:{:5.3f})'.format(epsilon)
self.runEpoch(interactions, epochSize)
self.task.flushStats()
self.doValidation(epoch)
s = cu.toc('Epoch done in ',s)
epoch += 1
maxEpochs = config.geti('exploitLearningEpochs') + exEpochs + egEpochs
while epoch <= maxEpochs:
s = cu.tic()
print 'Epoch',epoch,'(exploitation mode: epsilon={:5.3f})'.format(epsilon)
self.runEpoch(interactions, epochSize)
self.task.flushStats()
self.doValidation(epoch)
s = cu.toc('Epoch done in ',s)
shutil.copy(networkFile, networkFile + '.' + str(epoch))
epoch += 1
def balanceTrainingExamples(self, candidates):
pos, neg = {}, {}
psc, ngc = 0, 0
for k in candidates.keys():
for box in candidates[k]:
if box[5] > config.getf(
'minPositiveOverlap'): # and box[6] >= 1.0:
try:
pos[k].append(box)
except:
pos[k] = [box]
psc += 1
else:
try:
neg[k].append(box)
except:
neg[k] = [box]
ngc += 1
self.pos = ImageBoxIndex(pos, True)
self.neg = ImageBoxIndex(neg, True)
self.probNeg = max(float(psc) / float(ngc), 0.05)
print 'Positives:', psc, 'Negatives:', ngc
def updatePostReward(self):
if len(self.state
) == self.terminalCounts or self.episodeMoves >= config.geti(
'maxMovesAllowed'):
self.loadNextEpisode()
def updatePostReward(self):
if len(self.state) == self.terminalCounts or self.episodeMoves >= config.geti('maxMovesAllowed'):
self.loadNextEpisode()
def __init__(self, alpha=0.5, gamma=0.99):
ValueBasedLearner.__init__(self)
self.alpha = alpha
self.gamma = gamma
self.netManager = cnm.CaffeConvNetManagement(config.get('networkDir'))
scoredDetections = bse.loadScores(config.get('testMemory'), catI)
groundTruthFile = config.get('testGroundTruth')
#ps,rs = bse.evaluateCategory(scoredDetections, 'scores', groundTruthFile)
pl,rl = bse.evaluateCategory(scoredDetections, 'landmarks', groundTruthFile)
line = lambda x,y,z: x + '\t{:5.3f}\t{:5.3f}\n'.format(y,z)
#print line('Validation Scores:',ps,rs)
print line('Validation Landmarks:',pl,rl)
if __name__ == "__main__":
if len(sys.argv) < 2:
print 'Use: ReinforcementLearningRunner.py configFile'
sys.exit()
## Load Global Configuration
config.readConfiguration(sys.argv[1])
from QNetwork import QNetwork
from QLearning import QLearning
from BoxSearchEnvironment import BoxSearchEnvironment
from BoxSearchTask import BoxSearchTask
from BoxSearchAgent import BoxSearchAgent
import BoxSearchEvaluation as bse
## Run Training and Testing
rl = BoxSearchRunner('train')
rl.run()
rl = BoxSearchRunner('test')
rl.run()
def __init__(self, alpha=0.5):
ValueBasedLearner.__init__(self)
self.alpha = alpha
self.gamma = config.getf('gammaDiscountReward')
self.netManager = CaffeMultiLayerPerceptronManagement(config.get('networkDir'))
def test(self):
interactions = config.geti('testInteractions')
self.controller.setEpsilonGreedy(config.getf('testEpsilon'))
self.runEpoch(interactions, len(self.environment.imageList))
__author__ = "Juan C. Caicedo, [email protected]" import os import utils as cu import numpy as np import caffe from caffe import wrapperv0 import RLConfig as config LAYER = config.get('convnetLayer') class ConvNet(): def __init__(self): self.net = None self.image = '' self.id = 0 self.loadNetwork() def loadNetwork(self): self.imgDim = config.geti('imageDim') self.cropSize = config.geti('cropSize') self.contextPad = config.geti('contextPad') modelFile = config.get('convnetDir') + config.get('convNetDef') networkFile = config.get('convnetDir') + config.get('trainedConvNet') self.net = wrapperv0.ImageNetClassifier( modelFile, networkFile, IMAGE_DIM=self.imgDim,
def prepareImage(self, image):
if self.image != '':
self.net.caffenet.ReleaseImageData()
self.image = config.get('imageDir') + image + '.jpg'
self.net.caffenet.InitializeImage(self.image, self.imgDim,
self.imageMean, self.cropSize)
import random
import numpy as np
import json
import utils as cu
import libDetection as det
import RLConfig as config
def sigmoid(x, a=1.0, b=0.0):
return 1.0/(1.0 + np.exp(-a*x + b))
def tanh(x, a=5, b=0.5, c=2.0):
return c*np.tanh(a*x + b)
TEST_TIME_OUT = config.geti('testTimeOut')
class BoxSearchEnvironment(Environment, Named):
def __init__(self, imageList, mode, groundTruthFile=None):
self.mode = mode
self.cnn = cn.ConvNet()
self.testRecord = None
self.idx = -1
self.imageList = [x.strip() for x in open(imageList)]
self.groundTruth = cu.loadBoxIndexFile(groundTruthFile)
#self.imageList = self.rankImages()
#self.imageList = self.imageList[0:10]
allImgs = set([x.strip() for x in open(config.get('allImagesList'))])
self.negativeSamples = list(allImgs.difference(set(self.groundTruth.keys())))
self.negativeEpisode = False
