def on_epoch_end(self, callback_data, model, epoch):
_eil = self._get_cached_epoch_loss(callback_data, model, epoch, 'loss')
if _eil:
if _eil['cost'] < self.best_cost or self.best_cost is None:
# TODO: switch this to a general seralization op
save_obj(model.serialize(keep_states=True), self.best_path)
self.best_cost = _eil['cost']
def save_history(self, epoch, model):
# if history > 1, this function will save the last N checkpoints
# where N is equal to self.history. The files will have the form
# of save_path with the epoch added to the filename before the ext
if len(self.checkpoint_files) > self.history:
# remove oldest checkpoint file when max count have been saved
fn = self.checkpoint_files.popleft()
try:
os.remove(fn)
logger.info("removed old checkpoint %s" % fn)
except OSError:
logger.warn("Could not delete old checkpoint file %s" % fn)
path_split = os.path.splitext(self.save_path)
save_path = "%s_%d%s" % (path_split[0], epoch, path_split[1])
# add the current file to the deque
self.checkpoint_files.append(save_path)
save_obj(model.serialize(keep_states=True), save_path)
# maintain a symlink pointing to the latest model params
try:
if os.path.islink(self.save_path):
os.remove(self.save_path)
os.symlink(os.path.split(save_path)[-1], self.save_path)
except OSError:
logger.warn("Could not create latest model symlink %s -> %s" % (self.save_path, save_path))
def save_history(self, epoch):
# if history > 1, this function will save the last N checkpoints
# where N is equal to self.history. The files will have the form
# of save_path with the epoch added to the filename before the ext
if len(self.checkpoint_files) > self.history:
# remove oldest checkpoint file when max count have been saved
fn = self.checkpoint_files.popleft()
try:
os.remove(fn)
logger.info('removed old checkpoint %s' % fn)
except OSError:
logger.warn('Could not delete old checkpoint file %s' % fn)
path_split = os.path.splitext(self.save_path)
save_path = '%s_%d%s' % (path_split[0], epoch, path_split[1])
# add the current file to the deque
self.checkpoint_files.append(save_path)
save_obj(self.model.serialize(keep_states=True), save_path)
# maintain a symlink pointing to the latest model params
try:
if os.path.islink(self.save_path):
os.remove(self.save_path)
os.symlink(os.path.split(save_path)[-1], self.save_path)
except OSError:
logger.warn('Could not create latest model symlink %s -> %s' %
(self.save_path, save_path))
def on_epoch_end(self, callback_data, model, epoch):
_eil = self._get_cached_epoch_loss(callback_data, model, epoch, "loss")
if _eil:
if _eil["cost"] < self.best_cost or self.best_cost is None:
# TODO: switch this to a general seralization op
save_obj(model.serialize(keep_states=True), self.best_path)
self.best_cost = _eil["cost"]
def on_epoch_end(self, epoch):
_eil = self._get_cached_epoch_loss(epoch, 'loss')
if _eil:
if _eil['cost'] < self.best_cost or self.best_cost is None:
save_obj(self.model.serialize(keep_states=True),
self.best_path)
self.best_cost = _eil['cost']
def serialize(self, fn=None, keep_states=True):
"""
Creates a dictionary storing the layer parameters and epochs complete.
Arguments:
fn (str): file to save pkl formatted model dictionary
keep_states (bool): Whether to save optimizer states.
Returns:
dict: Model data including layer parameters and epochs complete.
"""
# get the model dict with the weights
pdict = self.get_description(get_weights=True, keep_states=keep_states)
pdict['epoch_index'] = self.epoch_index + 1
if self.initialized:
if not hasattr(self.layers, 'decoder'):
pdict['train_input_shape'] = self.layers.in_shape
else:
# serialize shapes both for encoder and decoder
pdict['train_input_shape'] = (self.layers.encoder.in_shape +
self.layers.decoder.in_shape)
if fn is not None:
save_obj(pdict, fn)
return
return pdict
def save_params(self, param_path, keep_states=True):
"""
Serializes and saves model parameters to the path specified.
Arguments:
param_path (str): File to write serialized parameter dict to.
keep_states (bool): Whether to save optimizer states too.
Defaults to True.
"""
save_obj(self.serialize(keep_states), param_path)
def on_epoch_end(self, epoch):
if 'cost/validation' in self.callback_data:
val_freq = self.callback_data['cost/validation'].attrs['epoch_freq']
if (epoch + 1) % val_freq == 0:
validation_cost = self.callback_data['cost/validation'][epoch/val_freq]
if validation_cost < self.best_cost or self.best_cost is None:
save_obj(self.model.serialize(keep_states=True), self.best_path)
self.best_cost = validation_cost
def save_params(self, param_path, keep_states=True):
"""
Serializes and saves model parameters to the path specified.
Arguments:
param_path (str): File to write serialized parameter dict to.
keep_states (bool): Whether to save optimizer states too.
Defaults to True.
"""
save_obj(self.serialize(keep_states), param_path)
def on_epoch_end(self, epoch):
if 'cost/validation' in self.callback_data:
val_freq = self.callback_data['cost/validation'].attrs[
'epoch_freq']
if (epoch + 1) % val_freq == 0:
validation_cost = self.callback_data['cost/validation'][
epoch / val_freq]
if validation_cost < self.best_cost or self.best_cost is None:
save_obj(self.model.serialize(keep_states=True),
self.best_path)
self.best_cost = validation_cost
def save_meta(self):
save_obj({'ntrain': self.ntrain,
'nval': self.nval,
'train_start': self.train_start,
'val_start': self.val_start,
'macro_size': self.macro_size,
'batch_prefix': self.batch_prefix,
'global_mean': self.global_mean,
'label_dict': self.label_dict,
'label_names': self.label_names,
'val_nrec': self.val_nrec,
'train_nrec': self.train_nrec,
'img_size': self.target_size,
'nclass': self.nclass}, self.meta_file)
def save_meta(self):
save_obj({'ntrain': self.ntrain,
'nval': self.nval,
'train_start': self.train_start,
'val_start': self.val_start,
'macro_size': self.macro_size,
'batch_prefix': self.batch_prefix,
'global_mean': self.global_mean,
'label_dict': self.label_dict,
'label_names': self.label_names,
'val_nrec': self.val_nrec,
'train_nrec': self.train_nrec,
'img_size': self.target_size,
'nclass': self.nclass}, self.meta_file)
def on_sigint_catch(self, epoch, minibatch):
"""
Callback to handle SIGINT events
Arguments:
epoch (int): index of current epoch
minibatch (int): index of minibatch that is ending
"""
# restore the orignal handler
signal.signal(signal.SIGINT, signal.SIG_DFL)
# save the model
if self.save_path is not None:
save_obj(self.model().serialize(keep_states=True), self.save_path)
raise KeyboardInterrupt('Checkpoint file saved to {0}'.format(self.save_path))
else:
raise KeyboardInterrupt
def on_sigint_catch(self, epoch, minibatch):
"""
Callback to handle SIGINT events
Arguments:
epoch (int): index of current epoch
minibatch (int): index of minibatch that is ending
"""
# restore the orignal handler
signal.signal(signal.SIGINT, signal.SIG_DFL)
# save the model
if self.save_path is not None:
save_obj(self.model().serialize(keep_states=True), self.save_path)
raise KeyboardInterrupt("Checkpoint file saved to {0}".format(self.save_path))
else:
raise KeyboardInterrupt
def save_meta(self):
save_obj(
{
"ntrain": self.ntrain,
"nval": self.nval,
"train_start": self.train_start,
"val_start": self.val_start,
"macro_size": self.macro_size,
"batch_prefix": self.batch_prefix,
"global_mean": self.global_mean,
"label_dict": self.label_dict,
"label_names": self.label_names,
"val_nrec": self.val_nrec,
"train_nrec": self.train_nrec,
"img_size": self.target_size,
"nclass": self.nclass,
},
self.meta_file,
)
def save_history(self, epoch):
# if history > 1, this function will save the last N checkpoints
# where N is equal to self.history. The files will have the form
# of save_path with the epoch added to the filename before the ext
if len(self.checkpoint_files) > self.history:
# remove oldest checkpoint file when max count have been saved
fn = self.checkpoint_files.popleft()
try:
os.remove(fn)
logger.info('removed old checkpoint %s' % fn)
except OSError:
logger.warn('Could not delete old checkpoint file %s' % fn)
path_split = os.path.splitext(self.save_path)
save_path = '%s_%d%s' % (path_split[0], epoch, path_split[1])
# add the current file to the deque
self.checkpoint_files.append(save_path)
save_obj(self.model.serialize(keep_states=True), save_path)
def serialize(self, fn=None, keep_states=True):
"""
Creates a dictionary storing the layer parameters and epochs complete.
Arguments:
fn (str): file to save pkl formatted model dictionary
keep_states (bool): Whether to save optimizer states.
Returns:
dict: Model data including layer parameters and epochs complete.
"""
# get the model dict with the weights
pdict = self.get_description(get_weights=True, keep_states=keep_states)
pdict['epoch_index'] = self.epoch_index + 1
if fn is not None:
save_obj(pdict, fn)
return
return pdict
def get_w2v_vocab(fname, max_vocab_size, cache=True):
"""
Get ordered dict of vocab from google word2vec
"""
if cache:
cache_fname = fname.split('.')[0] + ".vocab"
if os.path.isfile(cache_fname):
vocab, vocab_size = load_obj(cache_fname)
neon_logger.display(
"Word2Vec vocab cached, size is: {}".format(vocab_size))
return vocab, vocab_size
with open(fname, 'rb') as f:
header = f.readline()
vocab_size, embed_dim = map(int, header.split())
binary_len = np.dtype('float32').itemsize * embed_dim
neon_logger.display("Word2Vec vocab size is: {}".format(vocab_size))
vocab_size = min(max_vocab_size, vocab_size)
neon_logger.display("Reducing vocab size to: {}".format(vocab_size))
vocab = OrderedDict()
for i, line in enumerate(range(vocab_size)):
word = []
while True:
ch = f.read(1)
if ch == b' ':
word = (b''.join(word)).decode('utf-8')
break
if ch != b'\n':
word.append(ch)
f.read(binary_len)
vocab[word] = i
if cache:
save_obj((vocab, vocab_size), cache_fname)
return vocab, vocab_size
def get_w2v_vocab(fname, max_vocab_size, cache=True):
"""
Get ordered dict of vocab from google word2vec
"""
if cache:
cache_fname = fname.split('.')[0] + ".vocab"
if os.path.isfile(cache_fname):
vocab, vocab_size = load_obj(cache_fname)
neon_logger.display("Word2Vec vocab cached, size is: {}".format(vocab_size))
return vocab, vocab_size
with open(fname, 'rb') as f:
header = f.readline()
vocab_size, embed_dim = map(int, header.split())
binary_len = np.dtype('float32').itemsize * embed_dim
neon_logger.display("Word2Vec vocab size is: {}".format(vocab_size))
vocab_size = min(max_vocab_size, vocab_size)
neon_logger.display("Reducing vocab size to: {}".format(vocab_size))
vocab = OrderedDict()
for i, line in enumerate(range(vocab_size)):
word = []
while True:
ch = f.read(1)
if ch == b' ':
word = (b''.join(word)).decode('utf-8')
break
if ch != b'\n':
word.append(ch)
f.read(binary_len)
vocab[word] = i
if cache:
save_obj((vocab, vocab_size), cache_fname)
return vocab, vocab_size
def __init__(self,
path='.',
n_mb=None,
img_per_batch=None,
conv_size=None,
rpn_rois_per_img=None,
frcn_rois_per_img=None,
add_flipped=False,
shuffle=False,
deterministic=False,
rebuild_cache=False,
subset_pct=100,
mock_db=None):
self.batch_index = 0
self.path = path
self.mock_db = mock_db
# how many ROIs per image
self.rois_per_img = rpn_rois_per_img if rpn_rois_per_img else self.RPN_ROI_PER_IMAGE
self.img_per_batch = img_per_batch if img_per_batch else self.IMG_PER_BATCH
self.rois_per_batch = self.rois_per_img * self.img_per_batch
# how many ROIs to use to train frcnn
self.frcn_rois_per_img = frcn_rois_per_img if frcn_rois_per_img \
else self.FRCNN_ROI_PER_IMAGE
assert self.img_per_batch == 1, "Only a minibatch of 1 is supported."
self.num_classes = len(self.CLASSES)
self._class_to_index = dict(
list(zip(self.CLASSES, list(range(self.num_classes)))))
# shape of the final conv layer
if conv_size:
self._conv_size = conv_size
else:
self._conv_size = int(np.floor(self.MAX_SIZE * self.SCALE))
self._feat_stride = 1 / float(self.SCALE)
self._num_scales = len(self.SCALES) * len(self.RATIOS)
self._total_anchors = self._conv_size * self._conv_size * self._num_scales
self.shuffle = shuffle
self.deterministic = deterministic
self.add_flipped = add_flipped
# load the configure the dataset paths
self.config = self.load_data()
# annotation metadata
self._annotation_file_ext = '.xml'
self._annotation_obj_tag = 'object'
self._annotation_class_tag = 'name'
self._annotation_xmin_tag = 'xmin'
self._annotation_xmax_tag = 'xmax'
self._annotation_ymin_tag = 'ymin'
self._annotation_ymax_tag = 'ymax'
# self.rois_per_batch is 128 (2*64) ROIs
# But the image path batch size is self.img_per_batch
# need to control the batch size here
assert self.img_per_batch is 1, "Only a batch size of 1 image is supported"
neon_logger.display("Backend batchsize is changed to be {} "
"from Object Localization dataset".format(
self.img_per_batch))
self.be.bsz = self.img_per_batch
# 0. allocate buffers
self.allocate()
if not self.mock_db:
# 1. read image index file
assert os.path.exists(self.config['image_path']), \
'Image index file does not exist: {}'.format(self.config['image_path'])
with open(self.config['index_path']) as f:
self.image_index = [x.strip() for x in f.readlines()]
num_images = len(self.image_index)
self.num_image_entries = num_images * 2 if self.add_flipped else num_images
self.ndata = self.num_image_entries * self.rois_per_img
else:
self.num_image_entries = 1
self.ndata = self.num_image_entries * self.rois_per_img
assert (subset_pct > 0 and
subset_pct <= 100), ('subset_pct must be between 0 and 100')
if n_mb is not None:
self.nbatches = n_mb
else:
self.nbatches = int(self.num_image_entries / self.img_per_batch *
subset_pct / 100)
self.cache_file = self.config['cache_path']
if os.path.exists(
self.cache_file) and not rebuild_cache and not self.mock_db:
self.roi_db = load_obj(self.cache_file)
neon_logger.display('ROI dataset loaded from file {}'.format(
self.cache_file))
elif not self.mock_db:
# 2. read object Annotations (XML)
roi_db = self.load_roi_groundtruth()
if (self.add_flipped):
roi_db = self.add_flipped_db(roi_db)
# 3. construct acnhor targets
self.roi_db = self.add_anchors(roi_db)
if NORMALIZE_BBOX_TARGETS:
# 4. normalize bbox targets by class
self.roi_db = self.normalize_bbox_targets(self.roi_db)
save_obj(self.roi_db, self.cache_file)
neon_logger.display('wrote ROI dataset to {}'.format(
self.cache_file))
else:
assert self.mock_db is not None
roi_db = [self.mock_db]
self.roi_db = self.add_anchors(roi_db)
# 4. map anchors back to full canvas.
# This is neccessary because the network outputs reflect the full canvas.
# We cache the files in the unmapped state (above) to save memory.
self.roi_db = unmap(self.roi_db)
lr_sched = PolySchedule(total_epochs=10, power=0.5)
opt_gdm = GradientDescentMomentum(0.01, 0.9, wdecay=0.0002, schedule=lr_sched)
opt_biases = GradientDescentMomentum(0.02, 0.9, schedule=lr_sched)
opt = MultiOptimizer({'default': opt_gdm, 'Bias': opt_biases})
if not args.resume:
# fit the model for 3 epochs
model.fit(train, optimizer=opt, num_epochs=3, cost=cost, callbacks=callbacks)
train.reset()
# get 1 image
for im, l in train:
break
train.exit_batch_provider()
save_obj((im.get(), l.get()), 'im1.pkl')
im_save = im.get().copy()
if args.resume:
(im2, l2) = load_obj('im1.pkl')
im.set(im2)
l.set(l2)
# run fprop and bprop on this minibatch save the results
out_fprop = model.fprop(im)
out_fprop_save = [x.get() for x in out_fprop]
im.set(im_save)
out_fprop = model.fprop(im)
out_fprop_save2 = [x.get() for x in out_fprop]
for x, y in zip(out_fprop_save, out_fprop_save2):
assert np.max(np.abs(x - y)) == 0.0, '2 fprop iterations do not match'
from neon.util.argparser import NeonArgparser
from neon.layers import Pooling
from neon.models import Model
from neon.data import ImageLoader
from neon.util.persist import save_obj, load_obj
# parse the command line arguments (generates the backend)
parser = NeonArgparser(__doc__)
args = parser.parse_args()
scales = [112, 128, 160, 240]
for scale in scales:
print scale
test = ImageLoader(set_name='validation', shuffle=False, do_transforms=False, inner_size=scale,
scale_range=scale, repo_dir=args.data_dir)
model_desc = load_obj(args.model_file)
model_desc['model']['config']['layers'].insert(-1, Pooling('all', op='avg').get_description())
model = Model(model_desc, test, inference=True)
softmaxes = model.get_outputs(test)
save_obj(softmaxes, "bigfeat_dropout_SM_{}.pkl".format(scale))
if args.model_file is None:
load_vgg_weights(model, args.data_dir)
cost = Multicost(costs=[GeneralizedCostMask(costfunc=CrossEntropyMulti()),
GeneralizedCostMask(costfunc=SmoothL1Loss())],
weights=[1, 1])
callbacks = Callbacks(model, eval_set=test_set, **args.callback_args)
model.fit(train_set, optimizer=optimizer,
num_epochs=num_epochs, cost=cost, callbacks=callbacks)
# Fast R-CNN model requires scale the bbox regression branch linear layer weights
# before saving the model
model = scale_bbreg_weights(
model, train_set.bbtarget_means, train_set.bbtarget_stds)
save_obj(model.serialize(keep_states=True), args.save_path)
neon_logger.display('running eval...')
metric_train = model.eval(train_set, metric=ObjectDetection())
neon_logger.display(
'Train: label accuracy - {}%, object detection logloss - {}'.format(metric_train[0] * 100,
metric_train[1]))
metric_test = model.eval(test_set, metric=ObjectDetection())
neon_logger.display(
'Test: label accuracy - {}%, object detection logloss - {}'.format(metric_test[0] * 100,
metric_test[1]))
h5train_text = h5f['report_train']
if os.path.exists(args.vector_name):
neon_logger.display("cached encoded vectors exists: {}".format(args.vector_name))
(train_vec, sentences) = load_obj(args.vector_name)
model.initialize(dataset=(max_len, 1))
else:
neon_logger.display("Encoding the entire training set....")
# encode all the training sentences
model.initialize(dataset=(max_len, 1))
train_set = SentenceEncode(h5train, h5train_text, h5train.attrs['nsample'], vocab_size_layer,
max_len=max_len, index_from=index_from)
sentences = h5train_text[:train_set.ndata].reshape(-1, 1)
train_vec = model.get_outputs(train_set)
neon_logger.display("Encoding complete. Saving to {}".format(args.vector_name))
save_obj((train_vec, sentences), args.vector_name)
s2v = SentenceVector(train_vec, sentences)
xdev = be.zeros((max_len, 1), dtype=np.int32) # bsz is 1, feature size
xbuf = np.zeros((max_len, 1), dtype=np.int32)
# Add python 2 & 3 support for raw_input
try:
input = raw_input
except NameError:
pass
while True:
line = input('\nEnter a new sentence for inference: \n')
xbuf = prep_data(line, 'text', max_len, vocab)
def save_weights(self, save_path): save_obj(self.model.serialize(keep_states = True), save_path)
rng_seed=rng_seed,
device_id=args.device_id,
default_dtype=args.datatype,
)
(X_train, y_train), (X_test, y_test), nout = load_data(DATASET_NAME)
if VALIDATION:
(X_train, y_train), (X_valid, y_valid) = split_train_set(X_train, y_train)
model, opt, cost = get_custom_vgg(nout=nout)
train_set = DataIterator(X_train, y_train, nclass=nout, lshape=(3, 32, 32))
test_set = DataIterator(X_test, y_test, nclass=nout, lshape=(3, 32, 32))
callbacks = Callbacks(model, train_set, args, eval_set=test_set)
if VALIDATION:
valid_set = DataIterator(X_valid, y_valid, nclass=nout, lshape=(3, 32, 32))
callbacks = Callbacks(model, train_set, args, eval_set=valid_set)
model.fit(train_set, optimizer=opt, num_epochs=num_epochs, cost=cost, callbacks=callbacks)
print 'Validation: ', VALIDATION
print 'Train misclassification error: ', model.eval(train_set, metric=Misclassification())
if VALIDATION:
print 'Valid misclassification error: ', model.eval(valid_set, metric=Misclassification())
print 'Test misclassification error: ', model.eval(test_set, metric=Misclassification())
if args.save_path is not None:
save_obj(model.serialize(), EXPERIMENT_DIR + args.save_path)
def train(self, minibatch, epoch = 0):
# expand components of minibatch
prestates, steers, speeds, rewards, poststates, terminals = minibatch
assert len(prestates.shape) == 2
assert len(poststates.shape) == 2
assert len(steers.shape) == 1
assert len(speeds.shape) == 1
assert len(rewards.shape) == 1
assert len(terminals.shape) == 1
assert prestates.shape == poststates.shape
assert prestates.shape[0] == steers.shape[0] == speeds.shape[0] == rewards.shape[0] == poststates.shape[0] == terminals.shape[0]
if self.target_steps and self.train_iterations % self.target_steps == 0:
# HACK: serialize network to disk and read it back to clone
filename = self.save_weights_prefix + "_target.pkl"
save_obj(self.model.serialize(keep_states = False), filename)
self.target_model.load_weights(filename)
# feed-forward pass for poststates to get Q-values
self._setInput(poststates)
postq = self.target_model.fprop(self.input, inference = True)
assert postq.shape == (self.num_actions, self.batch_size)
# calculate max Q-value for each poststate
postq = postq.asnumpyarray()
maxsteerq = np.max(postq[:self.num_steers,:], axis=0)
assert maxsteerq.shape == (self.batch_size,), "size: %s" % str(maxsteerq.shape)
maxspeedq = np.max(postq[-self.num_speeds:,:], axis=0)
assert maxspeedq.shape == (self.batch_size,)
# feed-forward pass for prestates
self._setInput(prestates)
preq = self.model.fprop(self.input, inference = False)
assert preq.shape == (self.num_actions, self.batch_size)
# make copy of prestate Q-values as targets
# HACK: copy() was needed to make it work on CPU
targets = preq.asnumpyarray().copy()
# update Q-value targets for actions taken
for i, (steer, speed) in enumerate(zip(steers, speeds)):
if terminals[i]:
targets[steer, i] = float(rewards[i])
targets[self.num_steers + speed, i] = float(rewards[i])
else:
targets[steer, i] = float(rewards[i]) + self.discount_rate * maxsteerq[i]
targets[self.num_steers + speed, i] = float(rewards[i]) + self.discount_rate * maxspeedq[i]
# copy targets to GPU memory
self.targets.set(targets)
# calculate errors
deltas = self.cost.get_errors(preq, self.targets)
assert deltas.shape == (self.num_actions, self.batch_size)
#assert np.count_nonzero(deltas.asnumpyarray()) == 2 * self.batch_size, str(np.count_nonzero(deltas.asnumpyarray()))
# calculate cost, just in case
cost = self.cost.get_cost(preq, self.targets)
assert cost.shape == (1,1)
#print "cost:", cost.asnumpyarray()
# clip errors
if self.clip_error:
self.be.clip(deltas, -self.clip_error, self.clip_error, out = deltas)
# perform back-propagation of gradients
self.model.bprop(deltas)
# perform optimization
self.optimizer.optimize(self.model.layers_to_optimize, epoch)
'''
if np.any(rewards < 0):
preqq = preq.asnumpyarray().copy()
self._setInput(prestates)
qvalues = self.model.fprop(self.input, inference = True).asnumpyarray().copy()
indexes = rewards < 0
print "indexes:", indexes
print "preq:", preqq[:, indexes].T
print "preq':", qvalues[:, indexes].T
print "diff:", (qvalues[:, indexes]-preqq[:, indexes]).T
print "steers:", steers[indexes]
print "speeds:", speeds[indexes]
print "rewards:", rewards[indexes]
print "terminals:", terminals[indexes]
print "preq[0]:", preqq[:, 0]
print "preq[0]':", qvalues[:, 0]
print "diff:", qvalues[:, 0] - preqq[:, 0]
print "deltas:", deltas.asnumpyarray()[:, indexes].T
raw_input("Press Enter to continue...")
'''
# increase number of weight updates (needed for target clone interval)
self.train_iterations += 1
def train(self, minibatch, epoch):
# expand components of minibatch
prestates, actions, speed_actions, rewards, poststates, terminals = minibatch
assert len(prestates.shape) == 2
assert len(poststates.shape) == 2
assert len(actions.shape) == 1
assert len(rewards.shape) == 1
assert len(terminals.shape) == 1
assert prestates.shape == poststates.shape
assert prestates.shape[0] == actions.shape[0] == rewards.shape[0] == poststates.shape[0] == terminals.shape[0]
#print "WE ARE ACTUALLY TRAINING IN HERE"
if self.target_steps and self.train_iterations % self.target_steps == 0:
# HACK: serialize network to disk and read it back to clone
filename = self.save_weights_prefix + "_target.pkl"
save_obj(self.model.serialize(keep_states = False), filename)
self.target_model.load_weights(filename)
# feed-forward pass for poststates to get Q-values
self._setInput(poststates)
postq = self.target_model.fprop(self.input, inference = True)
assert postq.shape == (self.num_actions, self.batch_size)
# calculate max Q-value for each poststate
postq = postq.asnumpyarray()
maxpostq = np.max(postq, axis=0)
#print maxpostq.shape
assert maxpostq.shape == (self.batch_size,)
# feed-forward pass for prestates
self._setInput(prestates)
preq = self.model.fprop(self.input, inference = False)
assert preq.shape == (self.num_actions, self.batch_size)
# make copy of prestate Q-values as targets
targets = preq.asnumpyarray().copy()
# update Q-value targets for actions taken
for i, action in enumerate(actions):
self.action_count[action] += 1
if terminals[i]:
targets[action, i] = float(rewards[i])
if rewards[i] == -1000:
print "######################### action ", action, "should never be sampled again"
print "sampled_terminal"
else:
targets[action, i] = float(rewards[i]) + self.discount_rate * maxpostq[i]
#targets[i,action] = float(rewards[i]) + self.discount_rate * maxpostq[i]
#print "action count", self.action_count
# copy targets to GPU memory
self.targets.set(targets)
# calculate errors
deltas = self.cost.get_errors(preq, self.targets)
assert deltas.shape == (self.num_actions, self.batch_size)
#assert np.count_nonzero(deltas.asnumpyarray()) == 32
print "nonzero deltas", np.count_nonzero(deltas.asnumpyarray())
# calculate cost, just in case
cost = self.cost.get_cost(preq, self.targets)
assert cost.shape == (1,1)
print "cost:", cost.asnumpyarray()
# clip errors
#if self.clip_error:
# self.be.clip(deltas, -self.clip_error, self.clip_error, out = deltas)
# perform back-propagation of gradients
self.model.bprop(deltas)
# perform optimization
self.optimizer.optimize(self.model.layers_to_optimize, epoch)
# increase number of weight updates (needed for target clone interval)
self.train_iterations += 1
def on_epoch_end(self, epoch):
_eil = self._get_cached_epoch_loss(epoch, 'loss')
if _eil:
if _eil['cost'] < self.best_cost or self.best_cost is None:
save_obj(self.model.serialize(keep_states=True), self.best_path)
self.best_cost = _eil['cost']
def __init__(self, path='.', n_mb=None, img_per_batch=None, conv_size=None,
rpn_rois_per_img=None, frcn_rois_per_img=None, add_flipped=False,
shuffle=False, deterministic=False, rebuild_cache=False, subset_pct=100,
mock_db=None):
self.batch_index = 0
self.path = path
self.mock_db = mock_db
# how many ROIs per image
self.rois_per_img = rpn_rois_per_img if rpn_rois_per_img else self.RPN_ROI_PER_IMAGE
self.img_per_batch = img_per_batch if img_per_batch else self.IMG_PER_BATCH
self.rois_per_batch = self.rois_per_img * self.img_per_batch
# how many ROIs to use to train frcnn
self.frcn_rois_per_img = frcn_rois_per_img if frcn_rois_per_img \
else self.FRCNN_ROI_PER_IMAGE
assert self.img_per_batch == 1, "Only a minibatch of 1 is supported."
self.num_classes = len(self.CLASSES)
self._class_to_index = dict(list(zip(self.CLASSES, list(range(self.num_classes)))))
# shape of the final conv layer
if conv_size:
self._conv_size = conv_size
else:
self._conv_size = int(np.floor(self.MAX_SIZE * self.SCALE))
self._feat_stride = 1 / float(self.SCALE)
self._num_scales = len(self.SCALES) * len(self.RATIOS)
self._total_anchors = self._conv_size * self._conv_size * self._num_scales
self.shuffle = shuffle
self.deterministic = deterministic
self.add_flipped = add_flipped
# load the configure the dataset paths
self.config = self.load_data()
# annotation metadata
self._annotation_file_ext = '.xml'
self._annotation_obj_tag = 'object'
self._annotation_class_tag = 'name'
self._annotation_xmin_tag = 'xmin'
self._annotation_xmax_tag = 'xmax'
self._annotation_ymin_tag = 'ymin'
self._annotation_ymax_tag = 'ymax'
# self.rois_per_batch is 128 (2*64) ROIs
# But the image path batch size is self.img_per_batch
# need to control the batch size here
assert self.img_per_batch is 1, "Only a batch size of 1 image is supported"
neon_logger.display("Backend batchsize is changed to be {} "
"from Object Localization dataset".format(
self.img_per_batch))
self.be.bsz = self.img_per_batch
# 0. allocate buffers
self.allocate()
if not self.mock_db:
# 1. read image index file
assert os.path.exists(self.config['image_path']), \
'Image index file does not exist: {}'.format(self.config['image_path'])
with open(self.config['index_path']) as f:
self.image_index = [x.strip() for x in f.readlines()]
num_images = len(self.image_index)
self.num_image_entries = num_images * 2 if self.add_flipped else num_images
self.ndata = self.num_image_entries * self.rois_per_img
else:
self.num_image_entries = 1
self.ndata = self.num_image_entries * self.rois_per_img
assert (subset_pct > 0 and subset_pct <= 100), ('subset_pct must be between 0 and 100')
if n_mb is not None:
self.nbatches = n_mb
else:
self.nbatches = int(self.num_image_entries / self.img_per_batch * subset_pct / 100)
self.cache_file = self.config['cache_path']
if os.path.exists(self.cache_file) and not rebuild_cache and not self.mock_db:
self.roi_db = load_obj(self.cache_file)
neon_logger.display('ROI dataset loaded from file {}'.format(self.cache_file))
elif not self.mock_db:
# 2. read object Annotations (XML)
roi_db = self.load_roi_groundtruth()
if(self.add_flipped):
roi_db = self.add_flipped_db(roi_db)
# 3. construct acnhor targets
self.roi_db = self.add_anchors(roi_db)
if NORMALIZE_BBOX_TARGETS:
# 4. normalize bbox targets by class
self.roi_db = self.normalize_bbox_targets(self.roi_db)
save_obj(self.roi_db, self.cache_file)
neon_logger.display('wrote ROI dataset to {}'.format(self.cache_file))
else:
assert self.mock_db is not None
roi_db = [self.mock_db]
self.roi_db = self.add_anchors(roi_db)
# 4. map anchors back to full canvas.
# This is neccessary because the network outputs reflect the full canvas.
# We cache the files in the unmapped state (above) to save memory.
self.roi_db = unmap(self.roi_db)
def train(self, minibatch, epoch):
# expand components of minibatch
prestates, actions, rewards, poststates, terminals = minibatch
assert len(prestates.shape) == 4
assert len(poststates.shape) == 4
assert len(actions.shape) == 1
assert len(rewards.shape) == 1
assert len(terminals.shape) == 1
assert prestates.shape == poststates.shape
assert prestates.shape[0] == actions.shape[0] == rewards.shape[
0] == poststates.shape[0] == terminals.shape[0]
if self.target_steps and self.train_iterations % self.target_steps == 0:
# HACK: push something through network, so that weights exist
self.model.fprop(self.tensor)
# HACK: serialize network to disk and read it back to clone
filename = os.path.join(self.save_weights_path,
"target_network.pkl")
save_obj(self.model.serialize(keep_states=False), filename)
self.target_model.load_weights(filename)
# feed-forward pass for poststates to get Q-values
self.setTensor(poststates)
postq = self.target_model.fprop(self.tensor, inference=True)
assert postq.shape == (self.num_actions, self.batch_size)
# calculate max Q-value for each poststate
maxpostq = self.be.max(postq, axis=0).asnumpyarray()
assert maxpostq.shape == (1, self.batch_size)
# feed-forward pass for prestates
self.setTensor(prestates)
preq = self.model.fprop(self.tensor, inference=False)
assert preq.shape == (self.num_actions, self.batch_size)
# make copy of prestate Q-values as targets
targets = preq.asnumpyarray()
# update Q-value targets for actions taken
for i, action in enumerate(actions):
if terminals[i]:
targets[action, i] = float(rewards[i])
else:
targets[action, i] = float(
rewards[i]) + self.discount_rate * maxpostq[0, i]
# copy targets to GPU memory
self.targets.set(targets)
# calculate errors
deltas = self.cost.get_errors(preq, self.targets)
assert deltas.shape == (self.num_actions, self.batch_size)
#assert np.count_nonzero(deltas.asnumpyarray()) == 32
# calculate cost, just in case
cost = self.cost.get_cost(preq, self.targets)
assert cost.shape == (1, 1)
# clip errors
if self.clip_error:
self.be.clip(deltas, -self.clip_error, self.clip_error, out=deltas)
# perform back-propagation of gradients
self.model.bprop(deltas)
# perform optimization
self.optimizer.optimize(self.model.layers_to_optimize, epoch)
# increase number of weight updates (needed for target clone interval)
self.train_iterations += 1
# calculate statistics
if self.callback:
self.callback.on_train(cost.asnumpyarray()[0, 0])
def train(self, minibatch, epoch):
# expand components of minibatch
prestates, actions, rewards, poststates, terminals = minibatch
assert len(prestates.shape) == 4
assert len(poststates.shape) == 4
assert len(actions.shape) == 1
assert len(rewards.shape) == 1
assert len(terminals.shape) == 1
assert prestates.shape == poststates.shape
assert prestates.shape[0] == actions.shape[0] == rewards.shape[0] == poststates.shape[0] == terminals.shape[0]
if self.target_steps and self.train_iterations % self.target_steps == 0:
# HACK: push something through network, so that weights exist
self.model.fprop(self.tensor)
# HACK: serialize network to disk and read it back to clone
filename = os.path.join(self.save_weights_path, "target_network.pkl")
save_obj(self.model.serialize(keep_states = False), filename)
self.target_model.load_weights(filename)
# feed-forward pass for poststates to get Q-values
self.setTensor(poststates)
postq = self.target_model.fprop(self.tensor, inference = True)
assert postq.shape == (self.num_actions, self.batch_size)
# calculate max Q-value for each poststate
maxpostq = self.be.max(postq, axis=0).asnumpyarray()
assert maxpostq.shape == (1, self.batch_size)
# feed-forward pass for prestates
self.setTensor(prestates)
preq = self.model.fprop(self.tensor, inference = False)
assert preq.shape == (self.num_actions, self.batch_size)
# make copy of prestate Q-values as targets
targets = preq.asnumpyarray()
# update Q-value targets for actions taken
for i, action in enumerate(actions):
if terminals[i]:
targets[action, i] = float(rewards[i])
else:
targets[action, i] = float(rewards[i]) + self.discount_rate * maxpostq[0,i]
# copy targets to GPU memory
self.targets.set(targets)
# calculate errors
deltas = self.cost.get_errors(preq, self.targets)
assert deltas.shape == (self.num_actions, self.batch_size)
#assert np.count_nonzero(deltas.asnumpyarray()) == 32
# calculate cost, just in case
cost = self.cost.get_cost(preq, self.targets)
assert cost.shape == (1,1)
# clip errors
if self.clip_error:
self.be.clip(deltas, -self.clip_error, self.clip_error, out = deltas)
# perform back-propagation of gradients
self.model.bprop(deltas)
# perform optimization
self.optimizer.optimize(self.model.layers_to_optimize, epoch)
# increase number of weight updates (needed for target clone interval)
self.train_iterations += 1
# calculate statistics
if self.callback:
self.callback.on_train(cost.asnumpyarray()[0,0])
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('cache_file', help='path to data cache file')
args = parser.parse_args()
cache_file = args.cache_file
# check for RW access to file
assert os.path.exists(cache_file), 'file does not exist %s' % cache_file
if not os.access(os.path.abspath(cache_file), os.R_OK | os.W_OK):
raise IOError('Need to add read and/or write permissions on file %s' % cache_file)
dc = load_obj(cache_file)
if 'global_mean' not in dc or 'img_size' not in dc:
raise ValueError('data cache file missing global_mean key')
sz = dc['img_size']
gm = dc['global_mean']
if len(gm.shape) != 2 or (gm.shape[0] != sz*sz*3 or gm.shape[1] != 1):
raise ValueError('global mean shape %s does not match format expected' % str(gm.shape))
# Collapse the full tensor mean into channel means and correct the order (RGB <-> BGR)
dc['global_mean'] = np.mean(gm.reshape(3, -1), axis=1).reshape(3, 1)[::-1]
save_obj(dc, cache_file)
print '%s updated to new format' % cache_file
def load_data(path,
file_ext=['txt'],
valid_split=None,
vocab_file_name=None,
max_vocab_size=None,
max_len_w=None,
output_path=None,
subset_pct=100):
"""
Given a path where data are saved, look for the ones with the right extensions
If a split factor is given, it will split all the files into training and valid
set. Then build vocabulary from the training and validation sets.
Arguments:
path: which directory to look for all the documents
file_ext: what extension of the files to look for
valid_split: to split the data into train/valid set. If None, no split
vocab_file_name: optional file name. If None, the script will decide a name
given path and split
max_vocab_size: maximum number of words to use in vocabulary (by most frequent)
max_len_w: maximum length of sentences in words
output_path: path used to save preprocessed data and resuts
subset_pct: subset of dataset to load into H5 file (percentage)
Returns:
The function saves 2 files:
h5 file with preprocessed data
vocabulary file with: vocab, reverse_vocab, word_count
"""
file_names = get_file_list(path, file_ext)
file_str = get_file_str(path,
len(file_names),
labelled=False,
valid_split=valid_split,
subset_pct=subset_pct)
# create output dir if needed
if not os.path.isdir(output_path):
os.makedirs(output_path)
# file name to store the vocabulary
if vocab_file_name is None:
vocab_file_name = file_str + '.vocab'
vocab_file_name = os.path.join(output_path, vocab_file_name)
# If max sizes arent set, assume no limit
if not max_len_w:
max_len_w = sys.maxsize
if not max_vocab_size:
max_vocab_size = sys.maxsize
# file name to store the pre-processed train/valid dataset
h5_file_name = os.path.join(output_path, file_str + '.h5')
if os.path.exists(h5_file_name) and os.path.exists(vocab_file_name):
neon_logger.display(
"dataset files {} and vocabulary file {} already exist. "
"will use cached data. ".format(h5_file_name, vocab_file_name))
return h5_file_name, vocab_file_name
# split into training/valid set
if valid_split is not None:
if 'json' in file_ext:
# Split based on number of files
train_split = int(np.ceil(len(file_names) * (1 - valid_split)))
train_files = file_names[:train_split]
valid_files = file_names[train_split:]
train_sent = load_json_sent(train_files, subset_pct)
valid_sent = load_json_sent(valid_files, subset_pct)
all_sent = train_sent + valid_sent
elif 'txt' in file_ext:
# Split based on number of lines (since only 2 files)
all_sent = load_txt_sent(file_names, subset_pct)
train_split = int(np.ceil(len(all_sent) * (1 - valid_split)))
train_sent = all_sent[:train_split]
valid_sent = all_sent[train_split:]
else:
neon_logger.display(
"Unsure how to load file_ext {}, please use 'json' or 'txt'.".
format(file_ext))
else:
train_files = file_names
if 'json' in file_ext:
train_sent = load_json_sent(train_files, subset_pct)
elif 'txt' in file_ext:
train_sent = load_txt_sent(train_files, subset_pct)
else:
neon_logger.display(
"Unsure how to load file_ext {}, please use 'json' or 'txt'.".
format(file_ext))
all_sent = train_sent
if os.path.exists(vocab_file_name):
neon_logger.display(
"open existing vocab file: {}".format(vocab_file_name))
vocab, rev_vocab, word_count = load_obj(vocab_file_name)
else:
neon_logger.display("Building vocab file")
# build vocab
word_count = defaultdict(int)
for sent in all_sent:
sent_words = tokenize(sent)
if len(sent_words) > max_len_w or len(sent_words) == 0:
continue
for word in sent_words:
word_count[word] += 1
# sort the word_count , re-assign ids by its frequency. Useful for downstream tasks
# only done for train vocab
vocab_sorted = sorted(word_count.items(),
key=lambda kv: kv[1],
reverse=True)
vocab = OrderedDict()
# get word count as array in same ordering as vocab (but with maximum length)
word_count_ = np.zeros((len(word_count), ), dtype=np.int64)
for i, t in enumerate(list(zip(*vocab_sorted))[0][:max_vocab_size]):
word_count_[i] = word_count[t]
vocab[t] = i
word_count = word_count_
# generate the reverse vocab
rev_vocab = dict((wrd_id, wrd) for wrd, wrd_id in vocab.items())
neon_logger.display("vocabulary from {} is saved into {}".format(
path, vocab_file_name))
save_obj((vocab, rev_vocab, word_count), vocab_file_name)
vocab_size = len(vocab)
neon_logger.display(
"\nVocab size from the dataset is: {}".format(vocab_size))
neon_logger.display(
"\nProcessing and saving training data into {}".format(h5_file_name))
# now process and save the train/valid data
h5f = h5py.File(h5_file_name, 'w', libver='latest')
shape, maxshape = (len(train_sent), ), (None)
dt = np.dtype([('text', h5py.special_dtype(vlen=str)),
('num_words', np.uint16)])
report_text_train = h5f.create_dataset('report_train',
shape=shape,
maxshape=maxshape,
dtype=dt,
compression='gzip')
report_train = h5f.create_dataset('train',
shape=shape,
maxshape=maxshape,
dtype=h5py.special_dtype(vlen=np.int32),
compression='gzip')
# map text to integers
wdata = np.zeros((1, ), dtype=dt)
ntrain = 0
for sent in train_sent:
text_int = [-1 if t not in vocab else vocab[t] for t in tokenize(sent)]
# enforce maximum sentence length
if len(text_int) > max_len_w or len(text_int) == 0:
continue
report_train[ntrain] = text_int
wdata['text'] = clean_string(sent)
wdata['num_words'] = len(text_int)
report_text_train[ntrain] = wdata
ntrain += 1
report_train.attrs['nsample'] = ntrain
report_train.attrs['vocab_size'] = vocab_size
report_text_train.attrs['nsample'] = ntrain
report_text_train.attrs['vocab_size'] = vocab_size
if valid_split:
neon_logger.display(
"\nProcessing and saving validation data into {}".format(
h5_file_name))
shape = (len(valid_sent), )
report_text_valid = h5f.create_dataset('report_valid',
shape=shape,
maxshape=maxshape,
dtype=dt,
compression='gzip')
report_valid = h5f.create_dataset(
'valid',
shape=shape,
maxshape=maxshape,
dtype=h5py.special_dtype(vlen=np.int32),
compression='gzip')
nvalid = 0
for sent in valid_sent:
text_int = [
-1 if t not in vocab else vocab[t] for t in tokenize(sent)
]
# enforce maximum sentence length
if len(text_int) > max_len_w or len(text_int) == 0:
continue
report_valid[nvalid] = text_int
wdata['text'] = clean_string(sent)
wdata['num_words'] = len(text_int)
report_text_valid[nvalid] = wdata
nvalid += 1
report_valid.attrs['nsample'] = nvalid
report_valid.attrs['vocab_size'] = vocab_size
report_text_valid.attrs['nsample'] = nvalid
report_text_valid.attrs['vocab_size'] = vocab_size
h5f.close()
return h5_file_name, vocab_file_name
opt = MultiOptimizer({'default': opt_gdm, 'Bias': opt_biases})
if not args.resume:
# fit the model for 3 epochs
model.fit(train,
optimizer=opt,
num_epochs=3,
cost=cost,
callbacks=callbacks)
train.reset()
# get 1 image
for im, l in train:
break
train.exit_batch_provider()
save_obj((im.get(), l.get()), 'im1.pkl')
im_save = im.get().copy()
if args.resume:
(im2, l2) = load_obj('im1.pkl')
im.set(im2)
l.set(l2)
# run fprop and bprop on this minibatch save the results
out_fprop = model.fprop(im)
out_fprop_save = [x.get() for x in out_fprop]
im.set(im_save)
out_fprop = model.fprop(im)
out_fprop_save2 = [x.get() for x in out_fprop]
for x, y in zip(out_fprop_save, out_fprop_save2):
assert np.max(np.abs(x - y)) == 0.0, '2 fprop iterations do not match'
def on_epoch_end(self, callback_data, model, epoch):
if self.history > 1:
self.save_history(epoch, model)
else:
save_obj(model.serialize(keep_states=True), self.save_path)
X_test, y_test, cluster)
spec_out = nout
spec_set = DataIterator(
X_spec, y_spec, nclass=spec_out, lshape=(3, 32, 32))
spec_test = DataIterator(
X_spec_test, y_spec_test, nclass=spec_out, lshape=(3, 32, 32))
# Train the specialist
specialist, opt, cost = spec_net(nout=spec_out, archive_path=gene_path)
callbacks = Callbacks(specialist, spec_set, args, eval_set=spec_test)
callbacks.add_early_stop_callback(early_stop)
callbacks.add_save_best_state_callback(path)
specialist.fit(spec_set, optimizer=opt,
num_epochs=specialist.epoch_index + num_epochs, cost=cost, callbacks=callbacks)
# Print results
print 'Specialist Train misclassification error: ', specialist.eval(spec_set, metric=Misclassification())
print 'Specialist Test misclassification error: ', specialist.eval(spec_test, metric=Misclassification())
print 'Generalist Train misclassification error: ', generalist.eval(spec_set, metric=Misclassification())
print 'Generalist Test misclassification error: ', generalist.eval(spec_test, metric=Misclassification())
# specialists.append(specialist)
save_obj(specialist.serialize(), path)
except:
path = confusion_matrix_name + '_' + clustering_name + '_' + str(num_clusters) + 'clusters/'
print 'Failed for ', path
failed.append(path)
for f in failed:
print f
(proposals, num_proposals) = proposalLayer.get_proposals()
# convert outputs to bounding boxes
boxes = faster_rcnn.get_bboxes(outputs, proposals,
num_proposals, num_classes,
im_shape.get(), im_scale, max_per_image,
thresh, nms_thresh)
all_boxes[mb_idx] = boxes
# retrieve gt boxes
# we add a extra column to track detections during the AP calculation
detected = np.array([False] * num_gt_boxes)
gt_boxes = np.hstack([
gt_boxes.get()[:num_gt_boxes] / im_scale,
gt_classes.get()[:num_gt_boxes],
difficult.get()[:num_gt_boxes], detected[:, np.newaxis]
])
all_gt_boxes[mb_idx] = gt_boxes
neon_logger.display('Evaluating detections')
avg_precision = voc_eval(all_boxes,
all_gt_boxes,
valid_set.CLASSES,
use_07_metric=True)
if args.output is not None:
neon_logger.display('Saving inference results to {}'.format(args.output))
save_obj([all_boxes, avg_precision], args.output)
def __init__(self,
image_set,
year,
path='.',
add_flipped=False,
overlap_thre=None,
output_type=0,
n_mb=None,
img_per_batch=None,
rois_per_img=None,
rois_random_sample=True,
shuffle=False):
self.isRoiDB = True
self.batch_index = 0
self.year = year
self.image_set = image_set
self.add_flipped = add_flipped
self.overlap_thre = overlap_thre if overlap_thre else FRCN_IOU_THRE
self.output_type = output_type
# how many ROIs per image
self.rois_per_image = rois_per_img if rois_per_img else FRCN_ROI_PER_IMAGE
self.img_per_batch = img_per_batch if img_per_batch else FRCN_IMG_PER_BATCH
self.fg_rois_per_image = FRCN_FG_FRAC * self.rois_per_image
self.bg_rois_per_image = self.rois_per_image - self.fg_rois_per_image
self.rois_per_batch = self.rois_per_image * self.img_per_batch
self.rois_random_sample = rois_random_sample
self.shuffle = shuffle
self.cache_file_name = 'voc_{}_{}_flip_{}_ovlp_{}.pkl'.format(
self.year, self.image_set, self.add_flipped, self.overlap_thre)
print 'prepare PASCAL VOC {} from year {}: add flipped image {} and overlap threshold {}'\
.format(self.image_set, self.year, self.add_flipped, self.overlap_thre)
# PASCAL class to index
self.num_classes = PASCAL_VOC_NUM_CLASSES
self._class_to_index = dict(
zip(PASCAL_VOC_CLASSES, xrange(self.num_classes)))
# load the voc dataset
self.voc_root = self.load_voc(image_set, year, path)
self.cache_file = os.path.join(self.voc_root, self.cache_file_name)
# load the precomputed ss results from voc data, it includes both 2007 and 2012 data
self.ss_path = self.load_voc('ss', None, path)
# VOC paths and infos
self.image_index_file = os.path.join(self.voc_root, 'ImageSets',
'Main', self.image_set + '.txt')
self.image_path = os.path.join(self.voc_root, 'JPEGImages')
self._image_file_ext = '.jpg'
self.annotation_path = os.path.join(self.voc_root, 'Annotations')
self._annotation_file_ext = '.xml'
self._annotation_obj_tag = 'object'
self._annotation_class_tag = 'name'
self._annotation_xmin_tag = 'xmin'
self._annotation_xmax_tag = 'xmax'
self._annotation_ymin_tag = 'ymin'
self._annotation_ymax_tag = 'ymax'
self._selective_search_ext = '.pkl'
self.selective_search_file = os.path.join(
self.ss_path,
'_'.join(['voc', year, self.image_set, 'selectivesearch.pkl']))
self._bb_xmin_idx = 0
self._bb_ymin_idx = 1
self._bb_xmax_idx = 2
self._bb_ymax_idx = 3
# self.rois_per_batch is 128 (2*64) ROIs
# But the image path batch size is self.img_per_batch
# need to control the batch size here
print "Backend batchsize is changed to be image_per_batch from PASCAL_VOC dataset"
self.be.bsz = self.img_per_batch
# backend tensor to push the data
self.image_shape = (3, FRCN_MAX_SCALE, FRCN_MAX_SCALE)
self.img_np = np.zeros(
(3, FRCN_MAX_SCALE, FRCN_MAX_SCALE, self.be.bsz), dtype=np.float32)
self.dev_X_img = self.be.iobuf(self.image_shape, dtype=np.float32)
self.dev_X_img_chw = self.dev_X_img.reshape(3, FRCN_MAX_SCALE,
FRCN_MAX_SCALE,
self.be.bsz)
# for rois, features are 4 + 1 (idx within the batch)
self.dev_X_rois = self.be.zeros((self.rois_per_batch, 5))
self.dev_y_labels_flat = self.be.zeros((1, self.rois_per_batch),
dtype=np.int32)
self.dev_y_labels = self.be.zeros(
(self.num_classes, self.rois_per_batch), dtype=np.int32)
self.dev_y_bbtargets = self.be.zeros(
(self.num_classes * 4, self.rois_per_batch))
self.dev_y_bbmask = self.be.zeros(
(self.num_classes * 4, self.rois_per_batch))
# the shape will indicate the shape for 1st path (ImageNet model), and
# 2nd path (ROIs)
self.shape = [self.image_shape, self.num_classes * 4]
# Need to do the following:
# 1. load the image index list
# 2. for each image, load the ground truth from pascal annotation
# 3. load the selective search ROIs (this step needs gt ROIs)
# 4.1. merge the ROIs
# 4.2. may have to add the flipped images for training
# 4.3. add the fields for max overlap and max overlapped classes
# 4.4. add the bounding box targets for regression
# 5. during minibatch feeding:
# - rescale images
# - rescale ROIs
# - random select foreground ROIs (bigger ones)
# - random select background ROIS (smaller ones)
# - clamp bg ROI labels (to be 0)
# - convert ROIs into the regression target (ROIs, 4*21)
# 1.
assert os.path.exists(self.image_index_file), \
'Image index file does not exist: {}'.format(self.image_index_file)
with open(self.image_index_file) as f:
self.image_index = [x.strip() for x in f.readlines()]
# self.image_index = image_index * 2 if self.add_flipped else image_index
self.num_images = len(self.image_index)
self.num_image_entries = self.num_images * \
2 if self.add_flipped else self.num_images
self.ndata = self.num_image_entries * self.rois_per_image
self.nbatches = self.num_image_entries / self.img_per_batch
if n_mb is not None:
self.nbatches = n_mb
if os.path.exists(self.cache_file):
self.roi_db = load_obj(self.cache_file)
print 'ROI dataset loaded from file {}'.format(self.cache_file)
else:
# 2.
self.roi_gt = self.load_pascal_roi_groundtruth()
# 3.
self.roi_ss = self.load_pascal_roi_selectivesearch()
# 4.
self.roi_db = self.combine_gt_ss_roi()
save_obj(self.roi_db, self.cache_file)
print 'wrote ROI dataset to {}'.format(self.cache_file)
def test_model_serialize(backend_default, data):
(X_train, y_train), (X_test, y_test), nclass = load_mnist(path=data)
train_set = DataIterator(
[X_train, X_train], y_train, nclass=nclass, lshape=(1, 28, 28))
init_norm = Gaussian(loc=0.0, scale=0.01)
# initialize model
path1 = Sequential([Conv((5, 5, 16), init=init_norm, bias=Constant(0), activation=Rectlin()),
Pooling(2),
Affine(nout=20, init=init_norm, bias=init_norm, activation=Rectlin())])
path2 = Sequential([Affine(nout=100, init=init_norm, bias=Constant(0), activation=Rectlin()),
Dropout(keep=0.5),
Affine(nout=20, init=init_norm, bias=init_norm, activation=Rectlin())])
layers = [MergeMultistream(layers=[path1, path2], merge="stack"),
Affine(nout=20, init=init_norm, batch_norm=True, activation=Rectlin()),
Affine(nout=10, init=init_norm, activation=Logistic(shortcut=True))]
tmp_save = 'test_model_serialize_tmp_save.pickle'
mlp = Model(layers=layers)
mlp.optimizer = GradientDescentMomentum(learning_rate=0.1, momentum_coef=0.9)
mlp.cost = GeneralizedCost(costfunc=CrossEntropyBinary())
mlp.initialize(train_set, cost=mlp.cost)
n_test = 3
num_epochs = 3
# Train model for num_epochs and n_test batches
for epoch in range(num_epochs):
for i, (x, t) in enumerate(train_set):
x = mlp.fprop(x)
delta = mlp.cost.get_errors(x, t)
mlp.bprop(delta)
mlp.optimizer.optimize(mlp.layers_to_optimize, epoch=epoch)
if i > n_test:
break
# Get expected outputs of n_test batches and states of all layers
outputs_exp = []
pdicts_exp = [l.get_params_serialize() for l in mlp.layers_to_optimize]
for i, (x, t) in enumerate(train_set):
outputs_exp.append(mlp.fprop(x, inference=True))
if i > n_test:
break
# Serialize model
save_obj(mlp.serialize(keep_states=True), tmp_save)
# Load model
mlp = Model(layers=layers)
mlp.load_weights(tmp_save)
outputs = []
pdicts = [l.get_params_serialize() for l in mlp.layers_to_optimize]
for i, (x, t) in enumerate(train_set):
outputs.append(mlp.fprop(x, inference=True))
if i > n_test:
break
# Check outputs, states, and params are the same
for output, output_exp in zip(outputs, outputs_exp):
assert np.allclose(output.get(), output_exp.get())
for pd, pd_exp in zip(pdicts, pdicts_exp):
for s, s_e in zip(pd['states'], pd_exp['states']):
if isinstance(s, list): # this is the batch norm case
for _s, _s_e in zip(s, s_e):
assert np.allclose(_s, _s_e)
else:
assert np.allclose(s, s_e)
for p, p_e in zip(pd['params'], pd_exp['params']):
assert type(p) == type(p_e)
if isinstance(p, list): # this is the batch norm case
for _p, _p_e in zip(p, p_e):
assert np.allclose(_p, _p_e)
elif isinstance(p, np.ndarray):
assert np.allclose(p, p_e)
else:
assert p == p_e
os.remove(tmp_save)
h5train_text = h5f['report_train']
if os.path.exists(args.vector_name):
neon_logger.display("cached encoded vectors exists: {}".format(args.vector_name))
(train_vec, sentences) = load_obj(args.vector_name)
model.initialize(dataset=(max_len, 1))
else:
neon_logger.display("Encoding the entire training set....")
# encode all the training sentences
model.initialize(dataset=(max_len, 1))
train_set = SentenceEncode(h5train, h5train_text, h5train.attrs['nsample'], vocab_size_layer,
max_len=max_len, index_from=index_from)
sentences = h5train_text[:train_set.ndata].reshape(-1, 1)
train_vec = model.get_outputs(train_set)
neon_logger.display("Encoding complete. Saving to {}".format(args.vector_name))
save_obj((train_vec, sentences), args.vector_name)
s2v = SentenceVector(train_vec, sentences)
xdev = be.zeros((max_len, 1), dtype=np.int32) # bsz is 1, feature size
xbuf = np.zeros((max_len, 1), dtype=np.int32)
# Add python 2 & 3 support for raw_input
try:
input = raw_input
except NameError:
pass
while True:
line = input('\nEnter a new sentence for inference: \n')
xbuf = prep_data(line, 'text', max_len, vocab)
def on_epoch_end(self, callback_data, model, epoch):
if self.history > 1:
self.save_history(epoch, model)
else:
save_obj(model.serialize(keep_states=True), self.save_path)
def test_model_serialize(backend):
(X_train, y_train), (X_test, y_test), nclass = load_mnist()
train_set = DataIterator([X_train, X_train],
y_train,
nclass=nclass,
lshape=(1, 28, 28))
init_norm = Gaussian(loc=0.0, scale=0.01)
# initialize model
path1 = [
Conv((5, 5, 16),
init=init_norm,
bias=Constant(0),
activation=Rectlin()),
Pooling(2),
Affine(nout=20, init=init_norm, bias=init_norm, activation=Rectlin())
]
path2 = [
Dropout(keep=0.5),
Affine(nout=20, init=init_norm, bias=init_norm, activation=Rectlin())
]
layers = [
MergeConcat([path1, path2]),
Affine(nout=20, init=init_norm, bias=init_norm, activation=Rectlin()),
BatchNorm(),
Affine(nout=10, init=init_norm, activation=Logistic(shortcut=True))
]
tmp_save = 'test_model_serialize_tmp_save.pickle'
mlp = Model(layers=layers)
mlp.optimizer = GradientDescentMomentum(learning_rate=0.1,
momentum_coef=0.9)
mlp.cost = GeneralizedCost(costfunc=CrossEntropyBinary())
n_test = 3
num_epochs = 3
# Train model for num_epochs and n_test batches
for epoch in range(num_epochs):
for i, (x, t) in enumerate(train_set):
x = mlp.fprop(x)
delta = mlp.cost.get_errors(x, t)
mlp.bprop(delta)
mlp.optimizer.optimize(mlp.layers_to_optimize, epoch=epoch)
if i > n_test:
break
# Get expected outputs of n_test batches and states of all layers
outputs_exp = []
pdicts_exp = [l.get_params_serialize() for l in mlp.layers_to_optimize]
for i, (x, t) in enumerate(train_set):
outputs_exp.append(mlp.fprop(x, inference=True))
if i > n_test:
break
# Serialize model
save_obj(mlp.serialize(keep_states=True), tmp_save)
# Load model
mlp = Model(layers=layers)
mlp.load_weights(tmp_save)
outputs = []
pdicts = [l.get_params_serialize() for l in mlp.layers_to_optimize]
for i, (x, t) in enumerate(train_set):
outputs.append(mlp.fprop(x, inference=True))
if i > n_test:
break
# Check outputs, states, and params are the same
for output, output_exp in zip(outputs, outputs_exp):
assert np.allclose(output.get(), output_exp.get())
for pd, pd_exp in zip(pdicts, pdicts_exp):
for s, s_e in zip(pd['states'], pd_exp['states']):
if isinstance(s, list): # this is the batch norm case
for _s, _s_e in zip(s, s_e):
assert np.allclose(_s, _s_e)
else:
assert np.allclose(s, s_e)
for p, p_e in zip(pd['params'], pd_exp['params']):
if isinstance(p, list): # this is the batch norm case
for _p, _p_e in zip(p, p_e):
assert np.allclose(_p, _p_e)
else:
assert np.allclose(p, p_e)
os.remove(tmp_save)
(im_shape, im_scale, gt_boxes, gt_classes,
num_gt_boxes, difficult) = valid_set.get_metadata_buffers()
num_gt_boxes = int(num_gt_boxes.get())
im_scale = float(im_scale.get())
# retrieve region proposals generated by the model
(proposals, num_proposals) = proposalLayer.get_proposals()
# convert outputs to bounding boxes
boxes = faster_rcnn.get_bboxes(outputs, proposals, num_proposals, num_classes,
im_shape.get(), im_scale, max_per_image, thresh, nms_thresh)
all_boxes[mb_idx] = boxes
# retrieve gt boxes
# we add a extra column to track detections during the AP calculation
detected = np.array([False] * num_gt_boxes)
gt_boxes = np.hstack([gt_boxes.get()[:num_gt_boxes] / im_scale,
gt_classes.get()[:num_gt_boxes],
difficult.get()[:num_gt_boxes], detected[:, np.newaxis]])
all_gt_boxes[mb_idx] = gt_boxes
neon_logger.display('Evaluating detections')
avg_precision = voc_eval(all_boxes, all_gt_boxes, valid_set.CLASSES, use_07_metric=True)
if args.output is not None:
neon_logger.display('Saving inference results to {}'.format(args.output))
save_obj([all_boxes, avg_precision], args.output)
frcn_tree_cost,
],
weights=[1, 1, 1])
# setup optimizer
schedule_w = StepSchedule(step_config=[10], change=[0.001 / 10])
schedule_b = StepSchedule(step_config=[10], change=[0.002 / 10])
opt_w = GradientDescentMomentum(0.001, 0.9, wdecay=0.0005, schedule=schedule_w)
opt_b = GradientDescentMomentum(0.002, 0.9, wdecay=0.0005, schedule=schedule_b)
opt_skip = GradientDescentMomentum(0.0, 0.0)
optimizer = MultiOptimizer({'default': opt_w, 'Bias': opt_b,
'skip': opt_skip, 'skip_bias': opt_skip})
# if training a new model, seed the image model conv layers with pre-trained weights
# otherwise, just load the model file
if args.model_file is None:
util.load_vgg_all_weights(model, cache_dir)
callbacks = Callbacks(model, eval_set=train_set, **args.callback_args)
model.fit(train_set, optimizer=optimizer, cost=cost, num_epochs=args.epochs, callbacks=callbacks)
# Scale the bbox regression branch linear layer weights before saving the model
model = util.scale_bbreg_weights(model, [0.0, 0.0, 0.0, 0.0],
[0.1, 0.1, 0.2, 0.2], train_set.num_classes)
if args.save_path is not None:
save_obj(model.serialize(keep_states=True), args.save_path)
def on_epoch_end(self, epoch):
if self.history > 1:
self.save_history(epoch)
else:
save_obj(self.model.serialize(keep_states=True), self.save_path)
def on_epoch_end(self, epoch):
if self.history > 1:
self.save_history(epoch)
else:
save_obj(self.model.serialize(keep_states=True), self.save_path)
def load_data(path, file_ext=['txt'], valid_split=None, vocab_file_name=None,
max_vocab_size=None, max_len_w=None, output_path=None, subset_pct=100):
"""
Given a path where data are saved, look for the ones with the right extensions
If a split factor is given, it will split all the files into training and valid
set. Then build vocabulary from the training and validation sets.
Arguments:
path: which directory to look for all the documents
file_ext: what extension of the files to look for
valid_split: to split the data into train/valid set. If None, no split
vocab_file_name: optional file name. If None, the script will decide a name
given path and split
max_vocab_size: maximum number of words to use in vocabulary (by most frequent)
max_len_w: maximum length of sentences in words
output_path: path used to save preprocessed data and resuts
subset_pct: subset of dataset to load into H5 file (percentage)
Returns:
The function saves 2 files:
h5 file with preprocessed data
vocabulary file with: vocab, reverse_vocab, word_count
"""
file_names = get_file_list(path, file_ext)
file_str = get_file_str(path, len(file_names), labelled=False,
valid_split=valid_split, subset_pct=subset_pct)
# create output dir if needed
if not os.path.isdir(output_path):
os.makedirs(output_path)
# file name to store the vocabulary
if vocab_file_name is None:
vocab_file_name = file_str + '.vocab'
vocab_file_name = os.path.join(output_path, vocab_file_name)
# If max sizes arent set, assume no limit
if not max_len_w:
max_len_w = sys.maxsize
if not max_vocab_size:
max_vocab_size = sys.maxsize
# file name to store the pre-processed train/valid dataset
h5_file_name = os.path.join(output_path, file_str + '.h5')
if os.path.exists(h5_file_name) and os.path.exists(vocab_file_name):
neon_logger.display("dataset files {} and vocabulary file {} already exist. "
"will use cached data. ".format(h5_file_name, vocab_file_name))
return h5_file_name, vocab_file_name
# split into training/valid set
if valid_split is not None:
if 'json' in file_ext:
# Split based on number of files
train_split = int(np.ceil(len(file_names) * (1 - valid_split)))
train_files = file_names[:train_split]
valid_files = file_names[train_split:]
train_sent = load_json_sent(train_files, subset_pct)
valid_sent = load_json_sent(valid_files, subset_pct)
all_sent = train_sent + valid_sent
elif 'txt' in file_ext:
# Split based on number of lines (since only 2 files)
all_sent = load_txt_sent(file_names, subset_pct)
train_split = int(np.ceil(len(all_sent) * (1 - valid_split)))
train_sent = all_sent[:train_split]
valid_sent = all_sent[train_split:]
else:
neon_logger.display("Unsure how to load file_ext {}, please use 'json' or 'txt'."
.format(file_ext))
else:
train_files = file_names
if 'json' in file_ext:
train_sent = load_json_sent(train_files, subset_pct)
elif 'txt' in file_ext:
train_sent = load_txt_sent(train_files, subset_pct)
else:
neon_logger.display("Unsure how to load file_ext {}, please use 'json' or 'txt'."
.format(file_ext))
all_sent = train_sent
if os.path.exists(vocab_file_name):
neon_logger.display("open existing vocab file: {}".format(vocab_file_name))
vocab, rev_vocab, word_count = load_obj(vocab_file_name)
else:
neon_logger.display("Building vocab file")
# build vocab
word_count = defaultdict(int)
for sent in all_sent:
sent_words = tokenize(sent)
if len(sent_words) > max_len_w or len(sent_words) == 0:
continue
for word in sent_words:
word_count[word] += 1
# sort the word_count , re-assign ids by its frequency. Useful for downstream tasks
# only done for train vocab
vocab_sorted = sorted(word_count.items(), key=lambda kv: kv[1], reverse=True)
vocab = OrderedDict()
# get word count as array in same ordering as vocab (but with maximum length)
word_count_ = np.zeros((len(word_count), ), dtype=np.int64)
for i, t in enumerate(list(zip(*vocab_sorted))[0][:max_vocab_size]):
word_count_[i] = word_count[t]
vocab[t] = i
word_count = word_count_
# generate the reverse vocab
rev_vocab = dict((wrd_id, wrd) for wrd, wrd_id in vocab.items())
neon_logger.display("vocabulary from {} is saved into {}".format(path, vocab_file_name))
save_obj((vocab, rev_vocab, word_count), vocab_file_name)
vocab_size = len(vocab)
neon_logger.display("\nVocab size from the dataset is: {}".format(vocab_size))
neon_logger.display("\nProcessing and saving training data into {}".format(h5_file_name))
# now process and save the train/valid data
h5f = h5py.File(h5_file_name, 'w', libver='latest')
shape, maxshape = (len(train_sent),), (None)
dt = np.dtype([('text', h5py.special_dtype(vlen=str)),
('num_words', np.uint16)])
report_text_train = h5f.create_dataset('report_train', shape=shape,
maxshape=maxshape, dtype=dt,
compression='gzip')
report_train = h5f.create_dataset('train', shape=shape, maxshape=maxshape,
dtype=h5py.special_dtype(vlen=np.int32),
compression='gzip')
# map text to integers
wdata = np.zeros((1, ), dtype=dt)
ntrain = 0
for sent in train_sent:
text_int = [-1 if t not in vocab else vocab[t] for t in tokenize(sent)]
# enforce maximum sentence length
if len(text_int) > max_len_w or len(text_int) == 0:
continue
report_train[ntrain] = text_int
wdata['text'] = clean_string(sent)
wdata['num_words'] = len(text_int)
report_text_train[ntrain] = wdata
ntrain += 1
report_train.attrs['nsample'] = ntrain
report_train.attrs['vocab_size'] = vocab_size
report_text_train.attrs['nsample'] = ntrain
report_text_train.attrs['vocab_size'] = vocab_size
if valid_split:
neon_logger.display("\nProcessing and saving validation data into {}".format(h5_file_name))
shape = (len(valid_sent),)
report_text_valid = h5f.create_dataset('report_valid', shape=shape,
maxshape=maxshape, dtype=dt,
compression='gzip')
report_valid = h5f.create_dataset('valid', shape=shape, maxshape=maxshape,
dtype=h5py.special_dtype(vlen=np.int32),
compression='gzip')
nvalid = 0
for sent in valid_sent:
text_int = [-1 if t not in vocab else vocab[t] for t in tokenize(sent)]
# enforce maximum sentence length
if len(text_int) > max_len_w or len(text_int) == 0:
continue
report_valid[nvalid] = text_int
wdata['text'] = clean_string(sent)
wdata['num_words'] = len(text_int)
report_text_valid[nvalid] = wdata
nvalid += 1
report_valid.attrs['nsample'] = nvalid
report_valid.attrs['vocab_size'] = vocab_size
report_text_valid.attrs['nsample'] = nvalid
report_text_valid.attrs['vocab_size'] = vocab_size
h5f.close()
return h5_file_name, vocab_file_name
def save_weights(self, save_path):
save_obj(self.model.serialize(keep_states=True), save_path)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("cache_file", help="path to data cache file")
args = parser.parse_args()
cache_file = args.cache_file
# check for RW access to file
assert os.path.exists(cache_file), "file does not exist %s" % cache_file
if not os.access(os.path.abspath(cache_file), os.R_OK | os.W_OK):
raise IOError("Need to add read and/or write permissions on file %s" % cache_file)
dc = load_obj(cache_file)
if "global_mean" not in dc or "img_size" not in dc:
raise ValueError("data cache file missing global_mean key")
sz = dc["img_size"]
gm = dc["global_mean"]
if len(gm.shape) != 2 or (gm.shape[0] != sz * sz * 3 or gm.shape[1] != 1):
raise ValueError("global mean shape {} does not match format expected".format(gm.shape))
# Collapse the full tensor mean into channel means and correct the order (RGB <-> BGR)
dc["global_mean"] = np.mean(gm.reshape(3, -1), axis=1).reshape(3, 1)[::-1]
save_obj(dc, cache_file)
neon_logger.display("%s updated to new format" % cache_file)
scales = [112, 128, 160, 240]
for scale in scales:
print scale
layers = []
layers += [Conv(**conv_params(7, 32, 2))]
for nfm, stride in zip(nfms, strides):
layers.append(module_factory(nfm, stride))
layers.append(Pooling(7, op='avg'))
layers.append(
Conv(fshape=(1, 1, 100), init=Kaiming(local=True), batch_norm=True))
layers.append(Pooling(fshape='all', op='avg'))
layers.append(Activation(Softmax()))
model = Model(layers=layers)
test = ImageLoader(set_name='validation',
shuffle=False,
do_transforms=False,
inner_size=scale,
scale_range=scale,
repo_dir=args.data_dir)
model.load_params("/home/users/hunter/bigfeat_dropout.pkl")
softmaxes = model.get_outputs(test)
from neon.util.persist import save_obj
save_obj(softmaxes, "bigfeat_dropout_SM_{}.pkl".format(scale))
def __init__(self, image_set, year, path='.', add_flipped=False,
overlap_thre=None, output_type=0, n_mb=None, img_per_batch=None,
rois_per_img=None, rois_random_sample=True, shuffle=False):
self.isRoiDB = True
self.batch_index = 0
self.year = year
self.image_set = image_set
self.add_flipped = add_flipped
self.overlap_thre = overlap_thre if overlap_thre else FRCN_IOU_THRE
self.output_type = output_type
# how many ROIs per image
self.rois_per_image = rois_per_img if rois_per_img else FRCN_ROI_PER_IMAGE
self.img_per_batch = img_per_batch if img_per_batch else FRCN_IMG_PER_BATCH
self.fg_rois_per_image = FRCN_FG_FRAC * self.rois_per_image
self.bg_rois_per_image = self.rois_per_image - self.fg_rois_per_image
self.rois_per_batch = self.rois_per_image * self.img_per_batch
self.rois_random_sample = rois_random_sample
self.shuffle = shuffle
self.cache_file_name = 'voc_{}_{}_flip_{}_ovlp_{}.pkl'.format(self.year,
self.image_set,
self.add_flipped,
self.overlap_thre)
print 'prepare PASCAL VOC {} from year {}: add flipped image {} and overlap threshold {}'\
.format(self.image_set, self.year, self.add_flipped, self.overlap_thre)
# PASCAL class to index
self.num_classes = PASCAL_VOC_NUM_CLASSES
self._class_to_index = dict(
zip(PASCAL_VOC_CLASSES, xrange(self.num_classes)))
# load the voc dataset
self.voc_root = self.load_voc(image_set, year, path)
self.cache_file = os.path.join(self.voc_root, self.cache_file_name)
# load the precomputed ss results from voc data, it includes both 2007 and 2012 data
self.ss_path = self.load_voc('ss', None, path)
# VOC paths and infos
self.image_index_file = os.path.join(self.voc_root, 'ImageSets', 'Main',
self.image_set + '.txt')
self.image_path = os.path.join(self.voc_root, 'JPEGImages')
self._image_file_ext = '.jpg'
self.annotation_path = os.path.join(self.voc_root, 'Annotations')
self._annotation_file_ext = '.xml'
self._annotation_obj_tag = 'object'
self._annotation_class_tag = 'name'
self._annotation_xmin_tag = 'xmin'
self._annotation_xmax_tag = 'xmax'
self._annotation_ymin_tag = 'ymin'
self._annotation_ymax_tag = 'ymax'
self._selective_search_ext = '.pkl'
self.selective_search_file = os.path.join(
self.ss_path,
'_'.join(['voc', year, self.image_set, 'selectivesearch.pkl']))
self._bb_xmin_idx = 0
self._bb_ymin_idx = 1
self._bb_xmax_idx = 2
self._bb_ymax_idx = 3
# self.rois_per_batch is 128 (2*64) ROIs
# But the image path batch size is self.img_per_batch
# need to control the batch size here
print "Backend batchsize is changed to be image_per_batch from PASCAL_VOC dataset"
self.be.bsz = self.img_per_batch
# backend tensor to push the data
self.image_shape = (3, FRCN_MAX_SCALE, FRCN_MAX_SCALE)
self.img_np = np.zeros(
(3, FRCN_MAX_SCALE, FRCN_MAX_SCALE, self.be.bsz), dtype=np.float32)
self.dev_X_img = self.be.iobuf(self.image_shape, dtype=np.float32)
self.dev_X_img_chw = self.dev_X_img.reshape(
3, FRCN_MAX_SCALE, FRCN_MAX_SCALE, self.be.bsz)
# for rois, features are 4 + 1 (idx within the batch)
self.dev_X_rois = self.be.zeros((self.rois_per_batch, 5))
self.dev_y_labels_flat = self.be.zeros(
(1, self.rois_per_batch), dtype=np.int32)
self.dev_y_labels = self.be.zeros(
(self.num_classes, self.rois_per_batch), dtype=np.int32)
self.dev_y_bbtargets = self.be.zeros(
(self.num_classes * 4, self.rois_per_batch))
self.dev_y_bbmask = self.be.zeros(
(self.num_classes * 4, self.rois_per_batch))
# the shape will indicate the shape for 1st path (ImageNet model), and
# 2nd path (ROIs)
self.shape = [self.image_shape, self.num_classes * 4]
# Need to do the following:
# 1. load the image index list
# 2. for each image, load the ground truth from pascal annotation
# 3. load the selective search ROIs (this step needs gt ROIs)
# 4.1. merge the ROIs
# 4.2. may have to add the flipped images for training
# 4.3. add the fields for max overlap and max overlapped classes
# 4.4. add the bounding box targets for regression
# 5. during minibatch feeding:
# - rescale images
# - rescale ROIs
# - random select foreground ROIs (bigger ones)
# - random select background ROIS (smaller ones)
# - clamp bg ROI labels (to be 0)
# - convert ROIs into the regression target (ROIs, 4*21)
# 1.
assert os.path.exists(self.image_index_file), \
'Image index file does not exist: {}'.format(self.image_index_file)
with open(self.image_index_file) as f:
self.image_index = [x.strip() for x in f.readlines()]
# self.image_index = image_index * 2 if self.add_flipped else image_index
self.num_images = len(self.image_index)
self.num_image_entries = self.num_images * \
2 if self.add_flipped else self.num_images
self.ndata = self.num_image_entries * self.rois_per_image
self.nbatches = self.num_image_entries/self.img_per_batch
if n_mb is not None:
self.nbatches = n_mb
if os.path.exists(self.cache_file):
self.roi_db = load_obj(self.cache_file)
print 'ROI dataset loaded from file {}'.format(self.cache_file)
else:
# 2.
self.roi_gt = self.load_pascal_roi_groundtruth()
# 3.
self.roi_ss = self.load_pascal_roi_selectivesearch()
# 4.
self.roi_db = self.combine_gt_ss_roi()
save_obj(self.roi_db, self.cache_file)
print 'wrote ROI dataset to {}'.format(self.cache_file)
