def on_epoch_end(self, callback_data, model, epoch): _eil = self._get_cached_epoch_loss(callback_data, model, epoch, 'loss') if _eil: if _eil['cost'] < self.best_cost or self.best_cost is None: # TODO: switch this to a general seralization op save_obj(model.serialize(keep_states=True), self.best_path) self.best_cost = _eil['cost']
def save_history(self, epoch, model): # if history > 1, this function will save the last N checkpoints # where N is equal to self.history. The files will have the form # of save_path with the epoch added to the filename before the ext if len(self.checkpoint_files) > self.history: # remove oldest checkpoint file when max count have been saved fn = self.checkpoint_files.popleft() try: os.remove(fn) logger.info("removed old checkpoint %s" % fn) except OSError: logger.warn("Could not delete old checkpoint file %s" % fn) path_split = os.path.splitext(self.save_path) save_path = "%s_%d%s" % (path_split[0], epoch, path_split[1]) # add the current file to the deque self.checkpoint_files.append(save_path) save_obj(model.serialize(keep_states=True), save_path) # maintain a symlink pointing to the latest model params try: if os.path.islink(self.save_path): os.remove(self.save_path) os.symlink(os.path.split(save_path)[-1], self.save_path) except OSError: logger.warn("Could not create latest model symlink %s -> %s" % (self.save_path, save_path))
def save_history(self, epoch): # if history > 1, this function will save the last N checkpoints # where N is equal to self.history. The files will have the form # of save_path with the epoch added to the filename before the ext if len(self.checkpoint_files) > self.history: # remove oldest checkpoint file when max count have been saved fn = self.checkpoint_files.popleft() try: os.remove(fn) logger.info('removed old checkpoint %s' % fn) except OSError: logger.warn('Could not delete old checkpoint file %s' % fn) path_split = os.path.splitext(self.save_path) save_path = '%s_%d%s' % (path_split[0], epoch, path_split[1]) # add the current file to the deque self.checkpoint_files.append(save_path) save_obj(self.model.serialize(keep_states=True), save_path) # maintain a symlink pointing to the latest model params try: if os.path.islink(self.save_path): os.remove(self.save_path) os.symlink(os.path.split(save_path)[-1], self.save_path) except OSError: logger.warn('Could not create latest model symlink %s -> %s' % (self.save_path, save_path))
def on_epoch_end(self, callback_data, model, epoch): _eil = self._get_cached_epoch_loss(callback_data, model, epoch, "loss") if _eil: if _eil["cost"] < self.best_cost or self.best_cost is None: # TODO: switch this to a general seralization op save_obj(model.serialize(keep_states=True), self.best_path) self.best_cost = _eil["cost"]
def on_epoch_end(self, epoch): _eil = self._get_cached_epoch_loss(epoch, 'loss') if _eil: if _eil['cost'] < self.best_cost or self.best_cost is None: save_obj(self.model.serialize(keep_states=True), self.best_path) self.best_cost = _eil['cost']
def serialize(self, fn=None, keep_states=True): """ Creates a dictionary storing the layer parameters and epochs complete. Arguments: fn (str): file to save pkl formatted model dictionary keep_states (bool): Whether to save optimizer states. Returns: dict: Model data including layer parameters and epochs complete. """ # get the model dict with the weights pdict = self.get_description(get_weights=True, keep_states=keep_states) pdict['epoch_index'] = self.epoch_index + 1 if self.initialized: if not hasattr(self.layers, 'decoder'): pdict['train_input_shape'] = self.layers.in_shape else: # serialize shapes both for encoder and decoder pdict['train_input_shape'] = (self.layers.encoder.in_shape + self.layers.decoder.in_shape) if fn is not None: save_obj(pdict, fn) return return pdict
def save_params(self, param_path, keep_states=True): """ Serializes and saves model parameters to the path specified. Arguments: param_path (str): File to write serialized parameter dict to. keep_states (bool): Whether to save optimizer states too. Defaults to True. """ save_obj(self.serialize(keep_states), param_path)
def on_epoch_end(self, epoch): if 'cost/validation' in self.callback_data: val_freq = self.callback_data['cost/validation'].attrs['epoch_freq'] if (epoch + 1) % val_freq == 0: validation_cost = self.callback_data['cost/validation'][epoch/val_freq] if validation_cost < self.best_cost or self.best_cost is None: save_obj(self.model.serialize(keep_states=True), self.best_path) self.best_cost = validation_cost
def on_epoch_end(self, epoch): if 'cost/validation' in self.callback_data: val_freq = self.callback_data['cost/validation'].attrs[ 'epoch_freq'] if (epoch + 1) % val_freq == 0: validation_cost = self.callback_data['cost/validation'][ epoch / val_freq] if validation_cost < self.best_cost or self.best_cost is None: save_obj(self.model.serialize(keep_states=True), self.best_path) self.best_cost = validation_cost
def save_meta(self): save_obj({'ntrain': self.ntrain, 'nval': self.nval, 'train_start': self.train_start, 'val_start': self.val_start, 'macro_size': self.macro_size, 'batch_prefix': self.batch_prefix, 'global_mean': self.global_mean, 'label_dict': self.label_dict, 'label_names': self.label_names, 'val_nrec': self.val_nrec, 'train_nrec': self.train_nrec, 'img_size': self.target_size, 'nclass': self.nclass}, self.meta_file)
def on_sigint_catch(self, epoch, minibatch): """ Callback to handle SIGINT events Arguments: epoch (int): index of current epoch minibatch (int): index of minibatch that is ending """ # restore the orignal handler signal.signal(signal.SIGINT, signal.SIG_DFL) # save the model if self.save_path is not None: save_obj(self.model().serialize(keep_states=True), self.save_path) raise KeyboardInterrupt('Checkpoint file saved to {0}'.format(self.save_path)) else: raise KeyboardInterrupt
def on_sigint_catch(self, epoch, minibatch): """ Callback to handle SIGINT events Arguments: epoch (int): index of current epoch minibatch (int): index of minibatch that is ending """ # restore the orignal handler signal.signal(signal.SIGINT, signal.SIG_DFL) # save the model if self.save_path is not None: save_obj(self.model().serialize(keep_states=True), self.save_path) raise KeyboardInterrupt("Checkpoint file saved to {0}".format(self.save_path)) else: raise KeyboardInterrupt
def save_meta(self): save_obj( { "ntrain": self.ntrain, "nval": self.nval, "train_start": self.train_start, "val_start": self.val_start, "macro_size": self.macro_size, "batch_prefix": self.batch_prefix, "global_mean": self.global_mean, "label_dict": self.label_dict, "label_names": self.label_names, "val_nrec": self.val_nrec, "train_nrec": self.train_nrec, "img_size": self.target_size, "nclass": self.nclass, }, self.meta_file, )
def save_history(self, epoch): # if history > 1, this function will save the last N checkpoints # where N is equal to self.history. The files will have the form # of save_path with the epoch added to the filename before the ext if len(self.checkpoint_files) > self.history: # remove oldest checkpoint file when max count have been saved fn = self.checkpoint_files.popleft() try: os.remove(fn) logger.info('removed old checkpoint %s' % fn) except OSError: logger.warn('Could not delete old checkpoint file %s' % fn) path_split = os.path.splitext(self.save_path) save_path = '%s_%d%s' % (path_split[0], epoch, path_split[1]) # add the current file to the deque self.checkpoint_files.append(save_path) save_obj(self.model.serialize(keep_states=True), save_path)
def serialize(self, fn=None, keep_states=True): """ Creates a dictionary storing the layer parameters and epochs complete. Arguments: fn (str): file to save pkl formatted model dictionary keep_states (bool): Whether to save optimizer states. Returns: dict: Model data including layer parameters and epochs complete. """ # get the model dict with the weights pdict = self.get_description(get_weights=True, keep_states=keep_states) pdict['epoch_index'] = self.epoch_index + 1 if fn is not None: save_obj(pdict, fn) return return pdict
def get_w2v_vocab(fname, max_vocab_size, cache=True): """ Get ordered dict of vocab from google word2vec """ if cache: cache_fname = fname.split('.')[0] + ".vocab" if os.path.isfile(cache_fname): vocab, vocab_size = load_obj(cache_fname) neon_logger.display( "Word2Vec vocab cached, size is: {}".format(vocab_size)) return vocab, vocab_size with open(fname, 'rb') as f: header = f.readline() vocab_size, embed_dim = map(int, header.split()) binary_len = np.dtype('float32').itemsize * embed_dim neon_logger.display("Word2Vec vocab size is: {}".format(vocab_size)) vocab_size = min(max_vocab_size, vocab_size) neon_logger.display("Reducing vocab size to: {}".format(vocab_size)) vocab = OrderedDict() for i, line in enumerate(range(vocab_size)): word = [] while True: ch = f.read(1) if ch == b' ': word = (b''.join(word)).decode('utf-8') break if ch != b'\n': word.append(ch) f.read(binary_len) vocab[word] = i if cache: save_obj((vocab, vocab_size), cache_fname) return vocab, vocab_size
def get_w2v_vocab(fname, max_vocab_size, cache=True): """ Get ordered dict of vocab from google word2vec """ if cache: cache_fname = fname.split('.')[0] + ".vocab" if os.path.isfile(cache_fname): vocab, vocab_size = load_obj(cache_fname) neon_logger.display("Word2Vec vocab cached, size is: {}".format(vocab_size)) return vocab, vocab_size with open(fname, 'rb') as f: header = f.readline() vocab_size, embed_dim = map(int, header.split()) binary_len = np.dtype('float32').itemsize * embed_dim neon_logger.display("Word2Vec vocab size is: {}".format(vocab_size)) vocab_size = min(max_vocab_size, vocab_size) neon_logger.display("Reducing vocab size to: {}".format(vocab_size)) vocab = OrderedDict() for i, line in enumerate(range(vocab_size)): word = [] while True: ch = f.read(1) if ch == b' ': word = (b''.join(word)).decode('utf-8') break if ch != b'\n': word.append(ch) f.read(binary_len) vocab[word] = i if cache: save_obj((vocab, vocab_size), cache_fname) return vocab, vocab_size
def __init__(self, path='.', n_mb=None, img_per_batch=None, conv_size=None, rpn_rois_per_img=None, frcn_rois_per_img=None, add_flipped=False, shuffle=False, deterministic=False, rebuild_cache=False, subset_pct=100, mock_db=None): self.batch_index = 0 self.path = path self.mock_db = mock_db # how many ROIs per image self.rois_per_img = rpn_rois_per_img if rpn_rois_per_img else self.RPN_ROI_PER_IMAGE self.img_per_batch = img_per_batch if img_per_batch else self.IMG_PER_BATCH self.rois_per_batch = self.rois_per_img * self.img_per_batch # how many ROIs to use to train frcnn self.frcn_rois_per_img = frcn_rois_per_img if frcn_rois_per_img \ else self.FRCNN_ROI_PER_IMAGE assert self.img_per_batch == 1, "Only a minibatch of 1 is supported." self.num_classes = len(self.CLASSES) self._class_to_index = dict( list(zip(self.CLASSES, list(range(self.num_classes))))) # shape of the final conv layer if conv_size: self._conv_size = conv_size else: self._conv_size = int(np.floor(self.MAX_SIZE * self.SCALE)) self._feat_stride = 1 / float(self.SCALE) self._num_scales = len(self.SCALES) * len(self.RATIOS) self._total_anchors = self._conv_size * self._conv_size * self._num_scales self.shuffle = shuffle self.deterministic = deterministic self.add_flipped = add_flipped # load the configure the dataset paths self.config = self.load_data() # annotation metadata self._annotation_file_ext = '.xml' self._annotation_obj_tag = 'object' self._annotation_class_tag = 'name' self._annotation_xmin_tag = 'xmin' self._annotation_xmax_tag = 'xmax' self._annotation_ymin_tag = 'ymin' self._annotation_ymax_tag = 'ymax' # self.rois_per_batch is 128 (2*64) ROIs # But the image path batch size is self.img_per_batch # need to control the batch size here assert self.img_per_batch is 1, "Only a batch size of 1 image is supported" neon_logger.display("Backend batchsize is changed to be {} " "from Object Localization dataset".format( self.img_per_batch)) self.be.bsz = self.img_per_batch # 0. allocate buffers self.allocate() if not self.mock_db: # 1. read image index file assert os.path.exists(self.config['image_path']), \ 'Image index file does not exist: {}'.format(self.config['image_path']) with open(self.config['index_path']) as f: self.image_index = [x.strip() for x in f.readlines()] num_images = len(self.image_index) self.num_image_entries = num_images * 2 if self.add_flipped else num_images self.ndata = self.num_image_entries * self.rois_per_img else: self.num_image_entries = 1 self.ndata = self.num_image_entries * self.rois_per_img assert (subset_pct > 0 and subset_pct <= 100), ('subset_pct must be between 0 and 100') if n_mb is not None: self.nbatches = n_mb else: self.nbatches = int(self.num_image_entries / self.img_per_batch * subset_pct / 100) self.cache_file = self.config['cache_path'] if os.path.exists( self.cache_file) and not rebuild_cache and not self.mock_db: self.roi_db = load_obj(self.cache_file) neon_logger.display('ROI dataset loaded from file {}'.format( self.cache_file)) elif not self.mock_db: # 2. read object Annotations (XML) roi_db = self.load_roi_groundtruth() if (self.add_flipped): roi_db = self.add_flipped_db(roi_db) # 3. construct acnhor targets self.roi_db = self.add_anchors(roi_db) if NORMALIZE_BBOX_TARGETS: # 4. normalize bbox targets by class self.roi_db = self.normalize_bbox_targets(self.roi_db) save_obj(self.roi_db, self.cache_file) neon_logger.display('wrote ROI dataset to {}'.format( self.cache_file)) else: assert self.mock_db is not None roi_db = [self.mock_db] self.roi_db = self.add_anchors(roi_db) # 4. map anchors back to full canvas. # This is neccessary because the network outputs reflect the full canvas. # We cache the files in the unmapped state (above) to save memory. self.roi_db = unmap(self.roi_db)
lr_sched = PolySchedule(total_epochs=10, power=0.5) opt_gdm = GradientDescentMomentum(0.01, 0.9, wdecay=0.0002, schedule=lr_sched) opt_biases = GradientDescentMomentum(0.02, 0.9, schedule=lr_sched) opt = MultiOptimizer({'default': opt_gdm, 'Bias': opt_biases}) if not args.resume: # fit the model for 3 epochs model.fit(train, optimizer=opt, num_epochs=3, cost=cost, callbacks=callbacks) train.reset() # get 1 image for im, l in train: break train.exit_batch_provider() save_obj((im.get(), l.get()), 'im1.pkl') im_save = im.get().copy() if args.resume: (im2, l2) = load_obj('im1.pkl') im.set(im2) l.set(l2) # run fprop and bprop on this minibatch save the results out_fprop = model.fprop(im) out_fprop_save = [x.get() for x in out_fprop] im.set(im_save) out_fprop = model.fprop(im) out_fprop_save2 = [x.get() for x in out_fprop] for x, y in zip(out_fprop_save, out_fprop_save2): assert np.max(np.abs(x - y)) == 0.0, '2 fprop iterations do not match'
from neon.util.argparser import NeonArgparser from neon.layers import Pooling from neon.models import Model from neon.data import ImageLoader from neon.util.persist import save_obj, load_obj # parse the command line arguments (generates the backend) parser = NeonArgparser(__doc__) args = parser.parse_args() scales = [112, 128, 160, 240] for scale in scales: print scale test = ImageLoader(set_name='validation', shuffle=False, do_transforms=False, inner_size=scale, scale_range=scale, repo_dir=args.data_dir) model_desc = load_obj(args.model_file) model_desc['model']['config']['layers'].insert(-1, Pooling('all', op='avg').get_description()) model = Model(model_desc, test, inference=True) softmaxes = model.get_outputs(test) save_obj(softmaxes, "bigfeat_dropout_SM_{}.pkl".format(scale))
if args.model_file is None: load_vgg_weights(model, args.data_dir) cost = Multicost(costs=[GeneralizedCostMask(costfunc=CrossEntropyMulti()), GeneralizedCostMask(costfunc=SmoothL1Loss())], weights=[1, 1]) callbacks = Callbacks(model, eval_set=test_set, **args.callback_args) model.fit(train_set, optimizer=optimizer, num_epochs=num_epochs, cost=cost, callbacks=callbacks) # Fast R-CNN model requires scale the bbox regression branch linear layer weights # before saving the model model = scale_bbreg_weights( model, train_set.bbtarget_means, train_set.bbtarget_stds) save_obj(model.serialize(keep_states=True), args.save_path) neon_logger.display('running eval...') metric_train = model.eval(train_set, metric=ObjectDetection()) neon_logger.display( 'Train: label accuracy - {}%, object detection logloss - {}'.format(metric_train[0] * 100, metric_train[1])) metric_test = model.eval(test_set, metric=ObjectDetection()) neon_logger.display( 'Test: label accuracy - {}%, object detection logloss - {}'.format(metric_test[0] * 100, metric_test[1]))
h5train_text = h5f['report_train'] if os.path.exists(args.vector_name): neon_logger.display("cached encoded vectors exists: {}".format(args.vector_name)) (train_vec, sentences) = load_obj(args.vector_name) model.initialize(dataset=(max_len, 1)) else: neon_logger.display("Encoding the entire training set....") # encode all the training sentences model.initialize(dataset=(max_len, 1)) train_set = SentenceEncode(h5train, h5train_text, h5train.attrs['nsample'], vocab_size_layer, max_len=max_len, index_from=index_from) sentences = h5train_text[:train_set.ndata].reshape(-1, 1) train_vec = model.get_outputs(train_set) neon_logger.display("Encoding complete. Saving to {}".format(args.vector_name)) save_obj((train_vec, sentences), args.vector_name) s2v = SentenceVector(train_vec, sentences) xdev = be.zeros((max_len, 1), dtype=np.int32) # bsz is 1, feature size xbuf = np.zeros((max_len, 1), dtype=np.int32) # Add python 2 & 3 support for raw_input try: input = raw_input except NameError: pass while True: line = input('\nEnter a new sentence for inference: \n') xbuf = prep_data(line, 'text', max_len, vocab)
def save_weights(self, save_path): save_obj(self.model.serialize(keep_states = True), save_path)
rng_seed=rng_seed, device_id=args.device_id, default_dtype=args.datatype, ) (X_train, y_train), (X_test, y_test), nout = load_data(DATASET_NAME) if VALIDATION: (X_train, y_train), (X_valid, y_valid) = split_train_set(X_train, y_train) model, opt, cost = get_custom_vgg(nout=nout) train_set = DataIterator(X_train, y_train, nclass=nout, lshape=(3, 32, 32)) test_set = DataIterator(X_test, y_test, nclass=nout, lshape=(3, 32, 32)) callbacks = Callbacks(model, train_set, args, eval_set=test_set) if VALIDATION: valid_set = DataIterator(X_valid, y_valid, nclass=nout, lshape=(3, 32, 32)) callbacks = Callbacks(model, train_set, args, eval_set=valid_set) model.fit(train_set, optimizer=opt, num_epochs=num_epochs, cost=cost, callbacks=callbacks) print 'Validation: ', VALIDATION print 'Train misclassification error: ', model.eval(train_set, metric=Misclassification()) if VALIDATION: print 'Valid misclassification error: ', model.eval(valid_set, metric=Misclassification()) print 'Test misclassification error: ', model.eval(test_set, metric=Misclassification()) if args.save_path is not None: save_obj(model.serialize(), EXPERIMENT_DIR + args.save_path)
def train(self, minibatch, epoch = 0): # expand components of minibatch prestates, steers, speeds, rewards, poststates, terminals = minibatch assert len(prestates.shape) == 2 assert len(poststates.shape) == 2 assert len(steers.shape) == 1 assert len(speeds.shape) == 1 assert len(rewards.shape) == 1 assert len(terminals.shape) == 1 assert prestates.shape == poststates.shape assert prestates.shape[0] == steers.shape[0] == speeds.shape[0] == rewards.shape[0] == poststates.shape[0] == terminals.shape[0] if self.target_steps and self.train_iterations % self.target_steps == 0: # HACK: serialize network to disk and read it back to clone filename = self.save_weights_prefix + "_target.pkl" save_obj(self.model.serialize(keep_states = False), filename) self.target_model.load_weights(filename) # feed-forward pass for poststates to get Q-values self._setInput(poststates) postq = self.target_model.fprop(self.input, inference = True) assert postq.shape == (self.num_actions, self.batch_size) # calculate max Q-value for each poststate postq = postq.asnumpyarray() maxsteerq = np.max(postq[:self.num_steers,:], axis=0) assert maxsteerq.shape == (self.batch_size,), "size: %s" % str(maxsteerq.shape) maxspeedq = np.max(postq[-self.num_speeds:,:], axis=0) assert maxspeedq.shape == (self.batch_size,) # feed-forward pass for prestates self._setInput(prestates) preq = self.model.fprop(self.input, inference = False) assert preq.shape == (self.num_actions, self.batch_size) # make copy of prestate Q-values as targets # HACK: copy() was needed to make it work on CPU targets = preq.asnumpyarray().copy() # update Q-value targets for actions taken for i, (steer, speed) in enumerate(zip(steers, speeds)): if terminals[i]: targets[steer, i] = float(rewards[i]) targets[self.num_steers + speed, i] = float(rewards[i]) else: targets[steer, i] = float(rewards[i]) + self.discount_rate * maxsteerq[i] targets[self.num_steers + speed, i] = float(rewards[i]) + self.discount_rate * maxspeedq[i] # copy targets to GPU memory self.targets.set(targets) # calculate errors deltas = self.cost.get_errors(preq, self.targets) assert deltas.shape == (self.num_actions, self.batch_size) #assert np.count_nonzero(deltas.asnumpyarray()) == 2 * self.batch_size, str(np.count_nonzero(deltas.asnumpyarray())) # calculate cost, just in case cost = self.cost.get_cost(preq, self.targets) assert cost.shape == (1,1) #print "cost:", cost.asnumpyarray() # clip errors if self.clip_error: self.be.clip(deltas, -self.clip_error, self.clip_error, out = deltas) # perform back-propagation of gradients self.model.bprop(deltas) # perform optimization self.optimizer.optimize(self.model.layers_to_optimize, epoch) ''' if np.any(rewards < 0): preqq = preq.asnumpyarray().copy() self._setInput(prestates) qvalues = self.model.fprop(self.input, inference = True).asnumpyarray().copy() indexes = rewards < 0 print "indexes:", indexes print "preq:", preqq[:, indexes].T print "preq':", qvalues[:, indexes].T print "diff:", (qvalues[:, indexes]-preqq[:, indexes]).T print "steers:", steers[indexes] print "speeds:", speeds[indexes] print "rewards:", rewards[indexes] print "terminals:", terminals[indexes] print "preq[0]:", preqq[:, 0] print "preq[0]':", qvalues[:, 0] print "diff:", qvalues[:, 0] - preqq[:, 0] print "deltas:", deltas.asnumpyarray()[:, indexes].T raw_input("Press Enter to continue...") ''' # increase number of weight updates (needed for target clone interval) self.train_iterations += 1
def train(self, minibatch, epoch): # expand components of minibatch prestates, actions, speed_actions, rewards, poststates, terminals = minibatch assert len(prestates.shape) == 2 assert len(poststates.shape) == 2 assert len(actions.shape) == 1 assert len(rewards.shape) == 1 assert len(terminals.shape) == 1 assert prestates.shape == poststates.shape assert prestates.shape[0] == actions.shape[0] == rewards.shape[0] == poststates.shape[0] == terminals.shape[0] #print "WE ARE ACTUALLY TRAINING IN HERE" if self.target_steps and self.train_iterations % self.target_steps == 0: # HACK: serialize network to disk and read it back to clone filename = self.save_weights_prefix + "_target.pkl" save_obj(self.model.serialize(keep_states = False), filename) self.target_model.load_weights(filename) # feed-forward pass for poststates to get Q-values self._setInput(poststates) postq = self.target_model.fprop(self.input, inference = True) assert postq.shape == (self.num_actions, self.batch_size) # calculate max Q-value for each poststate postq = postq.asnumpyarray() maxpostq = np.max(postq, axis=0) #print maxpostq.shape assert maxpostq.shape == (self.batch_size,) # feed-forward pass for prestates self._setInput(prestates) preq = self.model.fprop(self.input, inference = False) assert preq.shape == (self.num_actions, self.batch_size) # make copy of prestate Q-values as targets targets = preq.asnumpyarray().copy() # update Q-value targets for actions taken for i, action in enumerate(actions): self.action_count[action] += 1 if terminals[i]: targets[action, i] = float(rewards[i]) if rewards[i] == -1000: print "######################### action ", action, "should never be sampled again" print "sampled_terminal" else: targets[action, i] = float(rewards[i]) + self.discount_rate * maxpostq[i] #targets[i,action] = float(rewards[i]) + self.discount_rate * maxpostq[i] #print "action count", self.action_count # copy targets to GPU memory self.targets.set(targets) # calculate errors deltas = self.cost.get_errors(preq, self.targets) assert deltas.shape == (self.num_actions, self.batch_size) #assert np.count_nonzero(deltas.asnumpyarray()) == 32 print "nonzero deltas", np.count_nonzero(deltas.asnumpyarray()) # calculate cost, just in case cost = self.cost.get_cost(preq, self.targets) assert cost.shape == (1,1) print "cost:", cost.asnumpyarray() # clip errors #if self.clip_error: # self.be.clip(deltas, -self.clip_error, self.clip_error, out = deltas) # perform back-propagation of gradients self.model.bprop(deltas) # perform optimization self.optimizer.optimize(self.model.layers_to_optimize, epoch) # increase number of weight updates (needed for target clone interval) self.train_iterations += 1
def __init__(self, path='.', n_mb=None, img_per_batch=None, conv_size=None, rpn_rois_per_img=None, frcn_rois_per_img=None, add_flipped=False, shuffle=False, deterministic=False, rebuild_cache=False, subset_pct=100, mock_db=None): self.batch_index = 0 self.path = path self.mock_db = mock_db # how many ROIs per image self.rois_per_img = rpn_rois_per_img if rpn_rois_per_img else self.RPN_ROI_PER_IMAGE self.img_per_batch = img_per_batch if img_per_batch else self.IMG_PER_BATCH self.rois_per_batch = self.rois_per_img * self.img_per_batch # how many ROIs to use to train frcnn self.frcn_rois_per_img = frcn_rois_per_img if frcn_rois_per_img \ else self.FRCNN_ROI_PER_IMAGE assert self.img_per_batch == 1, "Only a minibatch of 1 is supported." self.num_classes = len(self.CLASSES) self._class_to_index = dict(list(zip(self.CLASSES, list(range(self.num_classes))))) # shape of the final conv layer if conv_size: self._conv_size = conv_size else: self._conv_size = int(np.floor(self.MAX_SIZE * self.SCALE)) self._feat_stride = 1 / float(self.SCALE) self._num_scales = len(self.SCALES) * len(self.RATIOS) self._total_anchors = self._conv_size * self._conv_size * self._num_scales self.shuffle = shuffle self.deterministic = deterministic self.add_flipped = add_flipped # load the configure the dataset paths self.config = self.load_data() # annotation metadata self._annotation_file_ext = '.xml' self._annotation_obj_tag = 'object' self._annotation_class_tag = 'name' self._annotation_xmin_tag = 'xmin' self._annotation_xmax_tag = 'xmax' self._annotation_ymin_tag = 'ymin' self._annotation_ymax_tag = 'ymax' # self.rois_per_batch is 128 (2*64) ROIs # But the image path batch size is self.img_per_batch # need to control the batch size here assert self.img_per_batch is 1, "Only a batch size of 1 image is supported" neon_logger.display("Backend batchsize is changed to be {} " "from Object Localization dataset".format( self.img_per_batch)) self.be.bsz = self.img_per_batch # 0. allocate buffers self.allocate() if not self.mock_db: # 1. read image index file assert os.path.exists(self.config['image_path']), \ 'Image index file does not exist: {}'.format(self.config['image_path']) with open(self.config['index_path']) as f: self.image_index = [x.strip() for x in f.readlines()] num_images = len(self.image_index) self.num_image_entries = num_images * 2 if self.add_flipped else num_images self.ndata = self.num_image_entries * self.rois_per_img else: self.num_image_entries = 1 self.ndata = self.num_image_entries * self.rois_per_img assert (subset_pct > 0 and subset_pct <= 100), ('subset_pct must be between 0 and 100') if n_mb is not None: self.nbatches = n_mb else: self.nbatches = int(self.num_image_entries / self.img_per_batch * subset_pct / 100) self.cache_file = self.config['cache_path'] if os.path.exists(self.cache_file) and not rebuild_cache and not self.mock_db: self.roi_db = load_obj(self.cache_file) neon_logger.display('ROI dataset loaded from file {}'.format(self.cache_file)) elif not self.mock_db: # 2. read object Annotations (XML) roi_db = self.load_roi_groundtruth() if(self.add_flipped): roi_db = self.add_flipped_db(roi_db) # 3. construct acnhor targets self.roi_db = self.add_anchors(roi_db) if NORMALIZE_BBOX_TARGETS: # 4. normalize bbox targets by class self.roi_db = self.normalize_bbox_targets(self.roi_db) save_obj(self.roi_db, self.cache_file) neon_logger.display('wrote ROI dataset to {}'.format(self.cache_file)) else: assert self.mock_db is not None roi_db = [self.mock_db] self.roi_db = self.add_anchors(roi_db) # 4. map anchors back to full canvas. # This is neccessary because the network outputs reflect the full canvas. # We cache the files in the unmapped state (above) to save memory. self.roi_db = unmap(self.roi_db)
def train(self, minibatch, epoch): # expand components of minibatch prestates, actions, rewards, poststates, terminals = minibatch assert len(prestates.shape) == 4 assert len(poststates.shape) == 4 assert len(actions.shape) == 1 assert len(rewards.shape) == 1 assert len(terminals.shape) == 1 assert prestates.shape == poststates.shape assert prestates.shape[0] == actions.shape[0] == rewards.shape[ 0] == poststates.shape[0] == terminals.shape[0] if self.target_steps and self.train_iterations % self.target_steps == 0: # HACK: push something through network, so that weights exist self.model.fprop(self.tensor) # HACK: serialize network to disk and read it back to clone filename = os.path.join(self.save_weights_path, "target_network.pkl") save_obj(self.model.serialize(keep_states=False), filename) self.target_model.load_weights(filename) # feed-forward pass for poststates to get Q-values self.setTensor(poststates) postq = self.target_model.fprop(self.tensor, inference=True) assert postq.shape == (self.num_actions, self.batch_size) # calculate max Q-value for each poststate maxpostq = self.be.max(postq, axis=0).asnumpyarray() assert maxpostq.shape == (1, self.batch_size) # feed-forward pass for prestates self.setTensor(prestates) preq = self.model.fprop(self.tensor, inference=False) assert preq.shape == (self.num_actions, self.batch_size) # make copy of prestate Q-values as targets targets = preq.asnumpyarray() # update Q-value targets for actions taken for i, action in enumerate(actions): if terminals[i]: targets[action, i] = float(rewards[i]) else: targets[action, i] = float( rewards[i]) + self.discount_rate * maxpostq[0, i] # copy targets to GPU memory self.targets.set(targets) # calculate errors deltas = self.cost.get_errors(preq, self.targets) assert deltas.shape == (self.num_actions, self.batch_size) #assert np.count_nonzero(deltas.asnumpyarray()) == 32 # calculate cost, just in case cost = self.cost.get_cost(preq, self.targets) assert cost.shape == (1, 1) # clip errors if self.clip_error: self.be.clip(deltas, -self.clip_error, self.clip_error, out=deltas) # perform back-propagation of gradients self.model.bprop(deltas) # perform optimization self.optimizer.optimize(self.model.layers_to_optimize, epoch) # increase number of weight updates (needed for target clone interval) self.train_iterations += 1 # calculate statistics if self.callback: self.callback.on_train(cost.asnumpyarray()[0, 0])
def train(self, minibatch, epoch): # expand components of minibatch prestates, actions, rewards, poststates, terminals = minibatch assert len(prestates.shape) == 4 assert len(poststates.shape) == 4 assert len(actions.shape) == 1 assert len(rewards.shape) == 1 assert len(terminals.shape) == 1 assert prestates.shape == poststates.shape assert prestates.shape[0] == actions.shape[0] == rewards.shape[0] == poststates.shape[0] == terminals.shape[0] if self.target_steps and self.train_iterations % self.target_steps == 0: # HACK: push something through network, so that weights exist self.model.fprop(self.tensor) # HACK: serialize network to disk and read it back to clone filename = os.path.join(self.save_weights_path, "target_network.pkl") save_obj(self.model.serialize(keep_states = False), filename) self.target_model.load_weights(filename) # feed-forward pass for poststates to get Q-values self.setTensor(poststates) postq = self.target_model.fprop(self.tensor, inference = True) assert postq.shape == (self.num_actions, self.batch_size) # calculate max Q-value for each poststate maxpostq = self.be.max(postq, axis=0).asnumpyarray() assert maxpostq.shape == (1, self.batch_size) # feed-forward pass for prestates self.setTensor(prestates) preq = self.model.fprop(self.tensor, inference = False) assert preq.shape == (self.num_actions, self.batch_size) # make copy of prestate Q-values as targets targets = preq.asnumpyarray() # update Q-value targets for actions taken for i, action in enumerate(actions): if terminals[i]: targets[action, i] = float(rewards[i]) else: targets[action, i] = float(rewards[i]) + self.discount_rate * maxpostq[0,i] # copy targets to GPU memory self.targets.set(targets) # calculate errors deltas = self.cost.get_errors(preq, self.targets) assert deltas.shape == (self.num_actions, self.batch_size) #assert np.count_nonzero(deltas.asnumpyarray()) == 32 # calculate cost, just in case cost = self.cost.get_cost(preq, self.targets) assert cost.shape == (1,1) # clip errors if self.clip_error: self.be.clip(deltas, -self.clip_error, self.clip_error, out = deltas) # perform back-propagation of gradients self.model.bprop(deltas) # perform optimization self.optimizer.optimize(self.model.layers_to_optimize, epoch) # increase number of weight updates (needed for target clone interval) self.train_iterations += 1 # calculate statistics if self.callback: self.callback.on_train(cost.asnumpyarray()[0,0])
if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('cache_file', help='path to data cache file') args = parser.parse_args() cache_file = args.cache_file # check for RW access to file assert os.path.exists(cache_file), 'file does not exist %s' % cache_file if not os.access(os.path.abspath(cache_file), os.R_OK | os.W_OK): raise IOError('Need to add read and/or write permissions on file %s' % cache_file) dc = load_obj(cache_file) if 'global_mean' not in dc or 'img_size' not in dc: raise ValueError('data cache file missing global_mean key') sz = dc['img_size'] gm = dc['global_mean'] if len(gm.shape) != 2 or (gm.shape[0] != sz*sz*3 or gm.shape[1] != 1): raise ValueError('global mean shape %s does not match format expected' % str(gm.shape)) # Collapse the full tensor mean into channel means and correct the order (RGB <-> BGR) dc['global_mean'] = np.mean(gm.reshape(3, -1), axis=1).reshape(3, 1)[::-1] save_obj(dc, cache_file) print '%s updated to new format' % cache_file
def load_data(path, file_ext=['txt'], valid_split=None, vocab_file_name=None, max_vocab_size=None, max_len_w=None, output_path=None, subset_pct=100): """ Given a path where data are saved, look for the ones with the right extensions If a split factor is given, it will split all the files into training and valid set. Then build vocabulary from the training and validation sets. Arguments: path: which directory to look for all the documents file_ext: what extension of the files to look for valid_split: to split the data into train/valid set. If None, no split vocab_file_name: optional file name. If None, the script will decide a name given path and split max_vocab_size: maximum number of words to use in vocabulary (by most frequent) max_len_w: maximum length of sentences in words output_path: path used to save preprocessed data and resuts subset_pct: subset of dataset to load into H5 file (percentage) Returns: The function saves 2 files: h5 file with preprocessed data vocabulary file with: vocab, reverse_vocab, word_count """ file_names = get_file_list(path, file_ext) file_str = get_file_str(path, len(file_names), labelled=False, valid_split=valid_split, subset_pct=subset_pct) # create output dir if needed if not os.path.isdir(output_path): os.makedirs(output_path) # file name to store the vocabulary if vocab_file_name is None: vocab_file_name = file_str + '.vocab' vocab_file_name = os.path.join(output_path, vocab_file_name) # If max sizes arent set, assume no limit if not max_len_w: max_len_w = sys.maxsize if not max_vocab_size: max_vocab_size = sys.maxsize # file name to store the pre-processed train/valid dataset h5_file_name = os.path.join(output_path, file_str + '.h5') if os.path.exists(h5_file_name) and os.path.exists(vocab_file_name): neon_logger.display( "dataset files {} and vocabulary file {} already exist. " "will use cached data. ".format(h5_file_name, vocab_file_name)) return h5_file_name, vocab_file_name # split into training/valid set if valid_split is not None: if 'json' in file_ext: # Split based on number of files train_split = int(np.ceil(len(file_names) * (1 - valid_split))) train_files = file_names[:train_split] valid_files = file_names[train_split:] train_sent = load_json_sent(train_files, subset_pct) valid_sent = load_json_sent(valid_files, subset_pct) all_sent = train_sent + valid_sent elif 'txt' in file_ext: # Split based on number of lines (since only 2 files) all_sent = load_txt_sent(file_names, subset_pct) train_split = int(np.ceil(len(all_sent) * (1 - valid_split))) train_sent = all_sent[:train_split] valid_sent = all_sent[train_split:] else: neon_logger.display( "Unsure how to load file_ext {}, please use 'json' or 'txt'.". format(file_ext)) else: train_files = file_names if 'json' in file_ext: train_sent = load_json_sent(train_files, subset_pct) elif 'txt' in file_ext: train_sent = load_txt_sent(train_files, subset_pct) else: neon_logger.display( "Unsure how to load file_ext {}, please use 'json' or 'txt'.". format(file_ext)) all_sent = train_sent if os.path.exists(vocab_file_name): neon_logger.display( "open existing vocab file: {}".format(vocab_file_name)) vocab, rev_vocab, word_count = load_obj(vocab_file_name) else: neon_logger.display("Building vocab file") # build vocab word_count = defaultdict(int) for sent in all_sent: sent_words = tokenize(sent) if len(sent_words) > max_len_w or len(sent_words) == 0: continue for word in sent_words: word_count[word] += 1 # sort the word_count , re-assign ids by its frequency. Useful for downstream tasks # only done for train vocab vocab_sorted = sorted(word_count.items(), key=lambda kv: kv[1], reverse=True) vocab = OrderedDict() # get word count as array in same ordering as vocab (but with maximum length) word_count_ = np.zeros((len(word_count), ), dtype=np.int64) for i, t in enumerate(list(zip(*vocab_sorted))[0][:max_vocab_size]): word_count_[i] = word_count[t] vocab[t] = i word_count = word_count_ # generate the reverse vocab rev_vocab = dict((wrd_id, wrd) for wrd, wrd_id in vocab.items()) neon_logger.display("vocabulary from {} is saved into {}".format( path, vocab_file_name)) save_obj((vocab, rev_vocab, word_count), vocab_file_name) vocab_size = len(vocab) neon_logger.display( "\nVocab size from the dataset is: {}".format(vocab_size)) neon_logger.display( "\nProcessing and saving training data into {}".format(h5_file_name)) # now process and save the train/valid data h5f = h5py.File(h5_file_name, 'w', libver='latest') shape, maxshape = (len(train_sent), ), (None) dt = np.dtype([('text', h5py.special_dtype(vlen=str)), ('num_words', np.uint16)]) report_text_train = h5f.create_dataset('report_train', shape=shape, maxshape=maxshape, dtype=dt, compression='gzip') report_train = h5f.create_dataset('train', shape=shape, maxshape=maxshape, dtype=h5py.special_dtype(vlen=np.int32), compression='gzip') # map text to integers wdata = np.zeros((1, ), dtype=dt) ntrain = 0 for sent in train_sent: text_int = [-1 if t not in vocab else vocab[t] for t in tokenize(sent)] # enforce maximum sentence length if len(text_int) > max_len_w or len(text_int) == 0: continue report_train[ntrain] = text_int wdata['text'] = clean_string(sent) wdata['num_words'] = len(text_int) report_text_train[ntrain] = wdata ntrain += 1 report_train.attrs['nsample'] = ntrain report_train.attrs['vocab_size'] = vocab_size report_text_train.attrs['nsample'] = ntrain report_text_train.attrs['vocab_size'] = vocab_size if valid_split: neon_logger.display( "\nProcessing and saving validation data into {}".format( h5_file_name)) shape = (len(valid_sent), ) report_text_valid = h5f.create_dataset('report_valid', shape=shape, maxshape=maxshape, dtype=dt, compression='gzip') report_valid = h5f.create_dataset( 'valid', shape=shape, maxshape=maxshape, dtype=h5py.special_dtype(vlen=np.int32), compression='gzip') nvalid = 0 for sent in valid_sent: text_int = [ -1 if t not in vocab else vocab[t] for t in tokenize(sent) ] # enforce maximum sentence length if len(text_int) > max_len_w or len(text_int) == 0: continue report_valid[nvalid] = text_int wdata['text'] = clean_string(sent) wdata['num_words'] = len(text_int) report_text_valid[nvalid] = wdata nvalid += 1 report_valid.attrs['nsample'] = nvalid report_valid.attrs['vocab_size'] = vocab_size report_text_valid.attrs['nsample'] = nvalid report_text_valid.attrs['vocab_size'] = vocab_size h5f.close() return h5_file_name, vocab_file_name
opt = MultiOptimizer({'default': opt_gdm, 'Bias': opt_biases}) if not args.resume: # fit the model for 3 epochs model.fit(train, optimizer=opt, num_epochs=3, cost=cost, callbacks=callbacks) train.reset() # get 1 image for im, l in train: break train.exit_batch_provider() save_obj((im.get(), l.get()), 'im1.pkl') im_save = im.get().copy() if args.resume: (im2, l2) = load_obj('im1.pkl') im.set(im2) l.set(l2) # run fprop and bprop on this minibatch save the results out_fprop = model.fprop(im) out_fprop_save = [x.get() for x in out_fprop] im.set(im_save) out_fprop = model.fprop(im) out_fprop_save2 = [x.get() for x in out_fprop] for x, y in zip(out_fprop_save, out_fprop_save2): assert np.max(np.abs(x - y)) == 0.0, '2 fprop iterations do not match'
def on_epoch_end(self, callback_data, model, epoch): if self.history > 1: self.save_history(epoch, model) else: save_obj(model.serialize(keep_states=True), self.save_path)
X_test, y_test, cluster) spec_out = nout spec_set = DataIterator( X_spec, y_spec, nclass=spec_out, lshape=(3, 32, 32)) spec_test = DataIterator( X_spec_test, y_spec_test, nclass=spec_out, lshape=(3, 32, 32)) # Train the specialist specialist, opt, cost = spec_net(nout=spec_out, archive_path=gene_path) callbacks = Callbacks(specialist, spec_set, args, eval_set=spec_test) callbacks.add_early_stop_callback(early_stop) callbacks.add_save_best_state_callback(path) specialist.fit(spec_set, optimizer=opt, num_epochs=specialist.epoch_index + num_epochs, cost=cost, callbacks=callbacks) # Print results print 'Specialist Train misclassification error: ', specialist.eval(spec_set, metric=Misclassification()) print 'Specialist Test misclassification error: ', specialist.eval(spec_test, metric=Misclassification()) print 'Generalist Train misclassification error: ', generalist.eval(spec_set, metric=Misclassification()) print 'Generalist Test misclassification error: ', generalist.eval(spec_test, metric=Misclassification()) # specialists.append(specialist) save_obj(specialist.serialize(), path) except: path = confusion_matrix_name + '_' + clustering_name + '_' + str(num_clusters) + 'clusters/' print 'Failed for ', path failed.append(path) for f in failed: print f
(proposals, num_proposals) = proposalLayer.get_proposals() # convert outputs to bounding boxes boxes = faster_rcnn.get_bboxes(outputs, proposals, num_proposals, num_classes, im_shape.get(), im_scale, max_per_image, thresh, nms_thresh) all_boxes[mb_idx] = boxes # retrieve gt boxes # we add a extra column to track detections during the AP calculation detected = np.array([False] * num_gt_boxes) gt_boxes = np.hstack([ gt_boxes.get()[:num_gt_boxes] / im_scale, gt_classes.get()[:num_gt_boxes], difficult.get()[:num_gt_boxes], detected[:, np.newaxis] ]) all_gt_boxes[mb_idx] = gt_boxes neon_logger.display('Evaluating detections') avg_precision = voc_eval(all_boxes, all_gt_boxes, valid_set.CLASSES, use_07_metric=True) if args.output is not None: neon_logger.display('Saving inference results to {}'.format(args.output)) save_obj([all_boxes, avg_precision], args.output)
def __init__(self, image_set, year, path='.', add_flipped=False, overlap_thre=None, output_type=0, n_mb=None, img_per_batch=None, rois_per_img=None, rois_random_sample=True, shuffle=False): self.isRoiDB = True self.batch_index = 0 self.year = year self.image_set = image_set self.add_flipped = add_flipped self.overlap_thre = overlap_thre if overlap_thre else FRCN_IOU_THRE self.output_type = output_type # how many ROIs per image self.rois_per_image = rois_per_img if rois_per_img else FRCN_ROI_PER_IMAGE self.img_per_batch = img_per_batch if img_per_batch else FRCN_IMG_PER_BATCH self.fg_rois_per_image = FRCN_FG_FRAC * self.rois_per_image self.bg_rois_per_image = self.rois_per_image - self.fg_rois_per_image self.rois_per_batch = self.rois_per_image * self.img_per_batch self.rois_random_sample = rois_random_sample self.shuffle = shuffle self.cache_file_name = 'voc_{}_{}_flip_{}_ovlp_{}.pkl'.format( self.year, self.image_set, self.add_flipped, self.overlap_thre) print 'prepare PASCAL VOC {} from year {}: add flipped image {} and overlap threshold {}'\ .format(self.image_set, self.year, self.add_flipped, self.overlap_thre) # PASCAL class to index self.num_classes = PASCAL_VOC_NUM_CLASSES self._class_to_index = dict( zip(PASCAL_VOC_CLASSES, xrange(self.num_classes))) # load the voc dataset self.voc_root = self.load_voc(image_set, year, path) self.cache_file = os.path.join(self.voc_root, self.cache_file_name) # load the precomputed ss results from voc data, it includes both 2007 and 2012 data self.ss_path = self.load_voc('ss', None, path) # VOC paths and infos self.image_index_file = os.path.join(self.voc_root, 'ImageSets', 'Main', self.image_set + '.txt') self.image_path = os.path.join(self.voc_root, 'JPEGImages') self._image_file_ext = '.jpg' self.annotation_path = os.path.join(self.voc_root, 'Annotations') self._annotation_file_ext = '.xml' self._annotation_obj_tag = 'object' self._annotation_class_tag = 'name' self._annotation_xmin_tag = 'xmin' self._annotation_xmax_tag = 'xmax' self._annotation_ymin_tag = 'ymin' self._annotation_ymax_tag = 'ymax' self._selective_search_ext = '.pkl' self.selective_search_file = os.path.join( self.ss_path, '_'.join(['voc', year, self.image_set, 'selectivesearch.pkl'])) self._bb_xmin_idx = 0 self._bb_ymin_idx = 1 self._bb_xmax_idx = 2 self._bb_ymax_idx = 3 # self.rois_per_batch is 128 (2*64) ROIs # But the image path batch size is self.img_per_batch # need to control the batch size here print "Backend batchsize is changed to be image_per_batch from PASCAL_VOC dataset" self.be.bsz = self.img_per_batch # backend tensor to push the data self.image_shape = (3, FRCN_MAX_SCALE, FRCN_MAX_SCALE) self.img_np = np.zeros( (3, FRCN_MAX_SCALE, FRCN_MAX_SCALE, self.be.bsz), dtype=np.float32) self.dev_X_img = self.be.iobuf(self.image_shape, dtype=np.float32) self.dev_X_img_chw = self.dev_X_img.reshape(3, FRCN_MAX_SCALE, FRCN_MAX_SCALE, self.be.bsz) # for rois, features are 4 + 1 (idx within the batch) self.dev_X_rois = self.be.zeros((self.rois_per_batch, 5)) self.dev_y_labels_flat = self.be.zeros((1, self.rois_per_batch), dtype=np.int32) self.dev_y_labels = self.be.zeros( (self.num_classes, self.rois_per_batch), dtype=np.int32) self.dev_y_bbtargets = self.be.zeros( (self.num_classes * 4, self.rois_per_batch)) self.dev_y_bbmask = self.be.zeros( (self.num_classes * 4, self.rois_per_batch)) # the shape will indicate the shape for 1st path (ImageNet model), and # 2nd path (ROIs) self.shape = [self.image_shape, self.num_classes * 4] # Need to do the following: # 1. load the image index list # 2. for each image, load the ground truth from pascal annotation # 3. load the selective search ROIs (this step needs gt ROIs) # 4.1. merge the ROIs # 4.2. may have to add the flipped images for training # 4.3. add the fields for max overlap and max overlapped classes # 4.4. add the bounding box targets for regression # 5. during minibatch feeding: # - rescale images # - rescale ROIs # - random select foreground ROIs (bigger ones) # - random select background ROIS (smaller ones) # - clamp bg ROI labels (to be 0) # - convert ROIs into the regression target (ROIs, 4*21) # 1. assert os.path.exists(self.image_index_file), \ 'Image index file does not exist: {}'.format(self.image_index_file) with open(self.image_index_file) as f: self.image_index = [x.strip() for x in f.readlines()] # self.image_index = image_index * 2 if self.add_flipped else image_index self.num_images = len(self.image_index) self.num_image_entries = self.num_images * \ 2 if self.add_flipped else self.num_images self.ndata = self.num_image_entries * self.rois_per_image self.nbatches = self.num_image_entries / self.img_per_batch if n_mb is not None: self.nbatches = n_mb if os.path.exists(self.cache_file): self.roi_db = load_obj(self.cache_file) print 'ROI dataset loaded from file {}'.format(self.cache_file) else: # 2. self.roi_gt = self.load_pascal_roi_groundtruth() # 3. self.roi_ss = self.load_pascal_roi_selectivesearch() # 4. self.roi_db = self.combine_gt_ss_roi() save_obj(self.roi_db, self.cache_file) print 'wrote ROI dataset to {}'.format(self.cache_file)
def test_model_serialize(backend_default, data): (X_train, y_train), (X_test, y_test), nclass = load_mnist(path=data) train_set = DataIterator( [X_train, X_train], y_train, nclass=nclass, lshape=(1, 28, 28)) init_norm = Gaussian(loc=0.0, scale=0.01) # initialize model path1 = Sequential([Conv((5, 5, 16), init=init_norm, bias=Constant(0), activation=Rectlin()), Pooling(2), Affine(nout=20, init=init_norm, bias=init_norm, activation=Rectlin())]) path2 = Sequential([Affine(nout=100, init=init_norm, bias=Constant(0), activation=Rectlin()), Dropout(keep=0.5), Affine(nout=20, init=init_norm, bias=init_norm, activation=Rectlin())]) layers = [MergeMultistream(layers=[path1, path2], merge="stack"), Affine(nout=20, init=init_norm, batch_norm=True, activation=Rectlin()), Affine(nout=10, init=init_norm, activation=Logistic(shortcut=True))] tmp_save = 'test_model_serialize_tmp_save.pickle' mlp = Model(layers=layers) mlp.optimizer = GradientDescentMomentum(learning_rate=0.1, momentum_coef=0.9) mlp.cost = GeneralizedCost(costfunc=CrossEntropyBinary()) mlp.initialize(train_set, cost=mlp.cost) n_test = 3 num_epochs = 3 # Train model for num_epochs and n_test batches for epoch in range(num_epochs): for i, (x, t) in enumerate(train_set): x = mlp.fprop(x) delta = mlp.cost.get_errors(x, t) mlp.bprop(delta) mlp.optimizer.optimize(mlp.layers_to_optimize, epoch=epoch) if i > n_test: break # Get expected outputs of n_test batches and states of all layers outputs_exp = [] pdicts_exp = [l.get_params_serialize() for l in mlp.layers_to_optimize] for i, (x, t) in enumerate(train_set): outputs_exp.append(mlp.fprop(x, inference=True)) if i > n_test: break # Serialize model save_obj(mlp.serialize(keep_states=True), tmp_save) # Load model mlp = Model(layers=layers) mlp.load_weights(tmp_save) outputs = [] pdicts = [l.get_params_serialize() for l in mlp.layers_to_optimize] for i, (x, t) in enumerate(train_set): outputs.append(mlp.fprop(x, inference=True)) if i > n_test: break # Check outputs, states, and params are the same for output, output_exp in zip(outputs, outputs_exp): assert np.allclose(output.get(), output_exp.get()) for pd, pd_exp in zip(pdicts, pdicts_exp): for s, s_e in zip(pd['states'], pd_exp['states']): if isinstance(s, list): # this is the batch norm case for _s, _s_e in zip(s, s_e): assert np.allclose(_s, _s_e) else: assert np.allclose(s, s_e) for p, p_e in zip(pd['params'], pd_exp['params']): assert type(p) == type(p_e) if isinstance(p, list): # this is the batch norm case for _p, _p_e in zip(p, p_e): assert np.allclose(_p, _p_e) elif isinstance(p, np.ndarray): assert np.allclose(p, p_e) else: assert p == p_e os.remove(tmp_save)
def test_model_serialize(backend): (X_train, y_train), (X_test, y_test), nclass = load_mnist() train_set = DataIterator([X_train, X_train], y_train, nclass=nclass, lshape=(1, 28, 28)) init_norm = Gaussian(loc=0.0, scale=0.01) # initialize model path1 = [ Conv((5, 5, 16), init=init_norm, bias=Constant(0), activation=Rectlin()), Pooling(2), Affine(nout=20, init=init_norm, bias=init_norm, activation=Rectlin()) ] path2 = [ Dropout(keep=0.5), Affine(nout=20, init=init_norm, bias=init_norm, activation=Rectlin()) ] layers = [ MergeConcat([path1, path2]), Affine(nout=20, init=init_norm, bias=init_norm, activation=Rectlin()), BatchNorm(), Affine(nout=10, init=init_norm, activation=Logistic(shortcut=True)) ] tmp_save = 'test_model_serialize_tmp_save.pickle' mlp = Model(layers=layers) mlp.optimizer = GradientDescentMomentum(learning_rate=0.1, momentum_coef=0.9) mlp.cost = GeneralizedCost(costfunc=CrossEntropyBinary()) n_test = 3 num_epochs = 3 # Train model for num_epochs and n_test batches for epoch in range(num_epochs): for i, (x, t) in enumerate(train_set): x = mlp.fprop(x) delta = mlp.cost.get_errors(x, t) mlp.bprop(delta) mlp.optimizer.optimize(mlp.layers_to_optimize, epoch=epoch) if i > n_test: break # Get expected outputs of n_test batches and states of all layers outputs_exp = [] pdicts_exp = [l.get_params_serialize() for l in mlp.layers_to_optimize] for i, (x, t) in enumerate(train_set): outputs_exp.append(mlp.fprop(x, inference=True)) if i > n_test: break # Serialize model save_obj(mlp.serialize(keep_states=True), tmp_save) # Load model mlp = Model(layers=layers) mlp.load_weights(tmp_save) outputs = [] pdicts = [l.get_params_serialize() for l in mlp.layers_to_optimize] for i, (x, t) in enumerate(train_set): outputs.append(mlp.fprop(x, inference=True)) if i > n_test: break # Check outputs, states, and params are the same for output, output_exp in zip(outputs, outputs_exp): assert np.allclose(output.get(), output_exp.get()) for pd, pd_exp in zip(pdicts, pdicts_exp): for s, s_e in zip(pd['states'], pd_exp['states']): if isinstance(s, list): # this is the batch norm case for _s, _s_e in zip(s, s_e): assert np.allclose(_s, _s_e) else: assert np.allclose(s, s_e) for p, p_e in zip(pd['params'], pd_exp['params']): if isinstance(p, list): # this is the batch norm case for _p, _p_e in zip(p, p_e): assert np.allclose(_p, _p_e) else: assert np.allclose(p, p_e) os.remove(tmp_save)
(im_shape, im_scale, gt_boxes, gt_classes, num_gt_boxes, difficult) = valid_set.get_metadata_buffers() num_gt_boxes = int(num_gt_boxes.get()) im_scale = float(im_scale.get()) # retrieve region proposals generated by the model (proposals, num_proposals) = proposalLayer.get_proposals() # convert outputs to bounding boxes boxes = faster_rcnn.get_bboxes(outputs, proposals, num_proposals, num_classes, im_shape.get(), im_scale, max_per_image, thresh, nms_thresh) all_boxes[mb_idx] = boxes # retrieve gt boxes # we add a extra column to track detections during the AP calculation detected = np.array([False] * num_gt_boxes) gt_boxes = np.hstack([gt_boxes.get()[:num_gt_boxes] / im_scale, gt_classes.get()[:num_gt_boxes], difficult.get()[:num_gt_boxes], detected[:, np.newaxis]]) all_gt_boxes[mb_idx] = gt_boxes neon_logger.display('Evaluating detections') avg_precision = voc_eval(all_boxes, all_gt_boxes, valid_set.CLASSES, use_07_metric=True) if args.output is not None: neon_logger.display('Saving inference results to {}'.format(args.output)) save_obj([all_boxes, avg_precision], args.output)
frcn_tree_cost, ], weights=[1, 1, 1]) # setup optimizer schedule_w = StepSchedule(step_config=[10], change=[0.001 / 10]) schedule_b = StepSchedule(step_config=[10], change=[0.002 / 10]) opt_w = GradientDescentMomentum(0.001, 0.9, wdecay=0.0005, schedule=schedule_w) opt_b = GradientDescentMomentum(0.002, 0.9, wdecay=0.0005, schedule=schedule_b) opt_skip = GradientDescentMomentum(0.0, 0.0) optimizer = MultiOptimizer({'default': opt_w, 'Bias': opt_b, 'skip': opt_skip, 'skip_bias': opt_skip}) # if training a new model, seed the image model conv layers with pre-trained weights # otherwise, just load the model file if args.model_file is None: util.load_vgg_all_weights(model, cache_dir) callbacks = Callbacks(model, eval_set=train_set, **args.callback_args) model.fit(train_set, optimizer=optimizer, cost=cost, num_epochs=args.epochs, callbacks=callbacks) # Scale the bbox regression branch linear layer weights before saving the model model = util.scale_bbreg_weights(model, [0.0, 0.0, 0.0, 0.0], [0.1, 0.1, 0.2, 0.2], train_set.num_classes) if args.save_path is not None: save_obj(model.serialize(keep_states=True), args.save_path)
def on_epoch_end(self, epoch): if self.history > 1: self.save_history(epoch) else: save_obj(self.model.serialize(keep_states=True), self.save_path)
def load_data(path, file_ext=['txt'], valid_split=None, vocab_file_name=None, max_vocab_size=None, max_len_w=None, output_path=None, subset_pct=100): """ Given a path where data are saved, look for the ones with the right extensions If a split factor is given, it will split all the files into training and valid set. Then build vocabulary from the training and validation sets. Arguments: path: which directory to look for all the documents file_ext: what extension of the files to look for valid_split: to split the data into train/valid set. If None, no split vocab_file_name: optional file name. If None, the script will decide a name given path and split max_vocab_size: maximum number of words to use in vocabulary (by most frequent) max_len_w: maximum length of sentences in words output_path: path used to save preprocessed data and resuts subset_pct: subset of dataset to load into H5 file (percentage) Returns: The function saves 2 files: h5 file with preprocessed data vocabulary file with: vocab, reverse_vocab, word_count """ file_names = get_file_list(path, file_ext) file_str = get_file_str(path, len(file_names), labelled=False, valid_split=valid_split, subset_pct=subset_pct) # create output dir if needed if not os.path.isdir(output_path): os.makedirs(output_path) # file name to store the vocabulary if vocab_file_name is None: vocab_file_name = file_str + '.vocab' vocab_file_name = os.path.join(output_path, vocab_file_name) # If max sizes arent set, assume no limit if not max_len_w: max_len_w = sys.maxsize if not max_vocab_size: max_vocab_size = sys.maxsize # file name to store the pre-processed train/valid dataset h5_file_name = os.path.join(output_path, file_str + '.h5') if os.path.exists(h5_file_name) and os.path.exists(vocab_file_name): neon_logger.display("dataset files {} and vocabulary file {} already exist. " "will use cached data. ".format(h5_file_name, vocab_file_name)) return h5_file_name, vocab_file_name # split into training/valid set if valid_split is not None: if 'json' in file_ext: # Split based on number of files train_split = int(np.ceil(len(file_names) * (1 - valid_split))) train_files = file_names[:train_split] valid_files = file_names[train_split:] train_sent = load_json_sent(train_files, subset_pct) valid_sent = load_json_sent(valid_files, subset_pct) all_sent = train_sent + valid_sent elif 'txt' in file_ext: # Split based on number of lines (since only 2 files) all_sent = load_txt_sent(file_names, subset_pct) train_split = int(np.ceil(len(all_sent) * (1 - valid_split))) train_sent = all_sent[:train_split] valid_sent = all_sent[train_split:] else: neon_logger.display("Unsure how to load file_ext {}, please use 'json' or 'txt'." .format(file_ext)) else: train_files = file_names if 'json' in file_ext: train_sent = load_json_sent(train_files, subset_pct) elif 'txt' in file_ext: train_sent = load_txt_sent(train_files, subset_pct) else: neon_logger.display("Unsure how to load file_ext {}, please use 'json' or 'txt'." .format(file_ext)) all_sent = train_sent if os.path.exists(vocab_file_name): neon_logger.display("open existing vocab file: {}".format(vocab_file_name)) vocab, rev_vocab, word_count = load_obj(vocab_file_name) else: neon_logger.display("Building vocab file") # build vocab word_count = defaultdict(int) for sent in all_sent: sent_words = tokenize(sent) if len(sent_words) > max_len_w or len(sent_words) == 0: continue for word in sent_words: word_count[word] += 1 # sort the word_count , re-assign ids by its frequency. Useful for downstream tasks # only done for train vocab vocab_sorted = sorted(word_count.items(), key=lambda kv: kv[1], reverse=True) vocab = OrderedDict() # get word count as array in same ordering as vocab (but with maximum length) word_count_ = np.zeros((len(word_count), ), dtype=np.int64) for i, t in enumerate(list(zip(*vocab_sorted))[0][:max_vocab_size]): word_count_[i] = word_count[t] vocab[t] = i word_count = word_count_ # generate the reverse vocab rev_vocab = dict((wrd_id, wrd) for wrd, wrd_id in vocab.items()) neon_logger.display("vocabulary from {} is saved into {}".format(path, vocab_file_name)) save_obj((vocab, rev_vocab, word_count), vocab_file_name) vocab_size = len(vocab) neon_logger.display("\nVocab size from the dataset is: {}".format(vocab_size)) neon_logger.display("\nProcessing and saving training data into {}".format(h5_file_name)) # now process and save the train/valid data h5f = h5py.File(h5_file_name, 'w', libver='latest') shape, maxshape = (len(train_sent),), (None) dt = np.dtype([('text', h5py.special_dtype(vlen=str)), ('num_words', np.uint16)]) report_text_train = h5f.create_dataset('report_train', shape=shape, maxshape=maxshape, dtype=dt, compression='gzip') report_train = h5f.create_dataset('train', shape=shape, maxshape=maxshape, dtype=h5py.special_dtype(vlen=np.int32), compression='gzip') # map text to integers wdata = np.zeros((1, ), dtype=dt) ntrain = 0 for sent in train_sent: text_int = [-1 if t not in vocab else vocab[t] for t in tokenize(sent)] # enforce maximum sentence length if len(text_int) > max_len_w or len(text_int) == 0: continue report_train[ntrain] = text_int wdata['text'] = clean_string(sent) wdata['num_words'] = len(text_int) report_text_train[ntrain] = wdata ntrain += 1 report_train.attrs['nsample'] = ntrain report_train.attrs['vocab_size'] = vocab_size report_text_train.attrs['nsample'] = ntrain report_text_train.attrs['vocab_size'] = vocab_size if valid_split: neon_logger.display("\nProcessing and saving validation data into {}".format(h5_file_name)) shape = (len(valid_sent),) report_text_valid = h5f.create_dataset('report_valid', shape=shape, maxshape=maxshape, dtype=dt, compression='gzip') report_valid = h5f.create_dataset('valid', shape=shape, maxshape=maxshape, dtype=h5py.special_dtype(vlen=np.int32), compression='gzip') nvalid = 0 for sent in valid_sent: text_int = [-1 if t not in vocab else vocab[t] for t in tokenize(sent)] # enforce maximum sentence length if len(text_int) > max_len_w or len(text_int) == 0: continue report_valid[nvalid] = text_int wdata['text'] = clean_string(sent) wdata['num_words'] = len(text_int) report_text_valid[nvalid] = wdata nvalid += 1 report_valid.attrs['nsample'] = nvalid report_valid.attrs['vocab_size'] = vocab_size report_text_valid.attrs['nsample'] = nvalid report_text_valid.attrs['vocab_size'] = vocab_size h5f.close() return h5_file_name, vocab_file_name
def save_weights(self, save_path): save_obj(self.model.serialize(keep_states=True), save_path)
if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("cache_file", help="path to data cache file") args = parser.parse_args() cache_file = args.cache_file # check for RW access to file assert os.path.exists(cache_file), "file does not exist %s" % cache_file if not os.access(os.path.abspath(cache_file), os.R_OK | os.W_OK): raise IOError("Need to add read and/or write permissions on file %s" % cache_file) dc = load_obj(cache_file) if "global_mean" not in dc or "img_size" not in dc: raise ValueError("data cache file missing global_mean key") sz = dc["img_size"] gm = dc["global_mean"] if len(gm.shape) != 2 or (gm.shape[0] != sz * sz * 3 or gm.shape[1] != 1): raise ValueError("global mean shape {} does not match format expected".format(gm.shape)) # Collapse the full tensor mean into channel means and correct the order (RGB <-> BGR) dc["global_mean"] = np.mean(gm.reshape(3, -1), axis=1).reshape(3, 1)[::-1] save_obj(dc, cache_file) neon_logger.display("%s updated to new format" % cache_file)
scales = [112, 128, 160, 240] for scale in scales: print scale layers = [] layers += [Conv(**conv_params(7, 32, 2))] for nfm, stride in zip(nfms, strides): layers.append(module_factory(nfm, stride)) layers.append(Pooling(7, op='avg')) layers.append( Conv(fshape=(1, 1, 100), init=Kaiming(local=True), batch_norm=True)) layers.append(Pooling(fshape='all', op='avg')) layers.append(Activation(Softmax())) model = Model(layers=layers) test = ImageLoader(set_name='validation', shuffle=False, do_transforms=False, inner_size=scale, scale_range=scale, repo_dir=args.data_dir) model.load_params("/home/users/hunter/bigfeat_dropout.pkl") softmaxes = model.get_outputs(test) from neon.util.persist import save_obj save_obj(softmaxes, "bigfeat_dropout_SM_{}.pkl".format(scale))
def __init__(self, image_set, year, path='.', add_flipped=False, overlap_thre=None, output_type=0, n_mb=None, img_per_batch=None, rois_per_img=None, rois_random_sample=True, shuffle=False): self.isRoiDB = True self.batch_index = 0 self.year = year self.image_set = image_set self.add_flipped = add_flipped self.overlap_thre = overlap_thre if overlap_thre else FRCN_IOU_THRE self.output_type = output_type # how many ROIs per image self.rois_per_image = rois_per_img if rois_per_img else FRCN_ROI_PER_IMAGE self.img_per_batch = img_per_batch if img_per_batch else FRCN_IMG_PER_BATCH self.fg_rois_per_image = FRCN_FG_FRAC * self.rois_per_image self.bg_rois_per_image = self.rois_per_image - self.fg_rois_per_image self.rois_per_batch = self.rois_per_image * self.img_per_batch self.rois_random_sample = rois_random_sample self.shuffle = shuffle self.cache_file_name = 'voc_{}_{}_flip_{}_ovlp_{}.pkl'.format(self.year, self.image_set, self.add_flipped, self.overlap_thre) print 'prepare PASCAL VOC {} from year {}: add flipped image {} and overlap threshold {}'\ .format(self.image_set, self.year, self.add_flipped, self.overlap_thre) # PASCAL class to index self.num_classes = PASCAL_VOC_NUM_CLASSES self._class_to_index = dict( zip(PASCAL_VOC_CLASSES, xrange(self.num_classes))) # load the voc dataset self.voc_root = self.load_voc(image_set, year, path) self.cache_file = os.path.join(self.voc_root, self.cache_file_name) # load the precomputed ss results from voc data, it includes both 2007 and 2012 data self.ss_path = self.load_voc('ss', None, path) # VOC paths and infos self.image_index_file = os.path.join(self.voc_root, 'ImageSets', 'Main', self.image_set + '.txt') self.image_path = os.path.join(self.voc_root, 'JPEGImages') self._image_file_ext = '.jpg' self.annotation_path = os.path.join(self.voc_root, 'Annotations') self._annotation_file_ext = '.xml' self._annotation_obj_tag = 'object' self._annotation_class_tag = 'name' self._annotation_xmin_tag = 'xmin' self._annotation_xmax_tag = 'xmax' self._annotation_ymin_tag = 'ymin' self._annotation_ymax_tag = 'ymax' self._selective_search_ext = '.pkl' self.selective_search_file = os.path.join( self.ss_path, '_'.join(['voc', year, self.image_set, 'selectivesearch.pkl'])) self._bb_xmin_idx = 0 self._bb_ymin_idx = 1 self._bb_xmax_idx = 2 self._bb_ymax_idx = 3 # self.rois_per_batch is 128 (2*64) ROIs # But the image path batch size is self.img_per_batch # need to control the batch size here print "Backend batchsize is changed to be image_per_batch from PASCAL_VOC dataset" self.be.bsz = self.img_per_batch # backend tensor to push the data self.image_shape = (3, FRCN_MAX_SCALE, FRCN_MAX_SCALE) self.img_np = np.zeros( (3, FRCN_MAX_SCALE, FRCN_MAX_SCALE, self.be.bsz), dtype=np.float32) self.dev_X_img = self.be.iobuf(self.image_shape, dtype=np.float32) self.dev_X_img_chw = self.dev_X_img.reshape( 3, FRCN_MAX_SCALE, FRCN_MAX_SCALE, self.be.bsz) # for rois, features are 4 + 1 (idx within the batch) self.dev_X_rois = self.be.zeros((self.rois_per_batch, 5)) self.dev_y_labels_flat = self.be.zeros( (1, self.rois_per_batch), dtype=np.int32) self.dev_y_labels = self.be.zeros( (self.num_classes, self.rois_per_batch), dtype=np.int32) self.dev_y_bbtargets = self.be.zeros( (self.num_classes * 4, self.rois_per_batch)) self.dev_y_bbmask = self.be.zeros( (self.num_classes * 4, self.rois_per_batch)) # the shape will indicate the shape for 1st path (ImageNet model), and # 2nd path (ROIs) self.shape = [self.image_shape, self.num_classes * 4] # Need to do the following: # 1. load the image index list # 2. for each image, load the ground truth from pascal annotation # 3. load the selective search ROIs (this step needs gt ROIs) # 4.1. merge the ROIs # 4.2. may have to add the flipped images for training # 4.3. add the fields for max overlap and max overlapped classes # 4.4. add the bounding box targets for regression # 5. during minibatch feeding: # - rescale images # - rescale ROIs # - random select foreground ROIs (bigger ones) # - random select background ROIS (smaller ones) # - clamp bg ROI labels (to be 0) # - convert ROIs into the regression target (ROIs, 4*21) # 1. assert os.path.exists(self.image_index_file), \ 'Image index file does not exist: {}'.format(self.image_index_file) with open(self.image_index_file) as f: self.image_index = [x.strip() for x in f.readlines()] # self.image_index = image_index * 2 if self.add_flipped else image_index self.num_images = len(self.image_index) self.num_image_entries = self.num_images * \ 2 if self.add_flipped else self.num_images self.ndata = self.num_image_entries * self.rois_per_image self.nbatches = self.num_image_entries/self.img_per_batch if n_mb is not None: self.nbatches = n_mb if os.path.exists(self.cache_file): self.roi_db = load_obj(self.cache_file) print 'ROI dataset loaded from file {}'.format(self.cache_file) else: # 2. self.roi_gt = self.load_pascal_roi_groundtruth() # 3. self.roi_ss = self.load_pascal_roi_selectivesearch() # 4. self.roi_db = self.combine_gt_ss_roi() save_obj(self.roi_db, self.cache_file) print 'wrote ROI dataset to {}'.format(self.cache_file)