def write_batches(args, macrodir, datadir, val_pct): if os.path.exists(macrodir): return print('Writing batches to %s' % macrodir) bw = BatchWriter(out_dir=macrodir, image_dir=os.path.join(args.data_dir, datadir), target_size=32, macro_size=1024, file_pattern='*.jpg', validation_pct=val_pct) bw.run()
def load(self, backend=None, experiment=None): ''' Imageset only supports nervanagpu based backends ''' if not hasattr(self.backend, 'ng'): raise DeprecationWarning("Only nervanagpu-based backends " "supported. For using cudanet backend, " "revert to neon 0.8.2 ") bdir = os.path.expanduser(self.save_dir) cachefile = os.path.join(bdir, 'dataset_cache.pkl') if not os.path.exists(cachefile): logger.error("Batch dir cache not found in %s:", cachefile) response = raw_input("Press Y to create, otherwise exit: ") if response == 'Y': from neon.util.batch_writer import (BatchWriter, BatchWriterImagenet) if self.imageset.startswith('I1K'): self.bw = BatchWriterImagenet(**self.__dict__) else: self.bw = BatchWriter(**self.__dict__) self.bw.run() logger.error('Done writing batches - please rerun to train.') else: logger.error('Exiting...') sys.exit() cstats = deserialize(cachefile, verbose=False) if cstats['macro_size'] != self.macro_size: raise NotImplementedError("Cached macro size %d different from " "specified %d, delete save_dir %s " "and try again.", cstats['macro_size'], self.macro_size, self.save_dir) # Set the max indexes of batches for each from the cache file self.maxval = cstats['nval'] + cstats['val_start'] - 1 self.maxtrain = cstats['ntrain'] + cstats['train_start'] - 1 # Make sure only those properties not by yaml are updated cstats.update(self.__dict__) self.__dict__.update(cstats) # Should also put (in addition to nclass), number of train/val images req_param(self, ['ntrain', 'nval', 'train_start', 'val_start', 'train_mean', 'val_mean', 'labels_dict'])
class Imageset(Dataset): """ Sets up a macro batched imageset dataset. Assumes you have the data already partitioned and in macrobatch format Attributes: backend (neon.backends.Backend): backend used for this data inputs (dict): structure housing the loaded train/test/validation input data targets (dict): structure housing the loaded train/test/validation target data Keyword Args: repo_path (str, optional): where to locally host this dataset on disk """ def __init__(self, **kwargs): opt_param(self, ['preprocess_done'], False) opt_param(self, ['dotransforms', 'square_crop'], False) opt_param(self, ['mean_norm', 'unit_norm'], False) opt_param(self, ['shuffle_macro'], False) opt_param(self, ['tdims'], 0) opt_param(self, ['label_list'], ['l_id']) opt_param(self, ['num_channels'], 3) opt_param(self, ['num_workers'], 6) opt_param(self, ['backend_type'], 'np.float32') self.__dict__.update(kwargs) if self.backend_type in ['float16', 'np.float16', 'numpy.float16']: self.backend_type = np.float16 elif self.backend_type in ['float32', 'np.float32', 'numpy.float32']: self.backend_type = np.float32 else: raise ValueError('Datatype not understood') logger.warning("Imageset initialized with dtype %s", self.backend_type) req_param(self, ['cropped_image_size', 'output_image_size', 'imageset', 'save_dir', 'repo_path', 'macro_size']) opt_param(self, ['image_dir'], os.path.join(self.repo_path, self.imageset)) self.rgb = True if self.num_channels == 3 else False self.norm_factor = 128. if self.mean_norm else 256. self.img_dtype = np.int8 def __getstate__(self): """ Defines what and how we go about serializing an instance of this class. """ self.macro_decode_thread = None return self.__dict__ def __setstate__(self, state): """ Defines how we go about deserializing into an instance of this class. """ self.__dict__.update(state) def load(self, backend=None, experiment=None): ''' Imageset only supports nervanagpu based backends ''' if not hasattr(self.backend, 'ng'): raise DeprecationWarning("Only nervanagpu-based backends " "supported. For using cudanet backend, " "revert to neon 0.8.2 ") bdir = os.path.expanduser(self.save_dir) cachefile = os.path.join(bdir, 'dataset_cache.pkl') if not os.path.exists(cachefile): logger.error("Batch dir cache not found in %s:", cachefile) response = raw_input("Press Y to create, otherwise exit: ") if response == 'Y': from neon.util.batch_writer import (BatchWriter, BatchWriterImagenet) if self.imageset.startswith('I1K'): self.bw = BatchWriterImagenet(**self.__dict__) else: self.bw = BatchWriter(**self.__dict__) self.bw.run() logger.error('Done writing batches - please rerun to train.') else: logger.error('Exiting...') sys.exit() cstats = deserialize(cachefile, verbose=False) if cstats['macro_size'] != self.macro_size: raise NotImplementedError("Cached macro size %d different from " "specified %d, delete save_dir %s " "and try again.", cstats['macro_size'], self.macro_size, self.save_dir) # Set the max indexes of batches for each from the cache file self.maxval = cstats['nval'] + cstats['val_start'] - 1 self.maxtrain = cstats['ntrain'] + cstats['train_start'] - 1 # Make sure only those properties not by yaml are updated cstats.update(self.__dict__) self.__dict__.update(cstats) # Should also put (in addition to nclass), number of train/val images req_param(self, ['ntrain', 'nval', 'train_start', 'val_start', 'train_mean', 'val_mean', 'labels_dict']) def get_macro_batch(self): self.macro_idx = (self.macro_idx + 1 - self.startb) \ % self.nmacros + self.startb fname = os.path.join(self.save_dir, 'data_batch_{:d}'.format(self.macro_idx)) return deserialize(os.path.expanduser(fname), verbose=False) def del_mini_batch_producer(self): if self.macro_decode_thread is not None: self.macro_decode_thread.join() del self.inp_be def init_mini_batch_producer(self, batch_size, setname, predict=False): # local shortcuts sbe = self.backend.empty ibetype = self.img_dtype sn = 'val' if (setname == 'validation') else setname osz = self.output_image_size csz = self.cropped_image_size self.npixels = csz * csz * self.num_channels self.startb = getattr(self, sn + '_start') self.nmacros = getattr(self, 'n' + sn) self.maxmacros = getattr(self, 'max' + sn) if self.startb + self.nmacros - 1 > self.maxmacros: self.nmacros = self.maxmacros - self.startb + 1 logger.warning("Truncating n%s to %d", sn, self.nmacros) self.endb = self.startb + self.nmacros - 1 if self.endb == self.maxmacros: nrecs = getattr(self, sn + '_nrec') % self.macro_size + \ (self.nmacros - 1) * self.macro_size else: nrecs = self.nmacros * self.macro_size num_batches = nrecs / batch_size self.mean_img = getattr(self, sn + '_mean') self.mean_img.shape = (self.num_channels, osz, osz) pad = (osz - csz) / 2 self.mean_crop = self.mean_img[:, pad:(pad + csz), pad:(pad + csz)] if self.mean_norm: self.mean_val = self.mean_crop.reshape( (1, self.npixels)).astype(np.uint8) else: self.mean_val = 127 # Control params for macrobatch decoding thread self.macro_active_buf_idx = 0 self.macro_decode_buf_idx = 0 self.macro_num_decode_buf = 2 self.macro_decode_thread = None self.batch_size = batch_size self.predict = predict self.minis_per_macro = [self.macro_size / self.batch_size for i in range(self.macro_num_decode_buf)] if self.macro_size % self.batch_size != 0: raise ValueError('self.macro_size not divisible by batch_size') self.macro_idx = self.endb self.mini_idx = -1 # Allocate space for host side image, targets and labels self.int_macro = [np.zeros((self.macro_size, self.npixels), dtype=np.uint8) for i in range(self.macro_num_decode_buf)] self.img_macro = [self.backend.alloc_host_mem( (self.macro_size, self.npixels), dtype=ibetype) for i in range(self.macro_num_decode_buf)] self.target_macro = [None for i in range(self.macro_num_decode_buf)] self.lbl_one_hot = [{lbl: self.backend.alloc_host_mem( (self.macro_size, self.nclass[lbl]), dtype=ibetype) for lbl in self.label_list} for i in range(self.macro_num_decode_buf)] # Allocate space for device side buffers inp_shape = (self.npixels, self.batch_size) self.inp_be = self.backend.allocate_fragment(inp_shape, dtype=ibetype) self.inp_be.name = "minibatch" self.inp_beT = sbe(self.inp_be.shape[::-1], dtype=ibetype) lbl_shape = {lbl: (self.nclass[lbl], self.batch_size) for lbl in self.label_list} self.lbl_be = {lbl: sbe(lbl_shape[lbl], dtype=ibetype) for lbl in self.label_list} self.lbl_beT = {lbl: sbe(self.lbl_be[lbl].shape[::-1], dtype=ibetype) for lbl in self.label_list} if self.backend.is_dist: for lbl in self.label_list: self.lbl_be[lbl].ptype = 'replica' self.lbl_beT[lbl].ptype = 'replica' self.target_be = None return num_batches def get_mini_batch(self, batch_idx): batch_idx = self.macro_active_buf_idx self.mini_idx = (self.mini_idx + 1) % self.minis_per_macro[batch_idx] # Decode macrobatches in a background thread, # except for the first one which blocks if self.mini_idx == 0: if self.macro_decode_thread is not None: # No-op unless all mini finish faster than one macro self.macro_decode_thread.join() else: # special case for first run through self.macro_decode_thread = MacrobatchDecodeThread(self) self.macro_decode_thread.start() self.macro_decode_thread.join() # usual case for kicking off a background macrobatch thread self.macro_active_buf_idx = self.macro_decode_buf_idx self.macro_decode_buf_idx = \ (self.macro_decode_buf_idx + 1) % self.macro_num_decode_buf self.macro_decode_thread = MacrobatchDecodeThread(self) self.macro_decode_thread.start() # All minibatches except for the 0th just copy pre-prepared data batch_idx = self.macro_active_buf_idx start_idx = self.mini_idx * self.batch_size end_idx = (self.mini_idx + 1) * self.batch_size host_lbl = None # Copy into device and then transpose on device host_img = self.img_macro[batch_idx][start_idx:end_idx] self.backend.scatter(host_img, self.inp_beT) self.backend.transpose(self.inp_beT, self.inp_be) # Assume that the reference has to be replicated as long as # the output layer is fully-connected for lbl in self.label_list: host_lbl = self.lbl_one_hot[batch_idx][lbl][start_idx:end_idx] self.backend.set(self.lbl_beT[lbl], host_lbl) for lbl in self.label_list: self.backend.transpose(self.lbl_beT[lbl], self.lbl_be[lbl]) if self.unit_norm: self.backend.divide(self.inp_be, self.norm_factor, self.inp_be) return self.inp_be, self.target_be, self.lbl_be def has_set(self, setname): return True if (setname in ['train', 'validation']) else False
class Imageset(Dataset): """ Sets up a macro batched imageset dataset. Assumes you have the data already partitioned and in macrobatch format Attributes: backend (neon.backends.Backend): backend used for this data inputs (dict): structure housing the loaded train/test/validation input data targets (dict): structure housing the loaded train/test/validation target data Keyword Args: repo_path (str, optional): where to locally host this dataset on disk """ def __init__(self, **kwargs): opt_param(self, ['preprocess_done'], False) opt_param(self, ['dotransforms', 'square_crop'], True) opt_param(self, ['mean_norm', 'unit_norm'], False) opt_param(self, ['shuffle_macro'], False) opt_param(self, ['tdims'], 0) opt_param(self, ['label_list'], ['l_id']) opt_param(self, ['num_channels'], 3) opt_param(self, ['num_workers'], 6) opt_param(self, ['backend_type'], 'np.float32') self.__dict__.update(kwargs) if self.backend_type in ['float16', 'np.float16', 'numpy.float16']: self.backend_type = np.float16 elif self.backend_type in ['float32', 'np.float32', 'numpy.float32']: self.backend_type = np.float32 else: raise ValueError('Datatype not understood') logger.warning("Imageset initialized with dtype %s", self.backend_type) req_param(self, ['cropped_image_size', 'output_image_size', 'imageset', 'save_dir', 'repo_path', 'macro_size']) opt_param(self, ['image_dir'], os.path.join(self.repo_path, self.imageset)) self.rgb = True if self.num_channels == 3 else False self.norm_factor = 128. if self.mean_norm else 256. self.img_dtype = np.int8 def __getstate__(self): """ Defines what and how we go about serializing an instance of this class. """ self.macro_decode_thread = None return self.__dict__ def __setstate__(self, state): """ Defines how we go about deserializing into an instance of this class. """ self.__dict__.update(state) def load(self, backend=None, experiment=None): ''' Imageset only supports nervanagpu based backends ''' if not hasattr(self.backend, 'ng'): raise DeprecationWarning("Only nervanagpu-based backends " "supported. For using cudanet backend, " "revert to neon 0.8.2 ") bdir = os.path.expanduser(self.save_dir) cachefile = os.path.join(bdir, 'dataset_cache.pkl') if not os.path.exists(cachefile): logger.error("Batch dir cache not found in %s:", cachefile) response = raw_input("Press Y to create, otherwise exit: ") if response == 'Y': from neon.util.batch_writer import (BatchWriter, BatchWriterImagenet) if self.imageset.startswith('I1K'): self.bw = BatchWriterImagenet(**self.__dict__) else: self.bw = BatchWriter(**self.__dict__) self.bw.run() logger.error('Done writing batches - please rerun to train.') else: logger.error('Exiting...') sys.exit() cstats = deserialize(cachefile, verbose=False) if cstats['macro_size'] != self.macro_size: raise NotImplementedError("Cached macro size %d different from " "specified %d, delete save_dir %s " "and try again.", cstats['macro_size'], self.macro_size, self.save_dir) # Set the max indexes of batches for each from the cache file self.maxval = cstats['nval'] + cstats['val_start'] - 1 self.maxtrain = cstats['ntrain'] + cstats['train_start'] - 1 # Make sure only those properties not by yaml are updated cstats.update(self.__dict__) self.__dict__.update(cstats) # Should also put (in addition to nclass), number of train/val images req_param(self, ['ntrain', 'nval', 'train_start', 'val_start', 'train_mean', 'val_mean', 'labels_dict']) def get_macro_batch(self): self.macro_idx = (self.macro_idx + 1 - self.startb) \ % self.nmacros + self.startb fname = os.path.join(self.save_dir, 'data_batch_{:d}'.format(self.macro_idx)) return deserialize(os.path.expanduser(fname), verbose=False) def del_mini_batch_producer(self): if self.macro_decode_thread is not None: self.macro_decode_thread.join() del self.inp_be def init_mini_batch_producer(self, batch_size, setname, predict=False): # local shortcuts sbe = self.backend.empty ibetype = self.img_dtype sn = 'val' if (setname == 'validation') else setname osz = self.output_image_size csz = self.cropped_image_size self.npixels = csz * csz * self.num_channels self.startb = getattr(self, sn + '_start') self.nmacros = getattr(self, 'n' + sn) self.maxmacros = getattr(self, 'max' + sn) if self.startb + self.nmacros - 1 > self.maxmacros: self.nmacros = self.maxmacros - self.startb + 1 logger.warning("Truncating n%s to %d", sn, self.nmacros) self.endb = self.startb + self.nmacros - 1 if self.endb == self.maxmacros: nrecs = getattr(self, sn + '_nrec') % self.macro_size + \ (self.nmacros - 1) * self.macro_size else: nrecs = self.nmacros * self.macro_size num_batches = nrecs / batch_size self.mean_img = getattr(self, sn + '_mean') self.mean_img.shape = (self.num_channels, osz, osz) pad = (osz - csz) / 2 self.mean_crop = self.mean_img[:, pad:(pad + csz), pad:(pad + csz)] if self.mean_norm: self.mean_val = self.mean_crop.reshape( (1, self.npixels)).astype(np.uint8) else: self.mean_val = 127 # Control params for macrobatch decoding thread self.macro_active_buf_idx = 0 self.macro_decode_buf_idx = 0 self.macro_num_decode_buf = 2 self.macro_decode_thread = None self.batch_size = batch_size self.predict = predict self.minis_per_macro = [self.macro_size / self.batch_size for i in range(self.macro_num_decode_buf)] if self.macro_size % self.batch_size != 0: raise ValueError('self.macro_size not divisible by batch_size') self.macro_idx = self.endb self.mini_idx = -1 # Allocate space for host side image, targets and labels self.int_macro = [np.zeros((self.macro_size, self.npixels), dtype=np.uint8) for i in range(self.macro_num_decode_buf)] self.img_macro = [self.backend.alloc_host_mem( (self.macro_size, self.npixels), dtype=ibetype) for i in range(self.macro_num_decode_buf)] self.target_macro = [None for i in range(self.macro_num_decode_buf)] self.lbl_one_hot = [{lbl: self.backend.alloc_host_mem( (self.macro_size, self.nclass[lbl]), dtype=ibetype) for lbl in self.label_list} for i in range(self.macro_num_decode_buf)] # Allocate space for device side buffers inp_shape = (self.npixels, self.batch_size) self.inp_be = self.backend.allocate_fragment(inp_shape, dtype=ibetype) self.inp_be.name = "minibatch" self.inp_beT = sbe(self.inp_be.shape[::-1], dtype=ibetype) lbl_shape = {lbl: (self.nclass[lbl], self.batch_size) for lbl in self.label_list} self.lbl_be = {lbl: sbe(lbl_shape[lbl], dtype=ibetype) for lbl in self.label_list} self.lbl_beT = {lbl: sbe(self.lbl_be[lbl].shape[::-1], dtype=ibetype) for lbl in self.label_list} if self.backend.is_dist: for lbl in self.label_list: self.lbl_be[lbl].ptype = 'replica' self.lbl_beT[lbl].ptype = 'replica' self.target_be = None return num_batches def get_mini_batch(self, batch_idx): batch_idx = self.macro_active_buf_idx self.mini_idx = (self.mini_idx + 1) % self.minis_per_macro[batch_idx] # Decode macrobatches in a background thread, # except for the first one which blocks if self.mini_idx == 0: if self.macro_decode_thread is not None: # No-op unless all mini finish faster than one macro self.macro_decode_thread.join() else: # special case for first run through self.macro_decode_thread = MacrobatchDecodeThread(self) self.macro_decode_thread.start() self.macro_decode_thread.join() # usual case for kicking off a background macrobatch thread self.macro_active_buf_idx = self.macro_decode_buf_idx self.macro_decode_buf_idx = \ (self.macro_decode_buf_idx + 1) % self.macro_num_decode_buf self.macro_decode_thread = MacrobatchDecodeThread(self) self.macro_decode_thread.start() # All minibatches except for the 0th just copy pre-prepared data batch_idx = self.macro_active_buf_idx start_idx = self.mini_idx * self.batch_size end_idx = (self.mini_idx + 1) * self.batch_size host_lbl = None # Copy into device and then transpose on device host_img = self.img_macro[batch_idx][start_idx:end_idx] self.backend.scatter(host_img, self.inp_beT) self.backend.transpose(self.inp_beT, self.inp_be) # Assume that the reference has to be replicated as long as # the output layer is fully-connected for lbl in self.label_list: host_lbl = self.lbl_one_hot[batch_idx][lbl][start_idx:end_idx] self.backend.set(self.lbl_beT[lbl], host_lbl) for lbl in self.label_list: self.backend.transpose(self.lbl_beT[lbl], self.lbl_be[lbl]) if self.unit_norm: self.backend.divide(self.inp_be, self.norm_factor, self.inp_be) return self.inp_be, self.target_be, self.lbl_be def has_set(self, setname): return True if (setname in ['train', 'validation']) else False