def load_cifar100(): base_dir = '/home/sainbar/data/cifar-100-python/' train_file = util.load(base_dir + 'train') train_data = train_file['data'] train_data = train_data.T.copy() train_data = train_data.astype(np.float32) test_file = util.load(base_dir + 'test') test_data = test_file['data'] test_data = test_data.T.copy() test_data = test_data.astype(np.float32) train_labels = np.asarray(train_file['fine_labels'], np.float32) test_labels = np.asarray(test_file['fine_labels'], np.float32) return train_data, train_labels, test_data, test_labels
def load_cifar10(): base_dir = get_data_path() + 'cifar-10/train/' batch_meta = util.load(base_dir + 'batches.meta') data_file1 = util.load(base_dir + 'data_batch_1') data_file2 = util.load(base_dir + 'data_batch_2') data_file3 = util.load(base_dir + 'data_batch_3') data_file4 = util.load(base_dir + 'data_batch_4') data_file5 = util.load(base_dir + 'data_batch_5') data_file6 = util.load(base_dir + 'data_batch_6') labels1 = np.array(data_file1['labels']) labels2 = np.array(data_file2['labels']) labels3 = np.array(data_file3['labels']) labels4 = np.array(data_file4['labels']) labels5 = np.array(data_file5['labels']) labels6 = np.array(data_file6['labels']) train_data = np.concatenate( (data_file1['data'], data_file2['data'], data_file3['data'], data_file4['data'], data_file5['data']), axis=1) train_labels = np.concatenate( (labels1, labels2, labels3, labels4, labels5), axis=1) test_data = data_file6['data'] test_labels = labels6 train_data = train_data.astype(np.float32).copy() test_data = test_data.astype(np.float32).copy() return train_data, train_labels, test_data, test_labels
def get_next_batch(self): self.get_next_index() filename = os.path.join(self.data_dir, 'data_batch_%d' % self.curr_batch) data = util.load(filename) img = data['data'] - self.batch_meta['data_mean'] return BatchData(np.require(img, requirements='C', dtype=np.float32), np.array(data['labels']), self.curr_epoch)
def get_next_batch(self): self.get_next_index() filename = os.path.join(self.data_dir, 'data_batch_%d' % self.curr_batch) data = util.load(filename) img = data['data'] - self.batch_meta['data_mean'] return BatchData(np.require(img, requirements='C', dtype=np.float32), np.array(data['labels']), self.curr_epoch, self.curr_batch_index)
def get_next_batch(self): self.get_next_index() filename = os.path.join(self.data_dir + '.%s' % self.curr_batch) util.log('reading from %s', filename) data_dic = util.load(filename) data = data_dic[self.data_name].transpose() labels = data_dic['labels'] data = np.require(data, requirements='C', dtype=np.float32) return BatchData(data, labels, self.curr_epoch)
def get_next_batch(self): self.get_next_index() filename = os.path.join(self.data_dir, 'data_batch_%d' % self.curr_batch) data = util.load(filename) img = data['data'] img = img.reshape((3, 32, 32, len(data['labels']))) img = img.transpose(3, 0, 1, 2) cropped = np.ndarray((self.data_dim, len(data['labels']) * self.num_view), dtype=np.float32) self._trim_borders(img, cropped) cropped -= self.batch_meta['data_mean'] return BatchData(np.require(cropped, requirements='C', dtype=np.float32), np.array(data['labels']), self.curr_epoch)
def get_next_batch(self): self.get_next_index() filename = os.path.join(self.data_dir, 'data_batch_%d' % self.curr_batch) data = util.load(filename) img = data['data'] img = img.reshape((3, 32, 32, len(data['labels']))) img = img.transpose(3, 0, 1, 2) cropped = np.ndarray( (self.data_dim, len(data['labels']) * self.num_view), dtype=np.float32) self._trim_borders(img, cropped) cropped -= self.batch_meta['data_mean'] return BatchData( np.require(cropped, requirements='C', dtype=np.float32), np.array(data['labels']), self.curr_epoch)
def __init__(self, data_dir='.', batch_range=None): self.data_dir = data_dir self.meta_file = os.path.join(data_dir, 'batches.meta') self.curr_batch_index = 0 self.curr_batch = None self.curr_epoch = 1 if os.path.exists(self.meta_file): self.batch_meta = util.load(self.meta_file) else: util.log_warn('Missing metadata for loader.') if batch_range is None: self.batch_range = self.get_batch_indexes() else: self.batch_range = batch_range random.shuffle(self.batch_range) self.index = 0
def __init__(self, data_dir='.', batch_range=None): self.data_dir = data_dir self.meta_file = os.path.join(data_dir, 'batches.meta') self.multiview = 0 self.curr_batch_index = 0 self.curr_batch = None self.curr_epoch = 1 if os.path.exists(self.meta_file): self.batch_meta = util.load(self.meta_file) else: print 'No default meta file \'batches.meta\', using another meta file' if batch_range is None: self.batch_range = self.get_batch_indexes() else: self.batch_range = batch_range random.shuffle(self.batch_range) self.index = 0
def __init__(self, data_dir, batch_range=None, multiview = False, category_range=None, scale=1, batch_size=1024): DataProvider.__init__(self, data_dir, batch_range) self.multiview = multiview self.batch_size = batch_size self.scale = scale self.img_size = ImageNetDataProvider.img_size / scale self.border_size = ImageNetDataProvider.border_size / scale self.inner_size = self.img_size - self.border_size * 2 if self.multiview: self.batch_size = 12 self.images = _prepare_images(data_dir, category_range, batch_range, self.batch_meta) self.num_view = 5 * 2 if self.multiview else 1 assert len(self.images) > 0 self._shuffle_batches() if 'data_mean' in self.batch_meta: data_mean = self.batch_meta['data_mean'] else: data_mean = util.load(data_dir + 'image-mean.pickle')['data'] self.data_mean = (data_mean .astype(np.single) .T .reshape((3, 256, 256))[:, self.border_size:self.border_size + self.inner_size, self.border_size:self.border_size + self.inner_size] .reshape((self.data_dim, 1))) util.log('Starting data provider with %d batches', len(self.batches))
def __init__(self, data_dir, batch_range=None, multiview=False, category_range=None, scale=1, batch_size=1024): DataProvider.__init__(self, data_dir, batch_range) self.multiview = multiview self.batch_size = batch_size self.scale = scale self.img_size = ImageNetDataProvider.img_size / scale self.border_size = ImageNetDataProvider.border_size / scale self.inner_size = self.img_size - self.border_size * 2 if self.multiview: self.batch_size = 12 self.images = _prepare_images(data_dir, category_range, batch_range, self.batch_meta) self.num_view = 5 * 2 if self.multiview else 1 assert len(self.images) > 0 self._shuffle_batches() if 'data_mean' in self.batch_meta: data_mean = self.batch_meta['data_mean'] else: data_mean = util.load(data_dir + 'image-mean.pickle')['data'] self.data_mean = (data_mean.astype(np.single).T.reshape( (3, 256, 256))[:, self.border_size:self.border_size + self.inner_size, self.border_size:self.border_size + self.inner_size].reshape((self.data_dim, 1))) util.log('Starting data provider with %d batches', len(self.batches))
def __init__(self, data_dir='.', batch_range=None): self.data_dir = data_dir self.meta_file = os.path.join(data_dir, 'batches.meta') self.curr_batch_index = 0 self.curr_batch = None self.curr_epoch = 1 if os.path.exists(self.meta_file): self.batch_meta = util.load(self.meta_file) else: util.log_warn('Missing metadata for loader.') if batch_range is None: self.batch_range = self.get_batch_indexes() else: self.batch_range = batch_range util.log('Batch range: %s', self.batch_range) random.shuffle(self.batch_range) self.index = 0 self._handle_new_epoch()
def load_cifar10(): base_dir = get_data_path() + 'cifar-10/train/' batch_meta = util.load(base_dir + 'batches.meta') data_file1 = util.load(base_dir + 'data_batch_1') data_file2 = util.load(base_dir + 'data_batch_2') data_file3 = util.load(base_dir + 'data_batch_3') data_file4 = util.load(base_dir + 'data_batch_4') data_file5 = util.load(base_dir + 'data_batch_5') data_file6 = util.load(base_dir + 'data_batch_6') labels1 = np.array(data_file1['labels']) labels2 = np.array(data_file2['labels']) labels3 = np.array(data_file3['labels']) labels4 = np.array(data_file4['labels']) labels5 = np.array(data_file5['labels']) labels6 = np.array(data_file6['labels']) train_data = np.concatenate((data_file1['data'],data_file2['data'],data_file3['data'],data_file4['data'],data_file5['data']), axis=1) train_labels = np.concatenate((labels1,labels2,labels3,labels4,labels5), axis=1) test_data = data_file6['data'] test_labels = labels6 train_data = train_data.astype(np.float32).copy() test_data = test_data.astype(np.float32).copy() return train_data, train_labels, test_data, test_labels