Ejemplo n.º 1
0
def load_cifar100():
    base_dir = '/home/sainbar/data/cifar-100-python/'
    train_file = util.load(base_dir + 'train')
    train_data = train_file['data']
    train_data = train_data.T.copy()
    train_data = train_data.astype(np.float32)

    test_file = util.load(base_dir + 'test')
    test_data = test_file['data']
    test_data = test_data.T.copy()
    test_data = test_data.astype(np.float32)

    train_labels = np.asarray(train_file['fine_labels'], np.float32)
    test_labels = np.asarray(test_file['fine_labels'], np.float32)

    return train_data, train_labels, test_data, test_labels
Ejemplo n.º 2
0
def load_cifar10():
    base_dir = get_data_path() + 'cifar-10/train/'
    batch_meta = util.load(base_dir + 'batches.meta')
    data_file1 = util.load(base_dir + 'data_batch_1')
    data_file2 = util.load(base_dir + 'data_batch_2')
    data_file3 = util.load(base_dir + 'data_batch_3')
    data_file4 = util.load(base_dir + 'data_batch_4')
    data_file5 = util.load(base_dir + 'data_batch_5')
    data_file6 = util.load(base_dir + 'data_batch_6')
    labels1 = np.array(data_file1['labels'])
    labels2 = np.array(data_file2['labels'])
    labels3 = np.array(data_file3['labels'])
    labels4 = np.array(data_file4['labels'])
    labels5 = np.array(data_file5['labels'])
    labels6 = np.array(data_file6['labels'])

    train_data = np.concatenate(
        (data_file1['data'], data_file2['data'], data_file3['data'],
         data_file4['data'], data_file5['data']),
        axis=1)
    train_labels = np.concatenate(
        (labels1, labels2, labels3, labels4, labels5), axis=1)
    test_data = data_file6['data']
    test_labels = labels6
    train_data = train_data.astype(np.float32).copy()
    test_data = test_data.astype(np.float32).copy()
    return train_data, train_labels, test_data, test_labels
Ejemplo n.º 3
0
  def get_next_batch(self):
    self.get_next_index()
    filename = os.path.join(self.data_dir, 'data_batch_%d' % self.curr_batch)

    data = util.load(filename)
    img = data['data'] - self.batch_meta['data_mean']
    return BatchData(np.require(img, requirements='C', dtype=np.float32),
                     np.array(data['labels']),
                     self.curr_epoch)
Ejemplo n.º 4
0
  def get_next_batch(self):
    self.get_next_index()
    filename = os.path.join(self.data_dir, 'data_batch_%d' % self.curr_batch)

    data = util.load(filename)
    img = data['data'] - self.batch_meta['data_mean']
    return BatchData(np.require(img, requirements='C', dtype=np.float32),
                     np.array(data['labels']),
                     self.curr_epoch, self.curr_batch_index)
Ejemplo n.º 5
0
  def get_next_batch(self):
    self.get_next_index()

    filename = os.path.join(self.data_dir + '.%s' % self.curr_batch)
    util.log('reading from %s', filename)

    data_dic = util.load(filename)
    data  = data_dic[self.data_name].transpose()
    labels = data_dic['labels']
    data = np.require(data, requirements='C', dtype=np.float32)
    return BatchData(data, labels, self.curr_epoch)
Ejemplo n.º 6
0
  def get_next_batch(self):
    self.get_next_index()

    filename = os.path.join(self.data_dir + '.%s' % self.curr_batch)
    util.log('reading from %s', filename)

    data_dic = util.load(filename)
    data  = data_dic[self.data_name].transpose()
    labels = data_dic['labels']
    data = np.require(data, requirements='C', dtype=np.float32)
    return BatchData(data, labels, self.curr_epoch)
Ejemplo n.º 7
0
  def get_next_batch(self):
    self.get_next_index()
    filename = os.path.join(self.data_dir, 'data_batch_%d' % self.curr_batch)

    data = util.load(filename)
    img = data['data']
    img = img.reshape((3, 32, 32, len(data['labels'])))
    img = img.transpose(3, 0, 1, 2)
    cropped = np.ndarray((self.data_dim, len(data['labels']) * self.num_view), dtype=np.float32)

    self._trim_borders(img, cropped)
    cropped -= self.batch_meta['data_mean']

    return BatchData(np.require(cropped, requirements='C', dtype=np.float32),
                     np.array(data['labels']),
                     self.curr_epoch)
Ejemplo n.º 8
0
    def get_next_batch(self):
        self.get_next_index()
        filename = os.path.join(self.data_dir,
                                'data_batch_%d' % self.curr_batch)

        data = util.load(filename)
        img = data['data']
        img = img.reshape((3, 32, 32, len(data['labels'])))
        img = img.transpose(3, 0, 1, 2)
        cropped = np.ndarray(
            (self.data_dim, len(data['labels']) * self.num_view),
            dtype=np.float32)

        self._trim_borders(img, cropped)
        cropped -= self.batch_meta['data_mean']

        return BatchData(
            np.require(cropped, requirements='C', dtype=np.float32),
            np.array(data['labels']), self.curr_epoch)
Ejemplo n.º 9
0
  def __init__(self, data_dir='.', batch_range=None):
    self.data_dir = data_dir
    self.meta_file = os.path.join(data_dir, 'batches.meta')

    self.curr_batch_index = 0
    self.curr_batch = None
    self.curr_epoch = 1

    if os.path.exists(self.meta_file):
      self.batch_meta = util.load(self.meta_file)
    else:
      util.log_warn('Missing metadata for loader.')

    if batch_range is None:
      self.batch_range = self.get_batch_indexes()
    else:
      self.batch_range = batch_range
    random.shuffle(self.batch_range)

    self.index = 0
Ejemplo n.º 10
0
  def __init__(self, data_dir='.', batch_range=None):
    self.data_dir = data_dir
    self.meta_file = os.path.join(data_dir, 'batches.meta')

    self.multiview = 0
    self.curr_batch_index = 0
    self.curr_batch = None
    self.curr_epoch = 1

    if os.path.exists(self.meta_file):
      self.batch_meta = util.load(self.meta_file)
    else:
      print 'No default meta file \'batches.meta\', using another meta file'

    if batch_range is None:
      self.batch_range = self.get_batch_indexes()
    else:
      self.batch_range = batch_range
    random.shuffle(self.batch_range)

    self.index = 0
Ejemplo n.º 11
0
  def __init__(self, data_dir='.', batch_range=None):
    self.data_dir = data_dir
    self.meta_file = os.path.join(data_dir, 'batches.meta')

    self.multiview = 0
    self.curr_batch_index = 0
    self.curr_batch = None
    self.curr_epoch = 1

    if os.path.exists(self.meta_file):
      self.batch_meta = util.load(self.meta_file)
    else:
      print 'No default meta file \'batches.meta\', using another meta file'

    if batch_range is None:
      self.batch_range = self.get_batch_indexes()
    else:
      self.batch_range = batch_range
    random.shuffle(self.batch_range)

    self.index = 0
Ejemplo n.º 12
0
  def __init__(self, data_dir,
               batch_range=None,
               multiview = False,
               category_range=None,
               scale=1,
               batch_size=1024):
    DataProvider.__init__(self, data_dir, batch_range)
    self.multiview = multiview
    self.batch_size = batch_size

    self.scale = scale

    self.img_size = ImageNetDataProvider.img_size / scale
    self.border_size = ImageNetDataProvider.border_size / scale
    self.inner_size = self.img_size - self.border_size * 2

    if self.multiview:
      self.batch_size = 12

    self.images = _prepare_images(data_dir, category_range, batch_range, self.batch_meta)
    self.num_view = 5 * 2 if self.multiview else 1

    assert len(self.images) > 0

    self._shuffle_batches()

    if 'data_mean' in self.batch_meta:
      data_mean = self.batch_meta['data_mean']
    else:
      data_mean = util.load(data_dir + 'image-mean.pickle')['data']

    self.data_mean = (data_mean
        .astype(np.single)
        .T
        .reshape((3, 256, 256))[:,
                                self.border_size:self.border_size + self.inner_size,
                                self.border_size:self.border_size + self.inner_size]
        .reshape((self.data_dim, 1)))
    util.log('Starting data provider with %d batches', len(self.batches))
Ejemplo n.º 13
0
    def __init__(self,
                 data_dir,
                 batch_range=None,
                 multiview=False,
                 category_range=None,
                 scale=1,
                 batch_size=1024):
        DataProvider.__init__(self, data_dir, batch_range)
        self.multiview = multiview
        self.batch_size = batch_size

        self.scale = scale

        self.img_size = ImageNetDataProvider.img_size / scale
        self.border_size = ImageNetDataProvider.border_size / scale
        self.inner_size = self.img_size - self.border_size * 2

        if self.multiview:
            self.batch_size = 12

        self.images = _prepare_images(data_dir, category_range, batch_range,
                                      self.batch_meta)
        self.num_view = 5 * 2 if self.multiview else 1

        assert len(self.images) > 0

        self._shuffle_batches()

        if 'data_mean' in self.batch_meta:
            data_mean = self.batch_meta['data_mean']
        else:
            data_mean = util.load(data_dir + 'image-mean.pickle')['data']

        self.data_mean = (data_mean.astype(np.single).T.reshape(
            (3, 256, 256))[:,
                           self.border_size:self.border_size + self.inner_size,
                           self.border_size:self.border_size +
                           self.inner_size].reshape((self.data_dim, 1)))
        util.log('Starting data provider with %d batches', len(self.batches))
Ejemplo n.º 14
0
    def __init__(self, data_dir='.', batch_range=None):
        self.data_dir = data_dir
        self.meta_file = os.path.join(data_dir, 'batches.meta')

        self.curr_batch_index = 0
        self.curr_batch = None
        self.curr_epoch = 1

        if os.path.exists(self.meta_file):
            self.batch_meta = util.load(self.meta_file)
        else:
            util.log_warn('Missing metadata for loader.')

        if batch_range is None:
            self.batch_range = self.get_batch_indexes()
        else:
            self.batch_range = batch_range

        util.log('Batch range: %s', self.batch_range)
        random.shuffle(self.batch_range)

        self.index = 0
        self._handle_new_epoch()
Ejemplo n.º 15
0
def load_cifar10():
    base_dir = get_data_path() + 'cifar-10/train/'
    batch_meta = util.load(base_dir + 'batches.meta')
    data_file1 = util.load(base_dir + 'data_batch_1')
    data_file2 = util.load(base_dir + 'data_batch_2')
    data_file3 = util.load(base_dir + 'data_batch_3')
    data_file4 = util.load(base_dir + 'data_batch_4')
    data_file5 = util.load(base_dir + 'data_batch_5')
    data_file6 = util.load(base_dir + 'data_batch_6')
    labels1 = np.array(data_file1['labels'])
    labels2 = np.array(data_file2['labels'])
    labels3 = np.array(data_file3['labels'])
    labels4 = np.array(data_file4['labels'])
    labels5 = np.array(data_file5['labels'])
    labels6 = np.array(data_file6['labels'])
    
    train_data = np.concatenate((data_file1['data'],data_file2['data'],data_file3['data'],data_file4['data'],data_file5['data']), axis=1)
    train_labels = np.concatenate((labels1,labels2,labels3,labels4,labels5), axis=1)
    test_data = data_file6['data']
    test_labels = labels6
    train_data = train_data.astype(np.float32).copy()
    test_data = test_data.astype(np.float32).copy()
    return train_data, train_labels, test_data, test_labels