def generate(self): while True: rand_idxs = np.arange(len(self.img_ids)) if self.random: self.rng.shuffle(rand_idxs) for pos in xrange(0, len(rand_idxs), self.batch_size): idxs_batch = rand_idxs[pos:pos + self.batch_size] nb = len(idxs_batch) # allocate batches x_batch = [] y_batch = [] batch_ids = [] for i, idx in enumerate(idxs_batch): img_id = self.img_ids[idx] batch_ids.append(img_id) try: img = app.read_compressed_image(self.dataset, img_id) except Exception: print 'cannot open ', img_id x_batch.append(self.data_prep_fun(x=img)) if 'train' in self.dataset: y_batch.append(self.label_prep_fun( self.labels[img_id])) if self.full_batch: if nb == self.batch_size: yield x_batch, y_batch, batch_ids else: yield x_batch, y_batch, batch_ids if not self.infinite: break
def generate(self): while True: rand_pos_idxs = np.arange(len(self.pos_ids)) rand_neg_idxs = np.arange(len(self.neg_ids)) if self.random: self.rng.shuffle(rand_pos_idxs) self.rng.shuffle(rand_neg_idxs) min_idxs = min(len(rand_pos_idxs), len(rand_neg_idxs)) for pos in xrange(0, min_idxs, self.batch_size // 2): pos_idxs_batch = rand_pos_idxs[pos:pos + self.batch_size // 2] neg_idxs_batch = rand_neg_idxs[pos:pos + self.batch_size // 2] idxs_batch = np.concatenate([pos_idxs_batch, neg_idxs_batch]) nb = len(idxs_batch) # allocate batches if self.p_transform['channels']: x_batch = np.zeros(( nb, self.p_transform['channels'], ) + self.p_transform['patch_size'], dtype='float32') else: x_batch = np.zeros((nb, ) + self.p_transform['patch_size'], dtype='float32') y_batch = np.zeros((nb, ), dtype='float32') batch_ids = [] for i, idx in enumerate(pos_idxs_batch): #print i, idx, self.pos_ids[idx] img_id = self.pos_ids[idx] batch_ids.append(img_id) for i, idx in zip( np.arange(len(pos_idxs_batch), len(pos_idxs_batch) + len(neg_idxs_batch)), neg_idxs_batch): #print '-', i, idx, self.neg_ids[idx] img_id = self.neg_ids[idx] batch_ids.append(img_id) for i, img_id in enumerate(batch_ids): # print img_id img = app.read_compressed_image(self.dataset, img_id) x_batch[i] = self.data_prep_fun(x=img) y_batch[i] = self.label_prep_fun(self.labels[img_id]) if self.full_batch: if nb == self.batch_size: yield x_batch, y_batch, batch_ids else: yield x_batch, y_batch, batch_ids if not self.infinite: break
def generate(self): while True: rand_idxs = np.arange(len(self.img_ids)) if self.random: self.rng.shuffle(rand_idxs) for pos in xrange(0, len(rand_idxs), self.batch_size): idxs_batch = rand_idxs[pos:pos + self.batch_size] nb = len(idxs_batch) # allocate batches if self.p_transform['channels']: x_batch = np.zeros(( nb, self.p_transform['channels'], ) + self.patch_size, dtype='float32') else: x_batch = np.zeros((nb, ) + self.patch_size, dtype='float32') if self.p_transform['n_labels'] > 1: y_batch = np.zeros((nb, self.p_transform['n_labels']), dtype='float32') else: y_batch = np.zeros((nb, ), dtype='float32') batch_ids = [] for i, idx in enumerate(idxs_batch): img_id = self.img_ids[idx] batch_ids.append(img_id) try: img = app.read_compressed_image(self.dataset, img_id) except Exception: print 'cannot open ', img_id x_batch[i] = self.data_prep_fun(x=img) if 'train' in self.dataset: y_batch[i] = self.label_prep_fun(self.labels[img_id]) #print 'i', i, 'img_id', img_id, y_batch[i] if self.full_batch: if nb == self.batch_size: yield x_batch, y_batch, batch_ids else: yield x_batch, y_batch, batch_ids if not self.infinite: break
def generate(self): rand_idxs = np.arange(len(self.img_ids)) if self.random: self.rng.shuffle(rand_idxs) for pos in xrange(len(rand_idxs)): imid = rand_idxs[pos] nb = self.batch_size # allocate batches if self.p_transform['channels']: x_batch = np.zeros(( nb, self.p_transform['channels'], ) + self.p_transform['patch_size'], dtype='float32') else: x_batch = np.zeros((nb, ) + self.p_transform['patch_size'], dtype='float32') if self.p_transform['n_labels'] > 1: y_batch = np.zeros((nb, self.p_transform['n_labels']), dtype='float32') else: y_batch = np.zeros((nb, ), dtype='float32') img_id = self.img_ids[imid] try: img = app.read_compressed_image(self.dataset, img_id) except Exception: print 'cannot open ', self.dataset, img_id # if self.p_transform['channels'] == 4: # x_batch = self.tta.make_augmentations(np.swapaxes(self.data_prep_fun(x=img),0,2)) # else: x_batch = self.tta.make_augmentations(self.data_prep_fun(x=img)) if self.duplicate_label: y_batch = self.tta.duplicate_label( self.label_prep_fun(self.labels[img_id])) yield x_batch, y_batch, img_id
def generate(self): while True: pos_rand_ids = np.arange(len(self.pos_ids)) neg_rand_ids = np.arange(len(self.neg_ids)) neg_ptr = 0 if self.random: self.rng.shuffle(pos_rand_ids) self.rng.shuffle(neg_rand_ids) for pos_, neg_ in itertools.izip( xrange(0, len(pos_rand_ids), self.pos_batch_size), xrange(0, len(neg_rand_ids), self.neg_batch_size)): pos_ids_batch = pos_rand_ids[pos_:pos_ + self.pos_batch_size] neg_ids_batch = neg_rand_ids[neg_:neg_ + self.neg_batch_size] nb_pos = len(pos_ids_batch) nb_neg = len(neg_ids_batch) nb = nb_pos + nb_neg pos_ids_batch = np.array( [self.pos_ids[pid] for pid in pos_ids_batch]) neg_ids_batch = np.array( [self.neg_ids[pid] for pid in neg_ids_batch]) idxs_batch = np.hstack([pos_ids_batch, neg_ids_batch]) # allocate batches if self.p_transform['channels']: x_batch = np.zeros(( nb, self.p_transform['channels'], ) + self.p_transform['patch_size'], dtype='float32') else: x_batch = np.zeros((nb, ) + self.p_transform['patch_size'], dtype='float32') if self.p_transform['n_labels'] > 1: y_batch = np.zeros((nb, self.p_transform['n_labels']), dtype='float32') else: y_batch = np.zeros((nb, ), dtype='float32') batch_ids = [] for i, idx in enumerate(idxs_batch): img_id = self.img_ids[idx] batch_ids.append(img_id) try: img = app.read_compressed_image(self.dataset, img_id) except Exception: print 'cannot open ', img_id x_batch[i] = self.data_prep_fun(x=img) if 'train' in self.dataset: y_batch[i] = self.label_prep_fun(self.labels[img_id]) #print 'i', i, 'img_id', img_id, y_batch[i] if self.full_batch: if nb == self.batch_size: yield x_batch, y_batch, batch_ids else: yield x_batch, y_batch, batch_ids if not self.infinite: break