def generate(self):
while True:
rand_idxs = np.arange(len(self.img_ids))
if self.random:
self.rng.shuffle(rand_idxs)
for pos in xrange(0, len(rand_idxs), self.batch_size):
idxs_batch = rand_idxs[pos:pos + self.batch_size]
nb = len(idxs_batch)
# allocate batches
x_batch = []
y_batch = []
batch_ids = []
for i, idx in enumerate(idxs_batch):
img_id = self.img_ids[idx]
batch_ids.append(img_id)
try:
img = app.read_compressed_image(self.dataset, img_id)
except Exception:
print 'cannot open ', img_id
x_batch.append(self.data_prep_fun(x=img))
if 'train' in self.dataset:
y_batch.append(self.label_prep_fun(
self.labels[img_id]))
if self.full_batch:
if nb == self.batch_size:
yield x_batch, y_batch, batch_ids
else:
yield x_batch, y_batch, batch_ids
if not self.infinite:
break
def generate(self):
while True:
rand_pos_idxs = np.arange(len(self.pos_ids))
rand_neg_idxs = np.arange(len(self.neg_ids))
if self.random:
self.rng.shuffle(rand_pos_idxs)
self.rng.shuffle(rand_neg_idxs)
min_idxs = min(len(rand_pos_idxs), len(rand_neg_idxs))
for pos in xrange(0, min_idxs, self.batch_size // 2):
pos_idxs_batch = rand_pos_idxs[pos:pos + self.batch_size // 2]
neg_idxs_batch = rand_neg_idxs[pos:pos + self.batch_size // 2]
idxs_batch = np.concatenate([pos_idxs_batch, neg_idxs_batch])
nb = len(idxs_batch)
# allocate batches
if self.p_transform['channels']:
x_batch = np.zeros((
nb,
self.p_transform['channels'],
) + self.p_transform['patch_size'],
dtype='float32')
else:
x_batch = np.zeros((nb, ) + self.p_transform['patch_size'],
dtype='float32')
y_batch = np.zeros((nb, ), dtype='float32')
batch_ids = []
for i, idx in enumerate(pos_idxs_batch):
#print i, idx, self.pos_ids[idx]
img_id = self.pos_ids[idx]
batch_ids.append(img_id)
for i, idx in zip(
np.arange(len(pos_idxs_batch),
len(pos_idxs_batch) + len(neg_idxs_batch)),
neg_idxs_batch):
#print '-', i, idx, self.neg_ids[idx]
img_id = self.neg_ids[idx]
batch_ids.append(img_id)
for i, img_id in enumerate(batch_ids):
# print img_id
img = app.read_compressed_image(self.dataset, img_id)
x_batch[i] = self.data_prep_fun(x=img)
y_batch[i] = self.label_prep_fun(self.labels[img_id])
if self.full_batch:
if nb == self.batch_size:
yield x_batch, y_batch, batch_ids
else:
yield x_batch, y_batch, batch_ids
if not self.infinite:
break
def generate(self):
while True:
rand_idxs = np.arange(len(self.img_ids))
if self.random:
self.rng.shuffle(rand_idxs)
for pos in xrange(0, len(rand_idxs), self.batch_size):
idxs_batch = rand_idxs[pos:pos + self.batch_size]
nb = len(idxs_batch)
# allocate batches
if self.p_transform['channels']:
x_batch = np.zeros((
nb,
self.p_transform['channels'],
) + self.patch_size,
dtype='float32')
else:
x_batch = np.zeros((nb, ) + self.patch_size,
dtype='float32')
if self.p_transform['n_labels'] > 1:
y_batch = np.zeros((nb, self.p_transform['n_labels']),
dtype='float32')
else:
y_batch = np.zeros((nb, ), dtype='float32')
batch_ids = []
for i, idx in enumerate(idxs_batch):
img_id = self.img_ids[idx]
batch_ids.append(img_id)
try:
img = app.read_compressed_image(self.dataset, img_id)
except Exception:
print 'cannot open ', img_id
x_batch[i] = self.data_prep_fun(x=img)
if 'train' in self.dataset:
y_batch[i] = self.label_prep_fun(self.labels[img_id])
#print 'i', i, 'img_id', img_id, y_batch[i]
if self.full_batch:
if nb == self.batch_size:
yield x_batch, y_batch, batch_ids
else:
yield x_batch, y_batch, batch_ids
if not self.infinite:
break
def generate(self):
rand_idxs = np.arange(len(self.img_ids))
if self.random:
self.rng.shuffle(rand_idxs)
for pos in xrange(len(rand_idxs)):
imid = rand_idxs[pos]
nb = self.batch_size
# allocate batches
if self.p_transform['channels']:
x_batch = np.zeros((
nb,
self.p_transform['channels'],
) + self.p_transform['patch_size'],
dtype='float32')
else:
x_batch = np.zeros((nb, ) + self.p_transform['patch_size'],
dtype='float32')
if self.p_transform['n_labels'] > 1:
y_batch = np.zeros((nb, self.p_transform['n_labels']),
dtype='float32')
else:
y_batch = np.zeros((nb, ), dtype='float32')
img_id = self.img_ids[imid]
try:
img = app.read_compressed_image(self.dataset, img_id)
except Exception:
print 'cannot open ', self.dataset, img_id
# if self.p_transform['channels'] == 4:
# x_batch = self.tta.make_augmentations(np.swapaxes(self.data_prep_fun(x=img),0,2))
# else:
x_batch = self.tta.make_augmentations(self.data_prep_fun(x=img))
if self.duplicate_label:
y_batch = self.tta.duplicate_label(
self.label_prep_fun(self.labels[img_id]))
yield x_batch, y_batch, img_id
def generate(self):
while True:
pos_rand_ids = np.arange(len(self.pos_ids))
neg_rand_ids = np.arange(len(self.neg_ids))
neg_ptr = 0
if self.random:
self.rng.shuffle(pos_rand_ids)
self.rng.shuffle(neg_rand_ids)
for pos_, neg_ in itertools.izip(
xrange(0, len(pos_rand_ids), self.pos_batch_size),
xrange(0, len(neg_rand_ids), self.neg_batch_size)):
pos_ids_batch = pos_rand_ids[pos_:pos_ + self.pos_batch_size]
neg_ids_batch = neg_rand_ids[neg_:neg_ + self.neg_batch_size]
nb_pos = len(pos_ids_batch)
nb_neg = len(neg_ids_batch)
nb = nb_pos + nb_neg
pos_ids_batch = np.array(
[self.pos_ids[pid] for pid in pos_ids_batch])
neg_ids_batch = np.array(
[self.neg_ids[pid] for pid in neg_ids_batch])
idxs_batch = np.hstack([pos_ids_batch, neg_ids_batch])
# allocate batches
if self.p_transform['channels']:
x_batch = np.zeros((
nb,
self.p_transform['channels'],
) + self.p_transform['patch_size'],
dtype='float32')
else:
x_batch = np.zeros((nb, ) + self.p_transform['patch_size'],
dtype='float32')
if self.p_transform['n_labels'] > 1:
y_batch = np.zeros((nb, self.p_transform['n_labels']),
dtype='float32')
else:
y_batch = np.zeros((nb, ), dtype='float32')
batch_ids = []
for i, idx in enumerate(idxs_batch):
img_id = self.img_ids[idx]
batch_ids.append(img_id)
try:
img = app.read_compressed_image(self.dataset, img_id)
except Exception:
print 'cannot open ', img_id
x_batch[i] = self.data_prep_fun(x=img)
if 'train' in self.dataset:
y_batch[i] = self.label_prep_fun(self.labels[img_id])
#print 'i', i, 'img_id', img_id, y_batch[i]
if self.full_batch:
if nb == self.batch_size:
yield x_batch, y_batch, batch_ids
else:
yield x_batch, y_batch, batch_ids
if not self.infinite:
break