def batch_gen(self, path, batch_size, crop_size):
content_path, mask_path = get_file_paths(path)
while True:
index = random.sample(range(1, len(content_path)), batch_size)
try:
offset_h = random.randint(0, (2448 - crop_size[0]))
offset_w = random.randint(0, (2448 - crop_size[1]))
offset = (offset_h, offset_w)
contents = [
vgg_sub_mean(
random_crop(get_image(content_path[i]), offset,
crop_size)) for i in index
]
masks = [
mask_preprocess(
random_crop(get_image(mask_path[i]), offset,
crop_size)) for i in index
]
contents = np.asarray(contents, dtype=np.float32)
masks = np.asarray(masks, dtype=np.uint8)
except Exception as err:
print("\nError: {}".format(err))
continue
yield contents, masks
def _get_one_triplet(input_data, input_labels):
input_labels = np.array(input_labels)
index = np.random.choice(n_labels, 2, replace=False)
label_positive = index[0]
label_negative = index[1]
indexes = utils.get_index(input_labels, index[0])
np.random.shuffle(indexes)
# print(indexes[0])
data_anchor = input_data[indexes[0], :, :, :]
data_anchor = utils.prewhiten(data_anchor)
data_anchor = utils.flip(data_anchor, random_flip=True)
data_anchor = utils.random_crop(data_anchor, image_size=299)
data_anchor = utils.random_rotate_image(data_anchor)
data_positive = input_data[indexes[1], :, :, :]
data_positive = utils.prewhiten(data_positive)
data_positive = utils.flip(data_positive, random_flip=True)
data_positive = utils.random_crop(data_positive, image_size=299)
data_positive = utils.random_rotate_image(data_positive)
indexes = utils.get_index(input_labels, index[1])
# print(indexes)
np.random.shuffle(indexes)
data_negative = input_data[indexes[0], :, :, :]
data_negative = utils.prewhiten(data_negative)
data_negative = utils.flip(data_negative, random_flip=True)
data_negative = utils.random_crop(data_negative, image_size=299)
data_negative = utils.random_rotate_image(data_negative)
# print(np.shape(data_negative))
return data_anchor, data_positive, data_negative, \
label_positive, label_positive, label_negative
def get_cropped_obs_batch(obs, next_obs):
obs = obs.astype(np.uint8)
next_obs = next_obs.astype(np.uint8)
cpu_obs_tmp = utils.random_crop(obs, args.image_size)
obs_tmp = torch.as_tensor(cpu_obs_tmp, device=device).float()
cpu_next_obs_tmp = utils.random_crop(next_obs, args.image_size)
next_obs_tmp = torch.as_tensor(cpu_next_obs_tmp, device=device).float()
return obs_tmp / 255, next_obs_tmp / 255
def sample(self, batch_size, cpc=False, noise=False): """We should probably handle the max_size<batch_size elsewhere?""" max_size = min(self.mem_cntr, self.mem_size) idxs = np.random.choice(max_size, size=batch_size, replace=False) if cpc: obses = self.s[idxs] next_obses = self.s_[idxs] pos = obses.detach().clone() obses = random_crop(obses.numpy(), 84) next_obses = random_crop(next_obses.numpy(), 84) pos = random_crop(pos.numpy(), 84) obses = torch.tensor(obses) next_obses = torch.tensor(next_obses) pos = torch.tensor(pos) cpc_kwargs = dict( obs_anchor=obses, obs_pos=pos, time_anchor=None, time_pos=None ) return obses, self.a[idxs], self.r[idxs], next_obses, self.t[idxs], cpc_kwargs if noise: obses = self.s[idxs] next_obses = self.s_[idxs] pos = obses.detach().clone() # Commented out for multi-experiments in a row without augmentations # obses = center_crop_image(obses.numpy(), 84) # next_obses = center_crop_image(next_obses.numpy(), 84) # pos = center_crop_image(pos.numpy(), 84) # # obses = torch.tensor(obses) # next_obses = torch.tensor(next_obses) # pos = torch.tensor(pos) cpc_kwargs = dict( obs_anchor=obses, obs_pos=pos, time_anchor=None, time_pos=None ) return obses, self.a[idxs], self.r[idxs], next_obses, self.t[idxs], cpc_kwargs if self.norm: self.mean = torch.mean(self.r[:max_size]) self.std = torch.std(self.r[:max_size]) r = self.r[idxs] r = (r - self.mean) / (self.std + 1e-8) return self.s[idxs], self.a[idxs], r, self.s_[idxs], self.t[idxs], None else: return self.s[idxs], self.a[idxs], self.r[idxs], self.s_[idxs], self.t[idxs], None
def dist_reward(self, next_obs, obs_goal):
next_obs = next_obs[np.newaxis]
obs_goal = obs_goal[np.newaxis]
next_obs = utils.random_crop(next_obs, self.image_size)
obs_goal = utils.random_crop(obs_goal, self.image_size)
next_obs = torch.as_tensor(next_obs, device=self.device).float()
obs_goal = torch.as_tensor(obs_goal, device=self.device).float()
z_next_obs = self.CFRL.encode(next_obs, action=None)
z_goal = self.CFRL.encode(obs_goal, action=None)
dist = self.CFRL.dist_score(z_next_obs, z_goal)
return dist
def __getitem__(self, idx): # Nawid - Obtains item from replay buffer
''' Remove the randomness in the dataloading of each sample as the dataloader itself should be able to find the different values
idx = np.random.randint(
0, self.capacity if self.full else self.idx, size=1
)
idx = idx[0]
'''
obses = np.expand_dims(
self.obses[idx], 0
) # Need to expand dim to allow it to be the shape for cropping, then need to squeeze so its a 4d tensor rather than 5d with an extra dim so it can be used with the dataloader
next_obses = np.expand_dims(self.next_obses[idx], 0)
pos = obses.copy()
#obs and next_obs
if self.rand_crop:
obses_input = random_crop(
obses,
self.image_size) #center_crop_image(obses,self.image_size) #
next_obses_input = random_crop(next_obses, self.image_size)
else:
obses_input = center_crop_image(obses, self.image_size)
next_obses_input = center_crop_image(next_obses, self.image_size)
# random crop images
obses_anc = random_crop(obses, self.image_size)
pos = random_crop(pos, self.image_size)
next_obses_anc = random_crop(
next_obses, self.image_size
) # Set anchor for the next observation in order to contrast with the contrastive loss
# Squeeze shape
obses_input = np.squeeze(obses_input)
next_obses_input = np.squeeze(next_obses_input)
obses_anc = np.squeeze(obses_anc)
pos = np.squeeze(pos)
next_obses_anc = np.squeeze(next_obses_anc)
action = self.actions[idx]
if self.transform:
obses_input = self.transform(obses_input)
next_obses_input = self.transform(next_obses_input)
obses_anc = self.transform(obses_anc)
pos = self.transform(pos)
next_obses_anc = self.transform(next_obses_anc)
cpc_kwargs = dict(
obs_anchor=obses_anc, obs_pos=pos, next_obs_anchor=next_obses_anc
) # Nawid Postitive example is pos whilst anchor is obses
return obses_input, action, next_obses_input, cpc_kwargs
def sample_cpc(self): # Nawid - samples images I believe
start = time.time()
idxs = np.random.randint(
0, self.capacity if self.full else self.idx,
size=self.batch_size) # Used to randomly sample indices
obses = self.obses[idxs] # Nawid - Samples observation
pos = obses.copy() # Nawid -
next_obses = self.next_obses[idxs]
# Random crop or centre crops the image
if self.rand_crop:
obses_input = random_crop(obses, self.image_size)
next_obses_input = random_crop(next_obses, self.image_size)
else:
obses_input = center_crop_image(obses, self.image_size)
next_obses_input = centre_crop_image(next_obses, self.image_size)
# Nawid - Crop images randomly
obses_anc = random_crop(obses, self.image_size)
pos = random_crop(pos, self.image_size)
obses_input, next_obses_input = np.transpose(
obses_input, (0, 3, 1, 2)), np.transpose(next_obses_input,
(0, 3, 1, 2))
obses_anc, pos = np.transpose(obses_anc, (0, 3, 1, 2)), np.transpose(
pos, (0, 3, 1, 2))
obses_input = torch.tensor(obses_input,
device=self.device).float() / 255
actions = torch.as_tensor(self.actions[idxs], device=self.device)
next_obses_input = torch.tensor(next_obses_input,
device=self.device).float() / 255
obses_anc = torch.as_tensor(obses_anc, device=self.device).float(
) / 255 # Random color jitter turns the values already into torch tenros
pos = torch.as_tensor(pos, device=self.device).float() / 255
obses_anc = random_color_jitter(obses_anc,
batch_size=self.batch_size,
frames=self.frames)
pos = random_color_jitter(pos,
batch_size=self.batch_size,
frames=self.frames)
cpc_kwargs = dict(
obs_anchor=obses_anc, obs_pos=pos, time_anchor=None, time_pos=None
) # Nawid Postitive example is pos whilst anchor is obses
return obses_input, actions, next_obses_input, cpc_kwargs
def testReturnCorrectCropOfSingleImage(self):
np.random.seed(0)
height, width = 10, 20
image = np.random.randint(0, 256, size=(height, width, 3))
crop_height, crop_width = 2, 4
image_placeholder = tf.placeholder(tf.int32, shape=(None, None, 3))
[cropped] = utils.random_crop([image_placeholder], crop_height,
crop_width)
with self.test_session():
cropped_image = cropped.eval(feed_dict={image_placeholder: image})
# Ensure we can find the cropped image in the original:
is_found = False
for x in range(0, width - crop_width + 1):
for y in range(0, height - crop_height + 1):
if np.isclose(image[y:y + crop_height, x:x + crop_width, :],
cropped_image).all():
is_found = True
break
self.assertTrue(is_found)
def testReturnConsistenCropsOfImagesInTheList(self):
tf.set_random_seed(0)
height, width = 10, 20
crop_height, crop_width = 2, 3
labels = np.linspace(0, height * width - 1, height * width)
labels = labels.reshape((height, width, 1))
image = np.tile(labels, (1, 1, 3))
image_placeholder = tf.placeholder(tf.int32, shape=(None, None, 3))
label_placeholder = tf.placeholder(tf.int32, shape=(None, None, 1))
[cropped_image, cropped_label
] = utils.random_crop([image_placeholder, label_placeholder],
crop_height, crop_width)
with self.test_session() as sess:
cropped_image, cropped_labels = sess.run(
[cropped_image, cropped_label],
feed_dict={
image_placeholder: image,
label_placeholder: labels
})
for i in range(3):
self.assertAllEqual(cropped_image[:, :, i],
cropped_labels.squeeze())
def preprocess_setup(self):
# if self.train:
# funcs = []
# # if self.args.strongAugment:
# # funcs += [U.random_scale(1.25)]
# funcs += [U.padding(self.args.inputLength // 2),
# U.random_crop(self.args.inputLength),
# U.normalize(32768.0),
# ]
# else:
# funcs = [U.padding(self.args.inputLength // 2),
# U.normalize(32768.0),
# U.multi_crop(self.args.inputLength, self.args.nCrops),
# ]
if self.train:
funcs = []
# if self.args.strongAugment:
# funcs += [U.random_scale(1.25)]
funcs += [
U.random_crop(self.args.inputLength),
U.normalize(32768.0),
]
else:
funcs = [
U.normalize(32768.0),
U.multi_crop(self.args.inputLength, self.args.nCrops),
]
return funcs
def manipulate_latent(model, n_class, out_dim, data, args):
x_true, y_true = data
index = np.argmax(y_true, 1) == args.manipulate
number = np.random.randint(low=0, high=sum(index) - 1)
x, y = x_true[index][number], y_true[index][number]
x, y = np.expand_dims(x, 0), np.expand_dims(y, 0)
if args.crop_x is not None and args.crop_y is not None:
x = utils.random_crop(x, [args.crop_x, args.crop_y])
noise = np.zeros([1, n_class, out_dim])
x_recons = []
# Change params of vect in 0.05 steps. See also [1]
for dim in range(out_dim):
r = -0.25
while r <= 0.25:
tmp = np.copy(noise)
tmp[:,:,dim] = r
x_recon = model.predict([x, y, tmp])
x_recons.append(x_recon[0])
r += 0.05
img = utils.stack_images(x_recons, out_dim)
img.show()
img.save(args.save_dir + "/manipulate-%d.png" % args.manipulate)
def __getitem__(self, index):
X = np.empty((self.batch_size, self.img_h, self.img_w, 3),
dtype=np.float32)
y = np.zeros((self.batch_size, 1), dtype=np.float32)
indexes = self.indexes[index * self.batch_size:(index + 1) *
self.batch_size]
for i, f in enumerate(self.df['ImageId'].iloc[indexes]):
self.info[index * self.batch_size + i] = f
img = np.asarray(Image.open(self.data_path + f))
for j in range(4):
y[i][0] += rle2class(self.df['e' +
str(j + 1)].iloc[indexes[i]])
y[i][0] = 1 if y[i][0] > 0 else 0
random_crop_indexes = util.get_random_crop_indexes(
(256, 1600), (self.img_h, self.img_w), img, None)
X[i, ], _ = util.random_crop(img, None, random_crop_indexes)
if self.preprocess != None:
X = self.preprocess(X)
#Data augmentation
if (self.augmentation_parameters is not None):
for i in range(len(X)):
affine_aug, color_aug = util.get_augmentation(
self.augmentation_parameters)
X[i] = util.augment(affine_aug, X[i].astype(np.uint8))
X[i] = util.augment(color_aug, X[i].astype(np.uint8))
return X, y
def dataset_input_fn(is_train, batch_size=64, split=1):
sounds, labels = train[split - 1] if is_train is True else val[split - 1]
labels = np.array(labels).reshape((-1, 1))
dataset = tf.data.Dataset.from_generator(
lambda: zip(sounds, labels),
output_types=(tf.float32, tf.int32),
output_shapes=(tf.TensorShape([None]), tf.TensorShape(1)))
# if is_train:
# if opt.strongAugment:
# dataset = dataset.map(U.random_scale(1.25))
dataset = dataset.map(U.padding(opt.inputLength // 2))
dataset = dataset.map(U.random_crop(opt.inputLength))
dataset = dataset.map(U.normalize(float(2**16 / 2)))
dataset = dataset.shuffle(1000)
# else:
# # if not opt.longAudio:
# dataset = dataset.map(U.padding(opt.inputLength // 2))
# dataset = dataset.map(U.random_crop(opt.inputLength))
# dataset = dataset.map(U.normalize(float(2 ** 16 / 2)))
# # dataset = dataset.map(U.multi_crop(opt.inputLength, opt.nCrops))
dataset = dataset.batch(batch_size)
dataset = dataset.map(U.reshape([batch_size, -1, 1]))
iterator = dataset.make_one_shot_iterator()
return iterator.get_next()
def __getitem__(self, idx):
i = idx * batch_size
out_img_rows, out_img_cols = img_size * self.scale, img_size * self.scale
length = min(batch_size, (len(self.names) - i))
batch_x = np.empty((length, img_size, img_size, channel), dtype=np.float32)
batch_y = np.empty((length, out_img_rows, out_img_cols, channel), dtype=np.float32)
for i_batch in range(length):
name = self.names[i + i_batch]
filename = os.path.join(image_folder, name)
# b: 0 <=b<=255, g: 0 <=g<=255, r: 0 <=r<=255.
image_bgr = cv.imread(filename)
gt = random_crop(image_bgr, self.scale)
if np.random.random_sample() > 0.5:
gt = np.fliplr(gt)
angle = random.choice((0, 90, 180, 270))
gt = imutils.rotate_bound(gt, angle)
x = cv.resize(gt, (img_size, img_size), cv.INTER_CUBIC)
batch_x[i_batch, :, :] = preprocess_input(x)
batch_y[i_batch, :, :] = gt
return batch_x, batch_y
def data_augmentation(input_image, output_image):
# Data augmentation
input_image, output_image = utils.random_crop(input_image, output_image, args.crop_height, args.crop_width)
if args.h_flip and random.randint(0,1):
input_image = cv2.flip(input_image, 1)
output_image = cv2.flip(output_image, 1)
if args.v_flip and random.randint(0,1):
input_image = cv2.flip(input_image, 0)
output_image = cv2.flip(output_image, 0)
if args.brightness:
factor = 1.0 + abs(random.gauss(mu=0.0, sigma=args.brightness))
if random.randint(0,1):
factor = 1.0/factor
table = np.array([((i / 255.0) ** factor) * 255 for i in np.arange(0, 256)]).astype(np.uint8)
input_image = cv2.LUT(input_image, table)
if args.rotation:
angle = args.rotation
else:
angle = 0.0
if args.zoom:
scale = args.zoom
else:
scale = 1.0
if args.rotation or args.zoom:
M = cv2.getRotationMatrix2D((input_image.shape[1]//2, input_image.shape[0]//2), angle, scale)
input_image = cv2.warpAffine(input_image, M, (input_image.shape[1], input_image.shape[0]))
output_image = cv2.warpAffine(output_image, M, (output_image.shape[1], output_image.shape[0]))
return input_image, output_image
def data_augmentation(input_image, output_image):
# Data augmentation
input_image, output_image = utils.random_crop(input_image, output_image,
args.crop_height,
args.crop_width)
if args.h_flip and random.randint(0, 1):
input_image = cv2.flip(input_image, 1)
output_image = cv2.flip(output_image, 1)
if args.v_flip and random.randint(0, 1):
input_image = cv2.flip(input_image, 0)
output_image = cv2.flip(output_image, 0)
if args.brightness:
factor = 1.0 + random.uniform(-1.0 * self.brightness, self.brightness)
table = np.array([((i / 255.0) * factor) * 255
for i in np.arange(0, 256)]).astype(np.uint8)
input_image = cv2.LUT(input_image, table)
if args.rotation:
angle = random.uniform(-1 * args.rotation, args.rotation)
if args.rotation:
M = cv2.getRotationMatrix2D(
(input_image.shape[1] // 2, input_image.shape[0] // 2), angle, 1.0)
input_image = cv2.warpAffine(
input_image,
M, (input_image.shape[1], input_image.shape[0]),
flags=cv2.INTER_NEAREST)
output_image = cv2.warpAffine(
output_image,
M, (output_image.shape[1], output_image.shape[0]),
flags=cv2.INTER_NEAREST)
return input_image, output_image
def eco_preprocessing(frames, batch_size, time_step, image_size,
input_channels, is_training):
res = []
selected_frames = []
chunks = np.array_split(frames, time_step)
for c in chunks:
selected_idx = np.random.randint(c.shape[0])
frame_sample = c[selected_idx]
selected_frames.append(frame_sample)
is_flip = random.randint(1, 10) % 2 == 0
for f in selected_frames:
if is_training:
resized = utils.random_crop(f, image_size[0], image_size[1])
if is_flip:
resized = np.fliplr(resized)
else:
resized = utils.crop_center(f, image_size[0], image_size[1])
resized = resized - [104, 117, 123]
resized = resized.astype(np.float32)
res.append(resized)
return np.array(res)
def __getitem__(self, i):
image = cv2.imread(self.images[i])
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype("float32")
objects = self.objects[i]
boxes = np.array(objects['boxes']).astype("float32")
labels = np.array(objects['labels'])
difficulties = np.array(objects['difficulties'])
if not self.keep_difficult:
boxes = boxes[1 - difficulties]
labels = labels[1 - difficulties]
difficulties = difficulties[1 - difficulties]
if self.split == 'TRAIN' and self.data_argu:
data_enhance = [
random_bright, random_contrast, random_saturation, random_hue
]
random.shuffle(data_enhance)
for d in data_enhance:
image = d(image)
if random.random() < 0.5:
image, boxes = random_expand(image, boxes)
image, boxes, labels, difficulties = random_crop(
image, boxes, labels, difficulties)
image, boxes = random_flip(image, boxes)
height, width, _ = image.shape
image = cv2.resize(image, (300, 300))
image /= 255.
image = (image - self.mean) / self.std
image = image.transpose((2, 0, 1)).astype("float32")
boxes[:, [0, 2]] /= width
boxes[:, [1, 3]] /= height
return image, boxes, labels, difficulties
def compose(self, mode, item):
ref_cloud = self.data[item, ...]
R, t = generate_random_rotation_matrix(
), generate_random_tranlation_vector()
if mode == 'clean':
ref_cloud = random_select_points(ref_cloud, m=self.npts)
src_cloud_points = transform(ref_cloud[:, :3], R, t)
src_cloud_normal = transform(ref_cloud[:, 3:], R)
src_cloud = np.concatenate([src_cloud_points, src_cloud_normal],
axis=-1)
return src_cloud, ref_cloud, R, t
elif mode == 'partial':
source_cloud = random_select_points(ref_cloud, m=self.npts)
ref_cloud = random_select_points(ref_cloud, m=self.npts)
src_cloud_points = transform(source_cloud[:, :3], R, t)
src_cloud_normal = transform(source_cloud[:, 3:], R)
src_cloud = np.concatenate([src_cloud_points, src_cloud_normal],
axis=-1)
src_cloud = random_crop(src_cloud, p_keep=0.7)
return src_cloud, ref_cloud, R, t
elif mode == 'noise':
source_cloud = random_select_points(ref_cloud, m=self.npts)
ref_cloud = random_select_points(ref_cloud, m=self.npts)
src_cloud_points = transform(source_cloud[:, :3], R, t)
src_cloud_normal = transform(source_cloud[:, 3:], R)
src_cloud = np.concatenate([src_cloud_points, src_cloud_normal],
axis=-1)
return src_cloud, ref_cloud, R, t
else:
raise NotImplementedError
def __getitem__(self, index):
X = np.empty((self.batch_size,self.img_h,self.img_w,3),dtype=np.float32)
y = np.empty((self.batch_size,self.img_h,self.img_w,self.channels_mask),dtype=np.int8)
mask = np.empty((256,1600,self.channels_mask),dtype=np.int8)
if (self.use_balanced_dataset):
df_batch = self.get_class_balanced_batch(self.batch_size)
else:
df_batch = self.get_standard_batch(index, self.batch_size)
#Generate random crop indexes, create full resoultion mask and then crop
for i in range (len(df_batch)):
df = df_batch[i]
img = np.asarray(Image.open(self.data_path + df['ImageId']))
for j in range(4):
mask[:,:,j] = util.rle2maskResize(df['e'+str(j+1)])
if (self.channels_mask > 4):
mask[:,:,4] = util.mask2Background(mask)
random_crop_indexes = util.get_random_crop_indexes((256,1600), (self.img_h,self.img_w), img, mask[:,:,:4])
X[i,], y[i,:,:,:] = util.random_crop(img, mask, random_crop_indexes)
#Data augmentation
if (self.augmentation_parameters is not None):
for i in range(len(X)):
affine_aug, color_aug = util.get_augmentation (self.augmentation_parameters)
X[i], y[i] = util.augment(affine_aug, X[i].astype(np.uint8), y[i].astype(np.uint8))
X[i] = util.augment(color_aug, X[i].astype(np.uint8))
#Apply data preprocessing according to the model chosen
if self.preprocess!=None:
X = self.preprocess(X)
return X, y
def _read_image(im):
im = cv2.imread(im)
im = utils.prewhiten(im)
im = utils.flip(im, random_flip=True)
im = utils.random_crop(im, image_size=299)
im = cv2.resize(im, (128, 128))
im = utils.random_rotate_image(im)
return im
def train_generator_with_augmentation(x, y, batch_size, shift_fraction=0.):
train_datagen = ImageDataGenerator(width_shift_range=shift_fraction,
height_shift_range=shift_fraction) # shift up to 2 pixel for MNIST
generator = train_datagen.flow(x, y, batch_size=batch_size)
while 1:
x_batch, y_batch = generator.next()
if args.crop_x is not None and args.crop_y is not None:
x_batch = utils.random_crop(x_batch, [args.crop_x, args.crop_y])
yield (x_batch, y_batch)
def test_generator_with_augmentation(x, batch_size, shift_range, rotation_range):
test_datagen = ImageDataGenerator(width_shift_range=shift_range,
height_shift_range=shift_range,
rotation_range=rotation_range)
generator = test_datagen.flow(x, batch_size=batch_size, shuffle=False)
while 1:
x_batch = generator.next()
if args.crop_x is not None and args.crop_y is not None:
x_batch = utils.random_crop(x_batch, [args.crop_x, args.crop_y])
yield (x_batch)
def dist_reward(self, next_obs, obs_goal):
if self.encoder_type == 'pixel':
next_obs = next_obs[np.newaxis]
obs_goal = obs_goal[np.newaxis]
next_obs = utils.random_crop(next_obs, self.image_size)
obs_goal = utils.random_crop(obs_goal, self.image_size)
next_obs = torch.as_tensor(next_obs, device=self.device).float()
obs_goal = torch.as_tensor(obs_goal, device=self.device).float()
z_next_obs = self.CURL.encode(next_obs)
z_goal = self.CURL.encode(obs_goal)
dist = self.CURL.dist_score(z_next_obs, z_goal)
return dist
else:
dist = -np.linalg.norm(next_obs - obs_goal)
reward = torch.as_tensor(dist, device=self.device).float()
return reward.item()
def preprocess_setup(self):
if self.train:
funcs = [U.normalize(self.mean, self.std),
U.horizontal_flip(),
U.padding(4),
U.random_crop(32),
]
else:
funcs = [U.normalize(self.mean, self.std)]
return funcs
def testReturnDifferentCropAreasOnTwoEvals(self):
tf.set_random_seed(0)
crop_height, crop_width = 2, 3
image = np.random.randint(0, 256, size=(100, 200, 3))
image_placeholder = tf.placeholder(tf.int32, shape=(None, None, 3))
[cropped] = utils.random_crop([image_placeholder], crop_height,
crop_width)
with self.test_session():
crop0 = cropped.eval(feed_dict={image_placeholder: image})
crop1 = cropped.eval(feed_dict={image_placeholder: image})
self.assertFalse(np.isclose(crop0, crop1).all())
def __data_generation(self, list_ids_temp):
'Generates data containing batch_size samples'
x = np.empty((self.batch_size, *self.dim, self.n_channels))
y = np.empty((self.batch_size, *self.dim))
dataset_mean = self.mean
dataset_std = self.std
for i, ID in enumerate(list_ids_temp):
img1 = skimage.img_as_float64(
imread(dataset_add + "GEE_mapbiomas/" + ID +
"_2019-01-01.tif"))
img2 = skimage.img_as_float64(
imread(dataset_add + "GEE_mapbiomas/" + ID +
"_2019-04-01.tif"))
img3 = skimage.img_as_float64(
imread(dataset_add + "GEE_mapbiomas/" + ID +
"_2019-07-01.tif"))
img4 = skimage.img_as_float64(
imread(dataset_add + "GEE_mapbiomas/" + ID +
"_2019-10-01.tif"))
img5 = skimage.img_as_float64(
imread(dataset_add + "GEE_mapbiomas/" + ID +
"_2020-01-01.tif"))
mask = skimage.img_as_float64(
imread(dataset_add + "GEE_mapbiomas_masks/" + ID + ".tif"))
img1 = U.normalization(img1, mean=dataset_mean, std=dataset_std)
img2 = U.normalization(img2, mean=dataset_mean, std=dataset_std)
img3 = U.normalization(img3, mean=dataset_mean, std=dataset_std)
img4 = U.normalization(img4, mean=dataset_mean, std=dataset_std)
img5 = U.normalization(img5, mean=dataset_mean, std=dataset_std)
img = np.concatenate((img1, img2, img3, img4, img5), axis=2)
# 32x32 random crop
if self.shuffle == True:
img, mask = U.random_crop(img, mask, 32, 32)
else:
img = img[:32, :32]
mask = mask[:32, :32]
x[i, ] = img
y[i, ] = mask / 255.
y = np.expand_dims(y, axis=3)
return x, y
def preprocess_setup(self):
if self.opt.plus:
normalize = U.zero_mean
else:
normalize = U.normalize
if self.train and self.opt.noDataAug != True:
funcs = [normalize(self.mean, self.std),
U.horizontal_flip(),
U.padding(4),
U.random_crop(32),
]
else:
funcs = [normalize(self.mean, self.std)]
return funcs
def testRandomCropMaintainsNumberOfChannels(self):
np.random.seed(0)
crop_height, crop_width = 10, 20
image = np.random.randint(0, 256, size=(100, 200, 3))
tf.set_random_seed(37)
image_placeholder = tf.placeholder(tf.int32, shape=(None, None, 3))
[cropped] = utils.random_crop([image_placeholder], crop_height,
crop_width)
with self.test_session():
cropped_image = cropped.eval(feed_dict={image_placeholder: image})
self.assertTupleEqual(cropped_image.shape,
(crop_height, crop_width, 3))
def train(model, data, args):
# unpacking the data
(x_train, y_train), (x_test, y_test) = data
# callbacks
log = callbacks.CSVLogger(args.save_dir + '/log.csv')
tb = callbacks.TensorBoard(log_dir=args.save_dir + '/tensorboard-logs',
batch_size=args.batch_size, histogram_freq=int(args.debug))
checkpoint = callbacks.ModelCheckpoint(args.save_dir + '/weights-{epoch:02d}.hdf5', monitor='val_capsnet_acc',
save_best_only=True, save_weights_only=True, verbose=1)
lr_decay = callbacks.LearningRateScheduler(schedule=lambda epoch: args.lr * (args.lr_decay ** epoch))
# compile the model
model.compile(optimizer=optimizers.Adam(lr=args.lr),
loss=[margin_loss, reconstruction_loss], # We scale down this reconstruction loss by 0.0005 so that
loss_weights=[1., args.scale_reconstruction_loss], # ...it does not dominate the margin loss during training.
metrics={'capsnet': 'accuracy'})
# Generator with data augmentation as used in [1]
def train_generator_with_augmentation(x, y, batch_size, shift_fraction=0.):
train_datagen = ImageDataGenerator(width_shift_range=shift_fraction,
height_shift_range=shift_fraction)
generator = train_datagen.flow(x, y, batch_size=batch_size)
while 1:
x_batch, y_batch = generator.next()
if args.crop_x is not None and args.crop_y is not None:
x_batch = utils.random_crop(x_batch, [args.crop_x, args.crop_y])
yield ([x_batch, y_batch], [y_batch, x_batch])
generator = train_generator_with_augmentation(x_train, y_train, args.batch_size, args.shift_fraction)
# Validation set is always cropped the same
if args.crop_x is not None and args.crop_y is not None:
x_test = utils.random_crop(x_test, [args.crop_x, args.crop_y])
model.fit_generator(generator=generator,
steps_per_epoch=int(y_train.shape[0] / args.batch_size),
epochs=args.epochs,
validation_data=[[x_test, y_test], [y_test, x_test]], # Note: For the decoder the input is the label and the output the image
callbacks=[log, tb, checkpoint, lr_decay])
model.save_weights(args.save_dir + '/trained_model.hdf5')
print('Trained model saved to \'%s/trained_model.hdf5\'' % args.save_dir)
utils.plot_log(args.save_dir + '/log.csv', show=True)
return model
print 'regenerating '+pair[0]+' with label '+pair[1]
image = skimage.io.imread(Image_Path+filename)
transformed_img = mirror(image)
skimage.io.imsave(Image_Path+str(next_image_num)+'.png',transformed_img)
rep_image_label_file.write(str(next_image_num)+'.png '+pair[1].split('\n')[0]+'\n')
next_image_num += 1
# do random crop
# int(augment_fraction*train_image_num)
aug_idxs = np.random.choice(train_image_num, int(augment_fraction*train_image_num), replace=False)+1
for augidx in aug_idxs:
filename = str(augidx)+'.png'
pair = train_image_labels[augidx-1].split(' ')
print 'regenerating '+pair[0]+' with label '+pair[1]
image = skimage.io.imread(Image_Path+filename)
transformed_img = random_crop(image)
skimage.io.imsave(Image_Path+str(next_image_num)+'.png',transformed_img)
rep_image_label_file.write(str(next_image_num)+'.png '+pair[1].split('\n')[0]+'\n')
next_image_num += 1
# do random rotate
# int(augment_fraction*train_image_num)
aug_idxs = np.random.choice(train_image_num, int(augment_fraction*train_image_num), replace=False)+1
for augidx in aug_idxs:
filename = str(augidx)+'.png'
pair = train_image_labels[augidx-1].split(' ')
print 'regenerating '+pair[0]+' with label '+pair[1]
image = skimage.io.imread(Image_Path+filename)
transformed_img = rotate(image)
skimage.io.imsave(Image_Path+str(next_image_num)+'.png',transformed_img)
rep_image_label_file.write(str(next_image_num)+'.png '+pair[1].split('\n')[0]+'\n')
