def __getitem__(self, index):
X = np.empty((self.batch_size, self.img_h, self.img_w, 3),
dtype=np.float32)
y = np.zeros((self.batch_size, 1), dtype=np.float32)
indexes = self.indexes[index * self.batch_size:(index + 1) *
self.batch_size]
for i, f in enumerate(self.df['ImageId'].iloc[indexes]):
self.info[index * self.batch_size + i] = f
img = np.asarray(Image.open(self.data_path + f))
for j in range(4):
y[i][0] += rle2class(self.df['e' +
str(j + 1)].iloc[indexes[i]])
y[i][0] = 1 if y[i][0] > 0 else 0
random_crop_indexes = util.get_random_crop_indexes(
(256, 1600), (self.img_h, self.img_w), img, None)
X[i, ], _ = util.random_crop(img, None, random_crop_indexes)
if self.preprocess != None:
X = self.preprocess(X)
#Data augmentation
if (self.augmentation_parameters is not None):
for i in range(len(X)):
affine_aug, color_aug = util.get_augmentation(
self.augmentation_parameters)
X[i] = util.augment(affine_aug, X[i].astype(np.uint8))
X[i] = util.augment(color_aug, X[i].astype(np.uint8))
return X, y
def __getitem__(self, index):
"""
Get image at index from image set A and image set B.
Args:
index: Index of image in the image path list
"""
# make sure index doesn't exceed number of images
image_path_A = self.image_paths_A[index % self.size_A]
image_path_B = self.image_paths_B[index % self.size_B]
# get image as numpy array
image_A = Image.open(image_path_A).convert('RGB')
image_B = Image.open(image_path_B).convert('RGB')
# perform data augmentation on images
A = utils.augment(self.opt,
image_A,
grayscale=(self.in_channels == 1),
normalize=self.is_training)
B = utils.augment(self.opt,
image_B,
grayscale=(self.out_channels == 1),
normalize=self.is_training)
return {'A': A, 'A_path': image_path_A, 'B': B, 'B_path': image_path_B}
def __getitem__(self, index):
X = np.empty((self.batch_size,self.img_h,self.img_w,3),dtype=np.float32)
y = np.empty((self.batch_size,self.img_h,self.img_w,self.channels_mask),dtype=np.int8)
mask = np.empty((256,1600,self.channels_mask),dtype=np.int8)
if (self.use_balanced_dataset):
df_batch = self.get_class_balanced_batch(self.batch_size)
else:
df_batch = self.get_standard_batch(index, self.batch_size)
#Generate random crop indexes, create full resoultion mask and then crop
for i in range (len(df_batch)):
df = df_batch[i]
img = np.asarray(Image.open(self.data_path + df['ImageId']))
for j in range(4):
mask[:,:,j] = util.rle2maskResize(df['e'+str(j+1)])
if (self.channels_mask > 4):
mask[:,:,4] = util.mask2Background(mask)
random_crop_indexes = util.get_random_crop_indexes((256,1600), (self.img_h,self.img_w), img, mask[:,:,:4])
X[i,], y[i,:,:,:] = util.random_crop(img, mask, random_crop_indexes)
#Data augmentation
if (self.augmentation_parameters is not None):
for i in range(len(X)):
affine_aug, color_aug = util.get_augmentation (self.augmentation_parameters)
X[i], y[i] = util.augment(affine_aug, X[i].astype(np.uint8), y[i].astype(np.uint8))
X[i] = util.augment(color_aug, X[i].astype(np.uint8))
#Apply data preprocessing according to the model chosen
if self.preprocess!=None:
X = self.preprocess(X)
return X, y
def inplace_augment(self, index):
tparams = copy.deepcopy(self.tparams)
tparams['keep_size'] = True
tparams['rotate'] = (0, 1)
tparams['hpad'] = (0, 1)
tparams['vpad'] = (0, 1)
datum = self.data_split[index]
img = self.images[index]
out = img.copy()
boxes = datum['gt_boxes']
labels = np.array([self.wtoi[r['label']] for r in datum['regions']])
boxes = datum['gt_boxes']
embeddings = np.array([self.wtoe[r['label']] for r in datum['regions']])
for i, r in enumerate(reversed(datum['regions'])):
b = [r['x'], r['y'], r['x'] + r['width'], r['y'] + r['height']]
try: #Some random values for weird boxes give value errors, just handle and ignore
b = utils.close_crop_box(img, b)
word = img[b[1]:b[3], b[0]:b[2]]
aug = utils.augment(word, tparams, self.augment_mode)
out[b[1]:b[3], b[0]:b[2]] = aug
except ValueError:
continue
return out, boxes, embeddings, labels
def train(epoch):
model.train()
correct = 0
for batch_idx, labeled in enumerate(train_loader):
data, target = labeled
if args.augment:
data = augment(data) # augment labeled data
data, target = Variable(data), Variable(target)
optimizer.zero_grad()
# supervised model
outputs, laterals = model(data, corrupted=True)
# supervised loss
loss = F.cross_entropy(outputs, target)
loss.backward()
optimizer.step()
# prediction for training accuracy
pred = outputs.data.max(1)[
1] # get the index of the max log-probability
correct += pred.eq(target.data).cpu().sum()
if batch_idx % 10 == 0:
print('Train Epoch: {} [{}/{} ({:.00f}%)]\tLoss: {:.6f}'.format(
epoch, batch_idx * len(data), len(train_loader.dataset),
100. * batch_idx / len(train_loader), loss.data))
print('Accuracy: {}/{} ({:.2f}%)'.format(
correct, len(train_loader.dataset),
100. * correct / len(train_loader.dataset)))
def __getitem__(self, index):
start_index = index * self.batch_size
end_index = start_index + self.batch_size
batch_paths = self.path_to_pictures[start_index:end_index]
input_shape = self.model.get_input_shape()
x_shape = (len(batch_paths), ) + (input_shape)
x = np.empty(shape=x_shape)
for i, path in enumerate(batch_paths):
# addition: hack
if isinstance(
path,
str) and path.startswith('collected_data_customized/'):
path = path[len('collected_data_customized/'):]
# apply augmentation to 60% of the images, if enabled
if APPLY_DATA_AUGMENTATION and np.random.rand() < 0.6:
# data augmentation
img = load_image(self.data_dir, path)
img = augment(img)
img = preprocess(img) # crop + resize + rgb2yuv
else:
img = load_image(self.data_dir, path)
img = resize(img)
x[i] = self.model.normalize_and_reshape(img)
return x, x
def train(self):
self.load_train_data()
if self.model is None:
self.build_model()
# prepare model model saving directory.
save_dir = os.path.join(os.getcwd(), 'weights')
if not os.path.isdir(save_dir):
os.makedirs(save_dir)
weights_name = 'pspu_skelnet.h5'
filepath = os.path.join(save_dir, weights_name)
# prepare callbacks for model saving and for learning rate adjustment.
checkpoint = ModelCheckpoint(filepath=filepath,
verbose=1,
save_weights_only=True)
lr_scheduler = LearningRateScheduler(self.lr_schedule)
callbacks = [checkpoint, lr_scheduler]
# train the model with input images and labels
xval = self.input_pix.astype('float32') / 255
yval = self.output_pix.astype('float32') / 255
x, y = augment(self.input_pix, self.output_pix, ntimes=self.ntimes)
x = np.concatenate((self.input_pix, x), axis=0)
y = np.concatenate((self.output_pix, y), axis=0)
print("Augmented input train data shape: ", x.shape)
print("Augmented output train data shape: ", y.shape)
x = x.astype('float32') / 255
y = y.astype('float32') / 255
self.model.fit([x, x, x, x],
y,
epochs=EPOCHS,
validation_data=([xval, xval, xval, xval], yval),
batch_size=self.batch_size,
callbacks=callbacks)
def __getitem__(self, index: int) -> List[Any]:
filename: str = self.filenames[index]
path_name: Path = Path(filename)
images: List[D]
#print('get',self.folders, filename)
files = SliceDataset.load_images(self.folders, [filename],
self.in_memory)
#print('files', files, self.bounds_generators)
#print('old files', self.files[0][index])
#print(self.files, filename)
#print(path_name)
if path_name.suffix == ".png":
images = [
Image.open(files[index]).convert('L') for files in self.files
]
elif path_name.suffix == ".nii":
#print(files)
try:
images = [read_nii_image(f[0]) for f in files]
#print("nm",[i.shape for i in images])
except:
images = [read_unknownformat_image(f[0]) for f in files]
#print("em",[i.shape for i in images])
elif path_name.suffix == ".npy":
images = [np.load(files[index]) for files in self.files]
else:
raise ValueError(filename)
if self.augment:
images = augment(*images)
assert self.check_files() # Make sure all file exists
# Final transforms and assertions
t_tensors: List[Tensor] = [
tr(e) for (tr, e) in zip(self.transforms, images)
]
assert 0 <= t_tensors[0].min() and t_tensors[0].max(
) <= 1, t_tensors[0].max() # main image is between 0 and 1
#print(t_tensors[0].max())
_, w, h = t_tensors[0].shape
for ttensor, is_hot in zip(
t_tensors[1:],
self.are_hots): # All masks (ground truths) are class encoded
if is_hot:
assert one_hot(ttensor, axis=0)
#assert ttensor.shape == (self.C, w, h)
img, gt = t_tensors[:2]
#print(gt.shape)
try:
bounds = [
f(img, gt, t, filename)
for f, t in zip(self.bounds_generators, t_tensors[2:])
]
except:
print(self.folders, filename, self.bounds_generator)
# return t_tensors + [filename] + bounds
return [filename] + t_tensors + bounds
def full_page_augment(self, index, augment=True):
tparams = copy.deepcopy(self.tparams)
tparams['keep_size'] = False
tparams['rotate'] = (-5, 5)
tparams['hpad'] = (0, 12)
tparams['vpad'] = (0, 12)
m = int(np.median(self.medians))
s = 3 #inter word space
x, y = s, s #Upper left corner of box
gt_boxes = []
gt_labels = []
gt_embeddings = []
si = np.random.randint(len(self.data_split))
shape = (self.original_heights[si], self.original_widths[si])
canvas = self.create_background(m + np.random.randint(0, 20) - 10, shape)
maxy = 0
for j in range(self.max_words):
ind = np.random.randint(self.vocab_size)
k = len(self.words_by_label[ind])
while k == 0:
ind = np.random.randint(self.vocab_size)
k = len(self.words_by_label[ind])
word = self.words_by_label[ind][np.random.randint(k)]
#randomly transform word and place on canvas
if augment:
try:
tword = utils.augment(word, tparams, self.augment_mode)
except:
tword = word
else:
tword = word
h, w = tword.shape
if x + w > shape[1]: #done with row?
x = s
y = maxy + s
if y + h > shape[0]: #done with page?
break
x1, y1, x2, y2 = x, y, x + w, y + h
canvas[y1:y2, x1:x2] = tword
b = [x1, y1, x2, y2]
gt_labels.append(ind)
gt_boxes.append(b)
gt_embeddings.append(self.wtoe[self.vocab[ind]])
x = x2 + s
maxy = max(maxy, y2)
H, W = shape
scale = float(self.image_size) / max(H, W)
boxes = np.array(gt_boxes)
scaled_boxes = torch.from_numpy(np.round(scale * (boxes + 1) - 1))
gt_boxes = box_utils.x1y1x2y2_to_xcycwh(scaled_boxes).numpy()
return canvas, gt_boxes, np.array(gt_embeddings), np.array(gt_labels)
def __getitem__(self, index):
try:
lr_file = self.lr_list[index]
hr_file = self.hr_list[index]
interval = random.choice(self.interval_list)
lr_path,lr_name = util.get_file_name(lr_file)
hr_path, _ = util.get_file_name(hr_file)
center_frame_idx = int(lr_name)
neighbor_list = getNeighbor(center_frame_idx,self.n_frames,interval)
if self.random_reverse and random.random() < 0.5:
neighbor_list.reverse()
file_text = '{:03d}.png'.format(center_frame_idx)
#### get lr images
lr_data_list = []
for v in neighbor_list:
lr_data_path = osp.join(lr_path, '{:03d}.png'.format(v))
lr_data_list.append(read_img(lr_data_path))
#### get hr images
hr_data_path = osp.join(hr_path, file_text)
hr_data = read_img(hr_data_path)
# randomly crop
height, width, channel = hr_data.shape
hr_size_w,hr_size_h = self.patch_size * self.scale, self.patch_size * self.scale
lr_height = height // self.scale
lr_width = width // self.scale
rnd_h = random.randint(0, max(0, lr_height - self.patch_size))
rnd_w = random.randint(0, max(0, lr_width - self.patch_size))
img_lr_list = [one_data[rnd_h:rnd_h + self.patch_size, rnd_w:rnd_w + self.patch_size, :] for one_data in lr_data_list]
rnd_h_hr, rnd_w_hr = int(rnd_h * self.scale), int(rnd_w * self.scale)
img_hr = hr_data[rnd_h_hr:rnd_h_hr + hr_size_h, rnd_w_hr:rnd_w_hr + hr_size_w, :]
# augmentation - flip, rotate
img_lr_list.append(img_hr)
rlt = util.augment(img_lr_list, hflip=True, rot=True)
img_lr_list = rlt[0:-1]
img_hr = rlt[-1]
# stack lr images to NHWC, N is the frame number
img_lrs = np.stack(img_lr_list, axis=0)
# HWC to CHW, numpy to tensor
img_hr = torch.from_numpy(np.ascontiguousarray(np.transpose(img_hr, (2, 0, 1)))).float()
# img_lrs = [torch.from_numpy(np.ascontiguousarray(np.transpose(img, (2, 0, 1)))).float() for img in img_lr_list]
img_lrs = torch.from_numpy(np.ascontiguousarray(np.transpose(img_lrs, (0, 3, 1, 2)))).float()
except Exception as e:
random_sum = random.randrange(0, self.__len__())
return self.__getitem__(random_sum)
return {'LRs': img_lrs, 'HR': img_hr}
def process_name(name: str, folders: List[Path], dest_folders: List[Path], n_aug: int) -> None:
images: List[Image.Image] = [Image.open(Path(folder, name)).convert('L') for folder in folders]
stem: str = Path(name).stem
# Save the unmodified images as _0
save(stem, 0, images, dest_folders)
for i in range(1, n_aug + 1):
augmented: List[Image.Image] = augment(*images)
save(stem, i, augmented, dest_folders)
def __getitem__(self, index: int) -> List[Any]:
filename: str = self.filenames[index]
path_name: Path = Path(filename)
images: List[D]
if path_name.suffix == ".png":
images = [Image.open(files[index]) for files in self.files]
elif path_name.suffix == ".npy":
images = [np.load(files[index]) for files in self.files]
else:
raise ValueError(filename)
if self.spacing_dict:
dx, dy = self.spacing_dict[path_name.stem]
arrs = [np.array(im) for im in images]
w, h = arrs[0].shape
nw, nh = int(w / dx), int(h / dy)
images = [resizing_fn(arr, (nw, nh)) for arr in arrs]
if self.augment:
images = augment(*images)
# Final transforms and assertions
assert len(images) == len(self.folders) == len(self.transforms)
t_tensors: List[Tensor] = [
tr(e) for (tr, e) in zip(self.transforms, images)
]
# main image is between 0 and 1
if not self.ignore_norm:
assert 0 <= t_tensors[0].min() and t_tensors[0].max() <= 1, (
t_tensors[0].min(), t_tensors[0].max())
_, w, h = t_tensors[0].shape
for ttensor in t_tensors[
1:]: # Things should be one-hot or at least have the shape
assert ttensor.shape == (self.C, w, h), (ttensor.shape, self.C, w,
h)
for ttensor, is_hot in zip(
t_tensors,
self.are_hots): # All masks (ground truths) are class encoded
if is_hot:
assert one_hot(ttensor,
axis=0), torch.einsum("cwh->wh", ttensor)
img, gt = t_tensors[:2]
bounds = [
f(img, gt, t, filename)
for f, t in zip(self.bounds_generators, t_tensors[2:])
]
# return t_tensors + [filename] + bounds
return [filename] + t_tensors + bounds
def __getitem__(self, index):
# . . get one item from the dataset
# . . steering angle
steering = float(self.target[index])
# . . images from three cameras
cameras = self.data[index]
center = cameras[0]
left = cameras[1]
right = cameras[2]
# . . size of the processed image for the training: (width, height)
imsize = (70, 320)
# . . probability of fliping an image
pflip = 0.5
# . . correct steering for left and right directions
steering_correction = 0.4
# . . read and correct the steering for the left and right cameras
img_center, steering_center = utils.augment(center, steering, imsize,
pflip)
img_left, steering_left = utils.augment(left,
steering + steering_correction,
imsize, pflip)
img_right, steering_right = utils.augment(
right, steering - steering_correction, imsize, pflip)
# . . augment steering: there is not a single steering value that works. so add random noise to augment it
steering_augmentation = 0.25
# . . add zero mean Gaussian noise
steering_left += np.random.normal(loc=0, scale=steering_augmentation)
steering_center += np.random.normal(loc=0, scale=steering_augmentation)
steering_right += np.random.normal(loc=0, scale=steering_augmentation)
if self.transform is not None:
# . . scale images to the range between zero and one
img_center = self.transform(img_center)
img_left = self.transform(img_left)
img_right = self.transform(img_right)
return (img_center, steering_center), (img_left,
steering_left), (img_right,
steering_right)
def __getitem__(self, index):
if self.file_client is None:
self.file_client = FileClient(
self.io_opts_dict.pop('type'), **self.io_opts_dict
)
# random reverse
if self.opts_dict['random_reverse'] and random.random() < 0.5:
self.neighbor_list.reverse()
# ==========
# get frames
# ==========
# get the GT frame (im4.png)
gt_size = self.opts_dict['gt_size']
key = self.keys[index]
clip, seq, _ = key.split('/') # key example: 00001/0001/im1.png
img_gt_path = key
img_bytes = self.file_client.get(img_gt_path, 'gt')
img_gt = _bytes2img(img_bytes) # (H W 1)
# get the neighboring LQ frames
img_lqs = []
for neighbor in self.neighbor_list:
img_lq_path = f'{clip}/{seq}/im{neighbor}.png'
img_bytes = self.file_client.get(img_lq_path, 'lq')
img_lq = _bytes2img(img_bytes) # (H W 1)
img_lqs.append(img_lq)
# ==========
# data augmentation
# ==========
# randomly crop
img_gt, img_lqs = paired_random_crop(
img_gt, img_lqs, gt_size, img_gt_path
)
# flip, rotate
img_lqs.append(img_gt) # gt joint augmentation with lq
img_results = augment(
img_lqs, self.opts_dict['use_flip'], self.opts_dict['use_rot']
)
# to tensor
img_results = totensor(img_results)
img_lqs = torch.stack(img_results[0:-1], dim=0)
img_gt = img_results[-1]
return {
'lq': img_lqs, # (T [RGB] H W)
'gt': img_gt, # ([RGB] H W)
}
def __getitem__(self, index):
try:
lr_file = self.lr_list[index]
hr_file = self.hr_list[index]
# get the GT image (as the center frame)
hr_data = cv2.imread(hr_file)
hr_data = cv2.cvtColor(hr_data, cv2.COLOR_BGR2YCrCb)
hr_data = np.array(hr_data)
hr_data = hr_data.astype(np.float32)
height, width, channel = hr_data.shape
gt_size = self.patch_size * self.scale
# get LR image
lr_data = cv2.imread(lr_file)
lr_data = cv2.cvtColor(lr_data, cv2.COLOR_BGR2YCrCb)
lr_data = np.array(lr_data)
lr_data = lr_data.astype(np.float32)
# randomly crop
lr_height = height // self.scale
lr_width = width // self.scale
rnd_h = random.randint(0, max(0, lr_height - self.patch_size))
rnd_w = random.randint(0, max(0, lr_width - self.patch_size))
img_lr = lr_data[rnd_h:rnd_h + self.patch_size,
rnd_w:rnd_w + self.patch_size, :]
rnd_h_hr, rnd_w_hr = int(rnd_h * self.scale), int(rnd_w *
self.scale)
img_gt = hr_data[rnd_h_hr:rnd_h_hr + gt_size,
rnd_w_hr:rnd_w_hr + gt_size, :]
# augmentation - flip, rotate
# axis1 = np.random.randint(low=-1, high=3)
# img_lr = util.horizontal_flip(img_lr, axis=axis1)
# img_gt = util.horizontal_flip(img_gt, axis=axis1)
img_lr, img_gt = util.augment([img_lr, img_gt],
hflip=True,
rot=True)
# HWC to CHW, numpy to tensor
img_gt = torch.from_numpy(
np.ascontiguousarray(np.transpose(img_gt, (2, 0, 1)))).float()
img_lr = torch.from_numpy(
np.ascontiguousarray(np.transpose(img_lr, (2, 0, 1)))).float()
#可以这样认为，ascontiguousarray函数将一个内存不连续存储的数组转换为内存连续存储的数组，使得运行速度更快
except Exception as e:
random_sum = random.randrange(0, self.__len__())
# print(index,random_sum,'出现异常',e.__str__())
return self.__getitem__(random_sum)
return img_lr, img_gt
def train_semi(epoch):
model.train()
correct = 0
for batch_idx, (labeled, unlabeled) in enumerate(
zip(train_loader, train_unlabeled_loader)):
data, target = labeled
data_unlabeled, _ = unlabeled
if args.augment:
data = augment(data) # augment labeled data
data, target = to_gpu(Variable(data),
args.cuda), to_gpu(Variable(target), args.cuda)
data_unlabeled = to_gpu(Variable(data_unlabeled), args.cuda)
optimizer.zero_grad()
# supervised model
outputs, laterals = model(data, corrupted=True, cuda=args.cuda)
# clean and noisey version of model on unlabeled data
outputs_clean, laterals_clean = model(data_unlabeled,
corrupted=False,
cuda=args.cuda)
outputs_corrupted, laterals_corrupted = model(data_unlabeled,
corrupted=True,
cuda=args.cuda)
# supervised loss
loss = F.cross_entropy(outputs, target)
# unsupervised / reconstruction loss
unlabeled_loss = mse(outputs_corrupted, Variable(outputs_clean.data))
loss += args.semi_weight * unlabeled_loss
loss.backward()
optimizer.step()
# prediction for training accuracy
pred = outputs.data.max(1)[
1] # get the index of the max log-probability
correct += pred.eq(target.data).cpu().sum()
if batch_idx % 10 == 0:
print('Train Epoch: {} [{}/{} ({:.00f}%)]\tLoss: {:.6f}'.format(
epoch, batch_idx * len(data), len(train_loader.dataset),
100. * batch_idx / len(train_loader), loss.data[0]))
print('Accuracy: {}/{} ({:.2f}%)'.format(
correct, len(train_loader.dataset),
100. * correct / len(train_loader.dataset)))
def slice_patient(id_: str, dest_path: Path, source_path: Path, shape: Tuple[int, int],
n_augment: int):
id_path: Path = Path(source_path, id_)
for acq in id_path.glob("t0*"):
acq_id: int = int(acq.name[1:])
# print(id, acq, acq_id)
t1_path: Path = Path(acq, f"{id_}_t1w_deface_stx.nii.gz")
nib_obj = nib.load(str(t1_path))
t1: np.ndarray = np.asarray(nib_obj.dataobj)
# dx, dy, dz = nib_obj.header.get_zooms()
x, y, z = t1.shape
assert sanity_t1(t1, *t1.shape, *nib_obj.header.get_zooms())
# gt: np.ndarray = fuse_labels(t1, id_, acq, nib_obj)
gt, gt1 = fuse_labels(t1, id_, acq, nib_obj)
norm_img: np.ndarray = norm_arr(t1)
for idz in range(z):
padded_img: np.ndarray = center_pad(norm_img[:, :, idz], shape)
padded_gt: np.ndarray = center_pad(gt[:, :, idz], shape)
padded_gt1: np.ndarray = center_pad(gt1[:, :, idz], shape)
assert padded_img.shape == padded_gt.shape == shape
for k in range(n_augment + 1):
arrays: List[np.ndarray] = [padded_img, padded_gt, padded_gt1]
augmented_arrays: List[np.ndarray]
if k == 0:
augmented_arrays = arrays[:]
else:
augmented_arrays = map_(np.asarray, augment(*arrays))
subfolders: List[str] = ["img", "gt", "gt1"]
assert len(augmented_arrays) == len(subfolders)
for save_subfolder, data in zip(subfolders,
augmented_arrays):
filename = f"{id_}_{acq_id}_{idz}_{k}.png"
save_path: Path = Path(dest_path, save_subfolder)
save_path.mkdir(parents=True, exist_ok=True)
with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=UserWarning)
imsave(str(Path(save_path, filename)), data)
def __getitem__(self, index):
h5f_noisy = h5py.File(self.noisy_file_name, 'r')
h5f_target = h5py.File(self.target_file_name, 'r')
data_noisy = np.array(h5f_noisy['data'][index, :])
data_target = np.array(h5f_target['data'][index, :])
h5f_noisy.close()
h5f_target.close()
if random.random() < 0.5:
[data_noisy, data_target] = augment([data_noisy, data_target],
True, True)
data_noisy = data_noisy.copy()
data_target = data_target.copy()
return [torch.from_numpy(data_noisy), torch.from_numpy(data_target)]
def process_name(name: str, folders: List[Path], dest_folders: List[Path],
n_aug: int, args) -> None:
images: List[Image.Image] = [
Image.open(Path(folder, name)).convert('L') for folder in folders
]
stem: str = Path(name).stem
# Save the unmodified images as _0
save(stem, 0, images, dest_folders)
for i in range(1, n_aug + 1):
augmented: List[Image.Image] = augment(*images,
rotate_angle=args.rotate_angle,
flip=args.flip,
mirror=args.mirror,
rotate=args.rotate,
scale=args.scale)
save(stem, i, augmented, dest_folders)
def run_NNclassifier(params):
if params.out_dir is not None:
if not os.path.exists(params.out_dir): os.makedirs(params.out_dir)
for key, value in params.__dict__.items():
print str(key) + ': \t\t\t' + str(value)
# load data: x: ntrials x ntimepoints x nfeatures, y: ntrials
x = np.load(params.x_file)
y = np.load(params.y_file)
n_classes = len(np.unique(y))
# get train, val and test sets
n_folds = int(100 / params.test_pcnt)
Train, Val, Test = utils.make_kcrossvalidation(x, y, n_folds, shuffle=True)
Train, Val, Test, means, stds = utils.zscore_dataset(Train, Val, Test,
z_train=True, zscore_x=params.zscore, zscore_y=False)
Train, Val, Test = utils.dim_check(Train, Val, Test, nn_type=params.nn_type, nn_dim=params.n_dim)
# train model
Models = []
for kfold in range(n_folds):
print "Fold " + str(kfold)
NN = utils.make_NN(n_classes=n_classes, params=params)
M = NNClassifier(NN, lr = params.lr, w_decay=params.w_decay)
ktrain = utils.augment(Train[kfold], n_times=params.augment_times) if params.augment else Train[kfold]
M.train(ktrain, Val[kfold], n_epochs=params.n_epochs, batch_size=params.batch_size,
no_improve_lim=params.early_stop)
M.test(Test[kfold])
Models.append(utils.copy_model(M, copyall=params.save_weights))
# save models
pM = utils.concat_models(Models)
if params.out_dir is not None: utils.save_model(pM, params.out_dir + '/model' + str(n_folds) + '.p')
print "Total performance: " + \
str(round(sum(pM.test_correct) / float(sum(pM.test_n)), 4)) + " (" +\
str(sum(pM.test_correct)) + '/' + str(sum(pM.test_n)) + ")"
def read_data_from_csv(path_to_csv, n_epochs, height, width, training=True):
"""Read filenames and labels from a csv file. The csv file has to contain
one file name with complete path and one integer label, separated by comma. The
implementation follows Tensorflow input pipeline.
Parameters
----------
path_to_csv : string
The path of csv file to be read.
n_epochs : int
The number of iterations for which the complete dataset is to be iterated.
height: int
Height of an image
width: int
Width of an image
training : Bool [default: True]
Returns
--------
tf_label: Tensor
A tensor of labels with shape (batchsize, label)
tf_image: Tensor
A tensor of image shape(batchsize, height, width, channels)"""
csv_path = tf.train.string_input_producer([path_to_csv],
num_epochs=n_epochs)
textReader = tf.TextLineReader()
_, csv_content = textReader.read(csv_path)
im_name, im_label = tf.decode_csv(csv_content, record_defaults=[[""], [1]])
im_content = tf.read_file(im_name)
tf_image = tf.image.decode_jpeg(im_content, channels=3)
tf_image = tf.cast(tf_image, tf.float32) / 255.
if training == True:
tf_image = augment(tf_image)
size = tf.cast([height, width], tf.int32)
tf_image = tf.image.resize_images(tf_image, size)
tf_label = tf.cast(im_label, tf.int32)
return tf_image, tf_label
def __getitem__(self, index):
start_index = index * self.batch_size
end_index = start_index + self.batch_size
batch_paths = self.path_to_pictures[start_index:end_index]
steering_angles = self.steering_angles[start_index:end_index]
images = np.empty(
[len(batch_paths), IMAGE_HEIGHT, IMAGE_WIDTH, IMAGE_CHANNELS])
steers = np.empty([len(batch_paths)])
for i, paths in enumerate(batch_paths):
center, left, right = batch_paths[i]
steering_angle = steering_angles[i]
# argumentation
if self.is_training and np.random.rand() < 0.6:
image, steering_angle = augment(self.args.data_dir, center,
left, right, steering_angle)
else:
image = load_image(self.args.data_dir, center)
# add the image and steering angle to the batch
images[i] = preprocess(image)
steers[i] = steering_angle
return images, steers
def processData(self):
dataset = {}
for filename in os.listdir(self.datapath + self.chorpus + "/midi/"):
if filename[-3:] in ["mid", "MID", "SQU", "KPP", "squ", "kpp"]:
label = filename[:-4]
try:
dTseq, Tseq, Pseq, tpb = utils.parseMIDI(self.datapath +
self.chorpus +
"/midi/" +
filename)
except TypeError:
continue
dataset[label] = {}
dataset[label]['T'] = Tseq
dataset[label]['dT'] = dTseq
dataset[label]['P'] = Pseq
dataset[label]['TPB'] = tpb
dictionaries = utils.getDictionaries(dataset)
filtered_dataset = utils.filterRareRhythms(dataset, dictionaries)
filtered_dictionaries = utils.getDictionaries(filtered_dataset)
normalized_dataset, durations = utils.normalizeDuration(
filtered_dataset)
normalized_dictionaries = utils.getDictionaries(
normalized_dataset, durations)
augmented_dataset = utils.augment(normalized_dataset,
normalized_dictionaries)
augmented_dictionaries = utils.getDictionaries(augmented_dataset,
durations)
self.dataset = augmented_dataset
self.dictionaries = augmented_dictionaries
print(self.dictionaries)
def generator(X_train, y_train, batch_size, training=False):
'''
Data generator used for training the model.
- Reades images from path
- Preprocess images
- Augments training data only.
'''
number_samples = len(y_train)
while 1:
shuffle(X_train, y_train)
for offset in range(0, number_samples, batch_size):
X, y = X_train[offset:offset + batch_size], y_train[offset:offset +
batch_size]
X = [utils.open_image(x) for x in X]
X = [utils.pre_process(x) for x in X]
if training:
for i, (image, label) in enumerate(zip(X, y)):
X[i] = utils.augment(image)
X = np.array([keras_image.img_to_array(x) for x in X])
yield shuffle(X, y)
def save_slices(img_p: Path,
gt_p: Path,
dest_dir: Path,
shape: Tuple[int, int],
n_augment: int,
img_dir: str = "img",
gt_dir: str = "gt") -> Tuple[Any, Any, Any, Any]:
p_id: str = get_p_id(img_p)
assert "patient" in p_id
assert p_id == get_p_id(gt_p)
f_id: str = get_frame(img_p.name)
assert f_id == get_frame(gt_p.name)
# Load the data
dx, dy, dz = nib.load(str(img_p)).header.get_zooms()
assert dz in [5, 6.5, 7, 10], dz
img = np.asarray(nib.load(str(img_p)).dataobj)
gt = np.asarray(nib.load(str(gt_p)).dataobj)
nx, ny = shape
fx = nx / img.shape[0]
fy = ny / img.shape[1]
# print(f"Before dx {dx:.04f}, dy {dy:.04f}")
dx /= fx
dy /= fy
# print(f"After dx {dx:.04f}, dy {dy:.04f}")
# print(dx, dy, dz)
pixel_surface: float = dx * dy
voxel_volume: float = dx * dy * dz
assert img.shape == gt.shape
# assert img.shape[:-1] == shape
assert img.dtype in [np.uint8, np.int16, np.float32]
# Normalize and check data content
norm_img = norm_arr(
img) # We need to normalize the whole 3d img, not 2d slices
assert 0 == norm_img.min() and norm_img.max() == 255, (norm_img.min(),
norm_img.max())
assert gt.dtype == norm_img.dtype == np.uint8
resize_: Callable = partial(resize,
mode="constant",
preserve_range=True,
anti_aliasing=False)
save_dir_img: Path = Path(dest_dir, img_dir)
save_dir_gt: Path = Path(dest_dir, gt_dir)
sizes_2d: np.ndarray = np.zeros(img.shape[-1])
for j in range(img.shape[-1]):
img_s = norm_img[:, :, j]
gt_s = gt[:, :, j]
assert img_s.shape == gt_s.shape
assert gt_s.dtype == np.uint8
# Resize and check the data are still what we expect
r_img: np.ndarray = resize_(img_s, shape).astype(np.uint8)
r_gt: np.ndarray = resize_(gt_s, shape, order=0)
# r_gt: np.ndarray = np.array(Image.fromarray(gt_s, mode='L').resize(shape))
assert set(uniq(r_gt)).issubset(set(uniq(gt))), (r_gt.dtype,
uniq(r_gt))
r_gt = r_gt.astype(np.uint8)
assert r_img.dtype == r_gt.dtype == np.uint8
assert 0 <= r_img.min() and r_img.max(
) <= 255 # The range might be smaller
sizes_2d[j] = (r_gt == 3).astype(np.int64).sum()
for k in range(n_augment + 1):
if k == 0:
a_img, a_gt = r_img, r_gt
else:
a_img, a_gt = map_(np.asarray, augment(r_img, r_gt))
for save_dir, data in zip([save_dir_img, save_dir_gt],
[a_img, a_gt]):
filename = f"{p_id}_{f_id}_{k}_{j}.png"
save_dir.mkdir(parents=True, exist_ok=True)
with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=UserWarning)
imsave(str(Path(save_dir, filename)), data)
lv_gt = (gt == 3).astype(np.uint8)
assert set(np.unique(lv_gt)) <= set([0, 1])
assert lv_gt.shape == gt.shape
lv_gt = resize_(lv_gt, (*shape, img.shape[-1]), order=0)
assert set(np.unique(lv_gt)) <= set([0, 1])
slices_sizes_px = np.einsum("xyz->z", lv_gt.astype(np.int64))
assert np.array_equal(slices_sizes_px,
sizes_2d), (slices_sizes_px, sizes_2d)
# slices_sizes_px = sizes_2d[...]
slices_sizes_px = slices_sizes_px[slices_sizes_px > 0]
slices_sizes_mm2 = slices_sizes_px * pixel_surface
# volume_size_px = np.einsum("xyz->", lv_gt)
volume_size_px = slices_sizes_px.sum()
volume_size_mm3 = volume_size_px * voxel_volume
# print(f"{slices_sizes_px.mean():.0f}, {volume_size_px}")
return slices_sizes_px, slices_sizes_mm2, volume_size_px, volume_size_mm3
def __getitem__(self, index):
try:
lr_file = self.lr_list[index]
hr_file = self.hr_list[index]
lr_path, lr_name = util.get_file_name(lr_file)
hr_path, _ = util.get_file_name(hr_file)
center_frame_idx = int(lr_name)
if self.border_mode:
direction = 1 # 1: forward; 0: backward
if self.random_reverse and random.random() < 0.5:
direction = random.choice([0, 1])
if center_frame_idx + sum(self.frame_interval) > 99:
direction = 0
elif center_frame_idx - sum(self.frame_interval) < 0:
direction = 1
# get the neighbor list
if direction == 1:
neighbor_list = [
center_frame_idx + sum(self.frame_interval[0:i + 1])
for i in range(len(self.frame_interval))
]
else:
neighbor_list = [
center_frame_idx - sum(self.frame_interval[0:i + 1])
for i in range(len(self.frame_interval))
]
else:
while (center_frame_idx +
sum(self.frame_interval[self.half_N_frames:]) >
99) or (center_frame_idx - sum(
self.frame_interval[:self.half_N_frames + 1]) < 0):
center_frame_idx = random.randint(0, 99)
# get the neighbor list
neighbor_list = []
i = center_frame_idx
for x in self.frame_interval[0:self.half_N_frames]:
i -= x
neighbor_list.append(i)
neighbor_list.reverse()
neighbor_list.append(center_frame_idx)
i = center_frame_idx
for x in self.frame_interval[self.half_N_frames + 1:]:
i += x
neighbor_list.append(i)
if self.random_reverse and random.random() < 0.5:
neighbor_list.reverse()
#### get lr images
lr_data_list = []
for v in neighbor_list:
lr_data_path = osp.join(lr_path, '{:03d}.png'.format(v))
lr_data_list.append(self._read_img(lr_data_path))
#### get hr images
hr_data_list = []
for v in neighbor_list:
hr_data_path = osp.join(hr_path, '{:03d}.png'.format(v))
hr_data_list.append(self._read_img(hr_data_path))
# randomly crop
height, width, channel = hr_data_list[0].shape
hr_size_w, hr_size_h = self.size_w * self.scale, self.size_h * self.scale
lr_height = height // self.scale
lr_width = width // self.scale
rnd_h = random.randint(0, max(0, lr_height - self.size_h))
rnd_w = random.randint(0, max(0, lr_width - self.size_w))
img_lr_list = [
one_data[rnd_h:rnd_h + self.size_h,
rnd_w:rnd_w + self.size_w, :]
for one_data in lr_data_list
]
rnd_h_hr, rnd_w_hr = int(rnd_h * self.scale), int(rnd_w *
self.scale)
img_hr_list = [
one_data[rnd_h_hr:rnd_h_hr + hr_size_h,
rnd_w_hr:rnd_w_hr + hr_size_w, :]
for one_data in hr_data_list
]
# augmentation - flip, rotate
img_lr_list.extend(img_hr_list)
rlt = util.augment(img_lr_list, hflip=True, rot=True)
img_lr_list = rlt[0:self.n_frames]
img_hr_list = rlt[self.n_frames:]
# # stack lr images to NHWC, N is the frame number
img_lrs = np.stack(img_lr_list, axis=0)
img_hrs = np.stack(img_hr_list, axis=0)
#HWC to CHW, numpy to tensor
img_lrs = torch.from_numpy(
np.ascontiguousarray(np.transpose(img_lrs,
(0, 3, 1, 2)))).float()
img_hrs = torch.from_numpy(
np.ascontiguousarray(np.transpose(img_hrs,
(0, 3, 1, 2)))).float()
# RGB mean for DIV2K
img_lrs = self.sub_mean(img_lrs)
img_hrs = self.sub_mean(img_hrs)
#NCHW to CNHW
img_lrs = img_lrs.permute(1, 0, 2, 3).contiguous()
img_hrs = img_hrs.permute(1, 0, 2, 3).contiguous()
# contiguous() # 把tensor变成在内存中连续分布的形式。
# 判断是否contiguous用torch.Tensor.is_contiguous()函数。
except Exception as e:
random_sum = random.randrange(0, self.__len__())
return self.__getitem__(random_sum)
return {'LRs': img_lrs, 'HRs': img_hrs}
def save_slices(img_p: Path, gt_p: Path,
dest_dir: Path, shape: Tuple[int], n_augment: int,
img_dir: str = "img", gt_dir: str = "gt") -> Tuple[int, int, int]:
p_id: str = get_p_id(img_p)
assert "Case" in p_id
assert p_id == get_p_id(gt_p)
# Load the data
img = imread(str(img_p), plugin='simpleitk')
gt = imread(str(gt_p), plugin='simpleitk')
# print(img.shape, img.dtype, gt.shape, gt.dtype)
# print(img.min(), img.max(), len(np.unique(img)))
# print(np.unique(gt))
assert img.shape == gt.shape
assert img.dtype in [np.int16]
assert gt.dtype in [np.int8]
# Normalize and check data content
norm_img = norm_arr(img) # We need to normalize the whole 3d img, not 2d slices
assert 0 == norm_img.min() and norm_img.max() == 255, (norm_img.min(), norm_img.max())
assert norm_img.dtype == np.uint8
save_dir_img: Path = Path(dest_dir, img_dir)
save_dir_gt: Path = Path(dest_dir, gt_dir)
sizes_2d: np.ndarray = np.zeros(img.shape[-1])
for j in range(len(img)):
img_s = norm_img[j, :, :]
gt_s = gt[j, :, :]
assert img_s.shape == gt_s.shape
# Resize and check the data are still what we expect
resize_: Callable = partial(resize, mode="constant", preserve_range=True, anti_aliasing=False)
r_img: np.ndarray = resize_(img_s, shape).astype(np.uint8)
r_gt: np.ndarray = resize_(gt_s, shape).astype(np.uint8)
assert r_img.dtype == r_gt.dtype == np.uint8
assert 0 <= r_img.min() and r_img.max() <= 255 # The range might be smaller
assert set(uniq(r_gt)).issubset(set(uniq(gt)))
sizes_2d[j] = r_gt[r_gt == 1].sum()
# for save_dir, data in zip([save_dir_img, save_dir_gt], [r_img, r_gt]):
# save_dir.mkdir(parents=True, exist_ok=True)
# with warnings.catch_warnings():
# warnings.filterwarnings("ignore", category=UserWarning)
# imsave(str(Path(save_dir, filename)), data)
for k in range(n_augment + 1):
if k == 0:
a_img, a_gt = r_img, r_gt
else:
a_img, a_gt = map_(np.asarray, augment(r_img, r_gt))
for save_dir, data in zip([save_dir_img, save_dir_gt], [a_img, a_gt]):
filename = f"{p_id}_{k}_{j:02d}.png"
save_dir.mkdir(parents=True, exist_ok=True)
with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=UserWarning)
imsave(str(Path(save_dir, filename)), data)
return sizes_2d.sum(), sizes_2d[sizes_2d > 0].min(), sizes_2d.max()
def predict_image_set_with_augmentation(images, model):
augmented_images,_ = utils.augment(images, np.zeros(images.shape[0]))
#augmented_images,_ = train_model_mp.augment(images, np.zeros(images.shape[0]))
predictions = model.predict(augmented_images)
return np.exp(np.mean(np.log(predictions),axis=0,keepdims=True))
def cnn_model_fn(features, labels, mode,params):
"""Model function for CNN."""
# Input Layer
# Reshape X to 4-D tensor: [batch_size, width, height, channels]
# MNIST images are 28x28 pixels, and have one color channel
# input_layer = tf.reshape(features["x"], [-1, 28, 28, 1])
# iw = params['iw'] # image width (default, i.e. 28)
Kmnist = params['Kmnist'] # Number of classes
tw = params['tw'] # Total image width
if isinstance(features, tf.data.Dataset):
dataset = features
else :
dataset = None
images = features['x']
if dataset is not None :
dw = 6
dataset = dataset.batch(dw*dw+1)
iterator = dataset.make_initializable_iterator()
sess = tf.Session()
print("Global init")
init_op = tf.global_variables_initializer()
sess.run(init_op)
sess.run(iterator.initializer)
images, labels = iterator.get_next()
if images.shape[1].value == 28*28:
iw = 28
pad_size = (tw, tw)
else:
iw = tw
pad_size = None
images = tf.reshape(images,[-1,iw,iw,1])
## make compatible with MNIST in 28 x 28 format.
if mode == tf.estimator.ModeKeys.PREDICT: # Turn off input perturbation.
class_translation_minmax = None
else :
class_translation_minmax = []
for k in range(Kmnist):
dt = [0, (tw - iw)//2 + iw //3 ]
if k == Kmnist-1 and Kmnist == 11:
dt = [(tw + iw) // 2 - iw // 2, (tw + iw) // 2]
class_translation_minmax.append( dt )
class_translation_minmax = np.asarray(class_translation_minmax, dtype=np.float32)
[images, labels] = augment(images, labels, resize=None, pad_to_size=pad_size, class_translation_minmax=class_translation_minmax)
if dataset is not None:
im,la,RTr, arr = sess.run([images,labels, RT, ar])
for i in range(dw*dw):
mpl.subplot(dw, dw, i+1)
a = im[i].squeeze()
mpl.imshow(a)
mpl.title("Class: " + str( la[i]) )
mpl.show()
print("Show figures done.")
input_layer = images
# images = tf.placeholder(tf.uint8, shape=(None, None, None)) # add ,3) as last dim for RGB.
# labels = tf.placeholder(tf.uint64, shape=(None))
# Convolutional Layer #1
# Computes 32 features using a 5x5 filter with ReLU activation.
# Padding is added to preserve width and height.
# Input Tensor Shape: [batch_size, 28, 28, 1]
# Output Tensor Shape: [batch_size, 28, 28, 32]
conv1 = tf.layers.conv2d(
inputs=input_layer,
filters=32,
kernel_size=[5, 5],
padding="same",
activation=tf.nn.relu)
# Pooling Layer #1
# First max pooling layer with a 2x2 filter and stride of 2
# Input Tensor Shape: [batch_size, 28, 28, 32]
# Output Tensor Shape: [batch_size, 14, 14, 32] # 40 --> 20?
pool1 = tf.layers.max_pooling2d(inputs=conv1, pool_size=[2, 2], strides=2,name="pool1")
# Convolutional Layer #2
# Computes 64 features using a 5x5 filter.
# Padding is added to preserve width and height.
# Input Tensor Shape: [batch_size, 14, 14, 32]
# Output Tensor Shape: [batch_size, 14, 14, 64]
conv2 = tf.layers.conv2d(
inputs=pool1,
filters=64,
kernel_size=[5, 5],
padding="same",
activation=tf.nn.relu,name="conv2")
# Pooling Layer #2
# Second max pooling layer with a 2x2 filter and stride of 2
# Input Tensor Shape: [batch_size, 14, 14, 64]
# Output Tensor Shape: [batch_size, 7, 7, 64] # 20 --> 10
pool2 = tf.layers.max_pooling2d(inputs=conv2, pool_size=[2, 2], strides=2,name="pool2")
# Flatten tensor into a batch of vectors
# Input Tensor Shape: [batch_size, 7, 7, 64]
# Output Tensor Shape: [batch_size, 7 * 7 * 64]
nf = tw // 4
pool2_flat = tf.reshape(pool2, [-1, nf * nf * 64],name="pool2_flat")
# Dense Layer
# Densely connected layer with 1024 neurons
# Input Tensor Shape: [batch_size, 7 * 7 * 64]
# Output Tensor Shape: [batch_size, 1024]
dense1 = tf.layers.dense(inputs=pool2_flat, units=1024, activation=tf.nn.relu,name="densel1")
#dense2 = tf.layers.dense(inputs=dense1, units=1024, activation=tf.nn.relu, name="densel2")
# Add dropout operation; 0.6 probability that element will be kept
dropout = tf.layers.dropout(
inputs=dense1, rate=0.4, training=mode == tf.estimator.ModeKeys.TRAIN)
# Logits layer
# Input Tensor Shape: [batch_size, 1024]
# Output Tensor Shape: [batch_size, 10]
logits = tf.layers.dense(inputs=dropout, units=Kmnist)
predictions = {
# Generate predictions (for PREDICT and EVAL mode)
"classes": tf.argmax(input=logits, axis=1),
"probabilities": tf.nn.softmax(logits, name="softmax_tensor")
}
if mode == tf.estimator.ModeKeys.PREDICT:
return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions)
# Calculate Loss (for both TRAIN and EVAL modes)
loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits)
# Configure the Training Op (for TRAIN mode)
if mode == tf.estimator.ModeKeys.TRAIN:
optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.001)
train_op = optimizer.minimize(
loss=loss,
global_step=tf.train.get_global_step())
return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op)
# Add evaluation metrics (for EVAL mode)
eval_metric_ops = {
"accuracy": tf.metrics.accuracy(
labels=labels, predictions=predictions["classes"])}
return tf.estimator.EstimatorSpec(
mode=mode, loss=loss, eval_metric_ops=eval_metric_ops)
def main():
"""The main function."""
# Prepare the data for training
# - Load data
if 'eurosat' in DATASET:
x_train, x_val, x_test, _ = load_eurosat()
else: # 'ucm' in DATASET
x_train, x_val, x_test, _ = load_ucm()
# - Resize and rescale the images, and cache the training and the validation sets for faster training
x_train = x_train.map(
rescale_resize(image_size=INPUT_SHAPE[:-1]),
num_parallel_calls=tf.data.experimental.AUTOTUNE).cache()
x_val = x_val.map(
rescale_resize(image_size=INPUT_SHAPE[:-1]),
num_parallel_calls=tf.data.experimental.AUTOTUNE).cache()
x_test = x_test.map(rescale_resize(image_size=INPUT_SHAPE[:-1]),
num_parallel_calls=tf.data.experimental.AUTOTUNE)
# - Shuffle the training set and apply the augmentation (after caching to avoid caching randomness)
x_train = x_train.shuffle(BUFFER_SIZE).map(
augment(image_size=INPUT_SHAPE[:-1], augmentation_parameters=args),
num_parallel_calls=tf.data.experimental.AUTOTUNE)
# The model
with STRATEGY.scope():
# - A variable to be passed to the model as the simulation time resolution
dt_var = tf.Variable(DT, aggregation=tf.VariableAggregation.MEAN)
# - Functions to monitor the variables
# noinspection PyUnusedLocal
def dt_monitor(y_true, y_pred):
return dt_var.read_value()
# - Create a model
model = create_spiking_vgg16_model(model_path=MODEL_PATH,
input_shape=INPUT_SHAPE,
dt=dt_var,
l2=L2,
lower_hz=LOWER_HZ,
upper_hz=UPPER_HZ,
tau=TAU,
num_classes=NUM_CLASSES,
spiking_aware_training=True)
# - Compile the model
model.compile(
optimizer=tf.keras.optimizers.RMSprop(LR),
loss=tf.losses.SparseCategoricalCrossentropy(from_logits=True),
metrics=[tf.metrics.SparseCategoricalAccuracy(), dt_monitor])
# Show model's summary
model.summary()
# Iterate or not
start = 0 if args['iterate'] else EXPONENT
for n in range(start, EXPONENT + 1):
# The data
# - Add the temporal dimension
timesteps = 2**n
x_train_t = x_train.map(
add_temporal_dim(timesteps=timesteps),
num_parallel_calls=tf.data.experimental.AUTOTUNE)
x_val_t = x_val.map(add_temporal_dim(timesteps=timesteps),
num_parallel_calls=tf.data.experimental.AUTOTUNE)
x_test_t = x_test.map(add_temporal_dim(timesteps=timesteps),
num_parallel_calls=tf.data.experimental.AUTOTUNE)
# - Optional gradient-based input (Prewitt and Sobel filters can be applied here before batching, because we
# - already have an extra dimension - the temporal one)
x_train_t = x_train_t.map(
input_filter_map(filter_name=DATASET),
num_parallel_calls=tf.data.experimental.AUTOTUNE)
x_val_t = x_val_t.map(input_filter_map(filter_name=DATASET),
num_parallel_calls=tf.data.experimental.AUTOTUNE)
x_test_t = x_test_t.map(
input_filter_map(filter_name=DATASET),
num_parallel_calls=tf.data.experimental.AUTOTUNE)
# - Batch the data
batch_size = 2**(EXPONENT - n) * BATCH_SIZE
x_train_t = x_train_t.batch(batch_size=batch_size)
x_val_t = x_val_t.batch(batch_size=batch_size)
x_test_t = x_test_t.batch(batch_size=batch_size)
# - Prefetch data
x_train_t = x_train_t.prefetch(tf.data.experimental.AUTOTUNE)
x_val_t = x_val_t.prefetch(tf.data.experimental.AUTOTUNE)
x_test_t = x_test_t.prefetch(tf.data.experimental.AUTOTUNE)
# Learning rate
lr = LR * 2**(EXPONENT - n)
tf.keras.backend.set_value(model.optimizer.learning_rate, lr)
# Epochs
epochs = EPOCHS * 2**(EXPONENT - n)
# Callbacks
log_dir = 'logs/fit/' + datetime.datetime.now().strftime(
'%Y%m%d-%H%M%S')
callbacks = [
tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1),
tf.keras.callbacks.ReduceLROnPlateau(patience=epochs // 4,
verbose=True),
tf.keras.callbacks.EarlyStopping(patience=epochs // 2,
verbose=True)
]
if dt_var.value() > DT_TARGET:
callbacks.append(DTStop(dt=dt_var, dt_min=DT_TARGET))
# Print the training iteration parameters
print(
'\nStarting the training for {} epoch(s),'.format(epochs),
'with {} timestep(s)'.format(timesteps),
'on batches of {} example(s),'.format(batch_size),
'and the learning rate {}.'.format(LR * 2**(EXPONENT - n)),
'\nLearning rate reduced after {} epoch(s)'.format(epochs // 4),
'and early stopping after {} epoch(s) of no improvement in the validation loss.\n'
.format(epochs // 2))
# Train the model
print('Commencing the training on iteration {}/{}.'.format(
min(n + 1, EXPONENT), EXPONENT))
model.fit(x=x_train_t,
epochs=epochs,
validation_data=x_val_t,
callbacks=callbacks)
# Evaluate the model
results = model.evaluate(x=x_test_t,
batch_size=batch_size,
verbose=True)
try:
loss, acc, dt_stop = results
except ValueError:
loss, acc = results
dt_stop = DT_TARGET
print('\nModel\'s accuracy: {:.2f}%.\n'.format(acc * 100))
# New model to avoid serialization issued
with STRATEGY.scope():
new_model = create_spiking_vgg16_model(model_path=MODEL_PATH,
input_shape=INPUT_SHAPE,
dt=dt_stop,
l2=L2,
lower_hz=LOWER_HZ,
upper_hz=UPPER_HZ,
tau=TAU,
num_classes=NUM_CLASSES,
spiking_aware_training=True)
# - Load weights (skipping dt)
new_model.set_weights(
[w for w in model.get_weights() if w.shape != ()])
# - Compile the model
new_model.compile(
optimizer=tf.keras.optimizers.RMSprop(LR),
loss=tf.losses.SparseCategoricalCrossentropy(from_logits=True),
metrics=[tf.metrics.SparseCategoricalAccuracy()])
# Save model filepath
model_filepath = Path('models/spiking_vgg16').joinpath(DATASET)
os.makedirs(model_filepath, exist_ok=True)
model_filepath = model_filepath.joinpath(
's_{}'.format(SEED) + '_e_{}'.format(epochs) +
'_bs_{}'.format(batch_size) + '_lr_{}'.format(lr) +
'_t_{}'.format(timesteps) + '_dt_{:.4f}'.format(dt_stop) +
'_l2_{}'.format(L2) + '_lhz_{}'.format(LOWER_HZ) +
'_uhz_{}'.format(UPPER_HZ) + '_lz_{}'.format(args['lower_zoom']) +
'_uz_{}'.format(args['upper_zoom']) +
'_mbd_{}'.format(args['max_brightness_delta']) +
'_mhd_{}'.format(args['max_hue_delta']) +
'_lc_{}'.format(args['lower_contrast']) +
'_uc_{}'.format(args['upper_contrast']) +
'_ls_{}'.format(args['lower_saturation']) +
'_us_{}'.format(args['upper_saturation']) +
'_acc_{:.2f}.h5'.format(acc))
# Save model
print('\nSaving model weights to: ' + str(model_filepath))
new_model.save_weights(model_filepath)
# We stop optimising dt here
if dt_stop <= DT_TARGET:
model = new_model
del new_model
