def test_HWC2CHW(plot=False):
"""
Test HWC2CHW
"""
logger.info("Test HWC2CHW")
# First dataset
data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
decode_op = c_vision.Decode()
hwc2chw_op = c_vision.HWC2CHW()
data1 = data1.map(input_columns=["image"], operations=decode_op)
data1 = data1.map(input_columns=["image"], operations=hwc2chw_op)
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data2 = data2.map(input_columns=["image"], operations=decode_op)
image_transposed = []
image = []
for item1, item2 in zip(data1.create_dict_iterator(), data2.create_dict_iterator()):
image_transposed.append(item1["image"].copy())
image.append(item2["image"].copy())
# check if the shape of data is transposed correctly
# transpose the original image from shape (H,W,C) to (C,H,W)
mse = diff_mse(item1['image'], item2['image'].transpose(2, 0, 1))
assert mse == 0
if plot:
visualize(image, image_transposed)
def test_random_grayscale_valid_prob(plot=False):
"""
Test RandomGrayscale Op: valid input, expect to pass
"""
logger.info("test_random_grayscale_valid_prob")
# First dataset
data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
transforms1 = [
py_vision.Decode(),
# Note: prob is 1 so the output should always be grayscale images
py_vision.RandomGrayscale(1),
py_vision.ToTensor()
]
transform1 = py_vision.ComposeOp(transforms1)
data1 = data1.map(input_columns=["image"], operations=transform1())
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
transforms2 = [
py_vision.Decode(),
py_vision.ToTensor()
]
transform2 = py_vision.ComposeOp(transforms2)
data2 = data2.map(input_columns=["image"], operations=transform2())
image_gray = []
image = []
for item1, item2 in zip(data1.create_dict_iterator(), data2.create_dict_iterator()):
image1 = (item1["image"].transpose(1, 2, 0) * 255).astype(np.uint8)
image2 = (item2["image"].transpose(1, 2, 0) * 255).astype(np.uint8)
image_gray.append(image1)
image.append(image2)
if plot:
visualize(image, image_gray)
def test_center_crop_op(height=375, width=375, plot=False):
"""
Test CenterCrop
"""
logger.info("Test CenterCrop")
# First dataset
data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"])
decode_op = vision.Decode()
# 3 images [375, 500] [600, 500] [512, 512]
center_crop_op = vision.CenterCrop([height, width])
data1 = data1.map(input_columns=["image"], operations=decode_op)
data1 = data1.map(input_columns=["image"], operations=center_crop_op)
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"])
data2 = data2.map(input_columns=["image"], operations=decode_op)
image_cropped = []
image = []
for item1, item2 in zip(data1.create_dict_iterator(),
data2.create_dict_iterator()):
image_cropped.append(item1["image"].copy())
image.append(item2["image"].copy())
if plot:
visualize(image, image_cropped)
def main(vis=False):
X, Y = util.load_dataset(
'../kaggleData/sorted',
['mask_weared_incorrect', 'with_mask', 'without_mask'], 30)
X_train, X_test, Y_train, Y_test = train_test_split(X,
Y,
test_size=0.2,
random_state=0)
clf = LogisticRegression(random_state=0,
max_iter=1000,
multi_class='multinomial',
solver='lbfgs').fit(X_train, Y_train)
# predictedTrain = clf.predict(X_train)
# util.findF1Score(predictedTrain, Y_train, "On Train Set")
accOnTrain = clf.score(X_train, Y_train)
print("Acc on train: ", accOnTrain)
# predictedTest = clf.predict(X_test)
# util.findF1Score(predictedTest, Y_test, "On Test Set")
accOnTest = clf.score(X_test, Y_test)
print("Acc on test: ", accOnTest)
if vis: util.visualize(X_train, Y_train)
def summary_thick_seg(thick_seg, writer, flag, step):
image = thick_seg.planes[..., 0, 1:4]
rho = thick_seg.planes[..., 0, 6:7]
semantic = (thick_seg.planes[..., 0, 7] * args.num_classes).long()
predict = torch.argmax(thick_seg.predict2d, dim=-1)[..., 0]
writer.add_images(flag + '/image',
image,
global_step=step,
dataformats='NHWC')
writer.add_images(flag + '/rho', rho, global_step=step, dataformats='NHWC')
writer.add_images(flag + '/semantic',
util.visualize(semantic),
global_step=step,
dataformats='NHWC')
writer.add_images(flag + '/predict',
util.visualize(predict),
global_step=step,
dataformats='NHWC')
if thick_seg.labels2d is not None:
labels = thick_seg.labels2d[..., 0]
writer.add_images(flag + '/labels',
util.visualize(labels),
global_step=step,
dataformats='NHWC')
writer.add_scalar(flag + '/os', thick_seg.oa, global_step=step)
writer.add_scalar(flag + '/mean_iou',
thick_seg.mean_iou,
global_step=step)
writer.add_histogram(flag + '/iou', thick_seg.iou, global_step=step)
def test_random_perspective_op(plot=False):
"""
Test RandomPerspective in python transformations
"""
logger.info("test_random_perspective_op")
# define map operations
transforms1 = [
py_vision.Decode(),
py_vision.RandomPerspective(),
py_vision.ToTensor()
]
transform1 = py_vision.ComposeOp(transforms1)
transforms2 = [
py_vision.Decode(),
py_vision.ToTensor()
]
transform2 = py_vision.ComposeOp(transforms2)
# First dataset
data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data1 = data1.map(input_columns=["image"], operations=transform1())
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data2 = data2.map(input_columns=["image"], operations=transform2())
image_perspective = []
image_original = []
for item1, item2 in zip(data1.create_dict_iterator(), data2.create_dict_iterator()):
image1 = (item1["image"].transpose(1, 2, 0) * 255).astype(np.uint8)
image2 = (item2["image"].transpose(1, 2, 0) * 255).astype(np.uint8)
image_perspective.append(image1)
image_original.append(image2)
if plot:
visualize(image_original, image_perspective)
def test_random_crop_op_c(plot=False):
"""
Test RandomCrop Op in c transforms
"""
logger.info("test_random_crop_op_c")
# First dataset
data1 = ds.TFRecordDataset(DATA_DIR,
SCHEMA_DIR,
columns_list=["image"],
shuffle=False)
random_crop_op = c_vision.RandomCrop([512, 512], [200, 200, 200, 200])
decode_op = c_vision.Decode()
data1 = data1.map(input_columns=["image"], operations=decode_op)
data1 = data1.map(input_columns=["image"], operations=random_crop_op)
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR,
SCHEMA_DIR,
columns_list=["image"],
shuffle=False)
data2 = data2.map(input_columns=["image"], operations=decode_op)
image_cropped = []
image = []
for item1, item2 in zip(data1.create_dict_iterator(),
data2.create_dict_iterator()):
image1 = item1["image"]
image2 = item2["image"]
image_cropped.append(image1)
image.append(image2)
if plot:
visualize(image, image_cropped)
def main(args):
random.seed(229)
# load hyperparameters
json_path = os.path.join(args.model_dir, "params.json")
assert os.path.isfile(
json_path), "No json configuration file found at {}".format(json_path)
params = deep_net_utils.Params(json_path)
data_container = load_dataset(args, params)
X_train, Y_train = data_container['train']
X_val, Y_val = data_container['val']
X_test, Y_test = data_container['test']
clf = LogisticRegression(random_state=0,
max_iter=params.iter,
multi_class='multinomial',
solver='lbfgs').fit(X_train, Y_train)
predicted_train = clf.predict(X_train)
predicted_val = clf.predict(X_val)
train_confus_save_path = os.path.join(args.model_dir,
"confus_f1_train.json")
val_confus_save_path = os.path.join(args.model_dir, "confus_f1_val.json")
util.compute_and_save_f1(predicted_train, Y_train, train_confus_save_path)
util.compute_and_save_f1(predicted_val, Y_val, val_confus_save_path)
train_accur = clf.score(X_train, Y_train)
print("Acc on train: ", train_accur)
val_accur = clf.score(X_val, Y_val)
print("Acc on validation: ", val_accur)
if args.vis: util.visualize(X_train, Y_train)
def test_random_crop_op_py(plot=False):
"""
Test RandomCrop op in py transforms
"""
logger.info("test_random_crop_op_py")
# First dataset
data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
transforms1 = [
py_vision.Decode(),
py_vision.RandomCrop([512, 512], [200, 200, 200, 200]),
py_vision.ToTensor()
]
transform1 = py_vision.ComposeOp(transforms1)
data1 = data1.map(input_columns=["image"], operations=transform1())
# Second dataset
# Second dataset for comparison
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
transforms2 = [
py_vision.Decode(),
py_vision.ToTensor()
]
transform2 = py_vision.ComposeOp(transforms2)
data2 = data2.map(input_columns=["image"], operations=transform2())
crop_images = []
original_images = []
for item1, item2 in zip(data1.create_dict_iterator(), data2.create_dict_iterator()):
crop = (item1["image"].transpose(1, 2, 0) * 255).astype(np.uint8)
original = (item2["image"].transpose(1, 2, 0) * 255).astype(np.uint8)
crop_images.append(crop)
original_images.append(original)
if plot:
visualize(original_images, crop_images)
def test_random_crop_comp(plot=False):
"""
Test RandomCrop and compare between python and c image augmentation
"""
logger.info("Test RandomCrop with c_transform and py_transform comparison")
cropped_size = 512
# First dataset
data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
random_crop_op = c_vision.RandomCrop(cropped_size)
decode_op = c_vision.Decode()
data1 = data1.map(input_columns=["image"], operations=decode_op)
data1 = data1.map(input_columns=["image"], operations=random_crop_op)
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
transforms = [
py_vision.Decode(),
py_vision.RandomCrop(cropped_size),
py_vision.ToTensor()
]
transform = py_vision.ComposeOp(transforms)
data2 = data2.map(input_columns=["image"], operations=transform())
image_c_cropped = []
image_py_cropped = []
for item1, item2 in zip(data1.create_dict_iterator(), data2.create_dict_iterator()):
c_image = item1["image"]
py_image = (item2["image"].transpose(1, 2, 0) * 255).astype(np.uint8)
image_c_cropped.append(c_image)
image_py_cropped.append(py_image)
if plot:
visualize(image_c_cropped, image_py_cropped)
def test_center_crop_comp(height=375, width=375, plot=False):
"""
Test CenterCrop between python and c image augmentation
"""
logger.info("Test CenterCrop")
# First dataset
data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
decode_op = vision.Decode()
center_crop_op = vision.CenterCrop([height, width])
data1 = data1.map(input_columns=["image"], operations=decode_op)
data1 = data1.map(input_columns=["image"], operations=center_crop_op)
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
transforms = [
py_vision.Decode(),
py_vision.CenterCrop([height, width]),
py_vision.ToTensor()
]
transform = py_vision.ComposeOp(transforms)
data2 = data2.map(input_columns=["image"], operations=transform())
image_cropped = []
image = []
for item1, item2 in zip(data1.create_dict_iterator(), data2.create_dict_iterator()):
c_image = item1["image"]
py_image = (item2["image"].transpose(1, 2, 0) * 255).astype(np.uint8)
# Note: The images aren't exactly the same due to rounding error
assert diff_mse(py_image, c_image) < 0.001
image_cropped.append(c_image.copy())
image.append(py_image.copy())
if plot:
visualize(image, image_cropped)
def train(self, model, train_dataloader, eval_dataloader, val_dict):
device = self.device
model.to(device)
optimizer = AdamW(model.parameters(), lr=self.lr)
global_idx = 0
best_scores = {'F1': -1.0, 'EM': -1.0}
tensorboard_writer = SummaryWriter(self.save_dir)
for epoch_num in range(self.num_epochs):
self.log.info(f'Epoch: {epoch_num}')
with torch.enable_grad(), tqdm(total=len(train_dataloader.dataset),
position=0,
leave=True) as progress_bar:
for batch in train_dataloader:
optimizer.zero_grad()
model.train()
input_ids = batch['input_ids'].to(device)
attention_mask = batch['attention_mask'].to(device)
start_positions = batch['start_positions'].to(device)
end_positions = batch['end_positions'].to(device)
outputs = model(input_ids,
attention_mask=attention_mask,
start_positions=start_positions,
end_positions=end_positions)
loss = outputs[0]
loss.backward()
optimizer.step()
progress_bar.update(len(input_ids))
progress_bar.set_postfix(epoch=epoch_num, NLL=loss.item())
tensorboard_writer.add_scalar('train/NLL', loss.item(),
global_idx)
if (global_idx % self.eval_every) == 0:
self.log.info(f'Evaluating at step {global_idx}...')
preds, curr_score = self.evaluate(model,
eval_dataloader,
val_dict,
return_preds=True)
results_str = ', '.join(f'{k}: {v:05.2f}'
for k, v in curr_score.items())
self.log.info('Visualizing in TensorBoard...')
for k, v in curr_score.items():
tensorboard_writer.add_scalar(
f'val/{k}', v, global_idx)
self.log.info(f'Eval {results_str}')
if self.visualize_predictions:
util.visualize(tensorboard_writer,
pred_dict=preds,
gold_dict=val_dict,
step=global_idx,
split='val',
num_visuals=self.num_visuals)
if curr_score['F1'] >= best_scores['F1']:
best_scores = curr_score
self.save(model)
global_idx += 1
return best_scores
def test_linear_transformation_op(plot=False):
"""
Test LinearTransformation op: verify if images transform correctly
"""
logger.info("test_linear_transformation_01")
# Initialize parameters
height = 50
weight = 50
dim = 3 * height * weight
transformation_matrix = np.eye(dim)
mean_vector = np.zeros(dim)
# Define operations
transforms = [
py_vision.Decode(),
py_vision.CenterCrop([height, weight]),
py_vision.ToTensor()
]
transform = py_vision.ComposeOp(transforms)
# First dataset
data1 = ds.TFRecordDataset(DATA_DIR,
SCHEMA_DIR,
columns_list=["image"],
shuffle=False)
data1 = data1.map(input_columns=["image"], operations=transform())
# Note: if transformation matrix is diagonal matrix with all 1 in diagonal,
# the output matrix in expected to be the same as the input matrix.
data1 = data1.map(input_columns=["image"],
operations=py_vision.LinearTransformation(
transformation_matrix, mean_vector))
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR,
SCHEMA_DIR,
columns_list=["image"],
shuffle=False)
data2 = data2.map(input_columns=["image"], operations=transform())
image_transformed = []
image = []
for item1, item2 in zip(data1.create_dict_iterator(),
data2.create_dict_iterator()):
image1 = (item1["image"].transpose(1, 2, 0) * 255).astype(np.uint8)
image2 = (item2["image"].transpose(1, 2, 0) * 255).astype(np.uint8)
image_transformed.append(image1)
image.append(image2)
mse = diff_mse(image1, image2)
assert mse == 0
if plot:
visualize(image, image_transformed)
def test_five_crop_op(plot=False):
"""
Test FiveCrop
"""
logger.info("test_five_crop")
# First dataset
data1 = ds.TFRecordDataset(DATA_DIR,
SCHEMA_DIR,
columns_list=["image"],
shuffle=False)
transforms_1 = [
vision.Decode(),
vision.ToTensor(),
]
transform_1 = vision.ComposeOp(transforms_1)
data1 = data1.map(input_columns=["image"], operations=transform_1())
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR,
SCHEMA_DIR,
columns_list=["image"],
shuffle=False)
transforms_2 = [
vision.Decode(),
vision.FiveCrop(200),
lambda images: np.stack([vision.ToTensor()(image)
for image in images]) # 4D stack of 5 images
]
transform_2 = vision.ComposeOp(transforms_2)
data2 = data2.map(input_columns=["image"], operations=transform_2())
num_iter = 0
for item1, item2 in zip(data1.create_dict_iterator(),
data2.create_dict_iterator()):
num_iter += 1
image_1 = (item1["image"].transpose(1, 2, 0) * 255).astype(np.uint8)
image_2 = item2["image"]
logger.info("shape of image_1: {}".format(image_1.shape))
logger.info("shape of image_2: {}".format(image_2.shape))
logger.info("dtype of image_1: {}".format(image_1.dtype))
logger.info("dtype of image_2: {}".format(image_2.dtype))
if plot:
visualize(np.array([image_1] * 10),
(image_2 * 255).astype(np.uint8).transpose(0, 2, 3, 1))
# The output data should be of a 4D tensor shape, a stack of 5 images.
assert len(image_2.shape) == 4
assert image_2.shape[0] == 5
def util_test_ten_crop(crop_size, vertical_flip=False, plot=False):
"""
Utility function for testing TenCrop. Input arguments are given by other tests
"""
data1 = ds.TFRecordDataset(DATA_DIR,
SCHEMA_DIR,
columns_list=["image"],
shuffle=False)
transforms_1 = [
vision.Decode(),
vision.ToTensor(),
]
transform_1 = vision.ComposeOp(transforms_1)
data1 = data1.map(input_columns=["image"], operations=transform_1())
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR,
SCHEMA_DIR,
columns_list=["image"],
shuffle=False)
transforms_2 = [
vision.Decode(),
vision.TenCrop(crop_size, use_vertical_flip=vertical_flip),
lambda images: np.stack([vision.ToTensor()(image)
for image in images]) # 4D stack of 10 images
]
transform_2 = vision.ComposeOp(transforms_2)
data2 = data2.map(input_columns=["image"], operations=transform_2())
num_iter = 0
for item1, item2 in zip(data1.create_dict_iterator(),
data2.create_dict_iterator()):
num_iter += 1
image_1 = (item1["image"].transpose(1, 2, 0) * 255).astype(np.uint8)
image_2 = item2["image"]
logger.info("shape of image_1: {}".format(image_1.shape))
logger.info("shape of image_2: {}".format(image_2.shape))
logger.info("dtype of image_1: {}".format(image_1.dtype))
logger.info("dtype of image_2: {}".format(image_2.dtype))
if plot:
visualize(np.array([image_1] * 10),
(image_2 * 255).astype(np.uint8).transpose(0, 2, 3, 1))
# The output data should be of a 4D tensor shape, a stack of 10 images.
assert len(image_2.shape) == 4
assert image_2.shape[0] == 10
def test_random_choice_comp(plot=False):
"""
Test RandomChoice and compare with single CenterCrop results
"""
logger.info("test_random_choice_comp")
# define map operations
transforms_list = [py_vision.CenterCrop(64)]
transforms1 = [
py_vision.Decode(),
py_vision.RandomChoice(transforms_list),
py_vision.ToTensor()
]
transform1 = py_vision.ComposeOp(transforms1)
transforms2 = [
py_vision.Decode(),
py_vision.CenterCrop(64),
py_vision.ToTensor()
]
transform2 = py_vision.ComposeOp(transforms2)
# First dataset
data1 = ds.TFRecordDataset(DATA_DIR,
SCHEMA_DIR,
columns_list=["image"],
shuffle=False)
data1 = data1.map(input_columns=["image"], operations=transform1())
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR,
SCHEMA_DIR,
columns_list=["image"],
shuffle=False)
data2 = data2.map(input_columns=["image"], operations=transform2())
image_choice = []
image_original = []
for item1, item2 in zip(data1.create_dict_iterator(),
data2.create_dict_iterator()):
image1 = (item1["image"].transpose(1, 2, 0) * 255).astype(np.uint8)
image2 = (item2["image"].transpose(1, 2, 0) * 255).astype(np.uint8)
image_choice.append(image1)
image_original.append(image2)
mse = diff_mse(image1, image2)
assert mse == 0
if plot:
visualize(image_original, image_choice)
def test_random_crop_and_resize_op_py(plot=False):
"""
Test RandomCropAndResize op in py transforms
"""
logger.info("test_random_crop_and_resize_op_py")
# First dataset
data1 = ds.TFRecordDataset(DATA_DIR,
SCHEMA_DIR,
columns_list=["image"],
shuffle=False)
# With these inputs we expect the code to crop the whole image
transforms1 = [
py_vision.Decode(),
py_vision.RandomResizedCrop((256, 512), (2, 2), (1, 3)),
py_vision.ToTensor()
]
transform1 = py_vision.ComposeOp(transforms1)
data1 = data1.map(input_columns=["image"], operations=transform1())
# Second dataset
# Second dataset for comparison
data2 = ds.TFRecordDataset(DATA_DIR,
SCHEMA_DIR,
columns_list=["image"],
shuffle=False)
transforms2 = [py_vision.Decode(), py_vision.ToTensor()]
transform2 = py_vision.ComposeOp(transforms2)
data2 = data2.map(input_columns=["image"], operations=transform2())
num_iter = 0
crop_and_resize_images = []
original_images = []
for item1, item2 in zip(data1.create_dict_iterator(),
data2.create_dict_iterator()):
crop_and_resize = (item1["image"].transpose(1, 2, 0) * 255).astype(
np.uint8)
original = (item2["image"].transpose(1, 2, 0) * 255).astype(np.uint8)
original = cv2.resize(original, (512, 256))
mse = diff_mse(crop_and_resize, original)
# Due to rounding error the mse for Python is not exactly 0
assert mse <= 0.05
logger.info("random_crop_and_resize_op_{}, mse: {}".format(
num_iter + 1, mse))
num_iter += 1
crop_and_resize_images.append(crop_and_resize)
original_images.append(original)
if plot:
visualize(original_images, crop_and_resize_images)
def test_HWC2CHW_comp(plot=False):
"""
Test HWC2CHW between python and c image augmentation
"""
logger.info("Test HWC2CHW with c_transform and py_transform comparison")
# First dataset
data1 = ds.TFRecordDataset(DATA_DIR,
SCHEMA_DIR,
columns_list=["image"],
shuffle=False)
decode_op = c_vision.Decode()
hwc2chw_op = c_vision.HWC2CHW()
data1 = data1.map(input_columns=["image"], operations=decode_op)
data1 = data1.map(input_columns=["image"], operations=hwc2chw_op)
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR,
SCHEMA_DIR,
columns_list=["image"],
shuffle=False)
transforms = [
py_vision.Decode(),
py_vision.ToTensor(),
py_vision.HWC2CHW()
]
transform = py_vision.ComposeOp(transforms)
data2 = data2.map(input_columns=["image"], operations=transform())
image_c_transposed = []
image_py_transposed = []
for item1, item2 in zip(data1.create_dict_iterator(),
data2.create_dict_iterator()):
c_image = item1["image"]
py_image = (item2["image"].transpose(1, 2, 0) * 255).astype(np.uint8)
# compare images between that applying c_transform and py_transform
mse = diff_mse(py_image, c_image)
# the images aren't exactly the same due to rounding error
assert mse < 0.001
image_c_transposed.append(item1["image"].copy())
image_py_transposed.append(item2["image"].copy())
if plot:
visualize(image_c_transposed, image_py_transposed)
def test_random_vertical_comp(plot=False):
"""
Test test_random_vertical_flip and compare between python and c image augmentation ops
"""
logger.info("test_random_vertical_comp")
# First dataset
data1 = ds.TFRecordDataset(DATA_DIR,
SCHEMA_DIR,
columns_list=["image"],
shuffle=False)
decode_op = c_vision.Decode()
# Note: The image must be flipped if prob is set to be 1
random_horizontal_op = c_vision.RandomVerticalFlip(1)
data1 = data1.map(input_columns=["image"], operations=decode_op)
data1 = data1.map(input_columns=["image"], operations=random_horizontal_op)
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR,
SCHEMA_DIR,
columns_list=["image"],
shuffle=False)
transforms = [
py_vision.Decode(),
# Note: The image must be flipped if prob is set to be 1
py_vision.RandomVerticalFlip(1),
py_vision.ToTensor()
]
transform = py_vision.ComposeOp(transforms)
data2 = data2.map(input_columns=["image"], operations=transform())
images_list_c = []
images_list_py = []
for item1, item2 in zip(data1.create_dict_iterator(),
data2.create_dict_iterator()):
image_c = item1["image"]
image_py = (item2["image"].transpose(1, 2, 0) * 255).astype(np.uint8)
images_list_c.append(image_c)
images_list_py.append(image_py)
# Check if the output images are the same
mse = diff_mse(image_c, image_py)
assert mse < 0.001
if plot:
visualize(images_list_c, images_list_py)
def train(args, params):
random.seed(229)
# Check model_type is set (if 'model_type' is 'svm', 'kernel' is also set)
assert (hasattr(params, 'model_type'))
if params.model_type == 'svm': assert (hasattr(params, 'kernel'))
data_container = util.load_dataset_for_split(args, params)
X_train, Y_train = data_container['train']
X_val, Y_val = data_container['val']
X_test, Y_test = data_container['test']
if args.verbose:
print(
"We have {}, {}, and {} training examples for classes 0, 1, 2 respectively."
.format(np.sum(np.where(Y_train == 0, 1, 0)),
np.sum(np.where(Y_train == 1, 1, 0)),
np.sum(np.where(Y_train == 2, 1, 0))))
model = fit_softmax_or_svm(X_train, Y_train, params)
model_file_path = os.path.join(args.model_dir, "finalized_model.sav")
pickle.dump(model, open(model_file_path, 'wb'))
predicted_train = model.predict(X_train)
predicted_val = model.predict(X_val)
train_confus_save_path = os.path.join(args.model_dir,
"confus_f1_train.json")
val_confus_save_path = os.path.join(args.model_dir, "confus_f1_val.json")
util.compute_and_save_f1(predicted_train, Y_train, train_confus_save_path)
util.compute_and_save_f1(predicted_val, Y_val, val_confus_save_path)
if params.model_type == 'softmax' and args.verbose:
print("Model converged in {} iterations.".format(model.n_iter_))
train_accur = model.score(X_train, Y_train)
print("Acc on train: ", train_accur)
val_accur = model.score(X_val, Y_val)
print("Acc on validation: ", val_accur)
if args.vis: util.visualize(X_train, Y_train)
def test_random_crop_and_resize_op_c(plot=False):
"""
Test RandomCropAndResize op in c transforms
"""
logger.info("test_random_crop_and_resize_op_c")
# First dataset
data1 = ds.TFRecordDataset(DATA_DIR,
SCHEMA_DIR,
columns_list=["image"],
shuffle=False)
decode_op = c_vision.Decode()
# With these inputs we expect the code to crop the whole image
random_crop_and_resize_op = c_vision.RandomResizedCrop((256, 512), (2, 2),
(1, 3))
data1 = data1.map(input_columns=["image"], operations=decode_op)
data1 = data1.map(input_columns=["image"],
operations=random_crop_and_resize_op)
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR,
SCHEMA_DIR,
columns_list=["image"],
shuffle=False)
data2 = data2.map(input_columns=["image"], operations=decode_op)
num_iter = 0
crop_and_resize_images = []
original_images = []
for item1, item2 in zip(data1.create_dict_iterator(),
data2.create_dict_iterator()):
crop_and_resize = item1["image"]
original = item2["image"]
# Note: resize the original image with the same size as the one applied RandomResizedCrop()
original = cv2.resize(original, (512, 256))
mse = diff_mse(crop_and_resize, original)
assert mse == 0
logger.info("random_crop_and_resize_op_{}, mse: {}".format(
num_iter + 1, mse))
num_iter += 1
crop_and_resize_images.append(crop_and_resize)
original_images.append(original)
if plot:
visualize(original_images, crop_and_resize_images)
def main():
X, Y = util.load_dataset('../data/small/', ['incorrect', 'correct'], 96)
X_train, X_test, Y_train, Y_test = train_test_split(X,
Y,
test_size=0.2,
random_state=0)
clf = LogisticRegression(random_state=0,
max_iter=1000).fit(X_train, Y_train)
# clf = SGDClassifier(loss='log', max_iter=10000).fit(X_train, Y_train)
predictedTrain = clf.predict(X_train)
util.findF1Score(predictedTrain, Y_train, "On Train Set")
accOnTrain = clf.score(X_train, Y_train)
print("Acc on train: ", accOnTrain)
predictedTest = clf.predict(X_test)
util.findF1Score(predictedTest, Y_test, "On Test Set")
accOnTest = clf.score(X_test, Y_test)
print("Acc on test: ", accOnTest)
util.visualize(X_train, Y_train)
def main(vis = False):
X, Y = util.load_dataset('../FinalPhotosData', ['mask_weared_incorrect', 'with_mask', 'without_mask'], 224)
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2, random_state=0)
svm = SVC(kernel='rbf', probability=True, random_state=42)
# fit model
svm.fit(X_train, Y_train)
# predictedTrain = clf.predict(X_train)
# util.findF1Score(predictedTrain, Y_train, "On Train Set")
Y_pred = svm.predict(X_test)
# calculate accuracy
accuracy = accuracy_score(Y_test, Y_pred)
print('Model accuracy is: ', accuracy)
# # predictedTest = clf.predict(X_test)
# # util.findF1Score(predictedTest, Y_test, "On Test Set")
# accOnTest = clf.score(X_test, Y_test)
# print("Acc on test: ", accOnTest)
if vis: util.visualize(X_train, Y_train)
def main():
print("Analyzer has started..", flush=True)
if not os.path.exists(DATA_FOLDER_PATH):
os.mkdir(DATA_FOLDER_PATH)
if os.path.exists(DATA_ANALYZER_PATH):
with open(DATA_ANALYZER_PATH, 'r') as f:
data_analyzer = json.load(f)
else:
data_analyzer = data_template
data_analyzer = data_template
data_analyzer = addict.Dict(data_analyzer)
while True:
t0 = time.time()
last_id_processed = data_analyzer['last_id_processed']
user_answers = requests.get(
f"{URL_DATA_DISTRIBUTOR}/user_answers/{last_id_processed}").json()
print(user_answers)
if len(user_answers):
new_data_analyzer = parseUserAnswers(user_answers,
URL_DATA_DISTRIBUTOR)
data_analyzer = updateAnalyzerDict(data_analyzer,
new_data_analyzer)
visualize(data_analyzer, ANALYZE_PATH)
with open(DATA_ANALYZER_PATH, 'w') as f:
json.dump(data_analyzer, f)
dt = time.time() - t0
print(f"Analyzer took {dt} seconds\n", flush=True)
time.sleep(SLEEP_TIME - dt)
def print_net (self, epoch, display_flag = True ):
# saving down true images.
if self.main_img_visual is False:
imgs = self.train_set_x.reshape((self.train_set_x.shape[0].eval(),self.height,self.width,self.channels))
imgs = imgs.eval()[self.visualize_ind]
loc_im = '../visuals/images/image_'
imgs = util.visualize(imgs, prefix = loc_im, is_color = self.color_filter if self.channels == 3 else False)
self.main_img_visual = True
# visualizing activities.
activity_now = self.activities(0)
bar = progressbar.ProgressBar(maxval=len(self.nkerns), \
widgets=[progressbar.AnimatedMarker(), \
' visualizing ',
' ', progressbar.Percentage(), \
' ',progressbar.ETA(), \
]).start()
for m in xrange(len(self.nkerns)): #For each layer
loc_ac = '../visuals/activities/layer_' + str(m) + "/epoch_" + str(epoch)
if not os.path.exists(loc_ac):
os.makedirs(loc_ac)
loc_ac = loc_ac + "/filter_"
current_activity = activity_now[m]
current_activity = current_activity[self.visualize_ind]
imgs = util.visualize(current_activity, loc_ac, is_color = False)
current_weights = self.ConvLayers.weights[m] # for each layer
loc_we = '../visuals/filters/layer_' + str(m) + "/epoch_" + str(epoch)
if not os.path.exists(loc_we):
os.makedirs(loc_we)
loc_we = loc_we + "/filter_"
if len(current_weights.shape.eval()) == 5:
imgs = util.visualize(numpy.squeeze(current_weights.dimshuffle(0,3,4,1,2).eval()), prefix = loc_we, is_color = self.color_filter)
else:
imgs = util.visualize(current_weights.dimshuffle(0,2,3,1).eval(), prefix = loc_we, is_color = self.color_filter)
bar.update(m+1)
bar.finish()
def main(args):
# Set up logging
args.save_dir = util.get_save_dir(args.save_dir, args.name, training=False)
log = util.get_logger(args.save_dir, args.name)
device, gpu_ids = util.get_available_devices()
args.batch_size *= max(1, len(gpu_ids))
log.info(f'Args: {dumps(vars(args), indent=4, sort_keys=True)}')
# Get embeddings
log.info('Loading embeddings...')
word_vectors = util.torch_from_json(args.word_emb_file)
models = {}
if args.use_ensemble:
total_models = 0
for model_name in ['bidaf', 'bidafextra', 'fusionnet']:
models_list = []
for model_file in glob.glob(
f'{args.load_path}/{model_name}-*/{args.ensemble_models}'):
# Get model
log.info('Building model...')
if model_name == 'bidaf':
model = BiDAF(word_vectors=word_vectors,
hidden_size=args.hidden_size)
elif model_name == 'bidafextra':
model = BiDAFExtra(word_vectors=word_vectors, args=args)
elif model_name == 'fusionnet':
model = FusionNet(word_vectors=word_vectors, args=args)
model = nn.DataParallel(model, gpu_ids)
log.info(f'Loading checkpoint from {model_file}...')
model = util.load_model(model,
model_file,
gpu_ids,
return_step=False)
# Load each model on CPU (have plenty of RAM ...)
model = model.cpu()
model.eval()
models_list.append(model)
models[model_name] = models_list
total_models += len(models_list)
log.info(f'Using an ensemble of {total_models} models')
else:
device, gpu_ids = util.get_available_devices()
# Get model
log.info('Building model...')
if args.model == 'bidaf':
model = BiDAF(word_vectors=word_vectors,
hidden_size=args.hidden_size)
elif args.model == 'bidafextra':
model = BiDAFExtra(word_vectors=word_vectors, args=args)
elif args.model == 'fusionnet':
model = FusionNet(word_vectors=word_vectors, args=args)
model = nn.DataParallel(model, gpu_ids)
log.info(f'Loading checkpoint from {args.load_path}...')
model = util.load_model(model,
args.load_path,
gpu_ids,
return_step=False)
model = model.to(device)
model.eval()
models[args.model] = [model]
# Get data loader
log.info('Building dataset...')
record_file = vars(args)[f'{args.split}_record_file']
dataset = SQuAD(record_file, args)
data_loader = data.DataLoader(dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.num_workers,
collate_fn=collate_fn)
# Evaluate
log.info(f'Evaluating on {args.split} split...')
nll_meter = util.AverageMeter()
pred_dict = {} # Predictions for TensorBoard
sub_dict = {} # Predictions for submission
eval_file = vars(args)[f'{args.split}_eval_file']
with open(eval_file, 'r') as fh:
gold_dict = json_load(fh)
with torch.no_grad(), \
tqdm(total=len(dataset)) as progress_bar:
for cw_idxs, cc_idxs, qw_idxs, qc_idxs, cw_pos, cw_ner, cw_freq, cqw_extra, y1, y2, ids in data_loader:
# Setup for forward
cw_idxs = cw_idxs.to(device)
qw_idxs = qw_idxs.to(device)
batch_size = cw_idxs.size(0)
p1s = []
p2s = []
for model_name in models:
for model in models[model_name]:
# Move model to GPU to evaluate
model = model.to(device)
# Forward
if model_name == 'bidaf':
log_p1, log_p2 = model.to(device)(cw_idxs, qw_idxs)
else:
log_p1, log_p2 = model.to(device)(cw_idxs, qw_idxs,
cw_pos, cw_ner,
cw_freq, cqw_extra)
log_p1, log_p2 = log_p1.cpu(), log_p2.cpu()
if not args.use_ensemble:
y1, y2 = y1.to(device), y2.to(device)
log_p1, log_p2 = log_p1.to(device), log_p2.to(device)
loss = F.nll_loss(log_p1, y1) + F.nll_loss(log_p2, y2)
nll_meter.update(loss.item(), batch_size)
# Move model back to CPU to release GPU memory
model = model.cpu()
# Get F1 and EM scores
p1, p2 = log_p1.exp().unsqueeze(
-1).cpu(), log_p2.exp().unsqueeze(-1).cpu()
p1s.append(p1), p2s.append(p2)
best_ps = torch.max(
torch.cat([
torch.cat(p1s, -1).unsqueeze(-1),
torch.cat(p2s, -1).unsqueeze(-1)
], -1), -2)[0]
p1, p2 = best_ps[:, :, 0], best_ps[:, :, 1]
starts, ends = util.discretize(p1, p2, args.max_ans_len,
args.use_squad_v2)
# Log info
progress_bar.update(batch_size)
if args.split != 'test':
# No labels for the test set, so NLL would be invalid
progress_bar.set_postfix(NLL=nll_meter.avg)
idx2pred, uuid2pred = util.convert_tokens(gold_dict, ids.tolist(),
starts.tolist(),
ends.tolist(),
args.use_squad_v2)
pred_dict.update(idx2pred)
sub_dict.update(uuid2pred)
# Log results (except for test set, since it does not come with labels)
if args.split != 'test':
results = util.eval_dicts(gold_dict, pred_dict, args.use_squad_v2)
results_list = [('NLL', nll_meter.avg), ('F1', results['F1']),
('EM', results['EM'])]
if args.use_squad_v2:
results_list.append(('AvNA', results['AvNA']))
results = OrderedDict(results_list)
# Log to console
results_str = ', '.join(f'{k}: {v:05.2f}' for k, v in results.items())
log.info(f'{args.split.title()} {results_str}')
# Log to TensorBoard
tbx = SummaryWriter(args.save_dir)
util.visualize(tbx,
pred_dict=pred_dict,
eval_path=eval_file,
step=0,
split=args.split,
num_visuals=args.num_visuals)
# Write submission file
sub_path = join(args.save_dir, args.split + '_' + args.sub_file)
log.info(f'Writing submission file to {sub_path}...')
with open(sub_path, 'w', newline='', encoding='utf-8') as csv_fh:
csv_writer = csv.writer(csv_fh, delimiter=',')
csv_writer.writerow(['Id', 'Predicted'])
for uuid in sorted(sub_dict):
csv_writer.writerow([uuid, sub_dict[uuid]])
from sys import argv
import hierarchy_cluster
import util
data_set_location = "dataset/Hierarchical_2.csv"
if __name__ == "__main__":
# Load dataset
data = util.load_data_set(data_set_location)
# Visualization with no color
util.visualize(data)
# Argument Disimmiliarity method type clustering from CLI
# ex : python3 main 1
# 1 for single link
# 2 for complete link
# 3 for group average
# 4 for centroid based
try:
type = int(argv[1])
except:
type = 1 # default 1 for no argument
hierarchy_cluster.agglomerative_clustering(data, type=type)
def main(args):
device = torch.device(
"cuda:0" if torch.cuda.is_available() and args.cuda else "cpu")
results = []
columns = []
for num_quadrant_inputs in args.num_quadrant_inputs:
# adds an s in case of plural quadrants
maybes = "s" if num_quadrant_inputs > 1 else ""
print("Training with {} quadrant{} as input...".format(
num_quadrant_inputs, maybes))
# Dataset
datasets, dataloaders, dataset_sizes = get_data(
num_quadrant_inputs=num_quadrant_inputs, batch_size=128)
# Train baseline
baseline_net = baseline.train(
device=device,
dataloaders=dataloaders,
dataset_sizes=dataset_sizes,
learning_rate=args.learning_rate,
num_epochs=args.num_epochs,
early_stop_patience=args.early_stop_patience,
model_path="baseline_net_q{}.pth".format(num_quadrant_inputs),
)
# Train CVAE
cvae_net = cvae.train(
device=device,
dataloaders=dataloaders,
dataset_sizes=dataset_sizes,
learning_rate=args.learning_rate,
num_epochs=args.num_epochs,
early_stop_patience=args.early_stop_patience,
model_path="cvae_net_q{}.pth".format(num_quadrant_inputs),
pre_trained_baseline_net=baseline_net,
)
# Visualize conditional predictions
visualize(
device=device,
num_quadrant_inputs=num_quadrant_inputs,
pre_trained_baseline=baseline_net,
pre_trained_cvae=cvae_net,
num_images=args.num_images,
num_samples=args.num_samples,
image_path="cvae_plot_q{}.png".format(num_quadrant_inputs),
)
# Retrieve conditional log likelihood
df = generate_table(
device=device,
num_quadrant_inputs=num_quadrant_inputs,
pre_trained_baseline=baseline_net,
pre_trained_cvae=cvae_net,
num_particles=args.num_particles,
col_name="{} quadrant{}".format(num_quadrant_inputs, maybes),
)
results.append(df)
columns.append("{} quadrant{}".format(num_quadrant_inputs, maybes))
results = pd.concat(results, axis=1, ignore_index=True)
results.columns = columns
results.loc["Performance gap", :] = results.iloc[0, :] - results.iloc[1, :]
results.to_csv("results.csv")
def main(args):
# Set up logging
args.save_dir = util.get_save_dir(args.save_dir, args.name, training=False)
log = util.get_logger(args.save_dir, args.name)
log.info(f'Args: {dumps(vars(args), indent=4, sort_keys=True)}')
device, gpu_ids = util.get_available_devices()
args.batch_size *= max(1, len(gpu_ids))
# Get embeddings
log.info('Loading embeddings...')
word_vectors = util.torch_from_json(args.word_emb_file)
ch_vectors = util.torch_from_json(args.char_emb_file)
# Get model
log.info('Building model...')
model = BiDAF(word_vectors=word_vectors,
ch_vectors=ch_vectors,
hidden_size=args.hidden_size)
model = nn.DataParallel(model, gpu_ids)
log.info(f'Loading checkpoint from {args.load_path}...')
model = util.load_model(model, args.load_path, gpu_ids, return_step=False)
model = model.to(device)
model.eval()
# Get data loader
log.info('Building dataset...')
record_file = vars(args)[f'{args.split}_record_file']
dataset = SQuAD(record_file, args.use_squad_v2)
data_loader = data.DataLoader(dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.num_workers,
collate_fn=collate_fn)
# Evaluate
log.info(f'Evaluating on {args.split} split...')
nll_meter = util.AverageMeter()
pred_dict = {} # Predictions for TensorBoard
sub_dict = {} # Predictions for submission
eval_file = vars(args)[f'{args.split}_eval_file']
with open(eval_file, 'r') as fh:
gold_dict = json_load(fh)
with torch.no_grad(), \
tqdm(total=len(dataset)) as progress_bar:
for cw_idxs, cc_idxs, qw_idxs, qc_idxs, y1, y2, ids in data_loader:
# Setup for forward
cw_idxs = cw_idxs.to(device)
qw_idxs = qw_idxs.to(device)
cc_idxs = cc_idxs.to(device)
qc_idxs = qc_idxs.to(device)
batch_size = cw_idxs.size(0)
# Forward
log_p1, log_p2 = model(cw_idxs, qw_idxs, cc_idxs, qc_idxs)
y1, y2 = y1.to(device), y2.to(device)
loss = F.nll_loss(log_p1, y1) + F.nll_loss(log_p2, y2)
nll_meter.update(loss.item(), batch_size)
# Get F1 and EM scores
p1, p2 = log_p1.exp(), log_p2.exp()
starts, ends = util.discretize(p1, p2, args.max_ans_len,
args.use_squad_v2)
# Log info
progress_bar.update(batch_size)
if args.split != 'test':
# No labels for the test set, so NLL would be invalid
progress_bar.set_postfix(NLL=nll_meter.avg)
idx2pred, uuid2pred = util.convert_tokens(gold_dict, ids.tolist(),
starts.tolist(),
ends.tolist(),
args.use_squad_v2)
pred_dict.update(idx2pred)
sub_dict.update(uuid2pred)
# Log results (except for test set, since it does not come with labels)
if args.split != 'test':
results = util.eval_dicts(gold_dict, pred_dict, args.use_squad_v2)
results_list = [('NLL', nll_meter.avg), ('F1', results['F1']),
('EM', results['EM'])]
if args.use_squad_v2:
results_list.append(('AvNA', results['AvNA']))
results = OrderedDict(results_list)
# Log to console
results_str = ', '.join(f'{k}: {v:05.2f}' for k, v in results.items())
log.info(f'{args.split.title()} {results_str}')
# Log to TensorBoard
tbx = SummaryWriter(args.save_dir)
util.visualize(tbx,
pred_dict=pred_dict,
eval_path=eval_file,
step=0,
split=args.split,
num_visuals=args.num_visuals)
# Write submission file
sub_path = join(args.save_dir, args.split + '_' + args.sub_file)
log.info(f'Writing submission file to {sub_path}...')
with open(sub_path, 'w', newline='', encoding='utf-8') as csv_fh:
csv_writer = csv.writer(csv_fh, delimiter=',')
csv_writer.writerow(['Id', 'Predicted'])
for uuid in sorted(sub_dict):
csv_writer.writerow([uuid, sub_dict[uuid]])
def train(self, model, train_dataloader, eval_dataloader, val_dict, experts):
device = self.device
if experts is False:
model.to(device)
optim = AdamW(model.parameters(), lr=self.lr)
else:
model.gate.to(device)
optim = AdamW(model.gate.parameters(), lr=self.lr)
global_idx = 0
best_scores = {'F1': -1.0, 'EM': -1.0}
tbx = SummaryWriter(self.save_dir)
for epoch_num in range(self.num_epochs):
self.log.info(f'Epoch: {epoch_num}')
with torch.enable_grad(), tqdm(total=len(train_dataloader.dataset)) as progress_bar:
#remember stochastic for MoE
for batch in train_dataloader:
optim.zero_grad()
if experts:
model.gate.train()
model.expert1.eval()
model.expert2.eval()
model.expert3.eval()
else:
model.train()
input_ids = batch['input_ids'].to(device)
attention_mask = batch['attention_mask'].to(device)
start_positions = batch['start_positions'].to(device)
end_positions = batch['end_positions'].to(device)
if experts:
outputs1 = model.expert1(input_ids, attention_mask=attention_mask)
start_logits1, end_logits1 = outputs1.start_logits, outputs1.end_logits
outputs2 = model.expert2(input_ids, attention_mask=attention_mask)
start_logits2, end_logits2 = outputs2.start_logits, outputs2.end_logits
outputs3 = model.expert3(input_ids, attention_mask=attention_mask)
start_logits3, end_logits3 = outputs3.start_logits, outputs3.end_logits
expert_weights = model.gate.forward(example)
start_logits = start_logits1 * expert_weights[0] + start_logits2 * expert_weights[1] +
start_logits3 * expert_weights[2]
end_logits = end_logits1 * expert_weights[0] + end_logits2 * expert_weights[1] +
end_logits3 * expert_weights[2]
loss = -np.log(start_logits[0][start_positions[0]]) - np.log(end_logits[0][end_positions[0]])
else:
outputs = model(input_ids, attention_mask=attention_mask,
start_positions=start_positions,
end_positions=end_positions)
loss = outputs[0]
loss.backward()
optim.step()
progress_bar.update(len(input_ids))
progress_bar.set_postfix(epoch=epoch_num, NLL=loss.item())
tbx.add_scalar('train/NLL', loss.item(), global_idx)
if (global_idx % self.eval_every) == 0:
self.log.info(f'Evaluating at step {global_idx}...')
preds, curr_score = self.evaluate(model, eval_dataloader, val_dict, return_preds=True)
results_str = ', '.join(f'{k}: {v:05.2f}' for k, v in curr_score.items())
self.log.info('Visualizing in TensorBoard...')
for k, v in curr_score.items():
tbx.add_scalar(f'val/{k}', v, global_idx)
self.log.info(f'Eval {results_str}')
if self.visualize_predictions:
util.visualize(tbx,
pred_dict=preds,
gold_dict=val_dict,
step=global_idx,
split='val',
num_visuals=self.num_visuals)
if curr_score['F1'] >= best_scores['F1']:
best_scores = curr_score
if experts:
self.save(model.gate)
else:
self.save(model)
global_idx += 1
# MNISTデータ（手書き数字画像）を読み込む
# 初回はDLするので時間がかかる（53MB）
mnist = fetch_mldata('MNIST original')
# サンプルデータの読み込み
x_all = mnist.data.astype(np.float32) / 255
x_data = np.vstack([x_all[0]]*10)
# SaltAndPepperNoise
x = x_data.copy()
titles = []
for i in xrange(10):
rate = 0.1 * i
titles.append('%3.1f' % rate)
sap = SaltAndPepperNoise(rate=rate)
x[i] = sap.noise(x[i])
visualize(x, '../img/noise/s&p.jpg', (8, 2), (1, 10), titles=titles)
# GaussianNoise
x = x_data.copy()
titles = []
for i in xrange(10):
scale = 0.1 * i
titles.append('%3.1f' % scale)
if i == 0:
continue
gaus = GaussianNoise(scale=scale)
x[i] = gaus.noise(x[i])
visualize(x, '../img/noise/gaus.jpg', (8, 2), (1, 10), titles=titles)
def main(args):
print("in main")
print("args: ", args)
if True:
args.save_dir = util.get_save_dir(args.save_dir, args.name, training=True)
log = util.get_logger(args.save_dir, args.name)
tbx = SummaryWriter(args.save_dir)
device, args.gpu_ids = util.get_available_devices()
log.info('Args: {}'.format(dumps(vars(args), indent=4, sort_keys=True)))
args.batch_size *= max(1, len(args.gpu_ids))
# Set random seed
log.info('Using random seed {}...'.format(args.seed))
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
torch.cuda.manual_seed_all(args.seed)
# Get embeddings
log.info('Loading embeddings...')
# CHECK IF WE NEED TO USE ALL OF THESE????
word_vectors = util.torch_from_json(args.word_emb_file)
# Get model
log.info('Building model...')
model = BiDAF(word_vectors=word_vectors,
hidden_size=args.hidden_size,
drop_prob=args.drop_prob)
model = nn.DataParallel(model, args.gpu_ids)
if args.load_path:
log.info('Loading checkpoint from {}...'.format(args.load_path))
model, step = util.load_model(model, args.load_path, args.gpu_ids)
else:
step = 0
model = model.to(device)
model.train()
ema = util.EMA(model, args.ema_decay)
# Get saver
saver = util.CheckpointSaver(args.save_dir,
max_checkpoints=args.max_checkpoints,
metric_name=args.metric_name,
maximize_metric=args.maximize_metric,
log=log)
# Get optimizer and scheduler
optimizer = optim.Adadelta(model.parameters(), args.lr,
weight_decay=args.l2_wd)
scheduler = sched.LambdaLR(optimizer, lambda s: 1.) # Constant LR
# Get data loader
log.info('Building dataset...')
train_dataset = SQuAD(args.train_record_file, args.use_squad_v2)
print("train dataset!: ", train_dataset)
train_loader = data.DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.num_workers,
collate_fn=collate_fn)
dev_dataset = SQuAD(args.dev_record_file, args.use_squad_v2)
dev_loader = data.DataLoader(dev_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.num_workers,
collate_fn=collate_fn)
# Train
log.info('Training...')
steps_till_eval = args.eval_steps
epoch = step // len(train_dataset)
while epoch != args.num_epochs:
epoch += 1
log.info('Starting epoch {}...'.format(epoch))
with torch.enable_grad(), \
tqdm(total=len(train_loader.dataset)) as progress_bar:
for cw_idxs, cc_idxs, qw_idxs, qc_idxs, y1, y2, ids in train_loader:
# Setup for forward
cw_idxs = cw_idxs.to(device)
qw_idxs = qw_idxs.to(device)
batch_size = cw_idxs.size(0)
optimizer.zero_grad()
# Forward
log_p1, log_p2 = model(cw_idxs, qw_idxs)
y1, y2 = y1.to(device), y2.to(device)
loss = F.nll_loss(log_p1, y1) + F.nll_loss(log_p2, y2)
loss_val = loss.item()
# Backward
loss.backward()
nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm)
optimizer.step()
scheduler.step(step // batch_size)
ema(model, step // batch_size)
# Log info
step += batch_size
progress_bar.update(batch_size)
progress_bar.set_postfix(epoch=epoch,
NLL=loss_val)
tbx.add_scalar('train/NLL', loss_val, step)
tbx.add_scalar('train/LR',
optimizer.param_groups[0]['lr'],
step)
steps_till_eval -= batch_size
if steps_till_eval <= 0:
steps_till_eval = args.eval_steps
# Evaluate and save checkpoint
log.info('Evaluating at step {}...'.format(step))
ema.assign(model)
results, pred_dict = evaluate(model, dev_loader, device,
args.dev_eval_file,
args.max_ans_len,
args.use_squad_v2)
saver.save(step, model, results[args.metric_name], device)
ema.resume(model)
# Log to console
results_str = ', '.join('{}: {:05.2f}'.format(k, v)
for k, v in results.items())
log.info('Dev {}'.format(results_str))
# Log to TensorBoard
log.info('Visualizing in TensorBoard...')
for k, v in results.items():
tbx.add_scalar('dev/{}'.format(k), v, step)
util.visualize(tbx,
pred_dict=pred_dict,
eval_path=args.dev_eval_file,
step=step,
split='dev',
num_visuals=args.num_visuals)
def run_cnn( arch_params,
optimization_params ,
data_params,
filename_params,
visual_params,
verbose = False,
):
#####################
# Unpack Variables #
#####################
results_file_name = filename_params [ "results_file_name" ] # Files that will be saved down on completion Can be used by the parse.m file
error_file_name = filename_params [ "error_file_name" ]
cost_file_name = filename_params [ "cost_file_name" ]
confusion_file_name = filename_params [ "confusion_file_name" ]
network_save_name = filename_params [ "network_save_name" ]
dataset = data_params [ "loc" ]
height = data_params [ "height" ]
width = data_params [ "width" ]
batch_size = data_params [ "batch_size" ]
load_batches = data_params [ "load_batches" ] * batch_size
batches2train = data_params [ "batches2train" ]
batches2test = data_params [ "batches2test" ]
batches2validate = data_params [ "batches2validate" ]
channels = data_params [ "channels" ]
mom_start = optimization_params [ "mom_start" ]
mom_end = optimization_params [ "mom_end" ]
mom_epoch_interval = optimization_params [ "mom_interval" ]
mom_type = optimization_params [ "mom_type" ]
initial_learning_rate = optimization_params [ "initial_learning_rate" ]
learning_rate_decay = optimization_params [ "learning_rate_decay" ]
ada_grad = optimization_params [ "ada_grad" ]
fudge_factor = optimization_params [ "fudge_factor" ]
l1_reg = optimization_params [ "l1_reg" ]
l2_reg = optimization_params [ "l2_reg" ]
rms_prop = optimization_params [ "rms_prop" ]
rms_rho = optimization_params [ "rms_rho" ]
rms_epsilon = optimization_params [ "rms_epsilon" ]
squared_filter_length_limit = arch_params [ "squared_filter_length_limit" ]
n_epochs = arch_params [ "n_epochs" ]
validate_after_epochs = arch_params [ "validate_after_epochs" ]
mlp_activations = arch_params [ "mlp_activations" ]
cnn_activations = arch_params [ "cnn_activations" ]
dropout = arch_params [ "dropout" ]
column_norm = arch_params [ "column_norm" ]
dropout_rates = arch_params [ "dropout_rates" ]
nkerns = arch_params [ "nkerns" ]
outs = arch_params [ "outs" ]
filter_size = arch_params [ "filter_size" ]
pooling_size = arch_params [ "pooling_size" ]
num_nodes = arch_params [ "num_nodes" ]
use_bias = arch_params [ "use_bias" ]
random_seed = arch_params [ "random_seed" ]
svm_flag = arch_params [ "svm_flag" ]
visualize_flag = visual_params ["visualize_flag" ]
visualize_after_epochs = visual_params ["visualize_after_epochs" ]
n_visual_images = visual_params ["n_visual_images" ]
display_flag = visual_params ["display_flag" ]
# Random seed initialization.
rng = numpy.random.RandomState(random_seed)
#################
# Data Loading #
#################
print "... loading data"
# load matlab files as dataset.
if data_params["type"] == 'mat':
train_data_x, train_data_y, train_data_y1 = load_data_mat(dataset, batch = 1 , type_set = 'train')
test_data_x, test_data_y, valid_data_y1 = load_data_mat(dataset, batch = 1 , type_set = 'test') # Load dataset for first epoch.
valid_data_x, valid_data_y, test_data_y1 = load_data_mat(dataset, batch = 1 , type_set = 'valid') # Load dataset for first epoch.
train_set_x = theano.shared(numpy.asarray(train_data_x, dtype=theano.config.floatX), borrow=True)
train_set_y = theano.shared(numpy.asarray(train_data_y, dtype='int32'), borrow=True)
train_set_y1 = theano.shared(numpy.asarray(train_data_y1, dtype=theano.config.floatX), borrow=True)
test_set_x = theano.shared(numpy.asarray(test_data_x, dtype=theano.config.floatX), borrow=True)
test_set_y = theano.shared(numpy.asarray(test_data_y, dtype='int32'), borrow=True)
test_set_y1 = theano.shared(numpy.asarray(test_data_y1, dtype=theano.config.floatX), borrow=True)
valid_set_x = theano.shared(numpy.asarray(valid_data_x, dtype=theano.config.floatX), borrow=True)
valid_set_y = theano.shared(numpy.asarray(valid_data_y, dtype='int32'), borrow=True)
valid_set_y1 = theano.shared(numpy.asarray(valid_data_y1, dtype=theano.config.floatX), borrow=True)
# compute number of minibatches for training, validation and testing
n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size
n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] / batch_size
n_test_batches = test_set_x.get_value(borrow=True).shape[0] / batch_size
multi_load = True
# load pkl data as is shown in theano tutorials
elif data_params["type"] == 'pkl':
data = load_data_pkl(dataset)
train_set_x, train_set_y, train_set_y1 = data[0]
valid_set_x, valid_set_y, valid_set_y1 = data[1]
test_set_x, test_set_y, test_set_y1 = data[2]
# compute number of minibatches for training, validation and testing
n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size
n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] / batch_size
n_test_batches = test_set_x.get_value(borrow=True).shape[0] / batch_size
n_train_images = train_set_x.get_value(borrow=True).shape[0]
n_test_images = test_set_x.get_value(borrow=True).shape[0]
n_valid_images = valid_set_x.get_value(borrow=True).shape[0]
n_train_batches_all = n_train_images / batch_size
n_test_batches_all = n_test_images / batch_size
n_valid_batches_all = n_valid_images / batch_size
if (n_train_batches_all < batches2train) or (n_test_batches_all < batches2test) or (n_valid_batches_all < batches2validate): # You can't have so many batches.
print "... !! Dataset doens't have so many batches. "
raise AssertionError()
multi_load = False
# load skdata ( its a good library that has a lot of datasets)
elif data_params["type"] == 'skdata':
if (dataset == 'mnist' or
dataset == 'mnist_noise1' or
dataset == 'mnist_noise2' or
dataset == 'mnist_noise3' or
dataset == 'mnist_noise4' or
dataset == 'mnist_noise5' or
dataset == 'mnist_noise6' or
dataset == 'mnist_bg_images' or
dataset == 'mnist_bg_rand' or
dataset == 'mnist_rotated' or
dataset == 'mnist_rotated_bg') :
print "... importing " + dataset + " from skdata"
func = globals()['load_skdata_' + dataset]
data = func()
train_set_x, train_set_y, train_set_y1 = data[0]
valid_set_x, valid_set_y, valid_set_y1 = data[1]
test_set_x, test_set_y, test_set_y1 = data[2]
# compute number of minibatches for training, validation and testing
n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size
n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] / batch_size
n_test_batches = test_set_x.get_value(borrow=True).shape[0] / batch_size
n_train_images = train_set_x.get_value(borrow=True).shape[0]
n_test_images = test_set_x.get_value(borrow=True).shape[0]
n_valid_images = valid_set_x.get_value(borrow=True).shape[0]
n_train_batches_all = n_train_images / batch_size
n_test_batches_all = n_test_images / batch_size
n_valid_batches_all = n_valid_images / batch_size
if (n_train_batches_all < batches2train) or (n_test_batches_all < batches2test) or (n_valid_batches_all < batches2validate): # You can't have so many batches.
print "... !! Dataset doens't have so many batches. "
raise AssertionError()
multi_load = False
elif dataset == 'cifar10':
print "... importing cifar 10 from skdata"
data = load_skdata_cifar10()
train_set_x, train_set_y, train_set_y1 = data[0]
valid_set_x, valid_set_y, valid_set_y1 = data[1]
test_set_x, test_set_y, test_set_y1 = data[2]
# compute number of minibatches for training, validation and testing
n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size
n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] / batch_size
n_test_batches = test_set_x.get_value(borrow=True).shape[0] / batch_size
multi_load = False
elif dataset == 'caltech101':
print "... importing caltech 101 from skdata"
# shuffle the data
total_images_in_dataset = 9144
rand_perm = numpy.random.permutation(total_images_in_dataset) # create a constant shuffle, so that data can be loaded in batchmode with the same random shuffle
n_train_images = total_images_in_dataset / 3
n_test_images = total_images_in_dataset / 3
n_valid_images = total_images_in_dataset / 3
n_train_batches_all = n_train_images / batch_size
n_test_batches_all = n_test_images / batch_size
n_valid_batches_all = n_valid_images / batch_size
if (n_train_batches_all < batches2train) or (n_test_batches_all < batches2test) or (n_valid_batches_all < batches2validate): # You can't have so many batches.
print "... !! Dataset doens't have so many batches. "
raise AssertionError()
train_data_x, train_data_y = load_skdata_caltech101(batch_size = load_batches, rand_perm = rand_perm, batch = 1 , type_set = 'train' , height = height, width = width)
test_data_x, test_data_y = load_skdata_caltech101(batch_size = load_batches, rand_perm = rand_perm, batch = 1 , type_set = 'test' , height = height, width = width) # Load dataset for first epoch.
valid_data_x, valid_data_y = load_skdata_caltech101(batch_size = load_batches, rand_perm = rand_perm, batch = 1 , type_set = 'valid' , height = height, width = width) # Load dataset for first epoch.
train_set_x = theano.shared(train_data_x, borrow=True)
train_set_y = theano.shared(train_data_y, borrow=True)
test_set_x = theano.shared(test_data_x, borrow=True)
test_set_y = theano.shared(test_data_y, borrow=True)
valid_set_x = theano.shared(valid_data_x, borrow=True)
valid_set_y = theano.shared(valid_data_y, borrow=True)
# compute number of minibatches for training, validation and testing
n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size
n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] / batch_size
n_test_batches = test_set_x.get_value(borrow=True).shape[0] / batch_size
multi_load = True
# Just checking as a way to see if the intended dataset is indeed loaded.
assert height*width*channels == train_set_x.get_value( borrow = True ).shape[1]
assert batch_size >= n_visual_images
if ada_grad is True:
assert rms_prop is False
elif rms_prop is True:
assert ada_grad is False
fudge_factor = rms_epsilon
######################
# BUILD NETWORK #
######################
print '... building the network'
start_time = time.clock()
# allocate symbolic variables for the data
index = T.lscalar() # index to a [mini]batch
x = T.matrix('x') # the data is presented as rasterized images
y = T.ivector('y') # the labels are presented as 1D vector of [int]
if svm_flag is True:
y1 = T.matrix('y1') # [-1 , 1] labels in case of SVM
first_layer_input = x.reshape((batch_size, channels, height, width))
# Create first convolutional - pooling layers
activity = [] # to record Cnn activities
weights = []
conv_layers=[]
filt_size = filter_size[0]
pool_size = pooling_size[0]
if not nkerns == []:
conv_layers.append ( LeNetConvPoolLayer(
rng,
input = first_layer_input,
image_shape=(batch_size, channels , height, width),
filter_shape=(nkerns[0], channels , filt_size, filt_size),
poolsize=(pool_size, pool_size),
activation = cnn_activations[0],
verbose = verbose
) )
activity.append ( conv_layers[-1].output )
weights.append ( conv_layers[-1].filter_img)
# Create the rest of the convolutional - pooling layers in a loop
next_in_1 = ( height - filt_size + 1 ) / pool_size
next_in_2 = ( width - filt_size + 1 ) / pool_size
for layer in xrange(len(nkerns)-1):
filt_size = filter_size[layer+1]
pool_size = pooling_size[layer+1]
conv_layers.append ( LeNetConvPoolLayer(
rng,
input=conv_layers[layer].output,
image_shape=(batch_size, nkerns[layer], next_in_1, next_in_2),
filter_shape=(nkerns[layer+1], nkerns[layer], filt_size, filt_size),
poolsize=(pool_size, pool_size),
activation = cnn_activations[layer+1],
verbose = verbose
) )
next_in_1 = ( next_in_1 - filt_size + 1 ) / pool_size
next_in_2 = ( next_in_2 - filt_size + 1 ) / pool_size
weights.append ( conv_layers[-1].filter_img )
activity.append( conv_layers[-1].output )
# Assemble fully connected laters
if nkerns == []:
fully_connected_input = first_layer_input
else:
fully_connected_input = conv_layers[-1].output.flatten(2)
if len(dropout_rates) > 2 :
layer_sizes =[]
layer_sizes.append( nkerns[-1] * next_in_1 * next_in_2 )
for i in xrange(len(dropout_rates)-1):
layer_sizes.append ( num_nodes[i] )
layer_sizes.append ( outs )
elif len(dropout_rates) == 1:
layer_size = [ nkerns[-1] * next_in_1 * next_in_2, outs]
else :
layer_sizes = [ nkerns[-1] * next_in_1 * next_in_2, num_nodes[0] , outs]
assert len(layer_sizes) - 1 == len(dropout_rates) # Just checking.
""" Dropouts implemented from paper:
Srivastava, Nitish, et al. "Dropout: A simple way to prevent neural networks
from overfitting." The Journal of Machine Learning Research 15.1 (2014): 1929-1958.
"""
MLPlayers = MLP( rng=rng,
input=fully_connected_input,
layer_sizes=layer_sizes,
dropout_rates=dropout_rates,
activations=mlp_activations,
use_bias = use_bias,
svm_flag = svm_flag,
verbose = verbose)
# Build the expresson for the categorical cross entropy function.
if svm_flag is False:
cost = MLPlayers.negative_log_likelihood( y )
dropout_cost = MLPlayers.dropout_negative_log_likelihood( y )
else :
cost = MLPlayers.negative_log_likelihood( y1 )
dropout_cost = MLPlayers.dropout_negative_log_likelihood( y1 )
# create theano functions for evaluating the graphs
test_model = theano.function(
inputs=[index],
outputs=MLPlayers.errors(y),
givens={
x: test_set_x[index * batch_size:(index + 1) * batch_size],
y: test_set_y[index * batch_size:(index + 1) * batch_size]})
validate_model = theano.function(
inputs=[index],
outputs=MLPlayers.errors(y),
givens={
x: valid_set_x[index * batch_size:(index + 1) * batch_size],
y: valid_set_y[index * batch_size:(index + 1) * batch_size]})
prediction = theano.function(
inputs = [index],
outputs = MLPlayers.predicts,
givens={
x: test_set_x[index * batch_size: (index + 1) * batch_size]})
nll = theano.function(
inputs = [index],
outputs = MLPlayers.probabilities,
givens={
x: test_set_x[index * batch_size: (index + 1) * batch_size]})
# function to return activations of each image
activities = theano.function (
inputs = [index],
outputs = activity,
givens = {
x: train_set_x[index * batch_size: (index + 1) * batch_size]
})
# Compute cost and gradients of the model wrt parameter
params = []
for layer in conv_layers:
params = params + layer.params
params = params + MLPlayers.params
output = dropout_cost + l1_reg * MLPlayers.dropout_L1 + l2_reg * MLPlayers.dropout_L2 if dropout else cost + l1_reg * MLPlayers.L1 + l2_reg * MLPlayers.L2
gradients = []
for param in params:
gradient = T.grad( output ,param)
gradients.append ( gradient )
# TO DO: Try implementing Adadelta also.
# Compute momentum for the current epoch
epoch = T.scalar()
mom = ifelse(epoch <= mom_epoch_interval,
mom_start*(1.0 - epoch/mom_epoch_interval) + mom_end*(epoch/mom_epoch_interval),
mom_end)
# learning rate
eta = theano.shared(numpy.asarray(initial_learning_rate,dtype=theano.config.floatX))
# accumulate gradients for adagrad
grad_acc = []
for param in params:
eps = numpy.zeros_like(param.get_value(borrow=True), dtype=theano.config.floatX)
grad_acc.append(theano.shared(eps, borrow=True))
# accumulate velocities for momentum
velocities = []
for param in params:
velocity = theano.shared(numpy.zeros(param.get_value(borrow=True).shape,dtype=theano.config.floatX))
velocities.append(velocity)
# create updates for each combination of stuff
updates = OrderedDict()
print_flag = False
for velocity, gradient, acc , param in zip(velocities, gradients, grad_acc, params):
if ada_grad is True:
""" Adagrad implemented from paper:
John Duchi, Elad Hazan, and Yoram Singer. 2011. Adaptive subgradient methods
for online learning and stochastic optimization. JMLR
"""
current_acc = acc + T.sqr(gradient) # Accumulates Gradient
updates[acc] = current_acc # updates accumulation at timestamp
elif rms_prop is True:
""" Tieleman, T. and Hinton, G. (2012):
Neural Networks for Machine Learning, Lecture 6.5 - rmsprop.
Coursera. http://www.youtube.com/watch?v=O3sxAc4hxZU (formula @5:20)"""
current_acc = rms_rho * acc + (1 - rms_rho) * T.sqr(gradient)
updates[acc] = current_acc
else:
current_acc = 1
fudge_factor = 0
if mom_type == 0: # no momentum
updates[velocity] = -(eta / T.sqrt(current_acc + fudge_factor)) * gradient
#updates[velocity] = -1*eta*gradient
# perform adagrad velocity update
# this will be just added to parameters.
elif mom_type == 1: # if polyak momentum
""" Momentum implemented from paper:
Polyak, Boris Teodorovich. "Some methods of speeding up the convergence of iteration methods."
USSR Computational Mathematics and Mathematical Physics 4.5 (1964): 1-17.
Adapted from Sutskever, Ilya, Hinton et al. "On the importance of initialization and momentum in deep learning."
Proceedings of the 30th international conference on machine learning (ICML-13). 2013.
equation (1) and equation (2)"""
updates[velocity] = mom * velocity - (1.-mom) * ( eta / T.sqrt(current_acc+ fudge_factor)) * gradient
elif mom_type == 2: # Nestrov accelerated gradient beta stage...
"""Nesterov, Yurii. "A method of solving a convex programming problem with convergence rate O (1/k2)."
Soviet Mathematics Doklady. Vol. 27. No. 2. 1983.
Adapted from https://blogs.princeton.edu/imabandit/2013/04/01/acceleratedgradientdescent/
Instead of using past params we use the current params as described in this link
https://github.com/lisa-lab/pylearn2/pull/136#issuecomment-10381617,"""
updates[velocity] = mom * velocity - (1.-mom) * ( eta / T.sqrt(current_acc + fudge_factor)) * gradient
updates[param] = mom * updates[velocity]
else:
if print_flag is False:
print_flag = True
print "!! Unrecognized mometum type, switching to no momentum."
updates[velocity] = -( eta / T.sqrt(current_acc+ fudge_factor) ) * gradient
if mom_type != 2:
stepped_param = param + updates[velocity]
else:
stepped_param = param + updates[velocity] + updates[param]
if param.get_value(borrow=True).ndim == 2 and column_norm is True:
""" constrain the norms of the COLUMNs of the weight, according to
https://github.com/BVLC/caffe/issues/109 """
col_norms = T.sqrt(T.sum(T.sqr(stepped_param), axis=0))
desired_norms = T.clip(col_norms, 0, T.sqrt(squared_filter_length_limit))
scale = desired_norms / (1e-7 + col_norms)
updates[param] = stepped_param * scale
else:
updates[param] = stepped_param
if svm_flag is True:
train_model = theano.function(inputs= [index, epoch],
outputs=output,
updates=updates,
givens={
x: train_set_x[index * batch_size:(index + 1) * batch_size],
y1: train_set_y1[index * batch_size:(index + 1) * batch_size]},
on_unused_input='ignore'
)
else:
train_model = theano.function(inputs= [index, epoch],
outputs=output,
updates=updates,
givens={
x: train_set_x[index * batch_size:(index + 1) * batch_size],
y: train_set_y[index * batch_size:(index + 1) * batch_size]},
on_unused_input='ignore'
)
decay_learning_rate = theano.function(
inputs=[],
outputs=eta, # Just updates the learning rates.
updates={eta: eta * learning_rate_decay}
)
momentum_value = theano.function (
inputs =[epoch],
outputs = mom,
)
end_time = time.clock()
# setting up visualization stuff...
shuffle_batch_ind = numpy.arange(batch_size)
numpy.random.shuffle(shuffle_batch_ind)
visualize_ind = shuffle_batch_ind[0:n_visual_images]
#visualize_ind = range(n_visual_images)
main_img_visual = True
# create all directories required for saving results and data.
if visualize_flag is True:
if not os.path.exists('../visuals'):
os.makedirs('../visuals')
if not os.path.exists('../visuals/activities'):
os.makedirs('../visuals/activities')
for i in xrange(len(nkerns)):
os.makedirs('../visuals/activities/layer_'+str(i))
if not os.path.exists('../visuals/filters'):
os.makedirs('../visuals/filters')
for i in xrange(len(nkerns)):
os.makedirs('../visuals/filters/layer_'+str(i))
if not os.path.exists('../visuals/images'):
os.makedirs('../visuals/images')
if not os.path.exists('../results/'):
os.makedirs ('../results')
print "... -> building complete, took " + str((end_time - start_time)) + " seconds"
###############
# TRAIN MODEL #
###############
#pdb.set_trace()
print "... training"
start_time = time.clock()
patience = numpy.inf # look as this many examples regardless
patience_increase = 2 # wait this much longer when a new best is
# found
improvement_threshold = 0.995 # a relative improvement of this much is
# considered significant
this_validation_loss = []
best_validation_loss = numpy.inf
best_iter = 0
epoch_counter = 0
early_termination = False
cost_saved = []
best_params = None
iteration= 0
while (epoch_counter < n_epochs) and (not early_termination):
epoch_counter = epoch_counter + 1
for batch in xrange (batches2train):
if verbose is True:
print "... -> Epoch: " + str(epoch_counter) + " Batch: " + str(batch+1) + " out of " + str(batches2train) + " batches"
if multi_load is True:
iteration= (epoch_counter - 1) * n_train_batches * batches2train + batch
# Load data for this batch
if verbose is True:
print "... -> loading data for new batch"
if data_params["type"] == 'mat':
train_data_x, train_data_y, train_data_y1 = load_data_mat(dataset, batch = batch + 1 , type_set = 'train')
elif data_params["type"] == 'skdata':
if dataset == 'caltech101':
train_data_x, train_data_y = load_skdata_caltech101(batch_size = load_batches, batch = batch + 1 , type_set = 'train', rand_perm = rand_perm, height = height, width = width )
# Do not use svm_flag for caltech 101
train_set_x.set_value(train_data_x ,borrow = True)
train_set_y.set_value(train_data_y ,borrow = True)
for minibatch_index in xrange(n_train_batches):
if verbose is True:
print "... -> Mini Batch: " + str(minibatch_index + 1) + " out of " + str(n_train_batches)
cost_ij = train_model( minibatch_index, epoch_counter)
cost_saved = cost_saved +[cost_ij]
else:
iteration= (epoch_counter - 1) * n_train_batches + batch
cost_ij = train_model(batch, epoch_counter)
cost_saved = cost_saved +[cost_ij]
if epoch_counter % validate_after_epochs is 0:
# Load Validation Dataset here.
validation_losses = 0.
if multi_load is True:
# Load data for this batch
for batch in xrange ( batches2test ):
if data_params["type"] == 'mat':
valid_data_x, valid_data_y, valid_data_y1 = load_data_mat(dataset, batch = batch + 1 , type_set = 'valid')
elif data_params["type"] == 'skdata':
if dataset == 'caltech101':
valid_data_x, valid_data_y = load_skdata_caltech101(batch_size = load_batches, batch = batch + 1 , type_set = 'valid' , rand_perm = rand_perm, height = height, width = width )
# Do not use svm_flag for caltech 101
valid_set_x.set_value(valid_data_x,borrow = True)
valid_set_y.set_value(valid_data_y,borrow = True)
validation_losses = validation_losses + numpy.sum([[validate_model(i) for i in xrange(n_valid_batches)]])
this_validation_loss = this_validation_loss + [validation_losses]
if verbose is True:
if this_validation_loss[-1] < best_validation_loss :
print "... -> epoch " + str(epoch_counter) + ", cost: " + str(numpy.mean(cost_saved[-1*n_train_batches:])) +", validation accuracy :" + str(float( batch_size * n_valid_batches * batches2validate - this_validation_loss[-1])*100/(batch_size*n_valid_batches*batches2validate)) + "%, learning_rate = " + str(eta.get_value(borrow=True))+ ", momentum = " +str(momentum_value(epoch_counter)) + " -> best thus far "
else :
print "... -> epoch " + str(epoch_counter) + ", cost: " + str(numpy.mean(cost_saved[-1*n_train_batches:])) +", validation accuracy :" + str(float( batch_size * n_valid_batches * batches2validate - this_validation_loss[-1])*100/(batch_size*n_valid_batches*batches2validate)) + "%, learning_rate = " + str(eta.get_value(borrow=True)) + ", momentum = " +str(momentum_value(epoch_counter))
else:
if this_validation_loss[-1] < best_validation_loss :
print "... -> epoch " + str(epoch_counter) + ", cost: " + str(numpy.mean(cost_saved[-1*n_train_batches:])) +", validation accuracy :" + str(float( batch_size * n_valid_batches * batches2validate - this_validation_loss[-1])*100/(batch_size*n_valid_batches*batches2validate)) + "% -> best thus far "
else :
print "... -> epoch " + str(epoch_counter) + ", cost: " + str(numpy.mean(cost_saved[-1*n_train_batches:])) +", validation accuracy :" + str(float( batch_size * n_valid_batches * batches2validate - this_validation_loss[-1])*100/(batch_size*n_valid_batches*batches2validate)) + "%"
else:
validation_losses = [validate_model(i) for i in xrange(n_valid_batches)]
this_validation_loss = this_validation_loss + [numpy.sum(validation_losses)]
if verbose is True:
if this_validation_loss[-1] < best_validation_loss :
print "... -> epoch " + str(epoch_counter) + ", cost: " + str(cost_saved[-1]) +", validation accuracy :" + str(float(batch_size*n_valid_batches - this_validation_loss[-1])*100/(batch_size*n_valid_batches)) + "%, learning_rate = " + str(eta.get_value(borrow=True)) + ", momentum = " +str(momentum_value(epoch_counter)) + " -> best thus far "
else:
print "... -> epoch " + str(epoch_counter) + ", cost: " + str(cost_saved[-1]) +", validation accuracy :" + str(float(batch_size*n_valid_batches - this_validation_loss[-1])*100/(batch_size*n_valid_batches)) + "%, learning_rate = " + str(eta.get_value(borrow=True)) + ", momentum = " +str(momentum_value(epoch_counter))
else:
if this_validation_loss[-1] < best_validation_loss :
print "... -> epoch " + str(epoch_counter) + ", cost: " + str(cost_saved[-1]) +", validation accuracy :" + str(float(batch_size*n_valid_batches - this_validation_loss[-1])*100/(batch_size*n_valid_batches)) + "% -> best thus far "
else:
print "... -> epoch " + str(epoch_counter) + ", cost: " + str(cost_saved[-1]) +", validation accuracy :" + str(float(batch_size*n_valid_batches - this_validation_loss[-1])*100/(batch_size*n_valid_batches)) + "% "
#improve patience if loss improvement is good enough
if this_validation_loss[-1] < best_validation_loss * \
improvement_threshold:
patience = max(patience, iteration* patience_increase)
best_iter = iteration
best_validation_loss = min(best_validation_loss, this_validation_loss[-1])
new_leanring_rate = decay_learning_rate()
if visualize_flag is True:
if epoch_counter % visualize_after_epochs is 0:
# saving down images.
if main_img_visual is False:
for i in xrange(n_visual_images):
curr_img = numpy.asarray(numpy.reshape(train_set_x.get_value( borrow = True )[visualize_ind[i]],[height, width, channels] ) * 255., dtype='uint8' )
if verbose is True:
cv2.imshow("Image Number " +str(i) + "_label_" + str(train_set_y.eval()[visualize_ind[i]]), curr_img)
cv2.imwrite("../visuals/images/image_" + str(i)+ "_label_" + str(train_set_y.eval()[visualize_ind[i]]) + ".jpg", curr_img )
main_img_visual = True
# visualizing activities.
activity = activities(0)
for m in xrange(len(nkerns)): #For each layer
loc_ac = '../visuals/activities/layer_' + str(m) + "/epoch_" + str(epoch_counter) +"/"
if not os.path.exists(loc_ac):
os.makedirs(loc_ac)
current_activity = activity[m]
for i in xrange(n_visual_images): # for each randomly chosen image .. visualize its activity
visualize(current_activity[visualize_ind[i]], loc = loc_ac, filename = 'activity_' + str(i) + "_label_" + str(train_set_y.eval()[visualize_ind[i]]) +'.jpg' , show_img = display_flag)
# visualizing the filters.
for m in xrange(len(nkerns)):
if m == 0: # first layer outpus.
if channels == 3: # if the image is color, then first layer looks at color pictures and I can visualize the filters also as color.
curr_image = weights[m].eval()
if not os.path.exists('../visuals/filters/layer_'+str(m)+'/epoch_'+str(epoch_counter)):
os.makedirs('../visuals/filters/layer_'+str(m)+'/epoch_'+str(epoch_counter))
visualize_color_filters(curr_image, loc = '../visuals/filters/layer_' + str(m) + '/' + 'epoch_' + str(epoch_counter) + '/' , filename = 'kernel_0.jpg' , show_img = display_flag)
else: # visualize them as grayscale images.
for i in xrange(weights[m].shape.eval()[1]):
curr_image = weights[m].eval() [:,i,:,:]
if not os.path.exists('../visuals/filters/layer_'+str(m)+'/epoch_'+str(epoch_counter)):
os.makedirs('../visuals/filters/layer_'+str(m)+'/epoch_'+str(epoch_counter))
visualize(curr_image, loc = '../visuals/filters/layer_' + str(m) + '/' + 'epoch_' + str(epoch_counter) + '/' , filename = 'kernel_' + str(i) + '.jpg' , show_img = display_flag)
else:
for i in xrange(nkerns[m-1]):
curr_image = weights[m].eval()[:,i,:,:]
if not os.path.exists('../visuals/filters/layer_'+str(m)+'/epoch_'+str(epoch_counter)):
os.makedirs('../visuals/filters/layer_'+str(m)+'/epoch_'+str(epoch_counter))
visualize(curr_image, loc = '../visuals/filters/layer_' + str(m) + '/' + 'epoch_' + str(epoch_counter) + '/' , filename = 'kernel_' + str(i) + '.jpg' , show_img = display_flag)
if patience <= iteration:
early_termination = True
break
save_network( 'network.pkl.gz', params, arch_params, data_params )
end_time = time.clock()
print "... training complete, took " + str((end_time - start_time)/ 60.) +" minutes"
###############
# TEST MODEL #
###############
start_time = time.clock()
print "... testing"
wrong = 0
predictions = []
class_prob = []
labels = []
if multi_load is False:
labels = test_set_y.eval().tolist()
for mini_batch in xrange(batches2test):
#print ".. Testing batch " + str(mini_batch)
wrong = wrong + int(test_model(mini_batch))
predictions = predictions + prediction(mini_batch).tolist()
class_prob = class_prob + nll(mini_batch).tolist()
print "... -> Total test accuracy : " + str(float((batch_size*n_test_batches)-wrong )*100/(batch_size*n_test_batches)) + " % out of " + str(batch_size*n_test_batches) + " samples."
else:
for batch in xrange(batches2test):
print ".. Testing batch " + str(batch)
# Load data for this batch
if data_params["type"] == 'mat':
test_data_x, test_data_y, test_data_y1 = load_data_mat(dataset, batch = batch + 1 , type_set = 'test')
elif data_params["type"] == 'skdata':
if dataset == 'caltech101':
test_data_x, test_data_y = load_skdata_caltech101(batch_size = load_batches, batch = batch + 1 , type_set = 'test', rand_perm = rand_perm, height = height, width = width )
test_set_x.set_value(test_data_x,borrow = True)
test_set_y.set_value(test_data_y,borrow = True)
labels = labels + test_set_y.eval().tolist()
for mini_batch in xrange(n_test_batches):
wrong = wrong + int(test_model(mini_batch))
predictions = predictions + prediction(mini_batch).tolist()
class_prob = class_prob + nll(mini_batch).tolist()
print "... -> Total test accuracy : " + str(float((batch_size*n_test_batches*batches2test)-wrong )*100/(batch_size*n_test_batches*batches2test)) + " % out of " + str(batch_size*n_test_batches*batches2test) + " samples."
end_time = time.clock()
correct = 0
confusion = numpy.zeros((outs,outs), dtype = int)
for index in xrange(len(predictions)):
if labels[index] is predictions[index]:
correct = correct + 1
confusion[int(predictions[index]),int(labels[index])] = confusion[int(predictions[index]),int(labels[index])] + 1
# Save down data
f = open(results_file_name, 'w')
for i in xrange(len(predictions)):
f.write(str(i))
f.write("\t")
f.write(str(labels[i]))
f.write("\t")
f.write(str(predictions[i]))
f.write("\t")
for j in xrange(outs):
f.write(str(class_prob[i][j]))
f.write("\t")
f.write('\n')
f = open(error_file_name,'w')
for i in xrange(len(this_validation_loss)):
f.write(str(this_validation_loss[i]))
f.write("\n")
f.close()
f = open(cost_file_name,'w')
for i in xrange(len(cost_saved)):
f.write(str(cost_saved[i]))
f.write("\n")
f.close()
f = open(confusion_file_name, 'w')
f.write(confusion)
f.close()
save_network( network_save_name, params, arch_params, data_params )
end_time = time.clock()
print "Testing complete, took " + str((end_time - start_time)/ 60.) + " minutes"
print "Confusion Matrix with accuracy : " + str(float(correct)/len(predictions)*100)
print confusion
print "Done"
pdb.set_trace()
def print_net(self, epoch, display_flag=True):
# saving down true images.
if self.main_img_visual is False:
for i in xrange(self.n_visual_images):
curr_img = numpy.asarray(
numpy.reshape(
self.train_set_x.get_value(borrow=True)[self.visualize_ind[i]],
[self.height, self.width, self.channels],
)
* 255.0,
dtype="uint8",
)
if self.display_flag is True:
cv2.imshow(
"Image Number " + str(i) + "_label_" + str(self.train_set_y.eval()[self.visualize_ind[i]]),
curr_img,
)
cv2.imwrite(
"../visuals/images/image_"
+ str(i)
+ "_label_"
+ str(self.train_set_y.eval()[self.visualize_ind[i]])
+ ".jpg",
curr_img,
)
self.main_img_visual = True
# visualizing activities.
activity = self.activities(0)
for m in xrange(len(self.nkerns)): # For each layer
loc_ac = "../visuals/activities/layer_" + str(m) + "/epoch_" + str(epoch) + "/"
if not os.path.exists(loc_ac):
os.makedirs(loc_ac)
current_activity = activity[m]
for i in xrange(self.n_visual_images): # for each randomly chosen image .. visualize its activity
util.visualize(
current_activity[self.visualize_ind[i]],
loc=loc_ac,
filename="activity_"
+ str(i)
+ "_label_"
+ str(self.train_set_y.eval()[self.visualize_ind[i]])
+ ".jpg",
show_img=display_flag,
)
# visualizing the filters.
for m in xrange(len(self.nkerns)):
curr_weights = self.weights[m].eval()
if curr_weights.shape[1] == 3:
# if the image is color, then first layer looks at color pictures and
# I can visualize the filters also as color.
curr_image = curr_weights
if not os.path.exists("../visuals/filters/layer_" + str(m) + "/epoch_" + str(epoch)):
os.makedirs("../visuals/filters/layer_" + str(m) + "/epoch_" + str(epoch_counter))
util.visualize_color_filters(
curr_image,
loc="../visuals/filters/layer_" + str(m) + "/" + "epoch_" + str(epoch) + "/",
filename="kernel_0.jpg",
show_img=self.display_flag,
)
else: # visualize them as grayscale images.
for i in xrange(curr_weights.shape[1]):
curr_image = curr_weights[:, i, :, :]
if not os.path.exists("../visuals/filters/layer_" + str(m) + "/epoch_" + str(epoch)):
os.makedirs("../visuals/filters/layer_" + str(m) + "/epoch_" + str(epoch))
util.visualize(
curr_image,
loc="../visuals/filters/layer_" + str(m) + "/" + "epoch_" + str(epoch) + "/",
filename="kernel_" + str(i) + ".jpg",
show_img=self.display_flag,
)
if epoch == 0 and i == 0:
with open('../output/da/graph.dot', 'w') as o:
o.write(computational_graph.build_computational_graph((loss, )).dump())
with open('../output/da/graph.wo_split.dot', 'w') as o:
g = computational_graph.build_computational_graph((loss, ), remove_split=True)
o.write(g.dump())
print 'train mean loss={}'.format(sum_loss / N)
losses += [sum_loss / N]
# 評価
sum_loss = 0
for i in xrange(0, N_test, batchsize):
x_batch = x_test[i:i+batchsize]
loss = da.test(x_batch)
sum_loss += float(loss.data) * batchsize
print 'test mean loss={}'.format(sum_loss / N_test)
losses += [sum_loss / N_test]
all_loss.append(losses)
# 可視化
h = MinMaxScaler().fit_transform(da.model.encoder.W)
visualize(h, '../img/da/w/da_{0:04d}.jpg'.format(epoch + 1), (8, 8), (10, 10))
# モデルの保存
pickle.dump(da, open('../output/da/model.pkl', 'wb'))
# loss, accuracyの保存
pickle.dump(all_loss, open('../output/da/loss.pkl', 'wb'))
# 可視化
visualize(h, '../img/da/da_w.jpg', (8, 8), (10, 10))
loss = da1.train(x_batch)
sum_loss += float(loss.data) * batchsize
print 'train mean loss={}'.format(sum_loss / N)
losses += [sum_loss / N]
# 評価
sum_loss = 0
for i in xrange(0, N_test, batchsize):
x_batch = x_test[i:i+batchsize]
loss = da1.test(x_batch)
sum_loss += float(loss.data) * batchsize
print 'test mean loss={}'.format(sum_loss / N_test)
losses += [sum_loss / N_test]
all_loss.append(losses)
# 可視化
h1 = MinMaxScaler().fit_transform(da1.model.encoder.W)
visualize(h1, '../img/sda/da1/sda_da1_{0:04d}.jpg'.format(epoch + 1), (8, 8), (10, 10))
pickle.dump(da1, open(da1_filename, 'wb'))
pickle.dump(all_loss, open('../output/sda/loss_da1.pkl', 'wb'))
# 2層目
da2_filename = '../output/sda/model_da2.pkl'
try:
da2 = pickle.load(open(da2_filename))
except IOError:
da2 = DenoisingAutoencoder(100, 49, noise=SaltAndPepperNoise())
h1_train = da1.encode(Variable(x_train)).data
h1_test = da1.encode(Variable(x_test)).data
n_epoch = 30
all_loss = []
for epoch in xrange(n_epoch):
print 'epoch', epoch
