def create_simple_data_set(
n_training_points,
n_testing_points,
low=0,
high=3,
mode=training_testing_split.SEPERATE,
kernel=kernel_matern,
shuffle=True,
):
"""
This function uses GP to generate data
"""
gp = gaussian_process(kernel=kernel, verbose=True)
mid = (low + high) / 2
if mode == training_testing_split.SEPERATE_LONG:
x_training, x_testing = __seperate_long(
n_training_points, n_testing_points, low, high
)
elif mode == training_testing_split.SEPERATE:
x_training, x_testing = __seperate(
n_training_points, n_testing_points, low, high
)
elif mode == training_testing_split.INTERSPREAD:
x_training, x_testing = __interspread(
n_training_points, n_testing_points, low, high
)
elif mode == training_testing_split.RANDOM:
x_training, x_testing = __random(n_training_points, n_testing_points, low, high)
elif mode == training_testing_split.MIXED:
def r(z):
dist = np.random.randint(low=1, high=100, size=4)
λ = lambda x: x / dist.sum()
vfunc = np.vectorize(λ)
dist = vfunc(dist)
return (z * dist).round().astype(int)
training_dist = r(n_training_points)
testing_dist = r(n_testing_points)
x1, x2 = __random(training_dist[0], testing_dist[0], low, high)
x11, x22 = __interspread(training_dist[1], testing_dist[1], low, high)
x111, x222 = __interspread(training_dist[2], testing_dist[2], low, high)
x1111, x2222 = __seperate(training_dist[3], testing_dist[3], low, high)
x_training = np.vstack([x1, x11, x111, x1111])
x_testing = np.vstack([x2, x22, x222, x222])
y_samples = gp.sample(np.vstack([x_training, x_testing]), 1).squeeze()
y_training = y_samples[: len(x_training)].reshape(-1, 1)
y_testing = y_samples[len(x_training) :].reshape(-1, 1)
training_data_set = data_loader.DataSet(X=x_training, Y=y_training)
testing_data_set = data_loader.DataSet(X=x_testing, Y=y_testing)
if shuffle:
training_data_set.shuffle()
testing_data_set.shuffle()
return training_data_set, testing_data_set
def test_change_training_size(self):
conf = self.defaults._replace(train_rat=1.0)
ds = data_loader.DataSet(conf)
assert ds.lenOfEpoch('train') == 10
assert ds.lenOfEpoch('test') == 0
conf = self.defaults._replace(train_rat=0.0)
ds = data_loader.DataSet(conf)
assert ds.lenOfEpoch('train') == 0
assert ds.lenOfEpoch('test') == 10
def test_limit_dataset_size(self):
# We only have 3-4 features per label, which means asking for 10 will
# do nothing.
conf = self.defaults._replace(train_rat=1.0, num_samples=10)
ds = data_loader.DataSet(conf)
assert ds.lenOfEpoch('train') == 10
# Now there must be exactly 1 feature for each of the three labels,
# which means a total of 3 features in the data_loader.
conf = self.defaults._replace(train_rat=1.0, num_samples=1)
ds = data_loader.DataSet(conf)
assert ds.lenOfEpoch('train') == 3
def test_nextBatch_reset(self):
ds = data_loader.DataSet(self.defaults._replace(train_rat=0.8))
ds.nextBatch(2, 'train')
assert ds.posInEpoch('train') == 2
assert ds.posInEpoch('test') == 0
ds.nextBatch(1, 'test')
assert ds.posInEpoch('train') == 2
assert ds.posInEpoch('test') == 1
ds.reset('train')
assert ds.posInEpoch('train') == 0
assert ds.posInEpoch('test') == 1
ds.reset('test')
assert ds.posInEpoch('train') == 0
assert ds.posInEpoch('test') == 0
ds.nextBatch(2, 'train')
ds.nextBatch(2, 'test')
assert ds.posInEpoch('train') == 2
assert ds.posInEpoch('test') == 2
ds.reset()
assert ds.posInEpoch('train') == 0
assert ds.posInEpoch('test') == 0
def test_item_to_mat_conversion():
""" Test conversion mat > item > mat
NOT testing item > mat > item, because comparing items more tedious (first need association!),
"""
# get "original" mat
params = {'model': {'factor_downsample': 4,
'input_height': 512,
'input_width': 1536,
},
}
dataset = data_loader.DataSet(path_csv='../data/train.csv',
path_folder_images='../data/train_images',
path_folder_masks='../data/train_masks',
)
list_ids = dataset.list_ids
item_org = dataset.load_item(list_ids[0], flag_load_mask=False)
dataset_torch = data_loader_torch.DataSetTorch(dataset, params)
mat_org = dataset_torch.convert_item_to_mat(item_org)
# convert back
item = dataset_torch.convert_mat_to_item(mat_org)
item.img = item_org.img # need this for back conversion!
mat = dataset_torch.convert_item_to_mat(item)
assert np.allclose(mat, mat_org)
def test_nextBatch(self):
ds = data_loader.DataSet(self.defaults._replace(train_rat=0.8))
# Basic parameters.
assert ds.lenOfEpoch('train') == 8
assert ds.lenOfEpoch('test') == 2
assert ds.posInEpoch('train') == 0
assert ds.posInEpoch('test') == 0
# Fetch one feature/label/handle.
x, y, handles = ds.nextBatch(1, 'train')
assert len(x) == len(y) == len(handles) == 1
assert ds.posInEpoch('train') == 1
assert ds.posInEpoch('test') == 0
assert isinstance(x, np.ndarray) and isinstance(y, np.ndarray)
assert isinstance(handles, np.ndarray)
assert x.shape == (1, 1, 2, 2) and x.dtype == np.float32
assert y.shape == (1, ) and y.dtype == np.int32
assert handles.shape == (1,) and handles.dtype == np.int64
# Fetch the remaining 7 training data elements.
x, y, _ = ds.nextBatch(7, 'train')
assert len(x) == len(y) == len(_) == 7
assert ds.posInEpoch('train') == 8
assert ds.posInEpoch('test') == 0
# Another query must yield nothing because the epoch is exhausted.
x, y, _ = ds.nextBatch(1, 'train')
assert np.array_equal(x, np.zeros((0, 1, 2, 2)))
assert np.array_equal(y, [])
assert len(_) == 0
def evaluate(
model,
device,
params,
):
# define dataset
dataset = data_loader.DataSet(
path_csv=params['datasets']['valid']['path_csv'],
path_folder_images=params['datasets']['valid']['path_folder_images'],
path_folder_masks=params['datasets']['valid']['path_folder_masks'],
)
dataset_torch = data_loader_torch.DataSetTorch(dataset,
params,
flag_augment=False)
dataset_loader = torch.utils.data.DataLoader(
dataset=dataset_torch,
batch_size=params['train']['batch_size_eval'],
shuffle=False,
num_workers=0,
)
# set model to eval (affects e.g. dropout layers) and disable unnecessary grad computation
model.eval()
torch.set_grad_enabled(False)
torch.cuda.empty_cache() # empty cuda cache to prevent memory errors
gc.collect(
) # empty unreferenced objects at the end. Not sure whether better at the end?
# calculate loss for whole dataset
num_batches = len(dataset_loader)
loss_per_name = dict()
print("Evaluating")
for img_batch, mask_batch, heatmap_batch, regr_batch in tqdm(
dataset_loader):
# concat img and mask and perform inference
input = torch.cat([img_batch, mask_batch],
1) # nbatch, nchannels, height, width
output = model(input.to(device))
# calculate loss
batch_loss_per_name = calc_loss(
output,
heatmap_batch.to(device),
regr_batch.to(device),
params['train']['loss'],
)
for name, batch_loss in batch_loss_per_name.items():
if name not in loss_per_name:
loss_per_name[name] = 0
loss_per_name[name] += batch_loss.data
# calculate average
for name, loss in loss_per_name.items():
loss_per_name[name] = loss.cpu().numpy() / len(dataset_loader)
len_dataset = len(dataset_loader.dataset) # check difference
len_dataset2 = len(dataset_loader)
return loss_per_name
def test_nextBatch_invalid(self):
ds = data_loader.DataSet(self.defaults._replace(train_rat=0.8))
# Invalid value for N.
with pytest.raises(AssertionError):
ds.nextBatch(-1, 'train')
# Unknown dataset name (must be 'train' or 'test').
with pytest.raises(AssertionError):
ds.nextBatch(1, 'foo')
def test_basic(self):
conf = self.defaults._replace(train_rat=0.8)
ds = data_loader.DataSet(conf)
assert ds.lenOfEpoch('train') == 8
assert ds.lenOfEpoch('test') == 2
assert ds.posInEpoch('train') == 0
assert ds.posInEpoch('test') == 0
dim = ds.imageDimensions()
assert isinstance(dim, np.ndarray)
assert dim.dtype == np.uint32
assert dim.tolist() == [1, 2, 2]
assert ds.classNames() == {0: '0', 1: '1', 2: '2'}
def test_car_to_string():
""" Test that conversion string -> car -> string works
"""
dataset = data_loader.DataSet(
path_csv='../data/train.csv',
path_folder_images='../data/train_images',
path_folder_masks='../data/train_masks',
)
cars_as_string = dataset.df_cars.loc[0, 'PredictionString']
item = data_loader.DataItem()
item.set_cars_from_string(cars_as_string)
cars_as_string2 = item.get_cars_as_string()
# cars_as_string2 += '4'
assert cars_as_string == cars_as_string2
def test_limitSampleSize(self):
conf = self.defaults._replace(train_rat=1.0, num_samples=10)
ds = data_loader.DataSet(conf)
for i in range(1, 5):
x = np.arange(100)
y = x % 3
m = x % 5
x, y, m = ds.limitSampleSize(x, y, m, i)
assert len(x) == len(y) == len(m) == 3 * i
assert set(y) == {0, 1, 2}
x = np.arange(6)
y = x % 3
m = x % 5
x2, y2, m2 = ds.limitSampleSize(x, y, m, 0)
assert len(x2) == len(y2) == len(m2) == 0
x2, y2, m2 = ds.limitSampleSize(x, y, m, 100)
assert len(x2) == len(y2) == len(m2) == len(x)
def main():
log_path = "./tb_logs"
if (os.path.isdir(log_path)):
shutil.rmtree(log_path)
if FLAGS.dataset == "mnist":
data = dl.DataSet(28, 28, "mnist",
"../datasets/mnist/train_images.mat",
"../datasets/mnist/test_images.mat",
"../datasets/mnist/train_labels.mat",
"../datasets/mnist/test_labels.mat")
data.prepare_shape(FLAGS.num_categories)
elif FLAGS.dataset == "svhn":
data = dl.DataSet(32, 32, "svhn",
"../datasets/svhn_dataset/train_32x32.mat",
"../datasets/svhn_dataset/test_32x32.mat")
data.prepare_shape(FLAGS.num_categories)
data_height = data._height
data_width = data._width
num_dimensions = 1
x = tf.placeholder(tf.float32, [None, data_height * data_width])
y_ = tf.placeholder(tf.float32, [None, FLAGS.num_categories])
output, keep_prob = nns.nn_4layers_ccff(x, num_dimensions, data_height,
data_width, FLAGS.num_categories)
sess = tf.InteractiveSession()
with tf.name_scope("cross_entropy"):
cross_entropy = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=output))
with tf.name_scope("loss_optmizer"):
train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)
with tf.name_scope("accuracy"):
correct_prediction = tf.equal(tf.argmax(output, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
tf.summary.scalar("cross_entropy", cross_entropy)
tf.summary.scalar("accuracy", accuracy)
summarize_all = tf.summary.merge_all()
sess.run(tf.global_variables_initializer())
tbWriter = tf.summary.FileWriter(log_path, sess.graph)
start_time = time.time()
end_time = time.time()
for i in range(FLAGS.num_steps):
batch_data, batch_labels = data.load_batch(FLAGS.batch_size)
_, summary = sess.run([train_step, summarize_all],
feed_dict={
x: batch_data,
y_: batch_labels,
keep_prob: 0.5
})
if i % FLAGS.display_step == 0:
train_accuracy = accuracy.eval(feed_dict={
x: batch_data,
y_: batch_labels,
keep_prob: 1.0
})
end_time = time.time()
print(
"step {0}, elapsed time {1:.2f} seconds, training accuracy {2:.3f}%"
.format(i, end_time - start_time, train_accuracy * 100.0))
tbWriter.add_summary(summary, i)
end_time = time.time()
print("Total training time for {0} batches: {1:.2f} seconds".format(
i + 1, end_time - start_time))
test_data, test_labels = data.load_test_data()
print("Test accuracy {0:.3f}%".format(
accuracy.eval(feed_dict={
x: test_data,
y_: test_labels,
keep_prob: 1.0
}) * 100.0))
saver = tf.train.Saver()
saver.save(sess, '../trained_models/' + FLAGS.model_path)
saver.export_meta_graph('../trained_models/' + FLAGS.model_path + '.meta')
sess.close()
t=T_stride,
train_overlap=T_overlop,
noise_type=NOISE_TYPE[3],
train_divide=TRAIN_DIVIDE,
noise_proportion=noise_proportion,
noise_path=NOISE_PATH,
seed=seed)
train_X_features = feature_extractor.get_features(
FEATURES_TO_USE, train_X)
valid_features_dict = {}
for _, i in enumerate(val_dict):
X1 = feature_extractor.get_features(
FEATURES_TO_USE, val_dict[i]['X'])
valid_features_dict[i] = {'X': X1, 'y': val_dict[i]['y']}
train_data = data_loader.DataSet(train_X_features, train_y)
train_loader = DataLoader(train_data,
batch_size=BATCH_SIZE,
shuffle=True)
model = MODEL.MACNN(attention_head, attention_hidden)
if torch.cuda.is_available():
model = model.cuda()
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(),
lr=learning_rate,
weight_decay=1e-6)
maxWA = 0
def train(
model,
device,
params,
):
# define training dataset
dataset = data_loader.DataSet(
path_csv=params['datasets']['train']['path_csv'],
path_folder_images=params['datasets']['train']['path_folder_images'],
path_folder_masks=params['datasets']['train']['path_folder_masks'],
)
dataset_torch = data_loader_torch.DataSetTorch(
dataset,
params,
flag_load_label=True,
flag_augment=params['train']['use_augmentation'],
)
dataset_loader = torch.utils.data.DataLoader(
dataset=dataset_torch,
batch_size=params['train']['batch_size'],
shuffle=True,
num_workers=4,
pin_memory=True, # see https://pytorch.org/docs/stable/data.html
)
# define optimizer and decrease learning rate by 0.1 every 3 epochs
optimizer = torch.optim.AdamW(
model.parameters(),
lr=params['train']['learning_rate']['initial'],
)
lr_scheduler = torch.optim.lr_scheduler.StepLR(
optimizer,
step_size=params['train']['learning_rate']['num_epochs_const'],
gamma=params['train']['learning_rate']['factor_decrease'],
)
# for each epoch...
df_out = pd.DataFrame()
for idx_epoch in range(params['train']['num_epochs']):
print("Training epoch {}".format(idx_epoch))
# set model to train (affects e.g. dropout layers) and disable unnecessary grad computation
model.train()
torch.set_grad_enabled(True)
torch.cuda.empty_cache() # empty cuda cache to prevent memory errors
gc.collect(
) # empty unreferenced objects at the end. Not sure whether better at the end?
# calculate loss for whole dataset
num_batches = len(dataset_loader)
loss_per_name = dict()
dataset_tqdm = tqdm(dataset_loader)
for img_batch, mask_batch, heatmap_batch, regr_batch in dataset_tqdm:
# concat img and mask and perform inference
input = torch.cat([img_batch, mask_batch],
1) # nbatch, nchannels, height, width
output = model(input.to(device))
# calculate loss
batch_loss_per_name = calc_loss(
output,
heatmap_batch.to(device),
regr_batch.to(device),
params['train']['loss'],
)
for name, batch_loss in batch_loss_per_name.items():
if name not in loss_per_name:
loss_per_name[name] = 0
loss_per_name[name] += batch_loss.data
# change tqdm progress bar description
description = "loss: "
for name, batch_loss in batch_loss_per_name.items():
description += "{}={:.3f} ".format(
name,
batch_loss.data.cpu().numpy())
dataset_tqdm.set_description(description)
# perform optimization
batch_loss_per_name['tot'].backward(
) # computes x.grad += dloss/dx for all parameters x
optimizer.step() # updates values x += -lr * x.grad
optimizer.zero_grad() # set x.grad = 0, for next iteration
# step learning rate after each epoch (not after each batch)
lr_scheduler.step()
# calculate average and store results
for name in loss_per_name.keys():
loss_per_name[name] = loss_per_name[name].cpu().numpy() / len(
dataset_loader)
df_out.loc[idx_epoch, 'loss_' + name] = loss_per_name[name]
values_per_name = evaluate(model, device, params)
for key, value in values_per_name.items():
df_out.loc[idx_epoch, 'valid_' + key] = value
# save history
path_csv = os.path.join(
params['path_folder_out'],
'train_history_{}.csv'.format(idx_epoch),
)
os.makedirs(os.path.dirname(path_csv), exist_ok=True)
df_out.to_csv(path_csv, sep=';')
print(df_out)
# save model weights
path_weights = os.path.join(
params['path_folder_out'],
'model_{}.pth'.format(idx_epoch),
)
torch.save(model.state_dict(), path_weights)
return df_out
def test_basic_err(self):
with pytest.raises(AssertionError):
data_loader.DataSet(self.defaults._replace(train_rat=-1.0))
with pytest.raises(AssertionError):
data_loader.DataSet(self.defaults._replace(train_rat=1.1))
def predict(model,
device,
params,
):
# Create data generators - they will produce batches
dataset_test = data_loader.DataSet(
path_csv=params['datasets']['test']['path_csv'],
path_folder_images=params['datasets']['test']['path_folder_images'],
path_folder_masks=params['datasets']['test']['path_folder_masks'],
)
dataset_torch_test = data_loader_torch.DataSetTorch(
dataset_test, params,
flag_load_label=False,
flag_augment=False,
)
data_loader_test = torch.utils.data.DataLoader(
dataset=dataset_torch_test,
batch_size=params['predict']['batch_size'],
shuffle=False,
num_workers=4,
pin_memory=True, # see https://pytorch.org/docs/stable/data.html
)
# perform predictions
predictions = []
model.eval()
idx_batch = -1
for img, mask, _, _ in tqdm(data_loader_test):
idx_batch += 1
if idx_batch > params['predict']['num_batches_max']:
print("Ending early because of param num_batches_max={}".format(params['predict']['num_batches_max']))
break
# perform prediction
with torch.no_grad():
# concat img and mask and perform inference
input = torch.cat([img, mask], 1) # nbatch, nchannels, height, width
output = model(input.to(device))
output = output.data.cpu().numpy()
# extract cars as string from each element in batch
num_elems_in_batch = output.shape[0]
for idx_elem_in_batch in range(num_elems_in_batch):
idx_id = idx_batch * params['predict']['batch_size'] + idx_elem_in_batch
id = dataset_test.list_ids[idx_id]
# get mat from output and plot
mat = output[idx_elem_in_batch, ...]
mat = np.rollaxis(mat, 0, 3) # reverse rolling backwards
if params['predict']['flag_plot_mat']:
# convert image to numpy
img_numpy = img.data.cpu().numpy()
img_numpy = img_numpy[idx_elem_in_batch, ...]
img_numpy = np.rollaxis(img_numpy, 0, 3) # reverse rolling backwards
img_numpy = img_numpy[:, :, ::-1] # BGR to RGB
fig, ax = plt.subplots(2, 1, figsize=(10, 10))
ax[0].imshow(img_numpy)
ax_mask = ax[1].imshow(mat[:, :, 0], cmap='PiYG', vmin=-1, vmax=+1)
fig.colorbar(ax_mask, ax=ax[1])
if False: # only use in case of multiple axes, here x,y,z
ax[2].imshow(mat[:, :, 4])
ax[3].imshow(mat[:, :, 5])
ax[4].imshow(mat[:, :, 7])
for axi, label in zip(ax, ['img', 'mask']): # , 'x', 'y', 'z']):
axi.set_ylabel(label)
fig.suptitle('ImageID={}'.format(id))
# save
path_out = os.path.join(params['path_folder_out'],
'pred_mat',
'{:05d}.png'.format(idx_id),
)
os.makedirs(os.path.dirname(path_out), exist_ok=True)
fig.savefig(path_out)
plt.close()
# convert mat to item and plot.
item = dataset_torch_test.convert_mat_to_item(mat)
if params['predict']['flag_optimize']:
for idx_car, car in enumerate(item.cars):
x_new, y_new, z_new, is_marked = optimize_xyz(car.v, car.u, car.x, car.y, car.z, params)
item.cars[idx_car].x = x_new
item.cars[idx_car].y = y_new
item.cars[idx_car].z = z_new
item.cars[idx_car].is_marked = is_marked
if params['predict']['flag_plot_item']:
item_org = dataset_test.load_item(id)
item.img = item_org.img
item.mask = np.zeros((1, 1))
fig, ax = item.plot()
fig.suptitle('ImageID={}'.format(id))
if idx_batch == 2:
num_cars = len(item.cars)
plt.show()
# save
path_out = os.path.join(params['path_folder_out'],
'pred_item',
'{:05d}.png'.format(idx_id),
)
os.makedirs(os.path.dirname(path_out), exist_ok=True)
fig.savefig(path_out)
plt.close()
# extract prediction string from item
string = item.get_cars_as_string(flag_submission=True)
predictions.append(string)
# predictions to csv
df_out = pd.DataFrame()
df_out['ImageId'] = dataset_test.list_ids
df_out.loc[0:len(predictions) - 1, 'PredictionString'] = predictions
print(df_out.head())
path_csv = os.path.join(params['path_folder_out'], 'predictions.csv')
df_out.to_csv(path_csv, sep=',', index=False)
return df_out
# return
uv = np.array([u, v])
return uv
if __name__ == '__main__':
params = {'model': {'factor_downsample': 4,
'input_height': 512,
'input_width': 1536,
},
'train': {'loss': {'flag_focal_loss': 1}},
'datasets': {'flag_use_mask': 1},
}
dataset = data_loader.DataSet(path_csv='../data/train.csv',
path_folder_images='../data/train_images',
path_folder_masks='../data/train_masks',
)
dataset_torch = DataSetTorch(dataset, params, flag_augment=True)
num_items = len(dataset_torch)
for idx_item in tqdm(range(num_items)):
[img, mask, heatmap, regr] = dataset_torch[idx_item]
# reverse rolling backwards
img = np.rollaxis(img, 0, 3)
mask = np.rollaxis(mask, 0, 3)
regr = np.rollaxis(regr, 0, 3)
print(img.shape)
print(regr.shape)
# plot example
if False:
