def snow_analysis(self, seg_model):
"""
Complete snow analysis for each image of the directory and store the result in a text file.
:param seg_model: segmentation model
:type seg_model: keras.model
"""
# Variable initialization
self.snowy_list = []
# Creation of directory to store results
list_dir_path = os.path.join(self.img_dir, "Lists")
list_dir_path = safe_folder_creation(list_dir_path)
# Complete snow analysis for each image of the directory
pbar = progressbar.ProgressBar()
for img_filename in pbar(self.images_list):
img_path = os.path.join(self.img_dir, img_filename)
img = Ci.Image(img_path,
build_array=True,
build_snow_segmentation=True,
snow_seg_model=seg_model)
img.get_global_key_from_filename()
# img.show_image_snow_segmentation()
if img.is_snowy():
self.snowy_list.append(img_filename)
# Save in to files
write_list_to_textfile(
self.snowy_list, os.path.join(list_dir_path, "21-Snowy_list.txt"))
def simple_training(data_left, data_right, data_label, model_function,
model_function_args, folder_path, val_split, epochs,
batch_size):
"""
Train a siamese model and store results in a folder.
:param data_left: left images array
:type data_left: np.array
:param data_right: right images array
:type data_right: np.array
:param data_label: labels array
:type data_label: np.array
:param model_function: function which build the model to train with k fold
:type model_function: function
:param model_function_args: arguments of the model building function
:type model_function_args: list
:param folder_path: path of the folder where results are stored
:type folder_path: str
:param val_split: set proportion dedicated to the validation set
:type val_split: float
:param epochs: number of training epochs
:type epochs: int
:param batch_size: batch size
:type batch_size: int
:return:
:rtype:
"""
# Create folder to store results
folder_path = safe_folder_creation(folder_path)
# Build model
conv_model = model_function(*model_function_args)
train_data = np.stack([data_left, data_right], 0) #[data_left, data_right]
# Train model
model_fitted = conv_model.fit(train_data,
data_label,
epochs=epochs,
batch_size=batch_size,
validation_split=val_split)
# Save model and plots
model_path = os.path.join(folder_path, 'fitted_model.h5')
conv_model.save(model_path)
plot_validation_info(model_fitted, folder_path)
# Save also weights and structure for backup
weights_path = os.path.join(folder_path, 'weights.h5')
conv_model.save_weights(weights_path)
json_path = os.path.join(folder_path, 'structure.json')
save_structure_in_json(conv_model, json_path)
def create_images_from_mapillary_panos(self):
"""
Create for 4 simple images for all the downloaded Mapillary panoramas.
"""
# Creation of directory
self.mapillary_dir_images_from_pano = safe_folder_creation(self.mapillary_dir_images_from_pano)
# Display advancement of processing
pbar = progressbar.ProgressBar()
panos = os.listdir(self.mapillary_dir_pano)
for pano in pbar(panos):
# Create instance of panorama and its 4 images
p = Panorama(os.path.join(self.mapillary_dir_pano, pano))
p.create_4_images(self.mapillary_dir_images_from_pano)
# Print results
self.nb_mapillary_from_pano = 4 * len(panos)
print("Number of images produced : {}".format(self.nb_mapillary_from_pano))
def save_hyperas_results(best_model, best_run, result_folder, data, model):
"""
Create folder and file containing results of the Hyperas hyperparameters optimization.
:param best_model: best model found
:type best_model: keras.Model
:param best_run: dictionary of parameters of the best model
:type best_run: dict
:param result_folder:
:type result_folder:
:param data: loading data function
:type data: function
:param model: creation model function
:type model: function
"""
# Create result folder
result_folder = safe_folder_creation(result_folder)
# Save best model
best_model.save(os.path.join(result_folder, "best_model.h5"))
# Save also weights and structure for backup
weights_path = os.path.join(result_folder, 'weights.h5')
best_model.save_weights(weights_path)
json_path = os.path.join(result_folder, 'structure.json')
save_structure_in_json(best_model, json_path)
# Save model function an used data in a text file
s = "Grid search parameters Hyperas \n"
s += "\nData function code:\n\n" + inspect.getsource(data)
s += "\nModel source code:\n\n" + inspect.getsource(model)
s += "\nHyperas results:\n\n" + json.dumps(best_run)
# Save in a file
with open(os.path.join(result_folder, 'Hyperas_params.txt'), "w+") as f:
f.write(s)
def k_fold(data_left, data_right, data_label, k, model_function, model_function_args, folder_path, epochs,
batch_size):
"""
Execute a K-fold cross validation for a model.
:param data_left: left images array
:type data_left: np.array
:param data_right: right images array
:type data_right: np.array
:param data_label: labels array
:type data_label: np.array
:param k: number of fold
:type k: int
:param model_function: function which build the model to train with k fold
:type model_function: function
:param model_function_args: arguments of the model building function
:type model_function_args: list
:param folder_path: path of the folder where results are stored
:type folder_path: str
:param epochs: number of training epochs
:type epochs: int
:param batch_size: batch size
:type batch_size: int
"""
# Variable initialization
nb_comp = len(data_label)
num_val_samples = nb_comp // k
all_scores = []
# Create folder to store results
folder_path = safe_folder_creation(folder_path)
# Train model for each fold
for i in range(k):
print('Processing fold #', i)
# Select validation set
val_left = data_left[i * num_val_samples: (i + 1) * num_val_samples]
val_right = data_right[i * num_val_samples: (i + 1) * num_val_samples]
val_data = [val_left, val_right]
val_label = data_label[i * num_val_samples: (i + 1) * num_val_samples]
# Get the remaining data to create the training set
partial_train_left = np.concatenate(
[data_left[:i * num_val_samples], data_left[(i + 1) * num_val_samples:]])
partial_train_right = np.concatenate(
[data_right[:i * num_val_samples], data_right[(i + 1) * num_val_samples:]])
partial_train_data = [partial_train_left, partial_train_right]
partial_train_label = np.concatenate(
[data_label[:i * num_val_samples], data_label[(i + 1) * num_val_samples:]])
# Build model
bk.clear_session()
model = model_function(*model_function_args)
# Train model
model_fitted = model.fit(partial_train_data, partial_train_label, validation_data=[val_data, val_label],
epochs=epochs, batch_size=batch_size, verbose=1)
# Evaluate model and store result
val_mse, val_mae = model.evaluate(val_data, val_label, verbose=1)
all_scores.append(val_mae)
# Save model and plots
model_path = os.path.join(folder_path, 'fitted_model_k{}.h5'.format(i + 1))
model.save(model_path)
plot_validation_info_kfold(model_fitted, i + 1, folder_path)
# Save also weights and structure for backup
weights_path = os.path.join(folder_path, 'weights_k{}.h5'.format(i + 1))
model.save_weights(weights_path)
json_path = os.path.join(folder_path, 'structure_k{}.json'.format(i + 1))
save_structure_in_json(model, json_path)
# Save K-fold summary in a text file
filename = os.path.join(folder_path, "k_fold_report.txt")
summary = k_fold_summary(k, model_function, model_function_args, nb_comp, num_val_samples, all_scores, filename)
print(summary)
def green_analysis(self, green_seg_model, green_ratio, snow_seg_model):
"""
Execute all the following analyses at the same time to gain computation time:
- overexposure
- exposure from histogram
- blurriness
- under green ratio
- snow detection
:param green_seg_model: segmentation model for green ratio analysis
:type green_seg_model: keras.model
:param green_ratio: threshold value for green ratio
:type green_ratio: float
:param snow_seg_model: segmentation model for snow detection
:type snow_seg_model: keras.model
"""
# Variable initialization
self.green_ratio = green_ratio
# Creation of directories to store results
list_dir_path = os.path.join(self.img_dir, "Lists")
list_dir_path = safe_folder_creation(list_dir_path)
# Complete analysis for each image of the directory
pbar = progressbar.ProgressBar()
for img_filename in pbar(self.images_list):
img_path = os.path.join(self.img_dir, img_filename)
img = Ci.Image(img_path,
build_array=True,
build_green_segmentation=True,
green_seg_model=green_seg_model,
build_snow_segmentation=True,
snow_seg_model=snow_seg_model)
if img.is_overexposed():
self.overexpose_list.append(img_filename)
if img.is_wrong_exposed():
self.exposure_histogram_list.append(img_filename)
if img.is_under_green_ratio(green_ratio):
self.under_ratio_list.append(img_filename)
if img.is_blurry():
self.blurry_list.append(img_filename)
# Produce all and bad lists
self.bad_list = list_from_lists(self.overexpose_list,
self.exposure_histogram_list,
self.under_ratio_list,
self.blurry_list)
self.good_list = list(set(self.images_list) - set(self.bad_list))
# Save in to files
write_list_to_textfile(self.blurry_list,
os.path.join(list_dir_path, "01-Blurry.txt"))
write_list_to_textfile(
self.exposure_histogram_list,
os.path.join(list_dir_path, "02-Exposure_histogram.txt"))
write_list_to_textfile(
self.overexpose_list,
os.path.join(list_dir_path, "03-Overexposed_images.txt"))
write_list_to_textfile(
self.under_ratio_list,
os.path.join(list_dir_path,
"04-Under_Ratio_{}.txt".format(green_ratio)))
write_list_to_textfile(
self.bad_list, os.path.join(list_dir_path, "10-All_discarded.txt"))
write_list_to_textfile(self.good_list,
os.path.join(list_dir_path, "11-All_good.txt"))
def create_input_from_comparisons_predictions(model_path, img_dir, nb_creation,
save_folder):
"""
Create new data input for ranking model by predicting them with the comparisons model.
:param model_path: path of the trained comparisons model
:type model_path: str
:param img_dir: images directory path
:type img_dir: str
:param nb_creation: number of comparisons created
:type nb_creation: int
:param save_folder: path of the folder where the predictions dataset folder is stored
:type save_folder: str
:return: augmented dataset
:rtype: tuple(np.array)
"""
# Variable initialization
images_path = glob.glob(img_dir + "/*jpg")
nb_img = len(images_path) - 1
train_left_pred = []
train_right_pred = []
train_label_pred = []
train_label_score_pred = []
# Create output folder
pred_folder = safe_folder_creation(save_folder)
# Load comparisons model
comp_model = load_model(model_path)
img_size = comp_model.input_shape[0][1]
for _ in range(nb_creation):
# Select randomly two images
img_left = Ci.Image(images_path[random.randint(0, nb_img)],
build_array=True)
img_right = Ci.Image(images_path[random.randint(0, nb_img)],
build_array=True)
input_img_left = img_left.preprocess_image(img_size)
input_img_right = img_right.preprocess_image(img_size)
# Compute labels
proba = comp_model.predict(
[np.array([input_img_left]),
np.array([input_img_right])])
index_max = np.argmax(proba)
jump_next_pred = False
if index_max == 0:
winner = "left"
elif index_max == 1:
winner = "right"
else:
jump_next_pred = True
winner = "Draw"
if not jump_next_pred:
# Add labels to list
train_label_pred.append(get_label_comparison_from_string(winner))
train_label_score_pred.append(get_label_score_from_string(winner))
# Add images to list
train_left_pred.append(img_left.array)
train_right_pred.append(img_right.array)
# Convert to array
train_left_pred = np.array(train_left_pred)
train_right_pred = np.array(train_right_pred)
train_label_pred = np.array(train_label_pred)
train_label_score_pred = np.array(train_label_score_pred)
train_data_pred = [train_left_pred, train_right_pred]
# Save augmented training dataset as npy
np.save(os.path.join(pred_folder, "train_left_{}".format(img_size)),
train_left_pred)
np.save(os.path.join(pred_folder, "train_right_{}".format(img_size)),
train_right_pred)
np.save(os.path.join(pred_folder, "train_labels{}".format(img_size)),
train_label_pred)
np.save(os.path.join(pred_folder, "train_labels_score{}".format(img_size)),
train_label_score_pred)
return train_data_pred, train_label_pred, train_label_score_pred
def data_aug(train_left, train_right, train_label, train_label_score, nb,
save_folder):
"""
Create nb augmented comparisons from one comparison of the set. Save results in a folder.
:param train_left: left images array
:type train_left: np.array
:param train_right: right images array
:type train_right: np.array
:param train_label: comparisons labels array
:type train_label: np.array
:param train_label_score: ranking labels array
:type train_label_score: np.array
:param nb: number of augmented comparisons created for each original one
:type nb: int
:param save_folder: path of the folder where the augmented dataset folder is stored
:type save_folder: str
:return: augmented dataset
:rtype: tuple(np.array)
"""
# Create saving folder
aug_folder = safe_folder_creation(os.path.join(save_folder))
# Specify data generator parameters
datagenargs = {
'rotation_range': 2,
'width_shift_range': 0.2,
'height_shift_range': 0.2,
'shear_range': 0.1,
'zoom_range': 0.25,
'horizontal_flip': True,
'fill_mode': 'nearest'
}
# Create generators
left_datagen = ImageDataGenerator(**datagenargs)
right_datagen = ImageDataGenerator(**datagenargs)
# Initialization of data
train_left_aug = list(train_left)
train_right_aug = list(train_right)
train_label_aug = list(train_label)
train_label_score_aug = list(train_label_score)
img_size = train_left[0].shape[0]
# Display processing advancement
print("Creating new inputs...")
pbar = progressbar.ProgressBar()
# Create nb augmented images from an original one
for duel in pbar(range(len(train_label_score))):
for _ in range(nb):
# Create one augmented image from the left one
ori_left_img = train_left[duel]
left_img = ori_left_img.reshape((1, ) + ori_left_img.shape)
aug_img = left_datagen.flow(left_img, batch_size=1)
left_aug_img = aug_img[0].reshape(ori_left_img.shape)
# Create one augmented image from the right one
ori_right_img = train_right[duel]
right_img = ori_right_img.reshape((1, ) + ori_right_img.shape)
aug_img = right_datagen.flow(right_img, batch_size=1)
right_aug_img = aug_img[0].reshape(ori_right_img.shape)
# Add to list
train_left_aug.append(left_aug_img)
train_right_aug.append(right_aug_img)
train_label_aug.append(train_label[duel])
train_label_score_aug.append(train_label_score[duel])
# Convert to array
train_left_aug = np.array(train_left_aug)
train_right_aug = np.array(train_right_aug)
train_label_aug = np.array(train_label_aug)
train_label_score_aug = np.array(train_label_score_aug)
train_data_aug = [train_left_aug, train_right_aug]
print("Done\nSaving data ...")
# Save augmented training dataset as npy
np.save(os.path.join(aug_folder, "train_left_{}".format(img_size)),
np.array(train_left_aug))
np.save(os.path.join(aug_folder, "train_right_{}".format(img_size)),
np.array(train_right_aug))
np.save(os.path.join(aug_folder, "train_labels{}".format(img_size)),
np.array(train_label_aug))
np.save(os.path.join(aug_folder, "train_labels_score{}".format(img_size)),
np.array(train_label_score_aug))
print("Done")
return train_data_aug, train_label_aug, train_label_score_aug
def preprocessing_duels(csv_path, img_size, image_folder, save_folder, test):
"""
Create the inputs of the comparison network from the comparisons csv and save them as npy
A csv line has the following format:
image_key_1, image_key_2, winner, ip_address
the image keys are 22 character-long string
winner is one of the 3 following string : left, right, equivalent
:param csv_path: path of the comparisons csv
:type csv_path: str
:param img_size: input image size
:type img_size: int
:param image_folder: path of the image folder
:type image_folder: str
:param save_folder: path of the folder where npy are saved
:type save_folder: str
:param test: value to determine the size of the data dedicated to the test set.
It can either be a proportion inside [0,1] or the number of comparisons kept aside.
:type test: float
:return:
:rtype:
"""
# List initialization
left_images = []
right_images = []
labels = []
labels_score = []
print(csv_path)
# Get data from csv
with open(csv_path, 'r') as csvfileReader:
reader = csv.reader(csvfileReader, delimiter=',')
print("Creating inputs from csv ...")
pbar = progressbar.ProgressBar()
#for r in reader:
# print(r)
for line in reader: #pbar(reader):
# Do not include No preference comparisons
if line != [] and line[2] != 'No preference':
# Create Image instances
#print(line)
left_image_path = get_filename_from_key(line[0], image_folder)
right_image_path = get_filename_from_key(line[1], image_folder)
#print(left_image_path)
left_img = Ci.Image(left_image_path)
right_img = Ci.Image(right_image_path)
# Add images to list
left_images.append(left_img.preprocess_image(img_size))
right_images.append(right_img.preprocess_image(img_size))
# Add labels to list
labels.append(get_label_comparison_from_string(line[2]))
labels_score.append(get_label_score_from_string(line[2]))
# Compute number of comparisons kept for test set
if len(labels) > test > 1:
nb_test = int(test)
elif 0 <= test <= 1:
nb_test = int(test * len(labels))
else:
raise ValueError
print("Done\nSaving test set ...")
# Create test dataset
test_left = np.array(left_images[:nb_test])
test_right = np.array(right_images[:nb_test])
test_labels = np.array(labels[:nb_test])
test_labels_score = np.array(labels_score[:nb_test])
# Save testing dataset as npy
test_folder = os.path.join(save_folder, "test")
test_folder = safe_folder_creation(test_folder)
np.save(os.path.join(test_folder, "test_left_{}".format(img_size)),
np.array(test_left))
np.save(os.path.join(test_folder, "test_right_{}".format(img_size)),
np.array(test_right))
np.save(os.path.join(test_folder, "test_labels_{}".format(img_size)),
np.array(test_labels))
np.save(os.path.join(test_folder, "test_labels_score_{}".format(img_size)),
np.array(test_labels_score))
del test_left, test_right, test_labels, test_labels_score
print("Done\nSaving train set ...")
# Create training dataset
train_left = np.array(left_images[nb_test:])
train_right = np.array(right_images[nb_test:])
train_labels = np.array(labels[nb_test:])
train_labels_score = np.array(labels_score[nb_test:])
# Save training dataset as npy
train_folder = os.path.join(save_folder, "train")
train_folder = safe_folder_creation(train_folder)
np.save(os.path.join(train_folder, "train_left_{}".format(img_size)),
np.array(train_left))
del train_left
np.save(os.path.join(train_folder, "train_right_{}".format(img_size)),
np.array(train_right))
del train_right
np.save(os.path.join(train_folder, "train_labels_{}".format(img_size)),
np.array(train_labels))
np.save(
os.path.join(train_folder, "train_labels_score_{}".format(img_size)),
np.array(train_labels_score))
print("Done")
def download_gsv(self, gsv_key, start=0, end=-1):
"""
Downloads Google Street View (GSV) images points in Ottawa from a shapefile layer.
WARNING: the filename format is currently not compatible with the class Image definition.
:param gsv_key: GSV API key
:type gsv_key: str
:param start: index of the feature of the layer from where download starts
:type start: int
:param end: index of the feature of the layer from where download ends
:type end: int
"""
# Creation of directory
self.gsv_dir = safe_folder_creation(self.gsv_dir)
# Convert layer file
ds = ogr.Open(self.layer_path)
layer = ds.GetLayer()
# Determine the number of locations to download
loc_max = len(layer)
if start < end < len(layer):
stop = end
else:
stop = loc_max
n_loc = stop - start
# Display advancement of downloading
pbar = progressbar.ProgressBar()
for i in pbar(range(start, stop)):
# Get location
feature = layer[i]
lon = feature.GetGeometryRef().GetX()
lat = feature.GetGeometryRef().GetY()
# Get the closest panoramas from the location
pano_id = streetview.panoids(lat, lon, closest=True)
# Check if there is a pano
if len(pano_id):
# Create filename
image_key = pano_id[0]["panoid"]
if pano_id[0]["month"] < 10:
image_date = str(pano_id[0]["year"]) + "-0" + str(pano_id[0]["month"]) + "-01T00-00-00"
else:
image_date = str(pano_id[0]["year"]) + "-" + str(pano_id[0]["month"]) + "-01T00-00-00"
image_lon = "{0:.6f}".format(lon)
image_lat = "{0:.6f}".format(lat)
image_filename = '{}_{}_{}_{}'.format(image_lon, image_lat, image_key, image_date)
# Download one image
try:
streetview.api_download(image_key, 90, self.gsv_dir, gsv_key, fov=80, pitch=0,
fname=image_filename)
self.nb_gsv += 1
except Exception as err:
print(err)
print("Error on feature {}, lat = {}, lon = {} ".format(i, lat, lon))
continue
# Display information
print("Number of locations : {}".format(n_loc))
print("Number of images downloaded : {}".format(self.nb_gsv))
print("Ratio : {}%".format((self.nb_gsv / n_loc) * 100))
def download_mapillary(self, mapillary_key, start=0, end=-1):
"""
Downloads Mapillary images of points from a ArcGis layer
:param mapillary_key: Mapillary API key
:type mapillary_key: str
:param start: index of the feature of the layer from where download starts
:type start: int
:param end: index of the feature of the layer where download ends
:type end: int
"""
# Initialization of variables
self.nb_mapillary_jpg = 0
self.nb_mapillary_panos = 0
cmp_err = 0
err_list = []
# Creation of directory
self.mapillary_dir = safe_folder_creation(self.mapillary_dir)
self.mapillary_dir_pano = safe_folder_creation(self.mapillary_dir_pano)
# Convert layer file
ds = ogr.Open(self.layer_path)
layer = ds.GetLayer()
# Determine the number of locations to download
loc_max = len(layer)
if start < end < len(layer):
stop = end
else:
stop = loc_max
n_loc = stop - start
# Create a Mapillary Object
mapillary = Mapillary(mapillary_key)
# Display advancement of downloading
pbar = progressbar.ProgressBar()
for i in pbar(range(start, stop)):
# Get location
feature = layer[i]
lon = feature.GetGeometryRef().GetX()
lat = feature.GetGeometryRef().GetY()
close_to = "{},{}".format(lon, lat)
# Catch metadata search errors
try:
raw_json = mapillary.search_images(closeto=close_to,
per_page=1,
radius=10)
raw_json_pano = mapillary.search_images(closeto=close_to,
per_page=1,
radius=10,
pano="true")
except Exception as err:
print(err)
print("Error during metadata search on feature {}, lat = {}, lon = {} ".format(i, lat, lon))
cmp_err += 1
err_list.append(i)
continue
# Check if there is a image at this location and download it
try:
if len(raw_json['features']):
image_key = raw_json['features'][0]['properties']['key']
image_date = raw_json['features'][0]['properties']['captured_at'][:-5]
image_date = image_date.replace(":", "-")
image_lon = raw_json['features'][0]['geometry']['coordinates'][0]
image_lat = raw_json['features'][0]['geometry']['coordinates'][1]
image_lon = "{0:.6f}".format(image_lon)
image_lat = "{0:.6f}".format(image_lat)
image_filename = '{}_{}_{}_000_{}.jpg'.format(image_lon, image_lat, image_key, image_date)
image_path = os.path.join(self.mapillary_dir, image_filename)
download_image_by_key(image_key, 640, image_path)
self.nb_mapillary_jpg += 1
except Exception as err:
print(err)
print("Error when downloading image on feature {}, lat = {}, lon = {} ".format(i, lat, lon))
cmp_err += 1
err_list.append(i)
continue
# Check if there is a pano at this location and download it
try:
if len(raw_json_pano['features']):
pano_key = raw_json_pano['features'][0]['properties']['key']
pano_date = raw_json_pano['features'][0]['properties']['captured_at'][:-5]
pano_date = pano_date.replace(":", "-")
pano_lon = raw_json_pano['features'][0]['geometry']['coordinates'][0]
pano_lat = raw_json_pano['features'][0]['geometry']['coordinates'][1]
pano_lon = "{0:.6f}".format(pano_lon)
pano_lat = "{0:.6f}".format(pano_lat)
pano_filename = '{}_{}_{}_999_{}.jpg'.format(pano_lon, pano_lat, pano_key, pano_date)
pano_path = os.path.join(self.mapillary_dir_pano, pano_filename)
download_image_by_key(pano_key, 2048, pano_path)
self.nb_mapillary_panos += 1
except Exception as err:
print(err)
print("Error when downloading panorama on feature {}, lat = {}, lon = {} ".format(i, lat, lon))
cmp_err += 1
err_list.append(i)
continue
# Display information
print("Number of locations : {}".format(n_loc))
print("Number of images downloaded : {}".format(self.nb_mapillary_jpg))
print("Number of panoramas downloaded : {}".format(self.nb_mapillary_panos))
print("Number of errors caught : {}".format(cmp_err))
print("List of features index with errors : {}".format(err_list))
