def predict_using_pretrained_model(MP):
"""Top-level function that loads test data files and evaluates pretrained model on them.
Parameters
----------
MP : dict
Model Parameters.
"""
dir = MP['dir']
n_test = MP['n_test']
# pdb.set_trace()
# Load data
test = h5py.File('%smars_images_5k.hdf5' % dir, 'r')
Data = {
'test': [
test['input_images'][:n_test].astype('float32'),
test['target_masks'][:n_test].astype('float32')
]
}
print('printing dictionary : ###########################\n\n')
for key in Data:
for i in range(len(Data[key])):
print(key, i, type(Data[key][i]), Data[key][i].shape)
test.close()
# Rescale, normalize, add extra dim
proc.preprocess(Data)
# Load ground-truth craters
Craters = {'test': pd.HDFStore('%smars_craters.hdf5' % dir, 'r')}
# evaluate the model
test_model(Data, Craters, MP, i)
def get_model_preds(CP):
"""Reads in or generates model predictions.
Parameters
----------
CP : dict
Containins directory locations for loading data and storing
predictions.
Returns
-------
craters : h5py
Model predictions.
"""
n_imgs, dtype = CP['n_imgs'], CP['datatype']
data = h5py.File(CP['dir_data'], 'r')
Data = {
dtype: [data['input_images'][:n_imgs].astype('float32'),
data['target_masks'][:n_imgs].astype('float32')]
}
data.close()
proc.preprocess(Data)
model = load_model(CP['dir_model'])
preds = model.predict(Data[dtype][0])
# save
h5f = h5py.File(CP['dir_preds'], 'w')
h5f.create_dataset(dtype, data=preds)
print("Successfully generated and saved model predictions.")
return preds ,Data[dtype]
def main():
# pdb.set_trace()
zenodo_path = './mars_data/'
model = load_model('models/model_30k.h5')
#### TEST CRATERS
test_imgs = h5py.File(zenodo_path + '/mars_images_5k.hdf5', 'r')
test_data = {
'imgs': [
test_imgs['input_images'][...].astype('float32'),
test_imgs['target_masks'][...].astype('float32')
]
}
proc.preprocess(test_data)
sd_input_images = test_data['imgs'][0]
sd_target_masks = test_data['imgs'][1]
images = [25, 36]
plot_dir = "plots"
for iwant in images:
pred = model.predict(sd_input_images[iwant:iwant + 1])
extracted_rings = tmt.template_match_t(
pred[0].copy(), minrad=2.) # x coord, y coord, radius
fig = plt.figure(figsize=[16, 16])
plt.rcParams["font.size"] = 20
[[ax1, ax4], [ax2, ax3]] = fig.subplots(2, 2)
ax1.imshow(sd_input_images[iwant].squeeze(),
origin='upper',
cmap='Greys_r',
vmin=0,
vmax=1.1)
ax2.imshow(sd_target_masks[iwant].squeeze(),
origin='upper',
cmap='Greys_r')
ax3.imshow(pred[0], origin='upper', cmap='Greys_r', vmin=0, vmax=1)
ax4.imshow(sd_input_images[iwant].squeeze(),
origin='upper',
cmap="Greys_r")
for x, y, r in extracted_rings:
circle = plt.Circle((x, y),
r,
color='blue',
fill=False,
linewidth=2,
alpha=0.5)
ax4.add_artist(circle)
ax1.set_title('Mars DEM Image')
ax2.set_title('Ground-Truth Target Mask')
ax3.set_title('CNN Predictions')
ax4.set_title('Post-CNN Craters')
current_dir = os.path.dirname(os.path.abspath(__file__))
plot_location = os.path.join(
current_dir,
plot_dir + "/trained_model_results" + str(iwant) + ".png")
plt.savefig(plot_location)
def get_images_data_from_path(data_path):
images = h5py.File(data_path + '/train_images.hdf5', 'r')
sample_data = {'imgs': [images['input_images'][...].astype('float32'),
images['target_masks'][...].astype('float32')]}
proc.preprocess(sample_data)
sd_input_images = sample_data['imgs'][0]
sd_target_masks = sample_data['imgs'][1]
ctrs = pd.HDFStore(data_path + '/train_craters.hdf5', 'r')
return [sd_input_images,ctrs]
def main():
# pdb.set_trace()
path = './input_data/'
model = load_model('models/model_keras1.2.2.h5')
#### TEST CRATERS
test_imgs = h5py.File(path + '/Mercury_images.hdf5', 'r')
images = test_imgs['input_images'][:100, :, :].astype('float32')
test_data = {'imgs': [images[np.sum(images, axis=(1, 2)) > 0]]}
#for img in test_data['imgs'][0]:
# print(np.sum(img), img[img > 0].shape)
proc.preprocess(test_data)
sd_input_images = test_data['imgs'][0]
print(len(sd_input_images))
images = [2, 10, 20, 44]
plot_dir = "plots"
for iwant in images:
print("Predicting on image", iwant)
pred = model.predict(sd_input_images[iwant:iwant + 1])
extracted_rings = tmt.template_match_t(
pred[0].copy(), minrad=2.) # x coord, y coord, radius
fig = plt.figure(figsize=[9, 9])
[ax1, ax2, ax3] = fig.subplots(1, 3)
ax1.imshow(sd_input_images[iwant].squeeze(),
origin='upper',
cmap='Greys_r',
vmin=0,
vmax=1.1)
ax2.imshow(pred[0], origin='upper', cmap='Greys_r', vmin=0, vmax=1)
ax3.imshow(sd_input_images[iwant].squeeze(),
origin='upper',
cmap="Greys_r")
for x, y, r in extracted_rings:
circle = plt.Circle((x, y),
r,
color='blue',
fill=False,
linewidth=2,
alpha=0.5)
ax3.add_artist(circle)
ax1.set_title('Mercury DEM Image')
ax2.set_title('CNN Predictions')
ax3.set_title('Post-CNN Craters')
current_dir = os.path.dirname(os.path.abspath(__file__))
plot_location = os.path.join(
current_dir,
plot_dir + "/trained_model_results_Mercury" + str(iwant) + ".png")
plt.savefig(plot_location)
def get_models(MP):
"""Top-level function that loads data files and calls train_and_test_model.
Parameters
----------
MP : dict
Model Parameters.
"""
dir = MP['dir']
n_train, n_dev, n_test = MP['n_train'], MP['n_dev'], MP['n_test']
# Load data
train = h5py.File('%strain_images.hdf5' % dir, 'r')
dev = h5py.File('%sdev_images.hdf5' % dir, 'r')
test = h5py.File('%stest_images.hdf5' % dir, 'r')
Data = {
'train': [
train['input_images'][:n_train].astype('float32'),
train['target_masks'][:n_train].astype('float32')
],
'dev': [
dev['input_images'][:n_dev].astype('float32'),
dev['target_masks'][:n_dev].astype('float32')
],
'test': [
test['input_images'][:n_test].astype('float32'),
test['target_masks'][:n_test].astype('float32')
]
}
print('printing dictionary : ###########################\n\n')
for key in Data:
for i in range(len(Data[key])):
print(key, i, type(Data[key][i]), Data[key][i].shape)
train.close()
dev.close()
test.close()
# Rescale, normalize, add extra dim
proc.preprocess(Data)
# Load ground-truth craters
Craters = {
'train': pd.HDFStore('%strain_craters.hdf5' % dir, 'r'),
'dev': pd.HDFStore('%sdev_craters.hdf5' % dir, 'r'),
'test': pd.HDFStore('%stest_craters.hdf5' % dir, 'r')
}
# Iterate over parameters
for i in range(MP['N_runs']):
train_and_test_model(Data, Craters, MP, i)
def forward(self, image_paths):
x, img_sizes, img_scales = preprocess(image_paths)
cls_outs, box_outs = self.model(x.to(self.device))
cls_outs, box_outs, indices, classes = postprocess(cls_outs, box_outs)
batch_detections = generate_detections(cls_outs, box_outs,
self.anchor_boxes, indices,
classes, img_sizes, img_scales,
cfg.MAX_DETECTIONS_PER_IMAGE)
return batch_detections
def forward(self, image_paths, image_ids=None):
x, img_ids, image_scales = preprocess(image_paths, image_ids)
cls_outs, box_outs = self.model(x.to(self.device))
cls_outs, box_outs, indices, classes = postprocess(cls_outs, box_outs)
batch_detections = []
cls_outs = cls_outs.cpu().numpy()
box_outs = box_outs.cpu().numpy()
if self._anchor_cache is None:
anchor_boxes = self.anchors.boxes.cpu().numpy()
self._anchor_cache = anchor_boxes
else:
anchor_boxes = self._anchor_cache
indices = indices.cpu().numpy()
classes = classes.cpu().numpy()
for i in range(x.shape[0]):
detections = generate_detections(cls_outs[i], box_outs[i],
anchor_boxes, indices[i],
classes[i], image_ids[i],
image_scales[i], cfg.NUM_CLASSES)
batch_detections.append(detections)
return batch_detections
print("""absolute maximum detected pixel radius over all images =
%f""" % np.max(maxrad))
print("")
if __name__ == '__main__':
n_dev = 3000
n_test = 3000
dir = 'catalogues/'
dev = h5py.File('%sdev_images.hdf5' % dir, 'r')
test = h5py.File('%stest_images.hdf5' % dir, 'r')
Data = {
'dev': [
dev['input_images'][:n_dev].astype('float32'),
dev['target_masks'][:n_dev].astype('float32')
],
'test': [
test['input_images'][:n_test].astype('float32'),
test['target_masks'][:n_test].astype('float32')
]
}
proc.preprocess(Data)
Craters = {
'dev': pd.HDFStore('%sdev_craters.hdf5' % dir, 'r'),
'test': pd.HDFStore('%stest_craters.hdf5' % dir, 'r')
}
model = load_model('models/model_deepResUnet_30000_1.h5')
get_metrics(Data['dev'], Craters['dev'], 256, model)
import pandas as pd
import h5py
from keras.models import load_model
import sys
sys.path.append("/mnt/disks/disk0/deep_moon_working_dir/DeepCrater/utils")
sys.path.append("/mnt/disks/disk0/deep_moon_working_dir/DeepCrater")
import score_utils as sc
import utils.processing as proc
model = load_model("/mnt/disks/disk0/deep_moon_working_dir/data/Silburt/model_keras2.h5")
data_path = '/mnt/disks/disk0/deep_moon_working_dir/data/my_test_data'
images = h5py.File(data_path + '/train_images.hdf5', 'r')
sample_data = {'imgs': [images['input_images'][...].astype('float32'),
images['target_masks'][...].astype('float32')]}
proc.preprocess(sample_data)
sd_input_images = sample_data['imgs'][0]
sd_target_masks = sample_data['imgs'][1]
ctrs = pd.HDFStore(data_path + '/train_craters.hdf5', 'r')
def get_craters_from_database(ctrs, image_id):
my_craters = ctrs["/img_{:05d}".format(image_id)]
labeled_craters = []
for index, row in my_craters.iterrows():
labeled_craters.append([int(round(row['x'])), int(round(row['y'])), int(round(row['Diameter (pix)'])/2)])
return labeled_craters
from timeit import default_timer as timer
import math
import numpy as np
import template_match_target as tmt
def main():
pdb.set_trace()
zenodo_path = './data/'
deepmoon_path = os.path.dirname(os.getcwd())
train_imgs = h5py.File(zenodo_path + 'train_images.hdf5', 'r')
fig = plt.figure(figsize=[12, 6])
[ax1, ax2] = fig.subplots(1, 2)
ax1.imshow(train_imgs['input_images'][3][...],
origin='upper',
cmap='Greys_r',
vmin=120,
vmax=200)
ax2.imshow(train_imgs['target_masks'][3][...],
origin='upper',
cmap='Greys_r')
plt.show()
ctrs = pd.HDFStore(zenodo_path + 'train_craters.hdf5', 'r')
# pdb.set_trace()
Image.MAX_IMAGE_PIXELS = None
img = Image.open(zenodo_path +
"/LunarLROLrocKaguya_118mperpix.png").convert("L")
# Read and combine the LROC and Head datasets (stored under ../catalogues)
craters = igen.ReadLROCHeadCombinedCraterCSV(
filelroc="catalogues/LROCCraters.csv",
filehead="catalogues/HeadCraters.csv")
# Generate 100 image/target sets, and corresponding crater dataframes. np.random.seed is set for consistency.
igen.GenDataset(img,
craters,
zenodo_path + '/test_zenodo',
amt=25,
seed=1337)
gen_imgs = h5py.File(zenodo_path + '/test_zenodo_images.hdf5', 'r')
sample_data = {
'imgs': [
gen_imgs['input_images'][...].astype('float32'),
gen_imgs['target_masks'][...].astype('float32')
]
}
proc.preprocess(
sample_data
) #now, input images is shape 25 x 256 x 256 x 1, target_masks is shape 25 x 256 x 256
sd_input_images = sample_data['imgs'][0] #25 x 256 x 256 x 1
sd_target_masks = sample_data['imgs'][1] #25 x 256 x 256p
# Plot the data for fun.
fig = plt.figure(figsize=[12, 6])
[ax1, ax2] = fig.subplots(1, 2)
ax1.imshow(sd_input_images[5].squeeze(),
origin='upper',
cmap='Greys_r',
vmin=0,
vmax=1.1)
ax2.imshow(sd_target_masks[5].squeeze(), origin='upper', cmap='Greys_r')
plt.savefig("plots/processed_image_and_mask1.png")
fig = plt.figure(figsize=[12, 6])
[ax1, ax2] = fig.subplots(1, 2)
ax1.imshow(sd_input_images[8].squeeze(),
origin='upper',
cmap='Greys_r',
vmin=0,
vmax=1.1)
ax2.imshow(sd_target_masks[8].squeeze(), origin='upper', cmap='Greys_r')
plt.savefig("plots/processed_image_and_mask2.png")
fig = plt.figure(figsize=[12, 6])
[ax1, ax2] = fig.subplots(1, 2)
ax1.imshow(sd_input_images[12].squeeze(),
origin='upper',
cmap='Greys_r',
vmin=0,
vmax=1.1)
ax2.imshow(sd_target_masks[12].squeeze(), origin='upper', cmap='Greys_r')
plt.savefig("plots/processed_image_and_mask3.png")
fig = plt.figure(figsize=[12, 6])
[ax1, ax2] = fig.subplots(1, 2)
ax1.imshow(sd_input_images[16].squeeze(),
origin='upper',
cmap='Greys_r',
vmin=0,
vmax=1.1)
ax2.imshow(sd_target_masks[16].squeeze(), origin='upper', cmap='Greys_r')
plt.savefig("plots/processed_image_and_mask4.png")
sample_images = train_imgs['input_images'][0:20]
sample_masks = train_imgs['target_masks'][0:20]
pdb.set_trace()
model = load_model('models/model_30k.h5')
#### TEST CRATERS
test_imgs = h5py.File(zenodo_path + '/test_images.hdf5', 'r')
test_data = {
'imgs': [
test_imgs['input_images'][...].astype('float32'),
test_imgs['target_masks'][...].astype('float32')
]
}
proc.preprocess(test_data)
sd_input_images = test_data['imgs'][0]
sd_target_masks = test_data['imgs'][1]
iwant = 3
pred = model.predict(sd_input_images[iwant:iwant + 1])
extracted_rings = tmt.template_match_t(
pred[0].copy(), minrad=2.) # x coord, y coord, radius
fig = plt.figure(figsize=[16, 16])
[[ax1, ax2], [ax3, ax4]] = fig.subplots(2, 2)
ax1.imshow(sd_input_images[iwant].squeeze(),
origin='upper',
cmap='Greys_r',
vmin=0,
vmax=1.1)
ax2.imshow(sd_target_masks[iwant].squeeze(),
origin='upper',
cmap='Greys_r')
ax3.imshow(pred[0], origin='upper', cmap='Greys_r', vmin=0, vmax=1)
ax4.imshow(sd_input_images[iwant].squeeze(),
origin='upper',
cmap="Greys_r")
for x, y, r in extracted_rings:
circle = plt.Circle((x, y),
r,
color='blue',
fill=False,
linewidth=2,
alpha=0.5)
ax4.add_artist(circle)
ax1.set_title('Moon DEM Image')
ax2.set_title('Ground-Truth Target Mask')
ax3.set_title('CNN Predictions')
ax4.set_title('Post-CNN Craters')
plt.savefig("plots/trained_model_results1.png")
iwant = 6
pred = model.predict(sd_input_images[iwant:iwant + 1])
extracted_rings = tmt.template_match_t(
pred[0].copy(), minrad=2.) # x coord, y coord, radius
fig = plt.figure(figsize=[16, 16])
[[ax1, ax2], [ax3, ax4]] = fig.subplots(2, 2)
ax1.imshow(sd_input_images[iwant].squeeze(),
origin='upper',
cmap='Greys_r',
vmin=0,
vmax=1.1)
ax2.imshow(sd_target_masks[iwant].squeeze(),
origin='upper',
cmap='Greys_r')
ax3.imshow(pred[0], origin='upper', cmap='Greys_r', vmin=0, vmax=1)
ax4.imshow(sd_input_images[iwant].squeeze(),
origin='upper',
cmap="Greys_r")
for x, y, r in extracted_rings:
circle = plt.Circle((x, y),
r,
color='blue',
fill=False,
linewidth=2,
alpha=0.5)
ax4.add_artist(circle)
ax1.set_title('Moon DEM Image')
ax2.set_title('Ground-Truth Target Mask')
ax3.set_title('CNN Predictions')
ax4.set_title('Post-CNN Craters')
plt.savefig("plots/trained_model_results2.png")
iwant = 13
pred = model.predict(sd_input_images[iwant:iwant + 1])
extracted_rings = tmt.template_match_t(
pred[0].copy(), minrad=2.) # x coord, y coord, radius
fig = plt.figure(figsize=[16, 16])
[[ax1, ax2], [ax3, ax4]] = fig.subplots(2, 2)
ax1.imshow(sd_input_images[iwant].squeeze(),
origin='upper',
cmap='Greys_r',
vmin=0,
vmax=1.1)
ax2.imshow(sd_target_masks[iwant].squeeze(),
origin='upper',
cmap='Greys_r')
ax3.imshow(pred[0], origin='upper', cmap='Greys_r', vmin=0, vmax=1)
ax4.imshow(sd_input_images[iwant].squeeze(),
origin='upper',
cmap="Greys_r")
for x, y, r in extracted_rings:
circle = plt.Circle((x, y),
r,
color='blue',
fill=False,
linewidth=2,
alpha=0.5)
ax4.add_artist(circle)
ax1.set_title('Moon DEM Image')
ax2.set_title('Ground-Truth Target Mask')
ax3.set_title('CNN Predictions')
ax4.set_title('Post-CNN Craters')
plt.savefig("plots/trained_model_results3.png")
