def learning(self, bkg_image):
# PARAMETERS
self.image_parameters = Parameters({"pictures_dim": self.pictures_dim})
data = MosaicImage(bkg_image, self.image_parameters)
inputs_SOM = Parameters({"alpha": Variable(start=self.alpha_start, end=self.alpha_end, nb_steps=self.nb_epochs),
"sigma": Variable(start=self.sigma_start, end=self.sigma_end, nb_steps=self.nb_epochs),
"data": data.get_data(),
"neurons_nbr": self.neurons_nb,
"epochs_nbr": self.nb_epochs})
self.som = SOM(inputs_SOM)
# RUN
for i in range(self.nb_epochs):
# print('Epoch', i)
self.som.run_epoch()
self.initial_map = self.som.get_all_winners()
def runSOM(self):
if self.data is None:
self.load_dataset()
nb_epochs = self.model["nb_epochs"]
if "alpha_start" not in self.model: self.model["alpha_start"] = 0.2
if "alpha_end" not in self.model: self.model["alpha_end"] = 0.05
if "sigma_start" not in self.model: self.model["sigma_start"] = 0.7
if "sigma_end" not in self.model: self.model["sigma_end"] = 0.015
if "nb_images_evals" not in self.dataset:
self.dataset["nb_images_evals"] = 75
parameters = Parameters({
"alpha":
Variable(start=self.model["alpha_start"],
end=self.model["alpha_end"],
nb_steps=nb_epochs),
"sigma":
Variable(start=self.model["sigma_start"],
end=self.model["sigma_end"],
nb_steps=nb_epochs),
"data":
self.data.get_data(),
"neurons_nbr": (self.model["width"], self.model["height"]),
"epochs_nbr":
nb_epochs
})
if self.model["model"] == "standard":
self.map = SOM(parameters)
elif self.model["model"] == "fast":
self.map = FastSOM(parameters)
elif self.model["model"] == "recursive":
self.map = RecursiveSOM(parameters)
else:
print("Error : Unknown model !")
# if "initialisation" not in self.codebooks:
# self.codebooks["initialisation"] = self.som.neurons.tolist()
if "final" in self.codebooks:
self.map.neurons = np.asarray(self.codebooks["final"])
else:
self.map.run()
#for i in range(nb_epochs):
# self.map.run_epoch()
#self.codebooks["final"] = copy.deepcopy(self.map.neurons.tolist())
# for i in range(nb_epochs):
# print("Epoch "+str(i+1))
# if "Epoch "+str(i + 1) not in self.codebooks:
# if self.training_data is not None:
# self.som.data = self.training_data.get_data(self.som.data.shape[0])
# self.som.run_epoch()
# # self.codebooks["Epoch " + str(i + 1)] = copy.deepcopy(self.som.neurons.tolist())
# self.som.run_epoch()
self.map.data = self.data.get_data()
def __init__(self, parameters):
self.data = parameters["data"]
self.level_one = SOM(parameters)
self.level_two = SOM(parameters)
class RecursiveSOM:
def __init__(self, parameters):
self.data = parameters["data"]
self.level_one = SOM(parameters)
self.level_two = SOM(parameters)
def set_data(self, input_data):
self.data = input_data
self.level_one.set_data(self.data)
diff = self.level_one.get_difference()
diff = (np.asarray(diff) + 1) / 2
self.level_two.set_data(diff)
def run(self):
self.level_one.run()
diff = self.level_one.get_difference()
diff = (np.asarray(diff) + 1) / 2
self.level_two.set_data(diff)
self.level_two.run()
def get_reconstructed_data(self, winners=None):
rec_one = np.asarray(self.level_one.get_reconstructed_data(winners))
rec_two = np.asarray(self.level_two.get_reconstructed_data())
rec = rec_one + (rec_two * 2) - 1
for i in rec:
for j in np.ndindex(i.shape):
if i[j] < 0: i[j] = 0
elif i[j] > 1: i[j] = 1
return rec
def get_all_winners(self):
return self.level_one.get_all_winners()
def get_neural_distances(self, old_winners, new_winners):
return self.level_one.get_neural_distances(old_winners, new_winners)
def mean_error(self):
rec = self.get_reconstructed_data()
diff = rec - self.data
error = []
for i in diff:
error.append(np.mean(np.abs(i)))
return np.mean(error)
def square_error(self):
rec = self.get_reconstructed_data()
diff = rec - self.data
error = []
for i in diff:
error.append(np.mean(i**2))
return np.mean(error)
bkg = Image.open(os.path.join(path, name + "0.png"))
img_parameters = Parameters({"pictures_dim": pictures_dim})
data = MosaicImage(bkg, img_parameters)
nb_epochs = 100
parameters = Parameters({
"alpha":
Variable(start=0.6, end=0.05, nb_steps=nb_epochs),
"sigma":
Variable(start=0.5, end=0.001, nb_steps=nb_epochs),
"data":
data.get_data(),
"neurons_nbr": (10, 10),
"epochs_nbr":
nb_epochs
})
map = SOM(parameters)
for i in range(nb_epochs):
print("Epoch " + str(i + 1))
map.run_epoch()
map.data = data.get_data()
output_path = os.path.join("Data", "spikes", "office", "out")
os.makedirs(os.path.join(output_path, "difference"), exist_ok=True)
os.makedirs(os.path.join(output_path, "diff_winners"), exist_ok=True)
os.makedirs(os.path.join(output_path, "saliency"), exist_ok=True)
os.makedirs(os.path.join(output_path, "thresholded"), exist_ok=True)
initial_map = map.get_all_winners()
for i in range(828):
print("Extracting ", i)
pictures_dim = [10, 10]
parameters = Parameters({"pictures_dim": pictures_dim})
data = MosaicImage(bkg, parameters)
nb_epochs = 50
inputs_SOM = Parameters({
"alpha":
Variable(start=0.5, end=0.25, nb_steps=nb_epochs),
"sigma":
Variable(start=0.1, end=0.03, nb_steps=nb_epochs),
"data":
data.get_data(),
"neurons_nbr": (10, 10),
"epochs_nbr":
nb_epochs
})
som = SOM(inputs_SOM)
for i in range(nb_epochs):
print('Epoch ', i)
som.run_epoch()
original = data.image
reconstructed = Image.fromarray(
data.reconstruct(som.get_reconstructed_data()))
som_image = data.reconstruct(som.get_neural_list(), size=som.neurons_nbr)
difference_image = ImageChops.difference(original,
reconstructed).convert('L')
difference = np.asarray(difference_image)
# difference = np.sum(difference, axis=2)
# difference = np.divide(difference, 255*3)
difference = np.divide(difference, 255)
print(np.mean(np.square(difference)))
som_image = Image.fromarray(som_image)
class SaliencyGenerator:
def __init__(self, input_path, output_path, supplements_path, temporal_ROI, mask_ROI, parameters=None):
self.bkg = Image.open(os.path.join(input_path, "bkg.jpg"))
self.mask = mask_ROI
self.input_path = input_path
self.output_path = output_path
self.supplements_path = supplements_path
self.temporal_ROI = temporal_ROI
if parameters is None:
parameters = Parameters()
self.neurons_nb = parameters["neurons_nbr"] if parameters["neurons_nbr"] is not None else (10, 10)
self.pictures_dim = parameters["pictures_dim"] if parameters["pictures_dim"] is not None else [10, 10]
self.nb_epochs = parameters["nb_epochs"] if parameters["nb_epochs"] is not None else 50
self.alpha_start = parameters["alpha_start"] if parameters["alpha_start"] is not None else 0.09
self.alpha_end = parameters["alpha_end"] if parameters["alpha_end"] is not None else 0.22
self.sigma_start = parameters["sigma_start"] if parameters["sigma_start"] is not None else 0.94
self.sigma_end = parameters["sigma_end"] if parameters["sigma_end"] is not None else 0.56
self.threshold = parameters["threshold"] if parameters["threshold"] is not None else 10
self.step = parameters["step"] if parameters["step"] is not None else 30
def generate_all(self):
print("[Learning--", end='', flush=True)
self.learning(self.bkg)
print("--Tracking--", end = '', flush=True)
self.generate(self.input_path, self.output_path, self.supplements_path, self.temporal_ROI)
print("--Finished]", flush=True)
def optimize(self):
os.makedirs(os.path.join(self.supplements_path, "difference"), exist_ok=True)
os.makedirs(os.path.join(self.supplements_path, "diff_winners"), exist_ok=True)
os.makedirs(os.path.join(self.supplements_path, "saliency"), exist_ok=True)
os.makedirs(os.path.join(self.supplements_path, "thresholded"), exist_ok=True)
self.learning(self.bkg)
indexes = range(self.temporal_ROI[0], self.temporal_ROI[1]+1, self.step)
nb_runs = len(indexes)
for i in indexes:
self.extract_image(self.input_path, self.output_path, self.supplements_path, i)
def learning(self, bkg_image):
# PARAMETERS
self.image_parameters = Parameters({"pictures_dim": self.pictures_dim})
data = MosaicImage(bkg_image, self.image_parameters)
inputs_SOM = Parameters({"alpha": Variable(start=self.alpha_start, end=self.alpha_end, nb_steps=self.nb_epochs),
"sigma": Variable(start=self.sigma_start, end=self.sigma_end, nb_steps=self.nb_epochs),
"data": data.get_data(),
"neurons_nbr": self.neurons_nb,
"epochs_nbr": self.nb_epochs})
self.som = SOM(inputs_SOM)
# RUN
for i in range(self.nb_epochs):
# print('Epoch', i)
self.som.run_epoch()
self.initial_map = self.som.get_all_winners()
def generate(self, input_path, output_path, supplements_path, temporal_ROI):
os.makedirs(os.path.join(supplements_path, "difference"), exist_ok=True)
os.makedirs(os.path.join(supplements_path, "diff_winners"), exist_ok=True)
os.makedirs(os.path.join(supplements_path, "saliency"), exist_ok=True)
os.makedirs(os.path.join(supplements_path, "thresholded"), exist_ok=True)
indexes = range(temporal_ROI[0], temporal_ROI[1]+1)
nb_runs = len(indexes)
pool = mp.Pool(8)
a = pool.starmap(self.extract_image, zip(itertools.repeat(input_path, nb_runs), itertools.repeat(output_path, nb_runs),
itertools.repeat(supplements_path, nb_runs), indexes, itertools.repeat(True, nb_runs)))
pool.close()
pool.join()
def extract_image(self, input_path, output_path, supplements_path, image_nb, save=True):
current = Image.open(os.path.join(input_path, "in{0:06d}.jpg".format(image_nb)))
new_data = MosaicImage(current, self.image_parameters)
self.som.set_data(new_data.get_data())
winners = self.som.get_all_winners()
diff_winners = np.zeros(winners.shape)
for j in range(len(winners)):
diff_winners[j] = manhattan_distance(np.asarray(winners[j]), np.asarray(self.initial_map[j]))
diff_winners = diff_winners.reshape(new_data.nb_pictures)
diff_winners = np.kron(diff_winners, np.ones((self.pictures_dim[0], self.pictures_dim[1])))
# diff_winners *= 30 # Use this parameter ?
diff_winners = ImageOps.autocontrast(Image.fromarray(diff_winners).convert('L'))
reconstructed = Image.fromarray(new_data.reconstruct(self.som.get_reconstructed_data(winners)))
som_difference = ImageOps.autocontrast(ImageChops.difference(reconstructed, current).convert('L'))
som_difference_modulated = ImageChops.multiply(som_difference, diff_winners)
# Binarizing
fn = lambda x: 255 if x > self.threshold else 0
thresholded = som_difference_modulated.convert('L').point(fn, mode='1')
result = ImageChops.multiply(thresholded, self.mask)
# Saving
if save:
som_difference.save(os.path.join(supplements_path, "difference", "dif{0:06d}.png".format(image_nb)))
diff_winners.save(os.path.join(supplements_path, "diff_winners", "win{0:06d}.png".format(image_nb)))
som_difference_modulated.save(os.path.join(supplements_path, "saliency", "sal{0:06d}.png".format(image_nb)))
thresholded.save(os.path.join(supplements_path, "thresholded", "thr{0:06d}.png".format(image_nb)))
result.save(os.path.join(output_path, "bin{0:06d}.png".format(image_nb)))