def xavier_weight_initialization(self,
n_out: int,
n_in: int,
uniform: bool = False):
if uniform:
return torch.nn.init.xavier_uniform(
tensor=torch.zeros(int(n_out),
int(n_in),
dtype=torch.float,
requires_grad=True,
device=MyDevice().get()))
return torch.nn.init.xavier_normal_(
tensor=torch.zeros(int(n_out),
int(n_in),
dtype=torch.float,
requires_grad=True,
device=MyDevice().get()))
def __init__(self, layers: list):
self.layers = layers
self.weight = []
self.bias = []
self.momentum = []
self.bias_momentum = []
self.activation_function = []
for i in range(self.number_hidden_layers):
nodes_before = layers[i]
nodes_after = layers[i + 1]
self.weight.append(
self.xavier_weight_initialization(nodes_after, nodes_before))
self.bias.append(self.xavier_weight_initialization(1, nodes_after))
self.momentum.append(
torch.zeros(self.weight[i].shape,
dtype=torch.float,
device=MyDevice().get()))
self.bias_momentum.append(
torch.zeros(self.bias[i].shape,
dtype=torch.float,
device=MyDevice().get()))
self.activation_function.append(self.ACTIVATION_FUNCTION_SIGMOID)
nodes_before = layers[-2]
nodes_after = layers[-1]
self.output_weight = self.xavier_weight_initialization(
nodes_after, nodes_before)
self.output_bias = self.xavier_weight_initialization(1, nodes_after)
self.output_momentum = torch.zeros(self.output_weight.shape,
dtype=torch.float,
device=MyDevice().get())
self.output_bias_momentum = torch.zeros(self.output_bias.shape,
dtype=torch.float,
device=MyDevice().get())
self.output_activation_function = self.ACTIVATION_FUNCTION_SOFTMAX
self.loss_function = self.LOSS_FUNCTION_CROSS_ENTROPY
ElasticNodes.__init__(self, len(self.layers))
def add_element(tensor_data: torch.tensor,
momentum_tensor_data: torch.tensor, n_out: int):
tensor_data = torch.cat(
(tensor_data, self.xavier_weight_initialization(1, n_out)),
axis=1)
momentum_tensor_data = torch.cat(
(momentum_tensor_data,
torch.zeros(
1, n_out, dtype=torch.float, device=MyDevice().get())),
axis=1)
return tensor_data, momentum_tensor_data
def __init__(self, x):
self._number_features = x.shape[1]
self.center = x
self.variance = 0.01 + torch.zeros(int(self._number_features), dtype=torch.float, device=MyDevice().get())
self.variance = self.variance.view(1, self.variance.shape[0])
def compute_overlap_degree(self, gmm_winner_idx, maximum_limit=None, minimum_limit=None):
if maximum_limit is None:
maximum_limit = minimum_limit = 3
elif minimum_limit is None:
minimum_limit = maximum_limit
overlap_coefficient = torch.tensor(1 / self.M(), dtype=torch.float, device=MyDevice().get())
sigma_maximum_winner = maximum_limit * torch.sqrt(self.gmm_array[gmm_winner_idx].variance)
sigma_minimum_winner = minimum_limit * torch.sqrt(self.gmm_array[gmm_winner_idx].variance)
winner_center = self.gmm_array[gmm_winner_idx].center
if maximum_limit == minimum_limit:
mean_positive_sigma_winner = winner_center + sigma_maximum_winner
mean_negative_sigma_winner = winner_center - sigma_minimum_winner
else:
# FIXME This seems wrong
mean_positive_sigma_winner = winner_center + sigma_minimum_winner + sigma_maximum_winner
mean_negative_sigma_winner = winner_center - sigma_minimum_winner - sigma_maximum_winner
mean_positive_sigma = torch.zeros(int(self.M()), int(self.number_features), dtype=torch.float, device=MyDevice().get())
mean_negative_sigma = torch.zeros(int(self.M()), int(self.number_features), dtype=torch.float, device=MyDevice().get())
overlap_mins_mins = torch.zeros(int(self.M()), int(self.number_features), dtype=torch.float, device=MyDevice().get())
overlap_mins_plus = torch.zeros(int(self.M()), int(self.number_features), dtype=torch.float, device=MyDevice().get())
overlap_plus_mins = torch.zeros(int(self.M()), int(self.number_features), dtype=torch.float, device=MyDevice().get())
overlap_plus_plus = torch.zeros(int(self.M()), int(self.number_features), dtype=torch.float, device=MyDevice().get())
overlap_score = []
for i in range(self.M()):
sigma_maximum = maximum_limit * torch.sqrt(self.gmm_array[i].variance)
sigma_minimum = minimum_limit * torch.sqrt(self.gmm_array[i].variance)
if maximum_limit == minimum_limit:
mean_positive_sigma[i] = self.gmm_array[i].center + sigma_maximum
mean_negative_sigma[i] = self.gmm_array[i].center - sigma_maximum
else:
#FIXME This seems wrong
mean_positive_sigma[i] = sigma_maximum + sigma_minimum
mean_negative_sigma[i] = -sigma_minimum - sigma_maximum
overlap_mins_mins[i] = torch.mean(mean_negative_sigma[i] - mean_negative_sigma_winner)
overlap_mins_plus[i] = torch.mean(mean_positive_sigma[i] - mean_negative_sigma_winner)
overlap_plus_mins[i] = torch.mean(mean_negative_sigma[i] - mean_positive_sigma_winner)
overlap_plus_plus[i] = torch.mean(mean_positive_sigma[i] - mean_positive_sigma_winner)
condition1 = (overlap_mins_mins[i] >= 0).all() \
and (overlap_mins_plus[i] >= 0).all() \
and (overlap_plus_mins[i] <= 0).all() \
and (overlap_plus_plus[i] <= 0).all()
condition2 = (overlap_mins_mins[i] <= 0).all() \
and (overlap_mins_plus[i] >= 0).all() \
and (overlap_plus_mins[i] <= 0).all() \
and (overlap_plus_plus[i] >= 0).all()
condition3 = (overlap_mins_mins[i] > 0).all() \
and (overlap_mins_plus[i] > 0).all() \
and (overlap_plus_mins[i] < 0).all() \
and (overlap_plus_plus[i] > 0).all()
condition4 = (overlap_mins_mins[i] < 0).all() \
and (overlap_mins_plus[i] > 0).all() \
and (overlap_plus_mins[i] < 0).all() \
and (overlap_plus_plus[i] < 0).all()
if condition1 or condition2:
# full overlap, the cluster is inside the winning cluster
# the score is full score
overlap_score.append(overlap_coefficient)
elif condition3 or condition4:
# partial overlap, the score is the full score multiplied by the overlap degree
reward = MyUtil.norm_2(self.gmm_array[i].center - self.gmm_array[gmm_winner_idx].center) \
/ MyUtil.norm_2(self.gmm_array[i].center + self.gmm_array[gmm_winner_idx].center) \
+ MyUtil.norm_2(self.gmm_array[i].center - torch.sqrt(self.gmm_array[gmm_winner_idx].variance)) \
/ MyUtil.norm_2(self.gmm_array[i].center + torch.sqrt(self.gmm_array[gmm_winner_idx].variance))
overlap_score.append(overlap_coefficient * reward)
else:
# No overlap, then the score is 0
overlap_score.append(torch.zeros(1))
overlap_score.pop(gmm_winner_idx)
self.rho = torch.sum(torch.stack(overlap_score))
self.rho = torch.min(self.rho, torch.ones_like(self.rho))
self.rho = torch.max(self.rho, torch.ones_like(self.rho) * 0.1) # Do not let rho = zero
def delete_cluster(self):
if self.M() <= 1:
return
accumulated_inference = []
for gmm in self.gmm_array:
if gmm.survive_counter > 0:
accumulated_inference.append(gmm.inference_sum / gmm.survive_counter)
accumulated_inference = torch.stack(accumulated_inference)[torch.isnan(torch.stack(accumulated_inference)) == False]
delete_list = torch.where(accumulated_inference <= (torch.mean(accumulated_inference) - 0.5 * torch.std(accumulated_inference)))[0]
if len(delete_list) == len(self.gmm_array):
raise TypeError('problem') # FIXME if this happen, it means you have a great problem at your code
if len(delete_list):
self.gmm_array = np.delete(self.gmm_array, delete_list.cpu().numpy()).tolist()
accumulated_inference = torch.tensor(np.delete(accumulated_inference.cpu().numpy(), delete_list.cpu().numpy()).tolist(), dtype=torch.float, device=MyDevice().get())
sum_weight = 0
for gmm in self.gmm_array:
sum_weight += gmm.weight
if sum_weight == 0:
max_index = torch.argmax(accumulated_inference)
self.gmm_array[max_index].weight += 1
sum_weight = 0
for gmm in self.gmm_array:
sum_weight += gmm.weight
for gmm in self.gmm_array:
gmm.weight = gmm.weight / sum_weight
def run(self, x, bias2):
self.number_samples_feed += 1
if self.gmm_array is None:
self.gmm_array = [GMM(x)]
self.number_features = x.shape[1]
else:
self.compute_inference(x)
gmm_winner_idx = np.argmax(self.update_weights())
if self.M() > 1:
self.compute_overlap_degree(gmm_winner_idx, 3, 3)
denominator = 1.25 * torch.exp(-bias2) + 0.75 * self.number_features
numerator = 4 - 2 * torch.exp(torch.tensor(-self.number_features / 2.0, dtype=torch.float, device=MyDevice().get()))
threshold = torch.exp(- denominator / numerator)
condition1 = self.gmm_array[gmm_winner_idx].inference < threshold
condition2 = self.gmm_array[gmm_winner_idx].hyper_volume > self.rho * (self.hyper_volume - self.gmm_array[gmm_winner_idx].hyper_volume)
condition3 = self.number_samples_feed > 10
if condition1 and condition2 and condition3:
self.create_cluster(x)
self.gmm_array[-1].variance = (x - self.gmm_array[gmm_winner_idx].center) ** 2
else:
self.update_cluster(x, self.gmm_array[gmm_winner_idx])
def weight_sum(self):
weight_sum = torch.tensor(0, dtype=torch.float, device=MyDevice().get())
for i in range(self.M()):
weight_sum += self.gmm_array[i].weight
return weight_sum
def hyper_volume(self):
hyper_volume = torch.tensor(0, dtype=torch.float, device=MyDevice().get())
for i in range(len(self.gmm_array)):
hyper_volume += self.gmm_array[i].hyper_volume
return hyper_volume
def ATL(epochs: int = 1, n_batch: int = 1000, device='cpu'):
def print_metrics(minibatch, metrics, nn, ae, Xs, Xt):
print(
'Minibatch: %d | Execution time (dataset load/pre-processing + model run): %f'
% (minibatch, time.time() - metrics['start_execution_time']))
if minibatch > 1:
string_max = '' + Fore.GREEN + 'Max' + Style.RESET_ALL
string_mean = '' + Fore.YELLOW + 'Mean' + Style.RESET_ALL
string_min = '' + Fore.RED + 'Min' + Style.RESET_ALL
string_now = '' + Fore.BLUE + 'Now' + Style.RESET_ALL
string_accu = '' + Fore.MAGENTA + 'Accu' + Style.RESET_ALL
print(('Total of samples:' + Fore.BLUE + ' %d Source' +
Style.RESET_ALL + ' |' + Fore.RED + ' %d Target' +
Style.RESET_ALL) % (Xs.shape[0], Xt.shape[0]))
print(
('%s %s %s %s %s Training time:' + Fore.GREEN + ' %f' +
Fore.YELLOW + ' %f' + Fore.RED + ' %f' + Fore.BLUE + ' %f' +
Fore.MAGENTA + ' %f' + Style.RESET_ALL) %
(string_max, string_mean, string_min, string_now, string_accu,
np.max(metrics['train_time']), np.mean(
metrics['train_time']), np.min(metrics['train_time']),
metrics['train_time'][-1], np.sum(metrics['train_time'])))
print(
('%s %s %s %s %s Testing time:' + Fore.GREEN + ' %f' +
Fore.YELLOW + ' %f' + Fore.RED + ' %f' + Fore.BLUE + ' %f' +
Fore.MAGENTA + ' %f' + Style.RESET_ALL) %
(string_max, string_mean, string_min, string_now, string_accu,
np.max(metrics['test_time']), np.mean(
metrics['test_time']), np.min(metrics['test_time']),
metrics['test_time'][-1], np.sum(metrics['test_time'])))
print(
('%s %s %s %s CR Source:' + Fore.GREEN + ' %f%% ' + Back.BLUE +
Fore.YELLOW + Style.BRIGHT + '%f%%' + Style.RESET_ALL +
Fore.RED + ' %f%%' + Fore.BLUE + ' %f%%' + Style.RESET_ALL) %
(string_max, string_mean, string_min, string_now,
np.max(metrics['classification_rate_source']) * 100,
np.mean(metrics['classification_rate_source']) * 100,
np.min(metrics['classification_rate_source']) * 100,
metrics['classification_rate_source'][-1] * 100))
print(
('%s %s %s %s CR Target:' + Fore.GREEN + ' %f%% ' + Back.RED +
Fore.YELLOW + Style.BRIGHT + '%f%%' + Style.RESET_ALL +
Fore.RED + ' %f%%' + Fore.BLUE + ' %f%%' + Style.RESET_ALL) %
(string_max, string_mean, string_min, string_now,
np.max(metrics['classification_rate_target']) * 100,
np.mean(metrics['classification_rate_target']) * 100,
np.min(metrics['classification_rate_target']) * 100,
metrics['classification_rate_target'][-1] * 100))
print(('%s %s %s %s Classification Source Loss:' + Fore.GREEN +
' %f' + Fore.YELLOW + ' %f' + Fore.RED + ' %f' + Fore.BLUE +
' %f' + Style.RESET_ALL) %
(string_max, string_mean, string_min, string_now,
np.max(metrics['classification_source_loss']),
np.mean(metrics['classification_source_loss']),
np.min(metrics['classification_source_loss']),
metrics['classification_source_loss'][-1]))
print(('%s %s %s %s Classification Target Loss:' + Fore.GREEN +
' %f' + Fore.YELLOW + ' %f' + Fore.RED + ' %f' + Fore.BLUE +
' %f' + Style.RESET_ALL) %
(string_max, string_mean, string_min, string_now,
np.max(metrics['classification_target_loss']),
np.mean(metrics['classification_target_loss']),
np.min(metrics['classification_target_loss']),
metrics['classification_target_loss'][-1]))
print(('%s %s %s %s Reconstruction Target Loss:' + Fore.GREEN +
' %f' + Fore.YELLOW + ' %f' + Fore.RED + ' %f' + Fore.BLUE +
' %f' + Style.RESET_ALL) %
(string_max, string_mean, string_min, string_now,
np.max(metrics['reconstruction_target_loss']),
np.mean(metrics['reconstruction_target_loss']),
np.min(metrics['reconstruction_target_loss']),
metrics['reconstruction_target_loss'][-1]))
print(('%s %s %s %s Kullback-Leibler loss 1:' + Fore.GREEN +
' %f' + Fore.YELLOW + ' %f' + Fore.RED + ' %f' + Fore.BLUE +
' %f' + Style.RESET_ALL) %
(string_max, string_mean, string_min, string_now,
np.max(metrics['kl_loss']), np.mean(metrics['kl_loss']),
np.min(metrics['kl_loss']), metrics['kl_loss'][-1]))
print(('%s %s %s %s Nodes:' + Fore.GREEN + ' %d' + Fore.YELLOW +
' %f' + Fore.RED + ' %d' + Fore.BLUE + ' %d' +
Style.RESET_ALL) %
(string_max, string_mean, string_min, string_now,
np.max(metrics['node_evolution']),
np.mean(metrics['node_evolution']),
np.min(metrics['node_evolution']),
metrics['node_evolution'][-1]))
print(
('Network structure:' + Fore.BLUE +
' %s (Discriminative) %s (Generative)' + Style.RESET_ALL) %
(" ".join(map(str, nn.layers)), " ".join(map(str, ae.layers))))
print(Style.RESET_ALL)
metrics = {
'classification_rate_source': [],
'classification_rate_target': [],
'train_time': [],
'test_time': [],
'node_evolution': [],
'classification_target_loss': [],
'classification_source_loss': [],
'reconstruction_source_loss': [],
'reconstruction_target_loss': [],
'kl_loss': [],
'agmm_target_size_by_batch': [],
'agmm_source_size_by_batch': [],
'start_execution_time': time.time()
}
TorchDevice.instance().device = device
dm = DataManipulator('')
dm.load_custom_csv()
dm.normalize()
dm.split_as_source_target_streams(n_batch, 'dallas_2', 0.5)
nn = NeuralNetwork([dm.number_features, 1, dm.number_classes])
ae = DenoisingAutoEncoder([nn.layers[0], nn.layers[1], nn.layers[0]])
# I am building the greedy_layer_bias
x = dm.get_Xs(0)
x = torch.tensor(np.atleast_2d(x),
dtype=torch.float,
device=MyDevice().get())
ae.greedy_layer_wise_pretrain(x=x, number_epochs=0)
# I am building the greedy_layer_bias
agmm_source_discriminative = AGMM()
agmm_target_generative = AGMM()
for i in range(dm.number_minibatches):
Xs = torch.tensor(dm.get_Xs(i),
dtype=torch.float,
device=MyDevice().get())
ys = torch.tensor(dm.get_ys(i),
dtype=torch.float,
device=MyDevice().get())
Xt = torch.tensor(dm.get_Xt(i),
dtype=torch.float,
device=MyDevice().get())
yt = torch.tensor(dm.get_yt(i),
dtype=torch.float,
device=MyDevice().get())
if i > 0:
metrics['test_time'].append(time.time())
test(nn,
Xt,
yt,
is_source=False,
is_discriminative=True,
metrics=metrics)
metrics['test_time'][-1] = time.time() - metrics['test_time'][-1]
test(nn,
Xs,
ys,
is_source=True,
is_discriminative=True,
metrics=metrics)
test(ae,
Xt,
is_source=False,
is_discriminative=False,
metrics=metrics)
metrics['train_time'].append(time.time())
for epoch in range(epochs):
for x, y in [(x.view(1, x.shape[0]), y.view(1, y.shape[0]))
for x, y in zip(Xs, ys)]:
width_evolution(network=nn,
x=x,
y=y,
agmm=agmm_source_discriminative,
train_agmm=True if epoch == 1 else False)
if not grow_nodes(nn, ae): prune_nodes(nn, ae)
discriminative(network=nn,
x=x,
y=y,
agmm=agmm_source_discriminative)
copy_weights(source=nn, target=ae, layer_numbers=[1])
for x in [x.view(1, x.shape[0]) for x in Xt]:
width_evolution(network=ae,
x=x,
agmm=agmm_target_generative,
train_agmm=True if epoch == 1 else False)
if not grow_nodes(ae, nn): prune_nodes(ae, nn)
generative(network=ae, x=x, agmm=agmm_target_generative)
metrics['kl_loss'].append(kl(ae=ae, x_source=Xs, x_target=Xt))
copy_weights(source=ae, target=nn, layer_numbers=[1])
if agmm_target_generative.M() > 1:
agmm_target_generative.delete_cluster()
if agmm_source_discriminative.M() > 1:
agmm_source_discriminative.delete_cluster()
metrics['agmm_target_size_by_batch'].append(agmm_target_generative.M())
metrics['agmm_source_size_by_batch'].append(
agmm_source_discriminative.M())
metrics['train_time'][-1] = time.time() - metrics['train_time'][-1]
metrics['node_evolution'].append(nn.layers[1])
print_metrics(i + 1, metrics, nn, ae, Xs, Xt)
result = '%f (T) ' '| %f (S) \t %f | %d \t %f | %f' % (
np.mean(metrics['classification_rate_target']),
np.mean(metrics['classification_rate_source']),
np.mean(metrics['node_evolution']), metrics['node_evolution'][-1],
np.mean(metrics['train_time']), np.sum(metrics['train_time']))
print(result)
plot_time(metrics['train_time'], metrics['test_time'])
plot_node_evolution(metrics['node_evolution'])
plot_classification_rates(metrics['classification_rate_source'],
metrics['classification_rate_target'])
return result
def masking_noise(self, x: torch.tensor, noise_ratio=0.0):
return x.clone().masked_fill(
torch.rand(x.shape, device=MyDevice().get()) <= noise_ratio, 0)