def evaluate(net, valid_loader, device):
return cnn.predict(
model=net,
dataset=valid_loader,
device=device,
valid=True,
get_loss=True,
)
if "Dropout" in group:
net_copy.dropout = 0.2
if "Neural Noise" in group:
net_copy.neural_noise = (-0.2, 0.5)
net_copy.to(device)
parameters = group_config[group]
log, _ = cnn.train_model(
model=net_copy,
dataset=data_loader,
dtype=dtype,
device=device,
**parameters,
)
accuracy = cnn.predict(net_copy,
data_loader,
device=device,
valid=False)
train_accuracy = cnn.predict(net_copy, data_loader, train=True)
print(group, accuracy)
results.iloc[counter] = [
group, train_accuracy["Accuracy"][0], accuracy["Accuracy"][0], i
]
counter += 1
results.to_csv(EXP_RESULTS / "c10compare_best_params-{}-{}-{}-{}.csv".format(
START_TIME, NEUROGENESIS, FREQUENCY, TURNOVER))
print("Average Accuracy")
print(results.groupby("Group")["Test Accuracy"].mean())
print("Done")
print(f"Running trial {i+1}/{REPEATS}")
net = cnn.NgnCnn(args.neurons, seed=i, excite=args.excite)
for group in group_config:
net_copy = copy.deepcopy(net)
net_copy.to(device)
parameters = group_config[group]
log = cnn.train_model(
model=net_copy,
dataset=data_loader,
dtype=dtype,
device=device,
**parameters,
)
# accuracy['Accuracy'][0]
accuracy = cnn.predict(net_copy, data_loader)['Accuracy'][0]
start = cnn.predict(net_copy, data_loader, train=True)['Accuracy'][0]
ablate = cnn.ablation(net_copy, data_loader, mode=mode)
results = pd.DataFrame(ablate,
columns=['Proportion', 'Ablation Accuracy'])
end = results.iloc[-1]['Ablation Accuracy']
results['Group'] = group
results['Normalized Ablation Accuracy'] = (
results['Ablation Accuracy'] - end) / (start - end)
results['Test Accuracy'] = accuracy
dfs.append(results)
final = pd.concat(dfs)
final.to_csv(EXP_RESULTS / "{}.csv".format(RUN_SCRIPT_NAME))
# DATASET
data_loader = cnn.Cifar10_data(mode="test", batch_size=BATCH_SIZE)
for i in range(REPEATS):
print(f"Running trial {i+1}/{REPEATS}")
net = cnn.NgnCnn(args.neurons, seed=i)
for group in group_config:
net_copy = copy.deepcopy(net)
net_copy.to(device)
parameters = group_config[group]
log, _ = cnn.train_model(
model=net_copy,
dataset=data_loader,
dtype=dtype,
device=device,
**parameters,
)
accuracy = cnn.predict(net_copy,
data_loader,
device=device,
valid=False)
results.iloc[counter] = log + [group, accuracy["Accuracy"][0], i]
counter += 1
results.to_csv(EXP_RESULTS / "c10-compare_targeted-{}.csv".format(START_TIME))
print("Average Accuracy")
print(results.groupby("Group")["Test Accuracy"].mean())
print("Done")