def test_cost_function(train_data, test_data):
x, y, y_onehot, = train_data
x_test, y_test = test_data
network = NeuralNetwork(layers=[784, 25, 10])
network.fit(x, y_onehot, alpha=0.1, iterations=40)
prediction_test = network.predict(x_test)
accuracy_test = accuracy(prediction_test, y_test)
assert accuracy_test > 90
def test_network_feed_forward_computes_units_by_layer():
network = NeuralNetwork(layers=[2, 3, 2])
network.weights = [np.ones((3, 3)), np.ones((2, 4))]
a_, z_ = network.feed_forward([4, 6])
assert len(z_) == len(a_)
assert len(z_) == network.num_layers
assert len(a_[0]) == 2
assert len(z_[1]) == len(a_[1])
assert len(z_[1]) == 3
assert len(z_[2]) == len(a_[2])
assert len(z_[2]) == 2
def test_network_backpropagation():
network = NeuralNetwork(layers=[2, 3, 1])
network.weights = [
np.array([[1, 1, 2], [1, 2, 1], [1, 1, 1]]),
np.ones((1, 4))
]
x = np.array([[2, 3], [1, 1]])
y = np.array([2, 1]).reshape(2, 1)
gradients = network.backpropagation(x, y)
assert len(gradients) == network.num_layers - 1
assert np.shape(gradients[0]) == (3, 3)
assert np.shape(gradients[1]) == (1, 4)
def test_network_feed_forward_computes_hidden_units():
network = NeuralNetwork(layers=[2, 3, 2])
network.weights = [
np.array([[1, 1, 2], [1, 2, 1], [1, 1, 1]]),
np.ones((2, 4))
]
a_, z_ = network.feed_forward(np.array([2, 3]))
np.testing.assert_array_equal(a_[0], np.array([2, 3]))
np.testing.assert_array_equal(z_[1], np.array([9, 8, 6]))
np.testing.assert_array_equal(
a_[1], np.array([sigmoid(9), sigmoid(8),
sigmoid(6)]))
value = sigmoid(9) + sigmoid(8) + sigmoid(
6) + 1 # weights of theta2 are 1's
np.testing.assert_array_equal(z_[2], np.array([value, value]))
def test_pbgn(seed, reuse_block_inds):
"""
Test the Generalised Newton's method for optimisation, using parallel
block-diagonal approximations.
TODO: combine this and the gradient descent test (and any optimisers
implemented in future, EG adam, PSO) in to a single parametrised test
"""
# Set the random seed
np.random.seed(seed)
# Generate random number of iterations, network, data, and results file
n_iters = np.random.randint(10, 20)
n = NeuralNetwork(input_dim=1,
output_dim=1,
num_hidden_units=[4, 8, 6],
act_funcs=[activations.gaussian, activations.identity])
sin_data = data.Sinusoidal(input_dim=1, output_dim=1, freq=1)
name = "Test PBGN without line-search, reuse_block_inds={}".format(
reuse_block_inds)
results_filename = "{}.txt".format(name)
results_path = os.path.join(output_dir, results_filename)
results_file = open(results_path, "w")
result = optimisers.Result(name=name, verbose=True, file=results_file)
# Call gradient descent function
result_ls = optimisers.generalised_newton(
n,
sin_data,
terminator=optimisers.Terminator(i_lim=n_iters),
evaluator=optimisers.Evaluator(i_interval=1),
result=result,
reuse_block_inds=reuse_block_inds)
results_file.close()
def test_network_initialization():
network = NeuralNetwork(layers=[400, 25, 10])
assert network.num_layers == 3
assert network.layers[0] == 400
assert network.layers[1] == 25
assert network.layers[2] == 10
assert network.weights != []
def comparing_models(X_train, X_test, y_train, y_test):
AdaBoost(X_train, X_test, y_train, y_test)
Logistic_Regression(X_train, X_test, y_train, y_test)
NaiveBayes(X_train, X_test, y_train, y_test)
XGBoost(X_train, X_test, y_train, y_test)
RandomForest(X_train, X_test, y_train, y_test)
SVM(X_train, X_test, y_train, y_test)
NeuralNetwork(X_train, X_test, y_train, y_test)
def test_network_random_initialization_of_weights_by_layer():
network = NeuralNetwork(layers=[400, 25, 10])
weights_layer1 = network.weights[0]
weights_layer2 = network.weights[1]
init_epsilon1 = np.sqrt(6) / np.sqrt(425)
init_epsilon2 = np.sqrt(6) / np.sqrt(35)
assert np.shape(weights_layer1) == (25, 401)
assert np.shape(weights_layer2) == (10, 26)
assert are_values_in_range(weights_layer1, -init_epsilon1, init_epsilon1)
assert are_values_in_range(weights_layer2, -init_epsilon2, init_epsilon2)
def predict(self):
last_results = self.read_last_results()
configuration = last_results['configuration']
X_shape = len(configuration[0][0]['weights'])
Y_shape = len(configuration[-1])
network = NeuralNetwork(configuration, self.hidden_layers, X_shape, Y_shape, self.activation_func, self.dx_activation_func)
self.interactive_predict(network, X_shape)
def run_experiment(dataset, num_units, num_layers, log10_learning_rate,
max_block_size, log10_max_step, reuse_block_inds, act_func):
n = NeuralNetwork(input_dim=1,
output_dim=1,
num_hidden_units=[num_units for _ in range(num_layers)],
act_funcs=[act_func, activations.Identity()])
result = optimisers.generalised_newton(
n,
dataset,
learning_rate=pow(10, log10_learning_rate),
max_block_size=max_block_size,
max_step=pow(10, log10_max_step),
terminator=optimisers.Terminator(t_lim=3),
evaluator=optimisers.Evaluator(t_interval=0.1),
line_search=None)
return result
def train_net(train_loader):
lr = .0001
PATH = 'models/protein_net_{}_lr_{:.6f}.pth'.format(train_timestr, lr)
net = NeuralNetwork.NeuralNetwork()
criterion = nn.BCELoss()
optimizer = optim.Adam(net.parameters(), lr=lr)
lambda1 = lambda epoch: max(0.35**epoch, .0001)
scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=lambda1)
for epoch in range(2): # loop over the dataset multiple times
running_loss = 0.0
for i, data in enumerate(train_loader, 0):
# get the inputs; data is a list of [inputs, labels]
image, labels = data['image'].unsqueeze(1), data['localizations']
# zero the parameter gradients
optimizer.zero_grad()
# forward + backward + optimize
outputs = net(image)
loss = criterion(outputs, labels)
loss.backward()
optimizer.step()
# print statistics
running_loss += loss.item()
if i % 20 == 19: # print every 20 mini-batches
print('[%d, %5d] loss: %.3f lr: %.6f' %
(epoch + 1, i + 1, running_loss / 20,
optimizer.param_groups[0]["lr"]))
running_loss = 0.0
if i % 200 == 199:
print("saved model")
scheduler.step()
torch.save(net.state_dict(), PATH)
print('Finished Training')
torch.save(net.state_dict(), PATH)
def practica3():
x, y = rf.read_img()
theta1, theta2 = rf.read_weight()
np.place(y, y == 10, 0) # sustituimos los 10 con el 0, y.shape = (5000, 0)
num_etiquetas = 10
reg = np.zeros(shape=(num_etiquetas, x.shape[1])) # x.shape = (5000, 401)
for i in range(0, num_etiquetas):
theta = np.zeros((x.shape[1], 1)) #theta.shape = (401,)
reg = rl.oneVsAll(x, y, i, reg, theta)
prediction = rl.predOneVsAll(reg, x)
sumPrediction = sum(prediction[:,np.newaxis] == y)[0] / 5000 * 100
print("Accuracy Regression Log Multiple Clase: {}%".format(sumPrediction))
prediction_fp = nn.forwardPropagation(x, theta1, theta2)
sumPrediction = sum(prediction_fp[:,np.newaxis] == y) / prediction_fp.shape[0] * 100
print("Accuracy Forward Propagation: {}%".format(sumPrediction))
def main():
# get directory from path
directory = os.path.dirname('../models/file.h5')
# checks if directory exits
if not os.path.exists(directory):
# if it does not then create it
print('Creating directory \'../models/\'')
os.makedirs(directory)
# check number of arguments passed to script is correct
if len(sys.argv) != 3:
# helpful messages to help user
print('Error, was expecting two arguments: <model_type> <model_name>')
print('Found:', len(sys.argv) - 1)
return
# checks if first argument is a valid model_type
if sys.argv[1] == 'nn':
# save neural network model with filename as second argument
save(NeuralNetwork(), sys.argv[2])
print('File successfully saved at \'../models/' + sys.argv[2] +
'.h5\'')
elif sys.argv[1] == 'lstm':
# save lstm network model with filename as second argument
save(LSTMNetwork(), sys.argv[2])
print('File successfully saved at \'../models/' + sys.argv[2] +
'.h5\'')
elif sys.argv[1] == 'cnn':
# save cnn network model with filename as second argument
save(CNNNetwork(), sys.argv[2])
print('File successfully saved at \'../models/' + sys.argv[2] +
'.h5\'')
elif sys.argv[1] == 'cnn2':
# save cnn network model with filename as second argument
save(CNNNetwork2(), sys.argv[2])
print('File successfully saved at \'../models/' + sys.argv[2] +
'.h5\'')
else:
# first argument is not valid
# message displays list of possible model_types
print('Error, <model_type> was expecting: \'nn\', \'lstm\', \'cnn\'')
print('Found: \'' + sys.argv[1] + '\'')
from config import data_params, nn_params
#------------------------------------------------------------------------------
# Main execution
#------------------------------------------------------------------------------
# Dataset
trainA_loader = torch.utils.data.DataLoader(datasets.MNIST(
'MNIST_data',
train=True,
download=True,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
])),
shuffle=True,
**data_params)
# Model
model = NeuralNetwork(**nn_params)
model = model.to(model.device)
model.load("modelA.ckpt")
# Compute and save Fisher Information matrix
for X, Y in trainA_loader:
pass
X = X.view(-1, model.input_size).to(model.device)
Y = Y.to(model.device)
fishers = compute_fisher(model, X, Y)
for fim in fishers:
print(fim)
torch.save(fishers, "fisherA.pth")
if relevant_text.shape[0] > 0:
recall[i] += 1
recall = recall / total_query
return recall
if __name__ == "__main__":
# Read config file
with open('options.json', 'r') as f:
opt = json.load(f)
# Define model
text_encoder = NeuralNetwork(
input_dim=opt['text_dim'],
output_dim=opt['image_dim'],
hidden_units=opt['text_encoder_hidden'],
hidden_activation=opt['text_encoder_hidden_activation'],
output_activation=opt['text_encoder_output_activation'],
use_dropout=opt['use_dropout'],
use_batchnorm=opt['use_batchnorm']).to(device)
image_encoder = NeuralNetwork(
input_dim=opt['image_dim'],
output_dim=opt['common_dim'],
hidden_units=opt['image_encoder_hidden'],
hidden_activation=opt['image_encoder_hidden_activation'],
output_activation=opt['image_encoder_output_activation'],
use_dropout=opt['use_dropout'],
use_batchnorm=opt['use_batchnorm']).to(device)
image_mha = CustomSelfAttention(opt['common_dim'],
dropout=opt['mha_dropout']).to(device)
optimisers.warmup()
# Initialise data, number of iterations, and results list
np.random.seed(9251)
sin_data = data.SinusoidalDataSet1D1D(xlim=[-2, 2], freq=1)
n_iters = 10000
eval_every = n_iters // 20
results_list = []
for seed in [2295, 6997, 7681]:
# Set the random seed
np.random.seed(seed)
# Generate random network and store initial parameters
n = NeuralNetwork(1, 1, [10],
[activations.Gaussian(),
activations.Identity()])
w0 = n.get_parameter_vector().copy()
# Call gradient descent function
result_ls = optimisers.gradient_descent(n,
sin_data,
n_iters=n_iters,
eval_every=eval_every,
verbose=True,
name="SGD with line search",
line_search_flag=True)
results_list.append(result_ls)
# Try again without line search
n.set_parameter_vector(w0)
result_no_ls = optimisers.gradient_descent(n,
sin_data,
valid_features = torch.load('./cnn_train_feature/valid_cnn_features.pt').view(
-1, 2048)
train_val = torch.load('./cnn_train_feature/train_cnn_val.pt').type(
torch.LongTensor)
valid_val = torch.load('./cnn_train_feature/valid_cnn_val.pt').type(
torch.LongTensor)
print("train_features", train_features.shape)
print("train_val", train_val.shape)
print("valid_features", valid_features.shape)
print("valid_val", valid_val.shape)
# model, optimzer, loss function
device = torch.device("cuda") if torch.cuda.is_available() else torch.device(
"cpu")
feature_size = 2048
model = NeuralNetwork(feature_size).to(device)
model = torch.load("./best_cnn.pth")
optimizer = torch.optim.Adam(model.parameters(), lr=0.000025)
loss_function = nn.CrossEntropyLoss()
# some training parameters
batch_size = 64
num_epoch = 100
total_length = len(train_features)
max_accuracy = 0
logfile = open('log.txt', 'w')
now = datetime.datetime.now()
logfile.writelines("start training at:" + str(now) + "\n")
logfile.flush()
# start training
def test_invalid_error_function(repeat):
""" Test the errors raised by initialising an instance of the NeuralNetwork
class when there is an invalid value for the error_func argument """
set_random_seed_from_args("test_invalid_error_function", repeat)
n = NeuralNetwork(error_func=models.errors._SumOfSquares)
x, N_D = get_random_inputs(n.input_dim)
t = get_random_targets(n.output_dim, N_D)
n.forward_prop(x)
with pytest.raises(TypeError):
# Network is initialised with the SumOfSquares class, not an instance
n.mean_total_error(t)
n = NeuralNetwork(error_func=sum)
x, N_D = get_random_inputs(n.input_dim)
t = get_random_targets(n.output_dim, N_D)
n.forward_prop(x)
# Error function sum is callable, so mean_error is okay
n.mean_total_error(t)
with pytest.raises(AttributeError):
# Error function sum has no dydx method, so backprop fails
n.back_prop(x, t)
def test_invalid_act_function(repeat):
""" Test the errors raised by initialising an instance of the NeuralNetwork
class when there is an invalid value for the act_funcs argument """
set_random_seed_from_args("test_invalid_act_function", repeat)
with pytest.raises(TypeError):
# act_funcs argument should be a list of activation function objects
n = NeuralNetwork(act_funcs=models.activations.gaussian)
n = NeuralNetwork(act_funcs=[models.activations._Gaussian])
x, _ = get_random_inputs(n.input_dim)
with pytest.raises(TypeError):
# Network is initialised with the Gaussian class, not an instance
n.forward_prop(x)
n = NeuralNetwork(act_funcs=[None])
x, _ = get_random_inputs(n.input_dim)
with pytest.raises(TypeError):
# Activation function None is not callable
n.forward_prop(x)
n = NeuralNetwork(act_funcs=[abs])
x, N_D = get_random_inputs(n.input_dim)
t = get_random_targets(n.output_dim, N_D)
# Activation function abs is callable, so forward_prop is okay
n.forward_prop(x)
with pytest.raises(AttributeError):
# Activation function abs has no dydx method, so backprop fails
n.back_prop(x, t)
def test(data, optimizer_options, dropout_options, model_dir, log_dir):
for optimizer in optimizer_options:
for dropout in dropout_options:
print('\nOptimizer: {}\tDropout option: {}\n'.format(
optimizer, dropout))
# Initialize sub folders for multiple models
mode_name = optimizer + '_' + str(dropout)
sub_model_dir = os.path.join(model_dir, mode_name)
sess = tf.Session() # Initialize session
# Initialize model
model = None
if FLAGS.model == 'logistic':
model = Logistic(input_dim=data.img_size_flat,
output_dim=1,
optimizer=optimizer,
use_dropout=dropout,
lr=FLAGS.learning_rate,
random_seed=FLAGS.random_seed,
is_train=FLAGS.is_train,
log_dir=None,
name=mode_name)
elif FLAGS.model == 'neural_network':
model = NeuralNetwork(input_dim=data.img_size_flat,
output_dim=[1000, 1000, 10],
optimizer=optimizer,
use_dropout=dropout,
lr=FLAGS.learning_rate,
weight_decay=FLAGS.weight_decay,
random_seed=FLAGS.random_seed,
is_train=FLAGS.is_train,
log_dir=None,
name=mode_name)
elif FLAGS.model == 'cnn':
model = CNN(input_dim=data.img_shape,
output_dim=[128, 256, 512, 1000, 10],
optimizer=optimizer,
use_dropout=dropout,
lr=FLAGS.learning_rate,
weight_decay=FLAGS.weight_decay,
random_seed=FLAGS.random_seed,
is_train=FLAGS.is_train,
log_dir=None,
name=mode_name)
else:
raise NotImplementedError
# Initialize solver
solver = Solver(sess, model)
saver = tf.train.Saver(max_to_keep=1)
# Test process
if load_model(saver, solver, sub_model_dir, mode_name):
print(' [*] Load model: {} SUCCESS!'.format(mode_name))
else:
print(' [!] Load model: {} Failed...'.format(mode_name))
test_acc, _ = solver.evaluate(x=data.x_test,
y=data.y_test_cls if FLAGS.model
== 'logistic' else data.y_test,
batch_size=FLAGS.batch_size)
logger.info('Mode name: {}, Test acc: {}\n'.format(
mode_name, test_acc))
sess.close()
tf.reset_default_graph() # To release GPU memory
plot_loss(log_dir, optimizer_options, dropout_options, is_show=True)
def train(data, optimizer_options, dropout_options, model_dir, log_dir):
num_iters = int(round(FLAGS.epochs * data.num_train / FLAGS.batch_size))
iters_epoch = int(round(data.num_train / FLAGS.batch_size))
for optimizer in optimizer_options:
for dropout in dropout_options:
print('\nOptimizer: {}\tDropout option: {}\n'.format(
optimizer, dropout))
# Initialize sub folders for multiple models
mode_name = optimizer + '_' + str(dropout)
sub_model_dir = os.path.join(model_dir, mode_name)
sub_log_dir = os.path.join(log_dir, mode_name)
if not os.path.isdir(sub_model_dir):
os.makedirs(sub_model_dir)
if not os.path.isdir(sub_log_dir):
os.makedirs(sub_log_dir)
# Fix weight initialization of each model with different optimizers
tf.set_random_seed(FLAGS.random_seed)
sess = tf.Session() # Initialize session
# Initialize model
model = None
if FLAGS.model == 'logistic':
model = Logistic(input_dim=data.img_size_flat,
output_dim=1,
optimizer=optimizer,
use_dropout=dropout,
lr=FLAGS.learning_rate,
random_seed=FLAGS.random_seed,
is_train=FLAGS.is_train,
log_dir=sub_log_dir,
name=mode_name)
elif FLAGS.model == 'neural_network':
model = NeuralNetwork(input_dim=data.img_size_flat,
output_dim=[1000, 1000, 10],
optimizer=optimizer,
use_dropout=dropout,
lr=FLAGS.learning_rate,
weight_decay=FLAGS.weight_decay,
random_seed=FLAGS.random_seed,
is_train=FLAGS.is_train,
log_dir=sub_log_dir,
name=mode_name)
elif FLAGS.model == 'cnn':
model = CNN(input_dim=data.img_shape,
output_dim=[128, 256, 512, 1000, 10],
optimizer=optimizer,
use_dropout=dropout,
lr=FLAGS.learning_rate,
weight_decay=FLAGS.weight_decay,
random_seed=FLAGS.random_seed,
is_train=FLAGS.is_train,
log_dir=sub_log_dir,
name=mode_name)
# Initialize solver
solver = Solver(sess, model)
saver = tf.train.Saver(max_to_keep=1)
tb_writer = tf.summary.FileWriter(sub_log_dir,
graph_def=solver.sess.graph_def)
csvWriter = CSVWriter(path=log_dir, name=mode_name)
solver.init()
best_acc, num_epoch = 0., 0
# Training process
for iter_time in range(num_iters):
x_batch, y_batch, y_batch_cls = data.random_batch(
batch_size=FLAGS.batch_size)
_, loss, summary = solver.train(
x=x_batch,
y=y_batch_cls if FLAGS.model == 'logistic' else y_batch)
# Write to tensorboard
tb_writer.add_summary(summary, iter_time)
tb_writer.flush()
if iter_time % FLAGS.print_freq == 0:
print('{0:7}/{1:7}: Loss: {2:.3f}'.format(
iter_time, num_iters, loss))
csvWriter.update(iter_time, loss)
# Validation
if iter_time % iters_epoch == 0 or iter_time == (num_iters -
1):
# Evaluate train-batch accuracy
x_batch, y_batch, y_batch_cls = data.random_batch(
batch_size=FLAGS.batch_size)
_, train_summary = solver.evaluate(
x=x_batch,
y=y_batch_cls
if FLAGS.model == 'logistic' else y_batch,
is_train=True)
# Evaluate validation accuracy
val_acc, val_summary = solver.evaluate(
x=data.x_val,
y=data.y_val_cls
if FLAGS.model == 'logistic' else data.y_val,
batch_size=FLAGS.batch_size)
# Write to tensorboard
tb_writer.add_summary(train_summary, num_epoch)
tb_writer.add_summary(val_summary, num_epoch)
tb_writer.flush()
num_epoch += 1
if val_acc > best_acc:
logger.info('Acc: {1:.3f}, Best Acc: {2:.3f}'.format(
iter_time, val_acc, best_acc))
save_model(saver, solver, sub_model_dir, mode_name,
iter_time)
best_acc = val_acc
# Test process
if load_model(saver, solver, sub_model_dir, mode_name):
logger.info(' [*] Load model: {} SUCCESS!'.format(mode_name))
else:
logger.info(' [!] Load model: {} Failed...'.format(mode_name))
test_acc, _ = solver.evaluate(x=data.x_test,
y=data.y_test_cls if FLAGS.model
== 'logistic' else data.y_test,
batch_size=FLAGS.batch_size)
logger.info('Mode name: {}, Test acc: {}\n'.format(
mode_name, test_acc))
model.release_handles()
sess.close()
tf.reset_default_graph() # To release GPU memory
csvWriter.close()
plot_loss(log_dir, optimizer_options, dropout_options)
nx1=50,
x1lim=[-2, 2],
noise_std=0.1,
train_ratio=0.8,
output_dim=output_dim)
t_lim = 10
t_interval = t_lim / 50
results_list = []
for seed in [2295, 6997, 7681]:
# Set the random seed
np.random.seed(seed)
# Generate random network and store initial parameters
n = NeuralNetwork(input_dim=2,
output_dim=output_dim,
num_hidden_units=[20, 20],
act_funcs=[activations.Cauchy(),
activations.Identity()])
# Call gradient descent function
result = optimisers.gradient_descent(
n,
sin_data,
terminator=optimisers.Terminator(t_lim=t_lim),
evaluator=optimisers.Evaluator(t_interval=t_interval),
result=optimisers.Result(name="SGD with line search", verbose=True),
line_search=optimisers.LineSearch())
results_list.append(result)
# Get name of output directory
current_dir = os.path.dirname(os.path.abspath(__file__))
output_dir = os.path.join(current_dir, "Outputs")
have_train_num = 0
for i, begin in enumerate(
range(0, self.num_samples - batch_size, batch_size)):
yield self.X[indices[begin:begin + batch_size], :], one_hot(
self.y[indices[begin:begin + batch_size]], self.class_num)
have_train_num = (i + 1) * batch_size
if have_train_num < self.num_samples:
yield self.X[indices[have_train_num:], :], one_hot(
self.y[indices[have_train_num:]], self.class_num)
train_data = DataLoader(x_train, y_train, 3)
test_data = DataLoader(x_test, y_test, 3)
# 构建网络
model = NeuralNetwork()
model.add_layer(Layer(4, 8, 'relu'))
model.add_layer(Layer(8, 8, 'relu'))
model.add_layer(Layer(8, 3))
# 构建损失函数和优化器
lr = 0.01
loss = Loss(loss='cross_entropy_with_logits')
optimizer = Optimizers(optimizer='sgd', learning_rate=lr)
model.compile(loss=loss, optimizer=optimizer)
# 训练数据
num_epochs = 1600
batch_size = 64
train_loss = []
test_loss = []
model_ft = models.inception_v3(pretrained=True, aux_logits=False)
# model_ft = nn.Sequential(*list(model_ft.children())[:13])
print(model_ft(torch.randn(1, 3, 299, 299)).shape)
# print("model:", model_ft)
num_ftrs = 768
num_ftrs = model_ft.fc.in_features
print(num_ftrs)
model_ft.fc = nn.Linear(num_ftrs, len(classes))
model_ft = model_ft.to(device)
return model_ft
is_inception = False
if model_type == 'Simple':
is_inception = False
model_ft = NeuralNetwork(classes, image_size).to(device)
elif model_type == 'Inception3':
is_inception = True
model_ft = Inception3(num_classes=len(classes)).to(device)
elif model_type == 'Inception_transfer':
is_inception = False
model_ft = Inceptionnet(classes, aux_logits=True).to(device)
elif model_type == 'Resnet':
model_ft = Resnet(classes).to(device)
elif model_type == 'Alexnet':
model_ft = Alexnet(classes).to(device)
# print("model:", model_ft)
#train model
loss_fn = nn.CrossEntropyLoss()
optimizer_ft = torch.optim.SGD(model_ft.parameters(), lr=1e-3)
# dataset converted from label to binary category (with platform id, season id)
df_2 = df[['matchid', 'player', 'name', 'team_role', 'win']]
df_2 = df_2.pivot(index='matchid', columns='team_role', values='name')
df_2 = df_2.reset_index()
df_2 = df_2.merge(df[df['player'] == 1][['matchid', 'win', 'platformid', 'seasonid', 'version']],
left_on='matchid', right_on='matchid', how='left')
df_2 = df_2[df_2.columns.difference(['matchid', 'version'])]
df_2 = df_2.rename(columns={'win': 'T1 win'})
df['name'].unique()
df_2 = df_2.dropna()
le = preprocessing.LabelEncoder()
y_2 = df_2['T1 win']
X_2 = df_2[df_2.columns.difference(['T1 win'])]
le_t = X_2.apply(le.fit)
X_t_3 = X_2.apply(le.fit_transform)
enc = preprocessing.OneHotEncoder()
enc_t = enc.fit(X_t_3)
X_t_4 = enc_t.transform(X_t_3)
# split train & test
X_train, X_test, y_train, y_test = train_test_split(X_t_4, y_2, random_state=0)
# train using Neural Network
model = NeuralNetwork(X_train, X_test, y_train, y_test)
# Save model
model.save("Neural Network Model.h5")
print('Training model has been saved! ')
from models import NeuralNetwork
from utils import ProteinAtlasTestDataset
PATH = 'models/protein_net_05-08-21-1421_lr_0.000100.pth'
test_dir = 'data/test/'
test_csv_file = os.path.join(test_dir, 'sample_submission.csv')
testset = ProteinAtlasTestDataset.ProteinAtlasTestDataset(test_csv_file,
test_dir,
image_mean=13.42)
test_loader = DataLoader(testset, batch_size=64, num_workers=0, shuffle=False)
net = NeuralNetwork.NeuralNetwork()
net.load_state_dict(torch.load(PATH))
net.eval()
prediction_id = []
prediction_label = []
for i, data in enumerate(test_loader, 0):
# get the inputs; data is a list of [inputs, labels]
image, image_id = data['image'].unsqueeze(1), data['id']
# forward + backward + optimize
outputs = net(image)
array = outputs.detach().numpy()
transforms.Normalize((0.1307, ), (0.3081, ))])),
shuffle=True,
**data_params)
testA_loader = torch.utils.data.DataLoader(datasets.MNIST(
'MNIST_data',
train=False,
download=True,
transform=transforms.Compose(
[transforms.ToTensor(),
transforms.Normalize((0.1307, ), (0.3081, ))])),
shuffle=False,
**data_params)
# Model
model = NeuralNetwork(**nn_params)
model = model.to(model.device)
optimizer = optim.Adam(model.parameters(), **opt_params)
loss_fn = torch.nn.CrossEntropyLoss(reduction="elementwise_mean")
# Create callbacks
checkpoint = CheckPoint(model, "modelA.ckpt")
earlystop = EarlyStopping(**earlystop_params)
# Train and evaluate the model
flg_stop = False
for epoch in range(1, params["n_epochs"] + 1):
print("\n[EPOCH %d]" % (epoch))
loss_train = train(model,
trainA_loader,
optimizer,
if __name__ == '__main__':
# Reading data, Inintializing classes in the workspace
hlp, ft, als = Helpers(), Features(), Analysis()
all_data = hlp.saveReadToLocal('read', '_all_av45', None, '_cache')
all_data = all_data['av45']
keys = ['normal', 'mci', 'ad', 'labels_normal', 'labels_mci', 'labels_ad']
mixed_all_data = als.getTrainingSet(all_data, 0, 0, keys, do_split=False)
print('Features dimention before normalization: ',
len(mixed_all_data[0][0]))
mixed_all_data = ft.normalizeFeatures(mixed_all_data)
print('Features dimention after normalization: ', len(mixed_all_data[0][0]))
# mixed_all_data = als._cleanDataset(mixed_all_data[0], mixed_all_data[1])
# grid_params = runAnalysis(mixed_all_data, classifier='svm')
# a = runAnalysis(mixed_all_data, params=grid_params, classifier='svm')
# print('Overall accuracy: ', np.mean(a))
input_size = len(mixed_all_data[0][0])
nn = NeuralNetwork([input_size, input_size, 40, 40, 3])
input_data = np.asarray(mixed_all_data[0]).reshape(
(input_size, len(mixed_all_data[0])))
input_labels = np.asarray(mixed_all_data[1]).reshape(
(1, len(mixed_all_data[1])))
nn.run(input_data, input_labels, 0.01, 5000)
test = np.asarray(mixed_all_data[0][0]).reshape(
(input_size, 1))
print(nn.predict(test))
**data_params)
testB_loader = torch.utils.data.DataLoader(datasets.MNIST(
'MNIST_data',
train=False,
download=True,
transform=transforms.Compose([
transforms.Lambda(transform_permute),
transforms.ToTensor(),
transforms.Normalize((0.1307, ), (0.3081, ))
])),
shuffle=False,
**data_params)
# Model
model = NeuralNetwork(**nn_params)
model = model.to(model.device)
model.load("modelA.ckpt")
prev_opt_thetas = deepcopy(list(model.parameters()))
optimizer = optim.Adam(model.parameters(), **opt_params)
base_loss_fn = torch.nn.CrossEntropyLoss(reduction="elementwise_mean")
# Generate permute indices
ind_permute = np.arange(0, 28)
np.random.seed(0)
np.random.shuffle(ind_permute)
np.save("permuteB.npy", ind_permute)
# Load the previous FIM
fishers_cpu = torch.load("fisherA.pth")
# Dataset for evaluation
val_image_dataset = FeatureDataset(opt['feature_folder'], opt['val_file'], max_num_regions=opt["max_num_regions"], device=device)
val_image_dataloader = DataLoader(val_image_dataset, batch_size=opt['batch_size'], collate_fn=my_collate_fn)
val_text_dataset = TextDataset(opt['val_file'], tokenizer, opt['max_seq_len'])
val_text_dataloader = DataLoader(val_text_dataset, batch_size = opt['batch_size'], shuffle=False)
# Test dataset
test_image_dataset = FeatureDataset(opt['feature_folder'], opt['test_file'], max_num_regions=opt["max_num_regions"], device=device)
test_image_dataloader = DataLoader(test_image_dataset, batch_size=opt['batch_size'], collate_fn=my_collate_fn)
test_text_dataset = TextDataset(opt['test_file'], tokenizer, opt['max_seq_len'])
test_text_dataloader = DataLoader(test_text_dataset, batch_size = opt['batch_size'], shuffle=False)
# Define model
text_encoder = NeuralNetwork(input_dim=opt['text_dim'],
output_dim=opt['common_dim'],
hidden_units=opt['text_encoder_hidden'],
hidden_activation=opt['text_encoder_hidden_activation'],
output_activation=opt['text_encoder_output_activation'],
use_dropout=opt['use_dropout'],
use_batchnorm=opt['use_batchnorm']).to(device)
image_encoder = NeuralNetwork(input_dim=opt['image_dim'],
output_dim=opt['common_dim'],
hidden_units=opt['image_encoder_hidden'],
hidden_activation=opt['image_encoder_hidden_activation'],
output_activation=opt['image_encoder_output_activation'],
use_dropout=opt['use_dropout'],
use_batchnorm=opt['use_batchnorm']).to(device)
image_mha = MultiSelfAttention(opt['image_dim'], opt['image_dim'], opt['image_embed_dim'], num_layers=opt['num_attention_layers'], dropout=opt['mha_dropout']).to(device)
if opt['text_model_type'] == 'roberta':
bert = RobertaModel.from_pretrained(opt['text_model_pretrained'], output_hidden_states=True).to(device)