def fine_tune_model():
print(f'\n\nfine tuning model...')
data_manager = DataManager()
plotter = Plotter()
train_manager = get_train_manager(data_manager)
n_filters_list = []
train_losses, test_losses = [], []
for i in range(1, 11):
n_filters = 4 * i
net = Net(conv1_out_channels=n_filters)
optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)
train_manager.init_model(net, optimizer)
train_and_save_model(train_manager)
(train_loss,
train_accuracy), (test_loss,
test_accuracy) = train_manager.get_losses()
n_filters_list.append(n_filters)
train_losses.append(train_loss)
test_losses.append(test_loss)
plotter.plot_filters_losses(n_filters_list, train_losses, test_losses)
return train_losses, test_losses, n_filters_list
def main():
"""
main func
"""
text, x, y, char2idx, idx2char = getty()
T = 100
config = {
'dim_hidden': 300,
'l': T,
'clip': 5,
'mu': 0.9,
'step_size': 0.001
}
#np.random.seed(42)
r = RNN(config)
r.accept([27])
ttb = r.sample('f', char2idx, idx2char)
r.fit(x[:T], y[:T], 100, char2idx, idx2char)
tta = r.sample('f', char2idx, idx2char)
print(ttb)
print(tta)
print(text[:T])
return
(data, label) = cifar()
N = 10000
data = np.array(data, dtype=float)[:N, ]
label = np.array(label)[:N, ]
data = normalize(data)
config = {
'input_shape': [3, 32, 32],
'mu': 0.9,
'step_size': 0.000001,
'step_decay': 0.95
}
nn = Net(config)
conv1 = Conv([3, 3], 6)
relu1 = Relu()
conv2 = Conv([3, 3], 32)
relu2 = Relu()
pool = MaxPool()
fc = FC([10])
nn.add(conv1)
nn.add(relu1)
nn.add(pool)
nn.add(fc)
print(nn)
nn.fit(data, label, 200)
def main():
""" Step 1: make dataset """
random.seed()
# Make 3 inputs - 1 base and 2 added inputs
sig_len = 10
y_base = np.array([1, 2, 3, 2, 6, 5, 0, -1, 2, 4])
y_add1 = np.array([0, 0, 1, 0, -2, 0, 0, 1, 1, 0])
y_add2 = np.array([1, 0, 0, 1, 2, -1, 0, 0, 0, 0])
# Set up a bunch of random signals to detect
y_num = 100
signal1 = np.array([random.randint(0,1) for i in range(y_num)])
signal2 = np.array([random.randint(0,1) for i in range(y_num)])
signal = np.array([signal1, signal2])
# Add up the inputs accordingly
y_list = np.zeros([y_num, len(y_base)])
for i in range(y_num):
y_sum = np.array([y_base[j] + signal1[i]*y_add1[j] + signal2[i]*y_add2[j]
for j in range(sig_len)])
y_list[i] = y_sum
# Add noise
noise = np.random.random([y_num, len(y_base)]) / 10
y_list += noise
""" Step 2: train neural network """
# Set up input and signals
input = np.array(y_list)
signal = signal.transpose()
# Set up min and max for each input
# Can give the network a good idea of input ranges or just a rough range
limits = [[y_base[i]-2, y_base[i]+2] for i in range(10)]
#limits = [[-20, 20]]*10
# Make network
net = Net(limits, 2, 2)
errorList = net.train_many(input, signal, 0.1, 100, 0.001, True)
print "\n".join(map(str, errorList))
""" Step 3: check results """
# Print results by hand
#for i in range(y_num):
# print y_list[i]
# print signal1[i]
# print signal2[i]
# print net.sim(y_list[i, :])
# Plot error vs. training epochs
plt.semilogy(errorList)
plt.grid()
plt.xlabel('Epochs')
plt.ylabel('SSE')
plt.show()
def get_models(model_paths, device):
"""Return the contrusted models of the model paths given"""
hp = hp_dict()
n_in = 784
n_hidden = hp["width"][get_hp(model_paths[0], 3)]
n_layer = hp["depth"][get_hp(model_paths[0], 2)]
n_out = 10
seed = get_hp(model_paths[0], 4)
init = get_hp(model_paths[0], 10)
n_epoch = len(model_paths)
torch.manual_seed(seed)
# path = '../models/mnist/seed_{}_init_{}'.format(seed, init)
models = [
Net(n_in, n_hidden, n_out, n_layer, act='relu', init_val=2).to(device)
for _ in range(n_epoch)
]
for i, model in enumerate(models):
model.load_state_dict(torch.load(model_paths[i], map_location=device))
return models
def test_no_spatial_structure():
print(f'testing no spatial structure...')
fixed_shuffle_data_manager = DataManager(shuffle_type='fixed')
fresh_shuffle_data_manager = DataManager(shuffle_type='fresh')
for name, data_manager in [('fixed_shuffle', fixed_shuffle_data_manager),
('fresh_shuffle', fresh_shuffle_data_manager)]:
print(f'\ntesting {name}...')
train_manager = get_train_manager(data_manager)
net = Net()
optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)
train_manager.init_model(net, optimizer)
train_manager.train()
print(f'\ntesting no spatial...')
train_res, test_res = train_manager.get_losses()
print(f'train loss: {train_res[0]}, train accuracy: {train_res[1]}\n'
f'test loss: {test_res[0]}, test accuracy: {test_res[1]}')
def test_locality_of_receptive_field():
print(f'testing locality of receptive field...')
noshuffle_data_manager = DataManager(shuffle_type='none')
shuffled_data_manager = DataManager(shuffle_type='fixed')
for name, data_manager in [('no_shuffle', noshuffle_data_manager),
('shuffle', shuffled_data_manager)]:
print(f'\ntesting {name}...')
train_manager = get_train_manager(data_manager)
net = Net()
optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)
train_manager.init_model(net, optimizer)
train_manager.train()
print(f'\ntesting local field...')
train_res, test_res = train_manager.get_losses()
print(f'train loss: {train_res[0]}, train accuracy: {train_res[1]}\n'
f'test loss: {test_res[0]}, test accuracy: {test_res[1]}')
def recreate_model(path, dataset="mnist", act="relu"):
'''
Return the recreated model from the given model state
'''
device, dtype, _ = get_settings()
file_name = path.split("/")[-1]
noise_type, noise_level = path.split("/")[-4:-2]
hyperparams = file_name.split("_")
hp = []
for each in hyperparams[:11]:
hp.append(int(each))
hyperparams = get_hyperparameters(hp[:8], noise_type, noise_level)
# batch_size = hyperparams[1]
n_hidden = hyperparams[3]
n_layer = hyperparams[2]
# seed = hyperparams[4]
# print("Hyperparams", hyperparams)
# print("n_layer:", n_layer)
noise_type, noise_level, init_val = get_experiment([hp[8], hp[9], hp[10]], n_layer)
if noise_type == 'none':
noise_type = None
noise_level = None
# print(noise_type, noise_level, init_val)
n_in, n_out = get_data_dimensions(dataset)
model = Net(n_in, n_hidden, n_out, n_layer, act=act, noise_type=noise_type,
noise_level=noise_level, init_val=init_val).to(device, dtype)
model.load_state_dict(torch.load(path, map_location=device)) # NB double check that this doesn't break things on GPU
# model.load_state_dict(torch.load(path, map_location='cpu'))
# model.eval()
return model
def test_non_linearities():
print(f'testing non linearities...')
non_linear_net = Net(fc1_out_channels=120, non_linear=True)
linear_net = Net(fc1_out_channels=120, non_linear=False)
big_linear_net = Net(fc1_out_channels=240, non_linear=False)
nets = [
NetInfo('non_linear', non_linear_net),
NetInfo('linear', linear_net),
NetInfo('big_linear', big_linear_net)
]
for net in nets:
data_manager = DataManager()
train_manager = get_train_manager(data_manager)
# if net is not nets[1]: continue
print(f'\ntesting {net.name}...')
optimizer = optim.SGD(net.object.parameters(), lr=0.001, momentum=0.9)
train_manager.init_model(net.object, optimizer)
train_manager.train()
train_loss, test_loss = train_manager.get_losses()
print(f'train loss: {train_loss}\ntest loss: {test_loss}')
def get_score(n_iterations, init_val, model_name, n_epochs, eval_epochs,
train_loader, test_loader, device):
all_correlations = list()
for i in range(n_iterations):
net = Net(n_in,
n_hidden,
n_out,
n_layer,
act='relu',
init_val=init_val).to(device)
corrs = get_correlations(net, '{}_{}'.format(model_name,
i), n_epochs, eval_epochs,
train_loader, test_loader, device)
print(corrs)
all_correlations.append(corrs)
Y = np.array(all_correlations)
# trained_net = train(net, train_loader, criterion, optimiser,
# n_epochs, device, save=True, name=model_name, log_interval=1)
# test(trained_net, test_loader, criterion, device)y(all_correlations)
return Y
import torch
from src.net import Net
import json
import argparse
from src.load_dataset import load_dataset
PATH = './model.pth'
class_map = {
0: 'female',
1: 'male',
}
parser = argparse.ArgumentParser()
parser.add_argument('path', type=str)
args = parser.parse_args()
net = Net()
net.load_state_dict(torch.load(PATH))
loader, names = load_dataset(args.path)
predicted_general = []
for images in loader:
outputs = net(images)
_, predicted = torch.max(outputs, 1)
predicted_general += predicted.tolist()
data = {names[i]: class_map[predicted_general[i]] for i in range(len(names))}
with open('process_results.json', 'w') as outfile:
json.dump(data, outfile)
import numpy as np
from src.net import Net
from data.data import U0, U1, U2, U3, U4, U5, U6
if __name__ == "__main__":
network = Net([U0, U1, U2, U3, U4, U5, U6],
first_point_id=[0],
last_point_id=[5, 6])
result_representation = ""
for ind, result in enumerate(network.result):
result_representation += "Rozwiązanie " + str(ind + 1) + ":\n"
for point in result:
result_representation += "\tOptymalizator " + str(
point[0]) + " -> decyzja: " + str(point[1]) + "\n"
result_representation += "\nSuma kryterium: " + str(network.maximum)
print(result_representation)
"e2e_dump_settings": {
"enable": True,
"trans_flag": False
}
}
with open("./data_dump.json", "w", encoding="GBK") as f:
json.dump(data_dump, f)
os.environ['MINDSPORE_DUMP_CONFIG'] = abspath + "/data_dump.json"
def set_log_info():
os.environ['GLOG_v'] = '1'
os.environ['GLOG_logtostderr'] = '1'
os.environ['logger_maxBytes'] = '5242880'
os.environ['GLOG_log_dir'] = 'D:/' if os.name == "nt" else '/var/log/mindspore'
os.environ['logger_backupCount'] = '10'
print(logger.get_log_config())
if __name__ == "__main__":
set_dump_info()
set_log_info()
context.set_context(mode=context.GRAPH_MODE)
train_dataset = create_train_dataset()
eval_dataset = create_eval_dataset()
net = Net()
net_opt = Momentum(net.trainable_params(), 0.01, 0.9)
net_loss = SoftmaxCrossEntropyWithLogits(reduction='mean')
model = Model(network=net, loss_fn=net_loss, optimizer=net_opt, metrics={'Accuracy': nn.Accuracy()})
model.train(epoch=100,
train_dataset=train_dataset,
callbacks=[LossMonitor(), StopAtTime(3), SaveCallback(model, eval_dataset)])
from src.net import Net
from src.utils import randn, uniform, guass, read_mnist
import numpy as np
import time
inp_dim = 28
out_dim = 10
std = 1.
lr = 1e-3
inp_shape = (inp_dim, inp_dim, 1)
net = Net()
image = net.portal((28, 28, 1))
label = net.portal((10, ))
is_training = net.portal()
k1 = net.variable(guass(0., std, (5, 5, 1, 32)))
b1 = net.variable(np.ones((32, )) * .1)
k2 = net.variable(guass(0., std, (5, 5, 32, 64)))
b2 = net.variable(np.ones((64, )) * .1)
w3 = net.variable(guass(0., std, (7 * 7 * 64, 1024)))
b3 = net.variable(np.ones((1024, )) * .1)
w4 = net.variable(guass(0., std, (1024, 10)))
b4 = net.variable(np.ones((10, )) * .1)
conv1 = net.conv2d(image, k1, pad=(2, 2), stride=(1, 1))
conv1 = net.batch_norm(conv1, net.variable(guass(0., std, (32, ))),
is_training)
conv1 = net.plus_b(conv1, b1)
conv1 = net.relu(conv1)
pool1 = net.maxpool2(conv1)
shuffle=config.test_dataloader.shuffle,
num_workers=config.test_dataloader.num_workers)
# print("Trainset:")
# for i in range(len(trainset)):
# # print(trainset[i])
# print(trainset[i]["sensitive"])
# print("Testset:")
# for i in range(len(testset)):
# print("testset " + str(i))
# print(testset[i]["sensitive"])
#
writer = SummaryWriter("runs/noConv1D_ascad_desync_50_3")
#TODO: Change the model for the one of the paper
net = Net()
#TODO: propose NLLloss (Categorical Cross Entropy), Adam optimizer and Cyclic Learning Rate
criterion = nn.NLLLoss()
optimizer = optim.Adam(net.parameters(), lr=float(config.train.lr))
scheduler = optim.lr_scheduler.OneCycleLR(optimizer,
max_lr=float(config.train.lr),
epochs=50,
steps_per_epoch=len(trainloader))
# TODO: plot in tensorboard the curves loss and accuracy for train and val
for epoch in range(
config.train.epochs): # loop over the dataset multiple times
print('Epoch {}/{}'.format(epoch + 1, config.train.epochs))
print('-' * 10)
for phase in ["train", "val"]:
# Load data
train_loader, test_loader = load_data(dataset,
batch_size=batch_size,
path=os.path.join(
base_path, '../data'))
if start_epoch == 0:
print("results directory is empty, starting test from beginning")
# create network
torch.manual_seed(seed)
n_in, n_out = get_data_dimensions(dataset)
net = Net(n_in,
n_hidden,
n_out,
n_layer,
act=act,
noise_type=noise_type,
noise_level=noise_level,
init_val=init_val).to(device, dtype)
save_model(net,
experiment_name,
0,
noise_type,
noise_level,
model_dir=model_dir)
else:
print("starting from epoch {}".format(start_epoch))
net = recreate_model(model_to_load, dataset=dataset, act=act)
# optimiser parameters
w = net.variable(guass(0., .1, (inp_size, out_size)))
regularizer = net.l2_regularize(w, center)
b = net.variable(np.ones((out_size, )) * .1)
mul = net.matmul(inp, w)
biased = net.plus_b(mul, b)
if drop is not None:
biased = net.dropout(biased, drop)
return biased, regularizer
dat = TRECvn(holdout=0) # set holdout > 0 for early stopping
max_len, nclass = dat.max_len, dat.nclass
lstm_cell = 300
fc_size = 128
net = Net()
# Set up the portals and recurrent layer
x, y, keep, lens = portals(net, max_len, nclass)
lstmed = lstm_layer(net, dat.embeddings, x, True, lens, lstm_cell)
# Setup two fully-connected layer with middle dropout
# And L2 weight decay for both of them.
center = net.portal((1, )) # to be fed by zero.
penultimate, regularizer1 = fully_connected(lstmed,
lstm_cell,
fc_size,
center,
drop=keep)
penultimate = net.relu(penultimate)
predict, regularizer2 = fully_connected(penultimate, fc_size, nclass, center)
import numpy as np
from src.net import Net
batch = 8
out_dim = inp_dim = 10
hid_dim = 300
mem_size = 128
vec_size = 30
max_seq_len = 20
start_symbol = np.zeros([batch, 1, inp_dim])
stop_symbol = np.zeros([batch, 1, inp_dim])
start_symbol[:, 0, 0] = np.ones([batch])
stop_symbol[:, 0, 1] = np.ones([batch])
net = Net()
x = net.portal((inp_dim, ))
y = net.portal((inp_dim, ))
y = net.reshape(y, [-1, inp_dim], over_batch=True)
ntm_out = net.turing(x, out_dim, mem_size, vec_size, hid_dim, shift=1)
start, end = net.portal(), net.portal()
copy = net.dynamic_slice(ntm_out, start=start, end=end, axis=0)
logits = net.reshape(copy, [-1, inp_dim], over_batch=True)
loss = net.logistic(logits, y)
net.optimize(loss, 'adam', 1e-2)
def generate_random_input(batch, seq_length, inp_dim):
x = np.random.rand(batch, seq_length, inp_dim).round()
from src.net import Net from src.utils import randn, uniform, guass, read_mnist import numpy as np import time inp_dim = 784 hid_dim = 64 out_dim = 10 std = 1e-3 lr = 1e-2 batch = 128 net = Net() image = net.portal((784, )) keep_prob = net.portal() target = net.portal((10, )) w1 = net.variable(guass(0., std, (inp_dim, hid_dim))) b1 = net.variable(np.ones(hid_dim) * .1) w2 = net.variable(guass(0., std, (hid_dim, out_dim))) b2 = net.variable(np.ones(out_dim) * .1) fc1 = net.matmul(image, w1) bias = net.plus_b(fc1, b1) relu = net.relu(bias) dropped = net.dropout(relu, keep_prob) fc2 = net.matmul(dropped, w2) bias = net.plus_b(fc2, b2) loss = net.softmax_crossent(bias, target) net.optimize(loss, 'sgd', lr)
def get_saved_model(model_name):
print(f'loading model...')
model = Net()
model.load_state_dict(torch.load(TrainManager.get_model_path(model_name)))
return model
def build_infer(net, inp):
def _fc(inp, act):
weight, bias = next(infer_weights)
weight, bias = map(net.variable, [weight, bias])
lineared = net.matmul(inp, weight)
return act(net.plus_b(lineared, bias))
fc1 = _fc(inp, net.tanh)
fc2 = _fc(fc1, net.tanh)
fc3 = _fc(fc2, net.tanh)
fc4 = _fc(fc3, net.softmax)
return fc4
# Build the net.
net = Net()
image = net.portal((224, 224, 3))
question = net.portal((40, 300))
real_len = net.portal((1, ))
vgg16_feat = build_vgg16(net, image)
lstm3_feat = build_lstm3(net, question, real_len)
infer_feat = net.concat([lstm3_feat, vgg16_feat])
answer = build_infer(net, infer_feat)
image_feed = read_image('test.jpg')
queries = [u"What is the animal in the picture?"]
import time
query_feed = list()