def run(df):
y = df["target"].values
X = df.drop(["target", "kfold"], axis=1).values
train_dataset = dataset.TweetDataset(
tweets=X,
targets=y
)
train_dataloader = torch.utils.data.DataLoader(
train_dataset,
batch_size=config.TRAIN_BATCH_SIZE,
num_workers=2
)
device = "cuda" if torch.cuda.is_available() else "cpu"
print("Using {} device".format(device))
model = neural_net.NeuralNetwork().to(device)
optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)
loss_fn = torch.nn.BCELoss()
print("Training Model...")
for epoch in range(config.EPOCHS):
print(f"Epoch {epoch+1}\n--------------------")
engine.train(
train_dataloader,
model,
optimizer,
loss_fn,
device
)
torch.save(model.state_dict(), f"{config.MODEL_PATH}/{config.MODEL_NAME}.pth")
def KFoldCV(X, y, l, k=3):
t_size = X.shape[0]
tr_acc = 0
tst_acc = 0
for i in range(k):
st = int(i * t_size / 3.0)
end = int((i + 1) * t_size / 3.0)
X_train = np.concatenate((X[:st, :], X[end:, :]), axis=0)
y_train = np.concatenate((y[:st, :], y[end:, :]), axis=0)
X_test = X[st:end, :]
y_test = y[st:end, :]
neurons = [net.Layer(D, 256, 'l_relu'), net.Layer(256, m, 'softmax')]
NN = net.NeuralNetwork(2,
D,
m,
cost='crossent',
layers=neurons,
rate=lrate)
NN.train(X_train,
y_train,
n_epoch=1000,
batch=20,
verbose=False,
reg="l2",
l=l)
out = NN.predict(X_train)
tr_acc += net.accuracy(out, y_train)
out = NN.predict(X_test)
tst_acc += net.accuracy(out, y_test)
tr_acc = tr_acc / 3
tst_acc = tst_acc / 3
return tr_acc, tst_acc
def main():
print("DIGIT RECOGNITION")
print("\nLoading data...")
with np.load("mnist.npz") as data:
trainingSet = np.squeeze(data["training_images"])
trainingLabels = np.squeeze(data["training_labels"])
testingSet = np.squeeze(data["test_images"])
testingLabels = np.squeeze(data["test_labels"])
validationSet = np.squeeze(data["validation_images"])
validationLabels = np.squeeze(data["validation_labels"])
#showImage(testingSet[0].reshape(28, 28))
trainingSize = min(1000, trainingSet.shape[0])
architecture = (784, 24, 10)
nn = neural_net.NeuralNetwork(architecture, activation="sigmoid", cost="crossEntropy")
initialTestingAccuracy = nn.classificationAccuracy(testingSet, testingLabels)
initialGuess = np.argmax(nn.evaluate(testingSet[0]))
timer = time()
costs = nn.train(trainingSet[:trainingSize], trainingLabels[:trainingSize], alpha=3, iterations=3000, threshold=0.00001)
elapsed = round(time()-timer, 3)
trainingAccuracy = nn.classificationAccuracy(trainingSet[:trainingSize], trainingLabels[:trainingSize])
finalTestingAccuracy = nn.classificationAccuracy(testingSet, testingLabels)
print("\nTime elapsed:", elapsed, "sec")
print("\nInitial guess:", initialGuess)
print("Final guess:", np.argmax(nn.evaluate(testingSet[0])))
print("Correct guess:", np.argmax(testingLabels[0]))
print("\nCost always decreases:", all([costs[i+1] < costs[i] for i in range(len(costs)-1)]))
print("\nInitial testing accuracy:", round(initialTestingAccuracy, 3), "%")
print("Training accuracy:", round(trainingAccuracy, 3), "%")
print("Final testing accuracy:", round(finalTestingAccuracy, 3), "%")
t = np.linspace(0, len(costs)-1, len(costs))
plt.plot(t, costs)
plt.show()
save = input("\nOverwrite saved model? (y/n): ")
if save in {"y", "Y"}:
path = os.path.join(os.getcwd(), "saved_model.npz")
np.savez(path, weights=np.array(nn.weights, dtype=object), biases=np.array(nn.biases, dtype=object))
from data_collection import get_data, get_point
import neural_net
import numpy as np
# A newline separated list of tickers
portfolio_file = open('portfolio.txt', 'r')
portfolio = portfolio_file.read().split('\n')
portfolio_file.close()
# Data parameters
start_date = '2014-01-01'
end_date = '2016-06-21'
num_days_per_point = 7
traits = ['Volume', 'High', 'Low', 'Close', 'Open']
for stock in portfolio:
if len(stock) == 0:
continue
training_data = get_data(stock, start_date, end_date,
num_days_per_point, traits)
training_data = neural_net.standardize(training_data)
net = neural_net.NeuralNetwork([num_days_per_point*len(traits), 10, 1])
net.SGD(training_data, 1000, 10, 9, lmbda = 5.0)
tomorrow_input = get_point(stock, '2016-06-13', '2016-06-21', traits)
print net.feedforward(tomorrow_input)
def main():
# argument parser
parser = argparse.ArgumentParser(
description='Train and test a neural network for part-of-speech tagging'
)
parser.add_argument('-t',
'--test',
help='Select to test neural net, instead of training',
action='store_true',
default=False)
parser.add_argument('-i',
'--input',
help='File to read input from',
default=None)
parser.add_argument('-o',
'--output',
help='File to save output to',
default=None)
parser.add_argument('-e',
'--epochs',
help='number of epochs to train for',
type=int,
default=None)
args = parser.parse_args()
# argument errors and warnings
if args.test and args.input is None:
parser.error(
'Testing mode requires a trained neural network input file')
if args.test and args.output is not None:
print('Output option is not active in testing mode', file=sys.stderr)
if args.test and args.epochs is not None:
print('Epochs are not used in testing mode', file=sys.stderr)
# install the needed nltk module, if not present
nltk.download('averaged_perceptron_tagger')
# load the Word2Vec model. This can take a little while, so added text to clarify
print('Loading Word2Vec model, please wait...')
model = gensim.models.KeyedVectors.load_word2vec_format(
'trained-model.bin', binary=True)
print('Word2Vec model loaded.')
# setup for neural network
eta = .01
neural_nets = None
testing = args.test
# set number of epochs if specified, else 1000
if args.epochs is not None:
epochs = args.epochs
else:
epochs = 1000
# if neural net input file specified, open it and load it into the neural nets list
if args.input is not None:
with open(args.input, 'rb') as input_file:
neural_nets = pickle.load(input_file)
# else, create a new random-weighted list of 10 neural nets
if neural_nets is None:
neural_nets = [nn.NeuralNetwork(eta) for __ in range(10)]
# open the sentence input file, remove some words we aren't classifying
with open('sentences') as file:
sentences = [
line.rstrip('\n.,').replace(' to ',
' ').replace(' and ', ' ').replace(
' of ', ' ').replace('To ', '')
for line in file
]
# split sentences into training and testing sets
training_data = sentences[:650]
testing_data = sentences[650:]
# if testing, run through the testing data sequentially
if testing:
for line in testing_data:
# run the neural net with 0 for epoch because it is not needed
run(line, model, neural_nets, 0, testing)
# print accuracy results when done
pct = (correct_guesses / total_guesses) * 100.0
print('%.04f%% correct\n' % pct)
exit(0)
# if training, pick random sentences
else:
for epoch in range(epochs):
line = random.choice(training_data)
run(line, model, neural_nets, epoch, testing)
# if an output file is specified, save the network every 10000 epochs
if epoch % 10000 == 0:
if args.output is not None:
with open(args.output, 'wb') as output_file:
pickle.dump(neural_nets, output_file, -1)
# save the network after training, if output is specified
if args.output is not None and not testing:
with open(args.output, 'wb') as output_file:
pickle.dump(neural_nets, output_file, -1)
exit(0)
def run_cv(df, fold):
train_df = df[df["kfold"] != fold].reset_index(drop=True)
valid_df = df[df["kfold"] == fold].reset_index(drop=True)
#y_train = pd.get_dummies(train_df["target"], dtype="int64").values
y_train = train_df["target"].values
X_train = train_df.drop(["target", "kfold"], axis=1).values
#y_valid = pd.get_dummies(valid_df["target"], dtype="int64").values
y_valid = valid_df["target"].values
X_valid = valid_df.drop(["target", "kfold"], axis=1).values
train_dataset = dataset.TweetDataset(
tweets=X_train,
targets=y_train
)
valid_dataset = dataset.TweetDataset(
tweets=X_valid,
targets=y_valid
)
train_dataloader = torch.utils.data.DataLoader(
train_dataset,
batch_size=config.TRAIN_BATCH_SIZE,
num_workers=2
)
valid_dataloader = torch.utils.data.DataLoader(
valid_dataset,
batch_size=config.TRAIN_BATCH_SIZE,
num_workers=1
)
device = "cuda" if torch.cuda.is_available() else "cpu"
print("Using {} device".format(device))
model = neural_net.NeuralNetwork().to(device)
optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)
loss_fn = torch.nn.BCELoss()
print("Training Model...")
#early_stopping_counter = 0
for epoch in range(config.EPOCHS):
print(f"Epoch {epoch+1}\n--------------------")
engine.train(
train_dataloader,
model,
optimizer,
loss_fn,
device
)
outputs, targets = engine.evaluate(
valid_dataloader,
model,
loss_fn,
device
)
outputs = np.array(outputs).reshape(-1,)
outputs = list(map(lambda pred: 1 if pred>0.5 else 0, outputs))
valid_score = metrics.f1_score(targets, outputs)
print(f" F1 Score: {valid_score}\n")
# Scales training and validation data to be 0 < x <= 1.
scaled_training_data = (training_data - data_min) / (data_max - data_min)
scaled_training_solutions = (training_solutions - data_min) \
/ (data_max - data_min)
scaled_testing_data = (testing_data - data_min) / (data_max - data_min)
scaled_testing_solutions = (testing_solutions - data_min) \
/ (data_max - data_min)
# Lists for plotting loss. Records errors from training and validation.
training_record = []
validation_record = []
for test in range(len(test_runs)):
n = neural_net.NeuralNetwork(set_input_nodes, test_runs[test], set_output_nodes, \
set_learning_rate, set_hidden_layers)
for e in range(epoch):
loss = 0
validation_loss = 0
for record in range(training_size):
error = n.train(scaled_training_data[record, :], \
scaled_training_solutions[0, record])
error = error * (data_max - data_min) + data_min
loss += abs(error)
loss = loss / training_size
training_record.append(loss)
# Checks the trained network against the validation data and records the
# difference.
for record in range(testing_size):
output = n.query(scaled_testing_data[record, :])
import pandas as pd
import numpy as np
import config
import torch
import neural_net
if __name__ == "__main__":
df = pd.read_csv(config.DEV_TEST_FILE)
submission_df = pd.read_csv(config.ARCHIVE_SUBMISSION_FILE)
device = "cuda" if torch.cuda.is_available() else "cpu"
print("Using {} device".format(device))
model = neural_net.NeuralNetwork().to(device)
model.load_state_dict(torch.load(f"{config.MODEL_PATH}/{config.MODEL_NAME}.pth"))
model.eval()
X = torch.tensor(df.values)
X = X.to(device, dtype=torch.float)
predictions = model(X)
predictions = predictions.reshape(-1,)
predictions = list(map(lambda pred: 1 if pred>0.5 else 0, predictions))
submission_df["target"] = predictions
submission_df.to_csv(config.ARCHIVE_SUBMISSION_FILE, index=False)
labels,
train_size=0.8,
test_size=0.2)
X_train = np.array(X_train).reshape((53, 3))
X_test = np.array(X_test).reshape((14, 3))
y_train = np.array(y_train).reshape((53, 1))
y_test = np.array(y_test).reshape((14, 1))
X_train = preprocessing.normalize(X_train)
X_test = preprocessing.normalize(X_test)
y_train = preprocessing.normalize(y_train)
y_test = preprocessing.normalize(y_test)
start_time = time.time()
NN.train(X_train,
y_train,
iterations=1000,
learning_rate=0.01,
regularize=False,
display=True)
end_time = time.time()
total_time = end_time - start_time
print(NN.accuracy(X_test, y_test), 'total time:', total_time)
if __name__ == '__main__':
NN = neural_net.NeuralNetwork(3, 1, 10, 10)
stock_data(NN, "C:/Users/sam/Desktop/HistoricalQuotes.csv")
def simulate(samples, features=2, data_="blobs", architecture=[2, 2, 1]):
"""
"""
# Perhaps encode a forcing of an exception
# in case the data_ is not in a desired list
# of predefined strings
# By default, cluster_std = 1.0 in blobs and random_state = 2
# Just to link the code variables with my mathematical notations:
# Lplus2 = len(architecture)
datasets_ = {
'blobs':
datasets.make_blobs(n_samples=samples,
n_features=features,
centers=2,
cluster_std=1,
random_state=2),
'spirals':
datasets.make_moons(n_samples=samples, noise=0.2),
'chess':
generate_points(samples, [(0, 0), (0, 1), (1, 0), (1, 1)],
[0, 1, 1, 0], 0.005),
'q_random':
generate_points_1d(nb=samples, centers=[-1, 0, 1], labels=[1, 0, 1])
}
data = datasets_[data_]
if data_ == "chess" or data_ == "q_random":
X, y = data
else:
X = data[0].T
y = np.expand_dims(data[1], 1).T
network = nn.NeuralNetwork(architecture, seed=0)
history = network.train(X=X,
y=y,
batch_size=24,
epochs=20000,
learning_rate=0.2,
print_every=1000,
validation_split=0.2,
tqdm_=False,
plot_every=100)
weights, biases = history['weights'], history['biases']
# Initialize the neural network scheme
# np.shape(z0) = (input_dimension, samples)
# Donc c'est un "vecteur" de input_dimension lignes
# et samples colonnes. Géneralement input_dimension = 1, 2
z0 = X
# We begin by checking if it's 1d. If yes, then we cannot use
# scatter plot.
if np.shape(z0)[0] == 1:
# Red will contain the points whose labels correspond to 1,
# which we will paint in red
red = list()
# Blue will contain the points whose labels correspond to 0,
# which we will paint in blue
blue = list()
for i, x in enumerate(z0[0]):
if y[0][i] == 0:
blue.append(x)
elif y[0][i] == 1:
red.append(x)
else:
raise ValueError
#x_min, x_max = min(blue+red)-0.1, max(blue+red)+0.1
#major_ticks_x = np.linspace(x_min, x_max, 11)
#minor_ticks_x = np.linspace(x_min, x_max, 21)
#major_ticks_y = np.linspace(-0.5, 0.5, 11)
#minor_ticks_y = np.linspace(-0.5, 0.5, 21)
fig = plt.figure(figsize=(13.5, 5))
ax = fig.add_subplot(1, 1, 1)
x_min, x_max = min(blue + red) - 0.25, max(blue + red) + 0.25
major_ticks_x = np.linspace(x_min, x_max, 7)
minor_ticks_x = np.linspace(x_min, x_max, 13)
major_ticks_y = np.linspace(-0.2, 0.2, 3)
#minor_ticks_y = np.linspace(-0.2, 0.2, 5)
ax.set_xticks(major_ticks_x)
ax.set_xticks(minor_ticks_x, minor=True)
ax.set_yticks(major_ticks_y)
#ax.set_yticks(minor_ticks_y, minor=True)
ax.grid(which='minor', alpha=0.25, ls='-.')
ax.grid(which='major', alpha=0.75, ls='-.')
ax.plot(red, len(red) * [0.2], 'o', c='r', alpha=0.0)
ax.plot(red, len(red) * [-0.2], 'o', c='r', alpha=0.0)
plt.plot([x_min, x_max], [0, 0], c='black', alpha=0.35)
ax.plot(red, len(red) * [0], 'o', c='r', alpha=0.95, markersize=9)
ax.plot(blue, len(blue) * [0], 'o', c='b', alpha=0.95, markersize=9)
plt.xlim(x_min, x_max)
plt.ylim(-0.2, 0.2)
ax.xaxis.set_major_formatter(FormatStrFormatter('%.2f'))
ax.yaxis.set_major_formatter(FormatStrFormatter('%.2f'))
ax.tick_params(axis='both', which='major', labelsize=22)
ax.set_yticklabels(['', '0', ''])
plt.title(r'The N={} data points'.format(samples),
fontdict={'fontsize': 24})
elif np.shape(z0)[0] == 2:
x_min, x_max = min(z0.T[:, 0]), max(z0.T[:, 0])
y_min, y_max = min(z0.T[:, 1]), max(z0.T[:, 1])
major_ticks_x = np.linspace(x_min, x_max, 5)
minor_ticks_x = np.linspace(x_min, x_max, 9)
major_ticks_y = np.linspace(y_min, y_max, 5)
minor_ticks_y = np.linspace(y_min, y_max, 9)
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)
ax.set_xticks(major_ticks_x)
ax.set_xticks(minor_ticks_x, minor=True)
ax.set_yticks(major_ticks_y)
ax.set_yticks(minor_ticks_y, minor=True)
# Or if you want different settings for the grids:
ax.grid(which='minor', alpha=0.15)
ax.grid(which='major', alpha=0.3)
plt.xlim(x_min - 0.01, x_max + 0.01)
plt.ylim(y_min - 0.01, y_max + 0.01)
ax.tick_params(axis='both', which='major', labelsize=22)
plt.scatter(z0.T[:, 0],
z0.T[:, 1],
c=y.T.reshape(-1),
s=60,
cmap=plt.cm.coolwarm,
alpha=0.95)
plt.xlabel(r'$x_1$', fontdict={'fontsize': 16})
plt.ylabel(r'$x_2$', fontdict={'fontsize': 16})
plt.title(r'The N={} data points'.format(samples),
fontdict={'fontsize': 24})
else:
raise TypeError
# /!\ Faut que je repare ça /!\
Path('figures/visual_trans/{}/{}/{}d'.format(
data_, samples, architecture[1])).mkdir(parents=True, exist_ok=True)
plt.savefig('figures/visual_trans/{}/{}/{}d/0.svg'.format(
data_, samples, architecture[1]),
format='svg')
# We are now in a position to store the information from every transition.
# We store the linear transformations (\Lambda_1 z^0, \Lambda_2 z^1 etc.)
# in the following array
transitions = []
# We store in layers the activations of each linear transformation, i.e.
# z^0, z^1 = \sigma(\Lambda_1 z^0), z^2 = \sigma(\Lambda_2 z^1) etc.
# We also store z^{L+1}. (thus the list should be of length L+2)
layers = [z0]
# We have the scheme z^{k} = \sigma(A^{k-1} z^{k-1} + b^{k-1})
# for k = 1, ..., L; k represents the layers
for k in range(1, len(architecture)):
zkmoins1_list = copy.deepcopy(layers[k - 1])
# We initialize \Lambda_{k} as an array with
# N_{k} rows and samples columns, while
# z^{k} has N_k rows and samples columns.
# They both have samples columns, because we
# vectorize and compute over all of the data points
Lambdak_list = np.zeros((architecture[k], samples))
zk_list = np.zeros((architecture[k], samples))
for i in range(samples):
# We first compute the linear transitions
# The reshape has the effect of giving an array of N_{k-1} rows
# and 1 column (1 column because we do this for all i)
Lambdak_datai = network.Lambda(
k, zkmoins1_list[:, i].reshape(architecture[k - 1], 1))
# We fill the i-th column of every row with this linear transition
# Things should be consistent because Lambdak has N_k rows, and
# every element will have N_{k-1} columns
Lambdak_list[:, i] = Lambdak_datai.reshape(-1)
zk_list[:, i] = nn.sigmoid(Lambdak_datai).reshape(-1)
transitions.append(Lambdak_list)
layers.append(zk_list)
# We plot the transitions, up to dimension 3 (cannot do higher)
for i, lbd in enumerate(transitions):
if len(lbd) == 2:
x_min, x_max = min(lbd[0, :]) - 0.25, max(lbd[0, :]) + 0.25
y_min, y_max = min(lbd[1, :]) - 0.25, max(lbd[1, :]) + 0.25
major_ticks_x = np.linspace(x_min, x_max, 7)
minor_ticks_x = np.linspace(x_min, x_max, 13)
major_ticks_y = np.linspace(y_min, y_max, 7)
minor_ticks_y = np.linspace(y_min, y_max, 13)
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)
ax.set_xticks(major_ticks_x)
ax.set_xticks(minor_ticks_x, minor=True)
ax.set_yticks(major_ticks_y)
ax.set_yticks(minor_ticks_y, minor=True)
# Or if you want different settings for the grids:
ax.grid(which='minor', alpha=0.25, ls='-.')
ax.grid(which='major', alpha=0.75, ls='-.')
ax.tick_params(axis='both', which='major', labelsize=22)
plt.xlim(x_min, x_max + 0.01)
plt.ylim(y_min, y_max + 0.01)
ax.xaxis.set_major_formatter(FormatStrFormatter('%.2f'))
ax.yaxis.set_major_formatter(FormatStrFormatter('%.2f'))
plt.scatter(lbd.T[:, 0],
lbd.T[:, 1],
c=y.T.reshape(-1),
s=60,
cmap=plt.cm.coolwarm,
alpha=0.95)
plt.xlabel(r'$(\Lambda_{}z^{})_{{1}}$'.format(i + 1, i),
fontdict={'fontsize': 16},
labelpad=15)
plt.ylabel(r'$(\Lambda_{}z^{})_{{2}}$'.format(i + 1, i),
fontdict={'fontsize': 16})
elif len(lbd) == 3:
x_min, x_max = min(lbd[0, :]) - 0.15, max(lbd[0, :]) + 0.15
y_min, y_max = min(lbd[1, :]) - 0.15, max(lbd[1, :]) + 0.15
z_min, z_max = min(lbd[2, :]) - 0.15, max(lbd[2, :]) + 0.15
major_ticks_x = np.linspace(x_min, x_max, 5)
minor_ticks_x = np.linspace(x_min, x_max, 9)
major_ticks_y = np.linspace(y_min, y_max, 5)
minor_ticks_y = np.linspace(y_min, y_max, 9)
fig = plt.figure()
ax = Axes3D(fig)
ax.scatter(lbd.T[:, 0],
lbd.T[:, 1],
lbd.T[:, 2],
c=y.T.reshape(-1),
s=60,
cmap=plt.cm.coolwarm,
alpha=0.95)
plt.xlim(x_min, x_max)
plt.ylim(y_min, y_max)
plt.xlabel(r'$(\Lambda_{}z^{})_{{1}}$'.format(i + 1, i),
fontdict={'fontsize': 16},
labelpad=25)
plt.ylabel(r'$(\Lambda_{}z^{})_{{2}}$'.format(i + 1, i),
fontdict={'fontsize': 16},
labelpad=25)
ax.tick_params(axis='both', which='major', labelsize=22)
#plt.xlabel(r'$1$st coordinate', fontdict = {'fontsize' : 16})
#plt.ylabel(r'$2$nd coordinate', fontdict = {'fontsize' : 16})
plt.title(
r'Linear transformation nb.{}: $\Lambda_{}z^{} = A^{} z^{} + b^{}$'
.format(i + 1, i + 1, i, i, i, i),
fontdict={'fontsize': 24})
for angle in range(25, 70):
ax.view_init(elev=5, azim=-angle)
#plt.savefig('figures/visual_trans/{}/{}/{}d/{}angle{}.svg'.format(data_, samples, architecture[1], 2*i, angle), format='svg')
plt.savefig(
'figures/visual_trans/{}/{}/{}d/{}angle{}.png'.format(
data_, samples, architecture[1], 2 * i, angle))
elif len(lbd) == 1:
red = list()
blue = list()
lbd = copy.copy(lbd.reshape(-1))
for j, e in enumerate(lbd):
if y[0][j] == 0:
blue.append(e)
else:
red.append(e)
x_min, x_max = min(blue + red) - 0.25, max(blue + red) + 0.25
major_ticks_x = np.linspace(x_min, x_max, 7)
minor_ticks_x = np.linspace(x_min, x_max, 13)
major_ticks_y = np.linspace(-0.2, 0.2, 3)
#minor_ticks_y = np.linspace(-0.2, 0.2, 13)
fig = plt.figure(figsize=(13.5, 5))
ax = fig.add_subplot(1, 1, 1)
ax.set_xticks(major_ticks_x)
ax.set_xticks(minor_ticks_x, minor=True)
ax.set_yticks(major_ticks_y)
#ax.set_yticks(minor_ticks_y, minor=True)
# Or if you want different settings for the grids:
ax.grid(which='minor', alpha=0.25, ls='-.')
ax.grid(which='major', alpha=0.75, ls='-.')
ax.plot(red, len(red) * [0.2], 'o', c='r', alpha=0.0)
ax.plot(red, len(red) * [-0.2], 'o', c='r', alpha=0.0)
plt.plot([x_min, x_max], [0, 0], c='black', alpha=0.35)
ax.plot(blue,
len(blue) * [0],
'o',
c='b',
alpha=0.95,
markersize=9)
ax.plot(red, len(red) * [0], 'o', c='r', alpha=0.95, markersize=9)
plt.xlim(x_min - 0.01, x_max + 0.01)
plt.ylim(-0.2, 0.2)
ax.xaxis.set_major_formatter(FormatStrFormatter('%.2f'))
ax.yaxis.set_major_formatter(FormatStrFormatter('%.2f'))
ax.tick_params(axis='both', which='major', labelsize=22)
ax.set_yticklabels(['', '0', ''])
else:
pass
plt.title(
r'Linear transformation nb.{}: $\Lambda_{}z^{} = A^{} z^{} + b^{}$'
.format(i + 1, i + 1, i, i, i, i),
fontdict={'fontsize': 24})
plt.savefig('figures/visual_trans/{}/{}/{}d/{}.svg'.format(
data_, samples, architecture[1], 2 * i + 1),
format='svg')
for i, sig in enumerate(layers):
if i == 0:
pass
else:
if len(sig) == 2:
major_ticks = np.linspace(0.0, 1.0, 7)
minor_ticks = np.linspace(0.0, 1.0, 13)
fig = plt.figure(figsize=(10, 10))
ax = fig.add_subplot(1, 1, 1)
ax.set_xticks(major_ticks)
ax.set_xticks(minor_ticks, minor=True)
ax.set_yticks(major_ticks)
ax.set_yticks(minor_ticks, minor=True)
# Or if you want different settings for the grids:
ax.grid(which='minor', alpha=0.25, ls='-.')
ax.grid(which='major', alpha=0.75, ls='-.')
plt.xlim(-0.01, 1.01)
plt.ylim(-0.01, 1.01)
plt.scatter(sig.T[:, 0],
sig.T[:, 1],
c=y.T.reshape(-1),
s=60,
cmap=plt.cm.coolwarm,
alpha=0.95)
plt.xlabel(r'$z^{}_1$'.format(i),
fontdict={'fontsize': 16},
labelpad=15)
plt.ylabel(r'$z^{}_2$'.format(i), fontdict={'fontsize': 16})
plt.title(
r'Hidden layer nb.{}: $z^{} = \sigma(A^{} z^{} + b^{})$'.
format(i, i, i - 1, i - 1, i - 1),
fontdict={'fontsize': 24})
ax.tick_params(axis='both', which='major', labelsize=22)
ax.xaxis.set_major_formatter(FormatStrFormatter('%.2f'))
ax.yaxis.set_major_formatter(FormatStrFormatter('%.2f'))
elif len(sig) == 3:
fig = plt.figure()
ax = Axes3D(fig)
ax.scatter(sig.T[:, 0],
sig.T[:, 1],
sig.T[:, 2],
c=y.T.reshape(-1),
s=60,
cmap=plt.cm.coolwarm,
alpha=0.95)
plt.xlabel(r'$z^{}_1$'.format(i),
fontdict={'fontsize': 16},
labelpad=25)
plt.ylabel(r'$z^{}_2$'.format(i),
fontdict={'fontsize': 16},
labelpad=25)
plt.xlim(-0.05, 1.05)
plt.ylim(-0.05, 1.0)
ax.set_zlim(0.0, 1.0)
plt.title(
r'Hidden layer nb.{}: $z^{} = \sigma(A^{} z^{} + b^{})$'.
format(i, i, i - 1, i - 1, i - 1),
fontdict={'fontsize': 24})
ax.tick_params(axis='both', which='major', labelsize=22)
ax.xaxis.set_major_formatter(FormatStrFormatter('%.1f'))
ax.yaxis.set_major_formatter(FormatStrFormatter('%.1f'))
for angle in range(25, 70):
ax.view_init(elev=5, azim=-angle)
#plt.savefig('figures/visual_trans/{}/{}/{}d/{}angle{}.svg'.format(data_, samples, architecture[1], 2*i, angle), format='svg')
plt.savefig(
'figures/visual_trans/{}/{}/{}d/{}angle{}.png'.format(
data_, samples, architecture[1], 2 * i, angle))
elif len(sig) == 1:
red = list()
blue = list()
sig = copy.copy(sig.reshape(-1))
for j, e in enumerate(sig):
if y[0][j] == 0:
blue.append(e)
else:
red.append(e)
if i == len(layers) - 1:
fig = plt.figure(figsize=(13.5, 5))
plt.title(
r'Output of network: $z^{} = \sigma(A^{} z^{} + b^{}) \in [0, 1]$'
.format(i, i - 1, i - 1, i - 1, i - 1),
fontdict={'fontsize': 24})
else:
fig = plt.figure(figsize=(13.5, 5))
plt.title(
r'Hidden layer nb.{}: $z^{} = \sigma(A^{} z^{} + b^{}) \in [0, 1]^{{}}$'
.format(i, i, i - 1, i - 1, i - 1, architecture[i]),
fontdict={'fontsize': 24})
ax = fig.add_subplot(1, 1, 1)
major_ticks_x = np.linspace(-0.01, 1.01, 7)
minor_ticks_x = np.linspace(-0.01, 1.01, 13)
major_ticks_y = np.linspace(-0.2, 0.2, 3)
#minor_ticks_y = np.linspace(-0.2, 0.2, 13)
ax.set_xticks(major_ticks_x)
ax.set_xticks(minor_ticks_x, minor=True)
ax.set_yticks(major_ticks_y)
#ax.set_yticks(minor_ticks_y, minor=True)
ax.xaxis.set_major_formatter(FormatStrFormatter('%.2f'))
# Or if you want different settings for the grids:
ax.grid(which='minor', alpha=0.25, ls='-.')
ax.grid(which='major', alpha=0.75, ls='-.')
#plt.yticks(color='w')
plt.plot([-0.01, 1.01], [0, 0], c='black', alpha=0.35)
plt.plot(blue, len(blue) * [-0.2], 'o', c='b', alpha=0.0)
plt.plot(red, len(red) * [0.2], 'o', c='r', alpha=0.0)
plt.plot(blue,
len(blue) * [0],
'o',
c='b',
markersize=9,
alpha=0.95)
plt.plot(red,
len(red) * [0],
'o',
c='r',
markersize=9,
alpha=0.95)
ax.xaxis.set_major_formatter(FormatStrFormatter('%.2f'))
ax.yaxis.set_major_formatter(FormatStrFormatter('%.2f'))
#plt.xlabel(r'$z^{}$ coordinate'.format(i), fontdict = {'fontsize' : 16})
plt.xlim(-0.01, 1.01)
plt.ylim(-0.2, 0.2)
ax.tick_params(axis='both', which='major', labelsize=22)
#ax.set_xticklabels(['0.0', '', '', '0.5', '', '', '1.0'])
ax.xaxis.set_major_formatter(FormatStrFormatter('%.1f'))
ax.set_yticklabels(['', '0', ''])
else:
pass
plt.savefig('figures/visual_trans/{}/{}/{}d/{}.svg'.format(
data_, samples, architecture[1], 2 * i),
format='svg')
#### We generate a 2d grid
x_list = np.arange(-0.1, 1.1, 0.01)
y_list = np.arange(-0.1, 1.1, 0.01)
if architecture[1] == 2:
xx, yy = np.meshgrid(x_list, y_list)
f1 = lambda x, y: nn.sigmoid(network.Lambda(2, np.array([[x], [y]])))[
0, 0]
res = np.array([[f1(x, y) for x in x_list] for y in y_list])
major_ticks = np.arange(-0.1, 1.1, 0.2)
minor_ticks = np.arange(-0.1, 1.1, 0.1)
fig = plt.figure(figsize=(12, 10))
ax = fig.add_subplot(1, 1, 1)
ax.set_xticks(major_ticks)
ax.set_xticks(minor_ticks, minor=True)
ax.set_yticks(major_ticks)
ax.set_yticks(minor_ticks, minor=True)
# Or if you want different settings for the grids:
ax.grid(which='minor', alpha=0.15, ls='-.')
ax.grid(which='major', alpha=0.5, ls='-.')
plt.contourf(xx, yy, res, cmap=plt.cm.coolwarm, alpha=0.45)
plt.scatter(layers[1].T[:, 0],
layers[1].T[:, 1],
c=y.T.reshape(-1),
s=60,
cmap=plt.cm.coolwarm,
alpha=0.95)
cbar = plt.colorbar(plt.cm.ScalarMappable(cmap=plt.cm.coolwarm),
alpha=0.4)
#cbar.set_clim(0, 1)
droite = plt.contour(xx,
yy,
res,
levels=[0.5],
c='b',
linewidth=2.25,
alpha=0.75)
plt.clabel(droite, inline=1, fontsize=16)
droite.collections[0].set_label(
r'Level line $\{ x \in [0, 1]^2 \colon \sigma(A^1 x + b^1) = 0.5 \}$'
)
ax.tick_params(axis='both', which='major', labelsize=22)
plt.xlabel(
r'$x_1$',
fontdict={'fontsize': 16},
)
plt.ylabel(r'$x_2$', fontdict={'fontsize': 16})
plt.title(r'Level sets of $x \mapsto \sigma(A^1 x + b^1)$',
fontdict={'fontsize': 24})
plt.legend(loc='upper center',
bbox_to_anchor=(0.5, -0.075),
fancybox=True,
shadow=True,
ncol=5,
fontsize=18)
plt.savefig('figures/visual_trans/{}/{}/{}d/{}.svg'.format(
data_, samples, architecture[1], '2-5'),
format='svg')
if architecture[1] == 3:
x_list = np.arange(0.0, 1.0, 0.005)
y_list = np.arange(0.0, 1.0, 0.005)
a, b, c, d = weights[1][0][0], weights[1][0][1], weights[1][0][
2], biases[1][0][0]
X, Y = np.meshgrid(x_list, y_list)
fig = plt.figure(figsize=(12, 15))
ax = Axes3D(fig)
major_ticks = np.arange(0.0, 1.0, 0.2)
minor_ticks = np.arange(0.0, 1.0, 0.1)
x_ = np.linspace(-0.001, 1, 4)
y_ = np.linspace(-0.001, 1, 4)
X_, Y_ = np.meshgrid(x_, y_)
Z = (-d - a * X_ - b * Y_) / c
surf = ax.plot_surface(
X_,
Y_,
Z,
color='skyblue',
alpha=0.55,
linewidth=1,
label=
r'Level plane: $\{x \in [0, 1]^3 \colon \sigma(A^1 x+b^1)=0.5 \}$')
surf._facecolors2d = surf._facecolors3d
surf._edgecolors2d = surf._edgecolors3d
ax.scatter(layers[1].T[:, 0],
layers[1].T[:, 1],
layers[1].T[:, 2],
c=y.T.reshape(-1),
s=60,
cmap=plt.cm.coolwarm,
alpha=0.95)
plt.xlim(-0.05, 1.05)
plt.ylim(-0.05, 1.0)
ax.set_zlim(0.0, 1.0)
ax.tick_params(axis='both', which='major', labelsize=22)
ax.legend(loc='upper center',
bbox_to_anchor=(0.5, -0.001),
fancybox=True,
shadow=True,
ncol=5,
fontsize=18)
plt.xlabel(r'$z^{}_1$'.format(i),
fontdict={'fontsize': 16},
labelpad=25)
plt.ylabel(r'$z^{}_2$'.format(i),
fontdict={'fontsize': 16},
labelpad=25)
plt.title(
r'Separation before output layer: $z^{} = \sigma(A^{} z^{} + b^{})$'
.format(i, i - 1, i - 1, i - 1),
fontdict={'fontsize': 24})
for angle in range(25, 70):
ax.view_init(elev=5, azim=-angle)
#plt.savefig('figures/visual_trans/{}/{}/{}d/{}angle{}.svg'.format(data_, samples, architecture[1], '2-5', angle), format='svg')
plt.savefig('figures/visual_trans/{}/{}/{}d/{}angle{}.png'.format(
data_, samples, architecture[1], '2-5', angle))
