def main(model_name: str):
output_model = "models/" + model_name + ".pth"
writer = SummaryWriter()
# Init datasets and loaders
train_set = PneumoniaDataset(SetType.val)
val_set = PneumoniaDataset(SetType.val, shuffle=False) # for now.
final_check_set = PneumoniaDataset(SetType.test, shuffle=False) # for now.
train_loader = DataLoader(train_set,
batch_size=1,
shuffle=True,
num_workers=8)
val_loader = DataLoader(val_set,
batch_size=8,
shuffle=False,
num_workers=8)
final_loader = DataLoader(final_check_set,
batch_size=8,
shuffle=False,
num_workers=8)
for dataset in [train_set, val_set, final_check_set]:
print(f"Size of {dataset.set_type} set: {len(dataset)}")
# Init network, loss and optimizer
net = SimpleNet().cuda()
# There are twice as much pneumonia as healthy, offset the bias in the loss.
criterion = ContrastiveLoss(margin=1.0)
optimizer = optim.Adam(net.parameters(), lr=1e-3)
scheduler = ReduceLROnPlateau(optimizer,
factor=0.3,
mode="max",
verbose=True,
patience=15)
# Training Loop
train_iter = 0
for epoch in range(100):
running_loss = 0.0
for i, (input1, input2, distance) in enumerate(train_loader):
net.train()
optimizer.zero_grad()
vector1, vector2 = net(input1.float().cuda(),
input2.float().cuda())
loss = criterion(vector1.cpu(), vector2.cpu(), distance)
loss.backward()
optimizer.step()
running_loss += loss.item()
duration = 10
if i > 0 and (i) % duration == 0:
print("[%d, %5d] loss %.3f" %
(epoch, i, running_loss / duration))
writer.add_scalar("TrainLoss", (running_loss / duration),
train_iter)
train_iter += 1
running_loss = 0.0
def __init__(
self, model_name: str, epochs: int = 100, config: Optional[dict] = None
):
self.output_model = model_name + ".pth"
self.train_set = PneumoniaDataset(SetType.train)
self.train_loader = DataLoader(
PneumoniaDataset(SetType.train), batch_size=16, shuffle=True, num_workers=8
)
self.val_loader = DataLoader(
PneumoniaDataset(SetType.val, shuffle=False),
batch_size=16,
shuffle=False,
num_workers=8,
)
self.test_loader = DataLoader(
PneumoniaDataset(SetType.test, shuffle=False),
batch_size=16,
shuffle=False,
num_workers=8,
)
self.config = {
"pos_weight_bias": 0.5,
"starting_lr": 1e-2,
"momentum": 0.9,
"decay": 5e-4,
"lr_adjustment_factor": 0.3,
"scheduler_patience": 15,
"print_cadence": 100,
"comment": "Added large dense layer.",
"pos_weight": 1341 / 3875, # Number of negatives / positives.
}
self.epochs = epochs
self.device = torch.device("cuda:0")
self.writer = SummaryWriter(comment=self.config["comment"])
self.net = SimpleNet(1).to(self.device)
self.criterion = nn.BCEWithLogitsLoss(
pos_weight=torch.tensor(self.config["pos_weight"])
)
self.optimizer = optim.SGD(
self.net.parameters(),
lr=self.config["starting_lr"], # type: ignore
momentum=self.config["momentum"], # type: ignore
weight_decay=self.config["decay"], # type: ignore
)
self.scheduler = ReduceLROnPlateau(
self.optimizer,
factor=self.config["lr_adjustment_factor"], # type: ignore
mode="max",
verbose=True,
patience=self.config["scheduler_patience"], # type: ignore
)
print("Trainer Initialized.")
for dataset in [self.train_loader, self.test_loader, self.val_loader]:
print(f"Size of set: {len(dataset)}")
def execute_similarity_search(img_path, num_algs=32):
# this part can be refactored into a "setup model" func
# the func can have a flag for initing the hook or not.
path = "../saved_models/cifar_model_300_epochs.pth"
model = SimpleNet(10)
model.load_state_dict(torch.load(path))
# Init forward hook for exposing dense layer vectors
sf = SaveFeatureVectors(model.conv13[2])
model.eval()
is_correct = 0.0
path_list = []
for inputs, labels, paths in _get_single_image_batch(img_path):
outputs = model(inputs.float())
_, pred = outputs.max(1)
is_correct += (pred == labels.long()).sum().item()
print(f"Num correct: {is_correct}")
path_list.extend(paths)
# Generate Hash: todo-try stuff here.
m = MinHash(num_perm=num_algs)
m.update(sf.features.flatten())
def execute_inference(create_hashes=False,
num_algs=32,
hash_index="lsh-demo-3"):
path = "../saved_models/cifar_model_300_epochs.pth"
model = SimpleNet(10)
model.load_state_dict(torch.load(path))
# Init forward hook for exposing dense layer vectors
sf = SaveFeatureVectors(model.conv13[2])
model.eval()
is_correct = 0.0
path_list = []
for inputs, labels, paths in _get_test_batch():
outputs = model(inputs.float())
_, pred = outputs.max(1)
is_correct += (pred == labels.long()).sum().item()
print(f"Num correct: {is_correct}")
path_list.extend(paths)
print("Extracting Vectors")
if create_hashes:
generate_hash_tables(path_list, sf.features)
random_state=0) #print(X_test.shape , y_test.shape ) x_train = np.array(x_train, dtype=np.float32).reshape(len(x_train), 3) y_train = np.array(y_train, dtype=np.float32).reshape(len(y_train), 1) x_test = np.array(x_test, dtype=np.float32).reshape(len(x_test), 3) y_test = np.array(y_test, dtype=np.float32).reshape(len(y_test), 1) print(x_train.shape, y_train.shape) print(x_test.shape, y_test.shape) print(x_train[:10]) #print(type(x_train )) #quit() #quit() # network = SimpleNet(input_size=3, hidden_size=10, output_size=1) #iters_num = 30000 # 繰り返しの回数を適宜設定する iters_num = 10000 # 繰り返しの回数を適宜設定する train_size = x_train.shape[0] print(train_size) #quit() # global_start_time = time.time() # batch_size = 100 #batch_size = 32 batch_size = 8
class PneumoniaTrainer:
def __init__(
self, model_name: str, epochs: int = 100, config: Optional[dict] = None
):
self.output_model = model_name + ".pth"
self.train_set = PneumoniaDataset(SetType.train)
self.train_loader = DataLoader(
PneumoniaDataset(SetType.train), batch_size=16, shuffle=True, num_workers=8
)
self.val_loader = DataLoader(
PneumoniaDataset(SetType.val, shuffle=False),
batch_size=16,
shuffle=False,
num_workers=8,
)
self.test_loader = DataLoader(
PneumoniaDataset(SetType.test, shuffle=False),
batch_size=16,
shuffle=False,
num_workers=8,
)
self.config = {
"pos_weight_bias": 0.5,
"starting_lr": 1e-2,
"momentum": 0.9,
"decay": 5e-4,
"lr_adjustment_factor": 0.3,
"scheduler_patience": 15,
"print_cadence": 100,
"comment": "Added large dense layer.",
"pos_weight": 1341 / 3875, # Number of negatives / positives.
}
self.epochs = epochs
self.device = torch.device("cuda:0")
self.writer = SummaryWriter(comment=self.config["comment"])
self.net = SimpleNet(1).to(self.device)
self.criterion = nn.BCEWithLogitsLoss(
pos_weight=torch.tensor(self.config["pos_weight"])
)
self.optimizer = optim.SGD(
self.net.parameters(),
lr=self.config["starting_lr"], # type: ignore
momentum=self.config["momentum"], # type: ignore
weight_decay=self.config["decay"], # type: ignore
)
self.scheduler = ReduceLROnPlateau(
self.optimizer,
factor=self.config["lr_adjustment_factor"], # type: ignore
mode="max",
verbose=True,
patience=self.config["scheduler_patience"], # type: ignore
)
print("Trainer Initialized.")
for dataset in [self.train_loader, self.test_loader, self.val_loader]:
print(f"Size of set: {len(dataset)}")
def train(self):
training_pass = 0
for epoch in range(self.epochs):
running_loss = 0.0
for i, (inputs, labels, metadata) in enumerate(self.train_loader):
self.net.train()
self.optimizer.zero_grad()
outputs = self.net(inputs.float().to(self.device))
loss = self.criterion(
outputs, labels.unsqueeze(1).float().to(self.device)
)
loss.backward()
self.optimizer.step()
running_loss += loss.item()
if i > 0 and i % self.config["print_cadence"] == 0:
mean_loss = running_loss / self.config["print_cadence"]
print(
f'Epoch: {epoch}\tBatch: {i}\tLoss: {mean_loss}'
)
self.writer.add_scalar(
"Train/RunningLoss",
mean_loss,
training_pass,
)
running_loss = 0.0
training_pass += 1
train_accuracy = self.log_training_metrics(epoch)
self.log_validation_metrics(epoch)
self.scheduler.step(train_accuracy)
accuracy, metrics = self.calculate_accuracy(self.test_loader)
self.writer.add_text("Test/Accuracy", f"{accuracy}")
for key, val in metrics.items():
self.writer.add_text(f"Test/{key}", f"{val}")
self.save_model()
def log_training_metrics(self, epoch: int):
accuracy, metrics = self.calculate_accuracy(self.train_loader)
self.writer.add_scalar(f"Train/Accuracy", accuracy, epoch)
for key, val in metrics.items():
self.writer.add_scalar(f"Train/{key}", val, epoch)
return accuracy
def log_validation_metrics(self, epoch: int):
accuracy, metrics = self.calculate_accuracy(self.val_loader)
self.writer.add_scalar("Validation/Accuracy", accuracy, epoch)
for key, val in metrics.items():
self.writer.add_scalar(f"Validation/{key}", val, epoch)
return accuracy
def calculate_accuracy(self, loader: DataLoader):
truth_list: list = []
pred_list: list = []
with torch.no_grad():
self.net.eval()
correct = 0.0
total = 0.0
for inputs, labels, metadata in loader:
outputs = self.net(inputs.float().to(self.device))
sigmoid = torch.nn.Sigmoid()
preds = sigmoid(outputs)
preds = np.round(preds.detach().cpu().squeeze(1))
pred_list.extend(preds) # type: ignore
truth_list.extend(labels)
total += labels.size(0)
correct += preds.eq(labels.float()).sum().item()
print(f"Correct:\t{correct}, Incorrect:\t{total-correct}")
tn, fp, fn, tp = confusion_matrix(truth_list, pred_list).ravel()
metrics = {
"Recall": tp / (tp + fn),
"Precision": tp / (tp + fp),
"FalseNegativeRate": fn / (tn + fn),
"FalsePositiveRate": fp / (tp + fp),
}
return correct / total, metrics
def save_model(self):
print("saving...")
torch.save(self.net.state_dict(), self.output_model)
def _load_model(self, model_path: str) -> SimpleNet:
model = SimpleNet(1)
model.load_state_dict(torch.load(model_path))
return model
test_size=0.25,
random_state=0)
#print(X_test.shape , y_test.shape )
x_train = np.array(x_train, dtype=np.float32).reshape(len(x_train), 3)
y_train = np.array(y_train, dtype=np.float32).reshape(len(y_train), 1)
x_test = np.array(x_test, dtype=np.float32).reshape(len(x_test), 3)
y_test = np.array(y_test, dtype=np.float32).reshape(len(y_test), 1)
print(x_train.shape, y_train.shape)
print(x_test.shape, y_test.shape)
print(x_train[:10])
#quit()
# load model
# network = SimpleNet(input_size=1 , hidden_size=10, output_size=1 )
network = SimpleNet(input_size=3, hidden_size=10, output_size=1)
network.load_params("params.pkl")
#print( network.params["W1"] )
#pred
train_acc = network.accuracy(x_train, y_train)
test_acc = network.accuracy(x_test, y_test)
#
print("train acc, test acc | " + str(train_acc) + ", " + str(test_acc))
#
# x_test_dt= conv_num_date(x_test_pred )
# x_train_dt= conv_num_date(x_train )
#print(x_test_dt.shape )
print(x_test[:10])
y_val = network.predict(x_test[:10])
y_val = y_val * num_max_y
print(y_val)
from simple_net import SimpleNet
from predictFromArchive import predictFromArchive
import torch
from glob import glob
import os
networkWeights = 'trained_simplenet.torch'
wantedShape = (41, 53, 38, 6)
crop = (slice(4, 28), slice(20, 44), slice(7, 31))
resizeFactor = 2
dimIn = 12 * 12 * 12 * 6
dimOut = 4
net = SimpleNet(dimIn=dimIn, dimOut=dimOut).cpu()
net.load_state_dict(torch.load(networkWeights, map_location='cpu'))
archivPath = os.path.dirname(os.path.abspath(__file__))
for archiv in glob("/flywheel/v0/input/dtiInitArchive/dti*.zip"):
predictFromArchive(archiv, "/flywheel/v0/output/neuro-detect_report.txt",
net)
import numpy as np
import sys
sys.path.append('./src')
sys.path.append('./src/lib')
from simple_net import SimpleNet
from gradient import numerical_gradient
net = SimpleNet()
print(net.W)
# >>> [[-0.44439281 0.30789016 -1.50579685]
# [-0.93170709 0.08170439 -0.12740328]]
x = np.array([0.6, 0.9])
p = net.predict(x)
print(p)
# >>> [ 1.00824761 -1.47819523 0.03650346]
print(np.argmax(p))
# >> 1
t = np.array([0, 0, 1])
print(net.loss(x, t))
# >>> 1.704819611629646
f = lambda w: net.loss(x, t)
dW = numerical_gradient(f, net.W)
print(dW)
# >>> [[ 0.09999078 0.39092591 -0.49091668]
# [ 0.14998616 0.58638886 -0.73637502]]
def main():
es_staged_data_index = "cifar-metadata-1"
es_logging_index = "custom-net-cifar-12"
output_model = es_logging_index + ".pth"
es = Elasticsearch("localhost:9200")
data = [
doc["_source"] for doc in list(scan(es, index=es_staged_data_index))
]
np.random.seed(42)
np.random.shuffle(data)
training_data = [x for x in data if "train" in x["set_type"]]
testing_data = [x for x in data if "test" in x["set_type"]]
print(f"Size of training set: {len(training_data)}")
print(f"Size of testing set: {len(testing_data)}")
# didnt use this time around.
train_dataset_loader = _get_dataset_loader(training_data,
transform=transform_train,
shuffle=True)
test_dataset_loader = _get_dataset_loader(testing_data)
net = SimpleNet(10).cuda()
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(net.parameters(),
lr=1e-3,
momentum=0.9,
weight_decay=5e-4)
# Train
print("training...")
for epoch in range(300):
running_loss = 0.0
for i, (inputs, labels) in enumerate(train_dataset_loader):
optimizer.zero_grad()
outputs = net(inputs.float().cuda())
loss = criterion(outputs, labels.long().cuda())
loss.backward()
optimizer.step()
# print stats
running_loss += loss.item()
print_on = 100
if (i + 1) % print_on == 0:
record = {
"timestamp": datetime.utcnow().isoformat(),
"cross-entropy-loss": running_loss / print_on,
"model-name": "train-simplenet-8"
}
es.index(index=es_logging_index, body=record)
print('[%d, %5d] loss %.3f' % (epoch + 1, i + 1, running_loss /
(print_on + 1)))
running_loss = 0.0
# Test
if epoch + 1 % 10:
print("testing...")
with torch.no_grad():
correct = 0.0
total = 0.0
i = 0.0
for inputs, labels in test_dataset_loader:
outputs = net(inputs.float().cuda())
#_, predicted = torch.max(outputs.data, 1)
_, predicted = outputs.max(1)
total += labels.size(0)
correct += predicted.eq(labels.cuda()).sum().item()
#correct += (predicted == labels.long().cuda()).sum().item()
i += 1
test_accuracy = correct / total
print(f"Test Accuracy: {test_accuracy}")
print(f"Correct: {correct}, Incorrect: {total-correct}")
record = {
"accuracy": test_accuracy,
"correct": correct,
"incorrect": total - correct,
"timestamp": datetime.utcnow().isoformat()
}
es.index(index=es_logging_index, body=record)
# Save
print("saving...")
torch.save(net.state_dict(), output_model)
y_train = np.array(y_train, dtype=np.float32).reshape(len(y_train), 1)
x_test = np.array(x_test, dtype=np.float32).reshape(len(x_test), 5)
y_test = np.array(y_test, dtype=np.float32).reshape(len(y_test), 1)
#
#x_train =np.array(x_train, dtype = np.float64 ).reshape(len(x_train), 5)
#y_train =np.array(y_train, dtype = np.float64).reshape(len(y_train), 1)
#x_test =np.array(x_test, dtype = np.float64).reshape(len(x_test), 5 )
#y_test =np.array(y_test, dtype = np.float64).reshape(len(y_test), 1)
print(x_train.shape, y_train.shape)
print(x_test.shape, y_test.shape)
#quit()
# load model
#network = SimpleNet(input_size=1 , hidden_size=10, output_size=1 )
network = SimpleNet(input_size=5, hidden_size=10, output_size=1)
network.load_params("params.pkl")
#
#pred
print(y_test[:10] * num_max_y)
y_val = network.predict(x_test)
y_val = y_val * num_max_y
print(y_val[:10])
# quit()
#y_train = y_train * num_max_y
#y_val = y_val * num_max_y
print('time : ', time.time() - global_start_time)
#quit()
#print(y_val[:10] )
return m
(x_train, t_train), (x_test, t_test) = mnist.load_data()
t_train_one_hot = one_hot(t_train)
t_test_one_hot = one_hot(t_test)
x_train_reshape = x_train.reshape(60000, 784)
x_test_reshape = x_test.reshape(10000, 784) # 将之转化为我想要的数据格式
train_loss_list = []
iters_num = 10000
train_size = x_train.shape[0]
batch_size = 100
rate = 0.1
network = SimpleNet(input_size=784, hidden_size=50, output_size=10)
for i in range(iters_num):
batch_mask = np.random.choice(train_size, batch_size)
x_batch = x_train_reshape[batch_mask] # little bracket is using function
t_batch = t_train_one_hot[batch_mask] # 在样本中随机选择一小撮
grads = network.numerical_gradient(x_batch, t_batch)
for key in ('w1', 'b1', 'w2', 'b2'):
network.param[key] -= rate * grads[key] # 对参数进行了调整,发现梯度值太小
loss = network.loss_function(x_batch, t_batch)
print(loss)
train_loss_list.append(loss)
print(network.accuracy(x_batch, t_batch)) # 是随机概率，根本就没有提高
y = np.array(train_loss_list)
x = np.array(range(len(train_loss_list)))
y_train = np.array(y_train, dtype=np.float32).reshape(len(y_train), 1) x_test = np.array(x_test, dtype=np.float32).reshape(len(x_test), 5) y_test = np.array(y_test, dtype=np.float32).reshape(len(y_test), 1) # #x_train =np.array(x_train, dtype = np.float64 ).reshape(len(x_train), 5) #y_train =np.array(y_train, dtype = np.float64).reshape(len(y_train), 1) #x_test =np.array(x_test, dtype = np.float64).reshape(len(x_test), 5 ) #y_test =np.array(y_test, dtype = np.float64).reshape(len(y_test), 1) print(x_train.shape, y_train.shape) print(x_test.shape, y_test.shape) #print(x_train[: 10]) #print(type(x_train )) #quit() # network = SimpleNet(input_size=5, hidden_size=10, output_size=1) #iters_num = 30000 # 繰り返しの回数を適宜設定する iters_num = 10000 # 繰り返しの回数を適宜設定する train_size = x_train.shape[0] print(train_size) #quit() # global_start_time = time.time() #batch_size = 100 #batch_size = 32 batch_size = 16 learning_rate = 0.1
