def test_deep_model():
""" test that deep model has lower error than linear model. Example data is
y = x1 - x2 + x3^2, where features x1, x2, and x3 are normaly distributed
with zero mean and unit variance and features x1 and x2 are correlated with
value 0.5 while x3 is uncorrelated with both x1 and x2."""
import numpy as np
from crpm.setup_multicorrel import setup_multicorrel_deep_c
from crpm.setup_multicorrel import setup_multicorrel_c
from crpm.gradientdecent import gradientdecent
#init numpy seed
np.random.seed(40017)
#setup shallow model
model, data = setup_multicorrel_c()
#get data dimensions
nobv = data.shape[1]
# partition training and testing data
train = data[0:3, 0:nobv//2]
target = data[-1, 0:nobv//2]
vtrain = data[0:3, nobv//2:nobv]
vtarget = data[-1, nobv//2:nobv]
#train model with mean squared error
_, cost0, _ = gradientdecent(model, train, target, "mse", vtrain, vtarget,
finetune=8)
#save weights
#weight0 = model[1]["weight"]
#setup deep model
model, data = setup_multicorrel_deep_c()
#train model with mean squared error
_, cost1, _ = gradientdecent(model, train, target, "mse", vtrain, vtarget,
finetune=8)
#save weights
#weight1 = model[1]["weight"]
#print(cost0)
#print(cost1)
#print(weight0)
#print(weight1)
#print(model[2]["weight"])
assert cost0 > cost1
def test_solve_nestedcs_bce():
"""test nested cs can be solved
"""
import numpy as np
from crpm.setup_nestedcs import setup_nestedcs
from crpm.fwdprop import fwdprop
from crpm.lossfunctions import loss
from crpm.gradientdecent import gradientdecent
#init numpy seed
np.random.seed(40017)
#setup model
model, data = setup_nestedcs()
#calculate initial binary cross entropy error
pred, _ = fwdprop(data[0:2, ], model)
icost, _ = loss("bce", pred, data[-1, ])
#train model
pred, cost, _ = gradientdecent(model, data[0:2, ], data[-1, ], "bce")
#print(model)
#print(icost)
#print(cost)
assert icost > cost
assert cost < .29
def test_ffn_pre():
"""test static preprocessing block
number adder should have negative weights with inverting pre-processor
"""
import numpy as np
from crpm.ffn import FFN
from crpm.dataset import load_dataset
from crpm.gradientdecent import gradientdecent
#init numpy seed
np.random.seed(40017)
#create an inverter for number adder model input
inverter = FFN("crpm/data/numberadder_pre_bodyplan.csv")
#assert inverter is 5 by 5 square
assert inverter.body[1]["weight"].shape == (5, 5)
#manually set weight to define the negative of the identity matrix
inverter.body[1]["weight"] = -1 * np.identity(5)
#create number adder with inverter as pre-processor
model = FFN("crpm/data/numberadder_bodyplan.csv", pre=inverter.body)
#train numberadder model with mean squared error
_, data = load_dataset("crpm/data/numberadder.csv")
_, _, _ = gradientdecent(model, data[0:5, ], data[-1, ], "mse", finetune=7)
print(model.body[1]["weight"])
assert np.allclose(model.body[1]["weight"], -1.0, rtol=.005)
def will_roc_will_plot():
""" test if roc output will plot properly"""
import os
import matplotlib.pyplot as plt
from crpm.setup_nestedcs import setup_nestedcs
from crpm.gradientdecent import gradientdecent
from crpm.analyzebinaryclassifier import analyzebinaryclassifier
#setup model
model, data = setup_nestedcs()
#train model
pred, _, _ = gradientdecent(model, data[0:2, ], data[-1, ], "mse")
#analyze binary classifier
roc, _ = analyzebinaryclassifier(pred, data[-1, ])
plt.scatter(*zip(*roc))
#remove any previously saved files if they exist
if os.path.exists("nestedcs_roc.png"):
os.remove("nestedcs_roc.png")
#save file
plt.savefig("nestedcs_roc.png")
#plt.show()
#assert file was created then remove
assert os.path.exists("nestedcs_roc.png")
os.remove("nestedcs_roc.png")
def test_earlystopping_triggered():
""" test early stopping is triggered with overfitting dataset."""
import numpy as np
from crpm.setup_overfitting import setup_overfitting_shallow
from crpm.gradientdecent import gradientdecent
#init numpy seed
np.random.seed(40017)
#setup shallow model
model, _, train, valid = setup_overfitting_shallow()
#print(train.shape)
#print(valid.shape)
train = train[:, :NOBV]
#assert early stopping is triggered with dataset
_, _, ierr = gradientdecent(model,
train[:-1, :],
train[-1, :],
"mse",
valid[:-1, :],
valid[-1, :],
earlystop=True,
finetune=8)
#does early stopping message appears
#assert True
print(ierr)
assert ierr == 2
def test_ffn_post():
"""test static post processing block
number adder should have negative weights with inverting post-processor
"""
import numpy as np
from crpm.ffn import FFN
from crpm.dataset import load_dataset
from crpm.gradientdecent import gradientdecent
#init numpy seed
np.random.seed(40017)
#create an inverter for number adder model output
inverter = FFN("crpm/data/numberadder_post_bodyplan.csv")
#manually set weights to define the inverter
inverter.body[1]["weight"] = np.array([[-1 / 2], [-1 / 2]])
inverter.body[2]["weight"] = np.array([[1, 1]])
#create number adder with inverter as pre-processor
model = FFN("crpm/data/numberadder_bodyplan.csv", post=inverter.body)
#train numberadder model with mean squared error
_, data = load_dataset("crpm/data/numberadder.csv")
_, _, _ = gradientdecent(model, data[0:5, ], data[-1, ], "mse", finetune=7)
print(model.body[1]["weight"])
assert np.allclose(model.body[1]["weight"], -1.0, rtol=.005)
def test_solve_ffn_numberadder():
"""test FFN class number adder can be solved by gradient decent
"""
import numpy as np
from crpm.ffn import FFN
from crpm.dataset import load_dataset
from crpm.gradientdecent import gradientdecent
#init numpy seed
np.random.seed(40017)
#create number adder from file
model = FFN("crpm/data/numberadder_bodyplan.csv")
#train numberadder model with mean squared error
_, data = load_dataset("crpm/data/numberadder.csv")
_, _, _ = gradientdecent(model,
data[0:5, ],
data[-1, ],
"mse",
healforces=False,
finetune=7)
print(model.body[1]["weight"])
assert np.allclose(model.body[1]["weight"], 1.0, rtol=.005)
def test_solve_numberadder():
"""test number adder can be solved begining with weights = 1.1
"""
import numpy as np
from crpm.ffn_bodyplan import read_bodyplan
from crpm.ffn_bodyplan import init_ffn
from crpm.dataset import load_dataset
from crpm.gradientdecent import gradientdecent
#create shallow bodyplan with 5 inputs and 1 output for numebr adder data
bodyplan = read_bodyplan("crpm/data/numberadder_bodyplan.csv")
#create numberadder model
model = init_ffn(bodyplan)
#manually set layer weights to 1.1 and biases to 0
model[1]["weight"] = 1.1 * np.ones(model[1]["weight"].shape)
#train numberadder model with mean squared error
_, data = load_dataset("crpm/data/numberadder.csv")
_, _, _ = gradientdecent(model, data[0:5, ], data[-1, ], "mse")
print(model[1]["weight"])
assert np.allclose(model[1]["weight"], 1.0, rtol=.005)
def test_classify_spectra2():
"""test spectra2 can find two groups
"""
import numpy as np
from crpm.setup_spectra2 import setup_spectra2
from crpm.dynamics import computecost
from crpm.gradientdecent import gradientdecent
from crpm.analyzebinaryclassifier import analyzebinaryclassifier
#init numpy seed
np.random.seed(40017)
#setup model
discriminator, data = setup_spectra2()
#partition data (labels on first row)
nobv = data.shape[1]
cutoff = 2 * nobv // 3
target = data[0, :cutoff]
train = data[1:, :cutoff]
vtarget = data[0, cutoff:]
valid = data[1:, cutoff:]
#analyze untrained discriminator
pred, icost = computecost(discriminator, valid, vtarget, "bce")
roc, ireport = analyzebinaryclassifier(pred, vtarget)
if ireport["AreaUnderCurve"] < .5:
#flip labels
pred, icost = computecost(discriminator, valid, 1 - vtarget, "bce")
roc, ireport = analyzebinaryclassifier(pred, 1 - vtarget)
print(ireport)
#plotroc(roc)
#train discriminator
pred, cost, _ = gradientdecent(discriminator,
train,
target,
"bce",
valid,
vtarget,
earlystop=True,
finetune=7)
#analyze discriminator
print("analyze trained discriminator to iden subtype")
pred, cost = computecost(discriminator, valid, vtarget, "bce")
roc, report = analyzebinaryclassifier(pred, vtarget)
if report["AreaUnderCurve"] < .5:
#flip labels
pred, cost = computecost(discriminator, valid, 1 - vtarget, "bce")
roc, report = analyzebinaryclassifier(pred, 1 - vtarget)
print(report)
#plotroc(roc)
#assert discriminator can be trained by binary cross entropy error
assert icost > cost
assert report["AreaUnderCurve"] > ireport["AreaUnderCurve"]
assert report["AreaUnderCurve"] > .8
def test_solved_nestedcs_bce():
"""test solved nestedcs will produce optimal threashold
"""
from crpm.setup_nestedcs import setup_nestedcs
from crpm.gradientdecent import gradientdecent
from crpm.analyzebinaryclassifier import analyzebinaryclassifier
#setup model
model, data = setup_nestedcs()
#train model
pred, _, _ = gradientdecent(model, data[0:2, ], data[-1, ], "bce")
#analyze binary classifier
_, report = analyzebinaryclassifier(pred, data[-1, ])
print(report)
assert report["AreaUnderCurve"] >= .95
assert report["Accuracy"] >= .87
assert report["Accuracy"] < 1.0
assert report["F1Score"] >= .86
def test_solved_nestedcs():
"""test solved nestedcs will produce optimal threashold
"""
import numpy as np
from crpm.setup_nestedcs import setup_nestedcs
from crpm.gradientdecent import gradientdecent
from crpm.analyzebinaryclassifier import analyzebinaryclassifier
#init numpy seed
np.random.seed(40017)
#setup model
model, data = setup_nestedcs()
#train model
pred, _, _ = gradientdecent(model, data[0:2, ], data[-1, ], "mse")
#analyze binary classifier
_, report = analyzebinaryclassifier(pred, data[-1, ])
assert report["AreaUnderCurve"] >= .92
assert report["Accuracy"] >= .80
assert report["F1Score"] >= .80
def test_afnetwork():
"""test AF network patients can be encoded and generated
"""
#import matplotlib
#matplotlib.use('TkAgg')
#import matplotlib.pyplot as plt
#import matplotlib.patches as mpatches
import numpy as np
from crpm.setup_afmodel import setup_afmodel
from crpm.dynamics import computecost
from crpm.analyzebinaryclassifier import analyzebinaryclassifier
#from crpm.lossfunctions import loss
from crpm.analyzebinaryclassifier import plotroc
from crpm.gradientdecent import gradientdecent
from crpm.contrastivedivergence import contrastivedivergence
#from crpm.ffn import FFN
from crpm.ffn_bodyplan import stack_new_layer
from crpm.ffn_bodyplan import copy_ffn
from crpm.fwdprop import fwdprop
#from crpm.backprop import backprop
#from crpm.dynamics import computeforces
#from crpm.dynamics import maxforce
from crpm.gan import gan
#init numpy seed
np.random.seed(40017)
#setup model
prototype, train, target, valid, vtarget = setup_afmodel()
#trim data
#maxobv = 150
#train = train[:,:maxobv]
#valid = valid[:,:maxobv]
#target = target[:maxobv]
#vtarget = vtarget[:maxobv]
#get prototype depth
nlayer = len(prototype)
#get data dimensions
nfeat = train.shape[0]
nobv = train.shape[1]
#return untrained autoencoder
_, autoencoder = contrastivedivergence(prototype, train, maxepoch=0)
# ----- Discriminator -----
#create discriminator
discriminator = copy_ffn(autoencoder[0:len(prototype)])
discriminator = stack_new_layer(discriminator, n=1, activation="logistic")
print("analyze untrained discriminator to iden subtype")
pred, icost = computecost(discriminator, valid, vtarget, "bce")
roc, ireport = analyzebinaryclassifier(pred, vtarget)
if ireport["AreaUnderCurve"] < .5:
#flip labels
pred, icost = computecost(discriminator, valid, 1 - vtarget, "bce")
roc, ireport = analyzebinaryclassifier(pred, 1 - vtarget)
print(ireport)
#plotroc(roc)
#train discriminator
pred, cost, _ = gradientdecent(discriminator,
train,
target,
"bce",
valid,
vtarget,
earlystop=True,
finetune=7)
print("analyze trained discriminator to iden subtype")
pred, cost = computecost(discriminator, valid, vtarget, "bce")
roc, report = analyzebinaryclassifier(pred, vtarget)
if report["AreaUnderCurve"] < .5:
#flip labels
pred, cost = computecost(discriminator, valid, 1 - vtarget, "bce")
roc, report = analyzebinaryclassifier(pred, 1 - vtarget)
print(report)
#plotroc(roc)
#assert discriminator can be trained by binary cross entropy error
#assert icost > cost
#assert discriminator has potential to iden two classes
#assert report["AreaUnderCurve"] > ireport["AreaUnderCurve"]
#assert report["AreaUnderCurve"] > .55
# ----- GENERATOR -----
#create generator from decoder
generator = copy_ffn(autoencoder[len(prototype) - 1:len(autoencoder)])
#correct label idecies
idx = 0
for layer in generator:
generator[idx]["layer"] = idx
idx += 1
#assert False
#-- Main GAN training---
#ganerr = gan(generator, discriminator, train,
# maxepoch=100000, batchsize=1, finetune=6)
ganerr = gan(generator,
discriminator,
train,
maxepoch=100000,
batchsize=1,
finetune=6)
#def moving_average(a, n=3) :
# ret = np.cumsum(a, dtype=float)
# ret[n:] = ret[n:] - ret[:-n]
# return ret[n - 1:] / n
#ganerr[:,2] = np.log(ganerr[:,2]) #plot density error on logscale
#discerrbar = moving_average(ganerr[:, 0], n=20)
#generrbar = moving_average(ganerr[:, 1], n=20)
#autoerrbar = moving_average(ganerr[:, 2], n=20)
#assert generator fools discriminator at least some of the time bce<65%.
print(ganerr[-1, 1])
assert ganerr[-1, 1] < .65
#fig = plt.figure()
#plt.plot(ganerr[:, 0], ganerr[:, 1])
#plt.plot(discerrbar, generrbar)
#plt.plot(discerrbar[0], generrbar[0], marker="D", color="green", markersize=10)
#plt.plot(discerrbar[-1], generrbar[-1], marker="8", color="red", markersize=10)
#plt.xlabel("discriminator error")
#plt.ylabel("generator error")
#plt.show()
#fig = plt.figure()
#plt.plot(ganerr[:, 0], ganerr[:, 2])
#plt.plot(discerrbar, autoerrbar)
#plt.plot(discerrbar[0], autoerrbar[0], marker="D", color="green", markersize=10)
#plt.plot(discerrbar[-1], autoerrbar[-1], marker="8", color="red", markersize=10)
#plt.xlabel("discriminator error")
#plt.ylabel("encoder error")
#plt.show()
#generate fake data for every training sample
nsample = train.shape[1]
fake, _ = fwdprop(np.random.rand(generator[0]["n"], nsample), generator)
#merge training and fake data
gandata = np.hstack((train, fake))
ganlabels = np.hstack((np.repeat(1, nsample), np.repeat(0, nsample)))
print("analyze trained discriminator on fake vs training set")
pred, cost = computecost(discriminator, gandata, ganlabels, "bce")
roc, report = analyzebinaryclassifier(pred, ganlabels)
if report["AreaUnderCurve"] < .5:
#flip labels
pred, cost = computecost(discriminator, gandata, ganlabels, "bce")
roc, report = analyzebinaryclassifier(pred, 1 - ganlabels)
print(report)
#plotroc(roc)
#gen fake data for every validation sample
nsample = valid.shape[1]
fake, _ = fwdprop(np.random.rand(generator[0]["n"], nsample), generator)
#merge validation and fake data
gandata = np.hstack((valid, fake))
ganlabels = np.hstack((np.repeat(1, nsample), np.repeat(0, nsample)))
print("analyze trained discriminator on fake vs vaidation set")
pred, costv = computecost(discriminator, gandata, ganlabels, "bce")
roc, reportv = analyzebinaryclassifier(pred, ganlabels)
if reportv["AreaUnderCurve"] < .5:
#flip labels
pred, costv = computecost(discriminator, gandata, 1 - ganlabels, "bce")
roc, reportv = analyzebinaryclassifier(pred, 1 - ganlabels)
print(reportv)
#plotroc(roc)
#assert discriminator has poor potential to iden fake data
assert reportv["AreaUnderCurve"] < .55
#get fake data the discriminator thinks is real
pred, _ = fwdprop(fake, discriminator)
spoof = fake[:, pred[0, :] > report["OptimalThreshold"]]
def train(self, state, action, reward, new_state, validation=None):
""" will train deep network model by gradient decent """
#make sure input all have the same number of observations
nobv = state.shape[1]
failcheck = False
if new_state.shape[1] != nobv:
failcheck = True
if action.shape[0] != nobv:
failcheck = True
if reward.shape[0] != nobv:
failcheck = True
if validation is not None and validation.shape[0] != nobv:
failcheck = True
if failcheck:
print(
"runtime error in train: inconsistent number of observations!")
return
# Ask the model for the Q values of the current state (inference)
state_values = self.get_value(state)
#print("state_values")
#print(state_values)
# Ask the model for the Q values of the new state (target)
new_state_values = self.get_target_value(new_state)
#print("new_state_values")
#print(new_state_values)
#get network input size
nfeat = state.shape[0] #network input size
nlabels = state_values.shape[0] #network output size (actions)
#print("nfeat and nlabels(actions)")
#print(nfeat)
#print(nlabels)
#print("actions")
#print(action)
# loop over actions
for iact in range(nlabels):
#get patients who took this action
patients = np.where(action == iact, True, False)
#update Q Values if any patients took this action
if np.sum(patients) > 0:
state_values[iact, patients] = (
reward[patients] +
self.discount * np.amax(new_state_values[:, patients]))
# Train prediction network
if validation is None or np.sum(validation) < 1:
#get data from patients that participated
intrain = np.squeeze(~np.isnan(action))
nobv = np.sum(intrain)
#exit if too few participated
if nobv < 1:
print("too few participants found for training")
return
#otherwise proceed with training
traindata = state[:, intrain].reshape((nfeat, nobv))
#print("training data")
#print(traindata)
#print("training labels")
#print(state_values[:, intrain].reshape((nlabels, nobv)))
_, self.loss, _ = gradientdecent(self.prednet,
traindata,
state_values[:, intrain].reshape(
(nlabels, nobv)),
"mse",
maxepoch=1,
healforces=False)
else:
#partition out validation patients from dataset
intrain = np.logical_and(~validation, ~np.isnan(action))
invalid = np.logical_and(validation, ~np.isnan(action))
nobv = np.sum(intrain)
nobv_v = np.sum(invalid)
#exit if too few participated
if nobv < 1:
print("too few participants found for training")
return
if nobv_v < 1:
print("too few participants found for validation")
return
#otherwise proceed with training
traindata = state[:, intrain].reshape((nfeat, nobv))
validata = state[:, invalid].reshape((nfeat, nobv_v))
_, self.loss, _ = gradientdecent(
self.prednet,
traindata,
state_values[:, intrain].reshape((nlabels, nobv)),
"mse",
validata=validata,
valitargets=state_values[:, invalid].reshape(
(nlabels, nobv_v)),
earlystop=True,
healforces=True)
print("loss")
print(self.loss)
print("bias")
print(self.prednet[-1]["bias"])
print("weights")
print(self.prednet[-1]["weight"])
def test_regval_reduces_correl():
""" test that regularization term will reduce the weight assoctiated with
uncorrelated features compared with no regularization. Example function is
y = x1 - x2 + x3^2, where features x1, x2, and x3 are normaly distributed
with zero mean and unit variance and features x1 and x2 are correlated with
value 0.5 while x3 is uncorrelated with both x1 and x2."""
import numpy as np
from crpm.ffn_bodyplan import reinit_ffn
from crpm.setup_multicorrel import setup_multicorrel_c
from crpm.gradientdecent import gradientdecent
#init numpy seed
np.random.seed(40017)
#setup model with no regularization
model, data = setup_multicorrel_c()
#get data dimensions
nobv = data.shape[1]
# partition training and testing data
train = data[0:3, 0:nobv//2]
target = data[-1, 0:nobv//2]
vtrain = data[0:3, nobv//2:nobv]
vtarget = data[-1, nobv//2:nobv]
#train model with mean squared error
_, _, _ = gradientdecent(model, train, target, "mse", vtrain, vtarget,
finetune=8)
#save weights
weight0 = model[1]["weight"]
#switch to L1 regularization
model[1]["lreg"] = 1
#find regulaization value that minimizes cost by D&C
#lmin = 5
#lmax = 15
## Employ 8 rounds of D&C
#for round in range(0,8):
# #get halfway point for regval
# alpha = (lmax+lmin)/2.0
# #LEFT HAND SIDE
# #re-init model
# model = reinit_ffn(model)
# #set regularization term to between lmin and alpha
# model[1]["regval"] = (lmin+alpha)/2.0
# #train L1 regularized model
# __, costl = gradientdecent(model, train, target, "mse",
# validata=vtrain, valitargets=vtarget)
# #RIGHT HAND SIDE
# #re-init model
# model = reinit_ffn(model)
# #set regularization term to between lmax and alpha
# model[1]["regval"] = (lmax+alpha)/2.0
# #train L1 regularized model
# __, costr = gradientdecent(model, train, target, "mse",
# validata=vtrain, valitargets=vtarget)
# #set new boundaries
# if costl < costr:
# lmax = alpha #set right hand boundary to alpha
# else:
# lmin = alpha #set left hand boundary to alpha
#Calcualte L1 model between lmin and lmax
#re-init model
model = reinit_ffn(model)
#set regularization term to between lmin and lmax
model[1]["regval"] = 13.92578125#(lmin+lmax)/2.0
#model[1]["regval"] = (lmin+lmax)/2.0
#train L1 regularized model
_, _, _ = gradientdecent(model, train, target, "mse", vtrain, vtarget,
finetune=8)
#save weights and regval
#alpha1 = model[1]["regval"]
weight1 = model[1]["weight"]
#switch to L2 regularization
model[1]["lreg"] = 2
#find regulaization value that minimizes cost by D&C
#lmin = 0
#lmax = 10
## Employ 8 rounds of D&C
#for round in range(0,8):
# #get halfway point for regval
# alpha = (lmax+lmin)/2.0
# #LEFT HAND SIDE
# #re-init model
# model = reinit_ffn(model)
# #set regularization term to between lmin and alpha
# model[1]["regval"] = (lmin+alpha)/2.0
# #train L1 regularized model
# __, costl = gradientdecent(model, train, target, "mse",
# validata=vtrain, valitargets=vtarget)
# #RIGHT HAND SIDE
# #re-init model
# model = reinit_ffn(model)
# #set regularization term to between lmax and alpha
# model[1]["regval"] = (lmax+alpha)/2.0
# #train L1 regularized model
# __, costr = gradientdecent(model, train, target, "mse",
# validata=vtrain, valitargets=vtarget)
# #set new boundaries
# if costl < costr:
# lmax = alpha #set right hand boundary to alpha
# else:
# lmin = alpha #set left hand boundary to alpha
#Calcualte L2 model between lmin and lmax
#re-init model
model = reinit_ffn(model)
#set regularization term to between lmin and lmax
model[1]["regval"] = 0.76171875#(lmin+lmax)/2.0
#model[1]["regval"] = (lmin+lmax)/2.0
#train L1 regularized model
_, _, _ = gradientdecent(model, train, target, "mse", vtrain, vtarget,
finetune=9)
#save weights and regval
#alpha2 = model[1]["regval"]
weight2 = model[1]["weight"]
print("weights -------")
print(weight0)
print(weight1)
print(weight2)
#print("cost -------")
#print(cost0)
#print(cost1)
#print(cost2)
#print("reval -------")
#print(alpha1)
#print(alpha2)
assert abs(weight0[0, 2]) > abs(weight1[0, 2])
#assert abs(weight0[0, 2]) > abs(weight2[0, 2])
assert abs(weight2[0, 2]) != abs(weight1[0, 2])
def test_naive_earlystopping_gradient_decent():
""" test naive early stopping yeilds comperable outsample error using
overfitting dataset compared to training with no early stopping."""
import time
import numpy as np
from crpm.setup_overfitting import setup_overfitting_shallow
from crpm.ffn_bodyplan import reinit_ffn
from crpm.gradientdecent import gradientdecent
#init numpy seed
np.random.seed(40017)
#setup shallow model
model, _, train, valid = setup_overfitting_shallow()
train = train[:, :
NOBV] #reduce training data to ensure early stopping occours
#calculate initial error
_, cost0, _ = gradientdecent(model,
train[:-1, :],
train[-1, :],
"mse",
valid[:-1, :],
valid[-1, :],
maxepoch=0)
#calculate out-sample error with no early stopping
start_time = time.perf_counter()
_, cost, _ = gradientdecent(model,
train[:-1, :],
train[-1, :],
"mse",
valid[:-1, :],
valid[-1, :],
earlystop=False,
finetune=8)
run_time = time.perf_counter() - start_time
#reinit model
model = reinit_ffn(model)
#calculate out-sample error with early stopping
start_time = time.perf_counter()
_, cost2, _ = gradientdecent(model,
train[:-1, :],
train[-1, :],
"mse",
valid[:-1, :],
valid[-1, :],
earlystop=True,
finetune=8)
run_time2 = time.perf_counter() - start_time
#calculate learning efficiency defined as amount cost reduced per wall-clock time.
leff = (cost0 - cost) / run_time
leff2 = (cost0 - cost2) / run_time2
print("cost initial = " + str(cost0))
print("cost = " + str(cost))
print("cost earlystop = " + str(cost2))
print("runtime = " + str(run_time))
print("runtime earlystop = " + str(run_time2))
print("efficiency = " + str(leff))
print("efficiency earlystop= " + str(leff2))
#assert learning with early stopping is faster than without
#assert run_time2 < run_time
#assert cost decreases with early stopping
assert cost2 < cost0
#assert cost with early stopping is higher than without
assert cost2 > cost
def test_solve_toruscases_bce():
"""test toruscases can be solved
"""
import numpy as np
from crpm.setup_toruscases import setup_toruscases
from crpm.fwdprop import fwdprop
from crpm.lossfunctions import loss
from crpm.gradientdecent import gradientdecent
from crpm.analyzebinaryclassifier import analyzebinaryclassifier
#init numpy seed
np.random.seed(40017)
#setup model
model, data = setup_toruscases()
nx = data.shape[0]
nsample = data.shape[1]
#partition training and validation data
valid = data[1:data.shape[0], 0:nsample // 3]
validtargets = data[0, 0:nsample // 3]
train = data[1:data.shape[0], nsample // 3:nsample]
targets = data[0, nsample // 3:nsample]
#calculate initial binary cross entropy error
pred, _ = fwdprop(train, model)
icost, _ = loss("bce", pred, targets)
#analyze binary classifier
pred, _ = fwdprop(valid, model)
roc, ireport = analyzebinaryclassifier(pred, validtargets)
if ireport["AreaUnderCurve"] < .5:
pred = 1 - pred
icost, _ = loss("bce", pred, validtargets)
roc, ireport = analyzebinaryclassifier(pred, validtargets)
print(ireport)
#plotroc(roc)
#train model
pred, cost, _ = gradientdecent(model,
train,
targets,
"bce",
valid,
validtargets,
earlystop=True)
#analyze binary classifier
pred, _ = fwdprop(valid, model)
roc, report = analyzebinaryclassifier(pred, validtargets)
if report["AreaUnderCurve"] < .5:
pred = 1 - pred
cost, _ = loss("bce", pred, validtargets)
roc, report = analyzebinaryclassifier(pred, validtargets)
print(report)
#plotroc(roc)
#print(model)
print(icost)
print(cost)
assert icost > cost
assert cost < .4
assert report["MatthewsCorrCoef"] > .1
def r_test_spectra2():
"""test spectra2 can be encoded and generated
"""
import numpy as np
from crpm.setup_spectra2 import setup_spectra2
from crpm.dynamics import computecost
from crpm.analyzebinaryclassifier import analyzebinaryclassifier
#from crpm.lossfunctions import loss
#from crpm.analyzebinaryclassifier import plotroc
from crpm.gradientdecent import gradientdecent
from crpm.contrastivedivergence import contrastivedivergence
#from crpm.ffn import FFN
from crpm.ffn_bodyplan import stack_new_layer
from crpm.ffn_bodyplan import copy_ffn
from crpm.fwdprop import fwdprop
from crpm.backprop import backprop
#from crpm.dynamics import computeforces
#from crpm.dynamics import maxforce
from crpm.gan import gan
#import matplotlib
#matplotlib.use('TkAgg')
#import matplotlib.pyplot as plt
#init numpy seed
np.random.seed(40017)
#setup model
prototype, data = setup_spectra2()
#get prototype depth
nlayer = len(prototype)
#get data dimensions
nfeat = data.shape[0]
nobv = data.shape[1]
#zscore data
tdata = np.divide(data - np.mean(data, axis=1, keepdims=True),
np.std(data, axis=1, keepdims=True))
#transform features into boltzmann like probs
#tdata = np.exp(-data)
#partfunc = np.sum(tdata, axis=1, keepdims = True)
#tdata = np.divide(tdata,partfunc) #normalize
#tdata = np.divide(tdata, np.max(tdata, axis=1, keepdims=True))#scale features by maxintensity
#plt.plot(data[:,0])
#plt.show()
#plt.plot(tdata[:,0])
#plt.show()
#data = tdata
#partition data (labels on first row)
ntrain = 2 * nobv // 3
target = data[0, :ntrain]
train = data[1:, :ntrain]
vtarget = data[0, ntrain:]
valid = data[1:, ntrain:]
#return untrained autoencoder
_, autoencoder = contrastivedivergence(prototype, train, maxepoch=0)
#calculate initial reconstruction error
pred, ireconerr = computecost(autoencoder, valid, valid, "mse")
print("init recon error = " + str(ireconerr))
##train prototype
#_, autoencoder = contrastivedivergence(prototype, train,
# ncd=2,
# batchsize=50,
# nadj=10,
# maxepoch=100,
# momentum=0.1)
#train prototype
_, autoencoder = contrastivedivergence(prototype,
train,
validata=valid,
ncd=1,
batchsize=50,
nadj=10,
maxepoch=100,
momentum=0.0)
#calculate final reconstruction error
pred, reconerr = computecost(autoencoder, valid, valid, "mse")
print("pretrained recon error = " + str(reconerr))
#assert learning is taking place by reduced recon error.
assert ireconerr > reconerr
# ----- Discriminator -----
#create discriminator
discriminator = copy_ffn(autoencoder[0:len(prototype)])
discriminator = stack_new_layer(discriminator, n=1, activation="logistic")
#analyze trained binary classifier
pred, icost = computecost(discriminator, valid, vtarget, "bce")
roc, ireport = analyzebinaryclassifier(pred, vtarget)
if ireport["AreaUnderCurve"] < .5:
#flip labels
pred, icost = computecost(discriminator, valid, 1 - vtarget, "bce")
roc, ireport = analyzebinaryclassifier(pred, 1 - vtarget)
print(ireport)
#plotroc(roc)
#train discriminator
pred, cost, _ = gradientdecent(discriminator,
train,
target,
"bce",
valid,
vtarget,
earlystop=True,
finetune=6)
#analyze trained binary classifier
pred, cost = computecost(discriminator, valid, vtarget, "bce")
roc, report = analyzebinaryclassifier(pred, vtarget)
if report["AreaUnderCurve"] < .5:
#flip labels
pred, cost = computecost(discriminator, valid, 1 - vtarget, "bce")
roc, report = analyzebinaryclassifier(pred, 1 - vtarget)
print(report)
#plotroc(roc)
#assert discriminator can be trained by binary cross entropy error
assert icost > cost
#assert discriminator has potential to iden two calsses
assert report["AreaUnderCurve"] > ireport["AreaUnderCurve"]
#assert report["AreaUnderCurve"] > .6
# ----- generator -----
#create generator from decoder
generator = copy_ffn(autoencoder[len(prototype):len(autoencoder)])
#adjust regularization
for layer in generator:
layer["regval"] = 0 #.00001
#correct label idecies
idx = 0
for layer in generator:
generator[idx]["layer"] = idx
idx += 1
#generate fake samples
nfake = 600
ncode = generator[0]["n"]
fake, _ = fwdprop(np.random.rand(ncode, nfake), generator)
#calculate initial reconstruction error
pred, fkreconerr = computecost(autoencoder, fake, fake, "mse")
print("init fake recon error = " + str(fkreconerr))
#assert fake data recon error is better than untrained recon error
assert fkreconerr < ireconerr
#-- Start GAN training---
ganerr = gan(generator,
discriminator,
train,
maxepoch=20000,
batchsize=50,
finetune=6.3)
#assert generator fools discriminator at least some of the time bce<80%.
assert ganerr[-1, 1] < .8
#def moving_average(a, n=3) :
# ret = np.cumsum(a, dtype=float)
# ret[n:] = ret[n:] - ret[:-n]
# return ret[n - 1:] / n
#fig = plt.figure()
#plt.plot(ganerr[:, 0], ganerr[:, 1])
#plt.plot(moving_average(ganerr[:, 0], n=20), moving_average(ganerr[:, 1], n=20))
#plt.plot(ganerr[0, 0], ganerr[0, 1], marker="D", color="green", markersize=10)
#plt.plot(ganerr[-1, 0], ganerr[-1, 1], marker="8", color="red", markersize=10)
#plt.xlabel("discriminator error")
#plt.ylabel("generator error")
#plt.show()
#print("final report")
#print(report)
#plotroc(roc)
assert False
def r_test_pretrain_periodiccases_deep():
"""test pretained periodiccases model encodes better than non pretrained model
"""
import numpy as np
from crpm.setup_periodiccases import setup_periodiccases_deep
from crpm.dynamics import computecost
from crpm.gradientdecent import gradientdecent
from crpm.ffn_bodyplan import reinit_ffn
from crpm.contrastivedivergence import contrastivedivergence
from crpm.analyzebinaryclassifier import analyzebinaryclassifier
from crpm.ffn_bodyplan import stack_new_layer
from crpm.ffn_bodyplan import copy_ffn
#from crpm.analyzebinaryclassifier import plotroc
#init numpy seed
np.random.seed(40017)
#setup model
model, data = setup_periodiccases_deep()
nx = data.shape[0]
nsample = data.shape[1]
#partition training and validation data
valid = data[1:nx, 0:nsample // 3]
validtargets = data[0, 0:nsample // 3]
train = data[1:nx, nsample // 3:nsample]
targets = data[0, nsample // 3:nsample]
#remove discriminating layer
prototype = model[0:-1]
#re-init prototype
prototype = reinit_ffn(prototype)
#return untrained autoencoder
_, autoencoder = contrastivedivergence(prototype, train, maxepoch=0)
#calculate initial reconstruction error
pred, icost_encoder = computecost(autoencoder, valid, valid, "mse")
#conventional training autoencoder
pred, gcost_encoder, _ = gradientdecent(autoencoder,
train,
train,
"mse",
valid,
valid,
maxepoch=1E6,
earlystop=True,
healforces=False,
finetune=9)
#assert auto encoder can be conventionally trained
assert gcost_encoder < icost_encoder
#re-init prototype
prototype = reinit_ffn(prototype)
#CD train autoencoder
_, autoencoder = contrastivedivergence(prototype,
train,
validata=valid,
ncd=10,
batchsize=20,
nadj=10,
maxepoch=500,
momentum=0.05,
finetune=6)
#calculate reconstruction error
pred, cost_encoder = computecost(autoencoder, valid, valid, "mse")
print(cost_encoder)
#assert reconstruction error is less than initial recon error
assert cost_encoder < icost_encoder
#fine-tune autoencoder
pred, fcost_encoder, _ = gradientdecent(autoencoder,
train,
train,
"mse",
valid,
valid,
maxepoch=1E6,
earlystop=True,
healforces=False,
finetune=9)
#assert final reconstruction error is not greater than previous recon error
assert fcost_encoder <= cost_encoder
#assert final reconstruction error is not greater than with conventional training
assert fcost_encoder <= gcost_encoder
def test_regval_reduces_uncorrel():
""" test that regularization term will reduce the weight assoctiated with
uncorrelated features compared with no regularization. Example function is
y = 5x1 - x2 + 5x3^2, where features x1, x2, and x3 are indepently sampled from
normal distribution with zero mean and unit variance."""
import numpy as np
from crpm.setup_multicorrel import setup_multicorrel
from crpm.gradientdecent import gradientdecent
#init numpy seed
np.random.seed(40017)
#setup model with no regularization
model, data = setup_multicorrel()
#get data dimensions
nobv = data.shape[1]
# partition training and testing data
train = data[0:3, 0:nobv//2]
target = data[-1, 0:nobv//2]
vtrain = data[0:3, nobv//2:nobv]
vtarget = data[-1, nobv//2:nobv]
#train model with mean squared error
_, _, _ = gradientdecent(model, train, target, "mse", vtrain, vtarget,
finetune=8)
#save weights
weight0 = model[1]["weight"]
#re-init model
model, data = setup_multicorrel()
#manually set regularization term
model[1]["lreg"] = 1
model[1]["regval"] = 100#75
#train L1 regularized model
_, _, _ = gradientdecent(model, train, target, "mse", vtrain, vtarget,
finetune=8)
#save weights
weight1 = model[1]["weight"]
#re-init model
model, data = setup_multicorrel()
#manually set regularization term
model[1]["lreg"] = 2
model[1]["regval"] = 4
#train L2 regularized model
_, _, _ = gradientdecent(model, train, target, "mse", vtrain, vtarget,
finetune=8)
#save weights
weight2 = model[1]["weight"]
assert abs(weight0[0, 2]) > abs(weight1[0, 2])
assert abs(weight0[0, 2]) > abs(weight2[0, 2])
assert abs(weight2[0, 2]) != abs(weight1[0, 2])