def k_fold_validation(k, model, optimizer=optimizers.Adam(), tag='', batch=100, n_epoch=300):
loss = []
acc = []
acc_train = []
mnist = MNIST()
print str(model)
for i in range(k):
x_train, x_test, t_train, t_test = mnist.get_fold_i(k, i)
print 'fold:' + str(i)
l, at, a = validate_model(copy.deepcopy(model), copy.deepcopy(optimizer),
x_train, x_test, t_train, t_test,
batch, n_epoch)
loss.append(l)
acc.append(a)
acc_train.append(at)
def __dataloader(self, train):
# init data generators
transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307, ), (0.3081, ))
])
# dataset = CustomDataset(transform=transform)
dataset = MNIST(root='dataset',
train=train,
transform=transform,
download=True)
if not train:
device = torch.device(
"cuda:0" if torch.cuda.is_available() else "cpu")
self.support_set = torch.from_numpy(
dataset.support_set).float().to(device)
# when using multi-node (ddp) we need to add the datasampler
train_sampler = None
batch_size = self.hparams.batch_size
if self.use_ddp:
train_sampler = DistributedSampler(dataset)
should_shuffle = train_sampler is None
loader = DataLoader(dataset=dataset,
batch_size=batch_size,
shuffle=should_shuffle,
sampler=train_sampler,
num_workers=0,
drop_last=True)
return loader
def eval_cnn():
model = CNN()
config = ConfigManager(model).load()
optimizer = optim.SGD(model.parameters(),
lr=float(config["LEARNING_RATE"]),
momentum=float(config["MOMENTUM"]))
criterion = torch.nn.NLLLoss()
trainer = Trainer(model, MNIST(batch_size=10), optimizer, criterion, True)
trainer.evaluate()
def k_fold_validation(k,
model,
optimizer=optimizers.Adam(),
tag='',
batch=100,
n_epoch=300):
loss = []
acc = []
acc_train = []
mnist = MNIST()
print str(model)
for i in range(k):
x_train, x_test, t_train, t_test = mnist.get_fold_i(k, i)
print 'fold:' + str(i)
l, at, a = validate_model(copy.deepcopy(model),
copy.deepcopy(optimizer), x_train, x_test,
t_train, t_test, batch, n_epoch)
loss.append(l)
acc.append(a)
acc_train.append(at)
def MNIST_experiment():
"""I tried it with binary inputs but (at least with the current implementation) it is way too slow"""
tsetlin_machine = TsetlinMachine(number_clauses=1000,
number_action_states=1000,
precision=3.0,
threshold=10)
X, y, val_X, val_y = MNIST()
tsetlin_machine.fit(X, y, val_X, val_y, 300)
print('Final training accuracy:', tsetlin_machine.accuracy(X, y))
print('Final validation accuracy:', tsetlin_machine.accuracy(val_X, val_y))
def init_data_loader(dataset, data_path, batch_size, train=True, digits=None):
if dataset == "mnist":
if digits is not None:
return MNIST(data_path,
batch_size,
shuffle=False,
train=train,
condition_on=[digits])
else:
return MNIST(data_path, batch_size, shuffle=False, train=train)
elif dataset == "cifar10":
if digits is not None:
return CIFAR10(data_path,
batch_size,
shuffle=False,
train=train,
condition_on=[digits])
else:
return CIFAR10(data_path, batch_size, shuffle=False, train=train)
elif dataset == "fmnist":
if digits is not None:
return FMNIST(data_path,
batch_size,
shuffle=False,
train=train,
condition_on=[digits])
else:
return FMNIST(data_path, batch_size, shuffle=False, train=train)
elif dataset == "stl10":
if digits is not None:
return STL10(data_path,
batch_size,
shuffle=False,
train=train,
condition_on=[digits])
else:
return STL10(data_path, batch_size, shuffle=False, train=train)
from data import MNIST
from dbn import DBN
import numpy as np
d = DBN([28*28, 500,500, 1000], 10,0.1)
images = MNIST.load_images('images')
labels = MNIST.load_labels('labels')
img = images[0:56000]
lbl = labels[0:56000]
tst_img = images[56000:60000]
tst_lbl = labels[56000:60000]
d.pre_train(img,30)
d.train_labels(img, lbl,200)
#print d.sample(img[0:1],0)
#print d.sample(img[1:2],0)
#print 'layer 2'
#print d.sample(img[0:1],1)
#print d.sample(img[1:2],1)
#print 'layer 2'
#print d.sample(img[0:1],2)
#print d.sample(img[1:2],2)
#print 'layer 3'
#print d.sample(img[0:1],3)[0]
#print d.sample(img[1:2],3)[0]
#print 'labels'
print np.around(d.classify(img, 20), 2)[0:20]
print lbl[0:20]
from policy import json2policy
from models import Autoencoder, PSNR
policy_filename = "../models/policies/model_0_policy.json"
with open('../models/policies/model_0_policy.json', "r") as f:
policy_json = json.load(f)
policy = json2policy(policy_json)
n_epochs = 500
batch_size = 32
n_train_samples = 50000
lr = 0.02
decay = 0.1
mnist_gauss_05 = MNIST(batch_size=32,
n_train_samples=50,
noise_type="Gaussian",
intensity=0.5)
ae_pol0 = Autoencoder(mnist_gauss_05,
n_epochs=n_epochs,
n_stages=20,
lr=lr,
decay=decay)
ae_npol = Autoencoder(mnist_gauss_05,
n_epochs=n_epochs,
n_stages=20,
lr=lr,
decay=decay)
ae_pol0.train(do_valid=False, policy=policy)
ae_npol.train(do_valid=False, policy=policy)
def run(args):
print('\nSettings: \n', args, '\n')
args.model_signature = str(dt.datetime.now())[0:19].replace(' ', '_')
args.model_signature = args.model_signature.replace(':', '_')
########## Find GPUs
(gpu_config, n_gpu_used) = set_gpus(args.n_gpu)
########## Data, model, and optimizer setup
mnist = MNIST(args)
x = tf.placeholder(tf.float32, [None, 28, 28, 1])
if args.model == 'hvae':
if not args.K:
raise ValueError('Must set number of flow steps when using HVAE')
elif not args.temp_method:
raise ValueError('Must set tempering method when using HVAE')
model = HVAE(args, mnist.avg_logit)
elif args.model == 'cnn':
model = VAE(args, mnist.avg_logit)
else:
raise ValueError('Invalid model choice')
elbo = model.get_elbo(x, args)
nll = model.get_nll(x, args)
optimizer = AdamaxOptimizer(learning_rate=args.learn_rate,
eps=args.adamax_eps)
opt_step = optimizer.minimize(-elbo)
########## Tensorflow and saver setup
sess = tf.Session(config=gpu_config)
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
savepath = os.path.join(args.checkpoint_dir, args.model_signature,
'model.ckpt')
if not os.path.exists(args.checkpoint_dir):
os.makedirs(args.checkpoint_dir)
########## Test that GPU memory is sufficient
if n_gpu_used > 0:
try:
x_test = mnist.next_test_batch()
(t_e, t_n) = sess.run((elbo, nll), {x: x_test})
mnist.batch_idx_test = 0 # Reset batch counter if it works
except:
raise MemoryError("""
Likely insufficient GPU memory
Reduce test batch by lowering the -tbs parameter
""")
########## Training Loop
train_elbo_hist = []
val_elbo_hist = []
# For early stopping
best_elbo = -np.inf
es_epochs = 0
epoch = 0
train_times = []
for epoch in range(1, args.epochs + 1):
t0 = time.time()
train_elbo = train(epoch, mnist, opt_step, elbo, x, args, sess)
train_elbo_hist.append(train_elbo)
train_times.append(time.time() - t0)
print('One epoch took {:.2f} seconds'.format(time.time() - t0))
val_elbo = validate(mnist, elbo, x, sess)
val_elbo_hist.append(val_elbo)
if val_elbo > best_elbo:
# Save the model that currently generalizes best
es_epochs = 0
best_elbo = val_elbo
saver.save(sess, savepath)
best_model_epoch = epoch
elif args.early_stopping_epochs > 0:
es_epochs += 1
if es_epochs >= args.early_stopping_epochs:
print('***** STOPPING EARLY ON EPOCH {} of {} *****'.format(
epoch, args.epochs))
break
print('--> Early stopping: {}/{} (Best ELBO: {:.4f})'.format(
es_epochs, args.early_stopping_epochs, best_elbo))
print('\t Current val ELBO: {:.4f}\n'.format(val_elbo))
if np.isnan(val_elbo):
raise ValueError('NaN encountered!')
train_times = np.array(train_times)
mean_time = np.mean(train_times)
std_time = np.std(train_times)
print('Average train time per epoch: {:.2f} +/- {:.2f}'.format(
mean_time, std_time))
########## Evaluation
# Restore the best-performing model
saver.restore(sess, savepath)
test_elbos = np.zeros(args.n_nll_runs)
test_nlls = np.zeros(args.n_nll_runs)
for i in range(args.n_nll_runs):
print('\n---- Test run {} of {} ----\n'.format(i + 1, args.n_nll_runs))
(test_elbos[i], test_nlls[i]) = evaluate(mnist, elbo, nll, x, args,
sess)
mean_elbo = np.mean(test_elbos)
std_elbo = np.std(test_elbos)
mean_nll = np.mean(test_nlls)
std_nll = np.std(test_nlls)
print('\nTest ELBO: {:.2f} +/- {:.2f}'.format(mean_elbo, std_elbo))
print('Test NLL: {:.2f} +/- {:.2f}'.format(mean_nll, std_nll))
########## Logging, Saving, and Plotting
with open(args.logfile, 'a') as ff:
print('----------------- Test ID {} -----------------'.format(
args.model_signature),
file=ff)
print(args, file=ff)
print('Stopped after {} epochs'.format(epoch), file=ff)
print('Best model from epoch {}'.format(best_model_epoch), file=ff)
print('Average train time per epoch: {:.2f} +/- {:.2f}'.format(
mean_time, std_time),
file=ff)
print('FINAL VALIDATION ELBO: {:.2f}'.format(val_elbo_hist[-1]),
file=ff)
print('Test ELBO: {:.2f} +/- {:.2f}'.format(mean_elbo, std_elbo),
file=ff)
print('Test NLL: {:.2f} +/- {:.2f}\n'.format(mean_nll, std_nll),
file=ff)
if not os.path.exists(args.pickle_dir):
os.makedirs(args.pickle_dir)
train_dict = {
'train_elbo': train_elbo_hist,
'val_elbo': val_elbo_hist,
'args': args
}
pickle.dump(
train_dict,
open(os.path.join(args.pickle_dir, args.model_signature + '.p'), 'wb'))
if not os.path.exists(args.plot_dir):
os.makedirs(args.plot_dir)
tf_gen_samples = model.get_samples(args)
np_gen_samples = sess.run(tf_gen_samples)
plot_digit_samples(np_gen_samples, args)
plot_training_curve(train_elbo_hist, val_elbo_hist, args)
########## Email notification upon test completion
try:
msg_text = """Test completed for ID {0}.
Parameters: {1}
Test ELBO: {2:.2f} +/- {3:.2f}
Test NLL: {4:.2f} +/- {5:.2f} """.format(args.model_signature, args,
mean_elbo, std_elbo, mean_nll,
std_nll)
msg = MIMEText(msg_text)
msg['Subject'] = 'Test ID {0} Complete'.format(args.model_signature)
msg['To'] = args.receiver
msg['From'] = args.sender
s = smtplib.SMTP('localhost')
s.sendmail(args.sender, [args.receiver], msg.as_string())
s.quit()
except:
print('Unable to send email from sender {0} to receiver {1}'.format(
args.sender, args.receiver))
sys.path.append("../")
from data import MNIST
from transformations import json2policy
policy_files = [
None, 'model_0_policy.json', 'model_1_policy.json',
'model_2_policy.json'
]
policy_filename = policy_files[3]
with open(policy_filename, "r") as f:
policy_json = json.load(f)
policy = json2policy(policy_json)
n_epochs = 250
lr = 0.02
decay = 0.1
print("Training with policy {}".format(policy_filename))
print("** Number of Epochs:\t{}".format(n_epochs))
dataset = MNIST(batch_size=256, n_train_samples=500)
ae = Autoencoder(dataset,
n_epochs=n_epochs,
n_stages=20,
lr=lr,
decay=decay,
model_name="Autoencoder_nopol_500")
ae.train(do_valid=True, policy=None)
from data import MNIST
from dbn import DBN
import numpy as np
# Creates a DBN
d = DBN([28*28, 500,500, 2000], 10, 0.1)
# Loads the Images and Labels from the MNIST database
images = MNIST.load_images('images')
labels = MNIST.load_labels('labels')
# Gets the training set of images
img = images[0:60000]
lbl = labels[0:60000]
# Loads the test images from the MNIST database
tst_img = MNIST.load_images('test_images')
tst_lbl = MNIST.load_labels('test_labels')
# Pre trains the DBN
d.pre_train(img,5,50)
# Executes a 100 iterations of label training
# Attempts to classify and prints the error rate
for i in xrange(0, 100):
d.train_labels(img, lbl, 50, 50)
tst_class = d.classify(tst_img,10)
print 'Error over test data: {0}'.format(1 - (tst_class*tst_lbl).mean() * dbn.number_labels)
# Tests the DBN on test images
import pandas as pd
import util
from data import MNIST, FHR
import matplotlib.pyplot as plt
def getKurt(transformed):
tmp = pd.DataFrame(transformed)
tmp = tmp.kurt(axis=0)
plt.plot(tmp)
plt.show()
return tmp.abs().mean()
if __name__ == "__main__":
fhr = MNIST(1000)
ica = FastICA(n_components=2, random_state=0, max_iter=500, tol=0.001)
transformed = ica.fit_transform(fhr.X_train)
util.scatter_plot_2d(transformed, fhr.y_train,
"ICA scattered Plot (MNIST)")
plt.plot(ica.components_[0], 'r-')
plt.plot(ica.components_[1], 'g-')
plt.show()
# kurt = []
# for i in range(1, 42):
# ica.set_params(n_components=i)
# transformed = ica.fit_transform(fhr.X_train)
# kurt.append(getKurt(transformed))
# util.plot_regular(range(1, 42), kurt, "k component", "Kurtosis", "FHR-ICA: Kurtosis vs Number of Component")
net.apply(lambda m: initialize_params(m, args.no_act, 'normal'))
lr = args.lr
optimizer = torch.optim.SGD(net.parameters(), lr=lr, momentum=0.9)
best_va_acc = -1.
start_ep = -1
print('> Done.')
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
net = net.to(device)
torch.backends.cudnn.benchmark = True
print('> Loading dataset...')
if args.dataset == 'mnist':
tr_loader, va_loader, te_loader = MNIST(
root=args.datapath,
bs=args.bs,
valid_size=.1,
size=args.img_size,
normalize=(not args.raw_inputs))
elif args.dataset == 'cifar':
tr_loader, va_loader, te_loader = CIFAR10(
root=args.datapath,
bs=args.bs,
valid_size=.1,
size=args.img_size,
normalize=(not args.raw_inputs))
elif args.dataset == 'imgnet12':
tr_loader, va_loader, te_loader = IMGNET12(
root=args.datapath,
bs=args.bs,
valid_size=.1,
size=args.img_size,
out = out.reshape(out.size(0), -1)
out = self.drop_out(out)
out = self.fc1(out)
out = self.fc2(out)
return out
if __name__ == "__main__":
device = torch.device('cpu')
epochs = 5
classes = 10
batch_size = 100
lr = 0.001
train_data = MNIST("./MNIST/training/", "./MNIST/train_labels.csv")
test_data = MNIST("./MNIST/testing/", "./MNIST/test_labels.csv")
train_dataloader = torch.utils.data.DataLoader(dataset=train_data,
batch_size=batch_size,
shuffle=True)
test_dataloader = torch.utils.data.DataLoader(dataset=test_data,
batch_size=batch_size,
shuffle=False)
model = Conv_Net() # classes = 10 by default
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=lr)
def compute_vulnerability(args, attack_name, net, n_attacks=-1):
"""
Computes vulnerability using foolbox package of net
Parameters
----------
args : :class:`argparse.ArgumentParser`
The arguments passed to main.py
attack_name : string
The attack type. Must be one of
{'l1', 'l2', 'itl1', 'itl2', 'pgd', 'deepfool'}
net : :class:`torch.nn.Module`
The network whose vulnerability is computed.
n_attacks : int
The number of attacks to use for the computation of vulnerbaility.
If -1 or greater than dataset-size, uses the entire dataset.
Default: -1.
"""
print('\nStarting attacks of type ' + attack_name)
# Reload data without normalizing it
print('> Loading dataset %s...' % args.dataset)
if args.dataset == 'mnist':
_, loader = MNIST(root=args.datapath,
bs=args.bs,
valid_size=0.,
size=args.img_size,
normalize=False)
elif args.dataset == 'cifar':
_, loader = CIFAR10(root=args.datapath,
bs=args.bs,
valid_size=0.,
size=args.img_size,
normalize=False)
elif args.dataset == 'imgnet12':
_, loader = IMGNET12(root=args.datapath,
bs=args.bs,
valid_size=0.,
size=args.img_size,
normalize=False)
else:
raise NotImplementedError
print('> Done.')
# Image-normalizations (must be same as in data.py)
if args.raw_inputs:
means = [0., 0., 0.]
stds = [1., 1., 1.]
elif args.dataset == "mnist":
means = [0.1307]
stds = [0.3081]
elif args.dataset == "cifar":
means = [0.4914, 0.4822, 0.4465]
stds = [0.2023, 0.1994, 0.2010]
elif args.dataset == "imgnet12":
means = [.453, .443, .403]
stds = {
256: [.232, .226, .225],
128: [.225, .218, .218],
64: [.218, .211, .211],
32: [.206, .200, .200]
}[args.img_size]
else:
raise NotImplementedError
means = np.array(means).reshape(-1, 1, 1)
stds = np.array(stds).reshape(-1, 1, 1)
net.eval()
print('> Computing test accuracy...')
te_acc = do_pass(net, loader, args, means, stds)
print('> Done. Computed test accuracy: %5.2f' % te_acc)
# construct attack
bounds = (0, 1)
model = foolbox.models.PyTorchModel(net,
bounds=bounds,
preprocessing=(means, stds),
num_classes=args.categories)
# Choosing attack type
if attack_name == 'l1':
# vulnerability increases like sqrt(d) \propto img_size
# therefore, we divide the linfty-threshold by img_size
attack = partial(foolbox.attacks.FGSM(model, distance=Linfinity),
epsilons=1000,
max_epsilon=1. / args.img_size)
elif attack_name == 'l2':
# to be visually constant, the l2-threshold increases like sqrt d;
# but vulnerability also increases like sqrt d;
# therefore, use constant max_epsilon accross dimension d
attack = partial(foolbox.attacks.GradientAttack(model, distance=MSE),
epsilons=1000,
max_epsilon=1.)
elif attack_name == 'itl1':
it, eps = 10, 1. / args.img_size
attack = partial(foolbox.attacks.LinfinityBasicIterativeAttack(
model, distance=Linfinity),
iterations=it,
epsilon=eps,
stepsize=eps / (.75 * it),
binary_search=True)
elif attack_name == 'itl2':
it, eps = 10, 1.
attack = partial(foolbox.attacks.L2BasicIterativeAttack(model,
distance=MSE),
iterations=it,
epsilon=eps,
stepsize=eps / (.75 * it),
binary_search=True)
elif attack_name == 'pgd':
it, eps = 10, 1. / args.img_size
attack = partial(foolbox.attacks.RandomPGD(model, distance=Linfinity),
iterations=it,
epsilon=eps,
stepsize=eps / (.75 * it),
binary_search=True)
elif attack_name == 'deepFool':
attack = foolbox.attacks.DeepFoolAttack(model, distance=MSE)
elif attack_name == 'boundary':
attack = partial(foolbox.attacks.BoundaryAttack(model, distance=MSE),
iterations=2000,
log_every_n_steps=np.Infinity,
verbose=False)
else:
raise NotImplementedError(
"attack_name must be 'l1', 'l2', 'itl1', 'itl2', "
"'deepFool' or 'boundary'")
n_iterations = 0
results = {}
results['l2_snr'] = []
results['clean_grad'] = []
results['dirty_grad'] = []
results['l2_norm'] = []
results['linf_norm'] = []
n_fooled = 0
print('> Creating empty image-tensors')
n_saved = 64 if (n_attacks == -1) else min(n_attacks, 64)
clean_images = torch.zeros(n_saved, 3, args.img_size, args.img_size)
dirty_images = torch.zeros(n_saved, 3, args.img_size, args.img_size)
print('> Done.')
myPrint(("{:>15} " * 5).format("clean_grad", "dirty_grad", "linf_norm",
"l2_norm", "l2_snr"), args)
t0 = time.time()
for i, (images, labels) in enumerate(loader):
if n_iterations == n_attacks:
break
for i, clean_image in enumerate(images):
clean_label, clean_grad = classify(net, clean_image, args, means,
stds)
dirty_image_np = attack(clean_image.numpy(), clean_label)
if dirty_image_np is not None: # i.e. if adversarial was found
dirty_image = torch.Tensor(dirty_image_np)
_, dirty_grad = classify(net, dirty_image, args, means, stds)
if i < n_saved: # only save n_saved first images
dirty_images[i] = dirty_image.clone()
clean_images[i] = clean_image.clone()
l2_norm = (clean_image - dirty_image).norm().item()
linf_norm = (clean_image - dirty_image).abs().max().item()
l2_snr = 20. * math.log10(clean_image.norm().item() /
(l2_norm + 1e-6))
else:
l2_snr = dirty_grad = l2_norm = linf_norm = np.NaN
results['l2_snr'].append(l2_snr)
results['clean_grad'].append(clean_grad)
results['dirty_grad'].append(dirty_grad)
results['l2_norm'].append(l2_norm)
results['linf_norm'].append(linf_norm)
fmt_str = "{:>15.6f} " * 5
if ((attack.func._default_distance == MSE
and l2_norm < .005 * np.sqrt(args.img_size))
or (attack.func._default_distance == Linfinity
and linf_norm < .005)):
fmt_str += " * fooled!"
n_fooled += 1
myPrint(
fmt_str.format(clean_grad, dirty_grad, linf_norm, l2_norm,
l2_snr), args)
n_iterations += 1
if n_iterations == n_attacks:
break
# Printing summary
summary = {}
print("\n Summary for network in '{}' of test accuracy {}".format(
args.path, te_acc))
for key, value in results.items():
low95, high95 = conf95(value)
print("{:>10} mean:{:>10.5f} std:{:>10.5f} conf95:({:>10.5f}, "
"{:>10.5f}) minmax:({:>10.5f}, {:>10.5f})".format(
key, np.nanmean(value), np.nanstd(value), low95, high95,
np.nanmin(value), np.nanmax(value)))
summary[key] = [np.nanmean(value), np.nanstd(value), low95, high95]
percent = 100 * n_fooled / float(n_iterations)
print("{:>10} {:10d}s".format("Time", int(time.time() - t0)))
print("{:>10} {:10.1f}%".format("percent", percent))
# Preparing the output
output = dict()
output['summary'] = summary
output['results'] = results
output['clean_images'] = clean_images
output['dirty_images'] = dirty_images
output['percent'] = percent
output['te_acc'] = te_acc
return output
checkpoints = sorted(
checkpoints
) # pip install natsort: natsorted() would be a better choice..
assert len(checkpoints) != 0, 'No checkpoints found.'
checkpoint_file = checkpoints[-1]
print('Loading [{}]'.format(checkpoint_file))
model = load_model(checkpoint_file)
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
print(model.summary())
x_train, y_train, x_test, y_test = MNIST.get_mnist_data()
# checking that the accuracy is the same as before 99% at the first epoch.
# test_loss, test_acc = model.evaluate(x_test, y_test, verbose=0, batch_size=128)
# print('')
# assert test_acc > 0.98
utils.print_names_and_shapes(
keract.get_activations(model, x_test[0:200])) # with 200 samples.
utils.print_names_and_shapes(
keract.get_gradients_of_trainable_weights(model, x_train[0:10],
y_train[0:10]))
utils.print_names_and_shapes(
keract.get_gradients_of_activations(model, x_train[0:10],
y_train[0:10]))
from data import MNIST
from dbn import DBN
import numpy as np
d = DBN([28*28, 500, 2000], 10,0.1)
images = MNIST.load_images('test_images')
labels = MNIST.load_labels('test_labels')
img = images[0:100]
lbl = labels[0:100]
tst_img = images[9000:95000]
tst_lbl = labels[9000:95000]
d.pre_train(img,30,50)
for i in xrange(0, 100):
d.train_labels(img, lbl, 3, 50)
tst_class = d.classify(tst_img,20)
print 'Error kurwa: {0}'.format( 1 - ((tst_class * tst_lbl).mean()) * 10)
#print d.sample(img[0:1],0)
#print d.sample(img[1:2],0)
#print 'layer 2'
#print d.sample(img[0:1],1)
#print d.sample(img[1:2],1)
#print 'layer 2'
#print d.sample(img[0:1],2)
#print d.sample(img[1:2],2)
#print 'layer 3'
#print d.sample(img[0:1],3)[0]
#print d.sample(img[1:2],3)[0]
from data import MNIST
from dbn import DBN
import numpy as np
d = DBN([28 * 28, 500, 500, 2000], 10, 0.1)
images = MNIST.load_images("images")
labels = MNIST.load_labels("labels")
img = images[0:60000]
lbl = labels[0:60000]
tst_img = MNIST.load_images("test_images")
tst_lbl = MNIST.load_labels("test_labels")
d.pre_train(img, 5, 50)
for i in xrange(0, 100):
d.train_labels(img, lbl, 50, 50)
tst_class = d.classify(tst_img, 10)
print "Error over test data: {0}".format(1 - (tst_class * tst_lbl).mean() * 10)
# print d.sample(img[0:1],0)
# print d.sample(img[1:2],0)
# print 'layer 2'
# print d.sample(img[0:1],1)
# print d.sample(img[1:2],2)
# print 'layer 2'
# print d.sample(img[0:1],2)
# print d.sample(img[1:2],2)
# print 'layer 3'
# print d.sample(img[0:1],3)[0]
# print d.sample(img[1:2],3)[0]
num_labels = read32(f)
print('Number of labels:', num_labels)
buf = f.read(num_labels)
labels = np.frombuffer(buf, dtype=np.uint8)
# Convert to "one-hot" vectors
return (labels)
train_images = extract_images('train-images-idx3-ubyte.gz')
test_images = extract_images('t10k-images-idx3-ubyte.gz')
train_labels = extract_labels('train-labels-idx1-ubyte.gz')
test_labels = extract_labels('t10k-labels-idx1-ubyte.gz')
# Randmoze the order of training and test data
(train_images, train_labels) = sklearn.utils.shuffle(train_images,
train_labels,
random_state=0)
(test_images, test_labels) = sklearn.utils.shuffle(test_images,
test_labels,
random_state=0)
print('train images:', train_images.shape)
print('train labels:', train_labels.shape)
print('test images:', test_images.shape)
print('test labels:', test_labels.shape)
data = MNIST(train_images, test_images, train_labels, test_labels)
pickle.dump(data, open(pickle_file, "wb"), protocol=4)
mu = np.mean(X, axis=0)
pca = PCA()
pca.fit(X)
trandformedData = pca.transform(X)
reconstructed = np.dot(trandformedData[:, :ncomponent],
pca.components_[:ncomponent, :])
reconstructed += mu
errors = np.square(X - reconstructed)
return reconstructed, np.nanmean(errors)
if __name__ == "__main__":
mnist = MNIST(10000)
fhr = FHR()
# errors = []
# sampleRange= range(1, 41, 1)
# for k in sampleRange:
# reconstruced, error = PCA_Reconstruction(fhr.X_train, k)
# errors.append(error)
# util.plot_regular(sampleRange, errors, "n-component", "MSE", "PCA reconstruction error")
#
# reconstructed = PCA_Reconstruction(mnist.X_train, 784)
#
# util.plot_Original_Reconstructed(mnist.X_train[0].reshape(28, 28), reconstructed[0].reshape(28, 28))
start = time.time()
pca = PCA(random_state=0, n_components=400)
from data import MNIST
from dbn import DBN
import numpy as np
# An example of how to use the library
# Creates a DBN
d = DBN([28 * 28, 500, 500, 2000], 10, 0.1)
# Loads the Images and Labels from the MNIST database
images = MNIST.load_images('images')
labels = MNIST.load_labels('labels')
# Gets the training set of images
img = images[0:60000]
lbl = labels[0:60000]
# Permutes the data to ensure random batches
train_tuples = zip(img, lbl)
train_tuples = np.random.permutation(train_tuples)
(img, lbl) = zip(*train_tuples)
# Loads the test images from the MNIST database
tst_img = MNIST.load_images('test_images')
tst_lbl = MNIST.load_labels('test_labels')
# Pre trains the DBN
d.pre_train(img, 5, 50)
# Executes a 100 iterations of label training
# Attempts to classify and prints the error rate
from tensorflow.keras.layers import Dense
from tensorflow.keras.layers import LSTM
from tensorflow.keras.models import Sequential
import keract
import utils
from data import MNIST
if __name__ == '__main__':
# gradients requires no eager execution.
tf.compat.v1.disable_eager_execution()
# Check for GPUs and set them to dynamically grow memory as needed
# Avoids OOM from tensorflow greedily allocating GPU memory
utils.gpu_dynamic_mem_growth()
x_train, y_train, _, _ = MNIST.get_mnist_data()
# (60000, 28, 28, 1) to ((60000, 28, 28)
# LSTM has (batch, time_steps, input_dim)
x_train = x_train.squeeze()
model = Sequential()
model.add(LSTM(16, input_shape=(28, 28)))
model.add(Dense(MNIST.num_classes, activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
utils.print_names_and_shapes(keract.get_activations(model, x_train[:128]))
utils.print_names_and_shapes(
# ('hflip', {}),
# ('cutout', {'scale': (0.02, 0.15)}),]
# aug_ops = compose_augment_seq(aug_list, is_training=False)
aug_list = [('gaussian', {'std': 0.1}),
('translation', {'x_max': 0.0, 'y_max': 5.0}),
('affine', {'std': 0.5}),
('intensity_flip', {}),
('intensity_scaling', {'min': 0.25, 'max': 1.5}),
('intensity_offset', {'min': -0.5, 'max': 0.5})
]
from data import MNIST
tmp_root = 'tmp'
for aug in aug_list:
dsld = MNIST(batch_size=10, aug_list=[aug], train_shuffle=False)
dsld_ori = MNIST(batch_size=10, train_shuffle=False)
for (x_ori, _), (x, _) in zip(dsld_ori.train, dsld.train):
for i, (x_ori_b, x_b) in enumerate(zip(x_ori, x)):
x_concat = tf.concat((x_ori_b, x_b), 1)
x_png = tf.image.encode_png(tf.cast(x_concat*255, tf.uint8))
tf.io.write_file(os.path.join(tmp_root, aug[0], "%d.png"%(i)), x_png)
break
class Identity:
def __call__(self, x, y, is_training=True):
return x, y
