def main():
os.chdir("/home/swy/code/DRL/autoencoder_models/data")
L = []
for files in os.walk("/home/swy/code/DRL/autoencoder_models/data"):
for file in files:
L.append(file)
#autoencoder pretrain w1, b1
#if train:
autoencoder = Autoencoder(
n_input=10,
n_hidden=128,
transfer_function=tf.nn.softplus,
optimizer=tf.train.AdamOptimizer(learning_rate=0.001))
# train the whole file data
batchsize = 100
epoch = 1
#print(np.floor(len(Agent.dataBase)/batchsize))
for j in range(epoch):
for k in range(12):
Agent = Agent2(L[2][k], 10, 100, 2000)
for i in range(int(np.floor(len(Agent.dataBase) / batchsize))):
#print(len(Agent.dataBase))
state = Agent.get_state(i)
cost = autoencoder.partial_fit(state)
if i % 10 == 0:
print("cost")
print(cost)
w = autoencoder.getWeights()
b = autoencoder.getBiases()
if tf.gfile.Exists('/home/swy/code/DRL/tbencoder'):
tf.gfile.DeleteRecursively('/home/swy/code/DRL/tbencoder')
tf.gfile.MakeDirs('/home/swy/code/DRL/tbencoder')
config = get_config()
sess = tf.InteractiveSession()
train = False
if train:
out = lmmodel(config=config, sess=sess, W1=w, B1=b, FileList=L)
sess.run(tf.global_variables_initializer())
out.learn()
saver = tf.train.Saver(tf.global_variables())
save_path = out.saver.save(sess,
'/home/swy/code/DRL/cpencoder/model.ckpt')
t = True
if t:
out = lmmodel(config=config, sess=sess, W1=w, B1=b, FileList=L)
load_path = out.saver.restore(
sess, '/home/swy/code/DRL/cpencoder/model.ckpt')
out.learn()
def __init__(self, patch_size, stride=1, hidden=0.05):
self.n_input = patch_size**2 * 3
n_hidden = int(np.round(self.n_input * hidden))
self.patch_size = patch_size
self.stride = stride
self.padding = int(np.floor(self.patch_size / 2.0))
self.autoencoder = Autoencoder([self.n_input, n_hidden])
# Load the image file.
self.x = tf.compat.v1.placeholder(tf.string)
self.chunk = tf.compat.v1.placeholder(tf.int32, [4])
self.read_file = tf.io.read_file(self.x)
self.decode_image = tf.cast(tf.image.decode_image(self.read_file,
channels=3),
dtype=tf.float32)
# Slice image in chunks so really large images can be processed with constant
# memory consumption.
self.crop_chunk = tf.image.crop_to_bounding_box(
self.decode_image, self.chunk[0], self.chunk[1], self.chunk[2],
self.chunk[3])
# Extract image patches in the correct size and reshape the 2D array of patches
# to an 1D array that can be fed to the autoencoder.
self.extract_patches = tf.image.extract_patches(
images=[self.crop_chunk],
sizes=[1, self.patch_size, self.patch_size, 1],
strides=[1, self.stride, self.stride, 1],
rates=[1, 1, 1, 1],
padding='VALID')
self.patches_shape = tf.shape(input=self.extract_patches[0])
self.reshape = tf.reshape(self.extract_patches[0], [-1, self.n_input])
# Apply the autoencoder.
self.element_wise_cost = self.autoencoder.element_wise_cost(
self.reshape)
# Reshape the array back to the original image size (minus the invalid border).
# Collapse the grayscale channel so the result has shape [height, width] instead
# of [height, width, 1].
self.reshape_back = tf.reshape(self.element_wise_cost,
self.patches_shape[:2])
self.sess = self.autoencoder.sess
def getNetworkWithWeights(self, networkType, appliance, house):
print(('Loading weights for House ' + str(house)).ljust(40, '.'))
if networkType == 'vae':
network = Discova()
weightPath = 'weights/weights_0{}_{}.hdf5'.format(house, appliance)
else:
network = Autoencoder()
weightPath = 'weights/ae_0{}_{}.hdf5'.format(house, appliance)
print('Weights loaded. Compiling model.'.ljust(40, '.'))
network.construct_model()
network.compile_model()
network.model.load_weights(weightPath)
return network
def train_data (dataset_name,training_data, testing_data, run, cancer_training_size, cancer_testing_size):
# with tf.Session() as sess:
sess = tf.Session()
auto = Autoencoder(
sess,
run=run,
cancer_training_size=cancer_training_size,
cacner_testing_size=cancer_testing_size,
dataset_name=dataset_name,
epochs=FLAGS.epochs,
learning_rate=FLAGS.learning_rate,
batch_size=FLAGS.batch_size,
n_layers=FLAGS.n_layers,
training_data=training_data,
testing_data=testing_data,
checkpoint_dir=FLAGS.checkpoint_dir,
train=FLAGS.train,
generate=FLAGS.generate,
s_number=FLAGS.s_number)
sess.close()
'decoder_A': 'models/decoder_A.h5',
'decoder_B': 'models/decoder_B.h5'
}
INPUT_DIR = 'raw/clooney'
OUTPUT_DIR = 'output'
DIRECTION = 'AtoB' # conversion direction: {'AtoB', 'BtoA'}
# conversion params
size = 256 # aligned face size
input_size = 64 # model input size
crop = slice(48, 208) # for cropping (160 x 160) from aligned face
erosion_kernel_size = 10 # erosion inward size for masking output to input image
blur_size = 5 # blur size for masking output to input image
# Setup model and load weights
model = Autoencoder(MODEL_DIR)
model.load_weights()
autoencoder_A = model.autoencoder_A
autoencoder_B = model.autoencoder_B
encoder = model.encoder
decoder_A = model.decoder_A
decoder_B = model.decoder_B
def get_image_mask(face, new_face, mat, input_image, image_size):
# get mask with transformed input shape
face_mask = numpy.zeros(input_image.shape, dtype=float)
face_src = numpy.ones(new_face.shape, dtype=float)
cv2.warpAffine(face_src, mat * 64, image_size, face_mask,
cv2.WARP_INVERSE_MAP, cv2.BORDER_TRANSPARENT)
from tensorflow.keras.preprocessing import image
from Autoencoder import load_test_img, Autoencoder, setup_gpu
if __name__ == '__main__':
setup_gpu()
(color, black) = load_test_img('kwiat.png', (960, 640))
autoencoder = Autoencoder('Test', 64)
autoencoder.build()
autoencoder.load('models/Autoencoder/Autoencoder_0.h5')
y = autoencoder.predict(black)
img = image.array_to_img(y, scale=False)
img.save('colored.png')
black *= 255.0
img = image.array_to_img(black, scale=False)
img.save('black.png')
color *= 255.0
img = image.array_to_img(color, scale=False)
img.save('original.png')
from Dataset import Dataset
from Discova import Discova
from Autoencoder import Autoencoder
import numpy as np
import pandas as pd
results = pd.DataFrame()
# train house on rows, test house on cols
dfLabels = pd.read_csv('appliance_codes.csv')
applianceLabels = dfLabels['Washer Dryer'].tolist()
for train_house in [5]:
discova = Autoencoder() # Turn off variance for prediction
discova.construct_model()
discova.compile_model()
path = 'weights/ae_0' + str(train_house) + '_washerdryer.hdf5'
discova.model.load_weights(path)
print(('Loading weights for House ' + str(train_house)).ljust(40,'.'))
for test_house in range(1,7):
load_str = 'Loading test data for House ' + str(test_house)
print(load_str.ljust(40,'.'))
applianceLabel = applianceLabels[test_house - 1]
data = Dataset()
data.load_house_dataframe(test_house)
data.add_windows(test_house, '00 mains')
data.add_windows(test_house, applianceLabel)
data.add_statistics(test_house)
if __name__ == '__main__':
epochs = 50
batch_size = 100
latent_dim = 2
reg = True
dataloader = utils.get_dataloader(batch_size)
device = utils.get_device()
step_per_epoch = np.ceil(dataloader.dataset.__len__() / batch_size)
sample_dir = './samples'
checkpoint_dir = './checkpoints'
utils.makedirs(sample_dir, checkpoint_dir)
net = Autoencoder(latent_dim=latent_dim, enc_sig=False).to(device)
optim = utils.get_optim(net, 0.0002)
loss_log = []
criterion = nn.MSELoss()
result = None
for epoch_i in range(1, epochs + 1):
for step_i, (real_img, _) in enumerate(dataloader):
real_img = real_img.to(device)
if result is None:
result = real_img
code = net.encoder(real_img)
X_test = preprocessor.transform(X_test)
return X_train, X_test
def get_random_block_from_data(data, batch_size):
start_index = np.random.randint(0, len(data) - batch_size)
return data[start_index:(start_index + batch_size)]
X_train, X_test = standard_scale(mnist.train.images, mnist.test.images)
n_samples = int(mnist.train.num_examples)
training_epochs = 50
batch_size = 128
display_step = 1
autoencoder = Autoencoder(n_input = 784, # input vector dimension
n_hidden = 200, # hidden units num
transfer_function = tf.nn.softplus,
optimizer = tf.train.AdamOptimizer(learning_rate = 0.001))
for epoch in range(training_epochs): # how many times on the dataset
avg_cost = 0.
total_batch = int(n_samples / batch_size)
# Loop over all batches
for i in range(total_batch):
batch_xs = get_random_block_from_data(X_train, batch_size) # randomly generate batches
# Fit training using batch data
cost = autoencoder.partial_fit(batch_xs)
# Compute average loss
avg_cost += cost / n_samples * batch_size
# Display logs per epoch step
Dtest11=np.fromfile("Datapre/syn_A0.dat",dtype=np.float64)
Dtest11=Dtest11.reshape(60,1600)
n_samples=len(Dtrain)
training_epochs =10
batch_size = 15
display_step = 1
corruption_level = 0.0
sparse_reg=1
#
n_inputs = 1600
n_hidden = 3200
ae = Autoencoder(n_layers=[n_inputs, n_hidden],
transfer_function = tf.nn.sigmoid,
optimizer = tf.compat.v1.train.AdamOptimizer(learning_rate = 0.001),ae_para = [corruption_level, sparse_reg])
init = tf.compat.v1.global_variables_initializer()
sess = tf.compat.v1.Session()
sess.run(init)
for epoch in range(training_epochs):
avg_cost = 0
total_batch = int(n_samples / batch_size)
print("NO of Epochs",training_epochs,"N_samples",n_samples,"NO total_batches",total_batch,"batch size",batch_size)
# Loop over all batches
for i in range(total_batch):
# batch_xs= Dtrain[i*batch_size:(i+1)*batch_size,:] #just consider x, not ylabel
# ass=mnist.train.images
batch_xs=next_batch_random(Dtrain,batch_size)
# Fit training using batch data
#temp = ae.partial_fit()
def atexit_handler():
global AE
AE.save()
# Register callback to save on exit
atexit.register(atexit_handler)
# Read arguments
parser = argparse.ArgumentParser()
parser.add_argument('--dataset-dir', type=str, default='data/', help='')
parser.add_argument('-d',
'--debug',
action='store_true',
help='run in debug mode (no output files)')
parser.add_argument('--batch-size', type=int, default=64, help='')
parser.add_argument('--nb-epochs', type=int, default=3, help='')
args = parser.parse_args()
logger = Logger(debug=args.debug)
AE = Autoencoder((4 * 84 * 84, ), logger=logger)
# Create the batch iterator for the images
batches = batch_iterator(args.dataset_dir,
batch_size=args.batch_size,
nb_epochs=args.nb_epochs)
for idx, batch in enumerate(batches):
loss, accuracy = AE.train(batch)
print 'Autoencoder batch %d: loss %f - acc %f' % (idx, loss, accuracy)
def train(args, modelparam = ""):
# Load MNIST-M
# mnist = pkl.load(open(os.path.join(args.file_path,args.data_set), 'rb'))
mnist = input_data.read_data_sets('MNIST_data')
mnist_train_images = mnist.train.images
mnist_test_images = mnist.test.images
# save configuration
save_dir_modelparam = os.path.join(args.save_dir, modelparam)
if not os.path.isdir(save_dir_modelparam):
if not os.path.isdir(args.save_dir):
os.makedirs(args.save_dir)
os.makedirs(save_dir_modelparam)
with open(os.path.join(save_dir_modelparam, 'config.pkl'), 'wb') as f:
pkl.dump(args, f)
model = Autoencoder(args)
summariesAutoEncOp = tf.summary.merge(model.summaries)
with tf.Session() as sess:
# init
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(tf.global_variables())
print("Trainable")
names = [x.name for x in tf.trainable_variables()]
[print(n) for n in names]
start_time = time.strftime("%Y-%m-%d-%H-%M-%S")
# summary writer
writer = tf.summary.FileWriter(os.path.join(args.log_dir,
modelparam,
start_time),
sess.graph)
tf.get_default_graph().finalize()
# Batch generators
gen_train_batch = batch_generator([mnist_train_images], args.batch_size)
visualization = next(gen_train_batch)
visualization = visualization[0]
visualization = np.squeeze(visualization.reshape([args.batch_size, args.img_size_h, args.img_size_w, args.n_channels]))
num_imgs = int(np.floor(np.sqrt(visualization.shape[0])))
im = toimage(merge(visualization[:(num_imgs*num_imgs)],[num_imgs,num_imgs]))
im.save(os.path.join(save_dir_modelparam,"base_train"+modelparam+".jpg"))
# Batch generators
gen_test_batch = batch_generator([mnist_test_images], args.batch_size)
visualization = next(gen_test_batch)
visualization = visualization[0]
visualization = np.squeeze(visualization.reshape([args.batch_size, args.img_size_h, args.img_size_w, args.n_channels]))
im = toimage(merge(visualization[:(num_imgs*num_imgs)],[num_imgs,num_imgs]))
im.save(os.path.join(save_dir_modelparam,"base_test"+modelparam+".jpg"))
num_batches = int(len(mnist_train_images)/args.batch_size)
# run training
for e in range(args.num_epochs_ae):
print("\nEpoch " + str(e))
new_lr = model.lr.eval() * (args.decay_rate ** e)
sess.run(model.lr_update, feed_dict={model.new_lr: new_lr})
print("Learning rate: " + str(model.lr.eval()))
for b in range(num_batches):
global_step = e * num_batches + b
X = next(gen_train_batch)
batch = X[0].reshape(args.batch_size, args.img_size_h*args.img_size_w*args.n_channels)
start = time.time()
_, gen_loss, lat_loss, s = sess.run([model.train_op,
model.recon_loss,
model.lat_loss,
summariesAutoEncOp],
feed_dict={model.images_in: batch})
end = time.time()
writer.add_summary(s, global_step)
print("{}/{} (epoch {}), gen_loss = {:.3f}, lat_loss = {:.3f},\
time/batch = {:.3f}"
.format(global_step,
args.num_epochs_ae * num_batches,
e, np.mean(gen_loss), np.mean(lat_loss), end - start))
# save model and visualize
if global_step % args.save_every == 0 or (e==args.num_epochs_ae-1 and b == num_batches-1): # save for the last result
checkpoint_path = os.path.join(args.log_dir,
modelparam,
start_time,
'model.ckpt')
saver.save(sess, checkpoint_path, global_step = global_step)
print("model saved to {}".format(checkpoint_path))
# viz
generated_test= sess.run(model.generated_images_out,
feed_dict={model.images_in: visualization.reshape(args.batch_size, args.n_input)})
generated_test = np.squeeze(generated_test)
im = toimage(merge(generated_test[:(num_imgs*num_imgs)],[num_imgs,num_imgs]))
im.save(os.path.join(save_dir_modelparam,str(e)+modelparam+".jpg"))
def main():
model = Autoencoder(MODEL_DIR)
model.load_weights()
autoencoder_A = model.autoencoder_A
autoencoder_B = model.autoencoder_B
print('loading training data...')
images_A = get_image_paths(IMAGE_A_DIR)
images_B = get_image_paths(IMAGE_B_DIR)
images_A = load_images(images_A) / 255.0
images_B = load_images(images_B) / 255.0
# hack for FIXED_PREVIEW
fixed_preview_img_initialized = False
print("press 'q' to stop training and save model")
# open file for log
if not os.path.exists(os.path.dirname(TRAIN_LOG_PATH)):
os.mkdirs(os.path.dirname(TRAIN_LOG_PATH))
with open(TRAIN_LOG_PATH, 'a') as f:
for epoch in range(1000000):
batch_size = 64
warped_A, target_A = get_training_data(images_A, batch_size)
warped_B, target_B = get_training_data(images_B, batch_size)
# hack for FIXED_PREVIEW
if FIXED_PREVIEW and not fixed_preview_img_initialized:
test_A, test_B = load_fixed_preview_img(
target_A[0:14], target_B[0:14])
fixed_preview_img_initialized = True
loss_A = autoencoder_A.train_on_batch(warped_A, target_A)
loss_B = autoencoder_B.train_on_batch(warped_B, target_B)
msg = "time: {:.2f} epoch: {} loss_A: {} loss_B: {}".format(
time.time(), epoch, loss_A, loss_B)
print(msg)
f.write(msg + '\n')
f.flush()
if epoch % 100 == 0:
model.save_weights()
# hack for FIXED_PREVIEW
if not FIXED_PREVIEW:
test_A = target_A[0:14]
test_B = target_B[0:14]
figure_A = numpy.stack([
test_A,
autoencoder_A.predict(test_A),
autoencoder_B.predict(test_A),
],
axis=1)
figure_B = numpy.stack([
test_B,
autoencoder_B.predict(test_B),
autoencoder_A.predict(test_B),
],
axis=1)
figure = numpy.concatenate([figure_A, figure_B], axis=0)
figure = figure.reshape((4, 7) + figure.shape[1:])
figure = stack_images(figure)
figure = numpy.clip(figure * 255, 0, 255).astype('uint8')
if SHOW_PREVIEW:
cv2.imshow("", figure)
if SAVE_PREVIEW:
cv2.imwrite(
os.path.join(
MODEL_BASE_DIR,
'model_preview_epoch_{}.jpg'.format(epoch)),
figure)
key = cv2.waitKey(1)
if key == ord('q'):
print('saving weights...')
model.save_weights()
exit()
from Autoencoder import Autoencoder
from helpers import *
from Logger import Logger
def atexit_handler():
global AE
AE.save()
# Register callback to save on exit
atexit.register(atexit_handler)
# Read arguments
parser = argparse.ArgumentParser()
parser.add_argument('--dataset-dir', type=str, default='data/', help='')
parser.add_argument('-d', '--debug', action='store_true', help='run in debug mode (no output files)')
parser.add_argument('--batch-size', type=int, default=64, help='')
parser.add_argument('--nb-epochs', type=int, default=3, help='')
args = parser.parse_args()
logger = Logger(debug=args.debug)
AE = Autoencoder((4 * 84 * 84,), logger=logger)
# Create the batch iterator for the images
batches = batch_iterator(args.dataset_dir, batch_size=args.batch_size, nb_epochs=args.nb_epochs)
for idx, batch in enumerate(batches):
loss, accuracy = AE.train(batch)
print 'Autoencoder batch %d: loss %f - acc %f' % (idx, loss, accuracy)
img = Image(img_filename=scan, mask_filename=args.mask[index], atlas_filename=args.atlas[index], fwhm=5)
dim = img.image_mat_in_mask_atlas.shape[1]
#mat = PCA(whiten=True).fit_transform(img.image_mat_in_mask_atlas).T
mat = img.image_mat_in_mask_atlas.T
if all_atlas_mat is None:
all_atlas_mat = mat
all_atlas_time_dim = dim
else:
all_atlas_mat = np.concatenate((all_atlas_mat, mat))
all_atlas_time_dim = np.append(all_atlas_time_dim, dim)
all_atlas_mat = all_atlas_mat.T
all_atlas_mat=all_atlas_mat-all_atlas_mat.mean(axis=1)[:,None]
all_atlas_mat=all_atlas_mat-all_atlas_mat.std(axis=1)[:,None]
all_atlas_mat=all_atlas_mat.T
pca_group = PCA(whiten=True).fit_transform(all_atlas_mat.T)
graph_mat_2_atlas_nii(pca_group.T,args.atlasref,"group_atlas_pca.nii.gz")
ae_atlas = Autoencoder(args.n, max_iter=args.maxiter).fit(pca_group.T)
sio.savemat("group_atlas.mat", {'h':ae_atlas.h,'W1':ae_atlas.W1,'W2':ae_atlas.W2})
graph_mat_2_atlas_nii(ae_atlas.h,args.atlasref,"group_atlas.nii.gz")
import pdb; pdb.set_trace()
for index, scan in enumerate(args.img):
print scan
img = Image(img_filename=scan, mask_filename=args.mask[index], atlas_filename=args.atlas[index])
pca = PCA(n_components=50,whiten=True).fit_transform(img.image_mat_in_mask_normalised_atlas)
graph_mat_2_atlas_nii(pca.T,args.atlasref,(scan[0:scan.find('.nii')]+'.pca.nii.gz'))
Wpca = np.concatenate((pca.T,pca_group.T));
autoencoder = Autoencoder(args.n, max_iter=args.maxiter).fit(Wpca)
graph_mat_2_atlas_nii(autoencoder.h,args.atlasref,(scan[0:scan.find('.nii')]+'.autoencoder.nii.gz'))
from lib.process.Evaluation import Evaluation
from Autoencoder import Autoencoder
# CONSTANTS:
USE_CUDA = torch.cuda.is_available()
EPOCHs = 2
SMALL = True
DATASETMODE = 'autoencoderSpike'
netParams = snn.params('network_specs/autoencoder.yaml')
print(netParams)
device = torch.device("cuda" if USE_CUDA else "cpu")
# Create network instance.
net = Autoencoder(netParams).to(device)
# Define optimizer module.
optimizer = torch.optim.Adam(net.parameters(), lr=0.001, amsgrad=True)
# Learning stats instance.
stats = snn.learningStats()
trainingSet = SMNIST(datasetPath=netParams['training']['path']['in'],
samplingTime=netParams['simulation']['Ts'],
sampleLength=netParams['simulation']['tSample'],
mode=DATASETMODE,
small=SMALL)
testingSet = SMNIST(datasetPath=netParams['training']['path']['in'],
momentum = 0.1
display_step = 1
# activation function and optimizer
transfer_function = tf.nn.softplus
optimizer = tf.train.MomentumOptimizer(learning_rate, momentum = momentum)
# save image
save_image_name = 'mnist-momen.png'
# device config
device = '/gpu:2'
log_device_placement = True
with tf.device(device):
autoencoder = Autoencoder(device = device, n_input = n_input, n_hidden = n_hidden, transfer_function = transfer_function, optimizer = optimizer)
start_time = datetime.datetime.now()
for epoch in range(training_epochs):
avg_cost = 0.
total_batch = int(n_samples / batch_size)
# Loop over all batches
for i in range(total_batch):
batch_xs = get_random_block_from_data(X_train, batch_size)
# Fit training using batch data
cost = autoencoder.partial_fit(batch_xs)
# Compute average loss
avg_cost += cost / n_samples * batch_size
# Display logs per epoch step
num_batch = len(train_loader)
val_num_batch = len(val_loader)
test_num_batch = len(test_loader)
try:
os.makedirs(args.outf)
except OSError:
pass
#Encoder = Encoder(args.input_dim, [64, 128, 128, 256, args.latent_dim], 'relu')
#Decoder = Decoder(args.latent_dim, [256, 256, args.npoints * 3])
#Encoder = Encoder.cuda()
#Decoder = Decoder.cuda()
net = Autoencoder(args.input_dim, args.latent_dim,
[64, 128, 128, 256, args.latent_dim],
[256, 256, args.npoints * 3])
#device = torch.device('cuda: 0' if torch.cuda.is_available() else 'cpu')
net = net.cuda()
if args.optimizer == 'Adam':
optimizer = optim.Adam(list(net.parameters()),
lr=args.learn_rate,
betas=(0.9, 0.999))
elif args.optimizer == 'SGD':
optimizer = optim.SGD(list(net.parameters()),
lr=args.learn_rate,
momentum=args.momentum)
#scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=20, gamma=0.5)
if args.model != '':
import time
from Decoder import Decoder
from Encoder import Encoder
from Autoencoder import Autoencoder
import torch.optim as optim
from torch import nn
from torch.autograd import Variable
import sys
if __name__ == '__main__':
train_loader, test_loader = load_svhn('/Tmp/lavoiems', 32)
setup(sys.argv[1], sys.argv[2], env='test', use_tanh=True)
shape = train_loader.dataset.data.shape
encoder = Encoder(shape, 32, True, None, 'ReLU', 4).cuda()
decoder = Decoder(shape, 32, True, None, 'ReLU', 4).cuda()
autoencoder = Autoencoder(encoder, decoder).cuda()
optimizer = optim.Adam(autoencoder.parameters(),
lr=1e-4,
betas=(0.5, 0.999))
criterion = nn.MSELoss()
for i in range(200):
print('Epoch: %s' % i)
autoencoder.train()
start = time.time()
for train_data in train_loader:
inputs, labels = Variable(train_data[0].cuda()), Variable(
train_data[1].cuda())
inputs = 2 * inputs - 1.
prime = autoencoder(inputs)
loss = criterion(prime, inputs)
def get_random_block_from_data(data, batch_size):
start_index = np.random.randint(0, len(data) - batch_size)
return data[start_index:(start_index + batch_size)]
X_train = pd.read_csv('X_train.csv')
n_samples,_ = np.shape(X_train)
training_epochs = 1000
batch_size = **** #Adjustable parameters,Default is the number of samples subtract one
display_step = 1
autoencoder = Autoencoder(
n_input = ****, #Number of features
n_hidden1 = 800,
n_hidden2 = 400,
n_hidden3 = 800,
transfer_function=tf.nn.softplus,
optimizer=tf.train.AdamOptimizer(learning_rate=0.001))
for epoch in range(training_epochs):
avg_cost = 0.
total_batch = int(n_samples / batch_size)
# Loop over all batches
for i in range(total_batch):
batch_xs = get_random_block_from_data(X_train, batch_size)
# Fit training using batch data
cost = autoencoder.partial_fit(batch_xs)
weight_decay_param=args.weight_decay).fit(
img.image_mat_in_mask_normalised.T,
params_init_scan_rica,
params_bias_scan_rica,
optimization=True)
img.write_decomposition(maps=rica.components_,
filename=filename,
normalise=True)
W_all_new[:, i:j] = rica.W
filename = (scan[0:scan.find('.nii')] + '.fwhm' + str(args.fwhm) + 'mm.' +
'_backpropagation_groupbias.reference.nii.gz')
ae_voxels = Autoencoder(img.image_mat_in_mask_normalised.shape[1],
max_iter=args.maxiter,
weight_decay_param=args.weight_decay,
second_nonlinear=True).fit(
img.image_mat_in_mask_normalised.T,
params_init_scan,
optimization=False)
img.write_decomposition(maps=ae_voxels.components_,
filename=filename,
normalise=False)
filename = (scan[0:scan.find('.nii')] + '.fwhm' + str(args.fwhm) + 'mm.' +
'_backpropagation_groupbias.' + str(args.weight_decay) +
'.nii.gz')
ae_voxels = Autoencoder(img.image_mat_in_mask_normalised.shape[1],
max_iter=args.maxiter,
weight_decay_param=args.weight_decay,
second_nonlinear=True).fit(
img.image_mat_in_mask_normalised.T,
from Autoencoder import Autoencoder
import numpy as np
from dataset_utils import load_dataset
#x_train = load_dataset("../data/data_np50/1")
x_train = load_dataset("../data/data100_np/small")
print(x_train.shape)
ae = Autoencoder(100, 100, x_train, x_train)
ae.create_model()
#ae.load_model("hehehe.h5")
ae.train_model(32, 20, 1)
plt.axis('off')
plt.imshow(data.reshape(28, 28))
plt.savefig("./lerp/epoch_" + str(l) + "_" + suffix + '.png')
train, test = chainer.datasets.get_mnist()
train = train[0:1000]
test = test[47:63]
#plot_mnist_data(test, "base")
train = [i[0] for i in train]
train = tuple_dataset.TupleDataset(train, train)
train_iter = chainer.iterators.SerialIterator(train, 100)
enc = Autoencoder()
model = L.Classifier(enc, lossfun=F.mean_squared_error)
model.compute_accuracy = False
optimizer = chainer.optimizers.Adam()
optimizer.setup(model)
updater = training.StandardUpdater(train_iter, optimizer, device=-1)
trainer = training.Trainer(updater, (N_EPOCH, 'epoch'), out="result")
N_EPOCH = 990
chainer.serializers.load_npz(f'./result/snapshot_iter_{N_EPOCH}', trainer)
enc_data = []
for (data, label) in test:
pred_data = model.predictor(np.array([data]).astype(np.float32),
"ENC").data
enc_data.append((pred_data, label))
import chainer.functions as F
import chainer.links as L
from chainer import training
from chainer.training import extensions
from chainer.datasets import tuple_dataset
from Autoencoder import Autoencoder
train, test = chainer.datasets.get_mnist()
train = train[0:1000]
test = test[47:63]
#plot_mnist_data(test)
N_EPOCH = 100
train = [i[0] for i in train]
train = tuple_dataset.TupleDataset(train, train)
train_iter = chainer.iterators.SerialIterator(train, 100)
model = L.Classifier(Autoencoder(), lossfun=F.mean_squared_error)
model.compute_accuracy = False
optimizer = chainer.optimizers.Adam()
optimizer.setup(model)
updater = training.StandardUpdater(train_iter, optimizer, device=-1)
trainer = training.Trainer(updater, (N_EPOCH, 'epoch'), out="result")
trainer.extend(extensions.LogReport(), trigger=(1, 'epoch'))
trainer.extend(extensions.PrintReport(
entries=['epoch', 'main/loss', 'main/accuracy', 'elapsed_time']),
trigger=(1, 'epoch'))
trainer.extend(extensions.ProgressBar())
trainer.extend(extensions.snapshot(), trigger=(1, 'epoch'))
trainer.run()
plt.figure(figsize=(10, 10))
for i in range(len(imgs)):
plt.imshow(imgs[i])
plt.show()
test = []
for i in range(len(Xs)):
test.append(((Xs[i].astype('float32') / 255.0) - 0.5).reshape(12288))
# test = (((Xs[0] - np.mean(Xs[0])) / np.std(Xs[0])).reshape(-1, 12288))
# plt.imshow(test[0].reshape(64, 64, 3))
# plt.show()
hidden_layer = [192]
ae = Autoencoder(64 * 64 * 3, hidden_layer, 64, 0.00000002)
ae.train(test, test, 50, 0.0005, 5, 0.5, 10, 0.1, True)
outputs = []
latent_space = []
for inp in range(len(test)):
encoded_input = ae.encode(test[inp])
latent_space.append(encoded_input)
outputs.append(ae.decode(encoded_input))
show_image(outputs[inp].reshape(64, 64, 3))
bulb = latent_space[0]
caterpie = latent_space[1]
step = np.subtract(bulb, caterpie) / 9
print(ckpt_dir)
data_sets = ['mnist']
with open(os.path.join(save_dir, 'config.pkl'), 'rb') as f:
saved_args = cPickle.load(f)
save_dir_train = os.path.join(save_dir, 'train')
if not os.path.isdir(save_dir_train):
os.makedirs(save_dir_train)
save_dir_test = os.path.join(save_dir, 'test')
if not os.path.isdir(save_dir_test):
os.makedirs(os.path.join(save_dir_test))
model = Autoencoder(saved_args)
# sess = tf.InteractiveSession()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(tf.global_variables())
ckpt = tf.train.get_checkpoint_state(ckpt_dir)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, os.path.join(ckpt_dir, ckpt.model_checkpoint_path.split("/")[-1]))
for data_set in data_sets:
# Load data set
# mnistm = pkl.load(open(os.path.join(saved_args.file_path, data_set), 'rb'))
mnist = input_data.read_data_sets('MNIST_data')
#print("df", data)
input_numbers = []
for i in range(len(data)):
input_numbers.append("".join(np.squeeze(np.asarray(data[i]))))
for i in range(len(input_numbers)):
input_numbers[i] = list(input_numbers[i])
input_numbers[i] = [-1 if j == '0' else int(j) for j in input_numbers[i]]
norm = np.linalg.norm(input_numbers[i])
# if norm > 0:
# input_numbers[i] = input_numbers[i] / np.linalg.norm(input_numbers[i])
output_numbers = input_numbers
hidden_layer = [30,20, 10, 5]
# 7*5 pixeles
ae = Autoencoder(35, hidden_layer, 2, 0.0002)
ae.train(np.asarray(input_numbers), np.asarray(output_numbers), 15000, 0.0005, 10, 5, 10,0.1, True)
outputs = []
latent_space = []
labels = ["At", "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U",
"V", "W", "X", "Y", "Z", "[", "\\", "^", "_", "Space"]
for inp in range(len(input_numbers)):
encoded_input = ae.encode(input_numbers[inp])
latent_space.append(encoded_input)
outputs.append(ae.decode(encoded_input))
# Grafico del espacio latente en 2 dimensiones
x = []
y = []
for i in range(len(latent_space)):
random_prob = 0.1
random_func = np.vectorize(randomize)
output_numbers = np.copy(input_numbers)
for i in range(len(input_numbers)):
input_numbers[i] = random_func(input_numbers[i])
# outputs = []
# for inp in range(len(input_numbers)):
# encoded_input = ae.encode(input_numbers[inp])
# outputs.append(ae.decode(encoded_input))
hidden_layer = []
# 7*5 pixeles
ae = Autoencoder(35, hidden_layer, 30, 0.000002)
ae.train(np.asarray(input_numbers), np.asarray(output_numbers), 10000, 0.01,
10, 5, 10, 1, True)
# Plot the random inputs decodification
random_inputs_outputs = []
for inp in range(len(input_numbers)):
encoded_input = ae.encode(input_numbers[inp])
random_inputs_outputs.append(ae.decode(encoded_input))
for i, input_ in enumerate(input_numbers):
input_numbers[i] = np.array(input_).reshape((7, 5))
for i, out in enumerate(random_inputs_outputs):
random_inputs_outputs[i] = np.array(out).reshape((7, 5))
def main():
currDateTime = time.strftime('%Y%m%d_%H%M%S')
# Set up argparser.
parser = argparse.ArgumentParser()
parseTrainEval = parser.add_mutually_exclusive_group()
parseTrainEval.add_argument("-t",
"--train",
help="Use training mode",
action="store_true")
parseTrainEval.add_argument("-e",
"--evaluate",
help="Use evaluation mode",
action="store_true")
parser.add_argument(
"-b",
"--batch_size",
type=int,
default=8,
help=
"Batch size to use for training or evaluation depending on what mode you're in"
)
parser.add_argument("-s",
"--save_frequency",
type=int,
default=5,
help="Save a checkpoint every SAVE_FREQUENCY epochs")
parser.add_argument("-c",
"--checkpoint_directory",
type=str,
default="./checkpoints",
help="Directory to save checkpoints to")
parser.add_argument("-n",
"--num_epochs",
type=int,
default=50,
help="Number of epochs to train for")
parser.add_argument("-l",
"--load_checkpoint",
type=str,
help="Path of model checkpoint to load and use")
parser.add_argument(
"-f",
"--checkpoint_basename",
type=str,
default="checkpoint_" + currDateTime,
help=
"Basename to use for saved checkpoints. Gets appended with the epoch no. at saving"
)
parser.add_argument("--logfile_path",
type=str,
default="./logfile_" + currDateTime + ".csv",
help="Path to the logfile to use during training")
args = parser.parse_args()
print("Initialising...")
if (args.checkpoint_directory):
args.checkpoint_directory = os.path.dirname(args.checkpoint_directory)
if (args.load_checkpoint and not os.path.isfile(args.load_checkpoint)):
sys.exit(
"Error: specified checkpoint either doesn't exist or isn't a file."
)
# Transforms to put into a tensor and normalise the incoming Pillow images.
transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
])
print(" Loading datasets...")
# Set up datasets, model and loss/optimiser. If there's cuda available then send to the GPU.
trainset = NYUD2(root='/media/hdd1/Datasets',
split='train',
transform=transform)
testset = NYUD2(root='/media/hdd1/Datasets',
split='test',
transform=transform)
print(" Initialising model...")
model = Autoencoder()
epoch = 0 # This is used when resuming training and is overwritten on load.
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print(" Using device: " + str(device))
if torch.cuda.device_count() > 1:
print(" Using %d CUDA-capable devices" % torch.cuda.device_count())
model = nn.DataParallel(model)
model.to(device)
print(" Configuring optimiser...")
criterion = nn.MSELoss()
criterion = criterion.to(device)
optimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.9)
if (args.evaluate):
print("\n### Evaluation Mode ###\n")
if (args.load_checkpoint):
print("Loading model checkpoint for evaluation from " +
args.load_checkpoint)
model, epoch, optimizer, loss = utils.loadCheckpoint(
args.load_checkpoint, model)
print("Evaluating model with batch size %d..." % args.batch_size)
print(evaluate(model, criterion, testset, batch_size=args.batch_size))
elif (args.train):
print("\n### Training Mode ###\n")
if (args.load_checkpoint):
print("Training from checkpoint: " + args.load_checkpoint)
model, epoch, optimizer, loss = utils.loadCheckpoint(
args.load_checkpoint, model)
train(model,
optimizer,
criterion,
trainset,
logfile_path=args.logfile_path,
batch_size=args.batch_size,
epoch=epoch,
num_epochs=args.num_epochs,
save_freq=args.save_frequency,
checkpoint_dir=args.checkpoint_directory,
checkpoint_basename=args.checkpoint_basename)
else:
sys.exit(
"Error: No mode selected. Use `./main.py -h` for usage instructions."
)
from scipy import signal
from os.path import expanduser
from STFT import STFT
from Autoencoder import Autoencoder
from WavAnalysis import WavAnalysis
fft_len = 512
hidden_size = 256
code_len = 128
batch_size = 256
song = WavAnalysis(expanduser('~/Downloads/hybrid.wav'), fft_len)
data = song.analyze()
x = tf.placeholder(tf.float32)
ae = Autoencoder(data.shape[1], hidden_size, code_len, x=data)
optimizer = ae.optimizer
loss = ae.loss
decoder = ae.decode
num_steps = 200001
with tf.Session() as sess:
tf.global_variables_initializer().run()
writer = tf.summary.FileWriter('/tmp/tf', sess.graph)
for i in range(num_steps):
batch_inds = np.random.choice(range(data.shape[0]), batch_size)
batch = data[batch_inds, :]
_, l = sess.run([optimizer, loss], feed_dict={x: batch})
class AutoencoderSaliencyDetector(object):
def __init__(self, patch_size, stride=1, hidden=0.05):
self.n_input = patch_size**2 * 3
n_hidden = int(np.round(self.n_input * hidden))
self.patch_size = patch_size
self.stride = stride
self.padding = int(np.floor(self.patch_size / 2.0))
self.autoencoder = Autoencoder([self.n_input, n_hidden])
# Load the image file.
self.x = tf.compat.v1.placeholder(tf.string)
self.chunk = tf.compat.v1.placeholder(tf.int32, [4])
self.read_file = tf.io.read_file(self.x)
self.decode_image = tf.cast(tf.image.decode_image(self.read_file,
channels=3),
dtype=tf.float32)
# Slice image in chunks so really large images can be processed with constant
# memory consumption.
self.crop_chunk = tf.image.crop_to_bounding_box(
self.decode_image, self.chunk[0], self.chunk[1], self.chunk[2],
self.chunk[3])
# Extract image patches in the correct size and reshape the 2D array of patches
# to an 1D array that can be fed to the autoencoder.
self.extract_patches = tf.image.extract_patches(
images=[self.crop_chunk],
sizes=[1, self.patch_size, self.patch_size, 1],
strides=[1, self.stride, self.stride, 1],
rates=[1, 1, 1, 1],
padding='VALID')
self.patches_shape = tf.shape(input=self.extract_patches[0])
self.reshape = tf.reshape(self.extract_patches[0], [-1, self.n_input])
# Apply the autoencoder.
self.element_wise_cost = self.autoencoder.element_wise_cost(
self.reshape)
# Reshape the array back to the original image size (minus the invalid border).
# Collapse the grayscale channel so the result has shape [height, width] instead
# of [height, width, 1].
self.reshape_back = tf.reshape(self.element_wise_cost,
self.patches_shape[:2])
self.sess = self.autoencoder.sess
def train(self,
images,
number=10000,
reset=False,
display=True,
epochs=30,
batch_size=128,
display_step=10):
if reset: self.autoencoder._initialize()
X_train = images.random_patches(number=number, size=self.patch_size)
for epoch in range(epochs):
acc_cost = 0.
total_batch = int(number / batch_size)
# Loop over all batches
for i in range(total_batch):
batch_xs = self._get_random_block_from_data(
X_train, batch_size)
# Fit training using batch data
cost = self.autoencoder.partial_fit(batch_xs)
acc_cost += cost
# Display logs per epoch step
if display and epoch % display_step == 0:
# Compute average loss
avg_cost = acc_cost / number
print("Epoch: {:04d} ({:.5f})".format(epoch + 1, avg_cost))
def _get_random_block_from_data(self, data, batch_size):
start_index = np.random.randint(0, len(data) - batch_size)
return data[start_index:(start_index + batch_size)]
def apply(self, image_path, available_bytes=9e9):
img = PIL.Image.open(image_path)
# Determine how many patches fit into memory.
# *2 because the extracted patches are reshaped which duplicates the data
# *4 because the patches are cast to float32 (= 4 Bytes)
# *0.8 to account for "overhead", see: https://github.com/biigle/maia/issues/15
px_per_batch = 0.8 * available_bytes / (self.n_input * 2 * 4)
chunks = self._get_chunks(img.width, img.height, px_per_batch)
tmp_result = []
for chunk in chunks:
tmp_result.extend(
self.sess.run(self.reshape_back,
feed_dict={
self.x: image_path,
self.chunk: chunk
}))
# Upscale the result to the original size again because a stride != 1 shrinks
# it by factor "stride".
if self.stride != 1:
tmp_result = zoom(tmp_result, self.stride, order=0)
# Cut off any extraneous columns or rows that may be introduced in the
# scaling due to rounding.
diff = tmp_result.shape[0] - (img.height - 2 * self.padding)
if diff > 0: tmp_result = tmp_result[:-diff, :]
diff = tmp_result.shape[1] - (img.width - 2 * self.padding)
if diff > 0: tmp_result = tmp_result[:, :-diff]
result = np.zeros((img.height, img.width))
result[self.padding:-self.padding,
self.padding:-self.padding] = tmp_result
return result
"""
I've called these chunks so they are not confused with batches of mini batch
training.
"""
def _get_chunks(self, width, height, px_per_chunk):
# The effective (reduced) number of columns due to the stride.
# See: https://github.com/tensorflow/tensorflow/blob/1ad5e692e2fc218ca0b2a9a461c19762fdc9674b/tensorflow/core/framework/common_shape_fns.cc#L37
effective_width = (width - self.patch_size + self.stride) / self.stride
double_padding = 2 * self.padding
# Rows that should be processed per chunk. These rows contain exactly
# px_per_chunk pixels that are evaluated with the current stride.
rows_per_chunk = int(np.ceil(
px_per_chunk / effective_width)) * self.stride
# Finally add the padding to each chunk.
rows_per_chunk += double_padding
if rows_per_chunk <= double_padding:
raise Exception(
'Patch dimension of {} too large. Can only process {} rows in one chunk.'
.format(double_padding, rows_per_chunk))
start = 0
stop = 0
chunks = []
while stop < height:
stop = start + rows_per_chunk
if stop > height: stop = height
# Tuple with offset_height, offset_width, target_height, target_width for
# tf.image.crop_to_bounding_box().
chunks.append((start, 0, stop - start, width))
# Subtract padding because each chunk must be padded. So there are 2*padding
# lines in each chunk that aren't actually processed.
start = stop - double_padding
return chunks
print('shapes train; test; val:')
print(data_train.shape)
print(data_test.shape)
# corrompe o centro da imagem
cropy = 40
cropx = 40
x_train_noisy = crop_image(data_train, cropx, cropy)
x_test_noisy = crop_image(data_test, cropx, cropy)
#x_val_noisy = crop_image(data_val, cropx, cropy)
x_train_noisy = np.clip(x_train_noisy, 0., 1.)
x_test_noisy = np.clip(x_test_noisy, 0., 1.)
#x_val_noisy = np.clip(x_val_noisy, 0., 1.)
showOrigNoisy(data_test, x_test_noisy, num=4)
ae = Autoencoder()
if train:
ae.treina_modelo(x_train_noisy, data_train, x_test_noisy, data_test)
print(ae.history.history.keys())
plot_history(ae.history)
c10test = ae.prever(x_test_noisy)
#print("c10test: {0}\nc10val: {1}".format(np.average(c10test), np.average(c10val)))
showOrigNoisyRec(data_test, x_test_noisy, c10test)
#work with classes
dataCollection = DataCollection()
dataFrame = dataCollection.selectFeatures(1)
print dataFrame
#Read csv data
dataFrame = dataCollection.importData("testData.csv").head(100)
#Preprocess data - The first column is assumed to be an ID column
dataFramePreprocessed = dataCollection.preprocess(
dataFrame.drop([dataFrame.columns.values[0]], axis=1),
['gender_concept_id', 'measurement_type_concept_id'],
['year_of_birth', 'value_as_number', 'range_low', 'range_high'])
#Tune and Train model
autoencoder = Autoencoder()
hiddenOpt = [[50, 50], [100, 100], [5, 5, 5], [50, 50, 50]]
l2Opt = [1e-4, 1e-2]
hyperParameters = {"hidden": hiddenOpt, "l2": l2Opt}
bestModel = autoencoder.tuneAndTrain(
hyperParameters,
H2OAutoEncoderEstimator(activation="Tanh",
ignore_const_cols=False,
epochs=100), dataFramePreprocessed)
#Assign invalidity scores
invalidityScores = autoencoder.assignInvalidityScore(bestModel,
dataFramePreprocessed)
#Detect faulty records
testing = Testing()