def train(logpath, modeldir, batch_size=256, epochs=100):
modelpath = modeldir + 'model.h5'
dictpath = modeldir + 'word_dict.json'
for filepath in [logpath, modelpath, dictpath]:
check_validity(filepath)
check_file(logpath)
# load data
train_data = get_train_dataset(logpath)
# pre-process
from autoencoder import AutoEncoder # lazy load
pre_processor = Preprocessor(filepath=dictpath)
train_sr, time_sr = pre_processor.pre_process(train_data)
autoencoder = AutoEncoder(shape=(train_sr.shape[1], train_sr.shape[2]),
filepath=modelpath)
cluster_model = Cluster(dirpath=modeldir)
# train
autoencoder.fit(train_sr, batch_size=batch_size, epochs=epochs)
train_vector = autoencoder.transfer(train_sr)
predict_result, cluster_number, dist_tbl = cluster_model.classify(
train_vector)
top_index = get_topn_sql(dist_tbl, topn=1)
topn_sql = train_data[
top_index][:, -1] # typical SQL template for each cluster
cluster_model.get_cluster_info(predict_result, time_sr, cluster_number)
print("Train complete!")
return cluster_number, topn_sql
def test_mnist():
# Gradient check using MNIST
(train_x, _), (_, _) = mnist.load_data()
train_x = train_x / 255 # Normalizing images
# plotter.plot_mnist(train_x, "original") # Show original mnist images
num_img, img_dim, _ = train_x.shape # Get number of images and # pixels per square img
num_features = 500
mnist_in = np.reshape(
train_x, (img_dim * img_dim,
num_img)) # Reshape images to match autoencoder input
ga = Algorithm(x=mnist_in, num_features=num_features, debug=1, pop_size=20)
w_out, best_cost, logs = ga.run()
print(
f"Average time/generation (sec): {sum(logs['times']) / len(logs['times'])}"
)
print(f"Total time to run GA (sec): {logs['times']}")
ae = AutoEncoder(mnist_in, num_features, random_seed=1234, use_gpu=True)
z, _ = ae.psi(w_out)
phi_w_img = ae.phi(w_out) # Calculate phi(W)
new_mnist = z @ phi_w_img # Recreate original images using Z and phi(W)
new_imgs = np.reshape(
new_mnist, train_x.shape) # Reshape new images have original shape
plotter.plot_mnist(new_imgs,
f"{num_features}_features_ga") # Show new images
# print(loss_values)
plotter.plot_loss(logs['min'], "MNIST_Gradient_Loss_Over_Generations")
def train_ae(encoder_sizes,
decoder_sizes,
train_loader,
num_epochs=1,
bw=False):
"""
Pre-train Autoencoder on condition sketches
:return trained encoder nn.Module
"""
AE = AutoEncoder(encoder_sizes, decoder_sizes, bw=bw).to(device)
optimizer = torch.optim.Adam(AE.parameters(), lr=0.0005, weight_decay=3e-6)
for epoch in tqdm(range(num_epochs), "Encoder pretraining"):
epoch_loss = 0
for batch, (sketch, real,
label) in enumerate(tqdm(dataloader_train, "Batch")):
optimizer.zero_grad()
sketch, real, label = sketch.to(device), real.to(device), label.to(
device)
recon = AE(sketch)
loss = torch.mean((sketch - recon)**2)
loss.backward()
epoch_loss += loss.item() / len(dataloader_train)
optimizer.step()
print("AutoEncoder pretraining: Epoch {} Loss: {}".format(
epoch, epoch_loss))
del AE.decoder
optimizer.zero_grad()
del optimizer
return AE.encoder
def build_interaction_and_encoded_attr_mat(
self, interaction_and_attribute_info, attr_encoded_dim,
interaction_and_encoded_attr_mat_info_name):
print("start to build interaction and encode attribute matrix .... ")
attrs_vec = interaction_and_attribute_info["attrs_vec"]
attr_dim = len(attrs_vec[0])
print("encode attr embedding from {} to {}".format(
attr_dim, attr_encoded_dim))
print(attrs_vec.shape)
autoEncoder = AutoEncoder(attrs_vec, target_dim=attr_encoded_dim)
autoEncoder.train()
print("end to build interaction and encode attribute matrix .... ")
self.encoded_attr_vec = np.asarray(autoEncoder.get_coded_val())
print("encoded_attr_vec : {}".format(self.encoded_attr_vec.shape))
print(self.encoded_attr_vec)
interaction_and_attribute_info["attrs_vec"] = self.encoded_attr_vec
cache_var = CacheVar()
cache_var.build(interaction_and_encoded_attr_mat_info_name,
interaction_and_attribute_info)
def test_random():
# Sanity test to make sure that feature number positively impacts least squares error.
num_points = 100
num_data_per_point = 55
learning_rate = 0.5
x_in = np.random.normal(size=(num_data_per_point, num_points))
for num_features in [1, 5, 10, 15, 20, 40, 70]:
ae = AutoEncoder(x_in, num_features, random_seed=1234)
w_in = np.random.normal(size=(num_data_per_point, num_features))
z_out, least_squares_test = ae.psi(w_in)
print(
f"(# features : Least squares error = ({num_features} : {least_squares_test})"
)
print("Starting gradient decent...")
loss_values = [] # Keep track of loss values over epochs
for epoch in range(1000):
z_grd, ls_grd, grd = ae.calc_g(
w_in) # Calculate Z, Error, and Gradient Matrix
w_in = w_in - (learning_rate * grd
) # Update W using Gradient Matrix
loss_values.append(ls_grd) # Log loss
print(f"Epoch: {epoch}\t----------\tLoss: {ls_grd}")
# print(loss_values)
plotter.plot_loss(
loss_values,
f"Gradient Loss Over Epochs (test) (num_features: {num_features})")
def __init__(self):
self.edft = pd.read_csv(path + '/ML/data/pero_dft.csv', sep='\t')
self.comps_wdup = pd.read_csv(
path + '/ICSD_data/ICSD_all_data_with_all_phases.csv',
sep='\t') # compositions with duplicates for phase prediction
self.ae = AutoEncoder()
self.VAE = self.ae.build_AE(vae=True)
self.VAE.load_weights(path + '/saved_models/best_model_VAE.h5')
self.scaler = StandardScaler()
def sgd_da(corruption_level):
# Initialize RNGs
rng = numpy.random.RandomState(123)
theano_rng = RandomStreams(rng.randint(2 ** 30))
# Build the logistic regression class
# Images in MNIST are 28*28, there are 10 output classes
da = AutoEncoder(
numpy_rng=rng,
theano_rng=theano_rng,
input=x,
n_visible=28 * 28,
n_hidden=500
)
# Cost to minimize
cost = da.loss(corruption_level=corruption_level)
# Stochastic Gradient descent
updates = simple_sgd(cost, da.params, learning_rate)
train_da = theano.function(
inputs=[index],
outputs=cost,
updates=updates,
givens=[
(x, train_set_x[index * batch_size: (index + 1) * batch_size]),
]
)
################
# TRAIN MODEL #
################
start_time = timeit.default_timer()
print("... Training the model")
for epoch in range(n_epochs):
c = []
for batch_index in range(n_train_batches):
c.append(train_da(batch_index))
print('Training epoch {0}, cost {1}'.format(epoch, numpy.mean(c)))
end_time = timeit.default_timer()
training_time = (end_time - start_time)
print(('The {0} corruption code for file ' +
os.path.split(__file__)[1] +
' ran for {1:.2f}m').format(corruption_level * 100,
(training_time) / 60.))
image = Image.fromarray(
tile_raster_images(X=da.W.get_value(borrow=True).T,
img_shape=(28, 28), tile_shape=(10, 10),
tile_spacing=(1, 1))
)
image.save('filters_corrpution_' + str(int(corruption_level * 100))
+ '.png')
def main(options):
if options.num_classes == 2:
TRAINING_PATH = 'train_2classes.txt'
else:
TRAINING_PATH = 'train.txt'
IMG_PATH = '/Users/waz/JHU/CV-ADNI/ImageNoSkull'
dset_train = AD_3DRandomPatch(IMG_PATH, TRAINING_PATH)
train_loader = DataLoader(dset_train,
batch_size=options.batch_size,
shuffle=True,
num_workers=4,
drop_last=True)
sparsity = 0.05
beta = 0.5
mean_square_loss = nn.MSELoss()
kl_div_loss = nn.KLDivLoss()
use_gpu = len(options.gpuid) >= 1
autoencoder = AutoEncoder()
autoencoder = autoencoder.cpu()
optimizer = torch.optim.Adam(autoencoder.parameters(),
lr=options.learning_rate,
weight_decay=options.weight_decay)
train_loss = 0.
for epoch in range(options.epochs):
print("At {0}-th epoch.".format(epoch))
for i, patches in enumerate(train_loader):
for b, batch in enumerate(patches):
batch = Variable(batch)
output, mean_activitaion = autoencoder(batch)
loss1 = mean_square_loss(output, batch)
loss2 = kl_div_loss(mean_activitaion,
Variable(torch.Tensor([sparsity])))
print("loss1", loss1)
print("loss2", loss2)
loss = loss1 + loss2
train_loss += loss
logging.info(
"batch {0} training loss is : {1:.5f}, {1:.5f}".format(
b, loss1.data[0], loss2.data[0]))
optimizer.zero_grad()
loss.backward()
optimizer.step()
train_avg_loss = train_loss / len(train_loader * 1000)
print(
"Average training loss is {0:.5f} at the end of epoch {1}".format(
train_avg_loss.data[0], epoch))
torch.save(model.state_dict(), open("autoencoder_model", 'wb'))
def sgd_da(corruption_level):
# Initialize RNGs
rng = numpy.random.RandomState(123)
theano_rng = RandomStreams(rng.randint(2**30))
# Build the logistic regression class
# Images in MNIST are 28*28, there are 10 output classes
da = AutoEncoder(numpy_rng=rng,
theano_rng=theano_rng,
input=x,
n_visible=28 * 28,
n_hidden=500)
# Cost to minimize
cost = da.loss(corruption_level=corruption_level)
# Stochastic Gradient descent
updates = simple_sgd(cost, da.params, learning_rate)
train_da = theano.function(
inputs=[index],
outputs=cost,
updates=updates,
givens=[
(x, train_set_x[index * batch_size:(index + 1) * batch_size]),
])
################
# TRAIN MODEL #
################
start_time = timeit.default_timer()
print("... Training the model")
for epoch in range(n_epochs):
c = []
for batch_index in range(n_train_batches):
c.append(train_da(batch_index))
print('Training epoch {0}, cost {1}'.format(epoch, numpy.mean(c)))
end_time = timeit.default_timer()
training_time = (end_time - start_time)
print(
('The {0} corruption code for file ' + os.path.split(__file__)[1] +
' ran for {1:.2f}m').format(corruption_level * 100,
(training_time) / 60.))
image = Image.fromarray(
tile_raster_images(X=da.W.get_value(borrow=True).T,
img_shape=(28, 28),
tile_shape=(10, 10),
tile_spacing=(1, 1)))
image.save('filters_corrpution_' + str(int(corruption_level * 100)) +
'.png')
def test_mnist(num_epochs=None):
# Gradient check using MNIST
(train_x, _), (_, _) = mnist.load_data()
train_x = train_x / 255 # Normalizing images
# plotter.plot_mnist(train_x, "original") # Show original mnist images
num_img, img_dim, _ = train_x.shape # Get number of images and # pixels per square img
learning_rate = 0.5
num_features = 200
loss_values = [] # Keep track of loss values over epochs
loss_values_less = []
loss_diffs = []
w_in = np.random.normal(
size=(img_dim * img_dim,
num_features)) # Generate random W matrix to test
mnist_in = np.reshape(
train_x, (img_dim * img_dim,
num_img)) # Reshape images to match autoencoder input
ae = AutoEncoder(mnist_in, num_features, random_seed=1234, use_gpu=True)
start_time = time.time()
times = []
if num_epochs:
for epoch in range(num_epochs):
w_in, z_grd = do_epoch(ae, w_in, learning_rate, loss_values, times,
loss_values_less, loss_diffs, epoch,
start_time)
else:
epoch_history_check = 5
epoch = 0
loss_avg = 1000
tol = 0.03
while loss_avg > tol:
w_in, z_grd = do_epoch(ae, w_in, learning_rate, loss_values, times,
loss_values_less, loss_diffs, epoch,
start_time)
loss_check = loss_diffs[-epoch_history_check:]
loss_avg = sum(loss_check) / len(loss_check)
epoch += 1
print(
f"Total time to run gradient decent (sec): {time.time() - start_time}")
phi_w_img = ae.phi(w_in) # Calculate phi(W)
new_mnist = z_grd @ phi_w_img # Recreate original images using Z and phi(W)
new_imgs = np.reshape(
new_mnist, train_x.shape) # Reshape new images have original shape
plotter.plot_mnist(new_imgs,
f"{num_features}_features_gradient") # Show new images
# print(loss_values)
plotter.plot_loss(loss_values, "MNIST_Gradient_Loss_Over_Epochs")
plotter.plot_loss(
loss_values_less,
"MNIST_Gradient_Loss_Over_Epochs_all_epochs_except_zero")
def test_cifar10(num_epochs=None):
# (train_x, _), (_, _) = cifar10.load_data()
(_, _), (train_x, _) = cifar10.load_data()
print(train_x.shape)
plotter.plot_mnist(train_x, "original")
train_x = rgb2gray(train_x)
train_x = train_x / 255
plotter.plot_mnist(train_x, "grayscale")
num_img, img_h, img_w = train_x.shape
print(train_x.shape)
learning_rate = 0.5
num_features = 768;
loss_values = []
loss_values_less = []
loss_diffs = []
w_in = np.random.normal(size=(img_h * img_w, num_features))
cifar_in = np.reshape(train_x, (img_h * img_w, num_img))
# cifar_in = np.reshape(train_x, (img_h, img_w, num_img*img_ch))
print(cifar_in.shape)
ae = AutoEncoder(cifar_in, num_features, random_seed=1234, use_gpu=True)
start_time = time.time()
times = []
if num_epochs:
for epoch in range(num_epochs):
w_in, z_grd = do_epoch(ae, w_in, learning_rate, loss_values, times, loss_values_less, loss_diffs, epoch,
start_time)
else:
epoch_history_check = 5
epoch = 0
loss_avg = 1000
tol = 0.03
while loss_avg > tol:
w_in, z_grd = do_epoch(ae, w_in, learning_rate, loss_values, times, loss_values_less, loss_diffs, epoch,
start_time)
loss_check = loss_diffs[-epoch_history_check:]
loss_avg = sum(loss_check) / len(loss_check)
epoch += 1
print(f"Total time to run gradient decent (sec): {time.time() - start_time}")
phi_w_img = ae.phi(w_in) # Calculate phi(W)
new_cifar = z_grd @ phi_w_img # Recreate original images using Z and phi(W)
print(new_cifar.shape)
new_imgs = np.reshape(new_cifar, train_x.shape) # Reshape new images have original shape
plotter.plot_mnist(new_imgs, f"{num_features}_features_gradient") # Show new images
# print(loss_values)
plotter.plot_loss(loss_values, "CIFAR10_Gradient_Loss_Over_Epochs")
plotter.plot_loss(loss_values_less, "CIFAR10_Gradient_Loss_Over_Epochs_all_epochs_except_zero")
# return train_x
return new_imgs
def __init__(self, name, configuration, graph=None):
c = configuration
self.configuration = c
AutoEncoder.__init__(self, name, graph, configuration)
with tf.variable_scope(name):
self.z = c.encoder(self.x, **c.encoder_args)
self.vz = c.embedder(self.vx, **c.embedder_args)
self.bottleneck_size = int(self.z.get_shape()[1])
layer = c.decoder(self.z, **c.decoder_args)
c.decoder_args['reuse'] = True
vlayer = c.decoder(self.vz, **c.decoder_args)
c.decoder_args['reuse'] = False
if c.exists_and_is_not_none('close_with_tanh'):
layer = tf.nn.tanh(layer)
vlayer = tf.nn.tanh(vlayer)
self.x_reconstr = tf.reshape(
layer, [-1, self.n_output[0], self.n_output[1]])
self.vx_reconstr = tf.reshape(
vlayer, [-1, self.n_output[0], self.n_output[1]])
self.saver = tf.train.Saver(tf.global_variables(),
max_to_keep=c.saver_max_to_keep)
self._create_loss()
self._setup_optimizer()
# GPU configuration
if hasattr(c, 'allow_gpu_growth'):
growth = c.allow_gpu_growth
else:
growth = True
config = tf.ConfigProto()
config.gpu_options.allow_growth = growth
# Summaries
self.merged_summaries = tf.summary.merge_all()
self.train_writer = tf.summary.FileWriter(
osp.join(configuration.train_dir, 'summaries'), self.graph)
# Initializing the tensor flow variables
self.init = tf.global_variables_initializer()
# Launch the session
self.sess = tf.Session(config=config)
self.sess.run(self.init)
def train_ae(dataloaders_dict, device=0, num_epochs=5):
ae = AutoEncoder()
ae = ae.to(device)
distance = nn.MSELoss()
optimizer = optim.Adam(ae.parameters(), lr=0.001)
for epoch in range(num_epochs):
print('\nEpoch {}/{}'.format(epoch, num_epochs - 1))
print('-' * 10)
for phase in ["train", "val", "test"]:
if phase == "train":
ae.train() # Set ae to training mode
else:
ae.eval()
for data in dataloaders_dict[phase]:
inputs, _ = data
inputs = inputs.to(device)
with torch.set_grad_enabled(phase == "train"):
output = ae(inputs)
loss = distance(output, inputs)
optimizer.zero_grad()
if phase == "train":
loss.backward()
optimizer.step()
print("{} Loss: {:.4f}".format(phase, loss.item()))
return ae
def _load_aes(self, path):
autoencoders = [
AutoEncoder(30, False, False, 0.001, name)
for name in string.ascii_uppercase[:3]
]
baseline = AutoEncoder(30, False, False, 0.001, "baseline")
all_agents: List[AutoEncoder] = autoencoders + [baseline]
[
agent.load_state_dict(
torch.load(f"{path}/{agent.name}.pt", map_location=self.dev)
)
for agent in all_agents
]
return all_agents
def train_classifier(self, agent: AutoEncoder):
mlp = MLP(30).to(self.dev)
agent.to(self.dev)
for i in range(int(self.cfg.nsteps)):
X, y = map(
lambda x: x.to(self.dev),
self.dataset.sample_with_label(int(self.cfg.bsize)),
)
latent = agent.encode(X)
mlp.train(latent, y)
acc = mlp.compute_acc(latent, y)
self.tb.add_scalar("Accuracy-Post", acc, global_step=i)
# self.writer.add((acc.item(), agent.name), step=i)
return mlp
def run_model(data, in_memory=False):
"""
Runs the autoencoder model.
@param data is
if in_memory == True:
([[size, incoming]], [webpage_label])
else:
A list of paths
"""
tf.reset_default_graph()
tf.set_random_seed(123)
# Only print small part of array
np.set_printoptions(threshold=10)
with tf.Session() as session:
model = AutoEncoder(args.layers,
args.batch_size,
activation_func=args.activation_func,
learning_rate=args.learning_rate,
batch_norm=args.batch_norm)
session.run(tf.global_variables_initializer())
loss_track = train_on_copy_task(session,
model,
data,
batch_size=args.batch_size,
batches_in_epoch=100,
verbose=False)
def __init__(self, args):
super(DCN, self).__init__()
self.args = args
self.beta = args.beta # coefficient of the clustering term
self.lamda = args.lamda # coefficient of the reconstruction term
self.device = torch.device('cuda' if args.cuda else 'cpu')
# Validation check
if not self.beta > 0:
msg = 'beta should be greater than 0 but got value = {}.'
raise ValueError(msg.format(self.beta))
if not self.lamda > 0:
msg = 'lambda should be greater than 0 but got value = {}.'
raise ValueError(msg.format(self.lamda))
if len(self.args.hidden_dims) == 0:
raise ValueError('No hidden layer specified.')
self.kmeans = batch_KMeans(args)
self.autoencoder = AutoEncoder(args).to(self.device)
self.criterion = nn.MSELoss()
self.optimizer = torch.optim.Adam(self.parameters(),
lr=args.lr,
weight_decay=args.wd)
def test_gradient():
num_points = 30 # N
num_data_per_point = 20 # n
num_features = 12 # m
const = np.random.normal(size=(num_data_per_point, num_points)) # X
x = np.random.normal(size=(num_data_per_point, num_features)) # W
dx = x * 1e-3
ae = AutoEncoder(const, num_features, random_seed=1234)
def f(input):
return ae.psi(input)[1]
def df(input):
return ae.calc_g(input)[2] # G
# Test 1: Check norm(dx)
check1 = f(x + dx)
check2 = f(x - dx)
check3 = 2 * np.tensordot(df(x), dx, axes=2)
differror = np.linalg.norm(check1 - check2 - check3) / np.linalg.norm(dx)
print("Test 1 of gradient check: differror should be smaller than, or close to, norm(dx):")
print(f"differror: {differror}")
print(f"norm(dx): {np.linalg.norm(dx)}")
print()
# Test 2: drop dx by factor of 10 and see if differror drops by 10-100
new_dx = dx * 0.1
check1 = f(x + new_dx)
check2 = f(x - new_dx)
check3 = 2 * np.tensordot(df(x), new_dx, axes=2)
new_differror = np.linalg.norm(check1 - check2 - check3) / np.linalg.norm(new_dx)
print("Test 2 of gradient check: new_differror should be 10-100 factors smaller than differror:")
print(f"new_differror: {new_differror}")
print(f"differror: {differror}")
def main():
"""Load data, build autoencoder, train, and log data."""
train_loader, test_loader = get_data()
autoencoder = AutoEncoder(train_loader, latent_dim=2)
train_autoencoder_and_log(
autoencoder=autoencoder,
train_loader=train_loader,
test_loader=test_loader,
)
def test_autoencoder():
"""
Test that all components of the auto-encoder work correctly by executing a
training run against generated data.
"""
input_shape = (3, )
epochs = 1000
# Generate some data
x_train = np.random.rand(100, 3)
x_test = np.random.rand(30, 3)
# Define encoder and decoder model
def create_encoder_model(input_shape):
model_input = Input(shape=input_shape)
encoder = Dense(4)(model_input)
encoder = BatchNormalization()(encoder)
encoder = Activation(activation='relu')(encoder)
return Model(model_input, encoder)
def create_decoder_model(embedding_shape):
embedding_a = Input(shape=embedding_shape)
decoder = Dense(3)(embedding_a)
decoder = BatchNormalization()(decoder)
decoder = Activation(activation='relu')(decoder)
return Model(embedding_a, decoder)
# Create auto-encoder network
encoder_model = create_encoder_model(input_shape)
decoder_model = create_decoder_model(encoder_model.output_shape)
autoencoder = AutoEncoder(encoder_model, decoder_model)
# Prepare auto-encoder for training
autoencoder.compile(loss='binary_crossentropy', optimizer='adam')
# Evaluate network before training to establish a baseline
score_before = autoencoder.evaluate(x_train, x_train)
# Train network
autoencoder.fit(x_train,
x_train,
validation_data=(x_test, x_test),
epochs=epochs)
# Evaluate network
score_after = autoencoder.evaluate(x_train, x_train)
# Ensure that the training loss score improved as a result of the training
assert (score_before > score_after)
def plot_img_reconstructions(
root_path: str,
name_of_best_exp: str,
path_to_plot: str,
baseline: bool = False,
epoch: int = 49999,
):
dataset = MNISTDataset()
ae = AutoEncoder(30, False, False, 0.001, "test")
ae.load_state_dict(
torch.load(
os.path.join(
root_path,
name_of_best_exp,
f"params/step_{epoch}/rank_0/{'A' if not baseline else 'baseline'}.pt",
),
map_location=torch.device("cpu"),
)
)
digits: torch.Tensor = dataset.sample(50)
_, axes = plt.subplots(
nrows=10,
ncols=10,
figsize=(10, 8),
gridspec_kw=dict(
wspace=0.0, hspace=0.0, top=0.95, bottom=0.05, left=0.17, right=0.845
),
)
axes = axes.reshape(50, 2)
for digit, ax_column in zip(digits, axes):
ax_column[0].imshow(digit.squeeze().detach())
ax_column[0].set_axis_off()
rec = ae(digit.reshape(1, 1, 28, 28))
ax_column[1].imshow(rec.squeeze().detach())
ax_column[1].set_axis_off()
plt.show()
exit(1)
plt_path = f"plots/{path_to_plot}/reconstructions_baseline_{baseline}"
plt.savefig(plt_path + ".pdf")
plt.savefig(plt_path + ".svg")
plt.close()
def work(in_files, out_files, device_id, total_device, device_idx):
"""
Stylized the images
This function supports multi-GPU transformation
Arg: in_files - The path list of input images
out_files - The path list of output images
device_id - The name of device which is following the rule that tensorflow makes
total_device - The total number of devices
devices_idx - The index of the current device
"""
global adopt_revision
with tf.Graph().as_default():
tf_config = tf.ConfigProto(allow_soft_placement=True)
tf_config.gpu_options.allow_growth = True
# Construct graph
img_ph = tf.placeholder(tf.float32, shape=image_shape)
if adopt_revision == True:
logits = SmallAutoEncoder(img_ph)
else:
logits = AutoEncoder(img_ph)
# Run
with tf.Session(config=tf_config) as sess:
with tf.device(device_id): # Adopt multi-GPU to transfer the image
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
saver.restore(sess, model_path + model_name)
if total_device <= 1:
start = 0
end = int(len(in_files) // 1)
else:
start = device_idx * int(len(in_files) // total_device)
end = device_idx * int(
len(in_files) // total_device) + int(
len(in_files) // total_device)
conduct_time = time.time()
for i in range(start, end, 1):
# print("progress: ", i, ' / ', end - start, '\t proc: ', device_idx)
img_batch = np.ndarray(image_shape)
for j, img_path in enumerate(in_files[i:i + 1]):
img = get_img(img_path)
img_batch[j] = img
_style_result = sess.run(
[
logits,
], feed_dict={img_ph: img_batch / 255.0})
for j, img_path in enumerate(out_files[i:i + 1]):
save_img(img_path, _style_result[0][j])
conduct_time = time.time() - conduct_time
print("Conduct time: ", conduct_time)
def main():
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = AutoEncoder(channels_list, latent_dim).to(device)
optimizer = optim.Adam(model.parameters(), lr=lr)
train_loader = torch.utils.data.DataLoader(datasets.MNIST(
'./data', train=True, download=True, transform=transforms.ToTensor()),
batch_size=batch_size,
shuffle=True)
test_loader = torch.utils.data.DataLoader(datasets.MNIST(
'./data', train=False, transform=transforms.ToTensor()),
batch_size=batch_size,
shuffle=True)
runner = AERunner(model, optimizer, train_loader, test_loader, device)
for epoch in range(1, epochs + 1):
runner.train(epoch)
runner.test(epoch)
with torch.no_grad():
sample = model.sample(80, device)
save_image(
sample, './results_ae_' + str(latent_dim) + '/sample_' +
str(epoch) + '.png')
def main(options):
if options.num_classes == 2:
TRAINING_PATH = 'train_2classes.txt'
else:
TRAINING_PATH = 'train.txt'
IMG_PATH = './Image'
dset_train = AD_3DRandomPatch(IMG_PATH, TRAINING_PATH)
train_loader = DataLoader(dset_train,
batch_size=options.batch_size,
shuffle=True,
num_workers=4,
drop_last=True)
sparsity = 0.05
beta = 0.5
mean_square_loss = nn.MSELoss()
kl_div_loss = nn.KLDivLoss(reduce=False)
use_gpu = len(options.gpuid) >= 1
autoencoder = AutoEncoder()
if (use_gpu):
autoencoder = autoencoder.cuda()
else:
autoencoder = autoencoder.cpu()
optimizer = torch.optim.Adam(autoencoder.parameters(),
lr=options.learning_rate,
weight_decay=options.weight_decay)
train_loss = 0.
for epoch in range(options.epochs):
print("At {0}-th epoch.".format(epoch))
for i, patches in enumerate(train_loader):
print(i)
print(len(patches))
def run_model(data, in_memory=False):
"""
Runs a autoencoder model.
@param data is
if in_memory == True:
([[size, incoming]], [webpage_label])
else:
A list of paths
"""
tf.reset_default_graph()
tf.set_random_seed(123)
# Only print small part of array
np.set_printoptions(threshold=10)
with tf.Session() as session:
# with bidirectional encoder, decoder state size should be
# 2x encoder state size
model = AutoEncoder(args.layers,
args.batch_size,
activation_func=args.activation_func,
learning_rate=args.learning_rate,
saved_graph=args.graph_file,
sess=session,
batch_norm=args.batch_norm)
model.set_is_training(False)
# session.run(tf.global_variables_initializer())
get_vector_representations(session,
model,
data,
DATA_DIR + '/../ae_cells',
batch_size=args.batch_size,
max_batches=None,
batches_in_epoch=100,
extension=args.extension)
def test_get_variables(self):
with self.test_session() as sess:
ae_shape = [10, 20, 30, 2]
ae = AutoEncoder(ae_shape, sess)
with self.assertRaises(AssertionError):
ae.get_variables_to_init(0)
with self.assertRaises(AssertionError):
ae.get_variables_to_init(4)
v1 = ae.get_variables_to_init(1)
self.assertEqual(len(v1), 3)
v2 = ae.get_variables_to_init(2)
self.assertEqual(len(v2), 5)
v3 = ae.get_variables_to_init(3)
self.assertEqual(len(v3), 2)
def __init__(self, numpy_rng=None, input=None, n_visible=8, n_hidden=4,
corrupt_level=0.0,
W=None, bhid=None, bvis=None, theano_rng=None,
sparsity=0.05, beta=0.001):
AutoEncoder.__init__(self,
numpy_rng=numpy_rng,
input = input,
n_visible = n_visible,
n_hidden = n_hidden,
sparsity = sparsity,
beta = beta,
W = W,
bhid = bhid,
bvis = bvis
)
if not theano_rng:
theano_rng = RandomStreams(self.numpy_rng.randint(2 ** 3))
self.theano_rng = theano_rng
self.corrupt_level = corrupt_level
def predict(querypath, modeldir, ratio):
modelpath = modeldir + 'model.h5'
dictpath = modeldir + 'word_dict.json'
clusterpath = modeldir + 'Kmeans_model.pkl'
infopath = modeldir + 'cluster_info.json'
for filepath in [querypath, modelpath, dictpath, clusterpath, infopath]:
check_validity(filepath)
check_file(filepath)
# load data
predict_data = get_test_dataset(querypath)
# predict
from autoencoder import AutoEncoder # lazy load
pre_processor = Preprocessor(alpha=ratio, filepath=dictpath)
predict_sr = pre_processor.transform(predict_data)
autoencoder = AutoEncoder(shape=(predict_sr.shape[1], predict_sr.shape[2]),
filepath=modelpath)
cluster_model = Cluster(dirpath=modeldir)
predict_vector = autoencoder.transfer(predict_sr)
result = cluster_model.predict(predict_vector)
print('Predict result is: ')
print(result)
return result
def _load_aes(path):
autoencoders = [
AutoEncoder(30, bnorm=False, affine=False, name=name, lr=0.001)
for name in string.ascii_uppercase[:2]
]
baseline1 = AutoEncoder(30,
bnorm=False,
affine=False,
name="Base1",
lr=0.001)
baseline2 = AutoEncoder(30,
bnorm=False,
affine=False,
name="Base2",
lr=0.001)
all_agents: List[AutoEncoder] = autoencoders + [baseline1, baseline2]
[
agent.load_state_dict(
torch.load(f"{path}/{agent.name}.pt",
map_location=torch.device("cpu")))
for agent in all_agents
]
return all_agents
def _load_aes(self, path):
autoencoders = [
AutoEncoder(30, False, False, 0.001, name)
for name in string.ascii_uppercase[:3]
]
baseline1 = AutoEncoder(30, False, False, 0.001,
"baseline1").to(self.dev)
baseline2 = AutoEncoder(30, False, False, 0.001,
"baseline2").to(self.dev)
baselines = [baseline1, baseline2]
for agent in autoencoders:
agent.load_state_dict(
torch.load(
f"{path}/rank_{int(self.rank) % 3}/{agent.name}.pt",
map_location=self.dev,
))
for i, agent in enumerate(baselines):
agent.load_state_dict(
torch.load(
f"{path}/rank_{(int(self.rank) + i) % 3}/{'baseline'}.pt",
map_location=self.dev,
))
return autoencoders + baselines
def test_nets(self):
with self.test_session() as sess:
ae_shape = [10, 20, 30, 2]
ae = AutoEncoder(ae_shape, sess)
input_pl = tf.placeholder(tf.float32, shape=(100, 10))
with self.assertRaises(AssertionError):
ae.pretrain_net(input_pl, 0)
with self.assertRaises(AssertionError):
ae.pretrain_net(input_pl, 3)
net1 = ae.pretrain_net(input_pl, 1)
net2 = ae.pretrain_net(input_pl, 2)
self.assertEqual(net1.get_shape().dims[1].value, 10)
self.assertEqual(net2.get_shape().dims[1].value, 20)
net1_target = ae.pretrain_net(input_pl, 1, is_target=True)
self.assertEqual(net1_target.get_shape().dims[1].value, 10)
net2_target = ae.pretrain_net(input_pl, 2, is_target=True)
self.assertEqual(net2_target.get_shape().dims[1].value, 20)
sup_net = ae.supervised_net(input_pl)
self.assertEqual(sup_net.get_shape().dims[1].value, 2)
def test_get_variables(self):
with self.test_session() as sess:
ae_shape = [10, 20, 30, 2]
ae = AutoEncoder(ae_shape, sess)
with self.assertRaises(AssertionError):
ae.get_variables_to_init(0)
with self.assertRaises(AssertionError):
ae.get_variables_to_init(4)
v1 = ae.get_variables_to_init(1)
self.assertEqual(len(v1), 3)
v2 = ae.get_variables_to_init(2)
self.assertEqual(len(v2), 5)
v3 = ae.get_variables_to_init(3)
self.assertEqual(len(v3), 2)
def test_nets(self):
with self.test_session() as sess:
ae_shape = [10, 20, 30, 2]
ae = AutoEncoder(ae_shape, sess)
input_pl = tf.placeholder(tf.float32, shape=(100, 10))
with self.assertRaises(AssertionError):
ae.pretrain_net(input_pl, 0)
with self.assertRaises(AssertionError):
ae.pretrain_net(input_pl, 3)
net1 = ae.pretrain_net(input_pl, 1)
net2 = ae.pretrain_net(input_pl, 2)
self.assertEqual(net1.get_shape().dims[1].value, 10)
self.assertEqual(net2.get_shape().dims[1].value, 20)
net1_target = ae.pretrain_net(input_pl, 1, is_target=True)
self.assertEqual(net1_target.get_shape().dims[1].value, 10)
net2_target = ae.pretrain_net(input_pl, 2, is_target=True)
self.assertEqual(net2_target.get_shape().dims[1].value, 20)
sup_net = ae.supervised_net(input_pl)
self.assertEqual(sup_net.get_shape().dims[1].value, 2)