def get_data_loader(self):
data_phase = self.pipeline_config['data']
data_module = __import__('load_data')
if 'csv' in data_phase.keys():
# For reading 'manually' filtered data from a saved dataframe
csv_filename = data_phase['csv']
csv_file = self.config.get_output_data_dir() + csv_filename
self.dataloader = CSVLoader()
self.dataloader.set_csv_file(csv_file)
else:
data_func = getattr(data_module, data_phase['dataloader'])
data_file = data_phase['file']
data = data_func(data_file)
self.dataloader = DataLoader(label_identifier=data_phase['labels'])
self.dataloader.set_data(data)
if 'filters' in data_phase.keys():
# Every filter is a function name in load_data taking a data_loader and returning it filtered
data_filters = data_phase['filters']
for filter_function_name in data_filters:
filter_function_name = 'filter_' + filter_function_name
filter_func = getattr(data_module, filter_function_name)
self.dataloader = filter_func(self.dataloader)
return self.dataloader
def __init__(self):
# Input shape
self.img_rows = 256
self.img_cols = 256
self.channels = 3
self.img_shape = (self.img_rows, self.img_cols, self.channels)
# Configure data loader
print("Reading...")
filepath = [
"./data/trainImage256.txt", "./data/trainLabel256.txt",
"./data/testImage256.txt", "./data/testLabel256.txt"
]
self.data_loader = DataLoader(filepath,
img_res=(self.img_rows, self.img_cols))
# Calculate output shape of D (PatchGAN)
patch = int(self.img_rows / 2**4)
self.disc_patch = (patch, patch, 1)
# Number of filters in the first layer of G and D
self.gf = 64
self.df = 64
optimizer = Adam(0.0002, 0.5)
# Build and compile the discriminator
print("Building discriminator")
self.discriminator = self.build_discriminator()
self.discriminator.compile(loss='mse',
optimizer=optimizer,
metrics=['accuracy'])
#-------------------------
# Construct Computational
# Graph of Generator
#-------------------------
# Build the generator
self.generator = self.build_generator()
# Input images and their conditioning images
img_A = Input(shape=self.img_shape)
img_B = Input(shape=self.img_shape)
# By conditioning on B generate a fake version of A
fake_A = self.generator(img_B)
# For the combined model we will only train the generator
self.discriminator.trainable = False
# Discriminators determines validity of translated images / condition pairs
valid = self.discriminator([fake_A, img_B])
print("Building generator...")
self.combined = Model(inputs=[img_A, img_B], outputs=[valid, fake_A])
self.combined.compile(loss=['mse', 'mae'],
loss_weights=[1, 100],
optimizer=optimizer)
def val_predict(self):
loader = DataLoader()
_, val_gen = loader.load_data()
for i, (val_x, val_y) in enumerate(val_gen):
preds = self._predict(val_x)
np_img_list = [val_y[0], val_x[0], preds[0]]
self.save(i, np_img_list)
if i >= 3:
break
class TrainAutoColor(AutoColorEncoder, ImageStore):
def __init__(self, epochs=100):
self.autoencoder = self.build_model()
self.loader = DataLoader()
self.epochs = epochs
def train(self):
train_gen, val_gen, test_gen = self.loader.load_data()
train_steps, val_steps, test_steps = self.loader.cal_steps()
self._train(train_gen, train_steps, val_gen, val_steps)
self._predict(test_gen, test_steps, self.loader.test_list, self.loader.batch_size)
def _train(self, train_gen, train_steps, val_gen, val_steps):
self.autoencoder.fit_generator(
generator=train_gen,
steps_per_epoch=train_steps,
epochs=self.epochs,
validation_data=val_gen,
validation_steps=val_steps,
)
self.autoencoder.save_weights(os.path.join(settings.MODEL, 'auto_color_model.h5'))
def _predict(self, test_gen, test_steps, test_lists, batch_size):
preds = self.autoencoder.predict_generator(test_gen, steps=test_steps, verbose=0)
x_test = []
y_test = []
for i, (l, ab) in enumerate(self.loader.generator_with_preprocessing(test_lists, batch_size)):
x_test.append(l)
y_test.append(ab)
if i == test_steps - 1:
break
x_test = np.vstack(x_test)
y_test = np.vstack(y_test)
test_preds_lab = np.concatenate((x_test, preds), 3).astype(np.uint8)
test_preds_rgb = []
for i in range(test_preds_lab.shape[0]):
preds_rgb = lab_to_rgb(test_preds_lab[i, :, :, :])
test_preds_rgb.append(preds_rgb)
test_preds_rgb = np.stack(test_preds_rgb)
original_lab = np.concatenate((x_test, y_test), 3).astype(np.uint8)
original_rgb = []
for i in range(original_lab.shape[0]):
original_rgb.append(lab_to_rgb(original_lab[i, :, :, :]))
original_rgb = np.stack(original_rgb)
for i in range(test_preds_rgb.shape[0]):
gray_image = img_to_array(ImageOps.grayscale(array_to_img(test_preds_rgb[i])))
auto_colored_image = test_preds_rgb[i]
original_image = original_rgb[i]
np_img_list = [gray_image, auto_colored_image, original_image]
self.save(i, np_img_list)
def __init__(self, identity):
self.identity = identity
self.img_rows = 32
self.img_cols = 32
self.channels = 3
self.img_shape = (self.img_rows, self.img_cols, self.channels)
# Configure data loader
self.dataset_name = self.identity
self.data_loader = DataLoader(img_res=(self.img_rows, self.img_cols))
# Calculate output shape of D (PatchRAN)
patch = int(self.img_rows / 2**4)
self.disc_patch = (patch, patch, 1)
optimizer = Adam(0.0002, 0.5)
# Number of filters in the first layer of G and D
self.gf = 32
self.df = 64
# Build and compile the discriminator
self.discriminator = self.build_discriminator()
self.discriminator.compile(loss='mse',
optimizer=optimizer,
metrics=['accuracy'])
# Build and compile the reconstructor
self.reconstructor = self.build_reconstructor()
print(self.reconstructor.summary())
self.reconstructor.compile(loss='mse', optimizer=optimizer)
# The reconstructor takes noise as input and generated imgs
img = Input(shape=self.img_shape)
reconstr = self.reconstructor(img)
# For the combined model we will only train the reconstructor
self.discriminator.trainable = False
# The valid takes generated images as input and determines validity
valid = self.discriminator(reconstr)
# The combined model (stacked reconstructor and discriminator) takes
# images as input => reconstruct images => determines validity
self.combined = Model(img, [reconstr, valid])
self.combined.compile(loss=['mse', 'mse'],
loss_weights=[0.999, 0.001],
optimizer=optimizer)
def train(self):
(x_train, x_test), (noised_x_train, noised_x_test) = DataLoader().run(mode=self.mode, is_mnist=self.is_mnist)
self._train(noised_x_train, x_train)
noised_preds = self._predict(noised_x_test)
for i in range(1):
np_img_list = [x_test[i], noised_x_test[i], noised_preds[i]]
self.save(i, np_img_list)
def run(self, data_filename, do_eval=None, output_dir=None):
"""Run the two-steps ExreadCluster algorithm (no edge information).
Parameter:
row_header (String[]): column names
rows (String[[]]): array of string arrays.
Output:
data2rep (int array): cluster assignment for each data
r_emb (K * emb_size): representative embedding
emb_tensor (N * emb_size): fine-tuned input embedding
cls_loss (float): clustering loss
"""
accuracies = []
# Get data.
row_header, rows = FileLoader(data_filename)
target_rows, aux_rows, gold, spans = self.get_basics(row_header, rows)
# Generate batches.
row_iter, aux_c_sizes, aux_weights = DataLoader(target_rows, aux_rows,
self.batch_size, self.w2v_model)
# Get the initial embedding tensor as the average w2v embedding.
emb_tensor = self.w2v_model.get_emb_matrix(target_rows)
if do_eval is not None:
with open(os.path.join(output_dir, "emb_before"), "wb") as handle:
pickle.dump((emb_tensor, spans, gold), handle,
protocol=pickle.HIGHEST_PROTOCOL)
# Run the base cluster module
print("Run 1st module: clustering algorithm")
c_emb, labels, cls_loss = self.cluster_module.run(emb_tensor)
# Run the embedding fine-tuning module
print("Run 2nd module: refine embedding")
enc, emb_loss, emb_labels = self.emb_module.run(aux_c_sizes, row_iter,
LabelLoader(labels, self.batch_size), c_emb, spans, gold,
output_dir, aux_weights=aux_weights)
print("****cluster loss: %f; emb loss:%f****" % (cls_loss, emb_loss))
# Update embedding tensor
emb_tensor = enc.data
if do_eval is not None:
accuracies.append(Evaluate(labels, gold, do_eval))
print("Run 3rd module: refinement by clustering algorithm")
# Final refinement
c_emb, labels, cls_loss = self.cluster_module.run(emb_tensor)
labels = self.post_processing(row_header, rows, labels)
if do_eval is not None:
accuracies.append(Evaluate(labels, gold, do_eval))
with open(os.path.join(output_dir, "emb_after"), "wb") as handle:
pickle.dump((emb_tensor, spans, gold), handle,
protocol=pickle.HIGHEST_PROTOCOL)
return row_header, rows, labels, c_emb, emb_tensor, cls_loss, accuracies
def __init__(
self,
sdg_target_definitions_path='../data/SDG target definitions.xlsx',
sdg_target_document_matches_path='../data/SDG target - document text matches.xlsx',
training_test_split_path='../data/Training and test set division.xlsx',
documents_path='E:/documents3/',
results_path='../results/',
stemming=True,
embedding_dimensionality=300,
threads=10):
print('--------------------------------------------------')
print('Starting RIA experiments: embedding size ' +
str(embedding_dimensionality) + ', stemming: ' +
('yes' if stemming else 'no'))
print('--------------------------------------------------')
self._stemming = stemming
self._embedding_dims = embedding_dimensionality
self._data_loader = DataLoader(sdg_target_definitions_path,
sdg_target_document_matches_path,
training_test_split_path, stemming)
self._target_docs = None
corpus = list(self._data_loader.read_corpus(path=documents_path))
print('Creating word embeddings')
self._w2v = CustomParVec(corpus,
workers=threads,
dimensions=embedding_dimensionality,
min_word_count=30,
context=30,
sg=0,
iterations=5,
downsampling=0,
tfidf=False,
target_docs=None)
self._results_dir = results_path + str(embedding_dimensionality) + '/'
if stemming:
self._results_dir += 'stem/'
else:
self._results_dir += 'no_stem/'
os.makedirs(self._results_dir, exist_ok=True)
self._score_dict = None
self._matches_by_sent = None
self._avg_matches_by_sent = None
self._avg_sdg_matches_by_sent = None
def __init__(self, config):
self.data_path = r'F:\bma\project\data'
self.result_path = r'F:\bma\project\FactorAnalysis\MB_quant\results'
dataloader = DataLoader()
datadict = dataloader.load_eval_data()
self.tradeday = pd.read_csv(os.path.join(self.data_path,
'tradeday.csv'),
parse_dates=['date'])
self.tickers = pd.read_csv(os.path.join(self.data_path, 'tickers.csv'))
self.start_date = self.tradeday.iloc[0].date.strftime("%Y/%m/%d")
self.end_data = self.tradeday.iloc[-1].date.strftime("%Y/%m/%d")
self.close = datadict['close']
self.open = datadict['open']
self.suspendFlag = datadict['suspend']
self.upLimit = datadict['upperLimit']
self.downLimit = datadict['downLimit']
self.config = config
def __init__(self):
self.data_index = 0
self.min_count = 5 # 默认最低频次的单词
self.batch_size = 200 # 每次迭代训练选取的样本数目
self.embedding_size = 200 # 生成词向量的维度
self.window_size = 1 # 考虑前后几个词，窗口大小
self.num_steps = 100000 #定义最大迭代次数，创建并设置默认的session，开始实际训练
self.num_sampled = 100 # Number of negative examples to sample.
self.trainfilepath = './data/data'
self.modelpath = './model/cbow_wordvec.bin'
self.dataset = DataLoader().dataset
self.words = self.read_data(self.dataset)
def __init__(self):
self.data_index = 0
self.trainpath = './data/data.txt'
self.modelpath = './model/skipgram_wordvec.bin'
self.min_count = 5#最低词频，保留模型中的词表
self.batch_size = 200 # 每次迭代训练选取的样本数目
self.embedding_size = 200 # 生成词向量的维度
self.window_size = 5 # 考虑前后几个词，窗口大小, skipgram中的中心词-上下文pairs数目就是windowsize *2
self.num_sampled = 100 # 负样本采样.
self.num_steps = 100000#定义最大迭代次数，创建并设置默认的session，开始实际训练
self.dataset = DataLoader().dataset
self.words = self.read_data(self.dataset)
class TrainSuperResolution(SuperResolution):
def __init__(self, epochs=10):
self.model = self.build_model()
self.loader = DataLoader()
self.epochs = epochs
self.model_name = 'super_resolution_model'
def train(self):
train_gen, val_gen = self.loader.load_data()
train_steps, val_steps = self.loader.pre_calculation()
self._train(train_gen, train_steps, val_gen, val_steps)
def _train(self, train_gen, train_steps, val_gen, val_steps):
self.model.fit_generator(
generator=train_gen,
steps_per_epoch=train_steps,
epochs=self.epochs,
validation_data=val_gen,
validation_steps=val_steps,
)
self.model.save_weights(
os.path.join(settings.MODEL, f'{self.model_name}.h5'))
def readorg(self):
dic = {}
dishnames = DataLoader().load_dish_name()
basicwords = self.readcorpus()
regionnames = self.readregion()
embedding_text = Word2VecKeyedVectors.load_word2vec_format(self.txtfilepath, binary=False)
model = embedding_text
for word in basicwords:
if word in model.wv.vocab.keys():
dic[word] = model[word]
a = 0
temp =[]
#print(dic[word])
#print(dic[word][a])
while a < 200:
temp.append(float(dic[word][a]))
a +=1
dic[word] = temp
for dishname in dishnames:
for name in dishname:
if name in model.wv.vocab.keys():
dic[name] = model[name]
a = 0
temp = []
while a < 200:
temp.append(float(dic[name][a]))
a +=1
dic[name] = temp
for regionname in regionnames:
for name in regionname:
if name in model.wv.vocab.keys():
dic[name] = model[name]
a = 0
temp = []
while a < 200:
temp.append(float(dic[name][a]))
a +=1
dic[name] = temp
f = open('./data/ChineseFoodEmbedding.txt', 'w', encoding='utf-8')
index=0
while index < 10:
print(dic[basicwords[index]])
index += 1
for key in dic.keys():
pattern = re.compile(r'[\[\]\n\r\t]')
f.write(key+" "+re.sub(pattern, "", str(dic[key]))+'\n')
f.close()
def main():
dataDir = '/Users/Blair/PycharmProjects/CSE547/data'
dataTypes = ["train2014", "val2014", "test2014"]
for dataType in dataTypes[:2]:
loader = DataLoader(dataDir, dataType)
loader.load()
if dataType == dataTypes[0]:
train_dt = loader.feats
train_lab = loader.label
train_id = loader.id
elif dataType == dataTypes[1]:
val_dt = loader.feats
val_lab = loader.label
else:
test_dt = loader.feats
test_lab = loader.label
model = LSH()
search_time, avg_dist, mAP = model.nn_search(train_dt, train_lab, train_id,
val_dt, val_lab, 20, 5, 100,
20, 10, 1.5)
print(search_time)
print(avg_dist)
print(mAP)
def __init__(self):
self.dataset, self.cate_dict = DataLoader().load_data()
self.cate_nums = len(self.cate_dict)
self.catetype_dict = {
'0': '汽车',
'1': '财经',
'2': 'IT',
'3': '健康',
'4': '体育',
'5': '旅游',
'6': '教育',
'7': '招聘',
'8': '文化',
'9': '军事',
}
def filterEnbedding(self):
vocab_dict_word2vec = {}
dishnames = DataLoader().load_dish_name()
basicwords = self.readcorpus()
regionnames = self.readregion()
print("arrive position1!")
fv = fast2Vec()
fv.load_word2vec_format(word2vec_path=self.txtfilepath)
print("arrive position2!")
for dishname in dishnames:
basicwords +=dishname
for regionname in regionnames:
basicwords += regionname
print("arrive position3!")
for word in basicwords:
vocab_dict_word2vec[word] = fv.wordVec(word, min_n=1, max_n=3)
print("arrive position4!")
fv.wordvec_save2txt(vocab_dict_word2vec, save_path='./data/ChineseFoodEmbeddingModel.txt', encoding='utf-8-sig')
class TrainStyleTransfer(ImageStore):
def __init__(self, epochs=10):
st = StyleTransfer()
self.model = st.build_model()
self.model_gen = st.build_encoder_decoder()
self.loader = DataLoader()
self.epochs = epochs
def train(self):
gen, image_path_list = self.loader.load_data()
self._train(gen, image_path_list)
def _train(self, gen, image_path_list):
img_test = load_img(settings.TEST_IMAGE, target_size=settings.INPUT_SHAPE[:2])
img_arr_test = np.expand_dims(img_to_array(img_test), axis=0)
steps_per_epochs = math.ceil(len(image_path_list) / settings.BATCH_SIZE)
iters_vobose = 1000
iters_save_img = 1000
iters_save_model = steps_per_epochs
cur_epoch = 0
losses = []
path_tmp = 'epoch_{}_iters_{}_loss_{:.2f}{}'
for i, (x_train, y_train) in enumerate(gen):
if i % steps_per_epochs == 0:
cur_epoch += 1
loss = self.model.train_on_batch(x_train, y_train)
losses.append(loss)
if i % iters_vobose == 0:
print('epoch:{}\titers:{}\tloss:{:.2f}'.format(cur_epoch, i, loss[0]))
if i % iters_save_img == 0:
pred = self.model_gen.predict(img_arr_test)
img_pred = array_to_img(pred.squeeze())
path_trs_img = path_tmp.format(cur_epoch, i, loss[0], '.jpg')
img_pred.save(os.path.join(settings.DEBUG_IMG, path_trs_img))
print('saved {}'.format(path_trs_img))
if i % iters_save_model == 0:
self.model.save(os.path.join(settings.MODEL, path_tmp.format(cur_epoch, i, loss[0], '.h5')))
path_loss = os.path.join(settings.LOG, 'loss.pkl')
with open(path_loss, 'wb') as f:
pickle.dump(losses, f)
simple_value=np.sqrt(im_loss_avg / 100.0))
summary_writer.add_summary(summary, all_gif_num)
summary_writer.flush()
show_use_time(time.time() - start_time, head_str + ' use time:')
return im_loss_avg
def valid_batch(*arg, **kwargs):
return train_all_batch(*arg, **kwargs, training=False)
# Data Loader
train_dlr = DataLoader(_TRAIN_DATA,
img_size=cfg['image_height'],
load_num=20000)
valid_dlr = DataLoader(_VALID_DATA,
img_size=cfg['image_height'],
load_num=2000)
train_data = train_dlr.input_pipeline()
valid_data = valid_dlr.input_pipeline()
is_training = tf.placeholder(dtype=bool, shape=())
gif, fdb, cmd, name = tf.cond(is_training, lambda: (train_data), lambda:
(valid_data))
m = BehaviorClone()
m.set_network_property(drop_out=FLAGS.drop_out)
m.build_inputs_and_outputs(tf.squeeze(gif), tf.squeeze(fdb), tf.squeeze(cmd))
m.build_train_op()
def main(params):
""" param config for param_search """
args.alpha = params['alpha']
args.beta = params['beta']
args.sigma = params['sigma']
args.batch_size = params['batch_size']
args.semi_batchsize = params['semi_batch_size']
args.seed = params['seed']
args.output_name = params['output']
args.epochs = params['epochs']
args.data = params['data']
args.train_ratio = params['train_ratio']
args.gpu = params['gpu']
if args.data == 'news':
args.tfidf = params['tfidf']
args.reg_decay = params['rd']
print(args)
use_cuda = not args.no_cuda and torch.cuda.is_available()
torch.manual_seed(args.train_seed)
torch.cuda.manual_seed(args.train_seed)
np.random.seed(args.train_seed)
random.seed(args.train_seed)
torch.cuda.set_device(args.gpu)
device = torch.device(args.gpu if use_cuda else "cpu")
print('Seed: {}'.format(args.seed))
print('Data type: {}'.format(args.data))
Data = DataLoader(args=args)
train_loader = Data.train_loader
val_loader = Data.val_loader
in_loader = Data.in_loader
test_loader = Data.test_loader
model = Model(stds=[Data.treat_std, Data.cont_std],
p=args.dp,
in_dim=Data.in_dim,
alpha=args.alpha,
beta=args.beta,
features=torch.Tensor(Data.X),
device=device,
out_dim=args.dim,
W=Data.W).to(device)
weight_decay = 1e-8
clipping_value = 1
torch.nn.utils.clip_grad_norm_(model.parameters(), clipping_value)
optimizer = optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=weight_decay)
trainer = Trainer(model=model,
device=device,
args=args,
optimizer=optimizer,
train_loader=train_loader,
validation_loader=val_loader,
in_loader=in_loader,
test_loader=test_loader,
D=Data)
trainer.run()
print('Min (out, in): {}, {}'.format(np.min(trainer.test_losses),
np.min(trainer.in_losses)))
print('Validated min (out, in): {}, {}'.format(
trainer.test_losses[np.argsort(np.array(trainer.val_losses))[0]],
trainer.in_losses[np.argsort(np.array(trainer.val_losses))[0]]))
print('T mse: {}'.format(trainer.t_mse_losses[np.argsort(
np.array(trainer.val_losses))[0]]))
print('C mse: {}'.format(trainer.c_mse_losses[np.argsort(
np.array(trainer.val_losses))[0]]))
print('epoch: {}'.format(np.argsort(np.array(trainer.val_losses))[0] + 1))
if not os.path.exists(args.output_dir):
os.mkdir(args.output_dir)
f = open(args.output_dir + args.output_name, 'a')
f.write(
str(
float(trainer.test_losses[np.argsort(np.array(trainer.val_losses))
[0]])) + ',')
f.write(
str(
float(trainer.in_losses[np.argsort(np.array(trainer.val_losses))
[0]])) + ',')
f.write(str(float(np.min(np.array(trainer.val_losses)))) + '\n')
f.close()
def __init__(self):
self.dataset = DataLoader().dataset
self.min_count = 5
self.window_size = 5
self.word_demension = 200
self.embedding_path = 'model/word2doc_wordvec.bin'
import os
import argparse
import time
import gc
from pyspark.sql import SparkSession
from load_data import DataLoader
from pathlib import Path
from als_recommender import ALSRecommender
data_path = Path('data/ml-latest')
data_folder = Path('data/')
raw_file_path = Path('downloads/ml_latest.zip')
spark = SparkSession.builder.appName('Movie Recommender').master(
'local[*]').getOrCreate()
data_loader = DataLoader(spark)
folders = ['data', 'downloads', 'best_model', 'user_recs', 'archive']
for folder in folders:
if not os.path.exists(folder):
os.mkdir(folder)
if not os.path.exists(data_path):
if not os.path.exists(raw_file_path):
url = 'http://files.grouplens.org/datasets/movielens/ml-latest.zip'
data_loader.get_file(url, raw_file_path)
data_loader.unzip_file(data_folder, raw_file_path)
dfs = data_loader.load_to_DF(data_path, '.csv', 'movies', 'ratings')
ratings_df = dfs['ratings'].drop('timestamp')
movies_df = dfs['movies']
print('ratings_df size: {}\nmovies_df size: {}'.format(ratings_df.count(),
movies_df.count()))
def __init__(self, epochs=10):
self.model = self.build_model()
self.loader = DataLoader()
self.epochs = epochs
self.model_name = 'super_resolution_model'
return super().__call__(y_true, y_pred, sample_weight=sample_weight)
def train_model(model, train_dataset, val_dataset, num_classes, loss_fn, epochs=10):
sgd = tf.keras.optimizers.SGD(learning_rate=0.01, momentum=0.9)
model.compile(optimizer=sgd,loss=loss_fn(), weighted_metrics=["accuracy", MeanIoU(num_classes)])
# verbose=2 gives 1 line per epcoh, which is good when logging to a file
model.fit(x=train_dataset, verbose=1, epochs=epochs, validation_data=val_dataset, callbacks=[TensorBoard(), ModelCheckpoint("backup" + str(epochs)), DisplayCallback(model, train_dataset, saveimg=True)])
# Create dataset loader
# NOTE: A simple overfit-test can be made by loading only 1 image, but the generated
# images will look bad. This is due to poor estimation of batch norm parameters, and
# can be hotfixed by switching to training mode during image generation in display.py
loader = DataLoader('cityscapes', '50%', batch_size=8)
# Prepare all the data before usage. This includes:
# - Casting the images to float32.
# - Normalizing all the data according to the normalization strategy for ResNet50.
# - Applying a random flip to the training data every time it is encountered.
# - Batching the data.
# - Prefetching 1 batch to improve the data throughput.
loader.prepareDatasets()
# Load the datasets
train_ds = loader.getTrainData()
val_ds = loader.getValData()
# Define model
import numpy as np
from load_data import DataLoader
def loss_dec_test_BC(g, data_test, data_test_gt, size):
diffs=[]
for i in range(1,size):
rec_imgs = g[i].predict(data_test)
rec_imgs = np.reshape(rec_imgs,(rec_imgs.shape[0],32*32*3))
data_test_c = np.reshape(data_test,(data_test.shape[0],32*32*3))
diff = np.mean(np.square(rec_imgs- data_test_c), axis=-1)
diffs.append(diff)
return (diffs)
data_loader = DataLoader(img_res=(32, 32))
identities={}
g={}
NUM=34
# Load one-class classifiers of all users
for i in range(1,NUM):
identities[i]='user_%d.npz'%(i)
g[i] = load_model('SavedModels/%s/%s.h5'%(identities[i].split('.')[0],identities[i].split('.')[0]), compile=False)
print('The model of {} is loaded'.format(identities[i]))
# Prediction
positives=[]
for i in range(1,NUM):
class Pix2Pix():
def __init__(self):
# Input shape
self.img_rows = 256
self.img_cols = 256
self.channels = 3
self.img_shape = (self.img_rows, self.img_cols, self.channels)
# Configure data loader
print("Reading...")
filepath = [
"./data/trainImage256.txt", "./data/trainLabel256.txt",
"./data/testImage256.txt", "./data/testLabel256.txt"
]
self.data_loader = DataLoader(filepath,
img_res=(self.img_rows, self.img_cols))
# Calculate output shape of D (PatchGAN)
patch = int(self.img_rows / 2**4)
self.disc_patch = (patch, patch, 1)
# Number of filters in the first layer of G and D
self.gf = 64
self.df = 64
optimizer = Adam(0.0002, 0.5)
# Build and compile the discriminator
print("Building discriminator")
self.discriminator = self.build_discriminator()
self.discriminator.compile(loss='mse',
optimizer=optimizer,
metrics=['accuracy'])
#-------------------------
# Construct Computational
# Graph of Generator
#-------------------------
# Build the generator
self.generator = self.build_generator()
# Input images and their conditioning images
img_A = Input(shape=self.img_shape)
img_B = Input(shape=self.img_shape)
# By conditioning on B generate a fake version of A
fake_A = self.generator(img_B)
# For the combined model we will only train the generator
self.discriminator.trainable = False
# Discriminators determines validity of translated images / condition pairs
valid = self.discriminator([fake_A, img_B])
print("Building generator...")
self.combined = Model(inputs=[img_A, img_B], outputs=[valid, fake_A])
self.combined.compile(loss=['mse', 'mae'],
loss_weights=[1, 100],
optimizer=optimizer)
def build_generator(self):
"""U-Net Generator"""
# def conv2d(layer_input, filters, f_size=4, bn=True):
# """Layers used during downsampling"""
# d = Conv2D(filters, kernel_size=f_size, strides=2, padding='same')(layer_input)
# d = LeakyReLU(alpha=0.2)(d)
# if bn:
# d = BatchNormalization(momentum=0.8)(d)
# return d
# def deconv2d(layer_input, skip_input, filters, f_size=4, dropout_rate=0):
# """Layers used during upsampling"""
# u = UpSampling2D(size=2)(layer_input)
# u = Conv2D(filters, kernel_size=f_size, strides=1, padding='same', activation='relu')(u)
# if dropout_rate:
# u = Dropout(dropout_rate)(u)
# u = BatchNormalization(momentum=0.8)(u)
# u = Concatenate()([u, skip_input])
# return u
# Image input
# d0 = Input(shape=self.img_shape)
# # Downsampling
# d1 = conv2d(d0, self.gf, bn=False)
# d2 = conv2d(d1, self.gf*2)
# d3 = conv2d(d2, self.gf*4)
# d4 = conv2d(d3, self.gf*8)
# d5 = conv2d(d4, self.gf*8)
# d6 = conv2d(d5, self.gf*8)
# d7 = conv2d(d6, self.gf*8)
# # Upsampling
# u1 = deconv2d(d7, d6, self.gf*8)
# u2 = deconv2d(u1, d5, self.gf*8)
# u3 = deconv2d(u2, d4, self.gf*8)
# u4 = deconv2d(u3, d3, self.gf*4)
# u5 = deconv2d(u4, d2, self.gf*2)
# u6 = deconv2d(u5, d1, self.gf)
# u7 = UpSampling2D(size=2)(u6)
# output_img = Conv2D(self.channels, kernel_size=4, strides=1, padding='same', activation='tanh')(u7)
inputs = Input(shape=self.img_shape)
# encoding ##############
conv1_1 = Conv2D(64,
3,
activation="relu",
padding="same",
kernel_initializer="he_normal",
bias_initializer="zeros")(inputs)
conv1_2 = Conv2D(64,
3,
activation="relu",
padding="same",
kernel_initializer="he_normal",
bias_initializer="zeros")(conv1_1)
pool1 = MaxPooling2D(pool_size=(2, 2))(conv1_2)
conv2_1 = Conv2D(128,
3,
activation="relu",
padding="same",
kernel_initializer="he_normal",
bias_initializer="zeros")(pool1)
conv2_2 = Conv2D(128,
3,
activation="relu",
padding="same",
kernel_initializer="he_normal",
bias_initializer="zeros")(conv2_1)
pool2 = MaxPooling2D(pool_size=(2, 2))(conv2_2)
conv3_1 = Conv2D(256,
3,
activation="relu",
padding="same",
kernel_initializer="he_normal",
bias_initializer="zeros")(pool2)
conv3_2 = Conv2D(256,
3,
activation="relu",
padding="same",
kernel_initializer="he_normal",
bias_initializer="zeros")(conv3_1)
pool3 = MaxPooling2D(pool_size=(2, 2))(conv3_2)
conv4_1 = Conv2D(512,
3,
activation="relu",
padding="same",
kernel_initializer="he_normal",
bias_initializer="zeros")(pool3)
conv4_2 = Conv2D(512,
3,
activation="relu",
padding="same",
kernel_initializer="he_normal",
bias_initializer="zeros")(conv4_1)
#drop4 = Dropout(0.5)(conv4_2)
pool4 = MaxPooling2D(pool_size=(2, 2))(conv4_2)
conv5_1 = Conv2D(1024,
3,
activation="relu",
padding="same",
kernel_initializer="he_normal",
bias_initializer="zeros")(pool4)
conv5_2 = Conv2D(1024,
3,
activation="relu",
padding="same",
kernel_initializer="he_normal",
bias_initializer="zeros")(conv5_1)
#drop5 = Dropout(0.5)(conv5_2)
conv_up5 = Conv2D(512,
3,
activation="relu",
padding="same",
kernel_initializer="he_normal",
bias_initializer="zeros")(
UpSampling2D(size=(2, 2))(conv5_2))
concated5 = concatenate([conv4_2, conv_up5], axis=3)
# decoding ##############
conv6_1 = Conv2D(512,
3,
activation="relu",
padding="same",
kernel_initializer="he_normal",
bias_initializer="zeros")(concated5)
conv6_2 = Conv2D(512,
3,
activation="relu",
padding="same",
kernel_initializer="he_normal",
bias_initializer="zeros")(conv6_1)
conv_up6 = Conv2D(256,
3,
activation="relu",
padding="same",
kernel_initializer="he_normal",
bias_initializer="zeros")(
UpSampling2D(size=(2, 2))(conv6_2))
concated6 = concatenate([conv3_2, conv_up6], axis=3)
conv7_1 = Conv2D(256,
3,
activation="relu",
padding="same",
kernel_initializer="he_normal",
bias_initializer="zeros")(concated6)
conv7_2 = Conv2D(256,
3,
activation="relu",
padding="same",
kernel_initializer="he_normal",
bias_initializer="zeros")(conv7_1)
conv_up7 = Conv2D(128,
3,
activation="relu",
padding="same",
kernel_initializer="he_normal",
bias_initializer="zeros")(
UpSampling2D(size=(2, 2))(conv7_2))
concated7 = concatenate([conv2_2, conv_up7], axis=3)
conv8_1 = Conv2D(128,
3,
activation="relu",
padding="same",
kernel_initializer="he_normal",
bias_initializer="zeros")(concated7)
conv8_2 = Conv2D(128,
3,
activation="relu",
padding="same",
kernel_initializer="he_normal",
bias_initializer="zeros")(conv8_1)
conv_up8 = Conv2D(64,
3,
activation="relu",
padding="same",
kernel_initializer="he_normal",
bias_initializer="zeros")(
UpSampling2D(size=(2, 2))(conv8_2))
concated8 = concatenate([conv1_2, conv_up8], axis=3)
conv9_1 = Conv2D(64,
3,
activation="relu",
padding="same",
kernel_initializer="he_normal",
bias_initializer="zeros")(concated8)
conv9_2 = Conv2D(64,
3,
activation="relu",
padding="same",
kernel_initializer="he_normal",
bias_initializer="zeros")(conv9_1)
outputs = Conv2D(self.channels, 1, activation="tanh")(conv9_2)
return Model(input=inputs, output=outputs)
def build_discriminator(self):
def d_layer(layer_input, filters, f_size=4, bn=True):
"""Discriminator layer"""
d = Conv2D(filters, kernel_size=f_size, strides=2,
padding='same')(layer_input)
d = LeakyReLU(alpha=0.2)(d)
if bn:
d = BatchNormalization(momentum=0.8)(d)
return d
img_A = Input(shape=self.img_shape)
img_B = Input(shape=self.img_shape)
# Concatenate image and conditioning image by channels to produce input
combined_imgs = Concatenate(axis=-1)([img_A, img_B])
d1 = d_layer(combined_imgs, self.df, bn=False)
d2 = d_layer(d1, self.df * 2)
d3 = d_layer(d2, self.df * 4)
d4 = d_layer(d3, self.df * 8)
validity = Conv2D(1, kernel_size=4, strides=1, padding='same')(d4)
return Model([img_A, img_B], validity)
def train(self, epochs, batch_size=1, sample_interval=50):
start_time = datetime.datetime.now()
# Adversarial loss ground truths
valid = np.ones((batch_size, ) + self.disc_patch)
fake = np.zeros((batch_size, ) + self.disc_patch)
for epoch in range(epochs):
batchs = self.defineBatchComtents(batch_size)
for batch_i in range(
len(batchs)
): #, (imgs_A, imgs_B) in enumerate(self.data_loader.load_batch(batch_size)):
batch_sorted = sorted(batchs[batch_i])
imgs_A, imgs_B = self.data_loader.load_batch(batch_sorted)
# ---------------------
# Train Discriminator
# ---------------------
# Condition on B and generate a translated version
fake_A = self.generator.predict(imgs_B)
# Train the discriminators (original images = real / generated = Fake)
d_loss_real = self.discriminator.train_on_batch(
[imgs_A, imgs_B], valid)
d_loss_fake = self.discriminator.train_on_batch(
[fake_A, imgs_B], fake)
d_loss = 0.5 * np.add(d_loss_real, d_loss_fake)
# -----------------
# Train Generator
# -----------------
# Train the generators
g_loss = self.combined.train_on_batch([imgs_A, imgs_B],
[valid, imgs_A])
elapsed_time = datetime.datetime.now() - start_time
# Plot the progress
string = "[Epoch %d/%d] [Batch %d/%d] [D loss: %f, acc: %3d%%] [G loss: %f] time: %s" % (
epoch, epochs, batch_i, len(batchs), d_loss[0],
100 * d_loss[1], g_loss[0], elapsed_time)
print(string)
with open("./training.LOG", "a") as f:
f.write(string + "\n")
# If at save interval => save generated image samples
if batch_i % sample_interval == 0:
self.sample_images(epoch, batch_i)
def sample_images(self, epoch, batch_i):
os.makedirs('images/test', exist_ok=True)
r, c = 3, 3
imgs_A, imgs_B = self.data_loader.load_test_data(batch_size=3)
fake_A = self.generator.predict(imgs_B)
gen_imgs = np.concatenate([imgs_B, fake_A, imgs_A])
# Rescale images 0 - 1
gen_imgs = 0.5 * gen_imgs + 0.5
titles = ['Condition', 'Generated', 'Original']
fig, axs = plt.subplots(r, c)
cnt = 0
for i in range(r):
for j in range(c):
axs[i, j].imshow(gen_imgs[cnt])
axs[i, j].set_title(titles[i])
axs[i, j].axis('off')
cnt += 1
fig.savefig("images/test/%d_%d.png" % (epoch, batch_i))
plt.close()
def save(self, output_filename):
self.generator.save(output_filename[0])
self.discriminator.save(output_filename[1])
def defineBatchComtents(self, batch_size):
num = np.linspace(0, c.TRAIN_DATA_SIZE - 1, c.TRAIN_DATA_SIZE)
num = num.tolist()
COMPONENT = []
total_batch = int(c.TRAIN_DATA_SIZE / batch_size)
for i in range(total_batch):
component = random.sample(num, batch_size)
COMPONENT.append(component)
for j in range(batch_size):
cnt = 0
while True:
if (num[cnt] == component[j]):
num.pop(cnt)
break
else:
cnt += 1
return COMPONENT
class RAN():
def __init__(self, identity):
self.identity = identity
self.img_rows = 32
self.img_cols = 32
self.channels = 3
self.img_shape = (self.img_rows, self.img_cols, self.channels)
# Configure data loader
self.dataset_name = self.identity
self.data_loader = DataLoader(img_res=(self.img_rows, self.img_cols))
# Calculate output shape of D (PatchRAN)
patch = int(self.img_rows / 2**4)
self.disc_patch = (patch, patch, 1)
optimizer = Adam(0.0002, 0.5)
# Number of filters in the first layer of G and D
self.gf = 32
self.df = 64
# Build and compile the discriminator
self.discriminator = self.build_discriminator()
self.discriminator.compile(loss='mse',
optimizer=optimizer,
metrics=['accuracy'])
# Build and compile the reconstructor
self.reconstructor = self.build_reconstructor()
print(self.reconstructor.summary())
self.reconstructor.compile(loss='mse', optimizer=optimizer)
# The reconstructor takes noise as input and generated imgs
img = Input(shape=self.img_shape)
reconstr = self.reconstructor(img)
# For the combined model we will only train the reconstructor
self.discriminator.trainable = False
# The valid takes generated images as input and determines validity
valid = self.discriminator(reconstr)
# The combined model (stacked reconstructor and discriminator) takes
# images as input => reconstruct images => determines validity
self.combined = Model(img, [reconstr, valid])
self.combined.compile(loss=['mse', 'mse'],
loss_weights=[0.999, 0.001],
optimizer=optimizer)
def build_reconstructor(self):
"""reconstructor"""
def conv2d(layer_input, filters, f_size=4):
"""Layers used during downsampling"""
d = Conv2D(filters, kernel_size=f_size, strides=2,
padding='same')(layer_input)
d = LeakyReLU(alpha=0.2)(d)
d = InstanceNormalization()(d)
return d
def deconv2d(layer_input, filters, f_size=4, dropout_rate=0):
"""Layers used during upsampling"""
u = UpSampling2D(size=2)(layer_input)
u = Conv2D(filters,
kernel_size=f_size,
strides=1,
padding='same',
activation='relu')(u)
if dropout_rate:
u = Dropout(dropout_rate)(u)
u = InstanceNormalization()(u)
return u
# Image input
d0 = Input(shape=self.img_shape)
# Downsampling
d1 = conv2d(d0, self.gf)
d2 = conv2d(d1, self.gf * 2)
d3 = conv2d(d2, self.gf * 4)
d4 = conv2d(d3, self.gf * 4)
d5 = conv2d(d4, self.gf * 8)
# Upsampling
u1 = deconv2d(d5, self.gf * 8)
u2 = deconv2d(u1, self.gf * 8)
u3 = deconv2d(u2, self.gf * 8)
u4 = deconv2d(u3, self.gf * 4)
u5 = deconv2d(u4, self.gf * 2)
output_img = Conv2D(self.channels,
kernel_size=4,
strides=1,
padding='same',
activation='tanh')(u5)
return Model(d0, output_img)
def build_discriminator(self):
def d_layer(layer_input, filters, f_size=4, normalization=True):
"""Discriminator layer"""
d = Conv2D(filters, kernel_size=f_size, strides=2,
padding='same')(layer_input)
d = LeakyReLU(alpha=0.2)(d)
if normalization:
d = InstanceNormalization()(d)
return d
img = Input(shape=self.img_shape)
d1 = d_layer(img, self.df, normalization=False)
d2 = d_layer(d1, self.df * 2)
d3 = d_layer(d2, self.df * 4)
d4 = d_layer(d3, self.df * 8)
validity = Conv2D(1, kernel_size=4, strides=1, padding='same')(d4)
return Model(img, validity)
def train(self, epochs, batch_size=128, save_interval=50):
half_batch = int(batch_size / 2)
start_time = datetime.datetime.now()
imgsVal = self.data_loader.load_data(self.identity,
batch_size=half_batch,
is_testing=True)
TrainLoss = np.zeros(epochs)
ValLoss = np.ones(epochs)
for epoch in range(epochs):
# ---------------------
# Train Discriminator
# ---------------------
# Sample reconstructor input
img = self.data_loader.load_data(self.identity,
batch_size=half_batch)
# Reconstruct a batch of new images
reconstr = self.reconstructor.predict(img)
# Adversarial loss ground truths
valid = np.ones((half_batch, ) + self.disc_patch)
fake = np.zeros((half_batch, ) + self.disc_patch)
# Train the discriminator
d_loss_real = self.discriminator.train_on_batch(img, valid)
d_loss_fake = self.discriminator.train_on_batch(reconstr, fake)
d_loss = 0.5 * np.add(d_loss_real, d_loss_fake)
# ---------------------
# Train reconstructor
# ---------------------
# Sample reconstructor input
img = self.data_loader.load_data(self.identity,
batch_size=half_batch)
# Train the reconstructor
r_loss = self.combined.train_on_batch(img, [img, valid])
r_loss_val = self.combined.test_on_batch(imgsVal, [imgsVal, valid])
TrainLoss[epoch] = r_loss[0]
MinValLoss = ValLoss.min()
ValLoss[epoch] = r_loss_val[0]
# Plot the progress
print(
"%d [D loss: %f, acc.: %.2f%%] [R loss: %f] [R loss Val: %f] [Minimum: %f]"
% (epoch, d_loss[0], 100 * d_loss[1], r_loss[0], r_loss_val[0],
MinValLoss))
# If at save interval => save generated image samples
if ValLoss[epoch] < MinValLoss and MinValLoss < 0.04:
self.save_imgs(epoch)
self.reconstructor.save('SavedModel/%s/%s.h5' %
(self.identity, self.identity))
np.savez('loss/Loss_%s' % (self.dataset_name),
TrLoss=TrainLoss,
TeLoss=ValLoss)
def save_imgs(self, epoch):
r, c = 2, 2
imgs = self.data_loader.load_data(self.identity,
batch_size=1,
is_testing=False)
imgs_val = self.data_loader.load_data(self.identity,
batch_size=1,
is_testing=True)
# Translate images to the other domain
reconstr = self.reconstructor.predict(imgs)
reconstr_val = self.reconstructor.predict(imgs_val)
gen_imgs = np.concatenate([imgs, imgs_val, reconstr, reconstr_val])
# Rescale images 0 - 1
gen_imgs = 0.5 * gen_imgs + 0.5
titles = ['Train', 'Val', 'Reconstructed']
fig, axs = plt.subplots(r, c)
cnt = 0
for i in range(r):
for j in range(c):
axs[i, j].imshow(gen_imgs[cnt])
axs[i, j].set_title(titles[j])
axs[i, j].axis('off')
cnt += 1
fig.savefig("ReconstructedImages/%s/TrainValSamples_E%d.png" %
(self.dataset_name, epoch))
plt.close()
Conv2D(32, (3, 3),
padding='same',
kernel_regularizer=l2(),
activation='relu'))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(1024, kernel_regularizer=l2(), activation='relu'))
model.add(Dropout(0.1))
model.add(Dense(7, activation='softmax'))
return model
if __name__ == '__main__':
# Load Data, default csv path is data/fer2013.csv
train_x, train_y, test_x, test_y = DataLoader().load_data()
# data pre-processing
# we can subtract the mean image
# mean = train_x.mean(axis=0)
# or we can subtract the mean color
mean = np.mean(np.mean(np.mean(train_x, 2), 1), 0)
train_x = train_x - mean
test_x = test_x - mean
# get user choice
message = "Create model from scratch? [1]\nLoad a pre-trained model? [2]\n>>Please enter your choice: "
choice = input(message)
while not choice.isnumeric() or int(choice) < 1 or int(choice) > 2:
choice = input(">>Please enter a valid choice: ")
choice = int(choice)
def __init__(self, epochs=100):
self.autoencoder = self.build_model()
self.loader = DataLoader()
self.epochs = epochs
def __init__(self, epochs=10):
st = StyleTransfer()
self.model = st.build_model()
self.model_gen = st.build_encoder_decoder()
self.loader = DataLoader()
self.epochs = epochs
def run_vitc(params, x_data_train, y_data_train, x_data_val, y_data_val, x_data_test, y_data_test, y_data_test_noisefree, save_dir, truth_test, bounds, fixed_vals, bilby_samples, snrs_test=None):
epochs = params['num_iterations']
train_size = params['load_chunk_size']
batch_size = params['batch_size']
val_size = params['val_dataset_size']
test_size = params['r']
plot_dir = params['plot_dir']
plot_cadence = int(0.5*params['plot_interval'])
# Include the epoch in the file name (uses `str.format`)
checkpoint_path = "inverse_model_%s/model.ckpt" % params['run_label']
checkpoint_dir = os.path.dirname(checkpoint_path)
make_paper_plots = params['make_paper_plots']
hyper_par_tune = False
# if doing hour angle, use hour angle bounds on RA
bounds['ra_min'] = convert_ra_to_hour_angle(bounds['ra_min'],params,None,single=True)
bounds['ra_max'] = convert_ra_to_hour_angle(bounds['ra_max'],params,None,single=True)
print('... converted RA bounds to hour angle')
# load the training data
if not make_paper_plots:
train_dataset = DataLoader(params["train_set_dir"],params = params,bounds = bounds, masks = masks,fixed_vals = fixed_vals, chunk_batch = 40)
validation_dataset = DataLoader(params["val_set_dir"],params = params,bounds = bounds, masks = masks,fixed_vals = fixed_vals, chunk_batch = 2)
x_data_test, y_data_test_noisefree, y_data_test, snrs_test = load_data(params,bounds,fixed_vals,params['test_set_dir'],params['inf_pars'],test_data=True)
y_data_test = y_data_test[:params['r'],:,:]; x_data_test = x_data_test[:params['r'],:]
# load precomputed samples
bilby_samples = []
for sampler in params['samplers'][1:]:
bilby_samples.append(load_samples(params,sampler, bounds = bounds))
bilby_samples = np.array(bilby_samples)
#bilby_samples = np.array([load_samples(params,'dynesty'),load_samples(params,'ptemcee'),load_samples(params,'cpnest')])
test_dataset = (tf.data.Dataset.from_tensor_slices((x_data_test,y_data_test))
.batch(1))
train_loss_metric = tf.keras.metrics.Mean('train_loss', dtype=tf.float32)
train_summary_writer = tf.summary.create_file_writer(train_log_dir)
if params['resume_training']:
model = CVAE(x_data_train.shape[1], params['ndata'],
y_data_train.shape[2], params['z_dimension'], params['n_modes'], params, bounds = bounds, masks = masks)
# Load the previously saved weights
latest = tf.train.latest_checkpoint(checkpoint_dir)
model.load_weights(latest)
print('... loading in previous model %s' % checkpoint_path)
else:
model = CVAE(x_data_test.shape[1], params['ndata'],
y_data_test.shape[2], params['z_dimension'], params['n_modes'], params, bounds, masks)
# Make publication plots
if make_paper_plots:
print('... Making plots for publication.')
# Load the previously saved weights
latest = tf.train.latest_checkpoint(checkpoint_dir)
model.load_weights(latest)
print('... loading in previous model %s' % checkpoint_path)
paper_plots(test_dataset, y_data_test, x_data_test, model, params, plot_dir, run, bilby_samples)
return
# start the training loop
train_loss = np.zeros((epochs,3))
val_loss = np.zeros((epochs,3))
ramp_start = params['ramp_start']
ramp_length = params['ramp_end']
ramp_cycles = 1
KL_samples = []
optimizer = tf.keras.optimizers.Adam(1e-5)
# Keras hyperparameter optimization
if hyper_par_tune:
import keras_hyper_optim
del model
keras_hyper_optim.main(train_dataset, val_dataset)
exit()
# log params used for this run
path = params['plot_dir']
shutil.copy('./vitamin_c_new.py',path)
shutil.copy('./params_files/params.json',path)
print("Loading intitial data....")
train_dataset.load_next_chunk()
validation_dataset.load_next_chunk()
model.compile()
for epoch in range(1, epochs + 1):
train_loss_kl_q = 0.0
train_loss_kl_r1 = 0.0
start_time_train = time.time()
if params['resume_training']:
ramp = ramp = tf.convert_to_tensor(1.0)
print('... Not using ramp.')
else:
ramp = tf.convert_to_tensor(ramp_func(epoch,ramp_start,ramp_length,ramp_cycles), dtype=tf.float32)
for step in range(len(train_dataset)):
y_batch_train, x_batch_train = train_dataset[step]
if len(y_batch_train) == 0:
print("NO data: ", train_dataset.chunk_iter, np.shape(y_batch_train))
#print(step, np.shape(y_batch_train),np.shape(x_batch_train))
temp_train_r_loss, temp_train_kl_loss = model.train_step(x_batch_train, y_batch_train, optimizer, ramp=ramp)
train_loss[epoch-1,0] += temp_train_r_loss
train_loss[epoch-1,1] += temp_train_kl_loss
train_loss[epoch-1,2] = train_loss[epoch-1,0] + ramp*train_loss[epoch-1,1]
train_loss[epoch-1,:] /= float(step+1)
end_time_train = time.time()
with train_summary_writer.as_default():
tf.summary.scalar('loss', train_loss_metric.result(), step=epoch)
train_loss_metric.reset_states()
start_time_val = time.time()
for step in range(len(validation_dataset)):
y_batch_val, x_batch_val = validation_dataset[step]
temp_val_r_loss, temp_val_kl_loss = model.compute_loss(x_batch_val, y_batch_val)
val_loss[epoch-1,0] += temp_val_r_loss
val_loss[epoch-1,1] += temp_val_kl_loss
val_loss[epoch-1,2] = val_loss[epoch-1,0] + ramp*val_loss[epoch-1,1]
val_loss[epoch-1,:] /= float(step+1)
end_time_val = time.time()
print('Epoch: {}, Run {}, Training RECON: {}, KL: {}, TOTAL: {}, time elapsed: {}'
.format(epoch, run, train_loss[epoch-1,0], train_loss[epoch-1,1], train_loss[epoch-1,2], end_time_train - start_time_train))
print('Epoch: {}, Run {}, Validation RECON: {}, KL: {}, TOTAL: {}, time elapsed {}'
.format(epoch, run, val_loss[epoch-1,0], val_loss[epoch-1,1], val_loss[epoch-1,2], end_time_val - start_time_val))
if epoch % params['save_interval'] == 0:
# Save the weights using the `checkpoint_path` format
model.save_weights(checkpoint_path)
print('... Saved model %s ' % checkpoint_path)
# update loss plot
plot_losses(train_loss, val_loss, epoch, run=plot_dir)
if epoch > ramp_start + ramp_length + 2:
plot_losses_zoom(train_loss, val_loss, epoch, run=plot_dir, ind_start = ramp_start + ramp_length)
# generate and plot posterior samples for the latent space and the parameter space
if epoch % plot_cadence == 0:
for step, (x_batch_test, y_batch_test) in test_dataset.enumerate():
mu_r1, z_r1, mu_q, z_q = model.gen_z_samples(x_batch_test, y_batch_test, nsamples=1000)
plot_latent(mu_r1,z_r1,mu_q,z_q,epoch,step,run=plot_dir)
start_time_test = time.time()
samples = model.gen_samples(y_batch_test, ramp=ramp, nsamples=params['n_samples'])
end_time_test = time.time()
if np.any(np.isnan(samples)):
print('Epoch: {}, found nans in samples. Not making plots'.format(epoch))
for k,s in enumerate(samples):
if np.any(np.isnan(s)):
print(k,s)
KL_est = [-1,-1,-1]
else:
print('Epoch: {}, run {} Testing time elapsed for {} samples: {}'.format(epoch,run,params['n_samples'],end_time_test - start_time_test))
KL_est = plot_posterior(samples,x_batch_test[0,:],epoch,step,all_other_samples=bilby_samples[:,step,:],run=plot_dir)
_ = plot_posterior(samples,x_batch_test[0,:],epoch,step,run=plot_dir)
KL_samples.append(KL_est)
# plot KL evolution
#plot_KL(np.reshape(np.array(KL_samples),[-1,params['r'],len(params['samplers'])]),plot_cadence,run=plot_dir)
# iterate the chunk, i.e. load more noisefree data in
if epoch % 10 == 0:
print("Loading the next Chunk ...")
train_dataset.load_next_chunk()