def main():
# Parse the JSON arguments
config_args = parse_args()
# Create the experiment directories
_, config_args.summary_dir, config_args.checkpoint_dir = create_experiment_dirs(config_args.experiment_dir)
# Reset the default Tensorflow graph
tf.reset_default_graph()
# Tensorflow specific configuration
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
# Data loading
# The batch size is equal to 1 when testing to simulate the real experiment.
data_batch_size = config_args.batch_size if config_args.train_or_test == "train" else 1
data = DataLoader(data_batch_size, config_args.shuffle)
print("Loading Data...")
config_args.img_height, config_args.img_width, config_args.num_channels, \
config_args.train_data_size, config_args.test_data_size = data.load_data()
print("Data loaded\n\n")
# Model creation
print("Building the model...")
model = ShuffleNet(config_args)
print("Model is built successfully\n\n")
# Parameters visualization
show_parameters()
# Summarizer creation
summarizer = Summarizer(sess, config_args.summary_dir)
# Train class
trainer = Train(sess, model, data, summarizer)
if config_args.train_or_test == 'train':
try:
# print("FLOPs for batch size = " + str(config_args.batch_size) + "\n")
# calculate_flops()
print("Training...")
trainer.train()
print("Training Finished\n\n")
except KeyboardInterrupt:
trainer.save_model()
elif config_args.train_or_test == 'test':
# print("FLOPs for single inference \n")
# calculate_flops()
# This can be 'val' or 'test' or even 'train' according to the needs.
print("Testing...")
trainer.test('val')
print("Testing Finished\n\n")
else:
raise ValueError("Train or Test options only are allowed")
class Task:
"""
Core class of this volatility library
It schedules each component.
Read raw data.
Pre process data.
Analyze data.
Make prediction.
Output results.
"""
def __init__(self, path, start_date, interested_symbols, interested_start_date, interested_end_date,
num_std=3, frequency=FrequencyMap.Minute, forward=True, model_type='CloseToClose', clean=True):
"""
:param path: str
:param start_date: datetime.date
:param interested_symbols: list[str]
:param interested_start_date: datetime.date
:param interested_end_date: datetime.date
:param num_std: int
:param frequency: FrequencyMap
:param forward: boolean
:param model_type: str
:param clean: boolean
"""
self.interested_symbols = interested_symbols
self.interested_start_date = interested_start_date
self.interested_end_date = interested_end_date
self.frequency = frequency
self.analysis_window = 30 if frequency is not FrequencyMap.Month else 10
self.loader = DataLoader(path, start_date)
self.pre_process = DataPreProcessor(num_std, frequency, forward)
self.estimator = VolatilityEstimator(model_type, clean, frequency)
self.data_analyzer = DataAnalyzer()
self.model = Garch11('Constant', 'Garch')
self.error_estimator = ErrorEstimator(self.model, self.estimator, self.frequency)
def execute(self):
self.loader.load()
output = pd.DataFrame()
for symbol in self.interested_symbols:
df = self.loader.fetch(symbol, self.interested_start_date, self.interested_end_date)
df = self.pre_process.pre_process(df)
self.data_analyzer.analyze_data(df.copy())
self.estimator.analyze_realized_vol(df, self.interested_start_date, self.interested_end_date, self.analysis_window)
sample_size, error = self.error_estimator.get_best_sample_size(df)
predictions = self.model.get_predictions(df, sample_size, self.frequency)
output[symbol] = predictions[TimeSeriesDataFrameMap.Volatility]
index = predictions.index
output.set_index(index)
file_name = r'D:\programming\VOL\{frequency}_predictions.csv'.format(frequency=self.frequency)
output.to_csv(file_name)
def __init__(self, path, start_date, interested_symbols, interested_start_date, interested_end_date,
num_std=3, frequency=FrequencyMap.Minute, forward=True, model_type='CloseToClose', clean=True):
"""
:param path: str
:param start_date: datetime.date
:param interested_symbols: list[str]
:param interested_start_date: datetime.date
:param interested_end_date: datetime.date
:param num_std: int
:param frequency: FrequencyMap
:param forward: boolean
:param model_type: str
:param clean: boolean
"""
self.interested_symbols = interested_symbols
self.interested_start_date = interested_start_date
self.interested_end_date = interested_end_date
self.frequency = frequency
self.analysis_window = 30 if frequency is not FrequencyMap.Month else 10
self.loader = DataLoader(path, start_date)
self.pre_process = DataPreProcessor(num_std, frequency, forward)
self.estimator = VolatilityEstimator(model_type, clean, frequency)
self.data_analyzer = DataAnalyzer()
self.model = Garch11('Constant', 'Garch')
self.error_estimator = ErrorEstimator(self.model, self.estimator, self.frequency)
def main(_):
config = Config()
data_loader = DataLoader(config)
model = Model(config, True)
with tf.Session() as sess:
tf.initialize_all_variables().run()
for e in xrange(config.num_epochs):
print "epoch {} / {}".format(e, config.num_epochs)
sess.run(tf.assign(model.lr, config.learning_rate * (config.decay_rate ** e)))
state = model.initial_state.eval()
for b in xrange(data_loader.num_batches):
x, y = data_loader.next_batch()
feed = {model.input_data: x, model.target_data: y, model.initial_state: state}
train_loss, state, _ = sess.run([model.cost, model.final_state, model.train_op], feed_dict=feed)
print train_loss
def __init__(self):
# Input shape
self.img_rows = 256
self.img_cols = 256
self.channels = 3
self.img_shape = (self.img_rows, self.img_cols, self.channels)
# Configure data loader
self.dataset_name = 'facades'
self.data_loader = DataLoader(dataset_name=self.dataset_name,
img_res=(self.img_rows, self.img_cols))
# Calculate output shape of D (PatchGAN)
patch = int(self.img_rows / 2**4)
self.disc_patch = (patch, patch, 1)
# Number of filters in the first layer of G and D
self.gf = 64
self.df = 64
optimizer = Adam(0.0002, 0.5)
# Build and compile the discriminator
self.discriminator = self.build_discriminator()
self.discriminator.compile(loss='mse',
optimizer=optimizer,
metrics=['accuracy'])
#-------------------------
# Construct Computational
# Graph of Generator
#-------------------------
# Build the generator
self.generator = self.build_generator()
# Input images and their conditioning images
img_A = Input(shape=self.img_shape)
img_B = Input(shape=self.img_shape)
# By conditioning on B generate a fake version of A
fake_A = self.generator(img_B)
# For the combined model we will only train the generator
self.discriminator.trainable = False
# Discriminators determines validity of translated images / condition pairs
valid = self.discriminator([fake_A, img_B])
self.combined = Model(inputs=[img_A, img_B], outputs=[valid, fake_A])
self.combined.compile(loss=['mse', 'mae'],
loss_weights=[1, 100],
optimizer=optimizer)
def most_expensive_order(): data_loader = DataLoader() parser = Parser(data_loader.load_order_list()) the_user_list = UserList(data_loader.load_user_list()) most_expensive_order = parser.most_expensive_order() return 'Total price: ' + str(most_expensive_order.total_price()) + " ordered by: " + the_user_list.user_by_id(most_expensive_order.user_id)['username']
def __init__(self):
# Input shape
self.img_rows = 256
self.img_cols = 256
self.channels = 3
self.img_shape = (self.img_rows, self.img_cols, self.channels)
# Configure data loader
self.dataset_name = 'saree'
self.data_loader = DataLoader(dataset_name=self.dataset_name,
img_res=(self.img_rows, self.img_cols))
# Calculate output shape of D (PatchGAN)
patch = int(self.img_rows / 2**4)
self.disc_patch = (patch, patch, 1)
# Number of filters in the first layer of G and D
self.gf = 64
self.df = 64
optimizer = Adam(0.0002, 0.5)
# Build and compile the discriminators
self.d_A = self.build_discriminator()
self.d_B = self.build_discriminator()
self.d_A.compile(loss='mse', optimizer=optimizer, metrics=['accuracy'])
self.d_B.compile(loss='mse', optimizer=optimizer, metrics=['accuracy'])
#-------------------------
# Construct Computational
# Graph of Generators
#-------------------------
# Build the generators
self.g_AB = self.build_generator()
self.g_BA = self.build_generator()
# Input images from both domains
img_A = Input(shape=self.img_shape)
img_B = Input(shape=self.img_shape)
# Translate images to the other domain
fake_B = self.g_AB(img_A)
fake_A = self.g_BA(img_B)
# Translate images back to original domain
reconstr_A = self.g_BA(fake_B)
reconstr_B = self.g_AB(fake_A)
# For the combined model we will only train the generators
self.d_A.trainable = False
self.d_B.trainable = False
# Discriminators determines validity of translated images
valid_A = self.d_A(fake_A)
valid_B = self.d_B(fake_B)
# Objectives
# + Adversarial: Fool domain discriminators
# + Translation: Minimize MAE between e.g. fake B and true B
# + Cycle-consistency: Minimize MAE between reconstructed images and original
self.combined = Model(
inputs=[img_A, img_B],
outputs=[valid_A, valid_B, fake_B, fake_A, reconstr_A, reconstr_B])
self.combined.load_weights('saved_model/model15.h5')
self.g_AB.save_weights("saved_model/g_AB_60.h5")
self.g_BA.save_weights("saved_model/g_BA_60.h5")
self.combined.compile(loss=['mse', 'mse', 'mae', 'mae', 'mae', 'mae'],
optimizer=optimizer)
class DiscoGAN():
def __init__(self):
# Input shape
self.img_rows = 128
self.img_cols = 128
self.channels = 3
self.img_shape = (self.img_rows, self.img_cols, self.channels)
# Configure data loader
self.dataset_name = 'edges2shoes'
self.data_loader = DataLoader(dataset_name=self.dataset_name,
img_res=(self.img_rows, self.img_cols))
# Calculate output shape of D (PatchGAN)
patch = int(self.img_rows / 2**4)
self.disc_patch = (patch, patch, 1)
# Number of filters in the first layer of G and D
self.gf = 64
self.df = 64
optimizer = Adam(0.0002, 0.5)
# Build and compile the discriminators
self.d_A = self.build_discriminator()
self.d_B = self.build_discriminator()
self.d_A.compile(loss='mse', optimizer=optimizer, metrics=['accuracy'])
self.d_B.compile(loss='mse', optimizer=optimizer, metrics=['accuracy'])
# Build and compile the generators
self.g_AB = self.build_generator()
self.g_BA = self.build_generator()
self.g_AB.compile(loss='binary_crossentropy', optimizer=optimizer)
self.g_BA.compile(loss='binary_crossentropy', optimizer=optimizer)
# Input images from both domains
img_A = Input(shape=self.img_shape)
img_B = Input(shape=self.img_shape)
# Translate images to the other domain
fake_B = self.g_AB(img_A)
fake_A = self.g_BA(img_B)
# Translate images back to original domain
reconstr_A = self.g_BA(fake_B)
reconstr_B = self.g_AB(fake_A)
# For the combined model we will only train the generators
self.d_A.trainable = False
self.d_B.trainable = False
# Discriminators determines validity of translated images
valid_A = self.d_A(fake_A)
valid_B = self.d_B(fake_B)
# Objectives
# + Adversarial: Fool domain discriminators
# + Translation: Minimize MAE between e.g. fake B and true B
# + Cycle-consistency: Minimize MAE between reconstructed images and original
self.combined = Model([img_A, img_B], [valid_A, valid_B, \
fake_B, fake_A, \
reconstr_A, reconstr_B])
self.combined.compile(loss=['mse', 'mse', \
'mse', 'mse', \
'mse', 'mse'],
optimizer=optimizer)
def build_generator(self):
"""U-Net Generator"""
def conv2d(layer_input, filters, f_size=4, normalize=True):
"""Layers used during downsampling"""
d = Conv2D(filters, kernel_size=f_size, strides=2,
padding='same')(layer_input)
d = LeakyReLU(alpha=0.2)(d)
if normalize:
d = InstanceNormalization()(d)
return d
def deconv2d(layer_input,
skip_input,
filters,
f_size=4,
dropout_rate=0):
"""Layers used during upsampling"""
u = UpSampling2D(size=2)(layer_input)
u = Conv2D(filters,
kernel_size=f_size,
strides=1,
padding='same',
activation='relu')(u)
if dropout_rate:
u = Dropout(dropout_rate)(u)
u = InstanceNormalization()(u)
u = Concatenate()([u, skip_input])
return u
# Image input
d0 = Input(shape=self.img_shape)
# Downsampling
d1 = conv2d(d0, self.gf, normalize=False)
d2 = conv2d(d1, self.gf * 2)
d3 = conv2d(d2, self.gf * 4)
d4 = conv2d(d3, self.gf * 8)
d5 = conv2d(d4, self.gf * 8)
d6 = conv2d(d5, self.gf * 8)
d7 = conv2d(d6, self.gf * 8)
# Upsampling
u1 = deconv2d(d7, d6, self.gf * 8)
u2 = deconv2d(u1, d5, self.gf * 8)
u3 = deconv2d(u2, d4, self.gf * 8)
u4 = deconv2d(u3, d3, self.gf * 4)
u5 = deconv2d(u4, d2, self.gf * 2)
u6 = deconv2d(u5, d1, self.gf)
u7 = UpSampling2D(size=2)(u6)
output_img = Conv2D(self.channels,
kernel_size=4,
strides=1,
padding='same',
activation='tanh')(u7)
return Model(d0, output_img)
def build_discriminator(self):
def d_layer(layer_input, filters, f_size=4, normalization=True):
"""Discriminator layer"""
d = Conv2D(filters, kernel_size=f_size, strides=2,
padding='same')(layer_input)
d = LeakyReLU(alpha=0.2)(d)
if normalization:
d = InstanceNormalization()(d)
return d
img = Input(shape=self.img_shape)
d1 = d_layer(img, self.df, normalization=False)
d2 = d_layer(d1, self.df * 2)
d3 = d_layer(d2, self.df * 4)
d4 = d_layer(d3, self.df * 8)
validity = Conv2D(1, kernel_size=4, strides=1, padding='same')(d4)
return Model(img, validity)
def train(self, epochs, batch_size=128, save_interval=50):
start_time = datetime.datetime.now()
for epoch in range(epochs):
for batch_i, (imgs_A, imgs_B) in enumerate(
self.data_loader.load_batch(batch_size)):
# ----------------------
# Train Discriminators
# ----------------------
# Translate images to opposite domain
fake_B = self.g_AB.predict(imgs_A)
fake_A = self.g_BA.predict(imgs_B)
valid = np.ones((batch_size, ) + self.disc_patch)
fake = np.zeros((batch_size, ) + self.disc_patch)
# Train the discriminators (original images = real / translated = Fake)
dA_loss_real = self.d_A.train_on_batch(imgs_A, valid)
dA_loss_fake = self.d_A.train_on_batch(fake_A, fake)
dA_loss = 0.5 * np.add(dA_loss_real, dA_loss_fake)
dB_loss_real = self.d_B.train_on_batch(imgs_B, valid)
dB_loss_fake = self.d_B.train_on_batch(fake_B, fake)
dB_loss = 0.5 * np.add(dB_loss_real, dB_loss_fake)
# Total disciminator loss
d_loss = 0.5 * np.add(dA_loss, dB_loss)
# ------------------
# Train Generators
# ------------------
# The generators want the discriminators to label the translated images as real
valid = np.ones((batch_size, ) + self.disc_patch)
# Train the generators
g_loss = self.combined.train_on_batch([imgs_A, imgs_B], [valid, valid, \
imgs_B, imgs_A, \
imgs_A, imgs_B])
elapsed_time = datetime.datetime.now() - start_time
# Plot the progress
print("[%d] [%d/%d] time: %s, [d_loss: %f, g_loss: %f]" %
(epoch, batch_i, self.data_loader.n_batches,
elapsed_time, d_loss[0], g_loss[0]))
# If at save interval => save generated image samples
if batch_i % save_interval == 0:
self.save_imgs(epoch, batch_i)
def save_imgs(self, epoch, batch_i):
os.makedirs('images/%s' % self.dataset_name, exist_ok=True)
r, c = 2, 3
imgs_A, imgs_B = self.data_loader.load_data(batch_size=1,
is_testing=True)
# Translate images to the other domain
fake_B = self.g_AB.predict(imgs_A)
fake_A = self.g_BA.predict(imgs_B)
# Translate back to original domain
reconstr_A = self.g_BA.predict(fake_B)
reconstr_B = self.g_AB.predict(fake_A)
gen_imgs = np.concatenate(
[imgs_A, fake_B, reconstr_A, imgs_B, fake_A, reconstr_B])
# Rescale images 0 - 1
gen_imgs = 0.5 * gen_imgs + 0.5
titles = ['Original', 'Translated', 'Reconstructed']
fig, axs = plt.subplots(r, c)
cnt = 0
for i in range(r):
for j in range(c):
axs[i, j].imshow(gen_imgs[cnt])
axs[i, j].set_title(titles[j])
axs[i, j].axis('off')
cnt += 1
fig.savefig("images/%s/%d_%d.png" %
(self.dataset_name, epoch, batch_i))
plt.close()
break
else:
line += [vocab.itos[index]]
line = ' '.join(line)
lines += [line]
return lines
if __name__ == '__main__':
config = define_argparser()
loader = DataLoader(config.train,
config.valid,
batch_size=config.batch_size,
device=config.gpu_id,
max_length=config.max_length)
model = LM(len(loader.text.vocab),
word_vec_dim=config.word_vec_dim,
hidden_size=config.hidden_size,
n_layers=config.n_layers,
dropout_p=config.dropout,
max_length=config.max_length)
# Let criterion cannot count PAD as right prediction, because PAD is easy to predict.
loss_weight = torch.ones(len(loader.text.vocab))
loss_weight[data_loader.PAD] = 0
criterion = nn.NLLLoss(weight=loss_weight, size_average=False)
print(model)
from tensorflow.keras.optimizers import Adam
from speech_model import ModelSpeech
from speech_model_zoo import SpeechModel251
from data_loader import DataLoader
from speech_features import Spectrogram
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
AUDIO_LENGTH = 1600
AUDIO_FEATURE_LENGTH = 200
CHANNELS = 1
# 默认输出的拼音的表示大小是1428，即1427个拼音+1个空白块
OUTPUT_SIZE = 1428
sm251 = SpeechModel251(input_shape=(AUDIO_LENGTH, AUDIO_FEATURE_LENGTH,
CHANNELS),
output_size=OUTPUT_SIZE)
feat = Spectrogram()
train_data = DataLoader('train')
opt = Adam(lr=0.0001, beta_1=0.9, beta_2=0.999, decay=0.0, epsilon=10e-8)
ms = ModelSpeech(sm251, feat, max_label_length=64)
#ms.load_model('save_models/' + sm251.get_model_name() + '.model.h5')
ms.train_model(optimizer=opt,
data_loader=train_data,
epochs=50,
save_step=1,
batch_size=16,
last_epoch=0)
ms.save_model('save_models/' + sm251.get_model_name())
SAMPLING_SEED = 1234
# Get file paths and model type from arguments
if len(sys.argv) != 6:
print "Usage: evaluate.py FEATURES_FILE MODEL_FOLDER MODEL_TYPE DISTRICTS_FILE RESULT_PATH"
print " where model type one of: %s" % str(MODEL_TYPE_TO_CLASS.keys())
print " and the districts file is a text file with lines \"<lat>, <lon>\"."
sys.exit()
features_file, model_folder, model_type, districts_file, result_path = sys.argv[1:]
model_name = model_folder.split("/")[-1]
ModelClass = MODEL_TYPE_TO_CLASS[model_type]
spark_context, sql_context = create_spark_application("evaluate_%s_regression" % model_type)
data_loader = DataLoader(spark_context, sql_context, features_file)
#in case of linear regression the data_loader needs to do scaling and we use a one-hot-encoding
do_scaling = do_onehot = model_type == "linear"
data_loader.initialize(do_scaling=do_scaling, do_onehot=do_onehot)
results = []
# iterate over subset of districts given by longitude and latitude in districts_file
for district in read_districts_file(districts_file):
print("Evaluating district: %s" % str(district))
lat, lon = district
data = data_loader.train_df if MEASURE_TRAIN_ERROR else data_loader.test_df
if SAMPLING_FRACTION != 1.0:
data = data.sample(False, SAMPLING_FRACTION, SAMPLING_SEED)
class Pix2Pix():
def __init__(self):
# Input shape
self.img_rows = 256
self.img_cols = 256
self.channels = 3
self.img_shape = (self.img_rows, self.img_cols, self.channels)
# Configure data loader
self.dataset_name = 'facades'
self.data_loader = DataLoader(dataset_name=self.dataset_name,
img_res=(self.img_rows, self.img_cols))
# Calculate output shape of D (PatchGAN)
patch = int(self.img_rows / 2**4)
self.disc_patch = (patch, patch, 1)
# Number of filters in the first layer of G and D
self.gf = 64
self.df = 64
optimizer = Adam(0.0002, 0.5)
# Build and compile the discriminator
self.discriminator = self.build_discriminator()
self.discriminator.compile(loss='mse',
optimizer=optimizer,
metrics=['accuracy'])
#-------------------------
# Construct Computational
# Graph of Generator
#-------------------------
# Build the generator
self.generator = self.build_generator()
# Input images and their conditioning images
img_A = Input(shape=self.img_shape)
img_B = Input(shape=self.img_shape)
# By conditioning on B generate a fake version of A
fake_A = self.generator(img_B)
# For the combined model we will only train the generator
self.discriminator.trainable = False
# Discriminators determines validity of translated images / condition pairs
valid = self.discriminator([fake_A, img_B])
self.combined = Model(inputs=[img_A, img_B], outputs=[valid, fake_A])
self.combined.compile(loss=['mse', 'mae'],
loss_weights=[1, 100],
optimizer=optimizer)
def build_generator(self):
"""U-Net Generator"""
def conv2d(layer_input, filters, f_size=4, bn=True):
"""Layers used during downsampling"""
d = Conv2D(filters, kernel_size=f_size, strides=2, padding='same')(layer_input)
d = LeakyReLU(alpha=0.2)(d)
if bn:
d = BatchNormalization(momentum=0.8)(d)
return d
def deconv2d(layer_input, skip_input, filters, f_size=4, dropout_rate=0):
"""Layers used during upsampling"""
u = UpSampling2D(size=2)(layer_input)
u = Conv2D(filters, kernel_size=f_size, strides=1, padding='same', activation='relu')(u)
if dropout_rate:
u = Dropout(dropout_rate)(u)
u = BatchNormalization(momentum=0.8)(u)
u = Concatenate()([u, skip_input])
return u
# Image input
d0 = Input(shape=self.img_shape)
# Downsampling
d1 = conv2d(d0, self.gf, bn=False)
d2 = conv2d(d1, self.gf*2)
d3 = conv2d(d2, self.gf*4)
d4 = conv2d(d3, self.gf*8)
d5 = conv2d(d4, self.gf*8)
d6 = conv2d(d5, self.gf*8)
d7 = conv2d(d6, self.gf*8)
# Upsampling
u1 = deconv2d(d7, d6, self.gf*8)
u2 = deconv2d(u1, d5, self.gf*8)
u3 = deconv2d(u2, d4, self.gf*8)
u4 = deconv2d(u3, d3, self.gf*4)
u5 = deconv2d(u4, d2, self.gf*2)
u6 = deconv2d(u5, d1, self.gf)
u7 = UpSampling2D(size=2)(u6)
output_img = Conv2D(self.channels, kernel_size=4, strides=1, padding='same', activation='tanh')(u7)
return Model(d0, output_img)
def build_discriminator(self):
def d_layer(layer_input, filters, f_size=4, bn=True):
"""Discriminator layer"""
d = Conv2D(filters, kernel_size=f_size, strides=2, padding='same')(layer_input)
d = LeakyReLU(alpha=0.2)(d)
if bn:
d = BatchNormalization(momentum=0.8)(d)
return d
img_A = Input(shape=self.img_shape)
img_B = Input(shape=self.img_shape)
# Concatenate image and conditioning image by channels to produce input
combined_imgs = Concatenate(axis=-1)([img_A, img_B])
d1 = d_layer(combined_imgs, self.df, bn=False)
d2 = d_layer(d1, self.df*2)
d3 = d_layer(d2, self.df*4)
d4 = d_layer(d3, self.df*8)
validity = Conv2D(1, kernel_size=4, strides=1, padding='same')(d4)
return Model([img_A, img_B], validity)
def train(self, epochs, batch_size=1, sample_interval=50):
start_time = datetime.datetime.now()
# Adversarial loss ground truths
valid = np.ones((batch_size,) + self.disc_patch)
fake = np.zeros((batch_size,) + self.disc_patch)
for epoch in range(epochs):
for batch_i, (imgs_A, imgs_B) in enumerate(self.data_loader.load_batch(batch_size)):
# ---------------------
# Train Discriminator
# ---------------------
# Condition on B and generate a translated version
fake_A = self.generator.predict(imgs_B)
# Train the discriminators (original images = real / generated = Fake)
d_loss_real = self.discriminator.train_on_batch([imgs_A, imgs_B], valid)
d_loss_fake = self.discriminator.train_on_batch([fake_A, imgs_B], fake)
d_loss = 0.5 * np.add(d_loss_real, d_loss_fake)
# -----------------
# Train Generator
# -----------------
# Train the generators
g_loss = self.combined.train_on_batch([imgs_A, imgs_B], [valid, imgs_A])
elapsed_time = datetime.datetime.now() - start_time
# Plot the progress
print ("[Epoch %d/%d] [Batch %d/%d] [D loss: %f, acc: %3d%%] [G loss: %f] time: %s" % (epoch, epochs,
batch_i, self.data_loader.n_batches,
d_loss[0], 100*d_loss[1],
g_loss[0],
elapsed_time))
# If at save interval => save generated image samples
if batch_i % sample_interval == 0:
self.sample_images(epoch, batch_i)
def sample_images(self, epoch, batch_i):
os.makedirs('images/%s' % self.dataset_name, exist_ok=True)
r, c = 3, 3
imgs_A, imgs_B = self.data_loader.load_data(batch_size=3, is_testing=True)
fake_A = self.generator.predict(imgs_B)
gen_imgs = np.concatenate([imgs_B, fake_A, imgs_A])
# Rescale images 0 - 1
gen_imgs = 0.5 * gen_imgs + 0.5
titles = ['Condition', 'Generated', 'Original']
fig, axs = plt.subplots(r, c)
cnt = 0
for i in range(r):
for j in range(c):
axs[i,j].imshow(gen_imgs[cnt])
axs[i, j].set_title(titles[i])
axs[i,j].axis('off')
cnt += 1
fig.savefig("images/%s/%d_%d.png" % (self.dataset_name, epoch, batch_i))
plt.close()
def train(self):
X = tf.placeholder(tf.float32, [None, cf.Height, cf.Width, cf.Channel])
Y = tf.placeholder(tf.float32, [None, cf.Class_num])
keep_prob = tf.placeholder(tf.float32)
## Load network model
predictions = Model(x=X)
## Loss, Optimizer
loss = tf.losses.softmax_cross_entropy(Y, predictions)
opt = tf.train.AdamOptimizer(learning_rate=cf.Learning_rate)
train_opt = slim.learning.create_train_op(loss, opt)
## Accuracy
correct_pred = tf.equal(tf.argmax(predictions, 1), tf.argmax(Y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
## Prepare Training data
dl = DataLoader(phase='Train', shuffle=True)
## Prepare Test data
dl_test = DataLoader(phase='Test', shuffle=True)
## Start Train
print('--------\nTraining Start!!')
## Secure GPU Memory
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
sess.run(tf.global_variables_initializer())
fname = os.path.join(cf.Save_dir, 'loss.txt')
f = open(fname, 'w')
f.write("Step,Train_loss,Test_loss" + os.linesep)
saver = tf.train.Saver()
for ite in range(1, cf.Iteration+1):
x, y = dl.get_minibatch(shuffle=True)
trainL, trainA = sess.run([train_opt, accuracy], feed_dict={X:x, Y:y, keep_prob:1.0})
sess.run([train_opt], feed_dict={X:x, Y:y, keep_prob: 0.5})
con = '|'
if ite % cf.Save_train_step != 0:
for i in range(ite % cf.Save_train_step):
con += '>'
for i in range(cf.Save_train_step - ite % cf.Save_train_step):
con += ' '
else:
for i in range(cf.Save_train_step):
con += '>'
con += "| Iteration:{}, TrainL:{:.9f}, TrainA:{:.9f}".format(ite, trainL, trainA)
if ite % cf.Save_train_step == 0 or ite == 1 or ite == cf.Iteration:
saver.save(sess, cf.Save_path)
test_num = dl_test.mb
#accuracy_count = 0.
test_x, test_y = dl_test.get_minibatch(shuffle=True)
"""
for test_ind in range(test_num):
test_x, test_y = dl_test.get_minibatch(shuffle=False)
pred = logits.eval(feed_dict={X: test_x, keep_prob: 1.0})[0]
test_y = test_y[0]
pred_label = np.argmax(pred)
test_label = np.argmax(test_y)
if int(pred_label) == int(test_label):
accuracy_count += 1.
accuracy = accuracy_count / test_num
print('A:{:.9f} '.format(step, accuracy, accuracy_count, test_num))
"""
L, A = sess.run([loss, accuracy],
feed_dict={X: test_x, Y: test_y, keep_prob: 1.0})
con += ' TestL:{:.9f}, TestA:{:.9f}'.format(L, A)
con += '\n'
f.write("{},{},{},{},{}{}".format(ite, trainL, trainA, L, A, os.linesep))
sys.stdout.write("\r"+con)
## Save trained model
saver.save(sess, cf.Save_path)
print('\ntrained model was stored >> ', cf.Save_path)
def main(unused_argv):
assert FLAGS.output_dir, "--output_dir is required"
# Create training directory.
output_dir = FLAGS.output_dir
if not tf.gfile.IsDirectory(output_dir):
tf.logging.info("Creating training directory: %s", output_dir)
tf.gfile.MakeDirs(output_dir)
dl = DataLoader(FLAGS.data_dir)
dl.load_data()
dl.split()
x_dim = dl.get_X_dim()
y_dim = dl.get_Y_dim()
# Build the model.
model = NMC(x_dim, y_dim, cfgs)
if FLAGS.pretrained_fname:
try:
print('Resume from %s' % (FLAGS.pretrained_fname))
model.restore(FLAGS.pretrained_fname)
except:
print('Cannot resume model... Training from scratch')
lr = cfgs.initial_lr
epoch_counter = 0
ite = 0
while True:
start = time.time()
x, y, R, mask, flag = dl.next_batch(cfgs.batch_size_x,
cfgs.batch_size_y, 'train')
if np.sum(mask) == 0:
continue
load_data_time = time.time() - start
if flag:
epoch_counter += 1
# some boolean variables
do_log = (ite % cfgs.log_every_n_steps == 0) or flag
do_snapshot = flag and epoch_counter > 0 and epoch_counter % cfgs.save_every_n_epochs == 0
# train one step
start = time.time()
loss, recons, ite = model.partial_fit(x, y, R, mask, lr)
one_iter_time = time.time() - start
# writing outs
if do_log:
print('Iteration %d, (lr=%f) training loss : %f' %
(ite, lr, loss))
if do_snapshot:
print('Snapshotting')
model.save(FLAGS.output_dir)
if flag:
# decay learning rate during training
if epoch_counter % cfgs.num_epochs_per_decay == 0:
lr = lr * cfgs.lr_decay_factor
print('Decay learning rate to %f' % lr)
if epoch_counter == FLAGS.n_epochs:
if not do_snapshot:
print('Final snapshotting')
model.save(FLAGS.output_dir)
break
class WespeGAN():
def __init__(self, patch_size=(100, 100)):
# Input shape
self.img_rows = patch_size[0]
self.img_cols = patch_size[1]
self.channels = 3
self.img_shape = (self.img_rows, self.img_cols, self.channels)
# Calculate output shape of D (PatchGAN)
patch = int(np.ceil(self.img_shape[0] / 2**4))
self.disc_patch = (patch, patch, 1)
#details for gif creation featuring the progress of the training.
self.gif_batch_size = 5
self.gif_frames_per_sample_interval = 3
self.gif_images = [[] for i in range(self.gif_batch_size)]
#manual logs
#this will be changed using tensorboard
self.log_TrainingPoint = []
self.log_D_colorloss = []
self.log_D_textureloss = []
self.log_G_colorloss = []
self.log_G_textureloss = []
self.log_ReconstructionLoss = []
self.log_TotalVariance = []
self.log_sample_ssim_time_point = []
self.log_sample_ssim = []
self.log_sample_ssim_q = []
# Configure data loader
self.data_loader = DataLoader(img_res=(self.img_rows, self.img_cols))
#set the blurring and texture discriminator settings
self.kernel_size = 23
self.std = 3
self.blur_kernel_weights = gauss_kernel(self.kernel_size, self.std,
self.channels)
self.texture_weights = np.expand_dims(np.expand_dims(np.expand_dims(
np.array([0.2989, 0.5870, 0.1140]), axis=0),
axis=0),
axis=-1)
#print(self.texture_weights.shape)
#set the optimiser
#optimizerG = Adam(0.0001, beta_1=0.5)
optimizerD = Adam(0.0002, beta_1=0.5)
#optimizer = RAdam()
# Build and compile the discriminators
self.D_color = self.discriminator_network(name="Color_Discriminator",
preprocess="blur")
self.D_texture = self.discriminator_network(
name="Texture_Discriminator", preprocess="gray")
self.D_color.compile(loss='mse',
loss_weights=[0.5],
optimizer=optimizerD,
metrics=['accuracy'])
self.D_texture.compile(loss='mse',
loss_weights=[0.5],
optimizer=optimizerD,
metrics=['accuracy'])
#-------------------------
# Construct Computational
# Graph of Generators
#-------------------------
# Build the generators
self.G = self.generator_network(filters=64, name="Forward_Generator_G")
self.F = self.generator_network(filters=64,
name="Backward_Generator_F")
#instantiate the VGG model
self.vgg_model = VGG19(weights='imagenet',
include_top=False,
input_shape=self.img_shape)
self.layer_name = "block2_conv2"
self.inter_VGG_model = Model(inputs=self.vgg_model.input,
outputs=self.vgg_model.get_layer(
self.layer_name).output)
self.inter_VGG_model.trainable = False
# Input images from both domains
img_A = Input(shape=self.img_shape)
#img_A_vgg = vgg_model(img_A)
img_B = Input(shape=self.img_shape)
# Translate images to the other domain
fake_B = self.G(img_A)
#identity_B = self.G(img_B)
fake_A = self.F(img_B)
# Translate images back to original domain
reconstr_A = self.F(fake_B)
reconstr_A_vgg = self.inter_VGG_model(reconstr_A)
reconstr_B = self.G(fake_A)
reconstr_B_vgg = self.inter_VGG_model(reconstr_B)
# For the combined model we will only train the generators
self.D_color.trainable = False
self.D_texture.trainable = False
# Discriminators determines validity of translated images
valid_A_color = self.D_color(fake_B)
valid_A_texture = self.D_texture(fake_B)
# Combined model trains generators to fool discriminators
self.combined = Model(inputs=[img_A, img_B],
outputs=[
valid_A_color, valid_A_texture,
reconstr_A_vgg, reconstr_B_vgg, fake_B
])
#ssim_loss = DSSIMObjective()
self.combined.compile(
loss=['mse', 'mse', 'mae', 'mae', total_variation],
loss_weights=[0.5, 0.5, 1, 1, 0.5],
optimizer=optimizerD)
print(self.combined.summary())
def generator_network(self, filters, name):
def resblock(feature_in, filters, num):
init = RandomNormal(stddev=0.02)
temp = Conv2D(filters, (3, 3),
strides=1,
padding='SAME',
name=('resblock_%d_CONV_1' % num),
kernel_initializer=init)(feature_in)
temp = BatchNormalization(axis=-1)(temp)
temp = LeakyReLU(alpha=0.2)(temp)
temp = Conv2D(filters, (3, 3),
strides=1,
padding='SAME',
name=('resblock_%d_CONV_2' % num),
kernel_initializer=init)(temp)
temp = BatchNormalization(axis=-1)(temp)
temp = LeakyReLU(alpha=0.2)(temp)
return Add()([temp, feature_in])
init = RandomNormal(stddev=0.02)
image = Input(self.img_shape)
x = Lambda(lambda x: 2.0 * x - 1.0, output_shape=lambda x: x)(image)
b1_in = Conv2D(filters, (9, 9),
strides=1,
padding='SAME',
name='CONV_1',
activation='relu',
kernel_initializer=init)(x)
#b1_in = LeakyReLU(alpha=0.2)(b1_in)
b1_in = LeakyReLU(alpha=0.2)(b1_in)
#b1_in = relu()(b1_in)
# residual blocks
b1_out = resblock(b1_in, filters, 1)
b2_out = resblock(b1_out, filters, 2)
b3_out = resblock(b2_out, filters, 3)
b4_out = resblock(b3_out, filters, 4)
# conv. layers after residual blocks
temp = Conv2D(filters, (3, 3),
strides=1,
padding='SAME',
name='CONV_2',
kernel_initializer=init)(b4_out)
#temp = BatchNormalization()(temp)
temp = LeakyReLU(alpha=0.2)(temp)
#temp = LeakyReLU(alpha=0.2)(temp)
temp = Conv2D(filters, (3, 3),
strides=1,
padding='SAME',
name='CONV_3',
kernel_initializer=init)(temp)
#temp = BatchNormalization()(temp)
temp = LeakyReLU(alpha=0.2)(temp)
#temp = LeakyReLU(alpha=0.2)(temp)
temp = Conv2D(filters, (3, 3),
strides=1,
padding='SAME',
name='CONV_4',
kernel_initializer=init)(temp)
#temp = BatchNormalization()(temp)
temp = LeakyReLU(alpha=0.2)(temp)
#temp = LeakyReLU(alpha=0.2)(temp)
temp = Conv2D(3, (9, 9),
strides=1,
padding='SAME',
name='CONV_5',
kernel_initializer=init)(temp)
#temp = Activation('sigmoid')(temp)
#temp = Lambda(lambda x: K.clip(x, 0, 1), output_shape=lambda x:x)(temp)
temp = Activation('tanh')(temp)
temp = Lambda(lambda x: 0.5 * x + 0.5, output_shape=lambda x: x)(temp)
return Model(inputs=image, outputs=temp, name=name)
def discriminator_network(self, name, preprocess='gray'):
#The main modification from the original approach is the use of the InstanceNormalisation layer
image = Input(self.img_shape)
if preprocess == 'gray':
#convert to grayscale image
print("Discriminator-texture")
#output_shape=(image.shape[0], image.shape[1], 1)
gray_layer = Conv2D(1, (1, 1),
strides=1,
padding="SAME",
use_bias=False,
name="Gray_layer")
image_processed = gray_layer(image)
gray_layer.set_weights([self.texture_weights])
gray_layer.trainable = False
#image_processed=Lambda(rgb2gray, output_shape = output_gray_shape)(image)
#print(image_processed.shape)
#image_processed = rgb_to_grayscale(image)
elif preprocess == 'blur':
print("Discriminator-color (blur)")
g_layer = DepthwiseConv2D(self.kernel_size,
use_bias=False,
padding='same')
image_processed = g_layer(image)
g_layer.set_weights([self.blur_kernel_weights])
g_layer.trainable = False
else:
print("Discriminator-color (none)")
image_processed = image
# weight initialization
init = RandomNormal(stddev=0.02)
# source image input
#in_image = Input(shape=image_shape)
# C64
d = Lambda(lambda x: 2.0 * x - 1.0,
output_shape=lambda x: x)(image_processed)
d = Conv2D(64, (5, 5),
strides=(2, 2),
padding='same',
kernel_initializer=init)(d)
d = LeakyReLU(alpha=0.2)(d)
# C128
d = Conv2D(128, (5, 5),
strides=(2, 2),
padding='same',
kernel_initializer=init)(d)
#d = InstanceNormalization(axis=-1)(d)
d = BatchNormalization(axis=-1)(d)
d = LeakyReLU(alpha=0.2)(d)
# C256
d = Conv2D(256, (5, 5),
strides=(2, 2),
padding='same',
kernel_initializer=init)(d)
#d = InstanceNormalization(axis=-1)(d)
d = BatchNormalization(axis=-1)(d)
d = LeakyReLU(alpha=0.2)(d)
# C512
d = Conv2D(512, (3, 3),
strides=(2, 2),
padding='same',
kernel_initializer=init)(d)
#d = InstanceNormalization(axis=-1)(d)
d = BatchNormalization(axis=-1)(d)
d = LeakyReLU(alpha=0.2)(d)
# second last output layer
d = Conv2D(512, (3, 3), padding='same', kernel_initializer=init)(d)
#d = InstanceNormalization(axis=-1)(d)
d = BatchNormalization(axis=-1)(d)
d = LeakyReLU(alpha=0.2)(d)
# patch output
patch_out = Conv2D(1, (3, 3), padding='same',
kernel_initializer=init)(d)
# define model
return Model(inputs=image, outputs=patch_out, name=name)
def logger(self, ):
fig, axs = plt.subplots(2, 2, figsize=(6, 8))
ax = axs[0, 0]
ax.plot(self.log_TrainingPoint, self.log_D_colorloss, label="D_color")
ax.plot(self.log_TrainingPoint,
self.log_D_textureloss,
label="D_texture")
ax.legend()
ax.set_title("Discriminator Adv losses")
ax = axs[0, 1]
ax.plot(self.log_TrainingPoint, self.log_G_colorloss, label="G_color")
ax.plot(self.log_TrainingPoint,
self.log_G_textureloss,
label="G_texture")
ax.legend()
ax.set_title("Generator Adv losses")
ax = axs[1, 0]
ax.plot(self.log_TrainingPoint, self.log_ReconstructionLoss)
ax.set_title("Cycle-Content loss")
ax = axs[1, 1]
ax.plot(self.log_TrainingPoint, self.log_TotalVariance)
ax.set_title("Total Variation loss")
fig.savefig("progress/log.png")
fig, axs = plt.subplots(1, 1)
ax = axs
ax.plot(self.log_sample_ssim_time_point, self.log_sample_ssim)
#ax.plot(self.log_sample_ssim_time_point, self.log_sample_ssim_q, label="Quantised")
#ax.legend()
ax.set_title("sample SSIM value")
fig.savefig("progress/sample_ssim.png")
plt.close('all')
def train(self, epochs, batch_size=1, sample_interval=50):
#every sample_interval batches, the model is saved and sample images are generated and saved
start_time = datetime.datetime.now()
try:
#gif_batch = self.data_loader.load_data(domain="A", batch_size = self.gif_batch_size, is_testing = True)
# Adversarial loss ground truths
#valid = np.ones((batch_size,1))
#fake = np.zeros((batch_size,1))
valid = np.ones((batch_size, ) + self.disc_patch)
fake = np.zeros((batch_size, ) + self.disc_patch)
#instantiate the evaluator
performance_evaluator = evaluator(model=self.G,
img_shape=self.img_shape)
#evaluate the baseline SSIM value (mean SSIM between the phone dataset and canon dataset)
#baseline_SSIM = performance_evaluator.objective_test(baseline=True)
#print("Baseline SSIM value: %05f" % (baseline_SSIM))
for epoch in range(epochs):
for batch_i, (imgs_A, imgs_B) in enumerate(
self.data_loader.load_batch(batch_size)):
# ----------------------
# Train Discriminators
# ----------------------
# Translate images to opposite domain
fake_B = self.G.predict(imgs_A)
# Train the discriminators (original images = real / translated = Fake)
dcolor_loss_real = self.D_color.train_on_batch(
imgs_B, valid)
dcolor_loss_fake = self.D_color.train_on_batch(
fake_B, fake)
dcolor_loss = 0.5 * np.add(dcolor_loss_real,
dcolor_loss_fake)
self.log_D_colorloss.append(dcolor_loss[0])
dtexture_loss_real = self.D_texture.train_on_batch(
imgs_B, valid)
dtexture_loss_fake = self.D_texture.train_on_batch(
fake_B, fake)
dtexture_loss = 0.5 * np.add(dtexture_loss_real,
dtexture_loss_fake)
self.log_D_textureloss.append(dtexture_loss[0])
# Total disciminator loss
d_loss = 0.5 * np.add(dcolor_loss, dtexture_loss)
#d_loss = dcolor_loss
# ------------------
# Train Generators
# ------------------
# Train the generators
imgs_A_vgg = self.inter_VGG_model.predict(imgs_A)
imgs_B_vgg = self.inter_VGG_model.predict(imgs_B)
g_loss = self.combined.train_on_batch(
[imgs_A, imgs_B],
[valid, valid, imgs_A_vgg, imgs_B_vgg, imgs_A])
self.log_G_colorloss.append(g_loss[1])
self.log_G_textureloss.append(g_loss[2])
self.log_ReconstructionLoss.append(np.mean(g_loss[3:5]))
self.log_TotalVariance.append(g_loss[5])
training_time_point = epoch + batch_i / self.data_loader.n_batches
self.log_TrainingPoint.append(
np.around(training_time_point, 3))
elapsed_time = datetime.datetime.now() - start_time
# Plot the progress
print ("[Epoch %d/%d] [Batch %d/%d] [D loss: %f, acc: %3d%%] [G loss: %05f, adv: %05f, recon: %05f, TV: %05f] time: %s " \
% ( epoch, epochs,
batch_i, self.data_loader.n_batches,
d_loss[0], 100*d_loss[-1],
g_loss[0],
np.mean(g_loss[1:3]),
np.mean(g_loss[3:5]),
g_loss[5],
elapsed_time))
# If at save interval => save generated image samples
if batch_i % sample_interval == 0:
"""update the attributes of the performance_evaluator class"""
performance_evaluator.model = self.G
performance_evaluator.epoch = epoch
performance_evaluator.num_batch = batch_i
"""save the model"""
self.G.save("models/{}_{}.h5".format(epoch, batch_i))
print(
"Epoch: {} --- Batch: {} ---- model saved".format(
epoch, batch_i))
"""generation of perceptual results"""
#performance_evaluator.perceptual_test(5)
"""SSIM based evaluation on a batch of test data"""
#calculate mean SSIM on approximately 10% of the test data
#mean_sample_ssim_q = performance_evaluator.objective_test(500, quantiser=True)
mean_sample_ssim = performance_evaluator.objective_test(
500, quantiser=False)
print("Sample mean SSIM ---------%05f--------- " %
(mean_sample_ssim))
log_sample_ssim_time_point = epoch + batch_i / self.data_loader.n_batches
self.log_sample_ssim_time_point.append(
np.around(log_sample_ssim_time_point, 3))
#self.log_sample_ssim_q.append(mean_sample_ssim_q)
self.log_sample_ssim.append(mean_sample_ssim)
"""logger"""
self.logger()
if batch_i % int(self.data_loader.n_batches /
5) == 0 and not (epoch == 0
and batch_i == 0):
"""update the SSIM evolution graph saved in the file progress"""
#update the attributes of the performance_evaluator class
performance_evaluator.model = self.G
performance_evaluator.epoch = epoch
performance_evaluator.num_batch = batch_i
#calculate the mean SSIM on test data
total_mean_ssim = performance_evaluator.objective_test(
2000)
print(
"Mean SSIM (entire test dataset) ---------%05f--------- "
% (total_mean_ssim))
#save the value
performance_evaluator.ssim_vals.append(
np.abs(np.around(total_mean_ssim, decimals=3)))
#save the time point of the training
training_time_point = epoch + batch_i / self.data_loader.n_batches
performance_evaluator.training_points.append(
np.around(training_time_point, 2))
#update the SSIM evolution graph using the new point
fig = plt.figure()
#ax = fig.add_subplot(1, 1, 1)
num_values_saved = len(performance_evaluator.ssim_vals)
plt.plot(np.array(
performance_evaluator.training_points),
np.array(performance_evaluator.ssim_vals),
color='blue',
label="SSIM")
plt.plot(np.array(
performance_evaluator.training_points),
np.ones(num_values_saved) * 0.9,
color='red',
label="target SSIM")
plt.plot(np.array(
performance_evaluator.training_points),
np.ones(num_values_saved) * 0.75054,
color='green',
label="baseline SSIM")
plt.title("mean SSIM vs training epochs")
plt.legend()
fig.savefig("progress/ssim_curve.png")
except KeyboardInterrupt:
print("Training has been interrupted")
def __init__(self, patch_size=(100, 100)):
# Input shape
self.img_rows = patch_size[0]
self.img_cols = patch_size[1]
self.channels = 3
self.img_shape = (self.img_rows, self.img_cols, self.channels)
# Calculate output shape of D (PatchGAN)
patch = int(np.ceil(self.img_shape[0] / 2**4))
self.disc_patch = (patch, patch, 1)
#details for gif creation featuring the progress of the training.
self.gif_batch_size = 5
self.gif_frames_per_sample_interval = 3
self.gif_images = [[] for i in range(self.gif_batch_size)]
#manual logs
#this will be changed using tensorboard
self.log_TrainingPoint = []
self.log_D_colorloss = []
self.log_D_textureloss = []
self.log_G_colorloss = []
self.log_G_textureloss = []
self.log_ReconstructionLoss = []
self.log_TotalVariance = []
self.log_sample_ssim_time_point = []
self.log_sample_ssim = []
self.log_sample_ssim_q = []
# Configure data loader
self.data_loader = DataLoader(img_res=(self.img_rows, self.img_cols))
#set the blurring and texture discriminator settings
self.kernel_size = 23
self.std = 3
self.blur_kernel_weights = gauss_kernel(self.kernel_size, self.std,
self.channels)
self.texture_weights = np.expand_dims(np.expand_dims(np.expand_dims(
np.array([0.2989, 0.5870, 0.1140]), axis=0),
axis=0),
axis=-1)
#print(self.texture_weights.shape)
#set the optimiser
#optimizerG = Adam(0.0001, beta_1=0.5)
optimizerD = Adam(0.0002, beta_1=0.5)
#optimizer = RAdam()
# Build and compile the discriminators
self.D_color = self.discriminator_network(name="Color_Discriminator",
preprocess="blur")
self.D_texture = self.discriminator_network(
name="Texture_Discriminator", preprocess="gray")
self.D_color.compile(loss='mse',
loss_weights=[0.5],
optimizer=optimizerD,
metrics=['accuracy'])
self.D_texture.compile(loss='mse',
loss_weights=[0.5],
optimizer=optimizerD,
metrics=['accuracy'])
#-------------------------
# Construct Computational
# Graph of Generators
#-------------------------
# Build the generators
self.G = self.generator_network(filters=64, name="Forward_Generator_G")
self.F = self.generator_network(filters=64,
name="Backward_Generator_F")
#instantiate the VGG model
self.vgg_model = VGG19(weights='imagenet',
include_top=False,
input_shape=self.img_shape)
self.layer_name = "block2_conv2"
self.inter_VGG_model = Model(inputs=self.vgg_model.input,
outputs=self.vgg_model.get_layer(
self.layer_name).output)
self.inter_VGG_model.trainable = False
# Input images from both domains
img_A = Input(shape=self.img_shape)
#img_A_vgg = vgg_model(img_A)
img_B = Input(shape=self.img_shape)
# Translate images to the other domain
fake_B = self.G(img_A)
#identity_B = self.G(img_B)
fake_A = self.F(img_B)
# Translate images back to original domain
reconstr_A = self.F(fake_B)
reconstr_A_vgg = self.inter_VGG_model(reconstr_A)
reconstr_B = self.G(fake_A)
reconstr_B_vgg = self.inter_VGG_model(reconstr_B)
# For the combined model we will only train the generators
self.D_color.trainable = False
self.D_texture.trainable = False
# Discriminators determines validity of translated images
valid_A_color = self.D_color(fake_B)
valid_A_texture = self.D_texture(fake_B)
# Combined model trains generators to fool discriminators
self.combined = Model(inputs=[img_A, img_B],
outputs=[
valid_A_color, valid_A_texture,
reconstr_A_vgg, reconstr_B_vgg, fake_B
])
#ssim_loss = DSSIMObjective()
self.combined.compile(
loss=['mse', 'mse', 'mae', 'mae', total_variation],
loss_weights=[0.5, 0.5, 1, 1, 0.5],
optimizer=optimizerD)
print(self.combined.summary())
true_labels=test_labels)[1]
full_dataset.append((test_attrs, test_labels))
avg_accuracy += accuracy
self.train_accuracy = avg_accuracy / k
return self.train_accuracy
def summary(self):
print('=====Model Summary=====')
print('Class 1 label = 5.0')
print('Class 2 label = 6.0')
print('Pi =', self.pi1)
print('\nMu1 of size', end=' ')
print(self.mu1.shape, ':')
pprint.pprint(self.mu1)
print('\nMu2 of size', end=' ')
print(self.mu2.shape, ':')
pprint.pprint(self.mu2)
print('\nSigma of size', end=' ')
print(self.sigma.shape, ':')
for i in range(self.sigma.shape[0]):
pprint.pprint(self.sigma[i])
if self.train_accuracy:
print('\nTraining Accuracy = %.3f %%' % self.train_accuracy)
if __name__ == '__main__':
full_dataset = DataLoader.load_full_dataset('./dataset')
model = MOG(M=64)
model.learn(full_dataset, report_acc=True)
model.summary()
# dataX_hat=np.array([[[1,1,1,1,1],[1,1,1,1,1],[1,1,1,1,1]],[[1,1,1,1,1],[1,1,1,1,1],[1,1,1,1,1]]])
dataX_hat = np.array([[[1, 1, 0, 1, 0], [1, 1, 0, 1, 1], [0, 0, 0, 1, 0]],
[[1, 1, 0, 1, 1], [0, 0, 0, 0, 0], [1, 1, 0, 1, 1]],
[[1, 1, 1, 1, 1], [0, 0, 0, 0, 0], [1, 0, 0, 1, 1]]])
# dataX=np.array([[[0,0,0,0,0],[0,0,0,0,0],[0,0,0,0,0]],[[0,0,0,0,0],[0,0,0,0,0],[0,0,0,0,0]]])
dataX = np.array([[[1, 1, 0, 1, 0], [1, 1, 1, 1, 1], [0, 0, 0, 1, 0]],
[[1, 1, 1, 1, 1], [0, 0, 0, 0, 0], [1, 1, 0, 1, 1]],
[[1, 1, 1, 1, 1], [0, 0, 0, 0, 0], [1, 0, 0, 1, 1]]])
disc_score = discriminative_score_metrics(dataX, dataX_hat)
print(disc_score)
# load data once more
from data_loader import DataLoader
dl = DataLoader(csv_filename='raw_data.csv')
dl.del_position_key()
dl.output_csv()
# load data once more
from data_loader import DataLoader
dl = DataLoader()
robot_keys_mexico = dl.get_robot_keys_at_location()
robot_key = robot_keys_mexico[0]
dl.load_raw_data(robot_key)
dl.output_csv("mexico/key_1.csv")
# for i in range(1,5):
# robot_key = robot_keys_mexico[i]
# dl.load_raw_data(robot_key)
# dl.output_csv('mexico/key_'+str(robot_key)+'.csv')
def eval(opts):
"""
Evaluating function - evaluates on validation and test splits
"""
# Load the previous state
loaded_state = torch.load(opts.model_path)
opts.h5_path = loaded_state['opts'].h5_path
opts.dataset = loaded_state['opts'].dataset
opts.combineDropout = loaded_state['opts'].combineDropout
opts.featDropout = loaded_state['opts'].featDropout
opts.addExtraLinearFuse = loaded_state['opts'].addExtraLinearFuse
opts.T = loaded_state['opts'].T
opts.M = loaded_state['opts'].M
opts.N = loaded_state['opts'].N
opts.F = loaded_state['opts'].F
opts.dataset = loaded_state['opts'].dataset
if opts.delta_M == -1:
if hasattr(loaded_state['opts'], 'delta_M'):
opts.delta_M = loaded_state['opts'].delta_M
opts.delta_N = loaded_state['opts'].delta_N
else:
opts.delta_M = 5
opts.delta_N = 5
opts.rnn_type = loaded_state['opts'].rnn_type
opts.actOnElev = loaded_state['opts'].actOnElev
opts.actOnAzim = loaded_state['opts'].actOnAzim
opts.actOnTime = loaded_state['opts'].actOnTime
if opts.actorType == 'unset':
opts.actorType = loaded_state['opts'].actorType
opts.num_classes = loaded_state['opts'].num_classes
opts.wrap_azimuth = loaded_state['opts'].wrap_azimuth
opts.act_to_delta = loaded_state['opts'].act_to_delta
opts.delta_to_act = loaded_state['opts'].delta_to_act
opts.baselineType = loaded_state['opts'].baselineType
opts.wrap_elevation = loaded_state['opts'].wrap_elevation
opts.act_to_delta = loaded_state['opts'].act_to_delta
opts.delta_to_act = loaded_state['opts'].delta_to_act
opts.optimizer_type = loaded_state['opts'].optimizer_type
if hasattr(loaded_state['opts'], 'normalize_hidden'):
opts.normalize_hidden = loaded_state['opts'].normalize_hidden
else:
opts.normalize_hidden = True
if hasattr(loaded_state['opts'], 'nonlinearity'):
opts.nonlinearity = loaded_state['opts'].nonlinearity
else:
opts.nonlinearity = 'tanh'
opts.init = 'xavier'
opts.shuffle = 'False'
opts.seed = 123
opts.reward_scale = 1.0
opts.reward_scale_expert = 1e-2
opts.rewards_greedy = False
opts.A = opts.delta_M * opts.delta_N
# Set random seeds
set_random_seeds(opts.seed)
# Data loading
if opts.actorType == 'saved_trajectories' or opts.actorType == 'peek_saliency':
from data_loader import DataLoaderExpertPolicy as DataLoader
else:
from data_loader import DataLoaderSimple as DataLoader
loader = DataLoader(opts)
opts.frequent_class = loader.get_most_frequent_class()
# Create the agent
agent = Agent(opts, mode='eval')
agent.policy.load_state_dict(loaded_state['policy_state_dict'], strict=False)
for t in range(opts.T):
agent.classifiers[t].load_state_dict(loaded_state['classifier_state_dict'][t])
# Set networks to evaluate
agent.policy.eval()
for t in range(opts.T):
agent.classifiers[t].eval()
if opts.eval_val:
if opts.compute_all_times:
val_accuracy, val_accuracy_all_times, _, _ = evaluate(loader, agent, 'val', opts)
print('====> Validation accuracy (vs) time')
print(','.join(['{:.3f}'.format(_x * 100) for _x in val_accuracy_all_times]))
else:
val_accuracy, _, _ = evaluate(loader, agent, 'val', opts)
print('Validation accuracy: %.3f'%(val_accuracy*100))
else:
if opts.compute_all_times:
test_accuracy, test_accuracy_all_times, _, _ = evaluate(loader, agent, 'test', opts)
print('====> Testing accuracy (vs) time')
print(','.join(['{:.3f}'.format(_x * 100) for _x in test_accuracy_all_times]))
else:
test_accuracy, _, _ = evaluate(loader, agent, 'test', opts)
print('Testing accuracy: %.3f'%(test_accuracy*100))
def __init__(self):
# Input shape
self.img_rows = 128
self.img_cols = 128
self.channels = 3
self.img_shape = (self.img_rows, self.img_cols, self.channels)
# Configure data loader
self.dataset_name = 'apple2orange'
self.data_loader = DataLoader(dataset_name=self.dataset_name,
img_res=(self.img_rows, self.img_cols))
# Calculate output shape of D (PatchGAN)
patch = int(self.img_rows / 2**4)
self.disc_patch = (patch, patch, 1)
# Number of filters in the first layer of G and D
self.gf = 32
self.df = 64
# Loss weights
self.lambda_cycle = 10.0 # Cycle-consistency loss
self.lambda_id = 0.1 * self.lambda_cycle # Identity loss
optimizer = Adam(0.0002, 0.5)
# Build and compile the discriminators
self.d_A = self.build_discriminator()
self.d_B = self.build_discriminator()
self.d_A.compile(loss='mse',
optimizer=optimizer,
metrics=['accuracy'])
self.d_B.compile(loss='mse',
optimizer=optimizer,
metrics=['accuracy'])
#-------------------------
# Construct Computational
# Graph of Generators
#-------------------------
# Build the generators
self.g_AB = self.build_generator()
self.g_BA = self.build_generator()
# Input images from both domains
img_A = Input(shape=self.img_shape)
img_B = Input(shape=self.img_shape)
# Translate images to the other domain
fake_B = self.g_AB(img_A)
fake_A = self.g_BA(img_B)
# Translate images back to original domain
reconstr_A = self.g_BA(fake_B)
reconstr_B = self.g_AB(fake_A)
# Identity mapping of images
img_A_id = self.g_BA(img_A)
img_B_id = self.g_AB(img_B)
# For the combined model we will only train the generators
self.d_A.trainable = False
self.d_B.trainable = False
# Discriminators determines validity of translated images
valid_A = self.d_A(fake_A)
valid_B = self.d_B(fake_B)
# Combined model trains generators to fool discriminators
self.combined = Model(inputs=[img_A, img_B],
outputs=[ valid_A, valid_B,
reconstr_A, reconstr_B,
img_A_id, img_B_id ])
self.combined.compile(loss=['mse', 'mse',
'mae', 'mae',
'mae', 'mae'],
loss_weights=[ 1, 1,
self.lambda_cycle, self.lambda_cycle,
self.lambda_id, self.lambda_id ],
optimizer=optimizer)
def train(data_root_dir, context_files_root_dir, csv_files_root_dir, dump_model_para_root_dir):
"""
Train phase main process
:return:
"""
args = set_train_args()
# remove older summary log and model parameters
for folder in (args.train_summary, args.val_summary, dump_model_para_root_dir):
if os.path.exists(folder):
shutil.rmtree(folder)
dataloader = DataLoader(
data_root_dir=data_root_dir,
train_batch_size=args.batch_size,
val_batch_size=args.batch_size,
seq_length=args.seq_length,
frequency=args.frequency,
discrete_timestamps=args.discrete_timestamps,
patch_size=args.patch_size,
context_files_root_dir=context_files_root_dir,
csv_files_root_dir=csv_files_root_dir
)
print '\nDatasets loaded! \nReady to train.'
# create model
vrumodel = VRU2Model(args)
print '\n\nModel initialized successfully.'
# to save log
train_error = np.zeros(args.num_epochs)
valid_error = np.zeros(args.num_epochs)
step = 0
# configure GPU training, soft allocation.
gpuConfig = tf.ConfigProto(allow_soft_placement=True)
gpuConfig.gpu_options.allow_growth = True
with tf.Session(config=gpuConfig) as sess:
train_writer = tf.summary.FileWriter(args.train_summary, sess.graph)
val_writer = tf.summary.FileWriter(args.val_summary, sess.graph)
# initialize all variables in the graph
tf.global_variables_initializer().run()
# initialize a saver that saves all the variables in the graph
saver = tf.train.Saver(max_to_keep=None)
print '\n\nStart Training... \n'
for e in range(args.num_epochs):
dataloader.reset_train_pointer()
dataloader.reset_val_pointer()
# --- TRAIN ---
for batch in range(dataloader.train_batch_num):
# context data shape: (batch_size*seq_length, patch_size**2)
# input batch shape: (batch_size, seq_length, feature_size)
# target batch shape: (batch_size, discrete_steps, feature_size)
input_batch, target_batch, context_patch, __ = dataloader.next_batch(mode='train')
# reshape to 4D tensor [batch_size*seq_length, patch_size, patch_size, in_channel]
context_data_feed = context_patch.reshape((-1, args.patch_size, args.patch_size, 1))
complete_input, batch_error_train, summary, __ = sess.run(
fetches=[
vrumodel.complete_input,
vrumodel.loss,
vrumodel.summary_op,
vrumodel.train_op,
],
feed_dict={
vrumodel.input_data: input_batch,
vrumodel.target_data: target_batch,
vrumodel.context_input: context_data_feed
})
# add summary and accumulate stats
train_writer.add_summary(summary, step)
train_error[e] += batch_error_train
step += 1
# normalise running means by number of batches
train_error[e] /= dataloader.train_batch_num
# --- VALIDATION ---
for batch in range(dataloader.val_batch_num):
input_batch, target_batch, context_patch, __ = dataloader.next_batch(mode='val')
context_patch = context_patch.reshape((-1, args.patch_size, args.patch_size, 1))
summary, predicted_outputs, batch_error_val = sess.run(
fetches=[vrumodel.summary_op, vrumodel.predicted_outputs, vrumodel.loss],
feed_dict={
vrumodel.input_data: input_batch,
vrumodel.target_data: target_batch,
vrumodel.context_input: context_patch
})
val_writer.add_summary(summary, step)
valid_error[e] += batch_error_val
valid_error[e] /= dataloader.val_batch_num
# checkpoint model variable
if (e + 1) % args.save_every_epoch == 0:
model_name = 'epoch{}_{:2f}' \
'_{:2f}.ckpt'.format(e + 1, train_error[e], valid_error[e])
dump_model_full_path = os.path.join(dump_model_para_root_dir, model_name)
# save model
saver.save(sess=sess, save_path=dump_model_full_path)
tf.add_to_collection("predict", vrumodel.predicted_outputs)
print('Epoch {0:02d}: err(train)={1:.2f}, err(valid)={2:.2f}' .format(e + 1, train_error[e], valid_error[e]))
# close writer and session objects
train_writer.close()
val_writer.close()
sess.close()
return train_error, valid_error
class StockOptimizer:
def __init__(
self,
number_of_tickers,
models_and_parameters,
overfitting_threshold,
number_of_past_points):
self.number_of_tickers = number_of_tickers
self.data_loader = DataLoader()
self.choose_ticker = ChooseTicker()
self.graph_maker = CreateGraphs()
self.technical_analysis = TechnicalIndicators()
self.models_and_parameters = models_and_parameters
self.overfitting_threshold = overfitting_threshold
self.number_of_past_points = number_of_past_points
def run(self):
logger.info('Select tickers')
tickers = self.choose_ticker.random_tickers(self.number_of_tickers)
logger.info(f'The selected tickers are {tickers}')
for ticker in tickers:
# for ticker in ['MCHP']:
logger.info(f'Starting with ticker {ticker}')
data = self.data_loader.load(ticker)
self.graph_maker.plot_adjusted_prices(ticker, data)
data = self.technical_analysis.calculate(data)
logger.info(f'Technical analysis graphs')
train, test, validation, train_graph, test_graph, validation_graph = self.data_loader.transform(
data, number_of_past_points=self.number_of_past_points)
self.graph_maker.plot_train_test_val(
ticker, train_graph, test_graph, validation_graph)
for position, model_name in enumerate(
[element.get('model') for element in self.models_and_parameters]):
if model_name == 'feed_forward_neural_net':
model = FeedForwardNN(
dimension_of_first_layer=self.number_of_past_points * train[0][0].shape[1],
ticker=ticker,
overfitting_threshold=self.overfitting_threshold,
)
if model_name == 'random_forest':
model = RandomForest(
ticker=ticker, overfitting_threshold=self.overfitting_threshold)
if model_name == 'xgboost':
model = XGBoost(
ticker=ticker,
overfitting_threshold=self.overfitting_threshold)
if model_name == 'Arima':
model = Arima(
ticker=ticker,
overfitting_threshold=self.overfitting_threshold)
if model_name == 'regressors':
model = Regressors(
ticker=ticker,
overfitting_threshold=self.overfitting_threshold)
best_parameters, mse, trend_ratio, prediction, true_values, there_is_a_best_prediction = model.run(
train=train[::-1], val=validation[::-1], test=test[::-1], model_parameters=self.models_and_parameters[position])
logger.info(
f'The best scenario for a {model_name} is {best_parameters}, mse: {mse},'
f' ratio of trend {trend_ratio*100}')
if there_is_a_best_prediction:
self.graph_maker.plot_test_results(true_values, prediction, ticker, mse, model_name)
else:
logger.info(
f'No model could be fitted for {model_name} due to the '
f'overfitting threshold of {self.overfitting_threshold}')
# Set the random seed for reproducible experiments
random.seed(args.seed)
torch.manual_seed(args.seed)
if params.n_gpu > 0:
torch.cuda.manual_seed_all(args.seed) # set random seed for all GPUs
params.seed = args.seed
# Set the logger
utils.set_logger(os.path.join(args.model_dir, 'train.log'))
logging.info("device: {}, n_gpu: {}, 16-bits training: {}".format(params.device, params.n_gpu, args.fp16))
# Create the input data pipeline
logging.info("Loading the datasets...")
# Initialize the DataLoader
data_loader = DataLoader(args.data_dir, args.bert_model_dir, params, token_pad_idx=0)
# Load training data and test data
train_data = data_loader.load_data('train')
val_data = data_loader.load_data('val')
# Specify the training and validation dataset sizes
params.train_size = train_data['size']
params.val_size = val_data['size']
# Prepare model
model = BertForTokenClassification.from_pretrained(args.bert_model_dir, num_labels=len(params.tag2idx))
model.to(params.device)
if args.fp16:
model.half()
class DiscoGAN():
def __init__(self):
# Input shape
self.img_rows = 128
self.img_cols = 128
self.channels = 3
self.img_shape = (self.img_rows, self.img_cols, self.channels)
# Configure data loader
self.dataset_name = 'edges2shoes'
self.data_loader = DataLoader(dataset_name=self.dataset_name,
img_res=(self.img_rows, self.img_cols))
# Calculate output shape of D (PatchGAN)
patch = int(self.img_rows / 2**4)
self.disc_patch = (patch, patch, 1)
# Number of filters in the first layer of G and D
self.gf = 64
self.df = 64
optimizer = Adam(0.0002, 0.5)
# Build and compile the discriminators
self.d_A = self.build_discriminator()
self.d_B = self.build_discriminator()
self.d_A.compile(loss='mse',
optimizer=optimizer,
metrics=['accuracy'])
self.d_B.compile(loss='mse',
optimizer=optimizer,
metrics=['accuracy'])
#-------------------------
# Construct Computational
# Graph of Generators
#-------------------------
# Build the generators
self.g_AB = self.build_generator()
self.g_BA = self.build_generator()
# Input images from both domains
img_A = Input(shape=self.img_shape)
img_B = Input(shape=self.img_shape)
# Translate images to the other domain
fake_B = self.g_AB(img_A)
fake_A = self.g_BA(img_B)
# Translate images back to original domain
reconstr_A = self.g_BA(fake_B)
reconstr_B = self.g_AB(fake_A)
# For the combined model we will only train the generators
self.d_A.trainable = False
self.d_B.trainable = False
# Discriminators determines validity of translated images
valid_A = self.d_A(fake_A)
valid_B = self.d_B(fake_B)
# Objectives
# + Adversarial: Fool domain discriminators
# + Translation: Minimize MAE between e.g. fake B and true B
# + Cycle-consistency: Minimize MAE between reconstructed images and original
self.combined = Model(inputs=[img_A, img_B],
outputs=[ valid_A, valid_B,
fake_B, fake_A,
reconstr_A, reconstr_B ])
self.combined.compile(loss=['mse', 'mse',
'mae', 'mae',
'mae', 'mae'],
optimizer=optimizer)
def build_generator(self):
"""U-Net Generator"""
def conv2d(layer_input, filters, f_size=4, normalize=True):
"""Layers used during downsampling"""
d = Conv2D(filters, kernel_size=f_size, strides=2, padding='same')(layer_input)
d = LeakyReLU(alpha=0.2)(d)
if normalize:
d = InstanceNormalization()(d)
return d
def deconv2d(layer_input, skip_input, filters, f_size=4, dropout_rate=0):
"""Layers used during upsampling"""
u = UpSampling2D(size=2)(layer_input)
u = Conv2D(filters, kernel_size=f_size, strides=1, padding='same', activation='relu')(u)
if dropout_rate:
u = Dropout(dropout_rate)(u)
u = InstanceNormalization()(u)
u = Concatenate()([u, skip_input])
return u
# Image input
d0 = Input(shape=self.img_shape)
# Downsampling
d1 = conv2d(d0, self.gf, normalize=False)
d2 = conv2d(d1, self.gf*2)
d3 = conv2d(d2, self.gf*4)
d4 = conv2d(d3, self.gf*8)
d5 = conv2d(d4, self.gf*8)
d6 = conv2d(d5, self.gf*8)
d7 = conv2d(d6, self.gf*8)
# Upsampling
u1 = deconv2d(d7, d6, self.gf*8)
u2 = deconv2d(u1, d5, self.gf*8)
u3 = deconv2d(u2, d4, self.gf*8)
u4 = deconv2d(u3, d3, self.gf*4)
u5 = deconv2d(u4, d2, self.gf*2)
u6 = deconv2d(u5, d1, self.gf)
u7 = UpSampling2D(size=2)(u6)
output_img = Conv2D(self.channels, kernel_size=4, strides=1,
padding='same', activation='tanh')(u7)
return Model(d0, output_img)
def build_discriminator(self):
def d_layer(layer_input, filters, f_size=4, normalization=True):
"""Discriminator layer"""
d = Conv2D(filters, kernel_size=f_size, strides=2, padding='same')(layer_input)
d = LeakyReLU(alpha=0.2)(d)
if normalization:
d = InstanceNormalization()(d)
return d
img = Input(shape=self.img_shape)
d1 = d_layer(img, self.df, normalization=False)
d2 = d_layer(d1, self.df*2)
d3 = d_layer(d2, self.df*4)
d4 = d_layer(d3, self.df*8)
validity = Conv2D(1, kernel_size=4, strides=1, padding='same')(d4)
return Model(img, validity)
def train(self, epochs, batch_size=128, sample_interval=50):
start_time = datetime.datetime.now()
# Adversarial loss ground truths
valid = np.ones((batch_size,) + self.disc_patch)
fake = np.zeros((batch_size,) + self.disc_patch)
for epoch in range(epochs):
for batch_i, (imgs_A, imgs_B) in enumerate(self.data_loader.load_batch(batch_size)):
# ----------------------
# Train Discriminators
# ----------------------
# Translate images to opposite domain
fake_B = self.g_AB.predict(imgs_A)
fake_A = self.g_BA.predict(imgs_B)
# Train the discriminators (original images = real / translated = Fake)
dA_loss_real = self.d_A.train_on_batch(imgs_A, valid)
dA_loss_fake = self.d_A.train_on_batch(fake_A, fake)
dA_loss = 0.5 * np.add(dA_loss_real, dA_loss_fake)
dB_loss_real = self.d_B.train_on_batch(imgs_B, valid)
dB_loss_fake = self.d_B.train_on_batch(fake_B, fake)
dB_loss = 0.5 * np.add(dB_loss_real, dB_loss_fake)
# Total disciminator loss
d_loss = 0.5 * np.add(dA_loss, dB_loss)
# ------------------
# Train Generators
# ------------------
# Train the generators
g_loss = self.combined.train_on_batch([imgs_A, imgs_B], [valid, valid, \
imgs_B, imgs_A, \
imgs_A, imgs_B])
elapsed_time = datetime.datetime.now() - start_time
# Plot the progress
print ("[%d] [%d/%d] time: %s, [d_loss: %f, g_loss: %f]" % (epoch, batch_i,
self.data_loader.n_batches,
elapsed_time,
d_loss[0], g_loss[0]))
# If at save interval => save generated image samples
if batch_i % sample_interval == 0:
self.sample_images(epoch, batch_i)
def sample_images(self, epoch, batch_i):
os.makedirs('images/%s' % self.dataset_name, exist_ok=True)
r, c = 2, 3
imgs_A, imgs_B = self.data_loader.load_data(batch_size=1, is_testing=True)
# Translate images to the other domain
fake_B = self.g_AB.predict(imgs_A)
fake_A = self.g_BA.predict(imgs_B)
# Translate back to original domain
reconstr_A = self.g_BA.predict(fake_B)
reconstr_B = self.g_AB.predict(fake_A)
gen_imgs = np.concatenate([imgs_A, fake_B, reconstr_A, imgs_B, fake_A, reconstr_B])
# Rescale images 0 - 1
gen_imgs = 0.5 * gen_imgs + 0.5
titles = ['Original', 'Translated', 'Reconstructed']
fig, axs = plt.subplots(r, c)
cnt = 0
for i in range(r):
for j in range(c):
axs[i,j].imshow(gen_imgs[cnt])
axs[i, j].set_title(titles[j])
axs[i,j].axis('off')
cnt += 1
fig.savefig("images/%s/%d_%d.png" % (self.dataset_name, epoch, batch_i))
plt.close()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-data', required=True, help='Path to preprocessed data')
parser.add_argument('-name', required=True, help='Name of experiment')
parser.add_argument('-epoch', type=int, default=10)
parser.add_argument('-batch_size', type=int, default=512)
parser.add_argument('-dropout', type=float, default=0.3)
parser.add_argument('-log', action='store_true')
parser.add_argument('-save_model', action='store_true')
parser.add_argument('-no_cuda', action='store_true')
parser.add_argument('-one_fold', action='store_true', help='Train single fold')
opt = parser.parse_args()
opt.cuda = not opt.no_cuda
# ========= Loading Dataset =========#
data = torch.load(opt.data)
embeds = data['embeds']
opt.max_token_seq_len = data['settings'].max_token_seq_len
opt.src_vocab_size = len(data['dict'])
print(opt)
directory = 'predictions/'+opt.name
if not os.path.exists(directory):
os.makedirs(directory)
if opt.log:
log_train_file = directory + '/train.log'
log_valid_file = directory + '/valid.log'
print('[Info] Training performance will be written to file: {} and {}'.format(
log_train_file, log_valid_file))
with open(log_train_file, 'w') as log_tf, open(log_valid_file, 'w') as log_vf:
log_tf.write('fold,epoch,loss,auc\n')
log_vf.write('fold,epoch,loss,auc\n')
crit = nn.BCEWithLogitsLoss()
if opt.cuda:
crit = crit.cuda()
# =======================================#
n_train = len(data['train']['src'])
idx = list(range(n_train))
kf = KFold(n_splits=10, shuffle=True, random_state=42)
test_data = DataLoader(
data['dict'],
src=np.array(data['test']['src']),
batch_size=opt.batch_size,
shuffle=False,
test=True,
cuda=opt.cuda)
out = {}
i = 0
# ========= Preparing CrossVal =========#
for idx_train, idx_val in kf.split(idx):
#model = GMP(opt.src_vocab_size, embeds=embeds, dropout=opt.dropout)
model = GRUCnn(opt.src_vocab_size, embeds=embeds, dropout=opt.dropout)
if opt.cuda:
model = model.cuda()
optimizer = optim.Adam(model.get_trainable_parameters(), lr=0.001,
betas=(0.9, 0.98), eps=1e-09)
scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.2, patience=2, verbose=1)
# ========= DataLoaders =========#
training_data = DataLoader(
data['dict'],
src=np.array(data['train']['src'])[idx_train],
tgt=np.array(data['train']['tgt'])[idx_train],
batch_size=opt.batch_size,
cuda=opt.cuda)
validation_data = DataLoader(
data['dict'],
src=np.array(data['train']['src'])[idx_val],
tgt=np.array(data['train']['tgt'])[idx_val],
batch_size=opt.batch_size,
shuffle=False,
test=True,
cuda=opt.cuda)
fold_name = 'fold_'+str(i)
model_ft, val_proba = train(fold_name, model, training_data,
validation_data, crit, optimizer, scheduler, opt)
out[i] = {
'idx': idx_val,
'proba': val_proba
}
df_out = create_submit_df(model_ft, dataloader=test_data)
df_out.to_csv(directory+'/'+fold_name+'_test.csv', index=False)
if opt.one_fold:
break
i += 1
torch.save(out, directory+'/cv.prob')
from keras.callbacks import CSVLogger, ModelCheckpoint
from data_loader import DataLoader
from models import simple_CNN
from utils import preprocess_input
# parameters
batch_size = 128
num_epochs = 1000
training_split = .8
dataset_name = 'fer2013'
log_file_path = 'log_files/emotion_training.log'
trained_models_path = '../trained_models/emotion_models/simple_CNN'
# data loader
data_loader = DataLoader(dataset_name)
faces, emotions = data_loader.get_data()
print(len(faces))
faces = preprocess_input(faces)
num_classes = emotions.shape[1]
input_shape = faces.shape[1:]
# model parameters/compilation
model = simple_CNN(input_shape, num_classes)
model.compile(optimizer='adam', loss='categorical_crossentropy',
metrics=['accuracy'])
model.summary()
# model callbacks
csv_logger = CSVLogger(log_file_path, append=False)
model_names = trained_models_path + '.{epoch:02d}-{val_acc:.2f}.hdf5'
from torch.autograd import Variable
from unet import Unet
from resnetlike128 import ResnetLike128
import visdom #python -m visdom.server
viz = visdom.Visdom()
iterace = 9999999
init_lr = 0.0001
batch = 16
path_to_data = '../data_patch1'
lam = 10
if __name__ == '__main__':
loader = DataLoader(split='train', path_to_data=path_to_data, paired=False)
trainloader = data.DataLoader(loader,
batch_size=batch,
num_workers=4,
shuffle=True,
drop_last=True,
pin_memory=False)
loader = DataLoader(split='test', path_to_data=path_to_data, paired=True)
testloader = data.DataLoader(loader,
batch_size=batch,
num_workers=4,
shuffle=False,
drop_last=False,
pin_memory=False)
import os
import numpy as np
import pandas as pd
from data_loader import DataLoader
data_path = '../data/'
data_loader = DataLoader(data_path=os.path.join(data_path, 'train'),
common_path=os.path.join(data_path, 'volume'),
task_path=os.path.join(data_path, 'volume',
'local_test'),
measurement_normalize='mean',
pytest=True)
class Test_DataLoader():
def test_read_csv(self):
data_loader.cohort_df = data_loader.extract_outcome_cohort()
data_loader.person_df = data_loader.extract_person()
data_loader.condition_df = data_loader.extract_condition()
data_loader.measurement_df = data_loader.extract_measurement()
def test_group_hour(self):
data_loader.groupby_hour()
def test_make_data(self):
data_loader.make_person_sequence()
data_loader.make_data()
if data_loader.measurement_normalize == 'mean':
assert data_loader.x[0][0].shape == (84, )
else:
assert data_loader.x[0][0].shape == (98, )
@author: david.saltiel
"""
from data_loader import DataLoader
from OCA import OCAMethod
from RFE import RFEMethod
from BCA import BCAMethod
from graphics import create_graphic_for_method, create_figure2, create_figure3
''' This is the main function
It calls all the 3 features selection
method and compares them
'''
#%% reads the data
data = DataLoader(dataset_name = 'trade_selection_A229', extension = 'csv')
data.clean_data()
#%% OCA method
oca_method = OCAMethod(data.df, n_min=10, verbose = True,reload_feature_importance = False)
oca_method.select_features()
#%% RFE method
rfe_method = RFEMethod(data.df, verbose = True)
rfe_method.select_features()
rfe_dictionary = rfe_method.save_all_score()
#%% BCA method
bca_method = BCAMethod(data.df, verbose = True)
bca_method.select_features()
torch.manual_seed(args.seed)
use_cuda = torch.cuda.is_available() and args.cuda_able
if use_cuda:
torch.cuda.manual_seed(args.seed)
# ##############################################################################
# Load data
# ##############################################################################
data = torch.load(args.data)
args.max_word_len = data["max_word_len"]
training_data = DataLoader(data['train']['src'],
data['train']['tgt'],
batch_size=args.batch_size,
shuffle=False,
cuda=use_cuda)
validation_data = DataLoader(data['valid']['src'],
data['valid']['tgt'],
batch_size=args.batch_size,
shuffle=False,
evaluation=True,
cuda=use_cuda)
args.enc_vocab_size = data['dict']['src_size']
args.dec_vocab_size = data['dict']['tgt_size']
args.n_warmup_steps = args.n_warmup_steps if args.n_warmup_steps != 0 else training_data._stop_step
MODEL_FOLDER: the path where the models for each district should be stored (hdfs or s3)
DISTRICTS_FILE: the path to the file specifying the districts that should be trained on (e.g. districts.txt)
"""
import sys
from pyspark.mllib.regression import LinearRegressionWithSGD
from spark_application import create_spark_application
from data_loader import DataLoader
from reader import read_districts_file
# Get file paths from arguments
if len(sys.argv) != 4:
print "Usage: linear_regression.py FEATURES_FILE MODEL_FOLDER DISTRICTS_FILE"
sys.exit()
features_file, model_folder, districts_file = sys.argv[1:]
spark_context, sql_context = create_spark_application("train_linear_regression")
data_loader = DataLoader(spark_context, sql_context, features_file)
data_loader.initialize()
# train and store a model for each district in the districts file
for lat, lon in read_districts_file(districts_file):
print("Training District: %f, %f" % (lat, lon))
model = LinearRegressionWithSGD.train(data_loader.get_train_data((lat, lon)),
iterations=1000,
step=1e-1)
# save the model in the specified model_folder
model.save(spark_context,
'%s/model_%s_%s' % (model_folder, str(lat), str(lon)))
class GAN():
def __init__(self, dataset_name = 'terrain'):
# Input shape
self.img_rows = 512
self.img_cols = 512
self.channels = 3
self.img_shape = (self.img_rows, self.img_cols, self.channels)
# Configure data loader
self.dataset_name = dataset_name
self.data_loader = DataLoader(dataset_name=self.dataset_name,
img_res=(self.img_rows, self.img_cols))
# Calculate output shape of D (PatchGAN)
patch = int(self.img_rows / 2**4)
self.disc_patch = (patch, patch, 1)
# Number of filters in the first layer of G and D
self.gf = 16
self.df = 16
optimizer = Adam(0.0002, 0.5)
# Build and compile the discriminator
self.discriminator = self.build_discriminator()
self.discriminator.compile(loss='mse',
optimizer=optimizer,
metrics=['accuracy'])
#-------------------------
# Construct Computational
# Graph of Generator
#-------------------------
# Build the generator
self.generator = self.build_generator()
# Input images and their conditioning images
img_A = Input(shape=self.img_shape)
img_B = Input(shape=self.img_shape)
# By conditioning on B generate a fake version of A
fake_A = self.generator(img_B)
# For the combined model we will only train the generator
self.discriminator.trainable = False
# Discriminators determines validity of translated images / condition pairs
valid = self.discriminator([fake_A, img_B])
self.combined = Model(inputs=[img_A, img_B], outputs=[valid, fake_A])
self.combined.compile(loss=['mse', 'mae'],
loss_weights=[1, 100],
optimizer=optimizer)
def build_generator(self):
"""U-Net Generator"""
def conv2d(layer_input, filters, f_size=3, bn=True):
"""Layers used during downsampling"""
d = Conv2D(filters, kernel_size=f_size, strides=2, padding='same')(layer_input)
d = LeakyReLU(alpha=0.2)(d)
if bn:
d = BatchNormalization(momentum = 0.8)(d)
return d
def deconv2d(layer_input, skip_input, filters, f_size=3, dropout_rate=0):
"""Layers used during upsampling"""
u = UpSampling2D(size=2)(layer_input)
u = Conv2D(filters, kernel_size=f_size, strides=1, padding='same', activation='relu')(u)
if dropout_rate:
u = Dropout(dropout_rate)(u)
u = BatchNormalization(momentum = 0.8)(u)
u = Concatenate()([u, skip_input])
return u
# Image input
d0 = Input(shape=self.img_shape)
# Downsampling
d1 = conv2d(d0, self.gf, bn=False)
d2 = conv2d(d1, self.gf*2)
d3 = conv2d(d2, self.gf*4)
#d4 = conv2d(d3, self.gf*8)
#u5 = deconv2d(d4, d3, self.gf*4)
u6 = deconv2d(d3, d2, self.gf*2)
u7 = deconv2d(u6, d1, self.gf)
u8 = UpSampling2D(size=2)(u7)
#u8 = Conv2D(self.gf, kernel_size=4, strides=1, padding='same', activation='relu')(u7)
#u9 = BatchNormalization(momentum = 0.8)(u8)
#u10 = Concatenate()([u7, d0])
output_img = Conv2D(self.channels, kernel_size=3, strides=1, padding='same', activation='tanh')(u8)
return Model(d0, output_img)
def build_discriminator(self):
def d_layer(layer_input, filters, f_size=4, bn=True):
"""Discriminator layer"""
d = Conv2D(filters, kernel_size=f_size, strides=2, padding='same')(layer_input)
d = LeakyReLU(alpha=0.2)(d)
if bn:
d = BatchNormalization(momentum=0.8)(d)
return d
img_A = Input(shape=self.img_shape)
img_B = Input(shape=self.img_shape)
# Concatenate image and conditioning image by channels to produce input
combined_imgs = Concatenate(axis=-1)([img_A, img_B])
d1 = d_layer(combined_imgs, self.df, bn=False)
d2 = d_layer(d1, self.df*2)
d3 = d_layer(d2, self.df*4)
d4 = d_layer(d3, self.df*8)
#d5 = d_layer(d4, self.df*16)
validity = Conv2D(1, kernel_size=5, strides=1, padding='same')(d4)
return Model([img_A, img_B], validity)
def train(self, epochs, batch_size=1, sample_interval=50):
start_time = datetime.datetime.now()
# Adversarial loss ground truths
valid = np.ones((batch_size,) + self.disc_patch)
fake = np.zeros((batch_size,) + self.disc_patch)
for epoch in range(epochs):
for batch_i, (imgs_A, imgs_B) in enumerate(self.data_loader.load_batch(batch_size)):
# ---------------------
# Train Discriminator
# ---------------------
# Condition on B and generate a translated version
fake_A = self.generator.predict(imgs_B)
# Train the discriminators (original images = real / generated = Fake)
d_loss_real = self.discriminator.train_on_batch([imgs_A, imgs_B], valid)
d_loss_fake = self.discriminator.train_on_batch([fake_A, imgs_B], fake)
d_loss = 0.5 * np.add(d_loss_real, d_loss_fake)
# -----------------
# Train Generator
# -----------------
# Train the generators
g_loss = self.combined.train_on_batch([imgs_A, imgs_B], [valid, imgs_A])
g_loss = self.combined.train_on_batch([imgs_A, imgs_B], [valid, imgs_A])
elapsed_time = datetime.datetime.now() - start_time
# Plot the progress
print ("[Epoch %d/%d] [Batch %d/%d] [D loss: %f, acc: %3d%%] [G loss: %f] time: %s" % (epoch, epochs,
batch_i, self.data_loader.n_batches,
d_loss[0], 100*d_loss[1],
g_loss[0],
elapsed_time))
# If at save interval => save generated image samples
if batch_i % sample_interval == 0:
self.sample_images(epoch, batch_i)
def sample_images(self, epoch, batch_i):
os.makedirs('images/%s' % self.dataset_name, exist_ok=True)
r, c = 3, 3
imgs_A, imgs_B = self.data_loader.load_data(batch_size=3, is_testing=True)
fake_A = self.generator.predict(imgs_B)
gen_imgs = np.concatenate([imgs_B, fake_A, imgs_A])
# Rescale images 0 - 1
gen_imgs = 0.5 * gen_imgs + 0.5
titles = ['Condition', 'Generated', 'Original']
fig, axs = plt.subplots(r, c, figsize=(15,15))
cnt = 0
for i in range(r):
for j in range(c):
axs[i,j].imshow(gen_imgs[cnt][:,:,:3])
axs[i, j].set_title(titles[i])
axs[i,j].axis('off')
cnt += 1
fig.savefig("images/%s/%d_%d.png" % (self.dataset_name, epoch, batch_i))
plt.close()
def save_model(self):
model_json = self.generator.to_json()
with open("model/" + self.dataset_name + "_architecture.json", "w") as json_file:
json_file.write(model_json)
self.generator.save_weights("model/" + self.dataset_name + "_weight.h5")
print("Saved model to disk")
def __init__(self):
# Input shape
self.channels = 3
self.lr_height = 64 # Low resolution height
self.lr_width = 64 # Low resolution width
self.lr_shape = (self.lr_height, self.lr_width, self.channels)
self.hr_height = self.lr_height*4 # High resolution height
self.hr_width = self.lr_width*4 # High resolution width
self.hr_shape = (self.hr_height, self.hr_width, self.channels)
# Number of residual blocks in the generator
self.n_residual_blocks = 16
optimizer = Adam(0.0002, 0.5)
# We use a pre-trained VGG19 model to extract image features from the high resolution
# and the generated high resolution images and minimize the mse between them
self.vgg = self.build_vgg()
self.vgg.trainable = False
self.vgg.compile(loss='mse',
optimizer=optimizer,
metrics=['accuracy'])
# Configure data loader
self.dataset_name = 'img_align_celeba'
self.data_loader = DataLoader(dataset_name=self.dataset_name,
img_res=(self.hr_height, self.hr_width))
# Calculate output shape of D (PatchGAN)
patch = int(self.hr_height / 2**4)
self.disc_patch = (patch, patch, 1)
# Number of filters in the first layer of G and D
self.gf = 64
self.df = 64
# Build and compile the discriminator
self.discriminator = self.build_discriminator()
self.discriminator.compile(loss='mse',
optimizer=optimizer,
metrics=['accuracy'])
# Build the generator
self.generator = self.build_generator()
# High res. and low res. images
img_hr = Input(shape=self.hr_shape)
img_lr = Input(shape=self.lr_shape)
# Generate high res. version from low res.
fake_hr = self.generator(img_lr)
# Extract image features of the generated img
fake_features = self.vgg(fake_hr)
# For the combined model we will only train the generator
self.discriminator.trainable = False
# Discriminator determines validity of generated high res. images
validity = self.discriminator(fake_hr)
self.combined = Model([img_lr, img_hr], [validity, fake_features])
self.combined.compile(loss=['binary_crossentropy', 'mse'],
loss_weights=[1e-3, 1],
optimizer=optimizer)
if use_cuda:
torch.cuda.manual_seed(args.seed)
# ##############################################################################
# Load data
# ##############################################################################
from data_loader import DataLoader
data = torch.load(args.data)
args.max_len = data["max_len"]
args.tag_size = data["tag_size"]
args.vocab_size = data['dict']['vocab_size']
args.trains_score = data['trains_score']
training_data = DataLoader(
data['train']['src'],
data['train']['label'],
args.max_len,
batch_size=args.batch_size,
cuda=use_cuda)
validation_data = DataLoader(
data['valid']['src'],
data['valid']['label'],
args.max_len,
batch_size=args.batch_size,
shuffle=False,
cuda=use_cuda)
args.n_warmup_steps = args.n_warmup_steps if args.n_warmup_steps != 0 else training_data.sents_size // args.batch_size
# ##############################################################################
# Build model
"""Provided by MovieQA"""
from data_loader import DataLoader
from story_loader import StoryLoader
parser = argparse.ArgumentParser()
#parser.add_argument("-path",type=str,required=True)
parser.add_argument("-split",type=str, choices=['train','val','test','trainval','full'],required=True)
parser.add_argument("-type", type=str, choices=['plot','dvs','subtitle','script','split_plot'] , required=True)
args = parser.parse_args()
print "Calling word preprocessing on MoiveQA"+" split "+args.split+" type "+args.type
print "Loading MovieQA data..."
DL = DataLoader()
story, qa = DL.get_story_qa_data(args.split,args.type)
movie_list = DL.get_split_movies(args.split)
outpath = '../wholepass.'+args.split+'.'+args.type
print "Writing to "+outpath
fo = open(outpath,'wb')
count = 0
for imdb_key in movie_list:
print imdb_key
whole_pass = '******'.join(story[imdb_key])
fo.write(whole_pass+'\n')
count += 1
class CycleGAN():
def __init__(self):
# Input shape
self.img_rows = 128
self.img_cols = 128
self.channels = 3
self.img_shape = (self.img_rows, self.img_cols, self.channels)
# Configure data loader
self.dataset_name = 'apple2orange'
self.data_loader = DataLoader(dataset_name=self.dataset_name,
img_res=(self.img_rows, self.img_cols))
# Calculate output shape of D (PatchGAN)
patch = int(self.img_rows / 2**4)
self.disc_patch = (patch, patch, 1)
# Number of filters in the first layer of G and D
self.gf = 32
self.df = 64
# Loss weights
self.lambda_cycle = 10.0 # Cycle-consistency loss
self.lambda_id = 0.1 * self.lambda_cycle # Identity loss
optimizer = Adam(0.0002, 0.5)
# Build and compile the discriminators
self.d_A = self.build_discriminator()
self.d_B = self.build_discriminator()
self.d_A.compile(loss='mse',
optimizer=optimizer,
metrics=['accuracy'])
self.d_B.compile(loss='mse',
optimizer=optimizer,
metrics=['accuracy'])
#-------------------------
# Construct Computational
# Graph of Generators
#-------------------------
# Build the generators
self.g_AB = self.build_generator()
self.g_BA = self.build_generator()
# Input images from both domains
img_A = Input(shape=self.img_shape)
img_B = Input(shape=self.img_shape)
# Translate images to the other domain
fake_B = self.g_AB(img_A)
fake_A = self.g_BA(img_B)
# Translate images back to original domain
reconstr_A = self.g_BA(fake_B)
reconstr_B = self.g_AB(fake_A)
# Identity mapping of images
img_A_id = self.g_BA(img_A)
img_B_id = self.g_AB(img_B)
# For the combined model we will only train the generators
self.d_A.trainable = False
self.d_B.trainable = False
# Discriminators determines validity of translated images
valid_A = self.d_A(fake_A)
valid_B = self.d_B(fake_B)
# Combined model trains generators to fool discriminators
self.combined = Model(inputs=[img_A, img_B],
outputs=[ valid_A, valid_B,
reconstr_A, reconstr_B,
img_A_id, img_B_id ])
self.combined.compile(loss=['mse', 'mse',
'mae', 'mae',
'mae', 'mae'],
loss_weights=[ 1, 1,
self.lambda_cycle, self.lambda_cycle,
self.lambda_id, self.lambda_id ],
optimizer=optimizer)
def build_generator(self):
"""U-Net Generator"""
def conv2d(layer_input, filters, f_size=4):
"""Layers used during downsampling"""
d = Conv2D(filters, kernel_size=f_size, strides=2, padding='same')(layer_input)
d = LeakyReLU(alpha=0.2)(d)
d = InstanceNormalization()(d)
return d
def deconv2d(layer_input, skip_input, filters, f_size=4, dropout_rate=0):
"""Layers used during upsampling"""
u = UpSampling2D(size=2)(layer_input)
u = Conv2D(filters, kernel_size=f_size, strides=1, padding='same', activation='relu')(u)
if dropout_rate:
u = Dropout(dropout_rate)(u)
u = InstanceNormalization()(u)
u = Concatenate()([u, skip_input])
return u
# Image input
d0 = Input(shape=self.img_shape)
# Downsampling
d1 = conv2d(d0, self.gf)
d2 = conv2d(d1, self.gf*2)
d3 = conv2d(d2, self.gf*4)
d4 = conv2d(d3, self.gf*8)
# Upsampling
u1 = deconv2d(d4, d3, self.gf*4)
u2 = deconv2d(u1, d2, self.gf*2)
u3 = deconv2d(u2, d1, self.gf)
u4 = UpSampling2D(size=2)(u3)
output_img = Conv2D(self.channels, kernel_size=4, strides=1, padding='same', activation='tanh')(u4)
return Model(d0, output_img)
def build_discriminator(self):
def d_layer(layer_input, filters, f_size=4, normalization=True):
"""Discriminator layer"""
d = Conv2D(filters, kernel_size=f_size, strides=2, padding='same')(layer_input)
d = LeakyReLU(alpha=0.2)(d)
if normalization:
d = InstanceNormalization()(d)
return d
img = Input(shape=self.img_shape)
d1 = d_layer(img, self.df, normalization=False)
d2 = d_layer(d1, self.df*2)
d3 = d_layer(d2, self.df*4)
d4 = d_layer(d3, self.df*8)
validity = Conv2D(1, kernel_size=4, strides=1, padding='same')(d4)
return Model(img, validity)
def train(self, epochs, batch_size=1, sample_interval=50):
start_time = datetime.datetime.now()
# Adversarial loss ground truths
valid = np.ones((batch_size,) + self.disc_patch)
fake = np.zeros((batch_size,) + self.disc_patch)
for epoch in range(epochs):
for batch_i, (imgs_A, imgs_B) in enumerate(self.data_loader.load_batch(batch_size)):
# ----------------------
# Train Discriminators
# ----------------------
# Translate images to opposite domain
fake_B = self.g_AB.predict(imgs_A)
fake_A = self.g_BA.predict(imgs_B)
# Train the discriminators (original images = real / translated = Fake)
dA_loss_real = self.d_A.train_on_batch(imgs_A, valid)
dA_loss_fake = self.d_A.train_on_batch(fake_A, fake)
dA_loss = 0.5 * np.add(dA_loss_real, dA_loss_fake)
dB_loss_real = self.d_B.train_on_batch(imgs_B, valid)
dB_loss_fake = self.d_B.train_on_batch(fake_B, fake)
dB_loss = 0.5 * np.add(dB_loss_real, dB_loss_fake)
# Total disciminator loss
d_loss = 0.5 * np.add(dA_loss, dB_loss)
# ------------------
# Train Generators
# ------------------
# Train the generators
g_loss = self.combined.train_on_batch([imgs_A, imgs_B],
[valid, valid,
imgs_A, imgs_B,
imgs_A, imgs_B])
elapsed_time = datetime.datetime.now() - start_time
# Plot the progress
print ("[Epoch %d/%d] [Batch %d/%d] [D loss: %f, acc: %3d%%] [G loss: %05f, adv: %05f, recon: %05f, id: %05f] time: %s " \
% ( epoch, epochs,
batch_i, self.data_loader.n_batches,
d_loss[0], 100*d_loss[1],
g_loss[0],
np.mean(g_loss[1:3]),
np.mean(g_loss[3:5]),
np.mean(g_loss[5:6]),
elapsed_time))
# If at save interval => save generated image samples
if batch_i % sample_interval == 0:
self.sample_images(epoch, batch_i)
def sample_images(self, epoch, batch_i):
os.makedirs('images/%s' % self.dataset_name, exist_ok=True)
r, c = 2, 3
imgs_A = self.data_loader.load_data(domain="A", batch_size=1, is_testing=True)
imgs_B = self.data_loader.load_data(domain="B", batch_size=1, is_testing=True)
# Demo (for GIF)
#imgs_A = self.data_loader.load_img('datasets/apple2orange/testA/n07740461_1541.jpg')
#imgs_B = self.data_loader.load_img('datasets/apple2orange/testB/n07749192_4241.jpg')
# Translate images to the other domain
fake_B = self.g_AB.predict(imgs_A)
fake_A = self.g_BA.predict(imgs_B)
# Translate back to original domain
reconstr_A = self.g_BA.predict(fake_B)
reconstr_B = self.g_AB.predict(fake_A)
gen_imgs = np.concatenate([imgs_A, fake_B, reconstr_A, imgs_B, fake_A, reconstr_B])
# Rescale images 0 - 1
gen_imgs = 0.5 * gen_imgs + 0.5
titles = ['Original', 'Translated', 'Reconstructed']
fig, axs = plt.subplots(r, c)
cnt = 0
for i in range(r):
for j in range(c):
axs[i,j].imshow(gen_imgs[cnt])
axs[i, j].set_title(titles[j])
axs[i,j].axis('off')
cnt += 1
fig.savefig("images/%s/%d_%d.png" % (self.dataset_name, epoch, batch_i))
plt.close()
def __init__(self):
# Input shape
self.img_rows = 32
self.img_cols = 32
self.channels = 3
self.img_shape = (self.img_rows, self.img_cols, self.channels)
self.num_classes = 10
# Configure MNIST and MNIST-M data loader
self.data_loader = DataLoader(img_res=(self.img_rows, self.img_cols))
# Loss weights
lambda_adv = 10
lambda_clf = 1
# Calculate output shape of D (PatchGAN)
patch = int(self.img_rows / 2**4)
self.disc_patch = (patch, patch, 1)
# Number of residual blocks in the generator
self.residual_blocks = 6
optimizer = Adam(0.0002, 0.5)
# Number of filters in first layer of discriminator and classifier
self.df = 64
self.cf = 64
# Build and compile the discriminators
self.discriminator = self.build_discriminator()
self.discriminator.compile(loss='mse',
optimizer=optimizer,
metrics=['accuracy'])
# Build the generator
self.generator = self.build_generator()
# Build the task (classification) network
self.clf = self.build_classifier()
# Input images from both domains
img_A = Input(shape=self.img_shape)
img_B = Input(shape=self.img_shape)
# Translate images from domain A to domain B
fake_B = self.generator(img_A)
# Classify the translated image
class_pred = self.clf(fake_B)
# For the combined model we will only train the generator and classifier
self.discriminator.trainable = False
# Discriminator determines validity of translated images
valid = self.discriminator(fake_B)
self.combined = Model(img_A, [valid, class_pred])
self.combined.compile(loss=['mse', 'categorical_crossentropy'],
loss_weights=[lambda_adv, lambda_clf],
optimizer=optimizer,
metrics=['accuracy'])
task_path = os.path.join(data_path, 'volume', task_id)
log_path = os.path.join(data_path, 'volume', 'logs')
task_log_path = os.path.join(log_path, task_id)
if not os.path.exists(task_path):
os.mkdir(task_path)
if not os.path.exists(log_path):
os.mkdir(log_path)
if not os.path.exists(task_log_path):
os.mkdir(task_log_path)
print("Train Start")
data_loader = DataLoader(data_path=os.path.join(data_path, 'train'),
common_path=os.path.join(data_path, 'volume'),
task_path=task_path)
model = SimpleRNNModel(data_loader)
callbacks = [
ModelCheckpoint(filepath=os.path.join(task_path, 'model-{epoch:02d}-{val_loss:2f}.hdf5'),
monitor='val_loss',
checkpoint_mode='min',
save_best_only=False,
save_weights_only=False,
verbose=True
),
TensorBoard(log_dir=task_log_path,
write_graph=True
)
]
def get_data_loaders(region_feature_dim, tok2idx):
test_loader = DataLoader(args, region_feature_dim, 'test', tok2idx)
if args.test:
return test_loader, None, None
max_length = test_loader.max_length
train_loader = DataLoader(args, region_feature_dim, 'train', tok2idx)
max_length = max(max_length, train_loader.max_length)
val_loader = DataLoader(args, region_feature_dim, 'val', tok2idx,
set(train_loader.phrases))
max_length = max(max_length, val_loader.max_length)
test_loader.set_max_length(max_length)
train_loader.set_max_length(max_length)
val_loader.set_max_length(max_length)
return test_loader, train_loader, val_loader
class SRGAN():
def __init__(self):
# Input shape
self.channels = 3
self.lr_height = 64 # Low resolution height
self.lr_width = 64 # Low resolution width
self.lr_shape = (self.lr_height, self.lr_width, self.channels)
self.hr_height = self.lr_height*4 # High resolution height
self.hr_width = self.lr_width*4 # High resolution width
self.hr_shape = (self.hr_height, self.hr_width, self.channels)
# Number of residual blocks in the generator
self.n_residual_blocks = 16
optimizer = Adam(0.0002, 0.5)
# We use a pre-trained VGG19 model to extract image features from the high resolution
# and the generated high resolution images and minimize the mse between them
self.vgg = self.build_vgg()
self.vgg.trainable = False
self.vgg.compile(loss='mse',
optimizer=optimizer,
metrics=['accuracy'])
# Configure data loader
self.dataset_name = 'img_align_celeba'
self.data_loader = DataLoader(dataset_name=self.dataset_name,
img_res=(self.hr_height, self.hr_width))
# Calculate output shape of D (PatchGAN)
patch = int(self.hr_height / 2**4)
self.disc_patch = (patch, patch, 1)
# Number of filters in the first layer of G and D
self.gf = 64
self.df = 64
# Build and compile the discriminator
self.discriminator = self.build_discriminator()
self.discriminator.compile(loss='mse',
optimizer=optimizer,
metrics=['accuracy'])
# Build the generator
self.generator = self.build_generator()
# High res. and low res. images
img_hr = Input(shape=self.hr_shape)
img_lr = Input(shape=self.lr_shape)
# Generate high res. version from low res.
fake_hr = self.generator(img_lr)
# Extract image features of the generated img
fake_features = self.vgg(fake_hr)
# For the combined model we will only train the generator
self.discriminator.trainable = False
# Discriminator determines validity of generated high res. images
validity = self.discriminator(fake_hr)
self.combined = Model([img_lr, img_hr], [validity, fake_features])
self.combined.compile(loss=['binary_crossentropy', 'mse'],
loss_weights=[1e-3, 1],
optimizer=optimizer)
def build_vgg(self):
"""
Builds a pre-trained VGG19 model that outputs image features extracted at the
third block of the model
"""
vgg = VGG19(weights="imagenet")
# Set outputs to outputs of last conv. layer in block 3
# See architecture at: https://github.com/keras-team/keras/blob/master/keras/applications/vgg19.py
vgg.outputs = [vgg.layers[9].output]
img = Input(shape=self.hr_shape)
# Extract image features
img_features = vgg(img)
return Model(img, img_features)
def build_generator(self):
def residual_block(layer_input):
"""Residual block described in paper"""
d = Conv2D(64, kernel_size=3, strides=1, padding='same')(layer_input)
d = Activation('relu')(d)
d = BatchNormalization(momentum=0.8)(d)
d = Conv2D(64, kernel_size=3, strides=1, padding='same')(d)
d = BatchNormalization(momentum=0.8)(d)
d = Add()([d, layer_input])
return d
def deconv2d(layer_input):
"""Layers used during upsampling"""
u = UpSampling2D(size=2)(layer_input)
u = Conv2D(256, kernel_size=3, strides=1, padding='same')(u)
u = Activation('relu')(u)
return u
# Low resolution image input
img_lr = Input(shape=self.lr_shape)
# Pre-residual block
c1 = Conv2D(64, kernel_size=9, strides=1, padding='same')(img_lr)
c1 = Activation('relu')(c1)
# Propogate through residual blocks
r = residual_block(c1)
for _ in range(self.n_residual_blocks - 1):
r = residual_block(r)
# Post-residual block
c2 = Conv2D(64, kernel_size=3, strides=1, padding='same')(r)
c2 = BatchNormalization(momentum=0.8)(c2)
c2 = Add()([c2, c1])
# Upsampling
u1 = deconv2d(c2)
u2 = deconv2d(u1)
# Generate high resolution output
gen_hr = Conv2D(self.channels, kernel_size=9, strides=1, padding='same', activation='tanh')(u2)
return Model(img_lr, gen_hr)
def build_discriminator(self):
def d_block(layer_input, filters, strides=1, bn=True):
"""Discriminator layer"""
d = Conv2D(filters, kernel_size=3, strides=strides, padding='same')(layer_input)
d = LeakyReLU(alpha=0.2)(d)
if bn:
d = BatchNormalization(momentum=0.8)(d)
return d
# Input img
d0 = Input(shape=self.hr_shape)
d1 = d_block(d0, self.df, bn=False)
d2 = d_block(d1, self.df, strides=2)
d3 = d_block(d2, self.df*2)
d4 = d_block(d3, self.df*2, strides=2)
d5 = d_block(d4, self.df*4)
d6 = d_block(d5, self.df*4, strides=2)
d7 = d_block(d6, self.df*8)
d8 = d_block(d7, self.df*8, strides=2)
d9 = Dense(self.df*16)(d8)
d10 = LeakyReLU(alpha=0.2)(d9)
validity = Dense(1, activation='sigmoid')(d10)
return Model(d0, validity)
def train(self, epochs, batch_size=1, sample_interval=50):
start_time = datetime.datetime.now()
for epoch in range(epochs):
# ----------------------
# Train Discriminator
# ----------------------
# Sample images and their conditioning counterparts
imgs_hr, imgs_lr = self.data_loader.load_data(batch_size)
# From low res. image generate high res. version
fake_hr = self.generator.predict(imgs_lr)
valid = np.ones((batch_size,) + self.disc_patch)
fake = np.zeros((batch_size,) + self.disc_patch)
# Train the discriminators (original images = real / generated = Fake)
d_loss_real = self.discriminator.train_on_batch(imgs_hr, valid)
d_loss_fake = self.discriminator.train_on_batch(fake_hr, fake)
d_loss = 0.5 * np.add(d_loss_real, d_loss_fake)
# ------------------
# Train Generator
# ------------------
# Sample images and their conditioning counterparts
imgs_hr, imgs_lr = self.data_loader.load_data(batch_size)
# The generators want the discriminators to label the generated images as real
valid = np.ones((batch_size,) + self.disc_patch)
# Extract ground truth image features using pre-trained VGG19 model
image_features = self.vgg.predict(imgs_hr)
# Train the generators
g_loss = self.combined.train_on_batch([imgs_lr, imgs_hr], [valid, image_features])
elapsed_time = datetime.datetime.now() - start_time
# Plot the progress
print ("%d time: %s" % (epoch, elapsed_time))
# If at save interval => save generated image samples
if epoch % sample_interval == 0:
self.sample_images(epoch)
def sample_images(self, epoch):
os.makedirs('images/%s' % self.dataset_name, exist_ok=True)
r, c = 2, 2
imgs_hr, imgs_lr = self.data_loader.load_data(batch_size=2, is_testing=True)
fake_hr = self.generator.predict(imgs_lr)
# Rescale images 0 - 1
imgs_lr = 0.5 * imgs_lr + 0.5
fake_hr = 0.5 * fake_hr + 0.5
imgs_hr = 0.5 * imgs_hr + 0.5
# Save generated images and the high resolution originals
titles = ['Generated', 'Original']
fig, axs = plt.subplots(r, c)
cnt = 0
for row in range(r):
for col, image in enumerate([fake_hr, imgs_hr]):
axs[row, col].imshow(image[row])
axs[row, col].set_title(titles[col])
axs[row, col].axis('off')
cnt += 1
fig.savefig("images/%s/%d.png" % (self.dataset_name, epoch))
plt.close()
# Save low resolution images for comparison
for i in range(r):
fig = plt.figure()
plt.imshow(imgs_lr[i])
fig.savefig('images/%s/%d_lowres%d.png' % (self.dataset_name, epoch, i))
plt.close()
def build_train_graph(self):
opt = self.opt
loader = DataLoader(opt.dataset_dir, opt.batch_size, opt.img_height,
opt.img_width, opt.num_source, opt.num_scales)
with tf.name_scope("data_loading"):
tgt_image, src_image_stack, intrinsics = loader.load_train_batch()
tgt_image = self.preprocess_image(tgt_image)
src_image_stack = self.preprocess_image(src_image_stack)
with tf.name_scope("depth_prediction"):
pred_disp, depth_net_endpoints = disp_net(tgt_image,
is_training=True)
pred_depth = [1. / d for d in pred_disp]
with tf.name_scope("pose_and_explainability_prediction"):
pred_poses, pred_exp_logits, pose_exp_net_endpoints = \
pose_exp_net(tgt_image,
src_image_stack,
do_exp=(opt.explain_reg_weight > 0),
is_training=True)
with tf.name_scope("compute_loss"):
pixel_loss = 0
exp_loss = 0
smooth_loss = 0
tgt_image_all = []
src_image_stack_all = []
proj_image_stack_all = []
proj_error_stack_all = []
exp_mask_stack_all = []
for s in range(opt.num_scales):
if opt.explain_reg_weight > 0:
# Construct a reference explainability mask (i.e. all
# pixels are explainable)
ref_exp_mask = self.get_reference_explain_mask(s)
# Scale the source and target images for computing loss at the
# according scale.
curr_tgt_image = tf.image.resize_area(tgt_image, [
int(opt.img_height / (2**s)),
int(opt.img_width / (2**s))
])
curr_src_image_stack = tf.image.resize_area(
src_image_stack, [
int(opt.img_height / (2**s)),
int(opt.img_width / (2**s))
])
if opt.smooth_weight > 0:
smooth_loss += opt.smooth_weight/(2**s) * \
self.compute_smooth_loss(pred_disp[s])
for i in range(opt.num_source):
# Inverse warp the source image to the target image frame
curr_proj_image = projective_inverse_warp(
curr_src_image_stack[:, :, :, 3 * i:3 * (i + 1)],
tf.squeeze(pred_depth[s], axis=3), pred_poses[:, i, :],
intrinsics[:, s, :, :])
curr_proj_error = tf.abs(curr_proj_image - curr_tgt_image)
# Cross-entropy loss as regularization for the
# explainability prediction
if opt.explain_reg_weight > 0:
curr_exp_logits = tf.slice(pred_exp_logits[s],
[0, 0, 0, i * 2],
[-1, -1, -1, 2])
exp_loss += opt.explain_reg_weight * \
self.compute_exp_reg_loss(curr_exp_logits,
ref_exp_mask)
curr_exp = tf.nn.softmax(curr_exp_logits)
# Photo-consistency loss weighted by explainability
if opt.explain_reg_weight > 0:
pixel_loss += tf.reduce_mean(curr_proj_error * \
tf.expand_dims(curr_exp[:,:,:,1], -1))
else:
pixel_loss += tf.reduce_mean(curr_proj_error)
# Prepare images for tensorboard summaries
if i == 0:
proj_image_stack = curr_proj_image
proj_error_stack = curr_proj_error
if opt.explain_reg_weight > 0:
exp_mask_stack = tf.expand_dims(
curr_exp[:, :, :, 1], -1)
else:
proj_image_stack = tf.concat(
[proj_image_stack, curr_proj_image], axis=3)
proj_error_stack = tf.concat(
[proj_error_stack, curr_proj_error], axis=3)
if opt.explain_reg_weight > 0:
exp_mask_stack = tf.concat([
exp_mask_stack,
tf.expand_dims(curr_exp[:, :, :, 1], -1)
],
axis=3)
tgt_image_all.append(curr_tgt_image)
src_image_stack_all.append(curr_src_image_stack)
proj_image_stack_all.append(proj_image_stack)
proj_error_stack_all.append(proj_error_stack)
if opt.explain_reg_weight > 0:
exp_mask_stack_all.append(exp_mask_stack)
total_loss = pixel_loss + smooth_loss + exp_loss
with tf.name_scope("train_op"):
train_vars = [var for var in tf.trainable_variables()]
optim = tf.train.AdamOptimizer(opt.learning_rate, opt.beta1)
# self.grads_and_vars = optim.compute_gradients(total_loss,
# var_list=train_vars)
# self.train_op = optim.apply_gradients(self.grads_and_vars)
self.train_op = slim.learning.create_train_op(total_loss, optim)
self.global_step = tf.Variable(0,
name='global_step',
trainable=False)
self.incr_global_step = tf.assign(self.global_step,
self.global_step + 1)
# Collect tensors that are useful later (e.g. tf summary)
self.pred_depth = pred_depth
self.pred_poses = pred_poses
self.steps_per_epoch = loader.steps_per_epoch
self.total_loss = total_loss
self.pixel_loss = pixel_loss
self.exp_loss = exp_loss
self.smooth_loss = smooth_loss
self.tgt_image_all = tgt_image_all
self.src_image_stack_all = src_image_stack_all
self.proj_image_stack_all = proj_image_stack_all
self.proj_error_stack_all = proj_error_stack_all
self.exp_mask_stack_all = exp_mask_stack_all
def train():
seed = 8964
tf.set_random_seed(seed)
np.random.seed(seed)
random.seed(seed)
pp = pprint.PrettyPrinter()
pp.pprint(flags.FLAGS.__flags)
if not os.path.exists(opt.checkpoint_dir):
os.makedirs(opt.checkpoint_dir)
with tf.Graph().as_default():
# Data Loader
loader = DataLoader(opt)
tgt_image, src_image_stack, intrinsics = loader.load_train_batch()
# Build Model
model = GeoNetModel(opt, tgt_image, src_image_stack, intrinsics)
loss = model.total_loss
# Train Op
if opt.mode == 'train_flow' and opt.flownet_type == "residual":
# we pretrain DepthNet & PoseNet, then finetune ResFlowNetS
train_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, "flow_net")
vars_to_restore = slim.get_variables_to_restore(include=["depth_net", "pose_net"])
else:
train_vars = [var for var in tf.trainable_variables()]
vars_to_restore = slim.get_model_variables()
if opt.init_ckpt_file != None:
init_assign_op, init_feed_dict = slim.assign_from_checkpoint(
opt.init_ckpt_file, vars_to_restore)
optim = tf.train.AdamOptimizer(opt.learning_rate, 0.9)
train_op = slim.learning.create_train_op(loss, optim,
variables_to_train=train_vars)
# Global Step
global_step = tf.Variable(0,
name='global_step',
trainable=False)
incr_global_step = tf.assign(global_step,
global_step+1)
# Parameter Count
parameter_count = tf.reduce_sum([tf.reduce_prod(tf.shape(v)) \
for v in train_vars])
# Saver
saver = tf.train.Saver([var for var in tf.model_variables()] + \
[global_step],
max_to_keep=opt.max_to_keep)
# Session
sv = tf.train.Supervisor(logdir=opt.checkpoint_dir,
save_summaries_secs=0,
saver=None)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with sv.managed_session(config=config) as sess:
print('Trainable variables: ')
for var in train_vars:
print(var.name)
print("parameter_count =", sess.run(parameter_count))
if opt.init_ckpt_file != None:
sess.run(init_assign_op, init_feed_dict)
start_time = time.time()
for step in range(1, opt.max_steps):
fetches = {
"train": train_op,
"global_step": global_step,
"incr_global_step": incr_global_step
}
if step % 100 == 0:
fetches["loss"] = loss
results = sess.run(fetches)
if step % 100 == 0:
time_per_iter = (time.time() - start_time) / 100
start_time = time.time()
print('Iteration: [%7d] | Time: %4.4fs/iter | Loss: %.3f' \
% (step, time_per_iter, results["loss"]))
if step % opt.save_ckpt_freq == 0:
saver.save(sess, os.path.join(opt.checkpoint_dir, 'model'), global_step=step)
class SRGAN():
def __init__(self):
# Input shape
self.channels = 3
self.lr_height = 64 # Low resolution height
self.lr_width = 64 # Low resolution width
self.lr_shape = (self.lr_height, self.lr_width, self.channels)
self.hr_height = self.lr_height * 4 # High resolution height
self.hr_width = self.lr_width * 4 # High resolution width
self.hr_shape = (self.hr_height, self.hr_width, self.channels)
# Number of residual blocks in the generator
self.n_residual_blocks = 16
optimizer = Adam(0.0002, 0.5)
# We use a pre-trained VGG19 model to extract image features from the high resolution
# and the generated high resolution images and minimize the mse between them
self.vgg = self.build_vgg()
self.vgg.trainable = False
self.vgg.compile(loss='mse', optimizer=optimizer, metrics=['accuracy'])
# Configure data loader
self.dataset_name = 'img_align_celeba'
self.data_loader = DataLoader(dataset_name=self.dataset_name,
img_res=(self.hr_height, self.hr_width))
# Calculate output shape of D (PatchGAN)
patch = int(self.hr_height / 2**4)
self.disc_patch = (patch, patch, 1)
# Number of filters in the first layer of G and D
self.gf = 64
self.df = 64
# Build and compile the discriminator
self.discriminator = self.build_discriminator()
self.discriminator.compile(loss='mse',
optimizer=optimizer,
metrics=['accuracy'])
# Build and compile the generator
self.generator = self.build_generator()
self.generator.compile(loss='binary_crossentropy', optimizer=optimizer)
# High res. and low res. images
img_hr = Input(shape=self.hr_shape)
img_lr = Input(shape=self.lr_shape)
# Generate high res. version from low res.
fake_hr = self.generator(img_lr)
# Extract image features of the generated img
fake_features = self.vgg(fake_hr)
# For the combined model we will only train the generator
self.discriminator.trainable = False
# Discriminator determines validity of generated high res. images
validity = self.discriminator(fake_hr)
self.combined = Model([img_lr, img_hr], [validity, fake_features])
self.combined.compile(loss=['binary_crossentropy', 'mse'],
loss_weights=[1e-3, 1],
optimizer=optimizer)
def build_vgg(self):
"""
Builds a pre-trained VGG19 model that outputs image features extracted at the
third block of the model
"""
vgg = VGG19(weights="imagenet")
# Set outputs to outputs of last conv. layer in block 3
# See architecture at: https://github.com/keras-team/keras/blob/master/keras/applications/vgg19.py
vgg.outputs = [vgg.layers[9].output]
img = Input(shape=self.hr_shape)
# Extract image features
img_features = vgg(img)
return Model(img, img_features)
def build_generator(self):
def residual_block(layer_input):
"""Residual block described in paper"""
d = Conv2D(64, kernel_size=3, strides=1,
padding='same')(layer_input)
d = Activation('relu')(d)
d = BatchNormalization(momentum=0.8)(d)
d = Conv2D(64, kernel_size=3, strides=1, padding='same')(d)
d = BatchNormalization(momentum=0.8)(d)
d = Add()([d, layer_input])
return d
def deconv2d(layer_input):
"""Layers used during upsampling"""
u = UpSampling2D(size=2)(layer_input)
u = Conv2D(256, kernel_size=3, strides=1, padding='same')(u)
u = Activation('relu')(u)
return u
# Low resolution image input
img_lr = Input(shape=self.lr_shape)
# Pre-residual block
c1 = Conv2D(64, kernel_size=9, strides=1, padding='same')(img_lr)
c1 = Activation('relu')(c1)
# Propogate through residual blocks
r = residual_block(c1)
for _ in range(self.n_residual_blocks - 1):
r = residual_block(r)
# Post-residual block
c2 = Conv2D(64, kernel_size=3, strides=1, padding='same')(r)
c2 = BatchNormalization(momentum=0.8)(c2)
c2 = Add()([c2, c1])
# Upsampling
u1 = deconv2d(c2)
u2 = deconv2d(u1)
# Generate high resolution output
gen_hr = Conv2D(self.channels,
kernel_size=9,
strides=1,
padding='same',
activation='tanh')(u2)
return Model(img_lr, gen_hr)
def build_discriminator(self):
def d_block(layer_input, filters, strides=1, bn=True):
"""Discriminator layer"""
d = Conv2D(filters, kernel_size=3, strides=strides,
padding='same')(layer_input)
d = LeakyReLU(alpha=0.2)(d)
if bn:
d = BatchNormalization(momentum=0.8)(d)
return d
# Input img
d0 = Input(shape=self.hr_shape)
d1 = d_block(d0, self.df, bn=False)
d2 = d_block(d1, self.df, strides=2)
d3 = d_block(d2, self.df * 2)
d4 = d_block(d3, self.df * 2, strides=2)
d5 = d_block(d4, self.df * 4)
d6 = d_block(d5, self.df * 4, strides=2)
d7 = d_block(d6, self.df * 8)
d8 = d_block(d7, self.df * 8, strides=2)
d9 = Dense(self.df * 16)(d8)
d10 = LeakyReLU(alpha=0.2)(d9)
validity = Dense(1, activation='sigmoid')(d10)
return Model(d0, validity)
def train(self, epochs, batch_size=1, sample_interval=50):
start_time = datetime.datetime.now()
for epoch in range(epochs):
# ----------------------
# Train Discriminator
# ----------------------
# Sample images and their conditioning counterparts
imgs_hr, imgs_lr = self.data_loader.load_data(batch_size)
# From low res. image generate high res. version
fake_hr = self.generator.predict(imgs_lr)
valid = np.ones((batch_size, ) + self.disc_patch)
fake = np.zeros((batch_size, ) + self.disc_patch)
# Train the discriminators (original images = real / generated = Fake)
d_loss_real = self.discriminator.train_on_batch(imgs_hr, valid)
d_loss_fake = self.discriminator.train_on_batch(fake_hr, fake)
d_loss = 0.5 * np.add(d_loss_real, d_loss_fake)
# ------------------
# Train Generator
# ------------------
# Sample images and their conditioning counterparts
imgs_hr, imgs_lr = self.data_loader.load_data(batch_size)
# The generators want the discriminators to label the generated images as real
valid = np.ones((batch_size, ) + self.disc_patch)
# Extract ground truth image features using pre-trained VGG19 model
image_features = self.vgg.predict(imgs_hr)
# Train the generators
g_loss = self.combined.train_on_batch([imgs_lr, imgs_hr],
[valid, image_features])
elapsed_time = datetime.datetime.now() - start_time
# Plot the progress
print("%d time: %s" % (epoch, elapsed_time))
# If at save interval => save generated image samples
if epoch % sample_interval == 0:
self.sample_images(epoch)
def sample_images(self, epoch):
os.makedirs('images/%s' % self.dataset_name, exist_ok=True)
r, c = 2, 2
imgs_hr, imgs_lr = self.data_loader.load_data(batch_size=2,
is_testing=True)
fake_hr = self.generator.predict(imgs_lr)
# Rescale images 0 - 1
imgs_lr = 0.5 * imgs_lr + 0.5
fake_hr = 0.5 * fake_hr + 0.5
imgs_hr = 0.5 * imgs_hr + 0.5
# Save generated images and the high resolution originals
titles = ['Generated', 'Original']
fig, axs = plt.subplots(r, c)
cnt = 0
for row in range(r):
for col, image in enumerate([fake_hr, imgs_hr]):
axs[row, col].imshow(image[row])
axs[row, col].set_title(titles[col])
axs[row, col].axis('off')
cnt += 1
fig.savefig("images/%s/%d.png" % (self.dataset_name, epoch))
plt.close()
# Save low resolution images for comparison
for i in range(r):
fig = plt.figure()
plt.imshow(imgs_lr[i])
fig.savefig('images/%s/%d_lowres%d.png' %
(self.dataset_name, epoch, i))
plt.close()
class PixelDA():
def __init__(self):
# Input shape
self.img_rows = 32
self.img_cols = 32
self.channels = 3
self.img_shape = (self.img_rows, self.img_cols, self.channels)
self.num_classes = 10
# Configure MNIST and MNIST-M data loader
self.data_loader = DataLoader(img_res=(self.img_rows, self.img_cols))
# Loss weights
lambda_adv = 10
lambda_clf = 1
# Calculate output shape of D (PatchGAN)
patch = int(self.img_rows / 2**4)
self.disc_patch = (patch, patch, 1)
# Number of residual blocks in the generator
self.residual_blocks = 6
optimizer = Adam(0.0002, 0.5)
# Number of filters in first layer of discriminator and classifier
self.df = 64
self.cf = 64
# Build and compile the discriminators
self.discriminator = self.build_discriminator()
self.discriminator.compile(loss='mse',
optimizer=optimizer,
metrics=['accuracy'])
# Build the generator
self.generator = self.build_generator()
# Build the task (classification) network
self.clf = self.build_classifier()
# Input images from both domains
img_A = Input(shape=self.img_shape)
img_B = Input(shape=self.img_shape)
# Translate images from domain A to domain B
fake_B = self.generator(img_A)
# Classify the translated image
class_pred = self.clf(fake_B)
# For the combined model we will only train the generator and classifier
self.discriminator.trainable = False
# Discriminator determines validity of translated images
valid = self.discriminator(fake_B)
self.combined = Model(img_A, [valid, class_pred])
self.combined.compile(loss=['mse', 'categorical_crossentropy'],
loss_weights=[lambda_adv, lambda_clf],
optimizer=optimizer,
metrics=['accuracy'])
def build_generator(self):
"""Resnet Generator"""
def residual_block(layer_input):
"""Residual block described in paper"""
d = Conv2D(64, kernel_size=3, strides=1, padding='same')(layer_input)
d = BatchNormalization(momentum=0.8)(d)
d = Activation('relu')(d)
d = Conv2D(64, kernel_size=3, strides=1, padding='same')(d)
d = BatchNormalization(momentum=0.8)(d)
d = Add()([d, layer_input])
return d
# Image input
img = Input(shape=self.img_shape)
l1 = Conv2D(64, kernel_size=3, padding='same', activation='relu')(img)
# Propogate signal through residual blocks
r = residual_block(l1)
for _ in range(self.residual_blocks - 1):
r = residual_block(r)
output_img = Conv2D(self.channels, kernel_size=3, padding='same', activation='tanh')(r)
return Model(img, output_img)
def build_discriminator(self):
def d_layer(layer_input, filters, f_size=4, normalization=True):
"""Discriminator layer"""
d = Conv2D(filters, kernel_size=f_size, strides=2, padding='same')(layer_input)
d = LeakyReLU(alpha=0.2)(d)
if normalization:
d = InstanceNormalization()(d)
return d
img = Input(shape=self.img_shape)
d1 = d_layer(img, self.df, normalization=False)
d2 = d_layer(d1, self.df*2)
d3 = d_layer(d2, self.df*4)
d4 = d_layer(d3, self.df*8)
validity = Conv2D(1, kernel_size=4, strides=1, padding='same')(d4)
return Model(img, validity)
def build_classifier(self):
def clf_layer(layer_input, filters, f_size=4, normalization=True):
"""Classifier layer"""
d = Conv2D(filters, kernel_size=f_size, strides=2, padding='same')(layer_input)
d = LeakyReLU(alpha=0.2)(d)
if normalization:
d = InstanceNormalization()(d)
return d
img = Input(shape=self.img_shape)
c1 = clf_layer(img, self.cf, normalization=False)
c2 = clf_layer(c1, self.cf*2)
c3 = clf_layer(c2, self.cf*4)
c4 = clf_layer(c3, self.cf*8)
c5 = clf_layer(c4, self.cf*8)
class_pred = Dense(self.num_classes, activation='softmax')(Flatten()(c5))
return Model(img, class_pred)
def train(self, epochs, batch_size=128, sample_interval=50):
half_batch = int(batch_size / 2)
# Classification accuracy on 100 last batches of domain B
test_accs = []
# Adversarial ground truths
valid = np.ones((batch_size, *self.disc_patch))
fake = np.zeros((batch_size, *self.disc_patch))
for epoch in range(epochs):
# ---------------------
# Train Discriminator
# ---------------------
imgs_A, labels_A = self.data_loader.load_data(domain="A", batch_size=batch_size)
imgs_B, labels_B = self.data_loader.load_data(domain="B", batch_size=batch_size)
# Translate images from domain A to domain B
fake_B = self.generator.predict(imgs_A)
# Train the discriminators (original images = real / translated = Fake)
d_loss_real = self.discriminator.train_on_batch(imgs_B, valid)
d_loss_fake = self.discriminator.train_on_batch(fake_B, fake)
d_loss = 0.5 * np.add(d_loss_real, d_loss_fake)
# --------------------------------
# Train Generator and Classifier
# --------------------------------
# One-hot encoding of labels
labels_A = to_categorical(labels_A, num_classes=self.num_classes)
# Train the generator and classifier
g_loss = self.combined.train_on_batch(imgs_A, [valid, labels_A])
#-----------------------
# Evaluation (domain B)
#-----------------------
pred_B = self.clf.predict(imgs_B)
test_acc = np.mean(np.argmax(pred_B, axis=1) == labels_B)
# Add accuracy to list of last 100 accuracy measurements
test_accs.append(test_acc)
if len(test_accs) > 100:
test_accs.pop(0)
# Plot the progress
print ( "%d : [D - loss: %.5f, acc: %3d%%], [G - loss: %.5f], [clf - loss: %.5f, acc: %3d%%, test_acc: %3d%% (%3d%%)]" % \
(epoch, d_loss[0], 100*float(d_loss[1]),
g_loss[1], g_loss[2], 100*float(g_loss[-1]),
100*float(test_acc), 100*float(np.mean(test_accs))))
# If at save interval => save generated image samples
if epoch % sample_interval == 0:
self.sample_images(epoch)
def sample_images(self, epoch):
r, c = 2, 5
imgs_A, _ = self.data_loader.load_data(domain="A", batch_size=5)
# Translate images to the other domain
fake_B = self.generator.predict(imgs_A)
gen_imgs = np.concatenate([imgs_A, fake_B])
# Rescale images 0 - 1
gen_imgs = 0.5 * gen_imgs + 0.5
#titles = ['Original', 'Translated']
fig, axs = plt.subplots(r, c)
cnt = 0
for i in range(r):
for j in range(c):
axs[i,j].imshow(gen_imgs[cnt])
#axs[i, j].set_title(titles[i])
axs[i,j].axis('off')
cnt += 1
fig.savefig("images/%d.png" % (epoch))
plt.close()
def __init__(self):
# Input shape
self.channels = 3
self.lr_height = 64 # Low resolution height
self.lr_width = 64 # Low resolution width
self.lr_shape = (self.lr_height, self.lr_width, self.channels)
self.hr_height = self.lr_height * 4 # High resolution height
self.hr_width = self.lr_width * 4 # High resolution width
self.hr_shape = (self.hr_height, self.hr_width, self.channels)
# Number of residual blocks in the generator
self.n_residual_blocks = 16
optimizer = Adam(0.0002, 0.5)
# We use a pre-trained VGG19 model to extract image features from the high resolution
# and the generated high resolution images and minimize the mse between them
self.vgg = self.build_vgg()
self.vgg.trainable = False
self.vgg.compile(loss='mse', optimizer=optimizer, metrics=['accuracy'])
# Configure data loader
self.dataset_name = 'img_align_celeba'
self.data_loader = DataLoader(dataset_name=self.dataset_name,
img_res=(self.hr_height, self.hr_width))
# Calculate output shape of D (PatchGAN)
patch = int(self.hr_height / 2**4)
self.disc_patch = (patch, patch, 1)
# Number of filters in the first layer of G and D
self.gf = 64
self.df = 64
# Build and compile the discriminator
self.discriminator = self.build_discriminator()
self.discriminator.compile(loss='mse',
optimizer=optimizer,
metrics=['accuracy'])
# Build and compile the generator
self.generator = self.build_generator()
self.generator.compile(loss='binary_crossentropy', optimizer=optimizer)
# High res. and low res. images
img_hr = Input(shape=self.hr_shape)
img_lr = Input(shape=self.lr_shape)
# Generate high res. version from low res.
fake_hr = self.generator(img_lr)
# Extract image features of the generated img
fake_features = self.vgg(fake_hr)
# For the combined model we will only train the generator
self.discriminator.trainable = False
# Discriminator determines validity of generated high res. images
validity = self.discriminator(fake_hr)
self.combined = Model([img_lr, img_hr], [validity, fake_features])
self.combined.compile(loss=['binary_crossentropy', 'mse'],
loss_weights=[1e-3, 1],
optimizer=optimizer)
sys.path.append('/home/MLDickS/MovieQA_benchmark/.')
from data_loader import DataLoader
pickBestNum = 3
cosinSim = 0.75
split = 'val' #'train' OR 'val' OR 'test' OR 'full'
story_type='plot' #'plot', 'subtitle', 'dvs', 'script'
wordvec_file = '../GloVe/glove.6B.300d.txt'
#dataPickle_name = "../Memmap/"+str(split)+"."+str(story_type)+".lstm.memmap"
dataPickle_name = "../Pickle/"+str(split)+"."+str(story_type)+".best="+str(pickBestNum)+".pruned=300.lstm.pickle"
print "Saving to : ",dataPickle_name
DL = DataLoader()
story,qa = DL.get_story_qa_data(split,story_type)
def wordToVec(entry):
tempList = []
for i in range(1): #null loop, just for the right indent
temp_vector = np.zeros(300,dtype='float32')
for word in entry:
word = word.lower()
if word not in word_vec:
if '\'s' in word:
word = word.split('\'')[0]
elif 'n\'t' in word:
temp_vector = np.asarray(word_vec[word.split('n')[0]])
word = 'not'
elif '\'d' in word:
import numpy as np
import mxnet as mx
from mxnet import nd, autograd, gluon
from mxnet.gluon.data import dataset
from data_loader import DataLoader
from multiprocessing import cpu_count
import matplotlib.pyplot as plt
import random
import sys
args = sys.argv[1]
ctx = mx.cpu()
dl = DataLoader()
batch_size = 64
num_inputs = 784
num_outputs = 10
class CustomDataset(dataset.Dataset):
def __init__(self, mode, val):
self.X, self.y = dl.load_data(mode, val)
def __getitem__(self, idx):
return self.X[idx], self.y[idx]
def __len__(self):
return len(self.y)
iter = 0
while (iter < MIN_ITERATIONS) or (rmse < last_rmse - MIN_IMPROVEMENT):
last_rmse = rmse
squared_error = sgd_inner(feature, A_row, A_col, A_data, user_feature_matrix, movie_feature_matrix, NUM_FEATURES)
rmse = (squared_error / num_ratings) ** 0.5
iter += 1
print ('Squared error = %f' % squared_error)
print ('RMSE = %f' % rmse)
print ('Feature = %d' % feature)
return last_rmse
LAMBDA = 0.02
FEATURE_INIT_VALUE = 0.1
NUM_FEATURES = 40
movielens_file_path = '%s/ml-100k/u1.base' % os.environ['PWD']
A = DataLoader.create_review_matrix(movielens_file_path)
from scipy.io import mmread, mmwrite
mmwrite('./data/A', A)
user_feature_matrix = create_user_feature_matrix(A, NUM_FEATURES, FEATURE_INIT_VALUE)
movie_feature_matrix = create_movie_feature_matrix(A, NUM_FEATURES, FEATURE_INIT_VALUE)
users, movies = A.nonzero()
A = A.tocoo()
rmse = calculate_features(A.row, A.col, A.data, user_feature_matrix, movie_feature_matrix, NUM_FEATURES )
print ('rmse ', rmse)
class Pix2Pix():
def __init__(self):
# Input shape
self.img_rows = 256
self.img_cols = 256
self.channels = 3
self.img_shape = (self.img_rows, self.img_cols, self.channels)
# Configure data loader
self.dataset_name = 'facades'
self.data_loader = DataLoader(dataset_name=self.dataset_name,
img_res=(self.img_rows, self.img_cols))
# Calculate output shape of D (PatchGAN)
patch = int(self.img_rows / 2**4)
self.disc_patch = (patch, patch, 1)
# Number of filters in the first layer of G and D
self.gf = 64
self.df = 64
optimizer = Adam(0.0002, 0.5)
# Build and compile the discriminator
self.discriminator = self.build_discriminator()
self.discriminator.compile(loss='mse',
optimizer=optimizer,
metrics=['accuracy'])
# Build and compile the generator
self.generator = self.build_generator()
self.generator.compile(loss='binary_crossentropy', optimizer=optimizer)
# Input images and their conditioning images
img_A = Input(shape=self.img_shape)
img_B = Input(shape=self.img_shape)
# By conditioning on B generate a fake version of A
fake_A = self.generator(img_B)
# For the combined model we will only train the generator
self.discriminator.trainable = False
# Discriminators determines validity of translated images / condition pairs
valid = self.discriminator([fake_A, img_B])
self.combined = Model([img_A, img_B], [valid, fake_A])
self.combined.compile(loss=['mse', 'mae'],
loss_weights=[1, 100],
optimizer=optimizer)
def build_generator(self):
"""U-Net Generator"""
def conv2d(layer_input, filters, f_size=4, bn=True):
"""Layers used during downsampling"""
d = Conv2D(filters, kernel_size=f_size, strides=2,
padding='same')(layer_input)
d = LeakyReLU(alpha=0.2)(d)
if bn:
d = BatchNormalization(momentum=0.8)(d)
return d
def deconv2d(layer_input,
skip_input,
filters,
f_size=4,
dropout_rate=0):
"""Layers used during upsampling"""
u = UpSampling2D(size=2)(layer_input)
u = Conv2D(filters,
kernel_size=f_size,
strides=1,
padding='same',
activation='relu')(u)
if dropout_rate:
u = Dropout(dropout_rate)(u)
u = BatchNormalization(momentum=0.8)(u)
u = Concatenate()([u, skip_input])
return u
# Image input
d0 = Input(shape=self.img_shape)
# Downsampling
d1 = conv2d(d0, self.gf, bn=False)
d2 = conv2d(d1, self.gf * 2)
d3 = conv2d(d2, self.gf * 4)
d4 = conv2d(d3, self.gf * 8)
d5 = conv2d(d4, self.gf * 8)
d6 = conv2d(d5, self.gf * 8)
d7 = conv2d(d6, self.gf * 8)
# Upsampling
u1 = deconv2d(d7, d6, self.gf * 8)
u2 = deconv2d(u1, d5, self.gf * 8)
u3 = deconv2d(u2, d4, self.gf * 8)
u4 = deconv2d(u3, d3, self.gf * 4)
u5 = deconv2d(u4, d2, self.gf * 2)
u6 = deconv2d(u5, d1, self.gf)
u7 = UpSampling2D(size=2)(u6)
output_img = Conv2D(self.channels,
kernel_size=4,
strides=1,
padding='same',
activation='tanh')(u7)
return Model(d0, output_img)
def build_discriminator(self):
def d_layer(layer_input, filters, f_size=4, bn=True):
"""Discriminator layer"""
d = Conv2D(filters, kernel_size=f_size, strides=2,
padding='same')(layer_input)
d = LeakyReLU(alpha=0.2)(d)
if bn:
d = BatchNormalization(momentum=0.8)(d)
return d
img_A = Input(shape=self.img_shape)
img_B = Input(shape=self.img_shape)
# Concatenate image and conditioning image by channels to produce input
combined_imgs = Concatenate(axis=-1)([img_A, img_B])
d1 = d_layer(combined_imgs, self.df, bn=False)
d2 = d_layer(d1, self.df * 2)
d3 = d_layer(d2, self.df * 4)
d4 = d_layer(d3, self.df * 8)
validity = Conv2D(1, kernel_size=4, strides=1, padding='same')(d4)
return Model([img_A, img_B], validity)
def train(self, epochs, batch_size=1, save_interval=50):
start_time = datetime.datetime.now()
for epoch in range(epochs):
# ----------------------
# Train Discriminator
# ----------------------
# Sample images and their conditioning counterparts
imgs_A, imgs_B = self.data_loader.load_data(batch_size)
# Condition on B and generate a translated version
fake_A = self.generator.predict(imgs_B)
valid = np.ones((batch_size, ) + self.disc_patch)
fake = np.zeros((batch_size, ) + self.disc_patch)
# Train the discriminators (original images = real / generated = Fake)
d_loss_real = self.discriminator.train_on_batch([imgs_A, imgs_B],
valid)
d_loss_fake = self.discriminator.train_on_batch([fake_A, imgs_B],
fake)
d_loss = 0.5 * np.add(d_loss_real, d_loss_fake)
# ------------------
# Train Generator
# ------------------
# Sample images and their conditioning counterparts
imgs_A, imgs_B = self.data_loader.load_data(batch_size)
# The generators want the discriminators to label the generated images as real
valid = np.ones((batch_size, ) + self.disc_patch)
# Train the generators
g_loss = self.combined.train_on_batch([imgs_A, imgs_B],
[valid, imgs_A])
elapsed_time = datetime.datetime.now() - start_time
# Plot the progress
print("%d time: %s" % (epoch, elapsed_time))
# If at save interval => save generated image samples
if epoch % save_interval == 0:
self.save_imgs(epoch)
def save_imgs(self, epoch):
os.makedirs('images/%s' % self.dataset_name, exist_ok=True)
r, c = 3, 3
imgs_A, imgs_B = self.data_loader.load_data(batch_size=3,
is_testing=True)
fake_A = self.generator.predict(imgs_B)
gen_imgs = np.concatenate([imgs_B, fake_A, imgs_A])
# Rescale images 0 - 1
gen_imgs = 0.5 * gen_imgs + 0.5
titles = ['Condition', 'Generated', 'Original']
fig, axs = plt.subplots(r, c)
cnt = 0
for i in range(r):
for j in range(c):
axs[i, j].imshow(gen_imgs[cnt])
axs[i, j].set_title(titles[i])
axs[i, j].axis('off')
cnt += 1
fig.savefig("images/%s/%d.png" % (self.dataset_name, epoch))
plt.close()
def __init__(self):
# Input shape
self.img_rows = 128
self.img_cols = 128
self.channels = 3
self.img_shape = (self.img_rows, self.img_cols, self.channels)
# Configure data loader
self.dataset_name = 'edges2shoes'
self.data_loader = DataLoader(dataset_name=self.dataset_name,
img_res=(self.img_rows, self.img_cols))
# Calculate output shape of D (PatchGAN)
patch = int(self.img_rows / 2**4)
self.disc_patch = (patch, patch, 1)
# Number of filters in the first layer of G and D
self.gf = 64
self.df = 64
optimizer = Adam(0.0002, 0.5)
# Build and compile the discriminators
self.d_A = self.build_discriminator()
self.d_B = self.build_discriminator()
self.d_A.compile(loss='mse',
optimizer=optimizer,
metrics=['accuracy'])
self.d_B.compile(loss='mse',
optimizer=optimizer,
metrics=['accuracy'])
#-------------------------
# Construct Computational
# Graph of Generators
#-------------------------
# Build the generators
self.g_AB = self.build_generator()
self.g_BA = self.build_generator()
# Input images from both domains
img_A = Input(shape=self.img_shape)
img_B = Input(shape=self.img_shape)
# Translate images to the other domain
fake_B = self.g_AB(img_A)
fake_A = self.g_BA(img_B)
# Translate images back to original domain
reconstr_A = self.g_BA(fake_B)
reconstr_B = self.g_AB(fake_A)
# For the combined model we will only train the generators
self.d_A.trainable = False
self.d_B.trainable = False
# Discriminators determines validity of translated images
valid_A = self.d_A(fake_A)
valid_B = self.d_B(fake_B)
# Objectives
# + Adversarial: Fool domain discriminators
# + Translation: Minimize MAE between e.g. fake B and true B
# + Cycle-consistency: Minimize MAE between reconstructed images and original
self.combined = Model(inputs=[img_A, img_B],
outputs=[ valid_A, valid_B,
fake_B, fake_A,
reconstr_A, reconstr_B ])
self.combined.compile(loss=['mse', 'mse',
'mae', 'mae',
'mae', 'mae'],
optimizer=optimizer)
class CycleGAN():
def __init__(self):
self.img_rows = 128
self.img_cols = 128
self.channels = 3
self.img_shape = (self.img_rows, self.img_cols, self.channels)
self.dataset_name = 'apple2orange'
self.data_loader = DataLoader(dataset_name=self.dataset_name,
img_res=(self.img_rows, self.img_cols))
patch = int(self.img_rows / 2**4)
self.disc_patch = (patch, patch, 1)
self.gf = 32
self.df = 64
self.lambda_cycle = 10.0
self.lambda_id = 0.9 * self.lambda_cycle
optimizer = Adam(2e-4, 0.5)
self.d_A = self.build_discriminator()
self.d_B = self.build_discriminator()
self.d_A.compile(loss='mse', optimizer=optimizer, metrics=['accuracy'])
self.d_B.compile(loss='mse', optimizer=optimizer, metrics=['accuracy'])
self.g_AB = self.build_generator()
self.g_BA = self.build_generator()
img_A = Input(shape=self.img_shape)
img_B = Input(shape=self.img_shape)
fake_B = self.g_AB(img_A)
fake_A = self.g_BA(img_B)
reconstr_A = self.g_BA(fake_B)
reconstr_B = self.g_AB(fake_A)
img_A_id = self.g_BA(img_A)
img_B_id = self.g_AB(img_B)
self.d_A.trainable = False
self.d_B.trainable = False
valid_A = self.d_A(fake_A)
valid_B = self.d_B(fake_B)
self.combined = Model(inputs=[img_A, img_B],
outputs=[
valid_A, valid_B, reconstr_A, reconstr_B,
img_A_id, img_B_id
])
self.combined.compile(loss=['mse', 'mse', 'mae', 'mae', 'mae', 'mae'],
loss_weights=[
1, 1, self.lambda_cycle, self.lambda_cycle,
self.lambda_id, self.lambda_id
],
optimizer=optimizer)
def build_generator(self):
def conv2d(layer_input, filters, f_size=4):
d = Conv2D(filters, kernel_size=f_size, strides=2,
padding='same')(layer_input)
d = LeakyReLU(alpha=0.2)(d)
d = InstanceNormalization()(d)
return d
def deconv2d(layer_input,
skip_input,
filters,
f_size=4,
dropout_rate=0):
u = UpSampling2D(size=2)(layer_input)
u = Conv2D(filters,
kernel_size=f_size,
strides=1,
padding='same',
activation='relu')(u)
if dropout_rate:
u = Dropout(dropout_rate)(u)
u = InstanceNormalization()(u)
u = Concatenate()([u, skip_input])
return u
d0 = Input(shape=self.img_shape)
d1 = conv2d(d0, self.gf)
d2 = conv2d(d1, self.gf * 2)
d3 = conv2d(d2, self.gf * 4)
d4 = conv2d(d3, self.gf * 8)
u1 = deconv2d(d4, d3, self.gf * 4)
u2 = deconv2d(u1, d2, self.gf * 2)
u3 = deconv2d(u2, d1, self.gf)
u4 = UpSampling2D(size=2)(u3)
output_img = Conv2D(self.channels,
kernel_size=4,
strides=1,
padding='same',
activation='tanh')(u4)
return Model(d0, output_img)
def build_discriminator(self):
def d_layer(layer_input, filters, f_size=4, normalization=True):
d = Conv2D(filters, kernel_size=f_size, strides=2,
padding='same')(layer_input)
d = LeakyReLU(alpha=0.2)(d)
if normalization:
d = InstanceNormalization()(d)
return d
img = Input(shape=self.img_shape)
d1 = d_layer(img, self.df, normalization=False)
d2 = d_layer(d1, self.df * 2)
d3 = d_layer(d2, self.df * 4)
d4 = d_layer(d3, self.df * 8)
validity = Conv2D(1, kernel_size=4, strides=1, padding='same')(d4)
return Model(img, validity)
def sample_images(self, epoch, batch_i):
r, c = 2, 3
imgs_A = self.data_loader.load_data(domain='A',
batch_size=1,
is_testing=True)
imgs_B = self.data_loader.load_data(domain='B',
batch_size=1,
is_testing=True)
fake_B = self.g_AB.predict(imgs_A)
fake_A = self.g_BA.predict(imgs_B)
reconstr_A = self.g_BA.predict(fake_B)
reconstr_B = self.g_AB.predict(fake_A)
gen_imgs = np.concatenate(
[imgs_A, fake_B, reconstr_A, imgs_B, fake_A, reconstr_B])
gen_imgs = 0.5 * gen_imgs + 0.5
titles = ['Original', 'Translated', 'Reconstructed']
fig, axs = plt.subplots(r, c)
cnt = 0
for i in range(r):
for j in range(c):
axs[i, j].imshow(gen_imgs[cnt])
axs[i, j].set_title(titles[j])
axs[i, j].axis('off')
cnt += 1
fig.savefig('images/%s/%d_%d.png' %
(self.dataset_name, epoch, batch_i))
plt.show()
def train(self, epochs, batch_size=1, sample_interval=50):
start_time = datetime.datetime.now()
valid = np.ones((batch_size, ) + self.disc_patch)
fake = np.zeros((batch_size, ) + self.disc_patch)
for epoch in range(epochs):
for batch_i, (imgs_A, imgs_B) in enumerate(
self.data_loader.load_batch(batch_size)):
fake_B = self.g_AB.predict(imgs_A)
fake_A = self.g_BA.predict(imgs_B)
dA_loss_real = self.d_A.train_on_batch(imgs_A, valid)
dA_loss_fake = self.d_A.train_on_batch(fake_A, fake)
dA_loss = 0.5 * np.add(dA_loss_real, dA_loss_fake)
dB_loss_real = self.d_B.train_on_batch(imgs_B, valid)
dB_loss_fake = self.d_B.train_on_batch(fake_B, fake)
dB_loss = 0.5 * np.add(dB_loss_real, dB_loss_fake)
d_loss = 0.5 * np.add(dA_loss, dB_loss)
g_loss = self.combined.train_on_batch(
[imgs_A, imgs_B],
[valid, valid, imgs_A, imgs_B, imgs_A, imgs_B])
if batch_i % sample_interval == 0:
self.sample_images(epoch, batch_i)
criterionD = nn.BCEWithLogitsLoss().cuda()
criterionG = nn.BCEWithLogitsLoss().cuda()
real = Variable(torch.FloatTensor(batch_size, x_dim),
requires_grad=False).cuda()
noise = Variable(torch.FloatTensor(batch_size, z_dim),
requires_grad=False).cuda()
label_zero = Variable(torch.FloatTensor(batch_size, 1).fill_(0),
requires_grad=False).cuda()
label_one = Variable(torch.FloatTensor(batch_size, 1).fill_(1),
requires_grad=False).cuda()
for epoch in xrange(num_epochs):
data_loader = DataLoader(data_file,
batch_size=batch_size,
shuffle=True,
repeat=False).iterator()
batch_num = 0
while True:
try:
for p in d.parameters():
p.requires_grad = True
# train D
for _ in xrange(1):
# pdb.set_trace()
optimD.zero_grad()
batch = data_loader.next()
x = torch.from_numpy(batch)
real.data.copy_(x)
logit_real = d(real)
import time
import operator
from pyspark.mllib.tree import RandomForest
from spark_application import create_spark_application
from data_loader import DataLoader
from reader import read_districts_file
# Get file paths from arguments
if len(sys.argv) != 4:
print "Usage: random_forest.py FEATURES_FILE MODEL_FOLDER DISTRICTS_FILE"
sys.exit()
features_file, model_folder, districts_file = sys.argv[1:]
spark_context, sql_context = create_spark_application("train_random_forest")
data_loader = DataLoader(spark_context, sql_context, features_file)
#for the random forest scaling and onehot-encoding are disabled because they better fit to linear regression models
data_loader.initialize(do_scaling=False, do_onehot=False)
#in the case of decision trees categorical features make more sense
maxBins = 32
categorical_features_info = data_loader.get_categorical_features_info()
if categorical_features_info and max(categorical_features_info.values()) > maxBins:
maxBins = max(categorical_features_info.values())
# train and store a model for each district in the districts file
for lat, lon in read_districts_file(districts_file):
print("Training District: %f, %f" % (lat, lon))
start = time.time()
model = RandomForest.trainRegressor(data_loader.get_train_data((lat, lon)),
categoricalFeaturesInfo=categorical_features_info,
