def main():
#Architectures
from EcalEnergyGan import generator, discriminator
disc_weights = "ch_pionweights/params_discriminator_epoch_050.hdf5"
gen_weights = "ch_pionweights/params_generator_epoch_050.hdf5"
plots_dir = "ch_pion_s1000_ep51_2p1p1/"
latent = 200
num_data = 150000
num_events = 3000
m = 3
thresh = 1e-6
energies = [0, 50, 100, 200, 250, 300, 400, 500]
particle = 'ChPi'
datapath = '/bigdata/shared/LCD/NewV1/*scan/*.h5' #Training data path
#datapath = '/eos/project/d/dshep/LCD/V1/*scan/*.h5'
sortdir = 'SortedData'
gendir = 'Gen'
discdir = 'Disc'
Test = False
save_data = False # True if the sorted data is to be saved. It only saves when read_data is false
read_data = False # True if loading previously sorted data
save_gen = False # True if saving generated data.
read_gen = False # True if generated data is already saved and can be loaded
save_disc = False # True if discriminiator data is to be saved
read_disc = False # True if discriminated data is to be loaded from previously saved file
flags = [
Test, save_data, read_data, save_gen, read_gen, save_disc, read_disc
]
dweights = [disc_weights]
gweights = [gen_weights]
scales = [1000]
d = discriminator()
g = generator(latent)
var = perform_calculations(g,
d,
gweights,
dweights,
energies,
datapath,
sortdir,
gendir,
discdir,
num_data,
num_events,
m,
scales,
flags,
latent,
thresh,
particle=particle)
get_plots(var, plots_dir, energies, m, len(gweights))
def main():
#Architectures to import
from EcalEnergyGan import generator, discriminator
#Values to be set by user
parser = get_parser()
params = parser.parse_args()
nb_epochs = params.nbepochs #Total Epochs
batch_size = params.batchsize #batch size
latent_size = params.latentsize #latent vector size
verbose = params.verbose
datapath = '/bigdata/shared/LCD/NewV1/*scan/*.h5' #Data path on Caltech
#datapath = params.datapath#Data path on EOS CERN default
EventsperFile = params.nbperfile #Events in a file
nEvents = params.nbEvents #Total events for training
fitmod = params.mod
weightdir = params.weightsdir
xscale = params.xscale
pklfile = params.pklfile
print(params)
gan.safe_mkdir(weightdir)
# Analysis
analysis = True # if analysing
energies = [100, 200, 300, 400] # Bins
resultfile = 'results/3dgan_analysis.pkl' # analysis result
# Building discriminator and generator
d = discriminator()
g = generator(latent_size)
Gan3DTrain(d,
g,
datapath,
EventsperFile,
nEvents,
weightdir,
pklfile,
resultfile,
mod=fitmod,
nb_epochs=nb_epochs,
batch_size=batch_size,
latent_size=latent_size,
gen_weight=2,
aux_weight=0.1,
ecal_weight=0.1,
xscale=xscale,
analysis=analysis,
energies=energies)
def main():
#Architectures
from EcalEnergyGan import generator, discriminator
disc_weights = "params_discriminator_epoch_041.hdf5"
gen_weights = "params_generator_epoch_041.hdf5"
plots_dir = "correlation_plots/"
latent = 200
num_data = 100000
num_events = 2000
m = 3
energies = [0, 50, 100, 200, 250, 300, 400, 500]
particle = 'Ele'
#datapath = '/bigdata/shared/LCD/NewV1/*scan/*.h5' #Training data path caltech
datapath = '/eos/project/d/dshep/LCD/V1/*scan/*.h5' # Training data CERN EOS
sortdir = 'SortedData'
gendir = 'Gen'
discdir = 'Disc'
Test = True
stest = False
save_data = False # True if the sorted data is to be saved. It only saves when read_data is false
read_data = False # True if loading previously sorted data
save_gen = False # True if saving generated data.
read_gen = False # True if generated data is already saved and can be loaded
save_disc = False # True if discriminiator data is to be saved
read_disc = False # True if discriminated data is to be loaded from previously saved file
ifpdf = True # True if pdf are required. If false .C files will be generated
flags = [
Test, save_data, read_data, save_gen, read_gen, save_disc, read_disc
]
# Lists for different versions comparison. The weights, scales and labels will need to be provided for each version
dweights = [disc_weights]
gweights = [gen_weights]
scales = [100]
labels = ['']
d = discriminator()
g = generator(latent)
var = perform_calculations_multi(g, d, gweights, dweights, energies,
datapath, sortdir, gendir, discdir,
num_data, num_events, m, scales, flags,
latent, particle)
get_plots_multi(var, labels, plots_dir, energies, m, len(gweights), ifpdf,
stest)
def main():
#Values to be set by user
parser = get_parser()
params = parser.parse_args()
#nb_epochs = params.nbepochs #Total Epochs
nb_epochs = 50
batch_size = params.batchsize #batch size
latent_size = params.latentsize #latent vector size
verbose = params.verbose
datapath = '/bigdata/shared/LCD/NewV1/*scan/*.h5' #Data path on Caltech
#datapath = params.datapath#Data path on EOS CERN default
EventsperFile = params.nbperfile #Events in a file
nEvents = params.nbEvents #Total events for training
#fitmod = params.mod
fitmod = 1
#weightdir = params.weightsdir
weightdir = 'ch_pionweights'
#xscale = params.xscale
xscale = 1000
#pklfile = params.pklfile
pklfile = 'dcgan-ch-pion-history.pkl'
particle = params.particle
limit = params.limit
print(params)
from EcalEnergyGan import generator, discriminator
# Building discriminator and generator
d = discriminator()
g = generator(latent_size)
Gan3DTrain(d,
g,
datapath,
nEvents,
weightdir,
pklfile,
particle,
limit,
mod=fitmod,
nb_epochs=nb_epochs,
batch_size=batch_size,
gen_weight=2,
aux_weight=0.1,
ecal_weight=0.1,
xscale=xscale)
import setGPU
from EcalEnergyGan import generator, discriminator
#gStyle.SetOptStat(0)
gStyle.SetOptFit(1111) # superimpose fit results
c = TCanvas("c", "Ecal/Ep versus Ep for Data and Generated Events", 200, 10,
700, 500) #make nice
c.SetGrid()
gStyle.SetOptStat(0)
#c.SetLogx ()
hist_g4 = ROOT.TH2F("g4", "Ratio of Ecal and Ep;Ep;Ecal/Ep", 400, 100, 500,
100, 0, 0.04)
hist_g4.Sumw2()
num_events = 1000
latent = 200
g = generator(latent)
#gweight = 'gen_rootfit_2p1p1_ep33.hdf5'
gweight1 = 'params_generator_epoch_041.hdf5' # 1 gpu
gweight2 = 'params_generator_epoch_005.hdf5' # 8 gpu
gweight3 = 'params_generator_epoch_002.hdf5' # 16 gpu
g.load_weights(gweight1)
gweights = [gweight1, gweight2, gweight3]
label = ['1 gpu', '8 gpu', '16 gpu']
scales = [100, 1, 1]
filename = 'ecal_ratio_stats_500.pdf'
#Get Actual Data
d = h5py.File("/bigdata/shared/LCD/NewV1/EleEscan/EleEscan_1_1.h5")
#d=h5py.File("/afs/cern.ch/work/g/gkhattak/public/Ele_v1_1_2.h5",'r')
X = np.array(d.get('ECAL')[0:num_events], np.float64)
Y = np.array(d.get('target')[0:num_events][:, 1], np.float64)
X[X < 1e-6] = 0
def Gan3DTrain(discriminator,
generator,
datapath,
EventsperFile,
nEvents,
WeightsDir,
pklfile,
resultfile,
mod=0,
nb_epochs=30,
batch_size=128,
latent_size=200,
gen_weight=6,
aux_weight=0.2,
ecal_weight=0.1,
lr=0.001,
rho=0.9,
decay=0.0,
g_weights='params_generator_epoch_',
d_weights='params_discriminator_epoch_',
xscale=1,
analysis=False,
energies=[]):
start_init = time.time()
verbose = False
particle = 'Ele'
f = [0.9, 0.1]
print('[INFO] Building discriminator')
#discriminator.summary()
discriminator.compile(optimizer=RMSprop(),
loss=[
'binary_crossentropy',
'mean_absolute_percentage_error',
'mean_absolute_percentage_error'
],
loss_weights=[gen_weight, aux_weight, ecal_weight])
# build the generator
print('[INFO] Building generator')
#generator.summary()
generator.compile(optimizer=RMSprop(), loss='binary_crossentropy')
# build combined Model
latent = Input(shape=(latent_size, ), name='combined_z')
fake_image = generator(latent)
discriminator.trainable = False
fake, aux, ecal = discriminator(fake_image)
combined = Model(input=[latent],
output=[fake, aux, ecal],
name='combined_model')
combined.compile(
#optimizer=Adam(lr=adam_lr, beta_1=adam_beta_1),
optimizer=RMSprop(),
loss=[
'binary_crossentropy', 'mean_absolute_percentage_error',
'mean_absolute_percentage_error'
],
loss_weights=[gen_weight, aux_weight, ecal_weight])
# Getting Data
Trainfiles, Testfiles = gan.DivideFiles(datapath,
nEvents=nEvents,
EventsperFile=EventsperFile,
datasetnames=["ECAL"],
Particles=[particle])
print('The total data was divided in {} Train files and {} Test files'.
format(len(Trainfiles), len(Testfiles)))
nb_test = int(nEvents * f[1])
#Read test data into a single array
for index, dtest in enumerate(Testfiles):
if index == 0:
X_test, Y_test, ecal_test = GetprocData(dtest, xscale=xscale)
else:
if X_test.shape[0] < nb_test:
X_temp, Y_temp, ecal_temp = GetprocData(dtest, xscale=xscale)
X_test = np.concatenate((X_test, X_temp))
Y_test = np.concatenate((Y_test, Y_temp))
ecal_test = np.concatenate((ecal_test, ecal_temp))
X_test, Y_test, ecal_test = X_test[:
nb_test], Y_test[:
nb_test], ecal_test[:
nb_test]
nb_train = int(nEvents * f[0]) #
total_batches = int(nb_train / batch_size)
print(
'In this experiment {} events will be used for training as {}batches'.
format(nb_train, total_batches))
print('{} events will be used for Testing'.format(nb_test))
train_history = defaultdict(list)
test_history = defaultdict(list)
analysis_history = defaultdict(list)
init_time = time.time() - start_init
print('Initialization time is {} seconds'.format(init_time))
for epoch in range(nb_epochs):
epoch_start = time.time()
print('Epoch {} of {}'.format(epoch + 1, nb_epochs))
X_train, Y_train, ecal_train = GetprocData(Trainfiles[0],
xscale=xscale)
nb_file = 1
nb_batches = int(X_train.shape[0] / batch_size)
if verbose:
progress_bar = Progbar(target=total_batches)
epoch_gen_loss = []
epoch_disc_loss = []
file_index = 0
for index in np.arange(total_batches):
if verbose:
progress_bar.update(index)
else:
if index % 100 == 0:
print('processed {}/{} batches'.format(
index + 1, total_batches))
loaded_data = X_train.shape[0]
used_data = file_index * batch_size
if (loaded_data - used_data) < batch_size + 1 and (
nb_file < len(Trainfiles)):
X_temp, Y_temp, ecal_temp = GetprocData(Trainfiles[nb_file],
xscale=xscale)
print("\nData file loaded..........", Trainfiles[nb_file])
nb_file += 1
X_left = X_train[(file_index * batch_size):]
Y_left = Y_train[(file_index * batch_size):]
ecal_left = ecal_train[(file_index * batch_size):]
X_train = np.concatenate((X_left, X_temp))
Y_train = np.concatenate((Y_left, Y_temp))
ecal_train = np.concatenate((ecal_left, ecal_temp))
nb_batches = int(X_train.shape[0] / batch_size)
print("{} batches loaded..........".format(nb_batches))
file_index = 0
image_batch = X_train[(file_index * batch_size):(file_index + 1) *
batch_size]
energy_batch = Y_train[(file_index * batch_size):(file_index + 1) *
batch_size]
ecal_batch = ecal_train[(file_index *
batch_size):(file_index + 1) * batch_size]
file_index += 1
noise = np.random.normal(0, 1, (batch_size, latent_size))
sampled_energies = np.random.uniform(0.1, 5, (batch_size, 1))
generator_ip = np.multiply(sampled_energies, noise)
#ecal sum from fit
ecal_ip = gan.GetEcalFit(sampled_energies, particle, mod, xscale)
generated_images = generator.predict(generator_ip, verbose=0)
real_batch_loss = discriminator.train_on_batch(
image_batch,
[gan.BitFlip(np.ones(batch_size)), energy_batch, ecal_batch])
fake_batch_loss = discriminator.train_on_batch(
generated_images,
[gan.BitFlip(np.zeros(batch_size)), sampled_energies, ecal_ip])
epoch_disc_loss.append([
(a + b) / 2 for a, b in zip(real_batch_loss, fake_batch_loss)
])
trick = np.ones(batch_size)
gen_losses = []
for _ in np.arange(2):
noise = np.random.normal(0, 1, (batch_size, latent_size))
sampled_energies = np.random.uniform(0.1, 5, (batch_size, 1))
generator_ip = np.multiply(sampled_energies, noise)
ecal_ip = gan.GetEcalFit(sampled_energies, particle, mod,
xscale)
gen_losses.append(
combined.train_on_batch(
[generator_ip],
[trick,
sampled_energies.reshape((-1, 1)), ecal_ip]))
epoch_gen_loss.append([(a + b) / 2 for a, b in zip(*gen_losses)])
print('The training took {} seconds.'.format(time.time() -
epoch_start))
print('\nTesting for epoch {}:'.format(epoch + 1))
test_start = time.time()
noise = np.random.normal(0.1, 1, (nb_test, latent_size))
sampled_energies = np.random.uniform(0.1, 5, (nb_test, 1))
generator_ip = np.multiply(sampled_energies, noise)
generated_images = generator.predict(generator_ip,
verbose=False,
batch_size=batch_size)
ecal_ip = gan.GetEcalFit(sampled_energies, particle, mod, xscale)
sampled_energies = np.squeeze(sampled_energies, axis=(1, ))
X = np.concatenate((X_test, generated_images))
y = np.array([1] * nb_test + [0] * nb_test)
ecal = np.concatenate((ecal_test, ecal_ip))
aux_y = np.concatenate((Y_test, sampled_energies), axis=0)
discriminator_test_loss = discriminator.evaluate(X, [y, aux_y, ecal],
verbose=False,
batch_size=batch_size)
discriminator_train_loss = np.mean(np.array(epoch_disc_loss), axis=0)
noise = np.random.normal(0.1, 1, (2 * nb_test, latent_size))
sampled_energies = np.random.uniform(0.1, 5, (2 * nb_test, 1))
generator_ip = np.multiply(sampled_energies, noise)
ecal_ip = gan.GetEcalFit(sampled_energies, particle, mod, xscale)
trick = np.ones(2 * nb_test)
generator_test_loss = combined.evaluate(
generator_ip,
[trick, sampled_energies.reshape((-1, 1)), ecal_ip],
verbose=False,
batch_size=batch_size)
generator_train_loss = np.mean(np.array(epoch_gen_loss), axis=0)
train_history['generator'].append(generator_train_loss)
train_history['discriminator'].append(discriminator_train_loss)
test_history['generator'].append(generator_test_loss)
test_history['discriminator'].append(discriminator_test_loss)
print('{0:<22s} | {1:4s} | {2:15s} | {3:5s}| {4:5s}'.format(
'component', *discriminator.metrics_names))
print('-' * 65)
ROW_FMT = '{0:<22s} | {1:<4.2f} | {2:<15.2f} | {3:<5.2f}| {4:<5.2f}'
print(
ROW_FMT.format('generator (train)',
*train_history['generator'][-1]))
print(
ROW_FMT.format('generator (test)', *test_history['generator'][-1]))
print(
ROW_FMT.format('discriminator (train)',
*train_history['discriminator'][-1]))
print(
ROW_FMT.format('discriminator (test)',
*test_history['discriminator'][-1]))
# save weights every epoch
generator.save_weights(WeightsDir +
'/{0}{1:03d}.hdf5'.format(g_weights, epoch),
overwrite=True)
discriminator.save_weights(WeightsDir +
'/{0}{1:03d}.hdf5'.format(d_weights, epoch),
overwrite=True)
print(
"The Testing for {} epoch took {} seconds. Weights are saved in {}"
.format(epoch,
time.time() - test_start, WeightsDir))
pickle.dump({
'train': train_history,
'test': test_history
}, open(pklfile, 'wb'))
if analysis:
var = gan.sortEnergy([X_test, Y_test], ecal_test, energies, ang=0)
result = gan.OptAnalysisShort(var,
generated_images,
energies,
ang=0)
print('Analysing............')
# All of the results correspond to mean relative errors on different quantities
analysis_history['total'].append(result[0])
analysis_history['energy'].append(result[1])
analysis_history['moment'].append(result[2])
print('Result = ', result)
pickle.dump({'results': analysis_history}, open(resultfile, 'wb'))
import time
time.sleep(10 * hvd.local_rank())
import setGPU
#config.gpu_options.visible_device_list = str(hvd.local_rank())
tf.Session(config=config)
g_weights = 'params_generator_epoch_'
d_weights = 'params_discriminator_epoch_'
nb_epochs = 25
batch_size = 128
latent_size = 200
verbose = 'false'
nb_classes = 2
generator = generator(latent_size)
discriminator = discriminator()
print('[INFO] Building discriminator')
discriminator.summary()
#discriminator.load_weights('veganweights/params_discriminator_epoch_019.hdf5')
# Add Horovod Distributed Optimizer.
opt = hvd.DistributedOptimizer(RMSprop())
discriminator.compile(
#optimizer=Adam(lr=adam_lr, beta_1=adam_beta_1),
optimizer=opt,
loss=[
'binary_crossentropy', 'mean_absolute_percentage_error',
'mean_absolute_percentage_error'
def Gan3DTrain2(discriminator, generator, datapath, EventsperFile, nEvents, WeightsDir, mod=0, nb_epochs=30, batch_size=128, latent_size=200, gen_weight=6, aux_weight=0.2, ecal_weight=0.1, lr=0.001, rho=0.9, decay=0.0, g_weights='params_generator_epoch_', d_weights='params_discriminator_epoch_', xscale=1):
start_init = time.time()
verbose = False
print('[INFO] Building discriminator')
#discriminator.summary()
discriminator.compile(
optimizer=RMSprop(),
loss=['binary_crossentropy', 'mean_absolute_percentage_error', 'mean_absolute_percentage_error'],
loss_weights=[gen_weight, aux_weight, ecal_weight]
)
# build the generator
print('[INFO] Building generator')
#generator.summary()
generator.compile(
optimizer=RMSprop(),
loss='binary_crossentropy'
)
# build combined Model
latent = Input(shape=(latent_size, ), name='combined_z')
fake_image = generator( latent)
discriminator.trainable = False
fake, aux, ecal = discriminator(fake_image)
combined = Model(
input=[latent],
output=[fake, aux, ecal],
name='combined_model'
)
combined.compile(
#optimizer=Adam(lr=adam_lr, beta_1=adam_beta_1),
optimizer=RMSprop(),
loss=['binary_crossentropy', 'mean_absolute_percentage_error', 'mean_absolute_percentage_error'],
loss_weights=[gen_weight, aux_weight, ecal_weight]
)
# Getting Data
Trainfiles, Testfiles = DivideFiles(datapath, nEvents=nEvents, EventsperFile = EventsperFile, datasetnames=["ECAL"], Particles =["Ele"])
print(Trainfiles)
print(Testfiles)
#Read test data into a single array
for index, dtest in enumerate(Testfiles):
if index == 0:
X_test, Y_test, ecal_test = GetprocData(dtest, xscale=xscale, limit= 1e-5)
else:
X_temp, Y_temp, ecal_temp = GetprocData(dtest, xscale=xscale, limit= 1e-5)
X_test = np.concatenate((X_test, X_temp))
Y_test = np.concatenate((Y_test, Y_temp))
ecal_test = np.concatenate((ecal_test, ecal_temp))
print('Test Data loaded of shapes:')
print(X_test.shape)
print(Y_test.shape)
print(Y_test[:10])
print('*************************************************************************************')
nb_test = X_test.shape[0]
nb_train = EventsperFile * len(Trainfiles)# Total events in training files
total_batches = nb_train / batch_size
print('Total Training batches = {} with {} events'.format(total_batches, nb_train))
train_history = defaultdict(list)
test_history = defaultdict(list)
init_time = time.time()- start_init
print('Initialization time is {} seconds'.format(init_time))
for epoch in range(nb_epochs):
epoch_start = time.time()
print('Epoch {} of {}'.format(epoch + 1, nb_epochs))
X_train, Y_train, ecal_train = GetprocData(Trainfiles[0], xscale=xscale, limit= 1e-5)
print(Y_train[:10])
nb_file=1
nb_batches = int(X_train.shape[0] / batch_size)
if verbose:
progress_bar = Progbar(target=total_batches)
epoch_gen_loss = []
epoch_disc_loss = []
file_index = 0
real_label = np.ones(batch_size)
for index in range(total_batches):
if verbose:
progress_bar.update(index)
else:
if index % 100 == 0:
print('processed {}/{} batches'.format(index + 1, total_batches))
loaded_data = X_train.shape[0]
used_data = file_index * batch_size
if (loaded_data - used_data) < batch_size + 1 and (nb_file < len(Trainfiles)):
X_temp, Y_temp, ecal_temp = GetprocData(Trainfiles[nb_file], xscale=xscale, limit= 1e-5)
print("\nData file loaded..........",Trainfiles[nb_file])
nb_file+=1
X_left = X_train[(file_index * batch_size):]
Y_left = Y_train[(file_index * batch_size):]
ecal_left = ecal_train[(file_index * batch_size):]
X_train = np.concatenate((X_left, X_temp))
Y_train = np.concatenate((Y_left, Y_temp))
ecal_train = np.concatenate((ecal_left, ecal_temp))
nb_batches = int(X_train.shape[0] / batch_size)
print("{} batches loaded..........".format(nb_batches))
file_index = 0
image_batch = X_train[(file_index * batch_size):(file_index + 1) * batch_size]
energy_batch = Y_train[(file_index * batch_size):(file_index + 1) * batch_size]
ecal_batch = ecal_train[(file_index * batch_size):(file_index + 1) * batch_size]
file_index +=1
X_for_disc,Y_for_disc,X_for_combined,Y_for_combined = batch_transform(generator, image_batch, [real_label, energy_batch, ecal_batch], latent_size)
disc_batchloss = discriminator.train_on_batch(X_for_disc, Y_for_disc)
gen_batchloss = combined.train_on_batch(X_for_combined,Y_for_combined)
epoch_disc_loss.append(disc_batchloss)
epoch_gen_loss.append(gen_batchloss)
print('\nTesting for epoch {}:'.format(epoch + 1))
noise = np.random.normal(0.1, 1, (nb_test, latent_size))
sampled_energies = np.random.uniform(0.1, 5, (nb_test, 1))
generator_ip = np.multiply(sampled_energies, noise)
generated_images = generator.predict(generator_ip, verbose=False)
ecal_ip = GetEcalFit(sampled_energies, mod, xscale)
sampled_energies = np.squeeze(sampled_energies, axis=(1,))
X = np.concatenate((X_test, generated_images))
y = np.array([1] * nb_test + [0] * nb_test)
ecal = np.concatenate((ecal_test, ecal_ip))
aux_y = np.concatenate((Y_test, sampled_energies), axis=0)
discriminator_test_loss = discriminator.evaluate(
X, [y, aux_y, ecal], verbose=False, batch_size=batch_size)
discriminator_train_loss = np.mean(np.array(epoch_disc_loss), axis=0)
noise = np.random.normal(0.1, 1, (2 * nb_test, latent_size))
sampled_energies = np.random.uniform(0.1, 5, (2 * nb_test, 1))
generator_ip = np.multiply(sampled_energies, noise)
ecal_ip = GetEcalFit(sampled_energies, mod, xscale)
trick = np.ones(2 * nb_test)
generator_test_loss = combined.evaluate(generator_ip,
[trick, sampled_energies.reshape((-1, 1)), ecal_ip], verbose=False, batch_size=batch_size)
generator_train_loss = np.mean(np.array(epoch_gen_loss), axis=0)
train_history['generator'].append(generator_train_loss)
train_history['discriminator'].append(discriminator_train_loss)
test_history['generator'].append(generator_test_loss)
test_history['discriminator'].append(discriminator_test_loss)
print('{0:<22s} | {1:4s} | {2:15s} | {3:5s}| {4:5s}'.format(
'component', *discriminator.metrics_names))
print('-' * 65)
ROW_FMT = '{0:<22s} | {1:<4.2f} | {2:<15.2f} | {3:<5.2f}| {4:<5.2f}'
print(ROW_FMT.format('generator (train)',
*train_history['generator'][-1]))
print(ROW_FMT.format('generator (test)',
*test_history['generator'][-1]))
print(ROW_FMT.format('discriminator (train)',
*train_history['discriminator'][-1]))
print(ROW_FMT.format('discriminator (test)',
*test_history['discriminator'][-1]))
# save weights every epoch
generator.save_weights(WeightsDir + '/{0}{1:03d}.hdf5'.format(g_weights, epoch),
overwrite=True)
discriminator.save_weights(WeightsDir + '/{0}{1:03d}.hdf5'.format(d_weights, epoch),
overwrite=True)
epoch_time = time.time()-epoch_start
print("The {} epoch took {} seconds".format(epoch, epoch_time))
pickle.dump({'train': train_history, 'test': test_history},
open('dcgan-history_onepass.pkl', 'wb'))
config = tf.ConfigProto(log_device_placement=True)
#Architectures to import
from EcalEnergyGan import generator, discriminator
#Values to be set by user
parser = get_parser()
params = parser.parse_args()
nb_epochs = params.nbepochs #Total Epochs
batch_size = params.batchsize #batch size
latent_size = params.latentsize #latent vector size
verbose = params.verbose
datapath = '/bigdata/shared/LCD/NewV1/*scan/*.h5' #Data path on Caltech
#datapath = params.datapath#Data path on EOS CERN default
EventsperFile = params.nbperfile#Events in a file
nEvents = params.nbEvents#Total events for training
#fitmod = params.mod
fitmod = 1
#weightdir = params.weightsdir
#xscale = params.xscale
weightdir = 'veganweights_onepass'
xscale = 1
print(params)
# Building discriminator and generator
d=discriminator()
g=generator(latent_size)
Gan3DTrain2(d, g, datapath, EventsperFile, nEvents, weightdir, mod=fitmod, nb_epochs=nb_epochs, batch_size=batch_size, gen_weight=8, aux_weight=0.2, ecal_weight=0.1, xscale = xscale)
def GanTrain(discriminator,
generator,
opt,
global_batch_size,
warmup_epochs,
datapath,
EventsperFile,
nEvents,
WeightsDir,
mod=0,
nb_epochs=30,
batch_size=128,
latent_size=128,
gen_weight=6,
aux_weight=0.2,
ecal_weight=0.1,
lr=0.001,
rho=0.9,
decay=0.0,
g_weights='params_generator_epoch_',
d_weights='params_generator_epoch_',
xscale=1,
verbose=True):
start_init = time.time()
# verbose = False
if hvd.rank() == 0:
print('[INFO] Building discriminator')
#discriminator.summary()
discriminator.compile(optimizer=opt,
loss=[
'binary_crossentropy',
'mean_absolute_percentage_error',
'mean_absolute_percentage_error'
],
loss_weights=[gen_weight, aux_weight, ecal_weight])
# build the generator
if hvd.rank() == 0:
print('[INFO] Building generator')
#generator.summary()
generator.compile(optimizer=opt, loss='binary_crossentropy')
# build combined Model
latent = Input(shape=(latent_size, ), name='combined_z')
fake_image = generator(latent)
discriminator.trainable = False
fake, aux, ecal = discriminator(fake_image)
combined = Model(input=[latent],
output=[fake, aux, ecal],
name='combined_model')
# Getting Data
Trainfiles, Testfiles = DivideFiles(datapath,
nEvents=nEvents,
EventsperFile=EventsperFile,
datasetnames=["ECAL"],
Particles=["Ele"])
if hvd.rank() == 0:
print("Train files: {0} \nTest files: {1}".format(
Trainfiles, Testfiles))
#Read test data into a single array
for index, dtest in enumerate(Testfiles):
if index == 0:
X_test, Y_test, ecal_test = GetData(dtest)
else:
X_temp, Y_temp, ecal_temp = GetData(dtest)
X_test = np.concatenate((X_test, X_temp))
Y_test = np.concatenate((Y_test, Y_temp))
ecal_test = np.concatenate((ecal_test, ecal_temp))
for index, dtrain in enumerate(Trainfiles):
if index == 0:
X_train, Y_train, ecal_train = GetData(dtrain)
else:
X_temp, Y_temp, ecal_temp = GetData(dtrain)
X_train = np.concatenate((X_train, X_temp))
Y_train = np.concatenate((Y_train, Y_temp))
ecal_train = np.concatenate((ecal_train, ecal_temp))
nb_test = X_test.shape[0]
assert X_train.shape[0] == EventsperFile * len(
Trainfiles), "# Total events in training files"
nb_train = X_train.shape[0] # Total events in training files
total_batches = nb_train / global_batch_size
if hvd.rank() == 0:
print('Total Training batches = {} with {} events'.format(
total_batches, nb_train))
combined.compile(
#optimizer=Adam(lr=adam_lr, beta_1=adam_beta_1),
optimizer=opt,
loss=[
'binary_crossentropy', 'mean_absolute_percentage_error',
'mean_absolute_percentage_error'
],
loss_weights=[gen_weight, aux_weight, ecal_weight])
gcb = CallbackList( \
callbacks=[ \
hvd.callbacks.BroadcastGlobalVariablesCallback(0), \
hvd.callbacks.MetricAverageCallback(), \
hvd.callbacks.LearningRateWarmupCallback(warmup_epochs=warmup_epochs, verbose=1, steps_per_epoch=total_batches), \
hvd.callbacks.LearningRateScheduleCallback(start_epoch=warmup_epochs, end_epoch=nb_epochs, multiplier=1.), \
keras.callbacks.ReduceLROnPlateau(patience=10, verbose=1) \
])
dcb = CallbackList( \
callbacks=[ \
hvd.callbacks.BroadcastGlobalVariablesCallback(0), \
hvd.callbacks.MetricAverageCallback(), \
hvd.callbacks.LearningRateWarmupCallback(warmup_epochs=warmup_epochs, verbose=1, steps_per_epoch=total_batches), \
hvd.callbacks.LearningRateScheduleCallback(start_epoch=warmup_epochs, end_epoch=nb_epochs, multiplier=1.), \
keras.callbacks.ReduceLROnPlateau(patience=10, verbose=1) \
])
ccb = CallbackList( \
callbacks=[ \
hvd.callbacks.BroadcastGlobalVariablesCallback(0), \
hvd.callbacks.MetricAverageCallback(), \
hvd.callbacks.LearningRateWarmupCallback(warmup_epochs=warmup_epochs, verbose=1, steps_per_epoch=total_batches), \
hvd.callbacks.LearningRateScheduleCallback(start_epoch=warmup_epochs, end_epoch=nb_epochs, multiplier=1.), \
keras.callbacks.ReduceLROnPlateau(patience=10, verbose=1) \
])
gcb.set_model(generator)
dcb.set_model(discriminator)
ccb.set_model(combined)
gcb.on_train_begin()
dcb.on_train_begin()
ccb.on_train_begin()
print("On hostname {0} - After init using {1} memory".format(
socket.gethostname(),
psutil.Process(os.getpid()).memory_info()[0]))
train_history = defaultdict(list)
test_history = defaultdict(list)
if hvd.rank() == 0:
print('Initialization time was {} seconds'.format(time.time() -
start_init))
for epoch in range(nb_epochs):
epoch_start = time.time()
if hvd.rank() == 0:
print('Epoch {} of {}'.format(epoch + 1, nb_epochs))
randomize(X_train, Y_train, ecal_train)
epoch_gen_loss = []
epoch_disc_loss = []
image_batches = genbatches(X_train, batch_size)
energy_batches = genbatches(Y_train, batch_size)
ecal_batches = genbatches(ecal_train, batch_size)
for index in range(total_batches):
start = time.time()
image_batch = image_batches.next()
energy_batch = energy_batches.next()
ecal_batch = ecal_batches.next()
noise = np.random.normal(0, 1, (batch_size, latent_size))
sampled_energies = np.random.uniform(0.1, 5, (batch_size, 1))
generator_ip = np.multiply(sampled_energies, noise)
# ecal sum from fit
ecal_ip = GetEcalFit(sampled_energies, mod, xscale)
generated_images = generator.predict(generator_ip, verbose=0)
real_batch_loss = discriminator.train_on_batch(
image_batch,
[BitFlip(np.ones(batch_size)), energy_batch, ecal_batch])
fake_batch_loss = discriminator.train_on_batch(
generated_images,
[BitFlip(np.zeros(batch_size)), sampled_energies, ecal_ip])
epoch_disc_loss.append([
(a + b) / 2 for a, b in zip(real_batch_loss, fake_batch_loss)
])
trick = np.ones(batch_size)
gen_losses = []
for _ in range(2):
noise = np.random.normal(0, 1, (batch_size, latent_size))
sampled_energies = np.random.uniform(0.1, 5, (batch_size, 1))
generator_ip = np.multiply(sampled_energies, noise)
ecal_ip = GetEcalFit(sampled_energies, mod, xscale)
gen_losses.append(
combined.train_on_batch(
[generator_ip],
[trick,
sampled_energies.reshape((-1, 1)), ecal_ip]))
epoch_gen_loss.append([(a + b) / 2 for a, b in zip(*gen_losses)])
if (index % 1) == 0 and hvd.rank() == 0:
# progress_bar.update(index)
print('processed {}/{} batches in {}'.format(
index + 1, total_batches,
time.time() - start))
# save weights every epoch
if hvd.rank() == 0:
safe_mkdir(WeightsDir)
print("saving weights of gen")
generator.save_weights(
WeightsDir +
'/generator_{0}{1:03d}.hdf5'.format(g_weights, epoch),
overwrite=True)
print("saving weights of disc")
discriminator.save_weights(
WeightsDir +
'/discriminator_{0}{1:03d}.hdf5'.format(d_weights, epoch),
overwrite=True)
epoch_time = time.time() - epoch_start
print("The {} epoch took {} seconds".format(epoch, epoch_time))
print("Global batch size is: {0} / batch size is: {1}".format(
global_batch_size, batch_size))
latent_size = params.latentsize #latent vector size
verbose = params.verbose
#datapath = '/bigdata/shared/LCD/NewV1/*scan/*.h5' #Data path on Caltech
datapath = params.datapath #Data path on EOS CERN
EventsperFile = params.nbperfile #Events in a file
nEvents = params.nbEvents #Total events for training
fitmod = 1 #params.mod
weightdir = params.weightsdir
xscale = params.xscale
warmup_epochs = params.warmupepochs
mod = 0
opt = getattr(keras.optimizers, params.optimizer)
# Building discriminator and generator
g = generator(latent_size, keras_dformat=d_format)
g.load_weights('params_generator_epoch_041.hdf5')
Trainfiles, Testfiles = DivideFiles(datapath,
nEvents=nEvents,
EventsperFile=EventsperFile,
datasetnames=["ECAL"],
Particles=["Ele"])
for index, dtest in enumerate(Testfiles):
if index == 0:
X_test, Y_test, ecal_test = GetData(dtest, keras_dformat=d_format)
else:
X_temp, Y_temp, ecal_temp = GetData(dtest, keras_dformat=d_format)
X_test = np.concatenate((X_test, X_temp))
Y_test = np.concatenate((Y_test, Y_temp))
session = tf.Session(config=session_config)
K.set_session(session)
from EcalEnergyGan import generator, discriminator
#from EcalEnergyGan_16f import generator, discriminator
g_weights = 'params_generator_epoch_'
d_weights = 'params_discriminator_epoch_'
keras_dformat = 'channels_first'
nb_epochs = 1
batch_size = 128
latent_size = 200
verbose = 'false'
generator = generator(latent_size, keras_dformat=keras_dformat)
discriminator = discriminator(keras_dformat=keras_dformat)
nb_classes = 2
print(tf.__version__)
print('[INFO] Building discriminator')
discriminator.summary()
discriminator.compile(
#optimizer=Adam(lr=adam_lr, beta_1=adam_beta_1),
optimizer=RMSprop(),
loss=[
'binary_crossentropy', 'mean_absolute_percentage_error',
'mean_absolute_percentage_error'
],
loss_weights=[6, 0.2, 0.1]
#loss=['binary_crossentropy', 'kullback_leibler_divergence']