def __init__(self,str_create=None,loss = torch.nn.MSELoss()):
super(AE_NetForDIA,self).__init__()
cm = CreateModel()
if str_create is None:
str_create = 'I(3)-CBR(16,3)-' \
'Down(32)-CBR(32,3)-' \
'Down(64)-CBR(64,3)-' \
'Down(128)-CBR(128,3)-' \
'Down(256)-CBR(512,3)-' \
'Res(512,3)-Res(512,3)-' \
'Res(512,3)-Res(512,3)-' \
'Res(512,3)-Res(512,3)-' \
'Up(256,3)-CBR(256,3)-' \
'Up(128,3)-CBR(128,3)-' \
'Up(64,3)-CBR(64,3)-' \
'Up(32,3)-CBR(32,3)-' \
'CBR(32,3)-' \
'CBR(16,3)-' \
'Conv(3,3)-Tanh()'
self.model = cm.Create(str_create)
self.loss = loss
self.num = 28
def run_experiments(configfile):
"""
Run this script to do a simulation.
"""
params = ReadConfig(configfile)
# Create output folder
output_path = params['output_dir']
if not os.path.exists(output_path):
os.makedirs(output_path)
target_bofs = list()
for k in range(0, len(params['species'])):
target_bofs.append(
pd.read_csv(params['input_dir'] + '/' + params['biomass_comp'][k],
index_col=0))
cells_mL = pd.read_csv(params['input_dir'] + '/' + params['cellsmL_csv'],
squeeze=True,
header=None).to_list()
media = ReadMediumMetsConcen(params['input_dir'] + '/' +
params['csv_dir'] + '/' + params['media_csv'])
models = CreateModel(params['input_dir'] + '/' + params['model_dir'] + '/',
target_bofs, params['species'], params)
[biomass, chnops, chnops_new, ctns, initial_flux, allfluxes,
allstatus] = RunCulture(params, media, models, cells_mL, target_bofs)
# Save results
writer = pd.ExcelWriter(params['output_dir'] + '/' +
params['output_filename'] + '.xlsx',
engine='xlsxwriter')
ctns.to_excel(writer, sheet_name='ctns')
allstatus.to_excel(writer, sheet_name='status')
for k in range(0, len(params['species'])):
biomass[k].to_excel(writer, sheet_name='bio_comps' + str(k))
for k in range(0, len(params['species'])):
chnops[k].to_excel(writer, sheet_name='CHNOPS' + str(k))
for k in range(0, len(params['species'])):
chnops_new[k].to_excel(writer, sheet_name='CHNOPS_new' + str(k))
for k in range(0, len(params['species'])):
allfluxes[k].to_excel(writer, sheet_name='allfluxes' + str(k))
for k in range(0, len(params['species'])):
initial_flux[k].to_excel(writer, sheet_name='initial_flux' + str(k))
pd.DataFrame.from_dict(params,
orient='index').to_excel(writer,
sheet_name='params')
writer.save()
def mainTF(options):
import tensorflow as tf
from CreateModel import CreateModel
from DataManager import DataManager
from DataSet import DataSet
print "PROCESSING VALIDATION DATA"
dgSig = DataGetter.DefinedVariables(options.netOp.vNames, signal=True)
dgBg = DataGetter.DefinedVariables(options.netOp.vNames, background=True)
validDataSig = [
(("/cms/data/pastika/trainData_pt20_30_40_dRPi_tightMass_deepFlavor_v6p1/trainingTuple_0_division_1_TTbarSingleLepT_validation_0.h5",
), 1),
(("/cms/data/pastika/trainData_pt20_30_40_dRPi_tightMass_deepFlavor_v6p1/trainingTuple_0_division_1_TTbarSingleLepTbar_validation_0.h5",
), 1)
]
validDataBgTTbar = [
(("/cms/data/pastika/trainData_pt20_30_40_dRPi_tightMass_deepFlavor_v6/trainingTuple_0_division_1_TTbarSingleLepT_validation_0.h5",
), 1),
(("/cms/data/pastika/trainData_pt20_30_40_dRPi_tightMass_deepFlavor_v6/trainingTuple_0_division_1_TTbarSingleLepTbar_validation_0.h5",
), 1),
]
validDataBgQCDMC = [
(("/cms/data/pastika/trainData_pt20_30_40_dRPi_tightMass_deepFlavor_v6/trainingTuple_0_division_1_QCD_HT100to200_validation_0.h5",
), 1),
(("/cms/data/pastika/trainData_pt20_30_40_dRPi_tightMass_deepFlavor_v6/trainingTuple_0_division_1_QCD_HT200to300_validation_0.h5",
), 1),
(("/cms/data/pastika/trainData_pt20_30_40_dRPi_tightMass_deepFlavor_v6/trainingTuple_0_division_1_QCD_HT300to500_validation_0.h5",
), 1),
(("/cms/data/pastika/trainData_pt20_30_40_dRPi_tightMass_deepFlavor_v6/trainingTuple_0_division_1_QCD_HT500to700_validation_0.h5",
), 1),
(("/cms/data/pastika/trainData_pt20_30_40_dRPi_tightMass_deepFlavor_v6/trainingTuple_0_division_1_QCD_HT700to1000_validation_0.h5",
), 1),
(("/cms/data/pastika/trainData_pt20_30_40_dRPi_tightMass_deepFlavor_v6/trainingTuple_0_division_1_QCD_HT1000to1500_validation_0.h5",
), 1),
(("/cms/data/pastika/trainData_pt20_30_40_dRPi_tightMass_deepFlavor_v6/trainingTuple_0_division_1_QCD_HT1500to2000_validation_0.h5",
), 1),
(("/cms/data/pastika/trainData_pt20_30_40_dRPi_tightMass_deepFlavor_v6/trainingTuple_0_division_1_QCD_HT2000toInf_validation_0.h5",
), 1)
]
validDataBgQCDData = [((
"/cms/data/pastika/trainData_pt20_30_40_dRPi_tightMass_deepFlavor_v6/trainingTuple_0_division_1_Data_JetHT_2016_validation_0.h5",
), 1)]
print "Input Variables: ", len(dgSig.getList())
# Import data
#print options.runOp.validationSamples
validDataSig = getValidData(dgSig, validDataSig, options)
validDataBgTTbar = getValidData(dgBg, validDataBgTTbar, options)
validDataBgQCDMC = getValidData(dgBg, validDataBgQCDMC, options)
validDataBgQCDData = getValidData(dgBg, validDataBgQCDData, options)
validDataTTbar = combineValidationData(validDataSig, validDataBgTTbar)
validDataQCDMC = combineValidationData(validDataSig, validDataBgQCDMC)
validDataQCDData = combineValidationData(validDataSig, validDataBgQCDData)
#get input/output sizes
#print validData["data"].shape
nFeatures = validDataTTbar["data"].shape[1]
nLabels = validDataTTbar["labels"].shape[1]
nWeights = validDataTTbar["weights"].shape[1]
#Training parameters
l2Reg = options.runOp.l2Reg
MiniBatchSize = options.runOp.minibatchSize
nEpoch = options.runOp.nepoch
ReportInterval = options.runOp.reportInterval
validationCount = min(options.runOp.nValidationEvents,
validDataTTbar["data"].shape[0])
#scale data inputs to mean 0, stddev 1
categories = numpy.array(options.netOp.vCategories)
mins = numpy.zeros(categories.shape, dtype=numpy.float32)
ptps = numpy.zeros(categories.shape, dtype=numpy.float32)
for i in xrange(categories.max()):
selectedCategory = categories == i
mins[selectedCategory] = validDataTTbar["data"][:,
selectedCategory].mean(
)
ptps[selectedCategory] = validDataTTbar["data"][:,
selectedCategory].std(
)
ptps[ptps < 1e-10] = 1.0
##Create data manager, this class controls how data is fed to the network for training
# DataSet(fileGlob, xsec, Nevts, kFactor, sig, prescale, rescale)
signalDataSets = [
DataSet(
"/cms/data/pastika/trainData_pt20_30_40_dRPi_tightMass_deepFlavor_v6p1/trainingTuple_*_division_0_TTbarSingleLepT_training_*.h5",
365.4, 61878989, 1.0, True, 0, 1.0, 1.0, 8),
DataSet(
"/cms/data/pastika/trainData_pt20_30_40_dRPi_tightMass_deepFlavor_v6p1/trainingTuple_*_division_0_TTbarSingleLepTbar_training_*.h5",
365.4, 61901450, 1.0, True, 0, 1.0, 1.0, 8),
]
#pt reweighting histograms
ttbarRatio = (numpy.array([
0.7976347, 1.010679, 1.0329635, 1.0712056, 1.1147588, 1.0072196,
0.79854023, 0.7216115, 0.7717652, 0.851551, 0.8372917
]),
numpy.array([
0., 50., 100., 150., 200., 250., 300., 350., 400., 450.,
500., 1e10
]))
QCDDataRatio = (numpy.array([
0.50125164, 0.70985824, 1.007087, 1.6701245, 2.5925348, 3.6850858,
4.924969, 6.2674766, 7.5736594, 8.406105, 7.7529635
]),
numpy.array([
0., 50., 100., 150., 200., 250., 300., 350., 400.,
450., 500., 1e10
]))
QCDMCRatio = (numpy.array([
0.75231355, 1.0563549, 1.2571484, 1.3007764, 1.0678109, 0.83444154,
0.641499, 0.49130705, 0.36807108, 0.24333349, 0.06963781
]),
numpy.array([
0., 50., 100., 150., 200., 250., 300., 350., 400., 450.,
500., 1e10
]))
backgroundDataSets = [
DataSet(
"/cms/data/pastika/trainData_pt20_30_40_dRPi_tightMass_deepFlavor_v6/trainingTuple_*_division_0_TTbarSingleLepT_training_*.h5",
365.4, 61878989, 1.0, False, 0, 1.0, 1.0, 8, ttbarRatio),
DataSet(
"/cms/data/pastika/trainData_pt20_30_40_dRPi_tightMass_deepFlavor_v6/trainingTuple_*_division_0_TTbarSingleLepTbar_training_*.h5",
365.4, 61901450, 1.0, False, 0, 1.0, 1.0, 8, ttbarRatio),
DataSet(
"/cms/data/pastika/trainData_pt20_30_40_dRPi_tightMass_deepFlavor_v6/trainingTuple_*_division_0_Data_JetHT_2016_training_*.h5",
1.0,
1,
1.0,
False,
1,
1.0,
1.0,
8,
include=False), #QCDDataRatio),
DataSet(
"/cms/data/pastika/trainData_pt20_30_40_dRPi_tightMass_deepFlavor_v6/trainingTuple_*_division_0_QCD_HT100to200_training_*.h5",
27990000,
80684349,
0.0,
False,
2,
1.0,
1.0,
1,
include=False), #QCDMCRatio),
DataSet(
"/cms/data/pastika/trainData_pt20_30_40_dRPi_tightMass_deepFlavor_v6/trainingTuple_*_division_0_QCD_HT200to300_training_*.h5",
1712000,
57580393,
0.0,
False,
2,
1.0,
1.0,
1,
include=False), #QCDMCRatio),
DataSet(
"/cms/data/pastika/trainData_pt20_30_40_dRPi_tightMass_deepFlavor_v6/trainingTuple_*_division_0_QCD_HT300to500_training_*.h5",
347700,
54537903,
0.0,
False,
2,
1.0,
1.0,
1,
include=False), #QCDMCRatio),
DataSet(
"/cms/data/pastika/trainData_pt20_30_40_dRPi_tightMass_deepFlavor_v6/trainingTuple_*_division_0_QCD_HT500to700_training_*.h5",
32100,
62271343,
0.0,
False,
2,
1.0,
1.0,
1,
include=False), #QCDMCRatio),
DataSet(
"/cms/data/pastika/trainData_pt20_30_40_dRPi_tightMass_deepFlavor_v6/trainingTuple_*_division_0_QCD_HT700to1000_training_*.h5",
6831,
45232316,
0.0,
False,
2,
1.0,
1.0,
1,
include=False), #QCDMCRatio),
DataSet(
"/cms/data/pastika/trainData_pt20_30_40_dRPi_tightMass_deepFlavor_v6/trainingTuple_*_division_0_QCD_HT1000to1500_training_*.h5",
1207,
15127293,
0.0,
False,
2,
1.0,
1.0,
1,
include=False), #QCDMCRatio),
DataSet(
"/cms/data/pastika/trainData_pt20_30_40_dRPi_tightMass_deepFlavor_v6/trainingTuple_*_division_0_QCD_HT1500to2000_training_*.h5",
119.9,
11826702,
0.0,
False,
2,
1.0,
1.0,
1,
include=False), #QCDMCRatio),
DataSet(
"/cms/data/pastika/trainData_pt20_30_40_dRPi_tightMass_deepFlavor_v6/trainingTuple_*_division_0_QCD_HT2000toInf_training_*.h5",
25.24,
6039005,
0.0,
False,
2,
1.0,
1.0,
1,
include=False), #QCDMCRatio),
]
dm = DataManager(options.netOp.vNames, nEpoch, nFeatures, nLabels, 2,
nWeights, options.runOp.ptReweight, signalDataSets,
backgroundDataSets)
# Build the graph
denseNetwork = [nFeatures] + options.netOp.denseLayers + [nLabels]
convLayers = options.netOp.convLayers
rnnNodes = options.netOp.rnnNodes
rnnLayers = options.netOp.rnnLayers
mlp = CreateModel(options, denseNetwork, convLayers, rnnNodes, rnnLayers,
dm.inputDataQueue, MiniBatchSize, mins, 1.0 / ptps)
#summary writer
summary_writer = tf.summary.FileWriter(options.runOp.directory +
"log_graph",
graph=tf.get_default_graph())
print "TRAINING NETWORK"
with tf.Session(config=tf.ConfigProto(
intra_op_parallelism_threads=8)) as sess:
sess.run(tf.global_variables_initializer())
#start queue runners
dm.launchQueueThreads(sess)
print "Reporting validation loss every %i batches with %i events per batch for %i epochs" % (
ReportInterval, MiniBatchSize, nEpoch)
#preload the first data into staging area
sess.run([mlp.stagingOp],
feed_dict={
mlp.reg: l2Reg,
mlp.keep_prob: options.runOp.keepProb
})
i = 0
N_TRAIN_SUMMARY = 10
#flush queue until the sample fraction is approximately equal
flushctr = 200
while dm.continueTrainingLoop():
result = sess.run(dm.inputDataQueue.dequeue_many(MiniBatchSize))
signalCount = result[1][:, 0].sum()
bgCount = result[1][:, 1].sum()
signalFraction = signalCount / (signalCount + bgCount)
#the first this fraction drops below 0.5 means we are close enough to equal signal/bg fraction
if signalFraction < 0.5:
flushctr -= 1
if flushctr <= 0:
break
try:
while dm.continueTrainingLoop():
grw = 2 / (1 + exp(-i / 10000.0)) - 1
#run validation operations
if i == 0 or not i % ReportInterval:
#run validation operations
validation_loss, accuracy, summary_vl = sess.run(
[
mlp.loss_ph, mlp.accuracy,
mlp.merged_valid_summary_op
],
feed_dict={
mlp.x_ph: validDataTTbar["data"][:validationCount],
mlp.y_ph_:
validDataTTbar["labels"][:validationCount],
mlp.p_ph_:
validDataTTbar["domain"][:validationCount],
mlp.reg: l2Reg,
mlp.gradientReversalWeight: grw,
mlp.wgt_ph:
validDataTTbar["weights"][:validationCount]
})
summary_writer.add_summary(summary_vl, i / N_TRAIN_SUMMARY)
print(
'Interval %d, validation accuracy %0.6f, validation loss %0.6f'
% (i / ReportInterval, accuracy, validation_loss))
validation_loss, accuracy, summary_vl_QCDMC = sess.run(
[
mlp.loss_ph, mlp.accuracy,
mlp.merged_valid_QCDMC_summary_op
],
feed_dict={
mlp.x_ph: validDataQCDMC["data"][:validationCount],
mlp.y_ph_:
validDataQCDMC["labels"][:validationCount],
mlp.p_ph_:
validDataQCDMC["domain"][:validationCount],
mlp.reg: l2Reg,
mlp.gradientReversalWeight: grw,
mlp.wgt_ph:
validDataQCDMC["weights"][:validationCount]
})
summary_writer.add_summary(summary_vl_QCDMC,
i / N_TRAIN_SUMMARY)
validation_loss, accuracy, summary_vl_QCDData = sess.run(
[
mlp.loss_ph, mlp.accuracy,
mlp.merged_valid_QCDData_summary_op
],
feed_dict={
mlp.x_ph:
validDataQCDData["data"][:validationCount],
mlp.y_ph_:
validDataQCDData["labels"][:validationCount],
mlp.p_ph_:
validDataQCDData["domain"][:validationCount],
mlp.reg:
l2Reg,
mlp.gradientReversalWeight:
grw,
mlp.wgt_ph:
validDataQCDData["weights"][:validationCount]
})
summary_writer.add_summary(summary_vl_QCDData,
i / N_TRAIN_SUMMARY)
#run training operations
if i % N_TRAIN_SUMMARY == 0:
_, _, summary = sess.run(
[
mlp.stagingOp, mlp.train_step,
mlp.merged_train_summary_op
],
feed_dict={
mlp.reg: l2Reg,
mlp.keep_prob: options.runOp.keepProb,
mlp.training: True,
mlp.gradientReversalWeight: grw
})
summary_writer.add_summary(summary, i / N_TRAIN_SUMMARY)
else:
sess.run(
[mlp.stagingOp, mlp.train_step],
feed_dict={
mlp.reg: l2Reg,
mlp.keep_prob: options.runOp.keepProb,
mlp.training: True
})
i += 1
while dm.continueFlushingQueue():
sess.run(dm.inputDataQueue.dequeue_many(MiniBatchSize))
except Exception, e:
# Report exceptions to the coordinator.
dm.requestStop(e)
finally:
def mainTF(options):
import tensorflow as tf
from CreateModel import CreateModel
from FileNameQueue import FileNameQueue
from CustomQueueRunner import CustomQueueRunner
print "PROCESSING TRAINING DATA"
dg = DataGetter.DefinedVariables(options.netOp.vNames)
print "Input Variables: ", len(dg.getList())
# Import data
validData = dg.importData(samplesToRun=tuple(
options.runOp.validationSamples),
ptReweight=options.runOp.ptReweight)
#get input/output sizes
nFeatures = validData["data"].shape[1]
nLabels = validData["labels"].shape[1]
nWeigts = validData["weights"].shape[1]
#Training parameters
l2Reg = options.runOp.l2Reg
MiniBatchSize = options.runOp.minibatchSize
NEpoch = options.runOp.nepoch
ReportInterval = options.runOp.reportInterval
validationCount = min(options.runOp.nValidationEvents,
validData["data"].shape[0])
#scale data inputs to mean 0, stddev 1
categories = numpy.array(options.netOp.vCategories)
mins = numpy.zeros(categories.shape, dtype=numpy.float32)
ptps = numpy.zeros(categories.shape, dtype=numpy.float32)
for i in xrange(categories.max()):
selectedCategory = categories == i
mins[selectedCategory] = validData["data"][:, selectedCategory].mean()
ptps[selectedCategory] = validData["data"][:, selectedCategory].std()
ptps[ptps < 1e-10] = 1.0
#Create filename queue
fnq = FileNameQueue(options.runOp.trainingSamples, NEpoch, nFeatures,
nLabels, nWeigts, options.runOp.nReaders,
MiniBatchSize)
#Create CustomQueueRunner object to manage data loading
print "PT reweight: ", options.runOp.ptReweight
crs = [
CustomQueueRunner(MiniBatchSize,
dg.getList(),
fnq,
ptReweight=options.runOp.ptReweight)
for i in xrange(options.runOp.nReaders)
]
# Build the graph
denseNetwork = [nFeatures] + options.netOp.denseLayers + [nLabels]
convLayers = options.netOp.convLayers
rnnNodes = options.netOp.rnnNodes
rnnLayers = options.netOp.rnnLayers
mlp = CreateModel(options, denseNetwork, convLayers, rnnNodes, rnnLayers,
fnq.inputDataQueue, MiniBatchSize, mins, 1.0 / ptps)
#summary writer
summary_writer = tf.summary.FileWriter(options.runOp.directory +
"log_graph",
graph=tf.get_default_graph())
print "TRAINING MLP"
with tf.Session(config=tf.ConfigProto(
intra_op_parallelism_threads=8)) as sess:
sess.run(tf.global_variables_initializer())
#start queue runners
coord = tf.train.Coordinator()
# start the tensorflow QueueRunner's
qrthreads = tf.train.start_queue_runners(coord=coord, sess=sess)
# start the file queue running
fnq.startQueueProcess(sess)
# we must sleep to ensure that the file queue is filled before
# starting the feeder queues
sleep(2)
# start our custom queue runner's threads
for cr in crs:
cr.start_threads(sess, n_threads=options.runOp.nThreadperReader)
print "Reporting validation loss every %i batchces with %i events per batch for %i epochs" % (
ReportInterval, MiniBatchSize, NEpoch)
#preload the first data into staging area
sess.run([mlp.stagingOp],
feed_dict={
mlp.reg: l2Reg,
mlp.keep_prob: options.runOp.keepProb
})
i = 0
try:
while not coord.should_stop():
_, _, summary = sess.run([
mlp.stagingOp, mlp.train_step, mlp.merged_train_summary_op
],
feed_dict={
mlp.reg: l2Reg,
mlp.keep_prob:
options.runOp.keepProb
})
summary_writer.add_summary(summary, i)
i += 1
if i == 1 or not i % ReportInterval:
validation_loss, accuracy, summary_vl = sess.run(
[
mlp.loss_ph, mlp.accuracy,
mlp.merged_valid_summary_op
],
feed_dict={
mlp.x_ph: validData["data"][:validationCount],
mlp.y_ph_: validData["labels"][:validationCount],
mlp.reg: l2Reg,
mlp.wgt_ph: validData["weights"][:validationCount]
})
summary_writer.add_summary(summary_vl, i)
print(
'Interval %d, validation accuracy %0.6f, validation loss %0.6f'
% (i / ReportInterval, accuracy, validation_loss))
except tf.errors.OutOfRangeError:
print('Done training -- epoch limit reached')
finally:
# When done, ask the threads to stop.
coord.request_stop()
coord.join(qrthreads)
mlp.saveCheckpoint(sess, options.runOp.directory)
mlp.saveModel(sess, options.runOp.directory)
def mainTF(options):
import tensorflow as tf
from CreateModel import CreateModel
from DataManager import DataManager
from DataSet import DataSet
print "PROCESSING VALIDATION DATA"
dgSig = DataGetter.DefinedVariables(options.netOp.vNames, signal = True, background = False)
dgBg = DataGetter.DefinedVariables(options.netOp.vNames, signal = False, background = True)
validDataSig = [((dataPath + "/trainingTuple_0_division_1_rpv_stop_850_validation_0.h5", ), 2),]
validDataSig2 = [((dataPath + "/trainingTuple_0_division_1_stealth_stop_350_SHuHd_validation_0.h5", ), 2),]
validDataSig3 = [((dataPath + "/trainingTuple_0_division_1_rpv_stop_350_validation_0.h5", ), 2),]
validDataBgTTbar = [((dataPath + "/trainingTuple_20_division_1_TT_validation_0.h5", ), 1),
((dataPath + "/trainingTuple_2110_division_1_TT_validation_0.h5", ), 1),]
print "Input Variables: ",len(dgSig.getList())
# Import data
#print options.runOp.validationSamples
validDataSig = getValidData(dgSig, validDataSig, options)
validDataSig2 = getValidData(dgSig, validDataSig2, options)
validDataSig3 = getValidData(dgSig, validDataSig3, options)
validDataBgTTbar = getValidData(dgBg, validDataBgTTbar, options)
validDataTTbar = combineValidationData(validDataSig, validDataBgTTbar)
validDataQCDMC = combineValidationData(validDataSig2, validDataBgTTbar)
validDataQCDData = combineValidationData(validDataSig3, validDataBgTTbar)
#get input/output sizes
#print validData["data"].shape
nFeatures = validDataTTbar["data"].shape[1]
nLabels = validDataTTbar["labels"].shape[1]
nWeights = validDataTTbar["weights"].shape[1]
nDomain = validDataSig["domain"].shape[1]
#Training parameters
l2Reg = options.runOp.l2Reg
MiniBatchSize = options.runOp.minibatchSize
nEpoch = options.runOp.nepoch
ReportInterval = options.runOp.reportInterval
validationCount = min(options.runOp.nValidationEvents, validDataTTbar["data"].shape[0])
#scale data inputs to mean 0, stddev 1
categories = numpy.array(options.netOp.vCategories)
mins = numpy.zeros(categories.shape, dtype=numpy.float32)
ptps = numpy.zeros(categories.shape, dtype=numpy.float32)
for i in xrange(categories.max()):
selectedCategory = categories == i
mins[selectedCategory] = validDataTTbar["data"][:,selectedCategory].mean()
ptps[selectedCategory] = validDataTTbar["data"][:,selectedCategory].std()
ptps[ptps < 1e-10] = 1.0
##Create data manager, this class controls how data is fed to the network for training
# DataSet(fileGlob, xsec, Nevts, kFactor, sig, prescale, rescale)
signalDataSets = [
#DataSet(dataPath + "/trainingTuple_*_division_*_rpv_stop_350_training_0.h5", 365.4, 61878989, 1.0, True, 0, 1.0, 1.0, 1),
#DataSet(dataPath + "/trainingTuple_*_division_*_rpv_stop_450_training_0.h5", 365.4, 61878989, 1.0, True, 0, 1.0, 1.0, 1),
DataSet(dataPath + "/trainingTuple_*_division_*_rpv_stop_550_training_0.h5", 365.4, 61878989, 1.0, True, 0, 1.0, 1.0, 1),
DataSet(dataPath + "/trainingTuple_*_division_*_rpv_stop_650_training_0.h5", 365.4, 61878989, 1.0, True, 0, 1.0, 1.0, 1),
DataSet(dataPath + "/trainingTuple_*_division_*_rpv_stop_750_training_0.h5", 365.4, 61878989, 1.0, True, 0, 1.0, 1.0, 1),
DataSet(dataPath + "/trainingTuple_*_division_*_rpv_stop_850_training_0.h5", 365.4, 61878989, 1.0, True, 0, 1.0, 1.0, 1),
#DataSet(dataPath + "/trainingTuple_*_division_*_stealth_stop_350_SHuHd_training_0.h5", 365.4, 61878989, 1.0, True, 0, 1.0, 1.0, 1),
#DataSet(dataPath + "/trainingTuple_*_division_*_stealth_stop_450_SHuHd_training_0.h5", 365.4, 61878989, 1.0, True, 0, 1.0, 1.0, 1),
DataSet(dataPath + "/trainingTuple_*_division_*_stealth_stop_550_SHuHd_training_0.h5", 365.4, 61878989, 1.0, True, 0, 1.0, 1.0, 1),
DataSet(dataPath + "/trainingTuple_*_division_*_stealth_stop_650_SHuHd_training_0.h5", 365.4, 61878989, 1.0, True, 0, 1.0, 1.0, 1),
DataSet(dataPath + "/trainingTuple_*_division_*_stealth_stop_750_SHuHd_training_0.h5", 365.4, 61878989, 1.0, True, 0, 1.0, 1.0, 1),
DataSet(dataPath + "/trainingTuple_*_division_*_stealth_stop_850_SHuHd_training_0.h5", 365.4, 61878989, 1.0, True, 0, 1.0, 1.0, 1),
#DataSet(dataPath + "/trainingTuple_*_division_*_stealth_stop_350_SYY_training_0.h5", 365.4, 61878989, 1.0, True, 0, 1.0, 1.0, 1),
#DataSet(dataPath + "/trainingTuple_*_division_*_stealth_stop_450_SYY_training_0.h5", 365.4, 61878989, 1.0, True, 0, 1.0, 1.0, 1),
DataSet(dataPath + "/trainingTuple_*_division_*_stealth_stop_550_SYY_training_0.h5", 365.4, 61878989, 1.0, True, 0, 1.0, 1.0, 1),
DataSet(dataPath + "/trainingTuple_*_division_*_stealth_stop_650_SYY_training_0.h5", 365.4, 61878989, 1.0, True, 0, 1.0, 1.0, 1),
DataSet(dataPath + "/trainingTuple_*_division_*_stealth_stop_750_SYY_training_0.h5", 365.4, 61878989, 1.0, True, 0, 1.0, 1.0, 1),
DataSet(dataPath + "/trainingTuple_*_division_*_stealth_stop_850_SYY_training_0.h5", 365.4, 61878989, 1.0, True, 0, 1.0, 1.0, 1),
#DataSet(dataPath + "/trainingTuple_*_division_*_rpv_stop_*_training_0.h5", 365.4, 61878989, 1.0, True, 0, 1.0, 1.0, 1),
#DataSet(dataPath + "/trainingTuple_*_division_*_stealth_stop_*_SHuHd_training_0.h5", 365.4, 61901450, 1.0, True, 0, 1.0, 1.0, 1),
#DataSet(dataPath + "/trainingTuple_*_division_*_stealth_stop_*_SYY_training_0.h5", 365.4, 61901450, 1.0, True, 0, 1.0, 1.0, 1),
]
backgroundDataSets = [DataSet(dataPath + "/trainingTuple_*_division_0_TT_training_0.h5", 365.4, 61878989, 1.0, False, 0, 1.0, 1.0, len(signalDataSets)),]
dm = DataManager(options.netOp.vNames, nEpoch, nFeatures, nLabels, nDomain, nWeights, options.runOp.ptReweight, signalDataSets, backgroundDataSets)
# Build the graph
denseNetwork = [nFeatures]+options.netOp.denseLayers+[nLabels]
convLayers = options.netOp.convLayers
rnnNodes = options.netOp.rnnNodes
rnnLayers = options.netOp.rnnLayers
mlp = CreateModel(options, denseNetwork, convLayers, rnnNodes, rnnLayers, dm.inputDataQueue, MiniBatchSize, mins, 1.0/ptps)
#summary writer
summary_path = "/storage/local/data1/gpuscratch/%s"%(USER)
os.makedirs(summary_path)
summary_writer = tf.summary.FileWriter(summary_path + "/log_graph", graph=tf.get_default_graph())
print "TRAINING NETWORK"
with tf.Session(config=tf.ConfigProto(intra_op_parallelism_threads=8) ) as sess:
sess.run(tf.global_variables_initializer())
#start queue runners
dm.launchQueueThreads(sess)
print "Reporting validation loss every %i batches with %i events per batch for %i epochs"%(ReportInterval, MiniBatchSize, nEpoch)
#preload the first data into staging area
sess.run([mlp.stagingOp], feed_dict={mlp.reg: l2Reg, mlp.keep_prob:options.runOp.keepProb})
i = 0
N_TRAIN_SUMMARY = 10
#flush queue until the sample fraction is approximately equal
while dm.continueTrainingLoop():
result = sess.run(dm.inputDataQueue.dequeue_many(MiniBatchSize))
signalFraction = result[1][:,0].sum()/MiniBatchSize
#the first this fraction drops below 0.5 means we are close enough to equal signal/bg fraction
if signalFraction < 0.5:
break
try:
while dm.continueTrainingLoop():
grw = 1.0#1*(2/(1+exp(-i/10000.0)) - 1) #2/(1+exp(-i/10000.0)) - 1 #1000000000000.0*(2/(1+exp(-i/500000.0)) - 1)
#run validation operations
if i == 0 or not i % ReportInterval:
#run validation operations
validation_loss, accuracy, summary_vl = sess.run([mlp.loss_ph, mlp.accuracy, mlp.merged_valid_summary_op], feed_dict={mlp.x_ph: validDataTTbar["data"][:validationCount], mlp.y_ph_: validDataTTbar["labels"][:validationCount], mlp.p_ph_: validDataTTbar["domain"][:validationCount], mlp.reg: l2Reg, mlp.gradientReversalWeight:grw, mlp.wgt_ph: validDataTTbar["weights"][:validationCount]})
summary_writer.add_summary(summary_vl, i/N_TRAIN_SUMMARY)
print('Interval %d, validation accuracy %0.6f, validation loss %0.6f' % (i/ReportInterval, accuracy, validation_loss))
validation_loss, accuracy, summary_vl_QCDMC = sess.run([mlp.loss_ph, mlp.accuracy, mlp.merged_valid_QCDMC_summary_op], feed_dict={mlp.x_ph: validDataQCDMC["data"][:validationCount], mlp.y_ph_: validDataQCDMC["labels"][:validationCount], mlp.p_ph_: validDataQCDMC["domain"][:validationCount], mlp.reg: l2Reg, mlp.gradientReversalWeight:grw, mlp.wgt_ph: validDataQCDMC["weights"][:validationCount]})
summary_writer.add_summary(summary_vl_QCDMC, i/N_TRAIN_SUMMARY)
validation_loss, accuracy, summary_vl_QCDData = sess.run([mlp.loss_ph, mlp.accuracy, mlp.merged_valid_QCDData_summary_op], feed_dict={mlp.x_ph: validDataQCDData["data"][:validationCount], mlp.y_ph_: validDataQCDData["labels"][:validationCount], mlp.p_ph_: validDataQCDData["domain"][:validationCount], mlp.reg: l2Reg, mlp.gradientReversalWeight:grw, mlp.wgt_ph: validDataQCDData["weights"][:validationCount]})
summary_writer.add_summary(summary_vl_QCDData, i/N_TRAIN_SUMMARY)
#print(sess.run(mlp.x))
#run training operations
if i % N_TRAIN_SUMMARY == 0:
_, _, summary = sess.run([mlp.stagingOp, mlp.train_step, mlp.merged_train_summary_op], feed_dict={mlp.reg: l2Reg, mlp.keep_prob:options.runOp.keepProb, mlp.training: True, mlp.gradientReversalWeight:grw})
summary_writer.add_summary(summary, i/N_TRAIN_SUMMARY)
else:
sess.run([mlp.stagingOp, mlp.train_step], feed_dict={mlp.reg: l2Reg, mlp.keep_prob:options.runOp.keepProb, mlp.training: True})
i += 1
#Should fix bad end of training state
while dm.continueFlushingQueue():
sess.run(dm.inputDataQueue.dequeue_many(MiniBatchSize))
except Exception, e:
# Report exceptions to the coordinator.
dm.requestStop(e)
finally:
def gapso() -> None:
# 静态航迹规划采用遗传 - 粒子群算法
# # 问题设置
model = CreateModel() # 环境创建
model = model._replace(n=5) # 控制点数量
nVar = model.n # 决策变量数
VarSize = [1, nVar] # 决策变量矩阵的大小
VarMin = (model.xmin, model.ymin) #变量下限
VarMax = (model.xmax, model.ymax) #变量上限
# # 算法参数
MaxIt = 50 # 最大迭代次数
nPop = 250 # 种群数
w = 1 # 惯性因子
wdamp = 0.98 # 迭代惯性因子
c1 = 1.5 # 个体学习因子
c2 = 1.5 # 全体学习因子
alpha = 0.1
(vmax_x, vmax_y) = VarMax
(vmin_x, vmin_y) = VarMin
VelMax = (alpha * (vmax_x - vmin_x), alpha * (vmax_y - vmin_y)) #x,y方向最大速度
VelMin = (-vmax_x, -vmax_y) #x,y方向最小速度
# # 初始化
# 创建空结构
ep_best = Best(Position=None, Cost=None, Sol=None)
empty_particle = Particle(Position=None,
Velocity=None,
Cost=None,
Sol=None,
Best=ep_best)
# 初始化全体最优
GlobalBest = Best(Position=None, Cost=inf, Sol=None)
# 染色体存储空间初始化
#particle = repmat(empty_particle, nPop, 1)
particle = []
for i in range(nPop):
particle.append(empty_particle)
# 初始化循环
for i in range(nPop):
if i > 0:
(cx, cy) = CreateRandomSolution(model)
particle[i] = particle[i]._replace(
Position=(cx.reshape(-1), cy.reshape(-1))) # 初始化粒子
else:
# 初始化两点间直线
xx = linspace(model.xs, model.xt, model.n + 2)
yy = linspace(model.ys, model.yt, model.n + 2)
particle[i] = particle[i]._replace(Position=(xx[1:-1], yy[1:-1]))
# 初始化速度
particle[i] = particle[i]._replace(
Velocity=(np.zeros(VarSize).reshape(-1),
np.zeros(VarSize).reshape(-1)))
# 航迹代价计算
(pic, pis) = MyCost(particle[i].Position, model)
particle[i] = particle[i]._replace(Cost=pic, Sol=pis)
#[particle[i].Cost, particle[i].Sol] = MyCost(particle[i].Position, model)
# 更新个体最优
particle[i] = particle[i]._replace(
Best=Best(Position=particle[i].Position,
Cost=particle[i].Cost,
Sol=particle[i].Sol))
# 更新集体最优
if particle[i].Best.Cost.real < GlobalBest.Cost.real:
GlobalBest = particle[i].Best
# 保存最优航迹数据
BestCost = np.zeros(MaxIt)
# # 迭代过程
for it in range(MaxIt):
# 交叉变异 - 更新染色体
for i in range(nPop):
if i < nPop - it:
(sol1, sol2) = crossover(particle[i].Position,
particle[i + it].Position)
particle[i] = particle[i]._replace(Position=sol1) # 交叉
particle[i + it] = particle[i + it]._replace(Position=sol2)
else:
(sol1, sol2) = crossover(particle[i].Position,
particle[nPop - it].Position)
particle[i] = particle[i]._replace(Position=sol1)
particle[nPop - it] = particle[nPop -
it]._replace(Position=sol2)
particle[i] = particle[i]._replace(Position=mutation(
particle[i].Position)) # 变异
(VelMin_x, VelMin_y) = VelMin
(VelMax_x, VelMax_y) = VelMax
(VarMin_x, VarMin_y) = VarMin
(VarMax_x, VarMax_y) = VarMax
(vx, vy) = particle[i].Velocity
(px, py) = particle[i].Position
(bpx, bpy) = particle[i].Best.Position
(gpx, gpy) = GlobalBest.Position
# 更新速度
vx = list(vx)
vy = list(vy)
bpx = list(bpx)
bpy = list(bpy)
bpx_px = [bpx[i] - px[i] for i in range(len(bpx))]
bpy_py = [bpy[i] - py[i] for i in range(len(bpy))]
gpx_px = [gpx[i] - px[i] for i in range(len(gpx))]
gpy_py = [gpy[i] - py[i] for i in range(len(gpy))]
#velx = w * vx + c1 * np.random.rand(VarSize[0],VarSize[1]) * bpx_px + c2 * np.random.rand(VarSize[0],VarSize[1]) * gpx_px
velx = updatespeed(w, vx, c1, VarSize, bpx_px, c2, gpx_px)
# 更新速度范围
velx = clip(velx, VelMin_x, None)
velx = clip(velx, None, VelMax_x)
# 更新位置
posx = px + velx
# 速度镜像
OutOfTheRange = getOutTheRange(VarMin_x, VarMax_x, posx)
for i in OutOfTheRange:
velx[i] = -velx[i]
# 更新位置范围
posx = clip(posx, VarMin_x, None)
posx = clip(posx, None, VarMax_x)
#vely = w * vy + c1 * np.random.rand(VarSize[0],VarSize[1]) * bpy_py + c2 * np.random.rand(VarSize[0],VarSize[1]) * (gpy - py)
vely = updatespeed(w, vy, c1, VarSize, bpy_py, c2, gpy_py)
vely = clip(vely, VelMin_y, None)
vely = clip(vely, None, VelMax_y)
posy = py + vely
OutOfTheRange = getOutTheRange(VarMin_y, VarMax_y, posy)
for i in OutOfTheRange:
vely[i] = -vely[i]
posy = list(clip(posy, VarMin_y, None))
posy = list(clip(posy, None, VarMax_y))
particle[i] = particle[i]._replace(Position=(posx, posy),
Velocity=(velx, vely))
# 代价函数
(pic, pis) = MyCost(particle[i].Position, model)
particle[i] = particle[i]._replace(Cost=pic, Sol=pis)
#[particle[i] .Cost, particle[i] .Sol] = MyCost(particle[i].Position, model)
# 更新全局最佳
if particle[i].Cost.real < particle[i].Best.Cost.real:
particle[i] = particle[i]._replace(
Position=particle[i].Position,
Cost=particle[i].Cost,
Sol=particle[i].Sol)
#更新全局最佳
if particle[i].Best.Cost.real < GlobalBest.Cost.real:
GlobalBest = particle[i].Best
# 更新最优代价
BestCost[it] = GlobalBest.Cost.real
w = w * wdamp # 迭代更新惯性因子
# 显示最优代价
if GlobalBest.Sol.IsFeasible:
Flag = ' @'
else:
Flag = [', ', GlobalBest.Sol.Violation]
print('Iteration ', it, ': Best Cost = ', BestCost[it], Flag)
# 静态航迹绘图
PlotSolution(GlobalBest.Sol, model)
# # 结果输出静态航迹及总时间
"""
xoyo = ()
xoyo.x0 = GlobalBest.Sol.xx
xoyo.y0 = GlobalBest.Sol.yy
xoyo.T0 = GlobalBest.Sol.T
return xoyo
"""
return GlobalBest.Sol
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
import json
from tensorflow.keras.preprocessing.image import ImageDataGenerator, load_img, img_to_array
from tensorflow.keras import layers, models, optimizers, losses
from CreateModel import CreateModel
model = CreateModel()
checkpoint_dir = "training/"
latest = tf.train.latest_checkpoint(checkpoint_dir)
model.load_weights(latest)
with open('labels.json', 'r') as dic:
labels = json.load(dic)
def ShowFeatureMap(layer_num, img):
assert len(model.layers) > layer_num
assert type(
model.layers[layer_num]) == tf.python.keras.layers.convolutional.Conv2D
for i in range(layer_num + 1):
img = model.layers[i](img)
img = img[0].numpy()
height = 8
width = img.shape[-1] // height