def process(self, set, lock=None):
# Print channel being processed
self.message('Processing channel %s.' % set['channel'])
# Placeholder for winner
winner = None
# Variable Normalization
aves = {}; stds = {}
# Loop over the training sets
for counter in xrange(1,Common.NeatNumberTries+1):
# Setting the indir directory
indir = '%s/scratch/%s/Trainings/%s/Training%05d' % (
Common.NeatDirectory, self.getParameter('input'), set['channel'], counter
)
# Creating output directory
outdir = '%s/scratch/%s/Trainings/%s' % (
Common.NeatDirectory, self.getParameter('input'), set['channel']
)
# Look for missing files
files = ['neat.config', 'winner.dat']
missing = False
for file in files:
if not os.path.isfile("%s/%s" % (indir, file)):
self.message('%s does not exist in %s.' % (file, indir))
missing = True
if missing: continue
# Compute the normalization if needed
if len (aves) == 0 and self.getParameter('normalization','true') == 'true':
variables = open('%s/inputvars.txt' % indir).readlines()
variables = [variable.rstrip() for variable in variables]
# Reading training sample for computing normalization
training = TopovarReader(variables)
# Adding training files
for sample in Common.TrainingBackgrounds + Common.TrainingSignals:
training.add('%s/%s/%s.root' % (
Common.SampleLocation, Common.TrainingSample, filename(set, sample)
)
)
normalizer = VariableNormalizer(variables)
normalizer.add(training.sample(True))
normalizer.report()
for variable in variables:
ave, std = normalizer(variable)
aves[variable] = ave
stds[variable] = std
## Write winner files
# Read the winner
net = pickle.load(open('%s/winner.dat' % indir))
# Winner information
candidate = {
'training' : 'Training%05d' % counter,
'fitness' : net.fitness,
'variables' : variables,
'aves' : aves,
'stds' : stds
}
if not winner or candidate['fitness'] > winner['fitness']:
self.message('Winner candidate for channel %s found in training %s with fitness %.4g' % (
set['channel'], candidate['training'], candidate['fitness']
)
)
winner = candidate
file = open('%s/winner.info' % outdir,'w')
pickle.dump(winner, file)
)
# Get channel winner network
winner = pickle.load(open('%s/winner.info' % indir))
# Collect the list of variables
variables[key] = winner['variables']
# Collect the net
net = pickle.load(open('%s/%s/winner.dat' % (indir, winner['training'])))
# Create phenotype from genome
nets[key] = nn.create_ffphenotype(net)
# Creating a variable normalizers
normalizers[key] = VariableNormalizer(winner['variables'], winner['aves'], winner['stds'])
# Create a topowriter for adding neat outputs
topovars = None
if len(variables) > 1:
topovars = TopovarWriter(options.output, [], options.input)
else:
topovars = TopovarWriter(options.output, variables.itervalues().next(), options.input)
# Add neat outputs
for key in nets:
topovars.addVariable(key, 'double')
# Loop over the tree adding the neat output
for entry in xrange(topovars.getInTreeEntries()):
if entry % 1000 == 0 and entry != 0:
def Evaluate(set):
global signalYield, signalSample, backgroundYield, backgroundSample
# Reading variable list
variables = open('%s/inputvars.txt' % set['directory']).readlines()
variables = [variable.rstrip() for variable in variables]
# Setting the number of input and output in the neat config file
file = open ('%s/neat.config' % set['directory'])
template = string.Template(file.read())
file.close()
file = open ('%s/neat.config' % set['directory'], 'w')
file.write(template.safe_substitute(input_nodes = len(variables), output_nodes = 1))
file.close()
# Read neat configuration file
config.load('%s/neat.config' % set['directory'])
print('Training: Reading signal files')
# Signal topovars
signals = TopovarReader(variables)
# Adding signal files
for sample in Common.TrainingSignals:
signals.add('%s/%s/%s.root' % (
Common.SampleLocation, Common.TrainingSample, Common.filename(set, sample)
)
)
# Creating a variable normalizer
normalizer = VariableNormalizer(variables)
# Saving the population in buffer
signalSample = signals.sample(compress = True)
# Adding the sample to the normalizer
normalizer.add(signalSample)
# Compute the total weight
signalYield = normalizer.getTotalWeight()
print ('Training: Reading background files')
# Background topovars
backgrounds = TopovarReader(variables)
# Adding background files
for sample in Common.TrainingBackgrounds:
backgrounds.add('%s/%s/%s.root' % (
Common.SampleLocation, Common.TrainingSample, Common.filename(set, sample)
)
)
# Saving the population in buffer
backgroundSample = backgrounds.sample(compress = True)
# Adding the sample to the normalizer
normalizer.add(backgroundSample)
# Compute the total weight
backgroundYield = normalizer.getTotalWeight() - signalYield
# Reporting the normalization
normalizer.report()
# Normalization of the variables
normalizer.normalizeSample(signalSample)
normalizer.normalizeSample(backgroundSample)
# NEAT training
chromosome.node_gene_type = genome.NodeGene
population.Population.evaluate = FitnessFunctionWrapper
pop = population.Population()
pop.epoch(int(set['number_generations']), report=True, save_best=False, checkpoint_interval = None)
winner = pop.stats[0][-1]
print 'Training: Number of evaluations: %d' % winner.id
print 'Training: Best NN fitness: %0.2f' % winner.fitness
# Save the best network
file = open('%s/winner.dat' % set['directory'], 'w')
pickle.dump(winner, file)
file.close()
# Save the best network
file = open('%s/winner-fitness.txt' % set['directory'], 'w')
file.write('%f' % winner.fitness)
file.close()
def Evaluate(set):
global signalYield, signalSample, backgroundYield, backgroundSample
# Reading variable list
variables = open('%s/inputvars.txt' % set['directory']).readlines()
variables = [variable.rstrip() for variable in variables]
# Setting the number of input and output in the neat config file
file = open('%s/neat.config' % set['directory'])
template = string.Template(file.read())
file.close()
file = open('%s/neat.config' % set['directory'], 'w')
file.write(
template.safe_substitute(input_nodes=len(variables), output_nodes=1))
file.close()
# Read neat configuration file
config.load('%s/neat.config' % set['directory'])
print('Training: Reading signal files')
# Signal topovars
signals = TopovarReader(variables)
# Adding signal files
for sample in Common.TrainingSignals:
signals.add('%s/%s/%s.root' %
(Common.SampleLocation, Common.TrainingSample,
Common.filename(set, sample)))
# Creating a variable normalizer
normalizer = VariableNormalizer(variables)
# Saving the population in buffer
signalSample = signals.sample(compress=True)
# Adding the sample to the normalizer
normalizer.add(signalSample)
# Compute the total weight
signalYield = normalizer.getTotalWeight()
print('Training: Reading background files')
# Background topovars
backgrounds = TopovarReader(variables)
# Adding background files
for sample in Common.TrainingBackgrounds:
backgrounds.add('%s/%s/%s.root' %
(Common.SampleLocation, Common.TrainingSample,
Common.filename(set, sample)))
# Saving the population in buffer
backgroundSample = backgrounds.sample(compress=True)
# Adding the sample to the normalizer
normalizer.add(backgroundSample)
# Compute the total weight
backgroundYield = normalizer.getTotalWeight() - signalYield
# Reporting the normalization
normalizer.report()
# Normalization of the variables
normalizer.normalizeSample(signalSample)
normalizer.normalizeSample(backgroundSample)
# NEAT training
chromosome.node_gene_type = genome.NodeGene
population.Population.evaluate = FitnessFunctionWrapper
pop = population.Population()
pop.epoch(int(set['number_generations']),
report=True,
save_best=False,
checkpoint_interval=None)
winner = pop.stats[0][-1]
print 'Training: Number of evaluations: %d' % winner.id
print 'Training: Best NN fitness: %0.2f' % winner.fitness
# Save the best network
file = open('%s/winner.dat' % set['directory'], 'w')
pickle.dump(winner, file)
file.close()
# Save the best network
file = open('%s/winner-fitness.txt' % set['directory'], 'w')
file.write('%f' % winner.fitness)
file.close()