def process(options, args):
g1 = ID_BCell()
g2 = ID_BCell()
g1.load(args[0])
g2.load(args[1])
if len(args) > 2:
filepath = args[2]
else:
filepath = "compare"
f1 = open(filepath + ".txt", 'w')
f1.write(
"Type Shim no. Original Orientation Replacement Orientation\n"
)
for listkey in g1.genome.magnet_lists.keys():
for i in range(len(g1.genome.magnet_lists[listkey])):
if g1.genome.magnet_lists[listkey][i] != g2.genome.magnet_lists[
listkey][i]:
allwrite = ("%2s \t%3i \t\t%3s \t%1i \t\t%3s \t\t%1i" %
(listkey, i, g1.genome.magnet_lists[listkey][i][0],
g1.genome.magnet_lists[listkey][i][1],
g2.genome.magnet_lists[listkey][i][0],
g2.genome.magnet_lists[listkey][i][1]))
f1.write(allwrite)
f1.write("\n")
f1.close()
def process(options, args):
if options.seed:
random.seed(int(options.seed_value))
if options.singlethreaded:
rank = 0
size = 1
else:
rank = MPI.COMM_WORLD.rank # The process ID (integer 0-3 for 4-process run)
size = MPI.COMM_WORLD.size # The number of processes in the job.
# get the hostname
if options.singlethreaded:
ip = 'localhost'
else:
ip = socket.gethostbyname(socket.gethostname())
logging.debug("Process %d ip address is : %s" % (rank, ip))
f2 = open(options.id_filename, 'r')
info = json.load(f2)
f2.close()
logging.debug("Loading Lookup")
f1 = h5py.File(options.lookup_filename, 'r')
lookup = {}
for beam in info['beams']:
logging.debug("Loading beam %s" % (beam['name']))
lookup[beam['name']] = f1[beam['name']][...]
f1.close()
barrier(options.singlethreaded)
logging.debug("Loading Initial Bfield")
f1 = h5py.File(options.bfield_filename, 'r')
real_bfield = f1['id_Bfield'][...]
f1.close()
logging.debug(real_bfield)
barrier(options.singlethreaded)
logging.debug("Loading magnets")
mags = magnets.Magnets()
mags.load(options.magnets_filename)
logging.debug('mpi runenr calling fg.generate_reference_magnets()')
ref_mags = fg.generate_reference_magnets(mags)
logging.debug('mpi runenr calling MagLists()')
ref_maglist = magnets.MagLists(ref_mags)
logging.debug('after ref_maglist')
ref_total_id_field = fg.generate_id_field(info, ref_maglist, ref_mags,
lookup)
pherr, ref_trajectories = mt.calculate_phase_error(info,
ref_total_id_field)
barrier(options.singlethreaded)
#epoch_path = os.path.join(args[0], 'epoch')
#next_epoch_path = os.path.join(args[0], 'nextepoch')
# start by creating the directory to put the initial population in
population = []
estar = options.e
# Load the initial genome
initialgenome = ID_BCell()
initialgenome.load(options.genome_filename)
referencegenome = ID_BCell()
referencegenome.load(options.genome_filename)
# make the initial population
for i in range(options.setup):
# create a fresh maglist
newgenome = ID_Shim_BCell()
newgenome.create(info, lookup, mags, initialgenome.genome,
ref_trajectories, options.number_of_changes,
real_bfield)
population.append(newgenome)
# gather the population
trans = []
for i in range(size):
trans.append(population)
allpop = alltoall(options.singlethreaded, trans)
barrier(options.singlethreaded)
newpop = []
for pop in allpop:
newpop += pop
# Need to deal with replicas and old genomes
popdict = {}
for genome in newpop:
fitness_key = "%1.8E" % (genome.fitness)
if fitness_key in popdict.keys():
if popdict[fitness_key].age < genome.age:
popdict[fitness_key] = genome
else:
popdict[fitness_key] = genome
newpop = []
for genome in popdict.values():
if genome.age < options.max_age:
newpop.append(genome)
newpop.sort(key=lambda x: x.fitness)
newpop = newpop[options.setup * rank:options.setup * (rank + 1)]
for genome in newpop:
logging.debug("genome fitness: %1.8E Age : %2i Mutations : %4i" %
(genome.fitness, genome.age, genome.mutations))
#Checkpoint best solution
if rank == 0:
logging.debug("Best fitness so far is %f" % (newpop[0].fitness))
newpop[0].save(args[0])
# now run the processing
for i in range(options.iterations):
barrier(options.singlethreaded)
logging.debug("Starting itteration %i" % (i))
nextpop = []
for genome in newpop:
# now we have to create the offspring
# TODO this is for the moment
logging.debug("Generating children for %s" % (genome.uid))
number_of_children = options.setup
number_of_mutations = mutations(options.c, estar, genome.fitness,
options.scale)
children = genome.generate_children(number_of_children,
number_of_mutations,
info,
lookup,
mags,
ref_trajectories,
real_bfield=real_bfield)
# now save the children into the new file
for child in children:
nextpop.append(child)
# and save the original
nextpop.append(genome)
# gather the population
trans = []
for i in range(size):
trans.append(nextpop)
allpop = alltoall(options.singlethreaded, trans)
newpop = []
for pop in allpop:
newpop += pop
popdict = {}
for genome in newpop:
fitness_key = "%1.8E" % (genome.fitness)
if fitness_key in popdict.keys():
if popdict[fitness_key].age < genome.age:
popdict[fitness_key] = genome
else:
popdict[fitness_key] = genome
newpop = []
for genome in popdict.values():
if genome.age < options.max_age:
newpop.append(genome)
newpop.sort(key=lambda x: x.fitness)
estar = newpop[0].fitness * 0.99
logging.debug("new estar is %f" % (estar))
newpop = newpop[options.setup * rank:options.setup * (rank + 1)]
#Checkpoint best solution
if rank == 0:
initialgenome.genome.mutate_from_list(newpop[0].genome)
initialgenome.fitness = newpop[0].fitness
initialgenome.uid = "A" + newpop[0].uid
initialgenome.save(args[0])
saveh5(args[0], initialgenome, referencegenome, info, mags,
real_bfield, lookup)
# After the save reload the original data
initialgenome.load(options.genome_filename)
newpop[0].save(args[0])
for genome in newpop:
logging.debug(
"genome fitness: %1.8E Age : %2i Mutations : %4i" %
(genome.fitness, genome.age, genome.mutations))
barrier(options.singlethreaded)
barrier(options.singlethreaded)
# gather the population
trans = []
for i in range(size):
trans.append(nextpop)
allpop = alltoall(options.singlethreaded, trans)
newpop = []
for pop in allpop:
newpop += pop
newpop.sort(key=lambda x: x.fitness)
newpop = newpop[options.setup * rank:options.setup * (rank + 1)]
#Checkpoint best solution
if rank == 0:
initialgenome.genome.mutate_from_list(newpop[0].genome)
initialgenome.age_bcell()
initialgenome.save(args[0])
newpop[0].save(args[0])
from genome_tools import ID_BCell
if __name__ == "__main__":
import optparse
usage = "%prog [options] data"
parser = optparse.OptionParser(usage=usage)
(options, args) = parser.parse_args()
g1 = ID_BCell()
g2 = ID_BCell()
g1.load(args[0])
g2.load(args[1])
for listkey in g1.genome.magnet_lists.keys():
for i in range(len(g1.genome.magnet_lists[listkey])):
if g1.genome.magnet_lists[listkey][i] != g2.genome.magnet_lists[listkey][i]:
print listkey
print i
print g1.genome.magnet_lists[listkey][i]
print g2.genome.magnet_lists[listkey][i]
print ""
ref_mags = fg.generate_reference_magnets(mags)
ref_maglist = magnets.MagLists(ref_mags)
ref_total_id_field = fg.generate_id_field(info, ref_maglist, ref_mags, lookup)
pherr, ref_trajectories = mt.calculate_phase_error(info, ref_total_id_field)
MPI.COMM_WORLD.Barrier()
#epoch_path = os.path.join(args[0], 'epoch')
#next_epoch_path = os.path.join(args[0], 'nextepoch')
# start by creating the directory to put the initial population in
population = []
estar = options.e
# Load the initial genome
initialgenome = ID_BCell()
initialgenome.load(options.genome_filename)
referencegenome = ID_BCell()
referencegenome.load(options.genome_filename)
# make the initial population
for i in range(options.setup):
# create a fresh maglist
newgenome = ID_Shim_BCell()
newgenome.create(info, lookup, mags, initialgenome.genome, ref_trajectories, options.number_of_changes, real_bfield)
population.append(newgenome)
# gather the population
trans = []
for i in range(size):
dest="build",
help="Build output for input files",
action="store_true",
default=False)
(options, args) = parser.parse_args()
print("Loading magnets")
mags = magnets.Magnets()
mags.load(options.magnets_filename)
if options.build:
for filename in args[1::]:
print("Processing file %s" % (filename))
# load the genome
genome = ID_BCell()
genome.load(filename)
outfile = os.path.join(args[0], os.path.split(filename)[1] + '.h5')
fg.output_fields(outfile, options.id_filename,
options.lookup_filename, options.magnets_filename,
genome.genome)
sys.exit(0)
if options.setup > 0:
print("Running setup")
for i in range(options.setup):
# create a fresh maglist
maglist = magnets.MagLists(mags)
maglist.shuffle_all()
genome = ID_BCell(options.id_filename, options.lookup_filename,
MPI.COMM_WORLD.Barrier()
logging.debug("Loading magnets")
mags = magnets.Magnets()
mags.load(options.magnets_filename)
ref_mags = fg.generate_reference_magnets(mags)
ref_maglist = magnets.MagLists(ref_mags)
ref_total_id_field = fg.generate_id_field(info, ref_maglist, ref_mags, lookup)
pherr, ref_trajectories = mt.calculate_phase_error(info, ref_total_id_field)
MPI.COMM_WORLD.Barrier()
# Load the initial genome
genome = ID_BCell()
genome.load(options.genome_filename)
MPI.COMM_WORLD.Barrier()
# now run the processing
children = genome.generate_children(options.num_children, options.number_of_mutations, info, lookup, mags, ref_trajectories, real_bfield=real_bfield)
children.sort(key=lambda x: x.fitness)
# send the best member of the population for sorting
trans = []
for i in range(size):
trans.append(children[0])
parser.add_option("--param_c", dest="c", help="Set the OPT-AI parameter c", default=10.0, type='float')
parser.add_option("--param_e", dest="e", help="Set the OPT-AI parameter eStar", default=0.0, type='float')
parser.add_option("--param_scale", dest="scale", help="Set the OPT-AI parameter scale", default=10.0, type='float')
parser.add_option("-b", "--build", dest="build", help="Build output for input files", action="store_true", default=False)
(options, args) = parser.parse_args()
print("Loading magnets")
mags = magnets.Magnets()
mags.load(options.magnets_filename)
if options.build:
for filename in args[1::]:
print("Processing file %s" % (filename))
# load the genome
genome = ID_BCell()
genome.load(filename)
outfile = os.path.join(args[0], os.path.split(filename)[1]+'.h5')
fg.output_fields(outfile, options.id_filename, options.lookup_filename,
options.magnets_filename, genome.genome);
sys.exit(0)
if options.setup > 0:
print("Running setup")
for i in range(options.setup):
# create a fresh maglist
maglist = magnets.MagLists(mags)
maglist.shuffle_all()
genome = ID_BCell(options.id_filename, options.lookup_filename, options.magnets_filename)
genome.create(maglist)
MPI.COMM_WORLD.Barrier()
logging.debug("Loading magnets")
mags = magnets.Magnets()
mags.load(options.magnets_filename)
ref_mags = fg.generate_reference_magnets(mags)
ref_maglist = magnets.MagLists(ref_mags)
ref_total_id_field = fg.generate_id_field(info, ref_maglist, ref_mags, lookup)
pherr, ref_trajectories = mt.calculate_phase_error(info, ref_total_id_field)
MPI.COMM_WORLD.Barrier()
# Load the initial genome
genome = ID_BCell()
genome.load(options.genome_filename)
MPI.COMM_WORLD.Barrier()
# now run the processing
children = genome.generate_children(options.num_children,
options.number_of_mutations,
info,
lookup,
mags,
ref_trajectories,
real_bfield=real_bfield)
children.sort(key=lambda x: x.fitness)
def process(options, args):
if options.seed:
random.seed(int(options.seed_value))
if options.singlethreaded:
rank = 0
size = 1
else:
rank = MPI.COMM_WORLD.rank # The process ID (integer 0-3 for 4-process run)
size = MPI.COMM_WORLD.size # The number of processes in the job.
# get the hostname
if options.singlethreaded:
ip = 'localhost'
else:
ip = socket.gethostbyname(socket.gethostname())
logging.debug("Process %d ip address is : %s" % (rank, ip))
f2 = open(options.id_filename, 'r')
info = json.load(f2)
f2.close()
logging.debug("Loading Lookup")
f1 = h5py.File(options.lookup_filename, 'r')
lookup = {}
for beam in info['beams']:
logging.debug("Loading beam %s" % (beam['name']))
lookup[beam['name']] = f1[beam['name']][...]
f1.close()
barrier(options.singlethreaded)
logging.debug("Loading magnets")
mags = magnets.Magnets()
mags.load(options.magnets_filename)
ref_mags = fg.generate_reference_magnets(mags)
ref_maglist = magnets.MagLists(ref_mags)
ref_total_id_field = fg.generate_id_field(info, ref_maglist, ref_mags,
lookup)
#logging.debug("before phase calculate error call")
#logging.debug(ref_total_id_field.shape())
pherr, ref_trajectories = mt.calculate_phase_error(info,
ref_total_id_field)
barrier(options.singlethreaded)
#epoch_path = os.path.join(args[0], 'epoch')
#next_epoch_path = os.path.join(args[0], 'nextepoch')
# start by creating the directory to put the initial population in
population = []
estar = options.e
if options.restart and (rank == 0):
filenames = os.listdir(args[0])
# sort the genome filenames to ensure that when given the same set of
# files in a directory, population[0] is the same across different
# orderings of the listed directory contents: this is to fix the test
# MpiRunnerTest.test_process_initial_population() in mpi_runner_test.py
# when run on travis
filenames.sort()
for filename in filenames:
fullpath = os.path.join(args[0], filename)
try:
logging.debug("Trying to load %s" % (fullpath))
genome = ID_BCell()
genome.load(fullpath)
population.append(genome)
logging.debug("Loaded %s" % (fullpath))
except:
logging.debug("Failed to load %s" % (fullpath))
if len(population) < options.setup:
# Seed with children from first
children = population[0].generate_children(
options.setup - len(population), 20, info, lookup, mags,
ref_trajectories)
# now save the children into the new file
for child in children:
population.append(child)
else:
logging.debug("make the initial population")
for i in range(options.setup):
# create a fresh maglist
maglist = magnets.MagLists(mags)
maglist.shuffle_all()
genome = ID_BCell()
genome.create(info, lookup, mags, maglist, ref_trajectories)
population.append(genome)
logging.debug("Initial population created")
# gather the population
trans = []
for i in range(size):
trans.append(population)
allpop = alltoall(options.singlethreaded, trans)
barrier(options.singlethreaded)
newpop = []
for pop in allpop:
newpop += pop
# Need to deal with replicas and old genomes
popdict = {}
for genome in newpop:
fitness_key = "%1.8E" % (genome.fitness)
if fitness_key in popdict.keys():
if popdict[fitness_key].age < genome.age:
popdict[fitness_key] = genome
else:
popdict[fitness_key] = genome
newpop = []
for genome in popdict.values():
if genome.age < options.max_age:
newpop.append(genome)
newpop.sort(key=lambda x: x.fitness)
newpop = newpop[options.setup * rank:options.setup * (rank + 1)]
for genome in newpop:
logging.debug("genome fitness: %1.8E Age : %2i Mutations : %4i" %
(genome.fitness, genome.age, genome.mutations))
#Checkpoint best solution
if rank == 0:
newpop[0].save(args[0])
# now run the processing
for i in range(options.iterations):
barrier(options.singlethreaded)
logging.debug("Starting itteration %i" % (i))
nextpop = []
for genome in newpop:
# now we have to create the offspring
# TODO this is for the moment
logging.debug("Generating children for %s" % (genome.uid))
number_of_children = options.setup
number_of_mutations = mutations(options.c, estar, genome.fitness,
options.scale)
children = genome.generate_children(number_of_children,
number_of_mutations, info,
lookup, mags, ref_trajectories)
# now save the children into the new file
for child in children:
nextpop.append(child)
# and save the original
nextpop.append(genome)
# gather the population
trans = []
for i in range(size):
trans.append(nextpop)
allpop = alltoall(options.singlethreaded, trans)
newpop = []
for pop in allpop:
newpop += pop
popdict = {}
for genome in newpop:
fitness_key = "%1.8E" % (genome.fitness)
if fitness_key in popdict.keys():
if popdict[fitness_key].age < genome.age:
popdict[fitness_key] = genome
else:
popdict[fitness_key] = genome
newpop = []
for genome in popdict.values():
if genome.age < options.max_age:
newpop.append(genome)
newpop.sort(key=lambda x: x.fitness)
estar = newpop[0].fitness * 0.99
logging.debug("new estar is %f" % (estar))
newpop = newpop[options.setup * rank:options.setup * (rank + 1)]
#Checkpoint best solution
if rank == 0:
newpop[0].save(args[0])
for genome in newpop:
logging.debug(
"genome fitness: %1.8E Age : %2i Mutations : %4i" %
(genome.fitness, genome.age, genome.mutations))
barrier(options.singlethreaded)
barrier(options.singlethreaded)
# gather the population
trans = []
for i in range(size):
trans.append(nextpop)
allpop = alltoall(options.singlethreaded, trans)
newpop = []
for pop in allpop:
newpop += pop
newpop.sort(key=lambda x: x.fitness)
newpop = newpop[options.setup * rank:options.setup * (rank + 1)]
#Checkpoint best solution
if rank == 0:
newpop[0].save(args[0])
MPI.COMM_WORLD.Barrier()
#epoch_path = os.path.join(args[0], 'epoch')
#next_epoch_path = os.path.join(args[0], 'nextepoch')
# start by creating the directory to put the initial population in
population = []
estar = options.e
if options.restart and (rank == 0) :
for filename in os.listdir(args[0]):
fullpath = os.path.join(args[0],filename)
try :
logging.debug("Trying to load %s" % (fullpath))
genome = ID_BCell()
genome.load(fullpath)
population.append(genome)
logging.debug("Loaded %s" % (fullpath))
except :
logging.debug("Failed to load %s" % (fullpath))
if len(population) < options.setup:
# Seed with children from first
children = population[0].generate_children(options.setup-len(population), 20, info, lookup, mags, ref_trajectories)
# now save the children into the new file
for child in children:
population.append(child)
else :
# make the initial population
for i in range(options.setup):
# create a fresh maglist