def output_fields(filename, id_filename, lookup_filename, magnets_filename, maglist):
f2 = open(id_filename, 'r')
info = json.load(f2)
f2.close()
f1 = h5py.File(lookup_filename, 'r')
lookup = {}
for beam in info['beams']:
lookup[beam['name']] = f1[beam['name']][...]
f1.close()
mags = magnets.Magnets()
mags.load(magnets_filename)
ref_mags=generate_reference_magnets(mags)
f = h5py.File(filename, 'w')
per_beam_field = generate_per_beam_b_field(info, maglist, mags, lookup)
total_id_field = generate_id_field(info, maglist, mags, lookup)
for name in per_beam_field.keys():
f.create_dataset("%s_per_beam" % (name), data=per_beam_field[name])
f.create_dataset('id_Bfield', data=total_id_field)
trajectory_information=mt.calculate_phase_error(info, total_id_field)
f.create_dataset('id_phase_error', data = trajectory_information[0])
f.create_dataset('id_trajectory', data = trajectory_information[1])
per_beam_field = generate_per_beam_b_field(info, maglist, ref_mags, lookup)
total_id_field = generate_id_field(info, maglist, ref_mags, lookup)
for name in per_beam_field.keys():
f.create_dataset("%s_per_beam_perfect" % (name), data=per_beam_field[name])
f.create_dataset('id_Bfield_perfect', data=total_id_field)
trajectory_information=mt.calculate_phase_error(info, total_id_field)
f.create_dataset('id_phase_error_perfect', data = trajectory_information[0])
f.create_dataset('id_trajectory_perfect', data = trajectory_information[1])
f.close()
def generate_reference_magnets(mags):
ref_mags=magnets.Magnets()
for magtype in mags.magnet_sets.keys():
mag_dir = mags.magnet_sets[magtype].values()[0].argmax()
unit = np.zeros(3)
unit[mag_dir] = mags.mean_field[magtype]
#ref_mags.add_perfect_magnet_set(magtype, len(mags.magnet_sets[magtype]) , unit, mags.magnet_flip[magtype])
ref_mags.add_perfect_magnet_set_duplicate(magtype, mags.magnet_sets[magtype] , unit, mags.magnet_flip[magtype])
return ref_mags
def generate_reference_magnets(mags):
ref_mags = magnets.Magnets()
for magtype in mags.magnet_sets.keys():
mag_dir = mags.magnet_sets[magtype].values()[0].argmax()
unit = np.zeros(3)
unit[mag_dir] = mags.mean_field[magtype]
#ref_mags.add_perfect_magnet_set(magtype, len(mags.magnet_sets[magtype]) , unit, mags.magnet_flip[magtype])
ref_mags.add_perfect_magnet_set_duplicate(magtype,
mags.magnet_sets[magtype],
unit,
mags.magnet_flip[magtype])
#logging.debug("ref_mags shape %s"%(str(ref_mags.shape))) magnets object has no attribute shape
return ref_mags
def calculate_fitness(id_filename, lookup_filename, magnets_filename, maglist):
# TODO this will be slow, but should be optimizable with lookups
lookup = h5py.File(lookup_filename, 'r')
f2 = open(id_filename, 'r')
info = json.load(f2)
f2.close()
mags = magnets.Magnets()
mags.load(magnets_filename)
ref_mags = generate_reference_magnets(mags)
ref_maglist = magnets.MagLists(ref_mags)
ref_total_id_field = generate_id_field(info, ref_maglist, ref_mags, lookup)
result = calculate_cached_fitness(info, lookup, magnets, maglist, ref_total_id_field)
lookup.close()
return result
def process(options, args):
if options.seed:
random.seed(int(options.seed_value))
if options.singlethreaded:
rank = 0
size = 1
else:
rank = MPI.COMM_WORLD.rank # The process ID (integer 0-3 for 4-process run)
size = MPI.COMM_WORLD.size # The number of processes in the job.
# get the hostname
if options.singlethreaded:
ip = 'localhost'
else:
ip = socket.gethostbyname(socket.gethostname())
logging.debug("Process %d ip address is : %s" % (rank, ip))
f2 = open(options.id_filename, 'r')
info = json.load(f2)
f2.close()
logging.debug("Loading Lookup")
f1 = h5py.File(options.lookup_filename, 'r')
lookup = {}
for beam in info['beams']:
logging.debug("Loading beam %s" % (beam['name']))
lookup[beam['name']] = f1[beam['name']][...]
f1.close()
barrier(options.singlethreaded)
logging.debug("Loading Initial Bfield")
f1 = h5py.File(options.bfield_filename, 'r')
real_bfield = f1['id_Bfield'][...]
f1.close()
logging.debug(real_bfield)
barrier(options.singlethreaded)
logging.debug("Loading magnets")
mags = magnets.Magnets()
mags.load(options.magnets_filename)
logging.debug('mpi runenr calling fg.generate_reference_magnets()')
ref_mags = fg.generate_reference_magnets(mags)
logging.debug('mpi runenr calling MagLists()')
ref_maglist = magnets.MagLists(ref_mags)
logging.debug('after ref_maglist')
ref_total_id_field = fg.generate_id_field(info, ref_maglist, ref_mags,
lookup)
pherr, ref_trajectories = mt.calculate_phase_error(info,
ref_total_id_field)
barrier(options.singlethreaded)
#epoch_path = os.path.join(args[0], 'epoch')
#next_epoch_path = os.path.join(args[0], 'nextepoch')
# start by creating the directory to put the initial population in
population = []
estar = options.e
# Load the initial genome
initialgenome = ID_BCell()
initialgenome.load(options.genome_filename)
referencegenome = ID_BCell()
referencegenome.load(options.genome_filename)
# make the initial population
for i in range(options.setup):
# create a fresh maglist
newgenome = ID_Shim_BCell()
newgenome.create(info, lookup, mags, initialgenome.genome,
ref_trajectories, options.number_of_changes,
real_bfield)
population.append(newgenome)
# gather the population
trans = []
for i in range(size):
trans.append(population)
allpop = alltoall(options.singlethreaded, trans)
barrier(options.singlethreaded)
newpop = []
for pop in allpop:
newpop += pop
# Need to deal with replicas and old genomes
popdict = {}
for genome in newpop:
fitness_key = "%1.8E" % (genome.fitness)
if fitness_key in popdict.keys():
if popdict[fitness_key].age < genome.age:
popdict[fitness_key] = genome
else:
popdict[fitness_key] = genome
newpop = []
for genome in popdict.values():
if genome.age < options.max_age:
newpop.append(genome)
newpop.sort(key=lambda x: x.fitness)
newpop = newpop[options.setup * rank:options.setup * (rank + 1)]
for genome in newpop:
logging.debug("genome fitness: %1.8E Age : %2i Mutations : %4i" %
(genome.fitness, genome.age, genome.mutations))
#Checkpoint best solution
if rank == 0:
logging.debug("Best fitness so far is %f" % (newpop[0].fitness))
newpop[0].save(args[0])
# now run the processing
for i in range(options.iterations):
barrier(options.singlethreaded)
logging.debug("Starting itteration %i" % (i))
nextpop = []
for genome in newpop:
# now we have to create the offspring
# TODO this is for the moment
logging.debug("Generating children for %s" % (genome.uid))
number_of_children = options.setup
number_of_mutations = mutations(options.c, estar, genome.fitness,
options.scale)
children = genome.generate_children(number_of_children,
number_of_mutations,
info,
lookup,
mags,
ref_trajectories,
real_bfield=real_bfield)
# now save the children into the new file
for child in children:
nextpop.append(child)
# and save the original
nextpop.append(genome)
# gather the population
trans = []
for i in range(size):
trans.append(nextpop)
allpop = alltoall(options.singlethreaded, trans)
newpop = []
for pop in allpop:
newpop += pop
popdict = {}
for genome in newpop:
fitness_key = "%1.8E" % (genome.fitness)
if fitness_key in popdict.keys():
if popdict[fitness_key].age < genome.age:
popdict[fitness_key] = genome
else:
popdict[fitness_key] = genome
newpop = []
for genome in popdict.values():
if genome.age < options.max_age:
newpop.append(genome)
newpop.sort(key=lambda x: x.fitness)
estar = newpop[0].fitness * 0.99
logging.debug("new estar is %f" % (estar))
newpop = newpop[options.setup * rank:options.setup * (rank + 1)]
#Checkpoint best solution
if rank == 0:
initialgenome.genome.mutate_from_list(newpop[0].genome)
initialgenome.fitness = newpop[0].fitness
initialgenome.uid = "A" + newpop[0].uid
initialgenome.save(args[0])
saveh5(args[0], initialgenome, referencegenome, info, mags,
real_bfield, lookup)
# After the save reload the original data
initialgenome.load(options.genome_filename)
newpop[0].save(args[0])
for genome in newpop:
logging.debug(
"genome fitness: %1.8E Age : %2i Mutations : %4i" %
(genome.fitness, genome.age, genome.mutations))
barrier(options.singlethreaded)
barrier(options.singlethreaded)
# gather the population
trans = []
for i in range(size):
trans.append(nextpop)
allpop = alltoall(options.singlethreaded, trans)
newpop = []
for pop in allpop:
newpop += pop
newpop.sort(key=lambda x: x.fitness)
newpop = newpop[options.setup * rank:options.setup * (rank + 1)]
#Checkpoint best solution
if rank == 0:
initialgenome.genome.mutate_from_list(newpop[0].genome)
initialgenome.age_bcell()
initialgenome.save(args[0])
newpop[0].save(args[0])
parser = optparse.OptionParser(usage=usage)
(options, args) = parser.parse_args()
#f2 = open('/home/gdy32713/DAWN_stable/optid/Opt-ID/IDSort/src/v2/2015test.json', 'r')
f2 = open(args[0], 'r')
info = json.load(f2)
f2.close()
#f1 = h5py.File('/home/gdy32713/DAWN_stable/optid/Opt-ID/IDSort/src/v2/2015test.h5', 'r')
f1 = h5py.File(args[1], 'r')
lookup = {}
for beam in info['beams']:
lookup[beam['name']] = f1[beam['name']][...]
f1.close()
mags = magnets.Magnets()
#mags.load('/home/gdy32713/DAWN_stable/optid/Opt-ID/IDSort/src/v2/magnets.mag')
mags.load(args[2])
ref_mags = generate_reference_magnets(mags)
ref_maglist = magnets.MagLists(ref_mags)
ref_total_id_field = generate_id_field(info, ref_maglist, ref_mags, lookup)
ref_pherr, ref_trajectories = mt.calculate_phase_error(
info, ref_total_id_field)
maglist = magnets.MagLists(mags)
maglist.shuffle_all()
original_bfield, maglist_fitness = calculate_cached_trajectory_fitness(
info, lookup, mags, maglist, ref_trajectories)
mag_array = generate_per_magnet_array(info, maglist, mags)
def process(options, args):
if options.seed:
random.seed(int(options.seed_value))
if options.singlethreaded:
rank = 0
size = 1
else:
rank = MPI.COMM_WORLD.rank # The process ID (integer 0-3 for 4-process run)
size = MPI.COMM_WORLD.size # The number of processes in the job.
# get the hostname
if options.singlethreaded:
ip = 'localhost'
else:
ip = socket.gethostbyname(socket.gethostname())
logging.debug("Process %d ip address is : %s" % (rank, ip))
f2 = open(options.id_filename, 'r')
info = json.load(f2)
f2.close()
logging.debug("Loading Lookup")
f1 = h5py.File(options.lookup_filename, 'r')
lookup = {}
for beam in info['beams']:
logging.debug("Loading beam %s" % (beam['name']))
lookup[beam['name']] = f1[beam['name']][...]
f1.close()
barrier(options.singlethreaded)
logging.debug("Loading magnets")
mags = magnets.Magnets()
mags.load(options.magnets_filename)
ref_mags = fg.generate_reference_magnets(mags)
ref_maglist = magnets.MagLists(ref_mags)
ref_total_id_field = fg.generate_id_field(info, ref_maglist, ref_mags,
lookup)
#logging.debug("before phase calculate error call")
#logging.debug(ref_total_id_field.shape())
pherr, ref_trajectories = mt.calculate_phase_error(info,
ref_total_id_field)
barrier(options.singlethreaded)
#epoch_path = os.path.join(args[0], 'epoch')
#next_epoch_path = os.path.join(args[0], 'nextepoch')
# start by creating the directory to put the initial population in
population = []
estar = options.e
if options.restart and (rank == 0):
filenames = os.listdir(args[0])
# sort the genome filenames to ensure that when given the same set of
# files in a directory, population[0] is the same across different
# orderings of the listed directory contents: this is to fix the test
# MpiRunnerTest.test_process_initial_population() in mpi_runner_test.py
# when run on travis
filenames.sort()
for filename in filenames:
fullpath = os.path.join(args[0], filename)
try:
logging.debug("Trying to load %s" % (fullpath))
genome = ID_BCell()
genome.load(fullpath)
population.append(genome)
logging.debug("Loaded %s" % (fullpath))
except:
logging.debug("Failed to load %s" % (fullpath))
if len(population) < options.setup:
# Seed with children from first
children = population[0].generate_children(
options.setup - len(population), 20, info, lookup, mags,
ref_trajectories)
# now save the children into the new file
for child in children:
population.append(child)
else:
logging.debug("make the initial population")
for i in range(options.setup):
# create a fresh maglist
maglist = magnets.MagLists(mags)
maglist.shuffle_all()
genome = ID_BCell()
genome.create(info, lookup, mags, maglist, ref_trajectories)
population.append(genome)
logging.debug("Initial population created")
# gather the population
trans = []
for i in range(size):
trans.append(population)
allpop = alltoall(options.singlethreaded, trans)
barrier(options.singlethreaded)
newpop = []
for pop in allpop:
newpop += pop
# Need to deal with replicas and old genomes
popdict = {}
for genome in newpop:
fitness_key = "%1.8E" % (genome.fitness)
if fitness_key in popdict.keys():
if popdict[fitness_key].age < genome.age:
popdict[fitness_key] = genome
else:
popdict[fitness_key] = genome
newpop = []
for genome in popdict.values():
if genome.age < options.max_age:
newpop.append(genome)
newpop.sort(key=lambda x: x.fitness)
newpop = newpop[options.setup * rank:options.setup * (rank + 1)]
for genome in newpop:
logging.debug("genome fitness: %1.8E Age : %2i Mutations : %4i" %
(genome.fitness, genome.age, genome.mutations))
#Checkpoint best solution
if rank == 0:
newpop[0].save(args[0])
# now run the processing
for i in range(options.iterations):
barrier(options.singlethreaded)
logging.debug("Starting itteration %i" % (i))
nextpop = []
for genome in newpop:
# now we have to create the offspring
# TODO this is for the moment
logging.debug("Generating children for %s" % (genome.uid))
number_of_children = options.setup
number_of_mutations = mutations(options.c, estar, genome.fitness,
options.scale)
children = genome.generate_children(number_of_children,
number_of_mutations, info,
lookup, mags, ref_trajectories)
# now save the children into the new file
for child in children:
nextpop.append(child)
# and save the original
nextpop.append(genome)
# gather the population
trans = []
for i in range(size):
trans.append(nextpop)
allpop = alltoall(options.singlethreaded, trans)
newpop = []
for pop in allpop:
newpop += pop
popdict = {}
for genome in newpop:
fitness_key = "%1.8E" % (genome.fitness)
if fitness_key in popdict.keys():
if popdict[fitness_key].age < genome.age:
popdict[fitness_key] = genome
else:
popdict[fitness_key] = genome
newpop = []
for genome in popdict.values():
if genome.age < options.max_age:
newpop.append(genome)
newpop.sort(key=lambda x: x.fitness)
estar = newpop[0].fitness * 0.99
logging.debug("new estar is %f" % (estar))
newpop = newpop[options.setup * rank:options.setup * (rank + 1)]
#Checkpoint best solution
if rank == 0:
newpop[0].save(args[0])
for genome in newpop:
logging.debug(
"genome fitness: %1.8E Age : %2i Mutations : %4i" %
(genome.fitness, genome.age, genome.mutations))
barrier(options.singlethreaded)
barrier(options.singlethreaded)
# gather the population
trans = []
for i in range(size):
trans.append(nextpop)
allpop = alltoall(options.singlethreaded, trans)
newpop = []
for pop in allpop:
newpop += pop
newpop.sort(key=lambda x: x.fitness)
newpop = newpop[options.setup * rank:options.setup * (rank + 1)]
#Checkpoint best solution
if rank == 0:
newpop[0].save(args[0])
def process(options, args):
if options.create_genome:
for filename in args[0::]:
print("Turning file %s from Human Readable to Genome" % (filename))
f2 = open(filename, 'r')
buildlist = np.genfromtxt(filename, dtype=str)
f2.close()
mags = magnets.Magnets()
mags.load(options.magnets_filename)
maglist = magnets.MagLists(mags)
heswap = 0
veswap = 0
hhswap = 0
vvswap = 0
for line in range(buildlist.shape[0]):
if int(buildlist[line, 2]) == 4:
#maglist.magnet_lists['HE'][heswap][0]=buildlist[line,5]
maglist.swap(
'HE', maglist.magnet_lists['HE'].index(
[buildlist[line, 5], 1, 0]), heswap)
maglist.magnet_lists['HE'][heswap][1] = int(buildlist[line,
4])
heswap += 1
elif int(buildlist[line, 2]) == 3:
#maglist.magnet_lists['VE'][veswap][0]=buildlist[line,5]
maglist.swap(
'VE', maglist.magnet_lists['VE'].index(
[buildlist[line, 5], 1, 0]), veswap)
maglist.magnet_lists['VE'][veswap][1] = int(buildlist[line,
4])
veswap += 1
elif int(buildlist[line, 2]) == 2:
#maglist.magnet_lists['HH'][hhswap][0]=buildlist[line,5]
maglist.swap(
'HH', maglist.magnet_lists['HH'].index(
[buildlist[line, 5], 1, 0]), hhswap)
maglist.magnet_lists['HH'][hhswap][1] = int(buildlist[line,
4])
hhswap += 1
elif int(buildlist[line, 2]) == 1:
#maglist.magnet_lists['VV'][vvswap][0]=buildlist[line,5]
maglist.swap(
'VV', maglist.magnet_lists['VV'].index(
[buildlist[line, 5], 1, 0]), vvswap)
maglist.magnet_lists['VV'][vvswap][1] = int(buildlist[line,
4])
vvswap += 1
outfile = (os.path.split(filename)[1] + '.h5')
#fg.output_fields(outfile, options.id_filename, options.id_template, options.magnets_filename, maglist)
fp = open(os.path.split(filename)[1] + '.genome', 'w')
pickle.dump(maglist, fp)
fp.close()
if options.readable:
for filename in args[0::]:
print("Making file %s human readable." % (filename))
human_output(options.id_filename, filename)
if options.analysis:
for filename in args[0::]:
print("Processing file %s" % (filename))
# load the genome
maglists = pickle.load(open(filename, "rb"))
outfile = (os.path.split(filename)[1] + '.h5')
fg.output_fields(outfile, options.id_filename, options.id_template,
options.magnets_filename, maglists)
