def reduce_compute(tolerance, target_label, reactionModel, rmg, reaction_system_index):
"""
Reduces the model for the given tolerance and evaluates the
target conversion.
"""
# reduce model with the tolerance specified earlier:
important_reactions = find_important_reactions(rmg, tolerance)
original_size = len(reactionModel.core.reactions)
no_important_reactions = len(important_reactions)
logging.info('Number of important reactions: {}'.format(no_important_reactions))
#set the core reactions to the reduced reaction set:
original_reactions = reactionModel.core.reactions
rmg.reactionModel.core.reactions = important_reactions
#re-compute conversion:
conversion = compute_conversion(target_label, rmg.reactionModel,\
rmg.reactionSystems[reaction_system_index], reaction_system_index,\
rmg.absoluteTolerance, rmg.relativeTolerance)
#reset the reaction model to its original state:
rmg.reactionModel.core.reactions = original_reactions
logging.info('Conversion of reduced model ({} rxns): {:.2f}%'.format(no_important_reactions, conversion * 100))
return conversion
def _mate_chromosomes_with_current(self, neighbouring_individual):
father = neighbouring_individual.chromosome
logger.info("father " + str(father) + " mother " +
str(self._chromosome))
self._crossover(father, self._chromosome)
# mother
self._mutation(self._chromosome)
def __init__(self, server_user, server_password, server_ip_addr, exchange,
exchange_type, queue, subscription_routing_keys,
produced_messages, consumed_messages):
"""Create a new instance of the consumer class, passing in the AMQP
URL used to connect to RabbitMQ.
:param str amqp_url: The AMQP url to connect with
"""
logger.info("messenger" + str(server_ip_addr) + str(server_user) +
str(server_password))
self._exchange = exchange
self._exchange_type = exchange_type
self._queue = queue
self._subscription_routing_keys = subscription_routing_keys
self._connection = None
self._channel = None
self._deliveries = []
self._acked = 0
self._nacked = 0
self._message_number = 0
self._stopping = False
self._server_ip_addr = server_ip_addr
self._server_user = server_user
self._server_password = server_password
self._consumer_tag = None
self._produced_messages = produced_messages
self._consumed_messages = consumed_messages
threading.Thread.__init__(self)
def optimize_tolerance(target_label, reactionModel, rmg, reaction_system_index, error, orig_conv):
"""
Increment the trial tolerance from a very low value until the introduced error is greater than the parameter threshold.
"""
start = 1E-20
incr = 10
tolmax = 1
tol = start
trial = start
while True:
logging.info('Trial tolerance: {trial:.2E}'.format(**locals()))
Xred = reduce_compute(trial, target_label, reactionModel, rmg, reaction_system_index)
dev = np.abs((Xred - orig_conv) / orig_conv)
logging.info('Deviation: {dev:.2f}'.format(**locals()))
if dev > error or trial > tolmax:
break
tol = trial
trial = trial * incr
if tol == start:
logging.error('Starting value for tolerance was too high...')
return tol
def compute_conversion(target_label, reactionModel, reactionSystem,
reactionSystem_index, atol, rtol):
"""
Computes the conversion of a target molecule by
- searching the index of the target species in the core species
of the global reduction variable
- resetting the reaction system, initialing with empty variables
- fetching the initial moles variable y0
- running the simulation at the conditions stored in the reaction system
- fetching the computed moles variable y
- computing conversion
"""
target = search_target(target_label, reactionSystem)
target_index = reactionModel.core.species.index(target)
#reset reaction system variables:
logging.info('No. of rxns in core reactions: {}'.format(
len(reactionModel.core.reactions)))
reactionSystem.initializeModel(\
reactionModel.core.species, reactionModel.core.reactions,\
reactionModel.edge.species, reactionModel.edge.reactions, \
[], atol, rtol)
#get the initial moles:
y0 = reactionSystem.y.copy()
#run the simulation:
simulate_one(reactionModel, atol, rtol, reactionSystem)
#compute conversion:
conv = 1 - (reactionSystem.y[target_index] / y0[target_index])
return conv
def run_fine_grained_ga(population_size,
chromosome_size,
number_of_generations,
neighbourhood_size,
server_ip_addr,
fitness,
server_user,
server_password,
mate_best_neighbouring_individual=True):
ins = FineGrainedBase(
population_size=population_size,
chromosome_size=chromosome_size,
number_of_generations=number_of_generations,
neighbourhood_size=neighbourhood_size,
server_ip_addr=server_ip_addr,
fitness=fitness,
mate_best_neighbouring_individual=mate_best_neighbouring_individual,
server_user=server_user,
server_password=server_password)
populations = ins.initialize_population(None)
channels = ins.initialize_topology()
result = list(futures.map(ins, populations, channels))
dct = {}
while len(result):
fitness_val, vector = result.pop(0)
dct[fitness_val] = vector
logger.info("END RESULT " + str(sorted(dct.items()).pop()))
def run_coarse_grained_ga(population_size, deme_size, chromosome_size,
number_of_generations, neighbourhood_size,
server_ip_addr, server_user, server_password,
num_of_migrants, fitness):
ins = CoarseGrainedBase(population_size=population_size,
deme_size=deme_size,
chromosome_size=chromosome_size,
number_of_generations=number_of_generations,
neighbourhood_size=neighbourhood_size,
server_ip_addr=server_ip_addr,
server_user=server_user,
server_password=server_password,
num_of_migrants=num_of_migrants,
fitness=fitness)
populations = ins.initialize_population(deme_size)
print(str(populations))
channels = ins.initialize_topology()
results = list(futures.map(ins, populations, channels))
dct = {}
for data in results:
best_chromosome = data.pop(0)
fitness_val = best_chromosome.fit
vector = best_chromosome.chromosome
dct[fitness_val] = vector
logger.info("END RESULT" + str(sorted(dct.items()).pop()))
def add_on_channel_close_callback(self):
"""This method tells pika to call the on_channel_closed method if
RabbitMQ unexpectedly closes the channel.
"""
logger.info('Adding channel close callback')
self._channel.add_on_close_callback(self.on_channel_closed)
def _send_individuals_reproduce(self):
"""
Select individuals for reproduction with probability
based on fitness value. Weak individuals are removed
and replaced with newly generated ones.
"""
# retrieve best fitness of population
results = list(futures.map(self._fitness, self._population))
neighbours = self._Individuals()
for i in range(0, self._population_size):
fit_val = results.pop(0)
chromosome = self._population[i]
neighbours.append_object(self._Individual(fit_val, chromosome))
chosen_individuals = self._choose_individuals_based_on_fitness(
neighbours)
chromosomes_reproducing = chosen_individuals.sort_objects()
best_individual = chosen_individuals.best_individual
# it is sure that this is the right result
# but the algorithm needs to continue because of other demes
if best_individual is not None:
while len(self._population) <= self._population_size:
self._population.append(best_individual.chromosome)
return
best_individual = chromosomes_reproducing.pop(0)
# remove old population
del self._population[:]
logger.info("Number of individuals chosen for reproduction is " +
str(len(chromosomes_reproducing))+ " while best individuals has fitness "+
str(best_individual.fit))
# Reproducing requires two individuals.
# If number of selected individuals is even
# put the best individual to the new population.
# Otherwise, put him to individuals dedicated
# for reproduction
if len(chromosomes_reproducing) % 2 == 0:
self._population.append(best_individual.chromosome)
else:
# put the best individual to max index in order to not rewrite existing
chromosomes_reproducing.append(best_individual)
# randomly choose pairs for crossover
# then mutate new individuals and put them to new population
while len(chromosomes_reproducing) >= 2:
father = chromosomes_reproducing.pop(random.randrange(len(
chromosomes_reproducing))).chromosome
mother = chromosomes_reproducing.pop(random.randrange(len(
chromosomes_reproducing))).chromosome
self._crossover(father, mother)
# mutate
self._mutation(father)
self._mutation(mother)
self._population.append(father)
self._population.append(mother)
# Generate new individuals in order to make new population the same size
while len(self._population) != self._population_size:
self._population.append(self._gen_individual())
def __call__(self):
toReturn = []
logger.info("Process started")
for i in range(0, self._number_of_generations):
toReturn = self._process(None)
return toReturn
def _start_MPI(self, channels):
queue_to_produce = str(channels.pop(0))
queues_to_consume = list(map(str, channels))
logger.info("starting processing to queue: " + queue_to_produce +
" and consuming from: " + str(queues_to_consume))
connection = pika.BlockingConnection(
pika.ConnectionParameters(host=self._server_ip_addr,
credentials=pika.PlainCredentials(
"genetic1", "genetic1")))
channel = connection.channel()
channel.exchange_declare(exchange='direct_logs',
exchange_type='direct')
channel.basic_qos(prefetch_count=len(queues_to_consume))
result = channel.queue_declare(exclusive=True)
self._queue_name = result.method.queue
for queue in queues_to_consume:
channel.queue_bind(exchange='direct_logs',
queue=self._queue_name,
routing_key=queue)
self._queue_to_produce = queue_to_produce
self._queues_to_consume = queues_to_consume
self._channel = channel
self._connection = connection
time.sleep(5)
def _choose_individuals_based_on_fitness(self, evaluation_data):
ultimate_prob = 1
individuals_to_choose_from = evaluation_data.sort_objects()
best_individual = individuals_to_choose_from.pop(0)
fitness_min = best_individual.fit
chromosomes_reproducing = self._Individuals()
chromosomes_reproducing.append_object(best_individual)
if self.is_ultimate_solution(best_individual.fit):
chromosomes_reproducing.best_individual = best_individual
return chromosomes_reproducing
# at least two individuals are required to reproduce
for chromosome_data in individuals_to_choose_from:
# best individual has 100% probability to reproduce
# others probability is relative to his
# weak individuals are replaced with new ones
# this is because of division by zero
if self.is_ultimate_solution(chromosome_data.fit):
prob = ultimate_prob
else:
prob = fitness_min / chromosome_data.fit
if np.random.choice([True, False], p=[prob, 1 - prob]):
chromosomes_reproducing.append_object(
self._Individual(chromosome_data.fit,
chromosome_data.chromosome))
logger.info("LEN is " + str(len(chromosomes_reproducing.individuals)))
return chromosomes_reproducing
def _finish_processing(self, chromosome, mother):
logger.info("father " + str(chromosome) + " mother " + str(mother))
mother.pop(0)
self._crossover(chromosome, mother)
# mother
self._mutation(chromosome)
return self._fitness(chromosome), list(map(float, chromosome))
def reduce_compute(tolerance, target_label, reactionModel, rmg,
reaction_system_index):
"""
Reduces the model for the given tolerance and evaluates the
target conversion.
"""
# reduce model with the tolerance specified earlier:
important_reactions = find_important_reactions(rmg, tolerance)
original_size = len(reactionModel.core.reactions)
no_important_reactions = len(important_reactions)
logging.info(
'Number of important reactions: {}'.format(no_important_reactions))
#set the core reactions to the reduced reaction set:
original_reactions = reactionModel.core.reactions
rmg.reactionModel.core.reactions = important_reactions
#re-compute conversion:
conversion = compute_conversion(target_label, rmg.reactionModel,\
rmg.reactionSystems[reaction_system_index], reaction_system_index,\
rmg.absoluteTolerance, rmg.relativeTolerance)
#reset the reaction model to its original state:
rmg.reactionModel.core.reactions = original_reactions
logging.info('Conversion of reduced model ({} rxns): {:.2f}%'.format(
no_important_reactions, conversion * 100))
return conversion
def on_delivery_confirmation(self, method_frame):
"""Invoked by pika when RabbitMQ responds to a Basic.Publish RPC
command, passing in either a Basic.Ack or Basic.Nack frame with
the delivery tag of the message that was published. The delivery tag
is an integer counter indicating the message number that was sent
on the channel via Basic.Publish. Here we're just doing house keeping
to keep track of stats and remove message numbers that we expect
a delivery confirmation of from the list used to keep track of messages
that are pending confirmation.
:param pika.frame.Method method_frame: Basic.Ack or Basic.Nack frame
"""
confirmation_type = method_frame.method.NAME.split('.')[1].lower()
logger.info('Received %s for delivery tag: %i', confirmation_type,
method_frame.method.delivery_tag)
if confirmation_type == 'ack':
self._acked += 1
elif confirmation_type == 'nack':
self._nacked += 1
self._deliveries.remove(method_frame.method.delivery_tag)
logger.info(
'Published %i messages, %i have yet to be confirmed, '
'%i were acked and %i were nacked', self._message_number,
len(self._deliveries), self._acked, self._nacked)
def optimize_tolerance(target_label, reactionModel, rmg, reaction_system_index,
error, orig_conv):
"""
Increment the trial tolerance from a very low value until the introduced error is greater than the parameter threshold.
"""
start = 1E-20
incr = 10
tolmax = 1
tol = start
trial = start
while True:
logging.info('Trial tolerance: {trial:.2E}'.format(**locals()))
Xred = reduce_compute(trial, target_label, reactionModel, rmg,
reaction_system_index)
dev = np.abs((Xred - orig_conv) / orig_conv)
logging.info('Deviation: {dev:.2f}'.format(**locals()))
if dev > error or trial > tolmax:
break
tol = trial
trial = trial * incr
if tol == start:
logging.error('Starting value for tolerance was too high...')
return tol
def compute_conversion(target_label, reactionModel, reactionSystem, reactionSystem_index, atol, rtol):
"""
Computes the conversion of a target molecule by
- searching the index of the target species in the core species
of the global reduction variable
- resetting the reaction system, initialing with empty variables
- fetching the initial moles variable y0
- running the simulation at the conditions stored in the reaction system
- fetching the computed moles variable y
- computing conversion
"""
target = search_target(target_label, reactionSystem)
target_index = reactionModel.core.species.index(target)
#reset reaction system variables:
logging.info('No. of rxns in core reactions: {}'.format(len(reactionModel.core.reactions)))
reactionSystem.initializeModel(\
reactionModel.core.species, reactionModel.core.reactions,\
reactionModel.edge.species, reactionModel.edge.reactions, \
[], atol, rtol)
#get the initial moles:
y0 = reactionSystem.y.copy()
#run the simulation:
simulate_one(reactionModel, atol, rtol, reactionSystem)
#compute conversion:
conv = 1 - (reactionSystem.y[target_index] / y0[target_index])
return conv
def stop_consuming(self):
"""Tell RabbitMQ that you would like to stop consuming by sending the
Basic.Cancel RPC command.
"""
if self._channel:
logger.info('Sending a Basic.Cancel RPC command to RabbitMQ')
self._channel.basic_cancel(self.on_cancelok, self._consumer_tag)
def __call__(self):
to_return = []
logger.info("Process started")
for i in range(0, self._number_of_generations):
logger.info("GENERATION " + str(i))
to_return = self._process()
return to_return
def add_on_cancel_callback(self):
"""Add a callback that will be invoked if RabbitMQ cancels the consumer
for some reason. If RabbitMQ does cancel the consumer,
on_consumer_cancelled will be invoked by pika.
"""
logger.info('Adding consumer cancellation callback')
self._channel.add_on_cancel_callback(self.on_consumer_cancelled)
def close_channel(self):
"""Invoke this command to close the channel with RabbitMQ by sending
the Channel.Close RPC command.
"""
if self._channel is not None:
logger.info('Closing the channel')
self._channel.close()
def _send_individuals_reproduce(self):
"""
Select individuals for reproduction with probability
based on fitness value. Weak individuals are removed
and replaced with newly generated ones.
"""
# retrieve best fitness of population
evaluation_data = self._evaluate_population()
# choose individuals for reproduction based on probability
chosen_individuals = self._choose_individuals_based_on_fitness(
evaluation_data)
chromosomes_reproducing = chosen_individuals.sort_objects()
best_individual = chosen_individuals.best_individual
# it is sure that this is the right result
# but the algorithm needs to continue because of other demes
if best_individual is not None:
logger.info("Ultimate best individual was found.")
while len(self._population) <= self._deme_size:
self._population.append(best_individual.chromosome)
return
else:
best_individual = chromosomes_reproducing.pop(0)
# remove old population
del self._population[:]
# Reproducing requires two individuals.
# If number of selected individuals is even
# put the best individual to the new population.
# Otherwise, put him to individuals dedicated
# for reproduction
if len(chromosomes_reproducing) % 2 == 0:
self._population.append(best_individual.chromosome)
else:
# put the best individual to max index in order to not rewrite existing
chromosomes_reproducing.append(best_individual)
logger.info(
"Number of individuals chosen for reproduction is " + str(len(chromosomes_reproducing)))
# randomly choose pairs for crossover
# then mutate new individuals and put them to new population
while len(chromosomes_reproducing) >= 2:
father = chromosomes_reproducing.pop(random.randrange(len(
chromosomes_reproducing))).chromosome
mother = chromosomes_reproducing.pop(random.randrange(len(
chromosomes_reproducing))).chromosome
self._crossover(father, mother)
self._mutation(father)
self._mutation(mother)
self._population.append(father)
self._population.append(mother)
# Generate new individuals in order to make new population the same size
while len(self._population) <= self._deme_size:
self._population.append(self._gen_individual())
def run_master_slave_ga(population_size, chromosome_size,
number_of_generations, fitness):
ins = MasterSlaveBase(population_size=population_size,
chromosome_size=chromosome_size,
number_of_generations=number_of_generations,
fitness=fitness)
solution, sol_vec = ins()
logger.info("FINAL RESULT: fitness: " + str(solution) + " vector: " + str(sol_vec))
return solution, sol_vec
def on_exchange_declareok(self, unused_frame):
"""Invoked by pika when RabbitMQ has finished the Exchange.Declare RPC
command.
:param pika.Frame.Method unused_frame: Exchange.DeclareOk response frame
"""
logger.info('Exchange declared')
self.setup_queue(self._queue)
def acknowledge_message(self, delivery_tag):
"""Acknowledge the message delivery from RabbitMQ by sending a
Basic.Ack RPC method for the delivery tag.
:param int delivery_tag: The delivery tag from the Basic.Deliver frame
"""
logger.info('Acknowledging message %s', delivery_tag)
self._channel.basic_ack(delivery_tag)
def open_channel(self):
"""This method will open a new channel with RabbitMQ by issuing the
Channel.Open RPC command. When RabbitMQ confirms the channel is open
by sending the Channel.OpenOK RPC reply, the on_channel_open method
will be invoked.
"""
logger.info('Creating a new channel')
self._connection.channel(on_open_callback=self.on_channel_open)
def setup_queue(self, queue_name):
"""Setup the queue on RabbitMQ by invoking the Queue.Declare RPC
command. When it is complete, the on_queue_declareok method will
be invoked by pika.
:param str|unicode queue_name: The name of the queue to declare.
"""
logger.info('Declaring queue %s', queue_name)
self._channel.queue_declare(self.on_queue_declareok, queue_name)
def on_connection_open(self, unused_connection):
"""This method is called by pika once the connection to RabbitMQ has
been established. It passes the handle to the connection object in
case we need it, but in this case, we'll just mark it unused.
:type unused_connection: pika.SelectConnection
"""
logger.info('Connection opened')
self.open_channel()
def setup_exchange(self, exchange_name):
"""Setup the exchange on RabbitMQ by invoking the Exchange.Declare RPC
command. When it is complete, the on_exchange_declareok method will
be invoked by pika.
:param str|unicode exchange_name: The name of the exchange to declare
"""
logger.info('Declaring exchange %s', exchange_name)
self._channel.exchange_declare(self.on_exchange_declareok,
exchange_name, self._exchange_type)
def on_consumer_cancelled(self, method_frame):
"""Invoked by pika when RabbitMQ sends a Basic.Cancel for a consumer
receiving messages.
:param pika.frame.Method method_frame: The Basic.Cancel frame
"""
logger.info('Consumer was cancelled remotely, shutting down: %r',
method_frame)
if self._channel:
self._channel.close()
def on_cancelok(self, unused_frame):
"""This method is invoked by pika when RabbitMQ acknowledges the
cancellation of a consumer. At this point we will close the channel.
This will invoke the on_channel_closed method once the channel has been
closed, which will in-turn close the connection.
:param pika.frame.Method unused_frame: The Basic.CancelOk frame
"""
logger.info('RabbitMQ acknowledged the cancellation of the consumer')
self.close_channel()
def __call__(self, initial_data, channels):
to_return = []
logger.info("Process started with initial data " + str(initial_data) +
" and channels " + str(channels))
self._start_MPI(channels)
for i in range(0, self._number_of_generations):
data = self._process(initial_data)
self._send_data(data)
received_data = self._collect_data()
to_return = self._finish_processing(received_data, data)
return to_return
def wrapper(*args, **kwargs):
try:
logger.info(func.__name__ + " [ENTER] on " +
socket.gethostname())
return func(*args, **kwargs)
except:
# log the exception
err = "There was an exception in "
err += func.__name__
logger.exception(err)
# re-raise the exception
raise
def loadReductionInput(reductionFile):
"""
Load an reduction job from the input file located at `reductionFile`
"""
target = None
tolerance = -1
full_path = os.path.abspath(os.path.expandvars(reductionFile))
try:
f = open(full_path)
except IOError, e:
logging.error('The input file "{0}" could not be opened.'.format(full_path))
logging.info('Check that the file exists and that you have read access.')
raise e
def print_info(rxn, spc, important):
logging.info('Is reaction {0} important for species {1}: {2}'.format(rxn, spc, important))
"""
Load an reduction job from the input file located at `reductionFile`
"""
target = None
tolerance = -1
full_path = os.path.abspath(os.path.expandvars(reductionFile))
try:
f = open(full_path)
except IOError, e:
logging.error('The input file "{0}" could not be opened.'.format(full_path))
logging.info('Check that the file exists and that you have read access.')
raise e
logging.info('Reading input file "{0}"...'.format(full_path))
global_context = { '__builtins__': None }
local_context = {
'__builtins__': None,
'target': target,
'tolerance': tolerance
}
try:
exec f in global_context, local_context
target = local_context['target']
tolerance = local_context['tolerance']
except (NameError, TypeError, SyntaxError), e:
