def __init__(self, experiment_params):
"""
A class that can interact with the ESP service
:param experiment_params: the experiment parameters dictionary
"""
self.experiment_params = experiment_params
self.extension_packaging = ExtensionPackaging()
self.experiment_params_as_bytes = self.extension_packaging.to_extension_bytes(
experiment_params)
self.experiment_id = experiment_params["LEAF"]["experiment_id"]
self.version = experiment_params["LEAF"]["version"]
# gRPC connection
esp_host = experiment_params["LEAF"]["esp_host"]
esp_port = experiment_params["LEAF"]["esp_port"]
grpc_options = experiment_params["LEAF"].get(
"grpc_options", DEFAULT_GRPC_OPTIONS).items()
print("ESP service: {}:{}".format(esp_host, esp_port))
print("gRPC options:")
for pair in grpc_options:
print(" {}: {}".format(pair[0], pair[1]))
channel = grpc.insecure_channel('{}:{}'.format(esp_host, esp_port),
options=grpc_options)
print("Ready to connect.")
self.esp_service = PopulationServiceStub(channel)
def __init__(self, experiment_params, evaluator):
self.experiment_params = experiment_params
self.extension_packaging = ExtensionPackaging()
self.save_to_dir = self._generate_persistence_directory()
# Possible values are all, elites, best, none
self.candidates_to_persist = self.experiment_params["LEAF"].get("candidates_to_persist", "best").lower()
if self.candidates_to_persist not in ["all", "elites", "best", "none"]:
raise ValueError("Unknown value for experiment param [LEAF][candidates_to_persist]: {}".format(
self.candidates_to_persist))
self.persist_experiment_params(experiment_params)
self.evaluator = evaluator
class EspEvaluator(ABC):
"""
An abstract class to evaluate ESP populations.
"""
def __init__(self):
self._extension_packaging = ExtensionPackaging()
@abstractmethod
def evaluate_candidate(self, candidate):
"""
Evaluates a single Keras model.
:param candidate: a Keras model
:return: a dictionary of metrics
"""
pass
@abstractmethod
def evaluate_population(self, population_response):
"""
Evaluates the candidates in an ESP PopulationResponse
and updates the PopulationResponse with the candidates fitness.
:param population_response: an ESP PopulationResponse
:return: nothing. A dictionary containing the metrics is assigned to the candidates metrics as a UTF-8
encoded string (bytes) within the passed response
"""
pass
def _encode_metrics(self, metrics):
if not isinstance(metrics, dict):
# For backward compatibility: if the evaluation didn't return a dictionary,
# convert the returned value to a float and put it in a dictionary.
# metric might be a numpy object, like numpy.int64 or numpy.float32. Convert it to float.
score = float(metrics)
# Create a dictionary
metrics = {"score": score}
encoded_metrics = self._extension_packaging.to_extension_bytes(metrics)
return encoded_metrics
def _decode_metrics(self, encoded_metrics):
metrics = self._extension_packaging.from_extension_bytes(
encoded_metrics)
return metrics
def __init__(self):
self._extension_packaging = ExtensionPackaging()
class EspPersistor:
"""
A class to persist any kind of information from an experiment.
"""
def __init__(self, experiment_params, evaluator):
self.experiment_params = experiment_params
self.extension_packaging = ExtensionPackaging()
self.save_to_dir = self._generate_persistence_directory()
# Possible values are all, elites, best, none
self.candidates_to_persist = self.experiment_params["LEAF"].get("candidates_to_persist", "best").lower()
if self.candidates_to_persist not in ["all", "elites", "best", "none"]:
raise ValueError("Unknown value for experiment param [LEAF][candidates_to_persist]: {}".format(
self.candidates_to_persist))
self.persist_experiment_params(experiment_params)
self.evaluator = evaluator
def _generate_persistence_directory(self):
timestamp = time.strftime("%Y%m%d-%H%M%S")
persistence_dir = self.experiment_params["LEAF"]["persistence_dir"]
experiment_id = self.experiment_params["LEAF"]["experiment_id"]
dirname = os.path.join(persistence_dir, experiment_id)
version = self.experiment_params["LEAF"]["version"]
version = version + "_" + timestamp
dirname = os.path.join(dirname, version)
os.makedirs(dirname, exist_ok=True)
return dirname
def get_persistence_directory(self):
"""
Returns the name of the directory used for persistence.
:return: a string
"""
return self.save_to_dir
def persist_experiment_params(self, experiment_params):
"""
Persists the passed experiment parameters.
:param experiment_params: the experiment parameters to persist
:return: nothing. Saves a file called `experiment_params.json` to the persistence directory
"""
filename = os.path.join(self.save_to_dir, 'experiment_params.json')
with open(filename, 'w') as f:
f.write(json.dumps(experiment_params, indent=4))
def persist_response(self, response):
"""
Persists a generation's information.
:param response: an evaluated ESP PopulationResponse
:return: nothing. Saves files to the persistence directory
"""
gen = response.generation_count
checkpoint_id = response.checkpoint_id
candidates_info = self.persist_generation(response)
self.persist_stats(candidates_info, gen, checkpoint_id)
self.persist_candidates(candidates_info, gen)
stats_file = os.path.join(self.save_to_dir, 'experiment_stats.csv')
title = self.experiment_params["LEAF"]["experiment_id"]
EspPlotter.plot_stats(stats_file, title)
def persist_stats(self, candidates_info, generation, checkpoint_id):
"""
Collects statistics for the passed generation of candidates.
:param candidates_info: the candidates information
:param generation: the generation these candidates belong to
:param checkpoint_id: the checkpoint id corresponding to this generation
:return: nothing. Saves a file called `experiment_stats.csv` to the persistence directory
"""
filename = os.path.join(self.save_to_dir, 'experiment_stats.csv')
file_exists = os.path.exists(filename)
metrics_stats = {}
for metric_name in candidates_info[0]["metrics"].keys():
metric_values = [candidate["metrics"][metric_name] for candidate in candidates_info]
metrics_stats["max_" + metric_name] = max(metric_values)
metrics_stats["min_" + metric_name] = min(metric_values)
metrics_stats["mean_" + metric_name] = statistics.mean(metric_values)
candidates_info.sort(key=lambda k: k["metrics"][metric_name], reverse=False)
metrics_stats["cid_min_" + metric_name] = candidates_info[0]["id"]
metrics_stats["cid_max_" + metric_name] = candidates_info[-1]["id"]
# 'a+' Opens the file for appending; any data written to the file is automatically added to the end.
# The file is created if it does not exist.
with open(filename, 'a+') as stats_file:
writer = csv.writer(stats_file)
if not file_exists:
headers = ["generation", "checkpoint_id"]
headers.extend(metrics_stats.keys())
writer.writerow(headers)
generation_stats = [generation, checkpoint_id]
generation_stats.extend(metrics_stats.values())
writer.writerow(generation_stats)
def persist_generation(self, response):
"""
Persists the details of a generation to a file.
:param response: an evaluated ESP PopulationResponse
:return: nothing. Saves a file called `gen.csv` to the persistence directory (e.g. 1.csv for generation 1)
"""
gen = response.generation_count
gen_filename = os.path.join(self.save_to_dir, str(gen) + '.csv')
# 'w' to truncate the file if it already exists
candidates_info = []
with open(gen_filename, 'w') as stats_file:
writer = csv.writer(stats_file)
write_header = True
for candidate in response.population:
# Candidate's details
cid = candidate.id
identity = candidate.identity.decode('UTF-8')
metrics = self.extension_packaging.from_extension_bytes(candidate.metrics)
c = {"id": cid,
"identity": identity,
"metrics": metrics,
"model": candidate.interpretation}
candidates_info.append(c)
# Write the header if needed
if write_header:
# Unpack the metric names list
writer.writerow(["cid", "identity", *metrics.keys()])
write_header = False
# Write a row for this candidate
row_values = [cid, identity]
row_values.extend(metrics.values())
writer.writerow(row_values)
return candidates_info
def persist_candidates(self, candidates_info, gen):
"""
Persists the candidates in the response's population according to the experiment params.
Can be "all", "elites", "best", "none"
:param candidates_info: a PopulationResponse containing evaluated candidates
:param gen: the generation these candidates belong to
:return: nothing. Saves the candidates to a generation folder in the persistence directory
"""
if self.candidates_to_persist == "none":
return
gen_folder = os.path.join(self.save_to_dir, str(gen))
os.makedirs(gen_folder, exist_ok=True)
if self.candidates_to_persist == "all":
for candidate in candidates_info:
self.persist_candidate(candidate, gen_folder)
elif self.candidates_to_persist == "best":
# Save the best candidate, per objective
objectives = self.experiment_params['evolution'].get("fitness", DEFAULT_FITNESS)
for objective in objectives:
# Sort the candidates to figure out the best one for this objective
metric_name = objective["metric_name"]
candidates_info.sort(key=lambda k: k["metrics"][metric_name],
reverse=objective["maximize"])
# The best candidate for this objective is the first one
self.persist_candidate(candidates_info[0], gen_folder)
elif self.candidates_to_persist == "elites":
# Save the elites, according to the first objective only
nb_elites = self.experiment_params["evolution"]["nb_elites"]
objectives = self.experiment_params['evolution'].get("fitness", DEFAULT_FITNESS)
objective = objectives[0]
metric_name = objective["metric_name"]
candidates_info.sort(key=lambda k: k["metrics"][metric_name],
reverse=objective["maximize"])
for candidate in candidates_info[len(candidates_info) - nb_elites:]:
self.persist_candidate(candidate, gen_folder)
else:
print("Skipping candidates persistence: unknown candidates_to_persist attribute: {}".format(
self.candidates_to_persist))
def persist_candidate(self, candidate, gen_folder):
"""
Persists a candidates to a file
:param candidate: the candidates to persist
:param gen_folder: the folder to which to persist it
:return: nothing. Saves the candidate to a cid.h5 file in generation folder in the persistence directory
(where cid is the candidate id)
"""
cid = candidate["id"]
filename = cid + ".h5"
filename = os.path.join(gen_folder, filename)
representation = self.experiment_params["LEAF"]["representation"]
if representation == "KerasNN":
self.persist_keras_nn_model(candidate["model"], filename)
elif representation == "NNWeights":
self.persist_nn_weights_model(candidate["model"], filename)
else:
print("Persistor: Unknown representation: {}".format(representation))
@staticmethod
def persist_keras_nn_model(model_bytes, filename):
"""
Converts the passed bytes to a Keras model and saves it to a file
:param model_bytes: the bytes corresponding to a Keras model
:param filename: the name of the file to save to
:return: nothing
"""
# Convert the received bytes to a Keras model
import io
from keras.models import load_model
model_file = io.BytesIO(model_bytes)
keras_model = load_model(model_file)
# Save the model, without the optimizer (not used)
keras_model.save(filename, include_optimizer=False)
def persist_nn_weights_model(self, weights_bytes, filename):
"""
Creates a model from the passed weight bytes and saves it to a file
:param weights_bytes: the bytes corresponding to a Keras model weights
:param filename: the name of the file to save to
:return: nothing
"""
indy_weights = pickle.loads(weights_bytes)
model = self.evaluator.get_keras_model(indy_weights)
# Save the model, without the optimizer (not used)
model.save(filename, include_optimizer=False)
def setUp(self):
# Executed before each test
self.extension_packaging = ExtensionPackaging()
with open(EXPERIMENT_JSON) as json_data:
self.experiment_params = json.load(json_data)
class TestEspNNWeightsEvaluator(unittest.TestCase):
def setUp(self):
# Executed before each test
self.extension_packaging = ExtensionPackaging()
with open(EXPERIMENT_JSON) as json_data:
self.experiment_params = json.load(json_data)
# Mock where the class is used, i.e. in esp_evaluator
@patch('esp_sdk.v1_0.esp_evaluator.EspService', autospec=True)
def test_constructor(self, esp_service_mock):
evaluator = EspNNWeightsEvaluatorForTests(self.experiment_params)
self.assertIsNotNone(evaluator)
# Make sure we called the ESP service mock to get a base model
self.assertEqual(
1, esp_service_mock.return_value.request_base_model.call_count)
@patch('esp_sdk.v1_0.esp_evaluator.EspService', autospec=True)
def test_evaluate_population(self, esp_service_mock):
# If nothing is specified in the experiment_params, re-evaluate elites
self._evaluate_population(esp_service_mock, reevaluate_elites=True)
@patch('esp_sdk.v1_0.esp_evaluator.EspService', autospec=True)
def test_reevaluate_elites(self, esp_service_mock):
# Reevaluate elites
self.experiment_params["LEAF"]["reevaluate_elites"] = True
self._evaluate_population(esp_service_mock, reevaluate_elites=True)
@patch('esp_sdk.v1_0.esp_evaluator.EspService', autospec=True)
def test_do_not_reevaluate_elites(self, esp_service_mock):
# Do NOT reevaluate elites
self.experiment_params["LEAF"]["reevaluate_elites"] = False
self._evaluate_population(esp_service_mock, reevaluate_elites=False)
def _evaluate_population(self, esp_service_mock, reevaluate_elites):
evaluator = EspNNWeightsEvaluatorForTests(self.experiment_params)
# Make sure we called the ESP service mock to get a base model
self.assertEqual(
1, esp_service_mock.return_value.request_base_model.call_count)
# Create a population
response = self._create_population_response()
# And evaluate it
evaluator.evaluate_population(response)
# Check c1
c = response.population[0]
metrics_json = self.extension_packaging.from_extension_bytes(c.metrics)
score = metrics_json['score']
if reevaluate_elites:
# This candidate is an elite: we want to make sure it has been re-evaluated and it's score
# is the re-evaluated score, not the elite score.
self.assertEqual(
C1_SCORE, score,
"This elite candidate should have been re-evaluated")
else:
# This candidate is an elite, and we make sure we have NOT re-evaluated it
self.assertEqual(
E1_SCORE, score,
"This elite candidate should still have its elite score")
# Check c2
c = response.population[1]
score_json = self.extension_packaging.from_extension_bytes(c.metrics)
score = score_json['score']
self.assertEqual(C2_SCORE, score)
def _create_population_response(self):
population = []
# elite
c1 = Candidate()
c1.id = "1_1"
c1.interpretation = pickle.dumps(C1_MODEL)
# This is an elite: it has already been evaluated anc already contains a score
c1.metrics = self.extension_packaging.to_extension_bytes(
{"score": E1_SCORE})
c1.identity = pickle.dumps("C1 identity")
population.append(c1)
# new candidate
c2 = Candidate()
c2.id = "2_1"
c2.interpretation = pickle.dumps(C2_MODEL)
# c2.metrics = self.extension_packaging.to_extension_bytes(None)
c2.identity = pickle.dumps("C2 identity")
population.append(c2)
response = PopulationResponse(population=population)
return response
@patch('esp_sdk.v1_0.esp_evaluator.EspService', autospec=True)
def test_evaluate_population_multi_metrics(self, esp_service_mock):
evaluator = EspNNWeightsMultiMetricsEvaluatorForTests(
self.experiment_params)
# Make sure we called the ESP service mock to get a base model
self.assertEqual(
1, esp_service_mock.return_value.request_base_model.call_count)
# Create a population
response = self._create_population_response()
# And evaluate it
evaluator.evaluate_population(response)
# Check c1
c = response.population[0]
metrics_json = self.extension_packaging.from_extension_bytes(c.metrics)
score = metrics_json['score']
self.assertEqual(C1_SCORE, score)
time = metrics_json['time']
self.assertEquals(C1_TIME, time)
# Check c2
c = response.population[1]
score_json = self.extension_packaging.from_extension_bytes(c.metrics)
score = score_json['score']
self.assertEqual(C2_SCORE, score)
time = metrics_json['time']
self.assertEquals(C1_TIME, time)
class EspService(object):
def __init__(self, experiment_params):
"""
A class that can interact with the ESP service
:param experiment_params: the experiment parameters dictionary
"""
self.experiment_params = experiment_params
self.extension_packaging = ExtensionPackaging()
self.experiment_params_as_bytes = self.extension_packaging.to_extension_bytes(
experiment_params)
self.experiment_id = experiment_params["LEAF"]["experiment_id"]
self.version = experiment_params["LEAF"]["version"]
# gRPC connection
esp_host = experiment_params["LEAF"]["esp_host"]
esp_port = experiment_params["LEAF"]["esp_port"]
grpc_options = experiment_params["LEAF"].get(
"grpc_options", DEFAULT_GRPC_OPTIONS).items()
print("ESP service: {}:{}".format(esp_host, esp_port))
print("gRPC options:")
for pair in grpc_options:
print(" {}: {}".format(pair[0], pair[1]))
channel = grpc.insecure_channel('{}:{}'.format(esp_host, esp_port),
options=grpc_options)
print("Ready to connect.")
self.esp_service = PopulationServiceStub(channel)
def get_next_population(self, prev_response):
"""
Returns a new generation for a given experiment.
:param prev_response: the previous generation, *with* evaluation metrics for each Candidate
:return: a new generation, as a PopulationResponse object
"""
# Prepare a request for next generation
request_params = {
'version': self.version,
'experiment_id': self.experiment_id
}
request = ParseDict(request_params, PopulationRequest())
request.config = self.experiment_params_as_bytes
if prev_response:
request.evaluated_population_response.CopyFrom(prev_response)
# Ask for next generation
response = self._next_population_with_retry(request)
return response
def get_previous_population(self, experiment_id, checkpoint_id):
"""
Returns the population corresponding to the passed experiment_id and checkpoint_id
:param experiment_id: the experiment id
:param checkpoint_id: the checkpoint id returned by a previous call to get_next_population
:return: a previous generation, as a PopulationResponse object
"""
# Prepare a GetPopulation request
request_params = {
'version': self.version,
'experiment_id': experiment_id,
'checkpoint_id': checkpoint_id
}
request = ParseDict(request_params, ExistingPopulationRequest())
# Ask for a previous generation
response = self.esp_service.GetPopulation(request)
return response
@retry(stop=stop_after_attempt(NB_RETRIES),
wait=wait_random(1, 3),
retry=retry_if_exception_type(grpc.RpcError))
def _next_population_with_retry(self, request):
print("Sending NextPopulation request")
response = self.esp_service.NextPopulation(request)
print("NextPopulation response received.")
return response
@retry(stop=stop_after_attempt(NB_RETRIES),
wait=wait_random(1, 3),
retry=retry_if_exception_type(grpc.RpcError))
def _get_population_with_retry(self, request):
print("Sending GetPopulation request")
response = self.esp_service.GetPopulation(request)
print("GetPopulation response received.")
return response
def request_base_model(self):
# Update the request to query for 1 Keras model
params = copy.deepcopy(self.experiment_params)
params["evolution"] = {"population_size": 1}
params["LEAF"]["representation"] = "KerasNN"
# Prepare a request for next generation
request_params = {
'version': self.version,
'experiment_id': self.experiment_id
}
request = ParseDict(request_params, PopulationRequest())
request.config = self.extension_packaging.to_extension_bytes(params)
# Ask for the base model
response = self._next_population_with_retry(request)
# Convert the received bytes to a Keras model
model_bytes = response.population[0].interpretation
model_file = io.BytesIO(model_bytes)
keras_model = load_model(model_file)
# return the base model
return keras_model
def extract_candidates_info(self, response):
"""
Prints Candidate details from a population
:param response: a PopulationResponse from the ESP service
:return: a dictionary representing a candidate
"""
candidates_info = []
for candidate in response.population:
c = {
"id":
candidate.id,
"identity":
candidate.identity.decode('UTF-8'),
"metrics":
self.extension_packaging.from_extension_bytes(
candidate.metrics),
"model":
candidate.interpretation
}
candidates_info.append(c)
return candidates_info
@staticmethod
def print_population_response(response):
"""
Prints out the details of a population represented by a PopulationResponse object
:param response: a PopulationResponse object returned by the ESP API
"""
print("PopulationResponse:")
print(" Generation: {}".format(response.generation_count))
print(" Population size: {}".format(len(response.population)))
print(" Checkpoint id: {}".format(response.checkpoint_id))
print(" Evaluation stats: {}".format(
response.evaluation_stats.decode('UTF-8')))
def print_candidates(self, response, sort_candidates=True):
"""
Prints the candidates details
:param response: an evaluated PopulationResponse
:param sort_candidates: if True, sort the candidates by score, lowest first, to always see the best
candidates at the bottom of the logs
"""
# Interpret the response received from the ESP service
candidates_info = self.extract_candidates_info(response)
if sort_candidates:
# Sort the candidates by score, lowest first to always see the best candidates at the bottom of the log
candidates_info = sorted(candidates_info,
key=lambda k: k["metrics"]["score"],
reverse=False)
print("Evaluated candidates:")
for candidate in candidates_info:
print("Id: {} Identity: {} Metrics: {}".format(
candidate["id"], candidate["identity"], candidate["metrics"]))
print("")