def measure_task(configurations_sample: list, tasks_sample: list,
experiment_description: dict, search_space: Hyperparameter,
measured_tasks: int, config_type: Configuration.Type,
config_status: Configuration.Status):
"""
Test function for Repeater module.
Main steps:
0. Take default tasks sample.
1. Create instances of Repeater, Experiment, Default Configuration according to test requirements.
2. Create instance of current measurement.
3. Call Repeater function.
:param configurations_sample: default sample of measured configurations
:param tasks_sample: default sample of measured tasks
:param experiment_description: Experiment Description sample in json format
:param search_space: Search Space sample
:param measured_tasks: specify number of measured tasks in current measurement.
:param config_type: specify current measurement configuration type.
:param config_status: specify current measurement configuration status.
:return: list of configuration status and number of tasks to measure.
"""
experiment = Experiment(experiment_description, search_space)
Configuration.set_task_config(experiment.description["TaskConfiguration"])
configuration = Configuration(configurations_sample[1]["Params"],
config_type, experiment.unique_id)
configuration.status = config_status
for i in range(0, measured_tasks):
configuration.add_task(tasks_sample[i])
orchestrator = RepeaterOrchestration(experiment)
if config_type == Configuration.Type.DEFAULT:
orchestrator._type = orchestrator.get_repeater(True)
else:
orchestrator._type = orchestrator.get_repeater()
default_configuration = Configuration(
configurations_sample[0]["Params"], Configuration.Type.DEFAULT,
experiment.unique_id)
default_configuration.status = Configuration.Status.MEASURED
default_configuration._task_number = configurations_sample[0]["Tasks"]
default_configuration.results = configurations_sample[0]["Results"]
default_configuration._standard_deviation = configurations_sample[0][
"STD"]
experiment.default_configuration = default_configuration
task = json.dumps({"configuration": configuration.to_json()})
dummy_channel = None
dummy_method = None
dummy_properties = None
results_measurement = orchestrator.measure_configurations(
dummy_channel, dummy_method, dummy_properties, task)
return results_measurement
def get_default_configurations_results(self, ch, method, properties, body):
"""
Callback function for the result of default configuration
:param ch: pika.Channel
:param method: pika.spec.Basic.GetOk
:param properties: pika.spec.BasicProperties
:param body: result of measuring default configuration in bytes format
"""
default_configuration = Configuration.from_json(body.decode())
if default_configuration.status == Configuration.Status.BAD:
new_default_values = self.default_config_handler.get_new_default_config(
)
if new_default_values:
config = Configuration(new_default_values,
Configuration.Type.FROM_SELECTOR,
self.experiment.unique_id)
temp_msg = "New default configuration sampled."
self.logger.info(temp_msg)
self.sub.send('log', 'info', message=temp_msg)
self.consume_channel.basic_publish(
exchange='',
routing_key='measure_new_configuration_queue',
body=json.dumps({"configuration": config.to_json()}))
else:
self.logger.error(
"The specified default configuration is broken.")
self.stop()
self.sub.send(
'log',
'info',
message="The specified default configuration is broken.")
return
if self.experiment.is_configuration_evaluated(default_configuration):
self.experiment.default_configuration = default_configuration
self.database.update_record(
"Search_space", {"Exp_unique_ID": self.experiment.unique_id}, {
"Default_configuration":
default_configuration.get_configuration_record()
})
self.database.update_record(
"Search_space", {"Exp_unique_ID": self.experiment.unique_id}, {
"SearchspaceObject": pickle.dumps(
self.experiment.search_space)
})
temp_msg = f"Evaluated Default Configuration: {default_configuration}"
self.logger.info(temp_msg)
self.sub.send('log', 'info', message=temp_msg)
# starting main work: building model and choosing configuration for measuring
self.consume_channel.basic_publish(
exchange='', routing_key='get_worker_capacity_queue', body='')
def send_new_configurations_to_measure(self, ch, method, properties, body):
"""
This callback function will be triggered on arrival of ONE measured Configuration.
When there is new measured Configuration, following steps should be done:
- update and validate models;
- pick either by model, or by selection algorithm new Configuration(s) for evaluation;
Note: The amount of new Configurations are:
- 0 if number of available Worker nodes decreased;
- 1 if number of available Workers did not change;
- N + 1 if number of available Worker increased by N;
- send new Configuration to Repeater for evaluation.
"""
needed_configs = json.loads(body.decode()).get("worker_capacity", 1)
for _ in range(needed_configs):
config = self.predictor.predict(
self.experiment.measured_configurations)
if config not in self.experiment.evaluated_configurations:
temp_msg = f"Model predicted {config}."
else:
while config in self.experiment.evaluated_configurations and not self._is_interrupted:
if len(self.experiment.evaluated_configurations
) >= self.experiment.search_space.get_size():
msg = "Entire Search Space was evaluated. Shutting down."
self.logger.warning(msg)
self.consume_channel.basic_publish(
exchange='',
routing_key='stop_experiment_queue',
body=msg)
break
new_parameter_values = OrderedDict()
while not self.experiment.search_space.validate(
new_parameter_values, is_recursive=True):
self.experiment.search_space.generate(
new_parameter_values)
config = Configuration(new_parameter_values,
Configuration.Type.FROM_SELECTOR,
self.experiment.unique_id)
temp_msg = f"Fully randomly sampled {config}."
self.logger.info(temp_msg)
self.sub.send('log', 'info', message=temp_msg)
self.consume_channel.basic_publish(
exchange='',
routing_key='measure_new_configuration_queue',
body=json.dumps({"configuration": config.to_json()}))
def run(self):
"""
The entry point to the main node functionality - measuring default Configuration.
When the default Configuration finishes its evaluation, the first set of Configurations will be
sampled for evaluation (respectively, the queues for Configuration measurement results initialize).
"""
self._state = self.State.RUNNING
self.logger.info("Starting BRISE")
self.sub.send('log', 'info', message="Starting BRISE")
if not self.experiment_setup:
# Check if main.py running with a specified experiment description file path
if len(argv) > 1:
exp_desc_file_path = argv[1]
else:
exp_desc_file_path = './Resources/EnergyExperiment/EnergyExperiment.json'
log_msg = f"The Experiment Setup was not provided and the path to an experiment file was not specified." \
f" The default one will be executed: {exp_desc_file_path}"
self.logger.warning(log_msg)
self.sub.send('log', 'warning', message=log_msg)
experiment_description, search_space = load_experiment_setup(
exp_desc_file_path)
else:
experiment_description = self.experiment_setup[
"experiment_description"]
search_space = self.experiment_setup["search_space"]
validate_experiment_description(experiment_description)
os.makedirs(experiment_description["General"]["results_storage"],
exist_ok=True)
# Initializing instance of Experiment - main data holder.
self.experiment = Experiment(experiment_description, search_space)
search_space.experiment_id = self.experiment.unique_id
Configuration.set_task_config(
self.experiment.description["TaskConfiguration"])
# initialize connection to rabbitmq service
self.connection = pika.BlockingConnection(
pika.ConnectionParameters(
os.getenv("BRISE_EVENT_SERVICE_HOST"),
int(os.getenv("BRISE_EVENT_SERVICE_AMQP_PORT"))))
self.consume_channel = self.connection.channel()
# initialize connection to the database
self.database = MongoDB(os.getenv("BRISE_DATABASE_HOST"),
int(os.getenv("BRISE_DATABASE_PORT")),
os.getenv("BRISE_DATABASE_NAME"),
os.getenv("BRISE_DATABASE_USER"),
os.getenv("BRISE_DATABASE_PASS"))
# write initial settings to the database
self.database.write_one_record(
"Experiment_description",
self.experiment.get_experiment_description_record())
self.database.write_one_record(
"Search_space",
get_search_space_record(self.experiment.search_space,
self.experiment.unique_id))
self.experiment.send_state_to_db()
self.sub.send(
'experiment',
'description',
global_config=self.experiment.description["General"],
experiment_description=self.experiment.description,
searchspace_description=self.experiment.search_space.serialize(
True))
self.logger.debug(
"Experiment description and global configuration sent to the API.")
# Create and launch Stop Condition services in separate threads.
launch_stop_condition_threads(self.experiment.unique_id)
# Instantiate client for Worker Service, establish connection.
self.wsc_client = WSClient(
self.experiment.description["TaskConfiguration"],
os.getenv("BRISE_EVENT_SERVICE_HOST"),
int(os.getenv("BRISE_EVENT_SERVICE_AMQP_PORT")))
# Initialize Repeater - encapsulate Configuration evaluation process to avoid results fluctuations.
# (achieved by multiple Configuration evaluations on Workers - Tasks)
RepeaterOrchestration(self.experiment)
self.predictor: Predictor = Predictor(self.experiment.unique_id,
self.experiment.description,
self.experiment.search_space)
self.consume_channel.basic_consume(
queue='default_configuration_results_queue',
auto_ack=True,
on_message_callback=self.get_default_configurations_results)
self.consume_channel.basic_consume(
queue='configurations_results_queue',
auto_ack=True,
on_message_callback=self.get_configurations_results)
self.consume_channel.basic_consume(queue='stop_experiment_queue',
auto_ack=True,
on_message_callback=self.stop)
self.consume_channel.basic_consume(
queue="get_new_configuration_queue",
auto_ack=True,
on_message_callback=self.send_new_configurations_to_measure)
self.default_config_handler = get_default_config_handler(
self.experiment)
temp_msg = "Measuring default Configuration."
self.logger.info(temp_msg)
self.sub.send('log', 'info', message=temp_msg)
default_parameters = self.experiment.search_space.generate_default()
default_configuration = Configuration(default_parameters,
Configuration.Type.DEFAULT,
self.experiment.unique_id)
default_configuration.experiment_id = self.experiment.unique_id
dictionary_dump = {"configuration": default_configuration.to_json()}
body = json.dumps(dictionary_dump)
self.consume_channel.basic_publish(
exchange='',
routing_key='measure_new_configuration_queue',
body=body)
# listen all queues with responses until the _is_interrupted flag is False
try:
while not self._is_interrupted:
self.consume_channel.connection.process_data_events(
time_limit=1) # 1 second
finally:
if self.connection.is_open:
self.connection.close()
