def _hack_(self):
"""
TODO: please remove me at earliest convenience
HACK for the inconsistent proxy and backend behavior
Replace exc and bibi representation by reading from storage
Proxy might have changed something that backend is not aware of
"""
from hbp_nrp_backend.storage_client_api.StorageClient import StorageClient
import urllib
client = StorageClient()
latest_exc = client.get_file(self._sim_config.token,
urllib.quote_plus(
self._sim_config.experiment_id),
self._sim_config.exc_path.rel_path,
by_name=True)
latest_bibi = client.get_file(self._sim_config.token,
urllib.quote_plus(
self._sim_config.experiment_id),
self._sim_config.bibi_path.rel_path,
by_name=True)
try:
self._exc_dom = exc_parser.CreateFromDocument(latest_exc)
self._bibi_dom = bibi_parser.CreateFromDocument(latest_bibi)
except Exception as ex:
raise Exception(
"Something went horribly wrong while creating latest "
"config objects with following exception {}".format(str(ex)))
def put(self, sim_id):
"""
Set brain file of the simulation specified with simulation ID.
Depending on the type of brain file, it has to be transmitted as text or as base64
:param sim_id: The simulation ID
:< json string brain_type: Type of the brain file ('h5' or 'py')
:< json string data_type: type of the data field ('text' or 'base64')
:< json string data: Contents of the brain file. Encoding given in field data_type
:< json dict brain_populations: A dictionary indexed by population names and containing
neuron indices
:> json string error_message: Error Message if there is a syntax error in the code
:> json int error_line: Line of code, where error occurred
:> json int error_column: Column, where error occurred (if available)
:> json bool handle_population_change: a flag indicating if user wants to change transfer
functions according to population changes.
:status 500: {0}
:status 404: {1}
:status 401: {2}
:status 400: {3}
:status 200: Success. The experiment brain file was replaced
"""
simulation = _get_simulation_or_abort(sim_id)
storage_client = StorageClient()
if not UserAuthentication.can_modify(simulation):
raise NRPServicesWrongUserException()
body = request.get_json(force=True)
file_url = body.get('urls', {}).get('fileUrl')
if file_url:
# TODO: fix server certificate and remove verify
with requests.get(file_url, verify=False) as h5:
filename = os.path.basename(file_url)
with open(os.path.join(simulation.lifecycle.sim_dir, filename),
'w') as f:
f.write(h5.content)
storage_client.create_or_update(simulation.token,
simulation.experiment_id,
filename, h5.content,
"text/plain")
result = simulation.cle.set_simulation_brain(
brain_type=body['brain_type'],
data=body['data'],
data_type=body['data_type'],
brain_populations=json.dumps(body['brain_populations']))
if result.error_message:
# Error in given brain
return {
'error_message': result.error_message,
'error_line': result.error_line,
'error_column': result.error_column
}, 400
# Success
return {'message': "Success"}, 200
def __init__(self, sim_config):
"""
Creates a new simulation assembly to simulate an experiment using the CLE and Gazebo
:param sim_config: config of the simulation to be managed
"""
super(CLEGazeboSimulationAssembly, self).__init__(sim_config)
self.cle_server = None
self.simAssetsDir = os.path.join(sim_config.sim_dir, 'assets')
self._simResourcesDir = os.path.join(sim_config.sim_dir, 'resources')
self._storageClient = StorageClient()
self._storageClient.set_sim_dir(sim_config.sim_dir)
def __init__(self, simulation, initial_state='created'):
"""
Creates a new backend simulation lifecycle
:param simulation: The simulation for which the simulation lifecycle is created
"""
super(BackendSimulationLifecycle,
self).__init__(TOPIC_LIFECYCLE(simulation.sim_id), initial_state)
self.__simulation = simulation
self._sim_dir = None
self.__models_path = Settings.nrp_models_directory
self.__experiment_path = None
self.__textures_loaded = False
self.__storageClient = StorageClient()
def _get_robot_abs_path(self, robot_file):
"""
Gets the absolute path of the given robot file
:param robot_file: The robot file
:return: the absolute path to the robot file
"""
if 'storage://' in robot_file:
from hbp_nrp_backend.storage_client_api.StorageClient import StorageClient
client = StorageClient()
abs_file = os.path.join(client.get_temp_directory(),
os.path.basename(robot_file))
name, ext = os.path.splitext(abs_file)
ext = ext.lower()
zipped = False
if ext == '.zip':
zipped = True
with open(abs_file, "w") as f:
f.write(
client.get_file(self.token,
client.get_folder_uuid_by_name(
self.token, self.ctx_id, 'robots'),
os.path.basename(robot_file),
byname=True,
zipped=zipped))
elif self.__is_collab_hack():
abs_file = os.path.join(self.exc.dir, os.path.basename(robot_file))
else:
abs_file = os.path.join(models_path, robot_file)
name, ext = os.path.splitext(abs_file)
ext = ext.lower()
if ext == ".sdf":
return abs_file
elif ext == ".zip":
name = os.path.split(name)[1] + "/model.sdf"
with zipfile.ZipFile(abs_file) as robot_zip:
try:
robot_zip.getinfo(name)
except KeyError:
raise Exception(
"The robot zip archive must contain an sdf file named {0} "
"at the root of the archive, but does not.".format(
name))
self.__tmp_robot_dir = tempfile.mkdtemp(suffix="robot")
robot_zip.extractall(path=self.__tmp_robot_dir)
return os.path.join(self.__tmp_robot_dir, name)
def _check_and_extract_environment_zip(self, experiment):
"""
Checks for validity and extracts a zipped environment. First we
make sure that the zip referenced in the experiment exists in the
list of user environments, then we unzip it on the fly in the temporary
simulation directory. After the extraction we also make sure to copy
the sdf from the experiment folder cause the user may have modified it
:param experiment: The experiment object.
"""
from hbp_nrp_backend.storage_client_api.StorageClient import StorageClient
client = StorageClient()
environments_list = client.get_custom_models(
UserAuthentication.get_header_token(request),
self.simulation.ctx_id, 'environments')
# we use the paths of the uploaded zips to make sure the selected
# zip is there
paths_list = [environment['path'] for environment in environments_list]
# check if the zip is in the user storage
zipped_model_path = [
path for path in paths_list
if experiment.environmentModel.customModelPath in path
]
if len(zipped_model_path):
environment_path = os.path.join(
client.get_temp_directory(),
os.path.basename(experiment.environmentModel.src))
storage_env_zip_data = client.get_custom_model(
UserAuthentication.get_header_token(request),
self.simulation.ctx_id, zipped_model_path[0])
env_sdf_name = os.path.basename(experiment.environmentModel.src)
env_path = os.path.join(
client.get_temp_directory(),
experiment.environmentModel.customModelPath)
with open(env_path, 'w') as environment_zip:
environment_zip.write(storage_env_zip_data)
with zipfile.ZipFile(env_path) as env_zip_to_extract:
env_zip_to_extract.extractall(path=client.get_temp_directory())
# copy back the .sdf from the experiment folder, cause we don't want the one
# in the zip, cause the user might have made manual changes
client.clone_file(env_sdf_name,
UserAuthentication.get_header_token(request),
self.simulation.experiment_id)
# if the zip is not there, prompt the user to check his uploaded
# models
else:
raise NRPServicesGeneralException(
"Could not find selected zip %s in the list of uploaded models. Please make\
sure that it has been uploaded correctly" %
(os.path.dirname(experiment.environmentModel.src)),
"Zipped model retrieval failed")
return environment_path
def prepare_record_for_playback(self):
"""
Copy the record from user storage to tmp and unzip it.
:> experimentID: The experiment
:> storagePath: The storage path
:tmp path: The local path ready to be played
"""
client = StorageClient()
file_clone_destination = os.path.join(self.sim_dir,
self.simulation.playback_path)
dest_path = os.path.dirname(file_clone_destination)
try:
if not os.path.exists(dest_path):
os.makedirs(dest_path)
with open(file_clone_destination, "w") as file_clone:
file_contents = client.get_file(
UserAuthentication.get_header_token(),
urllib.quote_plus(self.simulation.experiment_id +
'/recordings'),
os.path.basename(
self.simulation.playback_path), # zip name
by_name=True,
zipped=True)
file_clone.write(file_contents)
ZipUtil.extractall(file_clone_destination, dest_path, True)
# Update sim object's playback path with folder name
self.simulation.playback_path = os.path.join(
dest_path, ZipUtil.get_rootname(file_clone_destination))
os.remove(file_clone_destination)
except Exception as ex:
SimUtil.delete_simulation_dir()
raise NRPServicesClientErrorException(
'Copying recording to backend tmp failed with {}'.format(
str(ex)),
error_code=404)
def __init__(self, assembly, interval=5, folder_name=None):
"""
The assembly object contains all the necessary information
to save the csv data, the token, experiment_id
:param SimulationAssembly assembly: contains all the
information tied to the running simulation
:param optional int interval: the interval between consequent saves
:param optional string folder_name: user-defined name for csv_data
"""
self._log_csv_thread = None
self._creation_time = get_date_and_time_string()
self._interval = interval
self._assembly = assembly
self._folder_name = folder_name
self._storage_client = StorageClient()
self.stop_flag = threading.Event()
def stop(self, state_change):
"""
Stops the simulation and releases all resources
:param state_change: The state change that led to releasing simulation resources
"""
self.simulation.kill_datetime = None
if self.simulation.cle is not None:
self.simulation.cle.stop_communication(
"Simulation server released")
logger.info(
"State machine outcomes: %s",
", ".join("%s: %s" % (sm.sm_id, str(sm.result))
for sm in self.simulation.state_machines))
self.simulation.state_machine_manager.shutdown()
from hbp_nrp_backend.storage_client_api.StorageClient import StorageClient
client = StorageClient()
client.remove_temp_directory()
def save_record_to_user_storage(self):
"""
Save the record to user storage
"""
client_record_folder = 'recordings'
record_path = self.simulation.cle.command_simulation_recorder(
SimulationRecorderRequest.STATE).message
file_name = 'recording_{timestamp}.{ext}'.format(
timestamp=time.strftime('%Y-%m-%d_%H-%M-%S'), ext='zip')
temp_dest = os.path.join(tempfile.gettempdir(), file_name)
ZipUtil.create_from_path(record_path, temp_dest)
client = StorageClient()
client.create_folder(self.simulation.token,
self.simulation.experiment_id,
client_record_folder)
try:
with open(temp_dest, 'rb') as record_file:
zip_data = record_file.read()
client.create_or_update(
self.simulation.token, self.simulation.experiment_id,
os.path.join(client_record_folder, file_name), zip_data,
"application/octet-stream")
finally:
os.remove(temp_dest)
return file_name
def _parse_env_path(self, environment_path, experiment, using_storage):
"""
Parses the environment path, depending if we are using a storage model from
a template experiment(where we have to fetch the model from the storage),
or we are running a storage experiment where the model is already there.
Default case is when we are not using a storage model
:param experiment: The experiment object.
:param environment_path: Path to the environment configuration.
:param using_storage: Private or template simulation
"""
from hbp_nrp_backend.storage_client_api.StorageClient import StorageClient
client = StorageClient()
if using_storage:
custom = experiment.environmentModel.customModelPath
if custom:
environment_path = self._check_and_extract_environment_zip(
experiment)
else:
if 'storage://' in environment_path:
environment_path = self._copy_storage_environment(
experiment)
else:
if not environment_path and 'storage://' in experiment.environmentModel.src:
environment_path = os.path.join(
client.get_temp_directory(),
os.path.basename(experiment.environmentModel.src))
with open(environment_path, "w") as f:
f.write(
client.get_file(
UserAuthentication.get_header_token(request),
client.get_folder_uuid_by_name(
UserAuthentication.get_header_token(request),
self.simulation.ctx_id, 'environments'),
os.path.basename(experiment.environmentModel.src),
byname=True))
else:
environment_path = os.path.join(
self.models_path, str(experiment.environmentModel.src))
return environment_path
def put(self, sim_id):
"""
Save the simulation CSV recorders' content to the storage.
:param sim_id: The simulation ID
:status 500: Error when saving recorder files
:status 404: {0}
:status 401: {1}
:status 200: Success. Files saved into storage
"""
simulation = _get_simulation_or_abort(sim_id)
if not UserAuthentication.matches_x_user_name_header(
request, simulation.owner):
raise NRPServicesWrongUserException()
csv_files = simulation.cle.get_simulation_CSV_recorders_files()
# Done here in order to avoid circular dependencies introduced by the
# way we __init__ the rest_server module.
from hbp_nrp_backend.storage_client_api.StorageClient \
import StorageClient
client = StorageClient()
time_string = get_date_and_time_string()
subfolder_name = string.join(['csv_records', time_string], '_')
folder_uuid = client.create_folder(
UserAuthentication.get_header_token(request),
simulation.experiment_id, subfolder_name)['uuid']
if csv_files:
for csv_file in csv_files:
with open(csv_file.temporary_path) as csvfile:
client.create_or_update(
UserAuthentication.get_header_token(request),
folder_uuid, csv_file.name, csvfile.read(),
'text/plain')
return 200
def _copy_storage_environment(self, experiment):
"""
Copies a storage environment from the storage environment models
to the running simulation temporary folder
:param experiment: The experiment object.
"""
from hbp_nrp_backend.storage_client_api.StorageClient import StorageClient
client = StorageClient()
environment_path = os.path.join(
client.get_temp_directory(),
os.path.basename(experiment.environmentModel.src))
with open(environment_path, "w") as f:
f.write(
client.get_file(
UserAuthentication.get_header_token(request),
client.get_folder_uuid_by_name(
UserAuthentication.get_header_token(request),
self.simulation.ctx_id, 'environments'),
os.path.basename(experiment.environmentModel.src),
byname=True))
return environment_path
def _load_brain(self, rng_seed):
"""
Loads the neural simulator, interfaces, and configuration
:param rng_seed RNG seed to spawn Nest with
"""
# Create interfaces to brain
self._notify("Loading neural simulator")
brainconfig.rng_seed = rng_seed
braincomm, braincontrol = self._create_brain_adapters()
self._notify("Loading brain and population configuration")
# load brain
brainfilepath = self.bibi.brainModel.file
if self.__is_collab_hack():
if self.exc.dir is not None:
brainfilepath = os.path.join(self.exc.dir, brainfilepath)
elif 'storage://' in brainfilepath:
from hbp_nrp_backend.storage_client_api.StorageClient import StorageClient
client = StorageClient()
brainfilepath = os.path.join(client.get_temp_directory(),
os.path.basename(brainfilepath))
with open(brainfilepath, "w") as f:
f.write(
client.get_file(self.token,
client.get_folder_uuid_by_name(
self.token, self.ctx_id, 'brains'),
os.path.basename(brainfilepath),
byname=True))
else:
brainfilepath = os.path.join(models_path, brainfilepath)
neurons_config = get_all_neurons_as_dict(
self.bibi.brainModel.populations)
return braincontrol, braincomm, brainfilepath, neurons_config
def post(self, sim_id):
"""
Save the current running experiment SDF back to the storage
:param sim_id: The sim_id
:param context_id: The context_id of the experiment
:status 500: Error saving file
:status 200: Success. File written.
"""
# pylint: disable=too-many-locals
simulation = _get_simulation_or_abort(sim_id)
try:
rospy.wait_for_service('/gazebo/export_world_sdf', 3)
except rospy.ROSException as exc:
raise NRPServicesUnavailableROSService(str(exc))
dump_sdf_world = rospy.ServiceProxy('/gazebo/export_world_sdf',
ExportWorldSDF)
try:
sdf_string = dump_sdf_world().sdf_dump
tree = ET.fromstring(sdf_string)
# Erase all robots from the SDF
robots = simulation.cle.get_simulation_robots()
for robot in robots:
for m in tree.findall(".//model[@name='" + robot.robot_id +
"']"):
m.getparent().remove(m)
sdf_string = ET.tostring(tree, encoding='utf8', method='xml')
except rospy.ServiceException as exc:
raise NRPServicesClientErrorException(
"Service did not process request:" + str(exc))
client = StorageClient()
# find the sdf world filename from the .exc
exp_xml_file_path = client.clone_file(
'experiment_configuration.exc',
UserAuthentication.get_header_token(), simulation.experiment_id)
experiment_file = client.parse_and_check_file_is_valid(
exp_xml_file_path, exp_conf_api_gen.CreateFromDocument,
exp_conf_api_gen.ExD_)
world_file_name = experiment_file.environmentModel.src
client.create_or_update(UserAuthentication.get_header_token(),
simulation.experiment_id, world_file_name,
sdf_string, "text/plain")
return 200
class SimulationResetStorage(Resource):
"""
This resource handles the reset of a simulation, forwarding all the reset requests to the
respective CLE instance.
"""
storage_client = StorageClient()
@swagger.model
class ResetRequest(object):
"""
Represents a request for the API implemented by SimulationResetStorage
"""
resource_fields = {
'resetType': fields.Integer,
'contextId': fields.String()
}
required = ['resetType']
@swagger.operation(
notes='Handles the reset of a given simulation.',
parameters=[
{
"name": "sim_id",
"description": "The ID of the simulation whose state shall be retrieved",
"required": True,
"paramType": "path",
"dataType": int.__name__
},
{
"name": "body",
"paramType": "body",
"dataType": ResetRequest.__name__,
"required": True
}
],
responseMessages=[
{
"code": 500,
"message": ErrorMessages.SERVER_ERROR_500
},
{
"code": 404,
"message": ErrorMessages.SIMULATION_NOT_FOUND_404
},
{
"code": 401,
"message": ErrorMessages.SIMULATION_PERMISSION_401
},
{
"code": 400,
"message": "Invalid request, the JSON parameters are incorrect"
},
{
"code": 200,
"message": "Success. The reset type requested was performed on the given "
"simulation"
}
]
)
@docstring_parameter(ErrorMessages.SERVER_ERROR_500,
ErrorMessages.SIMULATION_NOT_FOUND_404,
ErrorMessages.SIMULATION_PERMISSION_401)
def put(self, sim_id, experiment_id):
"""
Calls the CLE for resetting a given simulation to the last saved state in the storage.
:param sim_id: The simulation ID.
:param experiment_id: The experiment ID
:> json resetType: the reset type the user wants to be performed, details about possible
values are given in
GazeboRosPackages/src/cle_ros_msgs/srv/ResetSimulation.srv
:status 500: {0}
:status 404: {1}
:status 401: {2}
:status 400: Invalid request, the JSON parameters are incorrect
:status 200: The requested reset was performed successfully
"""
sim = _get_simulation_or_abort(sim_id)
if not UserAuthentication.can_modify(sim):
raise NRPServicesWrongUserException()
req_body = request.get_json(force=True)
context_id = req_body.get('contextId', None)
for missing_par in (par for par in self.ResetRequest.required if par not in req_body):
raise NRPServicesClientErrorException('Missing parameter {}'.format(missing_par))
for invalid_p in (par for par in req_body if par not in self.ResetRequest.resource_fields):
raise NRPServicesClientErrorException('Invalid parameter {}'.format(invalid_p))
reset_type = req_body.get('resetType')
rsr = srv.ResetSimulationRequest
try:
if reset_type == rsr.RESET_ROBOT_POSE:
sim.cle.reset(reset_type)
elif reset_type == rsr.RESET_WORLD:
sim.cle.reset(reset_type,
world_sdf=self._get_sdf_world_from_storage(experiment_id, context_id))
elif reset_type == rsr.RESET_FULL:
brain_path, populations, _ = \
self._get_brain_info_from_storage(experiment_id, context_id)
if sim.playback_path is None:
self.reset_from_storage_all(sim, experiment_id, context_id)
sim.cle.reset(reset_type,
world_sdf=self._get_sdf_world_from_storage(experiment_id, context_id),
brain_path=brain_path,
populations=populations)
else:
return {}, 400 # Other reset modes are unsupported
except ROSCLEClientException as e:
raise NRPServicesGeneralException(str(e), 'CLE error', 500)
return {}, 200
@classmethod
def reset_from_storage_all(cls, simulation, experiment_id, context_id):
"""
Reset states machines and transfer functions
:param: the simulation id
:param: the experiment id
:param: the context_id for collab based simulations
"""
SimUtil.clear_dir(simulation.lifecycle.sim_dir)
token = UserAuthentication.get_header_token()
cls.storage_client.clone_all_experiment_files(
token, experiment_id, destination_dir=simulation.lifecycle.sim_dir)
with open(simulation.lifecycle.experiment_path) as exc_file:
exc = exp_conf_api_gen.CreateFromDocument(exc_file.read())
bibi_path = os.path.join(os.path.dirname(simulation.lifecycle.experiment_path),
exc.bibiConf.src)
with open(bibi_path) as bibi_file:
bibi = bibi_api_gen.CreateFromDocument(bibi_file.read())
cls.reset_brain(simulation, experiment_id, context_id)
cls.reset_transfer_functions(simulation, bibi, simulation.lifecycle.sim_dir)
cls.reset_state_machines(simulation, exc, simulation.lifecycle.sim_dir)
@classmethod
def reset_brain(cls, simulation, experiment_id, context_id):
"""
Reset brain
:param simulation: simulation object
:param experiment_id: the related experiment id
:param context_id: the context ID for collab based simulations
"""
brain_path, _, neurons_config = cls._get_brain_info_from_storage(experiment_id, context_id)
# Convert the populations to a JSON dictionary
for name, s in neurons_config.iteritems():
# Convert slice to a dictionary for Non-Spinnaker brains
experiment_population = SimulationResetStorage._get_experiment_population(name, s)
if (experiment_population.type != ExperimentPopulationInfo.TYPE_POPULATION_SPINNAKER):
neurons_config[name] = {
'from': s.start,
'to': s.stop,
'step': s.step if s.step > 0 else 1}
with open(brain_path, 'r') as brain_file:
result_set_brain = simulation.cle.set_simulation_brain(
brain_type='py',
data_type='text',
data=brain_file.read(),
brain_populations=json.dumps(neurons_config))
if result_set_brain is not None and result_set_brain.error_message:
# Error in given brain
raise ROSCLEClientException('{err}, column: {col}'.format(
err=result_set_brain.error_message, col=result_set_brain.error_column))
@staticmethod
def reset_state_machines(sim, experiment, sm_base_path):
"""
Reset states machines
:param sim:
:param experiment: experiment conf
:param sm_base_path: base path of the experiment
"""
sim.delete_all_state_machines()
state_machine_paths = {}
if experiment.experimentControl is not None:
state_machine_paths.update({sm.id: os.path.join(sm_base_path, sm.src)
for sm in
experiment.experimentControl.stateMachine
if isinstance(sm, exp_conf_api_gen.SMACHStateMachine)})
if experiment.experimentEvaluation is not None:
state_machine_paths.update({sm.id: os.path.join(sm_base_path, sm.src)
for sm in
experiment.experimentEvaluation.stateMachine
if isinstance(sm, exp_conf_api_gen.SMACHStateMachine)})
sim.state_machine_manager.add_all(state_machine_paths, sim.sim_id, sim.lifecycle.sim_dir)
sim.state_machine_manager.initialize_all()
@staticmethod
def reset_transfer_functions(simulation, bibi_conf, base_path):
"""
Reset transfer functions
:param simulation: simulation object
:param bibi_conf: BIBI conf
:param base_path: base path of the experiment
"""
old_tfs, _ = simulation.cle.get_simulation_transfer_functions()
for old_tf_name in (get_tf_name(old_tf) for old_tf in old_tfs):
if old_tf_name is not None: # ignore broken TFs
simulation.cle.delete_simulation_transfer_function(old_tf_name)
for tf in bibi_conf.transferFunction:
tf_code = '{}\n'.format(correct_indentation(generate_tf(tf, base_path), 0).strip())
logger.debug(" RESET TF: {tf_name}\n{tf_code}\n"
.format(tf_name=get_tf_name(tf_code), tf_code=tf_code))
# adding original TFs from the bibi
# do not check the error message.
# CLE will handle also invalid TFs
simulation.cle.add_simulation_transfer_function(str(tf_code))
@classmethod
def _get_brain_info_from_storage(cls, experiment_id, context_id):
"""
Gathers from the storage the brain script and the populations by getting the BIBI
configuration file.
:param experiment_id: the id of the experiment in which to look for the brain information
:param context_id: the context ID for collab based simulations
:return: A tuple with the path to the brain file and a list of populations
"""
del context_id # Unused
request_token = UserAuthentication.get_header_token()
experiment_file = cls.storage_client.get_file(
request_token, experiment_id, 'experiment_configuration.exc', by_name=True)
bibi_filename = exp_conf_api_gen.CreateFromDocument(experiment_file).bibiConf.src
# find the brain filename from the bibi
bibi_file = cls.storage_client.get_file(
request_token, experiment_id, bibi_filename, by_name=True)
bibi_file_obj = bibi_api_gen.CreateFromDocument(bibi_file)
brain_filename = os.path.basename(bibi_file_obj.brainModel.file)
brain_filepath = cls.storage_client.clone_file(brain_filename, request_token, experiment_id)
neurons_config = get_all_neurons_as_dict(bibi_file_obj.brainModel.populations)
neurons_config_clean = \
[SimulationResetStorage._get_experiment_population(name, v)
for (name, v) in neurons_config.iteritems()]
return brain_filepath, neurons_config_clean, neurons_config
@staticmethod
def _get_experiment_population(name, value):
"""
Gets an ExperimentPopulation object for the given population
:param name: The population name
:param value: The value describing the population
:return:
"""
if isinstance(value, slice):
return msg.ExperimentPopulationInfo(name=name, type=1, ids=[],
start=value.start, stop=value.stop, step=value.step)
if isinstance(value, list):
return msg.ExperimentPopulationInfo(name=name, type=2, ids=value,
start=0, stop=0, step=0)
type_id = 3 if isinstance(value, basestring) else 0
return msg.ExperimentPopulationInfo(name=name, type=type_id, ids=[],
start=0, stop=0, step=0)
@classmethod
def _get_sdf_world_from_storage(cls, experiment_id, context_id):
"""
Download from the storage an sdf world file as a string.
The file belongs to the experiment identified by experiment_id
:param experiment_id: the ID of the experiment in which to look for the world sdf
:param context_id: the context ID for collab based simulations
:return: The content of the world sdf file
"""
del context_id # Unused
request_token = UserAuthentication.get_header_token()
# find the sdf filename from the .exc
experiment_file = cls.storage_client.get_file(
request_token, experiment_id, 'experiment_configuration.exc', by_name=True)
world_file_name = exp_conf_api_gen.CreateFromDocument(experiment_file).environmentModel.src
return cls.storage_client.get_file(
request_token, experiment_id, world_file_name, by_name=True)
def put(self, experiment_id):
"""
Save transfer functions of an experiment to the storage.
:param path experiment_id: The experiment_id of the experiment where the transfer functions
will be saved
:<json body json array of string transfer_functions: the transfer functions as python
:status 500: BIBI configuration file not found
:status 500: Error saving file
:status 404: The experiment_id with the given expreiment ID was not found
:status 404: The request body is malformed
:status 200: Success. File written.
"""
# pylint: disable=too-many-locals
# Done here in order to avoid circular dependencies introduced by the
# way we __init__ the rest_server module
from hbp_nrp_backend.storage_client_api.StorageClient \
import StorageClient
body = request.get_json(force=True)
if 'transfer_functions' not in body:
raise NRPServicesClientErrorException(
"Transfer functions code should be sent in "
"the body under the 'transfer_functions' key")
client = StorageClient()
experiment_file = client.get_file(
UserAuthentication.get_header_token(request),
experiment_id,
'experiment_configuration.exc',
byname=True)
bibi_filename = exp_conf_api_gen.CreateFromDocument(
experiment_file).bibiConf.src
bibi_file_path = client.clone_file(
bibi_filename, UserAuthentication.get_header_token(request),
experiment_id)
bibi = client.parse_and_check_file_is_valid(
bibi_file_path, bibi_api_gen.CreateFromDocument,
bibi_api_gen.BIBIConfiguration)
# Remove all transfer functions from BIBI. Then we save them in a
# separate python file.
del bibi.transferFunction[:]
threads = []
for transfer_function in body['transfer_functions']:
transfer_function_name = get_tf_name(transfer_function)
if transfer_function_name is not None:
transfer_function_node = bibi_api_gen.PythonTransferFunction()
transfer_function_node.src = transfer_function_name + ".py"
bibi.transferFunction.append(transfer_function_node)
t = Thread(target=client.create_or_update,
kwargs={
'token':
UserAuthentication.get_header_token(request),
'experiment':
experiment_id,
'filename':
transfer_function_name + ".py",
'content':
transfer_function,
'content_type':
'text/plain'
})
t.start()
threads.append(t)
# we need to prettify the parsed bibi
pretty_bibi = xml.dom.minidom.parseString(
bibi.toxml("utf-8")).toprettyxml()
t = Thread(target=client.create_or_update,
kwargs={
'token': UserAuthentication.get_header_token(request),
'experiment': experiment_id,
'filename': bibi_filename,
'content': pretty_bibi,
'content_type': 'text/plain'
})
t.start()
threads.append(t)
for x in threads:
x.join()
return 200
def initialize(self, state_change):
"""
Initializes the simulation
:param state_change: The state change that caused the simulation to be initialized
"""
# TODO: fix dependencies so these import are not necessary
# anymore
from hbp_nrp_backend.storage_client_api.StorageClient import StorageClient
simulation = self.simulation
try:
using_storage = simulation.private is not None
if using_storage:
client = StorageClient()
clone_folder, experiment_paths = client.clone_all_experiment_files(
UserAuthentication.get_header_token(request),
simulation.experiment_id)
self.__experiment_path = experiment_paths['experiment_conf']
self.__simulation_root_folder = clone_folder
environment_path = experiment_paths['environment_conf']
else:
self.__experiment_path = os.path.join(
self.__experiment_path, simulation.experiment_conf)
self.__simulation_root_folder = os.path.dirname(
self.__experiment_path)
environment_path = simulation.environment_conf
experiment, environment_path = self._parse_exp_and_initialize_paths(
self.__experiment_path, environment_path, using_storage)
simulation.kill_datetime = datetime.datetime.now(timezone) \
+ datetime.timedelta(seconds=experiment.timeout)
logger.info("simulation timeout initialized")
simulation_factory_client = ROSCLESimulationFactoryClient()
simulation_factory_client.create_new_simulation(
environment_path, self.__experiment_path,
simulation.gzserver_host, simulation.reservation,
simulation.brain_processes, simulation.sim_id,
str(simulation.kill_datetime), simulation.playback_path,
UserAuthentication.get_header_token(request),
self.simulation.ctx_id)
if not simulation.playback_path:
simulation.cle = ROSCLEClient(simulation.sim_id)
else:
simulation.cle = PlaybackClient(simulation.sim_id)
logger.info("simulation initialized")
except IOError as e:
raise NRPServicesGeneralException(
"Error while accessing simulation models (" + repr(e.message) +
")", "Models error")
except rospy.ROSException as e:
raise NRPServicesGeneralException(
"Error while communicating with the CLE (" + repr(e.message) +
")", "CLE error")
except rospy.ServiceException as e:
raise NRPServicesGeneralException(
"Error starting the simulation. (" + repr(e.message) + ")",
"rospy.ServiceException",
data=e.message)
def post(self, experiment_id):
"""
Save the current running experiment SDF back to the storage
:param experiment_id: The experiment ID
:param context_id: The context_id of the experiment
:status 500: Error saving file
:status 200: Success. File written.
"""
# pylint: disable=too-many-locals
body = request.get_json(force=True)
context_id = body.get('context_id', None)
# Done here in order to avoid circular dependencies introduced by the
# way we __init__ the rest_server module.
from hbp_nrp_backend.storage_client_api.StorageClient \
import StorageClient
try:
rospy.wait_for_service('/gazebo/export_world_sdf', 3)
except rospy.ROSException as exc:
raise NRPServicesUnavailableROSService(str(exc))
dump_sdf_world = rospy.ServiceProxy('/gazebo/export_world_sdf',
ExportWorldSDF)
robot_pose = []
try:
sdf_string = dump_sdf_world().sdf_dump
tree = ET.fromstring(sdf_string)
try:
robot_pose = tree.findall(
".//state/model[@name='robot']/pose")[0].text.split()
# pylint: disable=bare-except
except:
logger.error("Can't retrieve robot position.")
# Erase all robots from the SDF
for m in tree.findall(".//model[@name='robot']"):
m.getparent().remove(m)
sdf_string = ET.tostring(tree, encoding='utf8', method='xml')
except rospy.ServiceException as exc:
raise NRPServicesClientErrorException(
"Service did not process request:" + str(exc))
client = StorageClient()
# find the sdf world filename from the .exc
exp_xml_file_path = client.clone_file(
'experiment_configuration.exc',
UserAuthentication.get_header_token(request), experiment_id)
experiment_file = client.parse_and_check_file_is_valid(
exp_xml_file_path, exp_conf_api_gen.CreateFromDocument,
exp_conf_api_gen.ExD_)
world_file_name = experiment_file.environmentModel.src
if 'storage://' in world_file_name:
world_file_name = os.path.basename(world_file_name)
client.create_or_update(
UserAuthentication.get_header_token(request),
client.get_folder_uuid_by_name(
UserAuthentication.get_header_token(request), context_id,
'environments'), world_file_name, sdf_string, "text/plain")
else:
client.create_or_update(
UserAuthentication.get_header_token(request), experiment_id,
world_file_name, sdf_string, "text/plain")
# Save the robot position in the ExDConf file
if len(robot_pose) is 6: # We need 6 elements (from Gazebo)
experiment_file.environmentModel.robotPose.x = robot_pose[0]
experiment_file.environmentModel.robotPose.y = robot_pose[1]
experiment_file.environmentModel.robotPose.z = robot_pose[2]
quaternion = tf.transformations.quaternion_from_euler(
float(robot_pose[3]), float(robot_pose[4]),
float(robot_pose[5]))
experiment_file.environmentModel.robotPose.ux = quaternion[0]
experiment_file.environmentModel.robotPose.uy = quaternion[1]
experiment_file.environmentModel.robotPose.uz = quaternion[2]
experiment_file.environmentModel.robotPose.theta = quaternion[3]
client.create_or_update(
UserAuthentication.get_header_token(request),
experiment_id, 'experiment_configuration.exc',
experiment_file.toxml("utf-8"), "text/plain")
else:
logger.error("Malformed robot position tag in SDF: " + robot_pose)
return 200
class BackendSimulationLifecycle(SimulationLifecycle):
"""
This class implements the backend simulation lifecycle
"""
def __init__(self, simulation, initial_state='created'):
"""
Creates a new backend simulation lifecycle
:param simulation: The simulation for which the simulation lifecycle is created
"""
super(BackendSimulationLifecycle,
self).__init__(TOPIC_LIFECYCLE(simulation.sim_id), initial_state)
self.__simulation = simulation
self._sim_dir = None
self.__models_path = Settings.nrp_models_directory
self.__experiment_path = None
self.__textures_loaded = False
self.__storageClient = StorageClient()
@property
def simulation(self):
"""
Gets the simulation for which the lifecycle is controlled
:return:
"""
return self.__simulation
def _load_state_machines(self, exc): # pragma: no cover
"""
Parses the experiment configuration, loads state machines
:param experiment_path: Path the experiment configuration
"""
state_machine_paths = {}
if exc.experimentControl is not None and not self.simulation.playback_path:
state_machine_paths.update({
sm.id: os.path.join(self._sim_dir, sm.src)
for sm in exc.experimentControl.stateMachine
if isinstance(sm, exp_conf_api_gen.SMACHStateMachine)
})
if exc.experimentEvaluation is not None and not self.simulation.playback_path:
state_machine_paths.update({
sm.id: os.path.join(self._sim_dir, sm.src)
for sm in exc.experimentEvaluation.stateMachine
if isinstance(sm, exp_conf_api_gen.SMACHStateMachine)
})
self.simulation.state_machine_manager.add_all(state_machine_paths,
self.simulation.sim_id,
self.sim_dir)
self.simulation.state_machine_manager.initialize_all()
logger.info("Requesting simulation resources")
return exc
def _prepare_custom_environment(self, exc):
"""
Download and extracts zipped environment defined in the exc
:param exc: The exc DOM object
"""
# pylint: disable=too-many-locals
env_model = Model(exc.environmentModel.model, ResourceType.ENVIRONMENT)
data = self.__storageClient.get_model(
UserAuthentication.get_header_token(), self.simulation.ctx_id,
env_model)
# if the zip is not there, prompt the user to check his uploaded models
if not data:
raise NRPServicesGeneralException(
"Could not find selected zip {} in the list of uploaded custom models. Please make "
"sure that it has been uploaded correctly".format(
os.path.dirname(exc.environmentModel.model)),
"Zipped model retrieval failed")
ZipUtil.extractall(zip_abs_path=io.BytesIO(data),
extract_to=os.path.join(self._sim_dir, 'assets'),
overwrite=True)
def initialize(self, state_change):
"""
Initializes the simulation
:param state_change: The state change that caused the simulation to be initialized
"""
simulation = self.simulation
if not simulation.playback_path:
self._sim_dir = SimUtil.init_simulation_dir()
try:
if not simulation.private:
raise NRPServicesGeneralException(
"Only private experiments are supported", "CLE error", 500)
self.__storageClient.clone_all_experiment_files(
token=UserAuthentication.get_header_token(),
experiment=simulation.experiment_id,
destination_dir=self._sim_dir,
exclude=['recordings/']
if not simulation.playback_path else [])
# divine knowledge about the exc name
self.__experiment_path = os.path.join(
self._sim_dir, 'experiment_configuration.exc')
with open(self.__experiment_path) as exd_file:
exc = exp_conf_api_gen.CreateFromDocument(exd_file.read())
self._load_state_machines(exc)
if exc.environmentModel.model: # i.e., custom zipped environment
self._prepare_custom_environment(exc)
simulation.timeout_type = (
TimeoutType.SIMULATION if exc.timeout.time
== TimeoutType.SIMULATION else TimeoutType.REAL)
timeout = exc.timeout.value()
if simulation.timeout_type == TimeoutType.REAL:
timeout = datetime.datetime.now(timezone) + datetime.timedelta(
seconds=timeout)
simulation.kill_datetime = timeout
else:
simulation.kill_datetime = None
logger.info("simulation timeout initialized")
simulation_factory_client = ROSCLESimulationFactoryClient()
simulation_factory_client.create_new_simulation(
self.__experiment_path, simulation.gzserver_host,
simulation.reservation, simulation.brain_processes,
simulation.sim_id, str(timeout),
simulation.timeout_type, simulation.playback_path,
UserAuthentication.get_header_token(), self.simulation.ctx_id,
self.simulation.experiment_id)
if not simulation.playback_path:
simulation.cle = ROSCLEClient(simulation.sim_id)
else:
simulation.cle = PlaybackClient(simulation.sim_id)
logger.info("simulation initialized")
except IOError as e:
raise NRPServicesGeneralException(
"Error while accessing simulation models (" + repr(e.message) +
")", "Models error")
except rospy.ROSException as e:
raise NRPServicesGeneralException(
"Error while communicating with the CLE (" + repr(e.message) +
")", "CLE error")
except rospy.ServiceException as e:
raise NRPServicesGeneralException(
"Error starting the simulation. (" + repr(e.message) + ")",
"rospy.ServiceException",
data=e.message)
# pylint: disable=broad-except
except Exception as e:
raise NRPServicesGeneralException(
"Error starting the simulation. (" + repr(e) + ")",
"Unknown exception occured",
data=e.message)
def start(self, state_change):
"""
Starts the simulation
:param state_change: The state change that led to starting the simulation
"""
logger.info("Starting State Machines...")
try:
self.simulation.state_machine_manager.start_all(False)
except Exception as e: # pylint: disable=broad-except
logger.error("Starting state machines failed")
logger.exception(e)
# The frontend will be notified of any state machine issues directly
# over the cle_error topic
def stop(self, state_change):
"""
Stops the simulation and releases all resources
:param state_change: The state change that led to releasing simulation resources
"""
if self.simulation.cle is not None:
is_recording = self.simulation.cle.command_simulation_recorder(
SimulationRecorderRequest.STATE).value
if is_recording is True:
self.save_record_to_user_storage()
self.simulation.kill_datetime = None
if self.simulation.cle is not None:
self.simulation.cle.stop_communication(
"Simulation server released")
logger.info(
"State machine outcomes: %s",
", ".join("%s: %s" % (sm.sm_id, str(sm.result))
for sm in self.simulation.state_machines))
self.simulation.state_machine_manager.shutdown()
def pause(self, state_change):
"""
Pauses the simulation
:param state_change: The state change that paused the simulation
"""
# From the backend side, there is nothing to do because state machines
# keep running
pass
def fail(self, state_change):
"""
Fails the simulation
:param state_change: The state change which resulted in failing the simulation
"""
self.simulation.kill_datetime = None
if self.simulation.cle is not None:
self.simulation.cle.stop_communication("Simulation has failed")
self.simulation.state_machine_manager.terminate_all()
def reset(self, state_change):
"""
Resets the simulation
:param state_change: The state change that led to reseting the simulation
"""
logger.info(
"State machine outcomes: %s",
", ".join("%s: %s" % (sm.sm_id, str(sm.result))
for sm in self.simulation.state_machines))
self.simulation.state_machine_manager.terminate_all()
logger.info("simulation reset")
def save_record_to_user_storage(self):
"""
Save the record to user storage
"""
client_record_folder = 'recordings'
record_path = self.simulation.cle.command_simulation_recorder(
SimulationRecorderRequest.STATE).message
file_name = 'recording_{timestamp}.{ext}'.format(
timestamp=time.strftime('%Y-%m-%d_%H-%M-%S'), ext='zip')
temp_dest = os.path.join(tempfile.gettempdir(), file_name)
ZipUtil.create_from_path(record_path, temp_dest)
client = StorageClient()
client.create_folder(self.simulation.token,
self.simulation.experiment_id,
client_record_folder)
try:
with open(temp_dest, 'rb') as record_file:
zip_data = record_file.read()
client.create_or_update(
self.simulation.token, self.simulation.experiment_id,
os.path.join(client_record_folder, file_name), zip_data,
"application/octet-stream")
finally:
os.remove(temp_dest)
return file_name
@property
def experiment_path(self):
"""
Gets the experiment_path
:return: The experiment_path
"""
return self.__experiment_path
@experiment_path.setter
def experiment_path(self, value):
"""
Sets the experiment_path
"""
self.__experiment_path = value
@property
def sim_dir(self):
"""
Gets the simulation root folder
:return: The _sim_dir
"""
return self._sim_dir
class CSVLogger(object):
"""
Class that logs CSV data to the storage,
received from the CLE every n seconds
Runs in a separate killable thread
"""
def __init__(self, assembly, interval=5, folder_name=None):
"""
The assembly object contains all the necessary information
to save the csv data, the token, experiment_id
:param SimulationAssembly assembly: contains all the
information tied to the running simulation
:param optional int interval: the interval between consequent saves
:param optional string folder_name: user-defined name for csv_data
"""
self._log_csv_thread = None
self._creation_time = get_date_and_time_string()
self._interval = interval
self._assembly = assembly
self._folder_name = folder_name
self._storage_client = StorageClient()
self.stop_flag = threading.Event()
def initialize(self):
"""
Initializes a killable thread which runs the log_csv function
every interval seconds, and starts the thread
"""
self.stop_flag.clear()
def _log_csv_job(): # pragma: no cover
"""
The job to put in the thread
"""
while not self.stop_flag.isSet():
self._log_csv()
time.sleep(self._interval)
self._log_csv_thread = threading.Thread(target=_log_csv_job)
self._log_csv_thread.start()
def shutdown(self):
"""
Terminates the killable thread that saves the csv data
"""
if self._log_csv_thread:
self.stop_flag.set()
self._log_csv()
if self._log_csv_thread:
self._log_csv_thread.join() # wait till thread returns
self._log_csv_thread = None
def reset(self):
"""
Resets the killable thread that saves the csv data and updates
the creation time. This is done to create a new folder after reset
"""
self.shutdown()
# update the creation time to store the data in a separate folder upon reset
self._creation_time = get_date_and_time_string()
self.initialize()
def _log_csv(self):
"""
Appends the simulation CSV recorders' content to the storage
"""
csv_files = [
CSVRecordedFile(recorded_file[0], recorded_file[1],
recorded_file[2])
for recorded_file in tf_framework.dump_csv_recorder_to_files()
]
if csv_files:
time_string = self._creation_time if self._creation_time \
else get_date_and_time_string()
subfolder_name = self._folder_name if self._folder_name \
else '_'.join(['csv_records', time_string])
# no harm calling the function since the create_folder does
# nothing if the folder exists
folder_uuid = self._storage_client.create_folder(
self._assembly.sim_config.token,
self._assembly.sim_config.experiment_id,
subfolder_name)['uuid']
for csv_file in csv_files:
# if there is no lock file it means that for the currently running sim
# no csv files have been created, thus we create them in the storage and
# append the headers. To make sure that we don't do it every time in the
# context of the current simulation, we check if a lock file exists,
# if not, we create it
lock_filename = csv_file.name + '.lock'
lock_full_path = os.path.join(self._assembly.sim_dir,
subfolder_name, lock_filename)
dirname = os.path.dirname(lock_full_path)
lock = (SimUtil.find_file_in_paths(lock_filename, [dirname]) or
SimUtil.find_file_in_paths(csv_file.name, [dirname]))
if not lock:
content = ''.join(csv_file.headers) + \
''.join(csv_file.values)
if not os.path.exists(dirname):
os.makedirs(dirname)
with open(os.path.join(dirname, lock_filename), 'a'):
pass
else:
content = ''.join(csv_file.values)
self._storage_client.create_or_update(
self._assembly.sim_config.token,
folder_uuid,
csv_file.name,
content,
'text/plain',
append=lock)
class CLEGazeboSimulationAssembly(GazeboSimulationAssembly):
"""
This class assembles the simulation using the CLE
"""
def __init__(self, sim_config):
"""
Creates a new simulation assembly to simulate an experiment using the CLE and Gazebo
:param sim_config: config of the simulation to be managed
"""
super(CLEGazeboSimulationAssembly, self).__init__(sim_config)
self.cle_server = None
self.simAssetsDir = os.path.join(sim_config.sim_dir, 'assets')
self._simResourcesDir = os.path.join(sim_config.sim_dir, 'resources')
self._storageClient = StorageClient()
self._storageClient.set_sim_dir(sim_config.sim_dir)
@property
def storage_client(self):
"""
Gets the storage client handler
"""
return self._storageClient
def _initialize(self, except_hook):
"""
Internally initialize the simulation
:param environment: The environment that should be simulated
:param except_hook: A method that should be called when there is a critical error
"""
# pylint: disable=too-many-locals
# create the CLE server and lifecycle first to report any failures properly
# initialize the cle server and services
logger.info("Creating CLE Server")
self.cle_server = ROSCLEServer(self.sim_config.sim_id,
self.sim_config.timeout,
self.sim_config.timeout_type,
self.gzserver, self.ros_notificator)
self.cle_server.setup_handlers(self)
# Put the resources folder into the sys path for import
try:
SimUtil.makedirs(self._simResourcesDir)
with open(os.path.join(self._simResourcesDir, '__init__.py'),
'w+'):
pass # make sure the __init__.py exists
except IOError as err:
logger.info(
"Failed to setup resource directory due to {err}".format(
err=err))
sys.path.insert(0, self._simResourcesDir)
# start Gazebo simulator and bridge
self._start_gazebo(extra_models=self.simAssetsDir + ':' + self.sim_dir)
# load user textures in Gazebo
self._load_textures()
# load environment and robot models
models, lights = self._load_environment(
self.sim_config.world_model.resource_path.abs_path)
# find robot
self.robotManager.set_robot_dict(self.sim_config.robot_models)
self._load_robot()
robot_poses = {}
for rid, robot in self.robotManager.get_robot_dict().iteritems():
robot_poses[rid] = robot.pose
# load robot adapters
robotcomm, robotcontrol = self._create_robot_adapters()
# load the brain
braincontrol, braincomm, brainfile, brainconf = self._load_brain()
#load external modules
externalmodulearray = ExternalModuleManager()
# initialize the cle server and services
logger.info("Preparing CLE Server")
self.cle_server.cle = self.__load_cle(robotcontrol, robotcomm,
braincontrol, braincomm,
brainfile, brainconf,
externalmodulearray, robot_poses,
models, lights)
self.cle_server.prepare_simulation(except_hook)
# load transfer functions
self.__load_tfs()
# Wait for the backend rendering environment to load (for any sensors/cameras)
self._notify("Waiting for Gazebo simulated sensors to be ready")
self.robotManager.scene_handler().wait_for_backend_rendering()
# Spawns a new thread for the csv logger
# pylint: disable=protected-access
self.cle_server._csv_logger.initialize()
def _prepare_simconfig_robots(self):
"""
Reads robot list from bibi and poses from exc and populates robot manager
:return: -
"""
# pylint: disable=too-many-branches
if not self.sim_config.robot_models:
return
for robot in self.sim_config.robot_models.values():
if robot.isCustom:
# pylint: disable=protected-access
status, ret = self.cle_server._robotHandler.prepare_custom_robot(
robot.model)
if not status:
raise Exception(
"Could not prepare custom robot {err}".format(err=ret))
sdf_abs_path = ret
else:
sdf_abs_path = SimUtil.find_file_in_paths(
os.path.join(robot.id,
os.path.basename(robot.SDFFileAbsPath)),
self.sim_config.model_paths)
# Perhaps it's a previously coned experiment? Try with modelTag.value() BUT
# only look into the simulation_directory, as DELETE robot REST call, if called,
# would delete this file. Only for the exps without robotid folder in the storage
# TODO: backward compatibility code. Remove when we decide not to support anymore
if not sdf_abs_path:
sdf_abs_path = SimUtil.find_file_in_paths(
robot.SDFFileAbsPath, [self.sim_dir])
# still couldn't find the SDF, abort!
if not sdf_abs_path:
raise Exception("Could not find robot file: {0}".format(
robot.SDFFileAbsPath))
robot.SDFFileAbsPath = sdf_abs_path
# Find robot specific roslaunch file in the directory where the SDF resides
# Take the first one (by name) if multiple available
rosLaunchRelPath = next(
(f for f in os.listdir(os.path.dirname(robot.SDFFileAbsPath))
if f.endswith('.launch')), None)
robot.rosLaunchAbsPath = (
None if rosLaunchRelPath is None else os.path.join(
os.path.dirname(robot.SDFFileAbsPath), rosLaunchRelPath))
def _load_environment(self, world_file_abs_path):
"""
Loads the environment and robot in Gazebo
:param world_file_abs_path Path to the world sdf
"""
self._notify("Loading experiment environment")
return self.robotManager.scene_handler().parse_gazebo_world_file(
world_file_abs_path)
def _load_textures(self):
"""
Loads custom textures in Gazebo
"""
self._notify("Loading textures")
try:
textures = self.storage_client.get_textures(
self.sim_config.experiment_id, self.sim_config.token)
except: # pylint: disable=bare-except
logger.info("Non-existent textures or folder!")
return # ignore missing textures or texture folder
try:
if textures:
self.robotManager.scene_handler().load_textures(textures)
except: # pylint: disable=bare-except
logger.info("Timeout while trying to load textures.")
# pylint: disable-msg=too-many-branches
def _load_robot(self):
"""
Loads robots defined in the bibi and initializes any external controller
"""
# Set retina config for the robotManager
if self.sim_config.retina_config:
self._notify("Configuring Retina Camera Plugin")
self.robotManager.retina_config = self.sim_config.retina_config
self._notify("Loading robots")
self._prepare_simconfig_robots()
for robot in self.robotManager.get_robot_dict().values():
self.robotManager.initialize(robot)
# load external robot controller
if self.sim_config.ext_robot_controller is not None:
robot_controller_filepath = SimUtil.find_file_in_paths(
self.sim_config.ext_robot_controller,
self.sim_config.model_paths)
if not os.path.isfile(
robot_controller_filepath) and self.sim_dir is not None:
robot_controller_filepath = os.path.join(
self.sim_dir, self.sim_config.ext_robot_controller)
if os.path.isfile(robot_controller_filepath):
self._notify("Loading external robot controllers") # +1
res = subprocess.call([robot_controller_filepath, 'start'])
if res > 0:
logger.error(
"The external robot controller could not be loaded")
self.shutdown()
return
def _create_robot_adapters(self): # pragma: no cover
"""
Creates the adapter components for the robot side
:return: A tuple of the communication and control adapter for the robot side
"""
raise NotImplementedError(
"This method must be overridden in an implementation")
def _load_brain(self):
"""
Loads the neural simulator, interfaces, and configuration
"""
# Create interfaces to brain
self._notify("Loading neural simulator")
brainconfig.rng_seed = self.rng_seed
braincomm, braincontrol = self._create_brain_adapters()
self._notify("Loading brain and population configuration")
if not self.sim_config.brain_model:
return braincontrol, braincomm, None, None
if self.sim_config.brain_model.is_custom:
self._extract_brain_zip()
brain_abs_path = self.sim_config.brain_model.resource_path.abs_path
brain_rel_path = self.sim_config.brain_model.resource_path.rel_path
if not os.path.exists(brain_abs_path):
logger.info(
"Cannot find specified brain file {file} in {dir}. Searching in default "
"directories {default}".format(
file=brain_rel_path,
dir=self.sim_dir,
default=str(self.sim_config.model_paths)))
brain_abs_path = SimUtil.find_file_in_paths(
brain_rel_path, self.sim_config.model_paths)
if brain_abs_path:
self.sim_config.brain_model.resource_path.abs_path = brain_abs_path
else:
raise NRPServicesGeneralException(
"Could not find brain file: {}".format(brain_rel_path))
neurons_config = self.sim_config.get_populations_dict()
return braincontrol, braincomm, brain_abs_path, neurons_config
def _extract_brain_zip(self):
"""
Checks for validity, and extracts a zipped brain. First we
make sure that the zip referenced in the bibi exists in the
list of user brains, then we unzip it on the fly in the temporary
simulation directory. After the extraction we also make sure to copy
the .py from the experiment folder cause the user may have modified it
"""
# pylint: disable=too-many-locals
brain = Model(self.sim_config.brain_model.model, ResourceType.BRAIN)
storage_brain_zip_data = self._storageClient.get_model(
self.sim_config.token, self.sim_config.ctx_id, brain)
if storage_brain_zip_data:
# Get the data
# Write the zip in sim dir
zip_model_path = self._storageClient.get_model_path(
self.sim_config.token, self.sim_config.ctx_id, brain)
brain_abs_zip_path = os.path.join(self.sim_dir, zip_model_path)
if not os.path.exists(os.path.dirname(brain_abs_zip_path)):
os.makedirs(os.path.dirname(brain_abs_zip_path))
with open(brain_abs_zip_path, 'w') as brain_zip:
brain_zip.write(storage_brain_zip_data)
# Extract and flatten
# FixME: not sure exactly why flattening is required
ZipUtil.extractall(
zip_abs_path=self.sim_config.brain_model.zip_path.abs_path,
extract_to=self.sim_dir,
overwrite=False,
flatten=True)
# copy back the .py from the experiment folder, cause we don't want the one
# in the zip, cause the user might have made manual changes
# TODO: verify if this still required and why only one file is copied
brain_name = os.path.basename(
self.sim_config.brain_model.resource_path.rel_path)
self._storageClient.clone_file(brain_name, self.sim_config.token,
self.sim_config.experiment_id)
# if the zip is not there, prompt the user to check his uploaded models
else:
raise NRPServicesGeneralException(
"Could not find selected brain model {name} in the list of uploaded models. "
"Please make sure that it has been uploaded correctly".format(
name=self.sim_config.brain_model.model),
"Zipped model retrieval failed")
def _create_brain_adapters(self): # pragma: no cover
"""
Creates the adapter components for the neural simulator
:return: A tuple of the communication and control adapter for the neural simulator
"""
raise NotImplementedError(
"This method must be overridden in an implementation")
# pylint: disable=too-many-arguments
def __load_cle(self, roscontrol, roscomm, braincontrol, braincomm,
brain_file_path, neurons_config, externalmodulearray,
robot_poses, models, lights):
"""
Load the ClosedLoopEngine and initializes all interfaces
:param roscontrol Robot Control Adapter to use
:param roscomm Robot Communication Adapter to use
:param braincontrol Brain Control Adapter to use
:param braincomm Brain Communication Adapter to use
:param brain_file_path Accessible path to brain file
:param neurons_config Neuron configuration specified in the BIBI
:param robot_post Initial robot pose
:param models Initial models loaded into the environment
:param lights Initial lights loaded into the environment
"""
# Needed in order to cleanup global static variables
self._notify("Connecting brain simulator to robot")
tfm.start_new_tf_manager()
# Create transfer functions manager
tfmanager = tfm.config.active_node
# set adapters
tfmanager.robot_adapter = roscomm
tfmanager.brain_adapter = braincomm
# integration timestep between simulators, convert from ms to s (default to CLE value)
timestep = (ClosedLoopEngine.DEFAULT_TIMESTEP
if self.sim_config.timestep is None else
self.sim_config.timestep)
roscontrol.set_robots(self.robotManager.get_robot_dict())
# initialize CLE
self._notify("Initializing CLE")
cle = DeterministicClosedLoopEngine(roscontrol, roscomm, braincontrol,
braincomm, tfmanager,
externalmodulearray, timestep)
if brain_file_path:
cle.initialize(brain_file_path, **neurons_config)
else:
cle.initialize()
# Set initial pose
cle.initial_robot_poses = robot_poses
# Set initial models and lights
cle.initial_models = models
cle.initial_lights = lights
return cle
def __load_tfs(self):
"""
Loads and connects all transfer functions
"""
self._notify("Loading transfer functions")
for tf in self.sim_config.transfer_functions:
self._notify("Loading transfer function: {}".format(tf.name))
#tf.code = correct_indentation(tf.code, 0)
tf.code = tf.code.strip() + "\n"
logger.debug("TF: " + tf.name + "\n" + tf.code + '\n')
try:
new_code = compile_restricted(tf.code, '<string>', 'exec')
# pylint: disable=broad-except
except Exception as e:
logger.error(
"Error while compiling the transfer function {name} in restricted "
"mode with error {err}".format(name=tf.name, err=str(e)))
tfm.set_flawed_transfer_function(tf.code, tf.name, e)
continue
try:
tfm.set_transfer_function(tf.code, new_code, tf.name,
tf.active)
except tfm.TFLoadingException as loading_e:
logger.error(loading_e)
tfm.set_flawed_transfer_function(tf.code, tf.name, loading_e)
def _handle_gazebo_shutdown(self):
"""
Handles the case that Gazebo was shut down
:param close_error: Any exception happened while closing Gazebo
"""
super(CLEGazeboSimulationAssembly, self)._handle_gazebo_shutdown()
if self.cle_server is not None and self.cle_server.lifecycle is not None:
# Set the simulation to halted
self.cle_server.lifecycle.failed()
# If not already stopped, free simulation resources
self.cle_server.lifecycle.stopped()
def run(self):
"""
Runs the simulation
"""
self.cle_server.run()
def _shutdown(self, notifications):
"""
Shutdown the CLE and any hooks before shutting down Gazebo
:param notifications: A flag indicating whether notifications should be attempted to send
"""
try:
if notifications:
self.ros_notificator.update_task(
"Shutting down Closed Loop Engine",
update_progress=True,
block_ui=False)
self.robotManager.shutdown()
self.cle_server.shutdown()
# pylint: disable=broad-except
except Exception, e:
logger.error("The cle server could not be shut down")
logger.exception(e)
finally:
def put(self, experiment_id):
"""
Save a brain model PyNN of an experiment to the storage.
:param path experiment_id: The experiment id
:< json body json string data: PyNN script of the model
:< json body json string brain_populations: neuron populations
:< json body json string brain_populations: context_id of the sim
:status 500: {0}
:status 404: {1}
:status 400: The request body is malformed
:status 200: Success. File written.
"""
from hbp_nrp_backend.storage_client_api.StorageClient \
import StorageClient
body = request.get_json(force=True)
if 'data' not in body:
raise NRPServicesClientErrorException(
"Neural network python code should be sent in the body under the 'data' key"
)
context_id = body.get('context_id', None)
# no need to rewrite a get_header function since the user
# authentication already has one
# Read the request data
content_type = UserAuthentication.get_header(request, 'Content-type',
'text/plain')
data = body['data']
brain_populations = body.get('additional_populations')
# Instantiate the storage client
client = StorageClient()
# find the bibi filename from the .exc
experiment_file = client.get_file(
UserAuthentication.get_header_token(request),
experiment_id,
'experiment_configuration.exc',
byname=True)
bibi_filename = exp_conf_api_gen.CreateFromDocument(
experiment_file).bibiConf.src
# find the brain filename from the bibi
bibi_file = client.get_file(
UserAuthentication.get_header_token(request),
experiment_id,
bibi_filename,
byname=True)
bibi_file_obj = bibi_api_gen.CreateFromDocument(bibi_file)
brain_filename = bibi_file_obj.brainModel.file
if 'storage://' in brain_filename:
client.create_or_update(
UserAuthentication.get_header_token(request),
client.get_folder_uuid_by_name(
UserAuthentication.get_header_token(request), context_id,
'brains'), os.path.basename(brain_filename), data,
content_type)
else:
client.create_or_update(
UserAuthentication.get_header_token(request), experiment_id,
os.path.basename(brain_filename), data, content_type)
# remove all the populations
del bibi_file_obj.brainModel.populations[:]
if brain_populations is not None:
self.parsePopulations(brain_populations, bibi_file_obj)
# replace the bibi contents in the storage to match the new brain
# definition
client.create_or_update(
UserAuthentication.get_header_token(request), experiment_id,
bibi_filename,
xml.dom.minidom.parseString(
bibi_file_obj.toxml("utf-8")).toprettyxml(), "text/plain")
return 200
class UserAuthentication(object):
"""
Helper class to get the user, authenticated at the HBP Unified Portal, from a HTTP request.
This is done by reading the value of the 'X-User-Name' header. If this header is not present,
the user name 'default-owner' ist used to allow testing outside the portal, etc.
"""
HTTP_HEADER_USER_NAME = "X-User-Name"
HEADER_TOKEN = "Authorization"
DEFAULT_OWNER = "default-owner"
NO_TOKEN = "no_token"
client = StorageClient()
@staticmethod
def get_header(header_name, default_value):
"""
Gets the value of the headername header from the current HTTP
:param default_value: If nothing is found, this will be returned
:return: The value of the headername header or if not found default_value
"""
request_parser = reqparse.RequestParser()
request_parser.add_argument(header_name, type=str, location='headers')
header_value = request_parser.parse_args(request)[header_name]
if header_value is None:
header_value = default_value
return header_value
@staticmethod
def get_x_user_name_header():
"""
Gets the value of the 'X-User-Name' header from the current HTTP
:return: The value of the 'X-User-Name' header or if not found 'default-owner'
"""
return UserAuthentication.get_header(
UserAuthentication.HTTP_HEADER_USER_NAME,
UserAuthentication.DEFAULT_OWNER)
@staticmethod
def get_header_token():
"""
Gets the value of the 'Authorization' header from the current HTTP
:return: The value of the 'Authorization' header or if not found 'no-token'
"""
token_field = UserAuthentication.get_header(
UserAuthentication.HEADER_TOKEN, UserAuthentication.NO_TOKEN)
if (token_field != UserAuthentication.NO_TOKEN):
return token_field.split()[1]
else:
return token_field
@staticmethod
@lru_cache()
def get_token_owner(token):
"""
Gets the owner of an authentication token
:param token: The authentication token
:return: The user's id
"""
try:
user = UserAuthentication.client.get_user(token)
return user['id'] if user else None
# pylint: disable=broad-except
except Exception:
return None
@staticmethod
def get_user():
"""
Gets the owner of the current request, from the header data
It uses the x-user-name if specified, else by the authentication token
:return: The user's id
"""
username = UserAuthentication.get_x_user_name_header()
if username != UserAuthentication.DEFAULT_OWNER:
return username
token = UserAuthentication.get_header_token()
if token == UserAuthentication.NO_TOKEN:
return username
token_owner = UserAuthentication.get_token_owner(token)
return token_owner if token_owner else username
@staticmethod
@lru_cache()
def __user_can_access_experiment(token, context_id, experiment_id):
"""
Checkis if a user can access a simulation.
:param token: The authentication token
:param context_id: Optional context idenfifier
:param experiment_id: The simulation's experiment id
:return: Whether the user can access the simulation
"""
if token == UserAuthentication.NO_TOKEN:
return False
try:
return UserAuthentication.client.can_acess_experiment(
token, context_id, experiment_id)
# pylint: disable=broad-except
except Exception:
return False
@staticmethod
def can_view(simulation):
"""
Checks if the current HTTP request is authenticated by
someone the simulation was shared with
:param simulation: The simulation
:return: True if the simulation was shared with the request's user, False otherwise
"""
token = UserAuthentication.get_header_token()
return UserAuthentication.__user_can_access_experiment(
token, simulation.ctx_id, simulation.experiment_id)
@staticmethod
def can_modify(simulation):
"""
Checks if the current HTTP request is authenticated by the simulation owner
:param simulation: The simulation
:return: True if the request was triggered by a simulation owner, False otherwise
"""
user = simulation.owner
request_user = UserAuthentication.get_user()
if user == request_user:
return True
else:
logger.warn(
"Request from user '{request_user}' but simulation owned by '{user}'"
.format(request_user=request_user, user=user))
return False
def put(self, experiment_id):
"""
Save state machines to the storage
:param path experiment_id: The experiment_id id of the experiment
:param body source_code: Source code of the state machine as string.
:status 500: The experiment xml either could not be found or read
:status 200: Success. File written.
"""
# Done here in order to avoid circular dependencies introduced by the
# way we __init__ the rest_server module.
body = request.get_json(force=True)
if 'state_machines' not in body:
raise NRPServicesClientErrorException(
"State machine code should be sent in "
"the body under the 'state_machines' key")
from hbp_nrp_backend.storage_client_api.StorageClient \
import StorageClient
client = StorageClient()
exp_xml_file_path = client.clone_file(
'experiment_configuration.exc',
UserAuthentication.get_header_token(request), experiment_id)
if not exp_xml_file_path:
return {
"message": "Failed to clone experiment configuration file"
}, 500
experiment = client.parse_and_check_file_is_valid(
exp_xml_file_path, exp_conf_api_gen.CreateFromDocument,
exp_conf_api_gen.ExD_)
if not experiment:
return {
"message": "Failed to parse experiment configuration file"
}, 500
threads = []
for sm_name in body['state_machines']:
sm_node = exp_conf_api_gen.SMACHStateMachine()
sm_node.id = os.path.splitext(sm_name)[0]
sm_node.src = sm_name if sm_name.endswith(
".exd") else sm_name + ".exd"
exp_control = exp_conf_api_gen.ExperimentControl()
exp_control.stateMachine.append(sm_node)
experiment.experimentControl = exp_control
t = Thread(target=client.create_or_update,
kwargs={
'token':
UserAuthentication.get_header_token(request),
'experiment':
experiment_id,
'filename':
sm_node.src,
'content':
body['state_machines'][sm_name],
'content_type':
"application/hbp-neurorobotics.sm+python"
})
t.start()
threads.append(t)
t = Thread(target=client.create_or_update,
kwargs={
'token': UserAuthentication.get_header_token(request),
'experiment': experiment_id,
'filename': 'experiment_configuration.exc',
'content': experiment.toxml("utf-8"),
'content_type': "application/hbp-neurorobotics+xml"
})
t.start()
threads.append(t)
for thread in threads:
thread.join()
return {"message": "Success. Files written to the storage"}, 200