def _hack_(self):
"""
TODO: please remove me at earliest convenience
HACK for the inconsistent proxy and backend behavior
Replace exc and bibi representation by reading from storage
Proxy might have changed something that backend is not aware of
"""
from hbp_nrp_backend.storage_client_api.StorageClient import StorageClient
import urllib
client = StorageClient()
latest_exc = client.get_file(self._sim_config.token,
urllib.quote_plus(
self._sim_config.experiment_id),
self._sim_config.exc_path.rel_path,
by_name=True)
latest_bibi = client.get_file(self._sim_config.token,
urllib.quote_plus(
self._sim_config.experiment_id),
self._sim_config.bibi_path.rel_path,
by_name=True)
try:
self._exc_dom = exc_parser.CreateFromDocument(latest_exc)
self._bibi_dom = bibi_parser.CreateFromDocument(latest_bibi)
except Exception as ex:
raise Exception(
"Something went horribly wrong while creating latest "
"config objects with following exception {}".format(str(ex)))
def test_reset_is_called_properly(self, mock_lifecycle, mock_storage_client,
mock_get_experiment_data, mock_copy_tree,
mock_get_brain_info_from_storage):
simulations[0].cle = mock.MagicMock()
simulations[0].cle.set_simulation_transfer_function.return_value = None
mock_get_brain_info_from_storage.return_value = os.path.join(PATH,
'models/braitenberg.py'), {}, {}
experiment_file_path = os.path.join(PATH, 'experiments/experiment_data/test_1.exc')
with open(experiment_file_path) as exd_file:
try:
experiment = exp_conf_api_gen.CreateFromDocument(
exd_file.read())
except ValidationError, ve:
raise Exception("Could not parse experiment configuration {0:s} due to validation "
"error: {1:s}".format(experiment_file_path, str(ve)))
bibi_file = experiment.bibiConf.src
bibi_file_abs = os.path.join(EXPERIMENT_DATA_PATH, bibi_file)
with open(bibi_file_abs) as b_file:
try:
bibi = bibi_api_gen.CreateFromDocument(b_file.read())
except ValidationError, ve:
raise Exception("Could not parse brain configuration {0:s} due to validation "
"error: {1:s}".format(bibi_file_abs, str(ve)))
def test_experiment_world_sdf_put(self, mocked_get_sim_robots,
mocked_rospy, path_get_sim):
sim = MockedSimulation()
sim.experiment_id = '123456'
path_get_sim.return_value = sim
mocked_rospy.wait_for_service = MagicMock(return_value=None)
mocked_rospy.ServiceProxy = MagicMock(return_value=MockServiceResponse)
exd_conf_original_path = os.path.join(self.test_directory,
"experiments", "experiment_data",
"test_1.exc")
exd_conf_temp_path = os.path.join(self.temp_directory, "bibi_test.xml")
shutil.copyfile(exd_conf_original_path, exd_conf_temp_path)
with open(exd_conf_temp_path) as exp_xml:
exp = exp_conf_api_gen.CreateFromDocument(exp_xml.read())
def empty(a, b, c, d, e):
return
self.mock_storageClient_instance.clone_file.return_value = exd_conf_temp_path
self.mock_storageClient_instance.create_or_update = empty
self.mock_storageClient_instance.parse_and_check_file_is_valid.return_value = exp
response = self.client.post('/simulation/0/sdf_world')
self.assertEqual(response.status_code, 200)
def _get_brain_info_from_storage(cls, experiment_id, context_id):
"""
Gathers from the storage the brain script and the populations by getting the BIBI
configuration file.
:param experiment_id: the id of the experiment in which to look for the brain information
:param context_id: the context ID for collab based simulations
:return: A tuple with the path to the brain file and a list of populations
"""
del context_id # Unused
request_token = UserAuthentication.get_header_token()
experiment_file = cls.storage_client.get_file(
request_token, experiment_id, 'experiment_configuration.exc', by_name=True)
bibi_filename = exp_conf_api_gen.CreateFromDocument(experiment_file).bibiConf.src
# find the brain filename from the bibi
bibi_file = cls.storage_client.get_file(
request_token, experiment_id, bibi_filename, by_name=True)
bibi_file_obj = bibi_api_gen.CreateFromDocument(bibi_file)
brain_filename = os.path.basename(bibi_file_obj.brainModel.file)
brain_filepath = cls.storage_client.clone_file(brain_filename, request_token, experiment_id)
neurons_config = get_all_neurons_as_dict(bibi_file_obj.brainModel.populations)
neurons_config_clean = \
[SimulationResetStorage._get_experiment_population(name, v)
for (name, v) in neurons_config.iteritems()]
return brain_filepath, neurons_config_clean, neurons_config
def reset_from_storage_all(cls, simulation, experiment_id, context_id):
"""
Reset states machines and transfer functions
:param: the simulation id
:param: the experiment id
:param: the context_id for collab based simulations
"""
SimUtil.clear_dir(simulation.lifecycle.sim_dir)
token = UserAuthentication.get_header_token()
cls.storage_client.clone_all_experiment_files(
token, experiment_id, destination_dir=simulation.lifecycle.sim_dir)
with open(simulation.lifecycle.experiment_path) as exc_file:
exc = exp_conf_api_gen.CreateFromDocument(exc_file.read())
bibi_path = os.path.join(os.path.dirname(simulation.lifecycle.experiment_path),
exc.bibiConf.src)
with open(bibi_path) as bibi_file:
bibi = bibi_api_gen.CreateFromDocument(bibi_file.read())
cls.reset_brain(simulation, experiment_id, context_id)
cls.reset_transfer_functions(simulation, bibi, simulation.lifecycle.sim_dir)
cls.reset_state_machines(simulation, exc, simulation.lifecycle.sim_dir)
def test_brain_reset_from_storage(self, mock_get_exp_data,
mock_get_model_path,
mock_get_experiment_path):
simulations.append(
Simulation(3, 'experiments/experiment_data/test_5.exc', None,
'default-owner', 'created'))
simulations[2].cle = mock.MagicMock()
simulations[2].cle.set_simulation_brain.return_value = Mock(
error_message="")
mock_get_experiment_path.return_value = PATH
mock_get_model_path.return_value = os.path.join(PATH, 'models')
bibi_path = os.path.join(PATH,
'experiments/experiment_data/bibi_4.bibi')
experiment_file_path = os.path.join(
PATH, 'experiments/experiment_data/test_5.exc')
with open(experiment_file_path) as exd_file:
experiment = exp_conf_api_gen.CreateFromDocument(exd_file.read())
with open(bibi_path) as b_file:
bibi = bibi_api_gen.CreateFromDocument(b_file.read())
mock_get_exp_data.return_value = experiment, bibi
with patch("__builtin__.open", mock_open(read_data="data")) as mock_file, \
patch('os.listdir', return_value=['nrpTemp']) as mock_tempfile:
response = self.client.put('/simulation/2/reset',
data=json.dumps({
'resetType':
ResetSimulationRequest.RESET_BRAIN
}))
self.assertEqual(200, response.status_code)
def setUp(self):
dir = os.path.split(__file__)[0]
with open(os.path.join(
dir, "experiment_data/milestone2.bibi")) as bibi_file:
bibi = bibi_api_gen.CreateFromDocument(bibi_file.read())
with open(os.path.join(
dir, "experiment_data/ExDXMLExample.exc")) as exd_file:
exd = exp_conf_api_gen.CreateFromDocument(exd_file.read())
exd.path = "/somewhere/over/the/rainbow/exc"
exd.dir = "/somewhere/over/the/rainbow"
bibi.path = "/somewhere/over/the/rainbow/bibi"
models_path_patch = patch(
"hbp_nrp_cleserver.server.CLEGazeboSimulationAssembly.models_path",
"/somewhere/near/the/rainbow")
models_path_patch.start()
self.addCleanup(models_path_patch.stop)
with patch("hbp_nrp_cleserver.server.CLEGazeboSimulationAssembly.os",
MockOs):
self.launcher = CLEGazeboSimulationAssembly.CLEGazeboSimulationAssembly(
42, exd, bibi, gzserver_host="local")
self.launcher.cle_server = Mock()
self.launcher.gzweb = Mock()
self.launcher.gzserver = Mock()
self.launcher.gazebo_recorder = Mock()
self.launcher.ros_notificator = Mock()
self.launcher.ros_launcher = Mock()
def get(self, sim_id):
"""
Gets simulation resource files of the experiment running for the simulation ID
:param sim_id: The simulation ID
:> json string resources: Resource files
:status 500: {0}
:status 404: {1}. Or, {2}.
:status 200: Success. The simulation BIBI configuration files were retrieved
"""
simulation = _get_simulation_or_abort(sim_id)
experiment_file = simulation.lifecycle.experiment_path
if not os.path.isfile(experiment_file):
raise NRPServicesClientErrorException(
ErrorMessages.EXPERIMENT_CONF_FILE_NOT_FOUND_404,
error_code=404)
with open(experiment_file) as exd_file:
experiment_dom = exp_conf_api_gen.CreateFromDocument(
exd_file.read())
bibi_fullpath = os.path.join(
simulation.lifecycle.simulation_root_folder,
experiment_dom.bibiConf.src)
if not os.path.isfile(bibi_fullpath):
raise NRPServicesClientErrorException(
ErrorMessages.EXPERIMENT_BIBI_FILE_NOT_FOUND_404,
error_code=404)
resources = []
for conf in experiment_dom.configuration:
resources.append({'file': conf.src, 'type': conf.type})
with open(bibi_fullpath) as _file:
bibi_dom = bibi_api_gen.CreateFromDocument(_file.read())
for conf in bibi_dom.configuration:
resources.append({'file': conf.src, 'type': conf.type})
if simulation.private:
for conf in resources:
conf['file'] = os.path.join(
'/config-from-cloned-folder',
os.path.basename(
simulation.lifecycle.simulation_root_folder),
os.path.basename(conf['file']))
else:
# Get the template experiment folder name, e.g., "template_husky"
experiment_folder = os.path.basename(
os.path.dirname(experiment_file))
for conf in resources:
conf['file'] = os.path.join('/config-from-template-folder',
experiment_folder, conf['file'])
return {'resources': resources}, 200
def test_parse_env_path_template(self):
exp_path = os.path.join(PATH, self.simulation.experiment_conf)
with open(exp_path, 'r') as exp_file:
exp = exp_conf_api_gen.CreateFromDocument(exp_file.read())
env_path = self.lifecycle._parse_env_path(exp.environmentModel.src,
exp, False)
self.assertEqual(env_path,
os.path.join(PATH, 'virtual_room/virtual_room.sdf'))
def setUp(self):
dir = os.path.split(__file__)[0]
with open(os.path.join(dir, "experiment_data/milestone2.bibi")) as bibi_file:
bibi = bibi_api_gen.CreateFromDocument(bibi_file.read())
with open(os.path.join(dir, "experiment_data/ExDXMLExample.exc")) as exd_file:
exd = exp_conf_api_gen.CreateFromDocument(exd_file.read())
exd.path = "/somewhere/over/the/rainbow/exc"
exd.dir = "/somewhere/over/the/rainbow"
bibi.path = "/somewhere/over/the/rainbow/bibi"
self.models_path_patch=patch("hbp_nrp_cleserver.server.CLEGazeboSimulationAssembly.models_path", new="/somewhere/near/the/rainbow")
self.models_path_patch.start()
self.launcher = SynchronousNestSimulation(42, exd, bibi, gzserver_host="local")
def test_parse_env_path_template_storage(self, mock_storage):
exp_path = os.path.join(PATH, 'ExDXMLExample_2.xml')
with open(exp_path, 'r') as exp_file:
exp = exp_conf_api_gen.CreateFromDocument(exp_file.read())
mock_storage().get_temp_directory.return_value = PATH
mock_storage().get_file.return_value = '<sdf></sdf>'
mock_storage().get_folder_uuid_by_name.return_value = 'environments'
with patch(
"hbp_nrp_backend.simulation_control.__BackendSimulationLifecycle.UserAuthentication.get_header_token"
):
env_path = self.lifecycle._parse_env_path(None, exp, False)
self.assertEqual(env_path, os.path.join(PATH, 'virtual_room.sdf'))
def get_experiment_data(experiment_file_path):
"""
Parse experiment and bibi and return the objects
@param experiment_file: experiment file
@return experiment, bibi, parsed experiment and bibi
"""
with open(experiment_file_path) as exd_file:
try:
experiment = exp_conf_api_gen.CreateFromDocument(exd_file.read())
experiment.path = experiment_file_path
except ValidationError, ve:
raise Exception("Could not parse experiment configuration {0:s} due to validation "
"error: {1:s}".format(experiment_file_path, str(ve)))
def test_storage_experiment_get_ok(self, storage_mock, mock_bp0):
client_mock = MagicMock()
exp_temp_path = os.path.join(os.path.split(__file__)[0], "experiments", "experiment_data", "test_1.exc")
with open(exp_temp_path) as exp_xml:
exp = exp_conf_api_gen.CreateFromDocument(exp_xml.read())
client_mock.clone_file.return_value = exp_temp_path
client_mock.parse_and_check_file_is_valid.return_value = exp
storage_mock.return_value = client_mock
mock_bp0.return_value = PATH
response = self.client.get('/experiment/experiment_id')
assert(isinstance(response, Response))
self.assertEqual(response.status_code, 200)
self.assertNotEqual(response.get_data(), {})
def test_parse_env_path_custom_environment_throws(self, mock_storage):
with patch(
"hbp_nrp_backend.simulation_control.__BackendSimulationLifecycle.UserAuthentication.get_header_token"
) as user_auth:
mock_storage().get_custom_models.return_value = [{
'path': 'fakePath'
}]
exp_path = os.path.join(PATH, 'ExDXMLExampleZipped.exc')
with open(exp_path, 'r') as exp_file:
exp = exp_conf_api_gen.CreateFromDocument(exp_file.read())
with self.assertRaises(NRPServicesGeneralException) as context:
self.lifecycle._parse_env_path(exp.environmentModel.src, exp,
True)
self.assertEqual(NRPServicesGeneralException, context.expected)
def get(self, exp_id):
"""
Get preview image of the experiment specified with experiment ID.
:param exp_id: The experiment ID
:> json image_as_base64: The PNG image as base64
:status 500: {0}
:status 404: {1} or {2}
:status 200: Success. The preview image is returned
"""
# Check experiment
experiment_file_path = os.path.join(get_experiment_basepath(),
get_experiment_rel(exp_id))
if not os.path.isfile(experiment_file_path):
raise NRPServicesClientErrorException(
ErrorMessages.EXPERIMENT_CONF_FILE_NOT_FOUND_404,
error_code=404)
experiment_dir = os.path.split(experiment_file_path)[0]
# Parse the experiment XML and get the thumbnail path
with open(experiment_file_path) as exd_file:
try:
experiment_file = exp_conf_api_gen.CreateFromDocument(
exd_file.read())
preview_file = os.path.join(experiment_dir,
experiment_file.thumbnail)
except ValidationError:
raise NRPServicesClientErrorException(
ErrorMessages.EXPERIMENT_CONF_FILE_INVALID_500,
error_code=500)
# Check thumbnail
if not os.path.isfile(preview_file):
raise NRPServicesClientErrorException(
ErrorMessages.EXPERIMENT_PREVIEW_NOT_FOUND_404, error_code=404)
mime_type = mimetypes.guess_type(preview_file)[
0] # returns tuple (type, encoding)
if not mime_type or not mime_type.startswith('image'):
raise NRPServicesClientErrorException(
ErrorMessages.EXPERIMENT_PREVIEW_INVALID_500, error_code=500)
with open(preview_file, "rb") as _file:
data = _file.read()
# Base64
return dict(image_as_base64=base64.b64encode(data))
def _read_exc_and_bibi_dom_objects(self):
"""
Parse experiment and bibi and return the DOM objects
"""
# Read exc
with open(self._exc_path.abs_path) as excFile:
try:
self._exc_dom = exc_parser.CreateFromDocument(excFile.read())
except ValidationError as ve:
raise Exception(
"Could not parse experiment config {0} due to validation "
"error: {1}".format(self._exc_path.abs_path, str(ve)))
self._bibi_path = ResourcePath(self._exc_dom.bibiConf.src,
self.sim_dir)
logger.info("Bibi absolute path:" + self._bibi_path.abs_path)
# Read bibi
with open(self._bibi_path.abs_path) as bibiFile:
try:
self._bibi_dom = bibi_parser.CreateFromDocument(
bibiFile.read())
except ValidationError as ve:
raise Exception(
"Could not parse brain configuration {0:s} due to validation "
"error: {1:s}".format(self._bibi_path.abs_path, str(ve)))
except NamespaceError as ne:
# first check to see if the BIBI file appears to have a valid namespace
namespace = str(ne).split(" ", 1)[0]
if not namespace.startswith("http://schemas.humanbrainproject.eu/SP10") or \
not namespace.endswith("BIBI"):
raise Exception(
"Unknown brain configuration file format for: {0:s} with "
"namespace: {1:s}".format(self._bibi_path.abs_path,
namespace))
# notify the user that their file is out of date
raise Exception(
"The BIBI file for the requested experiment is out of date and no "
"longer supported. Please contact neurorobotics@humanbrainproject."
"eu with the following information for assistance in updating this"
" file.\n\nExperiment Configuration:\n\tName: {0:s}\n\tBIBI: {1:s}"
"\n\tVersion: {2:s}".format(self._exc_dom.name,
self._bibi_path.abs_path,
namespace))
# set config version based of something
self.exc_version = Version.CURRENT
def test_invalid_simulation(self):
dir = os.path.split(__file__)[0]
with open(os.path.join(dir, "experiment_data/milestone2.bibi")) as bibi_file:
bibi = bibi_api_gen.CreateFromDocument(bibi_file.read())
with open(os.path.join(dir, "experiment_data/ExDXMLExample.exc")) as exd_file:
exd = exp_conf_api_gen.CreateFromDocument(exd_file.read())
exd.path = "/somewhere/over/the/rainbow/exc"
exd.dir = "/somewhere/over/the/rainbow"
exd.physicsEngine = None
bibi.path = "/somewhere/over/the/rainbow/bibi"
models_path_patch = patch("hbp_nrp_cleserver.server.CLEGazeboSimulationAssembly.models_path",
new=None)
models_path_patch.start()
with self.assertRaises(Exception):
SynchronousNestSimulation(42, exd, bibi, gzserver_host="local")
models_path_patch.stop()
with self.assertRaises(Exception):
SynchronousNestSimulation(42, exd, bibi, gzserver_host="bullshit")
def clone_all_experiment_files(self, token, experiment, new_folder=False):
"""
Clones all the experiment files to a temporary folder.
The caller has then the responsibility of managing this folder.
:param token: The token of the request
:param experiment: The experiment to clone
:param new_folder: specifies whether we want to clone the files again
or use the existing temporary nrpTemp folder. Used in reset
:return: A dictionary containing the paths to the experiment files
as well as the path to the temporary folder
"""
experiment_paths = dict()
list_files_to_clone = self.list_files(token, experiment)
if new_folder:
destination_directory = tempfile.mkdtemp()
else:
destination_directory = self.get_temp_directory()
for file_under_experiment in list_files_to_clone:
file_clone_destination = os.path.join(
destination_directory, file_under_experiment['name'])
with open(file_clone_destination, "w") as f:
zipped = os.path.splitext(
file_under_experiment['name'])[1].lower() == '.zip'
file_contents = self.get_file(token,
experiment,
file_under_experiment['name'],
byname=True,
zipped=zipped)
# in order to return the environment and experiment path we have
# to read the .exc
if 'experiment_configuration.exc' in str(
file_clone_destination):
experiment_paths[
'experiment_conf'] = file_clone_destination
env_filename = exp_conf_api_gen.CreateFromDocument(
file_contents).environmentModel.src
experiment_paths['environment_conf'] = os.path.join(
destination_directory, env_filename)
f.write(file_contents)
return destination_directory, experiment_paths
def _get_sdf_world_from_storage(cls, experiment_id, context_id):
"""
Download from the storage an sdf world file as a string.
The file belongs to the experiment identified by experiment_id
:param experiment_id: the ID of the experiment in which to look for the world sdf
:param context_id: the context ID for collab based simulations
:return: The content of the world sdf file
"""
del context_id # Unused
request_token = UserAuthentication.get_header_token()
# find the sdf filename from the .exc
experiment_file = cls.storage_client.get_file(
request_token, experiment_id, 'experiment_configuration.exc', by_name=True)
world_file_name = exp_conf_api_gen.CreateFromDocument(experiment_file).environmentModel.src
return cls.storage_client.get_file(
request_token, experiment_id, world_file_name, by_name=True)
def test_experiment_storage_state_machines_put_sm_ok(
self, storage_mock, mock_bp0):
client_mock = MagicMock()
temp_directory = tempfile.mkdtemp()
exp_temp_path = os.path.join(temp_directory, "exp_test.xml")
exp_remote_path = os.path.join("collab_path", "exp_test.xml")
shutil.copyfile(
os.path.join(
os.path.split(__file__)[0], "experiments", "experiment_data",
"test_1.exc"), exp_temp_path)
with open(exp_temp_path) as exp_xml:
exp = exp_conf_api_gen.CreateFromDocument(exp_xml.read())
client_mock.clone_file.return_value = exp_temp_path
storage_mock.return_value = client_mock
mock_bp0.return_value = PATH
sm = "def test_sm():\n"\
" pass"
data = {"state_machines": {"test_sm": sm}}
response = self.client.put('/experiment/context_id/state-machines',
data=json.dumps(data))
self.assertEqual(response.status_code, 200)
def test_parse_env_path_custom_environment_ok(self, mock_zip,
mock_storage):
with patch(
"hbp_nrp_backend.simulation_control.__BackendSimulationLifecycle.UserAuthentication.get_header_token"
) as user_auth:
mock_storage().get_custom_models.return_value = [{
'path':
'virtual_room.zip'
}]
mock_storage().get_custom_model.return_value = 'test'
mock_storage().get_temp_directory.return_value = os.path.join(
os.path.dirname(os.path.realpath(__file__)), 'zipped_data')
import zipfile
mock_zip.ZipFile.return_value = zipfile.ZipFile(
os.path.join(os.path.dirname(os.path.realpath(__file__)),
'zipped_data', 'mouse_ymaze_world.zip'), 'r')
exp_path = os.path.join(PATH, 'ExDXMLExampleZipped.exc')
with open(exp_path, 'r') as exp_file:
exp = exp_conf_api_gen.CreateFromDocument(exp_file.read())
self.lifecycle._parse_env_path(exp.environmentModel.src, exp, True)
self.assertEqual(exp.environmentModel.src, 'virtual_room.sdf')
self.assertEqual(exp.environmentModel.customModelPath,
'virtual_room.zip')
def test_experiment_world_sdf_put(self, mocked_rospy):
experiment_id = '123456'
mocked_rospy.wait_for_service = MagicMock(return_value=None)
mocked_rospy.ServiceProxy = MagicMock(return_value=MockServiceResponse)
exd_conf_original_path = os.path.join(self.test_directory,
"experiments", "experiment_data",
"test_1.exc")
exd_conf_temp_path = os.path.join(self.temp_directory, "bibi_test.xml")
shutil.copyfile(exd_conf_original_path, exd_conf_temp_path)
with open(exd_conf_temp_path) as exp_xml:
exp = exp_conf_api_gen.CreateFromDocument(exp_xml.read())
def empty(a, b, c, d, e):
return
body = {'context_id': 'fake'}
self.mock_storageClient_instance.clone_file.return_value = exd_conf_temp_path
self.mock_storageClient_instance.create_or_update = empty
response = self.client.post('/experiment/' + experiment_id +
'/sdf_world',
data=json.dumps(body))
self.assertEqual(response.status_code, 200)
def _parse_exp_and_initialize_paths(self, experiment_path,
environment_path, using_storage):
"""
Parses the experiment configuration, loads state machines and updates the environment path
with the one in the configuration file if none is passed as an argument, for supporting
custom environments.
:param experiment_path: Path the experiment configuration.
:param environment_path: Path to the environment configuration.
:param using_storage: Private or template simulation
"""
# parse experiment
with open(experiment_path) as exd_file:
experiment = exp_conf_api_gen.CreateFromDocument(exd_file.read())
state_machine_paths = {}
if experiment.experimentControl is not None and not self.simulation.playback_path:
state_machine_paths.update({
sm.id: os.path.join(self.__simulation_root_folder, sm.src)
for sm in experiment.experimentControl.stateMachine
if isinstance(sm, exp_conf_api_gen.SMACHStateMachine)
})
if experiment.experimentEvaluation is not None and not self.simulation.playback_path:
state_machine_paths.update({
sm.id: os.path.join(self.__simulation_root_folder, sm.src)
for sm in experiment.experimentEvaluation.stateMachine
if isinstance(sm, exp_conf_api_gen.SMACHStateMachine)
})
self.simulation.state_machine_manager.add_all(state_machine_paths,
self.simulation.sim_id)
self.simulation.state_machine_manager.initialize_all()
logger.info("Requesting simulation resources")
return experiment, self._parse_env_path(environment_path, experiment,
using_storage)
def test_reset_from_storage_all(self,
mock_get_header_token,
mock_copy_tree,
mock_get_experiment_data,
mock_resetBrain,
mock_resetTFs,
mock_resetSMs,
mock_dirname):
self.mock_storageClient_instance.clone_all_experiment_files.return_value = '_', {
'experiment_conf': 'fakeFolder/fakeExp'}
mock_resetBrain.return_value = None
mock_resetTFs.return_value = None
mock_resetSMs.return_value = None
mock_dirname.return_value = os.path.join(PATH, 'experiments/experiment_data')
simulations[0].cle = mock.MagicMock()
simulations[0].cle.set_simulation_transfer_function.return_value = None
experiment_file_path = os.path.join(PATH, 'experiments/experiment_data/test_5.exc')
with open(experiment_file_path) as exd_file:
try:
experiment = exp_conf_api_gen.CreateFromDocument(
exd_file.read())
except ValidationError, ve:
raise Exception("Could not parse experiment configuration {0:s} due to validation "
"error: {1:s}".format(experiment_file_path, str(ve)))
bibi_file = experiment.bibiConf.src
bibi_file_abs = os.path.join(EXPERIMENT_DATA_PATH, bibi_file)
with open(bibi_file_abs) as b_file:
try:
bibi = bibi_api_gen.CreateFromDocument(b_file.read())
except ValidationError, ve:
raise Exception("Could not parse brain configuration {0:s} due to validation "
"error: {1:s}".format(bibi_file_abs, str(ve)))
def get_fake_experiment_file(token, headers):
with open(os.path.join(self.experiments_directory, "experiment_configuration.exc")) as exd_file:
exp_file_contents = exd_file.read()
return MockResponse(None, 200, exp_conf_api_gen.CreateFromDocument(exp_file_contents))
def initialize(self, state_change):
"""
Initializes the simulation
:param state_change: The state change that caused the simulation to be initialized
"""
simulation = self.simulation
if not simulation.playback_path:
self._sim_dir = SimUtil.init_simulation_dir()
try:
if not simulation.private:
raise NRPServicesGeneralException(
"Only private experiments are supported", "CLE error", 500)
self.__storageClient.clone_all_experiment_files(
token=UserAuthentication.get_header_token(),
experiment=simulation.experiment_id,
destination_dir=self._sim_dir,
exclude=['recordings/']
if not simulation.playback_path else [])
# divine knowledge about the exc name
self.__experiment_path = os.path.join(
self._sim_dir, 'experiment_configuration.exc')
with open(self.__experiment_path) as exd_file:
exc = exp_conf_api_gen.CreateFromDocument(exd_file.read())
self._load_state_machines(exc)
if exc.environmentModel.model: # i.e., custom zipped environment
self._prepare_custom_environment(exc)
simulation.timeout_type = (
TimeoutType.SIMULATION if exc.timeout.time
== TimeoutType.SIMULATION else TimeoutType.REAL)
timeout = exc.timeout.value()
if simulation.timeout_type == TimeoutType.REAL:
timeout = datetime.datetime.now(timezone) + datetime.timedelta(
seconds=timeout)
simulation.kill_datetime = timeout
else:
simulation.kill_datetime = None
logger.info("simulation timeout initialized")
simulation_factory_client = ROSCLESimulationFactoryClient()
simulation_factory_client.create_new_simulation(
self.__experiment_path, simulation.gzserver_host,
simulation.reservation, simulation.brain_processes,
simulation.sim_id, str(timeout),
simulation.timeout_type, simulation.playback_path,
UserAuthentication.get_header_token(), self.simulation.ctx_id,
self.simulation.experiment_id)
if not simulation.playback_path:
simulation.cle = ROSCLEClient(simulation.sim_id)
else:
simulation.cle = PlaybackClient(simulation.sim_id)
logger.info("simulation initialized")
except IOError as e:
raise NRPServicesGeneralException(
"Error while accessing simulation models (" + repr(e.message) +
")", "Models error")
except rospy.ROSException as e:
raise NRPServicesGeneralException(
"Error while communicating with the CLE (" + repr(e.message) +
")", "CLE error")
except rospy.ServiceException as e:
raise NRPServicesGeneralException(
"Error starting the simulation. (" + repr(e.message) + ")",
"rospy.ServiceException",
data=e.message)
# pylint: disable=broad-except
except Exception as e:
raise NRPServicesGeneralException(
"Error starting the simulation. (" + repr(e) + ")",
"Unknown exception occured",
data=e.message)
def put(self, experiment_id):
"""
Save transfer functions of an experiment to the storage.
:param path experiment_id: The experiment_id of the experiment where the transfer functions
will be saved
:<json body json array of string transfer_functions: the transfer functions as python
:status 500: BIBI configuration file not found
:status 500: Error saving file
:status 404: The experiment_id with the given expreiment ID was not found
:status 404: The request body is malformed
:status 200: Success. File written.
"""
# pylint: disable=too-many-locals
# Done here in order to avoid circular dependencies introduced by the
# way we __init__ the rest_server module
from hbp_nrp_backend.storage_client_api.StorageClient \
import StorageClient
body = request.get_json(force=True)
if 'transfer_functions' not in body:
raise NRPServicesClientErrorException(
"Transfer functions code should be sent in "
"the body under the 'transfer_functions' key")
client = StorageClient()
experiment_file = client.get_file(
UserAuthentication.get_header_token(request),
experiment_id,
'experiment_configuration.exc',
byname=True)
bibi_filename = exp_conf_api_gen.CreateFromDocument(
experiment_file).bibiConf.src
bibi_file_path = client.clone_file(
bibi_filename, UserAuthentication.get_header_token(request),
experiment_id)
bibi = client.parse_and_check_file_is_valid(
bibi_file_path, bibi_api_gen.CreateFromDocument,
bibi_api_gen.BIBIConfiguration)
# Remove all transfer functions from BIBI. Then we save them in a
# separate python file.
del bibi.transferFunction[:]
threads = []
for transfer_function in body['transfer_functions']:
transfer_function_name = get_tf_name(transfer_function)
if transfer_function_name is not None:
transfer_function_node = bibi_api_gen.PythonTransferFunction()
transfer_function_node.src = transfer_function_name + ".py"
bibi.transferFunction.append(transfer_function_node)
t = Thread(target=client.create_or_update,
kwargs={
'token':
UserAuthentication.get_header_token(request),
'experiment':
experiment_id,
'filename':
transfer_function_name + ".py",
'content':
transfer_function,
'content_type':
'text/plain'
})
t.start()
threads.append(t)
# we need to prettify the parsed bibi
pretty_bibi = xml.dom.minidom.parseString(
bibi.toxml("utf-8")).toprettyxml()
t = Thread(target=client.create_or_update,
kwargs={
'token': UserAuthentication.get_header_token(request),
'experiment': experiment_id,
'filename': bibi_filename,
'content': pretty_bibi,
'content_type': 'text/plain'
})
t.start()
threads.append(t)
for x in threads:
x.join()
return 200
def put(self, experiment_id):
"""
Save a brain model PyNN of an experiment to the storage.
:param path experiment_id: The experiment id
:< json body json string data: PyNN script of the model
:< json body json string brain_populations: neuron populations
:< json body json string brain_populations: context_id of the sim
:status 500: {0}
:status 404: {1}
:status 400: The request body is malformed
:status 200: Success. File written.
"""
from hbp_nrp_backend.storage_client_api.StorageClient \
import StorageClient
body = request.get_json(force=True)
if 'data' not in body:
raise NRPServicesClientErrorException(
"Neural network python code should be sent in the body under the 'data' key"
)
context_id = body.get('context_id', None)
# no need to rewrite a get_header function since the user
# authentication already has one
# Read the request data
content_type = UserAuthentication.get_header(request, 'Content-type',
'text/plain')
data = body['data']
brain_populations = body.get('additional_populations')
# Instantiate the storage client
client = StorageClient()
# find the bibi filename from the .exc
experiment_file = client.get_file(
UserAuthentication.get_header_token(request),
experiment_id,
'experiment_configuration.exc',
byname=True)
bibi_filename = exp_conf_api_gen.CreateFromDocument(
experiment_file).bibiConf.src
# find the brain filename from the bibi
bibi_file = client.get_file(
UserAuthentication.get_header_token(request),
experiment_id,
bibi_filename,
byname=True)
bibi_file_obj = bibi_api_gen.CreateFromDocument(bibi_file)
brain_filename = bibi_file_obj.brainModel.file
if 'storage://' in brain_filename:
client.create_or_update(
UserAuthentication.get_header_token(request),
client.get_folder_uuid_by_name(
UserAuthentication.get_header_token(request), context_id,
'brains'), os.path.basename(brain_filename), data,
content_type)
else:
client.create_or_update(
UserAuthentication.get_header_token(request), experiment_id,
os.path.basename(brain_filename), data, content_type)
# remove all the populations
del bibi_file_obj.brainModel.populations[:]
if brain_populations is not None:
self.parsePopulations(brain_populations, bibi_file_obj)
# replace the bibi contents in the storage to match the new brain
# definition
client.create_or_update(
UserAuthentication.get_header_token(request), experiment_id,
bibi_filename,
xml.dom.minidom.parseString(
bibi_file_obj.toxml("utf-8")).toprettyxml(), "text/plain")
return 200
