def delete_simulation_dir(): """ Removes simulation directory """ try: if os.path.exists(Settings.sim_dir_symlink): shutil.rmtree(os.path.realpath(Settings.sim_dir_symlink)) os.unlink(Settings.sim_dir_symlink) except IOError as err: raise NRPServicesGeneralException( "Could not create symlink to temp simulation folder. {err}".format(err=err)) except OSError as ex: raise NRPServicesGeneralException( "Could not create symlink to temp simulation folder. {err}".format(err=ex))
def delete(self, sim_id, state_machine_name): """ Delete a state machine :param sim_id: The simulation ID :param state_machine_name: The name of the state machine to be deleted :status 500: {0} :status 404: {1} :status 401: {2} :status 200: The delete operation was successfully called. This does not imply that the state machine function was correctly deleted though. """ simulation = _get_simulation_or_abort(sim_id) if not UserAuthentication.can_modify(simulation): raise NRPServicesWrongUserException() failure_message = "State machine destruction failed: " try: ok, response_message = simulation.delete_state_machine( state_machine_name) if ok: return "Success. The state machine was successfully deleted.", 200 except Exception as e: info = exc_info() raise NRPServicesGeneralException( failure_message + " {0}: {1}".format(e.__class__.__name__, e.message), "State machine error", ), None, info[2] raise NRPServicesStateMachineException( failure_message + "\n" + response_message, 404)
def get(self, sim_id): """ Gets the list of robots in the running simulation. """ # pylint: disable=no-self-use sim = _get_simulation_or_abort(sim_id) if not UserAuthentication.can_view(sim): raise NRPServicesWrongUserException() try: robots = SimulationRobots.__get_simulation_robots(sim) except ROSCLEClientException as e: raise NRPServicesGeneralException(str(e), 'CLE error', 500) result = { 'robots': [{ ParamNames.ROBOT_ID: robot.robot_id, ParamNames.ROBOT_ABS_PATH: robot.robot_model, ParamNames.IS_CUSTOM: robot.is_custom, ParamNames.ROBOT_POSE: SimulationRobots.__jsonisePose(robot.pose) } for robot in robots] } return result, 200
def delete(self, sim_id, robot_id): """ Delete a robot from the simulation. :param sim_id: The simulation ID. :param robot_id: The robot ID. :status 500: {0} :status 404: {1} :status 401: {2} :status 400: Invalid request, the JSON parameters are incorrect :status 200: The requested robot was deleted successfully """ # pylint: disable=no-self-use sim = _get_simulation_or_abort(sim_id) if not UserAuthentication.can_modify(sim): raise NRPServicesWrongUserException() try: res, err = SimulationRobot.__delete_robot(sim, robot_id) except ROSCLEClientException as e: raise NRPServicesGeneralException(str(e), 'CLE error', 500) if not res: return {'res': err}, 404 return {'res': 'success'}, 200
def post(self, sim_id, command): """ Issue user commands to the simulator recorder. See parameter description for supported query commands. :param sim_id: The simulation ID to command. :param command: The command to issue, supported: [start, stop, cancel, reset, save] :status 500: {0} :status 404: {1} :status 401: {2} :status 400: The command is invalid/refused by recorder - see message returned. :status 200: Success. The command was issued. """ # validate simulation id sim = _get_simulation_or_abort(sim_id) # only the simulation owner can command the recorder if not UserAuthentication.can_modify(sim): raise NRPServicesWrongUserException() # pure local command to save file to storage if command == 'save': try: file_name = sim.lifecycle.save_record_to_user_storage() return {'filename': file_name}, 200 except Exception as e: raise NRPServicesClientErrorException( 'Cannot copy record to client storage', error_code=404) else: # validate the remote command type valid_commands = { 'start': SimulationRecorderRequest.START, 'stop': SimulationRecorderRequest.STOP, 'cancel': SimulationRecorderRequest.CANCEL, 'reset': SimulationRecorderRequest.RESET } if command not in valid_commands: raise NRPServicesClientErrorException( 'Invalid recorder command: %s' % command, error_code=404) # command the recorder, if unsuccessful return the error message try: resp = sim.cle.command_simulation_recorder( valid_commands[command]) # check the command success, on failure return status 400 + error if not resp.value: raise NRPServicesClientErrorException(resp.message) # successful, return status 200 return 'success', 200 # internal CLE ROS error if service call fails, notify frontend except ROSCLEClientException as e: raise NRPServicesGeneralException(str(e), 'CLE error', 500)
def put(self, sim_id, experiment_id): """ Calls the CLE for resetting a given simulation to the last saved state in the storage. :param sim_id: The simulation ID. :param experiment_id: The experiment ID :> json resetType: the reset type the user wants to be performed, details about possible values are given in GazeboRosPackages/src/cle_ros_msgs/srv/ResetSimulation.srv :status 500: {0} :status 404: {1} :status 401: {2} :status 400: Invalid request, the JSON parameters are incorrect :status 200: The requested reset was performed successfully """ sim = _get_simulation_or_abort(sim_id) if not UserAuthentication.can_modify(sim): raise NRPServicesWrongUserException() req_body = request.get_json(force=True) context_id = req_body.get('contextId', None) for missing_par in (par for par in self.ResetRequest.required if par not in req_body): raise NRPServicesClientErrorException('Missing parameter {}'.format(missing_par)) for invalid_p in (par for par in req_body if par not in self.ResetRequest.resource_fields): raise NRPServicesClientErrorException('Invalid parameter {}'.format(invalid_p)) reset_type = req_body.get('resetType') rsr = srv.ResetSimulationRequest try: if reset_type == rsr.RESET_ROBOT_POSE: sim.cle.reset(reset_type) elif reset_type == rsr.RESET_WORLD: sim.cle.reset(reset_type, world_sdf=self._get_sdf_world_from_storage(experiment_id, context_id)) elif reset_type == rsr.RESET_FULL: brain_path, populations, _ = \ self._get_brain_info_from_storage(experiment_id, context_id) if sim.playback_path is None: self.reset_from_storage_all(sim, experiment_id, context_id) sim.cle.reset(reset_type, world_sdf=self._get_sdf_world_from_storage(experiment_id, context_id), brain_path=brain_path, populations=populations) else: return {}, 400 # Other reset modes are unsupported except ROSCLEClientException as e: raise NRPServicesGeneralException(str(e), 'CLE error', 500) return {}, 200
def init_simulation_dir(): """ Creates a temporary directory and links it to Settings.sim_dir_symlink """ sim_dir = tempfile.mkdtemp(prefix='nrp.') try: if os.path.exists(Settings.sim_dir_symlink): shutil.rmtree(os.path.realpath(Settings.sim_dir_symlink)) os.unlink(Settings.sim_dir_symlink) os.symlink(sim_dir, Settings.sim_dir_symlink) except IOError as err: raise NRPServicesGeneralException( "Could not create symlink to temp simulation folder. {err}".format(err=err)) except OSError as ex: raise NRPServicesGeneralException( "Could not create symlink to temp simulation folder. {err}".format(err=ex)) return sim_dir
def _check_and_extract_environment_zip(self, experiment): """ Checks for validity and extracts a zipped environment. First we make sure that the zip referenced in the experiment exists in the list of user environments, then we unzip it on the fly in the temporary simulation directory. After the extraction we also make sure to copy the sdf from the experiment folder cause the user may have modified it :param experiment: The experiment object. """ from hbp_nrp_backend.storage_client_api.StorageClient import StorageClient client = StorageClient() environments_list = client.get_custom_models( UserAuthentication.get_header_token(request), self.simulation.ctx_id, 'environments') # we use the paths of the uploaded zips to make sure the selected # zip is there paths_list = [environment['path'] for environment in environments_list] # check if the zip is in the user storage zipped_model_path = [ path for path in paths_list if experiment.environmentModel.customModelPath in path ] if len(zipped_model_path): environment_path = os.path.join( client.get_temp_directory(), os.path.basename(experiment.environmentModel.src)) storage_env_zip_data = client.get_custom_model( UserAuthentication.get_header_token(request), self.simulation.ctx_id, zipped_model_path[0]) env_sdf_name = os.path.basename(experiment.environmentModel.src) env_path = os.path.join( client.get_temp_directory(), experiment.environmentModel.customModelPath) with open(env_path, 'w') as environment_zip: environment_zip.write(storage_env_zip_data) with zipfile.ZipFile(env_path) as env_zip_to_extract: env_zip_to_extract.extractall(path=client.get_temp_directory()) # copy back the .sdf from the experiment folder, cause we don't want the one # in the zip, cause the user might have made manual changes client.clone_file(env_sdf_name, UserAuthentication.get_header_token(request), self.simulation.experiment_id) # if the zip is not there, prompt the user to check his uploaded # models else: raise NRPServicesGeneralException( "Could not find selected zip %s in the list of uploaded models. Please make\ sure that it has been uploaded correctly" % (os.path.dirname(experiment.environmentModel.src)), "Zipped model retrieval failed") return environment_path
def get(self, sim_id, resource_type, resource_path): """ Download the file in the resource_path for the give resource_type :param sim_id: The ID of the simulation whose files shall be handled :param resource_type: "robots", "brains", "environments", "files" :param resource_path: relative path the particular resource to be downloaded """ # pylint: disable=no-self-use sim = _get_simulation_or_abort(sim_id) if not UserAuthentication.can_view(sim): raise NRPServicesWrongUserException() if resource_type not in SimulationFiles.RESOURCE_TYPES: raise NRPServicesGeneralException("Wrong resource_type specified", 'Wrong Parameter', 500) try: ret, err = SimulationFiles.__download_file(sim, resource_type, resource_path) except ROSCLEClientException as e: raise NRPServicesGeneralException(str(e), 'CLE error', 500) return ({'res': 'success'}, 200) if ret else ({'res': err}, 500)
def _extract_brain_zip(self): """ Checks for validity, and extracts a zipped brain. First we make sure that the zip referenced in the bibi exists in the list of user brains, then we unzip it on the fly in the temporary simulation directory. After the extraction we also make sure to copy the .py from the experiment folder cause the user may have modified it """ # pylint: disable=too-many-locals brain = Model(self.sim_config.brain_model.model, ResourceType.BRAIN) storage_brain_zip_data = self._storageClient.get_model( self.sim_config.token, self.sim_config.ctx_id, brain) if storage_brain_zip_data: # Get the data # Write the zip in sim dir zip_model_path = self._storageClient.get_model_path( self.sim_config.token, self.sim_config.ctx_id, brain) brain_abs_zip_path = os.path.join(self.sim_dir, zip_model_path) if not os.path.exists(os.path.dirname(brain_abs_zip_path)): os.makedirs(os.path.dirname(brain_abs_zip_path)) with open(brain_abs_zip_path, 'w') as brain_zip: brain_zip.write(storage_brain_zip_data) # Extract and flatten # FixME: not sure exactly why flattening is required ZipUtil.extractall( zip_abs_path=self.sim_config.brain_model.zip_path.abs_path, extract_to=self.sim_dir, overwrite=False, flatten=True) # copy back the .py from the experiment folder, cause we don't want the one # in the zip, cause the user might have made manual changes # TODO: verify if this still required and why only one file is copied brain_name = os.path.basename( self.sim_config.brain_model.resource_path.rel_path) self._storageClient.clone_file(brain_name, self.sim_config.token, self.sim_config.experiment_id) # if the zip is not there, prompt the user to check his uploaded models else: raise NRPServicesGeneralException( "Could not find selected brain model {name} in the list of uploaded models. " "Please make sure that it has been uploaded correctly".format( name=self.sim_config.brain_model.model), "Zipped model retrieval failed")
def test_nrp_services_general_exception(self): simulations[0]._Simulation__lifecycle = mock.MagicMock() simulations[0]._Simulation__lifecycle.accept_command = \ mock.Mock(side_effect=NRPServicesGeneralException(\ "I am a NRPServicesGeneralException message", "I am a NRPServicesGeneralException type")) response = self.client.put('/simulation/0/state', data='{"state": "started"}') self.assertEqual(response.status_code, 500) response_object = json.loads(response.data) self.assertEqual(u"I am a NRPServicesGeneralException message", response_object['message']) self.assertEqual(u"I am a NRPServicesGeneralException type", response_object['type'])
def parse_and_check_file_is_valid(filepath, create_obj_function, instance_type): """ Parses a file and checks if it corresponds to its instance type and can be created into its object :param filepath: The path of the file :param create_obj_function: The function to create the object :param instance_type: The required instance type of the file :return: An object containing the file content """ with open(filepath) as file_content: try: file_obj = create_obj_function(file_content.read()) if not isinstance(file_obj, instance_type): raise NRPServicesGeneralException( "{filepath} configuration file content is not valid.". format(filepath=filepath), "File not valid") except (ValidationError, SAXParseException) as ve: raise NRPServicesGeneralException( "Could not parse file {filepath} due to validation error: {validation_error}" .format(filepath=filepath, validation_error=str(ve)), "File not valid") return file_obj
def put(self, sim_id, state_machine_name): """ Applies user changes to state machine code. If the simulation is paused or started, it will be paused. A stopped, created or failed simulation will fail the request with error code 403 :param sim_id: The simulation ID :param state_machine_name: The name of the state machine to be modified :< json string data: The source code of the state machine :status 404: {0} :status 401: {1} :status 400: {2} :status 200: Success. The code was successfully patched """ simulation = _get_simulation_or_abort(sim_id) assert simulation, Simulation if not UserAuthentication.can_modify(simulation): raise NRPServicesWrongUserException() state_machine_source = request.data try: simulation.set_state_machine_code(state_machine_name, state_machine_source) return "Success. The code was successfully patched.", 200 except (AttributeError, NameError) as e: info = exc_info() raise NRPServicesStateMachineException(e.message, 400), None, info[2] except SyntaxError as e: info = exc_info() args_txt = "" for text in e.args: args_txt += " {0}".format(text) raise NRPServicesStateMachineException( "The source code is invalid: " "SyntaxError in line {0}{1}.".format(e.lineno, args_txt), 400), None, info[2] except Exception as e: info = exc_info() raise NRPServicesGeneralException( "Update of state machine code failed. " "{0}: {1}".format(e.__class__.__name__, e.message), "State machine error"), None, info[2]
def post(self, sim_id, robot_id): """ Add a new robot to the simulation. :param sim_id: The simulation ID. :param robot_id: The robot string ID. :status 500: {0} :status 404: {1} :status 401: {2} :status 400: Invalid request, the JSON parameters are incorrect :status 200: The requested robot was created successfully """ # pylint: disable=no-self-use sim = _get_simulation_or_abort(sim_id) if not UserAuthentication.can_modify(sim): raise NRPServicesWrongUserException() body = request.get_json(force=True) missing_parameters = [ item for item in SimulationRobot.RobotPostRequest.required if item not in body ] if missing_parameters: raise NRPServicesClientErrorException('Missing parameter(s): ' + ' '.join(missing_parameters)) try: res, err = SimulationRobot.__add_new_robot( sim=sim, robot_id=robot_id, robot_model=body.get(ParamNames.ROBOT_MODEL), is_custom=(body.get(ParamNames.IS_CUSTOM, None) == 'True'), initial_pose=body.get(ParamNames.ROBOT_POSE, None)) except ROSCLEClientException as e: raise NRPServicesGeneralException(str(e), 'CLE error', 500) if not res: return {'res': err}, 404 return {'res': 'success'}, 200
def __set_material(self, visual_path, material): """ Sets the material of a particular visual :param visual_path: The visual path in the world description, e.g., 'left_screen::body::screen_glass', i.e., model_name::link_name::visual_name. :param material: The material :return: The response """ try: rospy.wait_for_service('/gazebo/set_visual_properties', 3) except rospy.ROSException as exc: raise NRPServicesUnavailableROSService(str(exc)) set_visual_properties = rospy.ServiceProxy( '/gazebo/set_visual_properties', SetVisualProperties ) names = visual_path.split('::') if len(names) < 3: raise NRPServicesClientErrorException( "Invalid visual path: " + visual_path + ".\n Expected: model_name::link_name::visual_name", error_code=404 ) assert(material is not None) try: set_visual_properties( model_name=names[0], link_name="::".join(names[1:-1]), visual_name=names[-1], property_name='material:script:name', property_value=material ) except rospy.ServiceException as exc: raise NRPServicesGeneralException( "Service did not process request: " + str(exc), "rospy service exception" ) return {'message': 'Material changed successfully'}, 200
def get(self, sim_id, command): """ Queries the simulation recorder for a value/status. See parameter description for supported query commands. :param sim_id: The simulation ID to command. :param command: The command to query, supported: [is-recording] :status 500: {0} :status 404: {1} :status 401: {2} :status 200: Success. The query was issued and value returned. """ # validate simulation id, allow any users/viewers to query recorder sim = _get_simulation_or_abort(sim_id) if not UserAuthentication.can_view(sim): raise NRPServicesWrongUserException() # validate the command type if command not in ['is-recording', 'is-playingback']: raise NRPServicesClientErrorException( 'Invalid recorder query: %s' % command, error_code=404) try: if command == 'is-recording': state = str( sim.cle.command_simulation_recorder( SimulationRecorderRequest.STATE).value) elif command == 'is-playingback': state = 'False' if sim.playback_path is None else 'True' return {'state': state}, 200 # internal CLE ROS error if service call fails, notify frontend except ROSCLEClientException as e: raise NRPServicesGeneralException(str(e), 'CLE error', 500)
def _load_brain(self): """ Loads the neural simulator, interfaces, and configuration """ # Create interfaces to brain self._notify("Loading neural simulator") brainconfig.rng_seed = self.rng_seed braincomm, braincontrol = self._create_brain_adapters() self._notify("Loading brain and population configuration") if not self.sim_config.brain_model: return braincontrol, braincomm, None, None if self.sim_config.brain_model.is_custom: self._extract_brain_zip() brain_abs_path = self.sim_config.brain_model.resource_path.abs_path brain_rel_path = self.sim_config.brain_model.resource_path.rel_path if not os.path.exists(brain_abs_path): logger.info( "Cannot find specified brain file {file} in {dir}. Searching in default " "directories {default}".format( file=brain_rel_path, dir=self.sim_dir, default=str(self.sim_config.model_paths))) brain_abs_path = SimUtil.find_file_in_paths( brain_rel_path, self.sim_config.model_paths) if brain_abs_path: self.sim_config.brain_model.resource_path.abs_path = brain_abs_path else: raise NRPServicesGeneralException( "Could not find brain file: {}".format(brain_rel_path)) neurons_config = self.sim_config.get_populations_dict() return braincontrol, braincomm, brain_abs_path, neurons_config
def _prepare_custom_environment(self, exc): """ Download and extracts zipped environment defined in the exc :param exc: The exc DOM object """ # pylint: disable=too-many-locals env_model = Model(exc.environmentModel.model, ResourceType.ENVIRONMENT) data = self.__storageClient.get_model( UserAuthentication.get_header_token(), self.simulation.ctx_id, env_model) # if the zip is not there, prompt the user to check his uploaded models if not data: raise NRPServicesGeneralException( "Could not find selected zip {} in the list of uploaded custom models. Please make " "sure that it has been uploaded correctly".format( os.path.dirname(exc.environmentModel.model)), "Zipped model retrieval failed") ZipUtil.extractall(zip_abs_path=io.BytesIO(data), extract_to=os.path.join(self._sim_dir, 'assets'), overwrite=True)
def test_nrp_services_general_exception(self): """ This method tests the class hbp_nrp_backend.rest_server.NRPServicesGeneralException """ nsge = NRPServicesGeneralException("StringA", "StringB") self.assertEqual(nsge.__str__(), "'StringA' (StringB)")
def initialize(self, state_change): """ Initializes the simulation :param state_change: The state change that caused the simulation to be initialized """ simulation = self.simulation if not simulation.playback_path: self._sim_dir = SimUtil.init_simulation_dir() try: if not simulation.private: raise NRPServicesGeneralException( "Only private experiments are supported", "CLE error", 500) self.__storageClient.clone_all_experiment_files( token=UserAuthentication.get_header_token(), experiment=simulation.experiment_id, destination_dir=self._sim_dir, exclude=['recordings/'] if not simulation.playback_path else []) # divine knowledge about the exc name self.__experiment_path = os.path.join( self._sim_dir, 'experiment_configuration.exc') with open(self.__experiment_path) as exd_file: exc = exp_conf_api_gen.CreateFromDocument(exd_file.read()) self._load_state_machines(exc) if exc.environmentModel.model: # i.e., custom zipped environment self._prepare_custom_environment(exc) simulation.timeout_type = ( TimeoutType.SIMULATION if exc.timeout.time == TimeoutType.SIMULATION else TimeoutType.REAL) timeout = exc.timeout.value() if simulation.timeout_type == TimeoutType.REAL: timeout = datetime.datetime.now(timezone) + datetime.timedelta( seconds=timeout) simulation.kill_datetime = timeout else: simulation.kill_datetime = None logger.info("simulation timeout initialized") simulation_factory_client = ROSCLESimulationFactoryClient() simulation_factory_client.create_new_simulation( self.__experiment_path, simulation.gzserver_host, simulation.reservation, simulation.brain_processes, simulation.sim_id, str(timeout), simulation.timeout_type, simulation.playback_path, UserAuthentication.get_header_token(), self.simulation.ctx_id, self.simulation.experiment_id) if not simulation.playback_path: simulation.cle = ROSCLEClient(simulation.sim_id) else: simulation.cle = PlaybackClient(simulation.sim_id) logger.info("simulation initialized") except IOError as e: raise NRPServicesGeneralException( "Error while accessing simulation models (" + repr(e.message) + ")", "Models error") except rospy.ROSException as e: raise NRPServicesGeneralException( "Error while communicating with the CLE (" + repr(e.message) + ")", "CLE error") except rospy.ServiceException as e: raise NRPServicesGeneralException( "Error starting the simulation. (" + repr(e.message) + ")", "rospy.ServiceException", data=e.message) # pylint: disable=broad-except except Exception as e: raise NRPServicesGeneralException( "Error starting the simulation. (" + repr(e) + ")", "Unknown exception occured", data=e.message)
def initialize(self, state_change): """ Initializes the simulation :param state_change: The state change that caused the simulation to be initialized """ # TODO: fix dependencies so these import are not necessary # anymore from hbp_nrp_backend.storage_client_api.StorageClient import StorageClient simulation = self.simulation try: using_storage = simulation.private is not None if using_storage: client = StorageClient() clone_folder, experiment_paths = client.clone_all_experiment_files( UserAuthentication.get_header_token(request), simulation.experiment_id) self.__experiment_path = experiment_paths['experiment_conf'] self.__simulation_root_folder = clone_folder environment_path = experiment_paths['environment_conf'] else: self.__experiment_path = os.path.join( self.__experiment_path, simulation.experiment_conf) self.__simulation_root_folder = os.path.dirname( self.__experiment_path) environment_path = simulation.environment_conf experiment, environment_path = self._parse_exp_and_initialize_paths( self.__experiment_path, environment_path, using_storage) simulation.kill_datetime = datetime.datetime.now(timezone) \ + datetime.timedelta(seconds=experiment.timeout) logger.info("simulation timeout initialized") simulation_factory_client = ROSCLESimulationFactoryClient() simulation_factory_client.create_new_simulation( environment_path, self.__experiment_path, simulation.gzserver_host, simulation.reservation, simulation.brain_processes, simulation.sim_id, str(simulation.kill_datetime), simulation.playback_path, UserAuthentication.get_header_token(request), self.simulation.ctx_id) if not simulation.playback_path: simulation.cle = ROSCLEClient(simulation.sim_id) else: simulation.cle = PlaybackClient(simulation.sim_id) logger.info("simulation initialized") except IOError as e: raise NRPServicesGeneralException( "Error while accessing simulation models (" + repr(e.message) + ")", "Models error") except rospy.ROSException as e: raise NRPServicesGeneralException( "Error while communicating with the CLE (" + repr(e.message) + ")", "CLE error") except rospy.ServiceException as e: raise NRPServicesGeneralException( "Error starting the simulation. (" + repr(e.message) + ")", "rospy.ServiceException", data=e.message)