def _operation_finished(operation, simulator_gid): op_ident = dao.get_operation_process_for_operation(operation.id) # TODO: Handle login job = Job( Transport(os.environ[HPCSchedulerClient.CSCS_LOGIN_TOKEN_ENV_KEY]), op_ident.job_id) operation = dao.get_operation_by_id(operation.id) folder = HPCSchedulerClient.storage_interface.get_project_folder( operation.project.name) storage_interface = StorageInterface() if storage_interface.encryption_enabled(): storage_interface.inc_project_usage_count(folder) storage_interface.sync_folders(folder) try: sim_h5_filenames, metric_op, metric_h5_filename = \ HPCSchedulerClient.stage_out_to_operation_folder(job.working_dir, operation, simulator_gid) operation.mark_complete(STATUS_FINISHED) dao.store_entity(operation) HPCSchedulerClient().update_db_with_results( operation, sim_h5_filenames, metric_op, metric_h5_filename) except OperationException as exception: HPCOperationService.LOGGER.error(exception) HPCOperationService._operation_error(operation) finally: if storage_interface.encryption_enabled(): storage_interface.sync_folders(folder) storage_interface.set_project_inactive(operation.project)
def _import_projects_from_folder(self, temp_folder): """ Process each project from the uploaded pack, to extract names. """ project_roots = [] for root, _, files in os.walk(temp_folder): if StorageInterface.TVB_PROJECT_FILE in files: project_roots.append(root) for temp_project_path in project_roots: update_manager = ProjectUpdateManager(temp_project_path) update_manager.run_all_updates() project = self.__populate_project(temp_project_path) # Populate the internal list of create projects so far, for cleaning up folders, in case of failure self.created_projects.append(project) # Ensure project final folder exists on disk project_path = self.storage_interface.get_project_folder(project.name) shutil.move(os.path.join(temp_project_path, StorageInterface.TVB_PROJECT_FILE), project_path) # Now import project operations with their results self.import_project_operations(project, temp_project_path) # Import images and move them from temp into target self._store_imported_images(project, temp_project_path, project.name) if StorageInterface.encryption_enabled(): StorageInterface.sync_folders(project_path) shutil.rmtree(project_path)
def run(self): """ Get the required data from the operation queue and launch the operation. """ operation_id = self.operation_id run_params = [TvbProfile.current.PYTHON_INTERPRETER_PATH, '-m', 'tvb.core.operation_async_launcher', str(operation_id), TvbProfile.CURRENT_PROFILE_NAME] current_operation = dao.get_operation_by_id(operation_id) storage_interface = StorageInterface() project_folder = storage_interface.get_project_folder(current_operation.project.name) in_usage = storage_interface.is_in_usage(project_folder) storage_interface.inc_running_op_count(project_folder) if not in_usage: storage_interface.sync_folders(project_folder) # In the exceptional case where the user pressed stop while the Thread startup is done, # We should no longer launch the operation. if self.stopped() is False: env = os.environ.copy() env['PYTHONPATH'] = os.pathsep.join(sys.path) # anything that was already in $PYTHONPATH should have been reproduced in sys.path launched_process = Popen(run_params, stdout=PIPE, stderr=PIPE, env=env) LOGGER.debug("Storing pid=%s for operation id=%s launched on local machine." % (operation_id, launched_process.pid)) op_ident = OperationProcessIdentifier(operation_id, pid=launched_process.pid) dao.store_entity(op_ident) if self.stopped(): # In the exceptional case where the user pressed stop while the Thread startup is done. # and stop_operation is concurrently asking about OperationProcessIdentity. self.stop_pid(launched_process.pid) subprocess_result = launched_process.communicate() LOGGER.info("Finished with launch of operation %s" % operation_id) returned = launched_process.wait() LOGGER.info("Return code: {}. Stopped: {}".format(returned, self.stopped())) LOGGER.info("Thread: {}".format(self)) if returned != 0 and not self.stopped(): # Process did not end as expected. (e.g. Segmentation fault) burst_service = BurstService() operation = dao.get_operation_by_id(self.operation_id) LOGGER.error("Operation suffered fatal failure! Exit code: %s Exit message: %s" % (returned, subprocess_result)) burst_service.persist_operation_state(operation, STATUS_ERROR, "Operation failed unexpectedly! Please check the log files.") del launched_process storage_interface.check_and_delete(project_folder) # Give back empty spot now that you finished your operation CURRENT_ACTIVE_THREADS.remove(self) LOCKS_QUEUE.put(1)
def editone(self, project_id=None, cancel=False, save=False, delete=False, **data): """ Create or change Project. When project_id is empty we create a new entity, otherwise we are to edit and existent one. """ if cherrypy.request.method == 'POST' and cancel: raise cherrypy.HTTPRedirect('/project') if cherrypy.request.method == 'POST' and delete: self._remove_project(project_id) raise cherrypy.HTTPRedirect('/project/viewall') current_user = common.get_logged_user() is_create = False if project_id is None or not int(project_id): is_create = True data["administrator"] = current_user.display_name admin_username = current_user.username else: current_project = self.project_service.find_project(project_id) if not save: # Only when we do not have submitted data, # populate fields with initial values for edit. data = dict(name=current_project.name, description=current_project.description) data["administrator"] = current_project.administrator.display_name admin_username = current_project.administrator.username self._mark_selected(current_project) data["project_id"] = project_id template_specification = dict(mainContent="project/editone", data=data, isCreate=is_create, title="Create new project" if is_create else "Edit " + data["name"], editUsersEnabled=(current_user.username == admin_username)) try: if cherrypy.request.method == 'POST' and save: data = EditForm().to_python(data) saved_project = self.project_service.store_project(current_user, is_create, project_id, **data) if StorageInterface.encryption_enabled() and is_create: project_folder = StorageInterface().get_project_folder(saved_project.name) StorageInterface.sync_folders(project_folder) shutil.rmtree(project_folder) self._mark_selected(saved_project) raise cherrypy.HTTPRedirect('/project/viewall') except formencode.Invalid as excep: self.logger.debug(str(excep)) template_specification[common.KEY_ERRORS] = excep.unpack_errors() except ProjectServiceException as excep: self.logger.debug(str(excep)) common.set_error_message(excep.message) raise cherrypy.HTTPRedirect('/project/viewall') all_users, members, pages = self.user_service.get_users_for_project(current_user.username, project_id) template_specification['usersList'] = all_users template_specification['usersMembers'] = [m.id for m in members] template_specification['usersPages'] = pages template_specification['usersCurrentPage'] = 1 return self.fill_default_attributes(template_specification, 'properties')