def _operation_finished(operation, simulator_gid):
op_ident = dao.get_operation_process_for_operation(operation.id)
# TODO: Handle login
job = Job(
Transport(os.environ[HPCSchedulerClient.CSCS_LOGIN_TOKEN_ENV_KEY]),
op_ident.job_id)
operation = dao.get_operation_by_id(operation.id)
folder = HPCSchedulerClient.storage_interface.get_project_folder(
operation.project.name)
storage_interface = StorageInterface()
if storage_interface.encryption_enabled():
storage_interface.inc_project_usage_count(folder)
storage_interface.sync_folders(folder)
try:
sim_h5_filenames, metric_op, metric_h5_filename = \
HPCSchedulerClient.stage_out_to_operation_folder(job.working_dir, operation, simulator_gid)
operation.mark_complete(STATUS_FINISHED)
dao.store_entity(operation)
HPCSchedulerClient().update_db_with_results(
operation, sim_h5_filenames, metric_op, metric_h5_filename)
except OperationException as exception:
HPCOperationService.LOGGER.error(exception)
HPCOperationService._operation_error(operation)
finally:
if storage_interface.encryption_enabled():
storage_interface.sync_folders(folder)
storage_interface.set_project_inactive(operation.project)
def _import_projects_from_folder(self, temp_folder):
"""
Process each project from the uploaded pack, to extract names.
"""
project_roots = []
for root, _, files in os.walk(temp_folder):
if StorageInterface.TVB_PROJECT_FILE in files:
project_roots.append(root)
for temp_project_path in project_roots:
update_manager = ProjectUpdateManager(temp_project_path)
update_manager.run_all_updates()
project = self.__populate_project(temp_project_path)
# Populate the internal list of create projects so far, for cleaning up folders, in case of failure
self.created_projects.append(project)
# Ensure project final folder exists on disk
project_path = self.storage_interface.get_project_folder(project.name)
shutil.move(os.path.join(temp_project_path, StorageInterface.TVB_PROJECT_FILE), project_path)
# Now import project operations with their results
self.import_project_operations(project, temp_project_path)
# Import images and move them from temp into target
self._store_imported_images(project, temp_project_path, project.name)
if StorageInterface.encryption_enabled():
StorageInterface.sync_folders(project_path)
shutil.rmtree(project_path)
def run(self):
"""
Get the required data from the operation queue and launch the operation.
"""
operation_id = self.operation_id
run_params = [TvbProfile.current.PYTHON_INTERPRETER_PATH, '-m', 'tvb.core.operation_async_launcher',
str(operation_id), TvbProfile.CURRENT_PROFILE_NAME]
current_operation = dao.get_operation_by_id(operation_id)
storage_interface = StorageInterface()
project_folder = storage_interface.get_project_folder(current_operation.project.name)
in_usage = storage_interface.is_in_usage(project_folder)
storage_interface.inc_running_op_count(project_folder)
if not in_usage:
storage_interface.sync_folders(project_folder)
# In the exceptional case where the user pressed stop while the Thread startup is done,
# We should no longer launch the operation.
if self.stopped() is False:
env = os.environ.copy()
env['PYTHONPATH'] = os.pathsep.join(sys.path)
# anything that was already in $PYTHONPATH should have been reproduced in sys.path
launched_process = Popen(run_params, stdout=PIPE, stderr=PIPE, env=env)
LOGGER.debug("Storing pid=%s for operation id=%s launched on local machine." % (operation_id,
launched_process.pid))
op_ident = OperationProcessIdentifier(operation_id, pid=launched_process.pid)
dao.store_entity(op_ident)
if self.stopped():
# In the exceptional case where the user pressed stop while the Thread startup is done.
# and stop_operation is concurrently asking about OperationProcessIdentity.
self.stop_pid(launched_process.pid)
subprocess_result = launched_process.communicate()
LOGGER.info("Finished with launch of operation %s" % operation_id)
returned = launched_process.wait()
LOGGER.info("Return code: {}. Stopped: {}".format(returned, self.stopped()))
LOGGER.info("Thread: {}".format(self))
if returned != 0 and not self.stopped():
# Process did not end as expected. (e.g. Segmentation fault)
burst_service = BurstService()
operation = dao.get_operation_by_id(self.operation_id)
LOGGER.error("Operation suffered fatal failure! Exit code: %s Exit message: %s" % (returned,
subprocess_result))
burst_service.persist_operation_state(operation, STATUS_ERROR,
"Operation failed unexpectedly! Please check the log files.")
del launched_process
storage_interface.check_and_delete(project_folder)
# Give back empty spot now that you finished your operation
CURRENT_ACTIVE_THREADS.remove(self)
LOCKS_QUEUE.put(1)
def editone(self, project_id=None, cancel=False, save=False, delete=False, **data):
"""
Create or change Project. When project_id is empty we create a
new entity, otherwise we are to edit and existent one.
"""
if cherrypy.request.method == 'POST' and cancel:
raise cherrypy.HTTPRedirect('/project')
if cherrypy.request.method == 'POST' and delete:
self._remove_project(project_id)
raise cherrypy.HTTPRedirect('/project/viewall')
current_user = common.get_logged_user()
is_create = False
if project_id is None or not int(project_id):
is_create = True
data["administrator"] = current_user.display_name
admin_username = current_user.username
else:
current_project = self.project_service.find_project(project_id)
if not save:
# Only when we do not have submitted data,
# populate fields with initial values for edit.
data = dict(name=current_project.name, description=current_project.description)
data["administrator"] = current_project.administrator.display_name
admin_username = current_project.administrator.username
self._mark_selected(current_project)
data["project_id"] = project_id
template_specification = dict(mainContent="project/editone", data=data, isCreate=is_create,
title="Create new project" if is_create else "Edit " + data["name"],
editUsersEnabled=(current_user.username == admin_username))
try:
if cherrypy.request.method == 'POST' and save:
data = EditForm().to_python(data)
saved_project = self.project_service.store_project(current_user, is_create, project_id, **data)
if StorageInterface.encryption_enabled() and is_create:
project_folder = StorageInterface().get_project_folder(saved_project.name)
StorageInterface.sync_folders(project_folder)
shutil.rmtree(project_folder)
self._mark_selected(saved_project)
raise cherrypy.HTTPRedirect('/project/viewall')
except formencode.Invalid as excep:
self.logger.debug(str(excep))
template_specification[common.KEY_ERRORS] = excep.unpack_errors()
except ProjectServiceException as excep:
self.logger.debug(str(excep))
common.set_error_message(excep.message)
raise cherrypy.HTTPRedirect('/project/viewall')
all_users, members, pages = self.user_service.get_users_for_project(current_user.username, project_id)
template_specification['usersList'] = all_users
template_specification['usersMembers'] = [m.id for m in members]
template_specification['usersPages'] = pages
template_specification['usersCurrentPage'] = 1
return self.fill_default_attributes(template_specification, 'properties')
