def _run_cluster_job(operation_identifier, user_name_label, adapter_instance): """ Threaded Popen It is the function called by the ClusterSchedulerClient in a Thread. This function starts a new process. """ # Load operation so we can estimate the execution time operation = dao.get_operation_by_id(operation_identifier) kwargs = parse_json_parameters(operation.parameters) time_estimate = int(adapter_instance.get_execution_time_approximation(**kwargs)) hours = int(time_estimate / 3600) minutes = (int(time_estimate) % 3600) / 60 seconds = int(time_estimate) % 60 # Anything lower than 2 hours just use default walltime if hours < 2: walltime = "02:00:00" elif hours > 23: walltime = "23:59:59" else: walltime = datetime.time(hours, minutes, seconds) walltime = walltime.strftime("%H:%M:%S") call_arg = config.CLUSTER_SCHEDULE_COMMAND % (walltime, operation_identifier, user_name_label) LOGGER.info(call_arg) process_ = Popen([call_arg], stdout=PIPE, shell=True) job_id = process_.stdout.read().replace('\n', '').split('OAR_JOB_ID=')[-1] LOGGER.debug("Got jobIdentifier = %s for CLUSTER operationID = %s" % (operation_identifier, job_id)) operation_identifier = model.OperationProcessIdentifier(operation_identifier, job_id=job_id) dao.store_entity(operation_identifier)
def _run_cluster_job(operation_identifier, user_name_label, adapter_instance): """ Threaded Popen It is the function called by the ClusterSchedulerClient in a Thread. This function starts a new process. """ # Load operation so we can estimate the execution time operation = dao.get_operation_by_id(operation_identifier) kwargs = parse_json_parameters(operation.parameters) kwargs = adapter_instance.prepare_ui_inputs(kwargs) time_estimate = int(adapter_instance.get_execution_time_approximation(**kwargs)) hours = int(time_estimate / 3600) minutes = (int(time_estimate) % 3600) / 60 seconds = int(time_estimate) % 60 # Anything lower than 5 hours just use default walltime if hours < 5: walltime = "05:00:00" else: if hours < 10: hours = "0%d" % hours else: hours = str(hours) walltime = "%s:%s:%s" % (hours, str(minutes), str(seconds)) call_arg = TvbProfile.current.cluster.SCHEDULE_COMMAND % (walltime, operation_identifier, user_name_label) LOGGER.info(call_arg) process_ = Popen([call_arg], stdout=PIPE, shell=True) job_id = process_.stdout.read().replace('\n', '').split('OAR_JOB_ID=')[-1] LOGGER.debug("Got jobIdentifier = %s for CLUSTER operationID = %s" % (operation_identifier, job_id)) operation_identifier = model.OperationProcessIdentifier(operation_identifier, job_id=job_id) dao.store_entity(operation_identifier)
def run(self): """ Get the required data from the operation queue and launch the operation. """ #Try to get a spot to launch own operation. LOCKS_QUEUE.get(True) operation_id = self.operation_id run_params = [TvbProfile.current.PYTHON_INTERPRETER_PATH, '-m', 'tvb.core.operation_async_launcher', str(operation_id), TvbProfile.CURRENT_PROFILE_NAME] # In the exceptional case where the user pressed stop while the Thread startup is done, # We should no longer launch the operation. if self.stopped() is False: env = os.environ.copy() env['PYTHONPATH'] = os.pathsep.join(sys.path) # anything that was already in $PYTHONPATH should have been reproduced in sys.path launched_process = Popen(run_params, stdout=PIPE, stderr=PIPE, env=env) LOGGER.debug("Storing pid=%s for operation id=%s launched on local machine." % (operation_id, launched_process.pid)) op_ident = model.OperationProcessIdentifier(operation_id, pid=launched_process.pid) dao.store_entity(op_ident) if self.stopped(): # In the exceptional case where the user pressed stop while the Thread startup is done. # and stop_operation is concurrently asking about OperationProcessIdentity. self.stop_pid(launched_process.pid) subprocess_result = launched_process.communicate() LOGGER.info("Finished with launch of operation %s" % operation_id) returned = launched_process.wait() if returned != 0 and not self.stopped(): # Process did not end as expected. (e.g. Segmentation fault) workflow_service = WorkflowService() operation = dao.get_operation_by_id(self.operation_id) LOGGER.error("Operation suffered fatal failure! Exit code: %s Exit message: %s" % (returned, subprocess_result)) workflow_service.persist_operation_state(operation, model.STATUS_ERROR, "Operation failed unexpectedly! Please check the log files.") burst_entity = dao.get_burst_for_operation_id(self.operation_id) if burst_entity: message = "Error in operation process! Possibly segmentation fault." workflow_service.mark_burst_finished(burst_entity, error_message=message) del launched_process #Give back empty spot now that you finished your operation CURRENT_ACTIVE_THREADS.remove(self) LOCKS_QUEUE.put(1)
def _run_cluster_job(operation_identifier, user_name_label): """ Threaded Popen It is the function called by the ClusterSchedulerClient in a Thread. This function starts a new process. """ call_arg = config.CLUSTER_SCHEDULE_COMMAND % (operation_identifier, user_name_label) process_ = Popen([call_arg], stdout=PIPE, shell=True) job_id = process_.stdout.read().replace('\n', '').split('OAR_JOB_ID=')[-1] LOGGER.debug( "Storing job identifier=%s for operation id=%s launched on cluster." % (operation_identifier, job_id)) operation_identifier = model.OperationProcessIdentifier(operation_identifier, job_id=job_id) dao.store_entity(operation_identifier)
def run(self): """ Get the required data from the operation queue and launch the operation. """ #Try to get a spot to launch own operation. LOCKS_QUEUE.get(True) operation_id = self.operation_id user_name_label = self.user_name_label run_params = [config().get_python_path(), '-m', 'tvb.core.cluster_launcher', str(operation_id), user_name_label] if tvb_profile.CURRENT_SELECTED_PROFILE is not None: run_params.extend([tvb_profile.SUBPARAM_PROFILE, tvb_profile.CURRENT_SELECTED_PROFILE]) # In the exceptional case where the user pressed stop while the Thread startup is done, # We should no longer launch the operation. if self.stopped() is False: launched_process = Popen(run_params, stdout=PIPE, stderr=PIPE) LOGGER.debug("Storing pid=%s for operation id=%s launched on local machine." % (operation_id, launched_process.pid)) op_ident = model.OperationProcessIdentifier(operation_id, pid=launched_process.pid) dao.store_entity(op_ident) if self.stopped(): # In the exceptional case where the user pressed stop while the Thread startup is done. # and stop_operation is concurrently asking about OperationProcessIdentity. self.stop_pid(launched_process.pid) launched_process.communicate() LOGGER.info("Finished with launch of operation %s" % operation_id) returned = launched_process.wait() if returned != 0 and not self.stopped(): # Process did not end as expected. (e.g. Segmentation fault) operation = dao.get_operation_by_id(self.operation_id) LOGGER.error("Operation suffered fatal failure with exit code: %s" % returned) operation.mark_complete(model.STATUS_ERROR, "Operation failed unexpectedly! Probably segmentation fault.") dao.store_entity(operation) burst_entity = dao.get_burst_for_operation_id(self.operation_id) if burst_entity: message = "Error on burst operation! Probably segmentation fault." WorkflowService().mark_burst_finished(burst_entity, error=True, error_message=message) del launched_process #Give back empty spot now that you finished your operation CURRENT_ACTIVE_THREADS.remove(self) LOCKS_QUEUE.put(1)