def send_message(self, method, args): try: write_framed_json((method, args), self.out_fp) self.out_fp.flush() except IOError: print >>sys.stderr, "RPC error when receiving message: process dying." sys.exit(-1)
def _guarded_run(self, task_private, task_descriptor, task_record): self.task_private = task_private self.task_record = task_record self.task_descriptor = task_descriptor self.expected_outputs = list(self.task_descriptor['expected_outputs']) self.error_details = None if "id" in task_private: id = task_private['id'] self.process_record = self.process_pool.get_process_record(id) else: self.process_record = self.process_pool.get_soft_cache_process(self.__class__, task_descriptor["dependencies"]) if self.process_record is None: self.process_record = self.process_pool.create_process_record(None, "json") if "command" in task_private: # XXX sos22 How is SWRealReference in scope here? if isinstance(task_private["command"], SWRealReference): # Command has been passed in as a reference. command = [retrieve_filename_for_ref(arg, self.task_record, False)] os.chmod(command_line[0], stat.S_IRWXU) else: command = [task_private["command"]] else: command = self.get_command() command.extend(["--write-fifo", self.process_record.get_read_fifo_name(), "--read-fifo", self.process_record.get_write_fifo_name()]) new_proc_env = os.environ.copy() new_proc_env.update(self.get_env()) new_proc = subprocess.Popen(command, env=new_proc_env, close_fds=True) self.process_record.set_pid(new_proc.pid) # XXX: This will block until the attached process opens the pipes. reader = self.process_record.get_read_fifo() writer = self.process_record.get_write_fifo() self.reader = reader self.writer = writer #ciel.log('Got reader and writer FIFOs', 'PROC', logging.INFO) write_framed_json(("start_task", task_private), writer) try: if self.process_record.protocol == 'line': finished = self.line_event_loop(reader, writer) elif self.process_record.protocol == 'json': finished = self.json_event_loop(reader, writer) else: raise BlameUserException('Unsupported protocol: %s' % self.process_record.protocol) except MissingInputException, mie: self.process_pool.delete_process_record(self.process_record) raise
def _unsubscribe_output(self, index): message = ("unsubscribe", {"index": index}) with self.transmit_lock: write_framed_json(message, self.writer)
def _subscribe_output(self, index, chunk_size): message = ("subscribe", {"index": index, "chunk_size": chunk_size}) with self.transmit_lock: write_framed_json(message, self.writer)
return PROC_ERROR except MissingInputException, mie: ciel.log("Task died due to missing input", 'PROC', logging.WARN) raise except TaskFailedError: raise except: ciel.log('Error during method handling in JSON event loop', 'PROC', logging.ERROR, True) return PROC_ERROR try: if response is not None: with self.transmit_lock: write_framed_json((method, response), writer) if response_fd is not None: socket_name = args["fd_socket_name"] sock = socket.socket(socket.AF_UNIX) sock.connect(socket_name) sendmsg.sendmsg(fd=sock.fileno(), data="FD", ancillary=(socket.SOL_SOCKET, sendmsg.SCM_RIGHTS, struct.pack("i", response_fd))) os.close(response_fd) sock.close() except: ciel.log('Error writing response in JSON event loop', 'PROC', logging.WARN, True) return PROC_ERROR return True
def kill(self): ciel.log("Garbage collecting process %s" % self.id, "PROCESSPOOL", logging.INFO) write_fp = self.get_write_fifo() write_framed_json(("die", {"reason": "Garbage collected"}), write_fp) self.cleanup()
def _guarded_run(self, task_private, task_descriptor, task_record): self.task_private = task_private self.task_record = task_record self.task_descriptor = task_descriptor self.expected_outputs = list(self.task_descriptor['expected_outputs']) self.error_details = None if "id" in task_private: id = task_private['id'] self.process_record = self.process_pool.get_process_record(id) else: self.process_record = self.process_pool.get_soft_cache_process( self.__class__, task_descriptor["dependencies"]) if self.process_record is None: self.process_record = self.process_pool.create_process_record( None, "json") if "command" in task_private: if isinstance(task_private["command"], SWRealReference): # Command has been passed in as a reference. command = [ retrieve_filename_for_ref(arg, self.task_record, False) ] os.chmod(command_line[0], stat.S_IRWXU) else: command = [task_private["command"]] else: command = self.get_command() command.extend([ "--write-fifo", self.process_record.get_read_fifo_name(), "--read-fifo", self.process_record.get_write_fifo_name() ]) new_proc_env = os.environ.copy() new_proc_env.update(self.get_env()) new_proc = subprocess.Popen(command, env=new_proc_env, close_fds=True) self.process_record.set_pid(new_proc.pid) # XXX: This will block until the attached process opens the pipes. reader = self.process_record.get_read_fifo() writer = self.process_record.get_write_fifo() self.reader = reader self.writer = writer #ciel.log('Got reader and writer FIFOs', 'PROC', logging.INFO) write_framed_json(("start_task", task_private), writer) try: if self.process_record.protocol == 'line': finished = self.line_event_loop(reader, writer) elif self.process_record.protocol == 'json': finished = self.json_event_loop(reader, writer) else: raise BlameUserException('Unsupported protocol: %s' % self.process_record.protocol) except MissingInputException, mie: self.process_pool.delete_process_record(self.process_record) raise
ciel.log("Task died due to missing input", 'PROC', logging.WARN) raise except TaskFailedError: raise except: ciel.log('Error during method handling in JSON event loop', 'PROC', logging.ERROR, True) return PROC_ERROR try: if response is not None: with self.transmit_lock: write_framed_json((method, response), writer) if response_fd is not None: socket_name = args["fd_socket_name"] sock = socket.socket(socket.AF_UNIX) sock.connect(socket_name) sendmsg.sendmsg(fd=sock.fileno(), data="FD", ancillary=(socket.SOL_SOCKET, sendmsg.SCM_RIGHTS, struct.pack("i", response_fd))) os.close(response_fd) sock.close() except: ciel.log('Error writing response in JSON event loop', 'PROC', logging.WARN, True) return PROC_ERROR