def tempControlProcTest(mode, cycle_time, duty_cycle, set_point, k_param, i_param, d_param, conn): p = current_process() print 'Starting:', p.name, p.pid parent_conn_temp, child_conn_temp = Pipe() ptemp = Process(name = "getrandProc", target=getrandProc, args=(child_conn_temp,)) #ptemp.daemon = True ptemp.start() parent_conn_heat, child_conn_heat = Pipe() pheat = Process(name = "heatProctest", target=heatProctest, args=(cycle_time, duty_cycle, child_conn_heat)) #pheat.daemon = True pheat.start() while (True): if parent_conn_temp.poll(): randnum = parent_conn_temp.recv() #non blocking receive conn.send([randnum, mode, cycle_time, duty_cycle, set_point, k_param, i_param, d_param]) if parent_conn_heat.poll(): cycle_time, duty_cycle = parent_conn_heat.recv() #duty_cycle = on_time/offtime*100.0 #cycle_time = on_time + off_time if conn.poll(): mode, cycle_time, duty_cycle, set_point, k_param, i_param, d_param = conn.recv() #conn.send([mode, cycle_time, duty_cycle]) #if mode == "manual": parent_conn_heat.send([cycle_time, duty_cycle])
class DataPicker: """ Class to flush data on a link, it continuously read the data received on a link, and return the last on get_data call. """ def __init__(self, pipe_in): self.pipe_in = pipe_in self.parent, self.child = Pipe() self.proc = Process(target=self.start, args=(self.child,)) self.proc.start() def start(self, child): try: while True: data_in = self.pipe_in.recv() # print "DATA_IN: ", data_in if child.poll(): data = child.recv() if data == "break": break child.send(data_in) except KeyboardInterrupt: self.close() def get_data(self): self.parent.send('ok') data = self.parent.recv() return data def close(self): self.parent.send("break")
class ClientProcess(Process): def __init__(self): Process.__init__(self) self.parent, self.child = Pipe() self.state = Lock() self.start() def run(self): while 1: active_clients = self.child.recv() self.state.acquire() client_socket = socket.fromfd(reduction.recv_handle(self.child), socket.AF_INET, socket.SOCK_STREAM) self.client_process(client_socket, active_clients) self.state.release() def client_process(self, sock, active_clients): while 1: data = protocol.recv(sock) if not data: return sock.close() encryption = TripleDESCipher(data["key"]) if data["encryption"] == 1: protocol.send(sock, (active_clients, encryption.encrypt(data["message"]))) else: protocol.send(sock, (active_clients, encryption.decrypt(data["message"]))) def add_task(self): if self.state.acquire(False): self.parent.send(True) self.state.release() return True return False
class gui: """ A multithreaded handler for the coincidence-count GUI """ def __init__(self): """ Constructor """ # Start up the GUI process and build the communication network self.pipe, their_pipe = Pipe() self.gui = Process(target=gui_head, name="gui_head", args=(their_pipe,)) self.gui.start() def collect(self): """ Collect all messages and act upon them """ messages = [] while self.pipe.poll(0): messages.append(self.pipe.recv()) return messages def send(self, key, value): """ Send a message to the threaded GUI """ self.pipe.send((key, value)) def kill(self): """ Send the message to shut down """ self.send("kill", None)
def main(): parent, child = Pipe() e = TestTask(control_pipe=child) uri = e.uris['rpc'] p = Process(target=e.run) try: p.start() z = ZmqJsonRpcProxy(uri.replace('*', 'localhost')) # Rather than calling the `hello_world` proxy method directly, we # instead call the `spawn` attribute of the proxy method. This causes # the proxy call to be performed using eventlet green threads. Here, # `d` is an `eventlet.event.Event` instance, which can be queried for # when the result is ready. The final result can be obtained by # calling the event's `wait` method. By calling `eventlet.sleep`, we # yield execution to the RPC green thread to check again for a # response. d = z.hello_world.spawn() while not d.ready(): print 'Something is happening in the "main" thread!' eventlet.sleep(0.5) result = d.wait() print 'result is ready:', result assert(result == 'hello, world') finally: parent.send('shutdown') time.sleep(0.1) p.terminate() del p
def calculateArea(feature,session_cookies,options): """ return:{ status { "invalid" : invalid message; "failed" : failed message; "overlapped" : overlap message } data: { total_area: 100 //exist if status_code = 1 other_area: 10 //exist if status_code = 1 and len(layers) > 0 layers: { //exist if status_code = 1 and len(layers) > 0 layer id: { total_area: 12 areas:[ {area:1, properties:{ name:value }} ] } } } } """ parent_conn,child_conn = Pipe(True) p = Process(target=calculateAreaInProcess,args=(child_conn,)) p.daemon = True p.start() parent_conn.send([feature,session_cookies,options]) result = parent_conn.recv() parent_conn.close() #p.join() #print("{}:get the area result from other process".format(datetime.now())) return result
class ProcessApplication(Application): """Run application class in subprocess.""" def __init__(self, reaktor, target, name=''): Application.__init__(self, None, name) self.target = target self._reaktor = reaktor self._process = None self.parent = None self.child = None self.transport = None self.runner = None def start(self): signal.signal(signal.SIGCHLD, self._sigchld) self.parent, self.child = Pipe(duplex=True) logging.debug("parent conn %d, child %d", self.parent.fileno(), self.child.fileno()) self.transport = ConnectedTransport(self._reaktor, self.parent, self) self.runner = _Subprocess(self.target, self.child, self.config) self._process = Process(target=self.runner.run) self._process.start() def _close(self, timeout): self._process.join(timeout) if self.transport is not None: # this closes also parent channel self.transport.close() self.transport.del_channel() self.child = self.parent = self.transport = None def _sigchld(self, signum, frame): self._close(0.5) def stop(self): self._process.terminate() self._close(2.0) def send(self, data): """Send data to application.""" self.parent.send(data) def recv(self, data): """Handle data received from subprocess application.""" if isinstance(data, Exception): self.log.error("Subprocess exception: %s", str(data)) raise data self.received(data) def event_readable(self, transport): assert(transport == self.transport) data = self.transport.socket.recv() self.log.debug("received from app: %s", str(data)) if data is not None: self.recv(data) def event_error(self, transport): typ, ex, tb = sys.exc_info() if typ != KeyboardInterrupt: self.log.debug("process: %s", str(ex))
def run(self): if not self.isRunnable(): print "Can't run thread '{}'".format(self.name) print self.target, self.EVT_ID, self.notifyWindow, self.process return try: parentPipe, childPipe = Pipe() #self.process = Process(target = self.target, args = (childPipe, self.args)) self.process = threading.Thread(target = self.target, name = "Serial communication thread", kwargs = {"pipe": childPipe, "args": self.args}) self.process.name = self.name self.process.daemon = True self.process.start() while self.process.isAlive() or parentPipe.poll(0.001): if parentPipe.poll(0.001): out = parentPipe.recv() wx.PostEvent(self.notifyWindow, ResultEvent(out, self.EVT_ID)) if self.stop: if self.stopSignalTime == 0: self.stopSignalTime = time() + 0.5 parentPipe.send([False]) if time() > self.stopSignalTime: self.process._Thread__stop() wx.YieldIfNeeded() #sleep(0.01) except OSError, e: wx.PostEvent(self.notifyWindow, ResultEvent(\ "Execution failed in thread '{}', message: {}".format(\ self.name, e), self.EVT_ID))
class StubExecuteTestsFunc: def __init__(self): self.main_conn, self.func_conn = Pipe() self._called = self._complete = None self.stub_reset() def stub_reset(self): self._called = self._complete = False def stub_complete(self): self._complete = True self.main_conn.send(StubExecuteTestsFuncConnMessages.COMPLETE) def stub_called(self): if not self._called and self.main_conn.poll(): conn_message = self.main_conn.recv() if conn_message == StubExecuteTestsFuncConnMessages.CALLED: self._called = True return self._called def __enter__(self): self.stub_reset() return self def __exit__(self, exc_type, exc_val, exc_tb): self.stub_complete() def __call__(self, *_): self._called = True self.func_conn.send(StubExecuteTestsFuncConnMessages.CALLED) while not self._complete: conn_message = self.func_conn.recv() if conn_message == StubExecuteTestsFuncConnMessages.COMPLETE: self._complete = True
class ActiveConn(object): """ Talks to the specific backend instance associated with a userid """ def __init__(self, userid, modules): self.userid = userid self.here, self.there = Pipe(duplex=True) self.proc = CoqProc() self.proc.start(modules) self.read() logging.debug("Coqtop Process started %s", self.proc) def read(self): """poll results from the coqtop backend""" res = None self.proc.run(self.there) if self.here.poll(): res = self.here.recv() logging.debug("Received content from process") return {'userid': self.userid, 'response': res} def send(self, data): """send results to our coqtop instance""" if self.proc.alive: logging.debug("sending stuff") self.here.send(data + " ") return True else: return False def quit(self): if self.proc.alive: return self.proc.terminate(True)
def recog_proc(self, child_recog: Pipe, e_recog: Event, yolo_type: str): """ Parallel process for object recognition Arguments: child_recog {Pipe} -- pipe for communication with parent process, sends bbox yolo type of recognized object e_recog {Event} -- event for indicating complete recognize in frame """ # initialize YOLO yolo = Yolo(yolo_type) e_recog.set() print("yolo defined") while True: frame = child_recog.recv() print("recog process frame recieved") if frame is None: print("FRAME NONE? R U SURE ABOUT THAT?!") return res = yolo.detect(frame, cvmat=True) print("recog send") e_recog.set() child_recog.send(res)
class ProcessStarter(object): def __init__(self): ''' Setup the shared memory data structure model and initialize the control parts. ''' self.running = True self.orprocess = None self.guiprocess = None self.pipeGUI, self.pipeOR = Pipe() self.StartProcesses() def StartProcesses(self): self.guiprocess = Process(target=self.__startGUI__) self.guiprocess.start() self.pipeGUI.send(["StartViewer", None]) self.orprocess = Process(target=ORServer,args=(self.pipeOR,)) self.orprocess.start() return True def terminate(self): try: self.pipeGUI.send(["Stop", None]) self.guiprocess.terminate() self.orprocess.terminate() except: pass def __startGUI__(self): app = QtGui.QApplication(sys.argv) form = pnpApp(self.pipeGUI, self) form.show() sys.exit(app.exec_())
def onExeBtn(self, event): srcFiles = self.projPane.getValue() self.analyzerOutDir = os.path.join(self.projRootDir, 'analyzer_output') oldcwd = os.getcwd() if not os.path.exists(self.analyzerOutDir): os.makedirs(self.analyzerOutDir) os.chdir(self.analyzerOutDir) rcv_pipe, snd_pipe = Pipe(duplex=True) self.dbname = ''.join([self.projRootDir.replace(os.sep, '_').strip('_'), '.sqlite']) self.exePane.dbPathTc.SetValue(os.path.join(self.analyzerOutDir, self.dbname)) self.exePane.ppLogFileTc.SetValue(os.path.join(self.analyzerOutDir, 'preprocessor.log')) self.exePane.parserLogFileTc.SetValue(os.path.join(self.analyzerOutDir, 'parser.log')) p = Process(target=analyze, args=(snd_pipe, os.path.join(self.analyzerOutDir, self.dbname), self.getPpCfg(), self.getParserCfg(), srcFiles, self.exePane.pipelineCb.GetValue(), self.exePane.numProcSc.GetValue(), self.exePane.numPpProcSc.GetValue(), self.exePane.numParserProcSc.GetValue(), )) p.start() dlg = wx.ProgressDialog('Executing', '0/%d' % len(srcFiles), parent=self, maximum=len(srcFiles)*10, style=wx.PD_CAN_ABORT | wx.PD_APP_MODAL | wx.PD_ELAPSED_TIME ) dlg.SetSize((500,150)) dlg.Layout() dlg.Show() result = None while True: i, total, result = rcv_pipe.recv() ret = dlg.Update(i*10, result if i == total else '[%d/%d] %s ... done' % (i+1, total, result)) if ret[0] == False: rcv_pipe.send('STOP') while result != 'STOPPED': result = rcv_pipe.recv() dlg.Update(total*10, 'Canceled') break if i == total: break p.join() self.exePane.dbPathTc.SetValue(os.path.join(self.analyzerOutDir, self.dbname)) os.chdir(oldcwd)
def main(): logging.basicConfig(level=logging.INFO) args = parse_args() print args echo_server = Process(target=EchoServer('ipc://ECHO:1', args.echo_delay).run) client = Process(target=JsonClient('ipc://PRODUCER_REP:1', 'ipc://PRODUCER_PUB:1', args.request_count, request_delay=args.request_delay).run) parent_pipe, child_pipe = Pipe(duplex=True) async_server_adapter = Process( target=AsyncJsonServerAdapter('ipc://ECHO:1', 'ipc://PRODUCER_REP:1', 'ipc://PRODUCER_PUB:1', child_pipe).run ) try: echo_server.start() async_server_adapter.start() client.start() client.join() parent_pipe.send('close') async_server_adapter.join() except KeyboardInterrupt: pass client.terminate() async_server_adapter.terminate() echo_server.terminate() # Since ipc:// creates files, delete them on exit cleanup_ipc_uris(('ipc://ECHO:1', 'ipc://PRODUCER_REP:1', 'ipc://PRODUCER_PUB:1'))
def master(): results = [] mcn, scn = Pipe() mon = Thread(target=monitor, args=(mcn, results)) mon.daemon = True mon.start() proc = Process(target=slave, args=(scn,)) proc.start() mcn.send('1') while not results: time.sleep(0.01) pid = results.pop() log.info('pid: %s', pid) mcn.send('die') try: for n in range(7): if mcn.poll(1): break if not proc.is_alive(): raise Exception('stop') except Exception: log.error('slave died', exc_info=True)
class ConsoleProxy(object): def __init__(self): self._read, self._write = Pipe(duplex=True) def fileno(self): return self._read.fileno() def read(self): data = self._read.recv() if data["type"] == "completer": result = command_root.match(data["line"], data["hints"]) elif data["type"] == "parser": try: result = command_root.parse(data["line"]) except Command.NotFound as e: if str(e) != "": result = str(e) else: result = "No such command '{:s}'".format(data["line"]) except Command.SyntaxError as e: result = str(e) else: result = "" self._read.send(result) def completer(self, line, hints): self._write.send({"type" : "completer", "line" : line, "hints" : hints}) return self._write.recv() def parser(self, line): self._write.send({"type" : "parser", "line" : line}) return self._write.recv()
def evaluate_expression(exp): ''' Evaluates given expression. ''' if not exp: return "No expression supplied." exp = str(exp) # Setup evaluation process if it's not present global eval_process, eval_pipe_parent, eval_pipe_child if not eval_process: eval_pipe_parent, eval_pipe_child = Pipe() eval_process = Process(name = "seejoo_eval", target = _eval_worker, args = (eval_pipe_child,)) eval_process.daemon = True eval_process.start() # Push expression through the pipe and wait for result eval_pipe_parent.send(exp) if eval_pipe_parent.poll(EVAL_TIMEOUT): res = str(eval_pipe_parent.recv()) res = filter(lambda x: ord(x) >= 32, res) # Sanitize result return res # Evaluation timed out; kill the process and return error os.kill(eval_process.pid, 9) os.waitpid(eval_process.pid, os.WUNTRACED) eval_process = None return "Operation timed out."
class MPPlugin(object): # this functionality needs to be implemented in the cython parent side def __init__(self): self.plot_pipe, plotter_pipe = Pipe() self.plotter = SimplePlotter(20000.) self.plot_process = Process(target=self.plotter, args=(plotter_pipe, )) self.plot_process.daemon = True self.plot_process.start() def bufferfunction(self, n_arr=None, finished=False): # print "entering plot" send = self.plot_pipe.send if finished: send(None) else: # print "sending data" if not n_arr: n_arr = np.random.random((11, 1000)) send({'data': n_arr}) while 1: if not self.plot_pipe.poll(): break e = self.plot_pipe.recv() print(e) def setparam(self, name, value): self.plot_pipe.send({'param': {name: value}})
class ProcessIA(): """ Classe qui va lancer une IA en subprocess. Gère aussi la communication avec les IA lancées via un pipe. """ def __init__(self, color_and_robot_list): self.__color = color_and_robot_list[0] if color_and_robot_list[1] is not None: self.__bigrobot = color_and_robot_list[1] if color_and_robot_list[2] is not None: self.__minirobot = color_and_robot_list[2] else: self.__minirobot = None else: self.__bigrobot =None if color_and_robot_list[2] is not None: self.__minirobot = color_and_robot_list[2] else: self.__minirobot = None self.__robots = color_and_robot_list self.__hokuyo = Hokuyo(self.__robots[1:]) self.__communication = Communication(self.__bigrobot, self.__minirobot, self.__hokuyo, self) #communication de data entre l'IA et le simu self.__parent_conn, self.__child_conn = Pipe() #lancement de l'ia if TEST_MODE == False: self.__process = Process(target=main.startIa, args=(self.__child_conn,self.__color)) elif TEST_MODE == True : self.__process = Process(target=test.testIa, args=(self.__child_conn,self.__color)) #pour les tests self.__process.start() time.sleep(0.1) #on démarre le thread de lecture des données IA renvoyées à travers le pipe self.__read_thread = threading.Thread(target=self.__readPipe) self.__read_thread.start() def writePipe(self, addresse, ordre, args): """ Envoie des données à l'IA à travers le Pipe """ self.__parent_conn.send((addresse, ordre, args)) def __readPipe(self): """ Méthode de lecture des données envoyées par l'IA via le pipe. recv est bloquant, donc lancé dans un thread """ while True: if self.__parent_conn.poll(): message = self.__parent_conn.recv() self.__parseDataIa(message) def __parseDataIa(self, data): """ Formate les données IA reçues pour les adapter à la méthode sendOrderAPI de Communication """ self.__communication.orderBalancing(data[0],data[1],data[2])
class Flow: def __init__ (self, debug_mode=False): from multiprocessing import Pipe, Process self.motions = [] self.debug_mode = debug_mode self.parent_conn, self.child_conn = Pipe() self.p = Process(target=self.do_motion_detect, args=(self.child_conn,)) self.p.start() # Send kill command to child process def kill (self): self.parent_conn.send('STOP') #self.parent_conn.recv() self.p.join() def _clear_pipe (self): while (self.parent_conn.poll(0.01) != False): reading = self.parent_conn.recv() self.motions.append(reading) # Checks if pipe has any data # If it does return that # Else return [] def get_motion (self): self._clear_pipe() if len(self.motions) == 0: return None else: copy = self.motions self.motions = [] return copy def do_motion_detect (self, child_conn): import picamera, subprocess, time camera = picamera.PiCamera(framerate=30) output = DetectMotion(camera) output.set_conn(child_conn) while True: try: #camera.resolution = (640, 480) camera.resolution = (200, 150) #camera.resolution = (240, 160) camera.start_recording('/dev/null', format='h264', motion_output=output) camera.wait_recording(0.25) #t = int(time.time()) #name = '{}-image.jpg'.format(t) name = 'image.jpg' os.remove(name) camera.capture(name, use_video_port=True) camera.stop_recording() if child_conn.poll(0.01): s = child_conn.recv() if s == 'STOP': break except KeyboardInterrupt: break camera.close()
def data_handler(spigot_data: SpigotData, lock: threading.Lock): """ Main thread for reading output from spigot IO worker. Also interprets data. spigot_data - shared SpigotData object """ client, child = Pipe() t = Process(target=r_w_worker, args=(child, spigot_data.close_event)) t.start() spigot_data.game.status = SpigotState.RUNNING running = True while running: while t.is_alive(): if client.poll(0.3): buf = client.recv() parse_event(spigot_data, buf) spigot_data.add_message(buf) logging.debug('<<OUTPUT>> {}'.format(buf)) while not spigot_data.commands.empty(): command = spigot_data.commands.get() client.send(command) logging.debug('<<COMMAND>> {}'.format(command)) # After java process is dead spigot_data.status = SpigotState.STOPPED spigot_data.add_message(info_message("""The server has stopped. Type 'start' to start. """ """Type 'quit' to close Spigot Monitor""")) # PEP8ers gonna hate spigot_data.game.players = {} # No players are available on a stopped server... while True and not AUTO_RESTART: command = spigot_data.commands.get().strip() # strip because commands have newline appended if command.lower() == 'start': t = Process(target=r_w_worker, args=(child, spigot_data.close_event)) t.start() logging.debug('Thread created.') break elif command.lower() == 'quit' or command.lower() == 'stop': message = info_message("KTHXBAI") spigot_data.add_message(message) logging.debug('Quitting program.') break if AUTO_RESTART: t = Process(target=r_w_worker, args=(child, spigot_data.close_event)) t.start() logging.debug('Thread created.') if not t.is_alive(): # thread hasn't started again running = False print('exiting data_handler loop')
class _PipeHandler(threading.Thread): """ Each PipeHandler holds an open pipe to a subprocess, to allow the sub-process to access the accounts and communicate status information. """ def __init__(self, account_manager): threading.Thread.__init__(self) self.daemon = True self.accm = account_manager self.to_child, self.to_parent = Pipe() def _send_account(self, account): if account is None: self.to_child.send(account) return response = ( account.__hash__(), account.get_name(), account.get_password(), account.get_authorization_password(), account.get_key(), ) self.to_child.send(response) def _handle_request(self, request): try: command, arg = request if command == "acquire-account-for-host": account = self.accm.acquire_account_for(arg, self) self._send_account(account) elif command == "acquire-account-from-hash": account = self.accm.get_account_from_hash(arg) if account is not None: account = self.accm.acquire_account(account, self) self._send_account(account) elif command == "acquire-account": account = self.accm.acquire_account(owner=self) self._send_account(account) elif command == "release-account": account = self.accm.get_account_from_hash(arg) account.release() self.to_child.send("ok") elif command == "log-add": log = _call_logger("add_log", *arg) self.to_child.send(log) elif command == "log-message": _call_logger("log", *arg) elif command == "log-aborted": _call_logger("log_aborted", *arg) elif command == "log-succeeded": _call_logger("log_succeeded", *arg) else: raise Exception("invalid command on pipe: " + repr(command)) except Exception, e: self.to_child.send(e) raise
def _proxy_loop(self, broker, *args): is_debug = partial(log.isEnabledFor, logging.DEBUG) pid = os.getpid() proc = None queue = self.queue def stop(): try: if proc is not None: child.send(STOP) child.close() proc.join() except Exception: log.error('%s failed to stop cleanly', str(self), exc_info=True) raise else: log.debug('terminated %s', str(self)) finally: self.pid = '%s-terminated' % self.pid while True: if proc is None or not proc.is_alive(): # start new worker process child, parent = Pipe() cx = _reduce_connection(parent) # HACK reduce for pickle proc = run_in_subprocess(worker_process, pid, cx, *args) self.pid = proc.pid task, return_to_pool = queue.get() if task == STOP: stop() break try: child.send(task) while not child.poll(task.heartrate): if not proc.is_alive(): broker.task_failed(task) raise Error('unknown cause of death') broker.heartbeat(task) (result, status) = child.recv() broker.set_result(task, result) except Exception: log.error('%s died unexpectedly', str(self), exc_info=True) child.close() proc.stdin.close() proc = None else: if is_debug(): log.debug('%s completed task', str(self)) if status == STOP: child.close() proc.stdin.close() proc = None finally: return_to_pool(self)
def __call__(self, *args, **kwargs): parent, child = Pipe() p = Process(target=worker, args=(self._function, child, 1)) p.daemon = True p.start() parent.send((args, kwargs)) child.close() return Task(next(self._counter), p, parent, self.callback, self.timeout)
class _Tracker(Process): """Background process for tracking resource usage""" def __init__(self, dt=1): Process.__init__(self) self.daemon = True self.dt = dt self.parent_pid = current_process().pid self.parent_conn, self.child_conn = Pipe() def shutdown(self): if not self.parent_conn.closed: self.parent_conn.send('shutdown') self.parent_conn.close() self.join() def _update_pids(self, pid): return [self.parent] + [p for p in self.parent.children() if p.pid != pid and p.status() != 'zombie'] def run(self): psutil = import_required("psutil", "Tracking resource usage requires " "`psutil` to be installed") self.parent = psutil.Process(self.parent_pid) pid = current_process() data = [] while True: try: msg = self.child_conn.recv() except KeyboardInterrupt: continue if msg == 'shutdown': break elif msg == 'collect': ps = self._update_pids(pid) while not data or not self.child_conn.poll(): tic = default_timer() mem = cpu = 0 for p in ps: try: mem2 = p.memory_info().rss cpu2 = p.cpu_percent() except Exception: # could be a few different exceptions pass else: # Only increment if both were successful mem += mem2 cpu += cpu2 data.append((tic, mem / 1e6, cpu)) sleep(self.dt) elif msg == 'send_data': self.child_conn.send(data) data = [] self.child_conn.close()
class guiconnector(): def __init__(self, console, observer, modelsready): self.console = console self.observer = observer self.modelsready = modelsready # Starts up data structures def start(self, params): self.a, self.b = Pipe(duplex=True) if __name__ == "projectb.maingui": self.p = Process(target=BayesianOptProcess, kwargs={ "params": parseintosimple(params), "pipein": self.a, "pipeout": self.b }) self.t = threading.Thread(target=self.bayesoptlistener) self.t.start() self.p.start() # Listnere for data from the BayesOpt Process def bayesoptlistener(self): pipein = self.a pipeout = self.b stop = False; while not stop: if pipein.poll(): output = pipein.recv() if output.has_key("stop_connector"): break if output.has_key("stopped_bayes"): self.modelsready(output["stopped_bayes"]) for k, v in output.items(): if k == "console": self.console.log(v["text"],v["verbose"]) else: self.observer.updatevar(k, v) # Request the BayesOpt Process for the posterior of the given model def queryposterior(self, xdata, modelid): self.a.send({"posterior": xdata, "modelid": modelid}) # Finish BayesOpt def endbayesopt(self): self.a.send({"stop": True, "stop_posterior_connector": True}) self.console.log("Attempting to stop the Bayesian Optimization") self.console.log("Please Wait...") self.console.log("Saving the data...") # Clear up the processes and close the pipes def closestage(self): self.b.send({"stop_connector": True}) self.endbayesopt() self.a.close() self.b.close() self.p.terminate()
def test_sending_raw(self): server_conn, worker_conn = Pipe() a, b = Pipe() process = Process(target=echo, args=(worker_conn,)) process.start() server_conn.send(b) self.assertEqual('error', server_conn.recv()) process.terminate()
def start_new_process(self, process_finished_notify, semaphore, target_method, task, build_config): result_val = Value(c_char_p, '') task_conn, task_conn_remote = Pipe() config_conn, config_conn_remote = Pipe() p = Process(target=target_method, args=[semaphore, process_finished_notify, task_conn_remote, config_conn_remote, result_val]) p.daemon = True logging.debug("Sending task: %s", task.name) task_conn.send(task) config_conn.send(build_config) task.result_proxy = result_val p.start()
def resource_test(self): response = ({'status': 200, 'content-type': 'text/plain; charset=utf-8'}, \ 'Hello') r = Response(response) pipe, child_pipe = Pipe() lazy_response = LazyResponse(child_pipe) pipe.send(r) assert lazy_response.resource() == 'Hello'
def simple_test(self): response = ({'status': 200}, 'Hello') r = Response(response) pipe, child_pipe = Pipe() lazy_response = LazyResponse(child_pipe) pipe.send(r) assert lazy_response.code == 200 assert lazy_response.body == 'Hello'
class AstroPrintPipeline(object): def __init__(self, device, size, source, encoding, onFatalError): self._logger = logging.getLogger(__name__) self._parentConn, self._processConn = Pipe(True) self._pendingReqs = {} self._lastReqId = 0 self._device = device self._source = source.lower() self._encoding = encoding.lower() self._size = tuple([int(x) for x in size.split('x')]) self._sendCondition = Condition() self._onFatalError = onFatalError self._responseListener = ProcessResponseListener( self._parentConn, self._onProcessResponse) self._responseListener.start() self._process = None self._listening = False def __del__(self): self._logger.debug('Pipeline Process Controller removed') def _kill(self): try: os.kill(self._process.pid, signal.SIGKILL) self._process.join() except OSError as e: # error 3: means the pid is not valid, # so the process has been killed if e.errno != 3: raise e self._process = None def startProcess(self): if self._process: # This should almost never happen (but it does) # Make sure the previous process is killed before a new one is started self._logger.warn( "A previous process was still running, killing it") self._kill() onListeningEvent = Event() # If True, it means the process had an error while starting errorState = Value('b', False) self._listening = False self._process = Process(target=startPipelineProcess, args=(self._device, self._size, self._source, self._encoding, onListeningEvent, errorState, (self._parentConn, self._processConn))) self._process.daemon = True self._process.start() if onListeningEvent.wait(5.0): if errorState.value: self._logger.error('Pipeline Failed to start.') self._kill() self._logger.debug('Pipeline Process killed.') else: self._logger.debug('Pipeline Process Started.') self._listening = True else: self._logger.debug( 'Timeout while waiting for pipeline process to start') self._kill() def stopProcess(self): if self._process: if self._listening: self._sendReqToProcess({'action': 'shutdown'}) self._process.join( 2.0) #Give it two seconds to exit and kill otherwise if self._process.exitcode is None: self._logger.warn( 'Process did not shutdown properly. Terminating...') self._process.terminate() # Give it another two secods to terminate, otherwise kill self._process.join(2.0) if self._process.exitcode is None: self._logger.warn('Process did not terminate properly. ' 'Sending KILL signal...') self._kill() self._logger.debug('Process terminated') self._process = None @property def processRunning(self): return self._process and self._process.is_alive() def stop(self): self._responseListener.stop() self.stopProcess() # It's possible that stop is called as a result of a response which is # executed in the self._responseListener Thread. # You can't join your own thread! if current_thread() != self._responseListener: self._responseListener.join() self._responseListener = None def startVideo(self, doneCallback=None): self._sendReqToProcess({'action': 'startVideo'}, doneCallback) def stopVideo(self, doneCallback=None): self._sendReqToProcess({'action': 'stopVideo'}, doneCallback) def isVideoPlaying(self, doneCallback): self._sendReqToProcess({'action': 'isVideoPlaying'}, doneCallback) def takePhoto(self, doneCallback, text=None): def postprocesing(resp): if resp: if 'error' in resp: self._logger.error('Error during photo capture: %s' % resp['error']) doneCallback(None) else: from base64 import b64decode try: doneCallback(b64decode(resp)) except TypeError as e: self._logger.error( 'Invalid returned photo. Received. Error: %s' % e) doneCallback(None) else: doneCallback(None) if text is not None: self._sendReqToProcess( { 'action': 'takePhoto', 'data': { 'text': text } }, postprocesing) else: self._sendReqToProcess({ 'action': 'takePhoto', 'data': None }, postprocesing) def _onProcessResponse(self, id, data): if id is 0: # this is a broadcast, likely an error. Inform all pending requests self._logger.warn( 'Broadcasting error to ALL pending requests [ %s ]' % \ repr(data)) if self._pendingReqs: for cb in self._pendingReqs.values(): if cb: cb(data) if data and 'error' in data and data['error'] == 'fatal_error': message = 'Fatal error occurred in video streaming (%s)' % \ data['details'] if 'details' in data else 'unkonwn' #signaling for remote peers manage_fatal_error_webrtc = blinkerSignal( 'manage_fatal_error_webrtc') manage_fatal_error_webrtc.send(self, message=message) #event for local peers eventManager().fire(Events.GSTREAMER_EVENT, {'message': message}) try: self._logger.info( "Trying to get list of formats supported by your " "camera...") self._logger.info( subprocess.Popen( "v4l2-ctl --list-formats-ext -d %s" % \ str(self._device), shell=True, stdout=subprocess.PIPE ).stdout.read()) except: self._logger.error("Unable to retrieve supported formats") #shutdown the process self._pendingReqs = {} self._onFatalError() elif id in self._pendingReqs: try: callback = self._pendingReqs[id] if callback: callback(data) del self._pendingReqs[id] if self._logger.isEnabledFor(logging.DEBUG): if sys.getsizeof(data) > 50: dataStr = "%d bytes" % sys.getsizeof(data) else: dataStr = repr(data) self._logger.debug('Response for %d handled [ %s ]' % (id, dataStr)) except Exception: self._logger.error("Problem executing callback response", exc_info=True) else: self._logger.error("There's no pending request for response %d" % id) def _sendReqToProcess(self, data, callback=None): if self.processRunning: with self._sendCondition: if self._listening: self._lastReqId += 1 id = self._lastReqId self._pendingReqs[id] = callback self._parentConn.send((id, data)) self._logger.debug('Sent request %d to process [ %s ]' % (id, repr(data))) else: self._logger.debug( 'Process not listening. There was a problem ' 'while starting it.') if callback: callback({ 'error': 'not_listening', 'details': 'The process is not currently ' 'listening to requests' }) else: self._logger.debug('Process not running. Trying to restart') self.startProcess() if self.processRunning: self._sendReqToProcess(data, callback) else: self._logger.error('Unable to re-start pipeline process.') if callback: callback({ 'error': 'no_process', 'details': 'Unable to re-start process' })
class EngDisplay: def __init__(self, src=None, use_light_theme=False, repeat_log=False): self.colors = EngColors(use_dark=(not use_light_theme)) self.parent_conn, self.child_conn = Pipe() self.parent_conn_serial_out, self.child_conn_serial_out = Pipe() self.data_src = src self.main = Main(src, multi_pipe=self.child_conn, serial_pipe=self.child_conn_serial_out, repeat_log=repeat_log) if src is None: self.using_serial = True else: self.using_serial = False self.proc = Process(target=self.main.run) self.proc.start() # self.width = 700 # self.height = 700 self.move_amt = 20 self.meas_to_map = 1 / 1000 self.universal_scale = 2 self.start_pos = [] self.measuring = False self.cur_line = None self.cur_line_txt = None self.closed = False self.popup_active = False self.cycle_counter = 0 self.paused = True self.not_cleared = True self.tk_version = tk.TclVersion self.create_eng_display() def create_eng_display(self): self.window = tk.Tk() self.myframe = Frame(self.window) self.myframe.pack(fill=BOTH, expand=YES) w, h = self.window.winfo_screenwidth(), self.window.winfo_screenheight( ) self.canvas = ResizingCanvas(self.myframe, width=w, height=h, borderwidth=0, bg=self.colors.background, highlightthickness=0) self.window.wm_title(f"MNSLAC Engineering Display") self.canvas.pack(fill=BOTH, expand=YES) self.zoom(3) # Add menu # self.menu_bar = Menu(self.window) # self.file_menu = Menu(self.menu_bar, tearoff=0) # self.file_menu.add_command(label="Exit", command=self.window.quit, accelerator="Cmd+q") # self.menu_bar.add_cascade(label="File", menu=self.file_menu) # self.help_menu = Menu(self.menu_bar, tearoff=0) # self.menu_bar.add_cascade(label="Help", menu=self.help_menu) # self.window.config(menu=self.menu_bar) # Bind these functions to motion, press, and release self.canvas.bind('<Motion>', self.measure) self.canvas.bind('<Button-1>', self.start_measure) self.canvas.bind('<Button-3>', lambda e: self.zoom(0.9)) self.canvas.bind('<Button-2>', lambda e: self.zoom(0.9)) self.window.bind('<Up>', lambda e: self.canvas.move_by(0, self.move_amt)) self.window.bind('<Down>', lambda e: self.canvas.move_by(0, -self.move_amt)) self.window.bind('<Left>', lambda e: self.canvas.move_by(self.move_amt, 0)) self.window.bind('<Right>', lambda e: self.canvas.move_by(-self.move_amt, 0)) self.window.bind('<Shift-Up>', lambda e: self.canvas.move_by(0, self.move_amt * 5)) self.window.bind('<Shift-Down>', lambda e: self.canvas.move_by(0, -self.move_amt * 5)) self.window.bind('<Shift-Left>', lambda e: self.canvas.move_by(self.move_amt * 5, 0)) self.window.bind('<Shift-Right>', lambda e: self.canvas.move_by(-self.move_amt * 5, 0)) self.canvas.bind('<ButtonRelease-1>', self.stop_measure) self.window.protocol("WM_DELETE_WINDOW", self.close_callback) # self.canvas.addtag_all("bg") # Create crosshairs in background scale = 25 * 1000 # Distance in mm between crosshairs number = 30 # Total number of crosshairs to be drawn min_val = -scale * number max_val = scale * number length = scale * number * self.universal_scale * self.meas_to_map for x in range(min_val, max_val, scale): x_tmp = x * self.universal_scale * self.meas_to_map + self.canvas.x_pos self.canvas.create_line(int(x_tmp), int(length), int(x_tmp), int(-length), fill=self.colors.crosshair, tags="obj-bg", dash=(3, 10)) # 1D1F25 for y in range(min_val, max_val, scale): y_tmp = y * self.universal_scale * self.meas_to_map + self.canvas.y_pos self.canvas.create_line(int(length), int(y_tmp), int(-length), int(y_tmp), fill=self.colors.crosshair, tags="obj-bg", dash=(3, 10)) # Create the MNSLAC icon mnslac_icon = PhotoImage(file=self.colors.mnslac_logo) self.icon = self.canvas.create_image(w - 10, 10, image=mnslac_icon, anchor=NE) # Create the hz update text self.update_hz = self.canvas.create_text( 10, 10, text="0.0 Hz", fill=self.colors.text, font=font.Font(family=self.colors.data_font, size=self.colors.text_size_large), anchor=tk.NW) # Create the no_connection rectangle and text self.no_connection_rect = self.canvas.create_rectangle( -300, h / 3 - 50, 300 + w, h / 3 + 50, fill=self.colors.background_accent, tag="del") self.no_connection = self.canvas.create_text( w / 2, h / 3, text="NO CONNECTION", fill=self.colors.text_warn, font=font.Font(family=self.colors.data_font, size=self.colors.text_size_xlarge), anchor=tk.CENTER, tag="del") # Create the connection details text based on whether or not we are using a serial connection or log file if self.using_serial is False: self.no_connection_details = self.canvas.create_text( w / 2, h / 3 + 20, text="Begin playback of log file by clicking 'Play'", fill=self.colors.text_details, font=font.Font(family=self.colors.data_font, size=self.colors.text_size_medium), anchor=tk.CENTER, tag="del") msg = f"Loaded from '{self.data_src}'" else: self.no_connection_details = self.canvas.create_text( w / 2, h / 3 + 20, text="Connect NODE and/or ensure proper serial configuration", fill=self.colors.text_details, font=font.Font(family=self.colors.data_font, size=self.colors.text_size_medium), anchor=tk.CENTER, tag="del") serial_port = config.SERIAL_PORT saving_log = self.main.log_file_name if saving_log is None: msg = f"Listening to Serial '{serial_port}', no logging'" else: msg = f"Listening to Serial '{serial_port}', logging to '{saving_log}'" self.file_details = self.canvas.create_text( 10, h - 10, text=f"{msg}", fill=self.colors.text, font=font.Font(family=self.colors.data_font, size=self.colors.text_size_small), anchor=tk.SW) # Initialize the main canvas if self.main.kill or not self.update_frame(): print('Error initializing the main canvas') return # Create the details canvas self.dh = 370 self.dw = 350 self.details = Canvas(self.window, width=self.dw, height=self.dh, bg=self.colors.background_accent, highlightthickness=0, borderwidth=0) self.details.create_rectangle(2, 2, self.dw - 4, self.dh - 4, fill="", outline=self.colors.pinstripe, dash=(1, 5)) self.d_title = self.details.create_text(20, 20, text="Node Details:", fill=self.colors.text_details, font=font.Font( family='Courier New', size=14), anchor=tk.NW) # Create the list for node details based on the node_list from main self.node_details_list = {} self.connection_list = {} y_pos = 50 for node_id, node_obj in self.main.node_list.items(): if node_obj.is_base: continue # Create the text objects for later use self.node_details_list[node_id] = { 'txt_id': self.details.create_text(20, y_pos, text=node_obj.name, fill=self.colors.text_details, font=font.Font(family='Courier New', size=12), anchor=tk.NW), 'name': node_obj.name, 'bat': 0, 'temp': 0, 'heading': 0, 'speed': 0 } y_pos += 20 # Create the node ranging section y_pos += 20 self.details.create_text(20, y_pos, text="Ranging List", fill=self.colors.text_details, font=font.Font(family='Courier New', size=14), anchor=tk.NW) y_pos += 30 x_pos = 20 y_pos_o = y_pos rows_per_col = 5 row_counter = 0 for name in self.main.name_arr: id1, id2 = name.split('-') n1 = self.main.node_list[id1].name n2 = self.main.node_list[id2].name n1 = n1[0] + n1.split()[1] n2 = n2[0] + n2.split()[1] self.connection_list[name] = { "counter": 0, "node_obj1": self.main.node_list[id1], "node_obj2": self.main.node_list[id2], "name": n1 + "↔" + n2, "txt_id": self.details.create_text(x_pos, y_pos, text=n1 + "↔" + n2 + ": 0", fill=self.colors.text_details, font=font.Font(family='Courier New', size=12), anchor=tk.NW) } y_pos += 20 row_counter += 1 if row_counter == rows_per_col: row_counter = 0 x_pos += 100 y_pos = y_pos_o # Skip below the ranging section y_pos = y_pos_o + 20 * rows_per_col + 30 x_pos = 75 # Create the section containing buttons, based on whether or not the data source is a file or serial connection if self.using_serial: # The data source is a serial connection, create the sleep and reset buttons button2 = Button(self.window, command=lambda: self.popup_msg(type="sleep"), text="Sleep (Seconds)", width=13, anchor=tk.CENTER, highlightbackground=self.colors.background_accent, bd=0, highlightthickness=0, relief=tk.FLAT) button2_window = self.details.create_window(x_pos, y_pos, anchor=tk.CENTER, window=button2) x_pos += 74 y_pos -= 15 self.sleep_time_entry = Entry( self.window, highlightbackground=self.colors.background_accent, bg=self.colors.entry_background, bd=2) e1_window = self.details.create_window( x_pos, y_pos, anchor=tk.NW, window=self.sleep_time_entry) y_pos += 44 x_pos -= 75 button1 = Button(self.window, command=lambda: self.popup_msg(type="reset"), text="Reset Network", width=13, anchor=tk.CENTER, highlightbackground=self.colors.background_accent, bd=0, highlightthickness=0, relief=tk.FLAT) button1_window = self.details.create_window(x_pos, y_pos, anchor=tk.CENTER, window=button1) else: # The data source is a file, create the play/pause button and playback speed slider y_pos += 10 self.play_pause_button_string = tk.StringVar() self.pause_play_button = Button( self.window, command=self.play_pause, textvariable=self.play_pause_button_string, width=13, anchor=tk.CENTER, highlightbackground=self.colors.background_accent, bd=0, highlightthickness=0, relief=tk.FLAT) self.play_pause_button_string.set("Play") button2_window = self.details.create_window( x_pos, y_pos, anchor=tk.CENTER, window=self.pause_play_button) x_pos += 75 y_pos -= 1 self.update_hz_limit_scale = Scale( self.window, from_=0, to=1, resolution=0.001, command=self.update_refresh_hz, orient=HORIZONTAL, troughcolor=self.colors.slider_trough, borderwidth=0, length=175, width=15, relief=tk.FLAT, activebackground="#AAA", sliderrelief=tk.FLAT, showvalue=False, fg="#FFF") self.update_hz_limit_scale.set(1) w_window = self.details.create_window( x_pos, y_pos, anchor=tk.W, window=self.update_hz_limit_scale) self.details.create_text(x_pos, y_pos - 15, text="Slow", fill=self.colors.text_details, font=font.Font(family='Courier New', size=10), anchor=tk.W) self.details.create_text(x_pos + 175, y_pos - 15, text="Fast", fill=self.colors.text_details, font=font.Font(family='Courier New', size=10), anchor=tk.E) self.update_hz_target_text = self.details.create_text( x_pos + 88, y_pos + 18, text="≈60Hz", fill=self.colors.text_details, font=font.Font(family='Courier New', size=10), anchor=tk.CENTER) self.canvas.assign_resize_callback(self.resize_event) # self.proc.terminate() self.main_loop() def popup_msg(self, type="reset"): if self.popup_active: return valid = False message = "" target = None if type == "reset": valid = True message = "Are you sure you want to reset?" target = self.reset_network elif type == "sleep": try: self.sleep_time = int(self.sleep_time_entry.get()) self.sleep_time_entry.config(fg=self.colors.text) message = "Sleep for {} seconds OK?".format( int(self.sleep_time)) target = self.sleep_network valid = True except: self.sleep_time_entry.config(fg=self.colors.text_error) print('Invalid value for sleep time: {}'.format( self.sleep_time_entry.get())) else: print('Unknown pop message type: {}'.format(type)) return if valid: ph = 80 pw = 270 self.popup = Canvas(self.window, width=pw, height=ph, bg=self.colors.background_dialog, highlightthickness=0, borderwidth=0) w, h = self.window.winfo_screenwidth( ), self.window.winfo_screenheight() self.popup.place(x=w / 2 - pw / 2, y=h / 3 - ph / 2) self.popup.create_rectangle(1, 1, pw - 3, ph - 3, fill="", outline="#888", dash=(1, 5)) self.popup.create_rectangle(0, 0, pw, 20, fill=self.colors.background_accent, outline="") self.popup.create_polygon([-1, ph, 4, ph, -1, ph - 5], outline='', fill=self.colors.background, width=0) self.popup.create_polygon([pw + 1, ph, pw - 4, ph, pw + 1, ph - 5], outline='', fill=self.colors.background, width=0) self.popup.create_polygon([-1, -1, 4, -1, -1, 5], outline='', fill=self.colors.background, width=0) self.popup.create_polygon([pw + 1, -1, pw - 4, -1, pw + 1, 5], outline='', fill=self.colors.background, width=0) self.popup.create_text(pw / 2, 11, text="NOTICE", fill=self.colors.text_notice, font=font.Font(family='Courier New', size=14), anchor=tk.CENTER) self.popup.create_text(pw / 2, 35, text=message, fill=self.colors.text_invert, font=font.Font(family='Helvetica', size=14), anchor=tk.CENTER) yes_button = Button( self.window, command=target, text="Yes", width=13, anchor=tk.CENTER, highlightbackground=self.colors.background_dialog, bd=0, highlightthickness=0, relief=tk.FLAT) yes_button_window = self.popup.create_window(70, ph - 20, anchor=tk.CENTER, window=yes_button) no_button = Button( self.window, command=self.destroy_popup, text="No", width=13, anchor=tk.CENTER, highlightbackground=self.colors.background_dialog, bd=0, highlightthickness=0, relief=tk.FLAT) no_button_window = self.popup.create_window(pw - 70, ph - 20, anchor=tk.CENTER, window=no_button) self.popup_active = True self.ph = ph self.pw = pw def destroy_popup(self): if self.popup_active: self.popup.destroy() self.popup_active = False def reset_network(self): self.destroy_popup() self.parent_conn_serial_out.send({'cmd': 'reset'}) def sleep_network(self): self.destroy_popup() self.parent_conn_serial_out.send({ 'cmd': 'sleep', 'time': self.sleep_time }) def play_pause(self): print('Play Pause!') if self.using_serial is False: self.paused = not self.paused if self.paused: self.parent_conn_serial_out.send({'cmd': 'pause'}) # self.details.itemconfig(self.pause_play_button, text="Play") self.play_pause_button_string.set("Play") else: self.parent_conn_serial_out.send({'cmd': 'play'}) # self.details.itemconfig(self.pause_play_button, text="Pause") self.play_pause_button_string.set("Pause") # 0 -> 1 # 0.5 -> 5 # 1 -> 60 def update_refresh_hz(self, value): try: refresh_hz = float(value) if refresh_hz <= 0.5: refresh_hz = refresh_hz * 8 + 1 else: refresh_hz = (refresh_hz - 0.5) * 110 + 5 if refresh_hz == 60: refresh_hz = 0 # Sets unlimited hz self.details.itemconfig(self.update_hz_target_text, text="∞".format(float(refresh_hz)), font=font.Font( family=self.colors.data_font, size=22)) else: self.details.itemconfig( self.update_hz_target_text, text="≈{:.2f}Hz".format(float(refresh_hz)), font=font.Font(family=self.colors.data_font, size=10)) self.parent_conn_serial_out.send({ 'cmd': 'set_speed', 'speed': refresh_hz }) except Exception as e: pass def resize_event(self, event): self.details.place(x=event.width - (self.dw + 25), y=event.height - (self.dh + 25)) if self.popup_active is True: self.popup.place(x=event.width / 2 - self.pw / 2, y=event.height / 3 - self.ph / 2) coords = self.canvas.coords(self.icon) self.canvas.move(self.icon, (event.width - 10) - coords[0], 0) coords = self.canvas.coords(self.file_details) self.canvas.move(self.file_details, 0, (event.height - 10) - coords[1]) if self.not_cleared: coords = self.canvas.coords(self.no_connection) self.canvas.move(self.no_connection, event.width / 2 - coords[0], event.height / 3 - coords[1]) coords = self.canvas.coords(self.no_connection_details) self.canvas.move(self.no_connection_details, event.width / 2 - coords[0], event.height / 3 + 20 - coords[1]) coords = self.canvas.coords(self.no_connection_details) self.canvas.move(self.no_connection_details, event.width / 2 - coords[0], event.height / 3 + 20 - coords[1]) coords = self.canvas.coords(self.no_connection_rect) self.canvas.move(self.no_connection_rect, -300 - coords[0], event.height / 3 - 40 - coords[1]) def close_callback(self): self.window.destroy() self.closed = True print('Window Closed!') self.proc.terminate() def main_loop(self): frame_end = False receiving = True last_update = 0 message_timer = 0 try: while True: if time.time() - message_timer > 0.5: message_timer = time.time() self.draw_update_hz(int(self.cycle_counter / 0.5)) self.cycle_counter = 0 if frame_end is True: if not self.update_frame(): return last_update = time.time() # self.clear_canvas() frame_end = False elif time.time() - last_update > 0.03333333333: if not self.update_frame(): return last_update = time.time() while receiving is True: if self.parent_conn.poll(): msg = self.parent_conn.recv() if type(msg) == dict and "cmd" in msg: if "args" not in msg: continue if msg['cmd'] == "frame_start": frame_end = False self.clear_canvas() self.cycle_counter += 1 elif msg['cmd'] == "frame_end": frame_end = True break elif msg['cmd'] == "clear_screen": self.clear_canvas() elif msg['cmd'] == "draw_circle": self.draw_circle(msg['args']) elif msg['cmd'] == "connect_points": self.connect_points(msg['args']) elif msg['cmd'] == "status_update": self.status_update(msg['args']) elif msg['cmd'] == "report_communication": self.report_communication(msg['args']) elif msg['cmd'] == "clear_connection_list": self.clear_connection_list(msg['args']) else: print(f"Unknown command: {msg['cmd']}") else: print(msg) else: receiving = False receiving = True except tk.TclError: print('Close detected. Exit!') exit() # Interactive features def draw_update_hz(self, hz_value): txt = "{} Hz".format(hz_value) self.canvas.itemconfig(self.update_hz, text=txt) def update_frame(self): try: self.window.update_idletasks() self.window.update() except: return False return True def measure(self, event): # txt = "Coordinates: ({}, {}) meters".format(round(x), round(y)) # self.details.itemconfig(self.d_mouse, text=txt) # Check to see if we are measuring if self.measuring: # Try to remove the old elements try: event.widget.delete(self.cur_line) event.widget.delete(self.cur_line_txt) except: pass (x, y) = self.translate_screen_pos_to_canvas_pos(event.x, event.y) x = x + self.canvas.x_pos y = y + self.canvas.y_pos # Calculate the rotation between the two points rotation = 180 - math.degrees( math.atan2(self.start_pos[1] - y, self.start_pos[0] - x)) # Normalize the rotation if 90 < rotation < 270: rotation -= 180 # Convert to radians rrotation = math.radians(rotation) # Calculate mid point + rotation offset midx = (self.start_pos[0] + x) / 2 - math.sin(rrotation) * 10 midy = (self.start_pos[1] + y) / 2 - math.cos(rrotation) * 10 # Calculate distance dist_num = math.sqrt( (self.start_pos[0] - x)**2 + (self.start_pos[1] - y)**2) / self.universal_scale # Calculate distance string dist = '{:.0f}m'.format(dist_num) # Create the text self.cur_line_txt = event.widget.create_text( midx, midy, text=dist, fill=self.colors.text, font=font.Font(family=self.colors.data_font, size=self.colors.text_size_large), justify=tk.LEFT, angle=rotation) # Create the line self.cur_line = event.widget.create_line( self.start_pos[0], self.start_pos[1], x, y, fill=self.colors.measure_line_color, dash=self.colors.measure_line_dash, arrow=tk.BOTH, width=self.colors.measure_line_width) def shrink(self, scale, x=None, y=None): if x is None or y is None: x = self.window.winfo_pointerx() - self.window.winfo_rootx() y = self.window.winfo_pointery() - self.window.winfo_rooty() # (x, y) = self.translate_screen_pos_to_canvas_pos(0, 0) # x = x + self.canvas.x_pos # y = y + self.canvas.y_pos # print(x, y) # x = 0 # y = 0 # self.canvas.scale("obj", x, y, scale, scale) # self.canvas.scale("obj-bg", x, y, scale, scale) old_scale = self.universal_scale self.universal_scale *= scale self.canvas.scale("obj", self.canvas.x_pos, self.canvas.y_pos, scale, scale) self.canvas.scale("obj-bg", self.canvas.x_pos, self.canvas.y_pos, scale, scale) def translate_screen_pos_to_canvas_pos(self, x, y): return x - self.canvas.x_pos - self.canvas.x_offset, y - self.canvas.y_pos - self.canvas.y_offset def translate_canvas_pos_to_screen_pos(self, x, y): return x + self.canvas.x_pos + self.canvas.x_offset, y + self.canvas.y_pos + self.canvas.y_offset def start_measure(self, event): # Save the initial point (x, y) = self.translate_screen_pos_to_canvas_pos(event.x, event.y) x = x + self.canvas.x_pos y = y + self.canvas.y_pos self.start_pos = (x, y) # Set measuring to True self.measuring = True def zoom(self, scale, center=False): if center is False: self.shrink(scale) else: self.shrink(scale, x=0, y=0) def stop_measure(self, event): # Include globals # Set measuring to False self.measuring = False now_pos = self.translate_screen_pos_to_canvas_pos(event.x, event.y) now_pos = (now_pos[0] + self.canvas.x_pos, now_pos[1] + self.canvas.y_pos) if self.start_pos[0] == now_pos[0] and self.start_pos[1] == now_pos[1]: self.zoom(1.1) # Try to remove the old elements try: event.widget.delete(self.cur_line) event.widget.delete(self.cur_line_txt) except: pass # Helper Functions def clear_canvas(self): self.canvas.delete("obj") self.canvas.delete("del") self.not_cleared = False @staticmethod def get_val_from_args(args, val): if val in args: return args[val] else: return None def clear_connection_list(self, args): for key in self.connection_list.keys(): self.connection_list[key]['counter'] = 0 def report_communication(self, args): key = self.get_val_from_args(args, "key") if key is None: print( f"Invalid args input for function 'report_communication': {args}" ) return if key in self.connection_list: self.connection_list[key]['counter'] += 1 txt = "{}: {:<5}".format(self.connection_list[key]['name'], self.connection_list[key]['counter']) self.details.itemconfig(self.connection_list[key]['txt_id'], text=txt) else: print( f"Nodes '{key}' not in the comm list. Command 'report_communication'." ) def status_update(self, args): node_id = self.get_val_from_args(args, "node_id") bat = self.get_val_from_args(args, "bat") temp = self.get_val_from_args(args, "temp") heading = self.get_val_from_args(args, "heading") if node_id is None or bat is None or temp is None: print(f"Invalid args input for function 'status_update': {args}") return if node_id not in self.node_details_list: if node_id != '0' and node_id != '1': print( f"Node '{node_id}' not in the details list. Command 'status_update'." ) return txt = "{:<7}| BAT {:<4}, TEMP {:<5}, HDG {}".format( self.node_details_list[node_id]['name'], str(round(bat)) + "%", str(round(temp)) + "°C", str(round(heading)) + "°") self.details.itemconfig(self.node_details_list[node_id]['txt_id'], text=txt) def draw_circle(self, args): x = self.get_val_from_args(args, "x") y = self.get_val_from_args(args, "y") r = self.get_val_from_args(args, "r") fill = self.get_val_from_args(args, "fill") tags = self.get_val_from_args(args, "tags") outline = self.get_val_from_args(args, "outline") width = self.get_val_from_args(args, "width") text = self.get_val_from_args(args, "text") text_color = self.get_val_from_args(args, "text_color") text_size = self.get_val_from_args(args, "text_size") text_y_bias = self.get_val_from_args(args, "text_y_bias") if x is None or y is None or r is None: print(f"Invalid args input for function 'draw_circle': {args}") return x = x * self.universal_scale y = y * self.universal_scale r = r * self.universal_scale (x, y) = self.translate_screen_pos_to_canvas_pos(x, y) if fill is None: fill = 'text' if tags is None: tags = [] if outline is None: outline = 'blank' if width is None: width = 3 x = x * self.meas_to_map y = y * self.meas_to_map r = r * self.meas_to_map self.create_circle(x, y, r, extra_tags=tags, fill=fill, width=width, outline=outline) if text is not None: if text_color is None: text_color = "text" if text_size is None: text_size = "text_size_large" if text_y_bias is None: ypos = y - r - 20 if ypos < 0: ypos = y + r + 20 else: ypos = text_y_bias self.create_text(x, ypos, text=text, color=text_color, size=text_size) def connect_points(self, args): pos1 = self.get_val_from_args(args, "pos1") pos2 = self.get_val_from_args(args, "pos2") dashed = self.get_val_from_args(args, "dashed") color = self.get_val_from_args(args, "color") text = self.get_val_from_args(args, "text") text_size = self.get_val_from_args(args, "text_size") text_color = self.get_val_from_args(args, "text_color") arrow = self.get_val_from_args(args, "arrow") if pos1 is None or pos2 is None: print(f"Invalid args input for function 'connect_points': {args}") return if dashed is None: dashed = True if arrow is "both": arrow = tk.BOTH else: arrow = None pos1_scaled = (pos1[0] * self.meas_to_map * self.universal_scale + self.canvas.x_pos, pos1[1] * self.meas_to_map * self.universal_scale + self.canvas.y_pos) pos2_scaled = (pos2[0] * self.meas_to_map * self.universal_scale + self.canvas.x_pos, pos2[1] * self.meas_to_map * self.universal_scale + self.canvas.y_pos) self._connect_points(pos1_scaled, pos2_scaled, text=text, text_size=text_size, text_color=text_color, dashed=dashed, color=color, arrow=arrow) def create_circle(self, x, y, r, extra_tags=[], fill=None, outline=None, **kwargs): fill = self.refrence_color(fill, default=self.colors.text) outline = self.refrence_color(outline, default=self.colors.blank) (x, y) = self.translate_canvas_pos_to_screen_pos(x, y) tags = ["obj"] return self.canvas.create_oval(x - r, y - r, x + r, y + r, tags=(tags + extra_tags), fill=fill, outline=outline, **kwargs) def create_text(self, x, y, text="", color=None, size=None, extra_tags=[], **kwargs): size = self.refrence_color(size, self.colors.text_size_large) color = self.refrence_color(color, default=self.colors.text) (x, y) = self.translate_canvas_pos_to_screen_pos(x, y) tags = ["obj"] self.canvas.create_text(x, y, text=text, fill=color, font=font.Font(family=self.colors.data_font, size=size), justify=tk.LEFT, tags=(tags + extra_tags)) def _connect_points(self, node1_pos, node2_pos, text=None, text_size=None, text_color=None, dashed=True, color="#3c4048", arrow=None): if node2_pos[0] is None or node2_pos[1] is None or node1_pos[ 0] is None or node1_pos[1] is None: return if text is not None: # Calculate the rotation between the two points rotation = 180 - math.degrees( math.atan2(node1_pos[1] - node2_pos[1], node1_pos[0] - node2_pos[0])) # node1_pos the rotation if 90 < rotation < 270: rotation -= 180 # Convert to radians rrotation = math.radians(rotation) # Calculate mid point + rotation offset midx = (node1_pos[0] + node2_pos[0]) / 2 - math.sin(rrotation) * 5 midy = (node1_pos[1] + node2_pos[1]) / 2 - math.cos(rrotation) * 5 text_size = self.refrence_color(text_size, self.colors.text_size_large) text_color = self.refrence_color(text_color, default=self.colors.text) if self.tk_version >= 8.6: self.canvas.create_text(midx, midy, text=text, fill=text_color, font=font.Font( family=self.colors.data_font, size=text_size), justify=tk.LEFT, tags=['scale', 'obj'], angle=rotation) else: self.canvas.create_text(midx, midy, text=text, fill=text_color, font=font.Font( family=self.colors.data_font, size=text_size), justify=tk.LEFT, tags=['scale', 'obj']) color = self.refrence_color(color, default=self.colors.main_line) if dashed is True: self.canvas.create_line(node1_pos[0], node1_pos[1], node2_pos[0], node2_pos[1], width=self.colors.line_width, fill=color, dash=self.colors.dash_type, arrow=arrow, tags="obj") else: self.canvas.create_line(node1_pos[0], node1_pos[1], node2_pos[0], node2_pos[1], width=self.colors.line_width, fill=color, arrow=arrow, tags="obj") def refrence_color(self, color, default=None): if color == default and default is not None: return color if color is not None and isinstance(color, str) and hasattr( self.colors, color): color = getattr(self.colors, color) else: if default is None: color = self.colors.text else: color = default return color
class BalloonVideo: def __init__(self): # get image resolution self.img_width = balloon_config.config.get_integer( 'camera', 'width', 640) self.img_height = balloon_config.config.get_integer( 'camera', 'height', 480) # get image center self.img_center_x = self.img_width / 2 self.img_center_y = self.img_height / 2 # define field of view self.cam_hfov = balloon_config.config.get_float( 'camera', 'horizontal-fov', 70.42) self.cam_vfov = balloon_config.config.get_float( 'camera', 'vertical-fov', 43.3) # define video output filename self.video_filename = balloon_config.config.get_string( 'camera', 'video_output_file', '~/balloon-%Y-%m-%d-%H-%M.avi') self.video_filename = expanduser(self.video_filename) self.video_filename = time.strftime(self.video_filename) # background image processing variables self.proc = None # background process object self.parent_conn = None # parent end of communicatoin pipe self.img_counter = 0 # num images requested so far # __str__ - print position vector as string def __str__(self): return "BalloonVideo Object W:%d H:%d" % (self.img_width, self.img_height) # get_camera - initialises camera and returns VideoCapture object def get_camera(self): # setup video capture self.camera = cv2.VideoCapture(0) self.camera.set(cv2.cv.CV_CAP_PROP_FRAME_WIDTH, self.img_width) self.camera.set(cv2.cv.CV_CAP_PROP_FRAME_HEIGHT, self.img_height) # check we can connect to camera if not self.camera.isOpened(): print "failed to open camera, exiting!" sys.exit(0) return self.camera # open_video_writer - begin writing to video file def open_video_writer(self): # Define the codec and create VideoWriter object # Note: setting ex to -1 will display pop-up requesting user choose the encoder ex = int(cv2.cv.CV_FOURCC('M', 'J', 'P', 'G')) self.video_writer = cv2.VideoWriter(self.video_filename, ex, 25, (self.img_width, self.img_height)) return self.video_writer # pixels_to_angle_x - converts a number of pixels into an angle in radians def pixels_to_angle_x(self, num_pixels): return num_pixels * math.radians(self.cam_hfov) / self.img_width # pixels_to_angle_y - converts a number of pixels into an angle in radians def pixels_to_angle_y(self, num_pixels): return num_pixels * math.radians(self.cam_vfov) / self.img_height # angle_to_pixels_x - converts a horizontal angle (i.e. yaw) to a number of pixels # angle : angle in radians def angle_to_pixels_x(self, angle): return int(angle * self.img_width / math.radians(self.cam_hfov)) # angle_to_pixels_y - converts a vertical angle (i.e. pitch) to a number of pixels # angle : angle in radians def angle_to_pixels_y(self, angle): return int(angle * self.img_height / math.radians(self.cam_vfov)) # # background image processing routines # # image_capture_background - captures all images from the camera in the background and returning the latest image via the pipe when the parent requests it def image_capture_background(self, imgcap_connection): # exit immediately if imgcap_connection is invalid if imgcap_connection is None: print "image_capture failed because pipe is uninitialised" return # open the camera camera = self.get_camera() # clear latest image latest_image = None while True: # constantly get the image from the webcam success_flag, image = camera.read() # if successful overwrite our latest image if success_flag: latest_image = image # check if the parent wants the image if imgcap_connection.poll(): recv_obj = imgcap_connection.recv() # if -1 is received we exit if recv_obj == -1: break # otherwise we return the latest image imgcap_connection.send(latest_image) # release camera when exiting camera.release() # start_background_capture - starts background image capture def start_background_capture(self): # create pipe self.parent_conn, imgcap_conn = Pipe() # create and start the sub process and pass it it's end of the pipe self.proc = Process(target=self.image_capture_background, args=(imgcap_conn, )) self.proc.start() def stop_background_capture(self): # send exit command to image capture process self.parent_conn.send(-1) # join process self.proc.join() # get_image - returns latest image from the camera captured from the background process def get_image(self): # return immediately if pipe is not initialised if self.parent_conn == None: return None # send request to image capture for image self.parent_conn.send(self.img_counter) # increment counter for next interation self.img_counter = self.img_counter + 1 # wait endlessly until image is returned recv_img = self.parent_conn.recv() # return image to caller return recv_img # main - tests BalloonVideo class def main(self): # start background process self.start_background_capture() while True: # send request to image capture for image img = self.get_image() # check image is valid if not img is None: # display image cv2.imshow('image_display', img) else: print "no image" # check for ESC key being pressed k = cv2.waitKey(5) & 0xFF if k == 27: break # take a rest for a bit time.sleep(0.1) # send exit command to image capture process self.stop_background_capture() print "a2p 10 = %f" % self.angle_to_pixels_x(10) print "p2a 10 = %f" % self.pixels_to_angle_x(10)
class PacketController(Process): # Constants WINDOW_SIZE = c.host.window_size ENABLE_TIMEOUT = c.host.enable_timeout INITIAL_TIMEOUT = c.host.initial_timeout SRTT_ALPHA = c.host.srtt_alpha RTTDEV_ALPHA = c.host.rttdev_alpha SRTT_GAIN = c.host.srtt_gain RTTDEV_GAIN = c.host.rttdev_gain SERIAL_RETRIES = c.host.serial_retries SERIAL_RETRY_TIMEOUT = c.host.serial_retry_timeout PACK_FORMAT = c.host.pack_format UNPACK_FORMAT = c.host.unpack_format RESET_MSG = c.host.reset_msg START_BYTE = c.packet.start_byte DEVICELIST_CODE = c.devices.devicelist.code HELLO_PAYLOAD = c.devices.devicelist.hello_code compare_pids = lambda self, a, b: np.int16(b - a) def __init__(self): self._stop = Event() self.tserial = None self.packet_parser = None self.pipe_inside, self.pipe_outside = Pipe() self.continuous_requests = set() self.weighted_tdma_list = [] self.reset() super(PacketController, self).__init__() def reset(self, clear_pipe=False): self.rtt_smoothed = None self.rtt_deviation = None self.timeout = self.INITIAL_TIMEOUT self.en_route = dict() # pid -> packet, time_sent self.receiving_buffer = dict() # pid -> sent_packet, payload self.next_send_pid = np.uint16(0) self.next_recv_pid = np.uint16(0) self.weighted_tdma_pos = 0 self.packets_received = 0 if clear_pipe: while self.pipe_inside.poll(): self.pipe_inside.recv() def connect(self): self.reset(True) raw_hello_packet = self.encode_raw_packet(0, self.DEVICELIST_CODE, self.HELLO_PAYLOAD) hello_response_length = len(raw_hello_packet) - 1 self.tserial = TAMPSerial(raw_hello_packet, hello_response_length) self.packet_parser = PacketParser(self.tserial) self.pipe_inside.send((None, self.RESET_MSG)) def encode_raw_packet(self, pid, dest, payload): pack_format = self.PACK_FORMAT.format(len(payload)) length = len(payload) + 5 return pack(pack_format, self.START_BYTE, pid, length, dest, payload) def get_new_packet_to_send(self): # process new requests while self.pipe_inside.poll(): packet_request = PacketRequest(*self.pipe_inside.recv()) if packet_request.is_continuous: if not packet_request.remove_continuous: self.continuous_requests.add(packet_request[:2]) self.weighted_tdma_list += ([packet_request[:2]] * packet_request.weight) else: self.continuous_requests.discard(packet_request[:2]) else: return packet_request[:2] # resend exiting continuous_requests while self.weighted_tdma_list: key = self.weighted_tdma_list[self.weighted_tdma_pos] if key in self.continuous_requests: self.weighted_tdma_pos += 1 self.weighted_tdma_pos %= len(self.weighted_tdma_list) return key else: # this item was removed self.weighted_tdma_list.pop(self.weighted_tdma_pos) if self.weighted_tdma_list: self.weighted_tdma_pos += 1 self.weighted_tdma_pos %= len(self.weighted_tdma_list) # nothing else to do return None def slide_window(self): if self.ENABLE_TIMEOUT: # resend all en_route packets that have timed out for pid in self.en_route: packet, time_sent = self.en_route[pid] dt = time() - time_sent if dt > self.timeout: print dt, self.timeout, self.rtt_smoothed, self.rtt_deviation self.en_route[pid] = (packet, time()) self.transmit(pid, *packet[:2]) return # transmit the next packet if len(self.en_route) < self.WINDOW_SIZE: packet = self.get_new_packet_to_send() if packet: new_pid = self.next_send_pid self.en_route[new_pid] = (packet, time()) self.transmit(new_pid, *packet) self.next_send_pid += np.uint16(1) def decode_raw_packet(self, raw_packet): payload_length = len(raw_packet) - 4 unpack_format = self.UNPACK_FORMAT.format(payload_length) start_byte, pid, length, payload = unpack(unpack_format, raw_packet) return (np.uint16(pid), payload) def transmit(self, pid, dest, payload): if self.tserial.out_waiting > 50: self.tserial.flush() raw_packet = self.encode_raw_packet(pid, dest, payload) self.tserial.write(raw_packet) def receive(self): raw_packets = self.packet_parser.receive() packets = [self.decode_raw_packet(p) for p in raw_packets] for pid, payload in packets: if payload != self.HELLO_PAYLOAD: self.process_packet(pid, payload) return len(packets) def process_packet(self, pid, payload): self.packets_received += 1 if pid not in self.en_route: print "retransmitted packet received" return sent_packet, time_sent = self.en_route.pop(pid) if self.ENABLE_TIMEOUT: self.update_timeout(time_sent) if self.next_recv_pid == pid: self.pipe_inside.send((sent_packet, payload)) i = 1 while pid + i in self.receiving_buffer: self.pipe_inside.send(self.receiving_buffer.pop(pid + i)) i += 1 self.next_recv_pid = pid + np.uint16(i) elif self.compare_pids(self.next_recv_pid, pid) > 0: self.receiving_buffer[pid] = (sent_packet, payload) else: self.pipe_inside.send((sent_packet, payload)) def update_timeout(self, time_sent): """ Update the timeout value to use for future packets, by combining the smoothed round trip time, and the deviation in the round trip time """ rtt = time() - time_sent if self.rtt_smoothed: self.rtt_smoothed = rtt * self.SRTT_ALPHA + ( 1 - self.SRTT_ALPHA) * self.rtt_smoothed else: self.rtt_smoothed = rtt if self.rtt_deviation: self.rtt_deviation = ( abs(rtt - self.rtt_smoothed) * self.RTTDEV_ALPHA + (1 - self.RTTDEV_ALPHA) * self.rtt_deviation) else: self.rtt_deviation = rtt self.timeout = self.SRTT_GAIN * self.rtt_smoothed + self.RTTDEV_GAIN * self.rtt_deviation def stop(self): self._stop.set() def run(self): i = self.SERIAL_RETRIES while i >= 0: try: self.connect() i = self.SERIAL_RETRIES while not self._stop.is_set(): self.slide_window() self.receive() return except (IOError, SerialException, SerialPortUnavailableException) as e: i -= 1 if i == 0: print "[SerialController] Giving up, hit maximum serial retries" return else: print "[SerialController] {}: {}".format( e.__class__.__name__, e) print( "[SerialController] ", "Retrying connection in 1 second, " "{} tries left".format(i)) sleep(self.SERIAL_RETRY_TIMEOUT) continue except SerialPortEstablishException as e: print e print "[SerialController] Quitting process..." return except KeyboardInterrupt: return
works.append(work) parent_conns.append(parent_conn) child_conns.append(child_conn) states = np.zeros([num_worker, 84, 84, 4]) while True: total_state, total_reward, total_done, total_next_state, total_action = [], [], [], [], [] global_update += 1 for _ in range(num_step): actions = agent.get_action(states) for parent_conn, action in zip(parent_conns, actions): parent_conn.send(action) next_states, rewards, dones, real_dones = [], [], [], [] for parent_conn in parent_conns: s, r, d, rd = parent_conn.recv() next_states.append(s) rewards.append(r) dones.append(d) real_dones.append(rd) next_states = np.stack(next_states) rewards = np.hstack(rewards) dones = np.hstack(dones) real_dones = np.hstack(real_dones) score += rewards[sample_idx]
# 管道 # 双向通信 # 如果管道的端点没有被使用,那就应该关闭管道的端点。如在生产者消费者模型中,生产者要关闭管道的输出端,消费者要关闭管道的输入端 # 如果没有执行关闭操作,程序可能在消费者中的 recv() 操作上挂起 # 管道用在进程之间会发生数据冲突的情况,数据不安全 # 在主进程中关闭管道不会影响管道在子进程中的使用 # 队列=管道+锁 from multiprocessing import Pipe from multiprocessing import Process def func(p): father, son = p #father.close() print(son.recv()) print(son.recv()) if __name__ == '__main__': father, son = Pipe() p = Process(target=func, args=((father, son), )) p.start() father.send('hello') father.close()
class MasterBlock(Process): """ This represent a Crappy block, it must be parent of all the blocks. Methods: main() It must not take any arg, it is where you define the main loop of the block. If not overriden, will raise an error add_[in/out]put(Link object) Add a link as [in/out]put prepare() This method will be called inside the new process but before actually starting the main loop of the program. Use it for all the tasks to be done before starting the main loop (can be empty). start() This is the same start method as Process.start: it starts the process, so the initialization (defined in prepare method) will be done, but NOT the main loop. launch(t0) Once the process is started, calling launch will set the starting time and actually start the main method. If the block was not started yet, it will be done automatically. t0: time to set as starting time of the block (mandatory, in seconds after epoch) status Property that can be accessed both in the process or from the parent "idle": Block not started yet "initializing": start was called and prepare is not over yet "ready": prepare is over, waiting to start main by calling launch "running": main is running "done": main is over "error": An error occured and the block stopped start and launch method will return instantly """ instances = WeakSet() def __init__(self): Process.__init__(self) # MasterBlock.instances.append(self) self.outputs = [] self.inputs = [] # This pipe allows to send 2 essential signals: # pipe1->pipe2 is to start the main function and set t0 # pipe2->pipe1 to set process status to the parent self.pipe1, self.pipe2 = Pipe() self._status = "idle" self.in_process = False # To know if we are in the process or not self.niceness = 0 def __new__(cls, *args, **kwargs): instance = super().__new__(cls) MasterBlock.instances.add(instance) return instance @classmethod def reset(cls): cls.instances = WeakSet() def run(self): self.in_process = True # we are in the process self.status = "initializing" try: self.prepare() self.status = "ready" # Wait for parent to tell me to start the main self.t0 = self.pipe2.recv() if self.t0 < 0: try: self.finish() except Exception: pass return self.status = "running" self.begin() self._MB_last_t = time() self._MB_last_FPS = self._MB_last_t self._MB_loops = 0 self.main() self.status = "done" except CrappyStop: print("[%r] Encountered CrappyStop Exception, terminating" % self) self.status = "done" self.stop_all() except KeyboardInterrupt: print("[%r] Keyboard interrupt received" % self) except Exception as e: print("[%r] Exception caught:" % self, e) try: self.finish() except Exception: pass self.status = "error" sleep(1) # To let downstream blocks process the data and avoid loss self.stop_all() raise self.finish() self.status = "done" @classmethod def get_status(cls): return [x.status for x in cls.instances] @classmethod def all_are(cls, s): """ Returns true only if all processes status are s """ l = cls.get_status() return len(set(l)) == 1 and s in l @classmethod def renice_all(cls, high_prio=True, verbose=True): """ Will renice all the blocks processes according to block.niceness value If high_prio is False, blocks with a negative niceness value will be ignored. This is to avoid asking for the sudo password since only root can lower the niceness of processes. """ if "win" in platform: # Not supported on Windows yet return for b in cls.instances: if b.niceness < 0 and high_prio or b.niceness > 0: print("[renice] Renicing", b.pid, "to", b.niceness) renice(b.pid, b.niceness) @classmethod def prepare_all(cls, verbose=True): """ Starts all the blocks processes (block.prepare), but not the main loop """ if verbose: def vprint(*args): print("[prepare]", *args) else: vprint = lambda *x: None vprint("Starting the blocks...") for instance in cls.instances: vprint("Starting", instance) instance.start() vprint("Started, PID:", instance.pid) vprint("All processes are started.") @classmethod def launch_all(cls, t0=None, verbose=True, bg=False): if verbose: def vprint(*args): print("[launch]", *args) else: vprint = lambda *x: None if not cls.all_are('ready'): vprint("Waiting for all blocks to be ready...") while not cls.all_are('ready'): sleep(.1) if not all([i in ['ready','initializing','idle'] for i in cls.get_status()]): print("Crappy failed to start!") for i in cls.instances: if i.status in ['ready','initializing']: i.launch(-1) cls.stop_all() return #raise RuntimeError("Crappy failed to start!") vprint("All blocks ready, let's go !") if not t0: t0 = time() vprint("Setting t0 to", strftime("%d %b %Y, %H:%M:%S", localtime(t0))) for instance in cls.instances: instance.launch(t0) t1 = time() vprint("All blocks loop started. It took", (t1 - t0) * 1000, "ms") if bg: return try: # Keep running l = cls.get_status() while not ("done" in l or "error" in l): l = cls.get_status() sleep(1) except KeyboardInterrupt: print("Main proccess got keyboard interrupt!") # It will automatically propagate to the blocks processes if not cls.all_are('running'): print('Waiting for all processes to finish') t = time() while 'running' in cls.get_status() and time() - t < 3: sleep(.1) if cls.all_are('done'): print("Crappy terminated gracefully") else: print("Crappy terminated, blocks status:") for b in cls.instances: print(b,b.status) @classmethod def start_all(cls, t0=None, verbose=True, bg=False, high_prio=False): cls.prepare_all(verbose) if high_prio and any([b.niceness < 0 for b in cls.instances]): print("[start] High prio: root premission needed to renice") cls.renice_all(high_prio, verbose=verbose) cls.launch_all(t0, verbose, bg) @classmethod def stop_all(cls, verbose=True): """ Stops all the blocks (crappy.stop) """ if verbose: def vprint(*args): print("[stop]", *args) else: vprint = lambda *x: None vprint("Stopping the blocks...") for instance in cls.instances: if instance.status == 'running': vprint("Stopping", instance) instance.stop() vprint("All blocks are stopped.") def begin(self): """ If main is not overriden, this method will be called first, before entering the main loop """ pass def finish(self): """ If main is not overriden, this method will be called upon exit or after a crash. """ pass def loop(self): raise NotImplementedError('You must override loop or main in' + str(self)) def main(self): while not self.pipe2.poll(): self.loop() self.handle_freq() print("[%r] Got stop signal, interrupting..." % self) def handle_freq(self): """ For block with a given number of loops/s (use freq attr to set it) """ self._MB_loops += 1 t = time() if hasattr(self, 'freq') and self.freq: d = t - self._MB_last_t + 1 / self.freq while d > 0: t = time() d = self._MB_last_t + 1 / self.freq - t sleep(max(0, d / 2 - 2e-3)) # Ugly, yet simple and pretty efficient self._MB_last_t = t if hasattr(self, 'verbose') and self.verbose and \ self._MB_last_t - self._MB_last_FPS > 2: print("[%r] loops/s:" % self, self._MB_loops / (self._MB_last_t - self._MB_last_FPS)) self._MB_loops = 0 self._MB_last_FPS = self._MB_last_t def launch(self, t0): """ To start the main method, will call start if needed """ if self.status == "idle": print(self, ": Called launch on unprepared process!") self.start() self.pipe1.send(t0) # asking to start main in the process @property def status(self): """ Returns the status of the block, from the process itself or the parent """ if not self.in_process: while self.pipe1.poll(): try: self._status = self.pipe1.recv() except EOFError: if self._status == 'running': self._status = 'done' # If another process tries to get the status if 'linux' in platform: self.pipe2.send(self._status) # Somehow the previous line makes crappy hang on Windows, no idea why # It is not critical, but it means that process status can now only # be read from the process itself and ONE other process. # Luckily, only the parent (the main process) needs the status for now return self._status @status.setter def status(self, s): assert self.in_process, "Cannot set status from outside of the process!" self.pipe2.send(s) self._status = s def prepare(self): """ This will be run when creating the process, but before the actual start The first code to be run in the new process, will only be called once and before the actual start of the main launch of the blocks. It can stay empty to do nothing. """ pass def send(self, data): """ To send the data to all blocks downstream Send has 2 ways to operate: you can either build the ordered dict yourself or you can define self.labels (usually time first) and call send with a list. It will then map them to the dict. Note that ONLY dict can go through links """ if isinstance(data, dict): pass elif isinstance(data, list): data = dict(zip(self.labels, data)) elif data == 'stop': pass else: raise IOError("Trying to send a " + str(type(data)) + " in a link!") for o in self.outputs: o.send(data) def get_last(self, num=None): """ To get the latest value of each labels from all inputs Unlike the recv methods of Link, get_last is NOT guaranteed to return all the data going through the links! It is meant to get the latest values, discarding all the previous one (for a displayer for example) Its mode of operation is completely different since it can operate on multiple inputs at once. num is a list containing all the concerned inputs. The first call may be blocking until it receives data, all the others will return instantaneously, giving the latest known reading If num is None, it will operate on all the input link at once """ if not hasattr(self, '_last_values'): self._last_values = [None] * len(self.inputs) if num is None: num = range(len(self.inputs)) elif not isinstance(num, list): num = [num] for i in num: if self._last_values[i] is None: self._last_values[i] = self.inputs[i].recv() while self.inputs[i].poll(): self._last_values[i] = self.inputs[i].recv() ret = {} for i in num: ret.update(self._last_values[i]) return ret def get_all_last(self, num=None): """ To get the data from all links of the block Almost the same as get_last, but will return all the data that goes through the links (in lists). Also, if multiple links have the same label, only the last link's value will be kept """ if not hasattr(self, '_all_last_values'): self._all_last_values = [None] * len(self.inputs) if num is None: num = range(len(self.inputs)) elif not isinstance(num, list): num = [num] for i in num: if self._all_last_values[i] is None or self.inputs[i].poll(): self._all_last_values[i] = self.inputs[i].recv_chunk() else: # Dropping all data (already sent on last call) except the last # to make sure the block has at least one value for key in self._all_last_values[i]: self._all_last_values[i][key][:-1] = [] ret = {} for i in num: ret.update(self._all_last_values[i]) return ret def drop(self, num=None): """ Will clear the inputs of the blocks This method performs like get_last but returns None instantly """ if num is None: num = range(len(self.inputs)) elif not isinstance(num,list): num = [num] for n in num: self.inputs[n].clear() def add_output(self, o): self.outputs.append(o) def add_input(self, i): self.inputs.append(i) def stop(self): if self.status != 'running': return print('[%r] Stopping' % self) self.pipe1.send(0) for i in self.inputs: i.send('stop') for i in range(10): if self.status == "done": break sleep(.05) #if self.status != "done": if self.status not in ['done','idle','error']: print('[%r] Could not stop properly, terminating' % self) try: self.terminate() except Exception: pass else: print("[%r] Stopped correctly" % self) def __repr__(self): return str(type(self)) + " (" + str(self.pid or "Not running") + ")"
GPIO.setup(3, GPIO.IN) GPIO.output(23, GPIO.HIGH) interuptProcess.start() interuptPID = p_conn.recv() vp.start() vpPID = pVS_conn.recv() phoneProcess.start() try: while True: #For starting the stream set this to 1 start = 1 GPIO.output(12, GPIO.HIGH) pVS_conn.send(1) print("waitng...") while True: if(pPhone_conn.recv() == "Connect"): break print("RUN!") while True: msg = "" GPIO.output(19, GPIO.HIGH) GPIO.output(25, GPIO.HIGH) #TODO: move these two lines to a new process ## date = str(datetime.date.today()) ## if((date +": " + msg) != (date +": Nothing")): ## f.write((date +": " + msg+"\n"))
class PSMPAgent(object): def __init__(self, model_func, exploration_schedule, obs_shape, input_type, lr=1e-4, buffer_size=1000000, num_actions=6, latent_dim=32, gamma=0.99, knn=4, eval_epsilon=0.01, queue_threshold=5e-5, tf_writer=None): self.conn, child_conn = Pipe() self.ec_buffer = PriorSweepProcess(num_actions, buffer_size, latent_dim, latent_dim, child_conn, gamma) self.obs = None self.z = None self.ind = -1 self.writer = tf_writer self.sequence = [] self.gamma = gamma self.queue_threshold = queue_threshold self.num_actions = num_actions self.exploration_schedule = exploration_schedule self.latent_dim = latent_dim self.knn = knn self.steps = 0 self.rmax = 100000 self.logger = logging.getLogger("ecbp") self.eval_epsilon = eval_epsilon self.hash_func, _, _ = build_train_dueling( make_obs_ph=lambda name: input_type(obs_shape, name=name), model_func=model_func, q_func=model, imitate=False, num_actions=num_actions, optimizer=tf.train.AdamOptimizer(learning_rate=lr, epsilon=1e-4), gamma=gamma, grad_norm_clipping=10, ) self.ec_buffer.start() def log(self, *args, logtype='debug', sep=' '): getattr(self.logger, logtype)(sep.join(str(a) for a in args)) def send_and_receive(self, msg, obj): self.conn.send((msg, obj)) if self.conn.poll(timeout=None): recv_msg, recv_obj = self.conn.recv() assert msg == recv_msg return recv_obj def act(self, obs, is_train=True): self.obs = obs self.z = np.array(self.hash_func(np.array(obs))).reshape( (self.latent_dim, )) if self.ind == -1: self.ind = self.send_and_receive(1, np.array([self.z])) self.steps += 1 epsilon = max(0, self.exploration_schedule.value( self.steps)) if is_train else self.eval_epsilon if np.random.random() < epsilon: action = np.random.randint(0, self.num_actions) return action else: finds = np.zeros((1, )) extrinsic_qs, intrinsic_qs, find = self.send_and_receive( 0, (np.array([self.z]), self.knn)) extrinsic_qs, intrinsic_qs = np.array(extrinsic_qs), np.array( intrinsic_qs) finds += sum(find) if is_train: q = intrinsic_qs + extrinsic_qs else: q = extrinsic_qs q = np.squeeze(q) q = np.squeeze(q) # q = np.nan_to_num(q) q_max = np.nanmax(q) if np.isnan(q_max): max_action = np.arange(self.num_actions) else: max_action = np.where(q >= q_max - 1e-7)[0] self.log("action selection", max_action, logtype='info') self.log("q", q, q_max, logtype='info') action_selected = np.random.randint(0, len(max_action)) return max_action[action_selected] def observe(self, action, reward, state_tp1, done, train=True): z_tp1 = np.array(self.hash_func( np.array(state_tp1)[np.newaxis, ...])).reshape((self.latent_dim, )) if train: self.ind = self.send_and_receive( 2, (self.ind, action, reward, z_tp1, done)) else: self.ind = self.send_and_receive(1, np.array([z_tp1])) if done: self.ind = -1 self.steps = 0 # def update_sequence(self): # self.ec_buffer.update_sequence(self.sequence, self.debug) # self.sequence = [] def finish(self): self.send_and_receive(3, (True, ))
class ValkkaProcess(Process): """ Semantics: Frontend: the part of the forked process that keeps running in the current, user virtual memory space Backend : the part of the forked process that runs in its own virtual memory space (e.g. "in the background") This class has both backend and frontend methods: - Backend methods should only be called from backend. They are designated with "_". - Frontend methods should only be called from frontend To avoid confusion, backend methods are designated with "_", except for the "run()" method, that's always in the backend Frontend methods use a pipe to send a signal to backend that then handles the signal with a backend method having the same name (but with "_" in the end) Backend methods can, in a similar fashion, send signals to the frontend using a pipe. In frontend, a listening thread is needed. That thread can then call the handleSignal method that chooses the correct frontend method to call TODO: add the possibility to bind the process to a certain processor """ # incoming signals : from frontend to backend incoming_signal_defs = { # each key corresponds to a front- and backend methods "test_": {"test_int": int, "test_str": str}, "stop_": [] } # outgoing signals : from back to frontend. Don't use same names as for # incoming signals .. outgoing_signal_defs = { "test_o": {"test_int": int, "test_str": str}, } def __init__(self, name, affinity=-1, **kwargs): super().__init__() self.pre = self.__class__.__name__ + " : " + name + \ " : " # auxiliary string for debugging output self.name = name self.affinity = affinity self.signal_in = Event() self.signal_out = Event() # communications pipe. Frontend uses self.pipe, backend self.childpipe self.pipe, self.childpipe = Pipe() self.signal_in.clear() self.signal_out.clear() # print(self.pre, "init") def getPipe(self): """Returns communication pipe for front-end """ return self.pipe def preRun_(self): """After the fork, but before starting the process loop """ if (self.affinity > -1): os.system("taskset -p -c %d %d" % (self.affinity, os.getpid())) def postRun_(self): """Just before process exit """ print(self.pre, "post: bye!") def cycle_(self): # Do whatever your process should be doing, remember timeout every now # and then time.sleep(5) print(self.pre, "hello!") def startAsThread(self): from threading import Thread t = Thread(target=self.run) t.start() def run(self): # No "_" in the name, but nevertheless, running in the backed """After the fork. Now the process starts running """ # print(self.pre," ==> run") self.preRun_() self.running = True while(self.running): self.cycle_() self.handleSignal_() self.postRun_() def handleSignal_(self): """Signals handling in the backend """ if (self.signal_in.is_set()): signal_dic = self.childpipe.recv() method_name = signal_dic.pop("name") method = getattr(self, method_name) method(**signal_dic) self.signal_in.clear() self.signal_out.set() def sendSignal(self, **kwargs): # sendSignal(name="test",test_int=1,test_str="kokkelis") """Incoming signals: this is used by frontend methods to send signals to the backend """ try: name = kwargs.pop("name") except KeyError: raise(AttributeError("Signal name missing")) # a dictionary: {"parameter_name" : parameter_type} model = self.incoming_signal_defs[name] for key in kwargs: # raises error if user is using undefined signal model_type = model[key] parameter_type = kwargs[key].__class__ if (model_type == parameter_type): pass else: raise(AttributeError("Wrong type for parameter " + str(key))) kwargs["name"] = name self.pipe.send(kwargs) self.signal_out.clear() self.signal_in. set() # indicate that there is a signal self.signal_out.wait() # wait for the backend to clear the signal def handleSignal(self, signal_dic): """Signal handling in the frontend """ method_name = signal_dic.pop("name") method = getattr(self, method_name) method(**signal_dic) def sendSignal_(self, **kwargs): # sendSignal_(name="test_out",..) """Outgoing signals: signals from backend to frontend """ try: name = kwargs.pop("name") except KeyError: raise(AttributeError("Signal name missing")) # a dictionary: {"parameter_name" : parameter_type} model = self.outgoing_signal_defs[name] for key in kwargs: # raises error if user is using undefined signal try: model_type = model[key] except KeyError: print("your outgoing_signal_defs for",name,"is:", model) print("you requested key:", key) raise parameter_type = kwargs[key].__class__ if (model_type == parameter_type): pass else: raise(AttributeError("Wrong type for parameter " + str(key))) kwargs["name"] = name self.childpipe.send(kwargs) # *** backend methods corresponding to each incoming signals *** def stop_(self): self.running = False def test_(self, test_int=0, test_str="nada"): print(self.pre, "test_ signal received with", test_int, test_str) # ** frontend methods corresponding to each incoming signal: these communicate with the backend via pipes ** def stop(self): self.sendSignal(name="stop_") def test(self, **kwargs): dictionaryCheck(self.incoming_signal_defs["test_"], kwargs) kwargs["name"] = "test_" self.sendSignal(**kwargs) # ** frontend methods corresponding to each outgoing signal ** # typically, there is a QThread in the frontend-side reading the process pipe # the QThread reads kwargs dictionary from the pipe, say # {"name":"test_o", "test_str":"eka", "test_int":1} # And calls handleSignal(kwargs) def test_o(self, **kwargs): pass
class virtualPortController(Thread): """ The class creates a Python thread which simulates control over a serial port. Commands for relaying via the serial port are received from separate Python threads/processes via Queues. N.B. To start the thread you must call start() from the parent Python thread. Args: serialWriteQueue (multiprocessing.Queue): a Queue for receiving commands to be written to the virtual Magstim unit via the serial port serialReadQueue (multiprocessing.Queue): a Queue for returning automated replies from the virtual Magstim unit when requested """ def __init__(self, magstimType, serialWriteQueue, serialReadQueue, **kwargs): Thread.__init__(self) self._serialWriteQueue = serialWriteQueue self._serialReadQueue = serialReadQueue self._portConn, self._magstimConn = Pipe() if magstimType == 'Magstim': self._magstim = virtualMagstim(self._magstimConn) elif magstimType == 'BiStim': self._magstim = virtualBiStim(self._magstimConn) elif magstimType == 'Rapid': self._magstim = virtualRapid(self._magstimConn, **kwargs) elif magstimType == 'Horizon': from horizonmagpy.virtual_horizon import virtualHorizon self._magstim = virtualHorizon(self._magstimConn, **kwargs) else: raise MagstimError('Unrecognised Magstim type.') self._magstim.daemon = True def run(self): """ Continuously monitor the serialWriteQueue for commands from other Python threads/processes to be sent to the virtual Magstim. When requested, will return the automated reply from the virtual Magstim unit to the calling thread via the serialReadQueue. N.B. This should be called via start() from the parent Python thread. """ #Start up virtual magstim self._magstim.start() #This continually monitors the serialWriteQueue for write requests while True: message, reply, readBytes = self._serialWriteQueue.get() #If the first part of the message is None this signals the thread to close the port and stop if message is None: self._portConn.send(None) break #If the first part of the message is a 1 or -1 this signals the thread to do something with the RTS pin, which we don't have - so pass elif message in {1, -1}: pass #Otherwise, the message is a command string else: #Try writing to the virtual port self._portConn.send(message) #Get reply if self._portConn.poll(0.3): #If we want a reply, read the response from the Magstim and place it in the serialReadQueue if reply: self._serialReadQueue.put([0, self._portConn.recv()]) #Otherwise just get rid of the reply from the pipe else: self._portConn.recv() else: self._serialReadQueue.put( [2, 'Timed out while waiting for response.']) #If we get here, it's time to shutdown the serial port controller self._portConn.close() return
class TableProcess(Process): def __init__(self, n_acts, discount, capacity, pri_cutoff, save_path): self.n_acts = n_acts self.discount = discount self.capacity = capacity self.pri_cutoff = pri_cutoff self.save_path = save_path self.killed = False self.lookup_conn, self.table_conn = Pipe() super(TableProcess, self).__init__(name='TableProcess') def start(self): self.daemon = True super(TableProcess, self).start() def run(self): try: logging.info("Starting sweeper table") self.ctable = SweeperTable(self.discount, self.capacity, self.n_acts, self.pri_cutoff, self.save_path) self._loop() logging.info("Exiting sweeper table.") except Exception as e: self.table_conn.send(ExceptionWrapper(e)) raise e def _loop(self): while not self.killed: #Check for q requests, experience updates or save requests self._empty_pipe() #Do a priority sweep self.ctable.priority_sweep() def __getitem__(self, key_slice): self.lookup_conn.send((0, key_slice)) return self.get_pipe_output() def add(self, ind, act, reward, ind_next): self.lookup_conn.send((1, (ind, act, reward, ind_next))) if self.lookup_conn.poll(): return self.get_pipe_output() def delete(self, ind, act, reward, ind_next): self.lookup_conn.send((2, (ind, act, reward, ind_next))) if self.lookup_conn.poll(): return self.get_pipe_output() def get_variables(self, *var_names): self.lookup_conn.send((3, var_names)) return self.get_pipe_output() def reset_summary_variables(self): self.lookup_conn.send((4, None)) if self.lookup_conn.poll(): return self.get_pipe_output() def save(self, filename, old_filename=None): self.lookup_conn.send((5, (filename, old_filename))) return self.get_pipe_output() def restore(self, filename): self.lookup_conn.send((6, filename)) return self.get_pipe_output() def clean_orphaned_states(self, filename, max_state): logging.info("Determining orphan states.") table_path = "%s/table.ckpt-%s" % (self.save_path, filename) with open(table_path, 'r') as handle: data = pickle.load(handle) transition_path = "%s/transitions.ckpt-%s.h5" % (self.save_path, filename) nsas, nsas_inv = h5table.restore(transition_path, self.n_acts) qs = data['qs'] rewards = data['rewards'] vs = data['vs'] us = data['us'] nsa = data['nsa'] priorities = data['priorities'] orphan_candidates = np.argwhere(nsa[:max_state].sum(axis=1)==0)[:, 0] orphans = [] for orphan_candidate in orphan_candidates: orphaned = True for act in range(self.n_acts): if len(nsas_inv[act][orphan_candidate]) > 0: orphaned = False break if orphaned: orphans.append(orphan_candidate) qs[orphan_candidate] = np.nan rewards[orphan_candidate] = 0 vs[orphan_candidate] = 0 us[orphan_candidate] = 0 priorities.pop(orphan_candidate, None) data = {'qs': qs, 'nsa': nsa, 'rewards': rewards, 'vs': vs, 'us': us, 'priorities': priorities} with open("%s/table.ckpt-%s" % (self.save_path, filename), 'w') as handle: pickle.dump(data, handle) return np.array(orphans) def reverse_rewards(self): self.lookup_conn.send((7, None)) return self.get_pipe_output() def kill(self): self.lookup_conn.send((8, None)) return self.get_pipe_output() def shutdown(self): if self.is_alive(): self.kill() self.join() def get_pipe_output(self): msg = self.lookup_conn.recv() if isinstance(msg, ExceptionWrapper): logging.exception("Exception in TableProcess") msg.re_raise() return msg def _empty_pipe(self): while self.table_conn.poll(): code, msg = self.table_conn.recv() result = self.ctable.parse_msg(code, msg) if result is None: self.killed = True result = True if result: self.table_conn.send(result)
break else: substitueix(n1, n2, s) if __name__ == '__main__': inici() s = Semaphore() a, b = Pipe() process = Process(target=fill, args=(s, a)) process.start() while True: print("valor 1: ") x = raw_input() a.send(x) if x == "q": break print("valor 2") y = raw_input() a.send(y) if y == "q": break time.sleep(1) process.join()
class RPCServerBase(object): """This is the base class for send and receive RPC server It uses a Pipe for IPC. >>> import grass.script as gscript >>> from grass.pygrass.rpc.base import RPCServerBase >>> import time >>> provider = RPCServerBase() >>> provider.is_server_alive() True >>> provider.is_check_thread_alive() True >>> provider.stop() >>> time.sleep(1) >>> provider.is_server_alive() False >>> provider.is_check_thread_alive() False >>> provider = RPCServerBase() >>> provider.is_server_alive() True >>> provider.is_check_thread_alive() True Kill the server process with an exception, it should restart >>> provider.client_conn.send([1]) >>> provider.is_server_alive() True >>> provider.is_check_thread_alive() True """ def __init__(self): self.client_conn = None self.server_conn = None self.queue = None self.server = None self.checkThread = None self.threadLock = threading.Lock() self.start_server() self.start_checker_thread() self.stopThread = False self.stopped = True # logging.basicConfig(level=logging.DEBUG) def is_server_alive(self): return self.server.is_alive() def is_check_thread_alive(self): return self.checkThread.is_alive() def start_checker_thread(self): if self.checkThread is not None and self.checkThread.is_alive(): self.stop_checker_thread() self.checkThread = threading.Thread(target=self.thread_checker) self.checkThread.daemon = True self.stopThread = False self.checkThread.start() def stop_checker_thread(self): self.threadLock.acquire() self.stopThread = True self.threadLock.release() self.checkThread.join(None) def thread_checker(self): """Check every 200 micro seconds if the server process is alive""" while True: time.sleep(0.2) self._check_restart_server(caller="Server check thread") self.threadLock.acquire() if self.stopThread is True: self.threadLock.release() return self.threadLock.release() def start_server(self): """This function must be re-implemented in the subclasses """ logging.debug("Start the libgis server") self.client_conn, self.server_conn = Pipe(True) self.lock = Lock() self.server = Process(target=dummy_server, args=(self.lock, self.server_conn)) self.server.daemon = True self.server.start() def check_server(self): self._check_restart_server() def _check_restart_server(self, caller="main thread"): """Restart the server if it was terminated """ logging.debug("Check libgis server restart") self.threadLock.acquire() if self.server.is_alive() is True: self.threadLock.release() return self.client_conn.close() self.server_conn.close() self.start_server() if self.stopped is not True: logging.warning("Needed to restart the libgis server, caller: %s" % (caller)) self.threadLock.release() self.stopped = False def safe_receive(self, message): """Receive the data and throw a FatalError exception in case the server process was killed and the pipe was closed by the checker thread""" logging.debug("Receive message: {message}") try: ret = self.client_conn.recv() if isinstance(ret, FatalError): raise ret return ret except (EOFError, IOError, FatalError) as e: # The pipe was closed by the checker thread because # the server process was killed raise FatalError("Exception raised: " + str(e) + " Message: " + message) def stop(self): """Stop the check thread, the libgis server and close the pipe This method should be called at exit using the package atexit """ logging.debug("Stop libgis server") self.stop_checker_thread() if self.server is not None and self.server.is_alive(): self.client_conn.send([0, ]) self.server.terminate() if self.client_conn is not None: self.client_conn.close() self.stopped = True
from multiprocessing import Process, Pipe def f(pipe): pipe.send(1) print pipe.recv() pipe.close() if __name__ == '__main__': parent_pipe, child_pipe = Pipe() p = Process(target=f, args=(child_pipe,)) p.start() print parent_pipe.recv() parent_pipe.send(2) p.join()
from multiprocessing import Pipe from multiprocessing.connection import Connection from threading import Thread from time import sleep def worker(conn: Connection, delay: int): sleep(delay) print("message from worker") print(conn.recv()) if __name__ == '__main__': parent_conn, child_conn = Pipe() Thread(target=worker, args=(child_conn, 3)).start() parent_conn.send("data from main")
def tempControlProc(myTempSensor, display, pinNum, readOnly, paramStatus, statusQ, conn): mode, cycle_time, duty_cycle, boil_duty_cycle, set_point, boil_manage_temp, num_pnts_smooth, \ k_param, i_param, d_param = unPackParamInitAndPost(paramStatus) p = current_process() print('Starting:', p.name, p.pid) #Pipe to communicate with "Get Temperature Process" parent_conn_temp, child_conn_temp = Pipe() #Start Get Temperature Process ptemp = Process(name="gettempProc", target=gettempProc, args=(child_conn_temp, myTempSensor)) ptemp.daemon = True ptemp.start() #Pipe to communicate with "Heat Process" parent_conn_heat, child_conn_heat = Pipe() #Start Heat Process pheat = Process(name="heatProcGPIO", target=heatProcGPIO, args=(cycle_time, duty_cycle, pinNum, child_conn_heat)) pheat.daemon = True pheat.start() temp_ma_list = [] manage_boil_trigger = False tempUnits = xml_root.find('Temp_Units').text.strip() numTempSensors = 0 for tempSensorId in xml_root.iter('Temp_Sensor_Id'): numTempSensors += 1 temp_ma = 0.0 #overwrite log file for new data log ff = open("brewery" + str(myTempSensor.sensorNum) + ".csv", "wb") ff.close() readyPIDcalc = False while (True): readytemp = False while parent_conn_temp.poll(): #Poll Get Temperature Process Pipe temp_C, tempSensorNum, elapsed = parent_conn_temp.recv( ) #non blocking receive from Get Temperature Process if temp_C == -99: print("Bad Temp Reading - retry") continue if (tempUnits == 'F'): temp = (9.0 / 5.0) * temp_C + 32 else: temp = temp_C temp_str = "%3.2f" % temp display.showTemperature(temp_str) readytemp = True if readytemp == True: if mode == "auto": temp_ma_list.append(temp) #smooth data temp_ma = 0.0 #moving avg init while (len(temp_ma_list) > num_pnts_smooth): temp_ma_list.pop(0) #remove oldest elements in list if (len(temp_ma_list) < num_pnts_smooth): for temp_pnt in temp_ma_list: temp_ma += temp_pnt temp_ma /= len(temp_ma_list) else: #len(temp_ma_list) == num_pnts_smooth for temp_idx in range(num_pnts_smooth): temp_ma += temp_ma_list[temp_idx] temp_ma /= num_pnts_smooth #print "len(temp_ma_list) = %d" % len(temp_ma_list) #print "Num Points smooth = %d" % num_pnts_smooth #print "temp_ma = %.2f" % temp_ma #print temp_ma_list #calculate PID every cycle if (readyPIDcalc == True): duty_cycle = pid.calcPID_reg4(temp_ma, set_point, True) #send to heat process every cycle parent_conn_heat.send([cycle_time, duty_cycle]) readyPIDcalc = False if mode == "boil": if (temp > boil_manage_temp) and (manage_boil_trigger == True): #do once manage_boil_trigger = False duty_cycle = boil_duty_cycle parent_conn_heat.send([cycle_time, duty_cycle]) #put current status in queue try: paramStatus = packParamGet(numTempSensors, myTempSensor.sensorNum, temp_str, tempUnits, elapsed, mode, cycle_time, duty_cycle, \ boil_duty_cycle, set_point, boil_manage_temp, num_pnts_smooth, k_param, i_param, d_param) statusQ.put(paramStatus) #GET request except Full: pass while (statusQ.qsize() >= 2): statusQ.get() #remove old status print("Current Temp: %3.2f deg %s, Heat Output: %3.1f%%" \ % (temp, tempUnits, duty_cycle)) logdata(myTempSensor.sensorNum, temp, duty_cycle) readytemp == False #if only reading temperature (no temp control) if readOnly: continue while parent_conn_heat.poll(): #Poll Heat Process Pipe cycle_time, duty_cycle = parent_conn_heat.recv( ) #non blocking receive from Heat Process display.showDutyCycle(duty_cycle) readyPIDcalc = True readyPOST = False while conn.poll( ): #POST settings - Received POST from web browser or Android device paramStatus = conn.recv() mode, cycle_time, duty_cycle_temp, boil_duty_cycle, set_point, boil_manage_temp, num_pnts_smooth, \ k_param, i_param, d_param = unPackParamInitAndPost(paramStatus) readyPOST = True if readyPOST == True: if mode == "auto": display.showAutoMode(set_point) print("auto selected") pid = PIDController.pidpy(cycle_time, k_param, i_param, d_param) #init pid duty_cycle = pid.calcPID_reg4(temp_ma, set_point, True) parent_conn_heat.send([cycle_time, duty_cycle]) if mode == "boil": display.showBoilMode() print("boil selected") boil_duty_cycle = duty_cycle_temp duty_cycle = 100 #full power to boil manage temperature manage_boil_trigger = True parent_conn_heat.send([cycle_time, duty_cycle]) if mode == "manual": display.showManualMode() print("manual selected") duty_cycle = duty_cycle_temp parent_conn_heat.send([cycle_time, duty_cycle]) if mode == "off": display.showOffMode() print("off selected") duty_cycle = 0 parent_conn_heat.send([cycle_time, duty_cycle]) readyPOST = False time.sleep(.01)
#print("Control: {}".format(cpPID)) #print("Instructions: {}".format(irPID)) manual = 0 auto = 0 power = 1 try: while (power == 1): GPIO.output(25, GPIO.LOW) #For starting the stream set this to 1 start = 0 GPIO.output(12, GPIO.HIGH) pVS_conn.send(1) print("waiting...") while True: if (pPhone_conn.recv() == "C"): print("WE HAVE CONNECTED") break while True: msg = "" GPIO.output(19, GPIO.HIGH) GPIO.output(25, GPIO.HIGH) if (GPIO.input(3) == True): msg = pPhone_conn.recv() pPhone_conn.send("done") GPIO.output(2, GPIO.LOW)
from multiprocessing import Process, Pipe def receiver(recv_pip): while True: try: res = recv_pip.recv() print('recv:', res) except EOFError: break if __name__ == '__main__': print('start') send_pip, recv_pip = Pipe() proc_recv = Process(target=receiver, args=(recv_pip, )) proc_recv.start() for i in range(1, 5): send_pip.send(i) print('send:', i) send_pip.close() proc_recv.join() print('end')
class LabEnvironment(environment.Environment): ACTION_LIST = [ _action(-20, 0, 0, 0, 0, 0, 0), # look_left _action(20, 0, 0, 0, 0, 0, 0), # look_right #_action( 0, 10, 0, 0, 0, 0, 0), # look_up #_action( 0, -10, 0, 0, 0, 0, 0), # look_down _action(0, 0, -1, 0, 0, 0, 0), # strafe_left _action(0, 0, 1, 0, 0, 0, 0), # strafe_right _action(0, 0, 0, 1, 0, 0, 0), # forward _action(0, 0, 0, -1, 0, 0, 0), # backward #_action( 0, 0, 0, 0, 1, 0, 0), # fire #_action( 0, 0, 0, 0, 0, 1, 0), # jump #_action( 0, 0, 0, 0, 0, 0, 1) # crouch ] @staticmethod def get_action_size(env_name): return len(LabEnvironment.ACTION_LIST) def __init__(self, env_name): environment.Environment.__init__(self) self.conn, child_conn = Pipe() self.proc = Process(target=worker, args=(child_conn, env_name)) self.proc.start() self.conn.recv() self.reset() def reset(self): self.conn.send([COMMAND_RESET, 0]) obs = self.conn.recv() self.last_state = self._preprocess_frame(obs) self.last_action = 0 self.last_reward = 0 def stop(self): self.conn.send([COMMAND_TERMINATE, 0]) ret = self.conn.recv() self.conn.close() self.proc.join() print("lab environment stopped") def _preprocess_frame(self, image): image = image.astype(np.float32) image = image / 255.0 return image def process(self, action): real_action = LabEnvironment.ACTION_LIST[action] self.conn.send([COMMAND_ACTION, real_action]) obs, reward, terminal = self.conn.recv() if not terminal: state = self._preprocess_frame(obs) else: state = self.last_state pixel_change = self._calc_pixel_change(state, self.last_state) self.last_state = state self.last_action = action self.last_reward = reward return state, reward, terminal, pixel_change
# -*- coding:utf-8 -*- __auth__ = 'christian' # The Pipe() function returns a pair of connection objects connected by a pipe which by default is duplex (two-way) ''' The two connection objects returned by Pipe() represent the two ends of the pipe. Each connection object has send() and recv() methods (among others). Note that data in a pipe may become corrupted if two processes (or threads) try to read from or write to the same end of the pipe at the same time. Of course there is no risk of corruption from processes using different ends of the pipe at the same time. ''' from multiprocessing import Process, Pipe def f(conn): conn.send([42, None, 'hello from child']) conn.send([42, None, 'hello from child']) print('from parent_conn:', conn.recv()) conn.close() if __name__ == '__main__': parent_conn, child_conn = Pipe() p = Process(target=f, args=(child_conn, )) p.start() print(parent_conn.recv()) # prints "[42, None, 'hello']" print(parent_conn.recv()) # prints "[42, None, 'hello']" parent_conn.send('hello chris') p.join()
class LabEnvironment(environment.Environment): ACTION_LIST = [ _action(-20, 0, 0, 0, 0, 0, 0), # look_left _action(20, 0, 0, 0, 0, 0, 0), # look_right #_action( 0, 10, 0, 0, 0, 0, 0), # look_up #_action( 0, -10, 0, 0, 0, 0, 0), # look_down _action(0, 0, -1, 0, 0, 0, 0), # strafe_left _action(0, 0, 1, 0, 0, 0, 0), # strafe_right _action(0, 0, 0, 1, 0, 0, 0), # forward _action(0, 0, 0, -1, 0, 0, 0), # backward #_action( 0, 0, 0, 0, 1, 0, 0), # fire #_action( 0, 0, 0, 0, 0, 1, 0), # jump #_action( 0, 0, 0, 0, 0, 0, 1) # crouch ] @staticmethod def get_action_size(): return len(LabEnvironment.ACTION_LIST) @staticmethod def inverse_actions(action1, action2): return np.all( np.equal( LabEnvironment.ACTION_LIST[action1] + LabEnvironment.ACTION_LIST[action2], np.zeros(len(LabEnvironment.ACTION_LIST[0])))) def __init__(self, maze_size, seed): environment.Environment.__init__(self) self.conn, child_conn = Pipe() self.proc = Process(target=worker, args=(child_conn, )) self.proc.start() self.conn.recv() self.reset(maze_size, seed) def reset(self, maze_size, seed): self.conn.send([COMMAND_RESET, [maze_size, seed]]) self.maze_size = maze_size obs, pos, ang = self.conn.recv() self.map = load_map(maze_size, seed) self.padded_map = np.pad(self._subsample( self.map - 0.5, 6), [[LOCAL_MAP_WIDTH // 6, LOCAL_MAP_WIDTH // 6], [LOCAL_MAP_WIDTH // 6, LOCAL_MAP_WIDTH // 6]], 'constant') position = self._preprocess_position(pos) angle = self._preprocess_angle(ang) self.last_state = { 'view': self._preprocess_frame(obs), 'position': position, 'angle': angle, 'egomotion': np.zeros(9), 'intended_egomotion': np.zeros(9), 'vlm': self._get_visual_local_map(self.padded_map, position[2], angle[1]) } self.last_action = 0 self.last_reward = 0 self.last_intrinsic_reward = 0 self.previous_short_term_goal = np.ones(5) def stop(self): self.conn.send([COMMAND_TERMINATE, 0]) ret = self.conn.recv() self.conn.close() self.proc.join() print("lab environment stopped") def _preprocess_frame(self, image): image = image.astype(np.float32) image = image / 255.0 return image def _preprocess_position(self, pos): #exact_position[0] = how much to the right, exact_position[1] = how much to the top exact_position = ((pos[:2] / 100)).astype(np.float32) #hit_wall = True if and only if the position is the closest the agent can get to a wall hit_wall = 1e-5 > np.min([(exact_position - 0.16125) % 1.0, (-exact_position - 0.16125) % 1.0]) #discretization to position indice pos = (pos[:2] * 3 / 100).astype(np.intc) pos[1] = 3 * self.maze_size - pos[1] - 1 position_indices = np.asarray([pos[1], pos[0]]) #one hot encoding for training position = np.zeros(shape=(63, 63)) position[pos[1]][pos[0]] = 1.0 return position, exact_position, position_indices, hit_wall def _preprocess_angle(self, ang): # 3-hot encoding angle_neurons = np.zeros(30) angle_neurons[int(ang[1] // 12) % 30] = 1.0 angle_neurons[int(ang[1] // 12 + 1) % 30] = 1.0 angle_neurons[int(ang[1] // 12 - 1) % 30] = 1.0 # converting to unit scale angle = ang[1] / 360.0 if angle < 0: angle = angle + 1.0 return angle_neurons, angle def _get_visual_local_map(self, padded_map, pos, ang): inner_pos = 2 - pos % 3 pos = pos // 3 local_map = padded_map[pos[0] - 1:pos[0] + LOCAL_MAP_WIDTH // 3 + 1, pos[1] - 1:pos[1] + LOCAL_MAP_WIDTH // 3 + 1] visual_field = np.asarray([[False] * (LOCAL_MAP_WIDTH + 6)] * (LOCAL_MAP_WIDTH + 6)) visible_local_map = np.zeros([LOCAL_MAP_WIDTH, LOCAL_MAP_WIDTH]) visible = [True] marks = [True, True, True, True] ang = ang - 0.5 if ang < 0: ang = ang + 1.0 # determine visual field for ring_radius in range(LOCAL_MAP_WIDTH // 2 + 4): ring_size = 8 * ring_radius if ring_size == 0: ring_size = 1 for i in range(2 * ring_radius + 1): ring_position = int(ang * ring_size + i) % ring_size if ring_position <= ring_size // 4: x = LOCAL_MAP_WIDTH // 2 - ring_radius + ring_position y = LOCAL_MAP_WIDTH // 2 - ring_radius elif ring_position <= 2 * ring_size // 4: x = LOCAL_MAP_WIDTH // 2 + ring_radius y = LOCAL_MAP_WIDTH // 2 - ring_radius + (ring_position - ring_size // 4) elif ring_position <= 3 * ring_size // 4: x = LOCAL_MAP_WIDTH // 2 + ring_radius - ( ring_position - 2 * ring_size // 4) y = LOCAL_MAP_WIDTH // 2 + ring_radius else: x = LOCAL_MAP_WIDTH // 2 - ring_radius y = LOCAL_MAP_WIDTH // 2 + ring_radius - ( ring_position - 3 * ring_size // 4) visual_field[x + 3][y + 3] = True # get visible local map elements for ring_radius in range(LOCAL_MAP_WIDTH // 3 + 2): ring_size = 4 * ring_radius if ring_size == 0: ring_size = 1 for i in range(ring_size): if visible[i]: x, y = self.ring_pos_to_xy(i, ring_size) in_visual_field = np.any( visual_field[3 * x + inner_pos[0]:3 * x + inner_pos[0] + 3, 3 * y + inner_pos[1]:3 * y + inner_pos[1] + 3]) if x <= LOCAL_MAP_WIDTH // 3 + 1 and x >= 0 and y <= LOCAL_MAP_WIDTH // 3 + 1 and y >= 0: value = local_map[x, y] else: value = 0 if in_visual_field: for l in range(3): x_ = 3 * (x - 1) + inner_pos[0] + l if x_ < LOCAL_MAP_WIDTH and x_ >= 0: for k in range(3): y_ = 3 * (y - 1) + inner_pos[1] + k if y_ < LOCAL_MAP_WIDTH and y_ >= 0: visible_local_map[x_][y_] = value if value < 0 or not in_visual_field: visible[i] = False visible_1 = np.concatenate([ visible[:ring_radius + 1], visible[ring_radius:2 * ring_radius + 1], visible[2 * ring_radius:3 * ring_radius + 1], visible[3 * ring_radius:], visible[:min(ring_radius, 1)] ]) visible_2 = np.concatenate([ visible[:min(ring_radius, 1)], visible[:ring_radius + 1], visible[ring_radius:2 * ring_radius + 1], visible[2 * ring_radius:3 * ring_radius + 1], visible[3 * ring_radius:] ]) visible = np.logical_and(np.logical_or(visible_1, visible_2), marks) marks = np.logical_and(visible_1, visible_2) r = ring_radius + 1 marks_1 = np.concatenate([ marks[:r + 1], marks[r:2 * r + 1], marks[2 * r:3 * r + 1], marks[3 * r:], [marks[0]] ]) marks_2 = np.concatenate([[marks[-1]], marks[:r + 1], marks[r:2 * r + 1], marks[2 * r:3 * r + 1], marks[3 * r:]]) marks = np.logical_or(marks_1, marks_2) return visible_local_map def ring_pos_to_xy(self, ring_position, ring_size): ring_radius = ring_size // 4 if ring_radius < 1: return LOCAL_MAP_WIDTH // 6 + 1, LOCAL_MAP_WIDTH // 6 + 1 if ring_position < ring_radius: x = LOCAL_MAP_WIDTH // 6 + ring_position y = LOCAL_MAP_WIDTH // 6 - ring_radius + ring_position elif ring_position < 2 * ring_radius: x = LOCAL_MAP_WIDTH // 6 + ring_radius - ring_position % ring_radius y = LOCAL_MAP_WIDTH // 6 + ring_position % ring_radius elif ring_position < 3 * ring_radius: x = LOCAL_MAP_WIDTH // 6 - ring_position % ring_radius y = LOCAL_MAP_WIDTH // 6 + ring_radius - ring_position % ring_radius else: x = LOCAL_MAP_WIDTH // 6 - ring_radius + ring_position % ring_radius y = LOCAL_MAP_WIDTH // 6 - ring_position % ring_radius return x + 1, y + 1 def _get_intrinsic_reward(self, location_uncertainty, shift_weights): short_term_goal_vector = np.asarray([ self.previous_short_term_goal[2] - self.previous_short_term_goal[0], self.previous_short_term_goal[3] - self.previous_short_term_goal[1] ]) egomotion_vector = np.asarray([ np.sum(shift_weights[:3] - shift_weights[6:9]), shift_weights[0] + shift_weights[3] + shift_weights[6] - shift_weights[2] - shift_weights[5] - shift_weights[8] ]) return np.dot(short_term_goal_vector, egomotion_vector) + ( self.previous_short_term_goal[4] - location_uncertainty) def process(self, action, short_term_goal, shift_weights): real_action = LabEnvironment.ACTION_LIST[action] self.conn.send([COMMAND_ACTION, real_action]) obs, pos, ang, reward, terminal = self.conn.recv() if not terminal: position = self._preprocess_position(pos) angle = self._preprocess_angle(ang) state = { 'view': self._preprocess_frame(obs), 'position': position, 'angle': angle, 'vlm': self._get_visual_local_map(self.padded_map, position[2], angle[1]) } if reward > 0: terminal = True intrinsic_reward = self._get_intrinsic_reward( short_term_goal[4], shift_weights) # small negative reward for running into a wall if state['position'][3]: reward = reward - 0.1 else: state = self.last_state intrinsic_reward = 0 self.previous_short_term_goal = short_term_goal self.last_state = state self.last_action = action self.last_reward = reward self.last_intrinsic_reward = intrinsic_reward return state, reward, intrinsic_reward, terminal
'out_format':'netxml', 'out_dest':output_dir, 'conn':airodump_child_conn, 'interval':'20'}) #This interval should be configurable by cmd line variable print "Creating process for folder watch" observer = Observer() #folder watchdog process to monitor outputxml from airodump-ng observer.schedule(MyHandler(), path=output_dir) airodump.start() scanning = True time_started = time.time() #print "time_started:%.0f" % time_started #debug print "Starting folder watchdog..." observer.start() while scanning == True: time.sleep(1) airodump_parent_conn.send(scanning) print "General scan now running for: %.0f seconds" % (time.time() - time_started) file_list = os.listdir(target_dir) if time.time() - time_started >= 60: print "Times up, aborting general scan..." scanning = False if file_list != []: #test for APs that havent previously been cracked/timed-out for _file in file_list: _xml = xml_machine(target_dir+_file) _xml.parse_deets() if str(_xml.cracked) == 'False': print "Targets detected, aborting general scan..." scanning = False break observer.stop()
class model(threading.Thread): def __init__(self): super(model, self).__init__() self.running = True self.title_status = "Awaiting printer status" self.sub_status = "" self.is_printing = False self.printer_received_cmd = False self.is_paused = False self.type_dict_regex = { "float": "([0-9]+\.?[0-9]+?)", "int": "([0-9]+)" } self.saved_variables = [ ["lift_speed", 5280, "int"], ["lift_amount", 25400, "int"], ["cali_min_time", 6.0, "float"], ["cali_max_time", 17.0, "float"], ["exposure_time", 13.0, "float"], ] # default the variable values for some_var, default_val, var_type in self.saved_variables: exec("self." + some_var + "=" + str(default_val)) self.load_printer_variables() self.object_paths = [] self.refresh_list() self.printer_conn, child_conn = Pipe() self.printer_process = Process(target=printer.run, args=(child_conn, )) self.printer_process.start() def run(self): try: while self.running: if self.printer_conn.poll(): self.title_status, self.sub_status = self.printer_conn.recv( ) if self.title_status == "SHUTDOWN": self.shutdown() if self.title_status == "Ready" and self.printer_received_cmd: self.is_printing = False if self.title_status != "Ready": self.printer_received_cmd = True time.sleep(0.5) except: pass printer.shutdown() self.printer_process.join() def refresh_list(self): folder_listing = os.listdir(os.getcwd()) self.object_paths = [] for listing in folder_listing: if re.match(".+\.slice$", listing): self.object_paths.append(listing) def send_command(self, command): # Save any variables coming from the command for some_var, default_val, var_type in self.saved_variables: if some_var in command: if re.match(self.type_dict_regex[var_type] + "$", command[some_var]) != None: exec("self.{0} = {1}(command['{0}'])".format( some_var, var_type)) else: # Argument is invalid, do not execute! return self.save_printer_variables() if command["command"] == "calibration": self.is_printing = True self.printer_received_cmd = False if command["command"] == "print_object": command["object_path"] = os.getcwd() + '/' + command["object_path"] self.is_printing = True self.printer_received_cmd = False if command["command"] == "Pause": self.is_paused = True if command["command"] == "Unpause": self.is_paused = False if command["command"] == "Abort": self.is_printing = False if command["command"] == "refresh_list": self.refresh_list() elif command["command"] == ["upload"]: # command["object_name"] is a tuple? command["command"].pop() self.upload(command) else: self.printer_conn.send(command) def shutdown(self): self.save_printer_variables() self.is_printing = False self.running = False def upload(self, command): f = open("temp.zip", 'w') f.write(command["datafile"][0]) f.close() call(["unzip", "-o", "temp.zip"]) call(["rm", "temp.zip"]) self.refresh_list() def load_printer_variables(self): try: f = open('printer_variables.txt') all_var = f.read() f.close() except: return for some_var, default_val, var_type in self.saved_variables: match_type = some_var + ": " + self.type_dict_regex[var_type] check_search = re.search(match_type, all_var, re.MULTILINE) if check_search != None: exec("self.{0}={1}({2})".format(some_var, var_type, check_search.group(1))) def save_printer_variables(self): try: f = open('printer_variables.txt', 'w') for some_var, default_val, var_type in self.saved_variables: f.write("{0}: {1}\r\n".format(some_var, str(eval("self." + some_var)))) f.close() except: pass
#多进程 管道 通信 from multiprocessing import Process, Pipe import time def chind(conn): conn.send('111') conn.send('222') print('from parent data', conn.recv()) conn.close() if __name__ == '__main__': parent_conn, child_conn = Pipe() p = Process(target=chind, args=(child_conn, )) p.start() print(parent_conn.recv()) print(parent_conn.recv()) time.sleep(5) parent_conn.send('333') p.join()
class Compiler: """The Compiler class.""" def __init__(self) -> None: """ Compiler Constructor. Starts a new backend compiler on the local machine with an empty WorkQueue, establishes a connection, and then starts running. Examples: >>> compiler = Compiler() >>> task = CompilationTask(...) >>> compiler.submit(task) >>> print(compiler.status(task)) TaskStatus.RUNNING """ self.conn, backend_conn = Pipe() self.process = Process(target=WorkQueue.run, args=(backend_conn, )) self.process.start() _logger.info('Started compiler process.') def __enter__(self) -> Compiler: """Enter a context for this compiler.""" return self def __exit__(self, type: Any, value: Any, traceback: Any) -> None: """Shutdowns compiler and closes connection.""" self.close() def close(self) -> None: """Shutdowns compiler and closes connection.""" self.conn.send('CLOSE') self.process.join() self.conn.close() _logger.info('Stopped compiler process.') def submit(self, task: CompilationTask) -> None: """Submit a CompilationTask to the Compiler.""" self.conn.send('SUBMIT') self.conn.send(task) okay_msg = self.conn.recv() # Block until response if (okay_msg != 'OKAY'): raise Exception('Failed to submit job.') _logger.info('Submitted task: %s' % task.task_id) def status(self, task: CompilationTask) -> TaskStatus: """Retrieve the status of the specified CompilationTask.""" self.conn.send('STATUS') self.conn.send(task.task_id) return self.conn.recv() # Block until response def result(self, task: CompilationTask) -> TaskResult: """Block until the CompilationTask is finished, return its result.""" self.conn.send('RESULT') self.conn.send(task.task_id) return self.conn.recv() # Block until response def remove(self, task: CompilationTask) -> None: """Remove a task from the compiler's workqueue.""" self.conn.send('REMOVE') self.conn.send(task.task_id) self.conn.recv() # Block until response _logger.info('Removed task: %s' % task.task_id) def compile(self, task: CompilationTask) -> Circuit: """Execute the CompilationTask.""" _logger.info('Compiling task: %s' % task.task_id) self.submit(task) result = self.result(task) return result.get_circuit()
class ProcessServer(Process, BaseImplServer): profile_filename = "profile.log" profile_processed_filename = "profile.log.processed" def __init__(self, command_channel, event_queue, featurelist): BaseImplServer.__init__(self) Process.__init__(self) self.command_channel = command_channel self.event_queue = event_queue self.event = EventAdapter(event_queue) self.featurelist = featurelist self.quit = False self.quitin, self.quitout = Pipe() self.event_handle = multiprocessing.Value("i") def run(self): for event in bb.event.ui_queue: self.event_queue.put(event) self.event_handle.value = bb.event.register_UIHhandler(self, True) bb.cooker.server_main(self.cooker, self.main) def main(self): # Ignore SIGINT within the server, as all SIGINT handling is done by # the UI and communicated to us self.quitin.close() signal.signal(signal.SIGINT, signal.SIG_IGN) bb.utils.set_process_name("Cooker") while not self.quit: try: if self.command_channel.poll(): command = self.command_channel.recv() self.runCommand(command) if self.quitout.poll(): self.quitout.recv() self.quit = True try: self.runCommand(["stateForceShutdown"]) except: pass self.idle_commands(.1, [self.command_channel, self.quitout]) except Exception: logger.exception('Running command %s', command) self.event_queue.close() bb.event.unregister_UIHhandler(self.event_handle.value) self.command_channel.close() self.cooker.shutdown(True) self.quitout.close() def idle_commands(self, delay, fds=None): nextsleep = delay if not fds: fds = [] for function, data in self._idlefuns.items(): try: retval = function(self, data, False) if retval is False: del self._idlefuns[function] nextsleep = None elif retval is True: nextsleep = None elif isinstance(retval, float): if (retval < nextsleep): nextsleep = retval elif nextsleep is None: continue else: fds = fds + retval except SystemExit: raise except Exception as exc: if not isinstance(exc, bb.BBHandledException): logger.exception('Running idle function') del self._idlefuns[function] self.quit = True if nextsleep is not None: select.select(fds,[],[],nextsleep) def runCommand(self, command): """ Run a cooker command on the server """ self.command_channel.send(self.cooker.command.runCommand(command)) def stop(self): self.quitin.send("quit") self.quitin.close()
def main(): """ The main() function. """ # Hold mp pipes mp.freeze_support() print("STARTING SPOT TRAINING ENV") seed = 0 max_timesteps = 4e6 eval_freq = 1e1 save_model = True file_name = "spot_ars_" if ARGS.HeightField: height_field = True else: height_field = False if ARGS.NoContactSensing: contacts = False else: contacts = True if ARGS.DontRandomize: env_randomizer = None else: env_randomizer = SpotEnvRandomizer() # Find abs path to this file my_path = os.path.abspath(os.path.dirname(__file__)) results_path = os.path.join(my_path, "../results") if contacts: models_path = os.path.join(my_path, "../models/contact") else: models_path = os.path.join(my_path, "../models/no_contact") if not os.path.exists(results_path): os.makedirs(results_path) if not os.path.exists(models_path): os.makedirs(models_path) env = spotBezierEnv(render=False, on_rack=False, height_field=height_field, draw_foot_path=False, contacts=contacts, env_randomizer=env_randomizer) # Set seeds env.seed(seed) np.random.seed(seed) state_dim = env.observation_space.shape[0] print("STATE DIM: {}".format(state_dim)) action_dim = env.action_space.shape[0] print("ACTION DIM: {}".format(action_dim)) max_action = float(env.action_space.high[0]) env.reset() g_u_i = GUI(env.spot.quadruped) spot = SpotModel() T_bf = spot.WorldToFoot bz_step = BezierStepper(dt=env._time_step) bzg = BezierGait(dt=env._time_step) # Initialize Normalizer normalizer = Normalizer(state_dim) # Initialize Policy policy = Policy(state_dim, action_dim) # Initialize Agent with normalizer, policy and gym env agent = ARSAgent(normalizer, policy, env, bz_step, bzg, spot) agent_num = 0 if os.path.exists(models_path + "/" + file_name + str(agent_num) + "_policy"): print("Loading Existing agent") agent.load(models_path + "/" + file_name + str(agent_num)) env.reset(agent.desired_velocity, agent.desired_rate) episode_reward = 0 episode_timesteps = 0 episode_num = 0 # Create mp pipes num_processes = policy.num_deltas processes = [] childPipes = [] parentPipes = [] # Store mp pipes for pr in range(num_processes): parentPipe, childPipe = Pipe() parentPipes.append(parentPipe) childPipes.append(childPipe) # Start multiprocessing # Start multiprocessing for proc_num in range(num_processes): p = mp.Process(target=ParallelWorker, args=(childPipes[proc_num], env, state_dim)) p.start() processes.append(p) print("STARTED SPOT TRAINING ENV") t = 0 while t < (int(max_timesteps)): # Maximum timesteps per rollout episode_reward, episode_timesteps = agent.train_parallel(parentPipes) t += episode_timesteps # episode_reward = agent.train() # +1 to account for 0 indexing. # +0 on ep_timesteps since it will increment +1 even if done=True print( "Total T: {} Episode Num: {} Episode T: {} Reward: {:.2f} REWARD PER STEP: {:.2f}" .format(t + 1, episode_num, episode_timesteps, episode_reward, episode_reward / float(episode_timesteps))) # Store Results (concat) if episode_num == 0: res = np.array( [[episode_reward, episode_reward / float(episode_timesteps)]]) else: new_res = np.array( [[episode_reward, episode_reward / float(episode_timesteps)]]) res = np.concatenate((res, new_res)) # Also Save Results So Far (Overwrite) # Results contain 2D numpy array of total reward for each ep # and reward per timestep for each ep np.save(results_path + "/" + str(file_name), res) # Evaluate episode if (episode_num + 1) % eval_freq == 0: if save_model: agent.save(models_path + "/" + str(file_name) + str(episode_num)) episode_num += 1 # Close pipes and hence envs for parentPipe in parentPipes: parentPipe.send([_CLOSE, "pay2"]) for p in processes: p.join()
class ProcessEngine(BaseEngine): """An engine executing the tasks it is sent in a different process. """ def perform(self, exec_infos): """Execute a given task. Parameters ---------- exec_infos : ExecutionInfos TaskInfos object describing the work to expected of the engine. Returns ------- exec_infos : ExecutionInfos Input object whose values have been updated. This is simply a convenience. Notes ----- IOError in pipe are raised only if an operation is attempted from the process that closed the pipe, but never when trying to poll from a different process. """ self.status = 'Running' # Clear all the flags. self._task_pause.clear() self._task_paused.clear() self._task_resumed.clear() self._task_stop.clear() self._force_stop.clear() self._stop_requested = False # If the process does not exist or is dead create a new one. if not self._process or not self._process.is_alive(): self._process_stop.clear() # Create the subprocess and the pipe. self._pipe, process_pipe = Pipe() self._process = TaskProcess(process_pipe, self._log_queue, self._monitor_queue, self._task_pause, self._task_paused, self._task_resumed, self._task_stop, self._process_stop) self._process.daemon = True # Create the logger thread in charge of dispatching log reports. self._log_thread = QueueLoggerThread(self._log_queue) self._log_thread.daemon = True logger.debug('Starting logging thread.') self._log_thread.start() # Create the monitor thread dispatching engine news to the monitor. self._monitor_thread = ThreadMeasureMonitor( self, self._monitor_queue) self._monitor_thread.daemon = True logger.debug('Starting monitoring thread.') self._monitor_thread.start() self._pause_thread = None # Start process. logger.debug('Starting subprocess') self._process.start() # Send the measurement. args = self._build_subprocess_args(exec_infos) try: self._pipe.send(args) except Exception: msg = ('Failed to send infos to subprocess :\n-infos : \n%s\n' '-errors :\n%s') logger.error(msg % (pformat(args), format_exc())) self._log_queue.put(None) self._monitor_queue.put((None, None)) self._cleanup(process=True) exec_infos.success = False exec_infos.errors['engine'] = msg self.status = 'Stopped' return exec_infos else: logger.debug('Task {} sent'.format(exec_infos.id)) # Check that the engine did receive the task. while not self._pipe.poll(2): if not self._process.is_alive(): msg = 'Subprocess was found dead unexpectedly' logger.debug(msg) self._log_queue.put(None) self._monitor_queue.put((None, None)) self._cleanup(process=False) exec_infos.success = False exec_infos.errors['engine'] = msg self.status = 'Stopped' return exec_infos # Simply empty the pipe the subprocess always send True if it answers self._pipe.recv() # Wait for the process to finish the measurement and check it has not # been killed. while not self._pipe.poll(1): if self._force_stop.is_set(): msg = 'Subprocess was terminated by the user.' logger.debug(msg) self._cleanup(process=False) exec_infos.errors['engine'] = msg self.status = 'Stopped' return exec_infos elif not self._process.is_alive(): msg = 'Subprocess was found dead unexpectedly' logger.debug(msg) self._log_queue.put(None) self._monitor_queue.put((None, None)) self._cleanup(process=False) exec_infos.success = False exec_infos.errors['engine'] = msg self.status = 'Stopped' return exec_infos # Here get message from process and react result, errors = self._pipe.recv() logger.debug('Subprocess done performing measurement') exec_infos.success = result exec_infos.errors.update(errors) self.status = 'Waiting' return exec_infos def pause(self): """Ask the engine to pause the current task execution. """ self.status = 'Pausing' self._task_resumed.clear() self._task_paused.clear() self._task_pause.set() self._pause_thread = Thread(target=self._wait_for_pause) self._pause_thread.start() def resume(self): """Ask the engine to resume the currently paused job. """ self.status = 'Resuming' self._task_pause.clear() def stop(self, force=False): """Ask the engine to stop the current job. This method should not wait for the job to stop save if a forced stop was requested. Parameters ---------- force : bool, optional Force the engine to stop the performing the task. This allow the engine to use any means necessary to stop, in this case only should the call to this method block. """ self.status = 'Stopping' self._stop_requested = True self._task_stop.set() if force: self._force_stop.set() # Stop running queues self._log_queue.put(None) self._monitor_queue.put((None, None)) # Terminate the process and make sure all threads stopped properly. self._process.terminate() self._log_thread.join() self._monitor_thread.join() # Discard the queues as they may have been corrupted when the # process was terminated. self._log_queue = Queue() self._monitor_queue = Queue() self.status = 'Stopped' def shutdown(self, force=False): """Ask the engine to stop completely. Parameters ---------- force : bool, optional Force the engine to stop the performing the task. This allow the engine to use any means necessary to stop, in this case only should the call to this method block. """ self.status = 'Shutting down' self._stop_requested = True self._task_stop.set() if not force: t = Thread(target=self._cleanup) t.start() else: self.stop(force=True) # ========================================================================= # --- Private API --------------------------------------------------------- # ========================================================================= #: Boolean indicating that the user requested the job to stop. _stop_requested = Bool() #: Interprocess event used to pause the subprocess current job. _task_pause = Value(factory=Event) #: Interprocess event signaling the subprocess current job is paused. _task_paused = Value(factory=Event) #: Interprocess event signaling the subprocess current job has resumed. _task_resumed = Value(factory=Event) #: Interprocess event used to stop the subprocess current measurement. _task_stop = Value(factory=Event) #: Interprocess event used to stop the subprocess. _process_stop = Value(factory=Event) #: Flag signaling that a forced exit has been requested _force_stop = Value(factory=tEvent) #: Current subprocess. _process = Typed(TaskProcess) #: Connection used to send and receive messages about execution (type #: ambiguous when the OS is not known) _pipe = Value() #: Inter-process queue used by the subprocess to transmit its log records. _log_queue = Value(factory=Queue) #: Thread in charge of collecting the log message coming from the #: subprocess. _log_thread = Typed(Thread) #: Inter-process queue used by the subprocess to send the values of the #: observed database entries. _monitor_queue = Value(factory=Queue) #: Thread in charge of collecting the values of the observed database #: entries. _monitor_thread = Typed(Thread) #: Thread in charge of notifying the engine that the engine did #: pause/resume after being asked to do so. _pause_thread = Typed(Thread) def _cleanup(self, process=True): """ Helper method taking care of making sure that everybody stops. Parameters ---------- process : bool Whether to join the worker process. Used when the process has been termintaed abruptly. """ logger.debug('Cleaning up') if process and self._process: self._process_stop.set() self._process.join() logger.debug('Subprocess joined') if self._pipe: self._pipe.close() if self._log_thread: self._log_thread.join() logger.debug('Log thread joined') if self._monitor_thread: self._monitor_thread.join() logger.debug('Monitor thread joined') if self._pause_thread: self._pause_thread.join() logger.debug('Pause thread joined') self.status = 'Stopped' def _build_subprocess_args(self, exec_infos): """Build the tuple to send to the subprocess. """ exec_infos.task.update_preferences_from_members() config = exec_infos.task.preferences database_root_state = exec_infos.task.database.copy_node_values() return (exec_infos.id, config, exec_infos.build_deps, exec_infos.runtime_deps, exec_infos.observed_entries, database_root_state, exec_infos.checks) def _wait_for_pause(self): """ Wait for the _task_paused event to be set. """ stop_sig = self._task_stop paused_sig = self._task_paused while not stop_sig.is_set(): if paused_sig.wait(0.1): self.status = 'Paused' break resuming_sig = self._task_resumed while not stop_sig.is_set(): if resuming_sig.wait(1): self.status = 'Running' break
class RTLSDR(Device): BYTES_PER_SAMPLE = 2 # RTLSDR device produces 8 bit unsigned IQ data def __init__(self, freq, gain, srate, device_number, is_ringbuffer=False): super().__init__(0, freq, gain, srate, is_ringbuffer) self.success = 0 self.is_receiving_p = Value('i', 0) """ Shared Value to communicate with the receiving process. """ self.bandwidth_is_adjustable = hasattr( rtlsdr, "set_tuner_bandwidth") # e.g. not in Manjaro Linux / Ubuntu 14.04 self._max_frequency = 6e9 self._max_sample_rate = 3200000 self._max_frequency = 6e9 self._max_bandwidth = 3200000 self._max_gain = 500 # Todo: Consider get_tuner_gains for allowed gains here self.device_number = device_number def open(self): pass # happens in start rx mode def close(self): pass # happens in stop tx mode def start_rx_mode(self): self.init_recv_buffer() self.is_open = True self.is_receiving = True self.receive_process = Process( target=receive_sync, args=(self.child_conn, self.device_number, self.frequency, self.sample_rate, self.gain)) self.receive_process.daemon = True self._start_read_rcv_buffer_thread() self.receive_process.start() def stop_rx_mode(self, msg): self.is_receiving = False self.parent_conn.send("stop") logger.info("RTLSDR: Stopping RX Mode: " + msg) if hasattr(self, "receive_process"): self.receive_process.join(0.3) if self.receive_process.is_alive(): logger.warning( "RTLSDR: Receive process is still alive, terminating it") self.receive_process.terminate() self.receive_process.join() self.parent_conn, self.child_conn = Pipe() if hasattr(self, "read_queue_thread" ) and self.read_recv_buffer_thread.is_alive(): try: self.read_recv_buffer_thread.join(0.001) logger.info("RTLSDR: Joined read_queue_thread") except RuntimeError: logger.error("RTLSDR: Could not join read_queue_thread") def set_device_frequency(self, frequency): self.parent_conn.send("center_freq:{}".format(int(frequency))) def set_device_sample_rate(self, sample_rate): self.parent_conn.send("sample_rate:{}".format(int(sample_rate))) def set_freq_correction(self, ppm): ret = rtlsdr.set_freq_correction(int(ppm)) self.log_retcode(ret, "Set frequency correction") def set_offset_tuning(self, on: bool): ret = rtlsdr.set_offset_tuning(on) self.log_retcode(ret, "Set offset tuning") def set_gain_mode(self, manual: bool): ret = rtlsdr.set_tuner_gain_mode(manual) self.log_retcode(ret, "Set gain mode manual") def set_if_gain(self, gain): ret = rtlsdr.set_tuner_if_gain(1, int(gain)) self.log_retcode(ret, "Set IF gain") def set_gain(self, gain): self.parent_conn.send("tuner_gain:{}".format(int(gain))) def set_device_gain(self, gain): self.set_gain(gain) def set_device_bandwidth(self, bandwidth): if hasattr(rtlsdr, "set_tuner_bandwidth"): self.parent_conn.send("tuner_bandwidth:{}".format(int(bandwidth))) else: logger.warning( "Setting the bandwidth is not supported by your RTL-SDR driver version." ) @staticmethod def unpack_complex(buffer, nvalues: int): """ The raw, captured IQ data is 8 bit unsigned data. :return: """ result = np.empty(nvalues, dtype=np.complex64) unpacked = np.frombuffer(buffer, dtype=[('r', np.uint8), ('i', np.uint8)]) result.real = (unpacked['r'] / 127.5) - 1.0 result.imag = (unpacked['i'] / 127.5) - 1.0 return result @staticmethod def pack_complex(complex_samples: np.ndarray): return (127.5 * (complex_samples.view(np.float32) + 1.0)).astype( np.uint8).tostring()