def tempControlProcTest(mode, cycle_time, duty_cycle, set_point, k_param, i_param, d_param, conn): p = current_process() print 'Starting:', p.name, p.pid parent_conn_temp, child_conn_temp = Pipe() ptemp = Process(name = "getrandProc", target=getrandProc, args=(child_conn_temp,)) #ptemp.daemon = True ptemp.start() parent_conn_heat, child_conn_heat = Pipe() pheat = Process(name = "heatProctest", target=heatProctest, args=(cycle_time, duty_cycle, child_conn_heat)) #pheat.daemon = True pheat.start() while (True): if parent_conn_temp.poll(): randnum = parent_conn_temp.recv() #non blocking receive conn.send([randnum, mode, cycle_time, duty_cycle, set_point, k_param, i_param, d_param]) if parent_conn_heat.poll(): cycle_time, duty_cycle = parent_conn_heat.recv() #duty_cycle = on_time/offtime*100.0 #cycle_time = on_time + off_time if conn.poll(): mode, cycle_time, duty_cycle, set_point, k_param, i_param, d_param = conn.recv() #conn.send([mode, cycle_time, duty_cycle]) #if mode == "manual": parent_conn_heat.send([cycle_time, duty_cycle])
def test_fork(self): """Test using a connection before and after a fork. """ if sys.platform == "win32": raise SkipTest() try: from multiprocessing import Process, Pipe except ImportError: raise SkipTest() db = Connection(self.host, self.port).pymongo_test # Failure occurs if the connection is used before the fork db.test.find_one() db.connection.end_request() def loop(pipe): while True: try: db.test.insert({"a": "b"}, safe=True) for _ in db.test.find(): pass except: pipe.send(True) os._exit(1) cp1, cc1 = Pipe() cp2, cc2 = Pipe() p1 = Process(target=loop, args=(cc1,)) p2 = Process(target=loop, args=(cc2,)) p1.start() p2.start() p1.join(1) p2.join(1) p1.terminate() p2.terminate() p1.join() p2.join() cc1.close() cc2.close() # recv will only have data if the subprocess failed try: cp1.recv() self.fail() except EOFError: pass try: cp2.recv() self.fail() except EOFError: pass
def QuickSortMPListArray(A,conn,NumProcs): print str(len(A))+' starting mplarray' if len(A)<=1 : conn.send(A) conn.close() elif int(NumProcs)<1: print 'proc limit reached, smp qs' conn.send(QuickSortListArray(A)) conn.close() else: pvalue=A[0] lesser=[x for x in A if int(x[0]) < int(pvalue[0])] greater=[x for x in A if int(x[0]) > int(pvalue[0])] pv=[x for x in A if int(x[0]) == int(pvalue[0])] Procs=int(NumProcs)-1 pConnLeft,cConnLeft=Pipe() leftProc=Process(target=QuickSortMPListArray,args=(lesser,cConnLeft,Procs)) pConnRight,cConnRight=Pipe() rightProc=Process(target=QuickSortMPListArray,args=(greater,cConnRight,Procs)) leftProc.start() rightProc.start() print 'mplarray send' conn.send(pConnRight.recv()+pv+pConnLeft.recv()) # conn.send(pConnLeft.recv()+[PivotValue]+pConnRight.recv()) conn.close() leftProc.join() rightProc.join() return
def QuickSortMP(A,conn,NumProcs): if len(A)<=1 : conn.send(A) conn.close() elif int(NumProcs)<1: conn.send(QuickSort(A)) conn.close() else: PivotIndex=random.randint(0,len(A)-1) PivotValue=A.pop(PivotIndex) pv=[] pv.append(int(PivotValue)) lesser=[int(x) for x in A if x < PivotValue] greater=[int(x) for x in A if x >= PivotValue] Procs=int(NumProcs)-1 pConnLeft,cConnLeft=Pipe() leftProc=Process(target=QuickSortMP,args=(lesser,cConnLeft,Procs)) pConnRight,cConnRight=Pipe() rightProc=Process(target=QuickSortMP,args=(greater,cConnRight,Procs)) leftProc.start() rightProc.start() leftStr=pConnLeft.recv() rightStr=pConnRight.recv() conn.send(leftStr+pv+rightStr) # conn.send(pConnLeft.recv()+[PivotValue]+pConnRight.recv()) conn.close() leftProc.join() rightProc.join() return
def process_pipe(): parent_conn, child_conn = Pipe() p = Process(target=pipe_test, args=(child_conn,)) p.start() print parent_conn.recv() p.join() parent_conn.close()
def startWith2Threads(trainData, labels, testData): print("startWith4Threads started") parent_conn1, child_conn1 = Pipe() parent_conn2, child_conn2 = Pipe() t1 = threading.Thread(target=RunAllFeatures, args = (trainData, labels, testData, child_conn1, 0, 20, )) t2 = threading.Thread(target=RunAllFeatures, args = (trainData, labels, testData, child_conn2, 20, 39, )) t1.start() t2.start() rr1 = parent_conn1.recv() rr2 = parent_conn2.recv() t1.join(); t2.join(); print("all tasks done") rr1 = np.row_stack((rr1, rr2)) print("result combined") res = np.transpose(rr1) pp = Normalization(res) print("normalization done") return pp
def test_fork(self): # Test using a client before and after a fork. if sys.platform == "win32": raise SkipTest("Can't fork on Windows") try: from multiprocessing import Process, Pipe except ImportError: raise SkipTest("No multiprocessing module") db = self._get_client().pymongo_test # Failure occurs if the client is used before the fork db.test.find_one() def loop(pipe): while True: try: db.test.insert({"a": "b"}) for _ in db.test.find(): pass except: traceback.print_exc() pipe.send(True) os._exit(1) cp1, cc1 = Pipe() cp2, cc2 = Pipe() p1 = Process(target=loop, args=(cc1,)) p2 = Process(target=loop, args=(cc2,)) p1.start() p2.start() p1.join(1) p2.join(1) p1.terminate() p2.terminate() p1.join() p2.join() cc1.close() cc2.close() # recv will only have data if the subprocess failed try: cp1.recv() self.fail() except EOFError: pass try: cp2.recv() self.fail() except EOFError: pass db.connection.close()
def _process_image(self, image): # Create pipes for process communication std_size_pipe, std_size_child_end = Pipe() thumb_pipe, thumb_child_end = Pipe() # Create the threads threads = [ Process( target=self._thumbnail, args=(image, thumb_child_end, (128, 128)) ), Process( target=self._thumbnail, args=(image, std_size_child_end, (1920, 1080)) ) ] # Start threads for thread in threads: thread.start() # Get processed images content and close threads images_bytes = ( thumb_pipe.recv(), std_size_pipe.recv() ) for thread in threads: thread.join() for obj in images_bytes: if type(obj) is IOError: raise obj return images_bytes
def run(self): logging.info('Visualizer thread started') parent_end, child_end = Pipe() # Sensible default value for max_process max_process = 2 process_count = 0 while not self.stop or not self.job_backlog.empty(): while parent_end.poll(0.1): parent_end.recv() ## currently not using the info... irrelevant ## TODO - a signal to notify the viewer that visuzaliztion job has been finished... #self.controller.view_update(self) process_count -= 1 if self.job_backlog.empty(): time.sleep(1) elif process_count < max_process: process_count += 1 run_name, function, snapshot = self.job_backlog.get_nowait() if not (run_name in self.remove_run_name): logging.info('Added job to visuzalizer Que: ' + str(run_name)) logging.info('No. of jobs in Que: ' + str(process_count)) p = Process(target=self.render_graph, args=(function, snapshot, run_name, child_end)) p.start() logging.info('Visualizer Finished')
def QuickSortMPListArray(A,conn,NumProcs): if len(A)<=1 : conn.send(A) conn.close() elif int(NumProcs)<1: conn.send(QuickSortListArray(A)) conn.close() else: lesser=[] greater=[] pv=[] Pivot=A.pop(0) pvVal=int(Pivot[0]) lesser=[x for x in A if x[0] < pvVal] greater=[x for x in A if x[0] > pvVal] pv=[x for x in A if x[0] == pvVal] pv.append(Pivot) Procs=int(NumProcs)-1 pConnLeft,cConnLeft=Pipe() leftProc=Process(target=QuickSortMPListArray,args=(lesser,cConnLeft,Procs)) pConnRight,cConnRight=Pipe() rightProc=Process(target=QuickSortMPListArray,args=(greater,cConnRight,Procs)) leftProc.start() rightProc.start() conn.send(pConnLeft.recv()+pv+pConnRight.recv()) conn.close() leftProc.join() rightProc.join() return
def onExeBtn(self, event): srcFiles = self.projPane.getValue() self.analyzerOutDir = os.path.join(self.projRootDir, 'analyzer_output') oldcwd = os.getcwd() if not os.path.exists(self.analyzerOutDir): os.makedirs(self.analyzerOutDir) os.chdir(self.analyzerOutDir) rcv_pipe, snd_pipe = Pipe(duplex=True) self.dbname = ''.join([self.projRootDir.replace(os.sep, '_').strip('_'), '.sqlite']) self.exePane.dbPathTc.SetValue(os.path.join(self.analyzerOutDir, self.dbname)) self.exePane.ppLogFileTc.SetValue(os.path.join(self.analyzerOutDir, 'preprocessor.log')) self.exePane.parserLogFileTc.SetValue(os.path.join(self.analyzerOutDir, 'parser.log')) p = Process(target=analyze, args=(snd_pipe, os.path.join(self.analyzerOutDir, self.dbname), self.getPpCfg(), self.getParserCfg(), srcFiles, self.exePane.pipelineCb.GetValue(), self.exePane.numProcSc.GetValue(), self.exePane.numPpProcSc.GetValue(), self.exePane.numParserProcSc.GetValue(), )) p.start() dlg = wx.ProgressDialog('Executing', '0/%d' % len(srcFiles), parent=self, maximum=len(srcFiles)*10, style=wx.PD_CAN_ABORT | wx.PD_APP_MODAL | wx.PD_ELAPSED_TIME ) dlg.SetSize((500,150)) dlg.Layout() dlg.Show() result = None while True: i, total, result = rcv_pipe.recv() ret = dlg.Update(i*10, result if i == total else '[%d/%d] %s ... done' % (i+1, total, result)) if ret[0] == False: rcv_pipe.send('STOP') while result != 'STOPPED': result = rcv_pipe.recv() dlg.Update(total*10, 'Canceled') break if i == total: break p.join() self.exePane.dbPathTc.SetValue(os.path.join(self.analyzerOutDir, self.dbname)) os.chdir(oldcwd)
def quicksort(self,arr, conn, procNum): if procNum <= 0 or len(arr)<= 1: conn.send(self.qck(arr)) conn.close() return #print 'Just in case you don't trust that this program works better than other quicksorts :3 FUBAR. process id:', os.getppid() pivot = arr.pop(random.randint(0, len(arr)-1)) leftSide = [x for x in arr if x < pivot] rightSide = [x for x in arr if x > pivot] pconnLeft, cconnLeft = Pipe() leftProc = Process(target= self.quicksort, args=(leftSide, cconnLeft,procNum -1)) pconnRight, cconnRight = Pipe() rightProc = Process(target=self.quicksort, args=(rightSide, cconnRight, procNum - 1)) leftProc.start() rightProc.start() conn.send(pconnLeft.recv() + [pivot] + pconnRight.recv()) conn.close() leftProc.join() rightProc.join()
class MMLWorker(object): """Represents the Daemon's connection to the subprocess""" def __init__(self,runner): """Creates and initalizes a subprocess and its connections.""" self.pipe, pipe = Pipe() self.proc = Process(target=worker, args=(pipe,runner)) self.proc.start(); self.pid = self.proc.pid def __del__(self): """Ensures the subprocess is correctly garbage collected.""" self.pipe.close(); self.proc.terminate(); def pump(self,block=False): """Returns a key,val pair from the subprocess, or None,None.""" key,val = None,None if block: (key,val) = self.pipe.recv() elif self.pipe.poll(): (key,val) = self.pipe.recv() return key,val def stop(self): """Sends the stop signal to the subprocess.""" self.pipe.send(('stop',())) def pause(self): """Sends the pause signal to the subprocess.""" self.pipe.send(('pause',())) def start(self,program): """Sends the start signal to the subprocess.""" self.pipe.send(('start',(program,)))
def smpScore(M1,M2,M3,D1SCORE,D2SCORE,D3SCORE,D4SCORE): st=time.time() pconn,cconn=Pipe() qconn,dconn=Pipe() rconn,econn=Pipe() nLIST=[] p=Process(target=smpScoFunction,args=(M1,cconn,D1SCORE,D2SCORE,D3SCORE,D4SCORE)) q=Process(target=smpScoFunction,args=(M2,dconn,D1SCORE,D2SCORE,D3SCORE,D4SCORE)) r=Process(target=smpScoFunction,args=(M3,econn,D1SCORE,D2SCORE,D3SCORE,D4SCORE)) p.start() q.start() r.start() p1LIST=pconn.recv() p2LIST=qconn.recv() p3LIST=rconn.recv() p.join() q.join() r.join() nLIST=p1LIST+p2LIST+p3LIST et=time.time() tt=et-st print 'smp scoring done in '+str(tt)+' seconds.' return nLIST
def usage3(): # The Pipe() function returns a pair of connection objects # connected by a pipe which by default is duplex (two-way) parent_conn, child_conn = Pipe() child_conn.send('wenychan3') child_conn.close() print parent_conn.recv() # prints "[42, None, 'hello']"
def main(): parent_conn, child_conn = Pipe() p = Process(target = f, args=(child_conn,)) p.start() # if you comment this line you can get the result after join print(parent_conn.recv()) p.join() print('Hello: ', parent_conn.recv())
def test_pool_with_fork(self): # Test that separate MongoClients have separate Pools, and that the # driver can create a new MongoClient after forking if sys.platform == "win32": raise SkipTest("Can't test forking on Windows") try: from multiprocessing import Process, Pipe except ImportError: raise SkipTest("No multiprocessing module") a = self.get_client(auto_start_request=False) a.pymongo_test.test.remove() a.pymongo_test.test.insert({'_id':1}) a.pymongo_test.test.find_one() self.assertEqual(1, len(get_pool(a).sockets)) a_sock = one(get_pool(a).sockets) def loop(pipe): c = self.get_client(auto_start_request=False) self.assertEqual(1,len(get_pool(c).sockets)) c.pymongo_test.test.find_one() self.assertEqual(1,len(get_pool(c).sockets)) pipe.send(one(get_pool(c).sockets).sock.getsockname()) cp1, cc1 = Pipe() cp2, cc2 = Pipe() p1 = Process(target=loop, args=(cc1,)) p2 = Process(target=loop, args=(cc2,)) p1.start() p2.start() p1.join(1) p2.join(1) p1.terminate() p2.terminate() p1.join() p2.join() cc1.close() cc2.close() b_sock = cp1.recv() c_sock = cp2.recv() self.assertTrue(a_sock.sock.getsockname() != b_sock) self.assertTrue(a_sock.sock.getsockname() != c_sock) self.assertTrue(b_sock != c_sock) # a_sock, created by parent process, is still in the pool d_sock = get_pool(a).get_socket() self.assertEqual(a_sock, d_sock) d_sock.close()
def test_mount_fstab_user_fail(self): if not self._can_create: self.skipTest('Cannot create %s filesystem' % self._fs_name) if not self._can_mount: self.skipTest('Cannot mount %s filesystem' % self._fs_name) # this test will change /etc/fstab, we might want to revert the changes after it finishes fstab = self.read_file('/etc/fstab') self.addCleanup(self.write_file, '/etc/fstab', fstab) disk = self.get_object('/block_devices/' + os.path.basename(self.vdevs[0])) self.assertIsNotNone(disk) # create filesystem disk.Format(self._fs_name, self.no_options, dbus_interface=self.iface_prefix + '.Block') self.addCleanup(self._clean_format, disk) # create user for our test self.addCleanup(self._remove_user, self.username) uid, gid = self._add_user(self.username) # add unmount cleanup now in case something wrong happens in the other process self.addCleanup(self._unmount, self.vdevs[0]) # create pipe to get error (if any) parent_conn, child_conn = Pipe() proc = Process(target=self._mount_as_user_fstab_fail, args=(child_conn, int(uid), int(gid), self.vdevs[0])) proc.start() res = parent_conn.recv() parent_conn.close() proc.join() if not res[0]: self.fail(res[1]) # now mount it as root and test that user can't unmount it mnt_path = disk.Mount(self.no_options, dbus_interface=self.iface_prefix + '.Filesystem') self.assertIsNotNone(mnt_path) self.assertTrue(os.path.ismount(mnt_path)) # create pipe to get error (if any) parent_conn, child_conn = Pipe() proc = Process(target=self._unmount_as_user_fstab_fail, args=(child_conn, int(uid), int(gid), self.vdevs[0])) proc.start() res = parent_conn.recv() parent_conn.close() proc.join() if not res[0]: self.fail(res[1]) self.assertTrue(os.path.ismount(mnt_path)) self._unmount(mnt_path)
def setUpClass(cls): parent_conn, child_conn = Pipe() proc = Process(target=cls._start_groonga, args=(child_conn,)) proc.daemon = True proc.start() cls.serverpid = parent_conn.recv() proc.join(1) # 生存戦略 if not proc.is_alive(): raise RuntimeError(parent_conn.recv())
def Compare(Original_DB, support, Sens_f, selection, fNames): time = ['']*len(selection) s_ef = [0]*len(selection) ch_raw_dat = [0]*len(selection) fil = [0]*len(selection) rev_t = ['']*len(selection) CPU_time = 0 if any(selection): out_fname = os.getcwd()+'\\Apriori_init.txt' parent, child = Pipe() p = Process(target=aprioriWorker, args=(Original_DB, support, out_fname, child,)) p.start() response = parent.recv() parent.close() p.join() p = None parent = None child = None if response == -1: return(response) elif type(response) == type(tuple): return(response) CPU_time = response for index in [i for i in xrange(len(selection)) if selection[i]]: parent, child = Pipe() p = Process(target=functionWorker, args=(fNames[index], out_fname, Sens_f, Original_DB, support, child,)) p.start() response = parent.recv() parent.close() p.join() if response == -1: return(response) elif type(response) == type(tuple) and len(response) == 2: return((response[0], fNames[index])) print(response) rev_t[index], ch_raw_dat[index], time[index] = response parent, child = Pipe() p = Process(target=metricWorker, args=(out_fname, fNames[index]+"_results.txt", Sens_f, support, child,)) p.start() s_ef[index], fil[index]= parent.recv() parent.close() p.join() p = None parent = None child = None gc.collect() return(s_ef, ch_raw_dat, time, rev_t, fil, CPU_time)
def test_invoke_runs_another_thread(self): q = Queue() parent, child = Pipe() def task(): q.put(current_process().name) time.sleep(1) child.send(1) sut = BackgroundTask(task) sut.invoke() parent.recv() self.assertNotEquals(q.get(), current_process().name)
def test_pool_with_fork(self): # Test that separate Connections have separate Pools, and that the # driver can create a new Connection after forking if sys.platform == "win32": raise SkipTest("Can't test forking on Windows") try: from multiprocessing import Process, Pipe except ImportError: raise SkipTest("No multiprocessing module") a = self.get_connection(auto_start_request=False) a.test.test.remove(safe=True) a.test.test.insert({'_id':1}, safe=True) a.test.test.find_one() self.assertEqual(1, len(a._Connection__pool.sockets)) a_sock = one(a._Connection__pool.sockets) def loop(pipe): c = self.get_connection(auto_start_request=False) self.assertEqual(1,len(c._Connection__pool.sockets)) c.test.test.find_one() self.assertEqual(1,len(c._Connection__pool.sockets)) pipe.send(one(c._Connection__pool.sockets).sock.getsockname()) cp1, cc1 = Pipe() cp2, cc2 = Pipe() p1 = Process(target=loop, args=(cc1,)) p2 = Process(target=loop, args=(cc2,)) p1.start() p2.start() p1.join(1) p2.join(1) p1.terminate() p2.terminate() p1.join() p2.join() cc1.close() cc2.close() b_sock = cp1.recv() c_sock = cp2.recv() self.assertTrue(a_sock.sock.getsockname() != b_sock) self.assertTrue(a_sock.sock.getsockname() != c_sock) self.assertTrue(b_sock != c_sock) self.assertEqual(a_sock, a._Connection__pool.get_socket((a.host, a.port)))
def test_pool_with_fork(self): if sys.platform == "win32": raise SkipTest() try: from multiprocessing import Process, Pipe except ImportError: raise SkipTest() a = get_connection() a.test.test.find_one() a.end_request() self.assertEqual(1, len(a._Connection__pool.sockets)) a_sock = a._Connection__pool.sockets[0] def loop(pipe): c = get_connection() self.assertEqual(1, len(c._Connection__pool.sockets)) c.test.test.find_one() self.assertEqual(0, len(c._Connection__pool.sockets)) c.end_request() self.assertEqual(1, len(c._Connection__pool.sockets)) pipe.send(c._Connection__pool.sockets[0].getsockname()) cp1, cc1 = Pipe() cp2, cc2 = Pipe() p1 = Process(target=loop, args=(cc1,)) p2 = Process(target=loop, args=(cc2,)) p1.start() p2.start() p1.join(1) p2.join(1) p1.terminate() p2.terminate() p1.join() p2.join() cc1.close() cc2.close() b_sock = cp1.recv() c_sock = cp2.recv() self.assert_(a_sock.getsockname() != b_sock) self.assert_(a_sock.getsockname() != c_sock) self.assert_(b_sock != c_sock) self.assertEqual(a_sock, a._Connection__pool.get_socket(a.host, a.port)[0])
def callThisFunction(): ## Create a multiprocessing pip connection ## It has a parent and child connection object ## Pass the child object to receive messages back ## parent_connection, child_connection = Pipe() ## Create the Process ## p = Process(target=printMe, args=(child_connection,'bob',)) p.start() print 'The parent knows the childs PID:' print parent_connection.recv() #prints the PID of the process p.join()
def test_sending(self): server_conn, worker_conn = Pipe() a, b = Pipe() process = Process(target=echo, args=(worker_conn,)) process.start() server_conn.send(SharedFile('test.txt', b)) self.assertEqual(('open', {'mode': 'r', 'filename': 'test.txt'}), a.recv()) a.send('a') self.assertEqual('a', a.recv()) process.terminate()
def get_ff(user): assert (type(user) == str) parent_friends_pi, friends_pi = Pipe() parent_followers_pi, followers_pi = Pipe() friend_pr = Process(target = get_friends_parallel, args=(friends_pi, user)) follower_pr = Process(target = get_followers_parallel, args=(followers_pi, user)) #Get friends and followers. friend_pr.start() follower_pr.start() friends = parent_friends_pi.recv() followers = parent_followers_pi.recv() friend_pr.join() follower_pr.join() return friends, followers
def main(): server_channel, client_channel = Pipe() my_process = MyProcess(client_channel) my_process.start() server_channel.send(('add', 4)) print(server_channel.recv()) server_channel.send(('add', 12)) print(server_channel.recv()) server_channel.send(('method', None)) print(("compiled" in server_channel.recv()) == ("compiled" in repr(MyProcess.run))) server_channel.send(('close', 0))
def fn_with_timeout(*args, **kwargs): conn1, conn2 = Pipe() kwargs['_conn'] = conn2 th = Process(target=fn, args=args, kwargs=kwargs) th.start() if conn1.poll(self.trial_timeout): fn_rval = conn1.recv() th.join() else: print 'TERMINATING DUE TO TIMEOUT' th.terminate() th.join() fn_rval = 'return', { 'status': hyperopt.STATUS_FAIL, 'failure': 'TimeOut' } assert fn_rval[0] in ('raise', 'return') if fn_rval[0] == 'raise': raise fn_rval[1] # -- remove potentially large objects from the rval # so that the Trials() object below stays small # We can recompute them if necessary, and it's usually # not necessary at all. if fn_rval[1]['status'] == hyperopt.STATUS_OK: fn_loss = float(fn_rval[1].get('loss')) fn_preprocs = fn_rval[1].pop('preprocs') fn_classif = fn_rval[1].pop('classifier') if fn_loss < self._best_loss: self._best_preprocs = fn_preprocs self._best_classif = fn_classif self._best_loss = fn_loss return fn_rval[1]
def run(self): logging.info('Visualizer thread started') parent_end, child_end = Pipe() # Sensible default value for max_process max_process = 3 process_count = 0 while not self.stop or not self.job_backlog.empty(): while parent_end.poll(0.1): (name, counter) = parent_end.recv() self.controller.find_trial(name).set_counter_plot(counter) process_count -= 1 if self.job_backlog.empty(): time.sleep(1) elif process_count < max_process: process_count += 1 function, snapshot, trial = self.job_backlog.get_nowait() logging.info('Visualizing {}'.format(trial.get_name())) p = Process(target=self.render_graph, args=(function, snapshot, trial.get_name(), child_end)) p.start() self.job_backlog.task_done() logging.info('Visualizer Finished')
def run(self): parent_pipe, child_pipe = Pipe(False) p = Process(target = run_metaheuristic, args = (child_pipe, self.model, self.pt, self.aa, self.algo, self.n, self.use_heur, self.worst, self.best)) p.start() for i in range(self.n + 1): if self.is_stopped() is True: parent_pipe.close() p.terminate() return try: result = parent_pipe.recv() except: break self.results.append(result[0]) self.fitness.append(result[1]) self.emit(QtCore.SIGNAL('update(int)'), i) if result[1] == 1: break parent_pipe.close() p.join()
def test_mount_fstab_user_fail(self): if not self._can_create: self.skipTest('Cannot create %s filesystem' % self._fs_name) if not self._can_mount: self.skipTest('Cannot mount %s filesystem' % self._fs_name) # this test will change /etc/fstab, we might want to revert the changes after it finishes fstab = self.read_file('/etc/fstab') self.addCleanup(self.write_file, '/etc/fstab', fstab) disk = self.get_object('/block_devices/' + os.path.basename(self.vdevs[0])) self.assertIsNotNone(disk) # create filesystem disk.Format(self._fs_name, self.no_options, dbus_interface=self.iface_prefix + '.Block') self.addCleanup(self._clean_format, disk) # create user for our test self.addCleanup(self._remove_user) uid, gid = self._add_user() # add unmount cleanup now in case something wrong happens in the other process self.addCleanup(self._unmount, self.vdevs[0]) # create pipe to get error (if any) parent_conn, child_conn = Pipe() proc = Process(target=self._mount_as_user_fstab_fail, args=(child_conn, int(uid), int(gid), self.vdevs[0])) proc.start() res = parent_conn.recv() parent_conn.close() proc.join() if not res[0]: self.fail(res[1]) # now mount it as root and test that user can't unmount it mnt_path = disk.Mount(self.no_options, dbus_interface=self.iface_prefix + '.Filesystem') self.assertIsNotNone(mnt_path) self.assertTrue(os.path.ismount(mnt_path)) # create pipe to get error (if any) parent_conn, child_conn = Pipe() proc = Process(target=self._unmount_as_user_fstab_fail, args=(child_conn, int(uid), int(gid), self.vdevs[0])) proc.start() res = parent_conn.recv() parent_conn.close() proc.join() if not res[0]: self.fail(res[1]) self.assertTrue(os.path.ismount(mnt_path)) self._unmount(mnt_path)
class Submarine: def __init__(self, SaveSensorData): # Create variables to hold the current position self._roll = 0 self._pitch = 0 self._yaw = 0 self._YPos = 0 self._angleFirstRead = True # Create variables to hold depth self._depth = 0 # Create variables to control the message passing between processes self.IMUParnetConn, self.IMUChildConn = Pipe() self.depthParnetConn, self.depthChildConn = Pipe() # Make the message board self._messageBoard = MessageBoard() # Create the object self._SubMotors = SubMotors() # Create the IMU object self.IMU = SubIMU(self._messageBoard._IMUFile, self.IMUChildConn, SaveSensorData) # Check if we were able to connect to the IMU if (self.IMU == None): # Don't bother updating the angles we don't have an IMU self._upateAngles = False else: self._upateAngles = True # Create the depth sensor object self.depthSensor = SubDepthSensor(self._messageBoard._DepthFile, self.depthChildConn, SaveSensorData) # Check if we were able to connect to the depth sensor if (self.depthSensor == None): # Don't bother updating the depth we don't have a sensor self._upateDepth = False else: self._upateDepth = True # Create the Maintain Forward object self._maintainForward = MaintainForward() # Create the Maintain Depth object self._maintainDepth = MaintainDepth() # Create the dive object self._goToDepth = GoToDepth() # Create a clockwise turn object self._clockWiseTurn = ClockWiseTurn() # Create a counterclockwise turn object self._counterClockWiseTurn = CounterClockWiseTurn() # Create the timing variables self._rateDiff = 25 # Create a method to leave self._exit = False # This method checks the IMU connection and update values def UpdateAngles(self): # Check if the IMU has posted a message if (self.IMUParnetConn.poll() or self._angleFirstRead): message = self.IMUParnetConn.recv() self._angleFirstRead = False if (str(message).find("[Log]") != -1): self._messageBoard.LogMessage(message) else: self._roll = message[0] self._pitch = message[1] self._yaw = message[2] self._YPos = message[3] # This method checks the IMU connection and update values def UpdateDepth(self): # Check if the Depth Sensor has posted a message if (self.depthParnetConn.poll()): message = self.depthParnetConn.recv() if (str(message).find("[Log]") != -1): self._messageBoard.LogMessage(message) else: self._depth = message def UpdateMotorSpeed(self, Packet): if (Packet == None or len(Packet) < 5): return if (Packet[0] != None): # if( abs(self._SubMotors._leftSpeed - Packet[0]) > SmallChange): self._leftSpeed = Packet[0] self._SubMotors.SetLeftSpeed(Packet[0], Packet[1]) if (Packet[2] != None): # if( abs(self._SubMotors._rightSpeed - Packet[2]) > SmallChange): self._rightSpeed = Packet[2] self._SubMotors.SetRightSpeed(Packet[2], Packet[3]) if (Packet[4] != None): # if( abs(self._SubMotors._servoAngle - Packet[4]) > SmallChange): self._servoAngle = Packet[4] self._SubMotors.SetServoAngle(Packet[4], Packet[5]) def UpdateSubState(self): if (self._upateAngles): self.UpdateAngles() if (self._upateDepth): self.UpdateDepth() # This will maintain a trajectory def Forward(self, length): self.UpdateSubState() counter = 0 startTime = time() # Capture the state of the submarine self._maintainForward.CaptureState( [self._roll, self._pitch, self._yaw, self._YPos], self._depth) self._maintainDepth.CaptureState( [self._roll, self._pitch, self._yaw, self._YPos], self._depth) while (time() - startTime < length): # Update the submarine state self.UpdateSubState() # Update the counter counter = counter + 1 # Send message if enough time has passed if (counter == self._rateDiff): # This will correct for an X-Y Variation self.UpdateMotorSpeed( self._maintainForward.UpdateState( [self._roll, self._pitch, self._yaw, self._YPos], self._depth)) # This will correct for an Z Variation self.UpdateMotorSpeed( self._maintainDepth.UpdateState( [self._roll, self._pitch, self._yaw, self._YPos], self._depth)) # Reset the counter counter = 0 def ChangeDepth(self, depth): self.UpdateSubState() counter = 0 startTime = time() # Capture the state of the submarine self._maintainForward.CaptureState( [self._roll, self._pitch, self._yaw, self._YPos], self._depth) self._goToDepth.CaptureState(self._depth, depth) # Set a flag to notify us when we reach our desired depth reachedDepth = False while (not reachedDepth): # Update the submarine state self.UpdateSubState() # Update the counter counter = counter + 1 # Send message if enough time has passed if (counter == self._rateDiff): # This will correct for an X-Y Variation self.UpdateMotorSpeed( self._maintainForward.UpdateState( [self._roll, self._pitch, self._yaw, self._YPos], self._depth)) # Set the angle of attack so we can dive response = self._goToDepth.UpdateState(self._depth) if (response != False): self.UpdateMotorSpeed(response) else: reachedDepth = True # Reset the counter counter = 0 def ClockWiseTurn(self, angle): self.UpdateSubState() counter = 0 StopAngle = self._yaw + angle while (abs(StopAngle - self._yaw) > 1): # Update the submarine state self.UpdateSubState() # Update the counter counter = counter + 1 # Send message if enough time has passed if (counter == self._rateDiff): if (not self._clockWiseTurn.StateCaptured): self._clockWiseTurn.CaptureState( [self._roll, self._pitch, self._yaw], angle) self.UpdateMotorSpeed( self._clockWiseTurn.UpdateState( [self._roll, self._pitch, self._yaw])) counter = 0 def CounterClockWiseTurn(self, angle): self.UpdateSubState() counter = 0 StopAngle = self._yaw - angle while (abs(StopAngle - self._yaw) > 1): # Update the submarine state self.UpdateSubState() # Update the counter counter = counter + 1 # Send message if enough time has passed if (counter == self._rateDiff): if (not self._counterClockWiseTurn.StateCaptured): self._counterClockWiseTurn.CaptureState( [self._roll, self._pitch, self._yaw], angle) self.UpdateMotorSpeed( self._counterClockWiseTurn.UpdateState( [self._roll, self._pitch, self._yaw])) counter = 0 def ShutDown(self): # Tell the IMU process to exits print("Here f**k # 2") self.IMUParnetConn.send("Close") self.depthParnetConn.send("Close") self._messageBoard.CloseBoard()
#listen mode s.listen(5) print ("socket is listening") dataObj = bytes(8) #accept conn,addr = s.accept() msg = "connected" print (msg) #For starting the stream set this to 1 vp.start() interuptProcess.start() cPid = parent_conn.recv() print("Child = {}".format(cPid)) while True: parent_conn.send(1) p_conn.send(1) dataObj = conn.recv(4096) decodeData = dataObj#.decode(encoding='utf-8', errors='strict') if (not decodeData): break data = '' print (decodeData) if(len(decodeData) > 4): data = (decodeData[3:4]) + (decodeData[4:5]) if (decodeData == b't\x00\x0211'):
class ProcessEngine(BaseEngine): """An engine executing the tasks it is sent in a different process. """ def perform(self, exec_infos): """Execute a given task. Parameters ---------- exec_infos : ExecutionInfos TaskInfos object describing the work to expected of the engine. Returns ------- exec_infos : ExecutionInfos Input object whose values have been updated. This is simply a convenience. Notes ----- IOError in pipe are raised only if an operation is attempted from the process that closed the pipe, but never when trying to poll from a different process. """ self.status = 'Running' # Clear all the flags. self._task_pause.clear() self._task_paused.clear() self._task_resumed.clear() self._task_stop.clear() self._force_stop.clear() self._stop_requested = False # If the process does not exist or is dead create a new one. if not self._process or not self._process.is_alive(): self._process_stop.clear() # Create the subprocess and the pipe. self._pipe, process_pipe = Pipe() self._process = TaskProcess(process_pipe, self._log_queue, self._monitor_queue, self._task_pause, self._task_paused, self._task_resumed, self._task_stop, self._process_stop) self._process.daemon = True # Create the logger thread in charge of dispatching log reports. self._log_thread = QueueLoggerThread(self._log_queue) self._log_thread.daemon = True logger.debug('Starting logging thread.') self._log_thread.start() # Create the monitor thread dispatching engine news to the monitor. self._monitor_thread = ThreadMeasureMonitor( self, self._monitor_queue) self._monitor_thread.daemon = True logger.debug('Starting monitoring thread.') self._monitor_thread.start() self._pause_thread = None # Start process. logger.debug('Starting subprocess') self._process.start() # Send the measure. args = self._build_subprocess_args(exec_infos) try: self._pipe.send(args) except Exception: msg = ('Failed to send infos to subprocess :\n-infos : \n%s\n' '-errors :\n%s') logger.error(msg % (pformat(args), format_exc())) self._log_queue.put(None) self._monitor_queue.put((None, None)) self._cleanup(process=True) exec_infos.success = False exec_infos.errors['engine'] = msg self.status = 'Stopped' return exec_infos else: logger.debug('Task {} sent'.format(exec_infos.id)) # Check that the engine did receive the task. while not self._pipe.poll(2): if not self._process.is_alive(): msg = 'Subprocess was found dead unexpectedly' logger.debug(msg) self._log_queue.put(None) self._monitor_queue.put((None, None)) self._cleanup(process=False) exec_infos.success = False exec_infos.errors['engine'] = msg self.status = 'Stopped' return exec_infos # Simply empty the pipe the subprocess always send True if it answers self._pipe.recv() # Wait for the process to finish the measure and check it has not # been killed. while not self._pipe.poll(1): if self._force_stop.is_set(): msg = 'Subprocess was terminated by the user.' logger.debug(msg) self._cleanup(process=False) exec_infos.errors['engine'] = msg self.status = 'Stopped' return exec_infos elif not self._process.is_alive(): msg = 'Subprocess was found dead unexpectedly' logger.debug(msg) self._log_queue.put(None) self._monitor_queue.put((None, None)) self._cleanup(process=False) exec_infos.success = False exec_infos.errors['engine'] = msg self.status = 'Stopped' return exec_infos # Here get message from process and react result, errors = self._pipe.recv() logger.debug('Subprocess done performing measure') exec_infos.success = result exec_infos.errors.update(errors) self.status = 'Waiting' return exec_infos def pause(self): """Ask the engine to pause the current task execution. """ self.status = 'Pausing' self._task_resumed.clear() self._task_paused.clear() self._task_pause.set() self._pause_thread = Thread(target=self._wait_for_pause) self._pause_thread.start() def resume(self): """Ask the engine to resume the currently paused job. """ self.status = 'Resuming' self._task_pause.clear() def stop(self, force=False): """Ask the engine to stop the current job. This method should not wait for the job to stop save if a forced stop was requested. Parameters ---------- force : bool, optional Force the engine to stop the performing the task. This allow the engine to use any means necessary to stop, in this case only should the call to this method block. """ self.status = 'Stopping' self._stop_requested = True self._task_stop.set() if force: self._force_stop.set() # Stop running queues self._log_queue.put(None) self._monitor_queue.put((None, None)) # Terminate the process and make sure all threads stopped properly. self._process.terminate() self._log_thread.join() self._monitor_thread.join() # Discard the queues as they may have been corrupted when the # process was terminated. self._log_queue = Queue() self._monitor_queue = Queue() self.status = 'Stopped' def shutdown(self, force=False): """Ask the engine to stop completely. Parameters ---------- force : bool, optional Force the engine to stop the performing the task. This allow the engine to use any means necessary to stop, in this case only should the call to this method block. """ self.status = 'Shutting down' self._stop_requested = True self._task_stop.set() if not force: t = Thread(target=self._cleanup) t.start() else: self.stop(force=True) # ========================================================================= # --- Private API --------------------------------------------------------- # ========================================================================= #: Boolean indicating that the user requested the job to stop. _stop_requested = Bool() #: Interprocess event used to pause the subprocess current job. _task_pause = Value(factory=Event) #: Interprocess event signaling the subprocess current job is paused. _task_paused = Value(factory=Event) #: Interprocess event signaling the subprocess current job has resumed. _task_resumed = Value(factory=Event) #: Interprocess event used to stop the subprocess current measure. _task_stop = Value(factory=Event) #: Interprocess event used to stop the subprocess. _process_stop = Value(factory=Event) #: Flag signaling that a forced exit has been requested _force_stop = Value(factory=tEvent) #: Current subprocess. _process = Typed(TaskProcess) #: Connection used to send and receive messages about execution (type #: ambiguous when the OS is not known) _pipe = Value() #: Inter-process queue used by the subprocess to transmit its log records. _log_queue = Value(factory=Queue) #: Thread in charge of collecting the log message coming from the #: subprocess. _log_thread = Typed(Thread) #: Inter-process queue used by the subprocess to send the values of the #: observed database entries. _monitor_queue = Value(factory=Queue) #: Thread in charge of collecting the values of the observed database #: entries. _monitor_thread = Typed(Thread) #: Thread in charge of notifying the engine that the engine did #: pause/resume after being asked to do so. _pause_thread = Typed(Thread) def _cleanup(self, process=True): """ Helper method taking care of making sure that everybody stops. Parameters ---------- process : bool Whether to join the worker process. Used when the process has been termintaed abruptly. """ logger.debug('Cleaning up') if process and self._process: self._process_stop.set() self._process.join() logger.debug('Subprocess joined') if self._pipe: self._pipe.close() if self._log_thread: self._log_thread.join() logger.debug('Log thread joined') if self._monitor_thread: self._monitor_thread.join() logger.debug('Monitor thread joined') if self._pause_thread: self._pause_thread.join() logger.debug('Pause thread joined') self.status = 'Stopped' def _build_subprocess_args(self, exec_infos): """Build the tuple to send to the subprocess. """ exec_infos.task.update_preferences_from_members() config = exec_infos.task.preferences database_root_state = exec_infos.task.database.copy_node_values() return (exec_infos.id, config, exec_infos.build_deps, exec_infos.runtime_deps, exec_infos.observed_entries, database_root_state, exec_infos.checks) def _wait_for_pause(self): """ Wait for the _task_paused event to be set. """ stop_sig = self._task_stop paused_sig = self._task_paused while not stop_sig.is_set(): if paused_sig.wait(0.1): self.status = 'Paused' break resuming_sig = self._task_resumed while not stop_sig.is_set(): if resuming_sig.wait(1): self.status = 'Running' break
def main(sub_p): global starttime # 创建一个“钩子”管理对象 hm = pyHook.HookManager() # 监听所有键盘事件 onActiveEvent = lambda sub_p=sub_p: onActiveEvent(sub_p=sub_p) hm.KeyDown = hm.MouseAll = onActiveEvent # 设置键盘“钩子” hm.HookKeyboard() # 监听所有鼠标事件 # hm.MouseAll = onMouseEvent # 设置鼠标“钩子” hm.HookMouse() # 进入循环,如不手动关闭,程序将一直处于监听状态 pythoncom.PumpMessages(10000) if __name__ == "__main__": sup_p, sub_p = Pipe() p = Process(target=main, args=(sub_p, )) p.start() while 1: if sup_p.recv(): starttime = 0 sleep(1) starttime += 1 print(starttime) if starttime > 20: ScreenSaver(10)
import multiprocessing as mp def teste(q): t = nquad(lambda x, y: x**2 + x + y**2 + y, [[-np.inf, np.inf], [-np.inf, np.inf]]) q.put(t) q = mp.Queue() p = mp.Process(target=teste, args=(q, )) p.start() print(q.get()) p.join() #%% from multiprocessing import Process, Pipe def f(conn): conn.send([42, None, 'hello']) conn.close() if __name__ == '__main__': parent_conn, child_conn = Pipe() p = Process(target=f, args=(child_conn, )) p.start() print parent_conn.recv() # prints "[42, None, 'hello']" p.join() #%% sys.exit(10)
class LabEnvironment(environment.Environment): ACTION_LIST = [ _action(-20, 0, 0, 0, 0, 0, 0), # look_left _action(20, 0, 0, 0, 0, 0, 0), # look_right #_action( 0, 10, 0, 0, 0, 0, 0), # look_up #_action( 0, -10, 0, 0, 0, 0, 0), # look_down _action(0, 0, -1, 0, 0, 0, 0), # strafe_left _action(0, 0, 1, 0, 0, 0, 0), # strafe_right _action(0, 0, 0, 1, 0, 0, 0), # forward _action(0, 0, 0, -1, 0, 0, 0), # backward #_action( 0, 0, 0, 0, 1, 0, 0), # fire #_action( 0, 0, 0, 0, 0, 1, 0), # jump #_action( 0, 0, 0, 0, 0, 0, 1) # crouch ] @staticmethod def get_action_size(env_name): return len(LabEnvironment.ACTION_LIST) def __init__(self, env_name): environment.Environment.__init__(self) self.conn, child_conn = Pipe() self.proc = Process(target=worker, args=(child_conn, env_name)) self.proc.start() self.conn.recv() self.reset() def reset(self): self.conn.send([COMMAND_RESET, 0]) obs = self.conn.recv() self.last_state = self._preprocess_frame(obs) self.last_action = 0 self.last_reward = 0 def stop(self): self.conn.send([COMMAND_TERMINATE, 0]) ret = self.conn.recv() self.conn.close() self.proc.join() print("lab environment stopped") def _preprocess_frame(self, image): image = image.astype(np.float32) image = image / 255.0 return image def process(self, action): real_action = LabEnvironment.ACTION_LIST[action] self.conn.send([COMMAND_ACTION, real_action]) obs, reward, terminal = self.conn.recv() if not terminal: state = self._preprocess_frame(obs) else: state = self.last_state pixel_change = self._calc_pixel_change(state, self.last_state) self.last_state = state self.last_action = action self.last_reward = reward return state, reward, terminal, pixel_change
def main(): parent_conn, child_conn = Pipe() proc = Process(target=f, args=(child_conn, )) proc.start() print(parent_conn.recv()) proc.join()
TXN = 0 DXN = 0 # !! end declaring global values !! #world generation with open("./output files/worldtypes") as wF: FCt = wF.read() #create save data with the mapGen script if ("a" not in FCt): #intitalizes the lists we look through for PXN's tile type worldgen() if __name__ == '__main__': reciever, sender = Pipe(False) p = Process(target=worldgen.fullgen, args=(sender,)) p.start() wat = reciever.recv() bun = reciever.recv() rou = reciever.recv() fai = reciever.recv() tow = reciever.recv() dun = reciever.recv() towN = reciever.recv() dunN = reciever.recv() else: #read from saved data with open("./output files/savedworld") as sD: sD.seek(0) strAll = sD.readline() listAll = strAll.split(";") wat = [] bun = []
def treater(pipe_in: Pipe, queue_ctrl: Queue, func: callable = None): """ :param pipe_in: Pipe # 消息管道 (MSG, ) :param queue_ctrl: Queue # 控制 ("is_active",):(bool,) :param func: callable # 自定义处理过程 :return: None """ def treat_msg(msg_orig): # --------- encryption ------------------------------------------ if encryption is None: msg_encryption = msg_orig else: msg_encryption = encryption.request(msg_orig) logging.debug("TEMPLE::TREATER::{0}::ENCRYPT request:{1}".format( msg_orig.encryption, msg_encryption)) # --------- treatment ------------------------------------------- if treatment is None: msg_treatment = msg_encryption else: msg_treatment = treatment.request(msg_encryption) logging.debug("TEMPLE::TREATER::{0}::TREAT request:{1}".format( msg_orig.treatment, msg_treatment)) del msg_encryption # --------- function -------------------------------------------- if func is None: msg_func = msg_treatment else: msg_func = func(msg_treatment) logging.debug("TEMPLE::TREATER func:{0}".format(msg_func)) del msg_treatment # --------- destination ----------------------------------------- if destination is None: logging.info( "TEMPLE::TREATER::NONE::DEST return:{0}".format(msg_func)) else: if isinstance(destination, MQ): destination.request(msg_func) logging.info("TEMPLE::TREATER::{0}::DEST request:{1}".format( msg_orig.destination, msg_func)) else: destination.write(msg_func) logging.info("TEMPLE::TREATER::{0}::DEST write:{1}".format( msg_orig.destination, msg_func)) return msg_func is_active = queue_ctrl.get() while is_active: if queue_ctrl.empty(): try: msg_treater = pipe_in.recv() logging.info( "TEMPLE::TREATER PIPE IN recv:{0}".format(msg_treater)) except EOFError: is_active = False continue origination = allocator(msg_treater.origination) # Data.read() treatment = allocator(msg_treater.treatment) # MessageQueue encryption = allocator(msg_treater.encryption) # MessageQueue destination = allocator(msg_treater.destination) # Data.write() # --------- origination ----------------------------------------- if isinstance(origination, MQ): origination.reply(func=treat_msg) else: msg_origination = origination.read(msg_treater) logging.info("TEMPLE::TREATER::{0}::ORIG read:{1}".format( msg_treater.origination, msg_origination)) treat_msg(msg_origination) else: is_active = queue_ctrl.get() else: while not queue_ctrl.empty(): queue_ctrl.get()
class TaskExecutionControl(Atom): """Store measurement in a queue of measures to be processed, take care of starting and communicating with the process performing the measure and handling user action (stopping single measure or whole process). Attributes ---------- running : bool Bool indicating whether or not the measurement process is running. task_stop : multiprocess.Event Event used to signal that the current measure should be stopped. process_stop : multiprocess.Event Event used to signal that the measurement process should be stopped. meas_holder : list(instance(Measure)) List containing all the enqueued measure. process : instance(Process) Measurement process. log_thread : instance(Thread) Thread dedicated to handling the log records coming from the measurement process log_queue : multiprocessing queue Queue in which the log records of the measurement process are sent from the measurement process to the main process. monitor_queue : multiprocessing queue Queue in which the value of the monitored parameters are sent from the measurement process to the main process. current_monitor : instance(MeasureMonitor) Monitor associated to the measurement being processed. pipe : multiprocessing double ended pipe Pipe used for communication between the two processus. Methods ------- append_task(new_task): Method handling the enqueuing of a measurement in the queue. """ running = Bool(False) task_stop = Instance(Event, ()) process_stop = Instance(Event, ()) meas_holder = ContainerList(Instance(Measure), []) process = Instance(Process) log_thread = Instance(Thread) log_queue = Instance(Queue, ()) pipe = Value( ) #Instance of Connection but ambiguous when the OS is not known monitor_queue = Instance(Queue, ()) current_monitor = Instance(MeasureMonitor) monitor_display = Instance(MonitorView, ()) def append_meas(self, new_meas): """Put a measure in the queue if it pass the tests. First the check method of the measure is called. If the tests pass, the user is asked to give a name to its measure (by default the name is 'Meas"i"' where is the index of the measure in the queue), then he can choose what entries he wants to monitor during the measure which is then enqueued and finally saved in the default folder ('default_path' attributes of the `RootTask` describing the measure). Otherwise the list of the failed tests is displayed to the user. Parameters ---------- new_task : instance(`RootTask`) Instance of `RootTask` representing the measure. Returns ------- bool : True is the measure was successfully enqueued, False otherwise. """ check = new_meas.root_task.check(test_instr=not self.running) if check[0]: path = os.path.join( new_meas.root_task.default_path, new_meas.monitor.measure_name + '_last_run.ini') new_meas.save_measure(path) meas = Measure() meas.load_measure(path) self.meas_holder.append(meas) return True else: TaskCheckDisplay(model=TaskCheckModel(check[1])).exec_() return False def _start_button_clicked(self): """Handle the `start_button` being pressed. Clear the event `task_stop` and `process_stop`, create the pipe and the measurement process. Start then the log thread and then the process. Finally start the thread handling the communication with the measurement process. """ print 'Starting process' self.task_stop.clear() self.process_stop.clear() self.pipe, process_pipe = Pipe() self.process = TaskProcess(process_pipe, self.log_queue, self.monitor_queue, self.task_stop, self.process_stop) self.log_thread = QueueLoggerThread(self.log_queue) self.log_thread.daemon = True self.log_thread.start() self.process.start() self.running = True Thread(group=None, target=self._process_listerner).start() def _stop_all_button_clicked(self): """Handle the `stop_button` being pressed. Set the `process_stop` event and signal in the pipe that not more measurement will be sent. Then wait for the process and the log thread to terminate. """ print 'Stopping process' self.process_stop.set() self.pipe.send((None, 'STOP', None)) self.task_stop.set() self.process.join() self.log_thread.join() self.running = False def _stop_button_clicked(self): """Handle the `stop_task_button` being pressed by setting the `task_stop` event. """ print 'Stopping task' self.task_stop.set() def _update_monitor_display_model(self, monitor): """ """ self.monitor_display.monitor = monitor if not self.monitor_display.visible: self.monitor_display.show() def _process_listerner(self): """Method called in a separated thread to handle communications with the measurement process. """ print 'Starting listener' meas = None while not self.process_stop.is_set(): self.pipe.poll(None) mess = self.pipe.recv() print 'Message received : {}'.format(mess) if mess == 'Need task': if self.meas_holder: i = 0 task = None # Look for a measure not being currently edited. while i < len(self.meas_holder): aux = self.meas_holder[i] if aux.monitor.status == 'EDITING' or aux == meas: i += 1 continue else: meas = self.meas_holder[i] task = meas.root_task monitor = meas.monitor name = meas.monitor.measure_name break # If one is found, stop the old monitor, if necessary start # a new one and send the measure in the pipe. if task is not None: meas.is_running = True task.update_preferences_from_members() if self.current_monitor: self.current_monitor.stop() self.current_monitor = None self.current_monitor = monitor deferred_call(setattr, monitor, 'status', 'RUNNING') # Leave a chance to the system to update the display sleep(0.1) if meas.use_monitor: monitor.start(self.monitor_queue) deferred_call(self._update_monitor_display_model, monitor) self.pipe.send( (name, task.task_preferences, self.current_monitor.database_values.keys())) else: deferred_call(self.monitor_display.close) self.pipe.send((name, task.task_preferences, None)) print 'Measurement sent' # If there is no measurement which can be sent, stop the # measurement process. else: self.process_stop.set() print 'The only task is the queue is being edited' self.pipe.send(('', 'STOP', '')) self.pipe.poll(None) self.pipe.close() self.process.join() self.log_thread.join() self.running = False break # If there is no measurement in the queue, stop the # measurement process. else: self.process_stop.set() print 'All measurements have been sent' self.pipe.send((None, 'STOP', None)) self.pipe.poll(None) self.pipe.close() self.process.join() self.log_thread.join() self.running = False break elif mess == 'Task processed': deferred_call(self.meas_holder.remove, meas) sleep(0.1) # If the measurement process sent a different message, # it means it will stop so we can clean up. else: self.pipe.close() self.process.join() self.log_thread.join() self.running = False break # Upon exit close the monitor if it is still opened. if self.current_monitor: deferred_call(setattr, self.current_monitor, 'status', 'STOPPED') self.current_monitor.stop()
#!/usr/bin/env python # -*- coding: utf-8 -*- from multiprocessing import Process, Pipe def f(conn): conn.send([42, None, 'hello']) conn.close() if __name__ == '__main__': parent_conn, child_conn = Pipe() p = Process(target=f, args=(child_conn, )) # send前にrecvを実行すると待機状態になるのでプロセスが停止する #print parent_conn.recv() p.start() print parent_conn.recv() p.join()
class UI: LOAD_TAB = 0 ALIGN_TAB = 1 STACK_TAB = 2 SHARPEN_TAB = 3 TITLE = "AstraStack" VERSION = "2.0.0" def __init__(self): self.parentConn, self.childConn = Pipe(duplex=True) self.pids = [] self.newVersionUrl = "" self.video = None self.align = None self.stack = None self.sharpen = None self.mousePosition = None self.clickedDriftP1 = False self.clickedDriftP2 = False self.clickedAreaOfInterest = False self.builder = Gtk.Builder() self.builder.add_from_file("ui/ui.glade") self.window = self.builder.get_object("mainWindow") self.saveDialog = self.builder.get_object("saveDialog") self.openDialog = self.builder.get_object("openDialog") self.tabs = self.builder.get_object("tabs") self.cpus = self.builder.get_object("cpus") self.progressBox = self.builder.get_object("progressBox") self.progress = self.builder.get_object("progress") self.frame = self.builder.get_object("frame") self.overlay = self.builder.get_object("overlay") self.frameSlider = self.builder.get_object("frameSlider") self.frameScale = self.builder.get_object("frameScale") self.startFrame = self.builder.get_object("startFrame") self.endFrame = self.builder.get_object("endFrame") self.normalize = self.builder.get_object("normalize") self.alignChannels = self.builder.get_object("alignChannels") self.autoCrop = self.builder.get_object("autoCrop") self.transformation = self.builder.get_object("transformation") self.drizzleFactor = self.builder.get_object("drizzleFactor") self.drizzleInterpolation = self.builder.get_object( "drizzleInterpolation") self.limit = self.builder.get_object("limit") self.limitPercent = self.builder.get_object("limitPercent") self.averageRadio = self.builder.get_object("averageRadio") self.medianRadio = self.builder.get_object("medianRadio") self.openDialog.set_preview_widget(Gtk.Image()) self.saveDialog.set_preview_widget(Gtk.Image()) self.builder.get_object("alignTab").set_sensitive(False) self.builder.get_object("stackTab").set_sensitive(False) self.builder.get_object("processTab").set_sensitive(False) self.builder.get_object("blackLevel").add_mark(0, Gtk.PositionType.TOP, None) self.builder.get_object("gamma").add_mark(100, Gtk.PositionType.TOP, None) self.builder.get_object("value").add_mark(100, Gtk.PositionType.TOP, None) self.builder.get_object("red").add_mark(100, Gtk.PositionType.TOP, None) self.builder.get_object("green").add_mark(100, Gtk.PositionType.TOP, None) self.builder.get_object("blue").add_mark(100, Gtk.PositionType.TOP, None) self.builder.get_object("saturation").add_mark(100, Gtk.PositionType.TOP, None) self.disableScroll() self.cpus.set_upper(min( 61, cpu_count())) # 61 is the maximum that Windows allows self.cpus.set_value(min(61, math.ceil(cpu_count() / 2))) g.pool = None self.processThread = None self.builder.connect_signals(self) # Needed so it can be temporarily removed self.limitPercentSignal = self.limitPercent.connect( "value-changed", self.setLimitPercent) g.driftP1 = (0, 0) g.driftP2 = (0, 0) g.areaOfInterestP1 = (0, 0) g.areaOfInterestP2 = (0, 0) self.window.show_all() self.checkNewVersion() self.setProgress() self.setNormalize() self.setAlignChannels() self.setTransformation() self.setDrizzleFactor() self.setDrizzleInterpolation() self.setAutoCrop() self.setThreads() self.frameScale.set_sensitive(False) g.reference = "0" # Cancels scroll event for widget def propagateScroll(self, widget, event): Gtk.propagate_event(widget.get_parent(), event) # Disables the scroll event from some fields def disableScroll(self): mask = Gdk.EventMask.BUTTON_MOTION_MASK | Gdk.EventMask.BUTTON_PRESS_MASK | Gdk.EventMask.BUTTON_RELEASE_MASK | Gdk.EventMask.KEY_RELEASE_MASK | Gdk.EventMask.TOUCH_MASK self.builder.get_object("denoiseWidget1").set_events(mask) self.builder.get_object("denoiseWidget2").set_events(mask) self.builder.get_object("denoiseWidget3").set_events(mask) self.builder.get_object("denoiseWidget4").set_events(mask) self.builder.get_object("denoiseWidget5").set_events(mask) self.builder.get_object("radiusWidget1").set_events(mask) self.builder.get_object("radiusWidget2").set_events(mask) self.builder.get_object("radiusWidget3").set_events(mask) self.builder.get_object("radiusWidget4").set_events(mask) self.builder.get_object("radiusWidget5").set_events(mask) # Sets up a listener so that processes can communicate with each other def createListener(self, function): def listener(function): while True: try: msg = self.parentConn.recv() except: return False if (msg == "stop"): g.ui.setProgress() return False function(msg) thread = Thread(target=listener, args=(function, )) thread.start() return thread # Shows the error dialog with the given title and message def showErrorDialog(self, message): dialog = self.builder.get_object("errorDialog") dialog.format_secondary_text(message) response = dialog.run() dialog.hide() return response # Shows the warning dialog with the given title and message def showWarningDialog(self, message): dialog = self.builder.get_object("warningDialog") dialog.format_secondary_text(message) response = dialog.run() dialog.hide() return response # Opens the About dialog def showAbout(self, *args): dialog = self.builder.get_object("about") dialog.set_program_name(UI.TITLE) dialog.set_version(UI.VERSION) response = dialog.run() dialog.hide() # Opens the user manual in the default pdf application def showManual(self, *args): if sys.platform.startswith('darwin'): subprocess.call(('open', "manual/Manual.pdf")) elif os.name == 'nt': # For Windows os.startfile("manual\Manual.pdf") elif os.name == 'posix': # For Linux, Mac, etc. subprocess.call(('xdg-open', "manual/Manual.pdf")) # Disable inputs def disableUI(self): self.builder.get_object("tabs").set_sensitive(False) self.builder.get_object("toolbar").set_sensitive(False) # Enable inputs def enableUI(self): self.builder.get_object("tabs").set_sensitive(True) self.builder.get_object("toolbar").set_sensitive(True) # The following is needed to forcibly refresh the value spacing of the slider def fixFrameSliderBug(self): self.frameScale.set_value_pos(Gtk.PositionType.RIGHT) self.frameScale.set_value_pos(Gtk.PositionType.LEFT) # Sets the number of threads to use def setThreads(self, *args): def initPool(method=None): if (method == "spawn"): GLib.idle_add(self.disableUI) g.nThreads = int(self.cpus.get_value()) if (g.pool is not None): g.pool.shutdown() g.pool = ProcessPoolExecutor(g.nThreads) # This seems like the most reliable way to get the pid of pool processes self.pids = [] before = list(map(lambda p: p.pid, active_children())) g.pool.submit(dummy, ()).result() after = list(map(lambda p: p.pid, active_children())) for pid in after: if (pid not in before): self.pids.append(pid) if (method == "spawn"): GLib.idle_add(self.enableUI) # Behave a bit differently depending on platform if (get_start_method() == "spawn"): thread = Thread(target=initPool, args=(get_start_method(), )) thread.start() else: initPool() # Checks github to see if there is a new version available def checkNewVersion(self): def callUrl(): try: contents = urllib.request.urlopen( "https://api.github.com/repos/Finalfantasykid/AstraStack/releases/latest" ).read() obj = json.loads(contents) if (version.parse(obj['name']) > version.parse(UI.VERSION)): self.newVersionUrl = obj['html_url'] button.show() except: return button = self.builder.get_object("newVersion") button.hide() thread = Thread(target=callUrl, args=()) thread.start() # Opens the GitHub releases page in a browser def clickNewVersion(self, *args): webbrowser.open(self.newVersionUrl) # Checks to see if there will be enough memory to process the image def checkMemory(self, w=0, h=0): if (Sharpen.estimateMemoryUsage(w, h) > psutil.virtual_memory().available): response = self.showWarningDialog( "Your system may not have enough memory to process this file, are you sure you want to continue?" ) return (response == Gtk.ResponseType.YES) return True # Shows preview image in file chooser dialog def updatePreview(self, dialog): path = dialog.get_preview_filename() pixbuf = None try: # First try as image pixbuf = GdkPixbuf.Pixbuf.new_from_file(path) except Exception: try: # Now try as video if ("video/" in mimetypes.guess_type(path)[0]): video = Video() img = cv2.cvtColor(video.getFrame(path, 0), cv2.COLOR_BGR2RGB) height, width = img.shape[:2] z = img.tobytes() Z = GLib.Bytes.new(z) pixbuf = GdkPixbuf.Pixbuf.new_from_bytes( Z, GdkPixbuf.Colorspace.RGB, False, 8, width, height, width * 3) except Exception: dialog.set_preview_widget_active(False) if (pixbuf is not None): # Valid pixbuf maxwidth, maxheight = 250, 250 width, height = pixbuf.get_width(), pixbuf.get_height() scale = min(maxwidth / width, maxheight / height) if (scale < 1): width, height = int(width * scale), int(height * scale) pixbuf = pixbuf.scale_simple(width, height, GdkPixbuf.InterpType.BILINEAR) dialog.get_preview_widget().set_size_request( width + 10, height + 10) dialog.get_preview_widget().set_from_pixbuf(pixbuf) dialog.set_preview_widget_active(True) # Opens the file chooser to open load a file def openVideo(self, *args): self.openDialog.set_current_folder(path.expanduser("~")) self.openDialog.set_select_multiple(False) self.openDialog.set_filter(self.builder.get_object("videoFilter")) response = self.openDialog.run() self.openDialog.hide() if (response == Gtk.ResponseType.OK): try: g.file = self.openDialog.get_filename() self.video = Video() self.video.checkMemory() thread = Thread(target=self.video.run, args=()) thread.start() self.disableUI() except MemoryError as error: self.enableUI() # Opens the file chooser to open load a file def openImageSequence(self, *args): self.openDialog.set_current_folder(path.expanduser("~")) self.openDialog.set_select_multiple(True) self.openDialog.set_filter(self.builder.get_object("imageFilter")) response = self.openDialog.run() self.openDialog.hide() if (response == Gtk.ResponseType.OK): try: g.file = self.openDialog.get_filenames() self.video = Video() self.video.checkMemory() thread = Thread(target=self.video.run, args=()) thread.start() self.disableUI() except MemoryError as error: self.enableUI() # Opens the file chooser to open load a file def openImage(self, *args): self.openDialog.set_current_folder(path.expanduser("~")) self.openDialog.set_select_multiple(False) self.openDialog.set_filter(self.builder.get_object("imageFilter")) response = self.openDialog.run() self.openDialog.hide() if (response == Gtk.ResponseType.OK): self.disableUI() g.file = self.openDialog.get_filename() try: self.video = Video() img = cv2.imread(g.file) h, w = img.shape[:2] if (not self.checkMemory(w, h)): raise MemoryError() self.window.set_title(path.split(g.file)[1] + " - " + UI.TITLE) self.saveDialog.set_current_name("") self.sharpen = Sharpen(g.file, True) self.builder.get_object("alignTab").set_sensitive(False) self.builder.get_object("stackTab").set_sensitive(False) self.builder.get_object("processTab").set_sensitive(True) self.tabs.set_current_page(UI.SHARPEN_TAB) self.frame.set_from_file(g.file) except MemoryError as error: pass except: # Open Failed self.showErrorDialog( "There was an error opening the image, make sure it is a valid image." ) self.enableUI() # Opens the file chooser to save the final image def saveFileDialog(self, *args): self.saveDialog.set_current_folder(path.expanduser("~")) if (self.saveDialog.get_current_name() == ""): # Set default file to save if empty if (isinstance(g.file, list)): sList = g.file self.saveDialog.set_current_name( Path(sList[0]).stem + "_" + Path(sList[-1]).stem + ".png") else: self.saveDialog.set_current_name(Path(g.file).stem + ".png") response = self.saveDialog.run() if (response == Gtk.ResponseType.OK): fileName = self.saveDialog.get_filename() try: cv2.imwrite( fileName, cv2.cvtColor(self.sharpen.finalImage.astype('uint8'), cv2.COLOR_RGB2BGR)) except: # Save Failed self.showErrorDialog( "There was an error saving the image, make sure it is a valid file extension." ) self.saveDialog.hide() # Called when the video is finished loading def finishedVideo(self): def update(): self.tabs.next_page() self.frameScale.set_sensitive(True) self.startFrame.set_lower(0) self.startFrame.set_upper(len(self.video.frames) - 1) self.startFrame.set_value(0) self.endFrame.set_upper(len(self.video.frames) - 1) self.endFrame.set_lower(0) self.endFrame.set_value(len(self.video.frames) - 1) g.driftP1 = (0, 0) g.driftP2 = (0, 0) g.areaOfInterestP1 = (0, 0) g.areaOfInterestP2 = (0, 0) g.reference = self.video.sharpest self.frameSlider.set_value(self.video.sharpest) self.setReference() self.setStartFrame() self.setEndFrame() self.setDriftPoint() self.enableUI() if (isinstance(g.file, list)): sList = g.file self.window.set_title( path.split(sList[0])[1] + " ... " + path.split(sList[-1])[1] + " - " + UI.TITLE) else: self.window.set_title(path.split(g.file)[1] + " - " + UI.TITLE) self.saveDialog.set_current_name("") self.builder.get_object("alignTab").set_sensitive(True) self.builder.get_object("stackTab").set_sensitive(False) self.builder.get_object("processTab").set_sensitive(False) self.stack = None GLib.idle_add(update) # Called when the tab is changed. Updates parts of the UI based on the tab def changeTab(self, notebook, page, page_num, user_data=None): if (page_num == UI.LOAD_TAB or page_num == UI.ALIGN_TAB): self.frameSlider.set_value(0) self.frameSlider.set_upper(len(self.video.frames) - 1) self.setStartFrame() self.setEndFrame() self.frameScale.show() self.updateImage(None, page_num) elif (page_num == UI.STACK_TAB): self.frameSlider.set_lower(0) self.frameSlider.set_upper(len(self.align.tmats) - 1) self.frameSlider.set_value(0) self.frameScale.show() self.updateImage(None, page_num) elif (page_num == UI.SHARPEN_TAB): self.frameScale.hide() self.sharpenImage() self.fixFrameSliderBug() # Changes the image frame to the frameSlider position def updateImage(self, adjustment=None, page_num=None): if (self.video is None): return if (page_num is None): page_num = self.tabs.get_current_page() if (page_num == UI.LOAD_TAB or page_num == UI.ALIGN_TAB): videoIndex = int(self.frameSlider.get_value()) img = cv2.cvtColor( self.video.getFrame(g.file, self.video.frames[videoIndex]), cv2.COLOR_BGR2RGB) height, width = img.shape[:2] z = img.tobytes() Z = GLib.Bytes.new(z) pixbuf = GdkPixbuf.Pixbuf.new_from_bytes(Z, GdkPixbuf.Colorspace.RGB, False, 8, width, height, width * 3) self.frame.set_from_pixbuf(pixbuf) elif (page_num == UI.STACK_TAB): tmat = self.stack.tmats[int(self.frameSlider.get_value())] videoIndex = tmat[0] M = tmat[1] img = self.video.getFrame(g.file, videoIndex) if (g.autoCrop): ref = self.stack.refBG.astype(np.uint8) else: ref = None img = transform(img, ref, M, self.align.minX, self.align.maxX, self.align.minY, self.align.maxY, g.drizzleFactor, g.drizzleInterpolation) img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) height, width = img.shape[:2] z = img.tobytes() Z = GLib.Bytes.new(z) pixbuf = GdkPixbuf.Pixbuf.new_from_bytes(Z, GdkPixbuf.Colorspace.RGB, False, 8, width, height, width * 3) self.frame.set_from_pixbuf(pixbuf) # Draws a rectangle where the area of interest is def drawOverlay(self, widget, cr): width = widget.get_allocated_width() height = widget.get_allocated_height() def drawPoint(cr, x, y): cr.new_sub_path() cr.set_line_width(2) cr.set_source_rgb(1, 1, 1) cr.arc(x, y, 2, 0, 2 * math.pi) cr.stroke_preserve() cr.set_source_rgb(1, 0, 0) cr.fill() def drawRect(cr, x1, y1, x2, y2): cr.rectangle(0, 0, x1, y1) cr.rectangle(0, y1, x1, (y2 - y1)) cr.rectangle(0, y1, x1, height * 2) cr.rectangle(x1, y2, (x2 - x1), height * 2) cr.rectangle(x2, y2, width * 2, height * 2) cr.rectangle(x2, y1, width * 2, (y2 - y1)) cr.rectangle(x2, 0, width * 2, y1) cr.rectangle(x1, 0, (x2 - x1), y1) cr.set_source_rgba(0, 0, 0, 0.25) cr.fill() cr.set_line_width(1) cr.set_source_rgb(1, 0, 0) cr.rectangle(x1, y1, (x2 - x1), (y2 - y1)) cr.stroke() if (self.tabs.get_current_page() == UI.ALIGN_TAB): current = self.frameSlider.get_value() # Area of Interest px1 = min(g.areaOfInterestP1[0], g.areaOfInterestP2[0]) py1 = min(g.areaOfInterestP1[1], g.areaOfInterestP2[1]) px2 = max(g.areaOfInterestP1[0], g.areaOfInterestP2[0]) py2 = max(g.areaOfInterestP1[1], g.areaOfInterestP2[1]) # Drift Points dx1 = g.driftP1[0] dy1 = g.driftP1[1] dx2 = g.driftP2[0] dy2 = g.driftP2[1] dx = 0 dy = 0 if (dx1 != 0 and dy1 != 0 and dx2 != 0 and dy2 != 0): dx = dx2 - dx1 dy = dy2 - dy1 if (px1 != 0 and py1 != 0 and px2 != 0 and py2 != 0): # Draw Area of Interest Rectangle drawRect( cr, px1 + (dx / (g.endFrame - g.startFrame)) * (current - g.startFrame), py1 + (dy / (g.endFrame - g.startFrame)) * (current - g.startFrame), px2 + (dx / (g.endFrame - g.startFrame)) * (current - g.startFrame), py2 + (dy / (g.endFrame - g.startFrame)) * (current - g.startFrame)) if (dx1 != 0 and dy1 != 0 and current == g.startFrame): # Draw point on first frame drawPoint(cr, dx1, dy1) if (dx1 != 0 and dy1 != 0 and current != g.startFrame and dx2 != 0 and dy2 != 0 and current != g.endFrame): # Draw interpolated point drawPoint( cr, dx1 + (dx / (g.endFrame - g.startFrame)) * (current - g.startFrame), dy1 + (dy / (g.endFrame - g.startFrame)) * (current - g.startFrame)) if (dx2 != 0 and dy2 != 0 and current == g.endFrame): # Draw point on last frame drawPoint(cr, dx2, dy2) # Sets the reference frame to the current visible frame def setReference(self, *args): g.reference = str(int(self.frameSlider.get_value())) self.builder.get_object("referenceLabel").set_text(g.reference) self.builder.get_object("alignButton").set_sensitive(True) # Updates the progress bar def setProgress(self, i=0, total=0, text=""): def update(): if (total == 0): self.progressBox.hide() else: self.progressBox.show() self.progress.set_fraction(i / total) self.progress.set_text(text + " " + str(round((i / total) * 100)) + "%") GLib.idle_add(update) # Sets the start frame for trimming def setStartFrame(self, *args): g.startFrame = int(self.startFrame.get_value()) self.endFrame.set_lower(g.startFrame + 1) self.frameSlider.set_lower(g.startFrame) self.frameSlider.set_value( max(g.startFrame, self.frameSlider.get_value())) if (int(g.startFrame) > int(g.reference)): # Reference is outside of the range, fix it g.reference = str(int(g.startFrame)) self.builder.get_object("referenceLabel").set_text(g.reference) self.fixFrameSliderBug() # Sets the end frame for trimming def setEndFrame(self, *args): g.endFrame = int(self.endFrame.get_value()) self.startFrame.set_upper(g.endFrame - 1) self.frameSlider.set_upper(g.endFrame) self.frameSlider.set_value( min(g.endFrame, self.frameSlider.get_value())) if (int(g.endFrame) < int(g.reference)): # Reference is outside of the range, fix it g.reference = str(int(g.endFrame)) self.builder.get_object("referenceLabel").set_text(g.reference) self.fixFrameSliderBug() # Drift Point 1 Button Clicked def clickDriftP1(self, *args): self.clickedDriftP1 = False self.clickedDriftP2 = False self.clickedAreaOfInterest = False self.setDriftPoint() self.frameSlider.set_value(g.startFrame) self.clickedDriftP1 = True self.window.get_window().set_cursor( Gdk.Cursor(Gdk.CursorType.CROSSHAIR)) # Drift Point 2 Button Clicked def clickDriftP2(self, *args): self.clickedDriftP1 = False self.clickedDriftP2 = False self.clickedAreaOfInterest = False self.setDriftPoint() self.frameSlider.set_value(g.endFrame) self.clickedDriftP2 = True self.window.get_window().set_cursor( Gdk.Cursor(Gdk.CursorType.CROSSHAIR)) # Reset Drift Point 1 to (0, 0) def resetDriftP1(self, widget, event): if (event.button == 3): # Right Click g.driftP1 = (0, 0) self.clickedDriftP1 = False self.clickedDriftP2 = False self.clickedAreaOfInterest = False self.window.get_window().set_cursor( Gdk.Cursor(Gdk.CursorType.LEFT_PTR)) self.overlay.queue_draw() # Reset Drift Point 2 to (0, 0) def resetDriftP2(self, widget, event): if (event.button == 3): # Right Click g.driftP2 = (0, 0) self.clickedDriftP1 = False self.clickedDriftP2 = False self.clickedAreaOfInterest = False self.window.get_window().set_cursor( Gdk.Cursor(Gdk.CursorType.LEFT_PTR)) self.overlay.queue_draw() # Updates the drift point def setDriftPoint(self, *args): if (self.clickedDriftP1 or self.clickedDriftP2): if (self.clickedDriftP1): g.driftP1 = self.mousePosition elif (self.clickedDriftP2): g.driftP2 = self.mousePosition self.clickedDriftP1 = False self.clickedDriftP2 = False self.window.get_window().set_cursor( Gdk.Cursor(Gdk.CursorType.LEFT_PTR)) self.overlay.queue_draw() # Area of Interest button clicked def clickAreaOfInterest(self, *args): g.areaOfInterestP1 = (0, 0) g.areaOfInterestP2 = (0, 0) self.clickedDriftP1 = False self.clickedDriftP2 = False self.clickedAreaOfInterest = True self.frameSlider.set_value(g.startFrame) self.window.get_window().set_cursor( Gdk.Cursor(Gdk.CursorType.CROSSHAIR)) self.overlay.queue_draw() # Reset Area of Interest to (0, 0) def resetAreaOfInterest(self, widget, event): if (event.button == 3): # Right Click g.areaOfInterestP1 = (0, 0) g.areaOfInterestP2 = (0, 0) self.clickedDriftP1 = False self.clickedDriftP2 = False self.clickedAreaOfInterest = False self.window.get_window().set_cursor( Gdk.Cursor(Gdk.CursorType.LEFT_PTR)) self.overlay.queue_draw() # First point int the Area of Interest clicked, drag started def dragBegin(self, *args): if (self.clickedAreaOfInterest): g.areaOfInterestP1 = self.mousePosition # Mouse released after dragging Area of Interest def dragEnd(self, *args): if (self.clickedAreaOfInterest): g.areaOfInterestP2 = self.mousePosition self.clickedAreaOfInterest = False self.window.get_window().set_cursor( Gdk.Cursor(Gdk.CursorType.LEFT_PTR)) # Called when the mouse moves over the frame def updateMousePosition(self, *args): pointer = self.frame.get_pointer() self.mousePosition = (min(max(0, pointer.x), self.frame.get_allocation().width), min(max(0, pointer.y), self.frame.get_allocation().height)) if (self.clickedAreaOfInterest): if (g.areaOfInterestP1 != (0, 0)): g.areaOfInterestP2 = self.mousePosition self.overlay.queue_draw() # Sets whether or not to normalize the frames during alignment def setNormalize(self, *args): g.normalize = self.normalize.get_active() # Sets whether or not to align channels separately def setAlignChannels(self, *args): g.alignChannels = self.alignChannels.get_active() # Sets the type of transformation def setTransformation(self, *args): text = self.transformation.get_active_text() if (text == "None"): g.transformation = -1 elif (text == "Translation"): g.transformation = StackReg.TRANSLATION elif (text == "Rigid Body"): g.transformation = StackReg.RIGID_BODY elif (text == "Scaled Rotation"): g.transformation = StackReg.SCALED_ROTATION elif (text == "Affine"): g.transformation = StackReg.AFFINE # Sets the drizzle scaling factor def setDrizzleFactor(self, *args): text = self.drizzleFactor.get_active_text() if (text == "0.25X"): g.drizzleFactor = 0.25 elif (text == "0.50X"): g.drizzleFactor = 0.50 elif (text == "0.75X"): g.drizzleFactor = 0.75 elif (text == "1.0X"): g.drizzleFactor = 1.0 elif (text == "1.5X"): g.drizzleFactor = 1.5 elif (text == "2.0X"): g.drizzleFactor = 2.0 elif (text == "2.5X"): g.drizzleFactor = 2.5 elif (text == "3.0X"): g.drizzleFactor = 3.0 if (self.stack is not None): self.stack.generateRefBG() self.updateImage() # Sets the drizzle scaling factor def setDrizzleInterpolation(self, *args): text = self.drizzleInterpolation.get_active_text() if (text == "Nearest Neighbor"): g.drizzleInterpolation = cv2.INTER_NEAREST elif (text == "Bilinear"): g.drizzleInterpolation = cv2.INTER_LINEAR elif (text == "Bicubic"): g.drizzleInterpolation = cv2.INTER_CUBIC elif (text == "Lanczos"): g.drizzleInterpolation = cv2.INTER_LANCZOS4 if (self.stack is not None): self.stack.generateRefBG() self.updateImage() # Sets whether or not to auto crop def setAutoCrop(self, *args): g.autoCrop = not self.autoCrop.get_active() self.updateImage() # Runs the Alignment step def clickAlign(self, *args): self.align = Align(self.video.frames[g.startFrame:g.endFrame + 1]) thread = Thread(target=self.align.run, args=()) thread.start() self.disableUI() # Kills all pool processes def killPool(self): for pid in self.pids: if (psutil.pid_exists(pid)): p = psutil.Process(pid) p.kill() # Stops the current action being performed def stopProcessing(self, *args): self.killPool() g.pool = None self.setThreads() self.setProgress() self.enableUI() # Called when the Alignment is complete def finishedAlign(self): def update(): self.stack = Stack(self.align.tmats) self.tabs.next_page() self.enableUI() self.builder.get_object("alignTab").set_sensitive(True) self.builder.get_object("stackTab").set_sensitive(True) self.builder.get_object("processTab").set_sensitive(False) self.limit.set_upper(len(self.align.tmats)) self.limit.set_value(int(len(self.align.tmats) / 2)) self.limitPercent.set_value( round(self.limit.get_value() / len(self.align.tmats) * 100)) self.setLimit() self.setLimitPercent() GLib.idle_add(update) # Sets the number of frames to use in the Stack def setLimit(self, *args): self.limitPercent.disconnect(self.limitPercentSignal) self.limit.set_upper(len(self.align.tmats)) g.limit = int(self.limit.get_value()) self.limitPercent.set_value( round(g.limit / len(self.align.tmats) * 100)) self.limitPercentSignal = self.limitPercent.connect( "value-changed", self.setLimitPercent) # Sets the number of frames to use in the Stack def setLimitPercent(self, *args): limitPercent = self.limitPercent.get_value() / 100 self.limit.set_value(round(limitPercent * len(self.align.tmats))) # Stack Button clicked def clickStack(self, *args): try: self.stack.checkMemory() thread = Thread(target=self.stack.run, args=()) thread.start() self.disableUI() except MemoryError as error: self.enableUI() # Called when the stack is complete def finishedStack(self): def update(): self.sharpen = Sharpen(self.stack.stackedImage) self.tabs.next_page() self.enableUI() self.builder.get_object("alignTab").set_sensitive(True) self.builder.get_object("stackTab").set_sensitive(True) self.builder.get_object("processTab").set_sensitive(True) GLib.idle_add(update) # Sharpens the final Stacked image def sharpenImage(self, *args): g.sharpen1 = self.builder.get_object("sharpen1").get_value() g.sharpen2 = self.builder.get_object("sharpen2").get_value() g.sharpen3 = self.builder.get_object("sharpen3").get_value() g.sharpen4 = self.builder.get_object("sharpen4").get_value() g.sharpen5 = self.builder.get_object("sharpen5").get_value() g.radius1 = self.builder.get_object("radius1").get_value() g.radius2 = self.builder.get_object("radius2").get_value() g.radius3 = self.builder.get_object("radius3").get_value() g.radius4 = self.builder.get_object("radius4").get_value() g.radius5 = self.builder.get_object("radius5").get_value() g.denoise1 = self.builder.get_object("denoise1").get_value() g.denoise2 = self.builder.get_object("denoise2").get_value() g.denoise3 = self.builder.get_object("denoise3").get_value() g.denoise4 = self.builder.get_object("denoise4").get_value() g.denoise5 = self.builder.get_object("denoise5").get_value() g.level1 = self.builder.get_object("level1").get_active() g.level2 = self.builder.get_object("level2").get_active() g.level3 = self.builder.get_object("level3").get_active() g.level4 = self.builder.get_object("level4").get_active() g.level5 = self.builder.get_object("level5").get_active() g.gamma = self.builder.get_object("gammaAdjust").get_value() g.blackLevel = self.builder.get_object("blackLevelAdjust").get_value() g.value = self.builder.get_object("valueAdjust").get_value() g.redAdjust = self.builder.get_object("redAdjust").get_value() g.greenAdjust = self.builder.get_object("greenAdjust").get_value() g.blueAdjust = self.builder.get_object("blueAdjust").get_value() g.saturation = self.builder.get_object("saturationAdjust").get_value() if (len(args) > 0 and (self.builder.get_object("gammaAdjust") == args[0] or self.builder.get_object("blackLevelAdjust") == args[0] or self.builder.get_object("redAdjust") == args[0] or self.builder.get_object("greenAdjust") == args[0] or self.builder.get_object("blueAdjust") == args[0] or self.builder.get_object("saturationAdjust") == args[0] or self.builder.get_object("valueAdjust") == args[0])): processAgain = self.sharpen.processAgain processColor = True else: processAgain = True processColor = False if (self.sharpen is None): if (self.stack is not None): self.sharpen = Sharpen(self.stack.stackedImage) else: self.sharpen = Sharpen(g.file, True) if (self.processThread != None and self.processThread.is_alive()): self.sharpen.processAgain = processAgain self.sharpen.processColorAgain = processColor else: self.processThread = Thread(target=self.sharpen.run, args=(processAgain, processColor)) self.processThread.start() # Called when sharpening is complete def finishedSharpening(self): def update(): z = self.sharpen.finalImage.astype('uint8').tobytes() Z = GLib.Bytes.new(z) pixbuf = GdkPixbuf.Pixbuf.new_from_bytes(Z, GdkPixbuf.Colorspace.RGB, False, 8, self.sharpen.w, self.sharpen.h, self.sharpen.w * 3) self.frame.set_from_pixbuf(pixbuf) GLib.idle_add(update) # Closes the application def close(self, *args): self.killPool() Gtk.main_quit() sys.exit()
def test_pipe(): parent_conn, child_conn = Pipe() p = Process(target=f2, args=(child_conn, )) p.start() print parent_conn.recv() p.join()
def advance_to_next_server(signum, frame): sys.stdout.write("#skip the current server...\n") skip[0] = True signal.signal(signal.SIGINT, advance_to_next_server) idx = 0 while idx < len(srv_list): ofs, conn = pick_ssh_server(srv_list[idx:], cmd, user, passwd, timeout, ob_key) idx += ofs if conn is not None: while conn.isalive(): if checker is not None and not checker.is_alive(): sorted_srv_name_list = checker_conn.recv() if opt.debug: sys.stdout.write("#got sorted server list\n %s\n" % "\n ".join(sorted_srv_name_list)) checker.join() checker = None with open(get_cfg_fpath(), "wb") as fd: fd.write("\n".join(sorted_srv_name_list)) fd.write("\n") if srv_list[idx] in sorted_srv_name_list[:3]: sys.stdout.write("#reconnection is not needed\n") else: srv_list = gen_server_list(sorted_srv_name_list, opt.port)
# multiProcessCommunicationByPipe.py from multiprocessing import Process,Pipe import os def sendMsg(conn,msg): conn.send("I am the child process %s send the message %s"%(os.getpid(),msg)) if __name__ == '__main__': parent_conn,child_conn = Pipe() p = Process(target = sendMsg,args=(child_conn,"hello kingwen",)) p.start() print("this is parent process %s ,msg from child is :%s"%(os.getpid(),parent_conn.recv())) p.join()
def run_base(exp_dir, **kwargs): config = ConfigProto() config.gpu_options.allow_growth = True config.gpu_options.per_process_gpu_memory_fraction = kwargs.get( 'gpu_frac', 0.95) # Instantiate classes set_seed(kwargs['seed']) baseline = kwargs['baseline']() if kwargs['env'] == 'Ant': env = AntEnv() simulation_sleep = 0.05 * kwargs['num_rollouts'] * kwargs[ 'max_path_length'] * kwargs['simulation_sleep_frac'] elif kwargs['env'] == 'HalfCheetah': env = HalfCheetahEnv() simulation_sleep = 0.05 * kwargs['num_rollouts'] * kwargs[ 'max_path_length'] * kwargs['simulation_sleep_frac'] elif kwargs['env'] == 'Hopper': env = HopperEnv() simulation_sleep = 0.008 * kwargs['num_rollouts'] * kwargs[ 'max_path_length'] * kwargs['simulation_sleep_frac'] elif kwargs['env'] == 'Walker2d': env = Walker2dEnv() simulation_sleep = 0.008 * kwargs['num_rollouts'] * kwargs[ 'max_path_length'] * kwargs['simulation_sleep_frac'] else: raise NotImplementedError policy = MetaGaussianMLPPolicy( name="meta-policy", obs_dim=np.prod(env.observation_space.shape), action_dim=np.prod(env.action_space.shape), meta_batch_size=kwargs['meta_batch_size'], hidden_sizes=kwargs['policy_hidden_sizes'], learn_std=kwargs['policy_learn_std'], hidden_nonlinearity=kwargs['policy_hidden_nonlinearity'], output_nonlinearity=kwargs['policy_output_nonlinearity'], ) dynamics_model = MLPDynamicsEnsemble( 'dynamics-ensemble', env=env, num_models=kwargs['num_models'], hidden_nonlinearity=kwargs['dyanmics_hidden_nonlinearity'], hidden_sizes=kwargs['dynamics_hidden_sizes'], output_nonlinearity=kwargs['dyanmics_output_nonlinearity'], learning_rate=kwargs['dynamics_learning_rate'], batch_size=kwargs['dynamics_batch_size'], buffer_size=kwargs['dynamics_buffer_size'], rolling_average_persitency=kwargs['rolling_average_persitency'], ) '''-------- dumps and reloads -----------------''' baseline_pickle = pickle.dumps(baseline) env_pickle = pickle.dumps(env) receiver, sender = Pipe() p = Process( target=init_vars, name="init_vars", args=(sender, config, policy, dynamics_model), daemon=True, ) p.start() policy_pickle, dynamics_model_pickle = receiver.recv() receiver.close() '''-------- following classes depend on baseline, env, policy, dynamics_model -----------''' worker_data_feed_dict = { 'env_sampler': { 'rollouts_per_meta_task': kwargs['real_env_rollouts_per_meta_task'], 'meta_batch_size': kwargs['meta_batch_size'], 'max_path_length': kwargs['max_path_length'], 'parallel': kwargs['parallel'], }, 'dynamics_sample_processor': { 'discount': kwargs['discount'], 'gae_lambda': kwargs['gae_lambda'], 'normalize_adv': kwargs['normalize_adv'], 'positive_adv': kwargs['positive_adv'], }, } worker_model_feed_dict = {} worker_policy_feed_dict = { 'model_sampler': { 'rollouts_per_meta_task': kwargs['rollouts_per_meta_task'], 'meta_batch_size': kwargs['meta_batch_size'], 'max_path_length': kwargs['max_path_length'], 'deterministic': kwargs['deterministic'], }, 'model_sample_processor': { 'discount': kwargs['discount'], 'gae_lambda': kwargs['gae_lambda'], 'normalize_adv': kwargs['normalize_adv'], 'positive_adv': kwargs['positive_adv'], }, 'algo': { 'step_size': kwargs['step_size'], 'inner_type': kwargs['inner_type'], 'inner_lr': kwargs['inner_lr'], 'meta_batch_size': kwargs['meta_batch_size'], 'num_inner_grad_steps': kwargs['num_inner_grad_steps'], 'exploration': kwargs['exploration'], } } trainer = ParallelTrainer( exp_dir=exp_dir, policy_pickle=policy_pickle, env_pickle=env_pickle, baseline_pickle=baseline_pickle, dynamics_model_pickle=dynamics_model_pickle, feed_dicts=[ worker_data_feed_dict, worker_model_feed_dict, worker_policy_feed_dict ], n_itr=kwargs['n_itr'], num_inner_grad_steps=kwargs['num_inner_grad_steps'], initial_random_samples=kwargs['initial_random_samples'], flags_need_query=kwargs['flags_need_query'], num_rollouts_per_iter=int(kwargs['meta_batch_size'] * kwargs['fraction_meta_batch_size']), config=config, simulation_sleep=simulation_sleep, ) trainer.train()
#!/usr/bin/python3 # -*- coding: utf-8 -*- from multiprocessing import Process, Pipe import time def f(conn): conn.send([42, None, 'hello from child']) conn.send([42, None, 'hello from child1']) conn.close() if __name__ == "__main__": parent_conn, child_conn = Pipe() p = Process(target=f, args=(child_conn, )) p.start() print(parent_conn.recv()) print(parent_conn.recv()) print(parent_conn.recv()) p.join()
class AillioR1: AILLIO_VID = 0x0483 AILLIO_PID = 0x5741 AILLIO_PID_REV3 = 0xa27e AILLIO_ENDPOINT_WR = 0x3 AILLIO_ENDPOINT_RD = 0x81 AILLIO_INTERFACE = 0x1 AILLIO_CONFIGURATION = 0x1 AILLIO_DEBUG = 1 AILLIO_CMD_INFO1 = [0x30, 0x02] AILLIO_CMD_INFO2 = [0x89, 0x01] AILLIO_CMD_STATUS1 = [0x30, 0x01] AILLIO_CMD_STATUS2 = [0x30, 0x03] AILLIO_CMD_PRS = [0x30, 0x01, 0x00, 0x00] AILLIO_CMD_HEATER_INCR = [0x34, 0x01, 0xaa, 0xaa] AILLIO_CMD_HEATER_DECR = [0x34, 0x02, 0xaa, 0xaa] AILLIO_CMD_FAN_INCR = [0x31, 0x01, 0xaa, 0xaa] AILLIO_CMD_FAN_DECR = [0x31, 0x02, 0xaa, 0xaa] AILLIO_STATE_OFF = 0x00 AILLIO_STATE_PH = 0x02 AILLIO_STATE_CHARGE = 0x04 AILLIO_STATE_ROASTING = 0x06 AILLIO_STATE_COOLING = 0x08 AILLIO_STATE_SHUTDOWN = 0x09 def __init__(self, debug=False): self.simulated = False self.AILLIO_DEBUG = debug self.__dbg('init') self.usbhandle = None self.bt = 0 self.dt = 0 self.heater = 0 self.fan = 0 self.bt_ror = 0 self.drum = 0 self.voltage = 0 self.exitt = 0 self.state_str = "" self.r1state = 0 self.worker_thread = None self.worker_thread_run = True self.roast_number = -1 self.fan_rpm = 0 def __del__(self): if not self.simulated: self.__close() def __dbg(self, msg): if self.AILLIO_DEBUG and self.simulated != True: try: print('AillioR1: ' + msg) except IOError: pass def __open(self): if self.simulated: return if self.usbhandle is not None: return self.usbhandle = usb.core.find(idVendor=self.AILLIO_VID, idProduct=self.AILLIO_PID) if self.usbhandle is None: self.usbhandle = usb.core.find(idVendor=self.AILLIO_VID, idProduct=self.AILLIO_PID_REV3) if self.usbhandle is None: raise IOError("not found or no permission") self.__dbg('device found!') if not system().startswith("Windows"): if self.usbhandle.is_kernel_driver_active(self.AILLIO_INTERFACE): try: self.usbhandle.detach_kernel_driver(self.AILLIO_INTERFACE) except Exception: self.usbhandle = None raise IOError("unable to detach kernel driver") try: config = self.usbhandle.get_active_configuration() if config.bConfigurationValue != self.AILLIO_CONFIGURATION: self.usbhandle.set_configuration( configuration=self.AILLIO_CONFIGURATION) except Exception: self.usbhandle = None raise IOError("unable to configure") try: usb.util.claim_interface(self.usbhandle, self.AILLIO_INTERFACE) except Exception: self.usbhandle = None raise IOError("unable to claim interface") self.__sendcmd(self.AILLIO_CMD_INFO1) reply = self.__readreply(32) sn = unpack('h', reply[0:2])[0] firmware = unpack('h', reply[24:26])[0] self.__dbg('serial number: ' + str(sn)) self.__dbg('firmware version: ' + str(firmware)) self.__sendcmd(self.AILLIO_CMD_INFO2) reply = self.__readreply(36) self.roast_number = unpack('>I', reply[27:31])[0] self.__dbg('number of roasts: ' + str(self.roast_number)) self.parent_pipe, self.child_pipe = Pipe() self.worker_thread = threading.Thread(target=self.__updatestate, args=(self.child_pipe, )) self.worker_thread.start() def __close(self): if self.simulated: return if self.usbhandle is not None: try: usb.util.release_interface(self.usbhandle, self.AILLIO_INTERFACE) usb.util.dispose_resources(self.usbhandle) except Exception: pass self.usbhandle = None if self.worker_thread: self.worker_thread_run = False self.worker_thread.join() self.parent_pipe.close() self.child_pipe.close() self.worker_thread = None def get_roast_number(self): self.__getstate() return self.roast_number def get_bt(self): self.__getstate() return self.bt def get_dt(self): self.__getstate() return self.dt def get_heater(self): self.__dbg('get_heater') self.__getstate() return self.heater def get_fan(self): self.__dbg('get_fan') self.__getstate() return self.fan def get_fan_rpm(self): self.__dbg('get_fan_rpm') self.__getstate() return self.fan_rpm def get_drum(self): self.__getstate() return self.drum def get_voltage(self): self.__getstate() return self.voltage def get_bt_ror(self): self.__getstate() return self.bt_ror def get_exit_temperature(self): self.__getstate() return self.exitt def get_state_string(self): self.__getstate() return self.state_str def get_state(self): self.__getstate() return self.r1state def set_heater(self, value): self.__dbg('set_heater ' + str(value)) value = int(value) if value < 0: value = 0 elif value > 9: value = 9 h = self.get_heater() d = abs(h - value) if d <= 0: return if d > 9: d = 9 if h > value: for _ in range(d): self.parent_pipe.send(self.AILLIO_CMD_HEATER_DECR) else: for _ in range(d): self.parent_pipe.send(self.AILLIO_CMD_HEATER_INCR) self.heater = value def set_fan(self, value): self.__dbg('set_fan ' + str(value)) value = int(value) if value < 1: value = 1 elif value > 12: value = 12 f = self.get_fan() d = abs(f - value) if d <= 0: return if d > 11: d = 11 if f > value: for _ in range(0, d): self.parent_pipe.send(self.AILLIO_CMD_FAN_DECR) else: for _ in range(0, d): self.parent_pipe.send(self.AILLIO_CMD_FAN_INCR) self.fan = value def set_drum(self, value): self.__dbg('set_drum ' + str(value)) value = int(value) if value < 1: value = 1 elif value > 9: value = 9 self.parent_pipe.send([0x32, 0x01, value, 0x00]) self.drum = value def prs(self): self.__dbg('PRS') self.parent_pipe.send(self.AILLIO_CMD_PRS) def __updatestate(self, p): while self.worker_thread_run: state1 = state2 = [] try: self.__dbg('updatestate') self.__sendcmd(self.AILLIO_CMD_STATUS1) state1 = self.__readreply(64) self.__sendcmd(self.AILLIO_CMD_STATUS2) state2 = self.__readreply(64) except Exception: pass if p.poll(): cmd = p.recv() self.__sendcmd(cmd) if len(state1) + len(state2) == 128: p.send(state1 + state2) time.sleep(0.1) def __getstate(self): self.__dbg('getstate') if self.simulated: if random.random() > 0.05: return self.bt += random.random() self.bt_ror += random.random() self.dt += random.random() self.exitt += random.random() self.fan = random.random() * 10 self.heater = random.random() * 8 self.drum = random.random() * 8 self.irt = random.random() self.pcbt = random.random() self.fan_rpm += random.random() self.voltage = 240 self.coil_fan = 0 self.coil_fan2 = 0 self.pht = 0 self.r1state = self.AILLIO_STATE_ROASTING self.state_str = "roasting" return self.__open() if not self.parent_pipe.poll(): return state = self.parent_pipe.recv() valid = state[41] # Heuristic to find out if the data is valid # It looks like we get a different message every 15 seconds # when we're not roasting. Ignore this message for now. if valid == 10: self.bt = round(unpack('f', state[0:4])[0], 1) self.bt_ror = round(unpack('f', state[4:8])[0], 1) self.dt = round(unpack('f', state[8:12])[0], 1) self.exitt = round(unpack('f', state[16:20])[0], 1) self.minutes = state[24] self.seconds = state[25] self.fan = state[26] self.heater = state[27] self.drum = state[28] self.r1state = state[29] self.irt = round(unpack('f', state[32:36])[0], 1) self.pcbt = round(unpack('f', state[36:40])[0], 1) self.fan_rpm = unpack('h', state[44:46])[0] self.voltage = unpack('h', state[48:50])[0] self.coil_fan = round(unpack('i', state[52:56])[0], 1) self.__dbg('BT: ' + str(self.bt)) self.__dbg('BT RoR: ' + str(self.bt_ror)) self.__dbg('DT: ' + str(self.dt)) self.__dbg('exit temperature ' + str(self.exitt)) self.__dbg('PCB temperature: ' + str(self.irt)) self.__dbg('IR temperature: ' + str(self.pcbt)) self.__dbg('voltage: ' + str(self.voltage)) self.__dbg('coil fan: ' + str(self.coil_fan)) self.__dbg('fan: ' + str(self.fan)) self.__dbg('heater: ' + str(self.heater)) self.__dbg('drum speed: ' + str(self.drum)) self.__dbg('time: ' + str(self.minutes) + ':' + str(self.seconds)) state = state[64:] self.coil_fan2 = round(unpack('i', state[32:36])[0], 1) self.pht = unpack('H', state[40:42])[0] self.__dbg('pre-heat temperature: ' + str(self.pht)) if self.r1state == self.AILLIO_STATE_OFF: self.state_str = "off" elif self.r1state == self.AILLIO_STATE_PH: self.state_str = "pre-heating to " + str(self.pht) + "C" elif self.r1state == self.AILLIO_STATE_CHARGE: self.state_str = "charge" elif self.r1state == self.AILLIO_STATE_ROASTING: self.state_str = "roasting" elif self.r1state == self.AILLIO_STATE_COOLING: self.state_str = "cooling" elif self.r1state == self.AILLIO_STATE_SHUTDOWN: self.state_str = "shutdown" self.__dbg('state: ' + self.state_str) self.__dbg('second coil fan: ' + str(self.coil_fan2)) def __sendcmd(self, cmd): self.__dbg('sending command: ' + str(cmd)) self.usbhandle.write(self.AILLIO_ENDPOINT_WR, cmd) def __readreply(self, length): return self.usbhandle.read(self.AILLIO_ENDPOINT_RD, length)
class BTgymRendering(): """ Handles BTgym Environment rendering. Note: Call `initialize_pyplot()` method before first render() call! """ # Here we'll keep last rendered image for each rendering mode: rgb_dict = dict() render_modes = ['episode', 'human'] params = dict( # Plotting controls, can be passed as kwargs: render_state_as_image=True, render_state_channel=0, render_size_human=(6, 3.5), render_size_state=(7, 3.5), render_size_episode=(12, 8), render_rowsmajor_episode=1, render_dpi=75, render_plotstyle='seaborn', render_cmap='PRGn', render_xlabel='Relative timesteps', render_ylabel='Value', render_title= 'local step: {}, state observation min: {:.4f}, max: {:.4f}', render_boxtext=dict( fontsize=12, fontweight='bold', color='w', bbox={ 'facecolor': 'k', 'alpha': 0.3, 'pad': 3 }, ), plt_backend='Agg', # Not used. ) enabled = True ready = False def __init__(self, render_modes, **kwargs): """ Plotting controls, can be passed as kwargs. Args: render_state_as_image=True, render_state_channel=0, render_size_human=(6, 3.5), render_size_state=(7, 3.5), render_size_episode=(12,8), render_dpi=75, render_plotstyle='seaborn', render_cmap='PRGn', render_xlabel='Relative timesteps', render_ylabel='Value', render_title='local step: {}, state observation min: {:.4f}, max: {:.4f}', render_boxtext=dict(fontsize=12, fontweight='bold', color='w', bbox={'facecolor': 'k', 'alpha': 0.3, 'pad': 3}, ) """ # Update parameters with relevant kwargs: for key, value in kwargs.items(): if key in self.params.keys(): self.params[key] = value # Unpack it as attributes: for key, value in self.params.items(): setattr(self, key, value) # Logging: if 'log_level' not in dir(self): self.log_level = WARNING StreamHandler(sys.stdout).push_application() self.log = Logger('BTgymRenderer', level=self.log_level) #from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas #self.FigureCanvas = FigureCanvas self.plt = None # Will set it inside server process when calling initialize_pyplot(). #self.plotter = BTgymPlotter() # Modified bt.Cerebro() plotter, to get episode renderings. # Set empty plugs for each render mode: self.render_modes = render_modes for mode in self.render_modes: self.rgb_dict[mode] = self.rgb_empty() def initialize_pyplot(self): """ Call me before use! [Supposed to be done inside already running server process] """ if not self.ready: from multiprocessing import Pipe self.out_pipe, self.in_pipe = Pipe() if self.plt is None: import matplotlib matplotlib.use(self.plt_backend, force=True) import matplotlib.pyplot as plt self.plt = plt self.ready = True def to_string(self, dictionary, excluded=[]): """ Converts given dictionary to more-or-less good looking `text block` string. """ text = '' for k, v in dictionary.items(): if k not in excluded: if type(v) in [float]: v = '{:.4f}'.format(v) text += '{}: {}\n'.format(k, v) return text[:-1] def rgb_empty(self): """ Returns empty 'plug' image. """ return (np.random.rand(100, 200, 3) * 255).astype(dtype=np.uint8) def parse_response( self, state, mode, reward, info, done, ): """ Converts environment response to plotting attributes: state, title, text. """ if len(state[mode].shape) <= 2: state = np.asarray(state[mode]) elif len(state[mode].shape) == 3: if state[mode].shape[1] == 1: # Assume 2nd dim (H) is fake expansion for 1D input, so can render all channels: state = np.asarray(state[mode][:, 0, :]) else: # Assume it is HWC 2D input, only can render single channel: state = np.asarray(state[mode][:, :, self.render_state_channel]) else: raise NotImplementedError( '2D rendering can be done for obs. state tensor with rank <= 3; ' +\ 'got state shape: {}'.format(np.asarray(state[mode]).shape)) # Figure out how to deal with info output: try: assert type(info[-1]) == dict info_dict = info[-1] except AssertionError: try: assert type(info) == dict info_dict = info except AssertionError: try: info_dict = {'info': str(dict)} except: info_dict = {} # Add records: info_dict.update( reward=reward, is_done=done, ) # Try to get step information: try: current_step = info_dict['step'] except: current_step = '--' # Set box text, excluding redundant fields: box_text = self.to_string(info_dict, excluded=['step']) # Set title output: title = self.render_title.format(current_step, state.min(), state.max()) return state, title, box_text def render(self, mode_list, cerebro=None, step_to_render=None, send_img=True): """ Renders given mode if possible, else just passes last already rendered image. Returns rgb image as numpy array. Logic: - If `cerebro` arg is received: render entire episode, using built-in backtrader plotting feature, update stored `episode` image. - If `step_to_render' arg is received: - if mode = 'raw_state': render current state observation in conventional 'price' format, update stored `raw_state` image; - if mode = something_else': visualise observation as 'seen' by agent, update stored 'agent' image. Returns: `mode` image. Note: It can actually return several modes in a single dict. It prevented by Gym modes convention, but done internally at the end of the episode. """ if type(mode_list) == str: mode_list = [mode_list] if cerebro is not None: self.rgb_dict['episode'] = self.draw_episode(cerebro) # Try to render given episode: #try: # Get picture of entire episode: #fig = cerebro.plot(plotter=self.plotter, # Modified plotter class, doesnt actually save anything. # savefig=True, # width=self.render_size_episode[0], # height=self.render_size_episode[1], # dpi=self.render_dpi, # use=None, #self.plt_backend, # iplot=False, # figfilename='_tmp_btgym_render.png', # )[0][0] #fig.canvas.draw() #rgb_array = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep='') #self.rgb_dict['episode'] = rgb_array.reshape(fig.canvas.get_width_height()[::-1] + (3,)) # Clean up: #self.plt.gcf().clear() #self.plt.close(fig) #except: # Just keep previous rendering # pass if step_to_render is not None: # Perform step rendering: # Unpack: raw_state, state, reward, done, info = step_to_render for mode in mode_list: if mode in self.render_modes and mode not in [ 'episode', 'human' ]: # Render user-defined (former agent) mode state: agent_state, title, box_text = self.parse_response( state, mode, reward, info, done) if self.render_state_as_image: self.rgb_dict[mode] = self.draw_image( agent_state, figsize=self.render_size_state, title='{} / {}'.format(mode, title), box_text=box_text, ylabel=self.render_ylabel, xlabel=self.render_xlabel, ) else: self.rgb_dict[mode] = self.draw_plot( agent_state, figsize=self.render_size_state, title='{} / {}'.format(mode, title), box_text=box_text, ylabel=self.render_ylabel, xlabel=self.render_xlabel, ) if 'human' in mode: # Render `human` state: human_state, title, box_text = self.parse_response( raw_state, mode, reward, info, done) self.rgb_dict['human'] = self.draw_plot( human_state, figsize=self.render_size_human, title=title, box_text=box_text, ylabel='Price', xlabel=self.render_xlabel, line_labels=['Open', 'High', 'Low', 'Close'], ) if send_img: return self.rgb_dict else: # this case is for internal use only; # now `mode` supposed to contain several modes, let's return dictionary of arrays: return_dict = dict() for entry in mode_list: if entry in self.rgb_dict.keys(): # ...and it is legal: return_dict[entry] = self.rgb_dict[entry] else: return_dict[entry] = self.rgb_empty() return return_dict def draw_plot(self, data, figsize=(10, 6), title='', box_text='', xlabel='X', ylabel='Y', line_labels=None): """ Visualises environment state as 2d line plot. Retrurns image as rgb_array. Args: data: np.array of shape [num_values, num_lines] figsize: figure size (in.) title: box_text: xlabel: ylabel: line_labels: iterable holding line legends as str Returns: rgb image as np.array of size [with, height, 3] """ if line_labels is None: # If got no labels - make it numbers: if len(data.shape) > 1: line_labels = [ 'line_{}'.format(i) for i in range(data.shape[-1]) ] else: line_labels = ['line_0'] data = data[:, None] else: assert len(line_labels) == data.shape[-1], \ 'Expected `line_labels` kwarg consist of {} names, got: {}'. format(data.shape[-1], line_labels) fig = self.plt.figure( figsize=figsize, dpi=self.render_dpi, ) #ax = fig.add_subplot(111) self.plt.style.use(self.render_plotstyle) self.plt.title(title) # Plot x axis as reversed time-step embedding: xticks = np.linspace(data.shape[0] - 1, 0, int(data.shape[0]), dtype=int) self.plt.xticks(xticks.tolist(), (-xticks[::-1]).tolist(), visible=False) # Set every 5th tick label visible: for tick in self.plt.xticks()[1][::5]: tick.set_visible(True) self.plt.xlabel(xlabel) self.plt.ylabel(ylabel) self.plt.grid(True) # Switch off antialiasing: #self.plt.setp([ax.get_xticklines() + ax.get_yticklines() + ax.get_xgridlines() + ax.get_ygridlines()],antialiased=False) #self.plt.rcParams['text.antialiased']=False # Add Info box: self.plt.text(0, data.min(), box_text, **self.render_boxtext) for line, label in enumerate(line_labels): self.plt.plot(data[:, line], label=label) self.plt.legend() self.plt.tight_layout() fig.canvas.draw() # Save it to a numpy array: rgb_array = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep='') # Clean up: self.plt.close(fig) #self.plt.gcf().clear() return rgb_array.reshape(fig.canvas.get_width_height()[::-1] + (3, )) def draw_image(self, data, figsize=(12, 6), title='', box_text='', xlabel='X', ylabel='Y', line_labels=None): """ Visualises environment state as image. Returns rgb_array. """ fig = self.plt.figure( figsize=figsize, dpi=self.render_dpi, ) #ax = fig.add_subplot(111) self.plt.style.use(self.render_plotstyle) self.plt.title(title) # Plot x axis as reversed time-step embedding: xticks = np.linspace(data.shape[0] - 1, 0, int(data.shape[0]), dtype=int) self.plt.xticks(xticks.tolist(), (-xticks[::-1]).tolist(), visible=False) # Set every 5th tick label visible: for tick in self.plt.xticks()[1][::5]: tick.set_visible(True) #self.plt.yticks(visible=False) self.plt.xlabel(xlabel) self.plt.ylabel(ylabel) self.plt.grid(False) # Switch off antialiasing: # self.plt.setp([ax.get_xticklines() + ax.get_yticklines() + ax.get_xgridlines() + ax.get_ygridlines()],antialiased=False) # self.plt.rcParams['text.antialiased']=False #self.log.warning('render_data_shape:{}'.format(data.shape)) # Add Info box: self.plt.text(0, data.shape[1] - 1, box_text, **self.render_boxtext) im = self.plt.imshow(data.T, aspect='auto', cmap=self.render_cmap) self.plt.colorbar(im, use_gridspec=True) self.plt.tight_layout() fig.canvas.draw() # Save it to a numpy array: rgb_array = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep='') # Clean up: self.plt.close(fig) #self.plt.gcf().clear() #ax.cla() return rgb_array.reshape(fig.canvas.get_width_height()[::-1] + (3, )) def draw_episode(self, cerebro): """ Hacky way to render episode. Due to backtrader/matplotlib memory leaks have to encapsulate it in separate process. Strange but reliable. PID's are driving crazy. Args: cerebro instance Returns: rgb array. """ draw_process = DrawCerebro( cerebro=cerebro, width=self.render_size_episode[0], height=self.render_size_episode[1], dpi=self.render_dpi, result_pipe=self.in_pipe, rowsmajor=self.render_rowsmajor_episode, ) draw_process.start() #print('Plotter PID: {}'.format(draw_process.pid)) try: rgb_array = self.out_pipe.recv() draw_process.terminate() draw_process.join() return rgb_array except: return self.rgb_empty()
def limit_function(func, wall_clock_time, mem_usage_limit, *args, **kwargs): """ :param func: the objective function to call. :param wall_clock_time: seconds. :param mem_usage_limit: megabytes. :param args: :param kwargs: :return: More Info about Memory Management [https://psutil.readthedocs.io/en/latest/#psutil.Process.memory_info]: rss: aka “Resident Set Size”, this is the non-swapped physical memory a process has used. On UNIX it matches “top“‘s RES column). On Windows this is an alias for wset field and it matches “Mem Usage” column of taskmgr.exe. vms: aka “Virtual Memory Size”, this is the total amount of virtual memory used by the process. On UNIX it matches “top“‘s VIRT column. On Windows this is an alias for pagefile field and it matches “Mem Usage” “VM Size” column of taskmgr.exe. """ _debug_mode = False if 'debug' in kwargs: _debug_mode = kwargs['debug'] # Deal with special case in Bayesian optimization. if len(args) == 0 and 'args' in kwargs: args = kwargs['args'] kwargs = kwargs['kwargs'] if _platform == 'Windows': freeze_support() parent_conn, child_conn = Pipe(False) func = dill.dumps(func) args = [func] + [child_conn] + [wall_clock_time] + [mem_usage_limit ] + list(args) p = Process(target=wrapper, args=tuple(args), kwargs=kwargs) p.start() # Special case on windows. if _platform in ['Windows', 'OSX', 'Linux']: p_id = p.pid exceed_mem_limit = False start_time = time.time() while time.time() <= start_time + wall_clock_time: if not psutil.pid_exists(p_id) or psutil.Process( p_id).status() == 'zombie': break rss_used = psutil.Process(p_id).memory_info().rss / 1024 / 1024 vms_used = psutil.Process(p_id).memory_info().vms / 1024 / 1024 if _debug_mode: print(psutil.Process(p_id).memory_info()) print('mem[rss]_used', rss_used) print('mem[vms]_used', vms_used) threshold = rss_used if _platform in ['OSX', 'Linux', 'Windows' ] else vms_used if threshold > mem_usage_limit: exceed_mem_limit = True break time.sleep(1.) if exceed_mem_limit: clean_processes(p) return Returns(status=False, result=OutOfMemoryLimitException) if p.is_alive(): clean_processes(p) return Returns(status=False, result=TimeoutException) result = parent_conn.recv() parent_conn.close() return result else: p.join(wall_clock_time) if p.is_alive(): clean_processes(p) return Returns(status=False, result=TimeoutException) result = parent_conn.recv() parent_conn.close() return result
class MarkupHandler: TAG_NAME_ITALIC = 'italic' TAG_NAME_BOLD = 'bold' TAG_NAME_BOLD_ITALIC = 'bold_italic' TAG_NAME_STRIKETHROUGH = 'strikethrough' TAG_NAME_CENTER = 'center' TAG_NAME_WRAP_NONE = 'wrap_none' TAG_NAME_PLAIN_TEXT = 'plain_text' TAG_NAME_GRAY_TEXT = 'gray_text' TAG_NAME_CODE_TEXT = 'code_text' TAG_NAME_CODE_BLOCK = 'code_block' TAG_NAME_UNFOCUSED_TEXT = 'unfocused_text' TAG_NAME_MARGIN_INDENT = 'margin_indent' def __init__(self, text_view): self.text_view = text_view self.text_buffer = text_view.get_buffer() self.marked_up_text = None # Tags. buffer = self.text_buffer self.tag_italic = buffer.create_tag(self.TAG_NAME_ITALIC, weight=Pango.Weight.NORMAL, style=Pango.Style.ITALIC) self.tag_bold = buffer.create_tag(self.TAG_NAME_BOLD, weight=Pango.Weight.BOLD, style=Pango.Style.NORMAL) self.tag_bold_italic = buffer.create_tag(self.TAG_NAME_BOLD_ITALIC, weight=Pango.Weight.BOLD, style=Pango.Style.ITALIC) self.tag_strikethrough = buffer.create_tag(self.TAG_NAME_STRIKETHROUGH, strikethrough=True) self.tag_center = buffer.create_tag( self.TAG_NAME_CENTER, justification=Gtk.Justification.CENTER) self.tag_wrap_none = buffer.create_tag(self.TAG_NAME_WRAP_NONE, wrap_mode=Gtk.WrapMode.NONE, pixels_above_lines=0, pixels_below_lines=0) self.tag_plain_text = buffer.create_tag( self.TAG_NAME_PLAIN_TEXT, weight=Pango.Weight.NORMAL, style=Pango.Style.NORMAL, strikethrough=False, justification=Gtk.Justification.LEFT) self.tag_gray_text = buffer.create_tag(self.TAG_NAME_GRAY_TEXT, foreground='gray', weight=Pango.Weight.NORMAL, style=Pango.Style.NORMAL) self.tag_code_text = buffer.create_tag(self.TAG_NAME_CODE_TEXT, weight=Pango.Weight.NORMAL, style=Pango.Style.NORMAL, strikethrough=False) self.tag_code_block = buffer.create_tag(self.TAG_NAME_CODE_BLOCK, weight=Pango.Weight.NORMAL, style=Pango.Style.NORMAL, strikethrough=False, indent=self.get_margin_indent( 0, 1)[1]) self.tags_markup = { self.TAG_NAME_ITALIC: lambda args: self.tag_italic, self.TAG_NAME_BOLD: lambda args: self.tag_bold, self.TAG_NAME_BOLD_ITALIC: lambda args: self.tag_bold_italic, self.TAG_NAME_STRIKETHROUGH: lambda args: self.tag_strikethrough, self.TAG_NAME_CENTER: lambda args: self.tag_center, self.TAG_NAME_WRAP_NONE: lambda args: self.tag_wrap_none, self.TAG_NAME_PLAIN_TEXT: lambda args: self.tag_plain_text, self.TAG_NAME_GRAY_TEXT: lambda args: self.tag_gray_text, self.TAG_NAME_CODE_TEXT: lambda args: self.tag_code_text, self.TAG_NAME_CODE_BLOCK: lambda args: self.tag_code_block, self.TAG_NAME_MARGIN_INDENT: lambda args: self.get_margin_indent_tag(*args) } # Focus mode. self.tag_unfocused_text = buffer.create_tag( self.TAG_NAME_UNFOCUSED_TEXT, foreground='gray', weight=Pango.Weight.NORMAL, style=Pango.Style.NORMAL) # Margin and indents. # A baseline margin is set to allow negative offsets for formatting headers, lists, etc. self.tags_margins_indents = {} self.baseline_margin = 0 self.char_width = 0 self.update_margins_indents() # Style. self.on_style_updated() # Worker process to handle parsing. self.parsing = False self.apply_pending = False self.parent_conn, child_conn = Pipe() Process(target=self.parse, args=(child_conn, ), daemon=True).start() GLib.io_add_watch(self.parent_conn.fileno(), GLib.PRIORITY_DEFAULT, GLib.IO_IN, self.on_parsed) def on_style_updated(self, *_): style_context = self.text_view.get_style_context() (found, color) = style_context.lookup_color('code_bg_color') if not found: (_, color) = style_context.lookup_color('background_color') self.tag_code_text.set_property("background", color.to_string()) self.tag_code_block.set_property("paragraph-background", color.to_string()) def apply(self): """Applies markup, parsing it in a worker process if the text has changed. In case parsing is already running, it will re-apply once it finishes. This ensure that the pipe doesn't fill (and block) if multiple requests are made in quick succession.""" if not self.parsing: self.parsing = True self.apply_pending = False text = self.text_buffer.get_slice( self.text_buffer.get_start_iter(), self.text_buffer.get_end_iter(), False) if text != self.marked_up_text: self.parent_conn.send(text) else: self.do_apply(text) else: self.apply_pending = True def parse(self, child_conn): """Parses markup in a worker process.""" while True: while True: try: text = child_conn.recv() if not child_conn.poll(): break except EOFError: child_conn.close() return # List of tuples in the form (tag_name, tag_args, tag_start, tag_end). result = [] # Find: # - "_italic_" (italic) # - "**bold**" (bold) # - "***bolditalic***" (bold/italic) # - "~~strikethrough~~" (strikethrough) # - "`code`" (colorize) # - "$math$" (colorize) # - "---" table (wrap/pixels) regexps = ((ITALIC, self.TAG_NAME_ITALIC), (BOLD, self.TAG_NAME_BOLD), (BOLD_ITALIC, self.TAG_NAME_BOLD_ITALIC), (STRIKETHROUGH, self.TAG_NAME_STRIKETHROUGH), (CODE, self.TAG_NAME_CODE_TEXT), (MATH, self.TAG_NAME_CODE_TEXT), (TABLE, self.TAG_NAME_WRAP_NONE)) for regexp, tag_name in regexps: matches = re.finditer(regexp, text) for match in matches: result.append((tag_name, (), match.start(), match.end())) # Find: # - "[description](url)" (gray out) # - "![description](image_url)" (gray out) regexps = ((LINK, self.TAG_NAME_GRAY_TEXT), (IMAGE, self.TAG_NAME_GRAY_TEXT)) for regexp, tag_name in regexps: matches = re.finditer(regexp, text) for match in matches: result.append( (tag_name, (), match.start(), match.start("text"))) result.append( (tag_name, (), match.end("text"), match.end())) # Find "<url>" links (gray out). matches = re.finditer(LINK_ALT, text) for match in matches: result.append( (self.TAG_NAME_GRAY_TEXT, (), match.start("text"), match.end("text"))) # Find "---" horizontal rule (center). matches = re.finditer(HORIZONTAL_RULE, text) for match in matches: result.append( (self.TAG_NAME_CENTER, (), match.start("symbols"), match.end("symbols"))) # Find "* list" (offset). matches = re.finditer(LIST, text) for match in matches: # Lists use character+space (eg. "* "). length = 2 nest = len(match.group("indent").replace(" ", "\t")) margin = -length - 2 * nest indent = -length - 2 * length * nest result.append((self.TAG_NAME_MARGIN_INDENT, (margin, indent), match.start("content"), match.end("content"))) # Find "1. ordered list" (offset). matches = re.finditer(ORDERED_LIST, text) for match in matches: # Ordered lists use numbers/letters+dot/parens+space (eg. "123. "). length = len(match.group("prefix")) + 1 nest = len(match.group("indent").replace(" ", "\t")) margin = -length - 2 * nest indent = -length - 2 * length * nest result.append((self.TAG_NAME_MARGIN_INDENT, (margin, indent), match.start("content"), match.end("content"))) # Find "> blockquote" (offset). matches = re.finditer(BLOCK_QUOTE, text) for match in matches: result.append((self.TAG_NAME_MARGIN_INDENT, (2, -2), match.start(), match.end())) # Find "# Header" (offset+bold). matches = re.finditer(HEADER, text) for match in matches: margin = -len(match.group("level")) - 1 result.append((self.TAG_NAME_MARGIN_INDENT, (margin, 0), match.start(), match.end())) result.append( (self.TAG_NAME_BOLD, (), match.start(), match.end())) # Find "=======" header underline (bold). matches = re.finditer(HEADER_UNDER, text) for match in matches: result.append( (self.TAG_NAME_BOLD, (), match.start(), match.end())) # Find "```" code block tag (offset + colorize paragraph). matches = re.finditer(markup_regex.CODE_BLOCK, text) for match in matches: result.append((self.TAG_NAME_CODE_BLOCK, (), match.start("block"), match.end("block"))) # Send parsed data back. child_conn.send((text, result)) def on_parsed(self, _source, _condition): """Reads the parsing result from the pipe and triggers any pending apply.""" self.parsing = False if self.apply_pending: self.apply() # self.apply clears the apply pending flag. try: if self.parent_conn.poll(): self.do_apply(*self.parent_conn.recv()) return True except EOFError: return False def do_apply(self, original_text, result=[]): """Applies the result of parsing if the current text matches the original text.""" buffer = self.text_buffer start = buffer.get_start_iter() end = buffer.get_end_iter() text = self.text_buffer.get_slice(start, end, False) # Apply markup tags. if text == original_text and text != self.marked_up_text: buffer.remove_tag(self.tag_italic, start, end) buffer.remove_tag(self.tag_bold, start, end) buffer.remove_tag(self.tag_bold_italic, start, end) buffer.remove_tag(self.tag_strikethrough, start, end) buffer.remove_tag(self.tag_center, start, end) buffer.remove_tag(self.tag_plain_text, start, end) buffer.remove_tag(self.tag_gray_text, start, end) buffer.remove_tag(self.tag_code_text, start, end) buffer.remove_tag(self.tag_code_block, start, end) buffer.remove_tag(self.tag_wrap_none, start, end) for tag in self.tags_margins_indents.values(): buffer.remove_tag(tag, start, end) for tag_name, tag_args, tag_start, tag_end in result: buffer.apply_tag(self.tags_markup[tag_name](tag_args), buffer.get_iter_at_offset(tag_start), buffer.get_iter_at_offset(tag_end)) # Apply focus mode tag (grey out before/after current sentence). buffer.remove_tag(self.tag_unfocused_text, start, end) if self.text_view.focus_mode: cursor_iter = buffer.get_iter_at_mark(buffer.get_insert()) start_sentence = cursor_iter.copy() start_sentence.backward_sentence_start() end_sentence = cursor_iter.copy() end_sentence.forward_sentence_end() buffer.apply_tag(self.tag_unfocused_text, start, start_sentence) buffer.apply_tag(self.tag_unfocused_text, end_sentence, end) # Margin and indent are cumulative. They differ in two ways: # * Margin is always in the beginning, which means it effectively only affects the first line # of multi-line text. Indent is applied to every line. # * Margin level can be negative, as a baseline margin exists from which it can be subtracted. # Indent is always positive, or 0. def get_margin_indent_tag(self, margin_level, indent_level): level = (margin_level, indent_level) if level not in self.tags_margins_indents: margin, indent = self.get_margin_indent(margin_level, indent_level) tag = self.text_buffer.create_tag("margin_indent_{}_{}".format( margin_level, indent_level), left_margin=margin, indent=indent) self.tags_margins_indents[level] = tag return tag else: return self.tags_margins_indents[level] def get_margin_indent(self, margin_level, indent_level, baseline_margin=None, char_width=None): if baseline_margin is None: baseline_margin = self.text_view.props.left_margin if char_width is None: char_width = helpers.get_char_width(self.text_view) margin = max(baseline_margin + char_width * margin_level, 0) indent = char_width * indent_level return margin, indent def update_margins_indents(self): baseline_margin = self.text_view.props.left_margin char_width = helpers.get_char_width(self.text_view) # Bail out if neither the baseline margin nor character width change if baseline_margin == self.baseline_margin and char_width == self.char_width: return self.baseline_margin = baseline_margin self.char_width = char_width # Adjust tab size tab_array = Pango.TabArray.new(1, True) tab_array.set_tab(0, Pango.TabAlign.LEFT, 4 * char_width) self.text_view.set_tabs(tab_array) # Adjust margins and indents for level, tag in self.tags_margins_indents.items(): margin, indent = self.get_margin_indent(*level, baseline_margin, char_width) tag.set_properties(left_margin=margin, indent=indent) def stop(self, *_): self.parent_conn.close()
def run_forked(suite): keep_alive_parent_end, keep_alive_child_end = Pipe(duplex=False) result_parent_end, result_child_end = Pipe(duplex=False) failed_parent_end, failed_child_end = Pipe(duplex=False) child = Process(target=test_runner_wrapper, args=(suite, keep_alive_child_end, result_child_end, failed_child_end)) child.start() last_test_temp_dir = None last_test_vpp_binary = None last_test = None result = None failed = set() last_heard = time.time() core_detected_at = None debug_core = os.getenv("DEBUG", "").lower() == "core" while True: readable = select.select([ keep_alive_parent_end.fileno(), result_parent_end.fileno(), failed_parent_end.fileno(), ], [], [], 1)[0] if result_parent_end.fileno() in readable: result = result_parent_end.recv() break if keep_alive_parent_end.fileno() in readable: while keep_alive_parent_end.poll(): last_test, last_test_vpp_binary,\ last_test_temp_dir, vpp_pid = keep_alive_parent_end.recv() last_heard = time.time() if failed_parent_end.fileno() in readable: while failed_parent_end.poll(): failed_test = failed_parent_end.recv() failed.add(failed_test.__name__) last_heard = time.time() fail = False if last_heard + test_timeout < time.time() and \ not os.path.isfile("%s/_core_handled" % last_test_temp_dir): fail = True global_logger.critical("Timeout while waiting for child test " "runner process (last test running was " "`%s' in `%s')!" % (last_test, last_test_temp_dir)) elif not child.is_alive(): fail = True global_logger.critical("Child python process unexpectedly died " "(last test running was `%s' in `%s')!" % (last_test, last_test_temp_dir)) elif last_test_temp_dir and last_test_vpp_binary: core_path = "%s/core" % last_test_temp_dir if os.path.isfile(core_path): if core_detected_at is None: core_detected_at = time.time() elif core_detected_at + core_timeout < time.time(): if not os.path.isfile( "%s/_core_handled" % last_test_temp_dir): global_logger.critical( "Child python process unresponsive and core-file " "exists in test temporary directory!") fail = True if fail: failed_dir = os.getenv('VPP_TEST_FAILED_DIR') lttd = last_test_temp_dir.split("/")[-1] link_path = '%s%s-FAILED' % (failed_dir, lttd) global_logger.error("Creating a link to the failed " + "test: %s -> %s" % (link_path, lttd)) try: os.symlink(last_test_temp_dir, link_path) except Exception: pass api_post_mortem_path = "/tmp/api_post_mortem.%d" % vpp_pid if os.path.isfile(api_post_mortem_path): global_logger.error("Copying api_post_mortem.%d to %s" % (vpp_pid, last_test_temp_dir)) shutil.copy2(api_post_mortem_path, last_test_temp_dir) if last_test_temp_dir and last_test_vpp_binary: core_path = "%s/core" % last_test_temp_dir if os.path.isfile(core_path): global_logger.error("Core-file exists in test temporary " "directory: %s!" % core_path) check_core_path(global_logger, core_path) global_logger.debug("Running `file %s':" % core_path) try: info = check_output(["file", core_path]) global_logger.debug(info) except CalledProcessError as e: global_logger.error( "Could not run `file' utility on core-file, " "rc=%s" % e.returncode) pass if debug_core: spawn_gdb(last_test_vpp_binary, core_path, global_logger) child.terminate() result = -1 break keep_alive_parent_end.close() result_parent_end.close() failed_parent_end.close() return result, failed
def test_fork(self): """Test using a connection before and after a fork. """ if sys.platform == "win32": raise SkipTest("Can't fork on Windows") try: from multiprocessing import Process, Pipe except ImportError: raise SkipTest("No multiprocessing module") db = self._get_connection().pymongo_test # Failure occurs if the connection is used before the fork db.test.find_one() #db.connection.end_request() def loop(pipe): while True: try: db.test.insert({"a": "b"}, safe=True) for _ in db.test.find(): pass except: traceback.print_exc() pipe.send(True) os._exit(1) cp1, cc1 = Pipe() cp2, cc2 = Pipe() p1 = Process(target=loop, args=(cc1, )) p2 = Process(target=loop, args=(cc2, )) p1.start() p2.start() p1.join(1) p2.join(1) p1.terminate() p2.terminate() p1.join() p2.join() cc1.close() cc2.close() # recv will only have data if the subprocess failed try: cp1.recv() self.fail() except EOFError: pass try: cp2.recv() self.fail() except EOFError: pass db.connection.close()
class mygui: def __init__(self): self.window = Tk() self.stop = Button(self.window, text="Stop Recording", command=self.stopclick) self.record = Button(self.window, text='Record', command=self.recordclick) self.terminate = Button(self.window, text="Exit", command=self.exitclick) self.input = Entry(self.window) self.stop['state'] = 'disabled' #processes self.training = None self.running = None #communication pipes self.trainingpipe = None self.runningpipe = None self.runningpipe, childpipe = Pipe() self.running = Process(target=mainloop, args=(childpipe, )) self.running.start() def pipelistener(self): print('running pipelistener') if self.runningpipe.poll(0.001) is not None: signal = self.runningpipe.recv() print(signal) def buildgui(self): self.record.place(x=1, y=50) self.input.place(x=1, y=75) self.stop.place(x=1, y=100) self.terminate.place(x=1, y=125) self.window.mainloop() def learnclick(self): print('learnclick') self.runningpipe.send('LEARN') def exitclick(self): print('stopclick') self.stop['state'] = 'disabled' self.record['state'] = 'normal' #other actions... if self.running is not None and self.running.exitcode is None: self.runningpipe.send('STOP') self.running.join() self.window.quit() def stopclick(self): print('stoprecording') self.stop['state'] = 'disabled' self.record['state'] = 'normal' self.runningpipe.send('STOPCACHE') print('STOPCACHE') print('Attempting to Learn') self.runningpipe.send('LEARN') def recordclick(self): print('recordclick') #other actions lbl = self.input.get() if lbl is None or len(lbl) == 0: return self.stop['state'] = 'normal' self.record['state'] = 'disabled' print('sending pipe') self.runningpipe.send('label:' + lbl)
)) agent = Process(target=agent_main, args=(0, info_pipe_agent, action_pipe_agent, input_agent, need_agent)) server.start() time.sleep(1) agent.start() if use_keyboard: actions_list = [] in_action = True while True: try: if in_action: x = need_main.recv() print('receive from client:', x) in_action = False if len(actions_list) == 0: if agent_action_type == 'continues': if continues_action_type == 'xy': actions = input( 'enter action or actions like \'-0.001 0.002\' or \'0.001 0.001 0.001 -0.001\':' ) elif continues_action_type == 'ar': actions = input( 'enter action or actions like \'92 0.002\' or \'-175 0.001 80 -0.001\':' ) if len(actions) > 0: actions = actions.split(' ')
from multiprocessing import Process, Pipe import os, time #创建管道 fd2, fd1 = Pipe(False) def fun(name): time.sleep(3) #向管道写入内容 fd1.send("hello" + str(name)) #传什么都可以 jobs = [] for i in range(5): p = Process(target=fun, args=(i, )) jobs.append(p) p.start() for i in range(5): #读取管道 data = fd2.recv() print(data) for i in jobs: i.join()
def run_forked(suite): keep_alive_parent_end, keep_alive_child_end = Pipe(duplex=False) result_parent_end, result_child_end = Pipe(duplex=False) failed_parent_end, failed_child_end = Pipe(duplex=False) child = Process(target=test_runner_wrapper, args=(suite, keep_alive_child_end, result_child_end, failed_child_end)) child.start() last_test_temp_dir = None last_test_vpp_binary = None last_test = None result = None failed = set() last_heard = time.time() while True: readable = select.select([ keep_alive_parent_end.fileno(), result_parent_end.fileno(), failed_parent_end.fileno(), ], [], [], 1)[0] if result_parent_end.fileno() in readable: result = result_parent_end.recv() break if keep_alive_parent_end.fileno() in readable: while keep_alive_parent_end.poll(): last_test, last_test_vpp_binary,\ last_test_temp_dir, vpp_pid = keep_alive_parent_end.recv() last_heard = time.time() if failed_parent_end.fileno() in readable: while failed_parent_end.poll(): failed_test = failed_parent_end.recv() failed.add(failed_test.__name__) last_heard = time.time() fail = False if last_heard + test_timeout < time.time(): fail = True global_logger.critical("Timeout while waiting for child test " "runner process (last test running was " "`%s' in `%s')!" % (last_test, last_test_temp_dir)) elif not child.is_alive(): fail = True global_logger.critical("Child process unexpectedly died (last " "test running was `%s' in `%s')!" % (last_test, last_test_temp_dir)) if fail: failed_dir = os.getenv('VPP_TEST_FAILED_DIR') lttd = last_test_temp_dir.split("/")[-1] link_path = '%s%s-FAILED' % (failed_dir, lttd) global_logger.error("Creating a link to the failed " + "test: %s -> %s" % (link_path, lttd)) os.symlink(last_test_temp_dir, link_path) api_post_mortem_path = "/tmp/api_post_mortem.%d" % vpp_pid if os.path.isfile(api_post_mortem_path): global_logger.error("Copying api_post_mortem.%d to %s" % (vpp_pid, last_test_temp_dir)) shutil.copy2(api_post_mortem_path, last_test_temp_dir) if last_test_temp_dir and last_test_vpp_binary: core_path = "%s/core" % last_test_temp_dir if os.path.isfile(core_path): global_logger.error("Core-file exists in test temporary " "directory: %s!" % core_path) if d and d.lower() == "core": spawn_gdb(last_test_vpp_binary, core_path, global_logger) child.terminate() result = -1 break keep_alive_parent_end.close() result_parent_end.close() failed_parent_end.close() return result, failed
def on_any_event(event): print("Watcher: Event found: {}".format(event.event_type)) logging.debug("Watcher: Event found: {}".format(event.event_type)) logging.debug("Directory: {}".format(os.path.basename(event.src_path))) if event.is_directory: return None # Windows is created # Linux is modified elif event.event_type == 'modified' or event.event_type == 'created': logging.debug( '-------------------------------------------------------------------' ) time.sleep(1) recv_end, send_end = Pipe(False) # Take any action here when a file is first created. print( "Received event {} for file {} - Beginning render job.".format( event.event_type, event.src_path)) logging.debug( "Received event {} for file {} - Beginning render job.".format( event.event_type, event.src_path)) # Testing print f = os.path.relpath(event.src_path, Watcher.DIRECTORY_TO_WATCH) pid, rend_type, other = f.split("_", 2) logging.debug("Project ID: {}".format(pid)) logging.debug("Render Type: {}".format(rend_type)) print(other) logging.debug("File found: {}".format(f)) logging.debug("Source path: {}".format(event.src_path)) logging.debug("Render type: {}".format(rend_type)) if rend_type == 'chunk': # Open the file for reading json_file = open(event.src_path, 'r') json_data = json.load(json_file) # If job hasn't been done if json_data['status'] == False: # Get the ID and run the render on it proj_id = json_data["id"] print("Project ID is {}".format(proj_id)) logging.debug("Project ID is {}".format(proj_id)) logging.debug("Starting render serivce...") compress = False chunk = True logging.debug( "Chunk render instance for {} requested".format( proj_id)) p = Process(target=chunk_render.get_chunk, args=( proj_id, send_end, compress, chunk, )) p.start() p.join() render_return = recv_end.recv() render_status = render_return[1] info = render_return[0] # Update the complete time at the end and dump it to file logging.debug("Updating JSON status file") json_data['dateCompleted'] = datetime.now().strftime( "%d-%b-%Y (%H:%M:%S)") json_data['status'] = True json_data['correctlyRendered'] = render_status json_data['otherInfo'] = info logging.debug("JSON Data for {}:".format(proj_id)) logging.debug(json_data) with open(event.src_path, "w") as json_write: json.dump(json_data, json_write) if render_status == 1: logging.debug("File written successfully.") print("File written successfully") return else: logging.debug( "Job completed, but an error may have occured during render service" ) return else: logging.debug("File already rendered") print("File already rendered") return else: print("[watcher_chunk.py] different render type") logging.debug("[watcher_chunk.py] different render type")
def main(use_multiprocessing=False, log_level=None): """ Create and run a tello controller : 1) get the video stream from the tello 2) wait for keyboard commands to pilot the tello 3) optionnally, process the video frames to track a body and pilot the tello accordingly. If use_multiprocessing is True, the parent process creates a child process ('worker') and the workload is shared between the 2 processes. The parent process job is to: - get the video stream from the tello and displays it in an OpenCV window, - write each frame in shared memory at destination of the child, each frame replacing the previous one (more efficient than a pipe or a queue), - read potential command from the child (currently, only one command:EXIT). Commands are transmitted by a Pipe. The child process is responsible of all the others tasks: - process the frames read in shared memory (openpose, write_hud), - if enable, do the tracking (calculate drone commands from position of body), - read keyboard commands, - transmit commands (from tracking or from keyboard) to the tello, and receive message from the tello. """ global tello if use_multiprocessing: # Create the pipe for the communication between the 2 processes parent_cnx, child_cnx = Pipe() else: child_cnx = None tello = TelloController(use_face_tracking=True, kbd_layout="AZERTY", write_log_data=False, log_level=log_level, child_cnx=child_cnx) first_frame = True frame_skip = 300 for frame in tello.container.decode(video=0): if 0 < frame_skip: frame_skip = frame_skip - 1 continue start_time = time.time() if frame.time_base < 1.0 / 60: time_base = 1.0 / 60 else: time_base = frame.time_base # Convert frame to cv2 image frame = cv2.cvtColor(np.array(frame.to_image(), dtype=np.uint8), cv2.COLOR_RGB2BGR) frame = cv2.resize(frame, (640, 480)) if use_multiprocessing: if first_frame: # Create the shared memory to share the current frame decoded by the parent process # and given to the child process for further processing (openpose, write_hud,...) frame_as_ctypes = np.ctypeslib.as_ctypes(frame) tello.shared_array = sharedctypes.RawArray( frame_as_ctypes._type_, frame_as_ctypes) tello.frame_shape = frame.shape first_frame = False # Launch process child p_worker = Process(target=openpose_worker) p_worker.start() # Write the current frame in shared memory tello.shared_array[:] = np.ctypeslib.as_ctypes(frame.copy()) # Check if there is some message from the child if parent_cnx.poll(): msg = parent_cnx.recv() if msg == "EXIT": print("MAIN EXIT") p_worker.join() tello.drone.quit() cv2.destroyAllWindows() exit(0) else: frame = tello.process_frame(frame) tello.sound_player.play() if not use_multiprocessing: tello.fps.update() # Display the frame cv2.imshow('Tello', frame) cv2.waitKey(1) frame_skip = int((time.time() - start_time) / time_base)