def tempControlProcTest(mode, cycle_time, duty_cycle, set_point, k_param, i_param, d_param, conn):
p = current_process()
print 'Starting:', p.name, p.pid
parent_conn_temp, child_conn_temp = Pipe()
ptemp = Process(name = "getrandProc", target=getrandProc, args=(child_conn_temp,))
#ptemp.daemon = True
ptemp.start()
parent_conn_heat, child_conn_heat = Pipe()
pheat = Process(name = "heatProctest", target=heatProctest, args=(cycle_time, duty_cycle, child_conn_heat))
#pheat.daemon = True
pheat.start()
while (True):
if parent_conn_temp.poll():
randnum = parent_conn_temp.recv() #non blocking receive
conn.send([randnum, mode, cycle_time, duty_cycle, set_point, k_param, i_param, d_param])
if parent_conn_heat.poll():
cycle_time, duty_cycle = parent_conn_heat.recv()
#duty_cycle = on_time/offtime*100.0
#cycle_time = on_time + off_time
if conn.poll():
mode, cycle_time, duty_cycle, set_point, k_param, i_param, d_param = conn.recv()
#conn.send([mode, cycle_time, duty_cycle])
#if mode == "manual":
parent_conn_heat.send([cycle_time, duty_cycle])
def test_fork(self):
"""Test using a connection before and after a fork.
"""
if sys.platform == "win32":
raise SkipTest()
try:
from multiprocessing import Process, Pipe
except ImportError:
raise SkipTest()
db = Connection(self.host, self.port).pymongo_test
# Failure occurs if the connection is used before the fork
db.test.find_one()
db.connection.end_request()
def loop(pipe):
while True:
try:
db.test.insert({"a": "b"}, safe=True)
for _ in db.test.find():
pass
except:
pipe.send(True)
os._exit(1)
cp1, cc1 = Pipe()
cp2, cc2 = Pipe()
p1 = Process(target=loop, args=(cc1,))
p2 = Process(target=loop, args=(cc2,))
p1.start()
p2.start()
p1.join(1)
p2.join(1)
p1.terminate()
p2.terminate()
p1.join()
p2.join()
cc1.close()
cc2.close()
# recv will only have data if the subprocess failed
try:
cp1.recv()
self.fail()
except EOFError:
pass
try:
cp2.recv()
self.fail()
except EOFError:
pass
def QuickSortMPListArray(A,conn,NumProcs):
print str(len(A))+' starting mplarray'
if len(A)<=1 :
conn.send(A)
conn.close()
elif int(NumProcs)<1:
print 'proc limit reached, smp qs'
conn.send(QuickSortListArray(A))
conn.close()
else:
pvalue=A[0]
lesser=[x for x in A if int(x[0]) < int(pvalue[0])]
greater=[x for x in A if int(x[0]) > int(pvalue[0])]
pv=[x for x in A if int(x[0]) == int(pvalue[0])]
Procs=int(NumProcs)-1
pConnLeft,cConnLeft=Pipe()
leftProc=Process(target=QuickSortMPListArray,args=(lesser,cConnLeft,Procs))
pConnRight,cConnRight=Pipe()
rightProc=Process(target=QuickSortMPListArray,args=(greater,cConnRight,Procs))
leftProc.start()
rightProc.start()
print 'mplarray send'
conn.send(pConnRight.recv()+pv+pConnLeft.recv())
# conn.send(pConnLeft.recv()+[PivotValue]+pConnRight.recv())
conn.close()
leftProc.join()
rightProc.join()
return
def QuickSortMP(A,conn,NumProcs):
if len(A)<=1 :
conn.send(A)
conn.close()
elif int(NumProcs)<1:
conn.send(QuickSort(A))
conn.close()
else:
PivotIndex=random.randint(0,len(A)-1)
PivotValue=A.pop(PivotIndex)
pv=[]
pv.append(int(PivotValue))
lesser=[int(x) for x in A if x < PivotValue]
greater=[int(x) for x in A if x >= PivotValue]
Procs=int(NumProcs)-1
pConnLeft,cConnLeft=Pipe()
leftProc=Process(target=QuickSortMP,args=(lesser,cConnLeft,Procs))
pConnRight,cConnRight=Pipe()
rightProc=Process(target=QuickSortMP,args=(greater,cConnRight,Procs))
leftProc.start()
rightProc.start()
leftStr=pConnLeft.recv()
rightStr=pConnRight.recv()
conn.send(leftStr+pv+rightStr)
# conn.send(pConnLeft.recv()+[PivotValue]+pConnRight.recv())
conn.close()
leftProc.join()
rightProc.join()
return
def process_pipe():
parent_conn, child_conn = Pipe()
p = Process(target=pipe_test, args=(child_conn,))
p.start()
print parent_conn.recv()
p.join()
parent_conn.close()
def startWith2Threads(trainData, labels, testData):
print("startWith4Threads started")
parent_conn1, child_conn1 = Pipe()
parent_conn2, child_conn2 = Pipe()
t1 = threading.Thread(target=RunAllFeatures, args = (trainData, labels, testData, child_conn1, 0, 20, ))
t2 = threading.Thread(target=RunAllFeatures, args = (trainData, labels, testData, child_conn2, 20, 39, ))
t1.start()
t2.start()
rr1 = parent_conn1.recv()
rr2 = parent_conn2.recv()
t1.join();
t2.join();
print("all tasks done")
rr1 = np.row_stack((rr1, rr2))
print("result combined")
res = np.transpose(rr1)
pp = Normalization(res)
print("normalization done")
return pp
def test_fork(self):
# Test using a client before and after a fork.
if sys.platform == "win32":
raise SkipTest("Can't fork on Windows")
try:
from multiprocessing import Process, Pipe
except ImportError:
raise SkipTest("No multiprocessing module")
db = self._get_client().pymongo_test
# Failure occurs if the client is used before the fork
db.test.find_one()
def loop(pipe):
while True:
try:
db.test.insert({"a": "b"})
for _ in db.test.find():
pass
except:
traceback.print_exc()
pipe.send(True)
os._exit(1)
cp1, cc1 = Pipe()
cp2, cc2 = Pipe()
p1 = Process(target=loop, args=(cc1,))
p2 = Process(target=loop, args=(cc2,))
p1.start()
p2.start()
p1.join(1)
p2.join(1)
p1.terminate()
p2.terminate()
p1.join()
p2.join()
cc1.close()
cc2.close()
# recv will only have data if the subprocess failed
try:
cp1.recv()
self.fail()
except EOFError:
pass
try:
cp2.recv()
self.fail()
except EOFError:
pass
db.connection.close()
def _process_image(self, image):
# Create pipes for process communication
std_size_pipe, std_size_child_end = Pipe()
thumb_pipe, thumb_child_end = Pipe()
# Create the threads
threads = [
Process(
target=self._thumbnail,
args=(image, thumb_child_end, (128, 128))
),
Process(
target=self._thumbnail,
args=(image, std_size_child_end, (1920, 1080))
)
]
# Start threads
for thread in threads:
thread.start()
# Get processed images content and close threads
images_bytes = (
thumb_pipe.recv(), std_size_pipe.recv()
)
for thread in threads:
thread.join()
for obj in images_bytes:
if type(obj) is IOError:
raise obj
return images_bytes
def run(self):
logging.info('Visualizer thread started')
parent_end, child_end = Pipe()
# Sensible default value for max_process
max_process = 2
process_count = 0
while not self.stop or not self.job_backlog.empty():
while parent_end.poll(0.1):
parent_end.recv() ## currently not using the info... irrelevant
## TODO - a signal to notify the viewer that visuzaliztion job has been finished...
#self.controller.view_update(self)
process_count -= 1
if self.job_backlog.empty():
time.sleep(1)
elif process_count < max_process:
process_count += 1
run_name, function, snapshot = self.job_backlog.get_nowait()
if not (run_name in self.remove_run_name):
logging.info('Added job to visuzalizer Que: ' + str(run_name))
logging.info('No. of jobs in Que: ' + str(process_count))
p = Process(target=self.render_graph,
args=(function, snapshot, run_name, child_end))
p.start()
logging.info('Visualizer Finished')
def QuickSortMPListArray(A,conn,NumProcs):
if len(A)<=1 :
conn.send(A)
conn.close()
elif int(NumProcs)<1:
conn.send(QuickSortListArray(A))
conn.close()
else:
lesser=[]
greater=[]
pv=[]
Pivot=A.pop(0)
pvVal=int(Pivot[0])
lesser=[x for x in A if x[0] < pvVal]
greater=[x for x in A if x[0] > pvVal]
pv=[x for x in A if x[0] == pvVal]
pv.append(Pivot)
Procs=int(NumProcs)-1
pConnLeft,cConnLeft=Pipe()
leftProc=Process(target=QuickSortMPListArray,args=(lesser,cConnLeft,Procs))
pConnRight,cConnRight=Pipe()
rightProc=Process(target=QuickSortMPListArray,args=(greater,cConnRight,Procs))
leftProc.start()
rightProc.start()
conn.send(pConnLeft.recv()+pv+pConnRight.recv())
conn.close()
leftProc.join()
rightProc.join()
return
def onExeBtn(self, event):
srcFiles = self.projPane.getValue()
self.analyzerOutDir = os.path.join(self.projRootDir, 'analyzer_output')
oldcwd = os.getcwd()
if not os.path.exists(self.analyzerOutDir):
os.makedirs(self.analyzerOutDir)
os.chdir(self.analyzerOutDir)
rcv_pipe, snd_pipe = Pipe(duplex=True)
self.dbname = ''.join([self.projRootDir.replace(os.sep, '_').strip('_'), '.sqlite'])
self.exePane.dbPathTc.SetValue(os.path.join(self.analyzerOutDir, self.dbname))
self.exePane.ppLogFileTc.SetValue(os.path.join(self.analyzerOutDir, 'preprocessor.log'))
self.exePane.parserLogFileTc.SetValue(os.path.join(self.analyzerOutDir, 'parser.log'))
p = Process(target=analyze,
args=(snd_pipe,
os.path.join(self.analyzerOutDir, self.dbname),
self.getPpCfg(),
self.getParserCfg(),
srcFiles,
self.exePane.pipelineCb.GetValue(),
self.exePane.numProcSc.GetValue(),
self.exePane.numPpProcSc.GetValue(),
self.exePane.numParserProcSc.GetValue(),
))
p.start()
dlg = wx.ProgressDialog('Executing',
'0/%d' % len(srcFiles),
parent=self,
maximum=len(srcFiles)*10,
style=wx.PD_CAN_ABORT |
wx.PD_APP_MODAL |
wx.PD_ELAPSED_TIME
)
dlg.SetSize((500,150))
dlg.Layout()
dlg.Show()
result = None
while True:
i, total, result = rcv_pipe.recv()
ret = dlg.Update(i*10, result if i == total else '[%d/%d] %s ... done' % (i+1, total, result))
if ret[0] == False:
rcv_pipe.send('STOP')
while result != 'STOPPED':
result = rcv_pipe.recv()
dlg.Update(total*10, 'Canceled')
break
if i == total:
break
p.join()
self.exePane.dbPathTc.SetValue(os.path.join(self.analyzerOutDir, self.dbname))
os.chdir(oldcwd)
def quicksort(self,arr, conn, procNum):
if procNum <= 0 or len(arr)<= 1:
conn.send(self.qck(arr))
conn.close()
return
#print 'Just in case you don't trust that this program works better than other quicksorts :3 FUBAR. process id:', os.getppid()
pivot = arr.pop(random.randint(0, len(arr)-1))
leftSide = [x for x in arr if x < pivot]
rightSide = [x for x in arr if x > pivot]
pconnLeft, cconnLeft = Pipe()
leftProc = Process(target= self.quicksort, args=(leftSide, cconnLeft,procNum -1))
pconnRight, cconnRight = Pipe()
rightProc = Process(target=self.quicksort, args=(rightSide, cconnRight, procNum - 1))
leftProc.start()
rightProc.start()
conn.send(pconnLeft.recv() + [pivot] + pconnRight.recv())
conn.close()
leftProc.join()
rightProc.join()
class MMLWorker(object):
"""Represents the Daemon's connection to the subprocess"""
def __init__(self,runner):
"""Creates and initalizes a subprocess and its connections."""
self.pipe, pipe = Pipe()
self.proc = Process(target=worker, args=(pipe,runner))
self.proc.start();
self.pid = self.proc.pid
def __del__(self):
"""Ensures the subprocess is correctly garbage collected."""
self.pipe.close();
self.proc.terminate();
def pump(self,block=False):
"""Returns a key,val pair from the subprocess, or None,None."""
key,val = None,None
if block:
(key,val) = self.pipe.recv()
elif self.pipe.poll():
(key,val) = self.pipe.recv()
return key,val
def stop(self):
"""Sends the stop signal to the subprocess."""
self.pipe.send(('stop',()))
def pause(self):
"""Sends the pause signal to the subprocess."""
self.pipe.send(('pause',()))
def start(self,program):
"""Sends the start signal to the subprocess."""
self.pipe.send(('start',(program,)))
def smpScore(M1,M2,M3,D1SCORE,D2SCORE,D3SCORE,D4SCORE): st=time.time() pconn,cconn=Pipe() qconn,dconn=Pipe() rconn,econn=Pipe() nLIST=[] p=Process(target=smpScoFunction,args=(M1,cconn,D1SCORE,D2SCORE,D3SCORE,D4SCORE)) q=Process(target=smpScoFunction,args=(M2,dconn,D1SCORE,D2SCORE,D3SCORE,D4SCORE)) r=Process(target=smpScoFunction,args=(M3,econn,D1SCORE,D2SCORE,D3SCORE,D4SCORE)) p.start() q.start() r.start() p1LIST=pconn.recv() p2LIST=qconn.recv() p3LIST=rconn.recv() p.join() q.join() r.join() nLIST=p1LIST+p2LIST+p3LIST et=time.time() tt=et-st print 'smp scoring done in '+str(tt)+' seconds.' return nLIST
def usage3():
# The Pipe() function returns a pair of connection objects
# connected by a pipe which by default is duplex (two-way)
parent_conn, child_conn = Pipe()
child_conn.send('wenychan3')
child_conn.close()
print parent_conn.recv() # prints "[42, None, 'hello']"
def main():
parent_conn, child_conn = Pipe()
p = Process(target = f, args=(child_conn,))
p.start()
# if you comment this line you can get the result after join
print(parent_conn.recv())
p.join()
print('Hello: ', parent_conn.recv())
def test_pool_with_fork(self):
# Test that separate MongoClients have separate Pools, and that the
# driver can create a new MongoClient after forking
if sys.platform == "win32":
raise SkipTest("Can't test forking on Windows")
try:
from multiprocessing import Process, Pipe
except ImportError:
raise SkipTest("No multiprocessing module")
a = self.get_client(auto_start_request=False)
a.pymongo_test.test.remove()
a.pymongo_test.test.insert({'_id':1})
a.pymongo_test.test.find_one()
self.assertEqual(1, len(get_pool(a).sockets))
a_sock = one(get_pool(a).sockets)
def loop(pipe):
c = self.get_client(auto_start_request=False)
self.assertEqual(1,len(get_pool(c).sockets))
c.pymongo_test.test.find_one()
self.assertEqual(1,len(get_pool(c).sockets))
pipe.send(one(get_pool(c).sockets).sock.getsockname())
cp1, cc1 = Pipe()
cp2, cc2 = Pipe()
p1 = Process(target=loop, args=(cc1,))
p2 = Process(target=loop, args=(cc2,))
p1.start()
p2.start()
p1.join(1)
p2.join(1)
p1.terminate()
p2.terminate()
p1.join()
p2.join()
cc1.close()
cc2.close()
b_sock = cp1.recv()
c_sock = cp2.recv()
self.assertTrue(a_sock.sock.getsockname() != b_sock)
self.assertTrue(a_sock.sock.getsockname() != c_sock)
self.assertTrue(b_sock != c_sock)
# a_sock, created by parent process, is still in the pool
d_sock = get_pool(a).get_socket()
self.assertEqual(a_sock, d_sock)
d_sock.close()
def test_mount_fstab_user_fail(self):
if not self._can_create:
self.skipTest('Cannot create %s filesystem' % self._fs_name)
if not self._can_mount:
self.skipTest('Cannot mount %s filesystem' % self._fs_name)
# this test will change /etc/fstab, we might want to revert the changes after it finishes
fstab = self.read_file('/etc/fstab')
self.addCleanup(self.write_file, '/etc/fstab', fstab)
disk = self.get_object('/block_devices/' + os.path.basename(self.vdevs[0]))
self.assertIsNotNone(disk)
# create filesystem
disk.Format(self._fs_name, self.no_options, dbus_interface=self.iface_prefix + '.Block')
self.addCleanup(self._clean_format, disk)
# create user for our test
self.addCleanup(self._remove_user, self.username)
uid, gid = self._add_user(self.username)
# add unmount cleanup now in case something wrong happens in the other process
self.addCleanup(self._unmount, self.vdevs[0])
# create pipe to get error (if any)
parent_conn, child_conn = Pipe()
proc = Process(target=self._mount_as_user_fstab_fail, args=(child_conn, int(uid), int(gid), self.vdevs[0]))
proc.start()
res = parent_conn.recv()
parent_conn.close()
proc.join()
if not res[0]:
self.fail(res[1])
# now mount it as root and test that user can't unmount it
mnt_path = disk.Mount(self.no_options, dbus_interface=self.iface_prefix + '.Filesystem')
self.assertIsNotNone(mnt_path)
self.assertTrue(os.path.ismount(mnt_path))
# create pipe to get error (if any)
parent_conn, child_conn = Pipe()
proc = Process(target=self._unmount_as_user_fstab_fail, args=(child_conn, int(uid), int(gid), self.vdevs[0]))
proc.start()
res = parent_conn.recv()
parent_conn.close()
proc.join()
if not res[0]:
self.fail(res[1])
self.assertTrue(os.path.ismount(mnt_path))
self._unmount(mnt_path)
def setUpClass(cls):
parent_conn, child_conn = Pipe()
proc = Process(target=cls._start_groonga, args=(child_conn,))
proc.daemon = True
proc.start()
cls.serverpid = parent_conn.recv()
proc.join(1)
# 生存戦略
if not proc.is_alive():
raise RuntimeError(parent_conn.recv())
def Compare(Original_DB, support, Sens_f, selection, fNames):
time = ['']*len(selection)
s_ef = [0]*len(selection)
ch_raw_dat = [0]*len(selection)
fil = [0]*len(selection)
rev_t = ['']*len(selection)
CPU_time = 0
if any(selection):
out_fname = os.getcwd()+'\\Apriori_init.txt'
parent, child = Pipe()
p = Process(target=aprioriWorker, args=(Original_DB, support, out_fname, child,))
p.start()
response = parent.recv()
parent.close()
p.join()
p = None
parent = None
child = None
if response == -1:
return(response)
elif type(response) == type(tuple):
return(response)
CPU_time = response
for index in [i for i in xrange(len(selection)) if selection[i]]:
parent, child = Pipe()
p = Process(target=functionWorker, args=(fNames[index], out_fname, Sens_f, Original_DB, support, child,))
p.start()
response = parent.recv()
parent.close()
p.join()
if response == -1:
return(response)
elif type(response) == type(tuple) and len(response) == 2:
return((response[0], fNames[index]))
print(response)
rev_t[index], ch_raw_dat[index], time[index] = response
parent, child = Pipe()
p = Process(target=metricWorker, args=(out_fname, fNames[index]+"_results.txt", Sens_f, support, child,))
p.start()
s_ef[index], fil[index]= parent.recv()
parent.close()
p.join()
p = None
parent = None
child = None
gc.collect()
return(s_ef, ch_raw_dat, time, rev_t, fil, CPU_time)
def test_invoke_runs_another_thread(self):
q = Queue()
parent, child = Pipe()
def task():
q.put(current_process().name)
time.sleep(1)
child.send(1)
sut = BackgroundTask(task)
sut.invoke()
parent.recv()
self.assertNotEquals(q.get(), current_process().name)
def test_pool_with_fork(self):
# Test that separate Connections have separate Pools, and that the
# driver can create a new Connection after forking
if sys.platform == "win32":
raise SkipTest("Can't test forking on Windows")
try:
from multiprocessing import Process, Pipe
except ImportError:
raise SkipTest("No multiprocessing module")
a = self.get_connection(auto_start_request=False)
a.test.test.remove(safe=True)
a.test.test.insert({'_id':1}, safe=True)
a.test.test.find_one()
self.assertEqual(1, len(a._Connection__pool.sockets))
a_sock = one(a._Connection__pool.sockets)
def loop(pipe):
c = self.get_connection(auto_start_request=False)
self.assertEqual(1,len(c._Connection__pool.sockets))
c.test.test.find_one()
self.assertEqual(1,len(c._Connection__pool.sockets))
pipe.send(one(c._Connection__pool.sockets).sock.getsockname())
cp1, cc1 = Pipe()
cp2, cc2 = Pipe()
p1 = Process(target=loop, args=(cc1,))
p2 = Process(target=loop, args=(cc2,))
p1.start()
p2.start()
p1.join(1)
p2.join(1)
p1.terminate()
p2.terminate()
p1.join()
p2.join()
cc1.close()
cc2.close()
b_sock = cp1.recv()
c_sock = cp2.recv()
self.assertTrue(a_sock.sock.getsockname() != b_sock)
self.assertTrue(a_sock.sock.getsockname() != c_sock)
self.assertTrue(b_sock != c_sock)
self.assertEqual(a_sock,
a._Connection__pool.get_socket((a.host, a.port)))
def test_pool_with_fork(self):
if sys.platform == "win32":
raise SkipTest()
try:
from multiprocessing import Process, Pipe
except ImportError:
raise SkipTest()
a = get_connection()
a.test.test.find_one()
a.end_request()
self.assertEqual(1, len(a._Connection__pool.sockets))
a_sock = a._Connection__pool.sockets[0]
def loop(pipe):
c = get_connection()
self.assertEqual(1, len(c._Connection__pool.sockets))
c.test.test.find_one()
self.assertEqual(0, len(c._Connection__pool.sockets))
c.end_request()
self.assertEqual(1, len(c._Connection__pool.sockets))
pipe.send(c._Connection__pool.sockets[0].getsockname())
cp1, cc1 = Pipe()
cp2, cc2 = Pipe()
p1 = Process(target=loop, args=(cc1,))
p2 = Process(target=loop, args=(cc2,))
p1.start()
p2.start()
p1.join(1)
p2.join(1)
p1.terminate()
p2.terminate()
p1.join()
p2.join()
cc1.close()
cc2.close()
b_sock = cp1.recv()
c_sock = cp2.recv()
self.assert_(a_sock.getsockname() != b_sock)
self.assert_(a_sock.getsockname() != c_sock)
self.assert_(b_sock != c_sock)
self.assertEqual(a_sock,
a._Connection__pool.get_socket(a.host, a.port)[0])
def callThisFunction(): ## Create a multiprocessing pip connection ## It has a parent and child connection object ## Pass the child object to receive messages back ## parent_connection, child_connection = Pipe() ## Create the Process ## p = Process(target=printMe, args=(child_connection,'bob',)) p.start() print 'The parent knows the childs PID:' print parent_connection.recv() #prints the PID of the process p.join()
def test_sending(self):
server_conn, worker_conn = Pipe()
a, b = Pipe()
process = Process(target=echo, args=(worker_conn,))
process.start()
server_conn.send(SharedFile('test.txt', b))
self.assertEqual(('open', {'mode': 'r', 'filename': 'test.txt'}), a.recv())
a.send('a')
self.assertEqual('a', a.recv())
process.terminate()
def get_ff(user):
assert (type(user) == str)
parent_friends_pi, friends_pi = Pipe()
parent_followers_pi, followers_pi = Pipe()
friend_pr = Process(target = get_friends_parallel, args=(friends_pi, user))
follower_pr = Process(target = get_followers_parallel, args=(followers_pi, user))
#Get friends and followers.
friend_pr.start()
follower_pr.start()
friends = parent_friends_pi.recv()
followers = parent_followers_pi.recv()
friend_pr.join()
follower_pr.join()
return friends, followers
def main():
server_channel, client_channel = Pipe()
my_process = MyProcess(client_channel)
my_process.start()
server_channel.send(('add', 4))
print(server_channel.recv())
server_channel.send(('add', 12))
print(server_channel.recv())
server_channel.send(('method', None))
print(("compiled" in server_channel.recv()) == ("compiled" in repr(MyProcess.run)))
server_channel.send(('close', 0))
def fn_with_timeout(*args, **kwargs):
conn1, conn2 = Pipe()
kwargs['_conn'] = conn2
th = Process(target=fn, args=args, kwargs=kwargs)
th.start()
if conn1.poll(self.trial_timeout):
fn_rval = conn1.recv()
th.join()
else:
print 'TERMINATING DUE TO TIMEOUT'
th.terminate()
th.join()
fn_rval = 'return', {
'status': hyperopt.STATUS_FAIL,
'failure': 'TimeOut'
}
assert fn_rval[0] in ('raise', 'return')
if fn_rval[0] == 'raise':
raise fn_rval[1]
# -- remove potentially large objects from the rval
# so that the Trials() object below stays small
# We can recompute them if necessary, and it's usually
# not necessary at all.
if fn_rval[1]['status'] == hyperopt.STATUS_OK:
fn_loss = float(fn_rval[1].get('loss'))
fn_preprocs = fn_rval[1].pop('preprocs')
fn_classif = fn_rval[1].pop('classifier')
if fn_loss < self._best_loss:
self._best_preprocs = fn_preprocs
self._best_classif = fn_classif
self._best_loss = fn_loss
return fn_rval[1]
def run(self):
logging.info('Visualizer thread started')
parent_end, child_end = Pipe()
# Sensible default value for max_process
max_process = 3
process_count = 0
while not self.stop or not self.job_backlog.empty():
while parent_end.poll(0.1):
(name, counter) = parent_end.recv()
self.controller.find_trial(name).set_counter_plot(counter)
process_count -= 1
if self.job_backlog.empty():
time.sleep(1)
elif process_count < max_process:
process_count += 1
function, snapshot, trial = self.job_backlog.get_nowait()
logging.info('Visualizing {}'.format(trial.get_name()))
p = Process(target=self.render_graph,
args=(function, snapshot, trial.get_name(),
child_end))
p.start()
self.job_backlog.task_done()
logging.info('Visualizer Finished')
def run(self):
parent_pipe, child_pipe = Pipe(False)
p = Process(target = run_metaheuristic,
args = (child_pipe, self.model, self.pt, self.aa,
self.algo, self.n, self.use_heur,
self.worst, self.best))
p.start()
for i in range(self.n + 1):
if self.is_stopped() is True:
parent_pipe.close()
p.terminate()
return
try:
result = parent_pipe.recv()
except:
break
self.results.append(result[0])
self.fitness.append(result[1])
self.emit(QtCore.SIGNAL('update(int)'), i)
if result[1] == 1:
break
parent_pipe.close()
p.join()
def test_mount_fstab_user_fail(self):
if not self._can_create:
self.skipTest('Cannot create %s filesystem' % self._fs_name)
if not self._can_mount:
self.skipTest('Cannot mount %s filesystem' % self._fs_name)
# this test will change /etc/fstab, we might want to revert the changes after it finishes
fstab = self.read_file('/etc/fstab')
self.addCleanup(self.write_file, '/etc/fstab', fstab)
disk = self.get_object('/block_devices/' +
os.path.basename(self.vdevs[0]))
self.assertIsNotNone(disk)
# create filesystem
disk.Format(self._fs_name,
self.no_options,
dbus_interface=self.iface_prefix + '.Block')
self.addCleanup(self._clean_format, disk)
# create user for our test
self.addCleanup(self._remove_user)
uid, gid = self._add_user()
# add unmount cleanup now in case something wrong happens in the other process
self.addCleanup(self._unmount, self.vdevs[0])
# create pipe to get error (if any)
parent_conn, child_conn = Pipe()
proc = Process(target=self._mount_as_user_fstab_fail,
args=(child_conn, int(uid), int(gid), self.vdevs[0]))
proc.start()
res = parent_conn.recv()
parent_conn.close()
proc.join()
if not res[0]:
self.fail(res[1])
# now mount it as root and test that user can't unmount it
mnt_path = disk.Mount(self.no_options,
dbus_interface=self.iface_prefix + '.Filesystem')
self.assertIsNotNone(mnt_path)
self.assertTrue(os.path.ismount(mnt_path))
# create pipe to get error (if any)
parent_conn, child_conn = Pipe()
proc = Process(target=self._unmount_as_user_fstab_fail,
args=(child_conn, int(uid), int(gid), self.vdevs[0]))
proc.start()
res = parent_conn.recv()
parent_conn.close()
proc.join()
if not res[0]:
self.fail(res[1])
self.assertTrue(os.path.ismount(mnt_path))
self._unmount(mnt_path)
class Submarine:
def __init__(self, SaveSensorData):
# Create variables to hold the current position
self._roll = 0
self._pitch = 0
self._yaw = 0
self._YPos = 0
self._angleFirstRead = True
# Create variables to hold depth
self._depth = 0
# Create variables to control the message passing between processes
self.IMUParnetConn, self.IMUChildConn = Pipe()
self.depthParnetConn, self.depthChildConn = Pipe()
# Make the message board
self._messageBoard = MessageBoard()
# Create the object
self._SubMotors = SubMotors()
# Create the IMU object
self.IMU = SubIMU(self._messageBoard._IMUFile, self.IMUChildConn,
SaveSensorData)
# Check if we were able to connect to the IMU
if (self.IMU == None):
# Don't bother updating the angles we don't have an IMU
self._upateAngles = False
else:
self._upateAngles = True
# Create the depth sensor object
self.depthSensor = SubDepthSensor(self._messageBoard._DepthFile,
self.depthChildConn, SaveSensorData)
# Check if we were able to connect to the depth sensor
if (self.depthSensor == None):
# Don't bother updating the depth we don't have a sensor
self._upateDepth = False
else:
self._upateDepth = True
# Create the Maintain Forward object
self._maintainForward = MaintainForward()
# Create the Maintain Depth object
self._maintainDepth = MaintainDepth()
# Create the dive object
self._goToDepth = GoToDepth()
# Create a clockwise turn object
self._clockWiseTurn = ClockWiseTurn()
# Create a counterclockwise turn object
self._counterClockWiseTurn = CounterClockWiseTurn()
# Create the timing variables
self._rateDiff = 25
# Create a method to leave
self._exit = False
# This method checks the IMU connection and update values
def UpdateAngles(self):
# Check if the IMU has posted a message
if (self.IMUParnetConn.poll() or self._angleFirstRead):
message = self.IMUParnetConn.recv()
self._angleFirstRead = False
if (str(message).find("[Log]") != -1):
self._messageBoard.LogMessage(message)
else:
self._roll = message[0]
self._pitch = message[1]
self._yaw = message[2]
self._YPos = message[3]
# This method checks the IMU connection and update values
def UpdateDepth(self):
# Check if the Depth Sensor has posted a message
if (self.depthParnetConn.poll()):
message = self.depthParnetConn.recv()
if (str(message).find("[Log]") != -1):
self._messageBoard.LogMessage(message)
else:
self._depth = message
def UpdateMotorSpeed(self, Packet):
if (Packet == None or len(Packet) < 5):
return
if (Packet[0] != None):
# if( abs(self._SubMotors._leftSpeed - Packet[0]) > SmallChange):
self._leftSpeed = Packet[0]
self._SubMotors.SetLeftSpeed(Packet[0], Packet[1])
if (Packet[2] != None):
# if( abs(self._SubMotors._rightSpeed - Packet[2]) > SmallChange):
self._rightSpeed = Packet[2]
self._SubMotors.SetRightSpeed(Packet[2], Packet[3])
if (Packet[4] != None):
# if( abs(self._SubMotors._servoAngle - Packet[4]) > SmallChange):
self._servoAngle = Packet[4]
self._SubMotors.SetServoAngle(Packet[4], Packet[5])
def UpdateSubState(self):
if (self._upateAngles):
self.UpdateAngles()
if (self._upateDepth):
self.UpdateDepth()
# This will maintain a trajectory
def Forward(self, length):
self.UpdateSubState()
counter = 0
startTime = time()
# Capture the state of the submarine
self._maintainForward.CaptureState(
[self._roll, self._pitch, self._yaw, self._YPos], self._depth)
self._maintainDepth.CaptureState(
[self._roll, self._pitch, self._yaw, self._YPos], self._depth)
while (time() - startTime < length):
# Update the submarine state
self.UpdateSubState()
# Update the counter
counter = counter + 1
# Send message if enough time has passed
if (counter == self._rateDiff):
# This will correct for an X-Y Variation
self.UpdateMotorSpeed(
self._maintainForward.UpdateState(
[self._roll, self._pitch, self._yaw, self._YPos],
self._depth))
# This will correct for an Z Variation
self.UpdateMotorSpeed(
self._maintainDepth.UpdateState(
[self._roll, self._pitch, self._yaw, self._YPos],
self._depth))
# Reset the counter
counter = 0
def ChangeDepth(self, depth):
self.UpdateSubState()
counter = 0
startTime = time()
# Capture the state of the submarine
self._maintainForward.CaptureState(
[self._roll, self._pitch, self._yaw, self._YPos], self._depth)
self._goToDepth.CaptureState(self._depth, depth)
# Set a flag to notify us when we reach our desired depth
reachedDepth = False
while (not reachedDepth):
# Update the submarine state
self.UpdateSubState()
# Update the counter
counter = counter + 1
# Send message if enough time has passed
if (counter == self._rateDiff):
# This will correct for an X-Y Variation
self.UpdateMotorSpeed(
self._maintainForward.UpdateState(
[self._roll, self._pitch, self._yaw, self._YPos],
self._depth))
# Set the angle of attack so we can dive
response = self._goToDepth.UpdateState(self._depth)
if (response != False):
self.UpdateMotorSpeed(response)
else:
reachedDepth = True
# Reset the counter
counter = 0
def ClockWiseTurn(self, angle):
self.UpdateSubState()
counter = 0
StopAngle = self._yaw + angle
while (abs(StopAngle - self._yaw) > 1):
# Update the submarine state
self.UpdateSubState()
# Update the counter
counter = counter + 1
# Send message if enough time has passed
if (counter == self._rateDiff):
if (not self._clockWiseTurn.StateCaptured):
self._clockWiseTurn.CaptureState(
[self._roll, self._pitch, self._yaw], angle)
self.UpdateMotorSpeed(
self._clockWiseTurn.UpdateState(
[self._roll, self._pitch, self._yaw]))
counter = 0
def CounterClockWiseTurn(self, angle):
self.UpdateSubState()
counter = 0
StopAngle = self._yaw - angle
while (abs(StopAngle - self._yaw) > 1):
# Update the submarine state
self.UpdateSubState()
# Update the counter
counter = counter + 1
# Send message if enough time has passed
if (counter == self._rateDiff):
if (not self._counterClockWiseTurn.StateCaptured):
self._counterClockWiseTurn.CaptureState(
[self._roll, self._pitch, self._yaw], angle)
self.UpdateMotorSpeed(
self._counterClockWiseTurn.UpdateState(
[self._roll, self._pitch, self._yaw]))
counter = 0
def ShutDown(self):
# Tell the IMU process to exits
print("Here f**k # 2")
self.IMUParnetConn.send("Close")
self.depthParnetConn.send("Close")
self._messageBoard.CloseBoard()
#listen mode
s.listen(5)
print ("socket is listening")
dataObj = bytes(8)
#accept
conn,addr = s.accept()
msg = "connected"
print (msg)
#For starting the stream set this to 1
vp.start()
interuptProcess.start()
cPid = parent_conn.recv()
print("Child = {}".format(cPid))
while True:
parent_conn.send(1)
p_conn.send(1)
dataObj = conn.recv(4096)
decodeData = dataObj#.decode(encoding='utf-8', errors='strict')
if (not decodeData): break
data = ''
print (decodeData)
if(len(decodeData) > 4):
data = (decodeData[3:4]) + (decodeData[4:5])
if (decodeData == b't\x00\x0211'):
class ProcessEngine(BaseEngine):
"""An engine executing the tasks it is sent in a different process.
"""
def perform(self, exec_infos):
"""Execute a given task.
Parameters
----------
exec_infos : ExecutionInfos
TaskInfos object describing the work to expected of the engine.
Returns
-------
exec_infos : ExecutionInfos
Input object whose values have been updated. This is simply a
convenience.
Notes
-----
IOError in pipe are raised only if an operation is attempted from the
process that closed the pipe, but never when trying to poll from a
different process.
"""
self.status = 'Running'
# Clear all the flags.
self._task_pause.clear()
self._task_paused.clear()
self._task_resumed.clear()
self._task_stop.clear()
self._force_stop.clear()
self._stop_requested = False
# If the process does not exist or is dead create a new one.
if not self._process or not self._process.is_alive():
self._process_stop.clear()
# Create the subprocess and the pipe.
self._pipe, process_pipe = Pipe()
self._process = TaskProcess(process_pipe, self._log_queue,
self._monitor_queue, self._task_pause,
self._task_paused, self._task_resumed,
self._task_stop, self._process_stop)
self._process.daemon = True
# Create the logger thread in charge of dispatching log reports.
self._log_thread = QueueLoggerThread(self._log_queue)
self._log_thread.daemon = True
logger.debug('Starting logging thread.')
self._log_thread.start()
# Create the monitor thread dispatching engine news to the monitor.
self._monitor_thread = ThreadMeasureMonitor(
self, self._monitor_queue)
self._monitor_thread.daemon = True
logger.debug('Starting monitoring thread.')
self._monitor_thread.start()
self._pause_thread = None
# Start process.
logger.debug('Starting subprocess')
self._process.start()
# Send the measure.
args = self._build_subprocess_args(exec_infos)
try:
self._pipe.send(args)
except Exception:
msg = ('Failed to send infos to subprocess :\n-infos : \n%s\n'
'-errors :\n%s')
logger.error(msg % (pformat(args), format_exc()))
self._log_queue.put(None)
self._monitor_queue.put((None, None))
self._cleanup(process=True)
exec_infos.success = False
exec_infos.errors['engine'] = msg
self.status = 'Stopped'
return exec_infos
else:
logger.debug('Task {} sent'.format(exec_infos.id))
# Check that the engine did receive the task.
while not self._pipe.poll(2):
if not self._process.is_alive():
msg = 'Subprocess was found dead unexpectedly'
logger.debug(msg)
self._log_queue.put(None)
self._monitor_queue.put((None, None))
self._cleanup(process=False)
exec_infos.success = False
exec_infos.errors['engine'] = msg
self.status = 'Stopped'
return exec_infos
# Simply empty the pipe the subprocess always send True if it answers
self._pipe.recv()
# Wait for the process to finish the measure and check it has not
# been killed.
while not self._pipe.poll(1):
if self._force_stop.is_set():
msg = 'Subprocess was terminated by the user.'
logger.debug(msg)
self._cleanup(process=False)
exec_infos.errors['engine'] = msg
self.status = 'Stopped'
return exec_infos
elif not self._process.is_alive():
msg = 'Subprocess was found dead unexpectedly'
logger.debug(msg)
self._log_queue.put(None)
self._monitor_queue.put((None, None))
self._cleanup(process=False)
exec_infos.success = False
exec_infos.errors['engine'] = msg
self.status = 'Stopped'
return exec_infos
# Here get message from process and react
result, errors = self._pipe.recv()
logger.debug('Subprocess done performing measure')
exec_infos.success = result
exec_infos.errors.update(errors)
self.status = 'Waiting'
return exec_infos
def pause(self):
"""Ask the engine to pause the current task execution.
"""
self.status = 'Pausing'
self._task_resumed.clear()
self._task_paused.clear()
self._task_pause.set()
self._pause_thread = Thread(target=self._wait_for_pause)
self._pause_thread.start()
def resume(self):
"""Ask the engine to resume the currently paused job.
"""
self.status = 'Resuming'
self._task_pause.clear()
def stop(self, force=False):
"""Ask the engine to stop the current job.
This method should not wait for the job to stop save if a forced stop
was requested.
Parameters
----------
force : bool, optional
Force the engine to stop the performing the task. This allow the
engine to use any means necessary to stop, in this case only should
the call to this method block.
"""
self.status = 'Stopping'
self._stop_requested = True
self._task_stop.set()
if force:
self._force_stop.set()
# Stop running queues
self._log_queue.put(None)
self._monitor_queue.put((None, None))
# Terminate the process and make sure all threads stopped properly.
self._process.terminate()
self._log_thread.join()
self._monitor_thread.join()
# Discard the queues as they may have been corrupted when the
# process was terminated.
self._log_queue = Queue()
self._monitor_queue = Queue()
self.status = 'Stopped'
def shutdown(self, force=False):
"""Ask the engine to stop completely.
Parameters
----------
force : bool, optional
Force the engine to stop the performing the task. This allow the
engine to use any means necessary to stop, in this case only should
the call to this method block.
"""
self.status = 'Shutting down'
self._stop_requested = True
self._task_stop.set()
if not force:
t = Thread(target=self._cleanup)
t.start()
else:
self.stop(force=True)
# =========================================================================
# --- Private API ---------------------------------------------------------
# =========================================================================
#: Boolean indicating that the user requested the job to stop.
_stop_requested = Bool()
#: Interprocess event used to pause the subprocess current job.
_task_pause = Value(factory=Event)
#: Interprocess event signaling the subprocess current job is paused.
_task_paused = Value(factory=Event)
#: Interprocess event signaling the subprocess current job has resumed.
_task_resumed = Value(factory=Event)
#: Interprocess event used to stop the subprocess current measure.
_task_stop = Value(factory=Event)
#: Interprocess event used to stop the subprocess.
_process_stop = Value(factory=Event)
#: Flag signaling that a forced exit has been requested
_force_stop = Value(factory=tEvent)
#: Current subprocess.
_process = Typed(TaskProcess)
#: Connection used to send and receive messages about execution (type
#: ambiguous when the OS is not known)
_pipe = Value()
#: Inter-process queue used by the subprocess to transmit its log records.
_log_queue = Value(factory=Queue)
#: Thread in charge of collecting the log message coming from the
#: subprocess.
_log_thread = Typed(Thread)
#: Inter-process queue used by the subprocess to send the values of the
#: observed database entries.
_monitor_queue = Value(factory=Queue)
#: Thread in charge of collecting the values of the observed database
#: entries.
_monitor_thread = Typed(Thread)
#: Thread in charge of notifying the engine that the engine did
#: pause/resume after being asked to do so.
_pause_thread = Typed(Thread)
def _cleanup(self, process=True):
""" Helper method taking care of making sure that everybody stops.
Parameters
----------
process : bool
Whether to join the worker process. Used when the process has been
termintaed abruptly.
"""
logger.debug('Cleaning up')
if process and self._process:
self._process_stop.set()
self._process.join()
logger.debug('Subprocess joined')
if self._pipe:
self._pipe.close()
if self._log_thread:
self._log_thread.join()
logger.debug('Log thread joined')
if self._monitor_thread:
self._monitor_thread.join()
logger.debug('Monitor thread joined')
if self._pause_thread:
self._pause_thread.join()
logger.debug('Pause thread joined')
self.status = 'Stopped'
def _build_subprocess_args(self, exec_infos):
"""Build the tuple to send to the subprocess.
"""
exec_infos.task.update_preferences_from_members()
config = exec_infos.task.preferences
database_root_state = exec_infos.task.database.copy_node_values()
return (exec_infos.id, config, exec_infos.build_deps,
exec_infos.runtime_deps, exec_infos.observed_entries,
database_root_state, exec_infos.checks)
def _wait_for_pause(self):
""" Wait for the _task_paused event to be set.
"""
stop_sig = self._task_stop
paused_sig = self._task_paused
while not stop_sig.is_set():
if paused_sig.wait(0.1):
self.status = 'Paused'
break
resuming_sig = self._task_resumed
while not stop_sig.is_set():
if resuming_sig.wait(1):
self.status = 'Running'
break
def main(sub_p):
global starttime
# 创建一个“钩子”管理对象
hm = pyHook.HookManager()
# 监听所有键盘事件
onActiveEvent = lambda sub_p=sub_p: onActiveEvent(sub_p=sub_p)
hm.KeyDown = hm.MouseAll = onActiveEvent
# 设置键盘“钩子”
hm.HookKeyboard()
# 监听所有鼠标事件
# hm.MouseAll = onMouseEvent
# 设置鼠标“钩子”
hm.HookMouse()
# 进入循环，如不手动关闭，程序将一直处于监听状态
pythoncom.PumpMessages(10000)
if __name__ == "__main__":
sup_p, sub_p = Pipe()
p = Process(target=main, args=(sub_p, ))
p.start()
while 1:
if sup_p.recv():
starttime = 0
sleep(1)
starttime += 1
print(starttime)
if starttime > 20:
ScreenSaver(10)
import multiprocessing as mp
def teste(q):
t = nquad(lambda x, y: x**2 + x + y**2 + y,
[[-np.inf, np.inf], [-np.inf, np.inf]])
q.put(t)
q = mp.Queue()
p = mp.Process(target=teste, args=(q, ))
p.start()
print(q.get())
p.join()
#%%
from multiprocessing import Process, Pipe
def f(conn):
conn.send([42, None, 'hello'])
conn.close()
if __name__ == '__main__':
parent_conn, child_conn = Pipe()
p = Process(target=f, args=(child_conn, ))
p.start()
print parent_conn.recv() # prints "[42, None, 'hello']"
p.join()
#%%
sys.exit(10)
class LabEnvironment(environment.Environment):
ACTION_LIST = [
_action(-20, 0, 0, 0, 0, 0, 0), # look_left
_action(20, 0, 0, 0, 0, 0, 0), # look_right
#_action( 0, 10, 0, 0, 0, 0, 0), # look_up
#_action( 0, -10, 0, 0, 0, 0, 0), # look_down
_action(0, 0, -1, 0, 0, 0, 0), # strafe_left
_action(0, 0, 1, 0, 0, 0, 0), # strafe_right
_action(0, 0, 0, 1, 0, 0, 0), # forward
_action(0, 0, 0, -1, 0, 0, 0), # backward
#_action( 0, 0, 0, 0, 1, 0, 0), # fire
#_action( 0, 0, 0, 0, 0, 1, 0), # jump
#_action( 0, 0, 0, 0, 0, 0, 1) # crouch
]
@staticmethod
def get_action_size(env_name):
return len(LabEnvironment.ACTION_LIST)
def __init__(self, env_name):
environment.Environment.__init__(self)
self.conn, child_conn = Pipe()
self.proc = Process(target=worker, args=(child_conn, env_name))
self.proc.start()
self.conn.recv()
self.reset()
def reset(self):
self.conn.send([COMMAND_RESET, 0])
obs = self.conn.recv()
self.last_state = self._preprocess_frame(obs)
self.last_action = 0
self.last_reward = 0
def stop(self):
self.conn.send([COMMAND_TERMINATE, 0])
ret = self.conn.recv()
self.conn.close()
self.proc.join()
print("lab environment stopped")
def _preprocess_frame(self, image):
image = image.astype(np.float32)
image = image / 255.0
return image
def process(self, action):
real_action = LabEnvironment.ACTION_LIST[action]
self.conn.send([COMMAND_ACTION, real_action])
obs, reward, terminal = self.conn.recv()
if not terminal:
state = self._preprocess_frame(obs)
else:
state = self.last_state
pixel_change = self._calc_pixel_change(state, self.last_state)
self.last_state = state
self.last_action = action
self.last_reward = reward
return state, reward, terminal, pixel_change
def main():
parent_conn, child_conn = Pipe()
proc = Process(target=f, args=(child_conn, ))
proc.start()
print(parent_conn.recv())
proc.join()
TXN = 0
DXN = 0
# !! end declaring global values !!
#world generation
with open("./output files/worldtypes") as wF:
FCt = wF.read()
#create save data with the mapGen script
if ("a" not in FCt):
#intitalizes the lists we look through for PXN's tile type
worldgen()
if __name__ == '__main__':
reciever, sender = Pipe(False)
p = Process(target=worldgen.fullgen, args=(sender,))
p.start()
wat = reciever.recv()
bun = reciever.recv()
rou = reciever.recv()
fai = reciever.recv()
tow = reciever.recv()
dun = reciever.recv()
towN = reciever.recv()
dunN = reciever.recv()
else:
#read from saved data
with open("./output files/savedworld") as sD:
sD.seek(0)
strAll = sD.readline()
listAll = strAll.split(";")
wat = []
bun = []
def treater(pipe_in: Pipe, queue_ctrl: Queue, func: callable = None):
"""
:param pipe_in: Pipe # 消息管道 （MSG, ）
:param queue_ctrl: Queue # 控制 ("is_active",):(bool,)
:param func: callable # 自定义处理过程
:return: None
"""
def treat_msg(msg_orig):
# --------- encryption ------------------------------------------
if encryption is None:
msg_encryption = msg_orig
else:
msg_encryption = encryption.request(msg_orig)
logging.debug("TEMPLE::TREATER::{0}::ENCRYPT request:{1}".format(
msg_orig.encryption, msg_encryption))
# --------- treatment -------------------------------------------
if treatment is None:
msg_treatment = msg_encryption
else:
msg_treatment = treatment.request(msg_encryption)
logging.debug("TEMPLE::TREATER::{0}::TREAT request:{1}".format(
msg_orig.treatment, msg_treatment))
del msg_encryption
# --------- function --------------------------------------------
if func is None:
msg_func = msg_treatment
else:
msg_func = func(msg_treatment)
logging.debug("TEMPLE::TREATER func:{0}".format(msg_func))
del msg_treatment
# --------- destination -----------------------------------------
if destination is None:
logging.info(
"TEMPLE::TREATER::NONE::DEST return:{0}".format(msg_func))
else:
if isinstance(destination, MQ):
destination.request(msg_func)
logging.info("TEMPLE::TREATER::{0}::DEST request:{1}".format(
msg_orig.destination, msg_func))
else:
destination.write(msg_func)
logging.info("TEMPLE::TREATER::{0}::DEST write:{1}".format(
msg_orig.destination, msg_func))
return msg_func
is_active = queue_ctrl.get()
while is_active:
if queue_ctrl.empty():
try:
msg_treater = pipe_in.recv()
logging.info(
"TEMPLE::TREATER PIPE IN recv:{0}".format(msg_treater))
except EOFError:
is_active = False
continue
origination = allocator(msg_treater.origination) # Data.read()
treatment = allocator(msg_treater.treatment) # MessageQueue
encryption = allocator(msg_treater.encryption) # MessageQueue
destination = allocator(msg_treater.destination) # Data.write()
# --------- origination -----------------------------------------
if isinstance(origination, MQ):
origination.reply(func=treat_msg)
else:
msg_origination = origination.read(msg_treater)
logging.info("TEMPLE::TREATER::{0}::ORIG read:{1}".format(
msg_treater.origination, msg_origination))
treat_msg(msg_origination)
else:
is_active = queue_ctrl.get()
else:
while not queue_ctrl.empty():
queue_ctrl.get()
class TaskExecutionControl(Atom):
"""Store measurement in a queue of measures to be processed, take care of
starting and communicating with the process performing the measure and
handling user action (stopping single measure or whole process).
Attributes
----------
running : bool
Bool indicating whether or not the measurement process is running.
task_stop : multiprocess.Event
Event used to signal that the current measure should be stopped.
process_stop : multiprocess.Event
Event used to signal that the measurement process should be stopped.
meas_holder : list(instance(Measure))
List containing all the enqueued measure.
process : instance(Process)
Measurement process.
log_thread : instance(Thread)
Thread dedicated to handling the log records coming from the measurement
process
log_queue : multiprocessing queue
Queue in which the log records of the measurement process are sent from
the measurement process to the main process.
monitor_queue : multiprocessing queue
Queue in which the value of the monitored parameters are sent from the
measurement process to the main process.
current_monitor : instance(MeasureMonitor)
Monitor associated to the measurement being processed.
pipe : multiprocessing double ended pipe
Pipe used for communication between the two processus.
Methods
-------
append_task(new_task):
Method handling the enqueuing of a measurement in the queue.
"""
running = Bool(False)
task_stop = Instance(Event, ())
process_stop = Instance(Event, ())
meas_holder = ContainerList(Instance(Measure), [])
process = Instance(Process)
log_thread = Instance(Thread)
log_queue = Instance(Queue, ())
pipe = Value(
) #Instance of Connection but ambiguous when the OS is not known
monitor_queue = Instance(Queue, ())
current_monitor = Instance(MeasureMonitor)
monitor_display = Instance(MonitorView, ())
def append_meas(self, new_meas):
"""Put a measure in the queue if it pass the tests.
First the check method of the measure is called. If the tests pass,
the user is asked to give a name to its measure (by default the name is
'Meas"i"' where is the index of the measure in the queue), then he can
choose what entries he wants to monitor during the measure which is then
enqueued and finally saved in the default folder ('default_path'
attributes of the `RootTask` describing the measure). Otherwise the list
of the failed tests is displayed to the user.
Parameters
----------
new_task : instance(`RootTask`)
Instance of `RootTask` representing the measure.
Returns
-------
bool :
True is the measure was successfully enqueued, False otherwise.
"""
check = new_meas.root_task.check(test_instr=not self.running)
if check[0]:
path = os.path.join(
new_meas.root_task.default_path,
new_meas.monitor.measure_name + '_last_run.ini')
new_meas.save_measure(path)
meas = Measure()
meas.load_measure(path)
self.meas_holder.append(meas)
return True
else:
TaskCheckDisplay(model=TaskCheckModel(check[1])).exec_()
return False
def _start_button_clicked(self):
"""Handle the `start_button` being pressed.
Clear the event `task_stop` and `process_stop`, create the pipe and
the measurement process. Start then the log thread and then the process.
Finally start the thread handling the communication with the measurement
process.
"""
print 'Starting process'
self.task_stop.clear()
self.process_stop.clear()
self.pipe, process_pipe = Pipe()
self.process = TaskProcess(process_pipe, self.log_queue,
self.monitor_queue, self.task_stop,
self.process_stop)
self.log_thread = QueueLoggerThread(self.log_queue)
self.log_thread.daemon = True
self.log_thread.start()
self.process.start()
self.running = True
Thread(group=None, target=self._process_listerner).start()
def _stop_all_button_clicked(self):
"""Handle the `stop_button` being pressed.
Set the `process_stop` event and signal in the pipe that not more
measurement will be sent. Then wait for the process and the log thread
to terminate.
"""
print 'Stopping process'
self.process_stop.set()
self.pipe.send((None, 'STOP', None))
self.task_stop.set()
self.process.join()
self.log_thread.join()
self.running = False
def _stop_button_clicked(self):
"""Handle the `stop_task_button` being pressed by setting the
`task_stop` event.
"""
print 'Stopping task'
self.task_stop.set()
def _update_monitor_display_model(self, monitor):
"""
"""
self.monitor_display.monitor = monitor
if not self.monitor_display.visible:
self.monitor_display.show()
def _process_listerner(self):
"""Method called in a separated thread to handle communications with the
measurement process.
"""
print 'Starting listener'
meas = None
while not self.process_stop.is_set():
self.pipe.poll(None)
mess = self.pipe.recv()
print 'Message received : {}'.format(mess)
if mess == 'Need task':
if self.meas_holder:
i = 0
task = None
# Look for a measure not being currently edited.
while i < len(self.meas_holder):
aux = self.meas_holder[i]
if aux.monitor.status == 'EDITING' or aux == meas:
i += 1
continue
else:
meas = self.meas_holder[i]
task = meas.root_task
monitor = meas.monitor
name = meas.monitor.measure_name
break
# If one is found, stop the old monitor, if necessary start
# a new one and send the measure in the pipe.
if task is not None:
meas.is_running = True
task.update_preferences_from_members()
if self.current_monitor:
self.current_monitor.stop()
self.current_monitor = None
self.current_monitor = monitor
deferred_call(setattr, monitor, 'status', 'RUNNING')
# Leave a chance to the system to update the display
sleep(0.1)
if meas.use_monitor:
monitor.start(self.monitor_queue)
deferred_call(self._update_monitor_display_model,
monitor)
self.pipe.send(
(name, task.task_preferences,
self.current_monitor.database_values.keys()))
else:
deferred_call(self.monitor_display.close)
self.pipe.send((name, task.task_preferences, None))
print 'Measurement sent'
# If there is no measurement which can be sent, stop the
# measurement process.
else:
self.process_stop.set()
print 'The only task is the queue is being edited'
self.pipe.send(('', 'STOP', ''))
self.pipe.poll(None)
self.pipe.close()
self.process.join()
self.log_thread.join()
self.running = False
break
# If there is no measurement in the queue, stop the
# measurement process.
else:
self.process_stop.set()
print 'All measurements have been sent'
self.pipe.send((None, 'STOP', None))
self.pipe.poll(None)
self.pipe.close()
self.process.join()
self.log_thread.join()
self.running = False
break
elif mess == 'Task processed':
deferred_call(self.meas_holder.remove, meas)
sleep(0.1)
# If the measurement process sent a different message,
# it means it will stop so we can clean up.
else:
self.pipe.close()
self.process.join()
self.log_thread.join()
self.running = False
break
# Upon exit close the monitor if it is still opened.
if self.current_monitor:
deferred_call(setattr, self.current_monitor, 'status', 'STOPPED')
self.current_monitor.stop()
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from multiprocessing import Process, Pipe
def f(conn):
conn.send([42, None, 'hello'])
conn.close()
if __name__ == '__main__':
parent_conn, child_conn = Pipe()
p = Process(target=f, args=(child_conn, ))
# send前にrecvを実行すると待機状態になるのでプロセスが停止する
#print parent_conn.recv()
p.start()
print parent_conn.recv()
p.join()
class UI:
LOAD_TAB = 0
ALIGN_TAB = 1
STACK_TAB = 2
SHARPEN_TAB = 3
TITLE = "AstraStack"
VERSION = "2.0.0"
def __init__(self):
self.parentConn, self.childConn = Pipe(duplex=True)
self.pids = []
self.newVersionUrl = ""
self.video = None
self.align = None
self.stack = None
self.sharpen = None
self.mousePosition = None
self.clickedDriftP1 = False
self.clickedDriftP2 = False
self.clickedAreaOfInterest = False
self.builder = Gtk.Builder()
self.builder.add_from_file("ui/ui.glade")
self.window = self.builder.get_object("mainWindow")
self.saveDialog = self.builder.get_object("saveDialog")
self.openDialog = self.builder.get_object("openDialog")
self.tabs = self.builder.get_object("tabs")
self.cpus = self.builder.get_object("cpus")
self.progressBox = self.builder.get_object("progressBox")
self.progress = self.builder.get_object("progress")
self.frame = self.builder.get_object("frame")
self.overlay = self.builder.get_object("overlay")
self.frameSlider = self.builder.get_object("frameSlider")
self.frameScale = self.builder.get_object("frameScale")
self.startFrame = self.builder.get_object("startFrame")
self.endFrame = self.builder.get_object("endFrame")
self.normalize = self.builder.get_object("normalize")
self.alignChannels = self.builder.get_object("alignChannels")
self.autoCrop = self.builder.get_object("autoCrop")
self.transformation = self.builder.get_object("transformation")
self.drizzleFactor = self.builder.get_object("drizzleFactor")
self.drizzleInterpolation = self.builder.get_object(
"drizzleInterpolation")
self.limit = self.builder.get_object("limit")
self.limitPercent = self.builder.get_object("limitPercent")
self.averageRadio = self.builder.get_object("averageRadio")
self.medianRadio = self.builder.get_object("medianRadio")
self.openDialog.set_preview_widget(Gtk.Image())
self.saveDialog.set_preview_widget(Gtk.Image())
self.builder.get_object("alignTab").set_sensitive(False)
self.builder.get_object("stackTab").set_sensitive(False)
self.builder.get_object("processTab").set_sensitive(False)
self.builder.get_object("blackLevel").add_mark(0, Gtk.PositionType.TOP,
None)
self.builder.get_object("gamma").add_mark(100, Gtk.PositionType.TOP,
None)
self.builder.get_object("value").add_mark(100, Gtk.PositionType.TOP,
None)
self.builder.get_object("red").add_mark(100, Gtk.PositionType.TOP,
None)
self.builder.get_object("green").add_mark(100, Gtk.PositionType.TOP,
None)
self.builder.get_object("blue").add_mark(100, Gtk.PositionType.TOP,
None)
self.builder.get_object("saturation").add_mark(100,
Gtk.PositionType.TOP,
None)
self.disableScroll()
self.cpus.set_upper(min(
61, cpu_count())) # 61 is the maximum that Windows allows
self.cpus.set_value(min(61, math.ceil(cpu_count() / 2)))
g.pool = None
self.processThread = None
self.builder.connect_signals(self)
# Needed so it can be temporarily removed
self.limitPercentSignal = self.limitPercent.connect(
"value-changed", self.setLimitPercent)
g.driftP1 = (0, 0)
g.driftP2 = (0, 0)
g.areaOfInterestP1 = (0, 0)
g.areaOfInterestP2 = (0, 0)
self.window.show_all()
self.checkNewVersion()
self.setProgress()
self.setNormalize()
self.setAlignChannels()
self.setTransformation()
self.setDrizzleFactor()
self.setDrizzleInterpolation()
self.setAutoCrop()
self.setThreads()
self.frameScale.set_sensitive(False)
g.reference = "0"
# Cancels scroll event for widget
def propagateScroll(self, widget, event):
Gtk.propagate_event(widget.get_parent(), event)
# Disables the scroll event from some fields
def disableScroll(self):
mask = Gdk.EventMask.BUTTON_MOTION_MASK | Gdk.EventMask.BUTTON_PRESS_MASK | Gdk.EventMask.BUTTON_RELEASE_MASK | Gdk.EventMask.KEY_RELEASE_MASK | Gdk.EventMask.TOUCH_MASK
self.builder.get_object("denoiseWidget1").set_events(mask)
self.builder.get_object("denoiseWidget2").set_events(mask)
self.builder.get_object("denoiseWidget3").set_events(mask)
self.builder.get_object("denoiseWidget4").set_events(mask)
self.builder.get_object("denoiseWidget5").set_events(mask)
self.builder.get_object("radiusWidget1").set_events(mask)
self.builder.get_object("radiusWidget2").set_events(mask)
self.builder.get_object("radiusWidget3").set_events(mask)
self.builder.get_object("radiusWidget4").set_events(mask)
self.builder.get_object("radiusWidget5").set_events(mask)
# Sets up a listener so that processes can communicate with each other
def createListener(self, function):
def listener(function):
while True:
try:
msg = self.parentConn.recv()
except:
return False
if (msg == "stop"):
g.ui.setProgress()
return False
function(msg)
thread = Thread(target=listener, args=(function, ))
thread.start()
return thread
# Shows the error dialog with the given title and message
def showErrorDialog(self, message):
dialog = self.builder.get_object("errorDialog")
dialog.format_secondary_text(message)
response = dialog.run()
dialog.hide()
return response
# Shows the warning dialog with the given title and message
def showWarningDialog(self, message):
dialog = self.builder.get_object("warningDialog")
dialog.format_secondary_text(message)
response = dialog.run()
dialog.hide()
return response
# Opens the About dialog
def showAbout(self, *args):
dialog = self.builder.get_object("about")
dialog.set_program_name(UI.TITLE)
dialog.set_version(UI.VERSION)
response = dialog.run()
dialog.hide()
# Opens the user manual in the default pdf application
def showManual(self, *args):
if sys.platform.startswith('darwin'):
subprocess.call(('open', "manual/Manual.pdf"))
elif os.name == 'nt': # For Windows
os.startfile("manual\Manual.pdf")
elif os.name == 'posix': # For Linux, Mac, etc.
subprocess.call(('xdg-open', "manual/Manual.pdf"))
# Disable inputs
def disableUI(self):
self.builder.get_object("tabs").set_sensitive(False)
self.builder.get_object("toolbar").set_sensitive(False)
# Enable inputs
def enableUI(self):
self.builder.get_object("tabs").set_sensitive(True)
self.builder.get_object("toolbar").set_sensitive(True)
# The following is needed to forcibly refresh the value spacing of the slider
def fixFrameSliderBug(self):
self.frameScale.set_value_pos(Gtk.PositionType.RIGHT)
self.frameScale.set_value_pos(Gtk.PositionType.LEFT)
# Sets the number of threads to use
def setThreads(self, *args):
def initPool(method=None):
if (method == "spawn"):
GLib.idle_add(self.disableUI)
g.nThreads = int(self.cpus.get_value())
if (g.pool is not None):
g.pool.shutdown()
g.pool = ProcessPoolExecutor(g.nThreads)
# This seems like the most reliable way to get the pid of pool processes
self.pids = []
before = list(map(lambda p: p.pid, active_children()))
g.pool.submit(dummy, ()).result()
after = list(map(lambda p: p.pid, active_children()))
for pid in after:
if (pid not in before):
self.pids.append(pid)
if (method == "spawn"):
GLib.idle_add(self.enableUI)
# Behave a bit differently depending on platform
if (get_start_method() == "spawn"):
thread = Thread(target=initPool, args=(get_start_method(), ))
thread.start()
else:
initPool()
# Checks github to see if there is a new version available
def checkNewVersion(self):
def callUrl():
try:
contents = urllib.request.urlopen(
"https://api.github.com/repos/Finalfantasykid/AstraStack/releases/latest"
).read()
obj = json.loads(contents)
if (version.parse(obj['name']) > version.parse(UI.VERSION)):
self.newVersionUrl = obj['html_url']
button.show()
except:
return
button = self.builder.get_object("newVersion")
button.hide()
thread = Thread(target=callUrl, args=())
thread.start()
# Opens the GitHub releases page in a browser
def clickNewVersion(self, *args):
webbrowser.open(self.newVersionUrl)
# Checks to see if there will be enough memory to process the image
def checkMemory(self, w=0, h=0):
if (Sharpen.estimateMemoryUsage(w, h) >
psutil.virtual_memory().available):
response = self.showWarningDialog(
"Your system may not have enough memory to process this file, are you sure you want to continue?"
)
return (response == Gtk.ResponseType.YES)
return True
# Shows preview image in file chooser dialog
def updatePreview(self, dialog):
path = dialog.get_preview_filename()
pixbuf = None
try:
# First try as image
pixbuf = GdkPixbuf.Pixbuf.new_from_file(path)
except Exception:
try:
# Now try as video
if ("video/" in mimetypes.guess_type(path)[0]):
video = Video()
img = cv2.cvtColor(video.getFrame(path, 0),
cv2.COLOR_BGR2RGB)
height, width = img.shape[:2]
z = img.tobytes()
Z = GLib.Bytes.new(z)
pixbuf = GdkPixbuf.Pixbuf.new_from_bytes(
Z, GdkPixbuf.Colorspace.RGB, False, 8, width, height,
width * 3)
except Exception:
dialog.set_preview_widget_active(False)
if (pixbuf is not None):
# Valid pixbuf
maxwidth, maxheight = 250, 250
width, height = pixbuf.get_width(), pixbuf.get_height()
scale = min(maxwidth / width, maxheight / height)
if (scale < 1):
width, height = int(width * scale), int(height * scale)
pixbuf = pixbuf.scale_simple(width, height,
GdkPixbuf.InterpType.BILINEAR)
dialog.get_preview_widget().set_size_request(
width + 10, height + 10)
dialog.get_preview_widget().set_from_pixbuf(pixbuf)
dialog.set_preview_widget_active(True)
# Opens the file chooser to open load a file
def openVideo(self, *args):
self.openDialog.set_current_folder(path.expanduser("~"))
self.openDialog.set_select_multiple(False)
self.openDialog.set_filter(self.builder.get_object("videoFilter"))
response = self.openDialog.run()
self.openDialog.hide()
if (response == Gtk.ResponseType.OK):
try:
g.file = self.openDialog.get_filename()
self.video = Video()
self.video.checkMemory()
thread = Thread(target=self.video.run, args=())
thread.start()
self.disableUI()
except MemoryError as error:
self.enableUI()
# Opens the file chooser to open load a file
def openImageSequence(self, *args):
self.openDialog.set_current_folder(path.expanduser("~"))
self.openDialog.set_select_multiple(True)
self.openDialog.set_filter(self.builder.get_object("imageFilter"))
response = self.openDialog.run()
self.openDialog.hide()
if (response == Gtk.ResponseType.OK):
try:
g.file = self.openDialog.get_filenames()
self.video = Video()
self.video.checkMemory()
thread = Thread(target=self.video.run, args=())
thread.start()
self.disableUI()
except MemoryError as error:
self.enableUI()
# Opens the file chooser to open load a file
def openImage(self, *args):
self.openDialog.set_current_folder(path.expanduser("~"))
self.openDialog.set_select_multiple(False)
self.openDialog.set_filter(self.builder.get_object("imageFilter"))
response = self.openDialog.run()
self.openDialog.hide()
if (response == Gtk.ResponseType.OK):
self.disableUI()
g.file = self.openDialog.get_filename()
try:
self.video = Video()
img = cv2.imread(g.file)
h, w = img.shape[:2]
if (not self.checkMemory(w, h)):
raise MemoryError()
self.window.set_title(path.split(g.file)[1] + " - " + UI.TITLE)
self.saveDialog.set_current_name("")
self.sharpen = Sharpen(g.file, True)
self.builder.get_object("alignTab").set_sensitive(False)
self.builder.get_object("stackTab").set_sensitive(False)
self.builder.get_object("processTab").set_sensitive(True)
self.tabs.set_current_page(UI.SHARPEN_TAB)
self.frame.set_from_file(g.file)
except MemoryError as error:
pass
except: # Open Failed
self.showErrorDialog(
"There was an error opening the image, make sure it is a valid image."
)
self.enableUI()
# Opens the file chooser to save the final image
def saveFileDialog(self, *args):
self.saveDialog.set_current_folder(path.expanduser("~"))
if (self.saveDialog.get_current_name() == ""):
# Set default file to save if empty
if (isinstance(g.file, list)):
sList = g.file
self.saveDialog.set_current_name(
Path(sList[0]).stem + "_" + Path(sList[-1]).stem + ".png")
else:
self.saveDialog.set_current_name(Path(g.file).stem + ".png")
response = self.saveDialog.run()
if (response == Gtk.ResponseType.OK):
fileName = self.saveDialog.get_filename()
try:
cv2.imwrite(
fileName,
cv2.cvtColor(self.sharpen.finalImage.astype('uint8'),
cv2.COLOR_RGB2BGR))
except: # Save Failed
self.showErrorDialog(
"There was an error saving the image, make sure it is a valid file extension."
)
self.saveDialog.hide()
# Called when the video is finished loading
def finishedVideo(self):
def update():
self.tabs.next_page()
self.frameScale.set_sensitive(True)
self.startFrame.set_lower(0)
self.startFrame.set_upper(len(self.video.frames) - 1)
self.startFrame.set_value(0)
self.endFrame.set_upper(len(self.video.frames) - 1)
self.endFrame.set_lower(0)
self.endFrame.set_value(len(self.video.frames) - 1)
g.driftP1 = (0, 0)
g.driftP2 = (0, 0)
g.areaOfInterestP1 = (0, 0)
g.areaOfInterestP2 = (0, 0)
g.reference = self.video.sharpest
self.frameSlider.set_value(self.video.sharpest)
self.setReference()
self.setStartFrame()
self.setEndFrame()
self.setDriftPoint()
self.enableUI()
if (isinstance(g.file, list)):
sList = g.file
self.window.set_title(
path.split(sList[0])[1] + " ... " +
path.split(sList[-1])[1] + " - " + UI.TITLE)
else:
self.window.set_title(path.split(g.file)[1] + " - " + UI.TITLE)
self.saveDialog.set_current_name("")
self.builder.get_object("alignTab").set_sensitive(True)
self.builder.get_object("stackTab").set_sensitive(False)
self.builder.get_object("processTab").set_sensitive(False)
self.stack = None
GLib.idle_add(update)
# Called when the tab is changed. Updates parts of the UI based on the tab
def changeTab(self, notebook, page, page_num, user_data=None):
if (page_num == UI.LOAD_TAB or page_num == UI.ALIGN_TAB):
self.frameSlider.set_value(0)
self.frameSlider.set_upper(len(self.video.frames) - 1)
self.setStartFrame()
self.setEndFrame()
self.frameScale.show()
self.updateImage(None, page_num)
elif (page_num == UI.STACK_TAB):
self.frameSlider.set_lower(0)
self.frameSlider.set_upper(len(self.align.tmats) - 1)
self.frameSlider.set_value(0)
self.frameScale.show()
self.updateImage(None, page_num)
elif (page_num == UI.SHARPEN_TAB):
self.frameScale.hide()
self.sharpenImage()
self.fixFrameSliderBug()
# Changes the image frame to the frameSlider position
def updateImage(self, adjustment=None, page_num=None):
if (self.video is None):
return
if (page_num is None):
page_num = self.tabs.get_current_page()
if (page_num == UI.LOAD_TAB or page_num == UI.ALIGN_TAB):
videoIndex = int(self.frameSlider.get_value())
img = cv2.cvtColor(
self.video.getFrame(g.file, self.video.frames[videoIndex]),
cv2.COLOR_BGR2RGB)
height, width = img.shape[:2]
z = img.tobytes()
Z = GLib.Bytes.new(z)
pixbuf = GdkPixbuf.Pixbuf.new_from_bytes(Z,
GdkPixbuf.Colorspace.RGB,
False, 8, width, height,
width * 3)
self.frame.set_from_pixbuf(pixbuf)
elif (page_num == UI.STACK_TAB):
tmat = self.stack.tmats[int(self.frameSlider.get_value())]
videoIndex = tmat[0]
M = tmat[1]
img = self.video.getFrame(g.file, videoIndex)
if (g.autoCrop):
ref = self.stack.refBG.astype(np.uint8)
else:
ref = None
img = transform(img, ref, M, self.align.minX, self.align.maxX,
self.align.minY, self.align.maxY, g.drizzleFactor,
g.drizzleInterpolation)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
height, width = img.shape[:2]
z = img.tobytes()
Z = GLib.Bytes.new(z)
pixbuf = GdkPixbuf.Pixbuf.new_from_bytes(Z,
GdkPixbuf.Colorspace.RGB,
False, 8, width, height,
width * 3)
self.frame.set_from_pixbuf(pixbuf)
# Draws a rectangle where the area of interest is
def drawOverlay(self, widget, cr):
width = widget.get_allocated_width()
height = widget.get_allocated_height()
def drawPoint(cr, x, y):
cr.new_sub_path()
cr.set_line_width(2)
cr.set_source_rgb(1, 1, 1)
cr.arc(x, y, 2, 0, 2 * math.pi)
cr.stroke_preserve()
cr.set_source_rgb(1, 0, 0)
cr.fill()
def drawRect(cr, x1, y1, x2, y2):
cr.rectangle(0, 0, x1, y1)
cr.rectangle(0, y1, x1, (y2 - y1))
cr.rectangle(0, y1, x1, height * 2)
cr.rectangle(x1, y2, (x2 - x1), height * 2)
cr.rectangle(x2, y2, width * 2, height * 2)
cr.rectangle(x2, y1, width * 2, (y2 - y1))
cr.rectangle(x2, 0, width * 2, y1)
cr.rectangle(x1, 0, (x2 - x1), y1)
cr.set_source_rgba(0, 0, 0, 0.25)
cr.fill()
cr.set_line_width(1)
cr.set_source_rgb(1, 0, 0)
cr.rectangle(x1, y1, (x2 - x1), (y2 - y1))
cr.stroke()
if (self.tabs.get_current_page() == UI.ALIGN_TAB):
current = self.frameSlider.get_value()
# Area of Interest
px1 = min(g.areaOfInterestP1[0], g.areaOfInterestP2[0])
py1 = min(g.areaOfInterestP1[1], g.areaOfInterestP2[1])
px2 = max(g.areaOfInterestP1[0], g.areaOfInterestP2[0])
py2 = max(g.areaOfInterestP1[1], g.areaOfInterestP2[1])
# Drift Points
dx1 = g.driftP1[0]
dy1 = g.driftP1[1]
dx2 = g.driftP2[0]
dy2 = g.driftP2[1]
dx = 0
dy = 0
if (dx1 != 0 and dy1 != 0 and dx2 != 0 and dy2 != 0):
dx = dx2 - dx1
dy = dy2 - dy1
if (px1 != 0 and py1 != 0 and px2 != 0 and py2 != 0):
# Draw Area of Interest Rectangle
drawRect(
cr, px1 + (dx / (g.endFrame - g.startFrame)) *
(current - g.startFrame),
py1 + (dy / (g.endFrame - g.startFrame)) *
(current - g.startFrame),
px2 + (dx / (g.endFrame - g.startFrame)) *
(current - g.startFrame),
py2 + (dy / (g.endFrame - g.startFrame)) *
(current - g.startFrame))
if (dx1 != 0 and dy1 != 0 and current == g.startFrame):
# Draw point on first frame
drawPoint(cr, dx1, dy1)
if (dx1 != 0 and dy1 != 0 and current != g.startFrame and dx2 != 0
and dy2 != 0 and current != g.endFrame):
# Draw interpolated point
drawPoint(
cr, dx1 + (dx / (g.endFrame - g.startFrame)) *
(current - g.startFrame),
dy1 + (dy / (g.endFrame - g.startFrame)) *
(current - g.startFrame))
if (dx2 != 0 and dy2 != 0 and current == g.endFrame):
# Draw point on last frame
drawPoint(cr, dx2, dy2)
# Sets the reference frame to the current visible frame
def setReference(self, *args):
g.reference = str(int(self.frameSlider.get_value()))
self.builder.get_object("referenceLabel").set_text(g.reference)
self.builder.get_object("alignButton").set_sensitive(True)
# Updates the progress bar
def setProgress(self, i=0, total=0, text=""):
def update():
if (total == 0):
self.progressBox.hide()
else:
self.progressBox.show()
self.progress.set_fraction(i / total)
self.progress.set_text(text + " " +
str(round((i / total) * 100)) + "%")
GLib.idle_add(update)
# Sets the start frame for trimming
def setStartFrame(self, *args):
g.startFrame = int(self.startFrame.get_value())
self.endFrame.set_lower(g.startFrame + 1)
self.frameSlider.set_lower(g.startFrame)
self.frameSlider.set_value(
max(g.startFrame, self.frameSlider.get_value()))
if (int(g.startFrame) > int(g.reference)):
# Reference is outside of the range, fix it
g.reference = str(int(g.startFrame))
self.builder.get_object("referenceLabel").set_text(g.reference)
self.fixFrameSliderBug()
# Sets the end frame for trimming
def setEndFrame(self, *args):
g.endFrame = int(self.endFrame.get_value())
self.startFrame.set_upper(g.endFrame - 1)
self.frameSlider.set_upper(g.endFrame)
self.frameSlider.set_value(
min(g.endFrame, self.frameSlider.get_value()))
if (int(g.endFrame) < int(g.reference)):
# Reference is outside of the range, fix it
g.reference = str(int(g.endFrame))
self.builder.get_object("referenceLabel").set_text(g.reference)
self.fixFrameSliderBug()
# Drift Point 1 Button Clicked
def clickDriftP1(self, *args):
self.clickedDriftP1 = False
self.clickedDriftP2 = False
self.clickedAreaOfInterest = False
self.setDriftPoint()
self.frameSlider.set_value(g.startFrame)
self.clickedDriftP1 = True
self.window.get_window().set_cursor(
Gdk.Cursor(Gdk.CursorType.CROSSHAIR))
# Drift Point 2 Button Clicked
def clickDriftP2(self, *args):
self.clickedDriftP1 = False
self.clickedDriftP2 = False
self.clickedAreaOfInterest = False
self.setDriftPoint()
self.frameSlider.set_value(g.endFrame)
self.clickedDriftP2 = True
self.window.get_window().set_cursor(
Gdk.Cursor(Gdk.CursorType.CROSSHAIR))
# Reset Drift Point 1 to (0, 0)
def resetDriftP1(self, widget, event):
if (event.button == 3): # Right Click
g.driftP1 = (0, 0)
self.clickedDriftP1 = False
self.clickedDriftP2 = False
self.clickedAreaOfInterest = False
self.window.get_window().set_cursor(
Gdk.Cursor(Gdk.CursorType.LEFT_PTR))
self.overlay.queue_draw()
# Reset Drift Point 2 to (0, 0)
def resetDriftP2(self, widget, event):
if (event.button == 3): # Right Click
g.driftP2 = (0, 0)
self.clickedDriftP1 = False
self.clickedDriftP2 = False
self.clickedAreaOfInterest = False
self.window.get_window().set_cursor(
Gdk.Cursor(Gdk.CursorType.LEFT_PTR))
self.overlay.queue_draw()
# Updates the drift point
def setDriftPoint(self, *args):
if (self.clickedDriftP1 or self.clickedDriftP2):
if (self.clickedDriftP1):
g.driftP1 = self.mousePosition
elif (self.clickedDriftP2):
g.driftP2 = self.mousePosition
self.clickedDriftP1 = False
self.clickedDriftP2 = False
self.window.get_window().set_cursor(
Gdk.Cursor(Gdk.CursorType.LEFT_PTR))
self.overlay.queue_draw()
# Area of Interest button clicked
def clickAreaOfInterest(self, *args):
g.areaOfInterestP1 = (0, 0)
g.areaOfInterestP2 = (0, 0)
self.clickedDriftP1 = False
self.clickedDriftP2 = False
self.clickedAreaOfInterest = True
self.frameSlider.set_value(g.startFrame)
self.window.get_window().set_cursor(
Gdk.Cursor(Gdk.CursorType.CROSSHAIR))
self.overlay.queue_draw()
# Reset Area of Interest to (0, 0)
def resetAreaOfInterest(self, widget, event):
if (event.button == 3): # Right Click
g.areaOfInterestP1 = (0, 0)
g.areaOfInterestP2 = (0, 0)
self.clickedDriftP1 = False
self.clickedDriftP2 = False
self.clickedAreaOfInterest = False
self.window.get_window().set_cursor(
Gdk.Cursor(Gdk.CursorType.LEFT_PTR))
self.overlay.queue_draw()
# First point int the Area of Interest clicked, drag started
def dragBegin(self, *args):
if (self.clickedAreaOfInterest):
g.areaOfInterestP1 = self.mousePosition
# Mouse released after dragging Area of Interest
def dragEnd(self, *args):
if (self.clickedAreaOfInterest):
g.areaOfInterestP2 = self.mousePosition
self.clickedAreaOfInterest = False
self.window.get_window().set_cursor(
Gdk.Cursor(Gdk.CursorType.LEFT_PTR))
# Called when the mouse moves over the frame
def updateMousePosition(self, *args):
pointer = self.frame.get_pointer()
self.mousePosition = (min(max(0, pointer.x),
self.frame.get_allocation().width),
min(max(0, pointer.y),
self.frame.get_allocation().height))
if (self.clickedAreaOfInterest):
if (g.areaOfInterestP1 != (0, 0)):
g.areaOfInterestP2 = self.mousePosition
self.overlay.queue_draw()
# Sets whether or not to normalize the frames during alignment
def setNormalize(self, *args):
g.normalize = self.normalize.get_active()
# Sets whether or not to align channels separately
def setAlignChannels(self, *args):
g.alignChannels = self.alignChannels.get_active()
# Sets the type of transformation
def setTransformation(self, *args):
text = self.transformation.get_active_text()
if (text == "None"):
g.transformation = -1
elif (text == "Translation"):
g.transformation = StackReg.TRANSLATION
elif (text == "Rigid Body"):
g.transformation = StackReg.RIGID_BODY
elif (text == "Scaled Rotation"):
g.transformation = StackReg.SCALED_ROTATION
elif (text == "Affine"):
g.transformation = StackReg.AFFINE
# Sets the drizzle scaling factor
def setDrizzleFactor(self, *args):
text = self.drizzleFactor.get_active_text()
if (text == "0.25X"):
g.drizzleFactor = 0.25
elif (text == "0.50X"):
g.drizzleFactor = 0.50
elif (text == "0.75X"):
g.drizzleFactor = 0.75
elif (text == "1.0X"):
g.drizzleFactor = 1.0
elif (text == "1.5X"):
g.drizzleFactor = 1.5
elif (text == "2.0X"):
g.drizzleFactor = 2.0
elif (text == "2.5X"):
g.drizzleFactor = 2.5
elif (text == "3.0X"):
g.drizzleFactor = 3.0
if (self.stack is not None):
self.stack.generateRefBG()
self.updateImage()
# Sets the drizzle scaling factor
def setDrizzleInterpolation(self, *args):
text = self.drizzleInterpolation.get_active_text()
if (text == "Nearest Neighbor"):
g.drizzleInterpolation = cv2.INTER_NEAREST
elif (text == "Bilinear"):
g.drizzleInterpolation = cv2.INTER_LINEAR
elif (text == "Bicubic"):
g.drizzleInterpolation = cv2.INTER_CUBIC
elif (text == "Lanczos"):
g.drizzleInterpolation = cv2.INTER_LANCZOS4
if (self.stack is not None):
self.stack.generateRefBG()
self.updateImage()
# Sets whether or not to auto crop
def setAutoCrop(self, *args):
g.autoCrop = not self.autoCrop.get_active()
self.updateImage()
# Runs the Alignment step
def clickAlign(self, *args):
self.align = Align(self.video.frames[g.startFrame:g.endFrame + 1])
thread = Thread(target=self.align.run, args=())
thread.start()
self.disableUI()
# Kills all pool processes
def killPool(self):
for pid in self.pids:
if (psutil.pid_exists(pid)):
p = psutil.Process(pid)
p.kill()
# Stops the current action being performed
def stopProcessing(self, *args):
self.killPool()
g.pool = None
self.setThreads()
self.setProgress()
self.enableUI()
# Called when the Alignment is complete
def finishedAlign(self):
def update():
self.stack = Stack(self.align.tmats)
self.tabs.next_page()
self.enableUI()
self.builder.get_object("alignTab").set_sensitive(True)
self.builder.get_object("stackTab").set_sensitive(True)
self.builder.get_object("processTab").set_sensitive(False)
self.limit.set_upper(len(self.align.tmats))
self.limit.set_value(int(len(self.align.tmats) / 2))
self.limitPercent.set_value(
round(self.limit.get_value() / len(self.align.tmats) * 100))
self.setLimit()
self.setLimitPercent()
GLib.idle_add(update)
# Sets the number of frames to use in the Stack
def setLimit(self, *args):
self.limitPercent.disconnect(self.limitPercentSignal)
self.limit.set_upper(len(self.align.tmats))
g.limit = int(self.limit.get_value())
self.limitPercent.set_value(
round(g.limit / len(self.align.tmats) * 100))
self.limitPercentSignal = self.limitPercent.connect(
"value-changed", self.setLimitPercent)
# Sets the number of frames to use in the Stack
def setLimitPercent(self, *args):
limitPercent = self.limitPercent.get_value() / 100
self.limit.set_value(round(limitPercent * len(self.align.tmats)))
# Stack Button clicked
def clickStack(self, *args):
try:
self.stack.checkMemory()
thread = Thread(target=self.stack.run, args=())
thread.start()
self.disableUI()
except MemoryError as error:
self.enableUI()
# Called when the stack is complete
def finishedStack(self):
def update():
self.sharpen = Sharpen(self.stack.stackedImage)
self.tabs.next_page()
self.enableUI()
self.builder.get_object("alignTab").set_sensitive(True)
self.builder.get_object("stackTab").set_sensitive(True)
self.builder.get_object("processTab").set_sensitive(True)
GLib.idle_add(update)
# Sharpens the final Stacked image
def sharpenImage(self, *args):
g.sharpen1 = self.builder.get_object("sharpen1").get_value()
g.sharpen2 = self.builder.get_object("sharpen2").get_value()
g.sharpen3 = self.builder.get_object("sharpen3").get_value()
g.sharpen4 = self.builder.get_object("sharpen4").get_value()
g.sharpen5 = self.builder.get_object("sharpen5").get_value()
g.radius1 = self.builder.get_object("radius1").get_value()
g.radius2 = self.builder.get_object("radius2").get_value()
g.radius3 = self.builder.get_object("radius3").get_value()
g.radius4 = self.builder.get_object("radius4").get_value()
g.radius5 = self.builder.get_object("radius5").get_value()
g.denoise1 = self.builder.get_object("denoise1").get_value()
g.denoise2 = self.builder.get_object("denoise2").get_value()
g.denoise3 = self.builder.get_object("denoise3").get_value()
g.denoise4 = self.builder.get_object("denoise4").get_value()
g.denoise5 = self.builder.get_object("denoise5").get_value()
g.level1 = self.builder.get_object("level1").get_active()
g.level2 = self.builder.get_object("level2").get_active()
g.level3 = self.builder.get_object("level3").get_active()
g.level4 = self.builder.get_object("level4").get_active()
g.level5 = self.builder.get_object("level5").get_active()
g.gamma = self.builder.get_object("gammaAdjust").get_value()
g.blackLevel = self.builder.get_object("blackLevelAdjust").get_value()
g.value = self.builder.get_object("valueAdjust").get_value()
g.redAdjust = self.builder.get_object("redAdjust").get_value()
g.greenAdjust = self.builder.get_object("greenAdjust").get_value()
g.blueAdjust = self.builder.get_object("blueAdjust").get_value()
g.saturation = self.builder.get_object("saturationAdjust").get_value()
if (len(args) > 0
and (self.builder.get_object("gammaAdjust") == args[0]
or self.builder.get_object("blackLevelAdjust") == args[0]
or self.builder.get_object("redAdjust") == args[0]
or self.builder.get_object("greenAdjust") == args[0]
or self.builder.get_object("blueAdjust") == args[0]
or self.builder.get_object("saturationAdjust") == args[0]
or self.builder.get_object("valueAdjust") == args[0])):
processAgain = self.sharpen.processAgain
processColor = True
else:
processAgain = True
processColor = False
if (self.sharpen is None):
if (self.stack is not None):
self.sharpen = Sharpen(self.stack.stackedImage)
else:
self.sharpen = Sharpen(g.file, True)
if (self.processThread != None and self.processThread.is_alive()):
self.sharpen.processAgain = processAgain
self.sharpen.processColorAgain = processColor
else:
self.processThread = Thread(target=self.sharpen.run,
args=(processAgain, processColor))
self.processThread.start()
# Called when sharpening is complete
def finishedSharpening(self):
def update():
z = self.sharpen.finalImage.astype('uint8').tobytes()
Z = GLib.Bytes.new(z)
pixbuf = GdkPixbuf.Pixbuf.new_from_bytes(Z,
GdkPixbuf.Colorspace.RGB,
False, 8, self.sharpen.w,
self.sharpen.h,
self.sharpen.w * 3)
self.frame.set_from_pixbuf(pixbuf)
GLib.idle_add(update)
# Closes the application
def close(self, *args):
self.killPool()
Gtk.main_quit()
sys.exit()
def test_pipe():
parent_conn, child_conn = Pipe()
p = Process(target=f2, args=(child_conn, ))
p.start()
print parent_conn.recv()
p.join()
def advance_to_next_server(signum, frame):
sys.stdout.write("#skip the current server...\n")
skip[0] = True
signal.signal(signal.SIGINT, advance_to_next_server)
idx = 0
while idx < len(srv_list):
ofs, conn = pick_ssh_server(srv_list[idx:], cmd, user, passwd, timeout,
ob_key)
idx += ofs
if conn is not None:
while conn.isalive():
if checker is not None and not checker.is_alive():
sorted_srv_name_list = checker_conn.recv()
if opt.debug:
sys.stdout.write("#got sorted server list\n %s\n" %
"\n ".join(sorted_srv_name_list))
checker.join()
checker = None
with open(get_cfg_fpath(), "wb") as fd:
fd.write("\n".join(sorted_srv_name_list))
fd.write("\n")
if srv_list[idx] in sorted_srv_name_list[:3]:
sys.stdout.write("#reconnection is not needed\n")
else:
srv_list = gen_server_list(sorted_srv_name_list,
opt.port)
# multiProcessCommunicationByPipe.py
from multiprocessing import Process,Pipe
import os
def sendMsg(conn,msg):
conn.send("I am the child process %s send the message %s"%(os.getpid(),msg))
if __name__ == '__main__':
parent_conn,child_conn = Pipe()
p = Process(target = sendMsg,args=(child_conn,"hello kingwen",))
p.start()
print("this is parent process %s ,msg from child is :%s"%(os.getpid(),parent_conn.recv()))
p.join()
def run_base(exp_dir, **kwargs):
config = ConfigProto()
config.gpu_options.allow_growth = True
config.gpu_options.per_process_gpu_memory_fraction = kwargs.get(
'gpu_frac', 0.95)
# Instantiate classes
set_seed(kwargs['seed'])
baseline = kwargs['baseline']()
if kwargs['env'] == 'Ant':
env = AntEnv()
simulation_sleep = 0.05 * kwargs['num_rollouts'] * kwargs[
'max_path_length'] * kwargs['simulation_sleep_frac']
elif kwargs['env'] == 'HalfCheetah':
env = HalfCheetahEnv()
simulation_sleep = 0.05 * kwargs['num_rollouts'] * kwargs[
'max_path_length'] * kwargs['simulation_sleep_frac']
elif kwargs['env'] == 'Hopper':
env = HopperEnv()
simulation_sleep = 0.008 * kwargs['num_rollouts'] * kwargs[
'max_path_length'] * kwargs['simulation_sleep_frac']
elif kwargs['env'] == 'Walker2d':
env = Walker2dEnv()
simulation_sleep = 0.008 * kwargs['num_rollouts'] * kwargs[
'max_path_length'] * kwargs['simulation_sleep_frac']
else:
raise NotImplementedError
policy = MetaGaussianMLPPolicy(
name="meta-policy",
obs_dim=np.prod(env.observation_space.shape),
action_dim=np.prod(env.action_space.shape),
meta_batch_size=kwargs['meta_batch_size'],
hidden_sizes=kwargs['policy_hidden_sizes'],
learn_std=kwargs['policy_learn_std'],
hidden_nonlinearity=kwargs['policy_hidden_nonlinearity'],
output_nonlinearity=kwargs['policy_output_nonlinearity'],
)
dynamics_model = MLPDynamicsEnsemble(
'dynamics-ensemble',
env=env,
num_models=kwargs['num_models'],
hidden_nonlinearity=kwargs['dyanmics_hidden_nonlinearity'],
hidden_sizes=kwargs['dynamics_hidden_sizes'],
output_nonlinearity=kwargs['dyanmics_output_nonlinearity'],
learning_rate=kwargs['dynamics_learning_rate'],
batch_size=kwargs['dynamics_batch_size'],
buffer_size=kwargs['dynamics_buffer_size'],
rolling_average_persitency=kwargs['rolling_average_persitency'],
)
'''-------- dumps and reloads -----------------'''
baseline_pickle = pickle.dumps(baseline)
env_pickle = pickle.dumps(env)
receiver, sender = Pipe()
p = Process(
target=init_vars,
name="init_vars",
args=(sender, config, policy, dynamics_model),
daemon=True,
)
p.start()
policy_pickle, dynamics_model_pickle = receiver.recv()
receiver.close()
'''-------- following classes depend on baseline, env, policy, dynamics_model -----------'''
worker_data_feed_dict = {
'env_sampler': {
'rollouts_per_meta_task':
kwargs['real_env_rollouts_per_meta_task'],
'meta_batch_size': kwargs['meta_batch_size'],
'max_path_length': kwargs['max_path_length'],
'parallel': kwargs['parallel'],
},
'dynamics_sample_processor': {
'discount': kwargs['discount'],
'gae_lambda': kwargs['gae_lambda'],
'normalize_adv': kwargs['normalize_adv'],
'positive_adv': kwargs['positive_adv'],
},
}
worker_model_feed_dict = {}
worker_policy_feed_dict = {
'model_sampler': {
'rollouts_per_meta_task': kwargs['rollouts_per_meta_task'],
'meta_batch_size': kwargs['meta_batch_size'],
'max_path_length': kwargs['max_path_length'],
'deterministic': kwargs['deterministic'],
},
'model_sample_processor': {
'discount': kwargs['discount'],
'gae_lambda': kwargs['gae_lambda'],
'normalize_adv': kwargs['normalize_adv'],
'positive_adv': kwargs['positive_adv'],
},
'algo': {
'step_size': kwargs['step_size'],
'inner_type': kwargs['inner_type'],
'inner_lr': kwargs['inner_lr'],
'meta_batch_size': kwargs['meta_batch_size'],
'num_inner_grad_steps': kwargs['num_inner_grad_steps'],
'exploration': kwargs['exploration'],
}
}
trainer = ParallelTrainer(
exp_dir=exp_dir,
policy_pickle=policy_pickle,
env_pickle=env_pickle,
baseline_pickle=baseline_pickle,
dynamics_model_pickle=dynamics_model_pickle,
feed_dicts=[
worker_data_feed_dict, worker_model_feed_dict,
worker_policy_feed_dict
],
n_itr=kwargs['n_itr'],
num_inner_grad_steps=kwargs['num_inner_grad_steps'],
initial_random_samples=kwargs['initial_random_samples'],
flags_need_query=kwargs['flags_need_query'],
num_rollouts_per_iter=int(kwargs['meta_batch_size'] *
kwargs['fraction_meta_batch_size']),
config=config,
simulation_sleep=simulation_sleep,
)
trainer.train()
#!/usr/bin/python3
# -*- coding: utf-8 -*-
from multiprocessing import Process, Pipe
import time
def f(conn):
conn.send([42, None, 'hello from child'])
conn.send([42, None, 'hello from child1'])
conn.close()
if __name__ == "__main__":
parent_conn, child_conn = Pipe()
p = Process(target=f, args=(child_conn, ))
p.start()
print(parent_conn.recv())
print(parent_conn.recv())
print(parent_conn.recv())
p.join()
class AillioR1:
AILLIO_VID = 0x0483
AILLIO_PID = 0x5741
AILLIO_PID_REV3 = 0xa27e
AILLIO_ENDPOINT_WR = 0x3
AILLIO_ENDPOINT_RD = 0x81
AILLIO_INTERFACE = 0x1
AILLIO_CONFIGURATION = 0x1
AILLIO_DEBUG = 1
AILLIO_CMD_INFO1 = [0x30, 0x02]
AILLIO_CMD_INFO2 = [0x89, 0x01]
AILLIO_CMD_STATUS1 = [0x30, 0x01]
AILLIO_CMD_STATUS2 = [0x30, 0x03]
AILLIO_CMD_PRS = [0x30, 0x01, 0x00, 0x00]
AILLIO_CMD_HEATER_INCR = [0x34, 0x01, 0xaa, 0xaa]
AILLIO_CMD_HEATER_DECR = [0x34, 0x02, 0xaa, 0xaa]
AILLIO_CMD_FAN_INCR = [0x31, 0x01, 0xaa, 0xaa]
AILLIO_CMD_FAN_DECR = [0x31, 0x02, 0xaa, 0xaa]
AILLIO_STATE_OFF = 0x00
AILLIO_STATE_PH = 0x02
AILLIO_STATE_CHARGE = 0x04
AILLIO_STATE_ROASTING = 0x06
AILLIO_STATE_COOLING = 0x08
AILLIO_STATE_SHUTDOWN = 0x09
def __init__(self, debug=False):
self.simulated = False
self.AILLIO_DEBUG = debug
self.__dbg('init')
self.usbhandle = None
self.bt = 0
self.dt = 0
self.heater = 0
self.fan = 0
self.bt_ror = 0
self.drum = 0
self.voltage = 0
self.exitt = 0
self.state_str = ""
self.r1state = 0
self.worker_thread = None
self.worker_thread_run = True
self.roast_number = -1
self.fan_rpm = 0
def __del__(self):
if not self.simulated:
self.__close()
def __dbg(self, msg):
if self.AILLIO_DEBUG and self.simulated != True:
try:
print('AillioR1: ' + msg)
except IOError:
pass
def __open(self):
if self.simulated:
return
if self.usbhandle is not None:
return
self.usbhandle = usb.core.find(idVendor=self.AILLIO_VID,
idProduct=self.AILLIO_PID)
if self.usbhandle is None:
self.usbhandle = usb.core.find(idVendor=self.AILLIO_VID,
idProduct=self.AILLIO_PID_REV3)
if self.usbhandle is None:
raise IOError("not found or no permission")
self.__dbg('device found!')
if not system().startswith("Windows"):
if self.usbhandle.is_kernel_driver_active(self.AILLIO_INTERFACE):
try:
self.usbhandle.detach_kernel_driver(self.AILLIO_INTERFACE)
except Exception:
self.usbhandle = None
raise IOError("unable to detach kernel driver")
try:
config = self.usbhandle.get_active_configuration()
if config.bConfigurationValue != self.AILLIO_CONFIGURATION:
self.usbhandle.set_configuration(
configuration=self.AILLIO_CONFIGURATION)
except Exception:
self.usbhandle = None
raise IOError("unable to configure")
try:
usb.util.claim_interface(self.usbhandle, self.AILLIO_INTERFACE)
except Exception:
self.usbhandle = None
raise IOError("unable to claim interface")
self.__sendcmd(self.AILLIO_CMD_INFO1)
reply = self.__readreply(32)
sn = unpack('h', reply[0:2])[0]
firmware = unpack('h', reply[24:26])[0]
self.__dbg('serial number: ' + str(sn))
self.__dbg('firmware version: ' + str(firmware))
self.__sendcmd(self.AILLIO_CMD_INFO2)
reply = self.__readreply(36)
self.roast_number = unpack('>I', reply[27:31])[0]
self.__dbg('number of roasts: ' + str(self.roast_number))
self.parent_pipe, self.child_pipe = Pipe()
self.worker_thread = threading.Thread(target=self.__updatestate,
args=(self.child_pipe, ))
self.worker_thread.start()
def __close(self):
if self.simulated:
return
if self.usbhandle is not None:
try:
usb.util.release_interface(self.usbhandle,
self.AILLIO_INTERFACE)
usb.util.dispose_resources(self.usbhandle)
except Exception:
pass
self.usbhandle = None
if self.worker_thread:
self.worker_thread_run = False
self.worker_thread.join()
self.parent_pipe.close()
self.child_pipe.close()
self.worker_thread = None
def get_roast_number(self):
self.__getstate()
return self.roast_number
def get_bt(self):
self.__getstate()
return self.bt
def get_dt(self):
self.__getstate()
return self.dt
def get_heater(self):
self.__dbg('get_heater')
self.__getstate()
return self.heater
def get_fan(self):
self.__dbg('get_fan')
self.__getstate()
return self.fan
def get_fan_rpm(self):
self.__dbg('get_fan_rpm')
self.__getstate()
return self.fan_rpm
def get_drum(self):
self.__getstate()
return self.drum
def get_voltage(self):
self.__getstate()
return self.voltage
def get_bt_ror(self):
self.__getstate()
return self.bt_ror
def get_exit_temperature(self):
self.__getstate()
return self.exitt
def get_state_string(self):
self.__getstate()
return self.state_str
def get_state(self):
self.__getstate()
return self.r1state
def set_heater(self, value):
self.__dbg('set_heater ' + str(value))
value = int(value)
if value < 0:
value = 0
elif value > 9:
value = 9
h = self.get_heater()
d = abs(h - value)
if d <= 0:
return
if d > 9:
d = 9
if h > value:
for _ in range(d):
self.parent_pipe.send(self.AILLIO_CMD_HEATER_DECR)
else:
for _ in range(d):
self.parent_pipe.send(self.AILLIO_CMD_HEATER_INCR)
self.heater = value
def set_fan(self, value):
self.__dbg('set_fan ' + str(value))
value = int(value)
if value < 1:
value = 1
elif value > 12:
value = 12
f = self.get_fan()
d = abs(f - value)
if d <= 0:
return
if d > 11:
d = 11
if f > value:
for _ in range(0, d):
self.parent_pipe.send(self.AILLIO_CMD_FAN_DECR)
else:
for _ in range(0, d):
self.parent_pipe.send(self.AILLIO_CMD_FAN_INCR)
self.fan = value
def set_drum(self, value):
self.__dbg('set_drum ' + str(value))
value = int(value)
if value < 1:
value = 1
elif value > 9:
value = 9
self.parent_pipe.send([0x32, 0x01, value, 0x00])
self.drum = value
def prs(self):
self.__dbg('PRS')
self.parent_pipe.send(self.AILLIO_CMD_PRS)
def __updatestate(self, p):
while self.worker_thread_run:
state1 = state2 = []
try:
self.__dbg('updatestate')
self.__sendcmd(self.AILLIO_CMD_STATUS1)
state1 = self.__readreply(64)
self.__sendcmd(self.AILLIO_CMD_STATUS2)
state2 = self.__readreply(64)
except Exception:
pass
if p.poll():
cmd = p.recv()
self.__sendcmd(cmd)
if len(state1) + len(state2) == 128:
p.send(state1 + state2)
time.sleep(0.1)
def __getstate(self):
self.__dbg('getstate')
if self.simulated:
if random.random() > 0.05:
return
self.bt += random.random()
self.bt_ror += random.random()
self.dt += random.random()
self.exitt += random.random()
self.fan = random.random() * 10
self.heater = random.random() * 8
self.drum = random.random() * 8
self.irt = random.random()
self.pcbt = random.random()
self.fan_rpm += random.random()
self.voltage = 240
self.coil_fan = 0
self.coil_fan2 = 0
self.pht = 0
self.r1state = self.AILLIO_STATE_ROASTING
self.state_str = "roasting"
return
self.__open()
if not self.parent_pipe.poll():
return
state = self.parent_pipe.recv()
valid = state[41]
# Heuristic to find out if the data is valid
# It looks like we get a different message every 15 seconds
# when we're not roasting. Ignore this message for now.
if valid == 10:
self.bt = round(unpack('f', state[0:4])[0], 1)
self.bt_ror = round(unpack('f', state[4:8])[0], 1)
self.dt = round(unpack('f', state[8:12])[0], 1)
self.exitt = round(unpack('f', state[16:20])[0], 1)
self.minutes = state[24]
self.seconds = state[25]
self.fan = state[26]
self.heater = state[27]
self.drum = state[28]
self.r1state = state[29]
self.irt = round(unpack('f', state[32:36])[0], 1)
self.pcbt = round(unpack('f', state[36:40])[0], 1)
self.fan_rpm = unpack('h', state[44:46])[0]
self.voltage = unpack('h', state[48:50])[0]
self.coil_fan = round(unpack('i', state[52:56])[0], 1)
self.__dbg('BT: ' + str(self.bt))
self.__dbg('BT RoR: ' + str(self.bt_ror))
self.__dbg('DT: ' + str(self.dt))
self.__dbg('exit temperature ' + str(self.exitt))
self.__dbg('PCB temperature: ' + str(self.irt))
self.__dbg('IR temperature: ' + str(self.pcbt))
self.__dbg('voltage: ' + str(self.voltage))
self.__dbg('coil fan: ' + str(self.coil_fan))
self.__dbg('fan: ' + str(self.fan))
self.__dbg('heater: ' + str(self.heater))
self.__dbg('drum speed: ' + str(self.drum))
self.__dbg('time: ' + str(self.minutes) + ':' + str(self.seconds))
state = state[64:]
self.coil_fan2 = round(unpack('i', state[32:36])[0], 1)
self.pht = unpack('H', state[40:42])[0]
self.__dbg('pre-heat temperature: ' + str(self.pht))
if self.r1state == self.AILLIO_STATE_OFF:
self.state_str = "off"
elif self.r1state == self.AILLIO_STATE_PH:
self.state_str = "pre-heating to " + str(self.pht) + "C"
elif self.r1state == self.AILLIO_STATE_CHARGE:
self.state_str = "charge"
elif self.r1state == self.AILLIO_STATE_ROASTING:
self.state_str = "roasting"
elif self.r1state == self.AILLIO_STATE_COOLING:
self.state_str = "cooling"
elif self.r1state == self.AILLIO_STATE_SHUTDOWN:
self.state_str = "shutdown"
self.__dbg('state: ' + self.state_str)
self.__dbg('second coil fan: ' + str(self.coil_fan2))
def __sendcmd(self, cmd):
self.__dbg('sending command: ' + str(cmd))
self.usbhandle.write(self.AILLIO_ENDPOINT_WR, cmd)
def __readreply(self, length):
return self.usbhandle.read(self.AILLIO_ENDPOINT_RD, length)
class BTgymRendering():
"""
Handles BTgym Environment rendering.
Note:
Call `initialize_pyplot()` method before first render() call!
"""
# Here we'll keep last rendered image for each rendering mode:
rgb_dict = dict()
render_modes = ['episode', 'human']
params = dict(
# Plotting controls, can be passed as kwargs:
render_state_as_image=True,
render_state_channel=0,
render_size_human=(6, 3.5),
render_size_state=(7, 3.5),
render_size_episode=(12, 8),
render_rowsmajor_episode=1,
render_dpi=75,
render_plotstyle='seaborn',
render_cmap='PRGn',
render_xlabel='Relative timesteps',
render_ylabel='Value',
render_title=
'local step: {}, state observation min: {:.4f}, max: {:.4f}',
render_boxtext=dict(
fontsize=12,
fontweight='bold',
color='w',
bbox={
'facecolor': 'k',
'alpha': 0.3,
'pad': 3
},
),
plt_backend='Agg', # Not used.
)
enabled = True
ready = False
def __init__(self, render_modes, **kwargs):
"""
Plotting controls, can be passed as kwargs.
Args:
render_state_as_image=True,
render_state_channel=0,
render_size_human=(6, 3.5),
render_size_state=(7, 3.5),
render_size_episode=(12,8),
render_dpi=75,
render_plotstyle='seaborn',
render_cmap='PRGn',
render_xlabel='Relative timesteps',
render_ylabel='Value',
render_title='local step: {}, state observation min: {:.4f}, max: {:.4f}',
render_boxtext=dict(fontsize=12,
fontweight='bold',
color='w',
bbox={'facecolor': 'k', 'alpha': 0.3, 'pad': 3},
)
"""
# Update parameters with relevant kwargs:
for key, value in kwargs.items():
if key in self.params.keys():
self.params[key] = value
# Unpack it as attributes:
for key, value in self.params.items():
setattr(self, key, value)
# Logging:
if 'log_level' not in dir(self):
self.log_level = WARNING
StreamHandler(sys.stdout).push_application()
self.log = Logger('BTgymRenderer', level=self.log_level)
#from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
#self.FigureCanvas = FigureCanvas
self.plt = None # Will set it inside server process when calling initialize_pyplot().
#self.plotter = BTgymPlotter() # Modified bt.Cerebro() plotter, to get episode renderings.
# Set empty plugs for each render mode:
self.render_modes = render_modes
for mode in self.render_modes:
self.rgb_dict[mode] = self.rgb_empty()
def initialize_pyplot(self):
"""
Call me before use!
[Supposed to be done inside already running server process]
"""
if not self.ready:
from multiprocessing import Pipe
self.out_pipe, self.in_pipe = Pipe()
if self.plt is None:
import matplotlib
matplotlib.use(self.plt_backend, force=True)
import matplotlib.pyplot as plt
self.plt = plt
self.ready = True
def to_string(self, dictionary, excluded=[]):
"""
Converts given dictionary to more-or-less good looking `text block` string.
"""
text = ''
for k, v in dictionary.items():
if k not in excluded:
if type(v) in [float]:
v = '{:.4f}'.format(v)
text += '{}: {}\n'.format(k, v)
return text[:-1]
def rgb_empty(self):
"""
Returns empty 'plug' image.
"""
return (np.random.rand(100, 200, 3) * 255).astype(dtype=np.uint8)
def parse_response(
self,
state,
mode,
reward,
info,
done,
):
"""
Converts environment response to plotting attributes:
state, title, text.
"""
if len(state[mode].shape) <= 2:
state = np.asarray(state[mode])
elif len(state[mode].shape) == 3:
if state[mode].shape[1] == 1:
# Assume 2nd dim (H) is fake expansion for 1D input, so can render all channels:
state = np.asarray(state[mode][:, 0, :])
else:
# Assume it is HWC 2D input, only can render single channel:
state = np.asarray(state[mode][:, :,
self.render_state_channel])
else:
raise NotImplementedError(
'2D rendering can be done for obs. state tensor with rank <= 3; ' +\
'got state shape: {}'.format(np.asarray(state[mode]).shape))
# Figure out how to deal with info output:
try:
assert type(info[-1]) == dict
info_dict = info[-1]
except AssertionError:
try:
assert type(info) == dict
info_dict = info
except AssertionError:
try:
info_dict = {'info': str(dict)}
except:
info_dict = {}
# Add records:
info_dict.update(
reward=reward,
is_done=done,
)
# Try to get step information:
try:
current_step = info_dict['step']
except:
current_step = '--'
# Set box text, excluding redundant fields:
box_text = self.to_string(info_dict, excluded=['step'])
# Set title output:
title = self.render_title.format(current_step, state.min(),
state.max())
return state, title, box_text
def render(self,
mode_list,
cerebro=None,
step_to_render=None,
send_img=True):
"""
Renders given mode if possible, else
just passes last already rendered image.
Returns rgb image as numpy array.
Logic:
- If `cerebro` arg is received:
render entire episode, using built-in backtrader plotting feature,
update stored `episode` image.
- If `step_to_render' arg is received:
- if mode = 'raw_state':
render current state observation in conventional 'price' format,
update stored `raw_state` image;
- if mode = something_else':
visualise observation as 'seen' by agent,
update stored 'agent' image.
Returns:
`mode` image.
Note:
It can actually return several modes in a single dict.
It prevented by Gym modes convention, but done internally at the end of the episode.
"""
if type(mode_list) == str:
mode_list = [mode_list]
if cerebro is not None:
self.rgb_dict['episode'] = self.draw_episode(cerebro)
# Try to render given episode:
#try:
# Get picture of entire episode:
#fig = cerebro.plot(plotter=self.plotter, # Modified plotter class, doesnt actually save anything.
# savefig=True,
# width=self.render_size_episode[0],
# height=self.render_size_episode[1],
# dpi=self.render_dpi,
# use=None, #self.plt_backend,
# iplot=False,
# figfilename='_tmp_btgym_render.png',
# )[0][0]
#fig.canvas.draw()
#rgb_array = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep='')
#self.rgb_dict['episode'] = rgb_array.reshape(fig.canvas.get_width_height()[::-1] + (3,))
# Clean up:
#self.plt.gcf().clear()
#self.plt.close(fig)
#except:
# Just keep previous rendering
# pass
if step_to_render is not None:
# Perform step rendering:
# Unpack:
raw_state, state, reward, done, info = step_to_render
for mode in mode_list:
if mode in self.render_modes and mode not in [
'episode', 'human'
]:
# Render user-defined (former agent) mode state:
agent_state, title, box_text = self.parse_response(
state, mode, reward, info, done)
if self.render_state_as_image:
self.rgb_dict[mode] = self.draw_image(
agent_state,
figsize=self.render_size_state,
title='{} / {}'.format(mode, title),
box_text=box_text,
ylabel=self.render_ylabel,
xlabel=self.render_xlabel,
)
else:
self.rgb_dict[mode] = self.draw_plot(
agent_state,
figsize=self.render_size_state,
title='{} / {}'.format(mode, title),
box_text=box_text,
ylabel=self.render_ylabel,
xlabel=self.render_xlabel,
)
if 'human' in mode:
# Render `human` state:
human_state, title, box_text = self.parse_response(
raw_state, mode, reward, info, done)
self.rgb_dict['human'] = self.draw_plot(
human_state,
figsize=self.render_size_human,
title=title,
box_text=box_text,
ylabel='Price',
xlabel=self.render_xlabel,
line_labels=['Open', 'High', 'Low', 'Close'],
)
if send_img:
return self.rgb_dict
else:
# this case is for internal use only;
# now `mode` supposed to contain several modes, let's return dictionary of arrays:
return_dict = dict()
for entry in mode_list:
if entry in self.rgb_dict.keys():
# ...and it is legal:
return_dict[entry] = self.rgb_dict[entry]
else:
return_dict[entry] = self.rgb_empty()
return return_dict
def draw_plot(self,
data,
figsize=(10, 6),
title='',
box_text='',
xlabel='X',
ylabel='Y',
line_labels=None):
"""
Visualises environment state as 2d line plot.
Retrurns image as rgb_array.
Args:
data: np.array of shape [num_values, num_lines]
figsize: figure size (in.)
title:
box_text:
xlabel:
ylabel:
line_labels: iterable holding line legends as str
Returns:
rgb image as np.array of size [with, height, 3]
"""
if line_labels is None:
# If got no labels - make it numbers:
if len(data.shape) > 1:
line_labels = [
'line_{}'.format(i) for i in range(data.shape[-1])
]
else:
line_labels = ['line_0']
data = data[:, None]
else:
assert len(line_labels) == data.shape[-1], \
'Expected `line_labels` kwarg consist of {} names, got: {}'. format(data.shape[-1], line_labels)
fig = self.plt.figure(
figsize=figsize,
dpi=self.render_dpi,
)
#ax = fig.add_subplot(111)
self.plt.style.use(self.render_plotstyle)
self.plt.title(title)
# Plot x axis as reversed time-step embedding:
xticks = np.linspace(data.shape[0] - 1,
0,
int(data.shape[0]),
dtype=int)
self.plt.xticks(xticks.tolist(), (-xticks[::-1]).tolist(),
visible=False)
# Set every 5th tick label visible:
for tick in self.plt.xticks()[1][::5]:
tick.set_visible(True)
self.plt.xlabel(xlabel)
self.plt.ylabel(ylabel)
self.plt.grid(True)
# Switch off antialiasing:
#self.plt.setp([ax.get_xticklines() + ax.get_yticklines() + ax.get_xgridlines() + ax.get_ygridlines()],antialiased=False)
#self.plt.rcParams['text.antialiased']=False
# Add Info box:
self.plt.text(0, data.min(), box_text, **self.render_boxtext)
for line, label in enumerate(line_labels):
self.plt.plot(data[:, line], label=label)
self.plt.legend()
self.plt.tight_layout()
fig.canvas.draw()
# Save it to a numpy array:
rgb_array = np.fromstring(fig.canvas.tostring_rgb(),
dtype=np.uint8,
sep='')
# Clean up:
self.plt.close(fig)
#self.plt.gcf().clear()
return rgb_array.reshape(fig.canvas.get_width_height()[::-1] + (3, ))
def draw_image(self,
data,
figsize=(12, 6),
title='',
box_text='',
xlabel='X',
ylabel='Y',
line_labels=None):
"""
Visualises environment state as image.
Returns rgb_array.
"""
fig = self.plt.figure(
figsize=figsize,
dpi=self.render_dpi,
)
#ax = fig.add_subplot(111)
self.plt.style.use(self.render_plotstyle)
self.plt.title(title)
# Plot x axis as reversed time-step embedding:
xticks = np.linspace(data.shape[0] - 1,
0,
int(data.shape[0]),
dtype=int)
self.plt.xticks(xticks.tolist(), (-xticks[::-1]).tolist(),
visible=False)
# Set every 5th tick label visible:
for tick in self.plt.xticks()[1][::5]:
tick.set_visible(True)
#self.plt.yticks(visible=False)
self.plt.xlabel(xlabel)
self.plt.ylabel(ylabel)
self.plt.grid(False)
# Switch off antialiasing:
# self.plt.setp([ax.get_xticklines() + ax.get_yticklines() + ax.get_xgridlines() + ax.get_ygridlines()],antialiased=False)
# self.plt.rcParams['text.antialiased']=False
#self.log.warning('render_data_shape:{}'.format(data.shape))
# Add Info box:
self.plt.text(0, data.shape[1] - 1, box_text, **self.render_boxtext)
im = self.plt.imshow(data.T, aspect='auto', cmap=self.render_cmap)
self.plt.colorbar(im, use_gridspec=True)
self.plt.tight_layout()
fig.canvas.draw()
# Save it to a numpy array:
rgb_array = np.fromstring(fig.canvas.tostring_rgb(),
dtype=np.uint8,
sep='')
# Clean up:
self.plt.close(fig)
#self.plt.gcf().clear()
#ax.cla()
return rgb_array.reshape(fig.canvas.get_width_height()[::-1] + (3, ))
def draw_episode(self, cerebro):
"""
Hacky way to render episode.
Due to backtrader/matplotlib memory leaks have to encapsulate it in separate process.
Strange but reliable. PID's are driving crazy.
Args:
cerebro instance
Returns:
rgb array.
"""
draw_process = DrawCerebro(
cerebro=cerebro,
width=self.render_size_episode[0],
height=self.render_size_episode[1],
dpi=self.render_dpi,
result_pipe=self.in_pipe,
rowsmajor=self.render_rowsmajor_episode,
)
draw_process.start()
#print('Plotter PID: {}'.format(draw_process.pid))
try:
rgb_array = self.out_pipe.recv()
draw_process.terminate()
draw_process.join()
return rgb_array
except:
return self.rgb_empty()
def limit_function(func, wall_clock_time, mem_usage_limit, *args, **kwargs):
"""
:param func: the objective function to call.
:param wall_clock_time: seconds.
:param mem_usage_limit: megabytes.
:param args:
:param kwargs:
:return:
More Info about Memory Management [https://psutil.readthedocs.io/en/latest/#psutil.Process.memory_info]:
rss: aka “Resident Set Size”, this is the non-swapped physical memory a process has used.
On UNIX it matches “top“‘s RES column).
On Windows this is an alias for wset field and it matches “Mem Usage” column of taskmgr.exe.
vms: aka “Virtual Memory Size”, this is the total amount of virtual memory used by the process.
On UNIX it matches “top“‘s VIRT column.
On Windows this is an alias for pagefile field and it matches “Mem Usage” “VM Size” column of taskmgr.exe.
"""
_debug_mode = False
if 'debug' in kwargs:
_debug_mode = kwargs['debug']
# Deal with special case in Bayesian optimization.
if len(args) == 0 and 'args' in kwargs:
args = kwargs['args']
kwargs = kwargs['kwargs']
if _platform == 'Windows':
freeze_support()
parent_conn, child_conn = Pipe(False)
func = dill.dumps(func)
args = [func] + [child_conn] + [wall_clock_time] + [mem_usage_limit
] + list(args)
p = Process(target=wrapper, args=tuple(args), kwargs=kwargs)
p.start()
# Special case on windows.
if _platform in ['Windows', 'OSX', 'Linux']:
p_id = p.pid
exceed_mem_limit = False
start_time = time.time()
while time.time() <= start_time + wall_clock_time:
if not psutil.pid_exists(p_id) or psutil.Process(
p_id).status() == 'zombie':
break
rss_used = psutil.Process(p_id).memory_info().rss / 1024 / 1024
vms_used = psutil.Process(p_id).memory_info().vms / 1024 / 1024
if _debug_mode:
print(psutil.Process(p_id).memory_info())
print('mem[rss]_used', rss_used)
print('mem[vms]_used', vms_used)
threshold = rss_used if _platform in ['OSX', 'Linux', 'Windows'
] else vms_used
if threshold > mem_usage_limit:
exceed_mem_limit = True
break
time.sleep(1.)
if exceed_mem_limit:
clean_processes(p)
return Returns(status=False, result=OutOfMemoryLimitException)
if p.is_alive():
clean_processes(p)
return Returns(status=False, result=TimeoutException)
result = parent_conn.recv()
parent_conn.close()
return result
else:
p.join(wall_clock_time)
if p.is_alive():
clean_processes(p)
return Returns(status=False, result=TimeoutException)
result = parent_conn.recv()
parent_conn.close()
return result
class MarkupHandler:
TAG_NAME_ITALIC = 'italic'
TAG_NAME_BOLD = 'bold'
TAG_NAME_BOLD_ITALIC = 'bold_italic'
TAG_NAME_STRIKETHROUGH = 'strikethrough'
TAG_NAME_CENTER = 'center'
TAG_NAME_WRAP_NONE = 'wrap_none'
TAG_NAME_PLAIN_TEXT = 'plain_text'
TAG_NAME_GRAY_TEXT = 'gray_text'
TAG_NAME_CODE_TEXT = 'code_text'
TAG_NAME_CODE_BLOCK = 'code_block'
TAG_NAME_UNFOCUSED_TEXT = 'unfocused_text'
TAG_NAME_MARGIN_INDENT = 'margin_indent'
def __init__(self, text_view):
self.text_view = text_view
self.text_buffer = text_view.get_buffer()
self.marked_up_text = None
# Tags.
buffer = self.text_buffer
self.tag_italic = buffer.create_tag(self.TAG_NAME_ITALIC,
weight=Pango.Weight.NORMAL,
style=Pango.Style.ITALIC)
self.tag_bold = buffer.create_tag(self.TAG_NAME_BOLD,
weight=Pango.Weight.BOLD,
style=Pango.Style.NORMAL)
self.tag_bold_italic = buffer.create_tag(self.TAG_NAME_BOLD_ITALIC,
weight=Pango.Weight.BOLD,
style=Pango.Style.ITALIC)
self.tag_strikethrough = buffer.create_tag(self.TAG_NAME_STRIKETHROUGH,
strikethrough=True)
self.tag_center = buffer.create_tag(
self.TAG_NAME_CENTER, justification=Gtk.Justification.CENTER)
self.tag_wrap_none = buffer.create_tag(self.TAG_NAME_WRAP_NONE,
wrap_mode=Gtk.WrapMode.NONE,
pixels_above_lines=0,
pixels_below_lines=0)
self.tag_plain_text = buffer.create_tag(
self.TAG_NAME_PLAIN_TEXT,
weight=Pango.Weight.NORMAL,
style=Pango.Style.NORMAL,
strikethrough=False,
justification=Gtk.Justification.LEFT)
self.tag_gray_text = buffer.create_tag(self.TAG_NAME_GRAY_TEXT,
foreground='gray',
weight=Pango.Weight.NORMAL,
style=Pango.Style.NORMAL)
self.tag_code_text = buffer.create_tag(self.TAG_NAME_CODE_TEXT,
weight=Pango.Weight.NORMAL,
style=Pango.Style.NORMAL,
strikethrough=False)
self.tag_code_block = buffer.create_tag(self.TAG_NAME_CODE_BLOCK,
weight=Pango.Weight.NORMAL,
style=Pango.Style.NORMAL,
strikethrough=False,
indent=self.get_margin_indent(
0, 1)[1])
self.tags_markup = {
self.TAG_NAME_ITALIC:
lambda args: self.tag_italic,
self.TAG_NAME_BOLD:
lambda args: self.tag_bold,
self.TAG_NAME_BOLD_ITALIC:
lambda args: self.tag_bold_italic,
self.TAG_NAME_STRIKETHROUGH:
lambda args: self.tag_strikethrough,
self.TAG_NAME_CENTER:
lambda args: self.tag_center,
self.TAG_NAME_WRAP_NONE:
lambda args: self.tag_wrap_none,
self.TAG_NAME_PLAIN_TEXT:
lambda args: self.tag_plain_text,
self.TAG_NAME_GRAY_TEXT:
lambda args: self.tag_gray_text,
self.TAG_NAME_CODE_TEXT:
lambda args: self.tag_code_text,
self.TAG_NAME_CODE_BLOCK:
lambda args: self.tag_code_block,
self.TAG_NAME_MARGIN_INDENT:
lambda args: self.get_margin_indent_tag(*args)
}
# Focus mode.
self.tag_unfocused_text = buffer.create_tag(
self.TAG_NAME_UNFOCUSED_TEXT,
foreground='gray',
weight=Pango.Weight.NORMAL,
style=Pango.Style.NORMAL)
# Margin and indents.
# A baseline margin is set to allow negative offsets for formatting headers, lists, etc.
self.tags_margins_indents = {}
self.baseline_margin = 0
self.char_width = 0
self.update_margins_indents()
# Style.
self.on_style_updated()
# Worker process to handle parsing.
self.parsing = False
self.apply_pending = False
self.parent_conn, child_conn = Pipe()
Process(target=self.parse, args=(child_conn, ), daemon=True).start()
GLib.io_add_watch(self.parent_conn.fileno(), GLib.PRIORITY_DEFAULT,
GLib.IO_IN, self.on_parsed)
def on_style_updated(self, *_):
style_context = self.text_view.get_style_context()
(found, color) = style_context.lookup_color('code_bg_color')
if not found:
(_, color) = style_context.lookup_color('background_color')
self.tag_code_text.set_property("background", color.to_string())
self.tag_code_block.set_property("paragraph-background",
color.to_string())
def apply(self):
"""Applies markup, parsing it in a worker process if the text has changed.
In case parsing is already running, it will re-apply once it finishes. This ensure that
the pipe doesn't fill (and block) if multiple requests are made in quick succession."""
if not self.parsing:
self.parsing = True
self.apply_pending = False
text = self.text_buffer.get_slice(
self.text_buffer.get_start_iter(),
self.text_buffer.get_end_iter(), False)
if text != self.marked_up_text:
self.parent_conn.send(text)
else:
self.do_apply(text)
else:
self.apply_pending = True
def parse(self, child_conn):
"""Parses markup in a worker process."""
while True:
while True:
try:
text = child_conn.recv()
if not child_conn.poll():
break
except EOFError:
child_conn.close()
return
# List of tuples in the form (tag_name, tag_args, tag_start, tag_end).
result = []
# Find:
# - "_italic_" (italic)
# - "**bold**" (bold)
# - "***bolditalic***" (bold/italic)
# - "~~strikethrough~~" (strikethrough)
# - "`code`" (colorize)
# - "$math$" (colorize)
# - "---" table (wrap/pixels)
regexps = ((ITALIC, self.TAG_NAME_ITALIC),
(BOLD, self.TAG_NAME_BOLD), (BOLD_ITALIC,
self.TAG_NAME_BOLD_ITALIC),
(STRIKETHROUGH, self.TAG_NAME_STRIKETHROUGH),
(CODE, self.TAG_NAME_CODE_TEXT),
(MATH, self.TAG_NAME_CODE_TEXT),
(TABLE, self.TAG_NAME_WRAP_NONE))
for regexp, tag_name in regexps:
matches = re.finditer(regexp, text)
for match in matches:
result.append((tag_name, (), match.start(), match.end()))
# Find:
# - "[description](url)" (gray out)
# - "" (gray out)
regexps = ((LINK, self.TAG_NAME_GRAY_TEXT),
(IMAGE, self.TAG_NAME_GRAY_TEXT))
for regexp, tag_name in regexps:
matches = re.finditer(regexp, text)
for match in matches:
result.append(
(tag_name, (), match.start(), match.start("text")))
result.append(
(tag_name, (), match.end("text"), match.end()))
# Find "<url>" links (gray out).
matches = re.finditer(LINK_ALT, text)
for match in matches:
result.append(
(self.TAG_NAME_GRAY_TEXT, (), match.start("text"),
match.end("text")))
# Find "---" horizontal rule (center).
matches = re.finditer(HORIZONTAL_RULE, text)
for match in matches:
result.append(
(self.TAG_NAME_CENTER, (), match.start("symbols"),
match.end("symbols")))
# Find "* list" (offset).
matches = re.finditer(LIST, text)
for match in matches:
# Lists use character+space (eg. "* ").
length = 2
nest = len(match.group("indent").replace(" ", "\t"))
margin = -length - 2 * nest
indent = -length - 2 * length * nest
result.append((self.TAG_NAME_MARGIN_INDENT, (margin, indent),
match.start("content"), match.end("content")))
# Find "1. ordered list" (offset).
matches = re.finditer(ORDERED_LIST, text)
for match in matches:
# Ordered lists use numbers/letters+dot/parens+space (eg. "123. ").
length = len(match.group("prefix")) + 1
nest = len(match.group("indent").replace(" ", "\t"))
margin = -length - 2 * nest
indent = -length - 2 * length * nest
result.append((self.TAG_NAME_MARGIN_INDENT, (margin, indent),
match.start("content"), match.end("content")))
# Find "> blockquote" (offset).
matches = re.finditer(BLOCK_QUOTE, text)
for match in matches:
result.append((self.TAG_NAME_MARGIN_INDENT, (2, -2),
match.start(), match.end()))
# Find "# Header" (offset+bold).
matches = re.finditer(HEADER, text)
for match in matches:
margin = -len(match.group("level")) - 1
result.append((self.TAG_NAME_MARGIN_INDENT, (margin, 0),
match.start(), match.end()))
result.append(
(self.TAG_NAME_BOLD, (), match.start(), match.end()))
# Find "=======" header underline (bold).
matches = re.finditer(HEADER_UNDER, text)
for match in matches:
result.append(
(self.TAG_NAME_BOLD, (), match.start(), match.end()))
# Find "```" code block tag (offset + colorize paragraph).
matches = re.finditer(markup_regex.CODE_BLOCK, text)
for match in matches:
result.append((self.TAG_NAME_CODE_BLOCK, (),
match.start("block"), match.end("block")))
# Send parsed data back.
child_conn.send((text, result))
def on_parsed(self, _source, _condition):
"""Reads the parsing result from the pipe and triggers any pending apply."""
self.parsing = False
if self.apply_pending:
self.apply() # self.apply clears the apply pending flag.
try:
if self.parent_conn.poll():
self.do_apply(*self.parent_conn.recv())
return True
except EOFError:
return False
def do_apply(self, original_text, result=[]):
"""Applies the result of parsing if the current text matches the original text."""
buffer = self.text_buffer
start = buffer.get_start_iter()
end = buffer.get_end_iter()
text = self.text_buffer.get_slice(start, end, False)
# Apply markup tags.
if text == original_text and text != self.marked_up_text:
buffer.remove_tag(self.tag_italic, start, end)
buffer.remove_tag(self.tag_bold, start, end)
buffer.remove_tag(self.tag_bold_italic, start, end)
buffer.remove_tag(self.tag_strikethrough, start, end)
buffer.remove_tag(self.tag_center, start, end)
buffer.remove_tag(self.tag_plain_text, start, end)
buffer.remove_tag(self.tag_gray_text, start, end)
buffer.remove_tag(self.tag_code_text, start, end)
buffer.remove_tag(self.tag_code_block, start, end)
buffer.remove_tag(self.tag_wrap_none, start, end)
for tag in self.tags_margins_indents.values():
buffer.remove_tag(tag, start, end)
for tag_name, tag_args, tag_start, tag_end in result:
buffer.apply_tag(self.tags_markup[tag_name](tag_args),
buffer.get_iter_at_offset(tag_start),
buffer.get_iter_at_offset(tag_end))
# Apply focus mode tag (grey out before/after current sentence).
buffer.remove_tag(self.tag_unfocused_text, start, end)
if self.text_view.focus_mode:
cursor_iter = buffer.get_iter_at_mark(buffer.get_insert())
start_sentence = cursor_iter.copy()
start_sentence.backward_sentence_start()
end_sentence = cursor_iter.copy()
end_sentence.forward_sentence_end()
buffer.apply_tag(self.tag_unfocused_text, start, start_sentence)
buffer.apply_tag(self.tag_unfocused_text, end_sentence, end)
# Margin and indent are cumulative. They differ in two ways:
# * Margin is always in the beginning, which means it effectively only affects the first line
# of multi-line text. Indent is applied to every line.
# * Margin level can be negative, as a baseline margin exists from which it can be subtracted.
# Indent is always positive, or 0.
def get_margin_indent_tag(self, margin_level, indent_level):
level = (margin_level, indent_level)
if level not in self.tags_margins_indents:
margin, indent = self.get_margin_indent(margin_level, indent_level)
tag = self.text_buffer.create_tag("margin_indent_{}_{}".format(
margin_level, indent_level),
left_margin=margin,
indent=indent)
self.tags_margins_indents[level] = tag
return tag
else:
return self.tags_margins_indents[level]
def get_margin_indent(self,
margin_level,
indent_level,
baseline_margin=None,
char_width=None):
if baseline_margin is None:
baseline_margin = self.text_view.props.left_margin
if char_width is None:
char_width = helpers.get_char_width(self.text_view)
margin = max(baseline_margin + char_width * margin_level, 0)
indent = char_width * indent_level
return margin, indent
def update_margins_indents(self):
baseline_margin = self.text_view.props.left_margin
char_width = helpers.get_char_width(self.text_view)
# Bail out if neither the baseline margin nor character width change
if baseline_margin == self.baseline_margin and char_width == self.char_width:
return
self.baseline_margin = baseline_margin
self.char_width = char_width
# Adjust tab size
tab_array = Pango.TabArray.new(1, True)
tab_array.set_tab(0, Pango.TabAlign.LEFT, 4 * char_width)
self.text_view.set_tabs(tab_array)
# Adjust margins and indents
for level, tag in self.tags_margins_indents.items():
margin, indent = self.get_margin_indent(*level, baseline_margin,
char_width)
tag.set_properties(left_margin=margin, indent=indent)
def stop(self, *_):
self.parent_conn.close()
def run_forked(suite):
keep_alive_parent_end, keep_alive_child_end = Pipe(duplex=False)
result_parent_end, result_child_end = Pipe(duplex=False)
failed_parent_end, failed_child_end = Pipe(duplex=False)
child = Process(target=test_runner_wrapper,
args=(suite, keep_alive_child_end, result_child_end,
failed_child_end))
child.start()
last_test_temp_dir = None
last_test_vpp_binary = None
last_test = None
result = None
failed = set()
last_heard = time.time()
core_detected_at = None
debug_core = os.getenv("DEBUG", "").lower() == "core"
while True:
readable = select.select([
keep_alive_parent_end.fileno(),
result_parent_end.fileno(),
failed_parent_end.fileno(),
], [], [], 1)[0]
if result_parent_end.fileno() in readable:
result = result_parent_end.recv()
break
if keep_alive_parent_end.fileno() in readable:
while keep_alive_parent_end.poll():
last_test, last_test_vpp_binary,\
last_test_temp_dir, vpp_pid = keep_alive_parent_end.recv()
last_heard = time.time()
if failed_parent_end.fileno() in readable:
while failed_parent_end.poll():
failed_test = failed_parent_end.recv()
failed.add(failed_test.__name__)
last_heard = time.time()
fail = False
if last_heard + test_timeout < time.time() and \
not os.path.isfile("%s/_core_handled" % last_test_temp_dir):
fail = True
global_logger.critical("Timeout while waiting for child test "
"runner process (last test running was "
"`%s' in `%s')!" %
(last_test, last_test_temp_dir))
elif not child.is_alive():
fail = True
global_logger.critical("Child python process unexpectedly died "
"(last test running was `%s' in `%s')!" %
(last_test, last_test_temp_dir))
elif last_test_temp_dir and last_test_vpp_binary:
core_path = "%s/core" % last_test_temp_dir
if os.path.isfile(core_path):
if core_detected_at is None:
core_detected_at = time.time()
elif core_detected_at + core_timeout < time.time():
if not os.path.isfile(
"%s/_core_handled" % last_test_temp_dir):
global_logger.critical(
"Child python process unresponsive and core-file "
"exists in test temporary directory!")
fail = True
if fail:
failed_dir = os.getenv('VPP_TEST_FAILED_DIR')
lttd = last_test_temp_dir.split("/")[-1]
link_path = '%s%s-FAILED' % (failed_dir, lttd)
global_logger.error("Creating a link to the failed " +
"test: %s -> %s" % (link_path, lttd))
try:
os.symlink(last_test_temp_dir, link_path)
except Exception:
pass
api_post_mortem_path = "/tmp/api_post_mortem.%d" % vpp_pid
if os.path.isfile(api_post_mortem_path):
global_logger.error("Copying api_post_mortem.%d to %s" %
(vpp_pid, last_test_temp_dir))
shutil.copy2(api_post_mortem_path, last_test_temp_dir)
if last_test_temp_dir and last_test_vpp_binary:
core_path = "%s/core" % last_test_temp_dir
if os.path.isfile(core_path):
global_logger.error("Core-file exists in test temporary "
"directory: %s!" % core_path)
check_core_path(global_logger, core_path)
global_logger.debug("Running `file %s':" % core_path)
try:
info = check_output(["file", core_path])
global_logger.debug(info)
except CalledProcessError as e:
global_logger.error(
"Could not run `file' utility on core-file, "
"rc=%s" % e.returncode)
pass
if debug_core:
spawn_gdb(last_test_vpp_binary, core_path,
global_logger)
child.terminate()
result = -1
break
keep_alive_parent_end.close()
result_parent_end.close()
failed_parent_end.close()
return result, failed
def test_fork(self):
"""Test using a connection before and after a fork.
"""
if sys.platform == "win32":
raise SkipTest("Can't fork on Windows")
try:
from multiprocessing import Process, Pipe
except ImportError:
raise SkipTest("No multiprocessing module")
db = self._get_connection().pymongo_test
# Failure occurs if the connection is used before the fork
db.test.find_one()
#db.connection.end_request()
def loop(pipe):
while True:
try:
db.test.insert({"a": "b"}, safe=True)
for _ in db.test.find():
pass
except:
traceback.print_exc()
pipe.send(True)
os._exit(1)
cp1, cc1 = Pipe()
cp2, cc2 = Pipe()
p1 = Process(target=loop, args=(cc1, ))
p2 = Process(target=loop, args=(cc2, ))
p1.start()
p2.start()
p1.join(1)
p2.join(1)
p1.terminate()
p2.terminate()
p1.join()
p2.join()
cc1.close()
cc2.close()
# recv will only have data if the subprocess failed
try:
cp1.recv()
self.fail()
except EOFError:
pass
try:
cp2.recv()
self.fail()
except EOFError:
pass
db.connection.close()
class mygui:
def __init__(self):
self.window = Tk()
self.stop = Button(self.window,
text="Stop Recording",
command=self.stopclick)
self.record = Button(self.window,
text='Record',
command=self.recordclick)
self.terminate = Button(self.window,
text="Exit",
command=self.exitclick)
self.input = Entry(self.window)
self.stop['state'] = 'disabled'
#processes
self.training = None
self.running = None
#communication pipes
self.trainingpipe = None
self.runningpipe = None
self.runningpipe, childpipe = Pipe()
self.running = Process(target=mainloop, args=(childpipe, ))
self.running.start()
def pipelistener(self):
print('running pipelistener')
if self.runningpipe.poll(0.001) is not None:
signal = self.runningpipe.recv()
print(signal)
def buildgui(self):
self.record.place(x=1, y=50)
self.input.place(x=1, y=75)
self.stop.place(x=1, y=100)
self.terminate.place(x=1, y=125)
self.window.mainloop()
def learnclick(self):
print('learnclick')
self.runningpipe.send('LEARN')
def exitclick(self):
print('stopclick')
self.stop['state'] = 'disabled'
self.record['state'] = 'normal'
#other actions...
if self.running is not None and self.running.exitcode is None:
self.runningpipe.send('STOP')
self.running.join()
self.window.quit()
def stopclick(self):
print('stoprecording')
self.stop['state'] = 'disabled'
self.record['state'] = 'normal'
self.runningpipe.send('STOPCACHE')
print('STOPCACHE')
print('Attempting to Learn')
self.runningpipe.send('LEARN')
def recordclick(self):
print('recordclick')
#other actions
lbl = self.input.get()
if lbl is None or len(lbl) == 0:
return
self.stop['state'] = 'normal'
self.record['state'] = 'disabled'
print('sending pipe')
self.runningpipe.send('label:' + lbl)
))
agent = Process(target=agent_main,
args=(0, info_pipe_agent, action_pipe_agent, input_agent,
need_agent))
server.start()
time.sleep(1)
agent.start()
if use_keyboard:
actions_list = []
in_action = True
while True:
try:
if in_action:
x = need_main.recv()
print('receive from client:', x)
in_action = False
if len(actions_list) == 0:
if agent_action_type == 'continues':
if continues_action_type == 'xy':
actions = input(
'enter action or actions like \'-0.001 0.002\' or \'0.001 0.001 0.001 -0.001\':'
)
elif continues_action_type == 'ar':
actions = input(
'enter action or actions like \'92 0.002\' or \'-175 0.001 80 -0.001\':'
)
if len(actions) > 0:
actions = actions.split(' ')
from multiprocessing import Process, Pipe
import os, time
#创建管道
fd2, fd1 = Pipe(False)
def fun(name):
time.sleep(3)
#向管道写入内容
fd1.send("hello" + str(name)) #传什么都可以
jobs = []
for i in range(5):
p = Process(target=fun, args=(i, ))
jobs.append(p)
p.start()
for i in range(5):
#读取管道
data = fd2.recv()
print(data)
for i in jobs:
i.join()
def run_forked(suite):
keep_alive_parent_end, keep_alive_child_end = Pipe(duplex=False)
result_parent_end, result_child_end = Pipe(duplex=False)
failed_parent_end, failed_child_end = Pipe(duplex=False)
child = Process(target=test_runner_wrapper,
args=(suite, keep_alive_child_end, result_child_end,
failed_child_end))
child.start()
last_test_temp_dir = None
last_test_vpp_binary = None
last_test = None
result = None
failed = set()
last_heard = time.time()
while True:
readable = select.select([
keep_alive_parent_end.fileno(),
result_parent_end.fileno(),
failed_parent_end.fileno(),
], [], [], 1)[0]
if result_parent_end.fileno() in readable:
result = result_parent_end.recv()
break
if keep_alive_parent_end.fileno() in readable:
while keep_alive_parent_end.poll():
last_test, last_test_vpp_binary,\
last_test_temp_dir, vpp_pid = keep_alive_parent_end.recv()
last_heard = time.time()
if failed_parent_end.fileno() in readable:
while failed_parent_end.poll():
failed_test = failed_parent_end.recv()
failed.add(failed_test.__name__)
last_heard = time.time()
fail = False
if last_heard + test_timeout < time.time():
fail = True
global_logger.critical("Timeout while waiting for child test "
"runner process (last test running was "
"`%s' in `%s')!" %
(last_test, last_test_temp_dir))
elif not child.is_alive():
fail = True
global_logger.critical("Child process unexpectedly died (last "
"test running was `%s' in `%s')!" %
(last_test, last_test_temp_dir))
if fail:
failed_dir = os.getenv('VPP_TEST_FAILED_DIR')
lttd = last_test_temp_dir.split("/")[-1]
link_path = '%s%s-FAILED' % (failed_dir, lttd)
global_logger.error("Creating a link to the failed " +
"test: %s -> %s" % (link_path, lttd))
os.symlink(last_test_temp_dir, link_path)
api_post_mortem_path = "/tmp/api_post_mortem.%d" % vpp_pid
if os.path.isfile(api_post_mortem_path):
global_logger.error("Copying api_post_mortem.%d to %s" %
(vpp_pid, last_test_temp_dir))
shutil.copy2(api_post_mortem_path, last_test_temp_dir)
if last_test_temp_dir and last_test_vpp_binary:
core_path = "%s/core" % last_test_temp_dir
if os.path.isfile(core_path):
global_logger.error("Core-file exists in test temporary "
"directory: %s!" % core_path)
if d and d.lower() == "core":
spawn_gdb(last_test_vpp_binary, core_path,
global_logger)
child.terminate()
result = -1
break
keep_alive_parent_end.close()
result_parent_end.close()
failed_parent_end.close()
return result, failed
def on_any_event(event):
print("Watcher: Event found: {}".format(event.event_type))
logging.debug("Watcher: Event found: {}".format(event.event_type))
logging.debug("Directory: {}".format(os.path.basename(event.src_path)))
if event.is_directory:
return None
# Windows is created
# Linux is modified
elif event.event_type == 'modified' or event.event_type == 'created':
logging.debug(
'-------------------------------------------------------------------'
)
time.sleep(1)
recv_end, send_end = Pipe(False)
# Take any action here when a file is first created.
print(
"Received event {} for file {} - Beginning render job.".format(
event.event_type, event.src_path))
logging.debug(
"Received event {} for file {} - Beginning render job.".format(
event.event_type, event.src_path))
# Testing print
f = os.path.relpath(event.src_path, Watcher.DIRECTORY_TO_WATCH)
pid, rend_type, other = f.split("_", 2)
logging.debug("Project ID: {}".format(pid))
logging.debug("Render Type: {}".format(rend_type))
print(other)
logging.debug("File found: {}".format(f))
logging.debug("Source path: {}".format(event.src_path))
logging.debug("Render type: {}".format(rend_type))
if rend_type == 'chunk':
# Open the file for reading
json_file = open(event.src_path, 'r')
json_data = json.load(json_file)
# If job hasn't been done
if json_data['status'] == False:
# Get the ID and run the render on it
proj_id = json_data["id"]
print("Project ID is {}".format(proj_id))
logging.debug("Project ID is {}".format(proj_id))
logging.debug("Starting render serivce...")
compress = False
chunk = True
logging.debug(
"Chunk render instance for {} requested".format(
proj_id))
p = Process(target=chunk_render.get_chunk,
args=(
proj_id,
send_end,
compress,
chunk,
))
p.start()
p.join()
render_return = recv_end.recv()
render_status = render_return[1]
info = render_return[0]
# Update the complete time at the end and dump it to file
logging.debug("Updating JSON status file")
json_data['dateCompleted'] = datetime.now().strftime(
"%d-%b-%Y (%H:%M:%S)")
json_data['status'] = True
json_data['correctlyRendered'] = render_status
json_data['otherInfo'] = info
logging.debug("JSON Data for {}:".format(proj_id))
logging.debug(json_data)
with open(event.src_path, "w") as json_write:
json.dump(json_data, json_write)
if render_status == 1:
logging.debug("File written successfully.")
print("File written successfully")
return
else:
logging.debug(
"Job completed, but an error may have occured during render service"
)
return
else:
logging.debug("File already rendered")
print("File already rendered")
return
else:
print("[watcher_chunk.py] different render type")
logging.debug("[watcher_chunk.py] different render type")
def main(use_multiprocessing=False, log_level=None):
"""
Create and run a tello controller :
1) get the video stream from the tello
2) wait for keyboard commands to pilot the tello
3) optionnally, process the video frames to track a body and pilot the tello accordingly.
If use_multiprocessing is True, the parent process creates a child process ('worker')
and the workload is shared between the 2 processes.
The parent process job is to:
- get the video stream from the tello and displays it in an OpenCV window,
- write each frame in shared memory at destination of the child,
each frame replacing the previous one (more efficient than a pipe or a queue),
- read potential command from the child (currently, only one command:EXIT).
Commands are transmitted by a Pipe.
The child process is responsible of all the others tasks:
- process the frames read in shared memory (openpose, write_hud),
- if enable, do the tracking (calculate drone commands from position of body),
- read keyboard commands,
- transmit commands (from tracking or from keyboard) to the tello, and receive message from the tello.
"""
global tello
if use_multiprocessing:
# Create the pipe for the communication between the 2 processes
parent_cnx, child_cnx = Pipe()
else:
child_cnx = None
tello = TelloController(use_face_tracking=True,
kbd_layout="AZERTY",
write_log_data=False,
log_level=log_level,
child_cnx=child_cnx)
first_frame = True
frame_skip = 300
for frame in tello.container.decode(video=0):
if 0 < frame_skip:
frame_skip = frame_skip - 1
continue
start_time = time.time()
if frame.time_base < 1.0 / 60:
time_base = 1.0 / 60
else:
time_base = frame.time_base
# Convert frame to cv2 image
frame = cv2.cvtColor(np.array(frame.to_image(), dtype=np.uint8),
cv2.COLOR_RGB2BGR)
frame = cv2.resize(frame, (640, 480))
if use_multiprocessing:
if first_frame:
# Create the shared memory to share the current frame decoded by the parent process
# and given to the child process for further processing (openpose, write_hud,...)
frame_as_ctypes = np.ctypeslib.as_ctypes(frame)
tello.shared_array = sharedctypes.RawArray(
frame_as_ctypes._type_, frame_as_ctypes)
tello.frame_shape = frame.shape
first_frame = False
# Launch process child
p_worker = Process(target=openpose_worker)
p_worker.start()
# Write the current frame in shared memory
tello.shared_array[:] = np.ctypeslib.as_ctypes(frame.copy())
# Check if there is some message from the child
if parent_cnx.poll():
msg = parent_cnx.recv()
if msg == "EXIT":
print("MAIN EXIT")
p_worker.join()
tello.drone.quit()
cv2.destroyAllWindows()
exit(0)
else:
frame = tello.process_frame(frame)
tello.sound_player.play()
if not use_multiprocessing: tello.fps.update()
# Display the frame
cv2.imshow('Tello', frame)
cv2.waitKey(1)
frame_skip = int((time.time() - start_time) / time_base)
