class MMLWorker(object):
"""Represents the Daemon's connection to the subprocess"""
def __init__(self,runner):
"""Creates and initalizes a subprocess and its connections."""
self.pipe, pipe = Pipe()
self.proc = Process(target=worker, args=(pipe,runner))
self.proc.start();
self.pid = self.proc.pid
def __del__(self):
"""Ensures the subprocess is correctly garbage collected."""
self.pipe.close();
self.proc.terminate();
def pump(self,block=False):
"""Returns a key,val pair from the subprocess, or None,None."""
key,val = None,None
if block:
(key,val) = self.pipe.recv()
elif self.pipe.poll():
(key,val) = self.pipe.recv()
return key,val
def stop(self):
"""Sends the stop signal to the subprocess."""
self.pipe.send(('stop',()))
def pause(self):
"""Sends the pause signal to the subprocess."""
self.pipe.send(('pause',()))
def start(self,program):
"""Sends the start signal to the subprocess."""
self.pipe.send(('start',(program,)))
def dispatch(self, fcn, timeout=0, params=()):
"""Start a new subrocess.
This function will detach 'fcn' as a process of its own,
passing params AND a pipe as arguments to 'fcn'.
If 'timeout' > 0 this function will block, waiting for the
detached process to send some data over the pipe.
If data i written to pipe in time, the pipe will be returned to the calling process.
On timeout, the pipe will be closed and 'None' will be returned to the calling process.
"""
conn, child_conn = Pipe()
p = Process(target=fcn, args=params + (child_conn,))
p.start()
if timeout > 0:
poll_status = conn.poll(timeout)
if poll_status == False:
print "Dispatched function %s did not send anything within the specified timeout (%d s)" % (fcn, timeout)
# FIXME: How to properly handle this case?
# - p.terminate() doesn't terminate any subprocesses the p might have spawned
# - conn.close(), i.e. closing the control channel, is probably better. Make sure p detects and handle it.
# - send a proper control message (set running to False) should work too.
# The proper way to implement a client is to handle connection.close() and stop running control message in the same way.
conn.close()
conn = None
return conn
def transcode(self, path, format='mp3', bitrate=False):
if self.stopping.is_set():
return
try:
stop = Event()
start_time = time.time()
parent_conn, child_conn = Pipe()
process = Process(target=transcode_process,
args=(child_conn, path, stop, format, bitrate))
process.start()
while not (self.stopping.is_set() or stop.is_set()):
data = parent_conn.recv()
if not data:
break
yield data
logger.debug("Transcoded %s in %0.2f seconds." % (path.encode(cfg['ENCODING']), time.time() - start_time))
except GeneratorExit:
stop.set()
logger.debug("User canceled the request during transcoding.")
except:
stop.set()
logger.warn("Some type of error occured during transcoding.")
finally:
parent_conn.close()
process.join()
class ProcessHandler:
'''
run(): The run() method is the entry point for a thread.
start(): The start() method starts a thread by calling the run method.
join([time]): The join() waits for threads to terminate.
isAlive(): The isAlive() method checks whether a thread is still executing.
'''
def __init__(self, daemonic, pipe):
self.daemonic = daemonic
self.pipe = pipe
if self.pipe:
self.parent_conn, self.child_conn = Pipe(duplex=False)
@abc.abstractmethod
def run(self, *args):
pass
def start(self, *args):
# Close write fd because parent not going to write
if not self.pipe:
self.process = Process(target=self.run, args=args)
else:
self.process = Process(
target=self.run, args=(self.child_conn,) + args)
if self.daemonic:
self.process.daemon = True
self.process.start()
def join(self):
if self.pipe:
self.parent_conn.close()
self.child_conn.close()
self.process.join()
"""
def process_pipe():
parent_conn, child_conn = Pipe()
p = Process(target=pipe_test, args=(child_conn,))
p.start()
print parent_conn.recv()
p.join()
parent_conn.close()
def test_queue_pipe():
print 'testing for pipe:'
for count in [10, 10**4, 10**5]:
output_p, input_p = Pipe()
#break
reader_p = Process(target=reader_pipe, args=((output_p, input_p),))
reader_p.start()
output_p.close()
_start = time.time()
writer_pipe(count, input_p)
input_p.close()
reader_p.join()
print 'Sending %s numbers to Pipe() took %s seconds' % (count, (time.time() - _start))
break
print "testing for queue:"
for count in [10**3, 10**4, 10**5]:
queue = Queue()
reader_p = Process(target=reader_queue, args=((queue),))
reader_p.daemon = True
reader_p.start()
_start = time.time()
writer_queue(count, queue)
reader_p.join()
print "Sending %s numbers to Queue() took %s seconds" % (count, (time.time() - _start))
def run(self):
parent_pipe, child_pipe = Pipe(False)
p = Process(target = run_metaheuristic,
args = (child_pipe, self.model, self.pt, self.aa,
self.algo, self.n, self.use_heur,
self.worst, self.best))
p.start()
for i in range(self.n + 1):
if self.is_stopped() is True:
parent_pipe.close()
p.terminate()
return
try:
result = parent_pipe.recv()
except:
break
self.results.append(result[0])
self.fitness.append(result[1])
self.emit(QtCore.SIGNAL('update(int)'), i)
if result[1] == 1:
break
parent_pipe.close()
p.join()
def calculateArea(feature,session_cookies,options):
"""
return:{
status {
"invalid" : invalid message;
"failed" : failed message;
"overlapped" : overlap message
}
data: {
total_area: 100 //exist if status_code = 1
other_area: 10 //exist if status_code = 1 and len(layers) > 0
layers: { //exist if status_code = 1 and len(layers) > 0
layer id: {
total_area: 12
areas:[
{area:1, properties:{
name:value
}}
]
}
}
}
}
"""
parent_conn,child_conn = Pipe(True)
p = Process(target=calculateAreaInProcess,args=(child_conn,))
p.daemon = True
p.start()
parent_conn.send([feature,session_cookies,options])
result = parent_conn.recv()
parent_conn.close()
#p.join()
#print("{}:get the area result from other process".format(datetime.now()))
return result
class PluginRunner:
def __init__(self, plugin):
self.name = plugin
self.proc = None
self.running = False
self.local_pipe, self.remote_pipe = Pipe()
def getConnection(self):
return self.local_pipe
def start(self):
assert not self.running, "Already running."
self.running = True
self.thread = Thread(target=self.run)
self.thread.start()
def restart(self):
self.proc.terminate()
def stop(self):
assert self.running, "Running"
self.running = False
self.proc.terminate()
self.thread.join()
self.remote_pipe.close()
self.local_pipe.close()
def run(self):
while self.running:
self.proc = Process(target=launch, args=('repeat', self.remote_pipe))
self.proc.start()
print("Waiting on proc to end")
self.proc.join()
class IndexProxy(Thread):
def __init__(self, network, indexdir, cmdprefix):
Thread.__init__(self)
self.module_p, index_p = Pipe()
self.proc = IndexProcess(network, indexdir, index_p)
self.proc.start()
self.inqueue = Queue() # thread.ident, query/data
self.waiting = {} #threadID : queue
self.cmdprefix = cmdprefix
def run(self):
procpipe = self.module_p
while True:
# process module calls
try: type, data = self.inqueue.get(timeout=0.2)
except Empty: pass
else:
try:
#process queued item
if type == STOP:
self.module_p.close()
break
elif type == QUERY:
resq, threadident = data[0:2]
data = data[1:]
self.waiting[threadident] = resq
procpipe.send((type, data))
else:
procpipe.send((type, data))
except:
print_exc()
prnt("IndexProxy Exception in pump.")
# process pipe data
while procpipe.poll():
tid, result = procpipe.recv()
try:
self.waiting.pop(tid).put(result)
except KeyError:
prnt("WAITING THREAD ID NOT FOUND FOR RESULT:"+repr(result))
for queue in self.waiting.itervalues():
queue.put(None)
def search(self, source, query, n, gb=None):
""" Will return None if shutdown before response ready."""
resultq = Queue()
self.inqueue.put((QUERY, (resultq, current_thread().ident, source, query, n, gb)))
return resultq.get()
def logmsg(self, *args):
# Ignore all lines that start with commandprefix, but allow things like "... (etc)"
if args[-1][0] == self.cmdprefix and args[-1][1] != self.cmdprefix: return
self.inqueue.put((LOG, args))
def stop(self):
self.inqueue.put((STOP, None))
# old, new
def rename(self, *args):
self.inqueue.put((RENAME, args))
def _ask(self, msg, args=(), kwargs={}):
i, o = Pipe()
reduced = reduce_connection(i)
self.inbox.put([msg, args, kwargs, reduced[1]])
ret = o.recv()
i.close()
o.close()
return ret
def _proxy_loop(self, broker, *args):
is_debug = partial(log.isEnabledFor, logging.DEBUG)
pid = os.getpid()
proc = None
queue = self.queue
def stop():
try:
if proc is not None:
child.send(STOP)
child.close()
proc.join()
except Exception:
log.error('%s failed to stop cleanly',
str(self), exc_info=True)
raise
else:
log.debug('terminated %s', str(self))
finally:
self.pid = '%s-terminated' % self.pid
while True:
if proc is None or not proc.is_alive():
# start new worker process
child, parent = Pipe()
cx = _reduce_connection(parent) # HACK reduce for pickle
proc = run_in_subprocess(worker_process, pid, cx, *args)
self.pid = proc.pid
task, return_to_pool = queue.get()
if task == STOP:
stop()
break
try:
child.send(task)
while not child.poll(task.heartrate):
if not proc.is_alive():
broker.task_failed(task)
raise Error('unknown cause of death')
broker.heartbeat(task)
(result, status) = child.recv()
broker.set_result(task, result)
except Exception:
log.error('%s died unexpectedly', str(self), exc_info=True)
child.close()
proc.stdin.close()
proc = None
else:
if is_debug():
log.debug('%s completed task', str(self))
if status == STOP:
child.close()
proc.stdin.close()
proc = None
finally:
return_to_pool(self)
def test_mount_fstab_user_fail(self):
if not self._can_create:
self.skipTest('Cannot create %s filesystem' % self._fs_name)
if not self._can_mount:
self.skipTest('Cannot mount %s filesystem' % self._fs_name)
# this test will change /etc/fstab, we might want to revert the changes after it finishes
fstab = self.read_file('/etc/fstab')
self.addCleanup(self.write_file, '/etc/fstab', fstab)
disk = self.get_object('/block_devices/' + os.path.basename(self.vdevs[0]))
self.assertIsNotNone(disk)
# create filesystem
disk.Format(self._fs_name, self.no_options, dbus_interface=self.iface_prefix + '.Block')
self.addCleanup(self._clean_format, disk)
# create user for our test
self.addCleanup(self._remove_user, self.username)
uid, gid = self._add_user(self.username)
# add unmount cleanup now in case something wrong happens in the other process
self.addCleanup(self._unmount, self.vdevs[0])
# create pipe to get error (if any)
parent_conn, child_conn = Pipe()
proc = Process(target=self._mount_as_user_fstab_fail, args=(child_conn, int(uid), int(gid), self.vdevs[0]))
proc.start()
res = parent_conn.recv()
parent_conn.close()
proc.join()
if not res[0]:
self.fail(res[1])
# now mount it as root and test that user can't unmount it
mnt_path = disk.Mount(self.no_options, dbus_interface=self.iface_prefix + '.Filesystem')
self.assertIsNotNone(mnt_path)
self.assertTrue(os.path.ismount(mnt_path))
# create pipe to get error (if any)
parent_conn, child_conn = Pipe()
proc = Process(target=self._unmount_as_user_fstab_fail, args=(child_conn, int(uid), int(gid), self.vdevs[0]))
proc.start()
res = parent_conn.recv()
parent_conn.close()
proc.join()
if not res[0]:
self.fail(res[1])
self.assertTrue(os.path.ismount(mnt_path))
self._unmount(mnt_path)
class _Tracker(Process):
"""Background process for tracking resource usage"""
def __init__(self, dt=1):
Process.__init__(self)
self.daemon = True
self.dt = dt
self.parent_pid = current_process().pid
self.parent_conn, self.child_conn = Pipe()
def shutdown(self):
if not self.parent_conn.closed:
self.parent_conn.send('shutdown')
self.parent_conn.close()
self.join()
def _update_pids(self, pid):
return [self.parent] + [p for p in self.parent.children()
if p.pid != pid and p.status() != 'zombie']
def run(self):
psutil = import_required("psutil", "Tracking resource usage requires "
"`psutil` to be installed")
self.parent = psutil.Process(self.parent_pid)
pid = current_process()
data = []
while True:
try:
msg = self.child_conn.recv()
except KeyboardInterrupt:
continue
if msg == 'shutdown':
break
elif msg == 'collect':
ps = self._update_pids(pid)
while not data or not self.child_conn.poll():
tic = default_timer()
mem = cpu = 0
for p in ps:
try:
mem2 = p.memory_info().rss
cpu2 = p.cpu_percent()
except Exception: # could be a few different exceptions
pass
else:
# Only increment if both were successful
mem += mem2
cpu += cpu2
data.append((tic, mem / 1e6, cpu))
sleep(self.dt)
elif msg == 'send_data':
self.child_conn.send(data)
data = []
self.child_conn.close()
class guiconnector():
def __init__(self, console, observer, modelsready):
self.console = console
self.observer = observer
self.modelsready = modelsready
# Starts up data structures
def start(self, params):
self.a, self.b = Pipe(duplex=True)
if __name__ == "projectb.maingui":
self.p = Process(target=BayesianOptProcess, kwargs={
"params": parseintosimple(params),
"pipein": self.a,
"pipeout": self.b
})
self.t = threading.Thread(target=self.bayesoptlistener)
self.t.start()
self.p.start()
# Listnere for data from the BayesOpt Process
def bayesoptlistener(self):
pipein = self.a
pipeout = self.b
stop = False;
while not stop:
if pipein.poll():
output = pipein.recv()
if output.has_key("stop_connector"):
break
if output.has_key("stopped_bayes"):
self.modelsready(output["stopped_bayes"])
for k, v in output.items():
if k == "console":
self.console.log(v["text"],v["verbose"])
else:
self.observer.updatevar(k, v)
# Request the BayesOpt Process for the posterior of the given model
def queryposterior(self, xdata, modelid):
self.a.send({"posterior": xdata, "modelid": modelid})
# Finish BayesOpt
def endbayesopt(self):
self.a.send({"stop": True, "stop_posterior_connector": True})
self.console.log("Attempting to stop the Bayesian Optimization")
self.console.log("Please Wait...")
self.console.log("Saving the data...")
# Clear up the processes and close the pipes
def closestage(self):
self.b.send({"stop_connector": True})
self.endbayesopt()
self.a.close()
self.b.close()
self.p.terminate()
class DataPump:
def __init__(self, fname, t0=0.0):
"""
"""
self.fname = fname
self._data_end, self._control_end = Pipe()
self.process = Process(target=self._read_data, args=())
self.process.start()
self._control_end.send((True, t0))
if self._control_end.poll(5.):
self.dt = self._control_end.recv()
else:
warnings.warn("dt could not be retrived from video, waited 5 s",
RuntimeWarning)
self.dt = np.nan
def _read_data(self):
"""
"""
vr = VideoReader(self.fname, color=False)
self._data_end.send(vr.dt_ns * 1e-9)
running = True
while running:
if self._data_end.poll(): # See if msg sent from get_data
running, next_t = self._data_end.recv() # get the time of the frame to read (from get_data)
if running:
data = vr.get_frame(next_t) # Read video frame
curr_t = vr.get_current_position(fmt='time') # get time of frame from video, should be very close to self.data_t
self._data_end.send((data, curr_t)) # Send data via the pipe to get_data
vr.close()
def get_data(self, next_t):
"""
Ask for a future frame and returns the previously asked for.
"""
# Get previous frame and time of frame via the pipe from self._read_data
data, curr_t = self._control_end.recv()
# Tell self._read_data to read a new frame at time next_t
self._control_end.send((True, next_t))
return data, curr_t
def close(self):
"""
"""
self._control_end.send((False, None))
self._control_end.close()
self._data_end.close()
self.process.join()
def _start(self, chunked_things):
for i in range(self.nb_process):
local_read_pipe, local_write_pipe = Pipe(duplex=False)
process_read_pipe, process_write_pipe = Pipe(duplex=False)
self.readers_pipes.append(local_read_pipe)
self.writers_pipes.append(process_write_pipe)
p = Process(target=run_keeped_process, args=(self.target, local_write_pipe, process_read_pipe, chunked_things[i]))
p.start()
self.processs.append(p)
local_write_pipe.close()
process_read_pipe.close()
def _start(self, pipe_package):
for i in range(self.nb_process):
local_read_pipe, local_write_pipe = Pipe(duplex=False)
process_read_pipe, process_write_pipe = Pipe(duplex=False)
self.readers_pipes.append(local_read_pipe)
self.writers_pipes.append(process_write_pipe)
pipe_package.setCurrentProcessId(i)
p = Process(target=run_keeped_process, args=(self.target, local_write_pipe, process_read_pipe, pipe_package))
p.start()
self.processs.append(p)
local_write_pipe.close()
process_read_pipe.close()
def main():
for count in [10**4, 10**5, 10**6]:
output_p, input_p = Pipe()
reader_p = Process(target=reader, args=((output_p, input_p),))
reader_p.start() # Launch the reader process
output_p.close() # We no longer need this part of the Pipe()
_start = time.time()
writer(count, input_p) # Send a lot of stuff to reader()
input_p.close() # Ask the reader to stop when it reads EOF
reader_p.join()
print "Sending %s numbers to Pipe() took %s seconds" % (count, (time.time() - _start))
def solve_paths(self, test_path_regex, *test_path_regexes, **kwargs):
# We need to create a new process to avoid importing the modules
# in the parent process
solve = self._build_solve_paths(test_path_regex, *test_path_regexes,
**kwargs)
parent_conn, child_conn = Pipe(duplex=False)
p = Process(target=solve, args=(child_conn,))
p.start()
test_paths, partial_reloads = parent_conn.recv()
parent_conn.close()
p.join()
return test_paths, partial_reloads
def start_kernel(self, kernel_id=None, config=None, resource_limits=None):
""" A function for starting new kernels by forking.
:arg str kernel_id: the id of the kernel to be started.
If no id is passed, a uuid will be generated.
:arg Ipython.config.loader config: kernel configuration.
:arg dict resource_limits: a dict with keys resource.RLIMIT_*
(see config_default documentation for explanation of valid options)
and values of the limit for the given resource to be set in the
kernel process
:returns: kernel id and connection information which includes the
kernel's ip, session key, and shell, heartbeat, stdin, and iopub
port numbers
:rtype: dict
"""
kernel_logger.debug("start_kernel with config %s", config)
if kernel_id is None:
kernel_id = str(uuid.uuid4())
if config is None:
config = Config({"ip": self.ip})
if resource_limits is None:
resource_limits = {}
config.HistoryManager.enabled = False
dir = os.path.join(self.dir, kernel_id)
try:
os.mkdir(dir)
except OSError:
# TODO: take care of race conditions and other problems with us
# using an 'unclean' directory
pass
currdir = os.getcwd()
os.chdir(dir)
p, q = Pipe()
proc = Process(target=self.fork_kernel, args=(config, q, resource_limits))
proc.start()
os.chdir(currdir)
# todo: yield back to the message processing while we wait
for i in range(5):
if p.poll(1):
connection = p.recv()
p.close()
self.kernels[kernel_id] = (proc, connection)
return {"kernel_id": kernel_id, "connection": connection}
else:
kernel_logger.info("Kernel %s did not start after %d seconds."
% (kernel_id[:4], i))
p.close()
self.kill_process(proc)
raise KernelError("Kernel start timeout.")
class Plugin:
def __init__(self, name, onMessage):
self.running = False
self.local_pipe, self.remote_pipe = Pipe()
self.name = name
self.onMessage = onMessage
self.start()
def start(self):
assert not self.running, "Already running."
self.running = True
self.thread = Thread(target=self.run)
self.thread.start()
Thread(target=self.reader).start()
def reader(self):
while self.running:
print("Checking for plugin output")
r, _, _ = select.select([self.local_pipe],[],[],.5)
print(r)
if r:
print("Available!")
got = self.local_pipe.recv()
print("Read:", got)
self.onMessage(got)
def restart(self):
self.proc.terminate()
def stop(self):
assert self.running, "Running"
self.running = False
self.remote_pipe.close()
self.local_pipe.close()
self.proc.terminate()
self.thread.join()
def run(self):
while self.running:
print("Staring %s" % self.name)
self.proc = multiprocessing.Process(
target=__pluginLauncher__,
args=(self.remote_pipe, self.name))
self.proc.start()
self.proc.join()
print("Exited %s" % self.name)
def send(self, line):
self.local_pipe.send(line)
def run(self):
to_child, to_self = Pipe()
try:
self.child.start(to_self)
result = to_child.recv()
self.child.join()
except:
result = sys.exc_info()
finally:
to_child.close()
to_self.close()
if result == '':
self.cb(None)
else:
self.cb(result)
def start_kernel(self, sage_dict=None, kernel_id=None, config=None):
random.seed()
if sage_dict is None:
sage_dict = {}
if kernel_id is None:
kernel_id = str(uuid.uuid4())
if config is None:
config = Config()
p, q = Pipe()
proc = Process(target=self.fork_kernel, args=(sage_dict, config, q))
proc.start()
connection = p.recv()
p.close()
self.kernels[kernel_id] = (proc, connection)
return {"kernel_id": kernel_id, "connection": connection}
class GELFHandler(BaseHandler):
def __init__(self, host, port=12201):
BaseHandler.__init__(self)
self.deactivated = False
self._record_pipe, process_pipe = Pipe(True)
self._process = Process(target=self._run, args=(process_pipe, host, port, root_logger._app_name), name='GELFHandler')
self._process.daemonhtop = True
self._process.start()
atexit.register(self._record_pipe.send, -1)
def _run(self, pipe, host, port, app):
signal.signal(signal.SIGINT, signal.SIG_IGN)
_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
_socket.connect((host, port))
_hostname = socket.gethostname()
for time, severity, name, message, kw in iter(pipe.recv, -1):
fields = {
'version': '1.0',
'host': _hostname,
'short_message': message.split('\n', 1)[0],
'full_message': message,
'timestamp': time,
'level': _logging2syslog[severity],
'facility': app,
'_name': name
}
for k, v in kw.items():
fields['_' + k] = v
raw = compress(dumps(fields))
try:
_socket.send(raw)
except socket.error as e:
print('socket error:', str(e), file=stdout)
def record(self, severity, name, refs, format, *a, **kw):
if self.deactivated:
return
for n, v in refs:
kw[n] = v
try:
a = tuple(i() if isinstance(i, collections.Callable) else i for i in a)
self._record_pipe.send((time.time(), severity, name, format % a if a else format, kw))
except IOError as e:
self._record_pipe.close()
self.deactivated = True
root_logger.log(50, 'reflogging.handlers.GELFHandler', [], 'GELFHandler\'s record_pipe went down: %s' % (str(e),))
class NodePair:
def __init__(self,txnode,rxnode):
self.txNode = txnode
self.rxNode = rxnode
self.ptx_conn, self.tx_conn = Pipe(True)
self.prx_conn, self.rx_conn = Pipe(True)
self.ptx = Process(target=self.txNode.runNode, args=(self.tx_conn,))
self.prx = Process(target=self.rxNode.runNode, args=(self.rx_conn,))
self.ptx.start()
self.prx.start()
def CompareResults(self):
self.prx_conn.send(StatusMessage())
passedB = False
ba = StatusMessage()
if self.prx_conn.poll(1):
b = self.prx_conn.recv()#clear buffer
print '%s checksums rx %x calc %x'%(self.rxNode.name, b.status[3], b.status[4])
passedB = True
if passedB :
if b.status[3] == b.status[4]:
return True
else:
return False
else:
print 'error found'
def GetDisplayName(self):
return self.txNode.name + ' to ' + self.rxNode.name
def StopNodePair(self):
self.ptx_conn.send(CloseMessage())
self.prx_conn.send(CloseMessage())
self.ptx.join()
self.ptx_conn.close()
self.tx_conn.close()
self.ptx.terminate()
self.prx.join()
self.prx_conn.close()
self.rx_conn.close()
self.prx.terminate()
def SetTranmitData(self,txData):
self.ptx_conn.send(TxDataMessage(txData))
if self.ptx_conn.poll(2):
a = self.ptx_conn.recv()#clear buffer
print 'inside checksum %x'%a.checksum
class ProcessHandler(object):
'''
run(): The run() method is the entry point for a thread.
start(): The start() method starts a thread by calling the run method.
join([time]): The join() waits for threads to terminate.
isAlive(): The isAlive() method checks whether a thread is still executing.
'''
process_pull = []
# max_process_pull = 10
def __init__(self, daemonic=True, pipe=False):
self.daemonic = daemonic
self.pipe = pipe
if pipe:
self.parent_conn, self.child_conn = Pipe(duplex=False)
@abc.abstractmethod
def run(self, *args):
pass
def start(self, *args):
# TODO: handle excep
# Close write fd because parent not going to write
self.process = Process(target=self.run, args=args)
if self.daemonic:
self.process.daemon = True
self.process.start()
self.__class__.process_pull.append(self.process)
def join(self):
self.parent_conn.close()
if pipe:
self.child_conn.close()
self.process.join()
@classmethod
def pull_size(cls):
return len(cls.process_pull)
@classmethod
def max_process_pull(cls):
return cls.max_process_pull
@classmethod
def list_process_pull(cls):
return cls.process_pull
def add_worker(self, name, target, args=(), kwargs={}):
worker_name = self.name + type(self).NAME_SEP + name
if worker_name in self.workers:
raise NameError("worker name already in use -- {!r}".format(worker_name))
input_recv, input_send = Pipe(duplex=False)
output_recv, output_send = Pipe(duplex=False)
worker_proc = Process(name=worker_name,
target=target,
args=(input_recv, output_send, self._log_queue)+args,
kwargs=kwargs)
worker_proc.start()
input_recv.close()
output_send.close()
worker_info = WorkerInfo(worker_name, input_send, output_recv, worker_proc)
self.workers[worker_name] = worker_info
self.available.append(worker_info)
self.forwarders.add(Forwarder(output_recv, self.messages, autorun=True))
def rss_spider():
parent_conn, child_conn = Pipe()
p = Process(target=main, args=(child_conn,))
p.start()
global lastLogTime
while True:
a = parent_conn.recv()
if(0 == a):
p.terminate()
parent_conn.close()
lastLogTime = time.time()
parent_conn, child_conn = Pipe()
p = Process(target=main, args=(child_conn,))
p.start()
logging.error(str(a) + "process:" + str(multiprocessing.active_children()))
time.sleep(1)
class WorkingThread(Thread): """WorkingThread and WorkingProcess provide the ability to defer a heavy task to another process, while keeping the GUI responsive. Because of the GIL, threads in python aren't really threads, so we need to spawn a new process. Besides, creating a new process make it possible to stop an already began calculation by killing the process. Launching an asynchronous task is done by creating a WorkingThread with two arguments. The first one is the heavy function, that will be executed in another process, and that should return its output. The second function will be executed when the first one terminates, and should take the first one's return value as input. """ def __init__(self, func, func_done): Thread.__init__(self) self.func = func self.func_done = func_done self.rd_conn, self.wr_conn = Pipe(False) self.valid_output = True self.p = WorkingProcess(self.func, self.rd_conn, self.wr_conn) def run(self): self.p.start() self.wr_conn.close() try: func_ret = self.rd_conn.recv() if not self.valid_output: # In case the process was terminated abruptly func_ret = None except EOFError: # In case the process was terminated abruptly func_ret = None finally: GObject.idle_add(self.func_done, func_ret) self.rd_conn.close() def stop_process(self): self.valid_output = False self.p.terminate()
def run_experiment(**kwargs):
print()
exp_dir = os.getcwd(
) + '/data/parallel_mb_ppo/' + EXP_NAME + '/' + kwargs.get('exp_name', '')
print("\n---------- experiment with dir {} ---------------------------".
format(exp_dir))
logger.configure(dir=exp_dir,
format_strs=['csv', 'stdout', 'log'],
snapshot_mode='last')
json.dump(kwargs,
open(exp_dir + '/params.json', 'w'),
indent=2,
sort_keys=True,
cls=ClassEncoder)
config = ConfigProto()
config.gpu_options.allow_growth = True
config.gpu_options.per_process_gpu_memory_fraction = kwargs.get(
'gpu_frac', 0.95)
# Instantiate classes
set_seed(kwargs['seed'])
baseline = kwargs['baseline']()
env = normalize(kwargs['env']()) # Wrappers?
policy = GaussianMLPPolicy(
name="meta-policy",
obs_dim=np.prod(env.observation_space.shape),
action_dim=np.prod(env.action_space.shape),
hidden_sizes=kwargs['policy_hidden_sizes'],
learn_std=kwargs['policy_learn_std'],
hidden_nonlinearity=kwargs['policy_hidden_nonlinearity'],
output_nonlinearity=kwargs['policy_output_nonlinearity'],
)
dynamics_model = MLPDynamicsEnsemble(
'dynamics-ensemble',
env=env,
num_models=kwargs['num_models'],
hidden_nonlinearity=kwargs['dyanmics_hidden_nonlinearity'],
hidden_sizes=kwargs['dynamics_hidden_sizes'],
output_nonlinearity=kwargs['dyanmics_output_nonlinearity'],
learning_rate=kwargs['dynamics_learning_rate'],
batch_size=kwargs['dynamics_batch_size'],
buffer_size=kwargs['dynamics_buffer_size'],
)
'''-------- dumps and reloads -----------------'''
baseline_pickle = pickle.dumps(baseline)
env_pickle = pickle.dumps(env)
receiver, sender = Pipe()
p = Process(
target=init_vars,
name="init_vars",
args=(sender, config, policy, dynamics_model),
daemon=False,
)
p.start()
policy_pickle, dynamics_model_pickle = receiver.recv()
receiver.close()
'''-------- following classes depend on baseline, env, policy, dynamics_model -----------'''
worker_data_feed_dict = {
'env_sampler': {
'num_rollouts': kwargs['num_rollouts'],
'max_path_length': kwargs['max_path_length'],
'n_parallel': kwargs['n_parallel'],
},
'dynamics_sample_processor': {
'discount': kwargs['discount'],
'gae_lambda': kwargs['gae_lambda'],
'normalize_adv': kwargs['normalize_adv'],
'positive_adv': kwargs['positive_adv'],
},
}
worker_model_feed_dict = {}
worker_policy_feed_dict = {
'model_sampler': {
'num_rollouts': kwargs['imagined_num_rollouts'],
'max_path_length': kwargs['max_path_length'],
'dynamics_model': dynamics_model,
'deterministic': kwargs['deterministic'],
},
'model_sample_processor': {
'discount': kwargs['discount'],
'gae_lambda': kwargs['gae_lambda'],
'normalize_adv': kwargs['normalize_adv'],
'positive_adv': kwargs['positive_adv'],
},
'algo': {
'learning_rate': kwargs['learning_rate'],
'clip_eps': kwargs['clip_eps'],
'max_epochs': kwargs['num_ppo_steps'],
}
}
trainer = ParallelTrainer(
policy_pickle=policy_pickle,
env_pickle=env_pickle,
baseline_pickle=baseline_pickle,
dynamics_model_pickle=dynamics_model_pickle,
feed_dicts=[
worker_data_feed_dict, worker_model_feed_dict,
worker_policy_feed_dict
],
n_itr=kwargs['n_itr'],
dynamics_model_max_epochs=kwargs['dynamics_max_epochs'],
log_real_performance=kwargs['log_real_performance'],
steps_per_iter=kwargs['steps_per_iter'],
flags_need_query=kwargs['flags_need_query'],
config=config,
simulation_sleep=kwargs['simulation_sleep'],
)
trainer.train()
class TestCaseWrapper(object):
def __init__(self, testcase_suite, manager):
self.keep_alive_parent_end, self.keep_alive_child_end = Pipe(
duplex=False)
self.finished_parent_end, self.finished_child_end = Pipe(duplex=False)
self.result_parent_end, self.result_child_end = Pipe(duplex=False)
self.testcase_suite = testcase_suite
if sys.version[0] == '2':
self.stdouterr_queue = manager.StreamQueue()
else:
from multiprocessing import get_context
self.stdouterr_queue = manager.StreamQueue(ctx=get_context())
self.logger = get_parallel_logger(self.stdouterr_queue)
self.child = Process(target=test_runner_wrapper,
args=(testcase_suite, self.keep_alive_child_end,
self.stdouterr_queue,
self.finished_child_end,
self.result_child_end, self.logger))
self.child.start()
self.last_test_temp_dir = None
self.last_test_vpp_binary = None
self._last_test = None
self.last_test_id = None
self.vpp_pid = None
self.last_heard = time.time()
self.core_detected_at = None
self.testcases_by_id = {}
self.testclasess_with_core = {}
for testcase in self.testcase_suite:
self.testcases_by_id[testcase.id()] = testcase
self.result = TestResult(testcase_suite, self.testcases_by_id)
@property
def last_test(self):
return self._last_test
@last_test.setter
def last_test(self, test_id):
self.last_test_id = test_id
if test_id in self.testcases_by_id:
testcase = self.testcases_by_id[test_id]
self._last_test = testcase.shortDescription()
if not self._last_test:
self._last_test = str(testcase)
else:
self._last_test = test_id
def add_testclass_with_core(self):
if self.last_test_id in self.testcases_by_id:
test = self.testcases_by_id[self.last_test_id]
class_name = unittest.util.strclass(test.__class__)
test_name = "'{}' ({})".format(
get_test_description(descriptions, test), self.last_test_id)
else:
test_name = self.last_test_id
class_name = re.match(
r'((tearDownClass)|(setUpClass)) '
r'\((.+\..+)\)', test_name).groups()[3]
if class_name not in self.testclasess_with_core:
self.testclasess_with_core[class_name] = (
test_name, self.last_test_vpp_binary, self.last_test_temp_dir)
def close_pipes(self):
self.keep_alive_child_end.close()
self.finished_child_end.close()
self.result_child_end.close()
self.keep_alive_parent_end.close()
self.finished_parent_end.close()
self.result_parent_end.close()
def was_successful(self):
return self.result.was_successful()
class Queue(object):
def __init__(self, maxsize=0):
if maxsize <= 0:
maxsize = _multiprocessing.SemLock.SEM_VALUE_MAX
self._maxsize = maxsize
self._reader, self._writer = Pipe(duplex=False)
self._rlock = Lock()
self._opid = os.getpid()
if sys.platform == 'win32':
self._wlock = None
else:
self._wlock = Lock()
self._sem = BoundedSemaphore(maxsize)
self._after_fork()
if sys.platform != 'win32':
register_after_fork(self, Queue._after_fork)
def __getstate__(self):
assert_spawning(self)
return (self._maxsize, self._reader, self._writer, self._rlock,
self._wlock, self._sem, self._opid)
def __setstate__(self, state):
(self._maxsize, self._reader, self._writer, self._rlock, self._wlock,
self._sem, self._opid) = state
self._after_fork()
def _after_fork(self):
debug('Queue._after_fork()')
self._notempty = threading.Condition(threading.Lock())
self._buffer = collections.deque()
self._thread = None
self._jointhread = None
self._joincancelled = False
self._closed = False
self._close = None
self._send = self._writer.send
self._recv = self._reader.recv
self._poll = self._reader.poll
def put(self, obj, block=True, timeout=None):
assert not self._closed
if not self._sem.acquire(block, timeout):
raise Full
self._notempty.acquire()
try:
if self._thread is None:
self._start_thread()
self._buffer.append(obj)
self._notempty.notify()
finally:
self._notempty.release()
def get(self, block=True, timeout=None):
if block and timeout is None:
self._rlock.acquire()
try:
res = self._recv()
self._sem.release()
return res
finally:
self._rlock.release()
else:
if block:
deadline = time.time() + timeout
if not self._rlock.acquire(block, timeout):
raise Empty
try:
if block:
timeout = deadline - time.time()
if timeout < 0 or not self._poll(timeout):
raise Empty
elif not self._poll():
raise Empty
res = self._recv()
self._sem.release()
return res
finally:
self._rlock.release()
def qsize(self):
# Raises NotImplementedError on Mac OSX because of broken sem_getvalue()
return self._maxsize - self._sem._semlock._get_value()
def empty(self):
return not self._poll()
def full(self):
return self._sem._semlock._is_zero()
def get_nowait(self):
return self.get(False)
def put_nowait(self, obj):
return self.put(obj, False)
def close(self):
self._closed = True
self._reader.close()
if self._close:
self._close()
def join_thread(self):
debug('Queue.join_thread()')
assert self._closed
if self._jointhread:
self._jointhread()
def cancel_join_thread(self):
debug('Queue.cancel_join_thread()')
self._joincancelled = True
try:
self._jointhread.cancel()
except AttributeError:
pass
def _start_thread(self):
debug('Queue._start_thread()')
# Start thread which transfers data from buffer to pipe
self._buffer.clear()
self._thread = threading.Thread(target=Queue._feed,
args=(self._buffer, self._notempty,
self._send, self._wlock,
self._writer.close),
name='QueueFeederThread')
self._thread.daemon = True
debug('doing self._thread.start()')
self._thread.start()
debug('... done self._thread.start()')
# On process exit we will wait for data to be flushed to pipe.
#
# However, if this process created the queue then all
# processes which use the queue will be descendants of this
# process. Therefore waiting for the queue to be flushed
# is pointless once all the child processes have been joined.
created_by_this_process = (self._opid == os.getpid())
if not self._joincancelled and not created_by_this_process:
self._jointhread = Finalize(self._thread,
Queue._finalize_join,
[weakref.ref(self._thread)],
exitpriority=-5)
# Send sentinel to the thread queue object when garbage collected
self._close = Finalize(self,
Queue._finalize_close,
[self._buffer, self._notempty],
exitpriority=10)
@staticmethod
def _finalize_join(twr):
debug('joining queue thread')
thread = twr()
if thread is not None:
thread.join()
debug('... queue thread joined')
else:
debug('... queue thread already dead')
@staticmethod
def _finalize_close(buffer, notempty):
debug('telling queue thread to quit')
notempty.acquire()
try:
buffer.append(_sentinel)
notempty.notify()
finally:
notempty.release()
@staticmethod
def _feed(buffer, notempty, send, writelock, close):
debug('starting thread to feed data to pipe')
from .util import is_exiting
nacquire = notempty.acquire
nrelease = notempty.release
nwait = notempty.wait
bpopleft = buffer.popleft
sentinel = _sentinel
if sys.platform != 'win32':
wacquire = writelock.acquire
wrelease = writelock.release
else:
wacquire = None
try:
while 1:
nacquire()
try:
if not buffer:
nwait()
finally:
nrelease()
try:
while 1:
obj = bpopleft()
if obj is sentinel:
debug('feeder thread got sentinel -- exiting')
close()
return
if wacquire is None:
send(obj)
else:
wacquire()
try:
send(obj)
finally:
wrelease()
except IndexError:
pass
except Exception as e:
# Since this runs in a daemon thread the resources it uses
# may be become unusable while the process is cleaning up.
# We ignore errors which happen after the process has
# started to cleanup.
try:
if is_exiting():
info('error in queue thread: %s', e)
else:
import traceback
traceback.print_exc()
except Exception:
pass
class RPCServerBase(object):
"""This is the base class for send and receive RPC server
It uses a Pipe for IPC.
>>> import grass.script as gscript
>>> from grass.pygrass.rpc.base import RPCServerBase
>>> import time
>>> provider = RPCServerBase()
>>> provider.is_server_alive()
True
>>> provider.is_check_thread_alive()
True
>>> provider.stop()
>>> time.sleep(1)
>>> provider.is_server_alive()
False
>>> provider.is_check_thread_alive()
False
>>> provider = RPCServerBase()
>>> provider.is_server_alive()
True
>>> provider.is_check_thread_alive()
True
Kill the server process with an exception, it should restart
>>> provider.client_conn.send([1])
>>> provider.is_server_alive()
True
>>> provider.is_check_thread_alive()
True
"""
def __init__(self):
self.client_conn = None
self.server_conn = None
self.queue = None
self.server = None
self.checkThread = None
self.threadLock = threading.Lock()
self.start_server()
self.start_checker_thread()
self.stopThread = False
self.stopped = True
# logging.basicConfig(level=logging.DEBUG)
def is_server_alive(self):
return self.server.is_alive()
def is_check_thread_alive(self):
return self.checkThread.is_alive()
def start_checker_thread(self):
if self.checkThread is not None and self.checkThread.is_alive():
self.stop_checker_thread()
self.checkThread = threading.Thread(target=self.thread_checker)
self.checkThread.daemon = True
self.stopThread = False
self.checkThread.start()
def stop_checker_thread(self):
self.threadLock.acquire()
self.stopThread = True
self.threadLock.release()
self.checkThread.join(None)
def thread_checker(self):
"""Check every 200 micro seconds if the server process is alive"""
while True:
time.sleep(0.2)
self._check_restart_server(caller="Server check thread")
self.threadLock.acquire()
if self.stopThread is True:
self.threadLock.release()
return
self.threadLock.release()
def start_server(self):
"""This function must be re-implemented in the subclasses
"""
logging.debug("Start the libgis server")
self.client_conn, self.server_conn = Pipe(True)
self.lock = Lock()
self.server = Process(target=dummy_server, args=(self.lock,
self.server_conn))
self.server.daemon = True
self.server.start()
def check_server(self):
self._check_restart_server()
def _check_restart_server(self, caller="main thread"):
"""Restart the server if it was terminated
"""
logging.debug("Check libgis server restart")
self.threadLock.acquire()
if self.server.is_alive() is True:
self.threadLock.release()
return
self.client_conn.close()
self.server_conn.close()
self.start_server()
if self.stopped is not True:
logging.warning("Needed to restart the libgis server, caller: %s" % (caller))
self.threadLock.release()
self.stopped = False
def safe_receive(self, message):
"""Receive the data and throw a FatalError exception in case the server
process was killed and the pipe was closed by the checker thread"""
logging.debug("Receive message: {message}")
try:
ret = self.client_conn.recv()
if isinstance(ret, FatalError):
raise ret
return ret
except (EOFError, IOError, FatalError) as e:
# The pipe was closed by the checker thread because
# the server process was killed
raise FatalError("Exception raised: " + str(e) + " Message: " + message)
def stop(self):
"""Stop the check thread, the libgis server and close the pipe
This method should be called at exit using the package atexit
"""
logging.debug("Stop libgis server")
self.stop_checker_thread()
if self.server is not None and self.server.is_alive():
self.client_conn.send([0, ])
self.server.terminate()
if self.client_conn is not None:
self.client_conn.close()
self.stopped = True
# from multiprocessing import Process,Queue
#
# def run(q):
# q.put("F**k!")
#
# if __name__ == "__main__":
# q = Queue()
# p = Process(target=run,args=(q,))
# p.start()
# print(q.get())
# p.join()
from multiprocessing import Process, Pipe
def f(conn):
conn.send("hello papa")
print("Son received: ", conn.recv())
conn.close()
if __name__ == "__main__":
p_conn, s_conn = Pipe()
s_process = Process(target=f, args=(s_conn, ))
s_process.start()
print("Parent received: ", p_conn.recv())
p_conn.send("hello son")
s_process.join()
p_conn.close()
def run_forked(suite):
keep_alive_parent_end, keep_alive_child_end = Pipe(duplex=False)
result_parent_end, result_child_end = Pipe(duplex=False)
failed_parent_end, failed_child_end = Pipe(duplex=False)
child = Process(target=test_runner_wrapper,
args=(suite, keep_alive_child_end, result_child_end,
failed_child_end))
child.start()
last_test_temp_dir = None
last_test_vpp_binary = None
last_test = None
result = None
failed = set()
last_heard = time.time()
core_detected_at = None
debug_core = os.getenv("DEBUG", "").lower() == "core"
while True:
readable = select.select([
keep_alive_parent_end.fileno(),
result_parent_end.fileno(),
failed_parent_end.fileno(),
], [], [], 1)[0]
if result_parent_end.fileno() in readable:
result = result_parent_end.recv()
break
if keep_alive_parent_end.fileno() in readable:
while keep_alive_parent_end.poll():
last_test, last_test_vpp_binary,\
last_test_temp_dir, vpp_pid = keep_alive_parent_end.recv()
last_heard = time.time()
if failed_parent_end.fileno() in readable:
while failed_parent_end.poll():
failed_test = failed_parent_end.recv()
failed.add(failed_test.__name__)
last_heard = time.time()
fail = False
if last_heard + test_timeout < time.time() and \
not os.path.isfile("%s/_core_handled" % last_test_temp_dir):
fail = True
global_logger.critical("Timeout while waiting for child test "
"runner process (last test running was "
"`%s' in `%s')!" %
(last_test, last_test_temp_dir))
elif not child.is_alive():
fail = True
global_logger.critical("Child process unexpectedly died (last "
"test running was `%s' in `%s')!" %
(last_test, last_test_temp_dir))
elif last_test_temp_dir and last_test_vpp_binary:
core_path = "%s/core" % last_test_temp_dir
if os.path.isfile(core_path):
if core_detected_at is None:
core_detected_at = time.time()
elif core_detected_at + core_timeout < time.time():
if not os.path.isfile(
"%s/_core_handled" % last_test_temp_dir):
global_logger.critical(
"Child unresponsive and core-file exists in test "
"temporary directory!")
fail = True
if fail:
failed_dir = os.getenv('VPP_TEST_FAILED_DIR')
lttd = last_test_temp_dir.split("/")[-1]
link_path = '%s%s-FAILED' % (failed_dir, lttd)
global_logger.error("Creating a link to the failed " +
"test: %s -> %s" % (link_path, lttd))
try:
os.symlink(last_test_temp_dir, link_path)
except:
pass
api_post_mortem_path = "/tmp/api_post_mortem.%d" % vpp_pid
if os.path.isfile(api_post_mortem_path):
global_logger.error("Copying api_post_mortem.%d to %s" %
(vpp_pid, last_test_temp_dir))
shutil.copy2(api_post_mortem_path, last_test_temp_dir)
if last_test_temp_dir and last_test_vpp_binary:
core_path = "%s/core" % last_test_temp_dir
if os.path.isfile(core_path):
global_logger.error("Core-file exists in test temporary "
"directory: %s!" % core_path)
if debug_core:
spawn_gdb(last_test_vpp_binary, core_path,
global_logger)
child.terminate()
result = -1
break
keep_alive_parent_end.close()
result_parent_end.close()
failed_parent_end.close()
return result, failed
def camera(yVal):
count = 1
countsp = 0
leftEyeSend, leftEyeRecv = Pipe()
p1 = Process(target=condEye, args=(yVal, leftEyeRecv))
p1.start()
#Load face detector and predictor, uses dlib shape predictor file
detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor('shape_predictor_68_face_landmarks.dat')
#Extract indexes of facial landmarks for the left and right eye
(lStart, lEnd) = face_utils.FACIAL_LANDMARKS_IDXS['left_eye']
(rStart, rEnd) = face_utils.FACIAL_LANDMARKS_IDXS['right_eye']
#Start webcam video capture
video_capture = cv2.VideoCapture(0)
try:
while True:
#Read each frame and flip it, and convert to grayscale
ret, frame = video_capture.read()
frame = cv2.flip(frame, 1)
frame = cv2.resize(frame, (300, 240))
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
faces = detector(gray, 0)
#making reference lines
frame = cv2.line(frame, (0, yVal - 40), (1279, yVal - 40),
(0, 0, 255), 2)
frame = cv2.line(frame, (0, yVal), (1279, yVal), (0, 0, 255), 2)
#Show video feed
cv2.imshow('EYE_DETECTION', frame)
if len(faces) == 0:
print("Faceless")
if count == 20:
print('count>>20')
sp.Popen([
"aplay /home/pi/Documents/Group4_SMART_TABLE/soundForSOT/noPeople.wav 2>/dev/null"
],
shell=True)
leftEyeSend.close()
p1.terminate()
time.sleep(0.1)
return ''
else:
count += 1
moveLinear('stop')
continue
if len(faces) > 1:
print('there are more than one person in the camera')
moveLinear('stop')
if countsp == 0:
sp.Popen([
"aplay /home/pi/Documents/Group4_SMART_TABLE/soundForSOT/morePeople.wav 2>/dev/null"
],
shell=True)
countsp += 1
continue
shape = predictor(gray, faces[0])
shape = face_utils.shape_to_np(shape)
#Get array of coordinates of leftEye and rightEye
leftEye = shape[lStart:lEnd]
leftEyeSend.send(leftEye[0][1])
count = 1
countsp = 0
# if not p1.is_alive():
# leftEyeSend.close()
# # p1.terminate()
# time.sleep(0.1)
# cv2.destroyAllWindows()
# print('p1 is terminated')
# return ''
if cv2.waitKey(1) & 0xFF == ord("q"):
p1.terminate()
time.sleep(0.1)
cv2.destroyAllWindows()
return
except KeyboardInterrupt:
moveLinear('stop')
leftEyeSend.close()
p1.terminate()
time.sleep(0.1)
video_capture.release()
cv2.destroyAllWindows()
return
class GetValue():
def __init__(self, com, dic=False, m_processing=True):
"""GetValue(com, dic=False, m_processing=True)
Reads the ICM42605 raw regesters
com: Com port to comunicate with the eval board
dic: If true return values as dictenary
m_processing: If true works with multiprosesing"""
self.com = com
self.i = 0
self.ser = None
self.dic = dic
if m_processing:
self.stop_flag = Value(c_bool, True)
self.m_processing = m_processing
def __enter__(self):
if self.m_processing:
self.stop_flag.value = 1
self.p_con, self.c_con = Pipe()
self.process = Process(target=m_serial,
args=(
self.com,
self.stop_flag,
self.c_con,
))
self.process.start()
else:
self.ser = Serial(self.com, baudrate=921600, timeout=1)
self.ser.reset_input_buffer()
self.ser.reset_output_buffer()
return self
def __exit__(self, exception_type, exception_value, traceback):
if self.m_processing:
self.stop_flag.value = 0 # Turn off process
sleep(0.01)
self.p_con.close()
if exception_type is None:
if self.ser:
self.ser.close()
if not self.ser.closed:
return False
return True
def __iter__(self):
if not self.m_processing:
self.ser.reset_input_buffer()
self.ser.reset_output_buffer()
return self
def __next__(self):
if self.m_processing:
if self.p_con.poll(0):
data = self.p_con.recv()
else:
data = None
else:
read = self.ser.readline()
while '[I]' != read[:3] or '\n' != read[-1]:
read = self.ser.readline()
data = read[3:].strip().replace(':', ',')
data = [int(x) for x in data.split(',')]
self.i += 1
if self.dic:
return {
'time': data[0],
'accx': data[1],
'accy': data[2],
'accz': data[3],
'temp': data[4],
'gyrox': data[5],
'gyroy': data[6],
'gyroz': data[7]
}
else:
return data
def start(self):
self.__enter__()
def stop(self):
self.__exit__(None, None, None)
def data_ready(self):
if self.p_con.poll(0):
return True
else:
return False
def read_line(self):
return self.__next__()
next = __next__
while True:
try:
msg = _out_pipe.recv()
print(msg)
except EOFError:
# 当out_pipe接受不到输出的时候且输入被关闭的时候，会抛出EORFError，可以捕获并且退出子进程
break
if __name__ == '__main__':
# 新建一个Pipe(duplex)的时候，如果duplex为True，那么创建的管道是双向的；
# 如果duplex为False，那么创建的管道是单向的。
out_pipe, in_pipe = Pipe(True)
son_p = Process(target=son_process, args=(100, (out_pipe, in_pipe)))
son_p.start()
"""
当第一次创建pipe的时候，主进程会连接着piped的输入和输出端，当fork到子进程的时候也是一样的。
所以第一次的时候会先关闭主进程的out端，然后关闭子进程的out端。这样主进程和子进程分别链接pipe的不同的两端。
"""
# 等pipe被fork 后，关闭主进程的输出端
# 这样，创建的Pipe一端连接着主进程的输入，一端连接着子进程的输出口
out_pipe.close()
for x in range(1000):
in_pipe.send(x)
#等待子进程执行完毕关闭主进程
in_pipe.close()
son_p.join()
print("主进程也结束了")
class ProcessServer(Process, BaseImplServer):
profile_filename = "profile.log"
profile_processed_filename = "profile.log.processed"
def __init__(self, command_channel, event_queue, featurelist):
BaseImplServer.__init__(self)
Process.__init__(self)
self.command_channel = command_channel
self.event_queue = event_queue
self.event = EventAdapter(event_queue)
self.featurelist = featurelist
self.quit = False
self.quitin, self.quitout = Pipe()
self.event_handle = multiprocessing.Value("i")
def run(self):
for event in bb.event.ui_queue:
self.event_queue.put(event)
self.event_handle.value = bb.event.register_UIHhandler(self, True)
bb.cooker.server_main(self.cooker, self.main)
def main(self):
# Ignore SIGINT within the server, as all SIGINT handling is done by
# the UI and communicated to us
self.quitin.close()
signal.signal(signal.SIGINT, signal.SIG_IGN)
bb.utils.set_process_name("Cooker")
while not self.quit:
try:
if self.command_channel.poll():
command = self.command_channel.recv()
self.runCommand(command)
if self.quitout.poll():
self.quitout.recv()
self.quit = True
try:
self.runCommand(["stateForceShutdown"])
except:
pass
self.idle_commands(.1, [self.command_channel, self.quitout])
except Exception:
logger.exception('Running command %s', command)
self.event_queue.close()
bb.event.unregister_UIHhandler(self.event_handle.value)
self.command_channel.close()
self.cooker.shutdown(True)
self.quitout.close()
def idle_commands(self, delay, fds=None):
nextsleep = delay
if not fds:
fds = []
for function, data in self._idlefuns.items():
try:
retval = function(self, data, False)
if retval is False:
del self._idlefuns[function]
nextsleep = None
elif retval is True:
nextsleep = None
elif isinstance(retval, float):
if (retval < nextsleep):
nextsleep = retval
elif nextsleep is None:
continue
else:
fds = fds + retval
except SystemExit:
raise
except Exception as exc:
if not isinstance(exc, bb.BBHandledException):
logger.exception('Running idle function')
del self._idlefuns[function]
self.quit = True
if nextsleep is not None:
select.select(fds,[],[],nextsleep)
def runCommand(self, command):
"""
Run a cooker command on the server
"""
self.command_channel.send(self.cooker.command.runCommand(command))
def stop(self):
self.quitin.send("quit")
self.quitin.close()
class Compiler:
"""The Compiler class."""
def __init__(self) -> None:
"""
Compiler Constructor. Starts a new backend compiler on the local machine
with an empty WorkQueue, establishes a connection, and then starts
running.
Examples:
>>> compiler = Compiler()
>>> task = CompilationTask(...)
>>> compiler.submit(task)
>>> print(compiler.status(task))
TaskStatus.RUNNING
"""
self.conn, backend_conn = Pipe()
self.process = Process(target=WorkQueue.run, args=(backend_conn, ))
self.process.start()
_logger.info('Started compiler process.')
def __enter__(self) -> Compiler:
"""Enter a context for this compiler."""
return self
def __exit__(self, type: Any, value: Any, traceback: Any) -> None:
"""Shutdowns compiler and closes connection."""
self.close()
def close(self) -> None:
"""Shutdowns compiler and closes connection."""
self.conn.send('CLOSE')
self.process.join()
self.conn.close()
_logger.info('Stopped compiler process.')
def submit(self, task: CompilationTask) -> None:
"""Submit a CompilationTask to the Compiler."""
self.conn.send('SUBMIT')
self.conn.send(task)
okay_msg = self.conn.recv() # Block until response
if (okay_msg != 'OKAY'):
raise Exception('Failed to submit job.')
_logger.info('Submitted task: %s' % task.task_id)
def status(self, task: CompilationTask) -> TaskStatus:
"""Retrieve the status of the specified CompilationTask."""
self.conn.send('STATUS')
self.conn.send(task.task_id)
return self.conn.recv() # Block until response
def result(self, task: CompilationTask) -> TaskResult:
"""Block until the CompilationTask is finished, return its result."""
self.conn.send('RESULT')
self.conn.send(task.task_id)
return self.conn.recv() # Block until response
def remove(self, task: CompilationTask) -> None:
"""Remove a task from the compiler's workqueue."""
self.conn.send('REMOVE')
self.conn.send(task.task_id)
self.conn.recv() # Block until response
_logger.info('Removed task: %s' % task.task_id)
def compile(self, task: CompilationTask) -> Circuit:
"""Execute the CompilationTask."""
_logger.info('Compiling task: %s' % task.task_id)
self.submit(task)
result = self.result(task)
return result.get_circuit()
class LabEnvironment(environment.Environment):
ACTION_LIST = [
_action(-20, 0, 0, 0, 0, 0, 0), # look_left
_action(20, 0, 0, 0, 0, 0, 0), # look_right
#_action( 0, 10, 0, 0, 0, 0, 0), # look_up
#_action( 0, -10, 0, 0, 0, 0, 0), # look_down
_action(0, 0, -1, 0, 0, 0, 0), # strafe_left
_action(0, 0, 1, 0, 0, 0, 0), # strafe_right
_action(0, 0, 0, 1, 0, 0, 0), # forward
_action(0, 0, 0, -1, 0, 0, 0), # backward
#_action( 0, 0, 0, 0, 1, 0, 0), # fire
#_action( 0, 0, 0, 0, 0, 1, 0), # jump
#_action( 0, 0, 0, 0, 0, 0, 1) # crouch
]
@staticmethod
def get_action_size(env_name):
return len(LabEnvironment.ACTION_LIST)
def __init__(self, env_name):
environment.Environment.__init__(self)
self.conn, child_conn = Pipe()
self.proc = Process(target=worker, args=(child_conn, env_name))
self.proc.start()
self.conn.recv()
self.reset()
def reset(self):
self.conn.send([COMMAND_RESET, 0])
obs = self.conn.recv()
self.last_state = self._preprocess_frame(obs)
self.last_action = 0
self.last_reward = 0
def stop(self):
self.conn.send([COMMAND_TERMINATE, 0])
ret = self.conn.recv()
self.conn.close()
self.proc.join()
print("lab environment stopped")
def _preprocess_frame(self, image):
image = image.astype(np.float32)
image = image / 255.0
return image
def process(self, action):
real_action = LabEnvironment.ACTION_LIST[action]
self.conn.send([COMMAND_ACTION, real_action])
obs, reward, terminal = self.conn.recv()
if not terminal:
state = self._preprocess_frame(obs)
else:
state = self.last_state
pixel_change = self._calc_pixel_change(state, self.last_state)
self.last_state = state
self.last_action = action
self.last_reward = reward
return state, reward, terminal, pixel_change
def test_mount_fstab_user_fail(self):
if not self._can_create:
self.skipTest('Cannot create %s filesystem' % self._fs_name)
if not self._can_mount:
self.skipTest('Cannot mount %s filesystem' % self._fs_name)
# this test will change /etc/fstab, we might want to revert the changes after it finishes
fstab = self.read_file('/etc/fstab')
self.addCleanup(self.write_file, '/etc/fstab', fstab)
disk = self.get_object('/block_devices/' +
os.path.basename(self.vdevs[0]))
self.assertIsNotNone(disk)
# create filesystem
disk.Format(self._fs_name,
self.no_options,
dbus_interface=self.iface_prefix + '.Block')
self.addCleanup(self._clean_format, disk)
# create user for our test
self.addCleanup(self._remove_user)
uid, gid = self._add_user()
# add unmount cleanup now in case something wrong happens in the other process
self.addCleanup(self._unmount, self.vdevs[0])
# create pipe to get error (if any)
parent_conn, child_conn = Pipe()
proc = Process(target=self._mount_as_user_fstab_fail,
args=(child_conn, int(uid), int(gid), self.vdevs[0]))
proc.start()
res = parent_conn.recv()
parent_conn.close()
proc.join()
if not res[0]:
self.fail(res[1])
# now mount it as root and test that user can't unmount it
mnt_path = disk.Mount(self.no_options,
dbus_interface=self.iface_prefix + '.Filesystem')
self.assertIsNotNone(mnt_path)
self.assertTrue(os.path.ismount(mnt_path))
# create pipe to get error (if any)
parent_conn, child_conn = Pipe()
proc = Process(target=self._unmount_as_user_fstab_fail,
args=(child_conn, int(uid), int(gid), self.vdevs[0]))
proc.start()
res = parent_conn.recv()
parent_conn.close()
proc.join()
if not res[0]:
self.fail(res[1])
self.assertTrue(os.path.ismount(mnt_path))
self._unmount(mnt_path)
from multiprocessing import Pipe
from sys import exit
from hook import VppDiedError
from framework import VppTestCase, KeepAliveReporter
class SanityTestCase(VppTestCase):
""" Sanity test case - verify if VPP is able to start """
pass
if __name__ == '__main__':
rc = 0
tc = SanityTestCase
x, y = Pipe()
reporter = KeepAliveReporter()
reporter.pipe = y
try:
tc.setUpClass()
except VppDiedError:
rc = -1
else:
try:
tc.tearDownClass()
except:
pass
x.close()
y.close()
exit(rc)
for i in range(10):
conn.send(i)
print('send {} to pipe '.format(i))
time.sleep(1)
def proc2(conn):
n = 9
while n > 0:
result = conn.recv()
print('receive {} from pip'.format(result))
n -= 1
if __name__ == '__main__':
q = Queue(20)
proc_produce = Process(target=produce, args=(q, ))
proc_consume = Process(target=consume, args=(q, ))
proc_produce.start()
proc_consume.start()
p1, p2 = Pipe(True) # True 表示两个管道是全双工的，均可收发
process1 = Process(target=proc1, args=(p1, ))
process2 = Process(target=proc2, args=(p2, ))
process1.start()
process2.start()
process1.join()
process2.join()
p1.close()
p2.close()
print('end')
# 管道
# 双向通信
# 如果管道的端点没有被使用，那就应该关闭管道的端点。如在生产者消费者模型中，生产者要关闭管道的输出端，消费者要关闭管道的输入端
# 如果没有执行关闭操作，程序可能在消费者中的 recv() 操作上挂起
# 管道用在进程之间会发生数据冲突的情况，数据不安全
# 在主进程中关闭管道不会影响管道在子进程中的使用
# 队列=管道+锁
from multiprocessing import Pipe
from multiprocessing import Process
def func(p):
father, son = p
#father.close()
print(son.recv())
print(son.recv())
if __name__ == '__main__':
father, son = Pipe()
p = Process(target=func, args=((father, son), ))
p.start()
father.send('hello')
father.close()
conn.close()
stream.stop_stream()
stream.close()
py_aud.terminate()
#
# This starts the audio input handling and builds the different threads
#
if __name__ == '__main__':
aud_receive, aud_send = Pipe(False)
cmd_pipe, frame_pipe = Pipe()
audio = Process(target=audio_decode, args=(aud_send,))
frames = Process(target=build_frames, args=(aud_receive, frame_pipe,))
audio.start()
frames.start()
console.STACetShell(cmd_pipe).cmdloop()
sock.close()
aud_receive.close()
aud_send.close()
audio.terminate()
frames.terminate()
time.sleep(.3)
audio.close()
frames.close()
from multiprocessing import Process, Pipe
def receiver(recv_pip):
while True:
try:
res = recv_pip.recv()
print('recv:', res)
except EOFError:
break
if __name__ == '__main__':
print('start')
send_pip, recv_pip = Pipe()
proc_recv = Process(target=receiver, args=(recv_pip, ))
proc_recv.start()
for i in range(1, 5):
send_pip.send(i)
print('send:', i)
send_pip.close()
proc_recv.join()
print('end')
class ProcessEngine(BaseEngine):
"""An engine executing the tasks it is sent in a different process.
"""
def perform(self, exec_infos):
"""Execute a given task.
Parameters
----------
exec_infos : ExecutionInfos
TaskInfos object describing the work to expected of the engine.
Returns
-------
exec_infos : ExecutionInfos
Input object whose values have been updated. This is simply a
convenience.
Notes
-----
IOError in pipe are raised only if an operation is attempted from the
process that closed the pipe, but never when trying to poll from a
different process.
"""
self.status = 'Running'
# Clear all the flags.
self._task_pause.clear()
self._task_paused.clear()
self._task_resumed.clear()
self._task_stop.clear()
self._force_stop.clear()
self._stop_requested = False
# If the process does not exist or is dead create a new one.
if not self._process or not self._process.is_alive():
self._process_stop.clear()
# Create the subprocess and the pipe.
self._pipe, process_pipe = Pipe()
self._process = TaskProcess(process_pipe, self._log_queue,
self._monitor_queue, self._task_pause,
self._task_paused, self._task_resumed,
self._task_stop, self._process_stop)
self._process.daemon = True
# Create the logger thread in charge of dispatching log reports.
self._log_thread = QueueLoggerThread(self._log_queue)
self._log_thread.daemon = True
logger.debug('Starting logging thread.')
self._log_thread.start()
# Create the monitor thread dispatching engine news to the monitor.
self._monitor_thread = ThreadMeasureMonitor(
self, self._monitor_queue)
self._monitor_thread.daemon = True
logger.debug('Starting monitoring thread.')
self._monitor_thread.start()
self._pause_thread = None
# Start process.
logger.debug('Starting subprocess')
self._process.start()
# Send the measurement.
args = self._build_subprocess_args(exec_infos)
try:
self._pipe.send(args)
except Exception:
msg = ('Failed to send infos to subprocess :\n-infos : \n%s\n'
'-errors :\n%s')
logger.error(msg % (pformat(args), format_exc()))
self._log_queue.put(None)
self._monitor_queue.put((None, None))
self._cleanup(process=True)
exec_infos.success = False
exec_infos.errors['engine'] = msg
self.status = 'Stopped'
return exec_infos
else:
logger.debug('Task {} sent'.format(exec_infos.id))
# Check that the engine did receive the task.
while not self._pipe.poll(2):
if not self._process.is_alive():
msg = 'Subprocess was found dead unexpectedly'
logger.debug(msg)
self._log_queue.put(None)
self._monitor_queue.put((None, None))
self._cleanup(process=False)
exec_infos.success = False
exec_infos.errors['engine'] = msg
self.status = 'Stopped'
return exec_infos
# Simply empty the pipe the subprocess always send True if it answers
self._pipe.recv()
# Wait for the process to finish the measurement and check it has not
# been killed.
while not self._pipe.poll(1):
if self._force_stop.is_set():
msg = 'Subprocess was terminated by the user.'
logger.debug(msg)
self._cleanup(process=False)
exec_infos.errors['engine'] = msg
self.status = 'Stopped'
return exec_infos
elif not self._process.is_alive():
msg = 'Subprocess was found dead unexpectedly'
logger.debug(msg)
self._log_queue.put(None)
self._monitor_queue.put((None, None))
self._cleanup(process=False)
exec_infos.success = False
exec_infos.errors['engine'] = msg
self.status = 'Stopped'
return exec_infos
# Here get message from process and react
result, errors = self._pipe.recv()
logger.debug('Subprocess done performing measurement')
exec_infos.success = result
exec_infos.errors.update(errors)
self.status = 'Waiting'
return exec_infos
def pause(self):
"""Ask the engine to pause the current task execution.
"""
self.status = 'Pausing'
self._task_resumed.clear()
self._task_paused.clear()
self._task_pause.set()
self._pause_thread = Thread(target=self._wait_for_pause)
self._pause_thread.start()
def resume(self):
"""Ask the engine to resume the currently paused job.
"""
self.status = 'Resuming'
self._task_pause.clear()
def stop(self, force=False):
"""Ask the engine to stop the current job.
This method should not wait for the job to stop save if a forced stop
was requested.
Parameters
----------
force : bool, optional
Force the engine to stop the performing the task. This allow the
engine to use any means necessary to stop, in this case only should
the call to this method block.
"""
self.status = 'Stopping'
self._stop_requested = True
self._task_stop.set()
if force:
self._force_stop.set()
# Stop running queues
self._log_queue.put(None)
self._monitor_queue.put((None, None))
# Terminate the process and make sure all threads stopped properly.
self._process.terminate()
self._log_thread.join()
self._monitor_thread.join()
# Discard the queues as they may have been corrupted when the
# process was terminated.
self._log_queue = Queue()
self._monitor_queue = Queue()
self.status = 'Stopped'
def shutdown(self, force=False):
"""Ask the engine to stop completely.
Parameters
----------
force : bool, optional
Force the engine to stop the performing the task. This allow the
engine to use any means necessary to stop, in this case only should
the call to this method block.
"""
self.status = 'Shutting down'
self._stop_requested = True
self._task_stop.set()
if not force:
t = Thread(target=self._cleanup)
t.start()
else:
self.stop(force=True)
# =========================================================================
# --- Private API ---------------------------------------------------------
# =========================================================================
#: Boolean indicating that the user requested the job to stop.
_stop_requested = Bool()
#: Interprocess event used to pause the subprocess current job.
_task_pause = Value(factory=Event)
#: Interprocess event signaling the subprocess current job is paused.
_task_paused = Value(factory=Event)
#: Interprocess event signaling the subprocess current job has resumed.
_task_resumed = Value(factory=Event)
#: Interprocess event used to stop the subprocess current measurement.
_task_stop = Value(factory=Event)
#: Interprocess event used to stop the subprocess.
_process_stop = Value(factory=Event)
#: Flag signaling that a forced exit has been requested
_force_stop = Value(factory=tEvent)
#: Current subprocess.
_process = Typed(TaskProcess)
#: Connection used to send and receive messages about execution (type
#: ambiguous when the OS is not known)
_pipe = Value()
#: Inter-process queue used by the subprocess to transmit its log records.
_log_queue = Value(factory=Queue)
#: Thread in charge of collecting the log message coming from the
#: subprocess.
_log_thread = Typed(Thread)
#: Inter-process queue used by the subprocess to send the values of the
#: observed database entries.
_monitor_queue = Value(factory=Queue)
#: Thread in charge of collecting the values of the observed database
#: entries.
_monitor_thread = Typed(Thread)
#: Thread in charge of notifying the engine that the engine did
#: pause/resume after being asked to do so.
_pause_thread = Typed(Thread)
def _cleanup(self, process=True):
""" Helper method taking care of making sure that everybody stops.
Parameters
----------
process : bool
Whether to join the worker process. Used when the process has been
termintaed abruptly.
"""
logger.debug('Cleaning up')
if process and self._process:
self._process_stop.set()
self._process.join()
logger.debug('Subprocess joined')
if self._pipe:
self._pipe.close()
if self._log_thread:
self._log_thread.join()
logger.debug('Log thread joined')
if self._monitor_thread:
self._monitor_thread.join()
logger.debug('Monitor thread joined')
if self._pause_thread:
self._pause_thread.join()
logger.debug('Pause thread joined')
self.status = 'Stopped'
def _build_subprocess_args(self, exec_infos):
"""Build the tuple to send to the subprocess.
"""
exec_infos.task.update_preferences_from_members()
config = exec_infos.task.preferences
database_root_state = exec_infos.task.database.copy_node_values()
return (exec_infos.id, config, exec_infos.build_deps,
exec_infos.runtime_deps, exec_infos.observed_entries,
database_root_state, exec_infos.checks)
def _wait_for_pause(self):
""" Wait for the _task_paused event to be set.
"""
stop_sig = self._task_stop
paused_sig = self._task_paused
while not stop_sig.is_set():
if paused_sig.wait(0.1):
self.status = 'Paused'
break
resuming_sig = self._task_resumed
while not stop_sig.is_set():
if resuming_sig.wait(1):
self.status = 'Running'
break
class TripletDataLayer(BasePythonDataLayer):
"""Triplet Data Layer:
Provide data batches for Triplet Network (using ranking loss)
Data: 3 * batch_size * channels * width * height
anchor image, positive and negative ones
Label: Relative Similarity (Optional)
Implemenation is based on BasePythonDataLayer,
need to implement:
1. get_next_minibatch(self) function
2. sampleing functions for randomly sampling and guided sampling
"""
def setup(self, bottom, top):
# setup functions from super class
super(TripletDataLayer, self).setup(bottom, top)
print("Using Triplet Python Data Layer")
# prefetch or not: default = False
self._sampling_type = self._layer_params.get('type', 'RANDOM')
self._prefetch = self._layer_params.get('prefetch', False)
"""Construct kwargs:
possible fields:
k - number of candidates when hard negative sampling
m - similarity graph filename for hard negative sampling
n - number of iterations before hard negative sampling
"""
kwargs = {}
for key, value in self._layer_params.iteritems():
if key.lower() in ['k', 'm', 'n']:
kwargs[key.lower()] = value
if self._prefetch:
# using prefetch to generate mini-batches
self._conn, conn = Pipe()
self._prefetch_process = TripletPrefetcher(
conn, self._label, self._data, self._mean, self._resize,
self._batch_size, self._sampling_type, **kwargs)
print("Start Prefetching Process...")
self._prefetch_process.start()
def cleanup():
print("Terminating Prefetching Processs...")
self._prefetch_process.terminate()
self._prefetch_process.join()
self._conn.close()
atexit.register(cleanup)
else:
self._sampler = TripletSampler(self._sampling_type, self._label,
**kwargs)
self.reshape(bottom, top)
def get_a_datum(self):
"""Get a datum:
Sampling -> decode images -> stack numpy array
"""
sample = self._sampler.sample()
if self._compressed:
datum_ = [
extract_sample(self._data[id], self._mean, self._resize)
for id in sample[:3]
]
else:
datum_ = [self._data[id] for id in sample[:3]]
if len(sample) == 4:
datum_.append(sample[-1])
return datum_
def get_next_minibatch(self):
if self._prefetch:
# get mini-batch from prefetcher
batch = self._conn.recv()
else:
# generate using in-thread functions
data = []
p_data = []
n_data = []
label = []
for i in range(self._batch_size):
datum_ = self.get_a_datum()
data.append(datum_[0])
p_data.append(datum_[1])
n_data.append(datum_[2])
if len(datum_) == 4:
# datum and label / margin
label.append(datum_[-1])
batch = [np.array(data), np.array(p_data), np.array(n_data)]
if len(label):
label = np.array(label).reshape(self._batch_size, 1, 1, 1)
batch.append(label)
return batch
class Engine(object):
def __init__(self, kernel_filename, scene_filename, width, height, noise,
obj, animation, record, no_gui):
self._run = self.animation_run if animation else self.normal_run
self.record = record
self.no_gui = no_gui
if self.record:
os.makedirs("animation")
self.frame = 0
self.wait = animation
self.width, self.height = width, height
self.actions_list = self.get_action_list()
scene = Scene(None, None)
scene.load_from_json(scene_filename)
if obj:
scene.load_from_mesh(obj)
self.camera = Camera(width, height)
scene.add_object(self.camera)
self.connector = Connector(kernel_filename, scene, width, height,
noise)
if not self.no_gui:
self.parent_conn, child_conn = Pipe()
self.gui_process = Process(target=gui_worker,
args=(child_conn, width, height))
self.gui_process.start()
self.running = True
self.denoiser = Denoiser.create("CnnAutoencoder", width, height)
self.next_frame = True
self.actions = {
a: False
for a in [
"w", "a", "s", "d", "Up", "Left", "Down", "Right", "q", "e",
"plus", "minus"
]
}
def collect_action(self, action):
value = True
if action[0] == "~":
action = action[1:]
value = False
if action in self.actions:
self.actions[action] = value
def update(self):
if self.actions["plus"]:
self.camera.speed += 0.001
if self.actions["minus"]:
self.camera.speed -= 0.001
if self.camera.speed < 0:
self.camera.speed = 0
self.camera.move(self.actions["w"], self.actions["s"],
self.actions["a"], self.actions["d"])
self.camera.rotate(self.actions["Up"], self.actions["Left"],
self.actions["Down"], self.actions["Right"])
self.camera.rotate_off_its_axis(self.actions["q"], self.actions["e"])
def normal_run(self):
self.current = datetime.datetime.now()
self.elapsed = (self.current - self.previous).total_seconds()
self.previous = self.current
self.lag += self.elapsed
if self.parent_conn.poll():
try:
msg = self.parent_conn.recv()
except EOFError:
self.running = False
return False
self.collect_action(msg)
while self.lag >= MS_PER_UPDATE:
self.update()
self.lag -= MS_PER_UPDATE
return True
def get_action_list(self):
action_list = []
action_list += ["Nothing"] * 10
action_list += ["q"] * 60
action_list += ["w"] * 250
action_list += ["Up"] * 30
action_list += ["Nothing"] * 10
action_list += ["Down"] * 30
action_list += ["q"] * 90
action_list += ["e"] * 150
action_list += ["w"] * 280
action_list += ["e"] * 45
action_list += ["Up"] * 30
action_list += ["Nothing"] * 10
action_list += ["Down"] * 30
action_list += ["e"] * 90
action_list += ["q"] * 45
action_list += ["w"] * 125
action_list += ["e"] * 90
action_list += ["w"] * 60
action_list += ["e"] * 365
return action_list
def action_generator(self):
if self.actions_list:
self.actions[self.actions_list.pop(0)] = False
else:
return False
if self.actions_list:
self.actions[self.actions_list[0]] = True
return True
def animation_run(self):
if not self.no_gui:
if self.parent_conn.poll():
try:
self.parent_conn.recv()
except EOFError:
self.running = False
return False
self.running = self.action_generator()
self.update()
return self.running
def run(self):
self.connector.run()
time_point_a = datetime.datetime.now()
total = 0
self.previous = datetime.datetime.now()
self.lag = 0
while self.running:
if not self._run():
break
image = self.connector.get_result(self.wait)
if image is not None:
image = self.denoiser.denoise(image, self.connector)
time_point_b = datetime.datetime.now()
diff = time_point_b - time_point_a
diff = diff.total_seconds()
total += diff
print("frame: ", diff)
time_point_a = time_point_b
if self.record:
self.save_frame(image)
if not self.no_gui:
try:
self.parent_conn.send(image.tobytes())
except BrokenPipeError:
self.running = False
break
self.connector.run()
print("Quitting")
self.parent_conn.close()
self.gui_process.terminate()
self.gui_process.join()
def send_and_query(self, status):
image = self.connector.get_result(self.wait)
try:
self.parent_conn.send(image)
except BrokenPipeError:
self.running = False
else:
self.connector.run(self.send_and_query)
def save_frame(self, data):
data = np.reshape(data, (self.height, self.width * 3))
with open("animation_cnn/{}.png".format(self.frame), "wb") as f:
w = png.Writer(self.width, self.height)
w.write(f, data)
self.frame += 1
file_list = os.listdir(target_dir) if time.time() - time_started >= 60: print "Times up, aborting general scan..." scanning = False if file_list != []: #test for APs that havent previously been cracked/timed-out for _file in file_list: _xml = xml_machine(target_dir+_file) _xml.parse_deets() if str(_xml.cracked) == 'False': print "Targets detected, aborting general scan..." scanning = False break observer.stop() airodump_parent_conn.send(scanning) airodump_parent_conn.close() if file_list != []: #parse xml exported previously with target deets sort_list = sort_by_power(target_dir+"*.xml") #print "sort_list:", sort_list ignore_aps = create_ignore_list() print "Ignoring previously scanned networks:", ignore_aps scan_list = [x for x in sort_list if x not in ignore_aps] print "Suitable Wifi APs for handshake detection:", scan_list for AP in scan_list: #print "target_dir+file:", (target_dir+file) #debug f_xml = xml_machine(target_dir+AP[0]+".xml") f_xml.parse_deets() #lat, lng, acc = geo_locate(f_xml.bssid, "0", "0") #power and snr to be added in future..... #print 'lat:', lat #print 'lng:', lng
if __name__ == '__main__':
send_pipe, receive_pipe = Pipe(True)
send_pipe2, receive_pipe2 = Pipe(True)
p1 = Process(target=env_step, args=(receive_pipe, ))
p2 = Process(target=env_step, args=(receive_pipe2, ))
start1 = time.time()
p1.start()
p2.start()
receive_pipe.close()
for i in range(1, 1000):
send_pipe.send(i)
send_pipe2.send(i)
send_pipe.send(0)
send_pipe2.send(0)
send_pipe.close()
send_pipe2.close()
p1.join()
p2.join()
print("close send pipe")
end1 = time.time()
start = time.time()
eng = matlab.engine.start_matlab() #start matlab in python
eng.addpath("/home/t630/Voltage_Control/test3",
nargout=0) #add voltage control path to the matlab workspace
eng.start_matpower() #start up matpower
eng.load_data(nargout=0)
N = 1000
eng.env_init(nargout=0)
for i in range(1, N):
class Device(QObject):
JOIN_TIMEOUT = 1.0
SEND_BUFFER_SIZE = 0
CONTINUOUS_SEND_BUFFER_SIZE = 0
class Command(Enum):
STOP = 0
SET_FREQUENCY = 1
SET_SAMPLE_RATE = 2
SET_BANDWIDTH = 3
SET_RF_GAIN = 4
SET_IF_GAIN = 5
SET_BB_GAIN = 6
SET_DIRECT_SAMPLING_MODE = 7
SET_FREQUENCY_CORRECTION = 8
SET_CHANNEL_INDEX = 9
SET_ANTENNA_INDEX = 10
rcv_index_changed = pyqtSignal(int, int)
ASYNCHRONOUS = False
DEVICE_LIB = None
DEVICE_METHODS = {
Command.SET_FREQUENCY.name: "set_center_freq",
Command.SET_SAMPLE_RATE.name: "set_sample_rate",
Command.SET_BANDWIDTH.name: "set_bandwidth",
Command.SET_RF_GAIN.name: "set_rf_gain",
Command.SET_IF_GAIN.name: {"rx": "set_if_rx_gain", "tx": "set_if_tx_gain"},
Command.SET_BB_GAIN.name: {"rx": "set_baseband_gain"}
}
@classmethod
def process_command(cls, command, ctrl_connection, is_tx: bool):
is_rx = not is_tx
if command == cls.Command.STOP.name:
return cls.Command.STOP.name
tag, value = command
try:
if isinstance(cls.DEVICE_METHODS[tag], str):
method_name = cls.DEVICE_METHODS[tag]
elif isinstance(cls.DEVICE_METHODS[tag], dict):
method_name = cls.DEVICE_METHODS[tag]["rx" if is_rx else "tx"]
else:
method_name = None
except KeyError:
method_name = None
if method_name:
try:
ret = getattr(cls.DEVICE_LIB, method_name)(value)
ctrl_connection.send("{0} to {1}:{2}".format(tag, value, ret))
except AttributeError as e:
logger.warning(str(e))
@classmethod
def setup_device(cls, ctrl_connection: Connection, device_identifier):
raise NotImplementedError("Overwrite this method in subclass!")
@classmethod
def init_device(cls, ctrl_connection: Connection, is_tx: bool, parameters: OrderedDict) -> bool:
if "identifier" in parameters:
identifier = parameters["identifier"]
else:
identifier = None
if cls.setup_device(ctrl_connection, device_identifier=identifier):
for parameter, value in parameters.items():
cls.process_command((parameter, value), ctrl_connection, is_tx)
return True
else:
return False
@classmethod
def adapt_num_read_samples_to_sample_rate(cls, sample_rate: float):
raise NotImplementedError("Overwrite this method in subclass!")
@classmethod
def shutdown_device(cls, ctrl_connection, is_tx: bool):
raise NotImplementedError("Overwrite this method in subclass!")
@classmethod
def enter_async_receive_mode(cls, data_connection: Connection, ctrl_connection: Connection) -> int:
raise NotImplementedError("Overwrite this method in subclass!")
@classmethod
def prepare_sync_receive(cls, ctrl_connection: Connection):
raise NotImplementedError("Overwrite this method in subclass!")
@classmethod
def receive_sync(cls, data_conn: Connection):
raise NotImplementedError("Overwrite this method in subclass!")
@classmethod
def enter_async_send_mode(cls, callback: object):
raise NotImplementedError("Overwrite this method in subclass!")
@classmethod
def prepare_sync_send(cls, ctrl_connection: Connection):
raise NotImplementedError("Overwrite this method in subclass!")
@classmethod
def send_sync(cls, data):
raise NotImplementedError("Overwrite this method in subclass!")
@classmethod
def device_receive(cls, data_connection: Connection, ctrl_connection: Connection, dev_parameters: OrderedDict):
if not cls.init_device(ctrl_connection, is_tx=False, parameters=dev_parameters):
ctrl_connection.send("failed to start rx mode")
return False
try:
cls.adapt_num_read_samples_to_sample_rate(dev_parameters[cls.Command.SET_SAMPLE_RATE.name])
except NotImplementedError:
# Many SDRs like HackRF or AirSpy do not need to calculate READ_SAMPLES
# as default values are either fine or given by the hardware
pass
if cls.ASYNCHRONOUS:
ret = cls.enter_async_receive_mode(data_connection, ctrl_connection)
else:
ret = cls.prepare_sync_receive(ctrl_connection)
if ret != 0:
ctrl_connection.send("failed to start rx mode")
return False
exit_requested = False
ctrl_connection.send("successfully started rx mode")
while not exit_requested:
if cls.ASYNCHRONOUS:
time.sleep(0.5)
else:
cls.receive_sync(data_connection)
while ctrl_connection.poll():
result = cls.process_command(ctrl_connection.recv(), ctrl_connection, is_tx=False)
if result == cls.Command.STOP.name:
exit_requested = True
break
cls.shutdown_device(ctrl_connection, is_tx=False)
data_connection.close()
ctrl_connection.close()
@classmethod
def device_send(cls, ctrl_connection: Connection, send_config: SendConfig, dev_parameters: OrderedDict):
if not cls.init_device(ctrl_connection, is_tx=True, parameters=dev_parameters):
ctrl_connection.send("failed to start tx mode")
return False
if cls.ASYNCHRONOUS:
ret = cls.enter_async_send_mode(send_config.get_data_to_send)
else:
ret = cls.prepare_sync_send(ctrl_connection)
if ret != 0:
ctrl_connection.send("failed to start tx mode")
return False
exit_requested = False
buffer_size = cls.CONTINUOUS_SEND_BUFFER_SIZE if send_config.continuous else cls.SEND_BUFFER_SIZE
if not cls.ASYNCHRONOUS and buffer_size == 0:
logger.warning("Send buffer size is zero!")
ctrl_connection.send("successfully started tx mode")
while not exit_requested and not send_config.sending_is_finished():
if cls.ASYNCHRONOUS:
time.sleep(0.5)
else:
cls.send_sync(send_config.get_data_to_send(buffer_size))
while ctrl_connection.poll():
result = cls.process_command(ctrl_connection.recv(), ctrl_connection, is_tx=True)
if result == cls.Command.STOP.name:
exit_requested = True
break
if exit_requested:
logger.debug("{}: exit requested. Stopping sending".format(cls.__class__.__name__))
if send_config.sending_is_finished():
logger.debug("{}: sending is finished.".format(cls.__class__.__name__))
cls.shutdown_device(ctrl_connection, is_tx=True)
ctrl_connection.close()
def __init__(self, center_freq, sample_rate, bandwidth, gain, if_gain=1, baseband_gain=1,
resume_on_full_receive_buffer=False):
super().__init__()
self.error_not_open = -4242
self.__bandwidth = bandwidth
self.__frequency = center_freq
self.__gain = gain # = rf_gain
self.__if_gain = if_gain
self.__baseband_gain = baseband_gain
self.__sample_rate = sample_rate
self.__channel_index = 0
self.__antenna_index = 0
self.__freq_correction = 0
self.__direct_sampling_mode = 0
self.bandwidth_is_adjustable = True
self.is_in_spectrum_mode = False
self.sending_is_continuous = False
self.continuous_send_ring_buffer = None
self.num_samples_to_send = None # None = get automatically. This value needs to be known in continuous send mode
self._current_sent_sample = Value("L", 0)
self._current_sending_repeat = Value("L", 0)
self.success = 0
self.error_codes = {}
self.device_messages = []
self.receive_process_function = self.device_receive
self.send_process_function = self.device_send
self.parent_data_conn, self.child_data_conn = Pipe(duplex=False)
self.parent_ctrl_conn, self.child_ctrl_conn = Pipe()
self.send_buffer = None
self.send_buffer_reader = None
self.samples_to_send = np.array([], dtype=np.complex64)
self.sending_repeats = 1 # How often shall the sending sequence be repeated? 0 = forever
self.resume_on_full_receive_buffer = resume_on_full_receive_buffer # for Spectrum Analyzer or Protocol Sniffing
self.current_recv_index = 0
self.is_receiving = False
self.is_transmitting = False
self.device_ip = "192.168.10.2" # For USRP and RTLSDRTCP
self.receive_buffer = None
self.spectrum_x = None
self.spectrum_y = None
def _start_read_rcv_buffer_thread(self):
self.read_recv_buffer_thread = threading.Thread(target=self.read_receiving_queue)
self.read_recv_buffer_thread.daemon = True
self.read_recv_buffer_thread.start()
def _start_read_message_thread(self):
self.read_dev_msg_thread = threading.Thread(target=self.read_device_messages)
self.read_dev_msg_thread.daemon = True
self.read_dev_msg_thread.start()
@property
def current_sent_sample(self):
return self._current_sent_sample.value // 2
@current_sent_sample.setter
def current_sent_sample(self, value: int):
self._current_sent_sample.value = value * 2
@property
def current_sending_repeat(self):
return self._current_sending_repeat.value
@current_sending_repeat.setter
def current_sending_repeat(self, value: int):
self._current_sending_repeat.value = value
@property
def device_parameters(self) -> OrderedDict:
return OrderedDict([(self.Command.SET_FREQUENCY.name, self.frequency),
(self.Command.SET_SAMPLE_RATE.name, self.sample_rate),
(self.Command.SET_BANDWIDTH.name, self.bandwidth),
(self.Command.SET_RF_GAIN.name, self.gain),
(self.Command.SET_IF_GAIN.name, self.if_gain),
(self.Command.SET_BB_GAIN.name, self.baseband_gain)])
@property
def send_config(self) -> SendConfig:
if self.num_samples_to_send is None:
total_samples = len(self.send_buffer)
else:
total_samples = 2 * self.num_samples_to_send
return SendConfig(self.send_buffer, self._current_sent_sample, self._current_sending_repeat,
total_samples, self.sending_repeats, continuous=self.sending_is_continuous,
pack_complex_method=self.pack_complex,
continuous_send_ring_buffer=self.continuous_send_ring_buffer)
@property
def receive_process_arguments(self):
return self.child_data_conn, self.child_ctrl_conn, self.device_parameters
@property
def send_process_arguments(self):
return self.child_ctrl_conn, self.send_config, self.device_parameters
def init_recv_buffer(self):
if self.receive_buffer is None:
num_samples = SettingsProxy.get_receive_buffer_size(self.resume_on_full_receive_buffer,
self.is_in_spectrum_mode)
self.receive_buffer = np.zeros(int(num_samples), dtype=np.complex64, order='C')
def log_retcode(self, retcode: int, action: str, msg=""):
msg = str(msg)
error_code_msg = self.error_codes[retcode] if retcode in self.error_codes else "Error Code: " + str(retcode)
if retcode == self.success:
if msg:
formatted_message = "{0}-{1} ({2}): Success".format(type(self).__name__, action, msg)
else:
formatted_message = "{0}-{1}: Success".format(type(self).__name__, action)
logger.info(formatted_message)
else:
if msg:
formatted_message = "{0}-{1} ({4}): {2} ({3})".format(type(self).__name__, action, error_code_msg,
retcode, msg)
else:
formatted_message = "{0}-{1}: {2} ({3})".format(type(self).__name__, action, error_code_msg, retcode)
logger.error(formatted_message)
self.device_messages.append(formatted_message)
@property
def received_data(self):
return self.receive_buffer[:self.current_recv_index]
@property
def sent_data(self):
return self.samples_to_send[:self.current_sent_sample]
@property
def sending_finished(self):
return self.current_sent_sample == len(self.samples_to_send)
@property
def bandwidth(self):
return self.__bandwidth
@bandwidth.setter
def bandwidth(self, value):
if not self.bandwidth_is_adjustable:
return
if value != self.__bandwidth:
self.__bandwidth = value
self.set_device_bandwidth(value)
def set_device_bandwidth(self, bw):
try:
self.parent_ctrl_conn.send((self.Command.SET_BANDWIDTH.name, int(bw)))
except (BrokenPipeError, OSError):
pass
@property
def frequency(self):
return self.__frequency
@frequency.setter
def frequency(self, value):
if value != self.__frequency:
self.__frequency = value
self.set_device_frequency(value)
def set_device_frequency(self, value):
try:
self.parent_ctrl_conn.send((self.Command.SET_FREQUENCY.name, int(value)))
except (BrokenPipeError, OSError):
pass
@property
def gain(self):
return self.__gain
@gain.setter
def gain(self, value):
if value != self.__gain:
self.__gain = value
self.set_device_gain(value)
def set_device_gain(self, gain):
try:
# Do not cast gain to int here, as it may be float e.g. for normalized USRP gain or LimeSDR gain
self.parent_ctrl_conn.send((self.Command.SET_RF_GAIN.name, gain))
except (BrokenPipeError, OSError):
pass
@property
def if_gain(self):
return self.__if_gain
@if_gain.setter
def if_gain(self, value):
if value != self.__if_gain:
self.__if_gain = value
self.set_device_if_gain(value)
def set_device_if_gain(self, if_gain):
try:
# Do not cast gain to int here, as it may be float e.g. for normalized USRP gain or LimeSDR gain
self.parent_ctrl_conn.send((self.Command.SET_IF_GAIN.name, if_gain))
except (BrokenPipeError, OSError):
pass
@property
def baseband_gain(self):
return self.__baseband_gain
@baseband_gain.setter
def baseband_gain(self, value):
if value != self.__baseband_gain:
self.__baseband_gain = value
self.set_device_baseband_gain(value)
def set_device_baseband_gain(self, baseband_gain):
try:
# Do not cast gain to int here, as it may be float e.g. for normalized USRP gain or LimeSDR gain
self.parent_ctrl_conn.send((self.Command.SET_BB_GAIN.name, baseband_gain))
except (BrokenPipeError, OSError):
pass
@property
def sample_rate(self):
return self.__sample_rate
@sample_rate.setter
def sample_rate(self, value):
if value != self.__sample_rate:
self.__sample_rate = value
self.set_device_sample_rate(value)
def set_device_sample_rate(self, sample_rate):
try:
self.parent_ctrl_conn.send((self.Command.SET_SAMPLE_RATE.name, int(sample_rate)))
except (BrokenPipeError, OSError):
pass
@property
def channel_index(self) -> int:
return self.__channel_index
@channel_index.setter
def channel_index(self, value: int):
if value != self.__channel_index:
self.__channel_index = value
self.set_device_channel_index(value)
def set_device_channel_index(self, value):
try:
self.parent_ctrl_conn.send((self.Command.SET_CHANNEL_INDEX.name, int(value)))
except (BrokenPipeError, OSError):
pass
@property
def antenna_index(self):
return self.__antenna_index
@antenna_index.setter
def antenna_index(self, value):
if value != self.__antenna_index:
self.__antenna_index = value
self.set_device_antenna_index(value)
def set_device_antenna_index(self, value):
try:
self.parent_ctrl_conn.send((self.Command.SET_ANTENNA_INDEX.name, int(value)))
except (BrokenPipeError, OSError):
pass
@property
def freq_correction(self):
return self.__freq_correction
@freq_correction.setter
def freq_correction(self, value):
if value != self.__freq_correction:
self.__freq_correction = value
self.set_device_freq_correction(value)
def set_device_freq_correction(self, value):
try:
self.parent_ctrl_conn.send((self.Command.SET_FREQUENCY_CORRECTION.name, int(value)))
except (BrokenPipeError, OSError):
pass
@property
def direct_sampling_mode(self):
return self.__direct_sampling_mode
@direct_sampling_mode.setter
def direct_sampling_mode(self, value):
if value != self.__direct_sampling_mode:
self.__direct_sampling_mode = value
self.set_device_direct_sampling_mode(value)
def set_device_direct_sampling_mode(self, value):
try:
self.parent_ctrl_conn.send((self.Command.SET_DIRECT_SAMPLING_MODE.name, int(value)))
except (BrokenPipeError, OSError):
pass
def start_rx_mode(self):
self.init_recv_buffer()
self.parent_data_conn, self.child_data_conn = Pipe(duplex=False)
self.parent_ctrl_conn, self.child_ctrl_conn = Pipe()
self.is_receiving = True
logger.info("{0}: Starting RX Mode".format(self.__class__.__name__))
self.receive_process = Process(target=self.receive_process_function,
args=self.receive_process_arguments)
self.receive_process.daemon = True
self._start_read_rcv_buffer_thread()
self._start_read_message_thread()
try:
self.receive_process.start()
except OSError as e:
logger.error(repr(e))
self.device_messages.append(repr(e))
def stop_rx_mode(self, msg):
try:
self.parent_ctrl_conn.send(self.Command.STOP.name)
except (BrokenPipeError, OSError) as e:
logger.debug("Closing parent control connection: " + str(e))
logger.info("{0}: Stopping RX Mode: {1}".format(self.__class__.__name__, msg))
if hasattr(self, "receive_process") and self.receive_process.is_alive():
self.receive_process.join(self.JOIN_TIMEOUT)
if self.receive_process.is_alive():
logger.warning("{0}: Receive process is still alive, terminating it".format(self.__class__.__name__))
self.receive_process.terminate()
self.receive_process.join()
self.is_receiving = False
self.parent_ctrl_conn.close()
self.parent_data_conn.close()
self.child_ctrl_conn.close()
self.child_data_conn.close()
def start_tx_mode(self, samples_to_send: np.ndarray = None, repeats=None, resume=False):
self.is_transmitting = True
self.parent_ctrl_conn, self.child_ctrl_conn = Pipe()
self.init_send_parameters(samples_to_send, repeats, resume=resume)
logger.info("{0}: Starting TX Mode".format(self.__class__.__name__))
self.transmit_process = Process(target=self.send_process_function,
args=self.send_process_arguments)
self.transmit_process.daemon = True
self._start_read_message_thread()
self.transmit_process.start()
def stop_tx_mode(self, msg):
try:
self.parent_ctrl_conn.send(self.Command.STOP.name)
except (BrokenPipeError, OSError) as e:
logger.debug("Closing parent control connection: " + str(e))
logger.info("{0}: Stopping TX Mode: {1}".format(self.__class__.__name__, msg))
if hasattr(self, "transmit_process") and self.transmit_process.is_alive():
self.transmit_process.join(self.JOIN_TIMEOUT)
if self.transmit_process.is_alive():
logger.warning("{0}: Transmit process is still alive, terminating it".format(self.__class__.__name__))
self.transmit_process.terminate()
self.transmit_process.join()
self.is_transmitting = False
self.parent_ctrl_conn.close()
self.child_ctrl_conn.close()
@staticmethod
def unpack_complex(buffer):
pass
@staticmethod
def pack_complex(complex_samples: np.ndarray):
pass
def read_device_messages(self):
while self.is_receiving or self.is_transmitting:
try:
message = self.parent_ctrl_conn.recv()
try:
action, return_code = message.split(":")
self.log_retcode(int(return_code), action)
except ValueError:
self.device_messages.append("{0}: {1}".format(self.__class__.__name__, message))
except (EOFError, UnpicklingError, OSError, ConnectionResetError) as e:
logger.info("Exiting read device message thread due to " + str(e))
break
self.is_transmitting = False
self.is_receiving = False
logger.debug("Exiting read device errors thread")
def read_receiving_queue(self):
while self.is_receiving:
try:
byte_buffer = self.parent_data_conn.recv_bytes()
samples = self.unpack_complex(byte_buffer)
n_samples = len(samples)
if n_samples > 0:
if self.current_recv_index + n_samples >= len(self.receive_buffer):
if self.resume_on_full_receive_buffer:
self.current_recv_index = 0
if n_samples >= len(self.receive_buffer):
n_samples = len(self.receive_buffer) - 1
else:
self.stop_rx_mode(
"Receiving buffer is full {0}/{1}".format(self.current_recv_index + n_samples,
len(self.receive_buffer)))
return
self.receive_buffer[self.current_recv_index:self.current_recv_index + n_samples] = samples[:n_samples]
old_index = self.current_recv_index
self.current_recv_index += n_samples
self.rcv_index_changed.emit(old_index, self.current_recv_index)
except (BrokenPipeError, OSError):
pass
except EOFError:
logger.info("EOF Error: Ending receive thread")
break
logger.debug("Exiting read_receive_queue thread.")
def init_send_parameters(self, samples_to_send: np.ndarray = None, repeats: int = None, resume=False):
if samples_to_send is not None:
self.samples_to_send = samples_to_send
self.send_buffer = None
if self.send_buffer is None:
self.send_buffer = self.pack_complex(self.samples_to_send)
elif not resume:
self.current_sending_repeat = 0
if repeats is not None:
self.sending_repeats = repeats
class MainFrame(gui.FrameMain):
# for the initialization of the GUI
def __init__(self, parent):
gui.FrameMain.__init__(self, parent)
print "Getting ready..."
self.setStatus("Ready to go. Select Port to begin.")
# initializing variables
self.start_time = 0.0 # start time for referencing
self.filename = "" # filename for file I/O
self.init_line = "" # the first line of file to be reused
self.initial_counter = 0 # the number of characters to be removed from the start of file
self.ports = [] # list of ports of arduino for serial
self.port = "" # port for selection
self.chart_num = 0 # charts that user want to display
self.pause = True # to decide whether acquisition stops
self.arduino = "" # arduino object
self.worker = "" # worker thread for updating voltage
self.ani_worker = "" # worker to run the animation
self.timer_thread = "" # timer thread
self.parent_conn = "" # parent_conn to child process
# initialize the ports
self.refresh()
# create a pubsub receiver
pub.subscribe(self.updateVoltage, "update_voltage")
pub.subscribe(self.updateTimer, "update_timer")
pub.subscribe(self.showMessage, "show_message")
pub.subscribe(self.setStatus, "set_status")
def updateVoltage(self, values):
if len(values):
x = "%.2f" % values[0]
y = "%.2f" % values[1]
z = "%.2f" % values[2]
a = "%.2f" % values[3]
if self.chart_num == 0:
y = "%.2f" % (values[1] - values[2])
else:
x = "-"
y = "-"
z = "-"
a = "-"
self.xDisplay.SetValue(x)
self.yDisplay.SetValue(y)
self.zDisplay.SetValue(z)
self.aDisplay.SetValue(a)
def updateTimer(self, time):
self.timerDisplay.SetValue(time)
def showMessage(self, message):
wx.MessageBox(message, "Note", wx.OK | wx.ICON_INFORMATION)
def setStatus(self, status):
self.statusText.SetLabel(status)
def selectPort(self, event=None):
self.port = self.ports[self.portChoice.GetCurrentSelection()]
# hide unused displays
if self.chart_num < 3:
self.aText.Hide()
self.aDisplay.Hide()
if self.chart_num < 2:
self.zText.Hide()
self.zDisplay.Hide()
# kills all workers and restart port
if self.worker:
self.worker.abort()
self.arduino.close()
# setup serial b/w ard and python
self.arduino = serial.Serial(self.port, 9600, timeout=1)
self.worker = VoltageThread(self, self.arduino)
self.worker.start()
# selection of chart. if user selects chart (2 - 3), show the difference voltage
def selectChart(self, event):
self.chart_num = self.chartChoice.GetCurrentSelection()
self.zText.Hide()
self.zDisplay.Hide()
self.aText.Hide()
self.aDisplay.Hide()
self.xText.Show()
self.xDisplay.Show()
self.yText.Show()
self.yDisplay.Show()
if self.chart_num > 1:
self.zText.Show()
self.zDisplay.Show()
if self.chart_num > 2:
self.aText.Show()
self.aDisplay.Show()
# check and update arduino ports
def refresh(self, event=None):
self.ports_selected = False
# check if arduino port is available
ports = list(list_ports.comports())
self.portChoice.Clear()
self.ports = []
for p in ports:
self.ports.append(p[0])
self.portChoice.Append(p[1])
# kill worker that updates the port
if self.worker:
self.worker.abort()
# when user clicks the "Start" to begin acquistion
def start(self, event):
global port_configured
# check if arduino port is available and selected. if not, cancel.
if port_configured:
print "Save file to begin data acquisition..."
self.statusText.SetLabel("Save file to begin data acquisition.")
# initialize and fire up the file dialog
fd = wx.FileDialog(self,
message="Save data as...",
style=wx.FD_SAVE,
wildcard="Dat file (*.dat)|*.dat")
fd.ShowModal()
# get filename if possible and proceed to save
self.filename = fd.GetPath()
# check if filename exists or user clicked cancel
if self.filename:
self.saveAndStart()
return
else:
self.showMessage(
"Port selected is not functioning or port is not selected. Select a functioning port to START."
)
return
def saveAndStart(self):
# get the first line
self.init_line = "#HD " + time.strftime("%Y %m %d %H %M")
# get the values from the 3 lines
line1 = self.input1.GetValue()
line2 = self.input2.GetValue()
line3 = self.input3.GetValue()
# line to be written into the file
to_be_written = "%s\n%s\n%s\n%s\n" % (self.init_line, line1, line2,
line3)
# update the counter
self.initial_counter = len(to_be_written)
# create the file
datafile = open(self.filename, "w+")
# write the first 3 lines
datafile.write(to_be_written)
# close the file to ensure values are saved
datafile.close()
# set the filePicker value to be the value of the file
self.filePicker1.SetPath(self.filename)
# disable start button
self.startButton.Disable()
# disable port choice
self.portChoice.Disable()
# disable chart choice
self.chartChoice.Disable()
# disable all displays
self.xDisplay.Disable()
self.yDisplay.Disable()
self.zDisplay.Disable()
self.aDisplay.Disable()
# enable end button
self.endButton.Enable()
# set timer to 0s
self.timerDisplay.SetValue("0")
# begin data acquisition
# set variables
self.chart_num = self.chartChoice.GetCurrentSelection()
# kill the existing thread for updating the voltage values
self.worker.abort()
# process pipe
self.parent_conn, child_conn = Pipe()
# start animation process
self.ani_worker = AnimationWorker(child_conn, self.port,
self.chart_num, self.filename)
self.ani_worker.start()
# start timer thread
self.timer_thread = TimerThread(self.parent_conn)
self.timer_thread.start()
self.statusText.SetLabel("Data acquisition started...")
# when user clicks the end button
def end(self, event):
# stop timer thread
self.timer_thread.abort()
# end all animation
if self.ani_worker.is_alive():
self.parent_conn.send(1)
self.parent_conn.recv()
self.parent_conn.close()
# get the values from the 3 lines
line1 = self.input1.GetValue()
line2 = self.input2.GetValue()
line3 = self.input3.GetValue()
# getting ready
to_be_written = "%s\n%s\n%s\n%s\n" % (self.init_line, line1, line2,
line3)
# open up the file
datafile = open(self.filename, "r+")
# read and replace
raw_data = datafile.read()
datafile.close()
# create new data
new_data = to_be_written + raw_data[self.initial_counter:]
# transferred new data
newfile = open(self.filename, "w")
newfile.write(new_data)
newfile.close()
# set the filePicker value to be the empty
self.filePicker1.SetPath("")
# enable start button
self.startButton.Enable()
# enable chart picker
self.portChoice.Enable()
# enable chart picker
self.chartChoice.Enable()
# enable all displays
self.xDisplay.Enable()
self.yDisplay.Enable()
self.zDisplay.Enable()
self.aDisplay.Enable()
# disable end button
self.endButton.Disable()
# restart worker to show voltage and difference
print "Saving data and restarting arduino port..."
self.selectPort(self)
data = conn2.recv() # 一次只能收一条 如果父进程发了很多 过来也要用循环来收
# 如果父进程只发了三条 range(4) 前面三条接收到之后 第四条 没收到就阻塞了
# 解决方法通过套在while 发送端发送结束之后关掉连接 客户端catch EOFError 退出 While
# 前提要求父进程关闭接收端 子进程关发送端
# 因为只有关了 系统才知道 管道 不会再有信息流过了 因为 都只有一端 要么发送 要么接收
# 才不会pend住 所以虽然管道既可以发送 也可接收 父子 肯定只有一方发送 一方接收 也就是砍掉每方的 一个端 才会抛出异常
print("子进程接收数据:", data)
except EOFError:
break
# 父进程代码
# 创建管道(双向)
conn1, conn2 = Pipe()
# 创建进程对象
p = Process(target=recive)
p.start()
# 在发送之前关闭父进程的接收端
conn2.close()
# 父进程向管道发送数据
# conn2.send(123456)
for content in ["聂风", "步惊云", "孔慈"]:
conn1.send(content)
# 关闭发送端：
conn1.close()
p.join()
# Service request
if msg_name == 'quit':
break
elif msg_name == 'request_position':
iip.receiveMessage()
position = TacticsCommon.Point(iip.getChariotX(),
iip.getChariotY())
orientation = iip.getOrientationFromCam()
# Send position and orientation
tactics_pipe.send(["position", position, orientation])
# TODO: Continually read data to ensure up to date information.
# else
#
# iip.receiveMessage()
# # Timestamp data?
if __name__ == '__main__':
mm_Main, mm_Msg = Pipe()
process = Process(target=msg_process, args=(mm_Msg, ))
process.start()
mm_Main.send(["quit"])
process.join()
time.sleep(1)
mm_Main.close()
mm_Msg.close()
process.terminate()
sys.exit()
class Launcher:
def __init__(self):
self.parent_conn, self.child_conn = Pipe()
self.p = Process(target=self.LoadGame, args=(self.child_conn, ))
self.p.start()
pygame.init()
self.FADE_IN_TIME = 1
self.FADE_OUT_TIME = 1
self.isGameLoaded = False
self.ST_FADEIN = 0
self.ST_FADEOUT = 1
self.clock = pygame.time.Clock()
infoObject = pygame.display.Info()
self.width, self.height = infoObject.current_w, infoObject.current_h
self.screen = pygame.display.set_mode((self.width, self.height) #,
#pygame.FULLSCREEN
)
self.font = pygame.font.SysFont('Comic Sans MS', 160, True)
self.rendered_text1 = self.font.render('Made in Python', True,
(255, 0, 0))
self.rendered_text2 = self.font.render('By SirBergue', True,
(255, 0, 0))
self.text_rect = self.rendered_text1.get_rect(center=(self.width // 2,
self.height //
2))
self.state = self.ST_FADEIN
self.last_state_change = time.time()
self.Run()
def PollEvents(self):
# Check for events
for event in pygame.event.get():
if event.type == pygame.QUIT:
pygame.quit()
quit()
if event.type == pygame.KEYDOWN:
if event.key == pygame.K_ESCAPE:
pygame.quit()
quit()
def Update(self):
# Update the state
self.state_time = time.time() - self.last_state_change
if self.state == self.ST_FADEIN:
if self.state_time >= self.FADE_IN_TIME:
self.state = self.ST_FADEOUT
self.alpha = 1.0
self.rt = self.rendered_text2
else:
self.alpha = (1.0 * self.state_time / self.FADE_IN_TIME)
self.rt = self.rendered_text1
elif self.state == self.ST_FADEOUT:
if self.isGameLoaded:
if self.state_time >= self.FADE_OUT_TIME:
self.parent_conn.send(True)
self.parent_conn.close()
pygame.quit()
quit()
self.alpha = 1.0 - (1.0 * self.state_time / self.FADE_OUT_TIME)
else:
if self.parent_conn.poll():
self.isGameLoaded = True
self.last_state_change = time.time()
self.surf2 = pygame.surface.Surface(
(self.text_rect.width, self.text_rect.height))
self.surf2.set_alpha(255 * self.alpha)
def Draw(self):
self.screen.fill((0, 0, 0))
self.surf2.blit(self.rt, (0, 0))
self.screen.blit(self.surf2, self.text_rect)
pygame.display.flip()
def Run(self):
while True:
self.PollEvents()
self.Update()
self.Draw()
self.clock.tick(60)
def LoadGame(self, conn):
from Sources import Loader
Loader.Loader(conn)
class LabEnvironment(environment.Environment):
ACTION_LIST = [
_action(-20, 0, 0, 0, 0, 0, 0), # look_left
_action(20, 0, 0, 0, 0, 0, 0), # look_right
#_action( 0, 10, 0, 0, 0, 0, 0), # look_up
#_action( 0, -10, 0, 0, 0, 0, 0), # look_down
_action(0, 0, -1, 0, 0, 0, 0), # strafe_left
_action(0, 0, 1, 0, 0, 0, 0), # strafe_right
_action(0, 0, 0, 1, 0, 0, 0), # forward
_action(0, 0, 0, -1, 0, 0, 0), # backward
#_action( 0, 0, 0, 0, 1, 0, 0), # fire
#_action( 0, 0, 0, 0, 0, 1, 0), # jump
#_action( 0, 0, 0, 0, 0, 0, 1) # crouch
]
@staticmethod
def get_action_size():
return len(LabEnvironment.ACTION_LIST)
@staticmethod
def inverse_actions(action1, action2):
return np.all(
np.equal(
LabEnvironment.ACTION_LIST[action1] +
LabEnvironment.ACTION_LIST[action2],
np.zeros(len(LabEnvironment.ACTION_LIST[0]))))
def __init__(self, maze_size, seed):
environment.Environment.__init__(self)
self.conn, child_conn = Pipe()
self.proc = Process(target=worker, args=(child_conn, ))
self.proc.start()
self.conn.recv()
self.reset(maze_size, seed)
def reset(self, maze_size, seed):
self.conn.send([COMMAND_RESET, [maze_size, seed]])
self.maze_size = maze_size
obs, pos, ang = self.conn.recv()
self.map = load_map(maze_size, seed)
self.padded_map = np.pad(self._subsample(
self.map - 0.5, 6), [[LOCAL_MAP_WIDTH // 6, LOCAL_MAP_WIDTH // 6],
[LOCAL_MAP_WIDTH // 6, LOCAL_MAP_WIDTH // 6]],
'constant')
position = self._preprocess_position(pos)
angle = self._preprocess_angle(ang)
self.last_state = {
'view':
self._preprocess_frame(obs),
'position':
position,
'angle':
angle,
'egomotion':
np.zeros(9),
'intended_egomotion':
np.zeros(9),
'vlm':
self._get_visual_local_map(self.padded_map, position[2], angle[1])
}
self.last_action = 0
self.last_reward = 0
self.last_intrinsic_reward = 0
self.previous_short_term_goal = np.ones(5)
def stop(self):
self.conn.send([COMMAND_TERMINATE, 0])
ret = self.conn.recv()
self.conn.close()
self.proc.join()
print("lab environment stopped")
def _preprocess_frame(self, image):
image = image.astype(np.float32)
image = image / 255.0
return image
def _preprocess_position(self, pos):
#exact_position[0] = how much to the right, exact_position[1] = how much to the top
exact_position = ((pos[:2] / 100)).astype(np.float32)
#hit_wall = True if and only if the position is the closest the agent can get to a wall
hit_wall = 1e-5 > np.min([(exact_position - 0.16125) % 1.0,
(-exact_position - 0.16125) % 1.0])
#discretization to position indice
pos = (pos[:2] * 3 / 100).astype(np.intc)
pos[1] = 3 * self.maze_size - pos[1] - 1
position_indices = np.asarray([pos[1], pos[0]])
#one hot encoding for training
position = np.zeros(shape=(63, 63))
position[pos[1]][pos[0]] = 1.0
return position, exact_position, position_indices, hit_wall
def _preprocess_angle(self, ang):
# 3-hot encoding
angle_neurons = np.zeros(30)
angle_neurons[int(ang[1] // 12) % 30] = 1.0
angle_neurons[int(ang[1] // 12 + 1) % 30] = 1.0
angle_neurons[int(ang[1] // 12 - 1) % 30] = 1.0
# converting to unit scale
angle = ang[1] / 360.0
if angle < 0:
angle = angle + 1.0
return angle_neurons, angle
def _get_visual_local_map(self, padded_map, pos, ang):
inner_pos = 2 - pos % 3
pos = pos // 3
local_map = padded_map[pos[0] - 1:pos[0] + LOCAL_MAP_WIDTH // 3 + 1,
pos[1] - 1:pos[1] + LOCAL_MAP_WIDTH // 3 + 1]
visual_field = np.asarray([[False] * (LOCAL_MAP_WIDTH + 6)] *
(LOCAL_MAP_WIDTH + 6))
visible_local_map = np.zeros([LOCAL_MAP_WIDTH, LOCAL_MAP_WIDTH])
visible = [True]
marks = [True, True, True, True]
ang = ang - 0.5
if ang < 0:
ang = ang + 1.0
# determine visual field
for ring_radius in range(LOCAL_MAP_WIDTH // 2 + 4):
ring_size = 8 * ring_radius
if ring_size == 0: ring_size = 1
for i in range(2 * ring_radius + 1):
ring_position = int(ang * ring_size + i) % ring_size
if ring_position <= ring_size // 4:
x = LOCAL_MAP_WIDTH // 2 - ring_radius + ring_position
y = LOCAL_MAP_WIDTH // 2 - ring_radius
elif ring_position <= 2 * ring_size // 4:
x = LOCAL_MAP_WIDTH // 2 + ring_radius
y = LOCAL_MAP_WIDTH // 2 - ring_radius + (ring_position -
ring_size // 4)
elif ring_position <= 3 * ring_size // 4:
x = LOCAL_MAP_WIDTH // 2 + ring_radius - (
ring_position - 2 * ring_size // 4)
y = LOCAL_MAP_WIDTH // 2 + ring_radius
else:
x = LOCAL_MAP_WIDTH // 2 - ring_radius
y = LOCAL_MAP_WIDTH // 2 + ring_radius - (
ring_position - 3 * ring_size // 4)
visual_field[x + 3][y + 3] = True
# get visible local map elements
for ring_radius in range(LOCAL_MAP_WIDTH // 3 + 2):
ring_size = 4 * ring_radius
if ring_size == 0: ring_size = 1
for i in range(ring_size):
if visible[i]:
x, y = self.ring_pos_to_xy(i, ring_size)
in_visual_field = np.any(
visual_field[3 * x + inner_pos[0]:3 * x +
inner_pos[0] + 3, 3 * y +
inner_pos[1]:3 * y + inner_pos[1] + 3])
if x <= LOCAL_MAP_WIDTH // 3 + 1 and x >= 0 and y <= LOCAL_MAP_WIDTH // 3 + 1 and y >= 0:
value = local_map[x, y]
else:
value = 0
if in_visual_field:
for l in range(3):
x_ = 3 * (x - 1) + inner_pos[0] + l
if x_ < LOCAL_MAP_WIDTH and x_ >= 0:
for k in range(3):
y_ = 3 * (y - 1) + inner_pos[1] + k
if y_ < LOCAL_MAP_WIDTH and y_ >= 0:
visible_local_map[x_][y_] = value
if value < 0 or not in_visual_field:
visible[i] = False
visible_1 = np.concatenate([
visible[:ring_radius + 1],
visible[ring_radius:2 * ring_radius + 1],
visible[2 * ring_radius:3 * ring_radius + 1],
visible[3 * ring_radius:], visible[:min(ring_radius, 1)]
])
visible_2 = np.concatenate([
visible[:min(ring_radius, 1)], visible[:ring_radius + 1],
visible[ring_radius:2 * ring_radius + 1],
visible[2 * ring_radius:3 * ring_radius + 1],
visible[3 * ring_radius:]
])
visible = np.logical_and(np.logical_or(visible_1, visible_2),
marks)
marks = np.logical_and(visible_1, visible_2)
r = ring_radius + 1
marks_1 = np.concatenate([
marks[:r + 1], marks[r:2 * r + 1], marks[2 * r:3 * r + 1],
marks[3 * r:], [marks[0]]
])
marks_2 = np.concatenate([[marks[-1]], marks[:r + 1],
marks[r:2 * r + 1],
marks[2 * r:3 * r + 1], marks[3 * r:]])
marks = np.logical_or(marks_1, marks_2)
return visible_local_map
def ring_pos_to_xy(self, ring_position, ring_size):
ring_radius = ring_size // 4
if ring_radius < 1:
return LOCAL_MAP_WIDTH // 6 + 1, LOCAL_MAP_WIDTH // 6 + 1
if ring_position < ring_radius:
x = LOCAL_MAP_WIDTH // 6 + ring_position
y = LOCAL_MAP_WIDTH // 6 - ring_radius + ring_position
elif ring_position < 2 * ring_radius:
x = LOCAL_MAP_WIDTH // 6 + ring_radius - ring_position % ring_radius
y = LOCAL_MAP_WIDTH // 6 + ring_position % ring_radius
elif ring_position < 3 * ring_radius:
x = LOCAL_MAP_WIDTH // 6 - ring_position % ring_radius
y = LOCAL_MAP_WIDTH // 6 + ring_radius - ring_position % ring_radius
else:
x = LOCAL_MAP_WIDTH // 6 - ring_radius + ring_position % ring_radius
y = LOCAL_MAP_WIDTH // 6 - ring_position % ring_radius
return x + 1, y + 1
def _get_intrinsic_reward(self, location_uncertainty, shift_weights):
short_term_goal_vector = np.asarray([
self.previous_short_term_goal[2] -
self.previous_short_term_goal[0],
self.previous_short_term_goal[3] - self.previous_short_term_goal[1]
])
egomotion_vector = np.asarray([
np.sum(shift_weights[:3] - shift_weights[6:9]),
shift_weights[0] + shift_weights[3] + shift_weights[6] -
shift_weights[2] - shift_weights[5] - shift_weights[8]
])
return np.dot(short_term_goal_vector, egomotion_vector) + (
self.previous_short_term_goal[4] - location_uncertainty)
def process(self, action, short_term_goal, shift_weights):
real_action = LabEnvironment.ACTION_LIST[action]
self.conn.send([COMMAND_ACTION, real_action])
obs, pos, ang, reward, terminal = self.conn.recv()
if not terminal:
position = self._preprocess_position(pos)
angle = self._preprocess_angle(ang)
state = {
'view':
self._preprocess_frame(obs),
'position':
position,
'angle':
angle,
'vlm':
self._get_visual_local_map(self.padded_map, position[2],
angle[1])
}
if reward > 0:
terminal = True
intrinsic_reward = self._get_intrinsic_reward(
short_term_goal[4], shift_weights)
# small negative reward for running into a wall
if state['position'][3]:
reward = reward - 0.1
else:
state = self.last_state
intrinsic_reward = 0
self.previous_short_term_goal = short_term_goal
self.last_state = state
self.last_action = action
self.last_reward = reward
self.last_intrinsic_reward = intrinsic_reward
return state, reward, intrinsic_reward, terminal
class virtualPortController(Thread):
"""
The class creates a Python thread which simulates control over a serial port. Commands for relaying via the serial port are received from separate Python threads/processes via Queues.
N.B. To start the thread you must call start() from the parent Python thread.
Args:
serialWriteQueue (multiprocessing.Queue): a Queue for receiving commands to be written to the virtual Magstim unit via the serial port
serialReadQueue (multiprocessing.Queue): a Queue for returning automated replies from the virtual Magstim unit when requested
"""
def __init__(self, magstimType, serialWriteQueue, serialReadQueue,
**kwargs):
Thread.__init__(self)
self._serialWriteQueue = serialWriteQueue
self._serialReadQueue = serialReadQueue
self._portConn, self._magstimConn = Pipe()
if magstimType == 'Magstim':
self._magstim = virtualMagstim(self._magstimConn)
elif magstimType == 'BiStim':
self._magstim = virtualBiStim(self._magstimConn)
elif magstimType == 'Rapid':
self._magstim = virtualRapid(self._magstimConn, **kwargs)
elif magstimType == 'Horizon':
from horizonmagpy.virtual_horizon import virtualHorizon
self._magstim = virtualHorizon(self._magstimConn, **kwargs)
else:
raise MagstimError('Unrecognised Magstim type.')
self._magstim.daemon = True
def run(self):
"""
Continuously monitor the serialWriteQueue for commands from other Python threads/processes to be sent to the virtual Magstim.
When requested, will return the automated reply from the virtual Magstim unit to the calling thread via the serialReadQueue.
N.B. This should be called via start() from the parent Python thread.
"""
#Start up virtual magstim
self._magstim.start()
#This continually monitors the serialWriteQueue for write requests
while True:
message, reply, readBytes = self._serialWriteQueue.get()
#If the first part of the message is None this signals the thread to close the port and stop
if message is None:
self._portConn.send(None)
break
#If the first part of the message is a 1 or -1 this signals the thread to do something with the RTS pin, which we don't have - so pass
elif message in {1, -1}:
pass
#Otherwise, the message is a command string
else:
#Try writing to the virtual port
self._portConn.send(message)
#Get reply
if self._portConn.poll(0.3):
#If we want a reply, read the response from the Magstim and place it in the serialReadQueue
if reply:
self._serialReadQueue.put([0, self._portConn.recv()])
#Otherwise just get rid of the reply from the pipe
else:
self._portConn.recv()
else:
self._serialReadQueue.put(
[2, 'Timed out while waiting for response.'])
#If we get here, it's time to shutdown the serial port controller
self._portConn.close()
return