def checkmultiprocess(ipqueue,cacheResult):
if ipqueue.qsize() == 0:
return
processlist = []
"如果ip数小于512，只使用一个子进程，否则则使用指定进程数，每个进程处理平均值的数量ip"
max_threads = g_maxthreads
maxprocess = g_useprocess
if ipqueue.qsize() < g_maxthreads:
max_threads = ipqueue.qsize()
maxprocess = 1
else:
max_threads = (ipqueue.qsize() + g_useprocess) / g_useprocess
if max_threads > g_maxthreads:
max_threads = g_maxthreads
#multiprocessing.log_to_stderr(logging.DEBUG)
for i in xrange(0,maxprocess):
p = Process(target=callsingleprocess,args=(ipqueue,cacheResult,max_threads))
p.daemon = True
processlist.append(p)
p.start()
try:
for p in processlist:
p.join()
except KeyboardInterrupt:
PRINT("need wait all process end...")
for p in processlist:
if p.is_alive():
p.terminate()
def fn_with_timeout(*args, **kwargs):
conn1, conn2 = Pipe()
kwargs["_conn"] = conn2
th = Process(target=partial(fn, best_loss=self._best_loss), args=args, kwargs=kwargs)
th.start()
if conn1.poll(self.trial_timeout):
fn_rval = conn1.recv()
th.join()
else:
self.info("TERMINATING DUE TO TIMEOUT")
th.terminate()
th.join()
fn_rval = "return", {"status": hyperopt.STATUS_FAIL, "failure": "TimeOut"}
assert fn_rval[0] in ("raise", "return")
if fn_rval[0] == "raise":
raise fn_rval[1]
# -- remove potentially large objects from the rval
# so that the Trials() object below stays small
# We can recompute them if necessary, and it's usually
# not necessary at all.
if fn_rval[1]["status"] == hyperopt.STATUS_OK:
fn_loss = float(fn_rval[1].get("loss"))
fn_preprocs = fn_rval[1].pop("preprocs")
fn_classif = fn_rval[1].pop("classifier")
fn_iters = fn_rval[1].pop("iterations")
if fn_loss < self._best_loss:
self._best_preprocs = fn_preprocs
self._best_classif = fn_classif
self._best_loss = fn_loss
self._best_iters = fn_iters
return fn_rval[1]
def load_link(browser, link):
''' Return true if load successful, false otherwise. '''
while True:
p = Process(target=browser_get, args=(browser, link))
p.start()
p.join(LOAD_TIME)
if p.is_alive():
p.terminate()
else:
break
while True:
wait_time = READY_TIME
start_time = time.time()
''' Wait for page to have completely loaded. '''
while True:
state = browser.execute_script('return document.readyState;')
if state == 'complete':
return True
if time.time() - start_time > wait_time:
logging.info("Document %s not ready after %ds", link, wait_time)
break
time.sleep(1)
wait_time = wait_time * READY_RATIO
if wait_time > MAX_READY_TIME * READY_RATIO:
logging.error("Skipping document %s. Was never ready.", link)
return False
else:
logging.info("Increasing wait time to %ds", wait_time)
class TCPServer(object):
def __init__(self, port):
self.port = int(port)
def start(self):
def go(port):
from httpretty import HTTPretty
HTTPretty.disable()
import socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(('localhost', port))
s.listen(True)
conn, addr = s.accept()
while True:
data = conn.recv(1024)
conn.send(b"RECEIVED: " + bytes(data))
conn.close()
args = [self.port]
self.process = Process(target=go, args=args)
self.process.start()
time.sleep(0.4)
def stop(self):
try:
os.kill(self.process.pid, 9)
except OSError:
self.process.terminate()
finally:
self.is_running = False
class SocketServer(asyncore.dispatcher): def __init__(self, host, port): self.clients = Clients() asyncore.dispatcher.__init__(self) self.port = port self.create_socket(socket.AF_INET, socket.SOCK_STREAM) self.set_reuse_addr() self.bind((host, port)) self.bind = host self.listen(5) def __sock_process(self, socket): self.clients.append(socket) self.socketHandler = SocketHandler(self, socket) #handle when a connection is established and a connect() has been issued, add client def handle_accept(self): pair = self.accept() if pair != None: socket, addr = pair self.s = Process(target=self.__sock_process(socket), args=[]) try: self.s.start() except: self.s.terminate() #handle when connection is closed and remove client def handle_close(self): self.clients.remove_all() self.s.close() print 'Sockets closed'
class ArtBox(object):
def __init__(self, width, height):
self._pen_comms = Pipe()
self._paper_comms = Pipe()
self._pen_ear, self._pen_mouth = Pipe()
self._paper_ear, self._paper_mouth = Pipe()
self._pen = pen.Pen()
self._paper = paper.Paper(width=width, height=height)
self._proc = Process(target=self._pen, args=(self._pen_comms, self._paper_comms))
self._proc.daemon = True
def add_resource_folder(self, folder_name):
pyglet.resource.path.append(folder_name)
pyglet.resource.reindex()
def precache(self, asset_dict):
for key in asset_dict:
attributes = asset_dict[key]
if len(attributes) == 1:
self._paper._handle_command(Nibs.Cache(key, attributes[0]))
elif len(attributes) == 2:
self._paper._handle_command(Nibs.Cache(key, attributes[0], attributes[1]))
def open(self):
self._proc.start()
self._paper.unfurl(self._pen_comms, self._paper_comms)
self._proc.join(1)
if self._proc.exitcode is None:
self._proc.terminate()
class FakeProcess:
'''
Runs an instance of multiprocessing.Process, which displays fake results based on PySystemMock.fakeCommandResult{},
or based on a generic countdown using the command string, in the event that fakeCommandResult{} doesn't match.
This class functions as an adapter from multiprocessing.Process() to subprocess.Popen(), which the caller will expect.
'''
stdout = FakeStdout() # can be read by callers as if it's a Process.stdout object
process = None
MOCK_STEPS_ITERATIONS = 5
def start(self, command, fakeCommandResults):
fakeCommandResult = self.getFakeResultForCommand(command, fakeCommandResults)
self.process = Process(target=writeFakeCommandResultsToPipe, args=(self.stdout.writer, fakeCommandResult))
self.process.start()
def getFakeResultForCommand(self, command, fakeCommandResults):
for regex in fakeCommandResults:
match = re.search(regex, command.__str__())
if match:
return fakeCommandResults[regex].split('\n')
return ["processing %s [%d]..." % (command, i) for i in range(self.MOCK_STEPS_ITERATIONS, 0, -1)]
def poll(self):
return self.process.exitcode
def wait(self):
return self.process.wait()
def terminate(self):
self.process.terminate()
def start_schedulers(options):
apps = [app.strip() for app in options.scheduler.split(',')]
try:
from multiprocessing import Process
except:
sys.stderr.write('Sorry, -K only supported for python 2.6-2.7\n')
return
processes = []
code = "from gluon import current;current._scheduler.loop()"
for app in apps:
if not check_existent_app(options, app):
print "Application '%s' doesn't exist, skipping" % (app)
continue
print 'starting scheduler for "%s"...' % app
args = (app,True,True,None,False,code)
logging.getLogger().setLevel(options.debuglevel)
p = Process(target=run, args=args)
processes.append(p)
print "Currently running %s scheduler processes" % (len(processes))
p.start()
print "Processes started"
for p in processes:
try:
p.join()
except (KeyboardInterrupt, SystemExit):
print "Processes stopped"
except:
p.terminate()
p.join()
def fn_with_timeout(*args, **kwargs):
conn1, conn2 = Pipe()
kwargs['_conn'] = conn2
th = Process(target=fn, args=args, kwargs=kwargs)
th.start()
if conn1.poll(self.trial_timeout):
fn_rval = conn1.recv()
th.join()
else:
print 'TERMINATING DUE TO TIMEOUT'
th.terminate()
th.join()
fn_rval = 'return', {
'status': hyperopt.STATUS_FAIL,
'failure': 'TimeOut'
}
assert fn_rval[0] in ('raise', 'return')
if fn_rval[0] == 'raise':
raise fn_rval[1]
# -- remove potentially large objects from the rval
# so that the Trials() object below stays small
# We can recompute them if necessary, and it's usually
# not necessary at all.
if fn_rval[1]['status'] == hyperopt.STATUS_OK:
fn_loss = float(fn_rval[1].get('loss'))
fn_preprocs = fn_rval[1].pop('preprocs')
fn_classif = fn_rval[1].pop('classifier')
if fn_loss < self._best_loss:
self._best_preprocs = fn_preprocs
self._best_classif = fn_classif
self._best_loss = fn_loss
return fn_rval[1]
class KeepAliveClientTest(TestCase):
server_address = ("127.0.0.1", 65535)
def __init__(self, *args, **kwargs):
super(KeepAliveClientTest, self).__init__(*args, **kwargs)
self.server_process = Process(target=self._run_server)
def setUp(self):
super(KeepAliveClientTest, self).setUp()
self.client = Client(["%s:%d" % self.server_address])
self.server_process.start()
time.sleep(.10)
def tearDown(self):
self.server_process.terminate()
super(KeepAliveClientTest, self).tearDown()
def _run_server(self):
self.server = BaseHTTPServer.HTTPServer(self.server_address, ClientAddressRequestHandler)
self.server.handle_request()
def test_client_keepalive(self):
for x in range(10):
result = self.client.sql("select * from fake")
another_result = self.client.sql("select again from fake")
self.assertEqual(result, another_result)
def cluster(nworkers=2):
_port[0] += 1
cport = _port[0]
center = Process(target=run_center, args=(cport,))
workers = []
for i in range(nworkers):
_port[0] += 1
port = _port[0]
proc = Process(target=run_worker, args=(port, cport), kwargs={"ncores": 1})
workers.append({"port": port, "proc": proc})
center.start()
for worker in workers:
worker["proc"].start()
sock = connect_sync("127.0.0.1", cport)
while True:
write_sync(sock, {"op": "ncores"})
ncores = read_sync(sock)
if len(ncores) == nworkers:
break
try:
yield {"proc": center, "port": cport}, workers
finally:
for port in [cport] + [w["port"] for w in workers]:
with ignoring(socket.error):
sock = connect_sync("127.0.0.1", port)
write_sync(sock, dict(op="terminate", close=True))
response = read_sync(sock)
sock.close()
for proc in [center] + [w["proc"] for w in workers]:
with ignoring(Exception):
proc.terminate()
class msListener:
def __init__(self, location, rules):
self.rules = rules
self.location = location
self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
self.sock.bind(location)
self.sock.listen(5) # backlog, queued connections, defalt&max is 5
self.start()
def start(self):
self.loop = Process(target=self.main)
self.loop.start()
print(" * Listener PID:", self.loop.pid)
def exit(self):
self.loop.terminate()
os.remove(self.location)
print(" * Listener stopped")
def main(self):
while True:
connection, address = self.sock.accept()
try:
connection.settimeout(5)
buf = connection.recv(1024).decode("utf-8")
print(' * Listener recieve:', buf)
Process(
target=self.rules.match,
args=(buf, ),
).start()
except socket.timeout:
print(' * Listener time out')
connection.close()
class ServerProc(object):
def __init__(self):
self.proc = None
self.daemon = None
self.stop = Event()
def start(self, init_func, config, paths, port):
self.proc = Process(target=self.create_daemon, args=(init_func, config, paths, port))
self.proc.daemon = True
self.proc.start()
def create_daemon(self, init_func, config, paths, port):
try:
self.daemon = init_func(config, paths, port)
except socket.error:
logger.error("Socket error on port %s" % port)
raise
if self.daemon:
self.daemon.start(block=False)
try:
self.stop.wait()
except KeyboardInterrupt:
pass
def wait(self):
self.stop.set()
self.proc.join()
def kill(self):
self.stop.set()
self.proc.terminate()
self.proc.join()
def test():
queue = Queue()
proc = Process(target=doNothing, args=(queue, ))
proc.start()
_logger.info("Started dummy process with PID %d", proc.pid)
startCodeCheckerServerAttachedToPid(proc.pid)
time.sleep(3)
_logger.info("Allowing the dummy process to finish")
queue.put(1)
proc.join()
if utils.isProcessRunning(proc.pid):
_logger.warning("Dummy process %d was still running", proc.pid)
proc.terminate()
time.sleep(1)
it.assertFalse(utils.isProcessRunning(proc.pid),
"Process %d is still running after terminating "
"it!" % proc.pid)
time.sleep(1)
_logger.info("Server should have died by now")
with it.assertRaises(requests.ConnectionError):
requests.post(it._url + '/get_diagnose_info')
def wrapper(*args, **kwargs):
process = Process(None, func, None, args, kwargs)
process.start()
process.join(seconds)
if process.is_alive():
process.terminate()
raise TimeoutError(error_message)
def test_fork(self):
# Test using a client before and after a fork.
if sys.platform == "win32":
raise SkipTest("Can't fork on Windows")
try:
from multiprocessing import Process, Pipe
except ImportError:
raise SkipTest("No multiprocessing module")
db = self._get_client().pymongo_test
# Failure occurs if the client is used before the fork
db.test.find_one()
def loop(pipe):
while True:
try:
db.test.insert({"a": "b"})
for _ in db.test.find():
pass
except:
traceback.print_exc()
pipe.send(True)
os._exit(1)
cp1, cc1 = Pipe()
cp2, cc2 = Pipe()
p1 = Process(target=loop, args=(cc1,))
p2 = Process(target=loop, args=(cc2,))
p1.start()
p2.start()
p1.join(1)
p2.join(1)
p1.terminate()
p2.terminate()
p1.join()
p2.join()
cc1.close()
cc2.close()
# recv will only have data if the subprocess failed
try:
cp1.recv()
self.fail()
except EOFError:
pass
try:
cp2.recv()
self.fail()
except EOFError:
pass
db.connection.close()
def watcher():
"""This little code snippet is from
http://greenteapress.com/semaphores/threading_cleanup.py (2012-07-31)
It's now possible to interrupt the testrunner via ctrl-c at any time
in a platform neutral way."""
if sys.platform == 'win32':
p = Process(target=main, name="MainProcess")
p.start()
try:
p.join()
rc = p.exitcode
if rc > 0:
sys.exit(rc)
except KeyboardInterrupt:
print 'KeyBoardInterrupt'
p.terminate()
else:
child = os.fork()
if child == 0:
main() # child runs test
try:
rc = os.waitpid(child, 0)[1] /256 # exit status is the high order byte of second member of the tuple
if rc > 0:
sys.exit( rc )
except KeyboardInterrupt:
print 'KeyBoardInterrupt'
try:
os.kill(child, signal.SIGKILL)
except OSError:
pass
except OSError:
pass
sys.exit()
def start_schedulers(apps='w2p_tvseries'):
try:
from multiprocessing import Process
except:
sys.stderr.write('Sorry, -K only supported for python 2.6-2.7\n')
return
processes = []
apps = [app.strip() for app in apps.split(',')]
code = "from gluon import current; current._scheduler.max_empty_runs=10; current._scheduler.loop()"
logging.getLogger().setLevel(logging.INFO)
if len(apps) == 1:
print 'starting single-scheduler for "%s"...' % apps[0]
run(apps[0], True, True, None, False, code)
return
for app in apps:
print 'starting scheduler for "%s"...' % app
args = (app, True, True, None, False, code)
p = Process(target=run, args=args)
processes.append(p)
print "Currently running %s scheduler processes" % (len(processes))
p.start()
print "Processes started"
for p in processes:
try:
p.join()
except (KeyboardInterrupt, SystemExit):
print "Processes stopped"
except:
p.terminate()
p.join()
def test_all_child_processes_should_be_killed_if_parent_is_killed(tmpdir, fork):
tmpdir = str(tmpdir)
p = Process(target=parent, args=(tmpdir,),
kwargs={'pause': True, 'fork': fork})
p.start()
for i in range(100):
time.sleep(0.1)
if glob(os.path.join(tmpdir, 'child-*')):
break
else:
raise Exception("no child process spawned")
p.terminate()
p.join()
cpids = [int(x.split('-')[1])
for x in os.listdir(tmpdir) if x.startswith('child-')]
assert len(cpids) == 3
for i in range(100):
time.sleep(0.1)
if not any(is_process_alive(pid) for pid in cpids):
break
else:
assert False, "child processes are not killed"
# ensure the parent logic is executed only once
assert len(glob(os.path.join(tmpdir, 'parent-*'))) == 1
def run(self):
parent_pipe, child_pipe = Pipe(False)
p = Process(target = run_metaheuristic,
args = (child_pipe, self.model, self.pt, self.aa,
self.algo, self.n, self.use_heur,
self.worst, self.best))
p.start()
for i in range(self.n + 1):
if self.is_stopped() is True:
parent_pipe.close()
p.terminate()
return
try:
result = parent_pipe.recv()
except:
break
self.results.append(result[0])
self.fitness.append(result[1])
self.emit(QtCore.SIGNAL('update(int)'), i)
if result[1] == 1:
break
parent_pipe.close()
p.join()
def execute_action(self, action):
event = Event()
queue = Queue()
proc = Process(
target=execute_action_proc,
args=(self.execute, action, event, queue))
proc.start()
# Send heartbeat.
heartbeat_retry = 0
while not event.is_set():
event.wait(config.ACTIVITY_HEARTBEAT_INTERVAL)
try:
res = self.heartbeat(self.task_token)
if res['cancelRequested']:
proc.terminate()
proc.join()
return Result('cancelled', -1, '', '', '', -1)
except Exception as err:
if heartbeat_retry <= config.ACTIVITY_HEARTBEAT_MAX_RETRY:
heartbeat_retry += 1
continue
else:
proc.terminate()
proc.join()
raise
# Evaluate the result.
result = queue.get_nowait()
proc.join()
return result
def handler():
req = bottle.request
content_type = req.get_header('content-type')
data = req.body.read()
if content_type == 'application/json':
data = req.json
event = {
'data': data,
'event-id': req.get_header('event-id'),
'event-type': req.get_header('event-type'),
'event-time': req.get_header('event-time'),
'event-namespace': req.get_header('event-namespace'),
'extensions': {
'request': req
}
}
method = req.method
func_calls.labels(method).inc()
with func_errors.labels(method).count_exceptions():
with func_hist.labels(method).time():
q = Queue()
p = Process(target=funcWrap, args=(q, event, function_context))
p.start()
p.join(timeout)
# If thread is still active
if p.is_alive():
p.terminate()
p.join()
return bottle.HTTPError(408, "Timeout while processing the function")
else:
res = q.get()
if isinstance(res, Exception):
raise res
return res
def query_results(query):
"""
Synchronously return results for an SQL query and return them as a dict to
be sent back to clients.
If the query takes too long, kill the query process and return an error
instead.
"""
conn = sqlite3.connect(config.DB_URI_READ_ONLY, uri=True)
result_queue = Queue()
query_process = Process(target=query_processor,
args=(conn, query, result_queue))
query_process.start()
query_process.join(config.QUERY_TIMEOUT_SECS)
if query_process.is_alive():
query_process.terminate()
err = {'error': 'Query took too long; max time %s seconds' %
config.QUERY_TIMEOUT_SECS}
return err
output = result_queue.get()
if type(output) is sqlite3.OperationalError:
err = {'error': repr(output)}
return err
return output
def main():
func.mysql_exec("insert into mysql_bigtable_history SELECT *,LEFT(REPLACE(REPLACE(REPLACE(create_time,'-',''),' ',''),':',''),8) from mysql_bigtable",'')
func.mysql_exec('delete from mysql_bigtable;','')
#get mysql servers list
servers = func.mysql_query('select id,host,port,username,password,tags,bigtable_size from db_servers_mysql where is_delete=0 and monitor=1 and bigtable_monitor=1;')
if servers:
print("%s: check mysql bigtable controller started." % (time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()),));
plist = []
for row in servers:
server_id=row[0]
host=row[1]
port=row[2]
username=row[3]
password=row[4]
tags=row[5]
bigtable_size=row[6]
p = Process(target = check_mysql_bigtable, args = (host,port,username,password,server_id,tags,bigtable_size))
plist.append(p)
for p in plist:
p.start()
time.sleep(15)
for p in plist:
p.terminate()
for p in plist:
p.join()
print("%s: check mysql bigtable controller finished." % (time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()),))
def run_parkinglot_expt(net, n):
"Run experiment"
seconds = args.time
# Start the bandwidth and cwnd monitors in the background
monitor = Process(target=monitor_devs_ng,
args=('%s/bwm.txt' % args.dir, 1.0))
monitor.start()
start_tcpprobe()
# Get receiver and clients
recvr = net.getNodeByName('receiver')
sender1 = net.getNodeByName('h1')
# Start the receiver
port = 5001
recvr.cmd('iperf -s -p', port,
'> %s/iperf_server.txt' % args.dir, '&')
waitListening(sender1, recvr, port)
# TODO: start the sender iperf processes and wait for the flows to finish
# Hint: Use getNodeByName() to get a handle on each sender.
# Hint: Use sendCmd() and waitOutput() to start iperf and wait for them to finish
# Hint: waitOutput waits for the command to finish allowing you to wait on a particular process on the host
# iperf command to start flow: 'iperf -c %s -p %s -t %d -i 1 -yc > %s/iperf_%s.txt' % (recvr.IP(), 5001, seconds, args.dir, node_name)
# Hint (not important): You may use progress(t) to track your experiment progress
recvr.cmd('kill %iperf')
# Shut down monitors
monitor.terminate()
stop_tcpprobe()
def __run_tests(self, tests):
logging.debug("Start __run_tests.")
logging.debug("__name__ = %s",__name__)
if self.os.lower() == 'windows':
logging.debug("Executing __run_tests on Windows")
for test in tests:
self.__run_test(test)
else:
logging.debug("Executing __run_tests on Linux")
# These features do not work on Windows
for test in tests:
if __name__ == 'benchmark.benchmarker':
print textwrap.dedent("""
-----------------------------------------------------
Running Test: {name} ...
-----------------------------------------------------
""".format(name=test.name))
test_process = Process(target=self.__run_test, args=(test,))
test_process.start()
test_process.join(self.run_test_timeout_seconds)
if(test_process.is_alive()):
logging.debug("Child process for {name} is still alive. Terminating.".format(name=test.name))
self.__write_intermediate_results(test.name,"__run_test timeout (="+ str(self.run_test_timeout_seconds) + " seconds)")
test_process.terminate()
logging.debug("End __run_tests.")
class StreamGatherer():
streamname = "mouseclicks"
streamschema = {"type": "integer"}
description = "Gathers the number of clicks made with the mouse"
datatype = "action.count"
def __init__(self):
self.click_number = Value('i',0)
self.clicklogger_process = None
def start(self,cache):
# Starts the background processes and stuff. The cache is passed, so that
# if the gatherer catches events, they can be logged as they come in
if self.clicklogger_process is None:
self.clicklogger_process = Process(target=log_click_count,args=(self.click_number,))
self.clicklogger_process.daemon = True
self.clicklogger_process.start()
def stop(self):
if self.clicklogger_process is not None:
self.clicklogger_process.terminate()
self.clicklogger_process = None
self.click_number.value = 0
def run(self,cache):
clk = self.clicks()
if clk > 0:
cache.insert(self.streamname,clk)
#Gets the number of keypresses that are logged, and reset the counter
def clicks(self):
v = self.click_number.value
self.click_number.value = 0
return v
def checkAfsCredentials(self):
'''
we do an ssh connection to an AFS computer and return true if we manage
to "land" on that computer
TODO: very pseudo-method >>> find a better way
'''
if not self.afsuser or not self.afspw:
self.parent.displayErrorMessage('Missing','No blank fields for AFS-credentials allowed')
return
comp = 'llc1'
pw = 'echo \\\"'+self.afspw+'\\\"'
p1 = subprocess.Popen(['echo "#!/bin/bash\n" > ~/pw.sh'],shell=True)
p2 = subprocess.Popen(['echo '+pw+' >> ~/pw.sh'],shell=True)
p3 = subprocess.Popen(['chmod a+x ~/pw.sh'],shell=True)
queue1 = Queue()
p = Process(target=self.checkAfsCredentialsHelper,args=(comp,queue1))
p.start()
p.join(2)
try:
if queue1.get(): # we try to get the queue >> will throw an error after timeout
subprocess.call(['rm ~/pw.sh'],shell=True)
return True
except IOError as e:
subprocess.call(['rm ~/pw.sh'],shell=True)
p.terminate()
self.parent.displayErrorMessage('Error','Wrong AFS Password/Username!')
return False
except ValueError:
print "value error"
return False
class ProcessStarter(object):
def __init__(self):
'''
Setup the shared memory data structure model and initialize the control parts.
'''
self.running = True
self.orprocess = None
self.guiprocess = None
self.pipeGUI, self.pipeOR = Pipe()
self.StartProcesses()
def StartProcesses(self):
self.guiprocess = Process(target=self.__startGUI__)
self.guiprocess.start()
self.pipeGUI.send(["StartViewer", None])
self.orprocess = Process(target=ORServer,args=(self.pipeOR,))
self.orprocess.start()
return True
def terminate(self):
try:
self.pipeGUI.send(["Stop", None])
self.guiprocess.terminate()
self.orprocess.terminate()
except:
pass
def __startGUI__(self):
app = QtGui.QApplication(sys.argv)
form = pnpApp(self.pipeGUI, self)
form.show()
sys.exit(app.exec_())
def main():
port = common.PORT
username = input("Choose a username : "******"", flush=True)
signal.signal(41, pause_handler)
signal.signal(42, chat_handler)
signal.pause()
try:
chat.chat(port, username)
except (KeyboardInterrupt, SystemExit): # Handle the ctrl+c
chat.broadcast(port, chat.encode(username, '{} leaved the chat'.format(username), special=True))
print('*** Quitting ***')
p.terminate()
def run_server():
server.run(host="0.0.0.0", port=int(os.environ.get('PORT', 5000)))
if __name__ == "__main__":
server_process = None
scheduler_process1 = None
try:
print("starting server")
server_process = Process(target=run_server)
server_process.start()
print(server_process)
print("starting schedule")
scheduler_process1 = Process(target=run_schedule)
scheduler_process1.start()
print(scheduler_process1)
bot.polling(none_stop=True)
finally:
if server_process:
server_process.terminate()
server_process.join()
if scheduler_process1:
scheduler_process1.terminate()
scheduler_process1.join()
print("Put {0} to Queue".format(i))
queue.put(i)
time.sleep(1)
def Read(queue):
print("Read Process {0}".format(os.getpid()))
while True:
msg = queue.get()
print("Get {0} from Queue".format(msg))
time.sleep(1)
# ==========================main==========================
try:
if __name__ == "__main__":
q = Queue()
p_W = Process(target = Write,args=(q,))
p_R = Process(target = Read,args=(q,))
p_W.start()
p_R.start()
p_W.join()
p_R.terminate()
except Exception as ex:
print("Error: {0}".format(ex))
#input("Press Enter to continue...\n")
def test_add_vote(self):
##
## run service
##
checker_service_process = Process(target=vote_contract.run_checker_service)
checker_service_process.start()
time.sleep(0.1)
##
## create transaction
##
# create keys
params = setup()
(tally_priv, tally_pub) = key_gen(params)
(voter1_priv, voter1_pub) = key_gen(params)
(_, voter2_pub) = key_gen(params)
(_, voter3_pub) = key_gen(params)
# set up options, particpants, and tally's key
options = ['alice', 'bob']
participants = [pack(voter1_pub), pack(voter2_pub), pack(voter3_pub)]
# get init token
init_transaction = vote.init()
token = init_transaction['transaction']['outputs'][0]
# initialise vote (all votes are zero)
create_vote_transaction = vote.create_vote(
(token,),
None,
None,
dumps(options),
dumps(participants),
pack(tally_priv),
pack(tally_pub)
)
old_vote = create_vote_transaction['transaction']['outputs'][1]
# add a vote
transaction = vote.add_vote(
(old_vote,),
None,
None,
dumps([1, 0]),
pack(voter1_priv),
pack(voter1_pub)
)
##
## submit transaction
##
response = requests.post(
'http://127.0.0.1:5000/' + vote_contract.contract_name + '/add_vote', json=transaction_to_solution(transaction)
)
self.assertTrue(response.json()['success'])
##
## stop service
##
checker_service_process.terminate()
checker_service_process.join()
def write(q):
print('Process to write: %s' % os.getpid())
for value in ['A', 'B', 'C']:
print('Put %s to queue...' % value)
q.put(value)
time.sleep(random.random())
# 读数据进程执行的代码:
def read(q):
print('Process to read: %s' % os.getpid())
while True:
value = q.get(True)
print('Get %s from queue.' % value)
if __name__ == '__main__':
# 父进程创建Queue，并传给各个子进程：
q = Queue()
pw = Process(target=write, args=(q, ))
pr = Process(target=read, args=(q, ))
# 启动子进程pw，写入:
pw.start()
# 启动子进程pr，读取:
pr.start()
# 等待pw结束:
pw.join()
# pr进程里是死循环，无法等待其结束，只能强行终止:
pr.terminate()
class TestRunnerManager(threading.Thread):
def __init__(self,
suite_name,
test_queue,
test_source_cls,
browser_cls,
browser_kwargs,
executor_cls,
executor_kwargs,
stop_flag,
rerun=1,
pause_after_test=False,
pause_on_unexpected=False,
restart_on_unexpected=True,
debug_info=None,
capture_stdio=True):
"""Thread that owns a single TestRunner process and any processes required
by the TestRunner (e.g. the Firefox binary).
TestRunnerManagers are responsible for launching the browser process and the
runner process, and for logging the test progress. The actual test running
is done by the TestRunner. In particular they:
* Start the binary of the program under test
* Start the TestRunner
* Tell the TestRunner to start a test, if any
* Log that the test started
* Log the test results
* Take any remedial action required e.g. restart crashed or hung
processes
"""
self.suite_name = suite_name
self.test_source = test_source_cls(test_queue)
self.browser_cls = browser_cls
self.browser_kwargs = browser_kwargs
self.executor_cls = executor_cls
self.executor_kwargs = executor_kwargs
# Flags used to shut down this thread if we get a sigint
self.parent_stop_flag = stop_flag
self.child_stop_flag = multiprocessing.Event()
self.rerun = rerun
self.run_count = 0
self.pause_after_test = pause_after_test
self.pause_on_unexpected = pause_on_unexpected
self.restart_on_unexpected = restart_on_unexpected
self.debug_info = debug_info
self.manager_number = next_manager_number()
self.command_queue = Queue()
self.remote_queue = Queue()
self.test_runner_proc = None
threading.Thread.__init__(self,
name="TestRunnerManager-%i" %
self.manager_number)
# This is started in the actual new thread
self.logger = None
self.test_count = 0
self.unexpected_count = 0
# This may not really be what we want
self.daemon = True
self.max_restarts = 5
self.browser = None
self.capture_stdio = capture_stdio
def run(self):
"""Main loop for the TestRunnerManager.
TestRunnerManagers generally receive commands from their
TestRunner updating them on the status of a test. They
may also have a stop flag set by the main thread indicating
that the manager should shut down the next time the event loop
spins."""
self.logger = structuredlog.StructuredLogger(self.suite_name)
with self.browser_cls(self.logger, **self.browser_kwargs) as browser:
self.browser = BrowserManager(self.logger,
browser,
self.command_queue,
no_timeout=self.debug_info
is not None)
dispatch = {
RunnerManagerState.before_init: self.start_init,
RunnerManagerState.initializing: self.init,
RunnerManagerState.running: self.run_test,
RunnerManagerState.restarting: self.restart_runner
}
self.state = RunnerManagerState.before_init()
end_states = (RunnerManagerState.stop, RunnerManagerState.error)
try:
while not isinstance(self.state, end_states):
f = dispatch.get(self.state.__class__)
while f:
self.logger.debug("Dispatch %s" % f.__name__)
if self.should_stop():
return
new_state = f()
if new_state is None:
break
self.state = new_state
self.logger.debug("new state: %s" %
self.state.__class__.__name__)
if isinstance(self.state, end_states):
return
f = dispatch.get(self.state.__class__)
new_state = None
while new_state is None:
new_state = self.wait_event()
if self.should_stop():
return
self.state = new_state
self.logger.debug("new state: %s" %
self.state.__class__.__name__)
except Exception as e:
self.logger.error(traceback.format_exc(e))
raise
finally:
self.logger.debug(
"TestRunnerManager main loop terminating, starting cleanup"
)
clean = isinstance(self.state, RunnerManagerState.stop)
self.stop_runner(force=not clean)
self.teardown()
self.logger.debug("TestRunnerManager main loop terminated")
def wait_event(self):
dispatch = {
RunnerManagerState.before_init: {},
RunnerManagerState.initializing: {
"init_succeeded": self.init_succeeded,
"init_failed": self.init_failed,
},
RunnerManagerState.running: {
"test_ended": self.test_ended,
"wait_finished": self.wait_finished,
},
RunnerManagerState.restarting: {},
RunnerManagerState.error: {},
RunnerManagerState.stop: {},
None: {
"runner_teardown": self.runner_teardown,
"log": self.log,
"error": self.error
}
}
try:
command, data = self.command_queue.get(True, 1)
self.logger.debug("Got command: %r" % command)
except IOError:
self.logger.error("Got IOError from poll")
return RunnerManagerState.restarting(0)
except Empty:
if (self.debug_info and self.debug_info.interactive
and self.browser.started and not self.browser.is_alive()):
self.logger.debug("Debugger exited")
return RunnerManagerState.stop()
if (isinstance(self.state, RunnerManagerState.running)
and not self.test_runner_proc.is_alive()):
if not self.command_queue.empty():
# We got a new message so process that
return
# If we got to here the runner presumably shut down
# unexpectedly
self.logger.info("Test runner process shut down")
if self.state.test is not None:
# This could happen if the test runner crashed for some other
# reason
# Need to consider the unlikely case where one test causes the
# runner process to repeatedly die
self.logger.critical("Last test did not complete")
return RunnerManagerState.error()
self.logger.warning(
"More tests found, but runner process died, restarting")
return RunnerManagerState.restarting(0)
else:
f = (dispatch.get(self.state.__class__, {}).get(command)
or dispatch.get(None, {}).get(command))
if not f:
self.logger.warning("Got command %s in state %s" %
(command, self.state.__class__.__name__))
return
return f(*data)
def should_stop(self):
return self.child_stop_flag.is_set() or self.parent_stop_flag.is_set()
def start_init(self):
test, test_group, group_metadata = self.get_next_test()
if test is None:
return RunnerManagerState.stop()
else:
return RunnerManagerState.initializing(test, test_group,
group_metadata, 0)
def init(self):
assert isinstance(self.state, RunnerManagerState.initializing)
if self.state.failure_count > self.max_restarts:
self.logger.error("Max restarts exceeded")
return RunnerManagerState.error()
self.browser.update_settings(self.state.test)
result = self.browser.init(self.state.group_metadata)
if result is Stop:
return RunnerManagerState.error()
elif not result:
return RunnerManagerState.initializing(
self.state.test, self.state.test_group,
self.state.group_metadata, self.state.failure_count + 1)
else:
self.executor_kwargs["group_metadata"] = self.state.group_metadata
self.start_test_runner()
def start_test_runner(self):
# Note that we need to be careful to start the browser before the
# test runner to ensure that any state set when the browser is started
# can be passed in to the test runner.
assert isinstance(self.state, RunnerManagerState.initializing)
assert self.command_queue is not None
assert self.remote_queue is not None
self.logger.info("Starting runner")
executor_browser_cls, executor_browser_kwargs = self.browser.browser.executor_browser(
)
args = (self.remote_queue, self.command_queue, self.executor_cls,
self.executor_kwargs, executor_browser_cls,
executor_browser_kwargs, self.capture_stdio,
self.child_stop_flag)
self.test_runner_proc = Process(target=start_runner,
args=args,
name="TestRunner-%i" %
self.manager_number)
self.test_runner_proc.start()
self.logger.debug("Test runner started")
# Now we wait for either an init_succeeded event or an init_failed event
def init_succeeded(self):
assert isinstance(self.state, RunnerManagerState.initializing)
self.browser.after_init()
return RunnerManagerState.running(self.state.test,
self.state.test_group,
self.state.group_metadata)
def init_failed(self):
assert isinstance(self.state, RunnerManagerState.initializing)
self.browser.check_crash(None)
self.browser.after_init()
self.stop_runner(force=True)
return RunnerManagerState.initializing(self.state.test,
self.state.test_group,
self.state.group_metadata,
self.state.failure_count + 1)
def get_next_test(self, test_group=None):
test = None
while test is None:
while test_group is None or len(test_group) == 0:
test_group, group_metadata = self.test_source.group()
if test_group is None:
self.logger.info("No more tests")
return None, None, None
test = test_group.popleft()
self.run_count = 0
return test, test_group, group_metadata
def run_test(self):
assert isinstance(self.state, RunnerManagerState.running)
assert self.state.test is not None
if self.browser.update_settings(self.state.test):
self.logger.info("Restarting browser for new test environment")
return RunnerManagerState.restarting(self.state.test,
self.state.test_group,
self.state.group_metadata)
self.logger.test_start(self.state.test.id)
if self.rerun > 1:
self.logger.info("Run %d/%d" % (self.run_count, self.rerun))
self.send_message("reset")
self.run_count += 1
self.send_message("run_test", self.state.test)
def test_ended(self, test, results):
"""Handle the end of a test.
Output the result of each subtest, and the result of the overall
harness to the logs.
"""
assert isinstance(self.state, RunnerManagerState.running)
assert test == self.state.test
# Write the result of each subtest
file_result, test_results = results
subtest_unexpected = False
for result in test_results:
if test.disabled(result.name):
continue
expected = test.expected(result.name)
is_unexpected = expected != result.status
if is_unexpected:
self.unexpected_count += 1
self.logger.debug("Unexpected count in this thread %i" %
self.unexpected_count)
subtest_unexpected = True
self.logger.test_status(test.id,
result.name,
result.status,
message=result.message,
expected=expected,
stack=result.stack)
# We have a couple of status codes that are used internally, but not exposed to the
# user. These are used to indicate that some possibly-broken state was reached
# and we should restart the runner before the next test.
# INTERNAL-ERROR indicates a Python exception was caught in the harness
# EXTERNAL-TIMEOUT indicates we had to forcibly kill the browser from the harness
# because the test didn't return a result after reaching the test-internal timeout
status_subns = {
"INTERNAL-ERROR": "ERROR",
"EXTERNAL-TIMEOUT": "TIMEOUT"
}
expected = test.expected()
status = status_subns.get(file_result.status, file_result.status)
if self.browser.check_crash(test.id):
status = "CRASH"
self.test_count += 1
is_unexpected = expected != status
if is_unexpected:
self.unexpected_count += 1
self.logger.debug("Unexpected count in this thread %i" %
self.unexpected_count)
if "assertion_count" in file_result.extra:
assertion_count = file_result.extra.pop("assertion_count")
if assertion_count > 0:
self.logger.assertion_count(test.id, int(assertion_count),
test.min_assertion_count,
test.max_assertion_count)
file_result.extra[
"test_timeout"] = test.timeout * self.executor_kwargs[
'timeout_multiplier']
self.logger.test_end(test.id,
status,
message=file_result.message,
expected=expected,
extra=file_result.extra,
stack=file_result.stack)
restart_before_next = (test.restart_after or file_result.status
in ("CRASH", "EXTERNAL-TIMEOUT",
"INTERNAL-ERROR")
or ((subtest_unexpected or is_unexpected)
and self.restart_on_unexpected))
if (not file_result.status == "CRASH" and self.pause_after_test
or (self.pause_on_unexpected and
(subtest_unexpected or is_unexpected))):
self.logger.info("Pausing until the browser exits")
self.send_message("wait")
else:
return self.after_test_end(test, restart_before_next)
def wait_finished(self):
assert isinstance(self.state, RunnerManagerState.running)
self.logger.debug("Wait finished")
# The browser should be stopped already, but this ensures we do any
# post-stop processing
return self.after_test_end(self.state.test, True)
def after_test_end(self, test, restart):
assert isinstance(self.state, RunnerManagerState.running)
if self.run_count == self.rerun:
test, test_group, group_metadata = self.get_next_test()
if test is None:
return RunnerManagerState.stop()
if test_group != self.state.test_group:
# We are starting a new group of tests, so force a restart
restart = True
else:
test_group = self.state.test_group
group_metadata = self.state.group_metadata
if restart:
return RunnerManagerState.restarting(test, test_group,
group_metadata)
else:
return RunnerManagerState.running(test, test_group, group_metadata)
def restart_runner(self):
"""Stop and restart the TestRunner"""
assert isinstance(self.state, RunnerManagerState.restarting)
self.stop_runner()
return RunnerManagerState.initializing(self.state.test,
self.state.test_group,
self.state.group_metadata, 0)
def log(self, action, kwargs):
getattr(self.logger, action)(**kwargs)
def error(self, message):
self.logger.error(message)
self.restart_runner()
def stop_runner(self, force=False):
"""Stop the TestRunner and the browser binary."""
if self.test_runner_proc is None:
return
if self.test_runner_proc.is_alive():
self.send_message("stop")
try:
self.browser.stop(force=force)
self.ensure_runner_stopped()
finally:
self.cleanup()
def teardown(self):
self.logger.debug("TestRunnerManager teardown")
self.test_runner_proc = None
self.command_queue.close()
self.remote_queue.close()
self.command_queue = None
self.remote_queue = None
def ensure_runner_stopped(self):
self.logger.debug("ensure_runner_stopped")
if self.test_runner_proc is None:
return
self.logger.debug("waiting for runner process to end")
self.test_runner_proc.join(10)
self.logger.debug("After join")
if self.test_runner_proc.is_alive():
# This might leak a file handle from the queue
self.logger.warning("Forcibly terminating runner process")
self.test_runner_proc.terminate()
# Multiprocessing queues are backed by operating system pipes. If
# the pipe in the child process had buffered data at the time of
# forced termination, the queue is no longer in a usable state
# (subsequent attempts to retrieve items may block indefinitely).
# Discard the potentially-corrupted queue and create a new one.
self.command_queue.close()
self.command_queue = Queue()
self.remote_queue.close()
self.remote_queue = Queue()
else:
self.logger.debug("Runner process exited with code %i" %
self.test_runner_proc.exitcode)
def runner_teardown(self):
self.ensure_runner_stopped()
return RunnerManagerState.stop()
def send_message(self, command, *args):
self.remote_queue.put((command, args))
def cleanup(self):
self.logger.debug("TestRunnerManager cleanup")
if self.browser:
self.browser.cleanup()
while True:
try:
cmd, data = self.command_queue.get_nowait()
except Empty:
break
else:
if cmd == "log":
self.log(*data)
elif cmd == "runner_teardown":
# It's OK for the "runner_teardown" message to be left in
# the queue during cleanup, as we will already have tried
# to stop the TestRunner in `stop_runner`.
pass
else:
self.logger.warning(
"Command left in command_queue during cleanup: %r, %r"
% (cmd, data))
while True:
try:
cmd, data = self.remote_queue.get_nowait()
self.logger.warning(
"Command left in remote_queue during cleanup: %r, %r" %
(cmd, data))
except Empty:
break
def test_get(self):
data = requests.get('http://0.0.0.0:8000/products',headers=headers)
self.assertEqual(data.json(), LocalData.products)
def test_post(self):
requests.post('http://0.0.0.0:8000/products',headers=headers,json={'id':'product2'})
data_get = requests.get('http://0.0.0.0:8000/products/product2',headers=headers)
self.assertTrue(data_get.json(),{'id':'product2'})
def test_put(self):
requests.post('http://0.0.0.0:8000/products/product2',headers=headers,json={'id':'product2','cost':'20'})
data_get = requests.get('http://0.0.0.0:8000/products/product2',headers=headers)
self.assertTrue(data_get.json(),{'id':'product2','cost':'20'})
def test_delete(self):
requests.delete('http://0.0.0.0:8000/products/product2',headers=headers)
data_get = requests.get('http://0.0.0.0:8000/products/product2',headers=headers)
self.assertTrue(data_get.json(),{"status": "200","msg": "record_deleted"})
# def test_search(self):
# pass
if __name__ == '__main__':
p1 = Process(target = run)
p1.start()
p2 = Process(target = unittest.main)
p2.start()
p2.join(timeout=1)
p1.terminate()
class FaceDetectorProcess:
def __init__(self, userId, cameraId):
self.process = Process(target=self.run, args=())
self.process.daemon = True
self.showVideoVariable = False
self.live = self.process.is_alive()
self.stop_running = False
self.suspectData = suspectData(userId, cameraId)
self.settings = self.suspectData.getSettings()
self.encodings, self.names = self.suspectData.readEncodings()
self.userId = userId
def start(self):
self.process.start()
self.process.join()
self.live = self.process.is_alive()
def stop(self):
self.process.terminate()
self.stop_running = True
return
def checkStatus(self):
# reads from DynamoDB table attribute req_Status to check if user requested stoping the server
check = self.suspectData.getServiceStatus()
if not check[0]['req_Status']:
logging.warning("Streamming will be terminated")
self.stop_running = True
# Update Server Status on Settings Table as (0) "Server Stopped"
self.suspectData.changeServerStatus(0, "Streaming will terminate")
return
def reportFinding(self, name, report):
# reportFind = suspectData(self.user, self.camera, name, report)
self.suspectData.reportFinding(name, report)
return
def checkData(self):
if self.settings == [] or self.names == [] or self.encodings == []:
return False
else:
return True
def createEncodings(self):
self.suspectData.createEncodings()
return
def snapShot(self, face_locations, face_names, frame, name):
for (top, right, bottom, left), name in zip(face_locations,
face_names):
# Scale back up face locations since the frame we detected in was scaled to 1/4 size
logging.warning("Snaphot")
top *= 4
right *= 4
bottom *= 4
left *= 4
# Draw a box around the face
cv2.rectangle(frame, (left, top), (right, bottom), (0, 0, 255), 2)
# Draw a label with a name below the face
cv2.rectangle(frame, (left, bottom - 35), (right, bottom),
(0, 0, 255), cv2.FILLED)
font = cv2.FONT_HERSHEY_DUPLEX
frame = cv2.putText(frame, name, (left + 6, bottom - 6), font, 1.0,
(255, 255, 255), 1)
snapshotId = uuid.uuid4()
cv2.imwrite(f'./tmp/{self.userId}/{snapshotId}.png', frame)
self.suspectData.uploadSnapShot(name, f'{snapshotId}.png')
def run(self):
logging.warning("Starting streamming")
# Initialize a timer
sched = BackgroundScheduler(daemon=True)
# check if user has request to stop server, every "seconds" ,
sched.add_job(self.checkStatus,
'interval',
seconds=int(os.environ['TIME_TO_CHECK']))
sched.start()
# read Data for specific user and camera
if self.checkData():
username = self.settings[0]['username']
password = self.settings[0]['password']
ip = self.settings[0]["ip"]
port = self.settings[0]["port"]
URL = self.settings[0]["url_path"]
else:
sched.shutdown()
logging.warning("User doesn't exists or Camera is not configured")
return "Server stopped"
try:
video_capture = cv2.VideoCapture(
f"rtsp://{username}:{password}@{ip}:{port}/{URL}")
if not video_capture.isOpened():
self.suspectData.changeServerStatus(0, "No stream available")
logging.warning("No stream available, check RTSP settings")
video_capture.release()
sched.shutdown()
return "Server stopped"
else:
self.suspectData.changeServerStatus(1, "Streaming")
# Initialize detection variables
face_locations = []
face_encodings = []
face_names = []
process_this_frame = True
log = {}
# Face Detaction process, exits if self.stop_running is True
logging.warning("Streamming")
while not self.stop_running:
try:
# Grab a single frame of video
ret, frame = video_capture.read()
if not (ret):
logging.warning("No Video capture")
st = time.time()
video_capture = cv2.VideoCapture(
f"rtsp://{username}:{password}@{ip}:{port}/{URL}")
ret, frame = video_capture.read()
continue
#if ret == True:
# Resize frame of video to 1/4 size for faster face recognition processing
small_frame = cv2.resize(frame, (0, 0), fx=0.25, fy=0.25)
# Convert the image from BGR color (which OpenCV uses) to RGB color (which face_recognition uses)
rgb_small_frame = small_frame[:, :, ::-1]
# Only process every other frame of video to save time
if process_this_frame:
# Find all the faces and face encodings in the current frame of video
face_locations = face_recognition.face_locations(
rgb_small_frame)
face_encodings = face_recognition.face_encodings(
rgb_small_frame, face_locations)
face_names = []
for face_encoding in face_encodings:
# See if the face is a match for the known face(s)
matches = face_recognition.compare_faces(
self.encodings, face_encoding)
name = "Unknown"
face_distances = face_recognition.face_distance(
self.encodings, face_encoding)
best_match_index = np.argmin(face_distances)
if matches[best_match_index]:
name = self.names[best_match_index]
now = datetime.now()
if log.get(name) != None:
if abs(now -
log[name]) < timedelta(minutes=10):
log[name] = now
logging.warning(
f'{name} seen no more than 10 min ago'
)
else:
face_names.append(name)
logging.warning(
f'{name} Return to site')
log[name] = now
date = now.strftime("%m/%d/%y, %H:%M")
self.snapShot(face_locations,
face_names, frame, name)
self.reportFinding(name, date)
else:
#logger(name, self.settings["phone"])
face_names.append(name)
logging.warning(f'{name} seen')
log[name] = now
print(name)
date = now.strftime("%m/%d/%y, %H:%M")
self.snapShot(face_locations, face_names,
frame, name)
self.reportFinding(name, date)
face_names.append(name)
# if cv2.waitKey(1) & 0xFF == ord('q'):
# break
if self.stop_running:
break
process_this_frame = not process_this_frame
if (self.showVideoVariable):
# Display the results
for (top, right, bottom,
left), name in zip(face_locations, face_names):
# Scale back up face locations since the frame we detected in was scaled to 1/4 size
top *= 4
right *= 4
bottom *= 4
left *= 4
# Draw a box around the face
cv2.rectangle(frame, (left, top), (right, bottom),
(0, 0, 255), 2)
# Draw a label with a name below the face
cv2.rectangle(frame, (left, bottom - 35),
(right, bottom), (0, 0, 255),
cv2.FILLED)
font = cv2.FONT_HERSHEY_DUPLEX
cv2.putText(frame, name, (left + 6, bottom - 6),
font, 1.0, (255, 255, 255), 1)
# Display the resulting image
cv2.imshow('Video', frame)
# Hit 'q' on the keyboard to quit!
if cv2.waitKey(1) & 0xFF == ord('q'):
break
# if self.stop_running:
# break
except cv2.error as e:
self.suspectData.changeServerStatus(
0, "No stream available")
logging.warning(
"CV2 Error - Streaming failed, check RTSP settings", e)
#video_capture.release()
sched.shutdown()
return "Server stopped"
# Release handle to the webcam
video_capture.release()
cv2.destroyAllWindows()
sched.shutdown()
logging.warning("Streamming terminated")
self.suspectData.changeServerStatus(0, "streamming terminated")
return "Server stopped"
except cv2.error as e:
self.suspectData.changeServerStatus(0, "No stream available")
logging.warning(
"CV2 Error - Streaming failed, check RTSP settings", e)
#video_capture.release()
sched.shutdown()
return "Server stopped"
class HttpViewer(object):
""" Class that process the and show the points received from Neato. """
def __init__(self, port, laser_queue, pose_queue):
"""
Constructor, port where the web will be served
laser_queue is the queue where the laser points has to be queued.
pose_queue is the queue where the pose point has to be queued.
"""
self.port = port
self.exit = False
self.mm_per_pixel = 20
self.laser_queue = laser_queue
self.pose_queue = pose_queue
self.process = Process(target=self.run)
self.process.start()
def main_http_server(self):
""" Start point of the thread that manages the http server. """
Handler = http_server.HttpServerHandler
self.httpd = SocketServer.TCPServer(("", self.port), Handler)
self.httpd.serve_forever()
def main_laser(self):
""" Start point of the thread that gets the laser points """
print "Laser thread"
laser_points = []
while True:
new_points = self.laser_queue.get()
laser_points.extend(new_points)
print("LASER POINTS:", laser_points)
if len(laser_points) > 1000:
laser_points = laser_points[360:]
print "Laser len:", len(laser_points)
write_points_json(laser_points, self.mm_per_pixel, "laserPoints", "laser.json")
def main_pose(self):
""" Start point of the thread that gets the pose points. """
print "Pose thread"
pose_points = []
while True:
new_points = self.pose_queue.get()
pose_points.extend(new_points)
if len(pose_points) > 5000:
pose_points = pose_points[360:]
write_points_json(pose_points, self.mm_per_pixel, "pose", "pose.json")
def run(self):
""" Method that start all the threads. """
sys.stdout = open('http_log.txt', 'w')
self.thread = threading.Thread(target=self.main_http_server)
self.thread.start()
self.thread_laser = threading.Thread(target=self.main_laser)
self.thread_laser.start()
self.thread_pose = threading.Thread(target=self.main_pose)
self.thread_pose.start()
while not self.exit:
time.sleep(0.4)
def quit(self):
""" Stops execution of the web server. """
self.exit = True
self.httpd.shutdown()
self.process.terminate()
from vmtk import pypes
except:
print "Unexpected error:", sys.exc_info()[0]
raise
if __name__=='__main__':
manager = Manager()
queue = manager.list()
if sys.platform != 'darwin':
pypeProcess = Process(target=pypeserver.PypeServer, args=(queue,None,None), kwargs={"returnIfEmptyQueue":True})
pypeProcess.start()
args = sys.argv[:]
if sys.argv[0].startswith('pyperun'):
args = sys.argv[1:]
queue.append(args)
try:
if sys.platform != 'darwin':
pypeProcess.join()
else:
pypeserver.PypeServer(queue,None,None,returnIfEmptyQueue=True)
except KeyboardInterrupt:
pypeProcess.terminate()
except BaseException, e:
print e
sys.exit(1)
class RemoteTest(unittest.TestCase):
"""
Test the Component, DataFlow, and VAs when shared remotely.
The test cases are run as "clients" and at start a server is started.
"""
container_name = "test"
def setUp(self):
# Use Thread for debug:
if USE_THREADS:
self.server = threading.Thread(target=ServerLoop, args=(self.container_name,))
else:
self.server = Process(target=ServerLoop, args=(self.container_name,))
self.server.start()
self.count = 0
self.data_arrays_sent = 0
time.sleep(0.1) # give it some time to start
self.rdaemon = Pyro4.Proxy("PYRO:Pyro.Daemon@./u:" + self.container_name)
self.comp = self.rdaemon.getObject("mycomp")
def tearDown(self):
self.comp.stopServer()
time.sleep(0.1) # give it some time to terminate
if self.server.is_alive():
if not USE_THREADS:
print("Warning: killing server still alive")
self.server.terminate()
# @unittest.skip("simple")
def test_simple(self):
"""
start a component, ping, and stop it
"""
ret = self.comp.ping()
self.assertEqual(ret, "pong", "Ping failed")
# @unittest.skip("simple")
def test_exception(self):
# test it raises
self.assertRaises(MyError, self.comp.bad_call)
# test it raises when wrong argument
self.assertRaises(TypeError, self.comp.ping, ("non needed arg",))
# non existing method
self.assertRaises(AttributeError, self.comp.non_existing_method)
# @unittest.skip("simple")
def test_roattributes(self):
"""
check roattributes
"""
val = self.comp.my_value
self.assertEqual(val, "ro", "Reading attribute failed")
# @unittest.skip("simple")
def test_async(self):
"""
test futures
MyComponent queues the future in order of request
"""
self.comp.set_number_futures(0)
ft1 = self.comp.do_long(2) # long enough we can cancel ft2
ft2 = self.comp.do_long(1) # shorter than ft1
self.assertFalse(ft1.done(), "Future finished too early")
self.assertFalse(ft2.done(), "Future finished too early")
self.assertFalse(ft2.cancelled(), "future doesn't claim being cancelled")
self.assertFalse(ft2.cancelled(), "future doesn't claim being cancelled")
self.assertGreater(ft2.result(), 1) # wait for ft2
self.assertFalse(ft2.cancel(), "could cancel the finished future")
self.assertTrue(ft1.done(), "Future not finished")
self.assertGreater(ft1.result(), 2)
self.assertEqual(self.comp.get_number_futures(), 2)
# @unittest.skip("simple")
def test_unref_futures(self):
"""
test many futures which don't even get referenced
It should behave as if the function does not return anything
"""
self.comp.set_number_futures(0)
expected = 100 # there was a bug with expected > threadpool size (=24)
start = time.time()
for i in range(expected):
self.comp.do_long(0.1)
ft_last = self.comp.do_long(0.1)
ft_last.result()
duration = time.time() - start
self.assertGreaterEqual(duration, expected * 0.1)
self.assertEqual(self.comp.get_number_futures(), expected + 1)
# @unittest.skip("simple")
def test_ref_futures(self):
"""
test many futures which get referenced and accumulated
It should behave as if the function does not return anything
"""
self.comp.set_number_futures(0)
small_futures = []
expected = 100 # there was a bug with expected > threadpool size (=24)
start = time.time()
for i in range(expected):
small_futures.append(self.comp.do_long(0.1))
ft_last = self.comp.do_long(0.1)
ft_last.result()
duration = time.time() - start
self.assertGreaterEqual(duration, expected * 0.1)
for f in small_futures:
self.assertTrue(f.done())
self.assertEqual(self.comp.get_number_futures(), expected + 1)
# @unittest.skip("simple")
def test_async_cancel(self):
"""
test futures
"""
self.comp.set_number_futures(0)
ft1 = self.comp.do_long(2) # long enough we can cancel ft2
ft2 = self.comp.do_long(1) # shorter than ft1
self.assertTrue(ft2.cancel(), "couldn't cancel the future")
self.assertTrue(ft2.cancelled(), "future doesn't claim being cancelled")
self.assertRaises(futures.CancelledError, ft2.result)
# wait for ft1
res1a = ft1.result()
self.assertGreater(res1a, 2)
self.assertTrue(ft1.done(), "Future not finished")
res1b = ft1.result()
self.assertEqual(res1a, res1b)
self.assertGreater(res1b, 2)
self.assertEqual(self.comp.get_number_futures(), 2)
def test_prog_future(self):
"""
Test ProgressiveFuture (remotely)
"""
self._done_calls = 0
self._progess_calls = 0
self._start = None
self._end = None
ft = self.comp.do_progressive_long(5)
ft.add_update_callback(self._on_future_proress)
ft.add_done_callback(self._on_future_done)
# we should have received at least one progress update already
self.assertGreaterEqual(self._progess_calls, 1)
ft.result()
self.assertGreaterEqual(self._progess_calls, 5)
self.assertEqual(self._done_calls, 1)
def _on_future_done(self, f):
self._done_calls += 1
def _on_future_proress(self, f, start, end):
self._progess_calls += 1
logging.info("Received future update for %f -> %f", start, end)
self._start = start
self._end = end
# @unittest.skip("simple")
def test_subcomponents(self):
# via method and via roattributes
# need to test cycles
p = self.rdaemon.getObject("parent")
self.assertEqual(len(p.children.value), 1, "parent doesn't have one child")
c = list(p.children.value)[0]
# self.assertEqual(c.parent, p, "Component and parent of child is different")
self.assertEqual(p.value, 42)
self.assertEqual(c.value, 43)
self.assertEqual(len(c.children.value), 0, "child shouldn't have children")
# @unittest.skip("simple")
def test_dataflow_subscribe(self):
self.count = 0
self.expected_shape = (2048, 2048)
self.data_arrays_sent = 0
self.comp.data.reset()
self.comp.data.subscribe(self.receive_data)
time.sleep(0.5)
self.comp.data.unsubscribe(self.receive_data)
count_end = self.count
print("received %d arrays over %d" % (self.count, self.data_arrays_sent))
time.sleep(0.1)
self.assertEqual(count_end, self.count)
self.assertGreaterEqual(count_end, 1)
def test_synchronized_df(self):
"""
Tests 2 dataflows, one synchronized on the event of acquisition started
on the other dataflow.
"""
number = 20
self.count = 0
self.left = number
self.expected_shape = (2, 2)
self.expected_shape_au = (2048, 2048)
self.data_arrays_sent = 0
dfe = self.comp.data
dfs = self.comp.datas
dfe.reset()
dfs.synchronizedOn(self.comp.startAcquire)
dfs.subscribe(self.receive_data)
time.sleep(0.2) # long enough to be after the first data if it generates data
# ensure that datas hasn't generated anything yet
self.assertEqual(self.count, 0)
dfe.subscribe(self.receive_data_auto_unsub)
for i in range(number):
# end early if it's already finished
if self.left == 0:
break
time.sleep(0.2) # 0.2s should be more than enough in any case
self.assertEqual(0, self.left)
self.assertEqual(number, self.count)
print("received %d arrays over %d" % (self.count, self.data_arrays_sent))
max_lat = dfs.get_max_lat()
if max_lat:
print("latency: %r, max= %f, avg= %f" % (max_lat, max(max_lat), sum(max_lat)/len(max_lat)))
time.sleep(0.1)
self.assertEqual(number, self.count)
dfs.unsubscribe(self.receive_data)
dfs.synchronizedOn(None)
time.sleep(0.1)
self.assertEqual(number, self.count)
# @unittest.skip("simple")
def test_dataflow_stridden(self):
# test that stridden array can be passed (even if less efficient)
self.count = 0
self.data_arrays_sent = 0
self.expected_shape = (2048, 2045)
self.comp.cut.value = 3
self.comp.data.reset()
self.comp.data.subscribe(self.receive_data)
time.sleep(0.5)
self.comp.data.unsubscribe(self.receive_data)
self.comp.cut.value = 0 # put it back
count_end = self.count
print("received %d stridden arrays over %d" % (self.count, self.data_arrays_sent))
time.sleep(0.1)
self.assertEqual(count_end, self.count)
self.assertGreaterEqual(count_end, 1)
def test_dataflow_empty(self):
"""
test passing empty DataArray
"""
self.count = 0
self.data_arrays_sent = 0
self.comp.data.setShape((0,), 16)
self.expected_shape = (0,)
self.comp.data.subscribe(self.receive_data)
time.sleep(0.5)
self.comp.data.unsubscribe(self.receive_data)
count_end = self.count
print("received %d stridden arrays over %d" % (self.count, self.data_arrays_sent))
time.sleep(0.1)
self.assertEqual(count_end, self.count)
self.assertGreaterEqual(count_end, 1)
def receive_data(self, dataflow, data):
self.count += 1
self.assertEqual(data.shape, self.expected_shape)
if data.ndim >= 2:
self.data_arrays_sent = data[0][0]
self.assertGreaterEqual(self.data_arrays_sent, self.count)
def receive_data_auto_unsub(self, dataflow, data):
"""
callback for df
"""
self.assertEqual(data.shape, self.expected_shape_au)
self.left -= 1
if self.left <= 0:
dataflow.unsubscribe(self.receive_data_auto_unsub)
def receive_data_and_unsubscribe(self, dataflow, data):
self.count += 1
self.assertEqual(data.shape, (2048, 2048))
self.data_arrays_sent = data[0][0]
self.assertGreaterEqual(self.data_arrays_sent, self.count)
dataflow.unsubscribe(self.receive_data_and_unsubscribe)
# @unittest.skip("simple")
def test_dataflow_unsubscribe_from_callback(self):
self.count = 0
self.data_arrays_sent = 0
self.comp.data.reset()
self.comp.data.subscribe(self.receive_data_and_unsubscribe)
time.sleep(0.3)
self.assertEqual(self.count, 1)
# It should be 1, or if the generation went very fast, it might be bigger
self.assertGreaterEqual(self.data_arrays_sent, 1)
# print "received %d arrays over %d" % (self.count, self.data_arrays_sent)
# data = comp.data
# del comp
# print gc.get_referrers(data)
# gc.collect()
# print gc.get_referrers(data)
# @unittest.skip("simple")
def test_dataflow_get(self):
self.comp.data.reset()
array = self.comp.data.get()
self.assertEqual(array.shape, (2048, 2048))
self.assertEqual(array[0][0], 0)
array = self.comp.data.get()
self.assertEqual(array.shape, (2048, 2048))
self.assertEqual(array[0][0], 0)
# @unittest.skip("simple")
def test_va_update(self):
prop = self.comp.prop
self.assertIsInstance(prop, VigilantAttributeBase)
self.assertEqual(prop.value, 42)
prop.value += 1
self.assertEqual(prop.value, 43)
self.called = 0
self.last_value = None
prop.subscribe(self.receive_va_update)
# now count
prop.value = 3 # +1
prop.value = 0 # +1
prop.value = 0 # nothing because same value
time.sleep(0.1) # give time to receive notifications
prop.unsubscribe(self.receive_va_update)
self.assertEqual(prop.value, 0)
self.assertEqual(self.last_value, 0)
# called once or twice depending if the brief 3 was seen
self.assertTrue(1 <= self.called <= 2)
called_before = self.called
# check we are not called anymore
prop.value = 3 # +1
self.assertEqual(self.called, called_before)
# re-subscribe
prop.subscribe(self.receive_va_update)
# change remotely
self.comp.change_prop(45)
time.sleep(0.1) # give time to receive notifications
self.assertEqual(prop.value, 45)
prop.unsubscribe(self.receive_va_update)
self.assertEqual(self.called, called_before + 1)
try:
prop.value = 7.5
self.fail("Assigning float to a int should not be allowed.")
except TypeError:
pass # as it should be
def receive_va_update(self, value):
self.called += 1
self.last_value = value
self.assertIsInstance(value, (int, float))
def test_va_override(self):
self.comp.prop.value = 42
with self.assertRaises(AttributeError):
# Simulate typo of "self.comp.prop.value = 42"
self.comp.prop = 42
# @unittest.skip("simple")
def test_enumerated_va(self):
# enumerated
self.assertEqual(self.comp.enum.value, "a")
self.assertEqual(self.comp.enum.choices, {"a", "c", "bfds"})
self.comp.enum.value = "c"
self.assertEqual(self.comp.enum.value, "c")
try:
self.comp.enum.value = "wfds"
self.fail("Assigning out of bound should not be allowed.")
except IndexError:
pass # as it should be
def test_continuous_va(self):
# continuous
self.assertEqual(self.comp.cont.value, 2)
self.assertEqual(self.comp.cont.range, (-1, 3.4))
self.comp.cont.value = 3.0
self.assertEqual(self.comp.cont.value, 3)
try:
self.comp.cont.value = 4.0
self.fail("Assigning out of bound should not be allowed.")
except IndexError:
pass # as it should be
def test_list_va(self):
# List
l = self.comp.listval
self.assertEqual(len(l.value), 2)
self.called = 0
l.subscribe(self.receive_listva_update)
l.value += [3]
self.assertEqual(len(l.value), 3)
time.sleep(0.01)
l.value[-1] = 4
self.assertEqual(l.value[-1], 4)
time.sleep(0.01)
l.value.reverse()
self.assertEqual(l.value[0], 4)
time.sleep(0.1)
self.assertEqual(self.called, 3)
l.unsubscribe(self.receive_listva_update)
def receive_listva_update(self, value):
self.called += 1
self.last_value = value
self.assertIsInstance(value, list)
class EnqueuedProcess(object):
"""The class that takes care of the handling of a single process. It provides
support for timeout check, exit status check and resource termination commits.
It implements methods to start and gently terminate processes, so that the
resource database will be updated with the current state.
- timeout check -- Check if a waiting process exceedes its timeout for waiting
to be run and terminate it.
A resource update will be send to the resource database.
- exits status check -- Check if the exit status of the process was 0, if not
check if the resource database acknowledged this with
a termination or error message, if not send a resource
update
- termination commits - Terminate the process and send an update to the resource
database about the termination
"""
def __init__(self, func, timeout, resource_logger, args):
self.process = Process(target=func, args=args)
self.timeout = timeout
self.config = args[0].config
self.resource_id = args[0].resource_id
self.iteration = args[0].iteration
self.user_id = args[0].user_id
self.api_info = args[0].api_info
self.resource_logger = resource_logger
self.init_time = time.time()
self.started = False
def __del__(self):
pass
# print("Process deleted", self.resource_id)
def start(self):
"""Start the process
:return:
"""
# print("Start job: ", self.api_info)
self.started = True
self.process.start()
def terminate(self, status, message):
"""Terminate the process
Send a termination response to the resource database.
Args:
status: The status why termination was requested
message: The message why the process was terminated by the server
(timeout, server shutdown, ...)
"""
# print("Terminate process with message: ", message)
if self.process.is_alive():
self.process.terminate()
self._send_resource_update(status=status, message=message)
def is_alive(self):
return self.process.is_alive()
def exitcode(self):
return self.process.exitcode
def check_timeout(self):
"""Check if the process waited longer for running then the timeout that was set
Terminate the process if the timeout limit was exceeded.
Returns:
False if within timeout, True if the process terminated itself
"""
# print("Check timeout for process: ", self.api_info)
if self.started is False:
current_time = time.time()
diff = current_time - self.init_time
if self.timeout < diff:
self.terminate(status="timeout",
message="Processes exceeded timeout (%i) in "
"waiting queue and was terminated." %
self.timeout)
return True
return False
def check_exit(self):
"""Check the exitcode, if a non-zero exit code was received then
send an update to the resource logger that something strange happened.
Send only if the status of the resource is not "error", "terminated"
or "timeout".
"""
if self.process.exitcode is not None and self.process.exitcode != 0:
# Check if the process noticed the error already
response_data = self.resource_logger.get(self.user_id,
self.resource_id,
self.iteration)
if response_data is not None:
http_code, response_model = pickle.loads(response_data)
if response_model["status"] != "error" and \
response_model["status"] != "terminated" and \
response_model["status"] != "timeout":
message = (
"The process unexpectedly terminated with exit code %i"
% self.process.exitcode)
self._send_resource_update(status="error",
message=message,
response_data=response_data)
def _send_resource_update(self, status, message, response_data=None):
"""Send a response to the resource logger about the current resource state
Args:
status: The status that should be set (terminated)
message: The message
"""
# print("Send resource update status: ", status, " message: ", message)
# Get the latest response and use it as template for the resource update
if response_data is None:
response_data = self.resource_logger.get(self.user_id,
self.resource_id,
self.iteartion)
# Send the termination response
if response_data is not None:
http_code, response_model = pickle.loads(response_data)
# print("Resource", http_code, response_model)
response_model["status"] = status
response_model["message"] = (
"The process was terminated by the server: %s" % message)
orig_time = response_model["accept_timestamp"]
response_model["timestamp"] = time.time()
response_model["datetime"] = str(datetime.now())
response_model[
"time_delta"] = response_model["timestamp"] - orig_time
document = pickle.dumps([http_code, response_model])
self.resource_logger.commit(
user_id=self.user_id,
resource_id=self.resource_id,
iteration=self.iteration,
document=document,
expiration=self.config.REDIS_RESOURCE_EXPIRE_TIME)
class ParallelGenerator(object):
def __init__(self,
orig_gen,
max_lookahead=None,
get_timeout=10):
"""
Creates a parallel generator from a normal one.
The elements will be prefetched up to max_lookahead
ahead of the consumer. If max_lookahead is None,
everything will be fetched.
The get_timeout parameter is the number of seconds
after which we check that the subprocess is still
alive, when waiting for an element to be generated.
Any exception raised in the generator will
be forwarded to this parallel generator.
"""
if max_lookahead:
self.queue = Queue(max_lookahead)
else:
self.queue = Queue()
def wrapped():
try:
for item in orig_gen:
self.queue.put(item)
raise StopIteration()
except Exception as e:
self.queue.put(ExceptionItem(e))
self.get_timeout = get_timeout
self.ppid = None
self.process = Process(target=wrapped)
self.process_started = False
def finish_if_possible(self):
if self.ppid == os.getpid() and self.process:# and self.process.is_alive():
self.process.terminate()
self.process = None
self.queue = None
self.ppid = None
def __enter__(self):
"""
Starts the process
"""
self.ppid = os.getpid()
self.process.start()
self.process_started = True
return self
def __exit__(self, exc_type, exc_val, exc_tb):
"""
Kills the process
"""
assert self.process_started and self.ppid == None or self.ppid == os.getpid()
self.finish_if_possible()
def __next__(self):
return self.next()
def __iter__(self):
return self
def __del__(self):
self.finish_if_possible()
def next(self):
if not self.process_started:
raise ParallelGeneratorException(
"""The generator has not been started.
Please use "with ParallelGenerator(..) as g:"
""")
try:
item_received = False
while not item_received:
try:
item = self.queue.get(timeout=self.get_timeout)
item_received = True
except Empty:
# check that the process is still alive
if not self.process.is_alive():
raise GeneratorDied(
"The generator died unexpectedly.")
if type(item) == ExceptionItem:
raise item.exception
return item
except Exception:
self.finish_if_possible()
raise
def minimize(
criterion,
params,
algorithm,
criterion_args=None,
criterion_kwargs=None,
constraints=None,
general_options=None,
algo_options=None,
dashboard=False,
db_options=None,
):
"""Minimize *criterion* using *algorithm* subject to *constraints* and bounds.
Args:
criterion (function):
Python function that takes a pandas DataFrame with parameters as the first
argument and returns a scalar floating point value.
params (pd.DataFrame):
See :ref:`params`.
algorithm (str):
specifies the optimization algorithm. See :ref:`list_of_algorithms`.
criterion_args (list or tuple):
additional positional arguments for criterion
criterion_kwargs (dict):
additional keyword arguments for criterion
constraints (list):
list with constraint dictionaries. See for details.
general_options (dict):
additional configurations for the optimization
algo_options (dict):
algorithm specific configurations for the optimization
dashboard (bool):
whether to create and show a dashboard
db_options (dict):
dictionary with kwargs to be supplied to the run_server function.
"""
# set default arguments
simplefilter(action="ignore", category=pd.errors.PerformanceWarning)
criterion_args = [] if criterion_args is None else criterion_args
criterion_kwargs = {} if criterion_kwargs is None else criterion_kwargs
constraints = [] if constraints is None else constraints
general_options = {} if general_options is None else general_options
algo_options = {} if algo_options is None else algo_options
db_options = {} if db_options is None else db_options
params = _process_params(params)
fitness_eval = criterion(params, *criterion_args, **criterion_kwargs)
constraints = process_constraints(constraints, params)
internal_params = reparametrize_to_internal(params, constraints, None)
scaling_factor = calculate_scaling_factor(
criterion,
params,
internal_params,
constraints,
general_options,
criterion_args,
criterion_kwargs,
)
queue = Queue() if dashboard else None
if dashboard:
stop_signal = Event()
outer_server_process = Process(
target=run_server,
kwargs={
"queue": queue,
"db_options": db_options,
"start_param_df": params,
"start_fitness": fitness_eval,
"stop_signal": stop_signal,
},
daemon=False,
)
outer_server_process.start()
result = _minimize(
criterion=criterion,
criterion_args=criterion_args,
criterion_kwargs=criterion_kwargs,
params=params,
internal_params=internal_params,
constraints=constraints,
scaling_factor=scaling_factor,
algorithm=algorithm,
algo_options=algo_options,
general_options=general_options,
queue=queue,
)
if dashboard:
stop_signal.set()
outer_server_process.terminate()
return result
q = ()
proc_writer1 = Process(target=proc_write,
args=(q, ['url_1', 'url_2', 'url_3']))
proc_writer2 = Process(target=proc_write,
args=(q, ['url_4', 'url_5', 'url_6']))
proc_reader = Process(target=proc_read, args=(q, ))
# 启动子进程proc_writer，写入:
proc_writer1.start()
proc_writer2.start()
# 启动子进程proc_reader，读取:
proc_reader.start()
# 等待proc_writer结束:
proc_writer1.join()
proc_writer2.join()
# proc_reader进程里是死循环，无法等待其结束，只能强行终止:
proc_reader.terminate()
"""
Queue 进程间通信
Queue 用来在多个进程间通信。Queue 有两个方法，get 和 put。
put 方法
Put 方法用来插入数据到队列中，有两个可选参数，blocked 和 timeout。
- blocked = True（默认值），timeout 为正
该方法会阻塞 timeout 指定的时间，直到该队列有剩余空间。如果超时，抛出 Queue.Full 异常。
blocked = False
如果 Queue 已满，立刻抛出 Queue.Full 异常
get 方法
get 方法用来从队列中读取并删除一个元素。有两个参数可选，blocked 和 timeout
class popup_ongoing_process(object):
"""
bfit: pointer to bfit object
do_disable: function to run to disable GUI elements on process start
do_enable: function to run to enable GUI elements on process end
kill_status:BooleanVar, if true, the process was terminated
message: string, summary of process which is ongoing
process: multiprocessing.Process
root: TopLevels
target: function handle, run this with no arguments
queue: multiprocessing.Queue: put output in here, gets returned
"""
def __init__(self,
bfit,
target,
message,
queue,
do_disable=None,
do_enable=None):
"""
bfit: pointer to bfit object
do_disable: function to run to disable GUI elements on process start
do_enable: function to run to enable GUI elements on process end
message: string, summary of process which is ongoing
root: TopLevels
target: function handle, run this with no arguments
queue: multiprocessing.Queue: put output in here, gets returned
"""
# variables
self.bfit = bfit
self.message = message
self.logger = bfit.logger
self.target = target
self.do_disable = do_disable
self.do_enable = do_enable
self.queue = queue
self.kill_status = BooleanVar()
self.kill_status.set(False)
# make window
root = Toplevel(bfit.root)
root.lift()
root.resizable(FALSE, FALSE)
# set icon
self.bfit.set_icon(root)
# set label
label = ttk.Label(root, text=message, justify='center', pad=0)
# make progress bar
pbar = ttk.Progressbar(root,
orient=HORIZONTAL,
mode='indeterminate',
length=200,
maximum=20)
pbar.start()
# make button to cancel the fit
cancel = ttk.Button(root, text="Cancel", command=self._kill, pad=0)
# grid
label.grid(column=0, row=0, padx=15, pady=5)
pbar.grid(column=0, row=1, padx=15, pady=5)
cancel.grid(column=0, row=2, padx=15, pady=5)
# set up close window behaviour
root.protocol("WM_DELETE_WINDOW", self._kill)
# update
bfit.root.update_idletasks()
# set window size
width = root.winfo_reqwidth()
height = root.winfo_reqheight()
rt_x = bfit.root.winfo_x()
rt_y = bfit.root.winfo_y()
rt_w = bfit.root.winfo_width()
rt_h = bfit.root.winfo_height()
x = rt_x + rt_w / 2 - (width / 2)
y = rt_y + rt_h / 3 - (width / 2)
root.geometry('{}x{}+{}+{}'.format(width, height, int(x), int(y)))
self.root = root
def _kill(self):
"""Terminate the processs"""
self.process.terminate()
self.kill_status.set(True)
self.logger.info('%s canceled', self.message)
def run(self):
# start fit
self.process = Process(target=self.target)
self.process.start()
# disable GUI
if self.do_disable is not None:
self.do_disable()
# get the output, checking for kill signal
try:
while True:
try:
output = self.queue.get(timeout=0.001)
except Empty:
# iterate the gui
try:
self.root.update()
# applicated destroyed
except TclError:
return
# check if fit cancelled
if self.kill_status.get():
if self.do_enable is not None:
self.do_enable()
return
# got someting in the queue
else:
self.process.join()
if self.do_enable is not None:
self.do_enable()
# fit success
return output
finally:
try:
# kill process, destroy fit window
self.process.terminate()
self.root.destroy()
del self.root
# window already destroyed case (main window closed)
except TclError:
pass
class AreaScreenshoter:
@staticmethod
def run_window(img):
"""
Create fullscreen window and print image given in param
:param img: Image to be printed in window
:return:
"""
# Create window root
root = Tk()
w = Label(root)
root.overrideredirect(True)
root.geometry("{0}x{1}+0+0".format(root.winfo_screenwidth(),
root.winfo_screenheight()))
root.bind('<Escape>', lambda e: root.destroy())
# root.after(1, lambda: root.focus_force())
root.lift()
root.attributes("-topmost", True)
root.focus_force()
photo = ImageTk.PhotoImage(img)
cv = Canvas()
cv.pack(side='top', fill='both', expand='yes')
cv.create_image(1, 1, image=photo, anchor='nw')
w.pack()
root.mainloop()
def start_window_process(self, background_image):
self.window_process = Process(target=self.run_window,
args=(background_image, ))
self.window_process.daemon = True
self.window_process.start()
def close_window(self):
if self.window_process.is_alive():
self.window_process.terminate()
def is_window_opened(self):
return self.window_process.is_alive()
def select_area(self):
"""
Attach mouse listener to select area
:return: Coordinates of selected area or False if selecting were interrupted
"""
def on_mouse_click(x, y, button, pressed):
nonlocal press_coordinates
nonlocal release_coordinates
# If button were pressed
if button == MouseButton.left:
if pressed:
print("Left button pressed")
press_coordinates = {
'detected': True,
'button': button,
'event': 'pressed',
'x': x,
'y': y
}
# If button were released
else:
print("Left button released")
release_coordinates = {
'detected': True,
'button': button,
'event': 'released',
'x': x,
'y': y
}
# Stop listening on button release
return False
# Cancel snapping if right mouse button clicked
elif button == MouseButton.right:
print("Right button pressed - listening canceled")
press_coordinates = {'detected': False}
release_coordinates = {'detected': False}
# Stop listening
return False
# Initialize local coordinates containers
press_coordinates = {'detected': False}
release_coordinates = {'detected': False}
# Get coordinates of selected area
print("Selecting area")
with MouseListener(on_click=on_mouse_click) as listener:
listener.join()
# Assign coordinates to objects variables
self.press_coordinates = press_coordinates
self.release_coordinates = release_coordinates
# Print detected area coordinates
if press_coordinates['detected'] and release_coordinates['detected']:
try:
print("Press: ", press_coordinates['x'],
press_coordinates['y'])
print("Release: ", release_coordinates['x'],
release_coordinates['y'])
except Exception as ex:
print(ex)
def take_screenshot_of_area(self):
# Take screenshot of area
"""
Take screenshot of previously selected area.
Coordinates are specified in object variables
:return: Screenshot of selected area or None if could not take screenshot
"""
selected_area_screenshot = None
if self.press_coordinates['detected'] and self.release_coordinates[
'detected']:
x1, y1 = self.press_coordinates['x'], self.press_coordinates['y']
x2, y2 = self.release_coordinates['x'], self.release_coordinates[
'y']
print("Taking screenshot of: ", x1, y1, x2, y2)
try:
selected_area_screenshot = ImageGrab.grab(bbox=(min(x1, x2),
min(y1, y2),
max(x1, x2),
max(y1, y2)))
except Exception as ex:
print(ex)
else:
print("Cannot take a screenshot without coordinates")
return selected_area_screenshot
def __init__(self):
# Coordinates of mouse press and release. It stands for coordinates of selected rectangle area.
self.press_coordinates = {}
self.release_coordinates = {}
# Initialize variable to store screenshoter window process
self.window_process = Process()
def main():
"""Start the Feedback Controller."""
# Get Options
description = """Feedback Controller"""
usage = "%prog [Options]"
version = """
Copyright (C) 2007-2014 Bastian Venthur <bastian.venthur at tu-berlin de>
Homepage: http://bbci.de/pyff
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
"""
parser = OptionParser(usage=usage,
version=version,
description=description)
parser.add_option(
'-l',
'--loglevel',
type='choice',
choices=['critical', 'error', 'warning', 'info', 'debug', 'notset'],
dest='loglevel',
help=
'Which loglevel to use for everything but the Feedbacks. Valid loglevels are: critical, error, warning, info, debug and notset. [default: warning]',
metavar='LEVEL')
parser.add_option(
'--fb-loglevel',
type='choice',
choices=['critical', 'error', 'warning', 'info', 'debug', 'notset'],
dest='fbloglevel',
help=
'Which loglevel to use for the Feedbacks. Valid loglevels are: critical, error, warning, info, debug and notset. [default: warning]',
metavar='LEVEL')
parser.add_option('--logserver',
dest='logserver',
action='store_true',
default=False,
help='Send log output to logserver.')
parser.add_option(
'-a',
'--additional-feedback-path',
dest='fbpath',
action='append',
help=
"Additional path to search for Feedbacks. Use this option several times if you want to add more than one path.",
metavar="DIR")
parser.add_option(
'--port',
dest='port',
help=
"Set the Parallel port address to use. Windows only. Should be in Hex (eg: 0x378)",
metavar="PORTNUM")
parser.add_option("--nogui",
action="store_true",
default=False,
help="Start without GUI.")
parser.add_option(
"--protocol",
dest='protocol',
type='choice',
help=
"Set the protocol to which Pyff listens to. Options are: json, bcixml and tobixml.",
choices=['bcixml', 'json', 'tobixml'],
default='bcixml')
options, args = parser.parse_args()
# Initialize logging
str2loglevel = {
'critical': logging.CRITICAL,
'error': logging.ERROR,
'warning': logging.WARNING,
'info': logging.INFO,
'debug': logging.DEBUG,
'notset': logging.NOTSET
}
loglevel = str2loglevel.get(options.loglevel, logging.WARNING)
fbloglevel = str2loglevel.get(options.fbloglevel, logging.WARNING)
if options.logserver:
rootLogger = logging.getLogger('')
socketHandler = logging.handlers.SocketHandler(
'localhost', logging.handlers.DEFAULT_TCP_LOGGING_PORT)
rootLogger.addHandler(socketHandler)
formatter = logging.Formatter(
'[%(process)-5d:%(threadName)-10s] %(name)-25s: %(levelname)-8s %(message)s'
)
socketHandler.setFormatter(formatter)
else:
logging.basicConfig(
level=loglevel,
format=
'[%(process)-5d:%(threadName)-10s] %(name)-25s: %(levelname)-8s %(message)s'
)
logging.info('Logger initialized with level %s.' % options.loglevel)
logging.getLogger("FB").setLevel(fbloglevel)
# get the rest
fbpath = options.fbpath
guiproc = None
if not options.nogui:
guiproc = Process(target=GUI.main, args=(options.protocol, ))
guiproc.start()
port = None
if options.port != None:
port = int(options.port, 16)
try:
fc = FeedbackController(fbpath, port, options.protocol)
except:
logging.exception(
"Could not start Feedback Controller, is another instance already running?"
)
return
try:
fc.start()
except (KeyboardInterrupt, SystemExit):
logging.debug("Caught keyboard interrupt or system exit; quitting")
except:
logging.exception("Caught an exception, quitting FeedbackController.")
finally:
print
print "Stopping FeedbackController...",
fc.stop()
print "Done."
if guiproc:
print "Stopping GUI...",
guiproc.terminate()
print "Done."
print "Stopping logging facilities...",
logging.shutdown()
print "Done."
from multiprocessing import Process, Queue
import os, time, random
def write(q):
print('Process to write %s' % os.getpid())
for i in range(10):
print('Put %d in queue' % i)
q.put(i)
time.sleep(random.random())
time.sleep(3)
def read(q):
while True:
value = q.get(True)
print('Process to read %s, Get %d from queue' % (os.getpid(), value))
if __name__ == '__main__':
q = Queue()
pw = Process(target=write, args=(q, ))
pr1 = Process(target=read, args=(q, ))
pr2 = Process(target=read, args=(q, ))
pw.start()
pr1.start()
pr2.start()
pw.join()
pr1.terminate()
pr2.terminate()
print("父进程：{0}".format(os.getpid()))
p = Process(target=startGame, args=(waitTime, ))
p.start()
print("子进程：启动成功")
event_handler = MyHandler()
observer = Observer()
observer.schedule(event_handler, path='.', recursive=False)
observer.start()
restartCount = 0
try:
while True:
if waitTime.value < 0:
p.terminate()
observer.stop()
break
elif waitTime.value == 0 and (datetime.now() -
lastChangeTime).seconds > MAX_TIME:
p.terminate()
p = Process(target=startGame, args=(waitTime, ))
logging.warning('进程重启.')
restartCount += 1
if restartCount == 5:
p.terminate()
observer.stop()
break
p.start()
lastChangeTime = datetime.now()
time.sleep(1)
"move_position.sh"))
sleep(2)
enterserach()
q = Queue()
process_one = Process(target=sensewalls, args=(q, ))
process_two = Process(target=move, args=(q, ))
process_one.start()
process_two.start()
q.close()
q.join_thread()
process_one.join()
process_two.join()
except KeyboardInterrupt:
print "exiting......."
speed(0)
turn(0)
process_one.terminate()
process_two.terminate()
while process_one.is_alive() and process_two.is_alive():
speed(0)
turn(0)
print process_one.is_alive()
print process_two.is_alive()
robot.close()
except Exception, e:
print e
def runExperiment():
global expStart
pygame.init()
#Global Variables
GPIO.setmode(GPIO.BOARD) #Sets GPIO pin numbering convention
GPIOrec, GPIOsend = Pipe() #Setup a duplex line of communication for processes to send data to log process
imageRec, imageSend = Pipe()
wheelRec, wheelSend = Pipe()
stopQueue = Value('i', 0) #Setup a shared variable to allow a keypress to flag all processes to stop
rewardActive = Value('i', 0) #Setup a shared variable to flag that a reward event is active
wheelActive = Value('i', 0) #Setup a shared variable to flag that a wheel event is active
#NOTE: A pipe can only connect two processes while a queue can connect multiple processes
#Also, a pipe is much faster than a queue. A SimpleQueue has a simplified instruction set
#Initialize Image, GPIO and logging sub processes
pLog = Process(target = logProcess, args=(GPIOrec, wheelRec, imageRec, stopQueue))
pGPIO = Process(target = GPIOprocess, args=(pinDoor, GPIOsend, stopQueue, rewardActive))
pWheel = Process(target = GPIOprocess, args=(pinWheel, wheelSend, stopQueue, wheelActive))
try:
lxprint("Starting")
expStart = time.time() #Record the start time of the experiemnt
pLog.start() #Start subprocesses before continuing with main thread, otherwise main thread will be too busy to start subprocesses
pGPIO.start()
pWheel.start()
imageProcess(imageSend, stopQueue, rewardActive, wheelActive) #PyGame does not support multi-processing, so it must stay in the main thread
pLog.terminate()
pGPIO.terminate()
pWheel.terminate()
pLog.join() #Verify that all subprocesses are successfully terminated
pGPIO.join()
pWheel.join()
except KeyboardInterrupt:
pLog.terminate()
pGPIO.terminate()
pWheel.terminate()
pLog.join() #Verify that all subprocesses are successfully terminated
pGPIO.join()
pWheel.join()
pygame.quit()
# GPIO.cleanup() # clean up GPIO on CTRL+C exit
#GPIO.cleanup() # clean up GPIO on normal exit
lxprint("End.")
class StreamVideo:
"""
"""
def __init__(self, *args, **kwargs):
self.video_src: str = vs.opt_kwargs(kwargs, 'video_src', '')
self.video_index: int = vs.opt_kwargs(kwargs, 'video_index', 0)
self.frame_format: str = vs.opt_kwargs(kwargs, 'frame_format',
vs.DEFAULT_FRAME_FORMAT)
self.frame_width: int = vs.opt_kwargs(kwargs, 'frame_width', 0)
self.frame_height: int = vs.opt_kwargs(kwargs, 'frame_height', 0)
self.frame_interpolation: str = vs.opt_kwargs(kwargs,
'frame_interpolation',
vs.INTERPOLATION_LIST[1])
self.options: dict = vs.opt_kwargs(kwargs, 'options', {})
self.container_options: dict = vs.opt_kwargs(kwargs,
'container_options', {})
self.stream_options: list = vs.opt_kwargs(kwargs, 'stream_options', [])
self.reconnect_sleep: float = vs.opt_kwargs(kwargs, 'reconnect_sleep',
vs.RECONNECT_SLEEP)
self.iteration_sleep: float = vs.opt_kwargs(kwargs, 'iteration_sleep',
vs.ITERATION_SLEEP)
self.verbose: bool = vs.opt_kwargs(kwargs, 'verbose', False)
self.low_delay: bool = vs.opt_kwargs(kwargs, 'low_delay', False)
self.refresh_error_threshold: int = vs.opt_kwargs(
kwargs, 'refresh_error_threshold', vs.REFRESH_ERROR_THRESHOLD)
self.max_queue_size: int = vs.opt_kwargs(kwargs, 'max_queue_size',
DEFAULT_MAX_QUEUE_SIZE)
self.exit_timeout_seconds: float = vs.opt_kwargs(
kwargs, 'exit_timeout_seconds', vs.DEFAULT_EXIT_TIMEOUT_SECONDS)
self.refresh_error_count = 0
self.refresh_flag: Synchronized = Value(c_bool, False)
self.process: Process = None # noqa
self.pid = UNKNOWN_PID
self.server_state: Synchronized = Value(c_int, vs.SERVER_STATE_DONE)
self.exit_flag: Synchronized = Value(c_bool, False)
self.queue: Queue = None # noqa
self.last_image = None
def on_set(self, key, val):
if key == 'video_src':
self.video_src = val
elif key == 'video_index':
self.video_index = int(val)
elif key == 'frame_format':
self.frame_format = val
elif key == 'frame_width':
self.frame_width = int(val)
elif key == 'frame_height':
self.frame_height = int(val)
elif key == 'frame_interpolation':
self.frame_interpolation = val
elif key == 'options':
self.options = str_to_dict(str(val))
elif key == 'container_options':
self.container_options = str_to_dict(str(val))
elif key == 'stream_options':
self.stream_options = list(
filter(lambda x: x.strip(),
str(val).split(',')))
elif key == 'reconnect_sleep':
self.reconnect_sleep = float(val)
elif key == 'iteration_sleep':
self.iteration_sleep = float(val)
elif key == 'verbose':
self.verbose = val.lower() in ['y', 'yes', 'true']
elif key == 'low_delay':
self.low_delay = val.lower() in ['y', 'yes', 'true']
elif key == 'max_queue_size':
self.max_queue_size = int(val)
elif key == 'exit_timeout_seconds':
self.exit_timeout_seconds = float(val)
elif key == 'refresh_error_threshold':
self.refresh_error_threshold = int(val)
def on_get(self, key):
if key == 'video_src':
return self.video_src
elif key == 'video_index':
return str(self.video_index)
elif key == 'frame_format':
return self.frame_format
elif key == 'frame_width':
return self.frame_width
elif key == 'frame_height':
return self.frame_height
elif key == 'frame_interpolation':
return self.frame_interpolation
elif key == 'options':
return dict_to_str(self.options)
elif key == 'container_options':
return dict_to_str(self.container_options)
elif key == 'stream_options':
return ','.join(list(map(lambda x: str(x), self.stream_options)))
elif key == 'reconnect_sleep':
return self.reconnect_sleep
elif key == 'iteration_sleep':
return self.iteration_sleep
elif key == 'verbose':
return str(self.verbose)
elif key == 'low_delay':
return str(self.low_delay)
elif key == 'max_queue_size':
return str(self.max_queue_size)
elif key == 'exit_timeout_seconds':
return str(self.exit_timeout_seconds)
elif key == 'refresh_error_threshold':
return str(self.refresh_error_threshold)
def _get_server_state(self):
with self.server_state.get_lock():
return self.server_state.value
def _set_server_state(self, value: int):
with self.server_state.get_lock():
self.server_state.value = value
def _set_exit_flag(self, value: bool):
with self.exit_flag.get_lock():
self.exit_flag.value = value
def _get_exit_flag(self):
with self.exit_flag.get_lock():
return self.exit_flag.value
def _set_refresh_flag(self, value: bool):
with self.refresh_flag.get_lock():
self.refresh_flag.value = value
def do_refresh_ok(self):
self.refresh_error_count = 0
self._set_refresh_flag(False)
def do_refresh_error(self):
if self.refresh_error_count < self.refresh_error_threshold:
self.refresh_error_count += 1
if self.verbose:
print_out(
f'StreamVideo.do_refresh_error({self.refresh_error_count}/{self.refresh_error_threshold})'
)
else:
self.refresh_error_count = 0
self._set_refresh_flag(True)
print_error(f'StreamVideo.do_refresh_error() Enable refresh_flag')
return self.last_image
def do_refresh_error_v2(self):
"""
Fix: camera power off -> and power on case ...
"""
if self.refresh_error_count < self.refresh_error_threshold:
self.refresh_error_count += 1
if self.verbose:
print_out(
f'StreamVideo.do_refresh_error_v2({self.refresh_error_count}/{self.refresh_error_threshold})'
)
else:
self.refresh_error_count = 0
self.last_image = self.get_empty_image(self.last_image)
self._set_refresh_flag(True)
if not self._get_exit_flag():
# assert self.process is not None
self.process.kill()
print_error(
f'StreamVideo.do_refresh_error_v2() Enable refresh_flag')
return self.last_image
def get_last_image(self):
try:
self.last_image = self.queue.get_nowait()
self.do_refresh_ok()
except Empty:
# self.do_refresh_error()
self.do_refresh_error_v2()
return self.last_image
def get_empty_image(self, image):
return np.zeros((image.shape[0], image.shape[1], image.shape[2]),
np.uint8)
def _create_process_impl(self):
assert self.queue is None
assert self.process is None
kwargs = {
'exit_flag': self.exit_flag,
'server_state': self.server_state,
'refresh_flag': self.refresh_flag,
'video_src': self.video_src,
'video_index': self.video_index,
'frame_format': self.frame_format,
'frame_width': self.frame_width,
'frame_height': self.frame_height,
'frame_interpolation': self.frame_interpolation,
'options': self.options,
'container_options': self.container_options,
'stream_options': self.stream_options,
'reconnect_sleep': self.reconnect_sleep,
'iteration_sleep': self.iteration_sleep,
'verbose': self.verbose,
'low_delay': self.low_delay,
}
self.queue = Queue(self.max_queue_size)
self.process = Process(target=vs.start_app,
args=(self.queue, ),
kwargs=kwargs)
self._set_server_state(vs.SERVER_STATE_OPENING)
self.process.start()
if self.process.is_alive():
self.pid = self.process.pid
print_out(
f'StreamVideo._create_process_impl() Server process PID: {self.pid}'
)
return True
else:
self._set_server_state(vs.SERVER_STATE_DONE)
print_error(
f'StreamVideo._create_process_impl() Server process is not alive.'
)
return False
def _create_process(self):
try:
return self._create_process_impl()
except Exception as e:
print_error(f'StreamVideo._create_process() Exception: {e}')
return False
def _close_process_impl(self):
self._set_exit_flag(True)
if self.process is not None:
timeout = self.exit_timeout_seconds
if self.process.is_alive():
request_begin = time.time()
print_out(
f'StreamVideo._close_process_impl() RequestExit(timeout={timeout}s)'
)
timeout = timeout - (time.time() - request_begin)
timeout = timeout if timeout >= 0.0 else 0.0
print_out(
f'StreamVideo._close_process_impl() Join(timeout={timeout}s) the sub-process.'
)
self.process.join(timeout=timeout)
if self.process.is_alive():
print_error(
f'StreamVideo._close_process_impl() Send a KILL signal to the server process.'
)
self.process.kill()
# A negative value -N indicates that the child was terminated by signal N.
print_out(
f'StreamVideo._close_process_impl() The exit code of sub-process is {self.process.exitcode}.'
)
if self.queue is not None:
self.queue.close()
self.queue.cancel_join_thread()
self.queue = None
if self.process is not None:
if self.process.is_alive():
self.process.terminate()
self.process.close()
self.process = None
if self.pid >= 1:
kill_process(self.pid)
self.pid = UNKNOWN_PID
self._set_server_state(vs.SERVER_STATE_DONE)
self._set_exit_flag(False)
self._set_refresh_flag(False)
assert self.queue is None
assert self.process is None
assert self.pid is UNKNOWN_PID
print_out(f'StreamVideo._close_process_impl() Done.')
def _close_process(self):
try:
self._close_process_impl()
except Exception as e:
print_error(f'StreamVideo._close_process() Exception: {e}')
finally:
self.queue = None
self.process = None
self.pid = UNKNOWN_PID
def create_process(self):
if self._create_process():
return True
self._close_process()
return False
def is_reopen(self):
if self.process is None:
return True
if not self.process.is_alive():
return True
return False
def reopen(self):
self._close_process()
if self._create_process():
print_out(f'Recreated Server process PID: {self.pid}')
else:
raise CreateProcessError
def on_init(self):
return self.create_process()
def on_valid(self):
return self.pid != UNKNOWN_PID
def on_run(self):
if self.is_reopen():
self.reopen()
state = self._get_server_state()
if state == vs.SERVER_STATE_DONE:
raise IllegalStateException
elif state == vs.SERVER_STATE_OPENING:
raise NotReadyException
elif state == vs.SERVER_STATE_RUNNING:
pass # OK!!
else:
raise InaccessibleException
frame = self.get_last_image()
if frame is None:
raise NullDataException
return {'frame': frame}
def on_destroy(self):
self._close_process()
def test_add_many_votes(self):
##
## run service
##
checker_service_process = Process(target=vote_contract.run_checker_service)
checker_service_process.start()
time.sleep(0.1)
##
## create transaction
##
# number of voters and values
options = ['alice', 'bob']
num_voters = 3
values = [[1, 0] for _ in range(0, num_voters)]
# create keys and particpants
params = setup()
(tally_priv, tally_pub) = key_gen(params)
keys = [key_gen(params) for _ in range(0, num_voters)]
participants = [pack(pub) for (_, pub) in keys]
# get init token
init_transaction = vote.init()
# get initial scores
create_vote_transaction = vote.create_vote(
(init_transaction['transaction']['outputs'][0],),
None,
None,
dumps(options),
dumps(participants),
pack(tally_priv),
pack(tally_pub)
)
vote_0 = create_vote_transaction['transaction']['outputs'][1]
# add votes
transaction = {}
input_obj = vote_0
for i in range(0, num_voters):
transaction = vote.add_vote(
(input_obj,),
None,
None,
dumps(values[i]), # votes' valu (0 or 1)
pack(keys[i][0]), # voter's priv key
pack(keys[i][1]) # voter's pub key
)
input_obj = transaction['transaction']['outputs'][0]
##
## submit transaction
##
response = requests.post(
'http://127.0.0.1:5000/' + vote_contract.contract_name + '/add_vote', json=transaction_to_solution(transaction)
)
self.assertTrue(response.json()['success'])
##
## stop service
##
checker_service_process.terminate()
checker_service_process.join()
def run(self):
p_fast_send = Process(target=self.fast_send)
p_HELLO = threading.Thread(target=self.broadcast_HELLO)
while True:
# 如果刚开机
if self.if_start_up:
self.if_start_up = False
print("%d开机，开始持续5s的监听" % self.my_port)
# 扫频慢收。5秒没有响应，则开始快发
self.recv_socket.settimeout(5)
try:
while True:
data, address = self.recv_socket.recvfrom(BUFSIZ)
des_address, src_address, content, master_ID, F, F_list, flag = parse_json(
data.decode('utf-8'))
# 如果收到了”快发“或者心跳信息，向其所在的中心节点申请入网
# TODO 如果几乎同时收到了两个中心节点的消息，待处理
if flag == "FAST_SEND" or flag == "HELLO":
print("%dReceive %s from address %d" %
(self.my_port, flag, src_address))
# 向master申请入网
msg = write_json(des_address=master_ID,
src_address=self.my_port,
content=[],
master_ID="",
F=F,
F_list=self.F_list,
flag="APPLY")
print("APPLY to master:", master_ID)
self.broadcast(msg)
# 等待中心节点的确认消息，如果收到ACK，入网成功。如果超时5s未收到ACK，独立建网
t1 = time.time()
while True:
# 收到的msg一直都不是ACK，3秒还没有收到ACK消息，break
if time.time() - t1 > 3:
recv_response = False
break
data, address = self.recv_socket.recvfrom(
BUFSIZ)
des_address, src_address, content, master_ID, F, F_list, flag = parse_json(
data.decode('utf-8'))
if flag == "ACK" and des_address == self.my_port:
print(
"Receive ACK from %d, connected to this master"
% src_address)
self.master = False
self.timeslot = content
self.timeslot_occupied.append(
self.timeslot)
self.timeslot_available = np.setdiff1d(
self.all_available,
self.timeslot_occupied)
self.master_ID = master_ID
self.master_ID_list.append(self.master_ID)
self.F = F
# F_list的长度如果达到3，不再加入新的子网
self.F_list.append(self.F)
print("%d连接至子网，时隙：%s，中心节点：%s，频率：%s" %
(self.my_port,
str(self.timeslot_occupied),
str(self.master_ID_list),
str(self.F_list)))
recv_response = True
self.recv_socket.settimeout(None)
break
break
except timeout:
print("没有监听到周围节点")
recv_response = False
# 没有发现别的节点，成为中心节点，进行扫频快发
if not recv_response:
print("开始持续20s的快发")
self.recv_socket.settimeout(40)
# 开启新进程，开始快发
p_fast_send.start()
# 20s没有人响应，则关机
try:
while True:
data, address = self.recv_socket.recvfrom(BUFSIZ)
des_address, src_address, content, master_ID, F, F_list, flag = parse_json(
data.decode('utf-8'))
# 收到申请
if flag == "APPLY" and des_address == self.my_port:
print("%d Receive APPLY from %d" %
(self.my_port, src_address))
# 结束快发
p_fast_send.terminate()
# 发送ACK，分配时隙，自己作为master
self.master = True
self.master_ID = self.my_port
self.master_ID_list.append(self.master_ID)
# 分配自己的时隙,更新可用时隙
self.timeslot = self.timeslot_available[0]
self.timeslot_occupied.append(self.timeslot)
self.timeslot_available = np.delete(
self.timeslot_available,
np.where(self.timeslot_available ==
self.timeslot))
# Change Here. 不考虑申请节点的时隙情况。总是以新的发送端发送入网申请。
# 分配申请入网节点的时隙,更新可用时隙
allocated_timeslot = self.timeslot_available[0]
self.timeslot_available = np.delete(
self.timeslot_available,
np.where(self.timeslot_available ==
allocated_timeslot))
# 发送ACK
to_port = src_address
self.children.append(to_port)
msg = write_json(des_address=to_port,
src_address=self.my_port,
F_list=self.F_list,
content=allocated_timeslot,
master_ID=self.my_port,
F=F,
flag="ACK")
self.F = F
self.F_list.append(self.F)
print("send back ACK， %d入网成功 :" % to_port)
print(
"子网建成，本端为中心节点%d，时隙：%s，频率：%s，网内节点：%s" %
(self.my_port, str(self.timeslot_occupied),
str(self.F_list), str(self.children)))
self.broadcast(msg)
self.recv_socket.settimeout(None)
break
# TODO 两个节点几乎同时开机，没有监听到后，都在快发的特殊情况，需要考虑。
# if flag == "APPLY" p_fast_send.terminate()
except timeout:
print("20秒内没有收到申请入网消息，关机")
p_fast_send.terminate()
self.recv_socket.close()
return 0
# 开始广播HELLO心跳信息
if not p_HELLO.is_alive():
print("开始定时广播")
p_HELLO.start()
# 正常运行模式，持续扫频慢收监听
data, address = self.recv_socket.recvfrom(BUFSIZ)
des_address, src_address, content, master_ID, F, F_list, flag = parse_json(
data.decode('utf-8'))
# 中心节点在运行过程中收到刚开机节点的入网申请
if self.master:
# 收到申请入网消息
if flag == "APPLY" and des_address == self.my_port:
# Change Here. 不考虑申请节点的时隙情况。
if len(self.timeslot_available) > 0:
print("%d Receive APPLY from %d" %
(self.my_port, src_address))
to_port = src_address
# 分配时隙
# content不为空，即为申请节点的占有时隙，可用时隙中去掉这些时隙
allocated_timeslot = self.timeslot_available[0]
self.timeslot_available = np.delete(
self.timeslot_available,
np.where(
self.timeslot_available == allocated_timeslot))
self.children.append(to_port)
msg = write_json(des_address=to_port,
src_address=self.my_port,
F_list=self.F_list,
content=allocated_timeslot,
master_ID=self.my_port,
F=self.F,
flag="ACK")
self.broadcast(msg)
print("send back ACK， %d入网成功 :" % to_port)
print("本端%d为中心节点%d，目前子网内有%d个子节点:%s" %
(self.my_port, self.F,
7 - self.timeslot_available.size - 1,
str(self.children)))
# 如果某新开机节点a无法加入本子网，a进行快发，该节点在新的频率上收到了a的快发消息。则该节点向a申请入网
if flag == "FAST_SEND" or flag == "HELLO":
# F_list的长度如果达到3，不再加入新的子网
# 1)首先判断是否是子网内其他节点的消息。
# 2)其次判断当前发送端是否占满了
# 3)多次收到其他子网的消息，不再重复发APPLY入网申请
# 4) DELETED，不再判断是否有可用时隙
if np.intersect1d(F_list, self.F_list).size == 0 \
and len(self.F_list) < 3 \
and master_ID not in self.applying:
print("%d Receive %s from address %d" %
(self.my_port, flag, src_address))
self.applying.append(master_ID)
# 阻塞，防止一次性申请进入多个子网，导致超过发送端口个数
self.F_list.append(F)
# 向master申请入网,content是自己已占有的时隙， master分配的时候排除这些时隙
msg = write_json(des_address=master_ID,
src_address=self.my_port,
F_list=self.F_list,
content=self.timeslot_occupied,
master_ID="",
F=F,
flag="APPLY")
print("APPLY to master:", master_ID)
self.broadcast(msg)
t_recv = time.time()
self.applying_pending[master_ID] = [t_recv, F]
if len(self.applying_pending) > 0:
pending_appling = self.applying_pending.copy()
tmp = pending_appling.popitem()
pending_ID, pending_time_F = tmp[0], tmp[1]
pending_time, pending_F = pending_time_F[0], pending_time_F[1]
if time.time() - pending_time > 4:
del self.applying_pending[pending_ID]
# 如果没有成功申请进入，释放该发送端口
self.F_list.remove(pending_F)
p_fast_send_running = Process(target=self.fast_send)
print("申请无响应，开始持续4s的快发")
# 开启新进程，开始快发
p_fast_send_running.start()
t_wait = time.time()
while True:
# 4s没有收到申请消息，回归原始状态
if time.time() - t_wait > 4:
print("4秒内没有收到邻近节点的申请入网消息，回归到原子网中，继续保持运行状态")
p_fast_send_running.terminate()
break
data, address = self.recv_socket.recvfrom(BUFSIZ)
des_address, src_address, content, master_ID, F, F_list, flag = parse_json(
data.decode('utf-8'))
# 收到申请
if flag == "APPLY" and des_address == self.my_port:
print("%d Receive APPLY from %d" %
(self.my_port, src_address))
# 结束快发
p_fast_send_running.terminate()
# 发送ACK，分配时隙，自己作为master
self.master = True
self.master_ID = self.my_port
self.master_ID_list.append(self.master_ID)
# 如果申请入网的节点已经在另外一个网内，排除掉其已占有的时隙
# content 是申请节点的所有占有时隙
applying_timeslot_occupied = content
applying_timeslot_available = np.setdiff1d(
self.all_available,
applying_timeslot_occupied).tolist()
timeslot_available = np.intersect1d(
applying_timeslot_available,
self.timeslot_available).tolist()
# 分配申请入网节点的时隙,更新可用时隙
allocated_timeslot = timeslot_available[0]
self.timeslot_available = np.delete(
self.timeslot_available,
np.where(self.timeslot_available ==
allocated_timeslot))
# 分配自己的时隙,更新可用时隙
self.timeslot = self.timeslot_available[0]
self.timeslot_occupied.append(self.timeslot)
self.timeslot_available = np.delete(
self.timeslot_available,
np.where(
self.timeslot_available == self.timeslot))
# 发送ACK
to_port = src_address
self.children.append(to_port)
msg = write_json(des_address=to_port,
src_address=self.my_port,
F_list=self.F_list,
content=allocated_timeslot,
master_ID=self.my_port,
F=F,
flag="ACK")
self.F = F
self.F_list.append(self.F)
print("send back ACK， %d入网成功 :" % to_port)
print("子网建成，本端为中心节点%d，时隙：%s，频率：%s，网内节点：%s" %
(self.my_port, str(self.timeslot_occupied),
str(self.F_list), str(self.children)))
self.broadcast(msg)
self.recv_socket.settimeout(None)
break
# 收到了该节点在正常运行过程中申请入网之后的确认消息
if flag == "ACK" and des_address == self.my_port:
print("Receive ACK from %d, connected to this master" %
src_address)
if master_ID in self.applying:
self.applying.remove(master_ID)
del self.applying_pending[master_ID]
self.timeslot = content
self.timeslot_occupied.append(self.timeslot)
self.timeslot_available = np.setdiff1d(self.all_available,
self.timeslot_occupied)
self.master_ID = master_ID
self.master_ID_list.append(self.master_ID)
self.F = F
print("%d连接至子网%d，时隙%s：，中心节点%s：，频率%s：" %
(self.my_port, F, str(self.timeslot_occupied),
str(self.master_ID_list), str(self.F_list)))
self.recv_socket.close()
class FitDataThread(QtCore.QThread):
"""
Fit data thread. This thread is used to fit data in the background, so that the UI isn't blocked. Each instance of
the thread fits two spots in a single image and terminates afterwards. The image data has to be supplied upon
construction of the thread object. The results are communicated through the 'fitted' signal.
If the thread is started with the 'killable' argument set to True, the thread may be killed by setting the 'kill'
attribute to True. Otherwise the thread will just continue execution until it is finished.
"""
fitted = QtCore.pyqtSignal(tuple, SpotFitting.FittingMethods, name="fitted")
"""
Signal emitted when the fitting process has finished
Signal args:
fitparams: tuple(fitparamsA, fitparamsB) of fitted parameters for both spots. See the data fitting module
for details on the fit parameters
fitting_method: fitting method that was applied
"""
def __init__(self, imgarray, spotA_rect, spotB_rect, cpopt, fitting_method, killable = True):
super().__init__()
# copy the image array so this thread has it's own copy and doesn't get upset if the image is changed
self.imgarray = np.copy(imgarray)
self.spotA_rect = spotA_rect
self.spotB_rect = spotB_rect
self.cpopt = cpopt
self.fitting_method = fitting_method
self.killable = killable
self.kill = False
self.queue = Queue()
self.subprocess = Process(target=self.subprocess_run, args=(self.imgarray, self.spotA_rect, self.spotB_rect,
self.cpopt, self.fitting_method, self.queue))
def subprocess_run(self, imgarray, spotA_rect, spotB_rect, cpopt, fitting_method, q=None):
converted_imgdata = ImageConversion.apply_color_processing(imgarray, cpopt)
spotA_imgarray, iy_A, ix_A, _, _ = ImageConversion.crop_image(converted_imgdata, spotA_rect)
spotB_imgarray, iy_B, ix_B, _, _ = ImageConversion.crop_image(converted_imgdata, spotB_rect)
fitparams = SpotFitting.fit_spots_from_array(spot_imgarrays=(spotA_imgarray, spotB_imgarray),
method=self.fitting_method,
crop_offsets=(np.array([ix_A, iy_A]), np.array([ix_B, iy_B])))
if q:
q.put(fitparams)
else:
return fitparams
def run(self):
if self.killable:
# if the fitting process should be killable, we launch it as a subprocess (which may be forcibly killed)
# communication with the subprocess is done using a Queue
self.subprocess.start()
while not self.kill:
time.sleep(0.01)
try:
fitparams = self.queue.get(False)
self.fitted.emit(fitparams, self.fitting_method)
return
except queue.Empty:
pass
# we have been killed!
self.subprocess.terminate()
self.queue = None
else:
# if the fitting process doesn't have to be killable, just launch it directly inside this thread
fitparams = self.subprocess_run(self.imgarray, self.spotA_rect, self.spotB_rect, self.cpopt, self.fitting_method)
self.fitted.emit(fitparams, self.fitting_method)
return
def run(self):
"""
Called when the process intializes.
"""
# Log management to prevent overwriting
# Allow the bin/<skyline_app>.d to manage the log
if os.path.isfile(skyline_app_logwait):
try:
os.remove(skyline_app_logwait)
except OSError:
logger.error('error - failed to remove %s, continuing' %
skyline_app_logwait)
pass
now = time()
log_wait_for = now + 5
while now < log_wait_for:
if os.path.isfile(skyline_app_loglock):
sleep(.1)
now = time()
else:
now = log_wait_for + 1
logger.info('starting %s run' % skyline_app)
if os.path.isfile(skyline_app_loglock):
logger.error(
'error - bin/%s.d log management seems to have failed, continuing'
% skyline_app)
try:
os.remove(skyline_app_loglock)
logger.info('log lock file removed')
except OSError:
logger.error('error - failed to remove %s, continuing' %
skyline_app_loglock)
pass
else:
logger.info('bin/%s.d log management done' % skyline_app)
logger.info('process intialized')
while 1:
now = time()
if settings.ENABLE_CRUCIBLE_DEBUG:
logger.info('process started - %s' % int(now))
# Make sure check_dir exists and has not been removed
try:
if settings.ENABLE_CRUCIBLE_DEBUG:
logger.info('checking check dir exists - %s' %
settings.CRUCIBLE_CHECK_PATH)
os.path.exists(settings.CRUCIBLE_CHECK_PATH)
except:
logger.error('error :: check dir did not exist - %s' %
settings.CRUCIBLE_CHECK_PATH)
mkdir_p(settings.CRUCIBLE_CHECK_PATH)
logger.info('check dir created - %s' %
settings.CRUCIBLE_CHECK_PATH)
os.path.exists(settings.CRUCIBLE_CHECK_PATH)
# continue
# Make sure Redis is up
try:
self.redis_conn.ping()
logger.info('connected to redis at socket path %s' %
settings.REDIS_SOCKET_PATH)
except:
logger.info(
'skyline can not connect to redis at socket path %s' %
settings.REDIS_SOCKET_PATH)
sleep(10)
logger.info('connecting to redis at socket path %s' %
settings.REDIS_SOCKET_PATH)
# @modified 20180519 - Feature #2378: Add redis auth to Skyline and rebrow
if settings.REDIS_PASSWORD:
self.redis_conn = StrictRedis(
password=settings.REDIS_PASSWORD,
unix_socket_path=settings.REDIS_SOCKET_PATH)
else:
self.redis_conn = StrictRedis(
unix_socket_path=settings.REDIS_SOCKET_PATH)
continue
"""
Determine if any metric has been added to test
"""
while True:
# Report app up
self.redis_conn.setex(skyline_app, 120, now)
metric_var_files = [
f for f in listdir(settings.CRUCIBLE_CHECK_PATH)
if isfile(join(settings.CRUCIBLE_CHECK_PATH, f))
]
# if len(metric_var_files) == 0:
if not metric_var_files:
logger.info('sleeping 10 no metric check files')
sleep(10)
# Discover metric to analyze
metric_var_files = ''
metric_var_files = [
f for f in listdir(settings.CRUCIBLE_CHECK_PATH)
if isfile(join(settings.CRUCIBLE_CHECK_PATH, f))
]
# if len(metric_var_files) > 0:
if metric_var_files:
break
metric_var_files_sorted = sorted(metric_var_files)
metric_check_file = settings.CRUCIBLE_CHECK_PATH + "/" + str(
metric_var_files_sorted[0])
logger.info('assigning check for processing - %s' %
str(metric_var_files_sorted[0]))
# Reset process_list
try:
self.process_list[:] = []
except:
logger.error('error :: failed to reset self.process_list')
# Spawn processes
pids = []
spawned_pids = []
pid_count = 0
run_timestamp = int(now)
for i in range(1, settings.CRUCIBLE_PROCESSES + 1):
p = Process(target=self.spin_process,
args=(i, run_timestamp, str(metric_check_file)))
pids.append(p)
pid_count += 1
logger.info('starting %s of %s spin_process/es' %
(str(pid_count), str(settings.CRUCIBLE_PROCESSES)))
p.start()
spawned_pids.append(p.pid)
# Send wait signal to zombie processes
# for p in pids:
# p.join()
# Self monitor processes and terminate if any spin_process has run
# for longer than CRUCIBLE_TESTS_TIMEOUT
p_starts = time()
while time() - p_starts <= settings.CRUCIBLE_TESTS_TIMEOUT:
if any(p.is_alive() for p in pids):
# Just to avoid hogging the CPU
sleep(.1)
else:
# All the processes are done, break now.
time_to_run = time() - p_starts
logger.info(
'%s :: %s spin_process/es completed in %.2f seconds' %
(skyline_app, str(
settings.CRUCIBLE_PROCESSES), time_to_run))
break
else:
# We only enter this if we didn't 'break' above.
logger.info(
'%s :: timed out, killing all spin_process processes' %
(skyline_app))
for p in pids:
p.terminate()
# p.join()
for p in pids:
if p.is_alive():
logger.info('%s :: stopping spin_process - %s' %
(skyline_app, str(p.is_alive())))
p.join()
while os.path.isfile(metric_check_file):
sleep(1)
class TestAgent(object):
"""Agent for testing purpose"""
def __init__(self):
q = Queue()
self._process = Process(target=self._setup, args=(q, ))
self._process.start()
self.port = q.get()
def terminate(self):
self._process.terminate()
def _setup(self, q):
"""Setup a new agent in a separate process.
The port the agent is listening too will be returned using the
provided queue.
"""
port = random.randrange(22000, 22989)
snmpEngine = engine.SnmpEngine()
config.addSocketTransport(
snmpEngine, udp.domainName,
udp.UdpTransport().openServerMode(('127.0.0.1', port)))
# Community is public and MIB is writable
config.addV1System(snmpEngine, 'read-write', 'public')
config.addVacmUser(snmpEngine, 1, 'read-write', 'noAuthNoPriv',
(1, 3, 6), (1, 3, 6))
config.addVacmUser(snmpEngine, 2, 'read-write', 'noAuthNoPriv',
(1, 3, 6), (1, 3, 6))
config.addV3User(snmpEngine, 'read-write',
config.usmHMACMD5AuthProtocol, 'authpass',
config.usmAesCfb128Protocol, 'privpass')
config.addVacmUser(snmpEngine, 3, 'read-write', 'authPriv', (1, 3, 6),
(1, 3, 6))
# Build MIB
def stringToOid(string):
return [ord(x) for x in string]
def flatten(*args):
result = []
for el in args:
if isinstance(el, (list, tuple)):
for sub in el:
result.append(sub)
else:
result.append(el)
return tuple(result)
snmpContext = context.SnmpContext(snmpEngine)
mibBuilder = snmpContext.getMibInstrum().getMibBuilder()
(MibTable, MibTableRow, MibTableColumn,
MibScalar, MibScalarInstance) = mibBuilder.importSymbols(
'SNMPv2-SMI', 'MibTable', 'MibTableRow', 'MibTableColumn',
'MibScalar', 'MibScalarInstance')
mibBuilder.exportSymbols(
'__MY_SNMPv2_MIB',
# SNMPv2-MIB::sysDescr
MibScalar((1, 3, 6, 1, 2, 1, 1, 1), v2c.OctetString()),
MibScalarInstance((1, 3, 6, 1, 2, 1, 1, 1), (0, ),
v2c.OctetString("Snimpy Test Agent")))
mibBuilder.exportSymbols(
'__MY_IF_MIB',
# IF-MIB::ifNumber
MibScalar((1, 3, 6, 1, 2, 1, 2, 1), v2c.Integer()),
MibScalarInstance((1, 3, 6, 1, 2, 1, 2, 1), (0, ), v2c.Integer(3)),
# IF-MIB::ifTable
MibTable((1, 3, 6, 1, 2, 1, 2, 2)),
MibTableRow((1, 3, 6, 1, 2, 1, 2, 2, 1)).setIndexNames(
(0, '__MY_IF_MIB', 'ifIndex')),
# IF-MIB::ifIndex
MibScalarInstance((1, 3, 6, 1, 2, 1, 2, 2, 1, 1), (1, ),
v2c.Integer(1)),
MibScalarInstance((1, 3, 6, 1, 2, 1, 2, 2, 1, 1), (2, ),
v2c.Integer(2)),
MibScalarInstance((1, 3, 6, 1, 2, 1, 2, 2, 1, 1), (3, ),
v2c.Integer(3)),
# IF-MIB::ifDescr
MibTableColumn((1, 3, 6, 1, 2, 1, 2, 2, 1, 2), v2c.OctetString()),
MibScalarInstance((1, 3, 6, 1, 2, 1, 2, 2, 1, 2), (1, ),
v2c.OctetString("lo")),
MibScalarInstance((1, 3, 6, 1, 2, 1, 2, 2, 1, 2), (2, ),
v2c.OctetString("eth0")),
MibScalarInstance((1, 3, 6, 1, 2, 1, 2, 2, 1, 2), (3, ),
v2c.OctetString("eth1")),
# IF-MIB::ifType
MibTableColumn((1, 3, 6, 1, 2, 1, 2, 2, 1, 3), v2c.Integer()),
MibScalarInstance((1, 3, 6, 1, 2, 1, 2, 2, 1, 3), (1, ),
v2c.Integer(24)),
MibScalarInstance((1, 3, 6, 1, 2, 1, 2, 2, 1, 3), (2, ),
v2c.Integer(6)),
MibScalarInstance((1, 3, 6, 1, 2, 1, 2, 2, 1, 3), (3, ),
v2c.Integer(6)),
# IF-MIB::ifIndex
ifIndex=MibTableColumn((1, 3, 6, 1, 2, 1, 2, 2, 1, 1),
v2c.Integer()))
mibBuilder.exportSymbols(
'__MY_SNIMPY-MIB',
# SNIMPY-MIB::snimpyIpAddress
MibScalar((1, 3, 6, 1, 2, 1, 45121, 1, 1),
v2c.OctetString()).setMaxAccess("readwrite"),
MibScalarInstance((1, 3, 6, 1, 2, 1, 45121, 1, 1), (0, ),
v2c.OctetString("AAAA")),
# SNIMPY-MIB::snimpyString
MibScalar((1, 3, 6, 1, 2, 1, 45121, 1, 2),
v2c.OctetString()).setMaxAccess("readwrite"),
MibScalarInstance((1, 3, 6, 1, 2, 1, 45121, 1, 2), (0, ),
v2c.OctetString("bye")),
# SNIMPY-MIB::snimpyInteger
MibScalar((1, 3, 6, 1, 2, 1, 45121, 1, 3),
v2c.Integer()).setMaxAccess("readwrite"),
MibScalarInstance((1, 3, 6, 1, 2, 1, 45121, 1, 3), (0, ),
v2c.Integer(19)),
# SNIMPY-MIB::snimpyEnum
MibScalar((1, 3, 6, 1, 2, 1, 45121, 1, 4),
v2c.Integer()).setMaxAccess("readwrite"),
MibScalarInstance((1, 3, 6, 1, 2, 1, 45121, 1, 4), (0, ),
v2c.Integer(2)),
# SNIMPY-MIB::snimpyObjectId
MibScalar((1, 3, 6, 1, 2, 1, 45121, 1, 5),
v2c.ObjectIdentifier()).setMaxAccess("readwrite"),
MibScalarInstance((1, 3, 6, 1, 2, 1, 45121, 1, 5), (0, ),
v2c.ObjectIdentifier((1, 3, 6, 4454, 0, 0))),
# SNIMPY-MIB::snimpyBoolean
MibScalar((1, 3, 6, 1, 2, 1, 45121, 1, 6),
v2c.Integer()).setMaxAccess("readwrite"),
MibScalarInstance((1, 3, 6, 1, 2, 1, 45121, 1, 6), (0, ),
v2c.Integer(1)),
# SNIMPY-MIB::snimpyCounter
MibScalar((1, 3, 6, 1, 2, 1, 45121, 1, 7),
v2c.Counter32()).setMaxAccess("readwrite"),
MibScalarInstance((1, 3, 6, 1, 2, 1, 45121, 1, 7), (0, ),
v2c.Counter32(47)),
# SNIMPY-MIB::snimpyGauge
MibScalar((1, 3, 6, 1, 2, 1, 45121, 1, 8),
v2c.Gauge32()).setMaxAccess("readwrite"),
MibScalarInstance((1, 3, 6, 1, 2, 1, 45121, 1, 8), (0, ),
v2c.Gauge32(18)),
# SNIMPY-MIB::snimpyTimeticks
MibScalar((1, 3, 6, 1, 2, 1, 45121, 1, 9),
v2c.TimeTicks()).setMaxAccess("readwrite"),
MibScalarInstance((1, 3, 6, 1, 2, 1, 45121, 1, 9), (0, ),
v2c.TimeTicks(12111100)),
# SNIMPY-MIB::snimpyCounter64
MibScalar((1, 3, 6, 1, 2, 1, 45121, 1, 10),
v2c.Counter64()).setMaxAccess("readwrite"),
MibScalarInstance((1, 3, 6, 1, 2, 1, 45121, 1, 10), (0, ),
v2c.Counter64(2**48 + 3)),
# SNIMPY-MIB::snimpyBits
MibScalar((1, 3, 6, 1, 2, 1, 45121, 1, 11),
v2c.OctetString()).setMaxAccess("readwrite"),
MibScalarInstance((1, 3, 6, 1, 2, 1, 45121, 1, 11), (0, ),
v2c.OctetString(b"\xa0")),
# SNIMPY-MIB::snimpyMacAddress
MibScalar((1, 3, 6, 1, 2, 1, 45121, 1, 15),
v2c.OctetString()).setMaxAccess("readwrite"),
MibScalarInstance((1, 3, 6, 1, 2, 1, 45121, 1, 15), (0, ),
v2c.OctetString(b"\x11\x12\x13\x14\x15\x16")),
# SNIMPY-MIB::snimpyIndexTable
MibTable((1, 3, 6, 1, 2, 1, 45121, 2, 3)),
MibTableRow((1, 3, 6, 1, 2, 1, 45121, 2, 3, 1)).setIndexNames(
(0, "__MY_SNIMPY-MIB", "snimpyIndexVarLen"),
(0, "__MY_SNIMPY-MIB", "snimpyIndexOidVarLen"),
(0, "__MY_SNIMPY-MIB", "snimpyIndexFixedLen"),
(1, "__MY_SNIMPY-MIB", "snimpyIndexImplied")),
# SNIMPY-MIB::snimpyIndexInt
MibScalarInstance((1, 3, 6, 1, 2, 1, 45121, 2, 3, 1, 6),
flatten(4, stringToOid('row1'), 3, 1, 2, 3,
stringToOid('alpha5'),
stringToOid('end of row1')),
v2c.Integer(4571)),
MibScalarInstance((1, 3, 6, 1, 2, 1, 45121, 2, 3, 1, 6),
flatten(4, stringToOid('row2'), 4, 1, 0, 2, 3,
stringToOid('beta32'),
stringToOid('end of row2')),
v2c.Integer(78741)),
MibScalarInstance((1, 3, 6, 1, 2, 1, 45121, 2, 3, 1, 6),
flatten(4, stringToOid('row3'), 4, 120, 1, 2, 3,
stringToOid('gamma7'),
stringToOid('end of row3')),
v2c.Integer(4110)),
# Indexes
snimpyIndexVarLen=MibTableColumn(
(1, 3, 6, 1, 2, 1, 45121, 2, 3, 1, 1),
v2c.OctetString()).setMaxAccess("noaccess"),
snimpyIndexIntIndex=MibTableColumn(
(1, 3, 6, 1, 2, 1, 45121, 2, 3, 1, 2),
v2c.Integer()).setMaxAccess("noaccess"),
snimpyIndexOidVarLen=MibTableColumn(
(1, 3, 6, 1, 2, 1, 45121, 2, 3, 1, 3),
v2c.ObjectIdentifier()).setMaxAccess("noaccess"),
snimpyIndexFixedLen=MibTableColumn(
(1, 3, 6, 1, 2, 1, 45121, 2, 3, 1, 4),
v2c.OctetString().setFixedLength(6)).setMaxAccess("noaccess"),
snimpyIndexImplied=MibTableColumn(
(1, 3, 6, 1, 2, 1, 45121, 2, 3, 1, 5),
v2c.OctetString()).setMaxAccess("noaccess"),
snimpyIndexInt=MibTableColumn(
(1, 3, 6, 1, 2, 1, 45121, 2, 3, 1, 6),
v2c.Integer()).setMaxAccess("readwrite"))
# Start agent
cmdrsp.GetCommandResponder(snmpEngine, snmpContext)
cmdrsp.SetCommandResponder(snmpEngine, snmpContext)
cmdrsp.NextCommandResponder(snmpEngine, snmpContext)
cmdrsp.BulkCommandResponder(snmpEngine, snmpContext)
q.put(port)
snmpEngine.transportDispatcher.jobStarted(1)
snmpEngine.transportDispatcher.runDispatcher()
print('====== multiprocessing.Queue ======')
def producer(q):
for fruit in ['apple', 'tangerine', 'orange', 'banana']:
print(' > produce "%s"' % fruit)
q.put(fruit)
time.sleep(random.random())
def consumer(q):
while True:
fruit = q.get(True)
print(' < eat "%s"' % fruit)
q = Queue()
p1 = Process(target=producer, args=(q, ))
c1 = Process(target=consumer, args=(q, ))
c2 = Process(target=consumer, args=(q, ))
p1.start()
c1.start()
c2.start()
p1.join() # 等待生产者结束
c1.terminate() # 进程里是死循环，无法等待其结束，只能强行终止:
c2.terminate() # 进程里是死循环，无法等待其结束，只能强行终止:
#############################
# 进程间通信: Pipes 管道
#############################
class Job(object):
"""An interruptable job.
Subclasses must implement `self.run()`. The implementation of `run` should
frequently check `self.stop_requested` and return when it becomes true.
Optionally, subclasses should implement `self.__str__()` to return a nice
name for the job.
Clients should invoke `.start()` after construction to start the Job. They
must invoke `.join()` at some point to clean up the Job's resources.
Sample usage:
class PrintLoop(Job):
def run(self):
while True:
if self.stop_requested:
return
print("still running!")
job = PrintLoop()
job.start()
sleep(10)
job.request_stop()
job.join()
assert job.done
print("success?", job.successful)
While a job is running, clients can check `job.done` to see if the job has
completed yet.
If .run() throws an uncaught exception, it is printed to standard error.
When a job completes, clients can check `job.successful` to see if the job
returned normally or threw an uncaught exception. There is currently no
way to retrieve the thrown exception, if any.
"""
def __init__(self):
self._thread = Process(target=self._run, daemon=True)
self._flags = Array("b", [False] * 3)
# flags[0] - stop_requested?
# flags[1] - done?
# flags[2] - true iff completed with no exception
def start(self):
"""Start the job by invoking its .run() method asynchronously."""
self._thread.start()
def run(self):
"""Subclasses should override this to implement the Job's behavior."""
raise NotImplementedError()
def _run(self):
"""Private helper that wraps .run() and sets various exit flags."""
try:
if do_profiling.value:
import cProfile
import tempfile
(fd, filename) = tempfile.mkstemp(suffix=".prof")
print("Profile info: {}".format(filename))
cProfile.runctx("self.run()",
globals(),
locals(),
filename=filename)
else:
self.run()
self._flags[2] = True
except Exception as e:
import traceback
traceback.print_exc()
self._flags[1] = True
@property
def stop_requested(self):
"""True if the job has been asked to stop.
The implementation of .run() should check this periodically and return
when it becomes True."""
return self._flags[0]
@property
def done(self):
"""True if the job has stopped."""
return self._flags[1] or (self._thread.exitcode is not None)
@property
def successful(self):
"""True if the job has stopped without throwing an uncaught exception."""
return self._flags[2]
def request_stop(self):
"""Request a graceful stop.
Causes this Job's .stop_requested property to become True.
Clients can call .join() to wait for the job to wrap up.
"""
print("requesting stop for {}".format(self))
self._flags[0] = True
def join(self, timeout=None):
"""Wait for the job to finish and clean up its resources.
This procedure may be called more than once, even on an already-joined
Job.
If `timeout` is provided, this procedure waits roughly `timeout`
seconds at most.
This procedure always returns None. If a timeout is provided, clients
should check the .done property to determine whether the job indeed
stopped.
NOTE: If a timeout was provided and the .done property was true after
completion, clients MUST call .join() again to ensure that the Job's
resources are properly cleaned up. This is because the Job may have
finished AFTER the call to .join() timed out but BEFORE the call to
.done completed.
Example of proper usage:
job.join(timeout=1)
if job.done:
job.join()
# the job is now complete and cleaned-up
else:
# the job is still running
"""
self._thread.join(timeout=timeout)
def kill(self):
"""Stop this job forcefully.
This is implemented using the `Process.terminate()` procedure in the
`multiprocessing` module. There are two important caveats:
- Clients should still call .join() afterwards to clean up the Job.
- If this Job spawned any Jobs of its own that it did not join, then
those jobs are now lost and can never be cleaned up.
- If this Job was killed while it was putting an object into a Queue
or SafeQueue or writing data to a pipe, then the queue or pipe may
become corrupted and unusable by the parent process.
- If this Job had acquired a shared lock, then it will not release the
lock and killing it may lead to deadlocks.
"""
self._thread.terminate()
@property
def pid(self):
"""Get the process ID of the process running this Job.
Clients should not call this before .start() or after .join().
"""
return self._thread.pid
# communicate with the real vehicles
serverThread.join(
timeout
) # implicitly controls the speed of the simulation; blocks the main program either until the server program terminates (if no timeout is defined) or until the timeout occurs
# process the information received from the real vehicles
if flag_hello_smartphone[0] == True:
addRealVehicle(ticket)
realVehicles.append('vehPrius{}'.format(ticket))
flag_hello_smartphone[0] = False
ticket += 1
for i in range(len(smartphone_broadcast)):
if 'vehPrius{}'.format(i + 1) in vehicles:
traci.vehicle.setSpeed('vehPrius{}'.format(i + 1),
smartphone_broadcast[i])
for i in range(len(flag_goodbye_smartphone)):
if flag_goodbye_smartphone[i] == True:
removeRealVehicle(i)
flag_goodbye_smartphone[i] = False
print("")
# goto the next time step
step += 100 # in milliseconds
traci.simulationStep(
10100 + step
) # perform the simulation until the time in the day indicated (in milliseconds) is reached; use for time steps of 0.1s
print("Shutting the server down.")
serverThread.terminate()
traci.close() # close the connection to SUMO
