def start_vision_process(manager: SyncManager) -> \
Tuple[Namespace, Event, Event]:
ns = manager.Namespace()
# init ns
setattr(ns, 'active_config', ConfigMode.GEARS)
evt = manager.Event()
sh_evt = manager.Event()
proc = mp.Process(target=vision_starter, args=(ns, evt, sh_evt))
proc.start()
return ns, evt, sh_evt
def __init__(self):
mgr = SyncManager()
mgr.start(signal.signal, (signal.SIGINT, signal.SIG_IGN))
self.ns_default = mgr.Namespace()
self.ns_default.error = None
self.ns_stats = mgr.Namespace()
self.input_queue = mgr.Queue(maxsize=100)
self.error_occurred = mgr.Event()
self.error_processed = mgr.Event()
self.batch_done = mgr.Event()
self.mgr = mgr
self.stats_lock = mgr.Lock()
self.main_lock = mgr.Lock()
def __init__(self, mp_manager: SyncManager, stop_event: threading.Event,
closing_event: threading.Event, connect_q: queue.Queue,
result_q: queue.Queue):
self.connect_q = connect_q
self.result_q = result_q
self._stop_event = stop_event
self._closing_event = closing_event
self.all_connections_created = mp_manager.Event()
self.all_connections_closed = mp_manager.Event()
self.last_all_closed = time.time()
self.connections: Dict[int, Connection] = {}
self.sock_filenos: List[int] = [] # cache this, select() takes a list
self.connected_targets: Dict[Tuple[int, str, int], Connection] = {}
def __init__(self, conf_path='config/pn_conf.yaml'):
# input param
self.conf_path = conf_path
self.conf = getConf(self.conf_path, root_key='audio')
if self.conf['target'] == 'pi':
self.t2s = Text2SpeechBaidu(self.conf_path) # sync
else:
self.t2s = Text2SpeechXunFei(self.conf_path) # sync
self.executor_t2s = concurrent.futures.ProcessPoolExecutor(2) # async
if self.conf['use_custom_manager']:
# create proxy manager
mgr = SyncManager(
(get_lan_ip(), self.conf['custom_manager_port']),
self.conf['custom_manager_authkey'].encode('utf8'))
sleep(0.5) # wait for manager to start
mgr.connect()
else:
mgr = multiprocessing.Manager()
self.q_audio = mgr.Queue()
#-# debug('audio data queue created. %s', self.q_audio)
self.event_exit = mgr.Event()
multiprocessing.current_process(
).authkey = self.conf['custom_manager_authkey'].encode(
'utf8') # https://bugs.python.org/issue7503
self.proc_play = multiprocessing.Process(target=self.playAudioFromQ,
args=(self.q_audio,
self.event_exit))
self.proc_play.start()
#-# debug('play background proc start. %s', self.proc_play)
# 触发进程池worker进程创建, 貌似提前创建的占用内存小些
self.executor_t2s.map(noop_func, (None, None))
def __init__(self, conf_path='config/pn_conf.yaml'):
self.conf_path = os.path.abspath(conf_path)
self.conf = getConf(self.conf_path, root_key='itchat')
self.thread_id = None
self.gid = None # 记录我们群的UserName
if self.conf['use_custom_manager']:
# create proxy manager
mgr = SyncManager(
(get_lan_ip(), self.conf['custom_manager_port']),
self.conf['custom_manager_authkey'].encode('utf8'))
#-# sleep(0.5) # wait for manager to start
mgr.connect()
else:
mgr = multiprocessing.Manager()
self.q_send = mgr.Queue()
self.event_exit = mgr.Event()
multiprocessing.current_process(
).authkey = self.conf['custom_manager_authkey'].encode(
'utf8') # https://bugs.python.org/issue7503
self.proc_wx = multiprocessing.Process(target=self.run,
args=(self.event_exit,
self.q_send))
self.proc_wx.start()
class Executor(Eventful):
'''
The executor guides the execution of a single state, handles state forking
and selection, maintains run statistics and handles all exceptional
conditions (system calls, memory faults, concretization, etc.)
'''
_published_events = {'enqueue_state', 'generate_testcase', 'fork_state', 'load_state', 'terminate_state'}
def __init__(self, initial=None, store=None, policy='random', context=None, **kwargs):
super().__init__(**kwargs)
# Signals / Callbacks handlers will be invoked potentially at different
# worker processes. State provides a local context to save data.
self.subscribe('did_load_state', self._register_state_callbacks)
# This is the global manager that will handle all shared memory access among workers
self.manager = SyncManager()
self.manager.start(lambda: signal.signal(signal.SIGINT, signal.SIG_IGN))
# The main executor lock. Acquire this for accessing shared objects
self._lock = self.manager.Condition()
# Shutdown Event
self._shutdown = self.manager.Event()
# States on storage. Shared dict state name -> state stats
self._states = self.manager.list()
# Number of currently running workers. Initially no running workers
self._running = self.manager.Value('i', 0)
self._workspace = Workspace(self._lock, store)
# Executor wide shared context
if context is None:
context = {}
self._shared_context = self.manager.dict(context)
# scheduling priority policy (wip)
# Set policy
policies = {'random': Random,
'uncovered': Uncovered,
'branchlimited': BranchLimited,
}
self._policy = policies[policy](self)
assert isinstance(self._policy, Policy)
if self.load_workspace():
if initial is not None:
logger.error("Ignoring initial state")
else:
if initial is not None:
self.add(initial)
def __del__(self):
self.manager.shutdown()
@contextmanager
def locked_context(self, key=None, default=dict):
''' Executor context is a shared memory object. All workers share this.
It needs a lock. Its used like this:
with executor.context() as context:
visited = context['visited']
visited.append(state.cpu.PC)
context['visited'] = visited
'''
assert default in (list, dict, set)
with self._lock:
if key is None:
yield self._shared_context
else:
sub_context = self._shared_context.get(key, None)
if sub_context is None:
sub_context = default()
yield sub_context
self._shared_context[key] = sub_context
def _register_state_callbacks(self, state, state_id):
'''
Install forwarding callbacks in state so the events can go up.
Going up, we prepend state in the arguments.
'''
# Forward all state signals
self.forward_events_from(state, True)
def enqueue(self, state):
'''
Enqueue state.
Save state on storage, assigns an id to it, then add it to the
priority queue
'''
# save the state to secondary storage
state_id = self._workspace.save_state(state)
self.put(state_id)
self._publish('did_enqueue_state', state_id, state)
return state_id
def load_workspace(self):
# Browse and load states in a workspace in case we are trying to
# continue from paused run
loaded_state_ids = self._workspace.try_loading_workspace()
if not loaded_state_ids:
return False
for id in loaded_state_ids:
self._states.append(id)
return True
###############################################
# Synchronization helpers
@sync
def _notify_start_run(self):
# notify siblings we are about to start a run()
self._running.value += 1
@sync
def _notify_stop_run(self):
# notify siblings we are about to stop this run()
self._running.value -= 1
if self._running is None or self._running.value < 0:
raise SystemExit
self._lock.notify_all()
################################################
# Public API
@property
def running(self):
''' Report an estimate of how many workers are currently running '''
return self._running.value
def shutdown(self):
''' This will stop all workers '''
self._shutdown.set()
def is_shutdown(self):
''' Returns True if shutdown was requested '''
return self._shutdown.is_set()
###############################################
# Priority queue
@sync
def put(self, state_id):
''' Enqueue it for processing '''
self._states.append(state_id)
self._lock.notify_all()
return state_id
@sync
def get(self):
''' Dequeue a state with the max priority '''
# A shutdown has been requested
if self.is_shutdown():
return None
# if not more states in the queue, let's wait for some forks
while len(self._states) == 0:
# if no worker is running, bail out
if self.running == 0:
return None
# if a shutdown has been requested, bail out
if self.is_shutdown():
return None
# if there ares actually some workers running, wait for state forks
logger.debug("Waiting for available states")
self._lock.wait()
state_id = self._policy.choice(list(self._states))
if state_id is None:
return None
del self._states[self._states.index(state_id)]
return state_id
def list(self):
''' Returns the list of states ids currently queued '''
return list(self._states)
def generate_testcase(self, state, message='Testcase generated'):
'''
Simply announce that we're going to generate a testcase. Actual generation
should be handled by the driver class (such as :class:`~manticore.Manticore`)
:param state: The state to generate information about
:param message: Accompanying message
'''
# broadcast test generation. This is the time for other modules
# to output whatever helps to understand this testcase
self._publish('will_generate_testcase', state, 'test', message)
def fork(self, state, expression, policy='ALL', setstate=None):
'''
Fork state on expression concretizations.
Using policy build a list of solutions for expression.
For the state on each solution setting the new state with setstate
For example if expression is a Bool it may have 2 solutions. True or False.
Parent
(expression = ??)
Child1 Child2
(expression = True) (expression = True)
setstate(True) setstate(False)
The optional setstate() function is supposed to set the concrete value
in the child state.
'''
assert isinstance(expression, Expression)
if setstate is None:
setstate = lambda x, y: None
# Find a set of solutions for expression
solutions = state.concretize(expression, policy)
if not solutions:
raise ExecutorError("Forking on unfeasible constraint set")
if len(solutions) == 1:
setstate(state, solutions[0])
return state
logger.info("Forking. Policy: %s. Values: %s",
policy,
', '.join(f'0x{sol:x}' for sol in solutions))
self._publish('will_fork_state', state, expression, solutions, policy)
# Build and enqueue a state for each solution
children = []
for new_value in solutions:
with state as new_state:
new_state.constrain(expression == new_value)
# and set the PC of the new state to the concrete pc-dest
#(or other register or memory address to concrete)
setstate(new_state, new_value)
self._publish('did_fork_state', new_state, expression, new_value, policy)
# enqueue new_state
state_id = self.enqueue(new_state)
# maintain a list of children for logging purpose
children.append(state_id)
logger.info("Forking current state into states %r", children)
return None
def run(self):
'''
Entry point of the Executor; called by workers to start analysis.
'''
# policy_order=self.policy_order
# policy=self.policy
current_state = None
current_state_id = None
with WithKeyboardInterruptAs(self.shutdown):
# notify siblings we are about to start a run
self._notify_start_run()
logger.debug("Starting Manticore Symbolic Emulator Worker (pid %d).", os.getpid())
solver = Z3Solver()
while not self.is_shutdown():
try: # handle fatal errors: exceptions in Manticore
try: # handle external (e.g. solver) errors, and executor control exceptions
# select a suitable state to analyze
if current_state is None:
with self._lock:
# notify siblings we are about to stop this run
self._notify_stop_run()
try:
# Select a single state_id
current_state_id = self.get()
# load selected state from secondary storage
if current_state_id is not None:
self._publish('will_load_state', current_state_id)
current_state = self._workspace.load_state(current_state_id)
self.forward_events_from(current_state, True)
self._publish('did_load_state', current_state, current_state_id)
logger.info("load state %r", current_state_id)
# notify siblings we have a state to play with
finally:
self._notify_start_run()
# If current_state is still None. We are done.
if current_state is None:
logger.debug("No more states in the queue, byte bye!")
break
assert current_state is not None
assert current_state.constraints is current_state.platform.constraints
# Allows to terminate manticore worker on user request
while not self.is_shutdown():
if not current_state.execute():
break
else:
# Notify this worker is done
self._publish('will_terminate_state', current_state, current_state_id, TerminateState('Shutdown'))
current_state = None
# Handling Forking and terminating exceptions
except Concretize as e:
# expression
# policy
# setstate()
logger.debug("Generic state fork on condition")
current_state = self.fork(current_state, e.expression, e.policy, e.setstate)
except TerminateState as e:
# Notify this worker is done
self._publish('will_terminate_state', current_state, current_state_id, e)
logger.debug("Generic terminate state")
if e.testcase:
self.generate_testcase(current_state, str(e))
current_state = None
except SolverException as e:
# raise
import traceback
trace = traceback.format_exc()
logger.error("Exception: %s\n%s", str(e), trace)
# Notify this state is done
self._publish('will_terminate_state', current_state, current_state_id, e)
if solver.check(current_state.constraints):
self.generate_testcase(current_state, "Solver failed" + str(e))
current_state = None
except (Exception, AssertionError) as e:
# raise
import traceback
trace = traceback.format_exc()
logger.error("Exception: %s\n%s", str(e), trace)
# Notify this worker is done
self._publish('will_terminate_state', current_state, current_state_id, e)
current_state = None
logger.setState(None)
assert current_state is None or self.is_shutdown()
# notify siblings we are about to stop this run
self._notify_stop_run()
def run_server():
from connectn.tournament import run_tournament_process
from multiprocessing.managers import SyncManager
cu.configure_logging(cu.SERVER_PROCESS_LOG)
logger = logging.getLogger(__name__)
cu.start_stunnel(True)
manager = SyncManager()
manager.start(_process_init)
sq = mp.Queue()
rq = manager.Queue()
shutdown = manager.Event()
rg = mp.Process(
target=_process_init,
args=(run_tournament_process, sq, rq, shutdown, cu.PLAY_ALL),
name="RunGames",
)
rg.start()
logger.info("Started run_games process")
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as ls:
try:
ls.settimeout(5.0)
ls.bind(("localhost", cu.LISTEN_PORT))
ls.listen(5)
logger.info("Started server listening socket")
except Exception:
logger.exception("Failure when binding to the listening port.")
else:
updated_agent_archives = []
running = True
while running:
try:
(cs, addr) = ls.accept()
logger.info("Accepted connection.")
handle_client(cs, updated_agent_archives)
except socket.timeout:
if len(updated_agent_archives):
logger.info(
f"Server sending {len(updated_agent_archives)} new agents for game-play."
)
logger.info(f"{updated_agent_archives}")
sq.put(updated_agent_archives)
updated_agent_archives = []
except cu.InactiveSocket:
logger.exception("Connection failed")
except KeyboardInterrupt:
inp = input("Shutdown? y/[n] ").lower()
while inp not in ("", "y", "n"):
inp = input("Shutdown? y/[n] ").lower()
if inp == "y":
logger.info("KeyboardInterrupt: Shutting down")
running = False
else:
inp = input("Play all games? y/[n] ").lower()
while inp not in ("", "y", "n"):
inp = input("Play all games? y/[n] ").lower()
if inp == "y":
sq.put("PLAY_ALL")
except Exception:
logger.exception("Unexpected error, will try to keep running.")
store_results_local(rq)
finally:
"""
If the port is orphaned use:
fuser -kn tcp <port>
"""
ls.shutdown(socket.SHUT_RDWR)
logger.info("Closed server socket")
logger.info("Telling run_games process to shutdown.")
shutdown.set()
rg.join()
logger.info("Finished server shutdown.")
vowel_point[0].set_ydata(vowel_point[0].get_ydata() + ydelta)
return (vowel_point, )
ani = animation.FuncAnimation(fig,
vowel_update,
blit=False,
repeat=True,
interval=10)
plt.show()
if __name__ == '__main__':
# some multiprocessing setup
thread_manager = SyncManager()
thread_manager.start(mgr_init)
exit_event = thread_manager.Event()
audio_queue = thread_manager.Queue()
# start audio input thread
audio_input = multiprocessing.Process(target=get_audio,
args=(audio_queue, exit_event))
audio_input.daemon = True
audio_input.start()
gui_process = multiprocessing.Process(target=do_gui, args=(audio_queue, ))
gui_process.daemon = True
gui_process.start()
while gui_process.exitcode == None:
time.sleep(0.1)
class SubprocessBase(object):
'''Base class for subprocess routines that use socket connection'''
__metaclass__ = abc.ABCMeta
_error_msg = ('This executable should only be called from '
'inside the main program.')
def __init__(self):
self._pid = os.getpid()
#abort event
self._mgr = SyncManager()
self._mgr.start(ignore_interrupt)
self._abort_event = self._mgr.Event()
#stdout/err
self._err = StreamEncoder(sys.stderr)
self._out = StreamEncoder(sys.stdout)
#connection information
self._port = None
self._con = None
self()
#end def
def __del__(self):
if self._con is not None:
self.terminate()
self._con.close()
self._mgr.shutdown()
#end def
def _check_tty(self):
if sys.stdin.isatty():
self._err.write(self._error_msg+'\n')
return True
return False
#end def
def _sig_handler(self, signal, frame):
if self._pid != os.getpid(): return
self._out.write('%d aborting...\n'%os.getpid())
self._abort_event.set(); sleep(0.1)
tmpStorage.clean_tmp_files()
#end def
def _set_sig_handlers(self):
signal.signal(signal.SIGINT, self._sig_handler)
signal.signal(signal.SIGTERM, self._sig_handler)
signal.signal(signal.SIGQUIT, self._sig_handler)
#end def
def _parse_args(self):
parser = argparse.ArgumentParser(self._error_msg)
conf_group = parser.add_argument_group('Preset configuration')
conf_group.add_argument('port', metavar='number',
type=int, nargs=1,
help='Port number to connect to.')
args = parser.parse_args()
self._port = args.port[0]
#end def
def _get_auth_key(self):
try: self._auth = sys.stdin.readline().strip('\n')
except: self._auth = None
#end def
def _connect(self):
if self._port is None: return False
try: self._con = mpc.Client(('localhost', self._port),
authkey=self._auth)
except mpc.AuthenticationError, e:
self._err.write('Cannot connect to the port %d\n%s\n' % (self._port,str(e)))
return False
except:
def main():
# global server
global ticker_condition, platforms
global get_dict, get_condition
# Only works with 3.6+, checking version
version = sys.version.split(' ')[0].split('.')
invalid = False
if int(version[0]) < 3:
invalid = True
elif int(version[0]) == 3:
if int(version[1]) < 6:
invalid = True
if invalid:
print('Invalid version of Python. Requires Python 3.6 or newer')
return
read_config()
manager = SyncManager()
manager.start()
running = manager.Event()
running.set()
platforms = manager.dict()
ticker_condition = manager.Condition()
r_queue = manager.Queue()
r_condition = manager.Condition()
# used for HTTP GET requests and their responses
get_dict = manager.dict()
get_condition = manager.Condition()
# used for HTTP PUT and POST to issue their responses
reply_queue = manager.Queue()
reply_condition = manager.Condition()
# A pool closes, don't want it to close.
reply_list = []
reply_args = (running, reply_queue, reply_condition)
for i in range(config['threads']['return_threads']):
r = Process(target=outbound, args=reply_args, name='Reply_%s' % i)
r.deamon = True
reply_list.append(r)
r.start()
# A pool closes, don't want it to close.
r_list = []
r_args = (running, api_key, platforms, r_queue, r_condition, get_dict,
get_condition, reply_queue, reply_condition, ticker_condition,
platform_lock)
for i in range(config['threads']['api_threads']):
r = Process(target=retriever, args=r_args, name='Retriever_%s' % i)
r.deamon = True
r_list.append(r)
r.start()
ticking_args = (running, platforms, ticker_condition, r_queue, r_condition)
ticking = Process(target=ticker, args=ticking_args, name='Ticker')
ticking.deamon = True
ticking.start()
poking_args = (running, platforms, ticker_condition)
poking = Process(target=poker, args=poking_args, name='Poker')
poking.daemon = True
poking.start()
handler = CreateSyncHTTPHandler(manager)
#server = http.server.HTTPServer( (config['server']['host'], config['server']['port']), MyHTTPHandler)
server = http.server.HTTPServer(
(config['server']['host'], config['server']['port']), handler)
# server.set_manager(manager) # Need another way to pass this in, since HTTPServer doesn't have it...
try:
server.serve_forever()
except KeyboardInterrupt:
print('\nStopping server...')