def ws_client(mocker): mocker.patch("print_nanny.clients.websocket.asyncio") m = aioprocessing.AioManager() return WebSocketWorker( "ws://localhost:8000/ws/predict/", "3a833ac48104772a349254690cae747e826886f1", m.Queue(), 1, )
def predict_worker(mocker): mocker.patch("print_nanny.predictor.threading") m = aioprocessing.AioManager() return PredictWorker( "http://localhost:8080/?action=snapshot", None, m.Queue(), m.Queue(), )
def __init__(self, plugin): self.plugin = plugin self.manager = aioprocessing.AioManager() self.shared = self.manager.Namespace() # proxy objects self.shared.printer_profile_id = None self.shared.print_job_id = None self.shared.calibration = None self.active = False # holds octoprint events to be uploaded via rest client self.tracking_queue = self.manager.AioQueue() # streamed to octoprint front-end over websocket self.octo_ws_queue = self.manager.AioQueue() # streamed to print nanny asgi over websocket self.pn_ws_queue = self.manager.AioQueue() self._tracking_event_handlers = { Events.PRINT_STARTED: self._handle_print_start, Events.PRINT_FAILED: self.stop, Events.PRINT_DONE: self.stop, Events.PRINT_CANCELLING: self.stop, Events.PRINT_CANCELLED: self.stop, Events.PRINT_PAUSED: self.stop, Events.PRINT_RESUMED: self.stop, Events.PRINT_PROGRESS: self._handle_print_progress_upload, } self._environment = {} # daemonized threads for rest api and octoprint websocket relay self.rest_api_thread = threading.Thread(target=self._rest_api_worker) self.rest_api_thread.daemon = True self.octo_ws_thread = threading.Thread( target=self._octo_ws_queue_worker) self.octo_ws_thread.daemon = True self.loop = None self.tracking_events = None
def test_executor(self): m = aioprocessing.AioManager() q = m.AioQueue() p = ProcessPoolExecutor(max_workers=1) val = 4 def submit(): yield p.submit(queue_put, q, val) next(submit()) @asyncio.coroutine def queue_get(): out = yield from q.coro_get() self.assertEqual(out, val) yield from q.coro_put(5) self.loop.run_until_complete(queue_get()) returned = q.get() self.assertEqual(returned, 5) p.shutdown()
async def retrieve_certificates(loop, download_concurrency, mass_concurrency, time_sec, once, anal_system_instance): async with aiohttp.ClientSession(loop=loop, conn_timeout=10) as session: while True: urls = get_ctls_from_mass(anal_system_instance) ctl_logs = await certlib.retrieve_all_ctls(session) for log in ctl_logs: if log['url'] not in urls: continue work_deque = deque() download_results_queue = asyncio.Queue( maxsize=DOWNLOAD_QUEUE_SIZE) manager = aioprocessing.AioManager() parse_results_queue = manager.Queue(maxsize=MASS_QUEUE_SIZE) logging.info("Downloading certificates for {}".format( log['description'])) try: log_info = await certlib.retrieve_log_info(log, session) except (aiohttp.ClientConnectorError, aiohttp.ServerTimeoutError, aiohttp.ClientOSError, aiohttp.ClientResponseError) as e: logging.error( "Failed to connect to CTL! -> {} - skipping.".format( e)) continue try: await certlib.populate_work(work_deque, log_info, start=0) except Exception as e: logging.error("Log needs no update - {}".format(e)) continue download_tasks = asyncio.gather(*[ download_worker(session, log_info, work_deque, download_results_queue, urls[log['url']]) for _ in range(download_concurrency) ]) processing_task = asyncio.ensure_future( processing_coro(download_results_queue, parse_results_queue)) asyncio.ensure_future(download_tasks) mass_task = asyncio.ensure_future( mass_worker(parse_results_queue, mass_concurrency)) await download_tasks await download_results_queue.put( None) # Downloads are done, processing can stop await processing_task for _ in range(0, mass_concurrency): parse_results_queue.put(None) print('Parsing complete. MASS Queue: {}'.format( parse_results_queue.qsize())) await mass_task if not once: logging.info( 'Completed. Sleeping for {} seconds.'.format(time_sec)) time.sleep(time_sec) else: logging.info('Completed.') break
def setUp(self): super().setUp() self.manager = aioprocessing.AioManager() self.Obj = self.manager.AioLock self.inst = self.Obj()
def setUp(self): super().setUp() self.type_ = MANAGER_TYPE self.manager = aioprocessing.AioManager() self.lock = self.manager.AioRLock()
sentence_probs[k] /= denominator return sentence_id, sentence_probs def best_sentence(sentences): sentence_id, sentence_probs = sentence_parsing(sentences) mbr = Mbr(sentence_probs) return sentence_id, mbr.mbr_calc() if __name__ == '__main__': readers = [ open(os.path.join(args.base_dir, input_path), 'r') for input_path in args.input ] m = aioprocessing.AioManager() in_queue_m = m.AioQueue() counter = 0 ret = [] for lines in zip(*[reader.readlines() for reader in readers]): counter += 1 if counter % args.beam_size == 0: ret.insert(0, '%d' % (int(counter / args.beam_size))) in_queue_m.put(ret) ret = [] else: for line in lines: ret.append(line.strip()) for reader in readers: reader.close()
def __init__(self): super().__init__() # have to set the level for logging logging.getLogger().setLevel(logging.INFO) # Set up item managers, unrelated to multiprocessing managers self.handlerManager = None self.blockManager = None self.sharedData = handlerData() # Load defaults self.loader = configLoader.configLoader(SETTINGS_FILE) # Set up multiprocessing items self.processDict = {} self.statusDict = {} self.routerProcess = None # This manager is for providing queues for the Router process self.AioManager = aioprocessing.AioManager() # This manager is for providing dill queues for the block processes self.dillBlockManager = dillMp.Manager() port = None authkey = None if "server.port" in self.loader.valueDict: port = int(self.loader.valueDict["server.port"]) else: port = 50000 # address is empty as the client will be acessing it address = ('', port) if "server.authkey" in self.loader.valueDict: authkey = str.encode(self.loader.valueDict["server.authkey"]) else: authkey = b"abc" # This manager is for cleint sessions to acess these queues self.clientSeverManager = qm.QueueManager(address=address, authkey=authkey) print( f"Starting up server manager Address ip: {address[0]} port: {address[1]} and authkey: {authkey}" ) # This queue is complicated as it's used both by local processes, that won't going through manager to get it # But it will also be used by queues that are only going to be acessing it by manager self.mainframeQueue = mp.Queue(-1) qm.QueueManager.register("getMainframeQueue", callable=lambda: self.mainframeQueue) # This queue will only be used by mainframe and ui main model self.uiQueue = mp.Queue(-1) qm.QueueManager.register("getUiQueue", callable=lambda: self.uiQueue) # start up the manager thread for serving its objects threading.Thread( target=self.clientSeverManager.get_server().serve_forever).start() # we use regular multiprocessing here because otherwise the Dill queue will send to log which # causes an infinite loop in our mpLogging module # we don't need dill for this queue so it's okay to just use regular multiprocessing queue # this queue is only used "locally" so it won't need to be connected to the manager self.loggingQueue = mp.Queue(-1) # set up flag variables self.uiConnected = False self.pendingUiMessages = [] self.uiLastTime = None # add commands for processor self.addCmdFunc(msg.CommandType.ADD_OUTPUT_VIEW, mainframe.addOutputView) self.addCmdFunc(msg.CommandType.UI_STARTUP, mainframe.sendStartupData) self.addCmdFunc(msg.CommandType.CHECK_UI_STATUS, mainframe.sendStatusCheck) # Get other config files to load config = configparser.ConfigParser() config.read(SETTINGS_FILE) blockConfigFile = "" handlerConfigFile = "" if 'Configs' in config: blockConfigFile = config.get('Configs', 'Block', fallback="") handlerConfigFile = config.get('Configs', 'Handler', fallback="") # init handler manager self.handlerManager = handlerManager(self.sharedData) self.loadHandlerConfig(handlerConfigFile) # init message router # we use an aio queue here as it needs to be compatible with asyncio # router and handlers use asyncio as handlers could have a lot of output operations # that are best suited to asyncio self.messageRouter = messageRouter( self.handlerManager.messageSubscriptions, self.sharedData, self.AioManager.AioQueue()) # init block manager self.blockManager = blockManager(self.messageRouter) self.loadBlockConfig(blockConfigFile) for _, block in self.blockManager.blocks.items(): block.blockQueue = self.dillBlockManager.Queue() block.mainframeQueue = self.mainframeQueue # this will set the current working directory from wherever to the directory this file is in # sys.path.append(os.path.dirname(os.path.abspath(sys.modules[__name__].__file__))) # removed this on 7/1/2021, if it doesn't have any issues after a month, permanently take out # dirPath = os.path.dirname(os.path.abspath(sys.modules[__name__].__file__)) # os.chdir(dirPath) mpLogging.info("Finished initializing mainframe")