def ws_client(mocker):
mocker.patch("print_nanny.clients.websocket.asyncio")
m = aioprocessing.AioManager()
return WebSocketWorker(
"ws://localhost:8000/ws/predict/",
"3a833ac48104772a349254690cae747e826886f1",
m.Queue(),
1,
)
def predict_worker(mocker):
mocker.patch("print_nanny.predictor.threading")
m = aioprocessing.AioManager()
return PredictWorker(
"http://localhost:8080/?action=snapshot",
None,
m.Queue(),
m.Queue(),
)
def __init__(self, plugin):
self.plugin = plugin
self.manager = aioprocessing.AioManager()
self.shared = self.manager.Namespace()
# proxy objects
self.shared.printer_profile_id = None
self.shared.print_job_id = None
self.shared.calibration = None
self.active = False
# holds octoprint events to be uploaded via rest client
self.tracking_queue = self.manager.AioQueue()
# streamed to octoprint front-end over websocket
self.octo_ws_queue = self.manager.AioQueue()
# streamed to print nanny asgi over websocket
self.pn_ws_queue = self.manager.AioQueue()
self._tracking_event_handlers = {
Events.PRINT_STARTED: self._handle_print_start,
Events.PRINT_FAILED: self.stop,
Events.PRINT_DONE: self.stop,
Events.PRINT_CANCELLING: self.stop,
Events.PRINT_CANCELLED: self.stop,
Events.PRINT_PAUSED: self.stop,
Events.PRINT_RESUMED: self.stop,
Events.PRINT_PROGRESS: self._handle_print_progress_upload,
}
self._environment = {}
# daemonized threads for rest api and octoprint websocket relay
self.rest_api_thread = threading.Thread(target=self._rest_api_worker)
self.rest_api_thread.daemon = True
self.octo_ws_thread = threading.Thread(
target=self._octo_ws_queue_worker)
self.octo_ws_thread.daemon = True
self.loop = None
self.tracking_events = None
def test_executor(self):
m = aioprocessing.AioManager()
q = m.AioQueue()
p = ProcessPoolExecutor(max_workers=1)
val = 4
def submit():
yield p.submit(queue_put, q, val)
next(submit())
@asyncio.coroutine
def queue_get():
out = yield from q.coro_get()
self.assertEqual(out, val)
yield from q.coro_put(5)
self.loop.run_until_complete(queue_get())
returned = q.get()
self.assertEqual(returned, 5)
p.shutdown()
async def retrieve_certificates(loop, download_concurrency, mass_concurrency,
time_sec, once, anal_system_instance):
async with aiohttp.ClientSession(loop=loop, conn_timeout=10) as session:
while True:
urls = get_ctls_from_mass(anal_system_instance)
ctl_logs = await certlib.retrieve_all_ctls(session)
for log in ctl_logs:
if log['url'] not in urls:
continue
work_deque = deque()
download_results_queue = asyncio.Queue(
maxsize=DOWNLOAD_QUEUE_SIZE)
manager = aioprocessing.AioManager()
parse_results_queue = manager.Queue(maxsize=MASS_QUEUE_SIZE)
logging.info("Downloading certificates for {}".format(
log['description']))
try:
log_info = await certlib.retrieve_log_info(log, session)
except (aiohttp.ClientConnectorError,
aiohttp.ServerTimeoutError, aiohttp.ClientOSError,
aiohttp.ClientResponseError) as e:
logging.error(
"Failed to connect to CTL! -> {} - skipping.".format(
e))
continue
try:
await certlib.populate_work(work_deque, log_info, start=0)
except Exception as e:
logging.error("Log needs no update - {}".format(e))
continue
download_tasks = asyncio.gather(*[
download_worker(session, log_info, work_deque,
download_results_queue, urls[log['url']])
for _ in range(download_concurrency)
])
processing_task = asyncio.ensure_future(
processing_coro(download_results_queue,
parse_results_queue))
asyncio.ensure_future(download_tasks)
mass_task = asyncio.ensure_future(
mass_worker(parse_results_queue, mass_concurrency))
await download_tasks
await download_results_queue.put(
None) # Downloads are done, processing can stop
await processing_task
for _ in range(0, mass_concurrency):
parse_results_queue.put(None)
print('Parsing complete. MASS Queue: {}'.format(
parse_results_queue.qsize()))
await mass_task
if not once:
logging.info(
'Completed. Sleeping for {} seconds.'.format(time_sec))
time.sleep(time_sec)
else:
logging.info('Completed.')
break
def setUp(self):
super().setUp()
self.manager = aioprocessing.AioManager()
self.Obj = self.manager.AioLock
self.inst = self.Obj()
def setUp(self):
super().setUp()
self.type_ = MANAGER_TYPE
self.manager = aioprocessing.AioManager()
self.lock = self.manager.AioRLock()
sentence_probs[k] /= denominator
return sentence_id, sentence_probs
def best_sentence(sentences):
sentence_id, sentence_probs = sentence_parsing(sentences)
mbr = Mbr(sentence_probs)
return sentence_id, mbr.mbr_calc()
if __name__ == '__main__':
readers = [
open(os.path.join(args.base_dir, input_path), 'r')
for input_path in args.input
]
m = aioprocessing.AioManager()
in_queue_m = m.AioQueue()
counter = 0
ret = []
for lines in zip(*[reader.readlines() for reader in readers]):
counter += 1
if counter % args.beam_size == 0:
ret.insert(0, '%d' % (int(counter / args.beam_size)))
in_queue_m.put(ret)
ret = []
else:
for line in lines:
ret.append(line.strip())
for reader in readers:
reader.close()
def __init__(self):
super().__init__()
# have to set the level for logging
logging.getLogger().setLevel(logging.INFO)
# Set up item managers, unrelated to multiprocessing managers
self.handlerManager = None
self.blockManager = None
self.sharedData = handlerData()
# Load defaults
self.loader = configLoader.configLoader(SETTINGS_FILE)
# Set up multiprocessing items
self.processDict = {}
self.statusDict = {}
self.routerProcess = None
# This manager is for providing queues for the Router process
self.AioManager = aioprocessing.AioManager()
# This manager is for providing dill queues for the block processes
self.dillBlockManager = dillMp.Manager()
port = None
authkey = None
if "server.port" in self.loader.valueDict:
port = int(self.loader.valueDict["server.port"])
else:
port = 50000
# address is empty as the client will be acessing it
address = ('', port)
if "server.authkey" in self.loader.valueDict:
authkey = str.encode(self.loader.valueDict["server.authkey"])
else:
authkey = b"abc"
# This manager is for cleint sessions to acess these queues
self.clientSeverManager = qm.QueueManager(address=address,
authkey=authkey)
print(
f"Starting up server manager Address ip: {address[0]} port: {address[1]} and authkey: {authkey}"
)
# This queue is complicated as it's used both by local processes, that won't going through manager to get it
# But it will also be used by queues that are only going to be acessing it by manager
self.mainframeQueue = mp.Queue(-1)
qm.QueueManager.register("getMainframeQueue",
callable=lambda: self.mainframeQueue)
# This queue will only be used by mainframe and ui main model
self.uiQueue = mp.Queue(-1)
qm.QueueManager.register("getUiQueue", callable=lambda: self.uiQueue)
# start up the manager thread for serving its objects
threading.Thread(
target=self.clientSeverManager.get_server().serve_forever).start()
# we use regular multiprocessing here because otherwise the Dill queue will send to log which
# causes an infinite loop in our mpLogging module
# we don't need dill for this queue so it's okay to just use regular multiprocessing queue
# this queue is only used "locally" so it won't need to be connected to the manager
self.loggingQueue = mp.Queue(-1)
# set up flag variables
self.uiConnected = False
self.pendingUiMessages = []
self.uiLastTime = None
# add commands for processor
self.addCmdFunc(msg.CommandType.ADD_OUTPUT_VIEW,
mainframe.addOutputView)
self.addCmdFunc(msg.CommandType.UI_STARTUP, mainframe.sendStartupData)
self.addCmdFunc(msg.CommandType.CHECK_UI_STATUS,
mainframe.sendStatusCheck)
# Get other config files to load
config = configparser.ConfigParser()
config.read(SETTINGS_FILE)
blockConfigFile = ""
handlerConfigFile = ""
if 'Configs' in config:
blockConfigFile = config.get('Configs', 'Block', fallback="")
handlerConfigFile = config.get('Configs', 'Handler', fallback="")
# init handler manager
self.handlerManager = handlerManager(self.sharedData)
self.loadHandlerConfig(handlerConfigFile)
# init message router
# we use an aio queue here as it needs to be compatible with asyncio
# router and handlers use asyncio as handlers could have a lot of output operations
# that are best suited to asyncio
self.messageRouter = messageRouter(
self.handlerManager.messageSubscriptions, self.sharedData,
self.AioManager.AioQueue())
# init block manager
self.blockManager = blockManager(self.messageRouter)
self.loadBlockConfig(blockConfigFile)
for _, block in self.blockManager.blocks.items():
block.blockQueue = self.dillBlockManager.Queue()
block.mainframeQueue = self.mainframeQueue
# this will set the current working directory from wherever to the directory this file is in
# sys.path.append(os.path.dirname(os.path.abspath(sys.modules[__name__].__file__)))
# removed this on 7/1/2021, if it doesn't have any issues after a month, permanently take out
# dirPath = os.path.dirname(os.path.abspath(sys.modules[__name__].__file__))
# os.chdir(dirPath)
mpLogging.info("Finished initializing mainframe")