async def gathering(serials, kwargs): class Done: pass ts = [] queue = asyncio.Queue() for serial in serials: ts.append( hp.async_as_background( self._follow(plans, serial, queue, **kwargs))) def on_finish(res): hp.async_as_background(queue.put(Done)) if not ts: on_finish(None) else: t = hp.async_as_background(asyncio.wait(ts)) t.add_done_callback(on_finish) while True: nxt = await queue.get() if nxt is Done: break yield nxt
async def run_with(self, references, args_for_run, **kwargs): do_raise = kwargs.get("error_catcher") is None error_catcher = [] if do_raise else kwargs["error_catcher"] kwargs["error_catcher"] = error_catcher queue = asyncio.Queue() futs = [] if isinstance(references, SpecialReference): try: _, references = await references.find( args_for_run, kwargs.get("broadcast", False), kwargs.get("find_timeout", 5)) except asyncio.CancelledError: raise except Exception as error: if do_raise: raise hp.add_error(error_catcher, error) return if type(references) is not list: references = [references] if not references: references = [[]] elif self.synchronized: references = [references] for reference in references: futs.append( hp.async_as_background(self.run_children( queue, reference, args_for_run, **kwargs), silent=True)) class Done: pass async def wait_all(): await asyncio.wait(futs, return_when=asyncio.ALL_COMPLETED) await queue.put(Done) waiter = hp.async_as_background(wait_all()) try: while True: nxt = await queue.get() if nxt is Done: break else: yield nxt finally: waiter.cancel() for f in futs: f.cancel() if do_raise and error_catcher: raise RunErrors(_errors=list(set(error_catcher)))
def on_done(packet, res): if res.cancelled(): hp.add_error( error_catcher, TimedOut("Message was cancelled", serial=packet.serial)) else: exc = res.exception() if exc: hp.add_error(error_catcher, exc) if all(f.done() for f in fs): hp.async_as_background(queue.put(Done))
def activate(*args): self.activate_listeners() if "stopped" not in info: info["stopped"] = time.time() self.removers.append( hp.async_as_background(remove_after_a_minute())) self.removers = [t for t in self.removers if not t.done()]
async def getter(self): gen = self.item.generator(self.generator_reference, self.sender, **self.kwargs) complete = None while True: try: msg = await gen.asend(complete) if isinstance(msg, Exception): hp.add_error(self.error_catcher, msg) continue if self.stop_fut.done(): break complete = asyncio.Future() f_for_items = [] for item in self.item.simplifier(msg): f = asyncio.Future() f_for_items.append(f) t = hp.async_as_background(self.retrieve(item, f)) self.ts.append(t) self.complete_on_all_done(f_for_items, complete) self.ts = [t for t in self.ts if not t.done()] except StopAsyncIteration: break await self.wait_for_ts()
def complete_on_all_done(self, fs, complete): def finish(res): if complete.done(): return if res.cancelled(): for f in fs: f.cancel() complete.cancel() return for f in fs: if f.cancelled(): complete.cancel() return exc = f.exception() if exc is not None: complete.set_result(False) return elif f.result() is False: complete.set_result(False) return complete.set_result(True) waiter = hp.async_as_background(asyncio.wait(fs)) waiter.add_done_callback(finish)
def update_found(self, found, query_new_devices=True): """ Update our idea of what devices on the network. We delete information about devices that are no longer on the network. """ for target in list(self.by_target): if target not in found: del self.by_target[target] if query_new_devices: for target in found: if target not in self.by_target: t = hp.async_as_background( self.device_finder_loops.add_new_device(target)) self.tasks_by_target[target] = t t.add_done_callback(partial(self.cleanup_task, target, t)) # Make sure our targets are in by_target # Must happen after calling add_new_device for those that aren't in by_target for target in found: self.by_target[target] self.found.reset() self.found.set_result(found)
def __await__(self): # Protect against starting multiple writings tasks if not hasattr(self, "_writings"): self._writings = hp.async_as_background(self.writings()) self.write_tasks.append(self._writings) return (yield from self.final_future)
def ensure_refresh_information_loop(self, sender, time_between_queries, collections): loop = getattr(self, "_refresh_information_loop", None) if not loop or loop.done(): self._refresh_information_loop = hp.async_as_background( self.refresh_information_loop(sender, time_between_queries, collections)) return self._refresh_information_loop
def __aiter__(self): self.getter_t = hp.async_as_background(self.getter()) def on_finish(res): hp.async_as_background(self.queue.put(self.Done)) self.getter_t.add_done_callback(on_finish) return self
def process(packet, res): if res.cancelled(): e = TimedOut("Message was cancelled" , serial=packet.serial ) hp.add_error(error_catcher, e) return if not res.cancelled(): exc = res.exception() if exc: hp.add_error(error_catcher, exc) full_number = len(gatherers) == len(writers) futs_done = all(f.done() for f in futures) gatherers_done = all(f.done() for f in gatherers) if full_number and futs_done and gatherers_done: hp.async_as_background(queue.put(Done))
async def change(self, serial, tile_index, left_x, top_y, target, afr): if serial not in self.serials: return {"serial": serial, "data": None} user_x = (left_x + 4) / 8 user_y = (top_y - 4) / 8 msg = TileMessages.SetUserPosition(tile_index=tile_index, user_x=user_x, user_y=user_y, res_required=False) errors = [] await target.script(msg).run_with_all(serial, afr, error_catcher=errors, message_timeout=5) hp.async_as_background( self.highlight(serial, tile_index, target, afr, error_catcher=[], message_timeout=3)) if errors: return {"serial": serial, "data": None} plans = make_plans(chain=ChainPlan(refresh=True)) gatherer = Gatherer(target) got = await gatherer.gather_all(plans, serial, afr, error_catcher=errors, message_timeout=5) if serial in got and got[serial][0]: pixel_coords = user_coords_to_pixel_coords( got[serial][1]["chain"]["coords_and_sizes"]) self.serials[serial]["coords"] = [xy for xy, _ in pixel_coords] return {"serial": serial, "data": self.info_for_browser(serial)}
async def create_receiver(self, conn, packet, addr): if packet.target not in self.receivers: async def receive(): while not self.stop_fut.finished(): if conn is True: await asyncio.sleep(0.1) else: nxt = await conn[0].get() self.received_data(nxt, addr, conn) self.receivers[packet.target] = hp.async_as_background(receive())
async def cypress(typ): try: import asynctest # noqa except ImportError: raise PhotonsAppError( 'You must `pip install -e ".[tests]"` before you can run integration tests' ) from whirlwind.test_helpers import free_port, port_connected port = free_port() env = {"CYPRESS_BASE_URL": f"http://127.0.0.1:{port}"} final_future = collector.configuration["photons_app"].final_future t = None collector.configuration["interactor"].host = "127.0.0.1" collector.configuration["interactor"].port = port collector.configuration["interactor"].fake_devices = True collector.configuration[ "interactor"].database.uri = "sqlite:///:memory:" t = hp.async_as_background(serve(collector)) start = time.time() while time.time() - start < 5: if port_connected(port): break await asyncio.sleep(0.01) if not port_connected(port): raise PhotonsAppError("Failed to start server for tests") loop = asyncio.get_event_loop() def doit(): if os.environ.get("NO_BUILD_ASSETS") != "1": log.info("Building assets") assets.run("run-script", "build") log.info("Running cypress") assets.run("run-script", f"cypress:{typ}", extra_env=env) try: await loop.run_in_executor(None, doit) finally: exc_info = sys.exc_info() photons_app = collector.configuration["photons_app"] if exc_info[1]: photons_app.graceful_final_future.set_exception(exc_info[1]) else: photons_app.graceful_final_future.set_result(None) await t
async def do_apply_theme(target, reference, afr, options): aps = appliers[options.theme] theme = Theme() for color in options.colors: theme.add_hsbk(color.hue, color.saturation, color.brightness, color.kelvin) tasks = [] async for pkt, _, _ in target.script(DeviceMessages.GetVersion()).run_with( reference, afr): serial = pkt.serial capability = capability_for_ids(pkt.product, pkt.vendor) if capability.has_multizone: log.info(hp.lc("Found a strip", serial=serial)) t = hp.async_as_background( apply_zone(aps["1d"], target, afr, pkt.serial, theme, options.overrides)) elif capability.has_chain: log.info(hp.lc("Found a tile", serial=serial)) t = hp.async_as_background( apply_tile(aps["2d"], target, afr, pkt.serial, theme, options.overrides)) else: log.info(hp.lc("Found a light", serial=serial)) t = hp.async_as_background( apply_light(aps["0d"], target, afr, pkt.serial, theme, options.overrides)) tasks.append((serial, t)) results = {} for serial, t in tasks: try: await t except Exception as error: results[serial] = error else: results[serial] = "ok" return results
async def _get_response(self, packet, timeout, waiter, limit=None): errf = hp.ResettableFuture() errf.add_done_callback(hp.silent_reporter) response = [] async def wait_for_responses(): async with (limit or NoLimit()): if hasattr(asyncio, "current_task"): current_task = asyncio.current_task() else: current_task = asyncio.Task.current_task() asyncio.get_event_loop().call_later(timeout, timeout_task, current_task, errf, packet.serial) try: for info in await waiter: response.append(info) finally: if hasattr(waiter, "finish"): await waiter.finish() f = hp.async_as_background(wait_for_responses(), silent=True) def process(res): if errf.done() and not errf.cancelled(): # Errf has an exception return elif res.cancelled(): errf.reset() errf.set_exception( TimedOut("Message was cancelled", serial=packet.serial)) return if not res.cancelled(): exc = res.exception() if exc: errf.set_exception(exc) return errf.set_result(True) f.add_done_callback(process) try: await errf finally: f.cancel() return response
async def retrieve(self, streamer): streamer.no_more_work() started = hp.create_future() async def retrieve(): results = [] started.set_result(True) async for result in streamer: results.append(result) return results return started, hp.async_as_background(retrieve())
async def execute(self): ts = [] result = chp.ResultBuilder() def get(db): info = [] for scene in db.queries.get_scenes(uuid=self.uuid).all(): info.append(scene.as_object()) if not info: raise NoSuchScene(uuid=self.uuid) return info for scene in await self.db_queue.request(get): fltr = chp.filter_from_matcher(scene.matcher, False) if scene.zones: multizonefltr = self.clone_fltr_with_cap(fltr, "multizone") ts.append(hp.async_as_background(self.apply_zones(multizonefltr, scene, result))) notmultizonefltr = self.clone_fltr_with_no_cap(fltr, "multizone") ts.append(hp.async_as_background(self.transform(notmultizonefltr, scene, result))) elif scene.chain: chainfltr = self.clone_fltr_with_cap(fltr, "chain") ts.append(hp.async_as_background(self.apply_chain(chainfltr, scene, result))) notchainfltr = self.clone_fltr_with_no_cap(fltr, "chain") ts.append(hp.async_as_background(self.transform(notchainfltr, scene, result))) else: ts.append(hp.async_as_background(self.transform(fltr, scene, result))) for t in ts: try: await t except Exception as error: result.error(error) return result
def connect(self): recvQueue = asyncio.Queue() resQueue = asyncio.Queue() async def receive(): while True: got = Messages.unpack(await recvQueue.get(), None) self.received.append(got) if got.ack_required: await resQueue.put(got.make_ack().tobytes()) if got.res_required: for res in got.make_res(): await resQueue.put(res.tobytes()) self.connections.append(hp.async_as_background(receive())) return resQueue, recvQueue
async def reboot(self): if callable(self.pre_reboot): await self.pre_reboot(self) self.attrs.online = False self.reboots.append(time.time()) self.wait_for_reboot_fut.reset() for responder in self.all_responders: if hasattr(responder, "shutdown"): await responder.shutdown(self) async def back_online(time_rebooting, power_on): await asyncio.sleep(time_rebooting) await power_on() if not self.wait_for_reboot_fut.done(): self.wait_for_reboot_fut.set_result(True) if self.time_rebooting >= 0: hp.async_as_background( back_online(self.time_rebooting, self.power_on)) else: self.wait_for_reboot_fut.set_result(True)
async def leave_arrange(self, ref, target, afr): log.info(hp.lc("Leaving arrange", ref=ref)) tasks = [] for serial, info in list(self.serials.items()): info["refs"] = [r for r in info["refs"] if r != ref] if not info["refs"]: tasks.append( hp.async_as_background( self.restore(serial, info["initial"], target, afr))) del self.serials[serial] for t in tasks: await t
def _spawn(self, address, fut, backoff, timeout): """Spawn the actual connection""" sock = self.make_socket(address) t = hp.async_as_background(self.connect_socket(sock, address, fut, backoff, timeout)) def pass_failure(res): if fut.done(): return if res.cancelled(): fut.cancel() return exc = res.exception() if exc is not None: fut.set_exception(exc) t.add_done_callback(pass_failure)
async def start_arrange(self, serials, ref, target, afr): log.info(hp.lc("Starting arrange", serials=serials, ref=ref)) all_errors = [] for serial in list(self.serials): if serial not in serials: del self.serials[serial] for serial in serials: tasks = [] info = {"refs": [], "highlightlock": asyncio.Lock()} if serial in self.serials: info = self.serials[serial] tasks.append( (serial, hp.async_as_background(self.start(serial, info, target, afr)))) for serial, t in tasks: try: errors, data = await t except Exception as error: errors = [error] if errors: all_errors.extend(errors) else: info.update(data) self.serials[serial] = info final = {"serials": {}} if all_errors: final["error"] = ", ".join(str(e) for e in all_errors) log.error(hp.lc("Failed to start arrange", errors=all_errors)) for serial in serials: if serial in self.serials: if ref not in self.serials[serial]["refs"]: self.serials[serial]["refs"].append(ref) final["serials"][serial] = self.info_for_browser(serial) return final
async def args_for_run(self): """Return an afr object. Multiple calls to this will return the same object""" if not hasattr(self, "afr_fut"): self.afr_fut = asyncio.Future() t = hp.async_as_background(self.target.args_for_run()) def transfer(res): if res.cancelled(): self.afr_fut.cancel() exc = res.exception() if exc: self.afr_fut.set_exception(exc) self.afr_fut.set_result(res.result()) t.add_done_callback(transfer) return await self.afr_fut
async def finish(self, exc_typ=None, exc=None, tb=None): await super().finish(exc_typ, exc, tb) ts = [ hp.async_as_background(t.close()) for t in self.broadcast_transports.values() ] await hp.cancel_futures_and_wait( *ts, name=f"{type(self).__name__}::finish[wait_for_broadcast_transports]" ) for t in ts: if not t.cancelled(): exc = t.exception() if exc: log.error( hp.lc("Failed to close broadcast transport", error=exc))
async def start(self): self.port = self.kwargs.get("port", None) or pytest.helpers.free_port() await pytest.helpers.wait_for_no_port(self.port) self.options = make_options(**{**self.kwargs, "port": self.port}) self._task = hp.async_as_background( self.server.serve( "127.0.0.1", self.options.port, self.options, tasks=self.ts, sender=self.sender, cleaners=self.cleaners, animation_options=self.animation_options, )) await pytest.helpers.wait_for_port(self.port) return self
async def spawn(self, packet, *, timeout=10, create=True): if self.transport is not None: if self.transport.done() and (self.transport.cancelled() or self.transport.exception()): self.transport = None if self.transport is not None and self.transport.done(): if not await self.is_transport_active(packet, self.transport.result()): self.transport = None if not create and (self.transport is None or not self.transport.done()): return None if self.transport is None: self.transport = hp.async_as_background( self.spawn_transport(timeout)) return await self.transport
async def spawn(self, packet, *, timeout=10, create=True): if self.transport is not None: if self.transport.done() and (self.transport.cancelled() or self.transport.exception()): self.transport = None if self.transport is not None and self.transport.done(): if not await self.is_transport_active(packet, self.transport.result()): self.transport = None if not create and (self.transport is None or not self.transport.done()): return None if self.transport is None: self.transport = hp.create_future( name= f"Transport::{self.session.__class__.__name__}::spawn[transport]" ) t = hp.async_as_background(self.spawn_transport(timeout)) t.add_done_callback(hp.transfer_result(self.transport)) return await self.transport
async def write_messages(self, sender, packets, kwargs): """Send all our packets and collect all the results""" fs = [] queue = asyncio.Queue() error_catcher = kwargs["error_catcher"] def on_done(packet, res): if res.cancelled(): hp.add_error( error_catcher, TimedOut("Message was cancelled", serial=packet.serial)) else: exc = res.exception() if exc: hp.add_error(error_catcher, exc) if all(f.done() for f in fs): hp.async_as_background(queue.put(Done)) for original, packet in packets: coro = self.do_send(sender, original, packet, queue, kwargs) f = hp.async_as_background(coro, silent=True) f.add_done_callback(partial(on_done, packet)) fs.append(f) try: while True: msg = await queue.get() if msg is Done: break yield msg finally: for f in fs: f.cancel()
await asyncio.sleep(2) except asyncio.CancelledError: assert False, "Expected it to just raise the error rather than cancelling first" start = time.time() with assertRaises(PhotonsAppError, "Blah"): await hp.async_with_timeout(func(), timeout=0.1, timeout_error=error) assert time.time() - start < 0.5 describe "async_as_background": async it "runs the coroutine in the background": async def func(one, two, three=None): return "{0}.{1}.{2}".format(one, two, three) t = hp.async_as_background(func(6, 5, three=9)) pytest.helpers.assertFutCallbacks(t, hp.reporter) assert isinstance(t, asyncio.Task) assert await t == "6.5.9" async it "uses silent_reporter if silent is True": async def func(one, two, three=None): return "{0}.{1}.{2}".format(one, two, three) t = hp.async_as_background(func(6, 5, three=9), silent=True) pytest.helpers.assertFutCallbacks(t, hp.silent_reporter) assert isinstance(t, asyncio.Task) assert await t == "6.5.9" describe "silent_reporter":