async def retrieve_all(self, msg, complete): try: async with hp.ResultStreamer( self.stop_fut, name="FromGenerator>Runner::retrieve_all[streamer]", error_catcher=squash, ) as streamer: for item in self.item.simplifier(msg): await streamer.add_generator(self.retrieve(item), context="retrieve") streamer.no_more_work() async for result in streamer: if result.value is hp.ResultStreamer.GeneratorComplete: continue if not result.successful: if not isinstance(result.value, self.Unsuccessful): hp.add_error(self.error_catcher, result.value) if not complete.done(): complete.set_result(False) else: yield result.value finally: if not complete.done(): exc_info = sys.exc_info() if exc_info[0] is None: complete.set_result(True) else: complete.set_result(False)
async def _process_pkt(self, pkt, label, info): """ Actually process a packet for this PlanInfo. If instance.process(pkt) returns True or if pkt is NoMessages then use instance.info() to get a result. We then mark the plan as done, cache the result, and return it. """ plankey = info.plankey instance = info.instance if pkt is NoMessages or instance.process(pkt): info.mark_done() try: result = await instance.info() except asyncio.CancelledError: raise except Exception as error: hp.add_error(self.error_catcher, error) return if plankey is not None: self.session.fill(plankey, instance.serial, result) return instance.serial, label, result
async def getter(self): gen = self.item.generator(self.generator_reference, self.sender, **self.kwargs) complete = None while True: try: msg = await gen.asend(complete) if isinstance(msg, Exception): hp.add_error(self.error_catcher, msg) continue if self.stop_fut.done(): break complete = asyncio.Future() f_for_items = [] for item in self.item.simplifier(msg): f = asyncio.Future() f_for_items.append(f) t = hp.async_as_background(self.retrieve(item, f)) self.ts.append(t) self.complete_on_all_done(f_for_items, complete) self.ts = [t for t in self.ts if not t.done()] except StopAsyncIteration: break await self.wait_for_ts()
def catch_errors(error_catcher=None): do_raise = error_catcher is None error_catcher = [] if do_raise else error_catcher try: yield error_catcher except asyncio.CancelledError: raise except StopPacketStream: pass except Exception as error: if not isinstance(error, (PhotonsAppError, UserQuit)): log.exception(error) hp.add_error(error_catcher, error) if not do_raise: return error_catcher = list(set(error_catcher)) if len(error_catcher) == 1: raise error_catcher[0] if error_catcher: raise RunErrors(_errors=error_catcher)
async def run_with(self, references, args_for_run, **kwargs): do_raise = kwargs.get("error_catcher") is None error_catcher = [] if do_raise else kwargs["error_catcher"] kwargs["error_catcher"] = error_catcher queue = asyncio.Queue() futs = [] if isinstance(references, SpecialReference): try: _, references = await references.find( args_for_run, kwargs.get("broadcast", False), kwargs.get("find_timeout", 5)) except asyncio.CancelledError: raise except Exception as error: if do_raise: raise hp.add_error(error_catcher, error) return if type(references) is not list: references = [references] if not references: references = [[]] elif self.synchronized: references = [references] for reference in references: futs.append( hp.async_as_background(self.run_children( queue, reference, args_for_run, **kwargs), silent=True)) class Done: pass async def wait_all(): await asyncio.wait(futs, return_when=asyncio.ALL_COMPLETED) await queue.put(Done) waiter = hp.async_as_background(wait_all()) try: while True: nxt = await queue.get() if nxt is Done: break else: yield nxt finally: waiter.cancel() for f in futs: f.cancel() if do_raise and error_catcher: raise RunErrors(_errors=list(set(error_catcher)))
async def wait_for_ts(self): for t in self.ts: if not t.done(): try: await t except asyncio.CancelledError: pass except Exception as error: hp.add_error(self.error_catcher, error)
def on_done(packet, res): if res.cancelled(): hp.add_error( error_catcher, TimedOut("Message was cancelled", serial=packet.serial)) else: exc = res.exception() if exc: hp.add_error(error_catcher, exc) if all(f.done() for f in fs): hp.async_as_background(queue.put(Done))
async def getter(self): gen = self.item.generator(self.generator_reference, self.sender, **self.kwargs) await self.streamer.add_coroutine(self.consume(gen, self.streamer), context="consume") self.streamer.no_more_work() async for result in self.streamer: if not result.successful: hp.add_error(self.error_catcher, result.value) elif result.context is self.Value: yield result.value
async def gather(self, plans, reference, error_catcher=None, **kwargs): """ This is an async generator that yields tuples of ``(serial, label, info)`` where ``serial`` is the serial of the device, ``label`` is the label of the plan, and ``info`` is the result of executing that plan. The error handling of this function is the same as the async generator behaviour in :ref:`sender <sender_interface>` API. """ if not plans: return async def gathering(serials, kwargs): class Done: pass ts = [] queue = asyncio.Queue() for serial in serials: ts.append( hp.async_as_background( self._follow(plans, serial, queue, **kwargs))) def on_finish(res): hp.async_as_background(queue.put(Done)) if not ts: on_finish(None) else: t = hp.async_as_background(asyncio.wait(ts)) t.add_done_callback(on_finish) while True: nxt = await queue.get() if nxt is Done: break yield nxt with catch_errors(error_catcher) as error_catcher: kwargs["error_catcher"] = error_catcher serials, missing = await find_serials(reference, self.sender, timeout=kwargs.get( "find_timeout", 20)) for serial in missing: hp.add_error(error_catcher, FailedToFindDevice(serial=serial)) async for item in gathering(serials, kwargs): yield item
async def _stop_ts(self, cancelled=False): d = None try: if hasattr(self, "getter_t"): if cancelled: self.getter_t.cancel() d, _ = await asyncio.wait([self.getter_t]) except asyncio.CancelledError: raise else: if d: exc = list(d)[0].exception() if exc: hp.add_error(self.error_catcher, exc)
def process(packet, res): if res.cancelled(): e = TimedOut("Message was cancelled" , serial=packet.serial ) hp.add_error(error_catcher, e) return if not res.cancelled(): exc = res.exception() if exc: hp.add_error(error_catcher, exc) full_number = len(gatherers) == len(writers) futs_done = all(f.done() for f in futures) gatherers_done = all(f.done() for f in gatherers) if full_number and futs_done and gatherers_done: hp.async_as_background(queue.put(Done))
async def gen(reference, sender, **kwargs): async with hp.tick(min_loop_time, min_wait=False, final_future=sender.stop_fut) as ticks: async for i, _ in ticks: try: await (yield msg) finally: if isinstance(reference, SpecialReference): reference.reset() if callable(on_done_loop): try: await on_done_loop() except Repeater.Stop: return except asyncio.CancelledError: raise except Exception as error: hp.add_error(kwargs["error_catcher"], error)
async def write_messages(self, sender, packets, kwargs): """Send all our packets and collect all the results""" error_catcher = kwargs["error_catcher"] async with hp.ResultStreamer( sender.stop_fut, error_catcher=silence_errors, name="Item::write_messages[streamer]") as streamer: count = 0 for original, packet in packets: count += 1 await streamer.add_coroutine(self.do_send( sender, original, packet, kwargs), context=packet) streamer.no_more_work() got = 0 async for result in streamer: got += 1 if result.successful: for msg in result.value: yield msg else: exc = result.value pkt = result.context if isinstance(exc, asyncio.CancelledError): hp.add_error( error_catcher, TimedOut( "Message was cancelled", sent_pkt_type=pkt.pkt_type, serial=pkt.serial, source=pkt.source, sequence=pkt.sequence, ), ) else: hp.add_error(error_catcher, exc)
async def consume(self, gen, streamer): complete = None try: while True: try: msg = await gen.asend(complete) if isinstance(msg, Exception): hp.add_error(self.error_catcher, msg) continue if self.stop_fut.done(): break complete = hp.create_future( name="FromGenerator>Runner::getter[complete]") await streamer.add_generator(self.retrieve_all( msg, complete), context=self.Value) except StopAsyncIteration: break finally: exc_info = sys.exc_info() if exc_info[0] not in (None, asyncio.CancelledError): hp.add_error(self.error_catcher, exc_info[1]) await streamer.add_coroutine( hp.stop_async_generator( gen, complete, name="FromGenerator>Runner::consume[finally_stop_gen]", exc=exc_info[1], ), force=True, ) if exc_info[0] is not asyncio.CancelledError: return False
async def gen(reference, sender, **kwargs): while True: start = time.time() f = yield msg await f if isinstance(reference, SpecialReference): reference.reset() if callable(on_done_loop): try: await on_done_loop() except Repeater.Stop: break except asyncio.CancelledError: raise except Exception as error: hp.add_error(kwargs["error_catcher"], error) took = time.time() - start diff = min_loop_time - took if diff > 0: await asyncio.sleep(diff)
async def gather(self, plans, reference, error_catcher=None, **kwargs): """ This is an async generator that yields tuples of ``(serial, label, info)`` where ``serial`` is the serial of the device, ``label`` is the label of the plan, and ``info`` is the result of executing that plan. The error handling of this function is the same as the async generator behaviour in :ref:`sender <sender_interface>` API. """ if not plans: return with catch_errors(error_catcher) as error_catcher: kwargs["error_catcher"] = error_catcher serials, missing = await find_serials(reference, self.sender, timeout=kwargs.get( "find_timeout", 20)) for serial in missing: hp.add_error(error_catcher, FailedToFindDevice(serial=serial)) async with hp.ResultStreamer( self.sender.stop_fut, error_catcher=error_catcher, exceptions_only_to_error_catcher=True, ) as streamer: for serial in serials: await streamer.add_generator( self._follow(plans, serial, **kwargs)) streamer.no_more_work() async for result in streamer: if result.successful: yield result.value
from photons_app import helpers as hp from delfick_project.errors_pytest import assertRaises describe "throw_error": it "passes on errors if error_catcher is a callable": es = [] def ec(e): es.append(e) e1 = ValueError("NOPE") e2 = ValueError("NUP") with catch_errors(ec) as error_catcher: hp.add_error(error_catcher, e1) raise e2 assert error_catcher is ec assert es == [e1, e2] it "passes on errors if error_catcher is a list": es = [] e1 = ValueError("NOPE") e2 = ValueError("NUP") with catch_errors(es) as error_catcher: hp.add_error(error_catcher, e1) raise e2
def new_error_catcher(e): data["found_error"] = True hp.add_error(original_ec, e)
async def write_messages(sender, packets, kwargs): yield res1 hp.add_error(kwargs["error_catcher"], error1) yield res2 hp.add_error(kwargs["error_catcher"], error2)
def pass_on_error(e): i["success"] = False hp.add_error(self.error_catcher, e)
async def write_messages(self, packets, check_packet, make_writer, make_waiter, timeout, error_catcher): """Make a bunch of writers and then use them to create waiters""" writers = [] writing_packets = [] errors = [] for (original, packet) in packets: error = check_packet(packet) if error: errors.append(error) else: try: writer = await make_writer(original, packet) except Exception as error: hp.add_error(error_catcher, error) else: writers.append(writer) writing_packets.append(packet) for error in set(errors): hp.add_error(error_catcher, error) if not writing_packets: return queue = asyncio.Queue() futures = [] gatherers = [] def process(packet, res): if res.cancelled(): e = TimedOut("Message was cancelled" , serial=packet.serial ) hp.add_error(error_catcher, e) return if not res.cancelled(): exc = res.exception() if exc: hp.add_error(error_catcher, exc) full_number = len(gatherers) == len(writers) futs_done = all(f.done() for f in futures) gatherers_done = all(f.done() for f in gatherers) if full_number and futs_done and gatherers_done: hp.async_as_background(queue.put(Done)) for writer, (_, packet) in zip(writers, packets): # We separate the waiter from waiting on the waiter # so we can cancel the waiter instead of the thing waiting on it # To avoid AssertionError: _step(): already done logs waiter = make_waiter(writer) async def doit(w): for info in await w: await queue.put(info) f = asyncio.ensure_future(doit(waiter)) gatherers.append(f) f.add_done_callback(partial(process, packet)) futures.append(waiter) def canceller(): for f, packet in zip(futures, writing_packets): if not f.done(): f.set_exception(TimedOut("Waiting for reply to a packet", serial=packet.serial)) asyncio.get_event_loop().call_later(timeout, canceller) while True: nxt = await queue.get() if nxt is Done: break yield nxt
# coding: spec from photons_app.test_helpers import TestCase from photons_app import helpers as hp import mock import os describe TestCase, "add_error": it "calls the error_catcher with the error if it's a callable": error = mock.Mock(name="error") catcher = mock.Mock(name="catcher") hp.add_error(catcher, error) catcher.assert_called_once_with(error) it "appends to the error catcher if it's a list": error = mock.Mock(name="error") catcher = [] hp.add_error(catcher, error) self.assertEqual(catcher, [error]) it "adds to the error catcher if it's a set": error = mock.Mock(name="error") catcher = set() hp.add_error(catcher, error) self.assertEqual(catcher, set([error])) describe TestCase, "a_temp_file": it "gives us the tmpfile": with hp.a_temp_file() as fle: fle.write(b"wassup")
def error(e): per_light_errors[serial].append(e) hp.add_error(original_error_catcher, e)
async def run_with(self, serials, args_for_run , broadcast=False, accept_found=False, found=None, error_catcher=None , **kwargs ): """ Entry point to this item, the idea is you create a `script` with the target and call `run_with` on the script, which ends up calling this We acknowledge the following keyword arguments. broadcast Whether we are broadcasting these messages or just unicasting directly to each device find_timeout timeout for finding devices connect_timeout timeout for connecting to devices timeout timeout for writing messages found A dictionary of ``{targetHex: (set([(ServiceType, addr), (ServiceType, addr), ...]), broadcastAddr)}`` If this is not provided, one is made for us accept_found Accept the found that was given and don't try to change it error_catcher A list that errors will be appended to instead of being raised. Or a callable that takes in the error as an argument. If this isn't specified then errors are raised after all the received messages have been yielded. Note that if there is only one serial that we sent messages to, then any error is raised as is. Otherwise we raise a ``photons_app.errors.RunErrors``, with all the errors in a list on the ``errors`` property of the RunErrors exception. * First we make the packets. * Then we find the devices (unless found is supplied) * Then we send the packets to the devices * Then we gather results and errors """ afr = args_for_run do_raise = error_catcher is None error_catcher = [] if do_raise else error_catcher broadcast_address = ( afr.default_broadcast if broadcast is True else broadcast ) or afr.default_broadcast if isinstance(serials, SpecialReference): try: found, serials = await serials.find(afr, broadcast, kwargs.get("find_timeout", 5)) accept_found = True except asyncio.CancelledError: raise except Exception as error: if do_raise: raise hp.add_error(error_catcher, error) return # Work out what and where to send # All the packets from here have targets on them packets = self.make_packets( afr , serials , broadcast ) # Determine found looked = False found = found if found is not None else afr.found if not accept_found and not broadcast: looked, found, catcher = await self.search( afr , found , packets , broadcast_address , kwargs.get("find_timeout", 20) ) if catcher: for error in set(catcher): hp.add_error(error_catcher, error) if do_raise: throw_error(serials, error_catcher) else: return # Work out where to send packets if type(broadcast) is tuple: addr = broadcast else: addr = None if broadcast is False else (broadcast_address, 56700) # Create our helpers that channel particular arguments into the correct places retry_options = afr.make_retry_options() def check_packet(packet): if packet.target is None or not looked or found and packet.target[:6] in found: return else: return FailedToFindDevice(serial=packet.serial) def make_waiter(writer): return afr.make_waiter(writer, retry_options=retry_options) async def make_writer(original, packet): return await afr.make_writer(original, packet , broadcast=broadcast, retry_options=retry_options , addr=addr, found=found, **kwargs ) writer_args = ( packets , check_packet, make_writer, make_waiter , kwargs.get("timeout", 10), error_catcher ) # Finally use our message_writer helper to get us some results async for thing in self.write_messages(*writer_args): yield thing if do_raise: throw_error(serials, error_catcher)