async def retrieve_all(self, msg, complete):
try:
async with hp.ResultStreamer(
self.stop_fut,
name="FromGenerator>Runner::retrieve_all[streamer]",
error_catcher=squash,
) as streamer:
for item in self.item.simplifier(msg):
await streamer.add_generator(self.retrieve(item),
context="retrieve")
streamer.no_more_work()
async for result in streamer:
if result.value is hp.ResultStreamer.GeneratorComplete:
continue
if not result.successful:
if not isinstance(result.value, self.Unsuccessful):
hp.add_error(self.error_catcher, result.value)
if not complete.done():
complete.set_result(False)
else:
yield result.value
finally:
if not complete.done():
exc_info = sys.exc_info()
if exc_info[0] is None:
complete.set_result(True)
else:
complete.set_result(False)
async def _process_pkt(self, pkt, label, info):
"""
Actually process a packet for this PlanInfo. If instance.process(pkt)
returns True or if pkt is NoMessages then use instance.info() to get a
result.
We then mark the plan as done, cache the result, and return it.
"""
plankey = info.plankey
instance = info.instance
if pkt is NoMessages or instance.process(pkt):
info.mark_done()
try:
result = await instance.info()
except asyncio.CancelledError:
raise
except Exception as error:
hp.add_error(self.error_catcher, error)
return
if plankey is not None:
self.session.fill(plankey, instance.serial, result)
return instance.serial, label, result
async def getter(self):
gen = self.item.generator(self.generator_reference, self.sender,
**self.kwargs)
complete = None
while True:
try:
msg = await gen.asend(complete)
if isinstance(msg, Exception):
hp.add_error(self.error_catcher, msg)
continue
if self.stop_fut.done():
break
complete = asyncio.Future()
f_for_items = []
for item in self.item.simplifier(msg):
f = asyncio.Future()
f_for_items.append(f)
t = hp.async_as_background(self.retrieve(item, f))
self.ts.append(t)
self.complete_on_all_done(f_for_items, complete)
self.ts = [t for t in self.ts if not t.done()]
except StopAsyncIteration:
break
await self.wait_for_ts()
def catch_errors(error_catcher=None):
do_raise = error_catcher is None
error_catcher = [] if do_raise else error_catcher
try:
yield error_catcher
except asyncio.CancelledError:
raise
except StopPacketStream:
pass
except Exception as error:
if not isinstance(error, (PhotonsAppError, UserQuit)):
log.exception(error)
hp.add_error(error_catcher, error)
if not do_raise:
return
error_catcher = list(set(error_catcher))
if len(error_catcher) == 1:
raise error_catcher[0]
if error_catcher:
raise RunErrors(_errors=error_catcher)
async def run_with(self, references, args_for_run, **kwargs):
do_raise = kwargs.get("error_catcher") is None
error_catcher = [] if do_raise else kwargs["error_catcher"]
kwargs["error_catcher"] = error_catcher
queue = asyncio.Queue()
futs = []
if isinstance(references, SpecialReference):
try:
_, references = await references.find(
args_for_run, kwargs.get("broadcast", False),
kwargs.get("find_timeout", 5))
except asyncio.CancelledError:
raise
except Exception as error:
if do_raise:
raise
hp.add_error(error_catcher, error)
return
if type(references) is not list:
references = [references]
if not references:
references = [[]]
elif self.synchronized:
references = [references]
for reference in references:
futs.append(
hp.async_as_background(self.run_children(
queue, reference, args_for_run, **kwargs),
silent=True))
class Done:
pass
async def wait_all():
await asyncio.wait(futs, return_when=asyncio.ALL_COMPLETED)
await queue.put(Done)
waiter = hp.async_as_background(wait_all())
try:
while True:
nxt = await queue.get()
if nxt is Done:
break
else:
yield nxt
finally:
waiter.cancel()
for f in futs:
f.cancel()
if do_raise and error_catcher:
raise RunErrors(_errors=list(set(error_catcher)))
async def wait_for_ts(self):
for t in self.ts:
if not t.done():
try:
await t
except asyncio.CancelledError:
pass
except Exception as error:
hp.add_error(self.error_catcher, error)
def on_done(packet, res):
if res.cancelled():
hp.add_error(
error_catcher,
TimedOut("Message was cancelled", serial=packet.serial))
else:
exc = res.exception()
if exc:
hp.add_error(error_catcher, exc)
if all(f.done() for f in fs):
hp.async_as_background(queue.put(Done))
async def getter(self):
gen = self.item.generator(self.generator_reference, self.sender,
**self.kwargs)
await self.streamer.add_coroutine(self.consume(gen, self.streamer),
context="consume")
self.streamer.no_more_work()
async for result in self.streamer:
if not result.successful:
hp.add_error(self.error_catcher, result.value)
elif result.context is self.Value:
yield result.value
async def gather(self, plans, reference, error_catcher=None, **kwargs):
"""
This is an async generator that yields tuples of
``(serial, label, info)`` where ``serial`` is the serial of the device,
``label`` is the label of the plan, and ``info`` is the result of
executing that plan.
The error handling of this function is the same as the async generator
behaviour in :ref:`sender <sender_interface>` API.
"""
if not plans:
return
async def gathering(serials, kwargs):
class Done:
pass
ts = []
queue = asyncio.Queue()
for serial in serials:
ts.append(
hp.async_as_background(
self._follow(plans, serial, queue, **kwargs)))
def on_finish(res):
hp.async_as_background(queue.put(Done))
if not ts:
on_finish(None)
else:
t = hp.async_as_background(asyncio.wait(ts))
t.add_done_callback(on_finish)
while True:
nxt = await queue.get()
if nxt is Done:
break
yield nxt
with catch_errors(error_catcher) as error_catcher:
kwargs["error_catcher"] = error_catcher
serials, missing = await find_serials(reference,
self.sender,
timeout=kwargs.get(
"find_timeout", 20))
for serial in missing:
hp.add_error(error_catcher, FailedToFindDevice(serial=serial))
async for item in gathering(serials, kwargs):
yield item
async def _stop_ts(self, cancelled=False):
d = None
try:
if hasattr(self, "getter_t"):
if cancelled:
self.getter_t.cancel()
d, _ = await asyncio.wait([self.getter_t])
except asyncio.CancelledError:
raise
else:
if d:
exc = list(d)[0].exception()
if exc:
hp.add_error(self.error_catcher, exc)
def process(packet, res):
if res.cancelled():
e = TimedOut("Message was cancelled"
, serial=packet.serial
)
hp.add_error(error_catcher, e)
return
if not res.cancelled():
exc = res.exception()
if exc:
hp.add_error(error_catcher, exc)
full_number = len(gatherers) == len(writers)
futs_done = all(f.done() for f in futures)
gatherers_done = all(f.done() for f in gatherers)
if full_number and futs_done and gatherers_done:
hp.async_as_background(queue.put(Done))
async def gen(reference, sender, **kwargs):
async with hp.tick(min_loop_time,
min_wait=False,
final_future=sender.stop_fut) as ticks:
async for i, _ in ticks:
try:
await (yield msg)
finally:
if isinstance(reference, SpecialReference):
reference.reset()
if callable(on_done_loop):
try:
await on_done_loop()
except Repeater.Stop:
return
except asyncio.CancelledError:
raise
except Exception as error:
hp.add_error(kwargs["error_catcher"], error)
async def write_messages(self, sender, packets, kwargs):
"""Send all our packets and collect all the results"""
error_catcher = kwargs["error_catcher"]
async with hp.ResultStreamer(
sender.stop_fut,
error_catcher=silence_errors,
name="Item::write_messages[streamer]") as streamer:
count = 0
for original, packet in packets:
count += 1
await streamer.add_coroutine(self.do_send(
sender, original, packet, kwargs),
context=packet)
streamer.no_more_work()
got = 0
async for result in streamer:
got += 1
if result.successful:
for msg in result.value:
yield msg
else:
exc = result.value
pkt = result.context
if isinstance(exc, asyncio.CancelledError):
hp.add_error(
error_catcher,
TimedOut(
"Message was cancelled",
sent_pkt_type=pkt.pkt_type,
serial=pkt.serial,
source=pkt.source,
sequence=pkt.sequence,
),
)
else:
hp.add_error(error_catcher, exc)
async def consume(self, gen, streamer):
complete = None
try:
while True:
try:
msg = await gen.asend(complete)
if isinstance(msg, Exception):
hp.add_error(self.error_catcher, msg)
continue
if self.stop_fut.done():
break
complete = hp.create_future(
name="FromGenerator>Runner::getter[complete]")
await streamer.add_generator(self.retrieve_all(
msg, complete),
context=self.Value)
except StopAsyncIteration:
break
finally:
exc_info = sys.exc_info()
if exc_info[0] not in (None, asyncio.CancelledError):
hp.add_error(self.error_catcher, exc_info[1])
await streamer.add_coroutine(
hp.stop_async_generator(
gen,
complete,
name="FromGenerator>Runner::consume[finally_stop_gen]",
exc=exc_info[1],
),
force=True,
)
if exc_info[0] is not asyncio.CancelledError:
return False
async def gen(reference, sender, **kwargs):
while True:
start = time.time()
f = yield msg
await f
if isinstance(reference, SpecialReference):
reference.reset()
if callable(on_done_loop):
try:
await on_done_loop()
except Repeater.Stop:
break
except asyncio.CancelledError:
raise
except Exception as error:
hp.add_error(kwargs["error_catcher"], error)
took = time.time() - start
diff = min_loop_time - took
if diff > 0:
await asyncio.sleep(diff)
async def gather(self, plans, reference, error_catcher=None, **kwargs):
"""
This is an async generator that yields tuples of
``(serial, label, info)`` where ``serial`` is the serial of the device,
``label`` is the label of the plan, and ``info`` is the result of
executing that plan.
The error handling of this function is the same as the async generator
behaviour in :ref:`sender <sender_interface>` API.
"""
if not plans:
return
with catch_errors(error_catcher) as error_catcher:
kwargs["error_catcher"] = error_catcher
serials, missing = await find_serials(reference,
self.sender,
timeout=kwargs.get(
"find_timeout", 20))
for serial in missing:
hp.add_error(error_catcher, FailedToFindDevice(serial=serial))
async with hp.ResultStreamer(
self.sender.stop_fut,
error_catcher=error_catcher,
exceptions_only_to_error_catcher=True,
) as streamer:
for serial in serials:
await streamer.add_generator(
self._follow(plans, serial, **kwargs))
streamer.no_more_work()
async for result in streamer:
if result.successful:
yield result.value
from photons_app import helpers as hp
from delfick_project.errors_pytest import assertRaises
describe "throw_error":
it "passes on errors if error_catcher is a callable":
es = []
def ec(e):
es.append(e)
e1 = ValueError("NOPE")
e2 = ValueError("NUP")
with catch_errors(ec) as error_catcher:
hp.add_error(error_catcher, e1)
raise e2
assert error_catcher is ec
assert es == [e1, e2]
it "passes on errors if error_catcher is a list":
es = []
e1 = ValueError("NOPE")
e2 = ValueError("NUP")
with catch_errors(es) as error_catcher:
hp.add_error(error_catcher, e1)
raise e2
def new_error_catcher(e):
data["found_error"] = True
hp.add_error(original_ec, e)
async def write_messages(sender, packets, kwargs):
yield res1
hp.add_error(kwargs["error_catcher"], error1)
yield res2
hp.add_error(kwargs["error_catcher"], error2)
def pass_on_error(e):
i["success"] = False
hp.add_error(self.error_catcher, e)
async def write_messages(self, packets, check_packet, make_writer, make_waiter, timeout, error_catcher):
"""Make a bunch of writers and then use them to create waiters"""
writers = []
writing_packets = []
errors = []
for (original, packet) in packets:
error = check_packet(packet)
if error:
errors.append(error)
else:
try:
writer = await make_writer(original, packet)
except Exception as error:
hp.add_error(error_catcher, error)
else:
writers.append(writer)
writing_packets.append(packet)
for error in set(errors):
hp.add_error(error_catcher, error)
if not writing_packets:
return
queue = asyncio.Queue()
futures = []
gatherers = []
def process(packet, res):
if res.cancelled():
e = TimedOut("Message was cancelled"
, serial=packet.serial
)
hp.add_error(error_catcher, e)
return
if not res.cancelled():
exc = res.exception()
if exc:
hp.add_error(error_catcher, exc)
full_number = len(gatherers) == len(writers)
futs_done = all(f.done() for f in futures)
gatherers_done = all(f.done() for f in gatherers)
if full_number and futs_done and gatherers_done:
hp.async_as_background(queue.put(Done))
for writer, (_, packet) in zip(writers, packets):
# We separate the waiter from waiting on the waiter
# so we can cancel the waiter instead of the thing waiting on it
# To avoid AssertionError: _step(): already done logs
waiter = make_waiter(writer)
async def doit(w):
for info in await w:
await queue.put(info)
f = asyncio.ensure_future(doit(waiter))
gatherers.append(f)
f.add_done_callback(partial(process, packet))
futures.append(waiter)
def canceller():
for f, packet in zip(futures, writing_packets):
if not f.done():
f.set_exception(TimedOut("Waiting for reply to a packet", serial=packet.serial))
asyncio.get_event_loop().call_later(timeout, canceller)
while True:
nxt = await queue.get()
if nxt is Done:
break
yield nxt
# coding: spec
from photons_app.test_helpers import TestCase
from photons_app import helpers as hp
import mock
import os
describe TestCase, "add_error":
it "calls the error_catcher with the error if it's a callable":
error = mock.Mock(name="error")
catcher = mock.Mock(name="catcher")
hp.add_error(catcher, error)
catcher.assert_called_once_with(error)
it "appends to the error catcher if it's a list":
error = mock.Mock(name="error")
catcher = []
hp.add_error(catcher, error)
self.assertEqual(catcher, [error])
it "adds to the error catcher if it's a set":
error = mock.Mock(name="error")
catcher = set()
hp.add_error(catcher, error)
self.assertEqual(catcher, set([error]))
describe TestCase, "a_temp_file":
it "gives us the tmpfile":
with hp.a_temp_file() as fle:
fle.write(b"wassup")
def error(e):
per_light_errors[serial].append(e)
hp.add_error(original_error_catcher, e)
async def run_with(self, serials, args_for_run
, broadcast=False, accept_found=False, found=None, error_catcher=None
, **kwargs
):
"""
Entry point to this item, the idea is you create a `script` with the
target and call `run_with` on the script, which ends up calling this
We acknowledge the following keyword arguments.
broadcast
Whether we are broadcasting these messages or just unicasting directly
to each device
find_timeout
timeout for finding devices
connect_timeout
timeout for connecting to devices
timeout
timeout for writing messages
found
A dictionary of
``{targetHex: (set([(ServiceType, addr), (ServiceType, addr), ...]), broadcastAddr)}``
If this is not provided, one is made for us
accept_found
Accept the found that was given and don't try to change it
error_catcher
A list that errors will be appended to instead of being raised.
Or a callable that takes in the error as an argument.
If this isn't specified then errors are raised after all the received
messages have been yielded.
Note that if there is only one serial that we sent messages to, then
any error is raised as is. Otherwise we raise a
``photons_app.errors.RunErrors``, with all the errors in a list on
the ``errors`` property of the RunErrors exception.
* First we make the packets.
* Then we find the devices (unless found is supplied)
* Then we send the packets to the devices
* Then we gather results and errors
"""
afr = args_for_run
do_raise = error_catcher is None
error_catcher = [] if do_raise else error_catcher
broadcast_address = (
afr.default_broadcast if broadcast is True else broadcast
) or afr.default_broadcast
if isinstance(serials, SpecialReference):
try:
found, serials = await serials.find(afr, broadcast, kwargs.get("find_timeout", 5))
accept_found = True
except asyncio.CancelledError:
raise
except Exception as error:
if do_raise:
raise
hp.add_error(error_catcher, error)
return
# Work out what and where to send
# All the packets from here have targets on them
packets = self.make_packets(
afr
, serials
, broadcast
)
# Determine found
looked = False
found = found if found is not None else afr.found
if not accept_found and not broadcast:
looked, found, catcher = await self.search(
afr
, found
, packets
, broadcast_address
, kwargs.get("find_timeout", 20)
)
if catcher:
for error in set(catcher):
hp.add_error(error_catcher, error)
if do_raise:
throw_error(serials, error_catcher)
else:
return
# Work out where to send packets
if type(broadcast) is tuple:
addr = broadcast
else:
addr = None if broadcast is False else (broadcast_address, 56700)
# Create our helpers that channel particular arguments into the correct places
retry_options = afr.make_retry_options()
def check_packet(packet):
if packet.target is None or not looked or found and packet.target[:6] in found:
return
else:
return FailedToFindDevice(serial=packet.serial)
def make_waiter(writer):
return afr.make_waiter(writer, retry_options=retry_options)
async def make_writer(original, packet):
return await afr.make_writer(original, packet
, broadcast=broadcast, retry_options=retry_options
, addr=addr, found=found, **kwargs
)
writer_args = (
packets
, check_packet, make_writer, make_waiter
, kwargs.get("timeout", 10), error_catcher
)
# Finally use our message_writer helper to get us some results
async for thing in self.write_messages(*writer_args):
yield thing
if do_raise:
throw_error(serials, error_catcher)
