async def gathering(serials, kwargs):
class Done:
pass
ts = []
queue = asyncio.Queue()
for serial in serials:
ts.append(
hp.async_as_background(
self._follow(plans, serial, queue, **kwargs)))
def on_finish(res):
hp.async_as_background(queue.put(Done))
if not ts:
on_finish(None)
else:
t = hp.async_as_background(asyncio.wait(ts))
t.add_done_callback(on_finish)
while True:
nxt = await queue.get()
if nxt is Done:
break
yield nxt
async def run_with(self, references, args_for_run, **kwargs):
do_raise = kwargs.get("error_catcher") is None
error_catcher = [] if do_raise else kwargs["error_catcher"]
kwargs["error_catcher"] = error_catcher
queue = asyncio.Queue()
futs = []
if isinstance(references, SpecialReference):
try:
_, references = await references.find(
args_for_run, kwargs.get("broadcast", False),
kwargs.get("find_timeout", 5))
except asyncio.CancelledError:
raise
except Exception as error:
if do_raise:
raise
hp.add_error(error_catcher, error)
return
if type(references) is not list:
references = [references]
if not references:
references = [[]]
elif self.synchronized:
references = [references]
for reference in references:
futs.append(
hp.async_as_background(self.run_children(
queue, reference, args_for_run, **kwargs),
silent=True))
class Done:
pass
async def wait_all():
await asyncio.wait(futs, return_when=asyncio.ALL_COMPLETED)
await queue.put(Done)
waiter = hp.async_as_background(wait_all())
try:
while True:
nxt = await queue.get()
if nxt is Done:
break
else:
yield nxt
finally:
waiter.cancel()
for f in futs:
f.cancel()
if do_raise and error_catcher:
raise RunErrors(_errors=list(set(error_catcher)))
def on_done(packet, res):
if res.cancelled():
hp.add_error(
error_catcher,
TimedOut("Message was cancelled", serial=packet.serial))
else:
exc = res.exception()
if exc:
hp.add_error(error_catcher, exc)
if all(f.done() for f in fs):
hp.async_as_background(queue.put(Done))
def activate(*args):
self.activate_listeners()
if "stopped" not in info:
info["stopped"] = time.time()
self.removers.append(
hp.async_as_background(remove_after_a_minute()))
self.removers = [t for t in self.removers if not t.done()]
async def getter(self):
gen = self.item.generator(self.generator_reference, self.sender,
**self.kwargs)
complete = None
while True:
try:
msg = await gen.asend(complete)
if isinstance(msg, Exception):
hp.add_error(self.error_catcher, msg)
continue
if self.stop_fut.done():
break
complete = asyncio.Future()
f_for_items = []
for item in self.item.simplifier(msg):
f = asyncio.Future()
f_for_items.append(f)
t = hp.async_as_background(self.retrieve(item, f))
self.ts.append(t)
self.complete_on_all_done(f_for_items, complete)
self.ts = [t for t in self.ts if not t.done()]
except StopAsyncIteration:
break
await self.wait_for_ts()
def complete_on_all_done(self, fs, complete):
def finish(res):
if complete.done():
return
if res.cancelled():
for f in fs:
f.cancel()
complete.cancel()
return
for f in fs:
if f.cancelled():
complete.cancel()
return
exc = f.exception()
if exc is not None:
complete.set_result(False)
return
elif f.result() is False:
complete.set_result(False)
return
complete.set_result(True)
waiter = hp.async_as_background(asyncio.wait(fs))
waiter.add_done_callback(finish)
def update_found(self, found, query_new_devices=True):
"""
Update our idea of what devices on the network.
We delete information about devices that are no longer on the network.
"""
for target in list(self.by_target):
if target not in found:
del self.by_target[target]
if query_new_devices:
for target in found:
if target not in self.by_target:
t = hp.async_as_background(
self.device_finder_loops.add_new_device(target))
self.tasks_by_target[target] = t
t.add_done_callback(partial(self.cleanup_task, target, t))
# Make sure our targets are in by_target
# Must happen after calling add_new_device for those that aren't in by_target
for target in found:
self.by_target[target]
self.found.reset()
self.found.set_result(found)
def __await__(self):
# Protect against starting multiple writings tasks
if not hasattr(self, "_writings"):
self._writings = hp.async_as_background(self.writings())
self.write_tasks.append(self._writings)
return (yield from self.final_future)
def ensure_refresh_information_loop(self, sender, time_between_queries,
collections):
loop = getattr(self, "_refresh_information_loop", None)
if not loop or loop.done():
self._refresh_information_loop = hp.async_as_background(
self.refresh_information_loop(sender, time_between_queries,
collections))
return self._refresh_information_loop
def __aiter__(self):
self.getter_t = hp.async_as_background(self.getter())
def on_finish(res):
hp.async_as_background(self.queue.put(self.Done))
self.getter_t.add_done_callback(on_finish)
return self
def process(packet, res):
if res.cancelled():
e = TimedOut("Message was cancelled"
, serial=packet.serial
)
hp.add_error(error_catcher, e)
return
if not res.cancelled():
exc = res.exception()
if exc:
hp.add_error(error_catcher, exc)
full_number = len(gatherers) == len(writers)
futs_done = all(f.done() for f in futures)
gatherers_done = all(f.done() for f in gatherers)
if full_number and futs_done and gatherers_done:
hp.async_as_background(queue.put(Done))
async def change(self, serial, tile_index, left_x, top_y, target, afr):
if serial not in self.serials:
return {"serial": serial, "data": None}
user_x = (left_x + 4) / 8
user_y = (top_y - 4) / 8
msg = TileMessages.SetUserPosition(tile_index=tile_index,
user_x=user_x,
user_y=user_y,
res_required=False)
errors = []
await target.script(msg).run_with_all(serial,
afr,
error_catcher=errors,
message_timeout=5)
hp.async_as_background(
self.highlight(serial,
tile_index,
target,
afr,
error_catcher=[],
message_timeout=3))
if errors:
return {"serial": serial, "data": None}
plans = make_plans(chain=ChainPlan(refresh=True))
gatherer = Gatherer(target)
got = await gatherer.gather_all(plans,
serial,
afr,
error_catcher=errors,
message_timeout=5)
if serial in got and got[serial][0]:
pixel_coords = user_coords_to_pixel_coords(
got[serial][1]["chain"]["coords_and_sizes"])
self.serials[serial]["coords"] = [xy for xy, _ in pixel_coords]
return {"serial": serial, "data": self.info_for_browser(serial)}
async def create_receiver(self, conn, packet, addr):
if packet.target not in self.receivers:
async def receive():
while not self.stop_fut.finished():
if conn is True:
await asyncio.sleep(0.1)
else:
nxt = await conn[0].get()
self.received_data(nxt, addr, conn)
self.receivers[packet.target] = hp.async_as_background(receive())
async def cypress(typ):
try:
import asynctest # noqa
except ImportError:
raise PhotonsAppError(
'You must `pip install -e ".[tests]"` before you can run integration tests'
)
from whirlwind.test_helpers import free_port, port_connected
port = free_port()
env = {"CYPRESS_BASE_URL": f"http://127.0.0.1:{port}"}
final_future = collector.configuration["photons_app"].final_future
t = None
collector.configuration["interactor"].host = "127.0.0.1"
collector.configuration["interactor"].port = port
collector.configuration["interactor"].fake_devices = True
collector.configuration[
"interactor"].database.uri = "sqlite:///:memory:"
t = hp.async_as_background(serve(collector))
start = time.time()
while time.time() - start < 5:
if port_connected(port):
break
await asyncio.sleep(0.01)
if not port_connected(port):
raise PhotonsAppError("Failed to start server for tests")
loop = asyncio.get_event_loop()
def doit():
if os.environ.get("NO_BUILD_ASSETS") != "1":
log.info("Building assets")
assets.run("run-script", "build")
log.info("Running cypress")
assets.run("run-script", f"cypress:{typ}", extra_env=env)
try:
await loop.run_in_executor(None, doit)
finally:
exc_info = sys.exc_info()
photons_app = collector.configuration["photons_app"]
if exc_info[1]:
photons_app.graceful_final_future.set_exception(exc_info[1])
else:
photons_app.graceful_final_future.set_result(None)
await t
async def do_apply_theme(target, reference, afr, options):
aps = appliers[options.theme]
theme = Theme()
for color in options.colors:
theme.add_hsbk(color.hue, color.saturation, color.brightness,
color.kelvin)
tasks = []
async for pkt, _, _ in target.script(DeviceMessages.GetVersion()).run_with(
reference, afr):
serial = pkt.serial
capability = capability_for_ids(pkt.product, pkt.vendor)
if capability.has_multizone:
log.info(hp.lc("Found a strip", serial=serial))
t = hp.async_as_background(
apply_zone(aps["1d"], target, afr, pkt.serial, theme,
options.overrides))
elif capability.has_chain:
log.info(hp.lc("Found a tile", serial=serial))
t = hp.async_as_background(
apply_tile(aps["2d"], target, afr, pkt.serial, theme,
options.overrides))
else:
log.info(hp.lc("Found a light", serial=serial))
t = hp.async_as_background(
apply_light(aps["0d"], target, afr, pkt.serial, theme,
options.overrides))
tasks.append((serial, t))
results = {}
for serial, t in tasks:
try:
await t
except Exception as error:
results[serial] = error
else:
results[serial] = "ok"
return results
async def _get_response(self, packet, timeout, waiter, limit=None):
errf = hp.ResettableFuture()
errf.add_done_callback(hp.silent_reporter)
response = []
async def wait_for_responses():
async with (limit or NoLimit()):
if hasattr(asyncio, "current_task"):
current_task = asyncio.current_task()
else:
current_task = asyncio.Task.current_task()
asyncio.get_event_loop().call_later(timeout, timeout_task,
current_task, errf,
packet.serial)
try:
for info in await waiter:
response.append(info)
finally:
if hasattr(waiter, "finish"):
await waiter.finish()
f = hp.async_as_background(wait_for_responses(), silent=True)
def process(res):
if errf.done() and not errf.cancelled():
# Errf has an exception
return
elif res.cancelled():
errf.reset()
errf.set_exception(
TimedOut("Message was cancelled", serial=packet.serial))
return
if not res.cancelled():
exc = res.exception()
if exc:
errf.set_exception(exc)
return
errf.set_result(True)
f.add_done_callback(process)
try:
await errf
finally:
f.cancel()
return response
async def retrieve(self, streamer):
streamer.no_more_work()
started = hp.create_future()
async def retrieve():
results = []
started.set_result(True)
async for result in streamer:
results.append(result)
return results
return started, hp.async_as_background(retrieve())
async def execute(self):
ts = []
result = chp.ResultBuilder()
def get(db):
info = []
for scene in db.queries.get_scenes(uuid=self.uuid).all():
info.append(scene.as_object())
if not info:
raise NoSuchScene(uuid=self.uuid)
return info
for scene in await self.db_queue.request(get):
fltr = chp.filter_from_matcher(scene.matcher, False)
if scene.zones:
multizonefltr = self.clone_fltr_with_cap(fltr, "multizone")
ts.append(hp.async_as_background(self.apply_zones(multizonefltr, scene, result)))
notmultizonefltr = self.clone_fltr_with_no_cap(fltr, "multizone")
ts.append(hp.async_as_background(self.transform(notmultizonefltr, scene, result)))
elif scene.chain:
chainfltr = self.clone_fltr_with_cap(fltr, "chain")
ts.append(hp.async_as_background(self.apply_chain(chainfltr, scene, result)))
notchainfltr = self.clone_fltr_with_no_cap(fltr, "chain")
ts.append(hp.async_as_background(self.transform(notchainfltr, scene, result)))
else:
ts.append(hp.async_as_background(self.transform(fltr, scene, result)))
for t in ts:
try:
await t
except Exception as error:
result.error(error)
return result
def connect(self):
recvQueue = asyncio.Queue()
resQueue = asyncio.Queue()
async def receive():
while True:
got = Messages.unpack(await recvQueue.get(), None)
self.received.append(got)
if got.ack_required:
await resQueue.put(got.make_ack().tobytes())
if got.res_required:
for res in got.make_res():
await resQueue.put(res.tobytes())
self.connections.append(hp.async_as_background(receive()))
return resQueue, recvQueue
async def reboot(self):
if callable(self.pre_reboot):
await self.pre_reboot(self)
self.attrs.online = False
self.reboots.append(time.time())
self.wait_for_reboot_fut.reset()
for responder in self.all_responders:
if hasattr(responder, "shutdown"):
await responder.shutdown(self)
async def back_online(time_rebooting, power_on):
await asyncio.sleep(time_rebooting)
await power_on()
if not self.wait_for_reboot_fut.done():
self.wait_for_reboot_fut.set_result(True)
if self.time_rebooting >= 0:
hp.async_as_background(
back_online(self.time_rebooting, self.power_on))
else:
self.wait_for_reboot_fut.set_result(True)
async def leave_arrange(self, ref, target, afr):
log.info(hp.lc("Leaving arrange", ref=ref))
tasks = []
for serial, info in list(self.serials.items()):
info["refs"] = [r for r in info["refs"] if r != ref]
if not info["refs"]:
tasks.append(
hp.async_as_background(
self.restore(serial, info["initial"], target, afr)))
del self.serials[serial]
for t in tasks:
await t
def _spawn(self, address, fut, backoff, timeout):
"""Spawn the actual connection"""
sock = self.make_socket(address)
t = hp.async_as_background(self.connect_socket(sock, address, fut, backoff, timeout))
def pass_failure(res):
if fut.done():
return
if res.cancelled():
fut.cancel()
return
exc = res.exception()
if exc is not None:
fut.set_exception(exc)
t.add_done_callback(pass_failure)
async def start_arrange(self, serials, ref, target, afr):
log.info(hp.lc("Starting arrange", serials=serials, ref=ref))
all_errors = []
for serial in list(self.serials):
if serial not in serials:
del self.serials[serial]
for serial in serials:
tasks = []
info = {"refs": [], "highlightlock": asyncio.Lock()}
if serial in self.serials:
info = self.serials[serial]
tasks.append(
(serial,
hp.async_as_background(self.start(serial, info, target,
afr))))
for serial, t in tasks:
try:
errors, data = await t
except Exception as error:
errors = [error]
if errors:
all_errors.extend(errors)
else:
info.update(data)
self.serials[serial] = info
final = {"serials": {}}
if all_errors:
final["error"] = ", ".join(str(e) for e in all_errors)
log.error(hp.lc("Failed to start arrange", errors=all_errors))
for serial in serials:
if serial in self.serials:
if ref not in self.serials[serial]["refs"]:
self.serials[serial]["refs"].append(ref)
final["serials"][serial] = self.info_for_browser(serial)
return final
async def args_for_run(self):
"""Return an afr object. Multiple calls to this will return the same object"""
if not hasattr(self, "afr_fut"):
self.afr_fut = asyncio.Future()
t = hp.async_as_background(self.target.args_for_run())
def transfer(res):
if res.cancelled():
self.afr_fut.cancel()
exc = res.exception()
if exc:
self.afr_fut.set_exception(exc)
self.afr_fut.set_result(res.result())
t.add_done_callback(transfer)
return await self.afr_fut
async def finish(self, exc_typ=None, exc=None, tb=None):
await super().finish(exc_typ, exc, tb)
ts = [
hp.async_as_background(t.close())
for t in self.broadcast_transports.values()
]
await hp.cancel_futures_and_wait(
*ts,
name=f"{type(self).__name__}::finish[wait_for_broadcast_transports]"
)
for t in ts:
if not t.cancelled():
exc = t.exception()
if exc:
log.error(
hp.lc("Failed to close broadcast transport",
error=exc))
async def start(self):
self.port = self.kwargs.get("port", None) or pytest.helpers.free_port()
await pytest.helpers.wait_for_no_port(self.port)
self.options = make_options(**{**self.kwargs, "port": self.port})
self._task = hp.async_as_background(
self.server.serve(
"127.0.0.1",
self.options.port,
self.options,
tasks=self.ts,
sender=self.sender,
cleaners=self.cleaners,
animation_options=self.animation_options,
))
await pytest.helpers.wait_for_port(self.port)
return self
async def spawn(self, packet, *, timeout=10, create=True):
if self.transport is not None:
if self.transport.done() and (self.transport.cancelled()
or self.transport.exception()):
self.transport = None
if self.transport is not None and self.transport.done():
if not await self.is_transport_active(packet,
self.transport.result()):
self.transport = None
if not create and (self.transport is None
or not self.transport.done()):
return None
if self.transport is None:
self.transport = hp.async_as_background(
self.spawn_transport(timeout))
return await self.transport
async def spawn(self, packet, *, timeout=10, create=True):
if self.transport is not None:
if self.transport.done() and (self.transport.cancelled()
or self.transport.exception()):
self.transport = None
if self.transport is not None and self.transport.done():
if not await self.is_transport_active(packet,
self.transport.result()):
self.transport = None
if not create and (self.transport is None
or not self.transport.done()):
return None
if self.transport is None:
self.transport = hp.create_future(
name=
f"Transport::{self.session.__class__.__name__}::spawn[transport]"
)
t = hp.async_as_background(self.spawn_transport(timeout))
t.add_done_callback(hp.transfer_result(self.transport))
return await self.transport
async def write_messages(self, sender, packets, kwargs):
"""Send all our packets and collect all the results"""
fs = []
queue = asyncio.Queue()
error_catcher = kwargs["error_catcher"]
def on_done(packet, res):
if res.cancelled():
hp.add_error(
error_catcher,
TimedOut("Message was cancelled", serial=packet.serial))
else:
exc = res.exception()
if exc:
hp.add_error(error_catcher, exc)
if all(f.done() for f in fs):
hp.async_as_background(queue.put(Done))
for original, packet in packets:
coro = self.do_send(sender, original, packet, queue, kwargs)
f = hp.async_as_background(coro, silent=True)
f.add_done_callback(partial(on_done, packet))
fs.append(f)
try:
while True:
msg = await queue.get()
if msg is Done:
break
yield msg
finally:
for f in fs:
f.cancel()
await asyncio.sleep(2)
except asyncio.CancelledError:
assert False, "Expected it to just raise the error rather than cancelling first"
start = time.time()
with assertRaises(PhotonsAppError, "Blah"):
await hp.async_with_timeout(func(), timeout=0.1, timeout_error=error)
assert time.time() - start < 0.5
describe "async_as_background":
async it "runs the coroutine in the background":
async def func(one, two, three=None):
return "{0}.{1}.{2}".format(one, two, three)
t = hp.async_as_background(func(6, 5, three=9))
pytest.helpers.assertFutCallbacks(t, hp.reporter)
assert isinstance(t, asyncio.Task)
assert await t == "6.5.9"
async it "uses silent_reporter if silent is True":
async def func(one, two, three=None):
return "{0}.{1}.{2}".format(one, two, three)
t = hp.async_as_background(func(6, 5, three=9), silent=True)
pytest.helpers.assertFutCallbacks(t, hp.silent_reporter)
assert isinstance(t, asyncio.Task)
assert await t == "6.5.9"
describe "silent_reporter":
