def initialize(self, *, path=None, sock=None, shutdown_timeout=60,
ssl_context=None, backlog=128, access_log_class=AccessLogger,
access_log_format=AccessLogger.LOG_FORMAT,
access_log=AccessLogger.access_logger, handle_signals=True,
reuse_address=None, reuse_port=None):
# 初始化模板
self._initialize_template()
# 初始化session
self._initialize_session()
# 初始化csrf
self._initialize_csrf()
self._initialize_runner(handle_signals, access_log_class,
access_log_format, access_log)
sites = []
try:
sites.append(web.TCPSite(runner, self.app.config["APPS"][self.app_name]['host'],
self.app.config["APPS"][
self.app_name]['port'],
shutdown_timeout=shutdown_timeout,
ssl_context=ssl_context,
backlog=backlog,
reuse_address=reuse_address,
reuse_port=reuse_port))
if path is not None:
if isinstance(path, (str, bytes, bytearray, memoryview)):
sites.append(web.UnixSite(runner, path,
shutdown_timeout=shutdown_timeout,
ssl_context=ssl_context,
backlog=backlog))
else:
for p in path:
sites.append(web.UnixSite(runner, p,
shutdown_timeout=shutdown_timeout,
ssl_context=ssl_context,
backlog=backlog))
if sock is not None:
if not isinstance(sock, Iterable):
sites.append(web.SockSite(runner, sock,
shutdown_timeout=shutdown_timeout,
ssl_context=ssl_context,
backlog=backlog))
else:
for s in sock:
sites.append(web.SockSite(runner, s,
shutdown_timeout=shutdown_timeout,
ssl_context=ssl_context,
backlog=backlog))
for site in sites:
self.loop.run_until_complete(site.start())
finally:
self.loop.run_until_complete(runner.cleanup())
self.show_info()
async def serve(app):
runner = web.AppRunner(app)
await runner.setup()
try:
# Try and open up the unix socket passed to us from systemd
site = web.SockSite(
runner, socket.fromfd(3, socket.AF_UNIX, socket.SOCK_STREAM))
await site.start()
logging.info("Using systemd socket activation")
except:
# We must be running in some other setup - bind the socket ourselves
site = web.UnixSite(runner, path='/run/docker/plugins/vlad.sock')
await site.start()
logging.info("Trying to open the plugin socket...")
try:
while True:
await asyncio.sleep(3600) # sleep forever by 1 hour intervals
except KeyboardInterrupt:
print("Got CTRL+C, shutting down...")
except Exception as e:
print(e)
finally:
await runner.cleanup()
# If we created the unix socket, clean it up
if isinstance(site, web.UnixSite):
os.unlink('/run/docker/plugins/vlad.sock')
async def runme():
loop = asyncio.get_event_loop()
signals = (signal.SIGHUP, signal.SIGTERM, signal.SIGINT)
for sig in signals:
loop.add_signal_handler(
sig, lambda sig=sig: asyncio.create_task(shutdown()))
# Make sure the notebook cells table exists
async with connect_to_database() as db:
await db.execute("PRAGMA foreign_keys=1")
hastable = True
async with db.execute(
"SELECT name FROM sqlite_master WHERE type='table' AND name='notebook_cells';"
) as cursor:
if await cursor.fetchone() is None:
hastable = False
if not hastable:
await db.execute(schema)
await db.commit()
# Runs the server over a unix domain socket. The socket is automatically placed in the data folder,
# and not the plugin folder.
runner = web.AppRunner(app)
await runner.setup()
site = web.UnixSite(runner, path=f"{p.name}.sock")
await site.start()
l.info("Notebook plugin ready")
def run(self):
self.loop = self.__loop = asyncio.get_event_loop()
if self.loop_monitor:
self.__loop.set_debug(True)
# loop.slow_callback_duration(0.2)
# Needs to happen after setting debug or may cause race condition
# http://bugs.python.org/issue30805
self.__loop.run_until_complete(self.__plugins_load())
if self.loop_monitor:
# Start monitor thread after plugins have been loaded
# because of the time spent doing I/O
t = threading.Thread(target=self._loop_monitor_thread)
t.setDaemon(True)
t.start()
self.__loop.add_signal_handler(signal.SIGINT, self.terminate)
self.__loop.add_signal_handler(signal.SIGTERM, self.terminate)
self.__loop.add_signal_handler(signal.SIGUSR1, self.pdb)
app = web.Application(loop=self.__loop)
app.router.add_route('GET', '/websocket', self.ws_handler)
app.router.add_route("*", "/api/docs{path_info:.*}", WSGIHandler(apidocs_app))
self.fileapp = FileApplication(self, self.__loop)
app.router.add_route('*', '/_download{path_info:.*}', self.fileapp.download)
app.router.add_route('*', '/_upload{path_info:.*}', self.fileapp.upload)
shellapp = ShellApplication(self)
app.router.add_route('*', '/_shell{path_info:.*}', shellapp.ws_handler)
restful_api = RESTfulAPI(self, app)
self.__loop.run_until_complete(
asyncio.ensure_future(restful_api.register_resources())
)
asyncio.ensure_future(self.jobs.run())
self.__setup_periodic_tasks()
# Start up middleware worker process pool
self.__procpool._start_queue_management_thread()
runner = web.AppRunner(app, handle_signals=False, access_log=None)
self.__loop.run_until_complete(runner.setup())
self.__loop.run_until_complete(
web.TCPSite(runner, '0.0.0.0', 6000, reuse_address=True, reuse_port=True).start()
)
self.__loop.run_until_complete(web.UnixSite(runner, '/var/run/middlewared.sock').start())
self.logger.debug('Accepting connections')
try:
self.__loop.run_forever()
except RuntimeError as e:
if e.args[0] != "Event loop is closed":
raise
async def start():
runner = web.AppRunner(app)
await runner.setup()
if app_path is not None:
site = web.UnixSite(runner, app_path)
else:
site = web.TCPSite(runner, listen_host, listen_port)
await site.start()
async def make_server_at_path(socket_path, endpoint, controller):
app = web.Application()
bind(app.router, endpoint, controller)
runner = web.AppRunner(app)
await runner.setup()
site = web.UnixSite(runner, socket_path)
await site.start()
return site
async def test_addresses(make_runner, tmpdir):
runner = make_runner()
await runner.setup()
tcp = web.TCPSite(runner)
await tcp.start()
path = str(tmpdir / 'tmp.sock')
unix = web.UnixSite(runner, path)
await unix.start()
addrs = runner.addresses
assert addrs == [('0.0.0.0', mock.ANY), path]
async def make_server_at_path(socket_path, endpoint, controller, **kw):
app = web.Application(**kw)
bind(app.router, endpoint, controller)
runner = web.AppRunner(app)
await runner.setup()
site = web.UnixSite(runner, socket_path)
await site.start()
# It is intended that a non-root client can connect.
os.chmod(socket_path, 0o666)
return site
async def start(self):
await self._runner.setup()
if "path" in self.config:
log.debug("Starting server on socket %s", self.config["path"])
self._site = web.UnixSite(self._runner, self.config["path"])
else:
log.debug("Starting server on host %s, port %d",
self.config["host"], self.config["port"])
self._site = web.TCPSite(self._runner, self.config["host"],
self.config["port"])
await self._site.start()
async def start_api_server(self):
app = web.Application(middlewares=[self.middleware])
bind(app.router, API.meta, MetaController(self))
bind(app.router, API.errors, ErrorController(self))
if self.opts.dry_run:
from .dryrun import DryRunController
bind(app.router, API.dry_run, DryRunController(self))
runner = web.AppRunner(app)
await runner.setup()
site = web.UnixSite(runner, self.opts.socket)
await site.start()
async def test_addresses(make_runner, unix_sockname) -> None:
_sock = get_unused_port_socket("127.0.0.1")
runner = make_runner()
await runner.setup()
tcp = web.SockSite(runner, _sock)
await tcp.start()
unix = web.UnixSite(runner, unix_sockname)
await unix.start()
actual_addrs = runner.addresses
expected_host, expected_post = _sock.getsockname()[:2]
assert actual_addrs == [(expected_host, expected_post), unix_sockname]
async def test_addresses(make_runner, shorttmpdir) -> None:
_sock = get_unused_port_socket('127.0.0.1')
runner = make_runner()
await runner.setup()
tcp = web.SockSite(runner, _sock)
await tcp.start()
path = str(shorttmpdir / 'tmp.sock')
unix = web.UnixSite(runner, path)
await unix.start()
actual_addrs = runner.addresses
expected_host, expected_post = _sock.getsockname()[:2]
assert actual_addrs == [(expected_host, expected_post), path]
async def runme():
await p.fire({
"user": "******",
"event": "LOL"
})
# Runs the server over a unix domain socket. The socket is automatically placed in the data folder,
# and not the plugin folder.
runner = web.AppRunner(app)
await runner.setup()
site = web.UnixSite(runner, path=f"{p.name}.sock")
await site.start()
print("Plugin Ready")
def web_site_from_url(runner: web.AppRunner, url: str) -> web.BaseSite:
parsed = urlparse(url)
if parsed.scheme == "http":
assert parsed.hostname is not None
assert parsed.port is not None
return web.TCPSite(runner, parsed.hostname, parsed.port)
elif parsed.scheme == "unix":
return web.UnixSite(runner, parsed.path)
else:
logger.warning(
"Ignoring web listen url %s: scheme %r not supported",
url,
parsed.scheme,
)
raise ValueError(url)
async def start_runner(runner, conf: config.Config):
await runner.setup()
site: Optional[Union[web.TCPSite, web.UnixSite, web.SockSite]] = None
if conf.port is not None:
print(f"Starting HTTP server on localhost:{conf.port}")
site = web.TCPSite(runner, 'localhost', conf.port)
elif conf.socket is not None:
print(f"Starting UNIX socket server on {conf.socket}")
site = web.UnixSite(runner, conf.socket)
elif conf.socket_fd is not None:
print(f"Starting UNIX socket server on FD {conf.socket_fd}")
sock = socket.socket(fileno=conf.socket_fd)
site = web.SockSite(runner, sock)
assert site is not None, "Invalid config, no listening address"
return await site.start()
async def start_api_server(self):
app = web.Application(middlewares=[self.middleware])
bind(app.router, API.meta, MetaController(self))
bind(app.router, API.errors, ErrorController(self))
if self.opts.dry_run:
from .dryrun import DryRunController
bind(app.router, API.dry_run, DryRunController(self))
for controller in self.controllers.instances:
controller.add_routes(app)
runner = web.AppRunner(app)
await runner.setup()
site = web.UnixSite(runner, self.opts.socket)
await site.start()
# It is intended that a non-root client can connect.
os.chmod(self.opts.socket, 0o666)
async def main(is_port: bool, destination):
logging.basicConfig(level=logging.WARNING)
app = init_app()
runner = web.AppRunner(app)
await runner.setup()
if is_port:
site = web.TCPSite(runner, port=destination)
else:
site = web.UnixSite(runner, path=destination)
await site.start()
await games_cleanup_routine()
while True:
await asyncio.sleep(5) # sleep forever
async def start_webapp(webapp, config):
logger.warning("CONFIG: %s", config)
runner = web.AppRunner(webapp)
await runner.setup()
if 'http_host' in config or 'http_port' in config:
tcp_site = web.TCPSite(runner, config.get('http_host'),
config.get('http_port'))
logger.info("Will start http on: %s", tcp_site.name)
await tcp_site.start()
if 'unix_socket' in config:
unix_socket = Path(config['unix_socket']).absolute()
unix_site = web.UnixSite(runner, unix_socket)
logger.info("Will start unix on: %s", unix_site.name)
await unix_site.start()
return runner
async def start():
runner = web.AppRunner(app)
tasks = []
await runner.setup()
if app_path is not None:
site = web.UnixSite(runner, app_path)
else:
site = web.TCPSite(runner, listen_host, listen_port)
tasks.append(site.start())
# Websocket
log.server_logger.info(
f"Attempting to open WS connection to {options.websocket_url}")
ws = WebsocketClient(app, options.websocket_url, callback_ws)
await ws.setup()
tasks.append(ws.loop())
await asyncio.wait(tasks)
async def simpleWebServer ():
with TemporaryDirectory () as socketDir:
app = web.Application()
app.add_routes([web.get('/', hello)])
app.add_routes([web.get('/large', largeResponse)])
socketPath = os.path.join (socketDir, 'socket')
runner = web.AppRunner(app)
await runner.setup()
site = web.UnixSite (runner, socketPath)
await site.start ()
yield socketPath, runner
# destroy application
await runner.cleanup()
async def run(self):
self.runner = web.AppRunner(self.app)
await self.runner.setup()
if self.socktype == 'unix':
site = web.UnixSite(self.runner, self.sockpath)
elif self.socktype == 'tcp':
site = web.TCPSite(self.runner, self.host, self.port)
else:
raise Exception(f"unknown socktype '{self.socktype!r}'")
await site.start()
socket = site._server.sockets[0]
if self.socktype == 'unix':
addr = self.sockpath
elif self.socktype == 'tcp':
addr = socket.getsockname()
else:
raise Exception(f"impossible socktype ({self.socktype!r})")
os.write(self.fd, bytes(repr((self.socktype, addr, time.time())), 'utf8'))
os.close(self.fd)
self.logger.info(f"now running and listening. socktype={self.socktype}, addr={addr}")
cors.add(control.add_route('POST', player.post_control))
player_app.router.add_route('GET', '/ping', ping)
player_app.router.add_routes([
web.get("/auth", auth.get_is_valid_invite),
web.get("/auth/{provider}/register", auth.get_register_url),
web.get("/auth/{provider}/login", auth.get_login_url),
web.get("/auth/{provider}/register/callback", auth.get_register_callback),
web.get("/auth/{provider}/login/callback", auth.get_login_callback)
])
player_app_runner = web.AppRunner(player_app)
stream = StreamView(config, player)
stream_app = web.Application()
stream_app.add_routes([web.get('/', stream.get_ws)])
stream_app_runner = web.AppRunner(stream_app)
loop.run_until_complete(player_app_runner.setup())
loop.run_until_complete(stream_app_runner.setup())
sites = []
sites.append(web.UnixSite(player_app_runner, config.player_socket))
sites.append(web.UnixSite(stream_app_runner, config.stream_socket))
for site in sites:
loop.run_until_complete(site.start())
log.info('Starting server...')
loop.run_forever()
def start(handle_func, uds, loop=None):
log.log("in http_stream.start")
app = setup_unix_server(handle_func, loop=loop)
socket_path = str(uds).lstrip("unix:")
if asyncio.iscoroutine(app):
app = loop.run_until_complete(app)
log.log("socket file exist? - {0}"
.format(os.path.exists(socket_path)))
app_runner = web.AppRunner(
app, handle_signals=True,
access_log=log.get_logger())
# setting up app runner
log.log("setting app_runner")
loop.run_until_complete(app_runner.setup())
# try to remove pre-existing UDS: ignore errors here
socket_dir, socket_file = os.path.split(socket_path)
phony_socket_path = os.path.join(
socket_dir, "phony" + socket_file)
log.log("deleting socket files if they exist")
try:
os.remove(socket_path)
os.remove(phony_socket_path)
except (FileNotFoundError, Exception, BaseException):
pass
log.log("starting unix socket site")
uds_sock = web.UnixSite(
app_runner, phony_socket_path,
shutdown_timeout=0.1)
loop.run_until_complete(uds_sock.start())
try:
try:
log.log("CHMOD 666 {0}".format(phony_socket_path))
os.chmod(phony_socket_path, 0o666)
log.log("phony socket permissions: {0}"
.format(oct(os.stat(phony_socket_path).st_mode)))
log.log("sym-linking {0} to {1}".format(
socket_path, phony_socket_path))
os.symlink(os.path.basename(phony_socket_path), socket_path)
log.log("socket permissions: {0}"
.format(oct(os.stat(socket_path).st_mode)))
except (Exception, BaseException) as ex:
log.log(str(ex))
raise ex
try:
log.log("starting infinite loop")
loop.run_forever()
except web.GracefulExit:
pass
finally:
loop.run_until_complete(app_runner.cleanup())
if hasattr(loop, 'shutdown_asyncgens'):
loop.run_until_complete(loop.shutdown_asyncgens())
loop.close()
async def _run_app(
self,
*,
host: Optional[Union[str, web.HostSequence]] = None,
port: Optional[int] = None,
path: Optional[str] = None,
sock: Optional[socket.socket] = None,
shutdown_timeout: float = 60.0,
ssl_context: Optional[SSLContext] = None,
backlog: int = 128,
access_log_class: Type[web.AbstractAccessLogger] = web.AccessLogger,
access_log_format: str = web.AccessLogger.LOG_FORMAT,
access_log: Optional[logging.Logger] = web.access_logger,
handle_signals: bool = True,
reuse_address: Optional[bool] = None,
reuse_port: Optional[bool] = None,
) -> None:
# This function is pulled from aiohttp.web._run_app
app = self
runner = web.AppRunner(
app,
handle_signals=handle_signals,
access_log_class=access_log_class,
access_log_format=access_log_format,
access_log=access_log,
)
await runner.setup()
sites = []
try:
if host is not None:
if isinstance(host, (str, bytes, bytearray, memoryview)):
sites.append(
web.TCPSite(
runner,
host,
port,
shutdown_timeout=shutdown_timeout,
ssl_context=ssl_context,
backlog=backlog,
reuse_address=reuse_address,
reuse_port=reuse_port,
))
else:
for h in host:
sites.append(
web.TCPSite(
runner,
h,
port,
shutdown_timeout=shutdown_timeout,
ssl_context=ssl_context,
backlog=backlog,
reuse_address=reuse_address,
reuse_port=reuse_port,
))
elif path is None and sock is None or port is not None:
sites.append(
web.TCPSite(
runner,
port=port,
shutdown_timeout=shutdown_timeout,
ssl_context=ssl_context,
backlog=backlog,
reuse_address=reuse_address,
reuse_port=reuse_port,
))
if path is not None:
if isinstance(path, (str, bytes, bytearray, memoryview)):
sites.append(
web.UnixSite(
runner,
path,
shutdown_timeout=shutdown_timeout,
ssl_context=ssl_context,
backlog=backlog,
))
else:
for p in path:
sites.append(
web.UnixSite(
runner,
p,
shutdown_timeout=shutdown_timeout,
ssl_context=ssl_context,
backlog=backlog,
))
if sock is not None:
if not isinstance(sock, Iterable):
sites.append(
web.SockSite(
runner,
sock,
shutdown_timeout=shutdown_timeout,
ssl_context=ssl_context,
backlog=backlog,
))
else:
for s in sock:
sites.append(
web.SockSite(
runner,
s,
shutdown_timeout=shutdown_timeout,
ssl_context=ssl_context,
backlog=backlog,
))
for site in sites:
await site.start()
names = sorted(str(s.name) for s in runner.sites)
logger.debug("Running EventSub server on {}".format(
", ".join(names)))
await self._closing.wait()
finally:
await runner.cleanup()
async def run(self):
# initialize streams in the data store and event store
try:
pool = await self.data_store.initialize(
self.db.query(DataStream).all())
await self.event_store.initialize(pool)
except DataError as e:
log.error("Database error: %s" % e)
exit(1)
# start the supervisor (runs the workers)
await self.supervisor.start()
# start inserters by subscribing to the streams
inserter_tasks = []
for stream in self.db.query(DataStream).filter(
DataStream.keep_us != DataStream.KEEP_NONE):
if not stream.is_destination:
continue # only subscribe to active streams
try:
task = asyncio.create_task(self._spawn_inserter(stream))
task.set_name("Daemon: spawn_inserter [%s]" % stream.name)
inserter_tasks.append(task)
except SubscriptionError as e:
logging.warning(e)
inserter_task_grp = asyncio.gather(*inserter_tasks)
# start the API server
middlewares = [
joule.middleware.sql_rollback,
joule.middleware.authorize(
exemptions=joule.controllers.insecure_routes),
]
app = web.Application(middlewares=middlewares)
app['module-connection-info'] = self.module_connection_info
app['supervisor'] = self.supervisor
app['data-store'] = self.data_store
app['event-store'] = self.event_store
app['db'] = self.db
# used to tell master's this node's info
# if the API is proxied the base_uri
# will be retrieved from the X-Api-Base-Uri header
app['base_uri'] = ""
# if the API is proxied the port
# will be retrieved from the X-Api-Port header
app['name'] = self.config.name
app['port'] = self.config.port
# note, if the API is proxied the scheme
# will be retrieved from the X-Api-Scheme header
if self.ssl_context is None:
app['scheme'] = 'http'
else:
app['scheme'] = 'https'
# for acting as a client when accessing remote streams and joining other nodes
app['cafile'] = self.cafile
app.add_routes(joule.controllers.routes)
runner = web.AppRunner(app)
await runner.setup()
if self.config.ip_address is not None:
site = web.TCPSite(runner,
self.config.ip_address,
self.config.port,
ssl_context=self.ssl_context)
await site.start()
sock_file = os.path.join(self.config.socket_directory, 'api')
sock_site = web.UnixSite(runner, sock_file)
await sock_site.start()
os.chmod(sock_file, 0o600)
# sleep and check for stop condition
while not self.stop_requested:
await asyncio.sleep(0.5)
# clean everything up
await self.supervisor.stop()
inserter_task_grp.cancel()
try:
await inserter_task_grp
except asyncio.CancelledError:
pass
await self.data_store.close()
try:
await asyncio.wait_for(runner.shutdown(), 5)
await asyncio.wait_for(runner.cleanup(), 5)
except asyncio.TimeoutError:
log.warning("unclean server shutdown, subscribed clients?")
self.db.close()
def test_passes_data_across_pipes(self):
loop = asyncio.get_event_loop()
# create worker connections
# child runs until stopped
self.module.exec_cmd = "/usr/bin/env python3 " + MODULE_SIMPLE_FILTER
interval1_data = helpers.create_data('float32_3', start=1000, step=100, length=100)
interval2_data = helpers.create_data('float32_3', start=1001 + 100 * 100, step=100, length=100)
# create a stub server to respond to API calls as the module starts up
app = web.Application()
node_stream_info_api_call_count = 0
async def stub_stream_info(request):
nonlocal node_stream_info_api_call_count
node_stream_info_api_call_count += 1
stream_id = int(request.query['id'])
return web.json_response(self.streams[stream_id].to_json())
app.add_routes([web.get('/stream.json', stub_stream_info)])
runner = web.AppRunner(app)
loop.run_until_complete(runner.setup())
tmp_dir = tempfile.TemporaryDirectory()
sock_file = os.path.join(tmp_dir.name, 'testing')
sock_site = web.UnixSite(runner, sock_file)
loop.run_until_complete(sock_site.start())
self.worker.API_SOCKET = sock_file
async def mock_producers():
# await asyncio.sleep(0.5)
subscribers = self.producers[0].subscribers[self.streams[0]]
while len(subscribers) == 0:
await asyncio.sleep(0.01)
# add two intervals of mock data to the producer queues
input1 = subscribers[0] # self.producers[0].subscribers[self.streams[0]][0]
await input1.write(interval1_data)
await input1.close_interval()
await input1.write(interval2_data)
await input1.close_interval()
await input1.close()
input2 = self.producers[1].subscribers[self.streams[1]][0]
await input2.write(interval1_data)
await input2.close_interval()
await input2.write(interval2_data)
await input2.close_interval()
await input2.close()
await asyncio.sleep(2)
# subscribe to the module outputs
output1 = pipes.LocalPipe(layout=self.streams[2].layout, name="output1", debug=False)
output2 = pipes.LocalPipe(layout=self.streams[3].layout, name="output2", debug=False)
# create a slow subscriber that times out
class SlowPipe(pipes.Pipe):
async def write(self, data):
await asyncio.sleep(10)
async def close_interval(self):
pass
slow_pipe = SlowPipe(stream=helpers.create_stream('slow stream', self.streams[2].layout), name='slow pipe')
self.worker.subscribe(self.streams[2], slow_pipe)
self.worker.SUBSCRIBER_TIMEOUT = 0.1
# create a subscriber that errors out
class ErrorPipe(pipes.Pipe):
async def write(self, data):
raise BrokenPipeError()
error_pipe = ErrorPipe(stream=helpers.create_stream('error stream', self.streams[2].layout), name='error pipe')
self.worker.subscribe(self.streams[3], error_pipe)
self.worker.subscribe(self.streams[3], output2)
self.worker.subscribe(self.streams[2], output1)
with self.assertLogs() as log:
loop.run_until_complete(asyncio.gather(
self.worker.run(self.supervisor.subscribe, restart=False),
mock_producers()))
loop.run_until_complete(asyncio.gather(
runner.shutdown(),
runner.cleanup()
))
# make sure the module queried the API endpoint for each stream
self.assertEqual(node_stream_info_api_call_count, 4)
tmp_dir.cleanup() # remove socket file and directory
log_dump = '\n'.join(log.output)
self.assertIn("subscriber write error", log_dump)
self.assertIn("timed out", log_dump)
# check stream2, should be stream0*2.0 [] stream0*2.0
output_data = output1.read_nowait()
output1.consume(len(output_data))
np.testing.assert_array_almost_equal(interval1_data['data'] * 2.0,
output_data['data'])
self.assertTrue(output1.end_of_interval)
output_data = output1.read_nowait()
output1.consume(len(output_data))
np.testing.assert_array_almost_equal(interval2_data['data'] * 2.0,
output_data['data'])
self.assertTrue(output1.end_of_interval)
# check stream3, should be stream1*3.0 [] stream1*3.0
output_data = output2.read_nowait()
output2.consume(len(output_data))
np.testing.assert_array_almost_equal(interval1_data['data'] * 3.0,
output_data['data'])
self.assertTrue(output2.end_of_interval)
output_data = output2.read_nowait()
output2.consume(len(output_data))
np.testing.assert_array_almost_equal(interval2_data['data'] * 3.0,
output_data['data'])
self.assertTrue(output2.end_of_interval)
