def run(self):
set_thread_name('mdns_discover')
if not mDNSDaemonMonitor.instance.mdnsd_running.wait(timeout=10):
return
sdRef = pybonjour.DNSServiceBrowse(
regtype=self.regtype,
callBack=self.on_discover
)
while True:
r, w, x = select.select([sdRef, self.pipe[0]], [], [])
if self.pipe[0] in r:
break
for ref in r:
try:
pybonjour.DNSServiceProcessResult(ref)
# mdnsd was probaly killed, so just continue since this thread will be restarted
except pybonjour.BonjourError:
continue
if self.finished.is_set():
break
if self.active(sdRef):
sdRef.close()
def run(self):
set_thread_name('mdns_resolve')
while True:
if not self.mdnsd.is_alive():
self.mdnsd.wait(timeout=10)
try:
obj = self.queue.get(block=True, timeout=self.timeout)
except queue.Empty:
if self.finished.is_set():
break
continue
sdRef = pybonjour.DNSServiceResolve(flags=obj.flags,
interfaceIndex=obj.interface,
name=obj.name,
regtype=obj.regtype,
domain=obj.domain,
callBack=self.on_resolve)
self.references.append(sdRef)
_references = list(filter(self.active, self.references))
r, w, x = select.select(_references, [], [])
for ref in r:
pybonjour.DNSServiceProcessResult(ref)
if self.finished.is_set():
break
for ref in self.references:
self.references.remove(ref)
if self.active(ref):
ref.close()
def run(self):
set_thread_name('mdns_discover')
if not self.mdnsd.is_alive():
self.mdnsd.wait(timeout=10)
sdRef = pybonjour.DNSServiceBrowse(regtype=self.regtype,
callBack=self.on_discover)
while True:
r, w, x = select.select([sdRef, self.pipe[0]], [], [])
if self.pipe[0] in r:
break
for ref in r:
try:
pybonjour.DNSServiceProcessResult(ref)
# mdnsd was probaly killed, so just continue since this thread will be restarted
except pybonjour.BonjourError:
continue
if self.finished.is_set():
break
if self.active(sdRef):
sdRef.close()
def run(self):
set_thread_name(f'mdns_svc_{self.service}')
try:
self.register()
except pybonjour.BonjourError:
self.logger.debug(
"ServiceThread: failed to register '%s', is mdnsd running?",
self.service)
def _target(self):
set_thread_name(self.name)
try:
while True:
work_item = self.executor.get_work_item(self)
if work_item is None:
return
work_item.run()
del work_item
except Exception:
logger.critical("Exception in worker", exc_info=True)
finally:
self.executor.remove_worker(self)
def __init__(self,
loop_debug=False,
loop_monitor=True,
overlay_dirs=None,
debug_level=None):
self.logger = logger.Logger('middlewared', debug_level).getLogger()
self.crash_reporting = logger.CrashReporting()
self.crash_reporting_semaphore = asyncio.Semaphore(value=2)
self.loop_debug = loop_debug
self.loop_monitor = loop_monitor
self.overlay_dirs = overlay_dirs or []
self.__loop = None
self.__thread_id = threading.get_ident()
# Spawn new processes for ProcessPool instead of forking
multiprocessing.set_start_method('spawn')
self.__procpool = concurrent.futures.ProcessPoolExecutor(
max_workers=2,
initializer=worker_init,
)
self.__threadpool = concurrent.futures.ThreadPoolExecutor(
initializer=lambda: set_thread_name('threadpool_ws'),
max_workers=10,
)
self.jobs = JobsQueue(self)
self.__schemas = Schemas()
self.__services = {}
self.__wsclients = {}
self.__event_sources = {}
self.__event_subs = defaultdict(list)
self.__hooks = defaultdict(list)
self.__server_threads = []
self.__init_services()
def __init__(
self, loop_debug=False, loop_monitor=True, overlay_dirs=None, debug_level=None,
log_handler=None, startup_seq_path=None,
):
super().__init__(overlay_dirs)
self.logger = logger.Logger('middlewared', debug_level).getLogger()
self.crash_reporting = logger.CrashReporting()
self.crash_reporting_semaphore = asyncio.Semaphore(value=2)
self.loop_debug = loop_debug
self.loop_monitor = loop_monitor
self.debug_level = debug_level
self.log_handler = log_handler
self.startup_seq = 0
self.startup_seq_path = startup_seq_path
self.app = None
self.__loop = None
self.__thread_id = threading.get_ident()
# Spawn new processes for ProcessPool instead of forking
multiprocessing.set_start_method('spawn')
self.__threadpool = concurrent.futures.ThreadPoolExecutor(
initializer=lambda: set_thread_name('threadpool_ws'),
max_workers=10,
)
self.__init_procpool()
self.__wsclients = {}
self.__events = Events()
self.__event_sources = {}
self.__event_subs = defaultdict(list)
self.__hooks = defaultdict(list)
self.__server_threads = []
self.__init_services()
self.__console_io = False if os.path.exists(self.CONSOLE_ONCE_PATH) else None
self.jobs = JobsQueue(self)
def __init__(
self, loop_debug=False, loop_monitor=True, overlay_dirs=None, debug_level=None,
log_handler=None, startup_seq_path=None,
):
super().__init__(overlay_dirs)
self.logger = logger.Logger('middlewared', debug_level).getLogger()
self.crash_reporting = logger.CrashReporting()
self.crash_reporting_semaphore = asyncio.Semaphore(value=2)
self.loop_debug = loop_debug
self.loop_monitor = loop_monitor
self.debug_level = debug_level
self.log_handler = log_handler
self.startup_seq = 0
self.startup_seq_path = startup_seq_path
self.app = None
self.__loop = None
self.__thread_id = threading.get_ident()
# Spawn new processes for ProcessPool instead of forking
multiprocessing.set_start_method('spawn')
self.__threadpool = concurrent.futures.ThreadPoolExecutor(
initializer=lambda: set_thread_name('threadpool_ws'),
max_workers=10,
)
self.__init_procpool()
self.__wsclients = {}
self.__events = Events()
self.__event_sources = {}
self.__event_subs = defaultdict(list)
self.__hooks = defaultdict(list)
self.__server_threads = []
self.__init_services()
self.__console_io = False if os.path.exists(self.CONSOLE_ONCE_PATH) else None
self.jobs = JobsQueue(self)
async def run_in_thread(self, method, *args, **kwargs):
executor = concurrent.futures.ThreadPoolExecutor(
max_workers=1,
initializer=lambda: set_thread_name('io_thread'),
)
try:
return await self.loop.run_in_executor(executor, functools.partial(method, *args, **kwargs))
finally:
executor.shutdown(wait=False)
async def run_in_thread(self, method, *args, **kwargs):
executor = concurrent.futures.ThreadPoolExecutor(
max_workers=1,
initializer=lambda: set_thread_name('io_thread'),
)
try:
return await self.loop.run_in_executor(executor, functools.partial(method, *args, **kwargs))
finally:
executor.shutdown(wait=False)
def run(self):
set_thread_name('mdns_services')
while True:
if not mDNSDaemonMonitor.instance.mdnsd_running.wait(timeout=10):
return
try:
obj = self.queue.get(block=True, timeout=self.timeout)
except queue.Empty:
if self.finished.is_set():
break
continue
regtype = self.to_regtype(obj)
if not regtype:
continue
sdRef = pybonjour.DNSServiceBrowse(
regtype=regtype,
callBack=self.on_discover
)
self.references.append(sdRef)
_references = list(filter(self.active, self.references))
r, w, x = select.select(_references + [self.pipe[0]], [], [])
if self.pipe[0] in r:
break
for ref in r:
pybonjour.DNSServiceProcessResult(ref)
if not (obj.flags & kDNSServiceFlagsAdd):
self.references.remove(sdRef)
if self.active(sdRef):
sdRef.close()
if self.finished.is_set():
break
for ref in self.references:
self.references.remove(ref)
if self.active(ref):
ref.close()
def _loop_monitor_thread(self):
"""
Thread responsible for checking current tasks that are taking too long
to finish and printing the stack.
DISCLAIMER/TODO: This is not free of race condition so it may show
false positives.
"""
set_thread_name('loop_monitor')
last = None
while True:
time.sleep(2)
current = asyncio.current_task(loop=self.__loop)
if current is None:
last = None
continue
if last == current:
frame = sys._current_frames()[self.__thread_id]
stack = traceback.format_stack(frame, limit=-10)
self.logger.warn(''.join(['Task seems blocked:\n'] + stack))
last = current
def _loop_monitor_thread(self):
"""
Thread responsible for checking current tasks that are taking too long
to finish and printing the stack.
DISCLAIMER/TODO: This is not free of race condition so it may show
false positives.
"""
set_thread_name('loop_monitor')
last = None
while True:
time.sleep(2)
current = asyncio.current_task(loop=self.__loop)
if current is None:
last = None
continue
if last == current:
frame = sys._current_frames()[self.__thread_id]
stack = traceback.format_stack(frame, limit=-10)
self.logger.warn(''.join(['Task seems blocked:'] + stack))
last = current
def run(self):
set_thread_name('mdns_services')
while True:
if not self.mdnsd.is_alive():
self.mdnsd.wait(timeout=10)
try:
obj = self.queue.get(block=True, timeout=self.timeout)
except queue.Empty:
if self.finished.is_set():
break
continue
regtype = self.to_regtype(obj)
if not regtype:
continue
sdRef = pybonjour.DNSServiceBrowse(regtype=regtype,
callBack=self.on_discover)
self.references.append(sdRef)
_references = list(filter(self.active, self.references))
r, w, x = select.select(_references + [self.pipe[0]], [], [])
if self.pipe[0] in r:
break
for ref in r:
pybonjour.DNSServiceProcessResult(ref)
if not (obj.flags & kDNSServiceFlagsAdd):
self.references.remove(sdRef)
if self.active(sdRef):
sdRef.close()
if self.finished.is_set():
break
for ref in self.references:
self.references.remove(ref)
if self.active(ref):
ref.close()
def run(self):
set_thread_name('mdnsd_monitor')
while True:
"""
If the system has not completely booted yet we need to way at least
for DNS to be configured.
In case middlewared is started after boot, system.ready will be set after this plugin
is loaded, hence the dns_sync timeout.
"""
if not self.middleware.call_sync('system.ready'):
if not self.dns_sync.wait(timeout=2):
continue
pid = self.is_alive()
if not pid:
self.start_mdnsd()
time.sleep(2)
continue
kqueue = select.kqueue()
try:
kqueue.control([
select.kevent(
pid,
filter=select.KQ_FILTER_PROC,
flags=select.KQ_EV_ADD,
fflags=select.KQ_NOTE_EXIT,
)
], 0, 0)
except ProcessLookupError:
continue
self.mdnsd_running.set()
self.middleware.call_sync('mdnsadvertise.restart')
self.middleware.call_sync('mdnsbrowser.restart')
kqueue.control(None, 1)
self.mdnsd_running.clear()
kqueue.close()
def run(self):
set_thread_name('mdnsd_monitor')
while True:
"""
If the system has not completely booted yet we need to way at least
for DNS to be configured.
In case middlewared is started after boot, system.ready will be set after this plugin
is loaded, hence the dns_sync timeout.
"""
if not self.middleware.call_sync('system.ready'):
if not self.dns_sync.wait(timeout=2):
continue
pid = self.is_alive()
if not pid:
self.start_mdnsd()
time.sleep(2)
continue
kqueue = select.kqueue()
try:
kqueue.control([
select.kevent(
pid,
filter=select.KQ_FILTER_PROC,
flags=select.KQ_EV_ADD,
fflags=select.KQ_NOTE_EXIT,
)
], 0, 0)
except ProcessLookupError:
continue
self.mdnsd_running.set()
self.middleware.call_sync('mdnsadvertise.restart')
kqueue.control(None, 1)
self.mdnsd_running.clear()
kqueue.close()
def run(self):
set_thread_name('mdns_resolve')
while True:
if not mDNSDaemonMonitor.instance.mdnsd_running.wait(timeout=10):
return
try:
obj = self.queue.get(block=True, timeout=self.timeout)
except queue.Empty:
if self.finished.is_set():
break
continue
sdRef = pybonjour.DNSServiceResolve(
flags=obj.flags,
interfaceIndex=obj.interface,
name=obj.name,
regtype=obj.regtype,
domain=obj.domain,
callBack=self.on_resolve
)
self.references.append(sdRef)
_references = list(filter(self.active, self.references))
r, w, x = select.select(_references, [], [])
for ref in r:
pybonjour.DNSServiceProcessResult(ref)
if self.finished.is_set():
break
for ref in self.references:
self.references.remove(ref)
if self.active(ref):
ref.close()
def run(self):
self._console_write('starting')
set_thread_name('asyncio_loop')
self.loop = self.__loop = asyncio.get_event_loop()
if self.loop_debug:
self.__loop.set_debug(True)
self.__loop.slow_callback_duration = 0.2
self.app = app = web.Application(middlewares=[
normalize_path_middleware(redirect_class=HTTPPermanentRedirect)
], loop=self.__loop)
# Needs to happen after setting debug or may cause race condition
# http://bugs.python.org/issue30805
self.__loop.run_until_complete(self.__plugins_load())
self._console_write('registering services')
if self.loop_monitor:
# Start monitor thread after plugins have been loaded
# because of the time spent doing I/O
t = threading.Thread(target=self._loop_monitor_thread)
t.setDaemon(True)
t.start()
self.__loop.add_signal_handler(signal.SIGINT, self.terminate)
self.__loop.add_signal_handler(signal.SIGTERM, self.terminate)
self.__loop.add_signal_handler(signal.SIGUSR1, self.pdb)
self.__loop.add_signal_handler(signal.SIGUSR2, self.log_threads_stacks)
app.router.add_route('GET', '/websocket', self.ws_handler)
app.router.add_route('*', '/api/docs{path_info:.*}', WSGIHandler(apidocs_app))
app.router.add_route('*', '/ui{path_info:.*}', WebUIAuth(self))
self.fileapp = FileApplication(self, self.__loop)
app.router.add_route('*', '/_download{path_info:.*}', self.fileapp.download)
app.router.add_route('*', '/_upload{path_info:.*}', self.fileapp.upload)
shellapp = ShellApplication(self)
app.router.add_route('*', '/_shell{path_info:.*}', shellapp.ws_handler)
restful_api = RESTfulAPI(self, app)
self.__loop.run_until_complete(
asyncio.ensure_future(restful_api.register_resources())
)
asyncio.ensure_future(self.jobs.run())
self.__setup_periodic_tasks()
# Start up middleware worker process pool
self.__procpool._start_queue_management_thread()
runner = web.AppRunner(app, handle_signals=False, access_log=None)
self.__loop.run_until_complete(runner.setup())
self.__loop.run_until_complete(
web.TCPSite(runner, '0.0.0.0', 6000, reuse_address=True, reuse_port=True).start()
)
self.__loop.run_until_complete(web.UnixSite(runner, '/var/run/middlewared.sock').start())
self.logger.debug('Accepting connections')
self._console_write('loading completed\n')
try:
self.__loop.run_forever()
except RuntimeError as e:
if e.args[0] != "Event loop is closed":
raise
def __init__(self, **kwargs):
kwargs['service'] = 'middleware'
super(mDNSServiceMiddlewareThread, self).__init__(**kwargs)
set_thread_name(f'mdns_{self.service}')
def run(self):
self._console_write('starting')
set_thread_name('asyncio_loop')
self.loop = self.__loop = asyncio.get_event_loop()
if self.loop_debug:
self.__loop.set_debug(True)
self.__loop.slow_callback_duration = 0.2
self.app = app = web.Application(middlewares=[
normalize_path_middleware(redirect_class=HTTPPermanentRedirect)
], loop=self.__loop)
# Needs to happen after setting debug or may cause race condition
# http://bugs.python.org/issue30805
self.__loop.run_until_complete(self.__plugins_load())
self._console_write('registering services')
if self.loop_monitor:
# Start monitor thread after plugins have been loaded
# because of the time spent doing I/O
t = threading.Thread(target=self._loop_monitor_thread)
t.setDaemon(True)
t.start()
self.__loop.add_signal_handler(signal.SIGINT, self.terminate)
self.__loop.add_signal_handler(signal.SIGTERM, self.terminate)
self.__loop.add_signal_handler(signal.SIGUSR1, self.pdb)
self.__loop.add_signal_handler(signal.SIGUSR2, self.log_threads_stacks)
app.router.add_route('GET', '/websocket', self.ws_handler)
app.router.add_route('*', '/api/docs{path_info:.*}', WSGIHandler(apidocs_app))
app.router.add_route('*', '/ui{path_info:.*}', WebUIAuth(self))
self.fileapp = FileApplication(self, self.__loop)
app.router.add_route('*', '/_download{path_info:.*}', self.fileapp.download)
app.router.add_route('*', '/_upload{path_info:.*}', self.fileapp.upload)
shellapp = ShellApplication(self)
app.router.add_route('*', '/_shell{path_info:.*}', shellapp.ws_handler)
restful_api = RESTfulAPI(self, app)
self.__loop.run_until_complete(
asyncio.ensure_future(restful_api.register_resources())
)
asyncio.ensure_future(self.jobs.run())
self.__setup_periodic_tasks()
# Start up middleware worker process pool
self.__procpool._start_queue_management_thread()
runner = web.AppRunner(app, handle_signals=False, access_log=None)
self.__loop.run_until_complete(runner.setup())
self.__loop.run_until_complete(
web.TCPSite(runner, '0.0.0.0', 6000, reuse_address=True, reuse_port=True).start()
)
self.__loop.run_until_complete(web.UnixSite(runner, '/var/run/middlewared.sock').start())
self.logger.debug('Accepting connections')
self._console_write('loading completed\n')
try:
self.__loop.run_forever()
except RuntimeError as e:
if e.args[0] != "Event loop is closed":
raise
def __init__(self, **kwargs):
kwargs['service'] = 'middleware'
super(mDNSServiceMiddlewareThread, self).__init__(**kwargs)
set_thread_name(f'mdns_{self.service}')
def run(self):
set_thread_name(f'mdns_svc_{self.service}')
try:
self.register()
except pybonjour.BonjourError:
self.logger.trace("ServiceThread: failed to register '%s', is mdnsd running?", self.service)
