def __init__(self):
self.event_loop = asyncio.get_event_loop()
self.broadcast_tasks = dict()
# parallel dict to broadcast_tasks that tracks stats
self.local_hostname = get_local_host()
self.stats_mgr = BroadcastWebsocketStatsManager(
self.event_loop, self.local_hostname)
def __init__(self):
self.event_loop = asyncio.get_event_loop()
'''
{
'hostname1': BroadcastWebsocketTask(),
'hostname2': BroadcastWebsocketTask(),
'hostname3': BroadcastWebsocketTask(),
}
'''
self.broadcast_tasks = dict()
self.local_hostname = get_local_host()
self.stats_mgr = BroadcastWebsocketStatsManager(self.event_loop, self.local_hostname)
class BroadcastWebsocketManager(object):
def __init__(self):
self.event_loop = asyncio.get_event_loop()
'''
{
'hostname1': BroadcastWebsocketTask(),
'hostname2': BroadcastWebsocketTask(),
'hostname3': BroadcastWebsocketTask(),
}
'''
self.broadcast_tasks = dict()
self.local_hostname = get_local_host()
self.stats_mgr = BroadcastWebsocketStatsManager(self.event_loop, self.local_hostname)
async def run_per_host_websocket(self):
while True:
known_hosts = await get_broadcast_hosts()
future_remote_hosts = known_hosts.keys()
current_remote_hosts = self.broadcast_tasks.keys()
deleted_remote_hosts = set(current_remote_hosts) - set(future_remote_hosts)
new_remote_hosts = set(future_remote_hosts) - set(current_remote_hosts)
remote_addresses = {k: v.remote_host for k, v in self.broadcast_tasks.items()}
for hostname, address in known_hosts.items():
if hostname in self.broadcast_tasks and address != remote_addresses[hostname]:
deleted_remote_hosts.add(hostname)
new_remote_hosts.add(hostname)
if deleted_remote_hosts:
logger.warning(f"Removing {deleted_remote_hosts} from websocket broadcast list")
if new_remote_hosts:
logger.warning(f"Adding {new_remote_hosts} to websocket broadcast list")
for h in deleted_remote_hosts:
self.broadcast_tasks[h].cancel()
del self.broadcast_tasks[h]
self.stats_mgr.delete_remote_host_stats(h)
for h in new_remote_hosts:
stats = self.stats_mgr.new_remote_host_stats(h)
broadcast_task = BroadcastWebsocketTask(name=self.local_hostname, event_loop=self.event_loop, stats=stats, remote_host=known_hosts[h])
broadcast_task.start()
self.broadcast_tasks[h] = broadcast_task
await asyncio.sleep(settings.BROADCAST_WEBSOCKET_NEW_INSTANCE_POLL_RATE_SECONDS)
def start(self):
self.stats_mgr.start()
self.async_task = self.event_loop.create_task(self.run_per_host_websocket())
return self.async_task
class BroadcastWebsocketManager(object):
def __init__(self):
self.event_loop = asyncio.get_event_loop()
self.broadcast_tasks = dict()
# parallel dict to broadcast_tasks that tracks stats
self.local_hostname = get_local_host()
self.stats_mgr = BroadcastWebsocketStatsManager(
self.event_loop, self.local_hostname)
async def run_per_host_websocket(self):
while True:
future_remote_hosts = get_broadcast_hosts()
current_remote_hosts = self.broadcast_tasks.keys()
deleted_remote_hosts = set(current_remote_hosts) - set(
future_remote_hosts)
new_remote_hosts = set(future_remote_hosts) - set(
current_remote_hosts)
if deleted_remote_hosts:
logger.warn(
f"{self.local_hostname} going to remove {deleted_remote_hosts} from the websocket broadcast list"
)
if new_remote_hosts:
logger.warn(
f"{self.local_hostname} going to add {new_remote_hosts} to the websocket broadcast list"
)
for h in deleted_remote_hosts:
self.broadcast_tasks[h].cancel()
del self.broadcast_tasks[h]
self.stats_mgr.delete_remote_host_stats(h)
for h in new_remote_hosts:
stats = self.stats_mgr.new_remote_host_stats(h)
broadcast_task = BroadcastWebsocketTask(
name=self.local_hostname,
event_loop=self.event_loop,
stats=stats,
remote_host=h)
broadcast_task.start()
self.broadcast_tasks[h] = broadcast_task
await asyncio.sleep(
settings.BROADCAST_WEBSOCKET_NEW_INSTANCE_POLL_RATE_SECONDS)
def start(self):
self.stats_mgr.start()
self.async_task = self.event_loop.create_task(
self.run_per_host_websocket())
return self.async_task
def handle(self, *arg, **options):
if options.get('status'):
try:
stats_all = BroadcastWebsocketStatsManager.get_stats_sync()
except redis.exceptions.ConnectionError as e:
print(
f"Unable to get Broadcast Websocket Status. Failed to connect to redis {e}"
)
return
data = {}
for family in stats_all:
if family.type == 'gauge' and len(family.samples) > 1:
for sample in family.samples:
if sample.value >= 1:
data[family.name] = sample.labels[family.name]
break
else:
data[family.name] = family.samples[0].value
me = Instance.objects.me()
hostnames = [
i.hostname for i in Instance.objects.exclude(
Q(hostname=me.hostname)
| Q(rampart_groups__controller__isnull=False))
]
host_stats = Command.get_connection_status(me, hostnames, data)
lines = Command._format_lines(host_stats)
print(
f'Broadcast websocket connection status from "{me.hostname}" to:'
)
print('\n'.join(lines))
host_stats = Command.get_connection_stats(me, hostnames, data)
lines = Command._format_lines(host_stats)
print(
f'\nBroadcast websocket connection stats from "{me.hostname}" to:'
)
print('\n'.join(lines))
return
try:
broadcast_websocket_mgr = BroadcastWebsocketManager()
task = broadcast_websocket_mgr.start()
loop = asyncio.get_event_loop()
loop.run_until_complete(task)
except KeyboardInterrupt:
logger.debug('Terminating Websocket Broadcaster')
def handle(self, *arg, **options):
# it's necessary to delay this import in case
# database migrations are still running
from awx.main.models.ha import Instance
executor = MigrationExecutor(connection)
migrating = bool(
executor.migration_plan(executor.loader.graph.leaf_nodes()))
registered = False
if not migrating:
try:
Instance.objects.me()
registered = True
except RuntimeError:
pass
if migrating or not registered:
# In containerized deployments, migrations happen in the task container,
# and the services running there don't start until migrations are
# finished.
# *This* service runs in the web container, and it's possible that it can
# start _before_ migrations are finished, thus causing issues with the ORM
# queries it makes (specifically, conf.settings queries).
# This block is meant to serve as a sort of bail-out for the situation
# where migrations aren't yet finished (similar to the migration
# detection middleware that the uwsgi processes have) or when instance
# registration isn't done yet
logger.error(
'AWX is currently installing/upgrading. Trying again in 5s...'
)
time.sleep(5)
return
if options.get('status'):
try:
stats_all = BroadcastWebsocketStatsManager.get_stats_sync()
except redis.exceptions.ConnectionError as e:
print(
f"Unable to get Broadcast Websocket Status. Failed to connect to redis {e}"
)
return
data = {}
for family in stats_all:
if family.type == 'gauge' and len(family.samples) > 1:
for sample in family.samples:
if sample.value >= 1:
data[family.name] = sample.labels[family.name]
break
else:
data[family.name] = family.samples[0].value
me = Instance.objects.me()
hostnames = [
i.hostname
for i in Instance.objects.exclude(hostname=me.hostname)
]
host_stats = Command.get_connection_status(me, hostnames, data)
lines = Command._format_lines(host_stats)
print(
f'Broadcast websocket connection status from "{me.hostname}" to:'
)
print('\n'.join(lines))
host_stats = Command.get_connection_stats(me, hostnames, data)
lines = Command._format_lines(host_stats)
print(
f'\nBroadcast websocket connection stats from "{me.hostname}" to:'
)
print('\n'.join(lines))
return
try:
broadcast_websocket_mgr = BroadcastWebsocketManager()
task = broadcast_websocket_mgr.start()
loop = asyncio.get_event_loop()
loop.run_until_complete(task)
except KeyboardInterrupt:
logger.debug('Terminating Websocket Broadcaster')