def __init__(self): self.event_loop = asyncio.get_event_loop() self.broadcast_tasks = dict() # parallel dict to broadcast_tasks that tracks stats self.local_hostname = get_local_host() self.stats_mgr = BroadcastWebsocketStatsManager( self.event_loop, self.local_hostname)
def __init__(self): self.event_loop = asyncio.get_event_loop() ''' { 'hostname1': BroadcastWebsocketTask(), 'hostname2': BroadcastWebsocketTask(), 'hostname3': BroadcastWebsocketTask(), } ''' self.broadcast_tasks = dict() self.local_hostname = get_local_host() self.stats_mgr = BroadcastWebsocketStatsManager(self.event_loop, self.local_hostname)
class BroadcastWebsocketManager(object): def __init__(self): self.event_loop = asyncio.get_event_loop() ''' { 'hostname1': BroadcastWebsocketTask(), 'hostname2': BroadcastWebsocketTask(), 'hostname3': BroadcastWebsocketTask(), } ''' self.broadcast_tasks = dict() self.local_hostname = get_local_host() self.stats_mgr = BroadcastWebsocketStatsManager(self.event_loop, self.local_hostname) async def run_per_host_websocket(self): while True: known_hosts = await get_broadcast_hosts() future_remote_hosts = known_hosts.keys() current_remote_hosts = self.broadcast_tasks.keys() deleted_remote_hosts = set(current_remote_hosts) - set(future_remote_hosts) new_remote_hosts = set(future_remote_hosts) - set(current_remote_hosts) remote_addresses = {k: v.remote_host for k, v in self.broadcast_tasks.items()} for hostname, address in known_hosts.items(): if hostname in self.broadcast_tasks and address != remote_addresses[hostname]: deleted_remote_hosts.add(hostname) new_remote_hosts.add(hostname) if deleted_remote_hosts: logger.warning(f"Removing {deleted_remote_hosts} from websocket broadcast list") if new_remote_hosts: logger.warning(f"Adding {new_remote_hosts} to websocket broadcast list") for h in deleted_remote_hosts: self.broadcast_tasks[h].cancel() del self.broadcast_tasks[h] self.stats_mgr.delete_remote_host_stats(h) for h in new_remote_hosts: stats = self.stats_mgr.new_remote_host_stats(h) broadcast_task = BroadcastWebsocketTask(name=self.local_hostname, event_loop=self.event_loop, stats=stats, remote_host=known_hosts[h]) broadcast_task.start() self.broadcast_tasks[h] = broadcast_task await asyncio.sleep(settings.BROADCAST_WEBSOCKET_NEW_INSTANCE_POLL_RATE_SECONDS) def start(self): self.stats_mgr.start() self.async_task = self.event_loop.create_task(self.run_per_host_websocket()) return self.async_task
class BroadcastWebsocketManager(object): def __init__(self): self.event_loop = asyncio.get_event_loop() self.broadcast_tasks = dict() # parallel dict to broadcast_tasks that tracks stats self.local_hostname = get_local_host() self.stats_mgr = BroadcastWebsocketStatsManager( self.event_loop, self.local_hostname) async def run_per_host_websocket(self): while True: future_remote_hosts = get_broadcast_hosts() current_remote_hosts = self.broadcast_tasks.keys() deleted_remote_hosts = set(current_remote_hosts) - set( future_remote_hosts) new_remote_hosts = set(future_remote_hosts) - set( current_remote_hosts) if deleted_remote_hosts: logger.warn( f"{self.local_hostname} going to remove {deleted_remote_hosts} from the websocket broadcast list" ) if new_remote_hosts: logger.warn( f"{self.local_hostname} going to add {new_remote_hosts} to the websocket broadcast list" ) for h in deleted_remote_hosts: self.broadcast_tasks[h].cancel() del self.broadcast_tasks[h] self.stats_mgr.delete_remote_host_stats(h) for h in new_remote_hosts: stats = self.stats_mgr.new_remote_host_stats(h) broadcast_task = BroadcastWebsocketTask( name=self.local_hostname, event_loop=self.event_loop, stats=stats, remote_host=h) broadcast_task.start() self.broadcast_tasks[h] = broadcast_task await asyncio.sleep( settings.BROADCAST_WEBSOCKET_NEW_INSTANCE_POLL_RATE_SECONDS) def start(self): self.stats_mgr.start() self.async_task = self.event_loop.create_task( self.run_per_host_websocket()) return self.async_task
def handle(self, *arg, **options): if options.get('status'): try: stats_all = BroadcastWebsocketStatsManager.get_stats_sync() except redis.exceptions.ConnectionError as e: print( f"Unable to get Broadcast Websocket Status. Failed to connect to redis {e}" ) return data = {} for family in stats_all: if family.type == 'gauge' and len(family.samples) > 1: for sample in family.samples: if sample.value >= 1: data[family.name] = sample.labels[family.name] break else: data[family.name] = family.samples[0].value me = Instance.objects.me() hostnames = [ i.hostname for i in Instance.objects.exclude( Q(hostname=me.hostname) | Q(rampart_groups__controller__isnull=False)) ] host_stats = Command.get_connection_status(me, hostnames, data) lines = Command._format_lines(host_stats) print( f'Broadcast websocket connection status from "{me.hostname}" to:' ) print('\n'.join(lines)) host_stats = Command.get_connection_stats(me, hostnames, data) lines = Command._format_lines(host_stats) print( f'\nBroadcast websocket connection stats from "{me.hostname}" to:' ) print('\n'.join(lines)) return try: broadcast_websocket_mgr = BroadcastWebsocketManager() task = broadcast_websocket_mgr.start() loop = asyncio.get_event_loop() loop.run_until_complete(task) except KeyboardInterrupt: logger.debug('Terminating Websocket Broadcaster')
def handle(self, *arg, **options): # it's necessary to delay this import in case # database migrations are still running from awx.main.models.ha import Instance executor = MigrationExecutor(connection) migrating = bool( executor.migration_plan(executor.loader.graph.leaf_nodes())) registered = False if not migrating: try: Instance.objects.me() registered = True except RuntimeError: pass if migrating or not registered: # In containerized deployments, migrations happen in the task container, # and the services running there don't start until migrations are # finished. # *This* service runs in the web container, and it's possible that it can # start _before_ migrations are finished, thus causing issues with the ORM # queries it makes (specifically, conf.settings queries). # This block is meant to serve as a sort of bail-out for the situation # where migrations aren't yet finished (similar to the migration # detection middleware that the uwsgi processes have) or when instance # registration isn't done yet logger.error( 'AWX is currently installing/upgrading. Trying again in 5s...' ) time.sleep(5) return if options.get('status'): try: stats_all = BroadcastWebsocketStatsManager.get_stats_sync() except redis.exceptions.ConnectionError as e: print( f"Unable to get Broadcast Websocket Status. Failed to connect to redis {e}" ) return data = {} for family in stats_all: if family.type == 'gauge' and len(family.samples) > 1: for sample in family.samples: if sample.value >= 1: data[family.name] = sample.labels[family.name] break else: data[family.name] = family.samples[0].value me = Instance.objects.me() hostnames = [ i.hostname for i in Instance.objects.exclude(hostname=me.hostname) ] host_stats = Command.get_connection_status(me, hostnames, data) lines = Command._format_lines(host_stats) print( f'Broadcast websocket connection status from "{me.hostname}" to:' ) print('\n'.join(lines)) host_stats = Command.get_connection_stats(me, hostnames, data) lines = Command._format_lines(host_stats) print( f'\nBroadcast websocket connection stats from "{me.hostname}" to:' ) print('\n'.join(lines)) return try: broadcast_websocket_mgr = BroadcastWebsocketManager() task = broadcast_websocket_mgr.start() loop = asyncio.get_event_loop() loop.run_until_complete(task) except KeyboardInterrupt: logger.debug('Terminating Websocket Broadcaster')