Example #1
0
 def emit_info(self, channel, info, wrap_in_list):
     # info can be a list (e.g. for leases)
     # or a dict, with or without an 'id' field
     if self.debug:
         msg = "len={}".format(len(info)) if isinstance(info, list) \
               else "id={}".format(info.get('id', '[none]'))
         logger.info("{} emitting {} -> {}"
                     .format(channel, msg, info))
                         
     # wrap info as single elt in a list
     if wrap_in_list:
         info = [ info ]
     message = json.dumps(info)
     if self.socketio is None:
         self.connect()
     try:
         self.socketio.emit(channel, message,
                            ReconnectableSocketIO.callback)
         self.counters.setdefault(channel, 0)
         self.counters[channel] += 1
     except Exception as e:
         # make sure we reconnect later on
         self.socketio = None
         label = "{}: {}".format(info.get('id', 'none'), one_char_summary(info))
         logger.warn("Dropped message on {} - channel {} - msg {}"
                     .format(self, channel, label))
         logger.exception("Dropped because of this exception")
Example #2
0
 async def keep_connected(self):
     """
     A continuous loop that keeps the connection open
     """
     while True:
         logger.debug(f"in keep_connected, proto={self.proto}")
         if self.proto and self.proto.open:
             pass
         else:
             # xxx should we close() our client ?
             self.proto = None
             # see if we need ssl
             secure = websockets.uri.parse_uri(self.url).secure
             kwds = {}
             if secure:
                 import ssl
                 kwds.update(dict(ssl=ssl.SSLContext()))
             try:
                 logger.info(f"(re)-connecting to {self.url} ...")
                 self.proto = await SidecarAsyncClient(self.url, **kwds)
                 logger.debug("connected !")
             except ConnectionRefusedError:
                 logger.warning(
                     f"Could not connect to {self.url} at this time")
             except:
                 logger.exception(
                     f"Could not connect to {self.url} at this time")
         await asyncio.sleep(self.keep_period)
Example #3
0
 async def probe_forever(self, cycle, ping_timeout, ssh_timeout):
     """
     runs forever, wait <cycle> seconds between 2 runs of probe()
     """
     while True:
         try:
             await self.probe(ping_timeout, ssh_timeout)
         except Exception:
             logger.exception("monitornodes oops 2")
         await asyncio.sleep(cycle)
Example #4
0
 async def probe_forever(self, cycle, ping_timeout, ssh_timeout):
     """
     runs forever, wait <cycle> seconds between 2 runs of probe()
     """
     while True:
         try:
             await self.probe(ping_timeout, ssh_timeout)
         except Exception as e:
             logger.exception("monitor oops 2")
         await asyncio.sleep(cycle)
Example #5
0
 async def watch_back_channel(self, category, callback):
     while True:
         if not self.proto:
             logger.debug(f"backing off for {self.keep_period}s")
             await asyncio.sleep(self.keep_period)
             continue
         try:
             incoming = await self.proto.recv()
             umbrella = json.loads(incoming)
             logger.info(
                 f"tmp - got incoming {umbrella['category']} x {umbrella['action']}"
             )
             if (umbrella['category'] == category
                     and umbrella['action'] == 'request'):
                 callback(umbrella)
         except Exception as exc:
             ### to review
             logger.exception("recv failed .. fix me")
             self.proto = None
Example #6
0
 async def emit_infos(self, infos):
     if not self.proto:
         logger.warning(f"dropping message {infos}")
         return False
     logger.debug(f"Sending {infos}")
     # xxx use Payload
     payload = dict(category=self.category, action='info', message=infos)
     # xxx try/except here
     try:
         await self.proto.send(json.dumps(payload))
         self.counter += 1
     except ConnectionRefusedError:
         logger.warning(f"Could not send {self.category} - dropped")
     except Exception as exc:
         # xxx to review
         logger.exception("send failed")
         self.proto = None
         return False
     return True
Example #7
0
 async def run_forever(self):
     leases = Leases(self.message_bus)
     while True:
         self.fast_track = False
         trigger = time.time() + self.cycle
         # check for back_channel every 15 ms
         while not self.fast_track and time.time() < trigger:
             await asyncio.sleep(self.step)
             # give a chance to socketio events to trigger
             self.reconnectable.wait(self.wait)
             
         try:
             await leases.refresh()
             # xxx this is fragile
             omf_leases = leases.resources
             self.reconnectable.emit_info(self.channel, omf_leases,
                                          wrap_in_list=False)
             logger.info("advertising {} leases".format(len(omf_leases)))
             if self.debug:
                 logger.info("Leases details: {}".format(omf_leases))
         except Exception as e:
             logger.exception("monitor could not get leases")
Example #8
0
    async def mainloop(self):
        leases = Leases(self.message_bus)
        if self.verbose:
            logger.info("Entering monitor on leases")
        while True:
            self.fast_track = False  # pylint: disable=w0201
            trigger = time.time() + self.cycle
            # check for back_channel every 50 ms
            while not self.fast_track and time.time() < trigger:
                await asyncio.sleep(self.step)

            try:
                if self.verbose:
                    logger.info("monitorleases mainloop")
                await leases.refresh()
                # xxx this is fragile
                omf_leases = leases.resources
                logger.info("advertising {} leases".format(len(omf_leases)))
                await self.reconnectable.emit_infos(omf_leases)
                if self.verbose:
                    logger.info("Leases details: {}".format(omf_leases))
            except Exception:
                logger.exception("monitornodes could not get leases")
Example #9
0
    async def probe(
            self,  # pylint: disable=r0912, r0914, r0915
            ping_timeout,
            ssh_timeout):
        """
        The logic for getting one node's info and send it to sidecar
        """
        if self.verbose:
            logger.info(f"entering pass1, info={self.info}")
        # pass1 : check for status
        padding_dict = {
            'control_ping': 'off',
            'control_ssh': 'off',
            # don't overwrite os_release though
        }
        # get USRP status no matter what - use "" if we receive None
        # to limit noise when the node is physically removed
        usrp_status = await self.node.get_usrpstatus() or 'fail'
        # replace usrpon and usrpoff with just on and off
        self.set_info({'usrp_on_off': usrp_status.replace('usrp', '')})
        # get CMC status
        status = await self.node.get_status()
        if status == "off":
            await self.set_info_and_report({'cmc_on_off': 'off'}, padding_dict)
            return
        elif status != "on":
            await self.set_info_and_report({'cmc_on_off': 'fail'},
                                           padding_dict)
            return
        if self.verbose:
            logger.info(f"entering pass2, info={self.info}")
        # pass2 : CMC status is ON - let's try to ssh it
        self.set_info({'cmc_on_off': 'on'})
        padding_dict = {
            'control_ping': 'on',
            'control_ssh': 'on',
        }

        remote_commands = [
            "cat /etc/lsb-release /etc/redhat-release /etc/gnuradio-release "
            "2> /dev/null | grep -i release",
            "echo -n GNURADIO: ; gnuradio-config-info --version "
            "2> /dev/null || echo none",
            # this trick allows to have the filename on each output line
            "grep . /etc/rhubarbe-image /dev/null",
            "echo -n UNAME: ; uname -r",
            "echo -n DOCKER: ; docker --version",
            "echo -n CONTAINER: ; docker inspect --format='{{.State.Running}} {{.Config.Image}}' container",
        ]
        # reconnect each time
        async with SshProxy(self.node) as ssh:
            if self.verbose:
                logger.info(
                    f"trying to ssh-connect to {self.node.control_hostname()} "
                    f"(timeout={ssh_timeout})")
            try:
                connected = await asyncio.wait_for(ssh.connect(),
                                                   timeout=ssh_timeout)
            except asyncio.TimeoutError:
                connected = False
                self.set_info({'control_ssh': 'off'})
            if self.verbose:
                logger.info(
                    f"{self.node.control_hostname()} ssh-connected={connected}"
                )
            if connected:
                try:
                    command = ";".join(remote_commands)
                    output = await asyncio.wait_for(ssh.run(command),
                                                    timeout=ssh_timeout)
                    # padding dict here sets control_ssh and control_ping to on
                    self.parse_ssh_probe_output(output, padding_dict)
                    # required as otherwise we leak openfiles
                    try:
                        await ssh.close()
                    except Exception:
                        logger.exception("monitornodes oops 1")
                except asyncio.TimeoutError:
                    if self.verbose:
                        logger.info(
                            f"received ssh timeout with {self.node.control_hostname()}"
                        )
                    self.set_info({'control_ssh': 'off'})
                except Exception:
                    logger.exception("monitornodes remote_command failed")
        if self.verbose:
            logger.info(f"{self.node.control_hostname()} ssh-based logic done "
                        f"ssh is deemed {self.info['control_ssh']}")

        # if we could ssh then we're done
        if self.info['control_ssh'] == 'on':
            await self.report_info()
            return

        if self.verbose:
            logger.info(f"entering pass3, info={self.info}")
        # pass3 : node is ON but could not ssh
        # check for ping
        # I don't know of an asyncio library to deal with icmp
        # so let's use asyncio.subprocess
        # xxx maybe a Ping class would be the way to go
        control = self.node.control_hostname()
        command = ["ping", "-c", "1", "-t", "1", control]
        try:
            subprocess = await asyncio.create_subprocess_exec(
                *command,
                stdout=asyncio.subprocess.DEVNULL,
                stderr=asyncio.subprocess.DEVNULL)
            # failure occurs through timeout
            await asyncio.wait_for(subprocess.wait(), timeout=ping_timeout)
            await self.set_info_and_report({'control_ping': 'on'})
            return
        except asyncio.TimeoutError:
            await self.set_info_and_report({'control_ping': 'off'})
            return
Example #10
0
    async def probe(self, ping_timeout, ssh_timeout):
        """
        The logic for getting one node's info and send it to sidecar
        """
        if self.debug:
            logger.info("entering pass1, info={}".format(self.info))
        # pass1 : check for status
        padding_dict = {
            'control_ping' : 'off',
            'control_ssh' : 'off',
            # don't overwrite os_release though
        }
        # get USRP status no matter what
        usrp_status = await self.node.get_usrpstatus()
        # replace usrpon and usrpoff with just on and off
        self.set_info({'usrp_on_off' : usrp_status.replace('usrp', '')})
        # get CMC status
        status = await self.node.get_status()
        if status == "off":
            self.set_info_and_report({'cmc_on_off' : 'off'}, padding_dict)
            return
        elif status != "on":
            self.set_info_and_report({'cmc_on_off' : 'fail'}, padding_dict)
            return
        if self.debug:
            logger.info("entering pass2, info={}".format(self.info))
        # pass2 : CMC status is ON - let's try to ssh it
        self.set_info({'cmc_on_off' : 'on'})
        padding_dict = {
            'control_ping' : 'on',
            'control_ssh' : 'on',
        }
        self.zero_wlan_infos()
        remote_commands = [
            "cat /etc/lsb-release /etc/fedora-release /etc/gnuradio-release 2> /dev/null | grep -i release",
            "echo -n GNURADIO: ; gnuradio-config-info --version 2> /dev/null || echo none",
            # this trick allows to have the filename on each output line
            "grep . /etc/rhubarbe-image /dev/null",
            "echo -n 'UNAME:' ; uname -r",
            ]
        if self.report_wlan:
            remote_commands.append(
                "head /sys/class/net/wlan?/statistics/[rt]x_bytes"                
            )
        # reconnect each time
        ssh = SshProxy(self.node)
        if self.debug:
            logger.info("trying to ssh-connect")
        try:
            connected = await asyncio.wait_for(ssh.connect(), timeout=ssh_timeout)
        except asyncio.TimeoutError as e:
            connected = False
        if self.debug:
            logger.info("ssh-connected={}".format(connected))
        if connected:
            try:
                command = ";".join(remote_commands)
                output = await ssh.run(command)
                # padding dict here sets control_ssh and control_ping to on
                self.parse_ssh_probe_output(output, padding_dict)
                # required as otherwise we leak openfiles
                try:
                    await ssh.close()
                except Exception as e:
                    logger.exception("monitor oops 1")
                    pass
            except Exception as e:
                logger.exception("monitor remote_command failed")
        else:
            self.set_info({'control_ssh': 'off'})

        # if we could ssh then we're done
        if self.info['control_ssh'] == 'on':
            self.report_info()
            return

        if self.debug:
            logger.info("entering pass3, info={}".format(self.info))
        # pass3 : node is ON but could not ssh
        # check for ping
        # I don't know of an asyncio library to deal with icmp
        # so let's use asyncio.subprocess
        # xxx maybe a Ping class would be the way to go
        control = self.node.control_hostname()
        command = [ "ping", "-c", "1", "-t", "1", control ]
        try:
            subprocess = await asyncio.create_subprocess_exec(
                *command,
                stdout = asyncio.subprocess.DEVNULL,
                stderr = asyncio.subprocess.DEVNULL)
            retcod = await asyncio.wait_for(subprocess.wait(), timeout=ping_timeout)
            self.set_info_and_report({'control_ping' : 'on'})
            return
        except asyncio.TimeoutError as e:
            self.set_info_and_report({'control_ping' : 'off'})
            return