Esempio n. 1
0
    def _send_heartbeat_telemetry(self, protocol):
        if self._last_telemetry_heartbeat is None:
            self._last_telemetry_heartbeat = datetime.utcnow(
            ) - UpdateHandler.TELEMETRY_HEARTBEAT_PERIOD

        if datetime.utcnow() >= (self._last_telemetry_heartbeat +
                                 UpdateHandler.TELEMETRY_HEARTBEAT_PERIOD):
            dropped_packets = self.osutil.get_firewall_dropped_packets(
                protocol.get_endpoint())
            auto_update_enabled = 1 if conf.get_autoupdate_enabled() else 0
            telemetry_msg = "{0};{1};{2};{3};{4}".format(
                self._heartbeat_counter, self._heartbeat_id, dropped_packets,
                self._heartbeat_update_goal_state_error_count,
                auto_update_enabled)

            add_event(name=AGENT_NAME,
                      version=CURRENT_VERSION,
                      op=WALAEventOperation.HeartBeat,
                      is_success=True,
                      message=telemetry_msg,
                      log_event=False)
            self._heartbeat_counter += 1
            self._heartbeat_update_goal_state_error_count = 0

            debug_log_msg = "[DEBUG HeartbeatCounter: {0};HeartbeatId: {1};DroppedPackets: {2};" \
                            "UpdateGSErrors: {3};AutoUpdate: {4}]".format(self._heartbeat_counter,
                                                                          self._heartbeat_id, dropped_packets,
                                                                          self._heartbeat_update_goal_state_error_count,
                                                                          auto_update_enabled)
            logger.info(
                u"[HEARTBEAT] Agent {0} is running as the goal state agent {1}",
                CURRENT_AGENT, debug_log_msg)
            self._last_telemetry_heartbeat = datetime.utcnow()
Esempio n. 2
0
    def _upgrade_available(self, protocol, base_version=CURRENT_VERSION):
        # Ignore new agents if updating is disabled
        if not conf.get_autoupdate_enabled():
            return False

        now = time.time()
        if self.last_attempt_time is not None:
            next_attempt_time = self.last_attempt_time + \
                                    conf.get_autoupdate_frequency()
        else:
            next_attempt_time = now
        if next_attempt_time > now:
            return False

        family = conf.get_autoupdate_gafamily()
        logger.info("Checking for agent updates (family: {0})", family)

        self.last_attempt_time = now

        try:
            manifest_list, etag = protocol.get_vmagent_manifests()

            manifests = [m for m in manifest_list.vmAgentManifests \
                            if m.family == family and len(m.versionsManifestUris) > 0]
            if len(manifests) == 0:
                logger.verbose(u"Incarnation {0} has no {1} agent updates",
                               etag, family)
                return False

            pkg_list = protocol.get_vmagent_pkgs(manifests[0])

            # Set the agents to those available for download at least as
            # current as the existing agent and remove from disk any agent
            # no longer reported to the VM.
            # Note:
            #  The code leaves on disk available, but blacklisted, agents
            #  so as to preserve the state. Otherwise, those agents could be
            #  again downloaded and inappropriately retried.
            host = self._get_host_plugin(protocol=protocol)
            self._set_agents(
                [GuestAgent(pkg=pkg, host=host) for pkg in pkg_list.versions])

            self._purge_agents()
            self._filter_blacklisted_agents()

            # Return True if current agent is no longer available or an
            # agent with a higher version number is available
            return not self._is_version_eligible(base_version) \
                or (len(self.agents) > 0 and self.agents[0].version > base_version)

        except Exception as e:
            msg = u"Exception retrieving agent manifests: {0}".format(
                ustr(traceback.format_exc()))
            add_event(AGENT_NAME,
                      op=WALAEventOperation.Download,
                      version=CURRENT_VERSION,
                      is_success=False,
                      message=msg)
            return False
Esempio n. 3
0
    def get_latest_agent(self):
        """
        If autoupdate is enabled, return the most current, downloaded,
        non-blacklisted agent (if any).
        Otherwise, return None (implying to use the installed agent).
        """

        if not conf.get_autoupdate_enabled():
            return None
        
        self._load_agents()
        available_agents = [agent for agent in self.agents if agent.is_available]
        return available_agents[0] if len(available_agents) >= 1 else None
Esempio n. 4
0
    def get_latest_agent(self):
        """
        If autoupdate is enabled, return the most current, downloaded,
        non-blacklisted agent (if any).
        Otherwise, return None (implying to use the installed agent).
        """

        if not conf.get_autoupdate_enabled():
            return None

        self._load_agents()
        available_agents = [
            agent for agent in self.agents if agent.is_available
        ]
        return available_agents[0] if len(available_agents) >= 1 else None
Esempio n. 5
0
    def get_latest_agent(self):
        """
        If autoupdate is enabled, return the most current, downloaded,
        non-blacklisted agent which is not the current version (if any).
        Otherwise, return None (implying to use the installed agent).
        """

        if not conf.get_autoupdate_enabled():
            return None
        
        self._load_agents()
        available_agents = [agent for agent in self.agents
                            if agent.is_available
                            and agent.version > FlexibleVersion(AGENT_VERSION)]

        return available_agents[0] if len(available_agents) >= 1 else None
Esempio n. 6
0
    def get_latest_agent(self):
        """
        If autoupdate is enabled, return the most current, downloaded,
        non-blacklisted agent which is not the current version (if any).
        Otherwise, return None (implying to use the installed agent).
        """

        if not conf.get_autoupdate_enabled():
            return None
        
        self._load_agents()
        available_agents = [agent for agent in self.agents
                            if agent.is_available
                            and agent.version > FlexibleVersion(AGENT_VERSION)]

        return available_agents[0] if len(available_agents) >= 1 else None
Esempio n. 7
0
    def _upgrade_available(self, base_version=CURRENT_VERSION):
        # Ignore new agents if updating is disabled
        if not conf.get_autoupdate_enabled():
            return False

        now = time.time()
        if self.last_attempt_time is not None:
            next_attempt_time = self.last_attempt_time + conf.get_autoupdate_frequency()
        else:
            next_attempt_time = now
        if next_attempt_time > now:
            return False

        family = conf.get_autoupdate_gafamily()
        logger.verbose("Checking for agent family {0} updates", family)

        self.last_attempt_time = now
        try:
            protocol = self.protocol_util.get_protocol()
            manifest_list, etag = protocol.get_vmagent_manifests()
        except Exception as e:
            msg = u"Exception retrieving agent manifests: {0}".format(ustr(e))
            logger.warn(msg)
            add_event(
                AGENT_NAME,
                op=WALAEventOperation.Download,
                version=CURRENT_VERSION,
                is_success=False,
                message=msg)
            return False

        manifests = [m for m in manifest_list.vmAgentManifests \
                        if m.family == family and len(m.versionsManifestUris) > 0]
        if len(manifests) == 0:
            logger.verbose(u"Incarnation {0} has no agent family {1} updates", etag, family)
            return False

        try:
            pkg_list = protocol.get_vmagent_pkgs(manifests[0])
        except ProtocolError as e:
            msg = u"Incarnation {0} failed to get {1} package list: " \
                  u"{2}".format(
                etag,
                family,
                ustr(e))
            logger.warn(msg)
            add_event(
                AGENT_NAME,
                op=WALAEventOperation.Download,
                version=CURRENT_VERSION,
                is_success=False,
                message=msg)
            return False

        # Set the agents to those available for download at least as current
        # as the existing agent and remove from disk any agent no longer
        # reported to the VM.
        # Note:
        #  The code leaves on disk available, but blacklisted, agents so as to
        #  preserve the state. Otherwise, those agents could be again
        #  downloaded and inappropriately retried.
        host = None
        if protocol and protocol.client:
            host = protocol.client.get_host_plugin()
        self._set_agents([GuestAgent(pkg=pkg, host=host) for pkg in pkg_list.versions])
        self._purge_agents()
        self._filter_blacklisted_agents()

        # Return True if agents more recent than the current are available
        return len(self.agents) > 0 and self.agents[0].version > base_version
Esempio n. 8
0
    def run_latest(self):
        """
        This method is called from the daemon to find and launch the most
        current, downloaded agent.

        Note:
        - Most events should be tagged to the launched agent (agent_version)
        """

        if self.child_process is not None:
            raise Exception("Illegal attempt to launch multiple goal state Agent processes")

        if self.signal_handler is None:
            self.signal_handler = signal.signal(signal.SIGTERM, self.forward_signal)

        latest_agent = self.get_latest_agent()
        if latest_agent is None:
            logger.info(u"Installed Agent {0} is the most current agent", CURRENT_AGENT)
            agent_cmd = "python -u {0} -run-exthandlers".format(sys.argv[0])
            agent_dir = os.getcwd()
            agent_name = CURRENT_AGENT
            agent_version = CURRENT_VERSION
        else:
            logger.info(u"Determined Agent {0} to be the latest agent", latest_agent.name)
            agent_cmd = latest_agent.get_agent_cmd()
            agent_dir = latest_agent.get_agent_dir()
            agent_name = latest_agent.name
            agent_version = latest_agent.version

        try:

            # Launch the correct Python version for python-based agents
            cmds = textutil.safe_shlex_split(agent_cmd)
            if cmds[0].lower() == "python":
                cmds[0] = get_python_cmd()
                agent_cmd = " ".join(cmds)

            self._evaluate_agent_health(latest_agent)

            self.child_process = subprocess.Popen(
                cmds,
                cwd=agent_dir,
                stdout=sys.stdout,
                stderr=sys.stderr,
                env=os.environ)

            logger.verbose(u"Agent {0} launched with command '{1}'", agent_name, agent_cmd)

            # If the most current agent is the installed agent and update is enabled,
            # assume updates are likely available and poll every second.
            # This reduces the start-up impact of finding / launching agent updates on
            # fresh VMs.
            if latest_agent is None and conf.get_autoupdate_enabled():
                poll_interval = 1
            else:
                poll_interval = CHILD_POLL_INTERVAL

            ret = None
            start_time = time.time()
            while (time.time() - start_time) < CHILD_HEALTH_INTERVAL:
                time.sleep(poll_interval)
                ret = self.child_process.poll()
                if ret is not None:
                    break

            if ret is None or ret <= 0:
                msg = u"Agent {0} launched with command '{1}' is successfully running".format(
                    agent_name,
                    agent_cmd)
                logger.info(msg)
                add_event(
                    AGENT_NAME,
                    version=agent_version,
                    op=WALAEventOperation.Enable,
                    is_success=True,
                    message=msg)

                if ret is None:
                    ret = self.child_process.wait()

            else:
                msg = u"Agent {0} launched with command '{1}' failed with return code: {2}".format(
                    agent_name,
                    agent_cmd,
                    ret)
                logger.warn(msg)
                add_event(
                    AGENT_NAME,
                    version=agent_version,
                    op=WALAEventOperation.Enable,
                    is_success=False,
                    message=msg)

            if ret is not None and ret > 0:
                msg = u"Agent {0} launched with command '{1}' returned code: {2}".format(
                    agent_name,
                    agent_cmd,
                    ret)
                logger.warn(msg)
                if latest_agent is not None:
                    latest_agent.mark_failure()

        except Exception as e:
            msg = u"Agent {0} launched with command '{1}' failed with exception: {2}".format(
                agent_name,
                agent_cmd,
                ustr(e))
            logger.warn(msg)
            add_event(
                AGENT_NAME,
                version=agent_version,
                op=WALAEventOperation.Enable,
                is_success=False,
                message=msg)
            if latest_agent is not None:
                latest_agent.mark_failure(is_fatal=True)

        self.child_process = None
        return
Esempio n. 9
0
    def _upgrade_available(self, base_version=CURRENT_VERSION):
        # Ignore new agents if updating is disabled
        if not conf.get_autoupdate_enabled():
            return False

        now = time.time()
        if self.last_attempt_time is not None:
            next_attempt_time = self.last_attempt_time + conf.get_autoupdate_frequency(
            )
        else:
            next_attempt_time = now
        if next_attempt_time > now:
            return False

        family = conf.get_autoupdate_gafamily()
        logger.info("Checking for agent family {0} updates", family)

        self.last_attempt_time = now
        try:
            protocol = self.protocol_util.get_protocol()
            manifest_list, etag = protocol.get_vmagent_manifests()
        except Exception as e:
            msg = u"Exception retrieving agent manifests: {0}".format(ustr(e))
            logger.warn(msg)
            add_event(AGENT_NAME,
                      op=WALAEventOperation.Download,
                      version=CURRENT_VERSION,
                      is_success=False,
                      message=msg)
            return False

        manifests = [m for m in manifest_list.vmAgentManifests \
                        if m.family == family and len(m.versionsManifestUris) > 0]
        if len(manifests) == 0:
            logger.info(u"Incarnation {0} has no agent family {1} updates",
                        etag, family)
            return False

        try:
            pkg_list = protocol.get_vmagent_pkgs(manifests[0])
        except ProtocolError as e:
            msg = u"Incarnation {0} failed to get {1} package list: " \
                  u"{2}".format(
                etag,
                family,
                ustr(e))
            logger.warn(msg)
            add_event(AGENT_NAME,
                      op=WALAEventOperation.Download,
                      version=CURRENT_VERSION,
                      is_success=False,
                      message=msg)
            return False

        # Set the agents to those available for download at least as current
        # as the existing agent and remove from disk any agent no longer
        # reported to the VM.
        # Note:
        #  The code leaves on disk available, but blacklisted, agents so as to
        #  preserve the state. Otherwise, those agents could be again
        #  downloaded and inappropriately retried.
        self._set_agents([GuestAgent(pkg=pkg) for pkg in pkg_list.versions])
        self._purge_agents()
        self._filter_blacklisted_agents()

        # Return True if agents more recent than the current are available
        return len(self.agents) > 0 and self.agents[0].version > base_version
Esempio n. 10
0
    def _upgrade_available(self, base_version=CURRENT_VERSION):
        # Emit an event expressing the state of AutoUpdate
        # Note:
        # - Duplicate events get suppressed; state transitions always emit
        add_event(AGENT_NAME,
                  version=CURRENT_VERSION,
                  op=WALAEventOperation.AutoUpdate,
                  is_success=conf.get_autoupdate_enabled())

        # Ignore new agents if updating is disabled
        if not conf.get_autoupdate_enabled():
            return False

        now = time.time()
        if self.last_attempt_time is not None:
            next_attempt_time = self.last_attempt_time + \
                                    conf.get_autoupdate_frequency()
        else:
            next_attempt_time = now
        if next_attempt_time > now:
            return False

        family = conf.get_autoupdate_gafamily()
        logger.verbose("Checking for agent family {0} updates", family)

        self.last_attempt_time = now
        protocol = self.protocol_util.get_protocol()

        for update_goal_state in [False, True]:
            try:
                if update_goal_state:
                    protocol.update_goal_state(forced=True)

                manifest_list, etag = protocol.get_vmagent_manifests()

                manifests = [m for m in manifest_list.vmAgentManifests \
                                if m.family == family and \
                                    len(m.versionsManifestUris) > 0]
                if len(manifests) == 0:
                    logger.verbose(u"Incarnation {0} has no {1} agent updates",
                                   etag, family)
                    return False

                pkg_list = protocol.get_vmagent_pkgs(manifests[0])

                # Set the agents to those available for download at least as
                # current as the existing agent and remove from disk any agent
                # no longer reported to the VM.
                # Note:
                #  The code leaves on disk available, but blacklisted, agents
                #  so as to preserve the state. Otherwise, those agents could be
                #  again downloaded and inappropriately retried.
                host = self._get_host_plugin(protocol=protocol)
                self._set_agents([GuestAgent(pkg=pkg, host=host) \
                                     for pkg in pkg_list.versions])

                self._purge_agents()
                self._evaluate_deployments()
                self._filter_blacklisted_agents()

                # Return True if current agent is no longer available or an
                # agent with a higher version number is available
                return not self._is_version_eligible(base_version) \
                    or (len(self.agents) > 0 \
                        and self.agents[0].version > base_version)

            except Exception as e:
                if isinstance(e, ResourceGoneError):
                    continue

                msg = u"Exception retrieving agent manifests: {0}".format(
                    ustr(e))
                logger.warn(msg)
                add_event(AGENT_NAME,
                          op=WALAEventOperation.Download,
                          version=CURRENT_VERSION,
                          is_success=False,
                          message=msg)
                return False
Esempio n. 11
0
    def run_latest(self, child_args=None):
        """
        This method is called from the daemon to find and launch the most
        current, downloaded agent.

        Note:
        - Most events should be tagged to the launched agent (agent_version)
        """

        if self.child_process is not None:
            raise Exception(
                "Illegal attempt to launch multiple goal state Agent processes"
            )

        if self.signal_handler is None:
            self.signal_handler = signal.signal(signal.SIGTERM,
                                                self.forward_signal)

        latest_agent = self.get_latest_agent()
        if latest_agent is None:
            logger.info(u"Installed Agent {0} is the most current agent",
                        CURRENT_AGENT)
            agent_cmd = "python -u {0} -run-exthandlers".format(sys.argv[0])
            agent_dir = os.getcwd()
            agent_name = CURRENT_AGENT
            agent_version = CURRENT_VERSION
        else:
            logger.info(u"Determined Agent {0} to be the latest agent",
                        latest_agent.name)
            agent_cmd = latest_agent.get_agent_cmd()
            agent_dir = latest_agent.get_agent_dir()
            agent_name = latest_agent.name
            agent_version = latest_agent.version

        if child_args is not None:
            agent_cmd = "{0} {1}".format(agent_cmd, child_args)

        try:

            # Launch the correct Python version for python-based agents
            cmds = textutil.safe_shlex_split(agent_cmd)
            if cmds[0].lower() == "python":
                cmds[0] = get_python_cmd()
                agent_cmd = " ".join(cmds)

            self._evaluate_agent_health(latest_agent)

            self.child_process = subprocess.Popen(cmds,
                                                  cwd=agent_dir,
                                                  stdout=sys.stdout,
                                                  stderr=sys.stderr,
                                                  env=os.environ)

            logger.verbose(u"Agent {0} launched with command '{1}'",
                           agent_name, agent_cmd)

            # If the most current agent is the installed agent and update is enabled,
            # assume updates are likely available and poll every second.
            # This reduces the start-up impact of finding / launching agent updates on
            # fresh VMs.
            if latest_agent is None and conf.get_autoupdate_enabled():
                poll_interval = 1
            else:
                poll_interval = CHILD_POLL_INTERVAL

            ret = None
            start_time = time.time()
            while (time.time() - start_time) < CHILD_HEALTH_INTERVAL:
                time.sleep(poll_interval)
                ret = self.child_process.poll()
                if ret is not None:
                    break

            if ret is None or ret <= 0:
                msg = u"Agent {0} launched with command '{1}' is successfully running".format(
                    agent_name, agent_cmd)
                logger.info(msg)
                add_event(AGENT_NAME,
                          version=agent_version,
                          op=WALAEventOperation.Enable,
                          is_success=True,
                          message=msg)

                if ret is None:
                    ret = self.child_process.wait()

            else:
                msg = u"Agent {0} launched with command '{1}' failed with return code: {2}".format(
                    agent_name, agent_cmd, ret)
                logger.warn(msg)
                add_event(AGENT_NAME,
                          version=agent_version,
                          op=WALAEventOperation.Enable,
                          is_success=False,
                          message=msg)

            if ret is not None and ret > 0:
                msg = u"Agent {0} launched with command '{1}' returned code: {2}".format(
                    agent_name, agent_cmd, ret)
                logger.warn(msg)
                if latest_agent is not None:
                    latest_agent.mark_failure(is_fatal=True)

        except Exception as e:
            # Ignore child errors during termination
            if self.running:
                msg = u"Agent {0} launched with command '{1}' failed with exception: {2}".format(
                    agent_name, agent_cmd, ustr(e))
                logger.warn(msg)
                add_event(AGENT_NAME,
                          version=agent_version,
                          op=WALAEventOperation.Enable,
                          is_success=False,
                          message=msg)
                if latest_agent is not None:
                    latest_agent.mark_failure(is_fatal=True)

        self.child_process = None
        return
Esempio n. 12
0
    def _upgrade_available(self, base_version=CURRENT_VERSION):
        # Emit an event expressing the state of AutoUpdate
        # Note:
        # - Duplicate events get suppressed; state transitions always emit
        add_event(
            AGENT_NAME,
            version=CURRENT_VERSION,
            op=WALAEventOperation.AutoUpdate,
            is_success=conf.get_autoupdate_enabled())

        # Ignore new agents if updating is disabled
        if not conf.get_autoupdate_enabled():
            return False

        now = time.time()
        if self.last_attempt_time is not None:
            next_attempt_time = self.last_attempt_time + \
                                    conf.get_autoupdate_frequency()
        else:
            next_attempt_time = now
        if next_attempt_time > now:
            return False

        family = conf.get_autoupdate_gafamily()
        logger.verbose("Checking for agent family {0} updates", family)

        self.last_attempt_time = now
        protocol = self.protocol_util.get_protocol()

        for update_goal_state in [False, True]:
            try:
                if update_goal_state:
                    protocol.update_goal_state(forced=True)

                manifest_list, etag = protocol.get_vmagent_manifests()

                manifests = [m for m in manifest_list.vmAgentManifests \
                                if m.family == family and \
                                    len(m.versionsManifestUris) > 0]
                if len(manifests) == 0:
                    logger.verbose(u"Incarnation {0} has no {1} agent updates",
                                    etag, family)
                    return False

                pkg_list = protocol.get_vmagent_pkgs(manifests[0])

                # Set the agents to those available for download at least as
                # current as the existing agent and remove from disk any agent
                # no longer reported to the VM.
                # Note:
                #  The code leaves on disk available, but blacklisted, agents
                #  so as to preserve the state. Otherwise, those agents could be
                #  again downloaded and inappropriately retried.
                host = self._get_host_plugin(protocol=protocol)
                self._set_agents([GuestAgent(pkg=pkg, host=host) \
                                     for pkg in pkg_list.versions])

                self._purge_agents()
                self._filter_blacklisted_agents()

                # Return True if current agent is no longer available or an
                # agent with a higher version number is available
                return not self._is_version_eligible(base_version) \
                    or (len(self.agents) > 0 \
                        and self.agents[0].version > base_version)

            except Exception as e:
                if isinstance(e, ResourceGoneError):
                    continue

                msg = u"Exception retrieving agent manifests: {0}".format(
                            ustr(traceback.format_exc()))
                logger.warn(msg)
                add_event(
                    AGENT_NAME,
                    op=WALAEventOperation.Download,
                    version=CURRENT_VERSION,
                    is_success=False,
                    message=msg)
                return False