def init_cgroups(): # Track metrics for the roll-up cgroup and for the agent cgroup try: CGroupsTelemetry.track_cgroup(CGroups.for_extension("")) CGroupsTelemetry.track_agent() except Exception as e: logger.error("monitor: Exception tracking wrapper and agent: {0} [{1}]", e, traceback.format_exc())
def init_cgroups(): # Track metrics for the roll-up cgroup and for the agent cgroup try: CGroupsTelemetry.track_cgroup(CGroups.for_extension("")) CGroupsTelemetry.track_agent() except Exception as e: logger.error( "monitor: Exception tracking wrapper and agent: {0} [{1}]", e, traceback.format_exc())
def init_cgroups(): # Track metrics for the wrapper cgroup and for the agent cgroup try: # This creates the wrapper cgroup for everything under agent, # /sys/fs/cgroup/{cpu,memory}/WALinuxAgent/ # There is no need in tracking this cgroup, as it only serves # as an umbrella for the agent and extensions cgroups CGroups.for_extension("") # This creates the agent's cgroup (for the daemon and extension handler) # /sys/fs/cgroup/{cpu,memory}/WALinuxAgent/WALinuxAgent # If the system is using systemd, it would have already been set up under /system.slice CGroupsTelemetry.track_agent() except Exception as e: # when a hierarchy is not mounted, we raise an exception # and we should therefore only issue a warning, since this # is not unexpected logger.warn("Monitor: cgroups not initialized: {0}", ustr(e)) logger.verbose(traceback.format_exc())
def init_cgroups(): # Track metrics for the roll-up cgroup and for the agent cgroup try: CGroupsTelemetry.track_cgroup(CGroups.for_extension("")) CGroupsTelemetry.track_agent() except Exception as e: # when a hierarchy is not mounted, we raise an exception # and we should therefore only issue a warning, since this # is not unexpected logger.warn("Monitor: cgroups not initialized: {0}", ustr(e)) logger.verbose(traceback.format_exc())
def init_cgroups(): # Track metrics for the roll-up cgroup and for the agent cgroup try: CGroupsTelemetry.track_cgroup(CGroups.for_extension("")) CGroupsTelemetry.track_agent() except Exception as e: # when a hierarchy is not mounted, we raise an exception # and we should therefore only issue a warning, since this # is not unexpected logger.warn("Monitor: cgroups not initialized: {0}", ustr(e)) logger.verbose(traceback.format_exc())
def launch_command(self, cmd, timeout=300, extension_error_code=1000, env=None): begin_utc = datetime.datetime.utcnow() self.logger.verbose("Launch command: [{0}]", cmd) base_dir = self.get_base_dir() if env is None: env = {} env.update(os.environ) try: # This should be .run(), but due to the wide variety # of Python versions we must support we must use .communicate(). # Some extensions erroneously begin cmd with a slash; don't interpret those # as root-relative. (Issue #1170) full_path = os.path.join(base_dir, cmd.lstrip(os.path.sep)) def pre_exec_function(): """ Change process state before the actual target process is started. Effectively, this runs between the fork() and the exec() of sub-process creation. :return: """ os.setsid() CGroups.add_to_extension_cgroup(self.ext_handler.name) process = subprocess.Popen(full_path, shell=True, cwd=base_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, preexec_fn=pre_exec_function) except OSError as e: raise ExtensionError("Failed to launch '{0}': {1}".format(full_path, e.strerror), code=extension_error_code) cg = CGroups.for_extension(self.ext_handler.name) CGroupsTelemetry.track_extension(self.ext_handler.name, cg) msg = capture_from_process(process, cmd, timeout, extension_error_code) ret = process.poll() if ret is None: raise ExtensionError("Process {0} was not terminated: {1}\n{2}".format(process.pid, cmd, msg), code=extension_error_code) if ret != 0: raise ExtensionError("Non-zero exit code: {0}, {1}\n{2}".format(ret, cmd, msg), code=extension_error_code) duration = elapsed_milliseconds(begin_utc) log_msg = "{0}\n{1}".format(cmd, "\n".join([line for line in msg.split('\n') if line != ""])) self.logger.verbose(log_msg) self.report_event(message=log_msg, duration=duration, log_event=False)
def launch_command(self, cmd, timeout=300, extension_error_code=1000, env=None): begin_utc = datetime.datetime.utcnow() self.logger.verbose("Launch command: [{0}]", cmd) base_dir = self.get_base_dir() if env is None: env = {} env.update(os.environ) try: # This should be .run(), but due to the wide variety # of Python versions we must support we must use .communicate(). # Some extensions erroneously begin cmd with a slash; don't interpret those # as root-relative. (Issue #1170) full_path = os.path.join(base_dir, cmd.lstrip(os.path.sep)) def pre_exec_function(): """ Change process state before the actual target process is started. Effectively, this runs between the fork() and the exec() of sub-process creation. :return: """ os.setsid() CGroups.add_to_extension_cgroup(self.ext_handler.name) process = subprocess.Popen(full_path, shell=True, cwd=base_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, preexec_fn=pre_exec_function) except OSError as e: raise ExtensionError("Failed to launch '{0}': {1}".format(full_path, e.strerror), code=extension_error_code) cg = CGroups.for_extension(self.ext_handler.name) CGroupsTelemetry.track_extension(self.ext_handler.name, cg) msg = capture_from_process(process, cmd, timeout, extension_error_code) ret = process.poll() if ret is None: raise ExtensionError("Process {0} was not terminated: {1}\n{2}".format(process.pid, cmd, msg), code=extension_error_code) if ret != 0: raise ExtensionError("Non-zero exit code: {0}, {1}\n{2}".format(ret, cmd, msg), code=extension_error_code) duration = elapsed_milliseconds(begin_utc) log_msg = "{0}\n{1}".format(cmd, "\n".join([line for line in msg.split('\n') if line != ""])) self.logger.verbose(log_msg) self.report_event(message=log_msg, duration=duration, log_event=False)
def assert_limits(self, _, patch_set_cpu, patch_set_memory_limit, patch_get_enforce, patch_add_event, ext_name, expected_cpu_limit, limits_enforced=True, exception_raised=False): should_limit = expected_cpu_limit > 0 patch_get_enforce.return_value = limits_enforced if exception_raised: patch_set_memory_limit.side_effect = CGroupsException( 'set_memory_limit error') try: cg = CGroups.for_extension(ext_name) cg.set_limits() if exception_raised: self.fail('exception expected') except CGroupsException: if not exception_raised: self.fail('exception not expected') self.assertEqual(should_limit, patch_set_cpu.called) self.assertEqual(should_limit, patch_set_memory_limit.called) self.assertEqual(should_limit, patch_add_event.called) if should_limit: actual_cpu_limit = patch_set_cpu.call_args[0][0] actual_memory_limit = patch_set_memory_limit.call_args[0][0] event_kw_args = patch_add_event.call_args[1] self.assertEqual(expected_cpu_limit, actual_cpu_limit) self.assertTrue(actual_memory_limit >= DEFAULT_MEM_LIMIT_MIN_MB) self.assertEqual(event_kw_args['op'], 'SetCGroupsLimits') self.assertEqual(event_kw_args['is_success'], not exception_raised) self.assertTrue( '{0}%'.format(expected_cpu_limit) in event_kw_args['message']) self.assertTrue(ext_name in event_kw_args['message']) self.assertEqual( exception_raised, 'set_memory_limit error' in event_kw_args['message'])
def test_telemetry_instantiation_as_superuser(self): """ Tracking a new cgroup for an extension; collect all metrics. """ # Record initial state initial_cgroup = make_self_cgroups() # Put the process into a different cgroup, consume some resources, ensure we see them end-to-end test_cgroup = CGroups.for_extension("agent_unittest") test_cgroup.add(os.getpid()) self.assertNotEqual(initial_cgroup.cgroups['cpu'], test_cgroup.cgroups['cpu']) self.assertNotEqual(initial_cgroup.cgroups['memory'], test_cgroup.cgroups['memory']) self.exercise_telemetry_instantiation(test_cgroup) # Restore initial state CGroupsTelemetry.stop_tracking("agent_unittest") initial_cgroup.add(os.getpid())
def test_telemetry_instantiation_as_superuser(self): """ Tracking a new cgroup for an extension; collect all metrics. """ # Record initial state initial_cgroup = make_self_cgroups() # Put the process into a different cgroup, consume some resources, ensure we see them end-to-end test_cgroup = CGroups.for_extension("agent_unittest") test_cgroup.add(os.getpid()) self.assertNotEqual(initial_cgroup.cgroups['cpu'], test_cgroup.cgroups['cpu']) self.assertNotEqual(initial_cgroup.cgroups['memory'], test_cgroup.cgroups['memory']) self.exercise_telemetry_instantiation(test_cgroup) # Restore initial state CGroupsTelemetry.stop_tracking("agent_unittest") initial_cgroup.add(os.getpid())
def assert_limits(self, _, patch_set_cpu, patch_set_memory_limit, patch_get_enforce, patch_add_event, ext_name, expected_cpu_limit, limits_enforced=True, exception_raised=False): should_limit = expected_cpu_limit > 0 patch_get_enforce.return_value = limits_enforced if exception_raised: patch_set_memory_limit.side_effect = CGroupsException('set_memory_limit error') try: cg = CGroups.for_extension(ext_name) cg.set_limits() if exception_raised: self.fail('exception expected') except CGroupsException: if not exception_raised: self.fail('exception not expected') self.assertEqual(should_limit, patch_set_cpu.called) self.assertEqual(should_limit, patch_set_memory_limit.called) self.assertEqual(should_limit, patch_add_event.called) if should_limit: actual_cpu_limit = patch_set_cpu.call_args[0][0] actual_memory_limit = patch_set_memory_limit.call_args[0][0] event_kw_args = patch_add_event.call_args[1] self.assertEqual(expected_cpu_limit, actual_cpu_limit) self.assertTrue(actual_memory_limit >= DEFAULT_MEM_LIMIT_MIN_MB) self.assertEqual(event_kw_args['op'], 'SetCGroupsLimits') self.assertEqual(event_kw_args['is_success'], not exception_raised) self.assertTrue('{0}%'.format(expected_cpu_limit) in event_kw_args['message']) self.assertTrue(ext_name in event_kw_args['message']) self.assertEqual(exception_raised, 'set_memory_limit error' in event_kw_args['message'])