def tearDownModule(): client = _get_salt_client() global ran_states_cntr logger.debug("Running tearDownModule") logger.info("COUNTER: Ran totally: %d States", (sum(ran_states_cntr.values()))) logger.info("COUNTER: By state declaration: %s", ran_states_cntr) logger.info("COUNTER: size of counter in bytes %d", sys.getsizeof(ran_states_cntr)) client("state.sls", "test.teardown")
def tearDownModule(): client = _get_salt_client() global ran_states_cntr logger.debug("Running tearDownModule") logger.info('COUNTER: Ran totally: %d States', (sum(ran_states_cntr.values()))) logger.info('COUNTER: By state declaration: %s', ran_states_cntr) logger.info('COUNTER: size of counter in bytes %d', sys.getsizeof(ran_states_cntr)) client('state.sls', 'test.teardown')
def add_role(target, role): target_grains = client().cmd(target, 'grains.items') for host, grains in target_grains.items(): roles = grains.get('roles', []) if role not in roles: roles.append(role) result = client().cmd( host, 'grains.setval', ['roles', json.dumps(roles)]) print result else: print {host: {'roles': roles}}
def add_role(target, role): target_grains = client().cmd(target, 'grains.items') for host, grains in target_grains.items(): roles = grains.get('roles', []) if role not in roles: roles.append(role) result = client().cmd(host, 'grains.setval', ['roles', json.dumps(roles)]) print result else: print {host: {'roles': roles}}
def run_salt_module(module_name, *args, **kwargs): try: output = client(module_name, *args, **kwargs) except Exception as err: logger.error("Catch error: %s", err, exc_info=True) raise Exception('Module:{0} error: {1}'.format(module_name, err)) return output
def sls(self, states): """ Apply specified list of states. """ logger.debug("Run states: %s", ', '.join(states)) try: output = client('state.sls', ','.join(states)) except Exception, err: logger.error("Catch error: %s", err, exc_info=True) self.fail('states: %s. error: %s' % ('.'.join(states), err))
def highstate(target, *args, **kwargs): kwargs.setdefault('timeout', 30 * 60) logging.info('%s:state.highstate args:%s kwargs:%s', target, args, kwargs) result = client().cmd(target, 'state.highstate', *args, **kwargs) for k, d in result.iteritems(): logging.info(k) if isinstance(d, dict): for key, value in d.iteritems(): logging.info(key) logging.info('%s', value) else: logging.info('%s', d)
def render_state_template(state): """ Return de-serialized data of specified state name """ tmp = tempfile.NamedTemporaryFile(delete=False) tmp.close() state_path = state.replace(".", "/") for path_template in ("salt://{0}.sls", "salt://{0}/init.sls"): source = path_template.format(state_path) client("cp.get_template", source, tmp.name) with open(tmp.name) as yaml_fh: try: data = yaml.safe_load(yaml_fh) if data: logger.debug("Found state %s, return dict size %d", source, len(data)) os.unlink(tmp.name) return data logger.debug("%s don't seem to exists", source) except Exception: logger.error("Can't parse YAML %s", source, exc_info=True) logger.error("Couldn't get content of %s", state) os.unlink(tmp.name) return {}
def render_state_template(state): """ Return de-serialized data of specified state name """ tmp = tempfile.NamedTemporaryFile(delete=False) tmp.close() state_path = state.replace('.', '/') for path_template in ('salt://{0}.sls', 'salt://{0}/init.sls'): source = path_template.format(state_path) client('cp.get_template', source, tmp.name) with open(tmp.name) as yaml_fh: try: data = yaml.safe_load(yaml_fh) if data: logger.debug("Found state %s, return dict size %d", source, len(data)) os.unlink(tmp.name) return data logger.debug("%s don't seem to exists", source) except Exception: logger.error("Can't parse YAML %s", source, exc_info=True) logger.error("Couldn't get content of %s", state) os.unlink(tmp.name) return {}
def list_user_space_processes(): """ return all running processes on minion """ client = _get_salt_client() result = client('status.procs') output = {} for pid in result: name = result[pid]['cmd'] # kernel process are like this: [xfs], ignore them if name.startswith('['): continue output[int(pid)] = name return output
def setUpModule(): """ Prepare minion for tests, this is executed only once time. """ client = _get_salt_client() logger.debug("Running setUpModule") # force HOME to be root directory os.environ["HOME"] = pwd.getpwnam("root").pw_dir client("saltutil.sync_all") client("saltutil.refresh_modules") logger.info("Rendering all *.test SLS to quickly find malformed ones") for sls in all_states: if sls.endswith(".test"): try: ret = client("state.show_sls", sls) except Exception as err: logger.error("Catch error: %s", err, exc_info=True) raise if isinstance(ret, list): # if render is okay, it returns dict, else, it returns list # of msg error raise Exception(ret) try: if client("pkg_installed.exists"): logger.info( "pkg_installed snapshot was found, skip setUpModule(). If you " "want to repeat the cleanup process, run " "'pkg_installed.forget'" ) return except KeyError, err: logger.debug("Catch error: %s", err, exc_info=True) # salt.client.Caller don't refresh the list of available modules # after running saltutil.sync_all, exit after doing it. client("saltutil.sync_all") logger.warning("Please re-execute: '%s'", " ".join(sys.argv)) sys.exit(0)
def run_cmd(target, *args, **kwargs): kwargs.setdefault('timeout', 30 * 60) result_required = kwargs.pop('result_required', True) logging.info('%s:cmd.run_all args:%s kwargs%s', target, args, kwargs) result = client().cmd(target, 'cmd.run_all', *args, **kwargs) if result_required and not result: raise RunCmdException('TIMEOUT: %s:cmd.run_all args:%s kwargs%s' % (target, args, kwargs)) for k, d in result.iteritems(): logging.info(k) try: stderr = d.get('stderr', '<not in result>') except: raise RunCmdException(d) # d is a traceback, not a dict else: logging.info('stderr:%s', stderr) logging.info('stdout:%s', d.get('stdout', '<not in result>')) retcode = d.get('retcode', '<not in result>') if retcode != 0: raise RunCmdException('nonzero retcode %s' % retcode)
def setUpModule(): """ Prepare minion for tests, this is executed only once time. """ client = _get_salt_client() logger.debug("Running setUpModule") # force HOME to be root directory os.environ['HOME'] = pwd.getpwnam('root').pw_dir client('saltutil.sync_all') client('saltutil.refresh_modules') logger.info("Rendering all *.test SLS to quickly find malformed ones") for sls in all_states: if sls.endswith('.test'): try: ret = client('state.show_sls', sls) except Exception as err: logger.error("Catch error: %s", err, exc_info=True) raise if isinstance(ret, list): # if render is okay, it returns dict, else, it returns list # of msg error raise Exception(ret) try: if client('pkg_installed.exists'): logger.info( "pkg_installed snapshot was found, skip setUpModule(). If you " "want to repeat the cleanup process, run " "'pkg_installed.forget'") return except KeyError, err: logger.debug("Catch error: %s", err, exc_info=True) # salt.client.Caller don't refresh the list of available modules # after running saltutil.sync_all, exit after doing it. client('saltutil.sync_all') logger.warning("Please re-execute: '%s'", ' '.join(sys.argv)) sys.exit(0)
def get_users(): client = _get_salt_client() return set(user['name'] for user in client('user.getent'))
def get_groups(): """ return a set of groups """ client = _get_salt_client() return set(group['name'] for group in client('group.getent', True))
def get_groups(): """ return a set of groups """ client = _get_salt_client() return set(group["name"] for group in client("group.getent", True))
unclean = set() NO_TEST_STRING = '-*- ci-automatic-discovery: off -*-' IGNORED_RESULTS = ('One or more requisite failed', ) def _get_salt_client(config_file='/root/salt/states/test/minion'): opts = salt.config.minion_config(config_file) opts['id'] = socket.gethostname() caller = salt.client.Caller(mopts=opts) return caller.function client = _get_salt_client() all_states = client('cp.list_states') ran_states_cntr = collections.Counter() def if_change(result): """ Check if changed occured in :func:`test_clean` :param result: result from :func:`salt.client.LocalClient.cmd` :type result: dict :return: True if any change :rtype: bool """ for key in result: try: if result[key]['changes'] != {}: return True
def celery_maintenance_mode(target, mode=1): target = target or pillar_get('consumeraffairs_celery_servers') return client().cmd( target, 'grains.setval', ['celery_maintenance_mode', mode])
class States(unittest.TestCase): """ Common logic to all Integration test class """ __metaclass__ = TestStateMeta def _check_same_status(self, original, function, messages): """ Checks that a set of things is invariant between runs. If `original` is None, `function` gets called and its return value saved in `original`. If not, `function` is called and its result value is compared against `original`. If they differ, the test fails. :params original: a set or None :params function: a function that returns a set or a generator function :params messages: a list of 3 strings used for logging. - the first element must take an integer and is used when the set is first filled in (original is None) - the second element must take an integer and is used when the current value of `function` is compared to `original` - the third value must take a string, and it's used for the failure message """ global clean_up_failed # check processes if not original: # function can be a generator function, so get a set of it original.update(set(function())) logger.debug(messages[0], len(original)) logger.debug('Stat: size of stored list: %d', sys.getsizeof(original)) else: current = function() global unclean for i, e in enumerate(current): if e not in original: unclean.add(e) logger.debug(messages[1], i) if unclean: clean_up_failed = True return messages[2] % os.linesep.join(unclean) return "" @staticmethod def get_rss(top=10): """Get processeses with highest memory usages """ procs = [] if HAS_PSUTIL: for p in psutil.process_iter(): try: if int(psutil.__version__.split('.')[0]) < 2: procs.append({ "name": p.name, "cmdline": p.cmdline, "memory_percent": p.get_memory_percent(), }) else: procs.append({ "name": p.name, "cmdline": p.cmdline(), "memory_percent": p.memory_percent(), }) except psutil.NoSuchProcess: logger.debug('Process %s termninated before return info', p.name) procs = sorted(procs, key=lambda p: p["memory_percent"], reverse=True)[0:top] return procs def setUp(self): """ Clean up the minion before each test. """ global is_clean, clean_up_failed, process_list global files_list, users_list, groups_list if clean_up_failed: self.skipTest("Previous cleanup failed") else: logger.debug("Go ahead, cleanup never failed before") if is_clean: logger.debug("Don't cleanup, it's already done") return try: self.sls(self.absent) except AssertionError, err: clean_up_failed = True logger.error("Can't run all .absent: %s", err) self.fail(err) # Go back on the same installed packages as after :func:`setUpClass` logger.info("Unfreeze installed packages") try: output = client('pkg_installed.revert', True) except Exception, err: clean_up_failed = True logger.error("Catch error: %s", err, exc_info=True) self.fail(err)
class States(unittest.TestCase): """ Common logic to all Integration test class """ __metaclass__ = TestStateMeta def _check_same_status(self, original, function, messages): """ Checks that a set of things is invariant between runs. If `original` is None, `function` gets called and its return value saved in `original`. If not, `function` is called and its result value is compared against `original`. If they differ, the test fails. :params original: a set or None :params function: a function that returns a set or a generator function :params messages: a list of 3 strings used for logging. - the first element must take an integer and is used when the set is first filled in (original is None) - the second element must take an integer and is used when the current value of `function` is compared to `original` - the third value must take a string, and it's used for the failure message """ global clean_up_failed # check processes if not original: # function can be a generator function, so get a set of it original.update(set(function())) logger.debug(messages[0], len(original)) logger.debug('Stat: size of stored list: %d', sys.getsizeof(original)) else: current = function() global unclean for i, e in enumerate(current): if e not in original: unclean.add(e) logger.debug(messages[1], i) if unclean: clean_up_failed = True return messages[2] % os.linesep.join(unclean) return "" @staticmethod def get_rss(): # http://man7.org/linux/man-pages/man5/proc.5.html with open('/proc/{0}/stat'.format(os.getpid())) as f: return int(f.readline().split()[23]) def setUp(self): """ Clean up the minion before each test. """ global is_clean, clean_up_failed, process_list global files_list, users_list, groups_list if clean_up_failed: self.skipTest("Previous cleanup failed") else: logger.debug("Go ahead, cleanup never failed before") if is_clean: logger.debug("Don't cleanup, it's already done") return try: self.sls(self.absent) except AssertionError, err: clean_up_failed = True logger.error("Can't run all .absent: %s", err) self.fail(err) # Go back on the same installed packages as after :func:`setUpClass` logger.info("Unfreeze installed packages") try: output = client('pkg_installed.revert', True) except Exception, err: clean_up_failed = True logger.error("Catch error: %s", err, exc_info=True) self.fail(err)
def get_users(): client = _get_salt_client() return set(user["name"] for user in client("user.getent"))
users_list = set() unclean = set() NO_TEST_STRING = '-*- ci-automatic-discovery: off -*-' IGNORED_RESULTS = ('One or more requisite failed',) def _get_salt_client(config_file='/root/salt/states/test/minion'): opts = salt.config.minion_config(config_file) opts['id'] = socket.gethostname() caller = salt.client.Caller(mopts=opts) return caller.function client = _get_salt_client() all_states = client('cp.list_states') ran_states_cntr = collections.Counter() def if_change(result): """ Check if changed occured in :func:`test_clean` :param result: result from :func:`salt.client.LocalClient.cmd` :type result: dict :return: True if any change :rtype: bool """ for key in result: try: if result[key]['changes'] != {}: return True
def celery_maintenance_mode(target, mode=1): target = target or pillar_get('consumeraffairs_celery_servers') return client().cmd(target, 'grains.setval', ['celery_maintenance_mode', mode])
unclean = set() NO_TEST_STRING = "-*- ci-automatic-discovery: off -*-" IGNORED_RESULTS = ("One or more requisite failed",) def _get_salt_client(config_file="/root/salt/states/test/minion"): opts = salt.config.minion_config(config_file) opts["id"] = socket.gethostname() caller = salt.client.Caller(mopts=opts) return caller.function client = _get_salt_client() all_states = client("cp.list_states") ran_states_cntr = collections.Counter() def if_change(result): """ Check if changed occured in :func:`test_clean` :param result: result from :func:`salt.client.LocalClient.cmd` :type result: dict :return: True if any change :rtype: bool """ for key in result: try: if result[key]["changes"] != {}: return True