def _recursive_compare(v1, v2): ''' Return v1 == v2. Compares list, dict, recursively. ''' if isinstance(v1, list): if v2 is None: v2 = [] if len(v1) != len(v2): return False v1.sort(key=_id_or_key) v2.sort(key=_id_or_key) for x, y in zip(v1, v2): if not _recursive_compare(x, y): return False return True elif isinstance(v1, dict): if v2 is None: v2 = {} v1 = dict(v1) v2 = dict(v2) if sorted(v1) != sorted(v2): return False for k in v1: if not _recursive_compare(v1[k], v2[k]): return False return True else: return v1 == v2
def test_trim_files(self): source = 'file.tar.gz' tmp_dir = 'temp' files = [ '\n'.join(['file1'] * 200), '\n'.join(['file2'] * 200), 'this\nthat\nother', 'this\nthat\nother', 'this\nthat\nother' ] trim_opt = [ True, False, 3, 1, 0 ] trim_100 = (['file1'] * 100) trim_100.append("List trimmed after 100 files.") expected = [ trim_100, ['file2'] * 200, ['this', 'that', 'other'], ['this', 'List trimmed after 1 files.'], ['List trimmed after 0 files.'], ] for test_files, test_trim_opts, test_expected in zip(files, trim_opt, expected): with patch.dict(archive.__salt__, {'cmd.run': MagicMock(return_value=test_files)}): ret = archive.unrar(source, tmp_dir, trim_output=test_trim_opts) self.assertEqual(ret, test_expected)
def threads(): ''' This tests the performance of the processor's scheduler CLI Example: .. code-block:: bash salt '*' sysbench.threads ''' # Test data thread_yields = [100, 200, 500, 1000] thread_locks = [2, 4, 8, 16] # Initializing the test variables test_command = 'sysbench --num-threads=64 --test=threads ' test_command += '--thread-yields={0} --thread-locks={1} run ' result = None ret_val = {} # Test begins! for yields, locks in zip(thread_yields, thread_locks): key = 'Yields: {0} Locks: {1}'.format(yields, locks) run_command = test_command.format(yields, locks) result = __salt__['cmd.run'](run_command) ret_val[key] = _parser(result) return ret_val
def sig2(method, endpoint, params, provider, aws_api_version): ''' Sign a query against AWS services using Signature Version 2 Signing Process. This is documented at: http://docs.aws.amazon.com/general/latest/gr/signature-version-2.html ''' timenow = datetime.datetime.utcnow() timestamp = timenow.strftime('%Y-%m-%dT%H:%M:%SZ') params_with_headers = params.copy() params_with_headers['AWSAccessKeyId'] = provider.get('id', '') params_with_headers['SignatureVersion'] = '2' params_with_headers['SignatureMethod'] = 'HmacSHA256' params_with_headers['Timestamp'] = '{0}'.format(timestamp) params_with_headers['Version'] = aws_api_version keys = sorted(params_with_headers.keys()) values = list(list(map(params_with_headers.get, keys))) querystring = urlencode(list(zip(keys, values))) canonical = '{0}\n{1}\n/\n{2}'.format( method.encode('utf-8'), endpoint.encode('utf-8'), querystring.encode('utf-8'), ) hashed = hmac.new(provider['key'], canonical, hashlib.sha256) sig = binascii.b2a_base64(hashed.digest()) params_with_headers['Signature'] = sig.strip() return params_with_headers
def blkid(device=None): ''' Return block device attributes: UUID, LABEL, etc. CLI Example: .. code-block:: bash salt '*' disk.blkid salt '*' disk.blkid /dev/sda ''' args = "" if device: args = " " + device ret = {} for line in __salt__['cmd.run_stdout']('blkid' + args).split('\n'): comps = line.split() device = comps[0][:-1] info = {} device_attributes = re.split(('\"*\"'), line.partition(' ')[2]) for key, value in zip(*[iter(device_attributes)]*2): key = key.strip('=').strip(' ') info[key] = value.strip('"') ret[device] = info return ret
def tables(schema): ''' Displays all the tables that are present in the given schema CLI Example:: salt '*' drizzle.tables schema_name ''' # Initializing the required variables ret_val = {} drizzle_db = _connect() cursor = drizzle_db.cursor() # Fetching tables try: cursor.execute('SHOW TABLES IN {0}'.format(schema)) except MySQLdb.OperationalError: return 'Unknown Schema' for iter, count in zip(list(range(cursor.rowcount)), list(range(1, cursor.rowcount+1))): table = cursor.fetchone() ret_val[count] = table[0] cursor.close() drizzle_db.close() return ret_val
def plugins(): ''' Fetches the plugins added to the database server CLI Example:: salt '*' drizzle.plugins ''' # Initializing the required variables ret_val = {} count = 1 drizzle_db = _connect() cursor = drizzle_db.cursor() # Fetching the plugins query = 'SELECT PLUGIN_NAME FROM DATA_DICTIONARY.PLUGINS WHERE IS_ACTIVE LIKE "YES"' cursor.execute(query) for iter, count in zip(list(range(cursor.rowcount)), list(range(1, cursor.rowcount+1))): table = cursor.fetchone() ret_val[count] = table[0] cursor.close() drizzle_db.close() return ret_val
def blkid(device=None): ''' Return block device attributes: UUID, LABEL, etc. This function only works on systems where blkid is available. CLI Example: .. code-block:: bash salt '*' disk.blkid salt '*' disk.blkid /dev/sda ''' args = "" if device: args = " " + device ret = {} blkid_result = __salt__['cmd.run_all']('blkid' + args, python_shell=False) if blkid_result['retcode'] > 0: return ret for line in blkid_result['stdout'].splitlines(): if not line: continue comps = line.split() device = comps[0][:-1] info = {} device_attributes = re.split(('\"*\"'), line.partition(' ')[2]) for key, value in zip(*[iter(device_attributes)]*2): key = key.strip('=').strip(' ') info[key] = value.strip('"') ret[device] = info return ret
def info(package, conn=None): ''' List info for a package ''' if conn is None: conn = init() fields = ( 'package', 'version', 'release', 'installed', 'os', 'os_family', 'dependencies', 'os_dependencies', 'os_family_dependencies', 'summary', 'description', ) data = conn.execute( 'SELECT {0} FROM packages WHERE package=?'.format(','.join(fields)), (package, ) ) row = data.fetchone() if not row: return None formula_def = dict(list(zip(fields, row))) formula_def['name'] = formula_def['package'] return formula_def
def _argspec_to_schema(mod, spec): args = spec["args"] defaults = spec["defaults"] or [] args_req = args[: len(args) - len(defaults)] args_defaults = list(zip(args[-len(defaults) :], defaults)) types = {"title": mod, "description": mod} for i in args_req: types[i] = S.OneOfItem( items=( S.BooleanItem(title=i, description=i, required=True), S.IntegerItem(title=i, description=i, required=True), S.NumberItem(title=i, description=i, required=True), S.StringItem(title=i, description=i, required=True), # S.ArrayItem(title=i, description=i, required=True), # S.DictItem(title=i, description=i, required=True), ) ) for i, j in args_defaults: types[i] = S.OneOfItem( items=( S.BooleanItem(title=i, description=i, default=j), S.IntegerItem(title=i, description=i, default=j), S.NumberItem(title=i, description=i, default=j), S.StringItem(title=i, description=i, default=j), # S.ArrayItem(title=i, description=i, default=j), # S.DictItem(title=i, description=i, default=j), ) ) return type(mod, (S.Schema,), types).serialize()
def get(self, obj, matches=None, mt=None, lt=None, eq=None): ''' Get objects from the table. :param table_name: :param matches: Regexp. :param mt: More than. :param lt: Less than. :param eq: Equals. :return: ''' objects = [] with gzip.open(os.path.join(self.db_path, obj._TABLE), 'rb') as table: header = None for data in csv.reader(table): if not header: header = data continue _obj = obj() for t_attr, t_data in zip(header, data): t_attr, t_type = t_attr.split(':') setattr(_obj, t_attr, self._to_type(t_data, t_type)) if self.__criteria(_obj, matches=matches, mt=mt, lt=lt, eq=eq): objects.append(_obj) return objects
def dict_from_line(cls, line): if line.startswith('#'): raise cls.ParseError("Comment!") comps = line.split() if len(comps) != 7: raise cls.ParseError("Invalid Entry!") return dict(zip(cls.vfstab_keys, comps))
def work(self, item): ret = {'channel': item['channel']} if isinstance(item['data'], six.integer_types): ret['code'] = item['data'] elif item['channel'] == '+switch-master': ret.update(dict(list(zip( ('master', 'old_host', 'old_port', 'new_host', 'new_port'), item['data'].split(' ') )))) elif item['channel'] in ('+odown', '-odown'): ret.update(dict(list(zip( ('master', 'host', 'port'), item['data'].split(' ')[1:] )))) else: ret = { 'channel': item['channel'], 'data': item['data'], } self.fire_master(ret, '{0}/{1}'.format(self.tag, item['channel']))
def _iostats_dict(header, stats): ''' Transpose collected data, average it, stomp it in dict using header Use Decimals so we can properly calc & round, convert to float 'caus' we can't transmit Decimals over 0mq ''' stats = [float((sum(stat) / len(stat)).quantize(decimal.Decimal('.01'))) for stat in zip(*stats)] stats = dict(zip(header, stats)) return stats
def test_extracted_tar(self): ''' archive.extracted tar options ''' source = '/tmp/file.tar.gz' tmp_dir = os.path.join(tempfile.gettempdir(), 'test_archive', '') test_tar_opts = [ '--no-anchored foo', 'v -p --opt', '-v -p', '--long-opt -z', 'z -v -weird-long-opt arg', ] ret_tar_opts = [ ['tar', 'x', '--no-anchored', 'foo', '-f'], ['tar', 'xv', '-p', '--opt', '-f'], ['tar', 'x', '-v', '-p', '-f'], ['tar', 'x', '--long-opt', '-z', '-f'], ['tar', 'xz', '-v', '-weird-long-opt', 'arg', '-f'], ] mock_true = MagicMock(return_value=True) mock_false = MagicMock(return_value=False) ret = {'stdout': ['saltines', 'cheese'], 'stderr': 'biscuits', 'retcode': '31337', 'pid': '1337'} mock_run = MagicMock(return_value=ret) mock_source_list = MagicMock(return_value=(source, None)) state_single_mock = MagicMock(return_value={'local': {'result': True}}) list_mock = MagicMock(return_value={ 'dirs': [], 'files': ['saltines', 'cheese'], 'top_level_dirs': [], 'top_level_files': ['saltines', 'cheese'], }) with patch.dict(archive.__opts__, {'test': False, 'cachedir': tmp_dir}): with patch.dict(archive.__salt__, {'file.directory_exists': mock_false, 'file.file_exists': mock_false, 'state.single': state_single_mock, 'file.makedirs': mock_true, 'cmd.run_all': mock_run, 'archive.list': list_mock, 'file.source_list': mock_source_list}): filename = os.path.join( tmp_dir, 'files/test/_tmp_file.tar.gz' ) for test_opts, ret_opts in zip(test_tar_opts, ret_tar_opts): ret = archive.extracted(tmp_dir, source, options=test_opts, enforce_toplevel=False) ret_opts.append(filename) mock_run.assert_called_with(ret_opts, cwd=tmp_dir, python_shell=False)
def dict_from_line(cls, line, keys=fstab_keys): if len(keys) != 6: raise ValueError('Invalid key array: {0}'.format(keys)) if line.startswith('#'): raise cls.ParseError("Comment!") comps = line.split() if len(comps) != 6: raise cls.ParseError("Invalid Entry!") return dict(zip(keys, comps))
def arg_lookup(fun, aspec=None): ''' Return a dict containing the arguments and default arguments to the function. ''' ret = {'kwargs': {}} if aspec is None: aspec = get_function_argspec(fun) if aspec.defaults: ret['kwargs'] = dict(zip(aspec.args[::-1], aspec.defaults[::-1])) ret['args'] = [arg for arg in aspec.args if arg not in ret['kwargs']] return ret
def freebsd_diskstats(): ''' freebsd specific implementation of diskstats ''' ret = {} iostat = __salt__['cmd.run']('iostat -xzd').splitlines() header = iostat[1] for line in iostat[2:]: comps = line.split() ret[comps[0]] = {} for metric, value in zip(header.split()[1:], comps[1:]): ret[comps[0]][metric] = _number(value) return ret
def test_extracted_tar(self): ''' archive.extracted tar options ''' source = 'file.tar.gz' tmp_dir = os.path.join(tempfile.gettempdir(), 'test_archive', '') test_tar_opts = [ '--no-anchored foo', 'v -p --opt', '-v -p', '--long-opt -z', 'z -v -weird-long-opt arg', ] ret_tar_opts = [ ['tar', 'x', '--no-anchored', 'foo', '-f'], ['tar', 'xv', '-p', '--opt', '-f'], ['tar', 'x', '-v', '-p', '-f'], ['tar', 'x', '--long-opt', '-z', '-f'], ['tar', 'xz', '-v', '-weird-long-opt', 'arg', '-f'], ] mock_true = MagicMock(return_value=True) mock_false = MagicMock(return_value=False) ret = {'stdout': ['saltines', 'cheese'], 'stderr': 'biscuits', 'retcode': '31337', 'pid': '1337'} mock_run = MagicMock(return_value=ret) with patch('os.path.exists', mock_true): with patch.dict(archive.__opts__, {'test': False, 'cachedir': tmp_dir}): with patch.dict(archive.__salt__, {'file.directory_exists': mock_false, 'file.file_exists': mock_false, 'file.makedirs': mock_true, 'cmd.run_all': mock_run}): if HAS_PWD: running_as = pwd.getpwuid(os.getuid()).pw_name else: running_as = 'root' filename = os.path.join( tmp_dir, 'files/test/_tmp{0}_test_archive_.tar'.format( '' if running_as == 'root' else '_{0}'.format(running_as) ) ) for test_opts, ret_opts in zip(test_tar_opts, ret_tar_opts): ret = archive.extracted(tmp_dir, source, 'tar', tar_options=test_opts) ret_opts.append(filename) mock_run.assert_called_with(ret_opts, cwd=tmp_dir, python_shell=False)
def generic_diskstats(): ''' generic implementation of diskstats note: freebsd and sunos ''' ret = {} iostat = __salt__['cmd.run']('iostat -xzd').splitlines() header = iostat[1] for line in iostat[2:]: comps = line.split() ret[comps[0]] = {} for metric, value in zip(header.split()[1:], comps[1:]): ret[comps[0]][metric] = _number(value) return ret
def _ip_ifaces(): ''' Parse output from 'ip a' ''' tmp = {} ret = {} if_ = None at_ = None out = __salt__['cmd.run']('ip a') for line in out.splitlines(): if not line.startswith(' '): comps = line.split(':') if_ = comps[1].strip() opts_comps = comps[2].strip().split() flags = opts_comps.pop(0).lstrip('<').rstrip('>').split(',') opts_iter = iter(opts_comps) ret[if_] = { 'flags': flags, 'options': dict(list(zip(opts_iter, opts_iter))) } else: if line.strip().startswith('link'): comps = iter(line.strip().split()) ret[if_]['link_layer'] = dict(list(zip(comps, comps))) elif line.strip().startswith('inet'): comps = line.strip().split() at_ = comps[0] if len(comps) % 2 != 0: last = comps.pop() comps[-1] += ' {0}'.format(last) ifi = iter(comps) ret[if_][at_] = dict(list(zip(ifi, ifi))) else: comps = line.strip().split() ifi = iter(comps) ret[if_][at_].update(dict(list(zip(ifi, ifi)))) return ret
def _ip_ifaces(): """ Parse output from 'ip a' """ tmp = {} ret = {} if_ = None at_ = None out = __salt__["cmd.run"]("ip a") for line in out.splitlines(): if not line.startswith(" "): comps = line.split(":") if_ = comps[1].strip() opts_comps = comps[2].strip().split() flags = opts_comps.pop(0).lstrip("<").rstrip(">").split(",") opts_iter = iter(opts_comps) ret[if_] = { "flags": flags, "options": dict(list(zip(opts_iter, opts_iter))), } else: if line.strip().startswith("link"): comps = iter(line.strip().split()) ret[if_]["link_layer"] = dict(list(zip(comps, comps))) elif line.strip().startswith("inet"): comps = line.strip().split() at_ = comps[0] if len(comps) % 2 != 0: last = comps.pop() comps[-1] += " {0}".format(last) ifi = iter(comps) ret[if_][at_] = dict(list(zip(ifi, ifi))) else: comps = line.strip().split() ifi = iter(comps) ret[if_][at_].update(dict(list(zip(ifi, ifi)))) return ret
def get_service_name(*args): ''' The Display Name is what is displayed in Windows when services.msc is executed. Each Display Name has an associated Service Name which is the actual name of the service. This function allows you to discover the Service Name by returning a dictionary of Display Names and Service Names, or filter by adding arguments of Display Names. If no args are passed, return a dict of all services where the keys are the service Display Names and the values are the Service Names. If arguments are passed, create a dict of Display Names and Service Names CLI Example: .. code-block:: bash salt '*' service.get_service_name salt '*' service.get_service_name 'Google Update Service (gupdate)' 'DHCP Client' ''' ret = {} services = [] display_names = [] cmd = [ 'sc', 'query', 'type=', 'service', 'state=', 'all', 'bufsize=', str(BUFFSIZE) ] lines = __salt__['cmd.run'](cmd, python_shell=False).splitlines() for line in lines: if 'SERVICE_NAME:' in line: comps = line.split(':', 1) if not len(comps) > 1: continue services.append(comps[1].strip()) if 'DISPLAY_NAME:' in line: comps = line.split(':', 1) if not len(comps) > 1: continue display_names.append(comps[1].strip()) if len(services) == len(display_names): service_dict = dict(zip(display_names, services)) else: return 'Service Names and Display Names mismatch' if len(args) == 0: return service_dict for arg in args: if arg in service_dict: ret[arg] = service_dict[arg] return ret
def sentinel_get_master_ip(master, host=None, port=None, password=None): ''' Get ip for sentinel master .. versionadded: 2016.3.0 CLI Example: .. code-block:: bash salt '*' redis.sentinel_get_master_ip 'mymaster' ''' server = _sconnect(host, port, password) ret = server.sentinel_get_master_addr_by_name(master) return dict(list(zip(('master_host', 'master_port'), ret)))
def dict_from_line(cls, line, keys=crypttab_keys): if len(keys) != 4: raise ValueError('Invalid key array: {0}'.format(keys)) if line.startswith('#'): raise cls.ParseError("Comment!") comps = line.split() # If there are only three entries, then the options have been omitted. if len(comps) == 3: comps += [''] if len(comps) != 4: raise cls.ParseError("Invalid Entry!") return dict(zip(keys, comps))
def sentinel_get_master_ip(master, host=None, port=None, password=None): ''' Get ip for sentinel master .. versionadded: Boron CLI Example: .. code-block:: bash salt '*' redis.sentinel_get_master_ip 'mymaster' ''' server = _sconnect(host, port, password) ret = server.sentinel_get_master_addr_by_name(master) return dict(list(zip(('master_host', 'master_port'), ret)))
def test_tree(self): test_map = ( 'ex1.nl', 'o.1.example.eu', 'a1a.b2b.c3c.example.com' ) res_map = ( ['ex1.nl'], ['o.1.example.eu', '1.example.eu', 'example.eu'], ['a1a.b2b.c3c.example.com', 'b2b.c3c.example.com', 'c3c.example.com', 'example.com'] ) for domain, result in zip(test_map, res_map): self.assertEqual(_tree(domain), result)
def test_base_protocol_settings(self): """ Tests default constants data. :return: """ base = ssdp.SSDPBase() v_keys = ["signature", "answer", "port", "listen_ip", "timeout"] v_vals = ["__salt_master_service", {}, 4520, "0.0.0.0", 3] for key in v_keys: assert key in base.DEFAULTS for key in base.DEFAULTS: assert key in v_keys for key, value in zip(v_keys, v_vals): assert base.DEFAULTS[key] == value
def get_master_ip(host=None, port=None, password=None): """ Get host information about slave .. versionadded: 2016.3.0 CLI Example: .. code-block:: bash salt '*' redis.get_master_ip """ server = _connect(host, port, password) srv_info = server.info() ret = (srv_info.get("master_host", ""), srv_info.get("master_port", "")) return dict(list(zip(("master_host", "master_port"), ret)))
def get_master_ip(host=None, port=None, password=None): ''' Get host information about slave .. versionadded: 2016.3.0 CLI Example: .. code-block:: bash salt '*' redis.get_master_ip ''' server = _connect(host, port, password) info = server.info() ret = (info.get('master_host', ''), info.get('master_port', '')) return dict(list(zip(('master_host', 'master_port'), ret)))
def get_master_ip(host=None, port=None, password=None): ''' Get host information about slave .. versionadded: Boron CLI Example: .. code-block:: bash salt '*' redis.get_master_ip ''' server = _connect(host, port, password) info = server.info() ret = (info.get('master_host', ''), info.get('master_port', '')) return dict(list(zip(('master_host', 'master_port'), ret)))
def test_base_protocol_settings(self): ''' Tests default constants data. :return: ''' base = ssdp.SSDPBase() v_keys = ['signature', 'answer', 'port', 'listen_ip', 'timeout'] v_vals = ['__salt_master_service', {}, 4520, '0.0.0.0', 3] for key in v_keys: assert key in base.DEFAULTS for key in base.DEFAULTS: assert key in v_keys for key, value in zip(v_keys, v_vals): assert base.DEFAULTS[key] == value
def store(self, obj, distinct=False): ''' Store an object in the table. :param obj: An object to store :param distinct: Store object only if there is none identical of such. If at least one field is different, store it. :return: ''' if distinct: fields = dict(zip(self._tables[obj._TABLE].keys(), obj._serialize(self._tables[obj._TABLE]))) db_obj = self.get(obj.__class__, eq=fields) if db_obj and distinct: raise Exception("Object already in the database.") with gzip.open(os.path.join(self.db_path, obj._TABLE), 'a') as table: csv.writer(table).writerow(self._validate_object(obj))
def get_service_name(*args): ''' The Display Name is what is displayed in Windows when services.msc is executed. Each Display Name has an associated Service Name which is the actual name of the service. This function allows you to discover the Service Name by returning a dictionary of Display Names and Service Names, or filter by adding arguments of Display Names. If no args are passed, return a dict of all services where the keys are the service Display Names and the values are the Service Names. If arguments are passed, create a dict of Display Names and Service Names CLI Example: .. code-block:: bash salt '*' service.get_service_name salt '*' service.get_service_name 'Google Update Service (gupdate)' 'DHCP Client' ''' ret = {} services = [] display_names = [] cmd = ['sc', 'query', 'type=', 'service', 'state=', 'all', 'bufsize=', str(BUFFSIZE)] lines = __salt__['cmd.run'](cmd, python_shell=False).splitlines() for line in lines: if 'SERVICE_NAME:' in line: comps = line.split(':', 1) if not len(comps) > 1: continue services.append(comps[1].strip()) if 'DISPLAY_NAME:' in line: comps = line.split(':', 1) if not len(comps) > 1: continue display_names.append(comps[1].strip()) if len(services) == len(display_names): service_dict = dict(zip(display_names, services)) else: return 'Service Names and Display Names mismatch' if len(args) == 0: return service_dict for arg in args: if arg in service_dict: ret[arg] = service_dict[arg] return ret
def holds(snapshot, **kwargs): """ Lists all existing user references for the given snapshot or snapshots. snapshot : string name of snapshot recursive : boolean lists the holds that are set on the named descendent snapshots also. .. versionadded:: 2016.3.0 CLI Example: .. code-block:: bash salt '*' zfs.holds myzpool/mydataset@baseline """ ## Configure command # NOTE: initialize the defaults flags = ["-H"] target = [] # NOTE: set extra config from kwargs if kwargs.get("recursive", False): flags.append("-r") # NOTE: update target target.append(snapshot) ## Lookup holds res = __salt__["cmd.run_all"]( __utils__["zfs.zfs_command"](command="holds", flags=flags, target=target,), python_shell=False, ) ret = __utils__["zfs.parse_command_result"](res) if res["retcode"] == 0: for hold in res["stdout"].splitlines(): hold_data = OrderedDict( list(zip(["name", "tag", "timestamp"], hold.split("\t"),)) ) ret[hold_data["tag"].strip()] = hold_data["timestamp"] return ret
def blkid(device=None, token=None): ''' Return block device attributes: UUID, LABEL, etc. This function only works on systems where blkid is available. device Device name from the system token Any valid token used for the search CLI Example: .. code-block:: bash salt '*' disk.blkid salt '*' disk.blkid /dev/sda salt '*' disk.blkid token='UUID=6a38ee5-7235-44e7-8b22-816a403bad5d' salt '*' disk.blkid token='TYPE=ext4' ''' cmd = ['blkid'] if device: cmd.append(device) elif token: cmd.extend(['-t', token]) ret = {} blkid_result = __salt__['cmd.run_all'](cmd, python_shell=False) if blkid_result['retcode'] > 0: return ret for line in blkid_result['stdout'].splitlines(): if not line: continue comps = line.split() device = comps[0][:-1] info = {} device_attributes = re.split(('\"*\"'), line.partition(' ')[2]) for key, value in zip(*[iter(device_attributes)] * 2): key = key.strip('=').strip(' ') info[key] = value.strip('"') ret[device] = info return ret
def freeze(name=None, force=False, **kwargs): ''' Save the list of package and repos in a freeze file. As this module is build on top of the pkg module, the user can send extra attributes to the underlying pkg module via kwargs. This function will call ``pkg.list_pkgs`` and ``pkg.list_repos``, and any additional arguments will be passed through to those functions. name Name of the frozen state. Optional. force If true, overwrite the state. Optional. CLI Example: .. code-block:: bash salt '*' freezer.freeze salt '*' freezer.freeze pre_install salt '*' freezer.freeze force=True root=/chroot ''' states_path = _states_path() try: if not os.path.exists(states_path): os.makedirs(states_path) except OSError as e: msg = 'Error when trying to create the freezer storage %s: %s' log.error(msg, states_path, e) raise CommandExecutionError(msg % (states_path, e)) if status(name) and not force: raise CommandExecutionError('The state is already present. Use ' 'force parameter to overwrite.') safe_kwargs = clean_kwargs(**kwargs) pkgs = __salt__['pkg.list_pkgs'](**safe_kwargs) repos = __salt__['pkg.list_repos'](**safe_kwargs) for fname, content in zip(_paths(name), (pkgs, repos)): with fopen(fname, 'w') as fp: json.dump(content, fp) return True
def store(self, obj, distinct=False): ''' Store an object in the table. :param obj: An object to store :param distinct: Store object only if there is none identical of such. If at least one field is different, store it. :return: ''' if distinct: fields = dict( zip(self._tables[obj._TABLE].keys(), obj._serialize(self._tables[obj._TABLE]))) db_obj = self.get(obj.__class__, eq=fields) if db_obj and distinct: raise Exception("Object already in the database.") with gzip.open(os.path.join(self.db_path, obj._TABLE), 'a') as table: csv.writer(table).writerow(self._validate_object(obj))
def token_list(kubeconfig=None, rootfs=None): ''' .. versionadded:: TBD List bootstrap tokens on the server kubeconfig The kubeconfig file to use when talking to the cluster. The default values in /etc/kubernetes/admin.conf rootfs The path to the real host root filesystem CLI Example: .. code-block:: bash salt '*' kubeadm.token_list ''' cmd = ['kubeadm', 'token', 'list'] parameters = [('kubeconfig', kubeconfig), ('rootfs', rootfs)] for parameter, value in parameters: if value: cmd.extend(['--{}'.format(parameter), str(value)]) lines = _cmd(cmd).splitlines() # Find the header and parse it. We do not need to validate the # content, as the regex will take care of future changes. header = lines.pop(0) header = [i.lower() for i in re.findall(r'(\w+(?:\s\w+)*)', header)] tokens = [] for line in lines: # TODO(aplanas): descriptions with multiple spaces can break # the parser. values = re.findall(r'(\S+(?:\s\S+)*)', line) if len(header) != len(values): log.error('Error parsing line: {}'.format(line)) continue tokens.append({key: value for key, value in zip(header, values)}) return tokens
def _data2rec(rschema, rdata): ''' OrderedDict({ 'prio': int, 'weight': int, 'port': to_port, 'name': str, }) '10 20 25 myawesome.nl' ''' try: rdata = rdata.split(' ', len(rschema)) rschema = rschema.items() return dict(((fname, rcb(rdata)) for (fname, rcb), rdata in zip(rschema, rdata))) except (AttributeError, TypeError, ValueError): log.error('Cant parse DNS record data: {0}'.format(rdata)) return False
def assertEqualXML(self, e1, e2): if six.PY3 and isinstance(e1, bytes): e1 = e1.decode('utf-8') if six.PY3 and isinstance(e2, bytes): e2 = e2.decode('utf-8') if isinstance(e1, six.string_types): e1 = etree.XML(e1) if isinstance(e2, six.string_types): e2 = etree.XML(e2) if e1.tag != e2.tag: return False if e1.text != e2.text: return False if e1.tail != e2.tail: return False if e1.attrib != e2.attrib: return False if len(e1) != len(e2): return False return all(self.assertEqualXML(c1, c2) for c1, c2 in zip(e1, e2))
def test_parse_version_string(self): test_parameters = [ ("> 1.0.0, < 15.0.0, != 14.0.1", [(">", "1.0.0"), ("<", "15.0.0"), ("!=", "14.0.1")]), ("> 1.0.0,< 15.0.0,!= 14.0.1", [(">", "1.0.0"), ("<", "15.0.0"), ("!=", "14.0.1")]), (">= 1.0.0, < 15.0.0", [(">=", "1.0.0"), ("<", "15.0.0")]), (">=1.0.0,< 15.0.0", [(">=", "1.0.0"), ("<", "15.0.0")]), ("< 15.0.0", [("<", "15.0.0")]), ("<15.0.0", [("<", "15.0.0")]), ("15.0.0", [("==", "15.0.0")]), ("", []) ] for version_string, expected_version_conditions in test_parameters: version_conditions = pkg._parse_version_string(version_string) self.assertEqual(len(expected_version_conditions), len(version_conditions)) for expected_version_condition, version_condition in zip(expected_version_conditions, version_conditions): self.assertEqual( expected_version_condition[0], version_condition[0]) self.assertEqual( expected_version_condition[1], version_condition[1])
def test_extracted_tar(self): ''' archive.extracted tar options ''' source = 'file.tar.gz' tmp_dir = os.path.join(tempfile.gettempdir(), 'test_archive') test_tar_opts = [ '--no-anchored foo', 'v -p --opt', '-v -p', '--long-opt -z', 'z -v -weird-long-opt arg', ] ret_tar_opts = [ ['tar', 'x', '--no-anchored', 'foo', '-f'], ['tar', 'xv', '-p', '--opt', '-f'], ['tar', 'x', '-v', '-p', '-f'], ['tar', 'x', '--long-opt', '-z', '-f'], ['tar', 'xz', '-v', '-weird-long-opt', 'arg', '-f'], ] mock_true = MagicMock(return_value=True) mock_false = MagicMock(return_value=False) ret = {'stdout': ['saltines', 'cheese'], 'stderr': 'biscuits', 'retcode': '31337', 'pid': '1337'} mock_run = MagicMock(return_value=ret) with patch('os.path.exists', mock_true): with patch.dict(archive.__opts__, {'test': False, 'cachedir': tmp_dir}): with patch.dict(archive.__salt__, {'file.directory_exists': mock_false, 'file.file_exists': mock_false, 'file.makedirs': mock_true, 'cmd.run_all': mock_run}): for test_opts, ret_opts in zip(test_tar_opts, ret_tar_opts): ret = archive.extracted(tmp_dir, source, 'tar', tar_options=test_opts) ret_opts.append(os.path.join(tmp_dir, 'files/test/_tmp_test_archive_.tar')) mock_run.assert_called_with(ret_opts, python_shell=False, cwd=os.path.join(tmp_dir, ''))
def test_data2group(self): right = [ ['10 mbox.example.com'], [ '10 mbox1.example.com', '20 mbox2.example.com', '20 mbox3.example.com', '30 mbox4.example.com', '30 mbox5.example.com', '30 mbox6.example.com', ], ] rschema = OrderedDict(( ('prio', int), ('srvr', str), )) results = [ OrderedDict([(10, [{ 'srvr': 'mbox.example.com' }])]), OrderedDict([(10, [{ 'srvr': 'mbox1.example.com' }]), (20, [{ 'srvr': 'mbox2.example.com' }, { 'srvr': 'mbox3.example.com' }]), (30, [{ 'srvr': 'mbox4.example.com' }, { 'srvr': 'mbox5.example.com' }, { 'srvr': 'mbox6.example.com' }])]), ] for rdata, res in zip(right, results): group = _data2rec_group(rschema, rdata, 'prio') self.assertEqual(group, res)
def list_nodes_full(conn=None, call=None): """ Return a list of the VMs that are on the provider, with all fields """ if call == "action": raise SaltCloudSystemExit("The list_nodes_full function must be called with -f or --function.") if not conn: conn = get_conn() # pylint: disable=E0602 nodes = conn.list_nodes() ret = {} for node in nodes: pairs = {} for key, value in zip(node.__dict__, six.itervalues(node.__dict__)): pairs[key] = value ret[node.name] = pairs del ret[node.name]["driver"] salt.utils.cloud.cache_node_list(ret, __active_provider_name__.split(":")[0], __opts__) return ret
def _dmi_parse(data, clean=True, fields=None): ''' Structurize DMI records into a nice list Optionally trash bogus entries and filter output ''' dmi = [] # Detect & split Handle records dmi_raw = iter( re.split('(handle [0-9]x[0-9a-f]+[^\n]+)\n', data, flags=re.MULTILINE + re.IGNORECASE)[1:]) for handle, dmi_raw in zip(dmi_raw, dmi_raw): handle, htype = [hline.split()[-1] for hline in handle.split(',')][0:2] dmi_raw = dmi_raw.split('\n') # log.debug('{0} record contains {1}'.format(handle, dmi_raw)) log.debug('Parsing handle {0}'.format(handle)) # The first line of a handle is a description of the type record = { 'handle': handle, 'description': dmi_raw.pop(0).strip(), 'type': int(htype) } if not len(dmi_raw): # empty record if not clean: dmi.append(record) continue # log.debug('{0} record contains {1}'.format(record, dmi_raw)) dmi_data = _dmi_data(dmi_raw, clean, fields) if len(dmi_data): record['data'] = dmi_data dmi.append(record) elif not clean: dmi.append(record) return dmi
def beacon(config): ''' Emit the load averages of this host. Specify thresholds for each load average and only emit a beacon if any of them are exceeded. .. code-block:: yaml beacons: - load: - 1m: - 0.0 - 2.0 - 5m: - 0.0 - 1.5 - 15m: - 0.1 - 1.0 ''' log.trace('load beacon starting') ret = [] if not os.path.isfile('/proc/loadavg'): return ret with salt.utils.fopen('/proc/loadavg', 'rb') as fp_: avgs = fp_.read().split()[:3] avg_keys = ['1m', '5m', '15m'] avg_dict = dict(zip(avg_keys, avgs)) # Check each entry for threshold if float(avgs[0]) < float(config[0]['1m'][0]) or \ float(avgs[0]) > float(config[0]['1m'][1]) or \ float(avgs[1]) < float(config[1]['5m'][0]) or \ float(avgs[1]) > float(config[1]['5m'][1]) or \ float(avgs[2]) < float(config[2]['15m'][0]) or \ float(avgs[2]) > float(config[2]['15m'][1]): ret.append(avg_dict) return ret
def beacon(config): ''' Emit the load averages of this host. Specify thresholds for each load average and only emit a beacon if any of them are exceeded. .. code-block:: yaml beacons: load: 1m: - 0.0 - 2.0 5m: - 0.0 - 1.5 15m: - 0.1 - 1.0 ''' log.trace('load beacon starting') ret = [] if not os.path.isfile('/proc/loadavg'): return ret with salt.utils.fopen('/proc/loadavg', 'rb') as fp_: avgs = fp_.read().split()[:3] avg_keys = ['1m', '5m', '15m'] avg_dict = dict(zip(avg_keys, avgs)) # Check each entry for threshold if float(avgs[0]) < float(config['1m'][0]) or \ float(avgs[0]) > float(config['1m'][1]) or \ float(avgs[1]) < float(config['5m'][0]) or \ float(avgs[1]) > float(config['5m'][1]) or \ float(avgs[2]) < float(config['15m'][0]) or \ float(avgs[2]) > float(config['15m'][1]): ret.append(avg_dict) return ret
def raid_info(): """ Return RAID info, takes no parameters, returns dict. """ field_names = ['slot', 'class', 'vendor', 'device', 'rev', 'subsystem_vendor', 'subsystem', 'driver'] field_values = [] if platform.system() == 'Linux': try: hwinfo = subprocess.Popen('lspci -m'.split(), stdout=subprocess.PIPE) for line in hwinfo.stdout.readlines(): if re.search('RAID|Serial Attached SCSI', line): # sanitize the output and load it into a list raw_fields = line.strip().split('"') fields = [field for field in raw_fields if not (field == ' ' or field == '')] field_values = [string.strip().rstrip() for string in fields] # append kmod name to data based on fetched slot id field_values.append(_kmod_name(field_values[0])) pci_data = dict(zip(field_names, field_values)) # for detected cards append kernel module info if pci_data: try: pci_data.update(_kmod_info(pci_data['driver'])) return {'raidcontroller': pci_data} except KeyError: log.debug('No RAID driver found') return {'raidcontroller': pci_data} else: log.debug('No RAID controllers detected') except OSError: pass else: log.debug('Not supported OS "{0}"'.format(platform.system()))
def _call_function(name, returner=None, **kwargs): ''' Calls a function from the specified module. :param name: :param kwargs: :return: ''' argspec = salt.utils.args.get_function_argspec(__salt__[name]) func_kw = dict( zip( argspec.args[-len(argspec.defaults or []):], # pylint: disable=incompatible-py3-code argspec.defaults or [])) func_args = [] for funcset in kwargs.get('func_args') or {}: if isinstance(funcset, dict): func_kw.update(funcset) else: func_args.append(funcset) missing = [] for arg in argspec.args: if arg not in func_kw: missing.append(arg) if missing: raise SaltInvocationError('Missing arguments: {0}'.format( ', '.join(missing))) mret = __salt__[name](*func_args, **func_kw) if returner is not None: returners = salt.loader.returners(__opts__, __salt__) if returner in returners: returners[returner]({ 'id': __opts__['id'], 'ret': mret, 'fun': name, 'jid': salt.utils.jid.gen_jid() }) return mret
def version(*names, **kwargs): ''' Common interface for obtaining the version of installed packages. Accepts full or partial FMRI. If called using pkg_resource, full FMRI is required. Partial FMRI is returned if the package is not installed. CLI Example: .. code-block:: bash salt '*' pkg.version vim salt '*' pkg.version foo bar baz salt '*' pkg_resource.version pkg://solaris/entire ''' if not names: return '' cmd = ['/bin/pkg', 'list', '-Hv'] cmd.extend(names) lines = __salt__['cmd.run_stdout'](cmd, ignore_retcode=True).splitlines() ret = {} for line in lines: ret[_ips_get_pkgname(line)] = _ips_get_pkgversion(line) # Append package names which are not installed/found unmatched = list([ name for name in names if not reduce(lambda x, y: x or name in y, ret, False) ]) # pylint: disable=W0640 ret.update(zip(unmatched, itertools.cycle(('', )))) # Return a string if only one package name passed if len(names) == 1: try: return next(six.itervalues(ret)) except StopIteration: return '' return ret
def _dmi_parse(data, clean=True, fields=None): """ Structurize DMI records into a nice list Optionally trash bogus entries and filter output """ dmi = [] # Detect & split Handle records dmi_split = re.compile( "(handle [0-9]x[0-9a-f]+[^\n]+)\n", re.MULTILINE + re.IGNORECASE ) dmi_raw = iter(re.split(dmi_split, data)[1:]) for handle, dmi_raw in zip(dmi_raw, dmi_raw): handle, htype = [hline.split()[-1] for hline in handle.split(",")][0:2] dmi_raw = dmi_raw.split("\n") # log.debug('%s record contains %s', handle, dmi_raw) log.debug("Parsing handle %s", handle) # The first line of a handle is a description of the type record = { "handle": handle, "description": dmi_raw.pop(0).strip(), "type": int(htype), } if not dmi_raw: # empty record if not clean: dmi.append(record) continue # log.debug('%s record contains %s', record, dmi_raw) dmi_data = _dmi_data(dmi_raw, clean, fields) if dmi_data: record['data'] = dmi_data dmi.append(record) elif not clean: dmi.append(record) return dmi
def decrypt(self, data): ''' verify HMAC-SHA256 signature and decrypt data with AES-CBC ''' aes_key, hmac_key = self.keys sig = data[-self.SIG_SIZE:] data = data[:-self.SIG_SIZE] mac_bytes = hmac.new(hmac_key, data, hashlib.sha256).digest() if len(mac_bytes) != len(sig): log.debug('Failed to authenticate message') raise AuthenticationError('message authentication failed') result = 0 for zipped_x, zipped_y in zip(mac_bytes, sig): result |= ord(zipped_x) ^ ord(zipped_y) if result != 0: log.debug('Failed to authenticate message') raise AuthenticationError('message authentication failed') iv_bytes = data[:self.AES_BLOCK_SIZE] data = data[self.AES_BLOCK_SIZE:] cypher = AES.new(aes_key, AES.MODE_CBC, iv_bytes) data = cypher.decrypt(data) return data[:-ord(data[-1])]
def _info(*pkgs): ''' Get all info brew can provide about a list of packages. Does not do any kind of processing, so the format depends entirely on the output brew gives. This may change if a new version of the format is requested. On failure, returns an empty dict and logs failure. On success, returns a dict mapping each item in pkgs to its corresponding object in the output of 'brew info'. Caveat: If one of the packages does not exist, no packages will be included in the output. ''' cmd = 'info --json=v1 {0}'.format(' '.join(pkgs)) brew_result = _call_brew(cmd) if brew_result['retcode']: log.error('Failed to get info about packages: %s', ' '.join(pkgs)) return {} output = salt.utils.json.loads(brew_result['stdout']) return dict(zip(pkgs, output))
def _info(self, args): ''' List info for a package ''' if len(args) < 2: log.error('A package must be specified') return False package = args[1] conn = sqlite3.connect(self.opts['spm_db'], isolation_level=None) cur = conn.cursor() fields = ( 'package', 'version', 'release', 'installed', 'os', 'os_family', 'dependencies', 'os_dependencies', 'os_family_dependencies', 'summary', 'description', ) data = conn.execute( 'SELECT {0} FROM packages WHERE package=?'.format(','.join(fields)), (package, ) ) row = data.fetchone() if not row: print('Package {0} not installed'.format(package)) return formula_def = dict(list(zip(fields, row))) formula_def['name'] = formula_def['package'] self._print_info(formula_def)
def _data2rec(schema, rec_data): ''' schema = OrderedDict({ 'prio': int, 'weight': int, 'port': to_port, 'name': str, }) rec_data = '10 20 25 myawesome.nl' res = {'prio': 10, 'weight': 20, 'port': 25 'name': 'myawesome.nl'} ''' try: rec_fields = rec_data.split(' ') assert len(rec_fields) == len(schema) return dict( ((field_name, rec_cast(rec_field)) for (field_name, rec_cast), rec_field in zip(schema.items(), rec_fields))) except (AssertionError, AttributeError, TypeError, ValueError) as e: raise ValueError('Unable to cast "{0}" as "{2}": {1}'.format( rec_data, e, ' '.join(schema.keys())))
def _info(*pkgs): ''' Get all info brew can provide about a list of packages. Does not do any kind of processing, so the format depends entirely on the output brew gives. This may change if a new version of the format is requested. On failure, returns an empty dict and logs failure. On success, returns a dict mapping each item in pkgs to its corresponding object in the output of 'brew info'. Caveat: If one of the packages does not exist, no packages will be included in the output. ''' cmd = 'brew info --json=v1 {0}'.format(' '.join(pkgs)) brew_result = _call_brew(cmd) if brew_result['retcode']: log.error('Failed to get info about packages: {0}'.format(' '.join(pkgs))) return {} output = json.loads(brew_result['stdout']) return dict(zip(pkgs, output))