class CaseInsensitiveDict(MutableMapping): """ Inspired by requests' case-insensitive dict implementation, but works with non-string keys as well. """ def __init__(self, init=None, **kwargs): """ Force internal dict to be ordered to ensure a consistent iteration order, irrespective of case. """ self._data = OrderedDict() self.update(init or {}, **kwargs) def __len__(self): return len(self._data) def __setitem__(self, key, value): # Store the case-sensitive key so it is available for dict iteration self._data[to_lowercase(key)] = (key, value) def __delitem__(self, key): del self._data[to_lowercase(key)] def __getitem__(self, key): return self._data[to_lowercase(key)][1] def __iter__(self): return (item[0] for item in self._data.values()) def __eq__(self, rval): if not isinstance(rval, Mapping): # Comparing to non-mapping type (e.g. int) is always False return False return dict(self.items_lower()) == dict(CaseInsensitiveDict(rval).items_lower()) def __repr__(self): return repr(dict(self.items())) def items_lower(self): """ Returns a generator iterating over keys and values, with the keys all being lowercase. """ return ((key, val[1]) for key, val in self._data.items()) def copy(self): """ Returns a copy of the object """ return CaseInsensitiveDict(self._data.items())
def _clean_vdev_config(config): ''' Return a simple vdev tree from zpool.status' config section ''' cln_config = OrderedDict() for label, sub_config in config.items(): if label not in ['state', 'read', 'write', 'cksum']: sub_config = _clean_vdev_config(sub_config) if sub_config and isinstance(cln_config, list): cln_config.append(OrderedDict([(label, sub_config)])) elif sub_config and isinstance(cln_config, OrderedDict): cln_config[label] = sub_config elif isinstance(cln_config, list): cln_config.append(label) elif isinstance(cln_config, OrderedDict): new_config = [] for old_label, old_config in cln_config.items(): new_config.append(OrderedDict([(old_label, old_config)])) new_config.append(label) cln_config = new_config else: cln_config = [label] return cln_config
def _prompt_choice(var_name, options): ''' Prompt the user to choose between a list of options, index each one by adding an enumerator based on https://github.com/audreyr/cookiecutter/blob/master/cookiecutter/prompt.py#L51 :param var_name: The question to ask the user :type var_name: ``str`` :param options: A list of options :type options: ``list`` of ``tupple`` :rtype: ``tuple`` :returns: The selected user ''' choice_map = OrderedDict((u'{0}'.format(i), value) for i, value in enumerate(options, 1) if value[0] != 'test') choices = choice_map.keys() default = u'1' choice_lines = [ u'{0} - {1} - {2}'.format(c[0], c[1][0], c[1][1]) for c in choice_map.items() ] prompt = u'\n'.join( (u'Select {0}:'.format(var_name), u'\n'.join(choice_lines), u'Choose from {0}'.format(u', '.join(choices)))) user_choice = click.prompt(prompt, type=click.Choice(choices), default=default) return choice_map[user_choice]
def _prompt_choice(var_name, options): """ Prompt the user to choose between a list of options, index each one by adding an enumerator based on https://github.com/audreyr/cookiecutter/blob/master/cookiecutter/prompt.py#L51 :param var_name: The question to ask the user :type var_name: ``str`` :param options: A list of options :type options: ``list`` of ``tupple`` :rtype: ``tuple`` :returns: The selected user """ choice_map = OrderedDict( ("{}".format(i), value) for i, value in enumerate(options, 1) if value[0] != "test" ) choices = choice_map.keys() default = "1" choice_lines = [ "{} - {} - {}".format(c[0], c[1][0], c[1][1]) for c in choice_map.items() ] prompt = "\n".join( ( "Select {}:".format(var_name), "\n".join(choice_lines), "Choose from {}".format(", ".join(choices)), ) ) user_choice = click.prompt(prompt, type=click.Choice(choices), default=default) return choice_map[user_choice]
def _prompt_choice(var_name, options): ''' Prompt the user to choose between a list of options, index each one by adding an enumerator based on https://github.com/audreyr/cookiecutter/blob/master/cookiecutter/prompt.py#L51 :param var_name: The question to ask the user :type var_name: ``str`` :param options: A list of options :type options: ``list`` of ``tupple`` :rtype: ``tuple`` :returns: The selected user ''' choice_map = OrderedDict( (u'{0}'.format(i), value) for i, value in enumerate(options, 1) if value[0] != 'test' ) choices = choice_map.keys() default = u'1' choice_lines = [u'{0} - {1} - {2}'.format(c[0], c[1][0], c[1][1]) for c in choice_map.items()] prompt = u'\n'.join(( u'Select {0}:'.format(var_name), u'\n'.join(choice_lines), u'Choose from {0}'.format(u', '.join(choices)) )) user_choice = click.prompt( prompt, type=click.Choice(choices), default=default ) return choice_map[user_choice]
def extract_certs(cert_string, common_name=None): if (cert_string and ('\n' not in cert_string) and os.path.exists(cert_string)): with open(cert_string) as fic: cert_string = fic.read() composants, cns, full_certs = OrderedDict(), [], [] if cert_string and cert_string.strip(): certstring = '' for i in cert_string.splitlines(): if (certstring or ('-----BEGIN CERTIFICATE-----' in i)): certstring += i.strip() if not certstring.endswith('\n'): certstring += '\n' if certstring and ('-----END CERTIFICATE-----' in i.strip()): ocert = load_cert(certstring) if ocert is not None: # valid cert full_certs.append(certstring) certstring = '' if full_certs: for ccert in full_certs: infos = ssl_infos(ccert) infos['cert'] = ccert try: CN = infos['subject'].CN except Exception: CN = '' if CN: composants[CN] = infos # we have certificates in, and not just one # we can compose an ssl authentication chain cert_cn = None if composants and (len(composants) > 1): # filter out the domain which will not be part of the ssl chain for cn, data in composants.items(): append = False # if we match the cert name subject, we got the cert # of this box if common_name is not None and domain_match(common_name, cn): cert_cn = cn # or we match exactly the common name else: append = True if append: cns.append(cn) # if we did not match the last routine, # assume that the real certificate is the first of the chain if not cns: for ix, cn in enumerate(composants): if ix == 0: cert_cn = cn cns.append(cn) return full_certs, composants, cns, cert_cn
def _doext_pillar(id_, prefixed, limited): _s = __salt__ all_vms = _s['mc_cloud_compute_node.get_vms']() targets = _s['mc_cloud_compute_node.get_targets']() vms, vts = OrderedDict(), [] target = None data = vm_registry(prefixed=prefixed) if prefixed: vts_pillar = data[PREFIX + '.vts'] vms_pillar = data[PREFIX + '.vms'] this_host = PREFIX + '.this_host' this_port = PREFIX + '.this_port' else: vts_pillar = data['vts'] vms_pillar = data['vms'] this_host = 'this_host' this_port = 'this_port' if id_ not in targets and id_ not in all_vms: return {} if id_ in all_vms: vt = all_vms[id_]['vt'] if vt not in vts: vts.append(vt) vms[id_] = all_vms[id_] target = all_vms[id_]['target'] if id_ in targets: target = id_ for vm_, vmdata_ in targets[id_]['vms'].items(): vms[vm_] = vmdata_ # pylint: disable=W0612 noecho = [ vts.append(i) for i in targets[id_]['vts'] if i not in vts ] for vt in vts: vts_pillar[vt] = _s['mc_cloud_vm.vt_extpillar'](target, vt, limited=limited) for vm, vmdata in vms.items(): vt = vmdata['vt'] vme_settings = _s['mc_cloud_vm.vm_extpillar'](vm, limited=limited) if id_ == vm: data[this_port] = vme_settings['ssh_reverse_proxy_port'] data[this_host] = vme_settings['target'] vme_settings['vt'] = vt vms_pillar[vm] = vme_settings return data
def gen_ini(self): yield "{0}[{1}]{0}".format(os.linesep, self.name) sections_dict = OrderedDict() for name, value in self.items(): # Handle Comment Lines if COM_REGX.match(name): yield "{}{}".format(value, os.linesep) # Handle Sections elif isinstance(value, _Section): sections_dict.update({name: value}) # Key / Value pairs # Adds spaces between the separator else: yield "{}{}{}{}".format( name, " {} ".format(self.sep) if self.sep != " " else self.sep, value, os.linesep, ) for name, value in sections_dict.items(): yield from value.gen_ini()
def _get_args(function: str) -> Dict: """ Given a function def, returns arguments and defaults """ # Generate list of arguments arg_strings = [] list_of_arguments = function.args.args if list_of_arguments: for arg in list_of_arguments: arg_strings.append(arg.arg) # Generate list of arg defaults # Values are only returned for populated items arg_default_strings = [] list_arg_defaults = function.args.defaults if list_arg_defaults: for arg_default in list_arg_defaults: if isinstance(arg_default, ast.NameConstant): arg_default_strings.append(arg_default.value) elif isinstance(arg_default, ast.Str): arg_default_strings.append(arg_default.s) elif isinstance(arg_default, ast.Num): arg_default_strings.append(arg_default.n) # Since only some args may have default values, need to zip in reverse order backwards_args = OrderedDict( itertools.zip_longest(reversed(arg_strings), reversed(arg_default_strings)) ) ordered_args = OrderedDict(reversed(list(backwards_args.items()))) try: ordered_args["args"] = function.args.vararg.arg except AttributeError: pass try: ordered_args["kwargs"] = function.args.kwarg.arg except AttributeError: pass return ordered_args
def _uncomment_if_commented(self, opt_key): # should be called only if opt_key is not already present # will uncomment the key if commented and create a place holder # for the key where the correct value can be update later # used to preserve the ordering of comments and commented options # and to make sure options without sectons go above any section options_backup = OrderedDict() comment_index = None for key, value in self.items(): if comment_index is not None: options_backup.update({key: value}) continue if "#comment" not in key: continue opt_match = self.opt_regx.match(value.lstrip("#")) if opt_match and opt_match.group(2) == opt_key: comment_index = key for key in options_backup: self.pop(key) self.pop(comment_index, None) super().update({opt_key: None}) for key, value in options_backup.items(): super().update({key: value})
# -*- coding: utf-8 -*- """ Application Kinds of Salt apps. These are used to indicate what kind of Application is using RAET """ from __future__ import absolute_import, unicode_literals from collections import namedtuple from salt.utils.odict import OrderedDict # Python equivalent of an enum APPL_KINDS = OrderedDict([("master", 0), ("minion", 1), ("syndic", 2), ("caller", 3)]) APPL_KIND_NAMES = OrderedDict( (v, k) for k, v in list(APPL_KINDS.items())) # inverse map ApplKind = namedtuple("ApplKind", list(APPL_KINDS.keys())) applKinds = ApplKind(**APPL_KINDS)
# -*- coding: utf-8 -*- ''' Application Kinds of Salt apps. These are used to indicate what kind of Application is using RAET ''' from __future__ import absolute_import from collections import namedtuple from salt.utils.odict import OrderedDict # Python equivalent of an enum APPL_KINDS = OrderedDict([('master', 0), ('minion', 1), ('syndic', 2), ('caller', 3)]) APPL_KIND_NAMES = OrderedDict((v, k) for k, v in list(APPL_KINDS.items())) # inverse map ApplKind = namedtuple('ApplKind', list(APPL_KINDS.keys())) applKinds = ApplKind(**APPL_KINDS)
def defaults(prefix, datadict, ignored_keys=None, overridden=None, noresolve=False, firstcall=True): ''' Magic defaults settings configuration getter - Get the "prefix" value from the configuration (pillar/grain) - Then overrides or append to it with the corresponding key in the given "datadict" if value is a dict or a list. - If we get from pillar/grains/local from the curent key in the form: "{prefix}-overrides: it overrides totally the original value. - if the datadict contains a key "{prefix}-append and the value is a list, it appends to the original value - If the datadict contains a key "{prefix}": - If a list: override to the list the default list in conf - Elif a dict: update the default dictionnary with the one in conf - Else take that as a value if the value is not a mapping or a list ''' if not ignored_keys: ignored_keys = [] if firstcall: global_pillar = copy.deepcopy( __salt__['mc_utils.get'](prefix)) if isinstance(global_pillar, dict): for k in [a for a in ignored_keys if a in global_pillar]: if k in global_pillar: del global_pillar[k] datadict = __salt__['mc_utils.dictupdate'](datadict, global_pillar) # if we overrided only keys of a dict # but this dict is an empty dict in the default mapping # be sure to load them inside this dict items = get_uniq_keys_for(prefix) dotedprefix = '{0}.'.format(prefix) for fullkey in items: key = dotedprefix.join(fullkey.split(dotedprefix)[1:]) val = items[fullkey] if isinstance(datadict, dict): curval = datadict.get(key, None) if isinstance(curval, dict): val = __salt__['mc_utils.dictupdate'](curval, val) elif isinstance(curval, (list, set)): if val is not None: for subitem in val: if subitem in curval: continue curval.append(subitem) val = curval datadict[key] = val if overridden is None: overridden = OrderedDict() if prefix not in overridden: overridden[prefix] = OrderedDict() pkeys = OrderedDict() for a in datadict: if a not in ignored_keys and isinstance(a, six.string_types): to_unicode = False for i in prefix, a: if isinstance(i, unicode): to_unicode = True break k = '{0}.{1}'.format(magicstring(prefix), magicstring(a)) if to_unicode: k = k.decode('utf-8') pkeys[a] = (k, datadict[a]) for key, value_data in pkeys.items(): value_key, default_value = value_data # special key to completly override the dictionnary avalue = _default_marker value = __salt__['mc_utils.get']( value_key + "-override", __salt__['mc_utils.get']( value_key + "-overrides", _default_marker) ) if isinstance(default_value, list): avalue = __salt__['mc_utils.get']( value_key + "-append", _default_marker) if value is not _default_marker: overridden[prefix][key] = value else: value = __salt__['mc_utils.get'](value_key, _default_marker) if not isinstance(default_value, list) and value is _default_marker: value = default_value if isinstance(default_value, list): if key in overridden[prefix]: value = overridden[prefix][key] else: nvalue = default_value[:] if ( value and (value != nvalue) and (value is not _default_marker) ): if nvalue is None: nvalue = [] for subitem in value: if subitem in nvalue: continue nvalue.append(subitem) value = nvalue if isinstance(avalue, list): for subitem in avalue: if subitem in value: continue value.append(subitem) elif isinstance(value, dict): # recurvive and conservative dictupdate ndefaults = defaults(value_key, value, overridden=overridden, noresolve=noresolve, firstcall=firstcall) # firstcall=False) if overridden[value_key]: for k, value in overridden[value_key].items(): default_value[k] = value # override specific keys values handle: # eg: makina-states.services.firewall.params.RESTRICTED_SSH = foo # eg: makina-states.services.firewall.params: # foo: var for k, subvalue in get_uniq_keys_for(value_key).items(): ndefaults[k.split('{0}.'.format(value_key))[1]] = subvalue value = __salt__['mc_utils.dictupdate'](default_value, ndefaults) datadict[key] = value for k, value in overridden[prefix].items(): datadict[k] = value if not noresolve: datadict = format_resolve(datadict) return datadict
""" Application Kinds of Salt apps. These are used to indicate what kind of Application is using RAET """ from collections import namedtuple from salt.utils.odict import OrderedDict # Python equivalent of an enum APPL_KINDS = OrderedDict([("master", 0), ("minion", 1), ("syndic", 2), ("caller", 3)]) APPL_KIND_NAMES = OrderedDict( (v, k) for k, v in list(APPL_KINDS.items()) ) # inverse map ApplKind = namedtuple("ApplKind", list(APPL_KINDS.keys())) applKinds = ApplKind(**APPL_KINDS)
def managed(name=HOSTS_FILE, admin_nodes=None, master_nodes=None, worker_nodes=None, other_nodes=None, caasp_hosts_file=CAASP_HOSTS_FILE, append={}, marker_start=None, marker_end=None, **kwargs): ''' Generate a /etc/hosts file. name The hosts file to load/save. admin_nodes The list of admin nodes. master_nodes The list of master nodes. worker_nodes The list of worker nodes. other_nodes The list of other nodes. .. code-block:: yaml /etc/hosts: caasp_hosts.managed ''' this_roles = __salt__['grains.get']('roles', []) infra_domain = __salt__['caasp_pillar.get'](PILLAR_INTERNAL_INFRA, 'infra.caasp.local') assert infra_domain def fqdn(name): return name + '.' + infra_domain # get the previous /etc/hosts file and save it on /etc/caasp/hosts # note that this must be done ony once in tthe first run of the # salt state orig_etc_hosts = name or __salt__['config.option']('hosts.file') if orig_etc_hosts is None: raise InvalidConfigError('Could not obtain current hosts file name') # Load the current /etc/hosts file (for calculating differences later on) orig_etc_hosts_contents = [] if os.path.exists(orig_etc_hosts): orig_etc_hosts_contents = _load_lines(orig_etc_hosts) hosts = OrderedDict() _load_hosts(hosts, MINIMAL_ETC_HOSTS.splitlines(), marker_start=marker_start, marker_end=marker_end) # copy the /etc/hosts to caasp_hosts_file the first time we run this if caasp_hosts_file: caasp_hosts_dir = os.path.dirname(caasp_hosts_file) _makedirs(caasp_hosts_dir) if not os.path.exists(caasp_hosts_file): __utils__['caasp_log.info']('hosts: saving %s in %s', orig_etc_hosts, caasp_hosts_file) _write_lines(caasp_hosts_file, orig_etc_hosts_contents) # TODO remove this file if something goes wrong... try: # remove any previous [marker_start, marker_end] block __salt__['file.blockreplace'](caasp_hosts_file, marker_start, marker_end, content='', backup=False) except Exception as e: __utils__['caasp_log.warn']( 'could not remove old blocks in {}: {}'.format( caasp_hosts_file, e)) assert os.path.exists(caasp_hosts_file) __utils__['caasp_log.info']('hosts: loading entries in "%s" file', caasp_hosts_file) if not os.path.isfile(caasp_hosts_file): raise EtcHostsRuntimeException( '{} cannot be loaded: it is not a file'.format( caasp_hosts_file)) _load_hosts_file(hosts, caasp_hosts_file, marker_start=marker_start, marker_end=marker_end) __utils__['caasp_log.debug']('hosts: custom /etc/hosts entries:') for k, v in hosts.items(): __utils__['caasp_log.debug']('hosts: %s %s', k, v) # get the admin, masters and workers def get_with_expr(expr): return __salt__['caasp_nodes.get_with_expr']( expr, grain='network.interfaces') admin_nodes = admin_nodes or get_with_expr(ADMIN_EXPR) master_nodes = master_nodes or get_with_expr(MASTER_EXPR) worker_nodes = worker_nodes or get_with_expr(WORKER_EXPR) other_nodes = other_nodes or get_with_expr(OTHER_EXPR) # add all the entries try: for nodes in [admin_nodes, master_nodes, worker_nodes, other_nodes]: _add_names_for(hosts, nodes, infra_domain) except Exception as e: raise EtcHostsRuntimeException( 'Could not add entries for roles in /etc/hosts: {}'.format(e)) try: for ip, names in append.items(): _add_names(hosts, ip, names) # add some extra names for the API servers and admin nodes if "kube-master" in this_roles or "admin" in this_roles: external_fqdn_name = __salt__['caasp_pillar.get']( PILLAR_EXTERNAL_FQDN) if not __salt__['caasp_filters.is_ip'](external_fqdn_name): _add_names(hosts, '127.0.0.1', external_fqdn_name) # set the ldap server at the Admin node if "admin" in this_roles: _add_names(hosts, '127.0.0.1', fqdn('ldap')) # try to make Salt happy by adding an ipv6 entry # for the local host (not really used for anything else) this_hostname = __salt__['grains.get']('localhost', '') _add_names(hosts, ['127.0.0.1', '::1'], [this_hostname, fqdn(this_hostname)]) __utils__['caasp_log.debug']( 'hosts: adding entry for the API server at 127.0.0.1') _add_names(hosts, '127.0.0.1', ['api', fqdn('api')]) except Exception as e: raise EtcHostsRuntimeException( 'Could not add special entries in /etc/hosts: {}'.format(e)) # sort the names for determinism for ip, names in hosts.items(): names.sort() # prepend the nodenames at the beginning of each entry try: for nodes in [admin_nodes, master_nodes, worker_nodes, other_nodes]: _add_nodenames_for(hosts, nodes, infra_domain) except Exception as e: raise EtcHostsRuntimeException( 'Could not add nodenames entries in /etc/hosts: {}'.format(e)) # (over)write the /etc/hosts try: preface = PREFACE.format(file=caasp_hosts_file).splitlines() new_etc_hosts_contents = [] for ip, names in hosts.items(): line = '{0} {1}'.format(ip, ' '.join(names)) new_etc_hosts_contents.append(line.strip().replace('\n', '')) new_etc_hosts_contents.sort() new_etc_hosts_contents = preface + new_etc_hosts_contents __utils__['caasp_log.info']('hosts: writing new content to %s', orig_etc_hosts) _write_lines(orig_etc_hosts, new_etc_hosts_contents) except Exception as e: raise EtcHostsRuntimeException('Could not write {} file: {}'.format( orig_etc_hosts, e)) if new_etc_hosts_contents != orig_etc_hosts_contents: # calculate the changes diff = difflib.unified_diff(orig_etc_hosts_contents, new_etc_hosts_contents, lineterm='') return list(diff) else: return []
def create_datagram_endpoint( loop, protocol_factory, local_addr=None, remote_addr=None, family=0, proto=0, flags=0, ): """ Create datagram connection. Based on code from Python 3.5 version, this method is used only in Python 2.7+ versions, since Trollius library did not ported UDP packets broadcast. """ if not (local_addr or remote_addr): if not family: raise ValueError("unexpected address family") addr_pairs_info = (((family, proto), (None, None)), ) else: addr_infos = OrderedDict() for idx, addr in ((0, local_addr), (1, remote_addr)): if addr is not None: assert (isinstance(addr, tuple) and len(addr) == 2), "2-tuple is expected" infos = yield asyncio.coroutines.From( loop.getaddrinfo(*addr, family=family, type=socket.SOCK_DGRAM, proto=proto, flags=flags)) if not infos: raise socket.error("getaddrinfo() returned empty list") for fam, _, pro, _, address in infos: key = (fam, pro) if key not in addr_infos: addr_infos[key] = [None, None] addr_infos[key][idx] = address addr_pairs_info = [(key, addr_pair) for key, addr_pair in addr_infos.items() if not ((local_addr and addr_pair[0] is None) or (remote_addr and addr_pair[1] is None))] if not addr_pairs_info: raise ValueError("can not get address information") exceptions = [] for ((family, proto), (local_address, remote_address)) in addr_pairs_info: sock = r_addr = None try: sock = socket.socket(family=family, type=socket.SOCK_DGRAM, proto=proto) for opt in [socket.SO_REUSEADDR, socket.SO_BROADCAST]: sock.setsockopt(socket.SOL_SOCKET, opt, 1) sock.setblocking(False) if local_addr: sock.bind(local_address) if remote_addr: yield asyncio.coroutines.From( loop.sock_connect(sock, remote_address)) r_addr = remote_address except socket.error as exc: if sock is not None: sock.close() exceptions.append(exc) except Exception: # pylint: disable=broad-except if sock is not None: sock.close() raise else: break else: raise exceptions[0] protocol = protocol_factory() waiter = asyncio.futures.Future(loop=loop) transport = loop._make_datagram_transport(sock, protocol, r_addr, waiter) try: yield asyncio.coroutines.From(waiter) except Exception: # pylint: disable=broad-except transport.close() raise raise asyncio.coroutines.Return(transport, protocol)
# -*- coding: utf-8 -*- ''' Application Kinds of Salt apps. These are used to indicate what kind of Application is using RAET ''' from __future__ import absolute_import, unicode_literals from collections import namedtuple from salt.utils.odict import OrderedDict # Python equivalent of an enum APPL_KINDS = OrderedDict([('master', 0), ('minion', 1), ('syndic', 2), ('caller', 3)]) APPL_KIND_NAMES = OrderedDict((v, k) for k, v in list(APPL_KINDS.items())) # inverse map ApplKind = namedtuple('ApplKind', list(APPL_KINDS.keys())) applKinds = ApplKind(**APPL_KINDS)