class Test_Security: """ This Test Class contains security checks on the configuration data based on our Security Policy. """ @staticmethod @pytest.mark.parametrize( "config_path, pool_members", iterutils.research(tenants, lambda p, k, v: k == "serverAddresses"), ) def test_pool_must_use_private_addresses(config_path, pool_members): """ The Security Policy requires pool members use private addresses / no public IPs """ print(f"currently processing: {config_path}, members: ", end="") for member in pool_members: print(f"member:{member} ", end="") assert IPv4Address( member).is_private # member IP address must be private @staticmethod @pytest.mark.parametrize( "config_path, servicePort", iterutils.research(tenants, lambda p, k, v: k == "servicePort"), ) def test_pool_allow_specific_ports(config_path, servicePort, allowed_ports=(80, 81, 8000, 8080)): """ Pool members must use a port in the allowed ports list """ print( f"currently processing: {config_path}, servicePort: {servicePort}", end=" ") assert isinstance(servicePort, int) # servicePort must be an integer assert (servicePort in allowed_ports ) # servicePort must be listed in allowed_ports @staticmethod @pytest.mark.parametrize( "config_path, virtualPort", iterutils.research(tenants, lambda p, k, v: k == "virtualPort"), ) def test_disallowed_service_ports( config_path, virtualPort, disallowed_ports=( 0, # no wildcard listeners 6667, # disallowed port )): """ Services must not listen on the disallowed_ports """ print( f"currently processing: {config_path}, virtualPort: {virtualPort}", end=" ") if virtualPort is not None: assert isinstance(virtualPort, int) # virtualPort must be an integer assert (virtualPort not in disallowed_ports ) # virtualPort must not be in disallowed_ports
def get_config_t(conf: Dict) -> Type: """ First create the transformation dictionary Then read the config dictionary and create the config type """ transname_dict = dict( research(conf, query=lambda p, k, v: k == _type_spec[9]) ) # construct dictionary mapping paths ('k1', ..., 'kn') to transformation name # remove last key -> 'transformation' and lookup the transformation with correct name trans_dict = { _path_to_glom_spec(k[:-1]): lookup_trans(v) for k, v in transname_dict.items() } paths = _nodes(conf) leaves = _leaves(paths) # create a copy of the dictionary, and recursively update the leaf nodes _conf = reduce(_update_inplace(_str_to_spec), leaves, deepcopy(conf)) # walk up the tree, and process the "new" leaf nodes. using a set takes # care of duplicates. branches: Set[_Path_t] = _leaves(paths - leaves) while branches: _conf = reduce(_update_inplace(_nested_type), branches, _conf) branches = {path[:-1] for path in branches if path[:-1]} config_t = _spec_to_type("config", _conf, bases=(_ConfigIO, )) config_t.trans_dict = trans_dict return config_t
def make_paths_relative(doc: Dict, base_directory: PurePath, allow_paths_outside_base=False): """ Find all pathlib.Path values in a document structure and make them relative to the given path. >>> from copy import deepcopy >>> base = PurePath('/tmp/basket') >>> doc = {'id': 1, 'fruits': [{'apple': PurePath('/tmp/basket/fruits/apple.txt')}]} >>> make_paths_relative(doc, base) >>> doc {'id': 1, 'fruits': [{'apple': 'fruits/apple.txt'}]} >>> # No change if repeated. (relative paths still relative) >>> previous = deepcopy(doc) >>> make_paths_relative(doc, base) >>> doc == previous True >>> # Relative pathlibs also become relative strings for consistency. >>> doc = {'villains': PurePath('the-baron.txt')} >>> make_paths_relative(doc, base) >>> doc {'villains': 'the-baron.txt'} """ for doc_path, value in iterutils.research( doc, lambda p, k, v: isinstance(v, PurePath)): value: PurePath value = relative_path( value, base_directory, allow_paths_outside_base=allow_paths_outside_base) docpath_set(doc, doc_path, value.as_posix())
def permute_nested_values(dicts: "List[dict]", gen_val: "Callable[[int], Any]"): """ This function permutes the values of a nested mapping, for testing that out merge method work regardless of the values types. Assumes the intial dictionary had integers for values. """ dicts = deepcopy(dicts) initial_values = [ x[1] for x in research(dicts, query=lambda p, k, v: isinstance(v, int)) ] mapping = {k: gen_val(k) for k in initial_values} return [remap(d, exit=partial(map_values, mapping)) for d in dicts]
def _nodes(conf: Dict) -> Set[_Path_t]: """Filter the list of paths for nodes Parameters ---------- conf : Dict Config dictionary Returns ------- Set[_Path_t] List of paths to nodes """ return {path for path, _ in research(conf, query=_is_node)}
def validate_item(item: Dict): validate_document(item, _ITEM_SCHEMA, schema_folder=_ITEM_SCHEMA_PATH.parent) # Should be a valid polygon assert "geometry" in item, "Item has no geometry" assert item["geometry"], "Item has blank geometry" with DebugContext(f"Failing shape:\n{pformat(item['geometry'])}"): shape = shapely_shape(item["geometry"]) assert shape.is_valid, f"Item has invalid geometry: {explain_validity(shape)}" assert shape.geom_type in ( "Polygon", "MultiPolygon", ), "Unexpected type of shape" # href should never be blank if present # -> The jsonschema enforces href as required, but it's not checking for emptiness. # (and we've had empty ones in previous prototypes) for offset, value in research(item, lambda p, k, v: k == "href"): viewable_offset = "→".join(map(repr, offset)) assert value.strip(), f"href has empty value: {repr(viewable_offset)}"
def make_paths_relative( doc: Dict, base_directory: PurePath, allow_paths_outside_base=False ): """ Find all pathlib.Path values in a document structure and make them relative to the given path. >>> base = PurePath('/tmp/basket') >>> doc = {'id': 1, 'fruits': [{'apple': PurePath('/tmp/basket/fruits/apple.txt')}]} >>> make_paths_relative(doc, base) >>> doc {'id': 1, 'fruits': [{'apple': 'fruits/apple.txt'}]} >>> # No change if repeated. (relative paths still relative) >>> previous = deepcopy(doc) >>> make_paths_relative(doc, base) >>> doc == previous True >>> # Relative pathlibs also become relative strings for consistency. >>> doc = {'villains': PurePath('the-baron.txt')} >>> make_paths_relative(doc, base) >>> doc {'villains': 'the-baron.txt'} """ for doc_path, value in iterutils.research( doc, lambda p, k, v: isinstance(v, PurePath) ): value: Path if value.is_absolute(): if base_directory not in value.parents: if not allow_paths_outside_base: raise ValueError( f"Path {value.as_posix()!r} is outside path {base_directory.as_posix()!r} " f"(allow_paths_outside_base={allow_paths_outside_base})" ) continue value = value.relative_to(base_directory) docpath_set(doc, doc_path, str(value))
class Test_RequiredPractices: """ These tests make sure the configuration follows required practices. Required practices usually ensure the configuration follows good/best practices to ensure they function as well as possible """ @staticmethod @pytest.mark.parametrize("config_path, _", iterutils.research(tenants, lambda p, k, v: v == "Pool")) def test_pools_have_monitors(config_path, _): """ Require monitor configuration for each service """ config_path = config_path[ 0: -1] # remove 'class' from config_path, which is always the last element monitors = iterutils.get_path(tenants, config_path).get("monitors") print(f"currently processing: {config_path}, monitors: {monitors}", end=" ") assert monitors # not empty and not None for monitor in monitors: assert monitor # make we don't have an empty entry in the list of monitors either!
def test_research(): root = {} with pytest.raises(TypeError): research(root, query=None) root = {'a': 'a'} res = research(root, query=lambda p, k, v: v == 'a') assert len(res) == 1 assert res[0] == (('a', ), 'a') def broken_query(p, k, v): raise RuntimeError() with pytest.raises(RuntimeError): research(root, broken_query, reraise=True) # empty results with default, reraise=False assert research(root, broken_query) == []
def test_research(): root = {} with pytest.raises(TypeError): research(root, query=None) root = {'a': 'a'} res = research(root, query=lambda p, k, v: v == 'a') assert len(res) == 1 assert res[0] == (('a',), 'a') def broken_query(p, k, v): raise RuntimeError() with pytest.raises(RuntimeError): research(root, broken_query, reraise=True) # empty results with default, reraise=False assert research(root, broken_query) == []
def serve( known, nginx_command, parsed_config, parsed_config_str, parsed_config_http, parsed_config_http_str, ): if not os.path.isdir(known.temp_dir): os.mkdir(known.temp_dir) logger.debug("temp_dir:\t{!r}".format(known.temp_dir)) _config_files = "nginx.conf", "mime.types" nginx_conf_join = partial( os.path.join, os.path.join( os.path.dirname( os.path.join( resource_filename(PythonPackageInfo().get_app_name(), "__init__.py"))), "_config", ), ) config_files = tuple(map(nginx_conf_join, _config_files)) deque(map(partial(copy, dst=known.temp_dir), config_files), maxlen=0) sites_available = os.path.join(known.temp_dir, "sites-available") if not os.path.isdir(sites_available): os.mkdir(sites_available) server_conf = os.path.join(sites_available, "server.conf") if parsed_config is not None: pp(parsed_config) pp( research( parsed_config, query=lambda p, k, v: is_directive(v) and print( "k:", k, ";\nv:", v, ";"), )) # pp(research(parsed_config, query=lambda p, k, v: is_directive(v))) with open(server_conf, "wt") as f: if parsed_config_http_str is not None: f.write(parsed_config_http_str) if parsed_config_str is not None: f.write(parsed_config_str) # Include this config in the new nginx.conf nginx_conf = os.path.join(known.temp_dir, _config_files[0]) nginx_conf_parsed = crossplane.parse(nginx_conf, catch_errors=False, comments=False) nginx_conf_parse = next( config for config in nginx_conf_parsed["config"] if os.path.basename(config["file"]) == "nginx.conf") line = count(nginx_conf_parse["parsed"][-1]["block"][-1]["line"]) del nginx_conf_parse["parsed"][-1]["block"][-1] nginx_conf_parse["parsed"].insert(1, { "args": ["off"], "directive": "daemon" }) nginx_conf_parse["parsed"][-1]["block"] += [ { "args": ["stderr", "warn"], "directive": "error_log", "line": next(line) }, { "args": ["/dev/stdout"], "directive": "access_log", "line": next(line) }, { "args": [os.path.join(sites_available, "*.conf")], "directive": "include", "includes": [2], "line": next(line), }, ] config_str = crossplane.build(nginx_conf_parse["parsed"]) os.remove(nginx_conf) with open(nginx_conf, "wt") as f: f.write(config_str + os.linesep) # logger.error print("nginx is running. Stop with: {}".format(" ".join( (known.nginx, "-c", nginx_conf, "-s", "stop")))) Popen([known.nginx, "-c", nginx_conf] + nginx_command)