def test_get_configuration_not_found(**kwargs): from python_moonutilities import configuration kwargs['mock'].get('http://consul:8500/v1/kv/components/port_start_wrong', json=[ ], status_code=500) with pytest.raises(Exception) as exception_info: configuration.get_configuration("components/port_start_wrong") assert str(exception_info.value) == '500: Consul error'
def test_get_configuration_invalid_response(**kwargs): from python_moonutilities import configuration kwargs['mock'].get('http://consul:8500/v1/kv/components/port_start', json=[ {"port_start":'port_start', 'Value': comp_util.get_b64_conf("components/port_start")} ]) with pytest.raises(Exception) as exception_info: configuration.get_configuration("components/port_start") assert str(exception_info.value) == '500: Consul Content error'
def __init__(self): super(K8S, self).__init__() config.load_kube_config() self.client = client.CoreV1Api() conf = configuration.get_configuration("components/orchestrator") self.orchestrator_hostname = conf["components/orchestrator"].get("hostname", "orchestrator") self.orchestrator_port = conf["components/orchestrator"].get("port", 80) conf = configuration.get_configuration("components/manager") self.manager_hostname = conf["components/manager"].get("hostname", "manager") self.manager_port = conf["components/manager"].get("port", 80)
def delete_pod(uuid): conf = configuration.get_configuration("components/orchestrator") hostname = conf["components/orchestrator"].get("hostname", "orchestrator") port = conf["components/orchestrator"].get("port", 80) proto = conf["components/orchestrator"].get("protocol", "http") # while True: # try: url = "{}://{}:{}/pods".format(proto, hostname, port) req = requests.get(url) # except requests.exceptions.ConnectionError: # logger.warning("Orchestrator is not ready, standby... {}".format(url)) # time.sleep(1) # else: # break for pod_key, pod_list in req.json().get("pods", {}).items(): for pod_value in pod_list: if "pdp_id" in pod_value: if pod_value["pdp_id"] == uuid: req = requests.delete( "{}://{}:{}/pods/{}".format(proto, hostname, port, pod_key)) if req.status_code != 200: logger.warning( "Cannot delete pod {} - {}".format(pod_key, pod_value['name'])) logger.debug(req.content) # Note (Asteroide): no need to go further if one match break
def create_wrappers(self, slave_name=None): contexts, active_context = self.get_contexts() logger.debug("contexts: {}".format(contexts)) logger.debug("active_context: {}".format(active_context)) conf = configuration.get_configuration("components/wrapper") hostname = conf["components/wrapper"].get( "hostname", "wrapper") port = conf["components/wrapper"].get("port", 80) container = conf["components/wrapper"].get( "container", "wukongsun/moon_wrapper:v4.3") for _ctx in contexts: if slave_name and slave_name != _ctx['name']: continue _config = config.new_client_from_config(context=_ctx['name']) logger.debug("_config={}".format(_config)) api_client = client.CoreV1Api(_config) ext_client = client.ExtensionsV1beta1Api(_config) data = [{ "name": hostname + "-" + get_random_name(), "container": container, "port": port, "namespace": "moon", "slave_name": _ctx['name'] }, ] self.load_deployment_and_service(data, api_client, ext_client, expose=True)
def add_pod(uuid, data): if not data.get("keystone_project_id"): return logger.info("Add a new pod {}".format(data)) if "pdp_id" not in data: data["pdp_id"] = uuid data['policies'] = PolicyManager.get_policies(user_id="admin") data['models'] = ModelManager.get_models(user_id="admin") conf = configuration.get_configuration("components/orchestrator") hostname = conf["components/orchestrator"].get("hostname", "orchestrator") port = conf["components/orchestrator"].get("port", 80) proto = conf["components/orchestrator"].get("protocol", "http") while True: try: req = requests.post( "{}://{}:{}/pods".format(proto, hostname, port), json=data, headers={"content-type": "application/json"}) except requests.exceptions.ConnectionError as e: logger.warning("add_pod: Orchestrator is not ready, standby...") logger.exception(e) time.sleep(1) else: break logger.info("Pod add request answer : {}".format(req.text))
def __init__(self, host="localhost", port=80, **kwargs): super(HTTPServer, self).__init__(host=host, port=port, **kwargs) self.app = Flask(__name__) conf = configuration.get_configuration("components/orchestrator") self.orchestrator_hostname = conf["components/orchestrator"].get( "hostname", "orchestrator") self.orchestrator_port = conf["components/orchestrator"].get( "port", 80) conf = configuration.get_configuration("components/manager") self.manager_hostname = conf["components/manager"].get( "hostname", "manager") self.manager_port = conf["components/manager"].get("port", 80) # TODO : specify only few urls instead of * # CORS(self.app) self.api = Api(self.app) self.driver = get_driver() logger.info("Driver = {}".format(self.driver.__class__)) self.__set_route() self.__hook_errors() pdp = None while True: try: pdp = requests.get("http://{}:{}/pdp".format( self.manager_hostname, self.manager_port)) except requests.exceptions.ConnectionError: logger.warning("Manager is not ready, standby...") time.sleep(1) except KeyError: logger.warning("Manager is not ready, standby...") time.sleep(1) else: if "pdps" in pdp.json(): break logger.debug("pdp={}".format(pdp)) self.driver.create_wrappers() for _pdp_key, _pdp_value in pdp.json()['pdps'].items(): if _pdp_value.get('keystone_project_id'): # TODO: select context to add security function self.driver.create_pipeline( keystone_project_id=_pdp_value.get('keystone_project_id'), pdp_id=_pdp_key, policy_ids=_pdp_value.get('security_pipeline', []))
def test_get_configuration_mutliple_list_success(**kwargs): from python_moonutilities import configuration kwargs['mock'].get('http://consul:8500/v1/kv/components/port_start', json=[ {'Key': 'port_start', 'Value': comp_util.get_b64_conf("components/port_start")}, {'Key': 'port_start', 'Value': comp_util.get_b64_conf("components/port_start")} ] ) assert len(configuration.get_configuration("components/port_start")) == 2
def __init__(self, host="localhost", port=80, **kwargs): super(HTTPServer, self).__init__(host=host, port=port, **kwargs) self.app = Flask(__name__) self.port = port conf = configuration.get_configuration("components/manager") self.manager_hostname = conf["components/manager"].get( "hostname", "manager") self.manager_port = conf["components/manager"].get("port", 80) self.api = Api(self.app) self.__set_route() self.__hook_errors()
def __init__(self, host="localhost", port=80, **kwargs): super(HTTPServer, self).__init__(host=host, port=port, **kwargs) self.app = Flask(__name__) self.app.config['TRAP_HTTP_EXCEPTIONS'] = True conf = configuration.get_configuration("components/manager") self.manager_hostname = conf["components/manager"].get( "hostname", "manager") self.manager_port = conf["components/manager"].get("port", 80) # TODO : specify only few urls instead of * CORS(self.app) self.api = CustomApi(self.app, catch_all_404s=True) self.__set_route()
def main(): configuration.init_logging() try: conf = configuration.get_configuration("components/wrapper") LOG.debug("wrapper.conf={}".format(conf)) hostname = conf["components/wrapper"].get("hostname", "wrapper") port = conf["components/wrapper"].get("port", 80) bind = conf["components/wrapper"].get("bind", "127.0.0.1") except exceptions.ConsulComponentNotFound: hostname = "wrapper" bind = "127.0.0.1" port = 80 configuration.add_component(uuid="wrapper", name=hostname, port=port, bind=bind) LOG.info("Starting server with IP {} on port {} bind to {}".format(hostname, port, bind)) return HTTPServer(host=bind, port=port)
def __init__(self, connector=None): self.driver = connector.driver Managers.KeystoneManager = self conf = configuration.get_configuration( "openstack/keystone")['openstack/keystone'] self.__url = conf['url'] self.__user = conf['user'] self.__password = conf['password'] self.__domain = conf['domain'] self.__project = conf['project'] try: os.environ.pop("http_proxy") os.environ.pop("https_proxy") except KeyError: pass
def __init__(self, host="localhost", port=80, **kwargs): super(HTTPServer, self).__init__(host=host, port=port, **kwargs) self.app = Flask(__name__) self.port = port conf = configuration.get_configuration("components/orchestrator") _hostname = conf["components/orchestrator"].get( "hostname", "orchestrator") _port = conf["components/orchestrator"].get("port", 80) _protocol = conf["components/orchestrator"].get("protocol", "http") self.orchestrator_url = "{}://{}:{}".format(_protocol, _hostname, _port) # Todo : specify only few urls instead of * # CORS(self.app) self.api = Api(self.app) self.__set_route() self.__hook_errors()
def create_server(): configuration.init_logging() try: conf = configuration.get_configuration("components/orchestrator") hostname = conf["components/orchestrator"].get("hostname", "orchestrator") port = conf["components/orchestrator"].get("port", 80) bind = conf["components/orchestrator"].get("bind", "127.0.0.1") except exceptions.ConsulComponentNotFound: hostname = "orchestrator" bind = "127.0.0.1" port = 80 configuration.add_component(uuid="orchestrator", name=hostname, port=port, bind=bind) logger.info("Starting server with IP {} on port {} bind to {}".format( hostname, port, bind)) return HTTPServer(host=bind, port=port)
def test_get_configuration_success(): from python_moonutilities import configuration assert configuration.get_configuration("components/port_start")["components/port_start"] == comp_util.CONF["components"]["port_start"]
super(PDPDriver, self).__init__(driver_name, engine_name) def update_pdp(self, pdp_id, value): raise NotImplementedError() # pragma: no cover def delete_pdp(self, pdp_id): raise NotImplementedError() # pragma: no cover def add_pdp(self, pdp_id=None, value=None): raise NotImplementedError() # pragma: no cover def get_pdp(self, pdp_id=None): raise NotImplementedError() # pragma: no cover class KeystoneDriver(Driver): def __init__(self, driver_name, engine_name): super(KeystoneDriver, self).__init__(driver_name, engine_name) conf = configuration.get_configuration("database")['database'] KeystoneManager = keystone.KeystoneManager( KeystoneDriver(conf['driver'], conf['url'])) ModelManager = model.ModelManager(ModelDriver(conf['driver'], conf['url'])) PolicyManager = policy.PolicyManager(PolicyDriver(conf['driver'], conf['url'])) PDPManager = pdp.PDPManager(PDPDriver(conf['driver'], conf['url']))
def __init__(self, **kwargs): conf = configuration.get_configuration("components/orchestrator") self.orchestrator_hostname = conf["components/orchestrator"].get( "hostname", "orchestrator") self.orchestrator_port = conf["components/orchestrator"].get( "port", 80)
# Copyright 2015 Open Platform for NFV Project, Inc. and its contributors # This software is distributed under the terms and conditions of the 'Apache-2.0' # license which can be found in the file 'LICENSE' in this package distribution # or at 'http://www.apache.org/licenses/LICENSE-2.0'. import os import requests import time from functools import wraps from flask import request from oslo_log import log as logging from python_moonutilities import exceptions, configuration logger = logging.getLogger(__name__) KEYSTONE_CONFIG = configuration.get_configuration( "openstack/keystone")["openstack/keystone"] TOKENS = {} def check_token(token, url=None): _verify = False if KEYSTONE_CONFIG['certificate']: _verify = KEYSTONE_CONFIG['certificate'] try: os.environ.pop("http_proxy") os.environ.pop("https_proxy") except KeyError: pass if not url: url = KEYSTONE_CONFIG['url'] headers = {
def create_pipeline(self, keystone_project_id, pdp_id, policy_ids, manager_data=None, active_context=None, slave_name=None): """ Create security functions :param keystone_project_id: the Keystone project id :param pdp_id: the PDP ID mapped to this pipeline :param policy_ids: the policy IDs mapped to this pipeline :param manager_data: data needed to create pods :param active_context: if present, add the security function in this context :param slave_name: if present, add the security function in this context name if active_context_name and active_context are not present, add the security function in all context (ie, in all slaves) :return: None """ if not manager_data: manager_data = dict() for key, value in self.get_pods().items(): for _pod in value: if _pod.get('keystone_project_id') == keystone_project_id: logger.warning("A pod for this Keystone project {} " "already exists.".format(keystone_project_id)) return plugins = configuration.get_plugins() conf = configuration.get_configuration("components/pipeline") # i_hostname = conf["components/pipeline"].get("interface").get("hostname", "interface") i_port = conf["components/pipeline"].get("interface").get("port", 80) i_container = conf["components/pipeline"].get("interface").get( "container", "wukongsun/moon_interface:v4.3") data = [ { "name": "pipeline-" + get_random_name(), "container": i_container, "port": i_port, 'pdp_id': pdp_id, 'genre': "interface", 'keystone_project_id': keystone_project_id, "namespace": "moon" }, ] logger.debug("data={}".format(data)) # When policies and models are empty, is it right that it returns 200 ? # Should it return no found policies or models ? policies = manager_data.get('policies') if not policies: logger.info("No policy data from Manager, trying to get them") policies = requests.get("http://{}:{}/policies".format( self.manager_hostname, self.manager_port)).json().get( "policies", dict()) logger.debug("policies={}".format(policies)) models = manager_data.get('models') if not models: logger.info("No models data from Manager, trying to get them") models = requests.get("http://{}:{}/models".format( self.manager_hostname, self.manager_port)).json().get( "models", dict()) logger.debug("models={}".format(models)) if not policy_ids: raise exceptions.PolicyUnknown for policy_id in policy_ids: if policy_id in policies: genre = policies[policy_id].get("genre", "authz") if genre in plugins: for meta_rule in models[policies[policy_id]['model_id']]['meta_rules']: data.append({ "name": genre + "-" + get_random_name(), "container": plugins[genre]['container'], 'pdp_id': pdp_id, "port": plugins[genre].get('port', 8080), 'genre': genre, 'policy_id': policy_id, 'meta_rule_id': meta_rule, 'keystone_project_id': keystone_project_id, "namespace": "moon" }) logger.debug("data={}".format(data)) contexts, _active_context = self.get_contexts() logger.debug("active_context_name={}".format(slave_name)) logger.debug("active_context={}".format(active_context)) if slave_name: for _context in contexts: if _context["name"] == slave_name: active_context = _context break if active_context: active_context = _active_context _config = config.new_client_from_config( context=active_context['name']) logger.debug("_config={}".format(_config)) api_client = client.CoreV1Api(_config) ext_client = client.ExtensionsV1beta1Api(_config) self.load_deployment_and_service(data, api_client, ext_client, expose=False) return logger.debug("contexts={}".format(contexts)) for _ctx in contexts: if slave_name and slave_name != _ctx['name']: continue _config = config.new_client_from_config(context=_ctx['name']) logger.debug("_config={}".format(_config)) api_client = client.CoreV1Api(_config) ext_client = client.ExtensionsV1beta1Api(_config) self.load_deployment_and_service(data, api_client, ext_client, expose=False)
# or at 'http://www.apache.org/licenses/LICENSE-2.0'. import re import os import types import requests import time from functools import wraps from flask import request import logging from python_moonutilities import exceptions, configuration logger = logging.getLogger("moon.utilities." + __name__) keystone_config = configuration.get_configuration("openstack/keystone")["openstack/keystone"] TOKENS = {} __targets = {} def filter_input(func_or_str): def __filter(string): if string and type(string) is str: return "".join(re.findall("[\w\- +]*", string)) return string def __filter_dict(arg): result = dict() for key in arg.keys(): if key == "email":
def init_engine(): db_conf = configuration.get_configuration("database")["database"] return create_engine(db_conf['url'])