def test_get_configuration_not_found(**kwargs):
from python_moonutilities import configuration
kwargs['mock'].get('http://consul:8500/v1/kv/components/port_start_wrong', json=[
], status_code=500)
with pytest.raises(Exception) as exception_info:
configuration.get_configuration("components/port_start_wrong")
assert str(exception_info.value) == '500: Consul error'
def test_get_configuration_invalid_response(**kwargs):
from python_moonutilities import configuration
kwargs['mock'].get('http://consul:8500/v1/kv/components/port_start', json=[
{"port_start":'port_start', 'Value': comp_util.get_b64_conf("components/port_start")}
])
with pytest.raises(Exception) as exception_info:
configuration.get_configuration("components/port_start")
assert str(exception_info.value) == '500: Consul Content error'
def __init__(self):
super(K8S, self).__init__()
config.load_kube_config()
self.client = client.CoreV1Api()
conf = configuration.get_configuration("components/orchestrator")
self.orchestrator_hostname = conf["components/orchestrator"].get("hostname", "orchestrator")
self.orchestrator_port = conf["components/orchestrator"].get("port", 80)
conf = configuration.get_configuration("components/manager")
self.manager_hostname = conf["components/manager"].get("hostname", "manager")
self.manager_port = conf["components/manager"].get("port", 80)
def delete_pod(uuid):
conf = configuration.get_configuration("components/orchestrator")
hostname = conf["components/orchestrator"].get("hostname", "orchestrator")
port = conf["components/orchestrator"].get("port", 80)
proto = conf["components/orchestrator"].get("protocol", "http")
# while True:
# try:
url = "{}://{}:{}/pods".format(proto, hostname, port)
req = requests.get(url)
# except requests.exceptions.ConnectionError:
# logger.warning("Orchestrator is not ready, standby... {}".format(url))
# time.sleep(1)
# else:
# break
for pod_key, pod_list in req.json().get("pods", {}).items():
for pod_value in pod_list:
if "pdp_id" in pod_value:
if pod_value["pdp_id"] == uuid:
req = requests.delete(
"{}://{}:{}/pods/{}".format(proto, hostname, port, pod_key))
if req.status_code != 200:
logger.warning(
"Cannot delete pod {} - {}".format(pod_key, pod_value['name']))
logger.debug(req.content)
# Note (Asteroide): no need to go further if one match
break
def create_wrappers(self, slave_name=None):
contexts, active_context = self.get_contexts()
logger.debug("contexts: {}".format(contexts))
logger.debug("active_context: {}".format(active_context))
conf = configuration.get_configuration("components/wrapper")
hostname = conf["components/wrapper"].get(
"hostname", "wrapper")
port = conf["components/wrapper"].get("port", 80)
container = conf["components/wrapper"].get(
"container",
"wukongsun/moon_wrapper:v4.3")
for _ctx in contexts:
if slave_name and slave_name != _ctx['name']:
continue
_config = config.new_client_from_config(context=_ctx['name'])
logger.debug("_config={}".format(_config))
api_client = client.CoreV1Api(_config)
ext_client = client.ExtensionsV1beta1Api(_config)
data = [{
"name": hostname + "-" + get_random_name(),
"container": container,
"port": port,
"namespace": "moon",
"slave_name": _ctx['name']
}, ]
self.load_deployment_and_service(data, api_client, ext_client, expose=True)
def add_pod(uuid, data):
if not data.get("keystone_project_id"):
return
logger.info("Add a new pod {}".format(data))
if "pdp_id" not in data:
data["pdp_id"] = uuid
data['policies'] = PolicyManager.get_policies(user_id="admin")
data['models'] = ModelManager.get_models(user_id="admin")
conf = configuration.get_configuration("components/orchestrator")
hostname = conf["components/orchestrator"].get("hostname", "orchestrator")
port = conf["components/orchestrator"].get("port", 80)
proto = conf["components/orchestrator"].get("protocol", "http")
while True:
try:
req = requests.post(
"{}://{}:{}/pods".format(proto, hostname, port),
json=data,
headers={"content-type": "application/json"})
except requests.exceptions.ConnectionError as e:
logger.warning("add_pod: Orchestrator is not ready, standby...")
logger.exception(e)
time.sleep(1)
else:
break
logger.info("Pod add request answer : {}".format(req.text))
def __init__(self, host="localhost", port=80, **kwargs):
super(HTTPServer, self).__init__(host=host, port=port, **kwargs)
self.app = Flask(__name__)
conf = configuration.get_configuration("components/orchestrator")
self.orchestrator_hostname = conf["components/orchestrator"].get(
"hostname", "orchestrator")
self.orchestrator_port = conf["components/orchestrator"].get(
"port", 80)
conf = configuration.get_configuration("components/manager")
self.manager_hostname = conf["components/manager"].get(
"hostname", "manager")
self.manager_port = conf["components/manager"].get("port", 80)
# TODO : specify only few urls instead of *
# CORS(self.app)
self.api = Api(self.app)
self.driver = get_driver()
logger.info("Driver = {}".format(self.driver.__class__))
self.__set_route()
self.__hook_errors()
pdp = None
while True:
try:
pdp = requests.get("http://{}:{}/pdp".format(
self.manager_hostname, self.manager_port))
except requests.exceptions.ConnectionError:
logger.warning("Manager is not ready, standby...")
time.sleep(1)
except KeyError:
logger.warning("Manager is not ready, standby...")
time.sleep(1)
else:
if "pdps" in pdp.json():
break
logger.debug("pdp={}".format(pdp))
self.driver.create_wrappers()
for _pdp_key, _pdp_value in pdp.json()['pdps'].items():
if _pdp_value.get('keystone_project_id'):
# TODO: select context to add security function
self.driver.create_pipeline(
keystone_project_id=_pdp_value.get('keystone_project_id'),
pdp_id=_pdp_key,
policy_ids=_pdp_value.get('security_pipeline', []))
def test_get_configuration_mutliple_list_success(**kwargs):
from python_moonutilities import configuration
kwargs['mock'].get('http://consul:8500/v1/kv/components/port_start',
json=[
{'Key': 'port_start', 'Value': comp_util.get_b64_conf("components/port_start")},
{'Key': 'port_start', 'Value': comp_util.get_b64_conf("components/port_start")}
]
)
assert len(configuration.get_configuration("components/port_start")) == 2
def __init__(self, host="localhost", port=80, **kwargs):
super(HTTPServer, self).__init__(host=host, port=port, **kwargs)
self.app = Flask(__name__)
self.port = port
conf = configuration.get_configuration("components/manager")
self.manager_hostname = conf["components/manager"].get(
"hostname", "manager")
self.manager_port = conf["components/manager"].get("port", 80)
self.api = Api(self.app)
self.__set_route()
self.__hook_errors()
def __init__(self, host="localhost", port=80, **kwargs):
super(HTTPServer, self).__init__(host=host, port=port, **kwargs)
self.app = Flask(__name__)
self.app.config['TRAP_HTTP_EXCEPTIONS'] = True
conf = configuration.get_configuration("components/manager")
self.manager_hostname = conf["components/manager"].get(
"hostname", "manager")
self.manager_port = conf["components/manager"].get("port", 80)
# TODO : specify only few urls instead of *
CORS(self.app)
self.api = CustomApi(self.app, catch_all_404s=True)
self.__set_route()
def main():
configuration.init_logging()
try:
conf = configuration.get_configuration("components/wrapper")
LOG.debug("wrapper.conf={}".format(conf))
hostname = conf["components/wrapper"].get("hostname", "wrapper")
port = conf["components/wrapper"].get("port", 80)
bind = conf["components/wrapper"].get("bind", "127.0.0.1")
except exceptions.ConsulComponentNotFound:
hostname = "wrapper"
bind = "127.0.0.1"
port = 80
configuration.add_component(uuid="wrapper", name=hostname, port=port, bind=bind)
LOG.info("Starting server with IP {} on port {} bind to {}".format(hostname, port, bind))
return HTTPServer(host=bind, port=port)
def __init__(self, connector=None):
self.driver = connector.driver
Managers.KeystoneManager = self
conf = configuration.get_configuration(
"openstack/keystone")['openstack/keystone']
self.__url = conf['url']
self.__user = conf['user']
self.__password = conf['password']
self.__domain = conf['domain']
self.__project = conf['project']
try:
os.environ.pop("http_proxy")
os.environ.pop("https_proxy")
except KeyError:
pass
def __init__(self, host="localhost", port=80, **kwargs):
super(HTTPServer, self).__init__(host=host, port=port, **kwargs)
self.app = Flask(__name__)
self.port = port
conf = configuration.get_configuration("components/orchestrator")
_hostname = conf["components/orchestrator"].get(
"hostname", "orchestrator")
_port = conf["components/orchestrator"].get("port", 80)
_protocol = conf["components/orchestrator"].get("protocol", "http")
self.orchestrator_url = "{}://{}:{}".format(_protocol, _hostname,
_port)
# Todo : specify only few urls instead of *
# CORS(self.app)
self.api = Api(self.app)
self.__set_route()
self.__hook_errors()
def create_server():
configuration.init_logging()
try:
conf = configuration.get_configuration("components/orchestrator")
hostname = conf["components/orchestrator"].get("hostname",
"orchestrator")
port = conf["components/orchestrator"].get("port", 80)
bind = conf["components/orchestrator"].get("bind", "127.0.0.1")
except exceptions.ConsulComponentNotFound:
hostname = "orchestrator"
bind = "127.0.0.1"
port = 80
configuration.add_component(uuid="orchestrator", name=hostname,
port=port, bind=bind)
logger.info("Starting server with IP {} on port {} bind to {}".format(
hostname, port, bind))
return HTTPServer(host=bind, port=port)
def test_get_configuration_success():
from python_moonutilities import configuration
assert configuration.get_configuration("components/port_start")["components/port_start"] == comp_util.CONF["components"]["port_start"]
super(PDPDriver, self).__init__(driver_name, engine_name)
def update_pdp(self, pdp_id, value):
raise NotImplementedError() # pragma: no cover
def delete_pdp(self, pdp_id):
raise NotImplementedError() # pragma: no cover
def add_pdp(self, pdp_id=None, value=None):
raise NotImplementedError() # pragma: no cover
def get_pdp(self, pdp_id=None):
raise NotImplementedError() # pragma: no cover
class KeystoneDriver(Driver):
def __init__(self, driver_name, engine_name):
super(KeystoneDriver, self).__init__(driver_name, engine_name)
conf = configuration.get_configuration("database")['database']
KeystoneManager = keystone.KeystoneManager(
KeystoneDriver(conf['driver'], conf['url']))
ModelManager = model.ModelManager(ModelDriver(conf['driver'], conf['url']))
PolicyManager = policy.PolicyManager(PolicyDriver(conf['driver'], conf['url']))
PDPManager = pdp.PDPManager(PDPDriver(conf['driver'], conf['url']))
def __init__(self, **kwargs):
conf = configuration.get_configuration("components/orchestrator")
self.orchestrator_hostname = conf["components/orchestrator"].get(
"hostname", "orchestrator")
self.orchestrator_port = conf["components/orchestrator"].get(
"port", 80)
# Copyright 2015 Open Platform for NFV Project, Inc. and its contributors
# This software is distributed under the terms and conditions of the 'Apache-2.0'
# license which can be found in the file 'LICENSE' in this package distribution
# or at 'http://www.apache.org/licenses/LICENSE-2.0'.
import os
import requests
import time
from functools import wraps
from flask import request
from oslo_log import log as logging
from python_moonutilities import exceptions, configuration
logger = logging.getLogger(__name__)
KEYSTONE_CONFIG = configuration.get_configuration(
"openstack/keystone")["openstack/keystone"]
TOKENS = {}
def check_token(token, url=None):
_verify = False
if KEYSTONE_CONFIG['certificate']:
_verify = KEYSTONE_CONFIG['certificate']
try:
os.environ.pop("http_proxy")
os.environ.pop("https_proxy")
except KeyError:
pass
if not url:
url = KEYSTONE_CONFIG['url']
headers = {
def create_pipeline(self, keystone_project_id,
pdp_id, policy_ids, manager_data=None,
active_context=None,
slave_name=None):
""" Create security functions
:param keystone_project_id: the Keystone project id
:param pdp_id: the PDP ID mapped to this pipeline
:param policy_ids: the policy IDs mapped to this pipeline
:param manager_data: data needed to create pods
:param active_context: if present, add the security function in this
context
:param slave_name: if present, add the security function in
this context name
if active_context_name and active_context are not present, add the
security function in all context (ie, in all slaves)
:return: None
"""
if not manager_data:
manager_data = dict()
for key, value in self.get_pods().items():
for _pod in value:
if _pod.get('keystone_project_id') == keystone_project_id:
logger.warning("A pod for this Keystone project {} "
"already exists.".format(keystone_project_id))
return
plugins = configuration.get_plugins()
conf = configuration.get_configuration("components/pipeline")
# i_hostname = conf["components/pipeline"].get("interface").get("hostname", "interface")
i_port = conf["components/pipeline"].get("interface").get("port", 80)
i_container = conf["components/pipeline"].get("interface").get(
"container",
"wukongsun/moon_interface:v4.3")
data = [
{
"name": "pipeline-" + get_random_name(),
"container": i_container,
"port": i_port,
'pdp_id': pdp_id,
'genre': "interface",
'keystone_project_id': keystone_project_id,
"namespace": "moon"
},
]
logger.debug("data={}".format(data))
# When policies and models are empty, is it right that it returns 200 ?
# Should it return no found policies or models ?
policies = manager_data.get('policies')
if not policies:
logger.info("No policy data from Manager, trying to get them")
policies = requests.get("http://{}:{}/policies".format(
self.manager_hostname, self.manager_port)).json().get(
"policies", dict())
logger.debug("policies={}".format(policies))
models = manager_data.get('models')
if not models:
logger.info("No models data from Manager, trying to get them")
models = requests.get("http://{}:{}/models".format(
self.manager_hostname, self.manager_port)).json().get(
"models", dict())
logger.debug("models={}".format(models))
if not policy_ids:
raise exceptions.PolicyUnknown
for policy_id in policy_ids:
if policy_id in policies:
genre = policies[policy_id].get("genre", "authz")
if genre in plugins:
for meta_rule in models[policies[policy_id]['model_id']]['meta_rules']:
data.append({
"name": genre + "-" + get_random_name(),
"container": plugins[genre]['container'],
'pdp_id': pdp_id,
"port": plugins[genre].get('port', 8080),
'genre': genre,
'policy_id': policy_id,
'meta_rule_id': meta_rule,
'keystone_project_id': keystone_project_id,
"namespace": "moon"
})
logger.debug("data={}".format(data))
contexts, _active_context = self.get_contexts()
logger.debug("active_context_name={}".format(slave_name))
logger.debug("active_context={}".format(active_context))
if slave_name:
for _context in contexts:
if _context["name"] == slave_name:
active_context = _context
break
if active_context:
active_context = _active_context
_config = config.new_client_from_config(
context=active_context['name'])
logger.debug("_config={}".format(_config))
api_client = client.CoreV1Api(_config)
ext_client = client.ExtensionsV1beta1Api(_config)
self.load_deployment_and_service(data, api_client, ext_client, expose=False)
return
logger.debug("contexts={}".format(contexts))
for _ctx in contexts:
if slave_name and slave_name != _ctx['name']:
continue
_config = config.new_client_from_config(context=_ctx['name'])
logger.debug("_config={}".format(_config))
api_client = client.CoreV1Api(_config)
ext_client = client.ExtensionsV1beta1Api(_config)
self.load_deployment_and_service(data, api_client, ext_client, expose=False)
# or at 'http://www.apache.org/licenses/LICENSE-2.0'.
import re
import os
import types
import requests
import time
from functools import wraps
from flask import request
import logging
from python_moonutilities import exceptions, configuration
logger = logging.getLogger("moon.utilities." + __name__)
keystone_config = configuration.get_configuration("openstack/keystone")["openstack/keystone"]
TOKENS = {}
__targets = {}
def filter_input(func_or_str):
def __filter(string):
if string and type(string) is str:
return "".join(re.findall("[\w\- +]*", string))
return string
def __filter_dict(arg):
result = dict()
for key in arg.keys():
if key == "email":
def init_engine():
db_conf = configuration.get_configuration("database")["database"]
return create_engine(db_conf['url'])
