def init_sys():
if get_config('system', 'banner-file', '') != '':
__print_banner(get_config('system', 'banner-file', ''))
# check if cork users and roles exist and create them if not
usernames_cork = [u[0] for u in Api.aaa.list_users()]
roles_cork = [r[0] for r in Api.aaa.list_roles()]
if 'portal' not in roles_cork:
__initialize_cork_role('portal', 100)
if 'admin' not in roles_cork:
__initialize_cork_role('admin', 101)
if 'experimenter' not in roles_cork:
__initialize_cork_role('experimenter', 60)
if 'admin' not in usernames_cork:
__initialize_cork_user(
'admin', 'admin', get_config('system', 'admin-password',
'softfire'))
if 'portal' not in usernames_cork:
__initialize_cork_user(
'portal', 'portal',
get_config('system', 'portal-password', 'softfire'))
t = threading.Thread(target=check_endpoint)
t.start()
return t
def get_openvpn_config(self):
openvpn_options = {
'openvpn_server':
get_config('openvpn', 'openvpn_server',
'softfire-vpn.av.tu-berlin.de'),
'openvpn_port':
int(get_config('openvpn', 'openvpn_port', "443")),
'protocol':
get_config('openvpn', 'openvpn_protocol', 'tcp'),
'certificate':
self.certificate.decode("utf-8"),
'key':
self.private_key.decode("utf-8")
}
return template(self.openvpn_config_tpl, openvpn_options)
def receive_forever():
server = grpc.server(
futures.ThreadPoolExecutor(
max_workers=int(get_config('system', 'server_threads', 5))))
messages_pb2_grpc.add_RegistrationServiceServicer_to_server(
RegistrationAgent(), server)
binding = '[::]:%s' % get_config('messaging', 'bind_port', 50051)
logger.info("Binding rpc registration server to: %s" % binding)
server.add_insecure_port(binding)
server.start()
try:
while True:
time.sleep(_ONE_DAY_IN_SECONDS)
except KeyboardInterrupt:
logger.debug("Stopping server")
server.stop(True)
def start_app():
"""
Start the ExperimentManager as application
"""
global executor, threads
threads.append(executor.submit(api.start_listening))
config_t = configuration.init_sys()
cancel = False
while True:
for t in threads:
try:
if not cancel:
t.result(timeout=3)
else:
if t.running():
t.cancel()
except TimeoutError:
continue
except KeyboardInterrupt:
logger.info("received ctrl-c, shutting down...")
cancel = True
if get_config('database', 'drop_on_exit',
False).lower() == 'true':
drop_tables()
configuration.stop.set()
config_t.join()
def setup_app() -> (SessionMiddleware, int, bool):
bottle.debug(True)
p = get_config(section='api', key='port', default=5080)
bottle.install(error_translation)
session_opts = {
'session.cookie_expires': True,
'session.encrypt_key': get_config('api', 'encrypt_key', 'softfire'),
'session.httponly': True,
'session.timeout': 3600 * 24, # 1 day
'session.type': 'cookie',
'session.validate_key': True,
}
a = SessionMiddleware(bottle.app(), session_opts)
qb = get_config('api', 'quiet', 'true').lower() == 'true'
logger.debug("Bootlepy quiet mode: %s" % qb)
return a, p, qb
def __initialize_cork_role(role_name, role_level):
cork = Cork(directory=get_config("api", "cork-files-path",
"/etc/softfire/users"),
initialize=False)
cork._store.roles[role_name] = role_level
cork._store.save_roles()
logger.debug('Created cork role: {} with level {}.'.format(
role_name, role_level))
def server_static(filename):
""" route to the css and static files"""
if ".." in filename:
return HTTPError(status=403)
return bottle.static_file(
filename,
root='%s/static' %
get_config('api', 'view-path', '/etc/softfire/views'))
def log_certificate_create(username, days=DEFAULT_CERT_VALIDITY):
cert_file = get_config('system', 'certificate-log-file',
'/etc/softfire/certificate-create.log')
try:
with open(cert_file, 'w+') as f:
f.write("common_name: %s, days: %s\n" % (username, days))
except:
logger.error("Can't write on cert create file, only logging")
logger.info("common_name: %s, days: %s\n" % (username, days))
def __init__(self):
self.key_length = 2048
self.digest = 'sha1'
with open(
get_config('system', 'cert-ca-file',
'/etc/softfire/softfire-ca.p12'), 'rb') as buf:
buffer = buf.read()
load_pkcs_ = crypto.load_pkcs12(buffer=buffer,
passphrase=get_config(
'system', 'cert-passphrase',
'softfire').encode())
self.ca_cert = load_pkcs_.get_certificate()
self.ca_key = load_pkcs_.get_privatekey()
self.certificate = None
self.private_key = None
with open(
get_config('system', 'openvpn-template-location',
'/etc/softfire/template_openvpn.tpl'), 'r') as f:
self.openvpn_config_tpl = f.read()
def add_authorized_experimenter(username):
if not os.path.exists(CONFIGURATION_FOLDER):
os.makedirs(CONFIGURATION_FOLDER)
authorized_experimenter_file = get_config(
'api', 'authorized-experimenters',
'/etc/softfire/authorized-experimenters.json')
with open(authorized_experimenter_file, 'w') as f:
authorized_exp = json.loads(f.read().encode("utf-8"))
authorized_exp[username] = True
f.write(json.dumps(authorized_exp))
def add_resource(username,
id,
node_type,
cardinality,
description,
testbed,
file=None):
"""
Creates a new ResourceMetadata object and stores it in the database.
If the file parameter is not None, it is expected to be of type FileUpload from the bottle module or str.
In the former case the file is stored where the available NSDs reside (/etc/softfire/experiment-nsd-csar).
In the latter case the file parameter should contain the file name so it can be added to the ResourceMetadata.
:param id:
:param node_type:
:param cardinality:
:param description:
:param testbed:
:param file:
:return: the ResourceMetadata ID
"""
logger.debug('Add new resource')
if username is None or username == '':
raise Exception('Username must not be None or empty.')
resource_metadata = ResourceMetadata()
resource_metadata.user = username
resource_metadata.id = id
resource_metadata.description = description
resource_metadata.cardinality = cardinality
resource_metadata.node_type = node_type
resource_metadata.testbed = testbed
resource_metadata.properties = {}
if file:
if isinstance(file, FileUpload): # method call comes directly from api
logger.debug('File is provided')
nsd_csar_location = get_config(
'system', 'temp-csar-location',
'/etc/softfire/experiment-nsd-csar').rstrip('/')
nsd_csar_location = '{}/{}'.format(nsd_csar_location, username)
if not os.path.exists(nsd_csar_location):
os.makedirs(nsd_csar_location)
logger.debug('Save file as {}/{}'.format(nsd_csar_location,
file.filename))
file.save('{}/{}'.format(nsd_csar_location, file.filename),
overwrite=True)
resource_metadata.properties['nsd_file_name'] = file.filename
elif isinstance(file, str): # only the file name is provided
logger.debug('File name is provided')
resource_metadata.properties['nsd_file_name'] = file
save(resource_metadata, ResourceMetadata)
logger.debug('Saved ResourceMetadata with ID: %s' % id)
return resource_metadata.id
def check_endpoint():
manager_unavailable_times = get_config('system', 'manager-unavailable',
'5')
while not stop.wait(int(get_config('system', 'manager-check-delay',
'20'))):
for endpoint in find(ManagerEndpoint):
try:
get_stub_from_manager_endpoint(endpoint).heartbeat(Empty())
except:
logger.error("Exception calling heartbeat for manager %s" %
endpoint.name)
if logger.isEnabledFor(logging.DEBUG):
traceback.print_exc()
if endpoint.unavailability >= int(manager_unavailable_times):
logger.error("Manager %s on endpoint %s is not running" %
(endpoint.name, endpoint.endpoint))
MessagingAgent.unregister_endpoint(endpoint.name)
else:
endpoint.unavailability += 1
save(endpoint)
def check_if_authorized(username):
authorized_experimenter_file = get_config(
'api', 'authorized-experimenters',
'/etc/softfire/authorized-experimenters.json')
if os.path.exists(authorized_experimenter_file) and os.path.isfile(
authorized_experimenter_file):
with open(authorized_experimenter_file, "r") as f:
authorized_exp = json.loads(f.read().encode("utf-8"))
return authorized_exp.get(username) and bool(
authorized_exp[username])
else:
return False
def __initialize_cork_user(username, role, password):
cork = Cork(directory=get_config("api", "cork-files-path",
"/etc/softfire/users"),
initialize=False)
user_cork = {
'role': role,
'hash': cork._hash(username, password),
'email_addr': username + '@localhost.local',
'desc': username + ' test user',
'creation_date': '{:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now())
}
cork._store.users[username] = user_cork
cork._store.save_users()
logger.debug('Created cork user {} with role {}.'.format(username, role))
def _create_user_cork():
logger.debug("Remote addr: %s" % request.remote_addr)
if "localhost" in request.remote_addr or "127.0.0.1" in request.remote_addr:
try:
aaa.login(username='******',
password=get_config('system', 'admin-password',
'softfire'))
aaa.create_user(request.json.get('username'),
request.json.get('role'),
request.json.get('password'))
except Exception as e:
return HTTPResponse(status=400, body=str(e))
return HTTPResponse(status=200)
else:
return HTTPResponse(status=404)
import requests
from beaker.middleware import SessionMiddleware
from bottle import request, post, get, HTTPError, HTTPResponse, hook
from cork import Cork
import eu.softfire.tub.exceptions.exceptions as exceptions
from eu.softfire.tub.core import CoreManagers
from eu.softfire.tub.core.CoreManagers import get_resources_dict, Experiment, \
get_experiment_dict, add_resource, get_other_resources
from eu.softfire.tub.core.calendar import CalendarManager
from eu.softfire.tub.core.certificate import CertificateGenerator, log_certificate_create
from eu.softfire.tub.utils.static_config import CONFIGURATION_FOLDER
from eu.softfire.tub.utils.utils import get_config, get_logger
logger = get_logger('eu.softfire.tub.api')
bottle.TEMPLATE_PATH = [get_config('api', 'view-path', '/etc/softfire/views')]
aaa = Cork(get_config("api", "cork-files-path", "/etc/softfire/users"))
authorize = aaa.make_auth_decorator(fail_redirect="/login")
create_user_thread = None
create_user_thread_pool = Pool(20)
@hook('after_request')
def maintenance():
try:
if request.environ.get(
'bottle.raw_path'
) == '/login' or aaa.current_user.role == 'admin':
return
except:
return
def provide_resources(username, experiment_id):
experiments_to_deploy = find_by_element_value(entities.Experiment,
entities.Experiment.id,
experiment_id)
if len(experiments_to_deploy) == 0:
logger.error("No experiment to be deployed....")
raise ExperimentNotFound("No experiment to be deployed....")
experiment_to_deploy = experiments_to_deploy[0]
res_in_error_or_deployed = 0
for ur in experiment_to_deploy.resources:
if ur.status in [
entities.ResourceStatus.ERROR.value,
entities.ResourceStatus.DEPLOYED.value
]:
res_in_error_or_deployed += 1
if res_in_error_or_deployed > 0:
logger.error("Deploying a resource in error or deployed state....")
raise ExperimentValidationError(
"You cannot deploy a resource in error or deployed state. Please delete it first"
)
for ur in experiment_to_deploy.resources:
today = datetime.today().date()
start_date = ur.start_date
end_date = ur.end_date
# to make sure that we don't run into any problems due to the wrong type, even though this should not happen
if isinstance(start_date, datetime):
start_date = start_date.date()
if isinstance(end_date, datetime):
end_date = end_date.date()
if start_date > today:
logger.error(
'The resource {} is reserved for a time period which begins in the future. You cannot use it yet.'
.format(ur.name))
raise ExperimentValidationError(
'The resource {} is reserved for a time period which begins in the future. You cannot use it yet.'
.format(ur.name))
if end_date < today:
logger.error(
'For the resource {} the reserved time period ended already.'.
format(ur.name))
raise ExperimentValidationError(
'For the resource {} the reserved time period ended already.'.
format(ur.name))
user_info = get_user_info(username)
if hasattr(user_info, 'name'):
un = user_info.name
else:
un = user_info.username
logger.debug("Received deploy resources from user %s" % un)
logger.debug("Received deploy resources %s" % experiment_to_deploy.name)
involved_managers = [
man_name for ur in experiment_to_deploy.resources
for man_name, node_types in get_mapping_managers().items()
if ur.node_type in node_types
]
manager_ordered = get_config('system', 'deployment-order', '').split(';')
remaining_managers = set(list(
get_mapping_managers().keys())) - set(manager_ordered)
value_to_pass = {}
for manager_name in manager_ordered:
if manager_name in involved_managers:
result = _provide_all_resources_for_manager(
experiment_to_deploy, manager_name, user_info, value_to_pass)
value_to_pass = {}
for ktp in keys_to_pass:
for v in result:
if ktp in v.keys():
value_to_pass[ktp] = v.get(ktp)
for manager_name in remaining_managers:
if manager_name in involved_managers:
_provide_all_resources_for_manager(experiment_to_deploy,
manager_name, user_info)
def __init__(self, file_path, username):
self.username = username
self.file_path = file_path
zf = zipfile.ZipFile(self.file_path)
for info in zf.infolist():
filename = info.filename
logger.debug("Filename: %s" % filename)
if filename.endswith(".yaml") and filename.startswith(
"Definitions/"):
logger.debug("parsing %s..." % filename)
try:
tpl = ToscaTemplate(
yaml_dict_tpl=yaml.load(zf.read(filename)))
except Exception as e:
if hasattr(e, 'message'):
raise ExperimentValidationError(e.message)
elif isinstance(e.args, list):
raise ExperimentValidationError(e.args[0])
elif isinstance(e.args, str):
raise ExperimentValidationError(e.args)
else:
raise ExperimentValidationError(
"no message available", e)
for node in tpl.nodetemplates:
logger.debug(
"Found node: %s of type %s with properties: %s" %
(node.name, node.type,
list(node.get_properties().keys())))
self.topology_template = tpl
if filename == 'TOSCA-Metadata/Metadata.yaml':
metadata = yaml.load(zf.read(filename))
self.name = metadata.get('name')
if find(entities.Experiment,
'{}_{}'.format(self.username, self.name)) is not None:
raise ExperimentValidationError(
'There is already an experiment with this name, please choose a different one.'
)
# 12/12/12 10:55
self.start_date = self.get_start_date(metadata)
# 12/12/12 11:55
self.end_date = self.get_end_date(metadata)
today = datetime.today().date()
logger.info("End date: date: %s" % self.end_date.date())
if self.end_date.date() < today:
logger.error(
'For the experiment {} the end date({}) is in the past.'
.format(self.name, self.end_date.date()))
raise ExperimentValidationError(
'For the experiment {} the end date({}) is in the past.'
.format(self.name, self.end_date.date()))
self.duration = self.end_date - self.start_date
logger.debug("Experiment duration %s" % self.duration)
if filename.startswith("Files/") and filename.endswith('.csar'):
experiment_nsd_csar_location = get_config(
'system', 'temp-csar-location',
'/etc/softfire/experiment-nsd-csar')
experiment_nsd_csar_location = '{}/{}'.format(
experiment_nsd_csar_location.rstrip('/'), username)
if not os.path.exists(experiment_nsd_csar_location):
os.makedirs(experiment_nsd_csar_location)
data = zf.read(filename)
nsd_file_location = "%s/%s" % (experiment_nsd_csar_location,
filename.split('/')[-1])
with open(nsd_file_location, 'wb+') as f:
f.write(data)
temp_ids = []
for node in self.topology_template.nodetemplates:
if node.type == 'NfvResource':
file_name = node.get_properties().get('file_name')
if file_name:
file_name = file_name.value
if file_name and file_name.startswith(
"Files/") and file_name.endswith(".csar"):
real_file_name = file_name[6:]
tmp_file_location = '{}/{}/{}'.format(
get_config(
'system', 'temp-csar-location',
'/etc/softfire/experiment-nsd-csar').rstrip('/'),
username, real_file_name)
# get the description
try:
zf = zipfile.ZipFile(tmp_file_location)
except FileNotFoundError:
logger.error(
"Please check that the file_name property is correctly set to the file name passed."
)
raise ExperimentValidationError(
"Please check that the file_name property is correctly set to the file name passed."
)
try:
filenames = [
zipinfo.filename for zipinfo in zf.filelist
]
if 'TOSCA-Metadata/Metadata.yaml' in filenames:
logger.debug(
"Found 'TOSCA-Metadata/Metadata.yaml'")
yaml_file = zf.read('TOSCA-Metadata/Metadata.yaml')
elif 'tosca-metadata/Metadata.yaml' in filenames:
# just for legacy support
# the correct file name following the specification is 'TOSCA-Metadata'
# to support experiments that were created before this fix we still check for 'tosca-metadata' though
logger.warning(
"'TOSCA-Metadata/Metadata.yaml' not found. Will try 'tosca-metadata/Metadata.yaml' to support older experiments."
)
yaml_file = zf.read('tosca-metadata/Metadata.yaml')
else:
raise ExperimentValidationError(
'The TOSCA-Metadata/Metadata.yaml file is missing in {}.'
.format(file_name))
except KeyError as e:
traceback.print_exc()
if hasattr(e, 'message'):
raise ExperimentValidationError(e.message)
else:
raise ExperimentValidationError(e.args)
if yaml_file:
yaml_content = yaml.load(yaml_file)
description = yaml_content.get('description')
else:
description = 'No description available'
testbeds = node.get_properties().get('testbeds').value
temp_ids.append(
add_resource(
username,
node.get_properties().get('resource_id').value,
"NfvResource", -1, description,
list(testbeds.keys())[0], real_file_name))
try:
self._validate()
except Exception as e:
for id in temp_ids:
delete(find(ResourceMetadata, _id=id))
raise ExperimentValidationError(e.args)
exp = entities.Experiment()
exp.id = '{}_{}'.format(self.username, self.name)
exp.username = self.username
exp.name = self.name
exp.start_date = self.start_date
exp.end_date = self.end_date
exp.resources = []
for node in self.topology_template.nodetemplates:
exp.resources.append(self._get_used_resource_by_node(node))
element_value = find_by_element_value(entities.Experiment,
entities.Experiment.username,
self.username)
max_no_experiment = get_config("System", "max-number-experiments", 3)
if len(element_value) >= max_no_experiment:
raise ExperimentValidationError(
"You cannot have more than %s experiments at the same time!" %
max_no_experiment)
logger.info("Saving experiment %s" % exp.name)
save(exp)
self.experiment = exp
for res in exp.resources:
_start_termination_thread_for_res(res)
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker, scoped_session
from sqlalchemy.orm.exc import NoResultFound
from sqlalchemy.pool import StaticPool
from eu.softfire.tub.entities import entities
from eu.softfire.tub.entities.entities import Base
from eu.softfire.tub.messaging.grpc import messages_pb2
from eu.softfire.tub.utils.utils import get_config, get_logger
logger = get_logger('eu.softfire.tub.repository')
lock = threading.RLock()
db_url = get_config('database', 'url', "sqlite:////tmp/experiment-manager.db")
is_sqlite = db_url.startswith("sqlite:")
if is_sqlite:
engine = create_engine(db_url,
poolclass=StaticPool,
connect_args={'check_same_thread': False})
else:
engine = create_engine(db_url)
debug_echo = (logger.getEffectiveLevel() == logging.DEBUG) and get_config(
'database', 'show_sql', False).lower() == 'true'
engine.echo = debug_echo
Base.metadata.create_all(engine)
if is_sqlite:
session_factory = sessionmaker(bind=engine)
_session = scoped_session(session_factory)
session = _session()