def add_resource(username,
id,
node_type,
cardinality,
description,
testbed,
file=None):
"""
Creates a new ResourceMetadata object and stores it in the database.
If the file parameter is not None, it is expected to be of type FileUpload from the bottle module or str.
In the former case the file is stored where the available NSDs reside (/etc/softfire/experiment-nsd-csar).
In the latter case the file parameter should contain the file name so it can be added to the ResourceMetadata.
:param id:
:param node_type:
:param cardinality:
:param description:
:param testbed:
:param file:
:return: the ResourceMetadata ID
"""
logger.debug('Add new resource')
if username is None or username == '':
raise Exception('Username must not be None or empty.')
resource_metadata = ResourceMetadata()
resource_metadata.user = username
resource_metadata.id = id
resource_metadata.description = description
resource_metadata.cardinality = cardinality
resource_metadata.node_type = node_type
resource_metadata.testbed = testbed
resource_metadata.properties = {}
if file:
if isinstance(file, FileUpload): # method call comes directly from api
logger.debug('File is provided')
nsd_csar_location = get_config(
'system', 'temp-csar-location',
'/etc/softfire/experiment-nsd-csar').rstrip('/')
nsd_csar_location = '{}/{}'.format(nsd_csar_location, username)
if not os.path.exists(nsd_csar_location):
os.makedirs(nsd_csar_location)
logger.debug('Save file as {}/{}'.format(nsd_csar_location,
file.filename))
file.save('{}/{}'.format(nsd_csar_location, file.filename),
overwrite=True)
resource_metadata.properties['nsd_file_name'] = file.filename
elif isinstance(file, str): # only the file name is provided
logger.debug('File name is provided')
resource_metadata.properties['nsd_file_name'] = file
save(resource_metadata, ResourceMetadata)
logger.debug('Saved ResourceMetadata with ID: %s' % id)
return resource_metadata.id
def refresh_resources(username, manager_name=None):
managers = []
if manager_name is None:
for man in find(ManagerEndpoint):
managers.append(man.name)
else:
managers.append(manager_name)
user_info = get_user_info(username)
result = []
if user_info:
for manager in managers:
stub = get_stub_from_manager_name(manager)
request_message = user_info
try:
response = stub.refresh_resources(request_message)
except _Rendezvous:
traceback.print_exc()
logger.error(
"Exception while calling gRPC, maybe %s Manager is down?" %
manager)
raise RpcFailedCall(
"Exception while calling gRPC, maybe %s Manager is down?" %
manager)
if response.result != 0:
logger.error("list resources returned %d: %s" %
(response.result, response.error_message))
raise RpcFailedCall("list resources returned %d: %s" %
(response.result, response.error_message))
result.extend(response.list_resource.resources)
logger.debug("Saving %d resources" % len(result))
for rm in find(entities.ResourceMetadata):
if rm.node_type in REFRESH_RES_NODE_TYPES:
delete(rm)
for rm in result:
resource_metadata = ResourceMetadata()
if rm.resource_id and (not hasattr(rm, 'user') or rm.user == username):
resource_metadata.id = rm.resource_id
resource_metadata.description = rm.description
resource_metadata.cardinality = rm.cardinality
resource_metadata.node_type = rm.node_type
if hasattr(rm, 'user') and rm.user:
resource_metadata.user = rm.user
else:
resource_metadata.user = ''
resource_metadata.testbed = list(TESTBED_MAPPING.keys())[list(
TESTBED_MAPPING.values()).index(rm.testbed)]
save(resource_metadata, ResourceMetadata)
return result
def reserve(self):
used_resources = []
for node in self.topology_template.nodetemplates:
used_resource = _get_used_resource_from_node(node, self.username)
used_resources.append(used_resource)
CalendarManager.check_availability_for_node(used_resource)
try:
CalendarManager.check_overlapping_resources(used_resources)
except ExperimentValidationError as e:
delete(self.experiment)
raise e
# all node have been granted
for us in self.experiment.resources:
us.status = ResourceStatus.RESERVED.value
save(self.experiment)
def register(self, request, context):
logger.info("registering %s" % request.name)
old_managers = find_by_element_value(ManagerEndpoint,
ManagerEndpoint.name,
request.name)
for old_man in old_managers:
delete(old_man)
logger.debug("Removed old manager endpoint: %s:%s" %
(old_man.name, old_man.endpoint))
manager_endpoint = ManagerEndpoint()
manager_endpoint.name = request.name
manager_endpoint.endpoint = request.endpoint
save(manager_endpoint, ManagerEndpoint)
list_resources()
response_message = messages_pb2.ResponseMessage()
response_message.result = 0
return response_message
def _save_or_create_experimenter(user_info, role="experimenter"):
old_experimenter = find_by_element_value(Experimenter,
Experimenter.username,
user_info.name)
if not old_experimenter:
experimenter = Experimenter()
else:
experimenter = old_experimenter[0]
experimenter.username = user_info.name
experimenter.password = user_info.password
experimenter.role = role
experimenter.testbed_tenants = {}
experimenter.ob_project_id = user_info.ob_project_id
for k, v in user_info.testbed_tenants.items():
experimenter.testbed_tenants[k] = v
save(experimenter)
return experimenter
def check_endpoint():
manager_unavailable_times = get_config('system', 'manager-unavailable',
'5')
while not stop.wait(int(get_config('system', 'manager-check-delay',
'20'))):
for endpoint in find(ManagerEndpoint):
try:
get_stub_from_manager_endpoint(endpoint).heartbeat(Empty())
except:
logger.error("Exception calling heartbeat for manager %s" %
endpoint.name)
if logger.isEnabledFor(logging.DEBUG):
traceback.print_exc()
if endpoint.unavailability >= int(manager_unavailable_times):
logger.error("Manager %s on endpoint %s is not running" %
(endpoint.name, endpoint.endpoint))
MessagingAgent.unregister_endpoint(endpoint.name)
else:
endpoint.unavailability += 1
save(endpoint)
def update_experiment(username, manager_name, resources):
experiments = find_by_element_value(entities.Experiment,
entities.Experiment.username, username)
try:
for new_res in resources:
new_res_dict = json.loads(new_res.content)
for experiment in experiments:
for ur in experiment.resources:
if manager_name == 'nfv-manager' and ur.node_type == 'NfvResource':
try:
val_dict = json.loads(ur.value)
except Exception as e:
logger.warning(
"Failed parsing of experiment: name: %s, username: %s"
% (experiment.name, experiment.username))
raise e
if ur.node_type in get_mapping_managers().get(manager_name) \
and val_dict.get('id') == new_res_dict.get('id'):
ur.value = json.dumps(new_res_dict)
elif manager_name == 'security-manager' and ur.node_type == 'SecurityResource':
try:
val_dict = json.loads(ur.value)
except Exception as e:
logger.warning(
"Failed parsing of experiment: name: %s, username: %s"
% (experiment.name, experiment.username))
raise e
if val_dict.get('random_id') == new_res_dict.get(
'random_id'):
ur.value = json.dumps(new_res_dict)
else:
if ur.node_type in get_mapping_managers().get(
manager_name):
ur.value = json.dumps(new_res_dict)
save(experiment)
except:
logger.warning(
"error while updating resource of manager %s. Error is: %s " %
(manager_name, traceback._cause_message))
def _provide_all_resources_for_manager(experiment_to_deploy,
manager_name,
user_info,
value_to_pass=None):
if value_to_pass is None:
value_to_pass = {}
stub = get_stub_from_manager_name(manager_name)
ret = []
for ur_to_deploy in experiment_to_deploy.resources:
node_types = get_mapping_managers().get(manager_name)
if ur_to_deploy.node_type in node_types:
if value_to_pass:
val = json.loads(ur_to_deploy.value)
val.get('properties').update(value_to_pass)
yaml_dump = json.dumps(val)
ur_to_deploy.value = yaml_dump
response = stub.execute(
messages_pb2.RequestMessage(
method=messages_pb2.PROVIDE_RESOURCES,
payload=ur_to_deploy.value,
user_info=user_info))
for ur in experiment_to_deploy.resources:
if ur.id == ur_to_deploy.id:
if response.result == messages_pb2.ERROR:
logger.error("provide resources returned %d: %s" %
(response.result, response.error_message))
ur.status = ResourceStatus.ERROR.value
ur.value = json.dumps(response.error_message)
else:
# TODO fix this in the api not here
for res in response.provide_resource.resources:
logger.debug("Received: %s" % str(res.content))
_res_dict = json.loads(res.content)
ur.value = json.dumps(_res_dict)
ret.append(_res_dict)
ur.status = ResourceStatus.DEPLOYED.value
save(ur)
return ret
def list_resources(username='', manager_name=None):
managers = []
if manager_name is None:
for man in find(ManagerEndpoint):
managers.append(man.name)
else:
managers.append(manager_name)
result = []
# List resources can be done in parallel
max_workers = len(managers)
tpe = ThreadPoolExecutor(max_workers=max_workers)
threads = []
for manager in managers:
threads.append(tpe.submit(_execute_rpc_list_res, manager))
for t in threads:
result.extend(t.result())
logger.debug("Saving %d resources" % len(result))
for rm in find(entities.ResourceMetadata):
for rm_to_del in result:
if rm.id == rm_to_del.resource_id:
delete(rm)
for rm in result:
if username == '' or rm.user == username:
resource_metadata = ResourceMetadata()
if not username is None:
resource_metadata.user = username
resource_metadata.id = rm.resource_id
resource_metadata.description = rm.description
resource_metadata.cardinality = rm.cardinality
resource_metadata.node_type = rm.node_type
resource_metadata.testbed = _get_testbed_string(rm.testbed)
save(resource_metadata, ResourceMetadata)
return result
def __init__(self, file_path, username):
self.username = username
self.file_path = file_path
zf = zipfile.ZipFile(self.file_path)
for info in zf.infolist():
filename = info.filename
logger.debug("Filename: %s" % filename)
if filename.endswith(".yaml") and filename.startswith(
"Definitions/"):
logger.debug("parsing %s..." % filename)
try:
tpl = ToscaTemplate(
yaml_dict_tpl=yaml.load(zf.read(filename)))
except Exception as e:
if hasattr(e, 'message'):
raise ExperimentValidationError(e.message)
elif isinstance(e.args, list):
raise ExperimentValidationError(e.args[0])
elif isinstance(e.args, str):
raise ExperimentValidationError(e.args)
else:
raise ExperimentValidationError(
"no message available", e)
for node in tpl.nodetemplates:
logger.debug(
"Found node: %s of type %s with properties: %s" %
(node.name, node.type,
list(node.get_properties().keys())))
self.topology_template = tpl
if filename == 'TOSCA-Metadata/Metadata.yaml':
metadata = yaml.load(zf.read(filename))
self.name = metadata.get('name')
if find(entities.Experiment,
'{}_{}'.format(self.username, self.name)) is not None:
raise ExperimentValidationError(
'There is already an experiment with this name, please choose a different one.'
)
# 12/12/12 10:55
self.start_date = self.get_start_date(metadata)
# 12/12/12 11:55
self.end_date = self.get_end_date(metadata)
today = datetime.today().date()
logger.info("End date: date: %s" % self.end_date.date())
if self.end_date.date() < today:
logger.error(
'For the experiment {} the end date({}) is in the past.'
.format(self.name, self.end_date.date()))
raise ExperimentValidationError(
'For the experiment {} the end date({}) is in the past.'
.format(self.name, self.end_date.date()))
self.duration = self.end_date - self.start_date
logger.debug("Experiment duration %s" % self.duration)
if filename.startswith("Files/") and filename.endswith('.csar'):
experiment_nsd_csar_location = get_config(
'system', 'temp-csar-location',
'/etc/softfire/experiment-nsd-csar')
experiment_nsd_csar_location = '{}/{}'.format(
experiment_nsd_csar_location.rstrip('/'), username)
if not os.path.exists(experiment_nsd_csar_location):
os.makedirs(experiment_nsd_csar_location)
data = zf.read(filename)
nsd_file_location = "%s/%s" % (experiment_nsd_csar_location,
filename.split('/')[-1])
with open(nsd_file_location, 'wb+') as f:
f.write(data)
temp_ids = []
for node in self.topology_template.nodetemplates:
if node.type == 'NfvResource':
file_name = node.get_properties().get('file_name')
if file_name:
file_name = file_name.value
if file_name and file_name.startswith(
"Files/") and file_name.endswith(".csar"):
real_file_name = file_name[6:]
tmp_file_location = '{}/{}/{}'.format(
get_config(
'system', 'temp-csar-location',
'/etc/softfire/experiment-nsd-csar').rstrip('/'),
username, real_file_name)
# get the description
try:
zf = zipfile.ZipFile(tmp_file_location)
except FileNotFoundError:
logger.error(
"Please check that the file_name property is correctly set to the file name passed."
)
raise ExperimentValidationError(
"Please check that the file_name property is correctly set to the file name passed."
)
try:
filenames = [
zipinfo.filename for zipinfo in zf.filelist
]
if 'TOSCA-Metadata/Metadata.yaml' in filenames:
logger.debug(
"Found 'TOSCA-Metadata/Metadata.yaml'")
yaml_file = zf.read('TOSCA-Metadata/Metadata.yaml')
elif 'tosca-metadata/Metadata.yaml' in filenames:
# just for legacy support
# the correct file name following the specification is 'TOSCA-Metadata'
# to support experiments that were created before this fix we still check for 'tosca-metadata' though
logger.warning(
"'TOSCA-Metadata/Metadata.yaml' not found. Will try 'tosca-metadata/Metadata.yaml' to support older experiments."
)
yaml_file = zf.read('tosca-metadata/Metadata.yaml')
else:
raise ExperimentValidationError(
'The TOSCA-Metadata/Metadata.yaml file is missing in {}.'
.format(file_name))
except KeyError as e:
traceback.print_exc()
if hasattr(e, 'message'):
raise ExperimentValidationError(e.message)
else:
raise ExperimentValidationError(e.args)
if yaml_file:
yaml_content = yaml.load(yaml_file)
description = yaml_content.get('description')
else:
description = 'No description available'
testbeds = node.get_properties().get('testbeds').value
temp_ids.append(
add_resource(
username,
node.get_properties().get('resource_id').value,
"NfvResource", -1, description,
list(testbeds.keys())[0], real_file_name))
try:
self._validate()
except Exception as e:
for id in temp_ids:
delete(find(ResourceMetadata, _id=id))
raise ExperimentValidationError(e.args)
exp = entities.Experiment()
exp.id = '{}_{}'.format(self.username, self.name)
exp.username = self.username
exp.name = self.name
exp.start_date = self.start_date
exp.end_date = self.end_date
exp.resources = []
for node in self.topology_template.nodetemplates:
exp.resources.append(self._get_used_resource_by_node(node))
element_value = find_by_element_value(entities.Experiment,
entities.Experiment.username,
self.username)
max_no_experiment = get_config("System", "max-number-experiments", 3)
if len(element_value) >= max_no_experiment:
raise ExperimentValidationError(
"You cannot have more than %s experiments at the same time!" %
max_no_experiment)
logger.info("Saving experiment %s" % exp.name)
save(exp)
self.experiment = exp
for res in exp.resources:
_start_termination_thread_for_res(res)