def _validate(self):
if not self.name:
raise ExperimentValidationError("Name must not be empty")
if self.start_date is None or self.end_date is None:
raise ExperimentValidationError(
"Not able to parse start or end date")
if self.duration <= timedelta(0, 1, 0):
raise ExperimentValidationError(
"Duration too short, modify start and end date")
resource_ids = [rm.id for rm in find(ResourceMetadata)]
threads = []
for node in self.topology_template.nodetemplates:
resource_id_ = node.get_properties()["resource_id"].value
if (resource_id_
not in resource_ids) and (node.type != 'NfvResource'):
raise ExperimentValidationError("resource id %s not allowed" %
resource_id_)
resource_metadata = find(ResourceMetadata, resource_id_)
thread = ExceptionHandlerThread(
target=_validate_resource,
args=[node, self.username, resource_metadata])
threads.append(thread)
thread.start()
for t in threads:
t.join()
if t.exception:
raise t.exception
def get_used_resources_by_experimenter(exp_name):
res = []
for ur in find(UsedResource):
experiment = find(Experiment, ur.parent_id)
if not experiment is None and experiment.username == exp_name:
res.append(ur)
return res
def _get_used_resource_from_node(node, username):
for e in find(entities.Experiment):
if e.username == username:
for ur in find(UsedResource):
if ur.parent_id == e.id and ur.name == node.name:
return ur
raise ResourceNotFound('Resource with name %s for user %s was not found' %
(node.name, username))
def delete_user(username):
for experimenter in find(Experimenter):
if experimenter.username == username:
for experiment in find(entities.Experiment):
if experiment.username == username:
try:
release_resources(username, experiment.id)
except ExperimentNotFound as e:
pass
except Exception as e:
logger.error(
'Exception while trying to delete experiment {} of user {}: {}'
.format(experiment.id, username, str(e)))
traceback.print_exc()
logger.warning(
'Removing the resources and experiment {} of user {} from the database so they might actually not be removed!'
.format(experiment.id, username))
try:
for ur in find(UsedResource):
if ur.parent_id == experiment.id:
logger.debug(
'Removing used resource {} ({}) from the database.'
.format(ur.id, ur.name))
try:
delete(ur)
except Exception as e:
logger.error(
'Exception while removing used resource {} ({}) from the database: {}'
.format(ur.id, ur.name, e))
traceback.print_exc()
logger.debug(
'Removing experiment {} from the database.'.
format(experiment.id))
try:
delete(experiment)
except Exception as e:
logger.error(
'Exception while removing experiment {} from the database: {}'
.format(experiment.id, e))
traceback.print_exc()
except Exception as e:
logger.error(
'Exception while removing the resources and experiment {} of user {} from the database.'
.format(experiment.id, username))
user_info = _create_user_info_from_experimenter(experimenter)
for man in MANAGERS_CREATE_USER:
try:
get_stub_from_manager_name(man).delete_user(user_info)
logger.info("Manager {} delete user {} finished".format(
man, experimenter.username))
except ManagerNotFound:
traceback.print_exc()
logger.error(
"Manager {} is not register and needs to delete user {}"
.format(man, experimenter.username))
delete(experimenter)
def refresh_resources(username, manager_name=None):
managers = []
if manager_name is None:
for man in find(ManagerEndpoint):
managers.append(man.name)
else:
managers.append(manager_name)
user_info = get_user_info(username)
result = []
if user_info:
for manager in managers:
stub = get_stub_from_manager_name(manager)
request_message = user_info
try:
response = stub.refresh_resources(request_message)
except _Rendezvous:
traceback.print_exc()
logger.error(
"Exception while calling gRPC, maybe %s Manager is down?" %
manager)
raise RpcFailedCall(
"Exception while calling gRPC, maybe %s Manager is down?" %
manager)
if response.result != 0:
logger.error("list resources returned %d: %s" %
(response.result, response.error_message))
raise RpcFailedCall("list resources returned %d: %s" %
(response.result, response.error_message))
result.extend(response.list_resource.resources)
logger.debug("Saving %d resources" % len(result))
for rm in find(entities.ResourceMetadata):
if rm.node_type in REFRESH_RES_NODE_TYPES:
delete(rm)
for rm in result:
resource_metadata = ResourceMetadata()
if rm.resource_id and (not hasattr(rm, 'user') or rm.user == username):
resource_metadata.id = rm.resource_id
resource_metadata.description = rm.description
resource_metadata.cardinality = rm.cardinality
resource_metadata.node_type = rm.node_type
if hasattr(rm, 'user') and rm.user:
resource_metadata.user = rm.user
else:
resource_metadata.user = ''
resource_metadata.testbed = list(TESTBED_MAPPING.keys())[list(
TESTBED_MAPPING.values()).index(rm.testbed)]
save(resource_metadata, ResourceMetadata)
return result
def _terminate_expired_resource(res: entities.UsedResource):
resource_to_delete = find(entities.UsedResource, _id=res.id)
exp_id = resource_to_delete.parent_id
if resource_to_delete and res == resource_to_delete:
_release_used_resource(res)
delete(resource_to_delete)
else:
logger.debug("The initial experiment was removed")
return
exp = find(entities.Experiment, _id=exp_id)
if not len(exp.resources):
delete(exp)
def get_month(cls):
result = []
for ur in find(UsedResource):
rm = find(ResourceMetadata, _id=ur.resource_id)
if rm and rm.cardinality > 0:
result.append({
"resource_id": ur.resource_id,
"start": ur.start_date,
"end": ur.end_date,
})
return result
def unregister_endpoint(manager_endpoint_name: str) -> bool:
deleted = False
for manager_endpoint in find(ManagerEndpoint):
if manager_endpoint.name == manager_endpoint_name:
for resource_type in get_mapping_managers().get(
manager_endpoint.name):
for rm in [
rm for rm in find(ResourceMetadata)
if rm.node_type.lower() == resource_type.lower()
]:
delete(rm)
delete(manager_endpoint)
deleted = True
return deleted
def check_overlapping_booking(cls, used_resource):
"""
Check calendar availability of this resource
:param used_resource: the used resource to book
:type used_resource: UsedResource
:return: True when availability is granted False otherwise
:rtype: bool
"""
resource_metadata = cls.get_metadata_from_usedresource(used_resource)
# if the resource metadata associated has infinite cardinality return true
if resource_metadata.cardinality <= 0:
return True
max_concurrent = resource_metadata.cardinality
counter = 0
# if not then i need to calculate the number of resource already booked for that period
for ur in find(UsedResource):
if ur.status == ResourceStatus.RESERVED.value:
if ur.start_date <= used_resource.start_date <= ur.end_date:
counter += 1
elif ur.start_date <= used_resource.end_date <= ur.end_date:
counter += 1
if counter < max_concurrent:
return True
return False
def _validate_resource(node, username, request_metadata):
for manager_endpoint in find(ManagerEndpoint):
if node.type in get_mapping_managers().get(manager_endpoint.name):
if request_metadata and request_metadata.properties and request_metadata.properties.get(
'nsd_file_name'):
nsd_file_name = request_metadata.properties.get(
'nsd_file_name')
node.entity_tpl.get(
'properties')['file_name'] = 'Files/%s' % nsd_file_name
request_message = messages_pb2.RequestMessage(
method=messages_pb2.VALIDATE_RESOURCES,
payload=yaml.dump(node.entity_tpl),
user_info=get_user_info(username))
try:
response = get_stub_from_manager_endpoint(
manager_endpoint).execute(request_message)
except Exception as e:
if hasattr(e, 'message'):
raise RpcFailedCall(e.message)
raise RpcFailedCall(e.args)
if response.result == messages_pb2.ERROR:
raise RpcFailedCall(response.error_message)
return
raise ManagerNotFound("manager handling resource %s was not found" %
node.type)
def get_resources_dict(username=None):
res = []
for rm in find(ResourceMetadata):
if rm.node_type != 'NfvImage' and rm.node_type != 'NfvNetwork' and rm.node_type != 'NfvFlavor':
if username:
if username == rm.user:
res.append(_get_resource_dict_from_rm(rm))
else:
if not rm.user:
res.append(_get_resource_dict_from_rm(rm))
return res
def list_resources(username='', manager_name=None):
managers = []
if manager_name is None:
for man in find(ManagerEndpoint):
managers.append(man.name)
else:
managers.append(manager_name)
result = []
# List resources can be done in parallel
max_workers = len(managers)
tpe = ThreadPoolExecutor(max_workers=max_workers)
threads = []
for manager in managers:
threads.append(tpe.submit(_execute_rpc_list_res, manager))
for t in threads:
result.extend(t.result())
logger.debug("Saving %d resources" % len(result))
for rm in find(entities.ResourceMetadata):
for rm_to_del in result:
if rm.id == rm_to_del.resource_id:
delete(rm)
for rm in result:
if username == '' or rm.user == username:
resource_metadata = ResourceMetadata()
if not username is None:
resource_metadata.user = username
resource_metadata.id = rm.resource_id
resource_metadata.description = rm.description
resource_metadata.cardinality = rm.cardinality
resource_metadata.node_type = rm.node_type
resource_metadata.testbed = _get_testbed_string(rm.testbed)
save(resource_metadata, ResourceMetadata)
return result
def check_overlapping_resources(cls, used_resources: list):
for ur_to_check in used_resources:
counter = 1
for other_res in used_resources:
if other_res != ur_to_check and other_res.node_type == ur_to_check.node_type:
if _time_overlap(ur_to_check.start_date,
ur_to_check.end_date,
other_res.start_date, other_res.end_date):
counter += 1
cardinality = find(ResourceMetadata,
_id=ur_to_check.resource_id).cardinality
if 0 < cardinality < counter:
raise ExperimentValidationError(
"Trying to book %d resources of resource_id %s" %
(counter, ur_to_check.resource_id))
def _release_used_resource(res: entities.UsedResource):
exp = find(entities.Experiment, _id=res.parent_id)
user_info = get_user_info(exp.username)
for man, node_types in get_mapping_managers().items():
if res.node_type in node_types:
response = get_stub_from_manager_name(man).execute(
messages_pb2.RequestMessage(
method=messages_pb2.RELEASE_RESOURCES,
payload=res.value,
user_info=user_info))
if response.result != 0:
logger.error("release resources returned %d: %s" %
(response.result, response.error_message))
raise RpcFailedCall("provide resources returned %d: %s" %
(response.result, response.error_message))
def get_other_resources():
images = []
networks = []
flavours = []
for rm in find(ResourceMetadata):
if rm.node_type == 'NfvImage':
tmp = {
'resource_id': rm.id,
'node_type': rm.node_type,
'testbed': rm.testbed,
'description': rm.description
}
if rm.cardinality < 0:
tmp['cardinality'] = 'infinite'
else:
tmp['cardinality'] = rm.cardinality
images.append(tmp)
if rm.node_type == 'NfvFlavor':
tmp = {
'resource_id': rm.id,
'node_type': rm.node_type,
'testbed': rm.testbed,
'description': rm.description
}
if rm.cardinality < 0:
tmp['cardinality'] = 'infinite'
else:
tmp['cardinality'] = rm.cardinality
flavours.append(tmp)
if rm.node_type == 'NfvNetwork':
tmp = {
'resource_id': rm.id,
'node_type': rm.node_type,
'testbed': rm.testbed,
'description': rm.description
}
if rm.cardinality < 0:
tmp['cardinality'] = 'infinite'
else:
tmp['cardinality'] = rm.cardinality
networks.append(tmp)
return images, networks, flavours
def create_user(username, password, role='experimenter'):
for experimenter in find(Experimenter):
if experimenter.username == username:
raise Exception('Username \'{}\' already exists.'.format(username))
user_info = messages_pb2.UserInfo(name=username, password=password)
for man in MANAGERS_CREATE_USER:
try:
user_info = get_stub_from_manager_name(man).create_user(user_info)
logger.info("Manager %s create user finished, new UserInfo: %s" %
(man, user_info))
except ManagerNotFound:
traceback.print_exc()
logger.error(
"one of the manager is not register and need to create user")
logger.info("Created user on the Managers")
experimenter = _save_or_create_experimenter(user_info, role)
logger.info("Stored new experimenter: %s" % experimenter.username)
def check_endpoint():
manager_unavailable_times = get_config('system', 'manager-unavailable',
'5')
while not stop.wait(int(get_config('system', 'manager-check-delay',
'20'))):
for endpoint in find(ManagerEndpoint):
try:
get_stub_from_manager_endpoint(endpoint).heartbeat(Empty())
except:
logger.error("Exception calling heartbeat for manager %s" %
endpoint.name)
if logger.isEnabledFor(logging.DEBUG):
traceback.print_exc()
if endpoint.unavailability >= int(manager_unavailable_times):
logger.error("Manager %s on endpoint %s is not running" %
(endpoint.name, endpoint.endpoint))
MessagingAgent.unregister_endpoint(endpoint.name)
else:
endpoint.unavailability += 1
save(endpoint)
def get_experiment_dict(username):
res = []
exp_names = []
exp_ids = []
for ex in find(entities.Experiment):
if not username or ex.username == username:
exp_ids.append(ex.id)
exp_names.append(ex.name)
for ur in ex.resources:
tmp = {
'resource_id': ur.resource_id,
'used_resource_id': ur.id,
'node_type': ur.node_type,
'status': ResourceStatus.from_int_to_enum(ur.status).name,
'value': ur.value,
'experiment_name': ex.name,
'experiment_id': ex.id,
'username': ex.username
}
res.append(tmp)
return exp_names, exp_ids, res
def get_metadata_from_usedresource(cls, used_resource):
return find(ResourceMetadata, _id=used_resource.resource_id)
def list_managers():
return [man.name for man in find(ManagerEndpoint)]
def list_experimenters():
return [man.username for man in find(Experimenter)]
def get_stub_from_manager_name(manager_name):
for manager_endpoint in find(ManagerEndpoint):
if manager_endpoint.name == manager_name:
logger.debug("Getting stub for manager: %s" % manager_name)
return get_stub_from_manager_endpoint(manager_endpoint)
raise ManagerNotFound("No manager found for name %s" % manager_name)
def __init__(self, file_path, username):
self.username = username
self.file_path = file_path
zf = zipfile.ZipFile(self.file_path)
for info in zf.infolist():
filename = info.filename
logger.debug("Filename: %s" % filename)
if filename.endswith(".yaml") and filename.startswith(
"Definitions/"):
logger.debug("parsing %s..." % filename)
try:
tpl = ToscaTemplate(
yaml_dict_tpl=yaml.load(zf.read(filename)))
except Exception as e:
if hasattr(e, 'message'):
raise ExperimentValidationError(e.message)
elif isinstance(e.args, list):
raise ExperimentValidationError(e.args[0])
elif isinstance(e.args, str):
raise ExperimentValidationError(e.args)
else:
raise ExperimentValidationError(
"no message available", e)
for node in tpl.nodetemplates:
logger.debug(
"Found node: %s of type %s with properties: %s" %
(node.name, node.type,
list(node.get_properties().keys())))
self.topology_template = tpl
if filename == 'TOSCA-Metadata/Metadata.yaml':
metadata = yaml.load(zf.read(filename))
self.name = metadata.get('name')
if find(entities.Experiment,
'{}_{}'.format(self.username, self.name)) is not None:
raise ExperimentValidationError(
'There is already an experiment with this name, please choose a different one.'
)
# 12/12/12 10:55
self.start_date = self.get_start_date(metadata)
# 12/12/12 11:55
self.end_date = self.get_end_date(metadata)
today = datetime.today().date()
logger.info("End date: date: %s" % self.end_date.date())
if self.end_date.date() < today:
logger.error(
'For the experiment {} the end date({}) is in the past.'
.format(self.name, self.end_date.date()))
raise ExperimentValidationError(
'For the experiment {} the end date({}) is in the past.'
.format(self.name, self.end_date.date()))
self.duration = self.end_date - self.start_date
logger.debug("Experiment duration %s" % self.duration)
if filename.startswith("Files/") and filename.endswith('.csar'):
experiment_nsd_csar_location = get_config(
'system', 'temp-csar-location',
'/etc/softfire/experiment-nsd-csar')
experiment_nsd_csar_location = '{}/{}'.format(
experiment_nsd_csar_location.rstrip('/'), username)
if not os.path.exists(experiment_nsd_csar_location):
os.makedirs(experiment_nsd_csar_location)
data = zf.read(filename)
nsd_file_location = "%s/%s" % (experiment_nsd_csar_location,
filename.split('/')[-1])
with open(nsd_file_location, 'wb+') as f:
f.write(data)
temp_ids = []
for node in self.topology_template.nodetemplates:
if node.type == 'NfvResource':
file_name = node.get_properties().get('file_name')
if file_name:
file_name = file_name.value
if file_name and file_name.startswith(
"Files/") and file_name.endswith(".csar"):
real_file_name = file_name[6:]
tmp_file_location = '{}/{}/{}'.format(
get_config(
'system', 'temp-csar-location',
'/etc/softfire/experiment-nsd-csar').rstrip('/'),
username, real_file_name)
# get the description
try:
zf = zipfile.ZipFile(tmp_file_location)
except FileNotFoundError:
logger.error(
"Please check that the file_name property is correctly set to the file name passed."
)
raise ExperimentValidationError(
"Please check that the file_name property is correctly set to the file name passed."
)
try:
filenames = [
zipinfo.filename for zipinfo in zf.filelist
]
if 'TOSCA-Metadata/Metadata.yaml' in filenames:
logger.debug(
"Found 'TOSCA-Metadata/Metadata.yaml'")
yaml_file = zf.read('TOSCA-Metadata/Metadata.yaml')
elif 'tosca-metadata/Metadata.yaml' in filenames:
# just for legacy support
# the correct file name following the specification is 'TOSCA-Metadata'
# to support experiments that were created before this fix we still check for 'tosca-metadata' though
logger.warning(
"'TOSCA-Metadata/Metadata.yaml' not found. Will try 'tosca-metadata/Metadata.yaml' to support older experiments."
)
yaml_file = zf.read('tosca-metadata/Metadata.yaml')
else:
raise ExperimentValidationError(
'The TOSCA-Metadata/Metadata.yaml file is missing in {}.'
.format(file_name))
except KeyError as e:
traceback.print_exc()
if hasattr(e, 'message'):
raise ExperimentValidationError(e.message)
else:
raise ExperimentValidationError(e.args)
if yaml_file:
yaml_content = yaml.load(yaml_file)
description = yaml_content.get('description')
else:
description = 'No description available'
testbeds = node.get_properties().get('testbeds').value
temp_ids.append(
add_resource(
username,
node.get_properties().get('resource_id').value,
"NfvResource", -1, description,
list(testbeds.keys())[0], real_file_name))
try:
self._validate()
except Exception as e:
for id in temp_ids:
delete(find(ResourceMetadata, _id=id))
raise ExperimentValidationError(e.args)
exp = entities.Experiment()
exp.id = '{}_{}'.format(self.username, self.name)
exp.username = self.username
exp.name = self.name
exp.start_date = self.start_date
exp.end_date = self.end_date
exp.resources = []
for node in self.topology_template.nodetemplates:
exp.resources.append(self._get_used_resource_by_node(node))
element_value = find_by_element_value(entities.Experiment,
entities.Experiment.username,
self.username)
max_no_experiment = get_config("System", "max-number-experiments", 3)
if len(element_value) >= max_no_experiment:
raise ExperimentValidationError(
"You cannot have more than %s experiments at the same time!" %
max_no_experiment)
logger.info("Saving experiment %s" % exp.name)
save(exp)
self.experiment = exp
for res in exp.resources:
_start_termination_thread_for_res(res)
