def _validate(self): if not self.name: raise ExperimentValidationError("Name must not be empty") if self.start_date is None or self.end_date is None: raise ExperimentValidationError( "Not able to parse start or end date") if self.duration <= timedelta(0, 1, 0): raise ExperimentValidationError( "Duration too short, modify start and end date") resource_ids = [rm.id for rm in find(ResourceMetadata)] threads = [] for node in self.topology_template.nodetemplates: resource_id_ = node.get_properties()["resource_id"].value if (resource_id_ not in resource_ids) and (node.type != 'NfvResource'): raise ExperimentValidationError("resource id %s not allowed" % resource_id_) resource_metadata = find(ResourceMetadata, resource_id_) thread = ExceptionHandlerThread( target=_validate_resource, args=[node, self.username, resource_metadata]) threads.append(thread) thread.start() for t in threads: t.join() if t.exception: raise t.exception
def get_used_resources_by_experimenter(exp_name): res = [] for ur in find(UsedResource): experiment = find(Experiment, ur.parent_id) if not experiment is None and experiment.username == exp_name: res.append(ur) return res
def _get_used_resource_from_node(node, username): for e in find(entities.Experiment): if e.username == username: for ur in find(UsedResource): if ur.parent_id == e.id and ur.name == node.name: return ur raise ResourceNotFound('Resource with name %s for user %s was not found' % (node.name, username))
def delete_user(username): for experimenter in find(Experimenter): if experimenter.username == username: for experiment in find(entities.Experiment): if experiment.username == username: try: release_resources(username, experiment.id) except ExperimentNotFound as e: pass except Exception as e: logger.error( 'Exception while trying to delete experiment {} of user {}: {}' .format(experiment.id, username, str(e))) traceback.print_exc() logger.warning( 'Removing the resources and experiment {} of user {} from the database so they might actually not be removed!' .format(experiment.id, username)) try: for ur in find(UsedResource): if ur.parent_id == experiment.id: logger.debug( 'Removing used resource {} ({}) from the database.' .format(ur.id, ur.name)) try: delete(ur) except Exception as e: logger.error( 'Exception while removing used resource {} ({}) from the database: {}' .format(ur.id, ur.name, e)) traceback.print_exc() logger.debug( 'Removing experiment {} from the database.'. format(experiment.id)) try: delete(experiment) except Exception as e: logger.error( 'Exception while removing experiment {} from the database: {}' .format(experiment.id, e)) traceback.print_exc() except Exception as e: logger.error( 'Exception while removing the resources and experiment {} of user {} from the database.' .format(experiment.id, username)) user_info = _create_user_info_from_experimenter(experimenter) for man in MANAGERS_CREATE_USER: try: get_stub_from_manager_name(man).delete_user(user_info) logger.info("Manager {} delete user {} finished".format( man, experimenter.username)) except ManagerNotFound: traceback.print_exc() logger.error( "Manager {} is not register and needs to delete user {}" .format(man, experimenter.username)) delete(experimenter)
def refresh_resources(username, manager_name=None): managers = [] if manager_name is None: for man in find(ManagerEndpoint): managers.append(man.name) else: managers.append(manager_name) user_info = get_user_info(username) result = [] if user_info: for manager in managers: stub = get_stub_from_manager_name(manager) request_message = user_info try: response = stub.refresh_resources(request_message) except _Rendezvous: traceback.print_exc() logger.error( "Exception while calling gRPC, maybe %s Manager is down?" % manager) raise RpcFailedCall( "Exception while calling gRPC, maybe %s Manager is down?" % manager) if response.result != 0: logger.error("list resources returned %d: %s" % (response.result, response.error_message)) raise RpcFailedCall("list resources returned %d: %s" % (response.result, response.error_message)) result.extend(response.list_resource.resources) logger.debug("Saving %d resources" % len(result)) for rm in find(entities.ResourceMetadata): if rm.node_type in REFRESH_RES_NODE_TYPES: delete(rm) for rm in result: resource_metadata = ResourceMetadata() if rm.resource_id and (not hasattr(rm, 'user') or rm.user == username): resource_metadata.id = rm.resource_id resource_metadata.description = rm.description resource_metadata.cardinality = rm.cardinality resource_metadata.node_type = rm.node_type if hasattr(rm, 'user') and rm.user: resource_metadata.user = rm.user else: resource_metadata.user = '' resource_metadata.testbed = list(TESTBED_MAPPING.keys())[list( TESTBED_MAPPING.values()).index(rm.testbed)] save(resource_metadata, ResourceMetadata) return result
def _terminate_expired_resource(res: entities.UsedResource): resource_to_delete = find(entities.UsedResource, _id=res.id) exp_id = resource_to_delete.parent_id if resource_to_delete and res == resource_to_delete: _release_used_resource(res) delete(resource_to_delete) else: logger.debug("The initial experiment was removed") return exp = find(entities.Experiment, _id=exp_id) if not len(exp.resources): delete(exp)
def get_month(cls): result = [] for ur in find(UsedResource): rm = find(ResourceMetadata, _id=ur.resource_id) if rm and rm.cardinality > 0: result.append({ "resource_id": ur.resource_id, "start": ur.start_date, "end": ur.end_date, }) return result
def unregister_endpoint(manager_endpoint_name: str) -> bool: deleted = False for manager_endpoint in find(ManagerEndpoint): if manager_endpoint.name == manager_endpoint_name: for resource_type in get_mapping_managers().get( manager_endpoint.name): for rm in [ rm for rm in find(ResourceMetadata) if rm.node_type.lower() == resource_type.lower() ]: delete(rm) delete(manager_endpoint) deleted = True return deleted
def check_overlapping_booking(cls, used_resource): """ Check calendar availability of this resource :param used_resource: the used resource to book :type used_resource: UsedResource :return: True when availability is granted False otherwise :rtype: bool """ resource_metadata = cls.get_metadata_from_usedresource(used_resource) # if the resource metadata associated has infinite cardinality return true if resource_metadata.cardinality <= 0: return True max_concurrent = resource_metadata.cardinality counter = 0 # if not then i need to calculate the number of resource already booked for that period for ur in find(UsedResource): if ur.status == ResourceStatus.RESERVED.value: if ur.start_date <= used_resource.start_date <= ur.end_date: counter += 1 elif ur.start_date <= used_resource.end_date <= ur.end_date: counter += 1 if counter < max_concurrent: return True return False
def _validate_resource(node, username, request_metadata): for manager_endpoint in find(ManagerEndpoint): if node.type in get_mapping_managers().get(manager_endpoint.name): if request_metadata and request_metadata.properties and request_metadata.properties.get( 'nsd_file_name'): nsd_file_name = request_metadata.properties.get( 'nsd_file_name') node.entity_tpl.get( 'properties')['file_name'] = 'Files/%s' % nsd_file_name request_message = messages_pb2.RequestMessage( method=messages_pb2.VALIDATE_RESOURCES, payload=yaml.dump(node.entity_tpl), user_info=get_user_info(username)) try: response = get_stub_from_manager_endpoint( manager_endpoint).execute(request_message) except Exception as e: if hasattr(e, 'message'): raise RpcFailedCall(e.message) raise RpcFailedCall(e.args) if response.result == messages_pb2.ERROR: raise RpcFailedCall(response.error_message) return raise ManagerNotFound("manager handling resource %s was not found" % node.type)
def get_resources_dict(username=None): res = [] for rm in find(ResourceMetadata): if rm.node_type != 'NfvImage' and rm.node_type != 'NfvNetwork' and rm.node_type != 'NfvFlavor': if username: if username == rm.user: res.append(_get_resource_dict_from_rm(rm)) else: if not rm.user: res.append(_get_resource_dict_from_rm(rm)) return res
def list_resources(username='', manager_name=None): managers = [] if manager_name is None: for man in find(ManagerEndpoint): managers.append(man.name) else: managers.append(manager_name) result = [] # List resources can be done in parallel max_workers = len(managers) tpe = ThreadPoolExecutor(max_workers=max_workers) threads = [] for manager in managers: threads.append(tpe.submit(_execute_rpc_list_res, manager)) for t in threads: result.extend(t.result()) logger.debug("Saving %d resources" % len(result)) for rm in find(entities.ResourceMetadata): for rm_to_del in result: if rm.id == rm_to_del.resource_id: delete(rm) for rm in result: if username == '' or rm.user == username: resource_metadata = ResourceMetadata() if not username is None: resource_metadata.user = username resource_metadata.id = rm.resource_id resource_metadata.description = rm.description resource_metadata.cardinality = rm.cardinality resource_metadata.node_type = rm.node_type resource_metadata.testbed = _get_testbed_string(rm.testbed) save(resource_metadata, ResourceMetadata) return result
def check_overlapping_resources(cls, used_resources: list): for ur_to_check in used_resources: counter = 1 for other_res in used_resources: if other_res != ur_to_check and other_res.node_type == ur_to_check.node_type: if _time_overlap(ur_to_check.start_date, ur_to_check.end_date, other_res.start_date, other_res.end_date): counter += 1 cardinality = find(ResourceMetadata, _id=ur_to_check.resource_id).cardinality if 0 < cardinality < counter: raise ExperimentValidationError( "Trying to book %d resources of resource_id %s" % (counter, ur_to_check.resource_id))
def _release_used_resource(res: entities.UsedResource): exp = find(entities.Experiment, _id=res.parent_id) user_info = get_user_info(exp.username) for man, node_types in get_mapping_managers().items(): if res.node_type in node_types: response = get_stub_from_manager_name(man).execute( messages_pb2.RequestMessage( method=messages_pb2.RELEASE_RESOURCES, payload=res.value, user_info=user_info)) if response.result != 0: logger.error("release resources returned %d: %s" % (response.result, response.error_message)) raise RpcFailedCall("provide resources returned %d: %s" % (response.result, response.error_message))
def get_other_resources(): images = [] networks = [] flavours = [] for rm in find(ResourceMetadata): if rm.node_type == 'NfvImage': tmp = { 'resource_id': rm.id, 'node_type': rm.node_type, 'testbed': rm.testbed, 'description': rm.description } if rm.cardinality < 0: tmp['cardinality'] = 'infinite' else: tmp['cardinality'] = rm.cardinality images.append(tmp) if rm.node_type == 'NfvFlavor': tmp = { 'resource_id': rm.id, 'node_type': rm.node_type, 'testbed': rm.testbed, 'description': rm.description } if rm.cardinality < 0: tmp['cardinality'] = 'infinite' else: tmp['cardinality'] = rm.cardinality flavours.append(tmp) if rm.node_type == 'NfvNetwork': tmp = { 'resource_id': rm.id, 'node_type': rm.node_type, 'testbed': rm.testbed, 'description': rm.description } if rm.cardinality < 0: tmp['cardinality'] = 'infinite' else: tmp['cardinality'] = rm.cardinality networks.append(tmp) return images, networks, flavours
def create_user(username, password, role='experimenter'): for experimenter in find(Experimenter): if experimenter.username == username: raise Exception('Username \'{}\' already exists.'.format(username)) user_info = messages_pb2.UserInfo(name=username, password=password) for man in MANAGERS_CREATE_USER: try: user_info = get_stub_from_manager_name(man).create_user(user_info) logger.info("Manager %s create user finished, new UserInfo: %s" % (man, user_info)) except ManagerNotFound: traceback.print_exc() logger.error( "one of the manager is not register and need to create user") logger.info("Created user on the Managers") experimenter = _save_or_create_experimenter(user_info, role) logger.info("Stored new experimenter: %s" % experimenter.username)
def check_endpoint(): manager_unavailable_times = get_config('system', 'manager-unavailable', '5') while not stop.wait(int(get_config('system', 'manager-check-delay', '20'))): for endpoint in find(ManagerEndpoint): try: get_stub_from_manager_endpoint(endpoint).heartbeat(Empty()) except: logger.error("Exception calling heartbeat for manager %s" % endpoint.name) if logger.isEnabledFor(logging.DEBUG): traceback.print_exc() if endpoint.unavailability >= int(manager_unavailable_times): logger.error("Manager %s on endpoint %s is not running" % (endpoint.name, endpoint.endpoint)) MessagingAgent.unregister_endpoint(endpoint.name) else: endpoint.unavailability += 1 save(endpoint)
def get_experiment_dict(username): res = [] exp_names = [] exp_ids = [] for ex in find(entities.Experiment): if not username or ex.username == username: exp_ids.append(ex.id) exp_names.append(ex.name) for ur in ex.resources: tmp = { 'resource_id': ur.resource_id, 'used_resource_id': ur.id, 'node_type': ur.node_type, 'status': ResourceStatus.from_int_to_enum(ur.status).name, 'value': ur.value, 'experiment_name': ex.name, 'experiment_id': ex.id, 'username': ex.username } res.append(tmp) return exp_names, exp_ids, res
def get_metadata_from_usedresource(cls, used_resource): return find(ResourceMetadata, _id=used_resource.resource_id)
def list_managers(): return [man.name for man in find(ManagerEndpoint)]
def list_experimenters(): return [man.username for man in find(Experimenter)]
def get_stub_from_manager_name(manager_name): for manager_endpoint in find(ManagerEndpoint): if manager_endpoint.name == manager_name: logger.debug("Getting stub for manager: %s" % manager_name) return get_stub_from_manager_endpoint(manager_endpoint) raise ManagerNotFound("No manager found for name %s" % manager_name)
def __init__(self, file_path, username): self.username = username self.file_path = file_path zf = zipfile.ZipFile(self.file_path) for info in zf.infolist(): filename = info.filename logger.debug("Filename: %s" % filename) if filename.endswith(".yaml") and filename.startswith( "Definitions/"): logger.debug("parsing %s..." % filename) try: tpl = ToscaTemplate( yaml_dict_tpl=yaml.load(zf.read(filename))) except Exception as e: if hasattr(e, 'message'): raise ExperimentValidationError(e.message) elif isinstance(e.args, list): raise ExperimentValidationError(e.args[0]) elif isinstance(e.args, str): raise ExperimentValidationError(e.args) else: raise ExperimentValidationError( "no message available", e) for node in tpl.nodetemplates: logger.debug( "Found node: %s of type %s with properties: %s" % (node.name, node.type, list(node.get_properties().keys()))) self.topology_template = tpl if filename == 'TOSCA-Metadata/Metadata.yaml': metadata = yaml.load(zf.read(filename)) self.name = metadata.get('name') if find(entities.Experiment, '{}_{}'.format(self.username, self.name)) is not None: raise ExperimentValidationError( 'There is already an experiment with this name, please choose a different one.' ) # 12/12/12 10:55 self.start_date = self.get_start_date(metadata) # 12/12/12 11:55 self.end_date = self.get_end_date(metadata) today = datetime.today().date() logger.info("End date: date: %s" % self.end_date.date()) if self.end_date.date() < today: logger.error( 'For the experiment {} the end date({}) is in the past.' .format(self.name, self.end_date.date())) raise ExperimentValidationError( 'For the experiment {} the end date({}) is in the past.' .format(self.name, self.end_date.date())) self.duration = self.end_date - self.start_date logger.debug("Experiment duration %s" % self.duration) if filename.startswith("Files/") and filename.endswith('.csar'): experiment_nsd_csar_location = get_config( 'system', 'temp-csar-location', '/etc/softfire/experiment-nsd-csar') experiment_nsd_csar_location = '{}/{}'.format( experiment_nsd_csar_location.rstrip('/'), username) if not os.path.exists(experiment_nsd_csar_location): os.makedirs(experiment_nsd_csar_location) data = zf.read(filename) nsd_file_location = "%s/%s" % (experiment_nsd_csar_location, filename.split('/')[-1]) with open(nsd_file_location, 'wb+') as f: f.write(data) temp_ids = [] for node in self.topology_template.nodetemplates: if node.type == 'NfvResource': file_name = node.get_properties().get('file_name') if file_name: file_name = file_name.value if file_name and file_name.startswith( "Files/") and file_name.endswith(".csar"): real_file_name = file_name[6:] tmp_file_location = '{}/{}/{}'.format( get_config( 'system', 'temp-csar-location', '/etc/softfire/experiment-nsd-csar').rstrip('/'), username, real_file_name) # get the description try: zf = zipfile.ZipFile(tmp_file_location) except FileNotFoundError: logger.error( "Please check that the file_name property is correctly set to the file name passed." ) raise ExperimentValidationError( "Please check that the file_name property is correctly set to the file name passed." ) try: filenames = [ zipinfo.filename for zipinfo in zf.filelist ] if 'TOSCA-Metadata/Metadata.yaml' in filenames: logger.debug( "Found 'TOSCA-Metadata/Metadata.yaml'") yaml_file = zf.read('TOSCA-Metadata/Metadata.yaml') elif 'tosca-metadata/Metadata.yaml' in filenames: # just for legacy support # the correct file name following the specification is 'TOSCA-Metadata' # to support experiments that were created before this fix we still check for 'tosca-metadata' though logger.warning( "'TOSCA-Metadata/Metadata.yaml' not found. Will try 'tosca-metadata/Metadata.yaml' to support older experiments." ) yaml_file = zf.read('tosca-metadata/Metadata.yaml') else: raise ExperimentValidationError( 'The TOSCA-Metadata/Metadata.yaml file is missing in {}.' .format(file_name)) except KeyError as e: traceback.print_exc() if hasattr(e, 'message'): raise ExperimentValidationError(e.message) else: raise ExperimentValidationError(e.args) if yaml_file: yaml_content = yaml.load(yaml_file) description = yaml_content.get('description') else: description = 'No description available' testbeds = node.get_properties().get('testbeds').value temp_ids.append( add_resource( username, node.get_properties().get('resource_id').value, "NfvResource", -1, description, list(testbeds.keys())[0], real_file_name)) try: self._validate() except Exception as e: for id in temp_ids: delete(find(ResourceMetadata, _id=id)) raise ExperimentValidationError(e.args) exp = entities.Experiment() exp.id = '{}_{}'.format(self.username, self.name) exp.username = self.username exp.name = self.name exp.start_date = self.start_date exp.end_date = self.end_date exp.resources = [] for node in self.topology_template.nodetemplates: exp.resources.append(self._get_used_resource_by_node(node)) element_value = find_by_element_value(entities.Experiment, entities.Experiment.username, self.username) max_no_experiment = get_config("System", "max-number-experiments", 3) if len(element_value) >= max_no_experiment: raise ExperimentValidationError( "You cannot have more than %s experiments at the same time!" % max_no_experiment) logger.info("Saving experiment %s" % exp.name) save(exp) self.experiment = exp for res in exp.resources: _start_termination_thread_for_res(res)