def check_vim_capacity(self, vim_request): LOG.debug("Invoking check_vim_capacity api") path = '/{}/{}'.format(self.version, 'check_vim_capacity') data = {} data['vCPU'] = vim_request['vCPU'] data['Memory'] = vim_request['Memory']['quantity'] data['Storage'] = vim_request['Storage']['quantity'] data['VIMs'] = vim_request['VIMs'] response = self._request('post', path=path, data=data, context="vim capacity", value="all") LOG.debug("Response check_vim_capacity api - {}".format(response)) if response is None or response.status_code != 200: return None body = response.json() if body: vims = body.get("VIMs") if vims is None: LOG.error( _LE("Unable to get VIMs with cpu-{}, memory-{}, disk-{}"). format(data['vCPU'], data['Memory'], data['Storage'])) return vims else: LOG.error( _LE("Unable to get VIMs from Multicloud with " "requirement {}").format(data)) return None
def _request(self, method='get', path='/', data=None, context=None, value=None): """Performs HTTP request.""" headers = { 'X-FromAppId': 'CONDUCTOR', 'X-TransactionId': str(uuid.uuid4()), } kwargs = { "method": method, "path": path, "headers": headers, "data": data, } start_time = time.time() response = self.rest.request(**kwargs) elapsed = time.time() - start_time LOG.debug("Total time for Multicloud request " "({0:}: {1:}): {2:.3f} sec".format(context, value, elapsed)) if response is None: LOG.error(_LE("No response from Multicloud ({}: {})"). format(context, value)) elif response.status_code != 200: LOG.error(_LE("Multicloud request ({}: {}) returned HTTP " "status {} {}, link: {}{}"). format(context, value, response.status_code, response.reason, self.base, path)) return response
def remote_api(aafUser): server_url = CONF.aaf_api.aaf_url+aafUser kwargs = { "server_url": server_url, "retries": CONF.aaf_api.aaf_retries, "username": CONF.aaf_api.username, "password": CONF.aaf_api.password, "log_debug": LOG.debug, "read_timeout": CONF.aaf_api.aaf_timeout, "cert_file": CONF.aaf_api.aaf_cert_file, "cert_key_file": CONF.aaf_api.aaf_cert_key_file, "ca_bundle_file": CONF.aaf_api.aaf_ca_bundle_file, } restReq = rest.REST(**kwargs) headers = {"Accept": "application/Perms+json;q=1.0;charset=utf-8;version=2.1,application/json;q=1.0;version=2.1,*/*;q=1.0"} rkwargs = { "method": 'GET', "path": '', "headers": headers, } response = restReq.request(**rkwargs) if response is None: LOG.error(_LE("No response from AAF ")) elif response.status_code != 200: LOG.error(_LE("AAF request returned HTTP " "status {} {}, link: {}"). format(response.status_code, response.reason, server_url)) return response.content
def _request(self, method='get', path='/', data=None, context=None, value=None): """Performs HTTP request.""" kwargs = { "method": method, "path": path, "data": data, } # TODO(jdandrea): Move timing/response logging into the rest helper? start_time = time.time() response = self.rest.request(**kwargs) elapsed = time.time() - start_time LOG.debug("Total time for SDN-C request " "({0:}: {1:}): {2:.3f} sec".format(context, value, elapsed)) if response is None: LOG.error( _LE("No response from SDN-C ({}: {})").format(context, value)) raise Exception('SDN-C query {} timed out'.format(path)) elif response.status_code != 200: LOG.error( _LE("SDN-C request ({}: {}) returned HTTP " "status {} {}, link: {}{}").format(context, value, response.status_code, response.reason, self.base, path)) return response
def resolve_location(self, ctx, arg): log_util.setLoggerFilter(LOG, ctx.get('keyspace'), ctx.get('plan_id')) error = False resolved_location = None host_name = arg.get('host_name') clli_code = arg.get('clli_code') if host_name: results = self.ip_ext_manager.map_method('resolve_host_location', host_name) elif clli_code: results = self.ip_ext_manager.map_method('resolve_clli_location', clli_code) else: results = None # unknown location response LOG.error( _LE("Unknown location type from the input template." "Expected location types are host_name" " or clli_code.")) if results and len(results) > 0: resolved_location = results[0] else: error = True return { 'response': { 'resolved_location': resolved_location }, 'error': error }
def call_reservation_operation(self, ctx, arg): result = True reserved_candidates = None method = arg["method"] candidate_list = arg["candidate_list"] reservation_name = arg["reservation_name"] reservation_type = arg["reservation_type"] controller = arg["controller"] request = arg["request"] if controller == "SDN-C": results = self.sc_ext_manager.map_method( 'call_reservation_operation', method=method, candidate_list=candidate_list, reservation_name=reservation_name, reservation_type=reservation_type, request=request) if results and len(results) > 0: reserved_candidates = results[0] else: LOG.error(_LE("Unknown service controller: {}").format(controller)) if reserved_candidates is None or not reserved_candidates: result = False LOG.debug( _LW("Unable to {} for " "candidate {}.").format(method, reserved_candidates)) return {'response': result, 'error': not result} else: LOG.debug("{} for the candidate: " "{}".format(method, reserved_candidates)) return {'response': result, 'error': not result}
def rollback_reservation(self, reservation_list): """Function to rollback(release) reservations""" # TODO(snarayanan): Need to test this once the API is ready for reservation in reservation_list: candidate_list = reservation['candidate_list'] reservation_name = reservation['reservation_name'] reservation_type = reservation['reservation_type'] controller = reservation['controller'] request = reservation['request'] is_success = self.try_reservation_call( method="release", candidate_list=candidate_list, reservation_name=reservation_name, reservation_type=reservation_type, controller=controller, request=request) if not is_success: # rollback failed report error to SDNC message = _LE("Unable to release reservation " "{}").format(reservation) LOG.error(message) return False # move to the next reserved candidate return True
def encode(user_id, password): """ Provide the basic authencation encoded value in an 'Authorization' Header """ user_pass = user_id + ':' + password base64_val = base64.b64encode(user_pass) authorization_val = _LE("Basic {}".format(base64_val)) return authorization_val
def get_inventory_group_candidates(self, ctx, arg): candidate_list = arg["candidate_list"] resolved_candidate = arg["resolved_candidate"] candidate_names = [] error = False service_description = 'DHV_VVIG_PAIR' results = self.ip_ext_manager.map_method( 'get_inventory_group_pairs', service_description=service_description) if not results or len(results) < 1: LOG.error( _LE("Empty inventory group response for service: {}").format( service_description)) error = True else: pairs = results[0] if not pairs or len(pairs) < 1: LOG.error( _LE("No inventory group candidates found for service: {}, " "inventory provider: {}").format( service_description, self.ip_ext_manager.names()[0])) error = True else: LOG.debug("Inventory group pairs: {}, service: {}, " "inventory provider: {}".format( pairs, service_description, self.ip_ext_manager.names()[0])) for pair in pairs: if resolved_candidate.get("candidate_id") == pair[0]: candidate_names.append(pair[1]) elif resolved_candidate.get("candidate_id") == pair[1]: candidate_names.append(pair[0]) candidate_list = [ c for c in candidate_list if c["candidate_id"] in candidate_names ] LOG.info( _LI("Inventory group candidates: {}, service: {}, " "inventory provider: {}").format( candidate_list, service_description, self.ip_ext_manager.names()[0])) return {'response': candidate_list, 'error': error}
def encode(user_id, password): """ Provide the basic authencation encoded value in an 'Authorization' Header """ user_pass = user_id + ":" + password # Here user_pass is str type but in python 3.x version, base64.b64encode is expecting byte # like object so we need to convert the str into bytes user_pass = user_pass.encode() # converting str into bytes form base64_val = base64.b64encode(user_pass).decode() authorization_val = _LE("Basic {}".format(base64_val)) return authorization_val
def _request(self, method='get', path='/', data=None, context=None, value=None): """Performs HTTP request.""" headers = { 'X-FromAppId': 'AAI', 'X-TransactionId': str(uuid.uuid4()), 'X-ECOMP-InstanceID': 'AAI', } kwargs = { "method": method, "path": path, "headers": headers, "data": data, "content_type": "application/octet-stream" } # TODO(jdandrea): Move timing/response logging into the rest helper? start_time = time.time() response = self.rest.request(**kwargs) elapsed = time.time() - start_time LOG.debug("Total time for SDC request " "({0:}: {1:}): {2:.3f} sec".format(context, value, elapsed)) if response is None: LOG.error( _LE("No response from SDC ({}: {})").format(context, value)) elif response.status_code != 200: LOG.error( _LE("SDC request ({}: {}) returned HTTP " "status {} {}, link: {}{}").format(context, value, response.status_code, response.reason, self.base, path)) return response
def translate(self, plan): """Translate the plan to a format the solver can use""" # Update the translation field and set status to TRANSLATED. try: LOG.info(_LI("Requesting plan {} translation").format(plan.id)) template_version = plan.template.get("homing_template_version") if template_version in VERSIONS['BASE']: trns = Translator(self.conf, plan.name, plan.id, plan.template) elif template_version in VERSIONS['GENERIC']: trns = GenericObjectiveTranslator(self.conf, plan.name, plan.id, plan.template, self.opt_schema) else: raise TranslatorException( "conductor_template_version must be one " "of: {}".format(', '.join( [x for v in VERSIONS.values() for x in v]))) trns.translate() if trns.ok: plan.translation = trns.translation plan.status = self.Plan.TRANSLATED LOG.info( _LI("Plan {} translated. Ready for solving").format( plan.id)) LOG.info(_LI("Plan name: {}").format(plan.name)) else: plan.message = trns.error_message plan.status = self.Plan.ERROR LOG.error( _LE("Plan {} translation error encountered").format( plan.id)) except Exception as ex: template = "An exception of type {0} occurred, arguments:\n{1!r}" plan.message = template.format(type(ex).__name__, ex.args) plan.status = self.Plan.ERROR _is_success = 'FAILURE' while 'FAILURE' in _is_success and (self.current_time_seconds() - self.millisec_to_sec(plan.updated)) \ <= self.conf.messaging_server.timeout: _is_success = plan.update( condition=self.translation_owner_condition) LOG.info( _LI("Changing the template status from translating to {}, " "atomic update response from MUSIC {}").format( plan.status, _is_success))
def lock_create(self, keyspace, table, pk_value): """Create and acquire a lock. Returns a lock name.""" # Generate the lock name, then create/acquire the lock id. lock_name = self._lock_name_generate(keyspace, table, pk_value) if CONF.music_api.debug: LOG.debug("Creating lock {}".format(lock_name)) lock_id = self._lock_id_create(lock_name) time_now = time.time() while not self._lock_id_acquire(lock_id): if time.time() - time_now > self.lock_timeout: raise IndexError( _LE('Lock id acquire timeout: %s') % lock_name) # Cache the lock name/id. self.lock_ids[lock_name] = lock_id return lock_name
def get_candidates_from_service(self, ctx, arg): candidate_list = arg["candidate_list"] constraint_name = arg["constraint_name"] constraint_type = arg["constraint_type"] controller = arg["controller"] request = arg["request"] request_type = arg["request_type"] error = False filtered_candidates = [] # call service and fetch candidates # TODO(jdandrea): Get rid of the SDN-C reference (outside of plugin!) if controller == "SDN-C": service_model = request.get("service_model") results = self.sc_ext_manager.map_method( 'filter_candidates', request=request, candidate_list=candidate_list, constraint_name=constraint_name, constraint_type=constraint_type, request_type=request_type) if results and len(results) > 0: filtered_candidates = results[0] else: LOG.warn( _LW("No candidates returned by service " "controller: {}; may be a new service " "instantiation.").format(controller)) else: LOG.error(_LE("Unknown service controller: {}").format(controller)) # if response from service controller is empty if filtered_candidates is None: if service_model == "ADIOD": LOG.error("No capacity found from SDN-GC for candidates: " "{}".format(candidate_list)) return {'response': [], 'error': error} else: LOG.debug("Filtered candidates: {}".format(filtered_candidates)) candidate_list = [ c for c in candidate_list if c in filtered_candidates ] return {'response': candidate_list, 'error': error}
def get_candidate_zone(self, ctx, arg): candidate = arg["candidate"] category = arg["category"] zone = None error = False if category == 'region': zone = candidate['location_id'] elif category == 'complex': zone = candidate['complex_name'] elif category == 'country': zone = candidate['country'] else: error = True if error: LOG.error(_LE("Unresolvable zone category {}").format(category)) else: LOG.info(_LI("Candidate zone is {}").format(zone)) return {'response': zone, 'error': error}
def solve(self, _decision_path, _candidate_list, _request): select_list = list() candidates_to_check = list() demand_name = _decision_path.current_demand.name # service-check candidates of the same inventory type # select candidate of all other types for candidate in _candidate_list: if self.inventory_type == "cloud": if candidate["inventory_type"] == "cloud": candidates_to_check.append(candidate) else: select_list.append(candidate) elif self.inventory_type == "service": if candidate["inventory_type"] == "service": candidates_to_check.append(candidate) else: select_list.append(candidate) elif self.inventory_type == "vfmodule": if candidate["inventory_type"] == "vfmodule": candidates_to_check.append(candidate) else: select_list.append(candidate) # call conductor data with request parameters if len(candidates_to_check) > 0: cei = _request.cei request_type = _request.request_type filtered_list = cei.get_candidates_from_service( self.name, self.constraint_type, candidates_to_check, self.controller, self.inventory_type, self.request, self.cost, demand_name, request_type) for c in filtered_list: select_list.append(c) else: LOG.error(_LE("Constraint {} ({}) has no candidates of " "inventory type {} for demand {}").format( self.name, self.constraint_type, self.inventory_type, demand_name) ) _candidate_list[:] = [c for c in _candidate_list if c in select_list] return _candidate_list
def translate(self, plan): """Translate the plan to a format the solver can use""" # Update the translation field and set status to TRANSLATED. try: LOG.info(_LI("Requesting plan {} translation").format(plan.id)) trns = translator.Translator(self.conf, plan.name, plan.id, plan.template) trns.translate() if trns.ok: plan.translation = trns.translation plan.status = self.Plan.TRANSLATED LOG.info( _LI("Plan {} translated. Ready for solving").format( plan.id)) LOG.info(_LI("Plan name: {}").format(plan.name)) else: plan.message = trns.error_message plan.status = self.Plan.ERROR LOG.error( _LE("Plan {} translation error encountered").format( plan.id)) except Exception as ex: template = "An exception of type {0} occurred, arguments:\n{1!r}" plan.message = template.format(type(ex).__name__, ex.args) plan.status = self.Plan.ERROR _is_success = 'FAILURE' while 'FAILURE' in _is_success and ( self.current_time_seconds() - self.millisec_to_sec( plan.updated)) <= self.conf.messaging_server.timeout: _is_success = plan.update( condition=self.translation_owner_condition) LOG.info( _LI("Changing the template status from translating to {}, " "atomic update response from MUSIC {}").format( plan.status, _is_success))
def solve(self, _decision_path, _candidate_list, _request): ''' Solver for HPA constraint type. :param _decision_path: decision tree :param _candidate_list: List of candidates :param _request: solver request :return: candidate_list with hpa features and flavor label mapping ''' # call conductor engine with request parameters cei = _request.cei demand_name = _decision_path.current_demand.name LOG.info(_LI("Solving constraint type '{}' for demand - [{}]").format( self.constraint_type, demand_name)) vm_label_list = self.properties.get('evaluate') for vm_demand in vm_label_list: id = vm_demand['id'] type = vm_demand['type'] directives = vm_demand['directives'] flavorProperties = vm_demand['flavorProperties'] response = (cei.get_candidates_with_hpa(id, type, directives, _candidate_list, flavorProperties)) _candidate_list = response if not response: LOG.error(_LE("No matching candidates for HPA exists").format( id)) # Metrics to Prometheus PC.HPA_CLOUD_REGION_UNSUCCESSFUL.labels('ONAP', 'N/A', 'ALL').inc() break # No need to continue. return _candidate_list
def test_basic_auth_util(self): self.authorization_val = _LE( BaseAuthUtil.encode(self.userId, self.userId)) self.assertEqual('Basic UGFzc3dvcmQ6UGFzc3dvcmQ=', self.authorization_val)
def get_candidates_with_hpa(self, ctx, arg): ''' RPC for getting candidates flavor mapping for matching hpa :param ctx: context :param arg: contains input passed from client side for RPC call :return: response candidate_list with matching label to flavor mapping ''' error = False candidate_list = arg["candidate_list"] id = arg["id"] type = arg["type"] directives = arg["directives"] attr = directives[0].get("attributes") label_name = attr[0].get("attribute_name") flavorProperties = arg["flavorProperties"] discard_set = set() for i in range(len(candidate_list)): # perform this check only for cloud candidates if candidate_list[i]["inventory_type"] != "cloud": continue # Check if flavor mapping for current label_name already # exists. This is an invalid condition. if candidate_list[i].get( "directives") and attr[0].get("attribute_value") != "": LOG.error( _LE("Flavor mapping for label name {} already" "exists").format(label_name)) continue # RPC call to inventory provider for matching hpa capabilities results = self.ip_ext_manager.map_method( 'match_hpa', candidate=candidate_list[i], features=flavorProperties) flavor_name = None if results and len(results) > 0 and results[0] is not None: LOG.debug("Find results {} and results length {}".format( results, len(results))) flavor_info = results[0].get("flavor_map") req_directives = results[0].get("directives") LOG.debug("Get directives {}".format(req_directives)) else: flavor_info = None LOG.info( _LW("No flavor mapping returned by " "inventory provider: {} for candidate: {}").format( self.ip_ext_manager.names()[0], candidate_list[i].get("candidate_id"))) # Metrics to Prometheus m_vim_id = candidate_list[i].get("vim-id") if not flavor_info: discard_set.add(candidate_list[i].get("candidate_id")) PC.HPA_CLOUD_REGION_UNSUCCESSFUL.labels( 'ONAP', 'N/A', m_vim_id).inc() else: if not flavor_info.get("flavor-name"): discard_set.add(candidate_list[i].get("candidate_id")) PC.HPA_CLOUD_REGION_UNSUCCESSFUL.labels( 'ONAP', 'N/A', m_vim_id).inc() else: if not candidate_list[i].get("flavor_map"): candidate_list[i]["flavor_map"] = {} # Create flavor mapping for label_name to flavor flavor_name = flavor_info.get("flavor-name") flavor_id = flavor_info.get("flavor-id") candidate_list[i]["flavor_map"][label_name] = flavor_name candidate_list[i]["flavor_map"]["flavorId"] = flavor_id # Create directives if not exist already if not candidate_list[i].get("all_directives"): candidate_list[i]["all_directives"] = {} candidate_list[i]["all_directives"]["directives"] = [] # Create flavor mapping and merge directives self.merge_directives(candidate_list, i, id, type, directives, req_directives) if not candidate_list[i].get("hpa_score"): candidate_list[i]["hpa_score"] = 0 candidate_list[i]["hpa_score"] += flavor_info.get("score") # Metrics to Prometheus PC.HPA_CLOUD_REGION_SUCCESSFUL.labels( 'ONAP', 'N/A', m_vim_id).inc() # return candidates not in discard set candidate_list[:] = [ c for c in candidate_list if c['candidate_id'] not in discard_set ] LOG.info( _LI("Candidates with matching hpa capabilities: {}, " "inventory provider: {}").format( candidate_list, self.ip_ext_manager.names()[0])) return {'response': candidate_list, 'error': error}
def get_candidates_by_attributes(self, ctx, arg): candidate_list = arg["candidate_list"] # demand_name = arg["demand_name"] properties = arg["properties"] discard_set = set() attributes_to_evaluate = properties.get('evaluate') for attrib, value in attributes_to_evaluate.items(): if value == '': continue if attrib == 'network_roles': role_candidates = dict() role_list = [] nrc_dict = value role_condition = '' if nrc_dict: if "all" in nrc_dict: role_list = nrc_dict.get("all") role_condition = "all" elif "any" in nrc_dict: role_list = nrc_dict.get("any") role_condition = "any" # if the role_list is empty do nothing if not role_list or role_list == '': LOG.error( _LE("No roles available, " "inventory provider: {}").format( self.ip_ext_manager.names()[0])) continue for role in role_list: # query inventory provider to check if # the candidate is in role results = self.ip_ext_manager.map_method( 'check_network_roles', network_role_id=role) if not results or len(results) < 1: LOG.error( _LE("Empty response from inventory " "provider {} for network role {}").format( self.ip_ext_manager.names()[0], role)) continue region_ids = results[0] if not region_ids: LOG.error( _LE("No candidates from inventory provider {} " "for network role {}").format( self.ip_ext_manager.names()[0], role)) continue LOG.debug("Network role candidates: {}, role: {}," "inventory provider: {}".format( region_ids, role, self.ip_ext_manager.names()[0])) role_candidates[role] = region_ids # find candidates that meet conditions for candidate in candidate_list: # perform this check only for cloud candidates if candidate["inventory_type"] != "cloud": continue c_any = False c_all = True for role in role_list: if role not in role_candidates: c_all = False continue rc = role_candidates.get(role) if rc and candidate.get("candidate_id") not in rc: c_all = False # discard even if one role is not met elif rc and candidate.get("candidate_id") in rc: c_any = True # include if any one role is met if role_condition == 'any' and not c_any: discard_set.add(candidate.get("candidate_id")) elif role_condition == 'all' and not c_all: discard_set.add(candidate.get("candidate_id")) elif attrib == 'replication_role': for candidate in candidate_list: host_id = candidate.get("host_id") if host_id: results = self.ip_ext_manager.map_method( 'check_candidate_role', host_id=host_id) if not results or len(results) < 1: LOG.error( _LE("Empty response for replication roles {}"). format(role)) discard_set.add(candidate.get("candidate_id")) continue # compare results from A&AI with the value in attribute constraint if value and results[0] != value.upper(): discard_set.add(candidate.get("candidate_id")) elif attrib == 'complex': v_discard_set = \ self.get_candidate_discard_set( value=value, candidate_list=candidate_list, value_attrib="complex_name") discard_set.update(v_discard_set) elif attrib == "country": v_discard_set = \ self.get_candidate_discard_set( value=value, candidate_list=candidate_list, value_attrib="country") discard_set.update(v_discard_set) elif attrib == "state": v_discard_set = \ self.get_candidate_discard_set( value=value, candidate_list=candidate_list, value_attrib="state") discard_set.update(v_discard_set) elif attrib == "region": v_discard_set = \ self.get_candidate_discard_set( value=value, candidate_list=candidate_list, value_attrib="region") discard_set.update(v_discard_set) elif attrib == "cloud-region": v_discard_set = \ self.get_candidate_discard_set_by_cloud_region( value=value, candidate_list=candidate_list, value_attrib="location_id") discard_set.update(v_discard_set) # return candidates not in discard set candidate_list[:] = [ c for c in candidate_list if c['candidate_id'] not in discard_set ] LOG.info("Available candidates after attribute checks: {}, " "inventory provider: {}".format( candidate_list, self.ip_ext_manager.names()[0])) return {'response': candidate_list, 'error': False}
def request(self, method='get', content_type='application/json', path='', headers=None, data=None): """Performs HTTP request. Returns a requests.Response object.""" if method not in ('post', 'get', 'put', 'delete'): method = 'get' method_fn = getattr(self.session, method) full_headers = { 'Accept': content_type, 'Content-Type': content_type, } if headers: full_headers.update(headers) full_url = '{}/{}'.format(self.server_url, path.lstrip('/').rstrip('/')) # Prepare the request args try: data_str = json.dumps(data) if data else None except (TypeError, ValueError): data_str = data kwargs = { 'data': data_str, 'headers': full_headers, 'timeout': self.timeout, 'cert': (self.cert, self.key), 'verify': self.verify, 'stream': False, } if self.username or self.password: LOG.debug("Using HTTPBasicAuth") kwargs['auth'] = HTTPBasicAuth(self.username, self.password) if self.cert and self.key: LOG.debug("Using SSL/TLS Certificate/Key") if self.log_debug: LOG.debug("Request: {} {}".format(method.upper(), full_url)) if data: LOG.debug("Request Body: {}".format(json.dumps(data))) response = None for attempt in range(self.retries): if attempt > 0: # No need to show 400 bad requests from Music - Ignorable when lock cannot be received at one particular point in time if "MUSIC" not in full_url: LOG.warn( _LW("Retry #{}/{}").format(attempt + 1, self.retries)) try: response = method_fn(full_url, **kwargs) # We shouldn't have to do this since stream is set to False, # but we're gonna anyway. See "Body Content Workflow" here: # http://docs.python-requests.org/en/master/user/advanced/ response.close() if not response.ok: # No need to show 400 bad requests from Music - Ignorable when lock cannot be received at one particular point in time if "MUSIC" not in full_url: LOG.warn("Response Status: {} {}".format( response.status_code, response.reason)) if self.log_debug and response.text: try: response_dict = json.loads(response.text) LOG.debug("Response JSON: {}".format( json.dumps(response_dict))) except ValueError: LOG.debug("Response Body: {}".format(response.text)) #response._content = response._content.decode() if response.ok: break except requests.exceptions.RequestException as err: LOG.error("Exception: %s", err.args) # Response.__bool__ returns false if status is not ok. Ruh roh! # That means we must check the object type vs treating it as a bool. # More info: https://github.com/kennethreitz/requests/issues/2002 if isinstance(response, requests.models.Response) and not response.ok: # No need to show 400 bad requests from Music - Ignorable when lock cannot be received at one particular point in time if "MUSIC" not in full_url: LOG.error( _LE("Status {} {} after {} retries for URL: {}").format( response.status_code, response.reason, self.retries, full_url)) return response
def run(self): """Run""" LOG.debug("{}".format(self.__class__.__name__)) # TODO(snarayanan): This is really meant to be a control loop # As long as self.running is true, we process another request. while self.running: # Delay time (Seconds) for MUSIC requests. time.sleep(self.conf.delay_time) # plans = Plan.query().all() # Find the first plan with a status of TRANSLATED. # Change its status to SOLVING. # Then, read the "translated" field as "template". json_template = None p = None requests_to_solve = dict() regions_maps = dict() country_groups = list() # Instead of using the query.all() method, now creating an index for 'status' # field in conductor.plans table, and query plans by status columns translated_plans = self.Plan.query.get_plan_by_col("status", self.Plan.TRANSLATED) solving_plans = self.Plan.query.get_plan_by_col("status", self.Plan.SOLVING) # combine the plans with status = 'translated' and 'solving' together plans = translated_plans + solving_plans found_translated_template = False for p in plans: if p.status == self.Plan.TRANSLATED: json_template = p.translation found_translated_template = True break elif p.status == self.Plan.SOLVING and (self.current_time_seconds() - self.millisec_to_sec(p.updated)) > self.conf.solver.timeout: p.status = self.Plan.TRANSLATED p.update(condition=self.solving_status_condition) break if not json_template: if found_translated_template: message = _LE("Plan {} status is translated, yet " "the translation wasn't found").format(p.id) LOG.error(message) p.status = self.Plan.ERROR p.message = message p.update(condition=self.translated_status_condition) continue if found_translated_template and p and p.solver_counter >= self.conf.solver.max_solver_counter: message = _LE("Tried {} times. Plan {} is unable to solve").format(self.conf.solver.max_solver_counter, p.id) LOG.error(message) p.status = self.Plan.ERROR p.message = message p.update(condition=self.translated_status_condition) continue log_util.setLoggerFilter(LOG, self.conf.keyspace, p.id) p.status = self.Plan.SOLVING p.solver_counter += 1 p.solver_owner = socket.gethostname() _is_updated = p.update(condition=self.translated_status_condition) if not _is_updated: continue # other VMs have updated the status and start solving the plan if 'FAILURE' in _is_updated: continue LOG.info(_LI("Sovling starts, changing the template status from translated to solving, " "atomic update response from MUSIC {}").format(_is_updated)) LOG.info(_LI("Plan {} with request id {} is solving by machine {}. Tried to solve it for {} times."). format(p.id, p.name, p.solver_owner, p.solver_counter)) _is_success = "FAILURE" request = parser.Parser() request.cei = self.cei request.request_id = p.name request.plan_id = p.id # getting the number of solutions need to provide num_solution = getattr(p, 'recommend_max', '1') if num_solution.isdigit(): num_solution = int(num_solution) # TODO(inam/larry): move this part of logic inside of parser and don't apply it to distance_between try: # getting region placeholders from database and insert/put into regions_maps dictionary region_placeholders = self.RegionPlaceholders.query.all() for region in region_placeholders: regions_maps.update(region.countries) # getting country groups from database and insert into the country_groups list customer_loc = '' location_list = json_template["conductor_solver"]["locations"] for location_id, location_info in location_list.items(): customer_loc = location_info['country'] countries = self.CountryLatency.query.get_plan_by_col("country_name", customer_loc) LOG.info("Customer Location for Latency Reduction " + customer_loc) if len(countries) == 0: LOG.info("country is not present is country latency table, looking for * wildcard entry") countries = self.CountryLatency.query.get_plan_by_col("country_name", "*") if len(countries) != 0: LOG.info("Found '*' wild card entry in country latency table") else: msg = "No '*' wild card entry found in country latency table. No solution will be provided" LOG.info(msg) p.message = msg for country in countries: country_groups = country.groups LOG.info("Done getting Latency Country DB Groups ") except Exception as error_msg: LOG.error("Exception thrown while reading region_placeholders and country groups information " "from database. Exception message: {}".format(error_msg)) try: request.parse_template(json_template, country_groups, regions_maps) request.assgin_constraints_to_demands() requests_to_solve[p.id] = request opt = optimizer.Optimizer(self.conf, _requests=requests_to_solve) solution_list = opt.get_solution(num_solution) except Exception as err: message = _LE("Plan {} status encountered a " "parsing error: {}").format(p.id, err) LOG.error(traceback.print_exc()) p.status = self.Plan.ERROR p.message = message while 'FAILURE' in _is_success: _is_success = p.update(condition=self.solver_owner_condition) LOG.info(_LI("Encountered a parsing error, changing the template status from solving to error, " "atomic update response from MUSIC {}").format(_is_success)) continue LOG.info("Preparing the recommendations ") # checking if the order is 'initial' or 'speed changed' one is_speed_change = False if request and request.request_type == 'speed changed': is_speed_change = True recommendations = [] if not solution_list or len(solution_list) < 1: # when order takes too much time to solve if (int(round(time.time())) - self.millisec_to_sec(p.updated)) > self.conf.solver.solver_timeout: message = _LI("Plan {} is timed out, exceed the expected " "time {} seconds").format(p.id, self.conf.solver.timeout) # when no solution found else: message = _LI("Plan {} search failed, no " "recommendations found by machine {}").format(p.id, p.solver_owner) LOG.info(message) # Update the plan status p.status = self.Plan.NOT_FOUND p.message = message # Metrics to Prometheus m_svc_name = p.template.get('parameters', {}).get('service_name', 'N/A') PC.VNF_FAILURE.labels('ONAP', m_svc_name).inc() while 'FAILURE' in _is_success: _is_success = p.update(condition=self.solver_owner_condition) LOG.info(_LI("Plan serach failed, changing the template status from solving to not found, " "atomic update response from MUSIC {}").format(_is_success)) else: # Assemble recommendation result JSON for solution in solution_list: current_rec = dict() for demand_name in solution: resource = solution[demand_name] if not is_speed_change: is_rehome = "false" else: is_rehome = "false" if resource.get("existing_placement") == 'true' else "true" location_id = "" if resource.get("cloud_region_version") == '2.5' \ else resource.get("location_id") rec = { # FIXME(shankar) A&AI must not be hardcoded here. # Also, account for more than one Inventory Provider. "inventory_provider": "aai", "service_resource_id": resource.get("service_resource_id"), "candidate": { "candidate_id": resource.get("candidate_id"), "inventory_type": resource.get("inventory_type"), "cloud_owner": resource.get("cloud_owner"), "location_type": resource.get("location_type"), "location_id": location_id, "is_rehome": is_rehome}, "attributes": { "physical-location-id": resource.get("physical_location_id"), "cloud_owner": resource.get("cloud_owner"), 'aic_version': resource.get("cloud_region_version")}, } if rec["candidate"]["inventory_type"] in ["nssi", "nsi", "slice_profiles"]: rec["candidate"] = resource if resource.get('vim-id'): rec["candidate"]['vim-id'] = resource.get('vim-id') if rec["candidate"]["inventory_type"] == "service": rec["attributes"]["host_id"] = resource.get("host_id") rec["attributes"]["service_instance_id"] = resource.get("candidate_id") rec["candidate"]["host_id"] = resource.get("host_id") if resource.get('vlan_key'): rec["attributes"]['vlan_key'] = resource.get('vlan_key') if resource.get('port_key'): rec["attributes"]['port_key'] = resource.get('port_key') if rec["candidate"]["inventory_type"] == "vfmodule": rec["attributes"]["host_id"] = resource.get("host_id") rec["attributes"]["service_instance_id"] = resource.get("service_instance_id") rec["candidate"]["host_id"] = resource.get("host_id") if resource.get('vlan_key'): rec["attributes"]['vlan_key'] = resource.get('vlan_key') if resource.get('port_key'): rec["attributes"]['port_key'] = resource.get('port_key') vf_module_data = rec["attributes"] vf_module_data['nf-name'] = resource.get("nf-name") vf_module_data['nf-id'] = resource.get("nf-id") vf_module_data['nf-type'] = resource.get("nf-type") vf_module_data['vnf-type'] = resource.get("vnf-type") vf_module_data['vf-module-id'] = resource.get("vf-module-id") vf_module_data['vf-module-name'] = resource.get("vf-module-name") vf_module_data['ipv4-oam-address'] = resource.get("ipv4-oam-address") vf_module_data['ipv6-oam-address'] = resource.get("ipv6-oam-address") vf_module_data['vservers'] = resource.get("vservers") elif rec["candidate"]["inventory_type"] == "cloud": if resource.get("all_directives") and resource.get("flavor_map"): rec["attributes"]["directives"] = \ self.set_flavor_in_flavor_directives( resource.get("flavor_map"), resource.get("all_directives")) # Metrics to Prometheus m_vim_id = resource.get("vim-id") m_hpa_score = resource.get("hpa_score", 0) m_svc_name = p.template['parameters'].get( 'service_name', 'N/A') for vnfc, flavor in resource.get("flavor_map").items(): PC.VNF_COMPUTE_PROFILES.labels('ONAP', m_svc_name, demand_name, vnfc, flavor, m_vim_id).inc() PC.VNF_SCORE.labels('ONAP', m_svc_name, demand_name, m_hpa_score).inc() if resource.get('conflict_id'): rec["candidate"]["conflict_id"] = resource.get("conflict_id") if resource.get('passthrough_attributes'): for key, value in resource.get('passthrough_attributes').items(): if key in rec["attributes"]: LOG.error('Passthrough attribute {} in demand {} already exist for candidate {}'. format(key, demand_name, rec['candidate_id'])) else: rec["attributes"][key] = value # TODO(snarayanan): Add total value to recommendations? # msg = "--- total value of decision = {}" # LOG.debug(msg.format(_best_path.total_value)) # msg = "--- total cost of decision = {}" # LOG.debug(msg.format(_best_path.total_cost)) current_rec[demand_name] = rec recommendations.append(current_rec) # Update the plan with the solution p.solution = { "recommendations": recommendations } # multiple spin-ups logic ''' go through list of recommendations in the solution for cloud candidates, check if (cloud-region-id + e2evnfkey) is in the order_locks table if so, insert the row with status 'parked' in order_locks, changes plan status to 'pending' in plans table (or other status value) otherwise, insert the row with status 'locked' in order_locks, and change status to 'solved' in plans table - continue reservation ''' # clean up the data/record in order_locks table, deleting all records that failed from MSO order_locks = self.OrderLock.query.all() for order_lock_record in order_locks: plans = getattr(order_lock_record, 'plans') for plan_id, plan_attributes in plans.items(): plan_dict = json.loads(plan_attributes) if plan_dict.get('status', None) == OrderLock.FAILED: order_lock_record.delete() LOG.info(_LI("The order lock record {} with status {} is deleted (due to failure" " spinup from MSO) from order_locks table"). format(order_lock_record, plan_dict.get('status'))) break inserted_order_records_dict = dict() available_dependenies_set = set() is_inserted_to_order_locks = True is_conflict_id_missing = False is_order_translated_before_spinup = False for solution in solution_list: for demand_name, candidate in solution.items(): if candidate.get('inventory_type') == 'cloud': conflict_id = candidate.get('conflict_id') service_resource_id = candidate.get('service_resource_id') # TODO(larry): add more logic for missing conflict_id in template if not conflict_id: is_conflict_id_missing = True break available_dependenies_set.add(conflict_id) # check if conflict_id exists in order_locks table order_lock_record = self.OrderLock.query.get_plan_by_col("id", conflict_id) if order_lock_record: is_spinup_completed = getattr(order_lock_record[0], 'is_spinup_completed') spinup_completed_timestamp = getattr(order_lock_record[0], 'spinup_completed_timestamp') if is_spinup_completed and spinup_completed_timestamp > p.translation_begin_timestamp: is_order_translated_before_spinup = True break elif not is_spinup_completed: inserted_order_records_dict[conflict_id] = service_resource_id if is_conflict_id_missing: message = _LE("Missing conflict identifier field for cloud candidates in the template, " "could not insert into order_locks table") LOG.debug(message) p.status = self.Plan.SOLVED elif is_order_translated_before_spinup: message = _LE("Retriggering Plan {} due to the new order arrives before the " "spinup completion of the old order ").format(p.id) LOG.debug(message) p.rehome_plan() elif len(inserted_order_records_dict) > 0: new_dependenies_set = available_dependenies_set - set(inserted_order_records_dict.keys()) dependencies = ','.join(str(s) for s in new_dependenies_set) for conflict_id, service_resource_id in inserted_order_records_dict.items(): plan = { p.id: { "status": OrderLock.UNDER_SPIN_UP, "created": self.current_time_millis(), "updated": self.current_time_millis(), "service_resource_id": service_resource_id } } if dependencies: plan[p.id]['dependencies'] = dependencies order_lock_row = self.OrderLock(id=conflict_id, plans=plan) response = order_lock_row.insert() # TODO(larry): add more logs for inserting order lock record (insert/update) LOG.info(_LI("Inserting the order lock record to order_locks table in MUSIC, " "conditional insert operation response from MUSIC {}").format(response)) if response and response.status_code == 200: body = response.json() LOG.info("Succcessfully inserted the record in order_locks table with " "the following response message {}".format(body)) else: is_inserted_to_order_locks = False else: for solution in solution_list: for demand_name, candidate in solution.items(): if candidate.get('inventory_type') == 'cloud': conflict_id = candidate.get('conflict_id') service_resource_id = candidate.get('service_resource_id') order_lock_record = self.OrderLock.query.get_plan_by_col("id", conflict_id) if order_lock_record: deleting_record = order_lock_record[0] plans = getattr(deleting_record, 'plans') is_spinup_completed = getattr(deleting_record, 'is_spinup_completed') spinup_completed_timestamp = getattr(deleting_record, 'spinup_completed_timestamp') if is_spinup_completed: # persist the record in order_locks_history table order_lock_history_record = \ self.OrderLockHistory(conflict_id=conflict_id, plans=plans, is_spinup_completed=is_spinup_completed, spinup_completed_timestamp=spinup_completed_timestamp ) LOG.debug("Inserting the history record with conflict id {}" " to order_locks_history table".format(conflict_id)) order_lock_history_record.insert() # remove the older record LOG.debug("Deleting the order lock record {} from order_locks table" .format(deleting_record)) deleting_record.delete() plan = { p.id: { "status": OrderLock.UNDER_SPIN_UP, "created": self.current_time_millis(), "updated": self.current_time_millis(), "service_resource_id": service_resource_id } } order_lock_row = self.OrderLock(id=conflict_id, plans=plan) response = order_lock_row.insert() # TODO(larry): add more logs for inserting order lock record (insert/update) LOG.info(_LI("Inserting the order lock record to order_locks table in MUSIC, " "conditional insert operation response from MUSIC {}").format(response)) if response and response.status_code == 200: body = response.json() LOG.info("Succcessfully inserted the record in order_locks table " "with the following response message {}".format(body)) else: is_inserted_to_order_locks = False if not is_inserted_to_order_locks: message = _LE("Plan {} status encountered an " "error while inserting order lock message to MUSIC.").format(p.id) LOG.error(message) p.status = self.Plan.ERROR p.message = message elif p.status == self.Plan.SOLVING: if len(inserted_order_records_dict) > 0: LOG.info(_LI("The plan with id {} is parked in order_locks table," "waiting for MSO release calls").format(p.id)) p.status = self.Plan.WAITING_SPINUP else: LOG.info(_LI("The plan with id {} is inserted in order_locks table."). format(p.id)) p.status = self.Plan.SOLVED while 'FAILURE' in _is_success \ and (self.current_time_seconds() - self.millisec_to_sec(p.updated)) <= self.conf.solver.timeout: _is_success = p.update(condition=self.solver_owner_condition) LOG.info(_LI("Plan search complete, changing the template status from solving to {}, " "atomic update response from MUSIC {}").format(p.status, _is_success)) LOG.info(_LI("Plan {} search complete, {} solution(s) found by machine {}"). format(p.id, len(solution_list), p.solver_owner)) LOG.debug("Plan {} detailed solution: {}". format(p.id, p.solution)) LOG.info("Plan name: {}".format(p.name))
def __set__(self, obj, value): """Set attribute""" if not self.fset: raise AttributeError(_LE("Can't set attribute")) type_ = type(obj) return self.fset.__get__(obj, type_)(value)
def run(self): """Run""" LOG.debug("%s" % self.__class__.__name__) # TODO(snarayanan): This is really meant to be a control loop # As long as self.running is true, we process another request. while self.running: # Delay time (Seconds) for MUSIC requests. time.sleep(self.conf.delay_time) # plans = Plan.query().all() # Find the first plan with a status of SOLVED. # Change its status to RESERVING. solution = None translation = None p = None # requests_to_reserve = dict() # Instead of using the query.all() method, now creating an index for 'status' # field in conductor.plans table, and query plans by status columns solved_plans = self.Plan.query.get_plan_by_col( "status", self.Plan.SOLVED) reserving_plans = self.Plan.query.get_plan_by_col( "status", self.Plan.RESERVING) # combine the plans with status = 'solved' and 'reserving' together plans = solved_plans + reserving_plans found_solved_template = False for p in plans: # when a plan is in RESERVING status more than timeout value if p.status == self.Plan.RESERVING and \ (self.current_time_seconds() - self.millisec_to_sec(p.updated)) > self.conf.reservation.timeout: # change the plan status to SOLVED for another VM to reserve p.status = self.Plan.SOLVED p.update(condition=self.reservating_status_condition) break elif p.status == self.Plan.SOLVED: solution = p.solution translation = p.translation found_solved_template = True break if not solution: if found_solved_template: message = _LE("Plan {} status is solved, yet " "the solution wasn't found").format(p.id) LOG.error(message) p.status = self.Plan.ERROR p.message = message p.update(condition=self.solved_status_condition) continue if found_solved_template and p and p.reservation_counter >= self.conf.reservation.max_reservation_counter: message = _LE("Tried {} times. Plan {} is unable to reserve") \ .format(self.conf.reservation.max_reservation_counter, p.id) LOG.error(message) p.status = self.Plan.ERROR p.message = message p.update(condition=self.solved_status_condition) continue log_util.setLoggerFilter(LOG, self.conf.keyspace, p.id) # update status to reserving p.status = self.Plan.RESERVING p.reservation_counter += 1 p.reservation_owner = socket.gethostname() _is_updated = p.update(condition=self.solved_status_condition) if not _is_updated: continue if 'FAILURE' in _is_updated: continue LOG.info( _LI("Reservation starts, changing the template status from solved to reserving, " "atomic update response from MUSIC {}").format( _is_updated)) LOG.info( _LI("Plan {} with request id {} is reserving by machine {}. Tried to reserve it for {} times." ).format(p.id, p.name, p.reservation_owner, p.reservation_counter)) # begin reservations # if plan needs reservation proceed with reservation # else set status to done. reservations = None _is_success = "FAILURE" if translation: conductor_solver = translation.get("conductor_solver") if conductor_solver: reservations = conductor_solver.get("reservations") else: LOG.error("no conductor_solver in " "translation for Plan {}".format(p.id)) if reservations: recommendations = solution.get("recommendations") reservation_list = list() # TODO(larry) combine the two reservation logic as one, make the code service independent sdwan_candidate_list = list() service_model = reservations.get("service_model") for reservation, resource in reservations.get("demands", {}).items(): candidates = list() reservation_demand = resource.get("demands") reservation_name = resource.get("name") reservation_type = resource.get("type") reservation_properties = resource.get("properties") if reservation_properties: controller = reservation_properties.get("controller") request = reservation_properties.get("request") for recommendation in recommendations: for demand, r_resource in recommendation.items(): if demand == reservation_demand: # get selected candidate from translation selected_candidate_id = r_resource.get( "candidate").get("candidate_id") demands = translation.get( "conductor_solver").get("demands") for demand_name, d_resource in demands.items(): if demand_name == demand: for candidate in d_resource.get( "candidates"): if candidate.get( "candidate_id" ) == selected_candidate_id: candidate['request'] = request candidates.append(candidate) sdwan_candidate_list.append( candidate) #TODO(larry) combine the two reservation logic as one, make the code service independent if service_model == "ADIOD": is_success = self.try_reservation_call( method="reserve", candidate_list=candidates, reservation_type=service_model, controller=controller, request=request, reservation_name=None) # if reservation succeed continue with next candidate if is_success: curr_reservation = dict() curr_reservation['candidate_list'] = candidates curr_reservation[ 'reservation_name'] = reservation_name curr_reservation[ 'reservation_type'] = reservation_type curr_reservation['controller'] = controller curr_reservation['request'] = request reservation_list.append(curr_reservation) else: # begin roll back of all reserved resources on # the first failed reservation rollback_status = \ self.rollback_reservation(reservation_list) # order_lock spin-up rollback for decision in solution.get('recommendations'): candidate = list(decision.values())[0].get( 'candidate' ) # Python 3 Conversion -- dict object to list object if candidate.get('inventory_type') == 'cloud': # TODO(larry) change the code to get('conflict_id') instead of 'location_id' conflict_id = candidate.get('conflict_id') order_record = self.OrderLock.query.get_plan_by_col( "id", conflict_id)[0] if order_record: order_record.delete() # statuses if rollback_status: # released all reservations, # move plan to translated if p.reservation_counter >= self.conf.reservation.max_reservation_counter: p.status = self.Plan.ERROR p.message = _LE( "Tried {} times. Plan {} is unable to reserve" ).format( self.conf.reservation. max_reservation_counter, p.id) LOG.error(p.message) else: p.status = self.Plan.TRANSLATED # TODO(larry): Should be replaced by the new api from MUSIC while 'FAILURE' in _is_success: _is_success = p.update( condition=self. reservation_owner_condition) LOG.info( _LI("Rolling back the template from reserving to {} status, " "atomic update response from MUSIC {}" ).format(p.status, _is_success)) del reservation_list[:] else: LOG.error("Reservation rollback failed") p.status = self.Plan.ERROR p.message = "Reservation release failed" # TODO(larry): Should be replaced by the new api from MUSIC while 'FAILURE' in _is_success: _is_success = p.update( condition=self. reservation_owner_condition) LOG.info( _LI("Rollback Failed, Changing the template status from reserving to error, " "atomic update response from MUSIC {}" ).format(_is_success)) break # reservation failed continue # continue with reserving the next candidate # TODO(larry) combine the two reservation logic as one, make the code service independent if service_model == "DHV": is_success = self.try_reservation_call( method="reserve", candidate_list=sdwan_candidate_list, reservation_type=service_model, controller=controller, request=request, reservation_name=None) if not is_success: # order_lock spin-up rollback for decision in solution.get('recommendations'): candidate = list(decision.values())[0].get( 'candidate' ) # Python 3 Conversion -- dict object to list object if candidate.get('inventory_type') == 'cloud': conflict_id = candidate.get('conflict_id') order_record = self.OrderLock.query.get_plan_by_col( "id", conflict_id)[0] if order_record: order_record.delete() if p.reservation_counter >= self.conf.reservation.max_reservation_counter: p.status = self.Plan.ERROR p.message = _LE( "Tried {} times. Plan {} is unable to reserve" ).format( self.conf.reservation.max_reservation_counter, p.id) LOG.error(p.message) else: p.status = self.Plan.TRANSLATED # TODO(larry): Should be replaced by the new api from MUSIC while 'FAILURE' in _is_success: _is_success = p.update( condition=self.reservation_owner_condition) LOG.info( _LI("Rolling back the template from reserving to {} status, " "atomic update response from MUSIC {}"). format(p.status, _is_success)) del reservation_list[:] # verify if all candidates have been reserved if p.status == self.Plan.RESERVING: # all reservations succeeded. LOG.info( _LI("Plan {} with request id {} Reservation complete"). format(p.id, p.name)) LOG.debug("Plan {} Reservation complete".format(p.id)) p.status = self.Plan.DONE while 'FAILURE' in _is_success and ( self.current_time_seconds() - self.millisec_to_sec( p.updated)) <= self.conf.reservation.timeout: _is_success = p.update( condition=self.reservation_owner_condition) LOG.info( _LI("Reservation is complete, changing the template status from reserving to done, " "atomic update response from MUSIC {}").format( _is_success)) continue
def __check_for_templates(self): """Wait for the polling interval, then do the real template check.""" # Wait for at least poll_interval sec polling_interval = self.conf.controller.polling_interval time.sleep(polling_interval) # Look for plans with the status set to TEMPLATE # Instead of using the query.all() method, now creating an index for 'status' # field in conductor.plans table, and query plans by status columns template_plans = self.Plan.query.get_plan_by_col( "status", self.Plan.TEMPLATE) translating_plans = self.Plan.query.get_plan_by_col( "status", self.Plan.TRANSLATING) # combine the plans with status = 'template' and 'translating' together plans = template_plans + translating_plans for plan in plans: # If there's a template to be translated, do it! if plan.status == self.Plan.TEMPLATE: if plan.translation_counter >= self.conf.controller.max_translation_counter: message = _LE("Tried {} times. Plan {} is unable to translate") \ .format(self.conf.controller.max_translation_counter, plan.id) plan.message = message plan.status = self.Plan.ERROR plan.update(condition=self.template_status_condition) LOG.error(message) break else: # change the plan status to "translating" and assign the current machine as translation owner plan.status = self.Plan.TRANSLATING plan.translation_counter += 1 plan.translation_owner = socket.gethostname() plan.translation_begin_timestamp = int( round(time.time() * 1000)) _is_updated = plan.update( condition=self.template_status_condition) log_util.setLoggerFilter(LOG, self.conf.keyspace, plan.id) LOG.info( _LE("Plan {} is trying to update the status from 'template' to 'translating'," " get {} response from MUSIC").format( plan.id, _is_updated)) if not _is_updated: break if _is_updated and 'SUCCESS' in _is_updated: self.translate(plan) break # TODO(larry): sychronized clock among Conducotr VMs, or use an offset elif plan.status == self.Plan.TRANSLATING and (self.current_time_seconds() - self.millisec_to_sec(plan.updated)) \ > self.conf.messaging_server.timeout: plan.status = self.Plan.TEMPLATE plan.update(condition=self.translating_status_condition) break elif plan.timedout: # TODO(jdandrea): How to tell all involved to stop working? # Not enough to just set status. continue
def _do(self): """Look for a new RPC call and serve it""" # Get all the messages in queue msgs = self.RPC.query.all() for msg in msgs: # Find the first msg marked as enqueued. if msg.working and \ (self.current_time_seconds() - self.millisec_to_sec(msg.updated))\ > self.conf.messaging_server.response_timeout: msg.status = message.Message.ENQUEUED msg.update(condition=self.working_status_condition) if not msg.enqueued: continue if 'plan_name' in msg.ctxt.keys(): LOG.info('Plan name: {}'.format(msg.ctxt['plan_name'])) elif 'plan_name' in msg.args.keys(): LOG.info('Plan name: {}'.format(msg.args['plan_name'])) # Change the status to WORKING (operation with a lock) msg.status = message.Message.WORKING msg.owner = socket.gethostname() # All update should have a condition (status == enqueued) _is_updated = msg.update(condition=self.enqueued_status_condition) if not _is_updated or 'FAILURE' in _is_updated: continue # RPC methods must not start/end with an underscore. if msg.method.startswith('_') or msg.method.endswith('_'): error_msg = _LE("Method {} must not start or end" "with underscores").format(msg.method) self._log_error_and_update_msg(msg, error_msg) return # The first endpoint that supports the method wins. method = None for endpoint in self.endpoints: if msg.method not in dir(endpoint): continue endpoint_method = getattr(endpoint, msg.method) if callable(endpoint_method): method = endpoint_method if self.conf.messaging_server.debug: LOG.debug("Message {} method {} is " "handled by endpoint {}".format( msg.id, msg.method, method.__str__.__name__)) break if not method: error_msg = _LE("Message {} method {} unsupported " "in endpoints.").format(msg.id, msg.method) self._log_error_and_update_msg(msg, error_msg) return # All methods must take a ctxt and args param. if inspect.getargspec(method).args != ['self', 'ctx', 'arg']: error_msg = _LE("Method {} must take three args: " "self, ctx, arg").format(msg.method) self._log_error_and_update_msg(msg, error_msg) return LOG.info( _LI("Message {} method {} received").format( msg.id, msg.method)) if self.conf.messaging_server.debug: LOG.debug( _LI("Message {} method {} context: {}, args: {}").format( msg.id, msg.method, msg.ctxt, msg.args)) failure = None try: # Add the template to conductor.plan table # Methods return an opaque dictionary result = method(msg.ctxt, msg.args) # FIXME(jdandrea): Remove response/error and make it opaque. # That means this would just be assigned result outright. msg.response = result.get('response', result) except Exception: # Current sys.exc_info() content can be overridden # by another exception raised by a log handler during # LOG.exception(). So keep a copy and delete it later. failure = sys.exc_info() # Do not log details about the failure here. It will # be returned later upstream. LOG.exception(_LE('Exception during message handling')) try: if failure is None: msg.status = message.Message.COMPLETED else: msg.failure = \ rpc_common.serialize_remote_exception(failure) msg.status = message.Message.ERROR LOG.info( _LI("Message {} method {}, status: {}").format( msg.id, msg.method, msg.status)) if self.conf.messaging_server.debug: LOG.debug("Message {} method {}, response: {}".format( msg.id, msg.method, msg.response)) _is_success = 'FAILURE' while 'FAILURE' in _is_success and ( self.current_time_seconds() - self.millisec_to_sec(msg.updated) ) <= self.conf.messaging_server.response_timeout: _is_success = msg.update() LOG.info( _LI("updating the message status from working to {}, " "atomic update response from MUSIC {}").format( msg.status, _is_success)) except Exception: LOG.exception( _LE("Can not send reply for message {} " "method {}").format(msg.id, msg.method)) finally: # Remove circular object reference between the current # stack frame and the traceback in exc_info. del failure
def call(self, ctxt, method, args): """Synchronous Call""" # # check if the call has a message saved in cache # # key: string concatenation of ctxt + method + args # # value: rpc response object # key = "" # for k, v in ctxt.items(): # key += str(k) # key += '#' + str(v) + '#' # key += '|' + str(method) + '|' # for k, v in args.items(): # key += str(k) # key += '#' + str(v) + '#' # # # check if the method has been called before # # and cached # if key in self.message_cache: # LOG.debug("Retrieved method {} with args " # "{} from cache".format(method, args)) # return self.message_cache[key] rpc_start_time = time.time() rpc = self.RPC(action=self.RPC.CALL, ctxt=ctxt, method=method, args=args) # TODO(jdandrea): Do something if the assert fails. assert (rpc.enqueued) rpc_id = rpc.id topic = self.target.topic LOG.info(_LI("Message {} on topic {} enqueued.").format(rpc_id, topic)) if self.conf.messaging_server.debug: LOG.debug("Calling method {} with args {}".format(method, args)) # Check message status within a thread executor = futurist.ThreadPoolExecutor() started_at = time.time() while (time.time() - started_at) <= self.conf.messaging_server.response_timeout: fut = executor.submit(self.__check_rpc_status, rpc_id, method) rpc = fut.result() if rpc and rpc.finished: if self.conf.messaging_server.debug: LOG.debug("Message {} method {} response received".format( rpc_id, method)) break executor.shutdown() # Get response, delete message, and return response if not rpc or not rpc.finished: LOG.error( _LE("Message {} on topic {} timed out at {} seconds").format( rpc_id, topic, self.conf.messaging_server.response_timeout)) elif not rpc.ok: LOG.error( _LE("Message {} on topic {} returned an error").format( rpc_id, topic)) response = rpc.response failure = rpc.failure rpc.delete() # TODO(jdandrea): Put a TTL on the msg instead? # self.message_cache[key] = response LOG.debug("Elapsed time: {0:.3f} sec".format(time.time() - rpc_start_time)) # If there's a failure, raise it as an exception allowed = [] if failure is not None and failure != '': # TODO(jdandrea): Do we need to populate allowed(_remote_exmods)? raise rpc_common.deserialize_remote_exception(failure, allowed) return response